Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(persons): split the divergence module into a package (#117388)

Eli Reisman01c3b352f2d7
diff --git a/posthog/api/person.py b/posthog/api/person.py
index 687fe08ee4ce..33a14be5fc84 100644
--- a/posthog/api/person.py
+++ b/posthog/api/person.py
@@ -61,7 +61,7 @@ from posthog.models.person.bulk_delete import (
queue_person_recording_deletion,
resolve_persons_for_deletion,
)
-from posthog.models.person.divergence import repair_distinct_id
+from posthog.models.person.divergence.repairs import repair_distinct_id
from posthog.models.person.missing_person import MissingPerson
from posthog.models.person.util import (
get_distinct_ids_for_persons,
diff --git a/posthog/api/test/test_person.py b/posthog/api/test/test_person.py
index 51f0dbcccc83..fdc340bf2b06 100644
--- a/posthog/api/test/test_person.py
+++ b/posthog/api/test/test_person.py
@@ -28,7 +28,8 @@ from parameterized import parameterized
from rest_framework import status
from rest_framework.exceptions import ValidationError
-import posthog.models.person.divergence
+import posthog.models.person.divergence.repairs
+import posthog.models.person.divergence.stale_merge
from posthog.api.person import tag_client_query_id
from posthog.clickhouse.client import sync_execute
from posthog.clickhouse.query_tagging import get_query_tag_value, reset_query_tags
@@ -2418,8 +2419,8 @@ class TestPerson(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):
assert person_after.version is not None and person_after.version > 105
@mock.patch(
- f"{posthog.models.person.divergence.__name__}.create_person_distinct_id",
- wraps=posthog.models.person.divergence.create_person_distinct_id,
+ f"{posthog.models.person.divergence.repairs.__name__}.create_person_distinct_id",
+ wraps=posthog.models.person.divergence.repairs.create_person_distinct_id,
)
@pytest.mark.flaky(reruns=2)
def test_reset_person_distinct_id_not_found(self, mocked_ch_call):
@@ -2698,7 +2699,7 @@ class TestPersonBatchRestrictedProperties(ClickhouseTestMixin, APIBaseTest):
team=self.team, property_definition=restricted, access_level=PropertyAccessLevel.NONE.value
)
- @mock.patch(f"{posthog.models.person.divergence.__name__}.capture_internal")
+ @mock.patch(f"{posthog.models.person.divergence.stale_merge.__name__}.capture_internal")
def test_reset_does_not_copy_a_restricted_property_into_an_event(self, capture: mock.MagicMock) -> None:
person = create_person(team=self.team, properties={"email": "[email protected]"}, version=5, uuid=str(uuid4()))
add_distinct_id(person=person, distinct_id="target", version=0)
diff --git a/posthog/management/commands/person_divergence.py b/posthog/management/commands/person_divergence.py
index c2a9174d9e0a..e201c2a601b5 100644
--- a/posthog/management/commands/person_divergence.py
+++ b/posthog/management/commands/person_divergence.py
@@ -27,24 +27,26 @@ from uuid import UUID
from django.core.management.base import BaseCommand, CommandError, CommandParser
from posthog.clickhouse.query_tagging import Feature, Product, tags_context
-from posthog.models.person.divergence import (
- HIDDEN_TEAM_STEP,
- SAMPLE_TEAM_STEP,
- STALE_TEAM_STEP,
- SWEPT_TEAM_STEP,
+from posthog.models.person.divergence.datatypes import (
DivergentPerson,
PersonRef,
RepairAction,
SampledPerson,
ScanSummary,
TeamCheck,
- check_team,
- repair_persons,
+)
+from posthog.models.person.divergence.fleet_scans import (
+ HIDDEN_TEAM_STEP,
+ SAMPLE_TEAM_STEP,
+ STALE_TEAM_STEP,
+ SWEPT_TEAM_STEP,
scan_hidden_persons,
scan_sample,
scan_stale_persons,
scan_swept_persons,
)
+from posthog.models.person.divergence.repairs import repair_persons
+from posthog.models.person.divergence.team_check import check_team
_DIVERGENT_SCANS: dict[str, tuple[Callable[..., ScanSummary], int, str]] = {
"hidden": (
diff --git a/posthog/models/person/divergence.py b/posthog/models/person/divergence.py
deleted file mode 100644
index 7c53d0a848ad..000000000000
--- a/posthog/models/person/divergence.py
+++ /dev/null
@@ -1,1299 +0,0 @@
-"""Find and repair persons whose ClickHouse rows disagree with the persons database.
-
-The persons database, read through personhog, is the source of truth. Scans only read. A repair
-republishes the Postgres state of a person and its distinct ids to ClickHouse. When ClickHouse
-holds a version at or above the Postgres one, the repair first raises the Postgres version above
-it, so the published row wins and the next ingestion update still outranks it.
-"""
-
-from __future__ import annotations
-
-import json
-import time
-from collections import Counter, defaultdict
-from collections.abc import Callable, Sequence
-from dataclasses import field
-from datetime import UTC, datetime
-from functools import partial
-from typing import Any, Literal, TypeVar
-from uuid import UUID
-
-import grpc
-
-from posthog.api.capture import capture_internal
-from posthog.clickhouse.client import sync_execute
-from posthog.clickhouse.workload import Workload
-from posthog.dataclasses import frozen
-from posthog.errors import CH_TRANSIENT_ERRORS
-from posthog.exceptions import ClickHouseQueryMemoryLimitExceeded, ClickHouseQueryTimeOut
-from posthog.kafka_client.client import ClickhouseProducer, ProduceResult
-from posthog.kafka_client.topics import KAFKA_PERSON
-from posthog.models.person import Person
-from posthog.models.person.sql import INSERT_PERSON_SQL
-from posthog.models.person.util import (
- _batched_get_distinct_ids_for_persons,
- _batched_get_persons_by_distinct_ids,
- _batched_get_persons_by_uuids,
- _flush_person_producers,
- _person_row,
- create_person_distinct_id,
- get_person_by_distinct_id,
- get_person_tombstones,
- get_persons_by_uuids,
-)
-from posthog.models.team import Team
-from posthog.personhog_client.client import personhog_call, require_personhog_client
-from posthog.personhog_client.interceptor import is_transient_rpc_error
-from posthog.personhog_client.proto import (
- CONSISTENCY_LEVEL_STRONG,
- GetDistinctIdsForPersonsRequest,
- ReadOptions,
- SetPersonDistinctIdVersionFloorRequest,
- SetPersonVersionFloorRequest,
-)
-
-from products.access_control.backend.facade.api import team_has_property_access_rules
-
-PersonDivergenceKind = Literal["hidden", "swept", "stale", "behind", "absent"]
-MappingDivergenceKind = Literal["hidden", "other_person", "stale", "absent"]
-SampleClassification = Literal["equal", "pg_below_ch", "pg_above_ch", "team_gone", "pg_tombstone", "pg_absent"]
-SampleEra = Literal["before_cutoff", "since_cutoff", "any"]
-RepairOutcome = Literal[
- "would_repair",
- "repaired",
- "skipped_not_divergent",
- "skipped_not_live",
- "skipped_team_gone",
- "skipped_tombstoned",
- "skipped_owner_changed",
- "skipped_mapping_gone",
- "skipped_reread_lagging",
- "skipped_stale",
- "skipped_stale_merge_failed",
- "skipped_stale_merge_too_large",
- "skipped_stale_merge_restricted",
- "merge_queued",
- "skipped_too_many_distinct_ids",
-]
-
-# The legacy delete path wrote ClickHouse tombstones at version + 100, so those rows sit at 100 or above.
-LEGACY_TOMBSTONE_MIN_VERSION = 100
-
-HIDDEN_TEAM_STEP = 10_000
-SWEPT_TEAM_STEP = 10_000
-STALE_TEAM_STEP = 5_000
-SAMPLE_TEAM_STEP = 10_000
-
-_HIDDEN_SETTINGS = {"max_execution_time": 900, "max_memory_usage": 32_000_000_000}
-_SWEPT_SETTINGS = {"apply_deleted_mask": 0, "max_execution_time": 1800, "max_memory_usage": 64_000_000_000}
-_STALE_SETTINGS = {
- "max_execution_time": 1800,
- "max_memory_usage": 48_000_000_000,
- "max_bytes_before_external_group_by": 20_000_000_000,
-}
-_SAMPLE_SETTINGS = {"max_execution_time": 3600, "max_memory_usage": 32_000_000_000}
-_TEAM_CHECK_SETTINGS = {"max_execution_time": 900, "max_memory_usage": 32_000_000_000}
-# Repair reads see lightweight-deleted rows, so a republish lands above every row a sweep masked.
-_REPAIR_READ_SETTINGS = {"apply_deleted_mask": 0, "max_execution_time": 60, "max_memory_usage": 4_000_000_000}
-
-# A scan reads only these fields, which keeps person properties out of the RPC payloads.
-_VERSION_ONLY_READ_OPTIONS = ReadOptions(field_mask=["id", "uuid", "team_id", "version"])
-
-_REPAIR_CHUNK_SIZE = 100
-_MAPPING_QUERY_CHUNK_SIZE = 1_000
-# A person with more distinct ids keeps its person repair, and its mappings are reported for a separate plan:
-# one read of every mapping can exceed the gRPC message limit, and their paced writes would run for hours.
-_MAX_REPAIR_DISTINCT_IDS_PER_PERSON = 1_000
-# The waits between re-reads give the replica about 0.5 s to show a raise, far above its usual lag of a few milliseconds.
-_REREAD_BACKOFF_SECONDS = (0.025, 0.05, 0.1, 0.15, 0.175)
-# Well under capture's 1,000,000-byte Kafka message limit and ingestion's 655,360-byte person properties limit
-# (trimmed to 512 KiB), so ingestion applies the merge whole.
-_STALE_MERGE_MAX_BYTES = 256 * 1024
-_TRANSIENT_RETRY_BACKOFF_SECONDS = (1.0, 2.0)
-_TRANSIENT_RPC_CODES = frozenset(
- {grpc.StatusCode.INTERNAL, grpc.StatusCode.UNAVAILABLE, grpc.StatusCode.DEADLINE_EXCEEDED}
-)
-_FLUSH_TIMEOUT_SECONDS = 5 * 60
-# Confirmed produce results are dropped this often, so a long repair does not hold one per published row.
-_DELIVERY_PRUNE_EVERY = 1_000
-
-_T = TypeVar("_T")
-
-_sleep = time.sleep
-
-
-@frozen
-class DivergentPerson:
- team_id: int
- person_uuid: str
- kind: PersonDivergenceKind
- ch_max_version: int
- pg_version: int
-
-
-@frozen
-class ScanSummary:
- candidates: int
- divergent: int
- skipped_team_ids: list[int]
-
-
-@frozen
-class SampledPerson:
- team_id: int
- person_uuid: str
- classification: SampleClassification
- era: SampleEra
- ch_max_version: int
- pg_version: int | None
-
-
-@frozen
-class SampleBucket:
- classification: SampleClassification
- era: SampleEra
-
-
-@frozen
-class SampleSummary:
- sampled: int
- counts: dict[SampleBucket, int]
- skipped_team_ids: list[int]
-
-
-@frozen
-class TeamCheck:
- team_id: int
- persons_sampled: int
- persons_live_in_postgres: int
- distinct_ids_sampled: int
- distinct_ids_live_in_postgres: int
-
-
-@frozen
-class PersonRef:
- team_id: int
- person_uuid: str
-
- def __post_init__(self) -> None:
- # Lookups key on the canonical form that ClickHouse toString(id) and the persons reads return.
- if str(UUID(self.person_uuid)) != self.person_uuid:
- raise ValueError(f"person_uuid must be a canonical lowercase UUID: {self.person_uuid!r}")
-
-
-@frozen
-class RepairAction:
- """One person row (``distinct_id`` is None) or one of its mappings, with what the repair did to it.
-
- A person with too many distinct ids to repair gets one extra row with no ``distinct_id`` and the
- outcome ``skipped_too_many_distinct_ids``, which stands for all of its mappings.
- """
-
- team_id: int
- person_uuid: str
- distinct_id: str | None
- kind: PersonDivergenceKind | MappingDivergenceKind | None
- pg_version: int | None
- ch_max_version: int | None
- target_version: int | None
- outcome: RepairOutcome
-
-
-@frozen
-class RepairSummary:
- applied: bool
- persons: int
- person_outcomes: dict[RepairOutcome, int]
- mapping_outcomes: dict[RepairOutcome, int]
- # Published rows that Kafka did not confirm: the delivery failed, or it was still queued at the deadline.
- undelivered: int
-
-
-def _ch(sql: str, args: dict[str, Any], settings: dict[str, int]) -> list[Any]:
- return sync_execute(sql, args, settings=settings, workload=Workload.OFFLINE, readonly=True)
-
-
-def _live_person_versions(team_id: int, person_uuids: list[str], operation: str) -> dict[str, int]:
- persons = personhog_call(
- operation,
- lambda: _batched_get_persons_by_uuids(
- team_id, person_uuids, operation, read_options=_VERSION_ONLY_READ_OPTIONS
- ),
- )
- return {str(UUID(p.uuid)): int(p.version) for p in persons}
-
-
-def _tombstoned_uuids(team_id: int, person_uuids: list[str]) -> set[str]:
- if not person_uuids:
- return set()
- return {str(t.uuid) for t in get_person_tombstones(team_id, [UUID(u) for u in person_uuids])}
-
-
-def _resolve_max_team_id(max_team_id: int | None) -> int:
- if max_team_id is not None:
- return max_team_id
- [[highest]] = _ch("SELECT max(team_id) FROM person", {}, _HIDDEN_SETTINGS)
- return int(highest or 0) + 1
-
-
-def _chunks(items: Sequence[_T], size: int) -> list[Sequence[_T]]:
- return [items[i : i + size] for i in range(0, len(items), size)]
-
-
-def _scan_team_ranges(
- query: Callable[[int, int], list[Any]],
- *,
- min_team_id: int,
- max_team_id: int,
- team_step: int,
- on_rows: Callable[[list[Any]], None],
- log: Callable[[str], None],
-) -> list[int]:
- """Return each team that runs out of memory or time even when scanned alone, so the caller can rerun it with higher limits."""
- skipped: list[int] = []
-
- def scan(lo: int, hi: int) -> None:
- try:
- rows = query(lo, hi)
- except (ClickHouseQueryMemoryLimitExceeded, ClickHouseQueryTimeOut) as exc:
- reason = "out of memory" if isinstance(exc, ClickHouseQueryMemoryLimitExceeded) else "timed out"
- if hi - lo == 1:
- log(f"team {lo}: {reason}, skipped")
- skipped.append(lo)
- return
- mid = (lo + hi) // 2
- log(f"teams [{lo}, {hi}): {reason}, bisecting")
- scan(lo, mid)
- scan(mid, hi)
- return
- on_rows(rows)
- log(f"teams [{lo}, {hi}): {len(rows)} ClickHouse candidates")
-
- for lo in range(min_team_id, max_team_id, team_step):
- scan(lo, min(lo + team_step, max_team_id))
- return skipped
-
-
-def _scan_divergent_persons(
- *,
- sql: str,
- settings: dict[str, int],
- extra_args: dict[str, Any],
- kind: PersonDivergenceKind,
- divergent: Callable[[int, int], bool],
- min_team_id: int,
- max_team_id: int | None,
- team_step: int,
- on_found: Callable[[DivergentPerson], None],
- log: Callable[[str], None],
-) -> ScanSummary:
- candidates = 0
- found = 0
-
- def on_rows(rows: list[Any]) -> None:
- nonlocal candidates, found
- by_team: dict[int, dict[str, int]] = defaultdict(dict)
- for team_id, person_uuid, ch_max_version in rows:
- by_team[int(team_id)][person_uuid] = int(ch_max_version)
- for team_id, ch_versions in sorted(by_team.items()):
- candidates += len(ch_versions)
- pg_versions = _live_person_versions(team_id, list(ch_versions), f"person_divergence_scan_{kind}")
- for person_uuid, pg_version in pg_versions.items():
- ch_max_version = ch_versions[person_uuid]
- if not divergent(pg_version, ch_max_version):
- continue
- on_found(
- DivergentPerson(
- team_id=team_id,
- person_uuid=person_uuid,
- kind=kind,
- ch_max_version=ch_max_version,
- pg_version=pg_version,
- )
- )
- found += 1
-
- skipped = _scan_team_ranges(
- lambda lo, hi: _ch(sql, {"min_team_id": lo, "max_team_id": hi, **extra_args}, settings),
- min_team_id=min_team_id,
- max_team_id=_resolve_max_team_id(max_team_id),
- team_step=team_step,
- on_rows=on_rows,
- log=log,
- )
- return ScanSummary(candidates=candidates, divergent=found, skipped_team_ids=skipped)
-
-
-_HIDDEN_SQL = """
-SELECT team_id, toString(id), max(version) AS max_version
-FROM person
-WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s
- AND (team_id, id) IN (
- SELECT team_id, id FROM person
- WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s
- AND is_deleted = 1 AND version >= %(min_tombstone_version)s
- )
-GROUP BY team_id, id
-HAVING argMax(is_deleted, version) = 1 AND max_version >= %(min_tombstone_version)s
-"""
-
-
-def scan_hidden_persons(
- *,
- min_team_id: int = 0,
- max_team_id: int | None = None,
- team_step: int = HIDDEN_TEAM_STEP,
- on_found: Callable[[DivergentPerson], None],
- log: Callable[[str], None],
-) -> ScanSummary:
- """Persons live in Postgres whose newest ClickHouse row is a legacy (version 100 or above) tombstone.
-
- Run it, then repair what it finds, right before every ClickHouse deletion sweep while such
- tombstones remain: the sweep deletes every ClickHouse row of these persons.
- """
- return _scan_divergent_persons(
- sql=_HIDDEN_SQL,
- settings=_HIDDEN_SETTINGS,
- extra_args={"min_tombstone_version": LEGACY_TOMBSTONE_MIN_VERSION},
- kind="hidden",
- divergent=lambda _pg_version, _ch_max_version: True,
- min_team_id=min_team_id,
- max_team_id=max_team_id,
- team_step=team_step,
- on_found=on_found,
- log=log,
- )
-
-
-# The ClickHouse deletion sweep deletes every row of a person whose highest-version row is a tombstone.
-# That includes a live row written after the tombstone at a lower version, so the person stays live in
-# Postgres with no visible ClickHouse row.
-_SWEPT_SQL = """
-SELECT team_id, toString(id), max(version) AS max_version
-FROM person
-WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s
- AND (team_id, id) IN (
- SELECT team_id, id FROM person
- WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s
- AND NOT _row_exists AND is_deleted = 1 AND version >= %(min_tombstone_version)s
- )
-GROUP BY team_id, id
-HAVING countIf(_row_exists) = 0
- AND maxIf(_timestamp, is_deleted = 0) > maxIf(_timestamp, is_deleted = 1 AND version >= %(min_tombstone_version)s)
-"""
-
-
-def scan_swept_persons(
- *,
- min_team_id: int = 0,
- max_team_id: int | None = None,
- team_step: int = SWEPT_TEAM_STEP,
- on_found: Callable[[DivergentPerson], None],
- log: Callable[[str], None],
-) -> ScanSummary:
- """Live Postgres persons with every ClickHouse row deleted, including a live row written after a legacy tombstone.
-
- Run it soon after a ClickHouse deletion sweep: once merges remove the deleted rows, no scan finds the person.
- """
- return _scan_divergent_persons(
- sql=_SWEPT_SQL,
- settings=_SWEPT_SETTINGS,
- extra_args={"min_tombstone_version": LEGACY_TOMBSTONE_MIN_VERSION},
- kind="swept",
- divergent=lambda _pg_version, _ch_max_version: True,
- min_team_id=min_team_id,
- max_team_id=max_team_id,
- team_step=team_step,
- on_found=on_found,
- log=log,
- )
-
-
-# A row with a lower version that arrives well after the winning row means Postgres restarted the
-# person below ClickHouse, so every later update loses to the old winner.
-_STALE_SQL = """
-SELECT team_id, toString(id), max(version) AS max_version
-FROM person
-WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s
- AND (team_id, id) IN (
- SELECT team_id, id FROM person
- WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s
- AND _timestamp > now() - toIntervalDay(%(window_days)s)
- )
-GROUP BY team_id, id
-HAVING argMax(is_deleted, version) = 0
- AND argMax(version, _timestamp) < max_version
- AND max(_timestamp) > argMax(_timestamp, version) + INTERVAL 1 HOUR
-"""
-
-
-def scan_stale_persons(
- *,
- window_days: int,
- min_team_id: int = 0,
- max_team_id: int | None = None,
- team_step: int = STALE_TEAM_STEP,
- on_found: Callable[[DivergentPerson], None],
- log: Callable[[str], None],
-) -> ScanSummary:
- """Persons whose live ClickHouse winner outranks Postgres, found through a late lower-version row."""
- return _scan_divergent_persons(
- sql=_STALE_SQL,
- settings=_STALE_SETTINGS,
- extra_args={"window_days": window_days},
- kind="stale",
- divergent=lambda pg_version, ch_max_version: pg_version < ch_max_version,
- min_team_id=min_team_id,
- max_team_id=max_team_id,
- team_step=team_step,
- on_found=on_found,
- log=log,
- )
-
-
-_SAMPLE_SQL = """
-SELECT team_id, toString(id), max(version) AS max_version, toUnixTimestamp(argMax(_timestamp, version)) AS written_at
-FROM person
-WHERE cityHash64(id) %% %(modulus)s = %(residue)s AND team_id >= %(min_team_id)s AND team_id < %(max_team_id)s
-GROUP BY team_id, id
-HAVING argMax(is_deleted, version) = 0
- AND (%(written_within_days)s = 0 OR argMax(_timestamp, version) >= now() - toIntervalDay(%(written_within_days)s))
-"""
-
-
-def _classify_sampled(
- pg_version: int | None, ch_max_version: int, *, team_exists: bool, tombstoned: bool
-) -> SampleClassification:
- if not team_exists:
- return "team_gone"
- if pg_version is None:
- return "pg_tombstone" if tombstoned else "pg_absent"
- if pg_version == ch_max_version:
- return "equal"
- return "pg_below_ch" if pg_version < ch_max_version else "pg_above_ch"
-
-
-def scan_sample(
- *,
- modulus: int,
- residue: int,
- written_within_days: int | None = None,
- cutoff: datetime | None = None,
- min_team_id: int = 0,
- max_team_id: int | None = None,
- team_step: int = SAMPLE_TEAM_STEP,
- on_sampled: Callable[[SampledPerson], None],
- log: Callable[[str], None],
-) -> SampleSummary:
- """Classify a uniform sample of live ClickHouse persons against Postgres."""
- if not 0 <= residue < modulus:
- raise ValueError("residue must be in [0, modulus)")
- cutoff_ts = cutoff.timestamp() if cutoff is not None else None
- counts: Counter[SampleBucket] = Counter()
- sampled_count = 0
-
- def query(lo: int, hi: int) -> list[Any]:
- return _ch(
- _SAMPLE_SQL,
- {
- "modulus": modulus,
- "residue": residue,
- "written_within_days": written_within_days or 0,
- "min_team_id": lo,
- "max_team_id": hi,
- },
- _SAMPLE_SETTINGS,
- )
-
- def on_rows(rows: list[Any]) -> None:
- nonlocal sampled_count
- if not rows:
- return
- sampled_count += len(rows)
- by_team: dict[int, dict[str, tuple[int, SampleEra]]] = defaultdict(dict)
- for team_id, person_uuid, ch_max_version, written_at in rows:
- era: SampleEra = (
- "any" if cutoff_ts is None else "before_cutoff" if written_at < cutoff_ts else "since_cutoff"
- )
- by_team[int(team_id)][person_uuid] = (int(ch_max_version), era)
-
- existing_teams = set(Team.objects.filter(id__in=list(by_team)).values_list("id", flat=True))
- for team_id, sampled in sorted(by_team.items()):
- team_exists = team_id in existing_teams
- # A deleted team's persons can outlive it in Postgres, so they are team_gone whatever their version.
- pg_versions = (
- _live_person_versions(team_id, list(sampled), "person_divergence_sample") if team_exists else {}
- )
- missing = [u for u in sampled if u not in pg_versions]
- tombstoned = _tombstoned_uuids(team_id, missing) if team_exists else set()
- for person_uuid, (ch_max_version, era) in sampled.items():
- pg_version = pg_versions.get(person_uuid)
- classification = _classify_sampled(
- pg_version, ch_max_version, team_exists=team_exists, tombstoned=person_uuid in tombstoned
- )
- counts[SampleBucket(classification=classification, era=era)] += 1
- on_sampled(
- SampledPerson(
- team_id=team_id,
- person_uuid=person_uuid,
- classification=classification,
- era=era,
- ch_max_version=ch_max_version,
- pg_version=pg_version,
- )
- )
-
- skipped = _scan_team_ranges(
- query,
- min_team_id=min_team_id,
- max_team_id=_resolve_max_team_id(max_team_id),
- team_step=team_step,
- on_rows=on_rows,
- log=log,
- )
- log(f"sampled {sampled_count} live ClickHouse persons")
- return SampleSummary(sampled=sampled_count, counts=dict(counts), skipped_team_ids=skipped)
-
-
-# The hash pre-filter keeps about this many times the sample, so the sample stays full size and uniform.
-_TEAM_CHECK_OVERSAMPLE = 10
-
-_TEAM_PERSON_ROWS_SQL = "SELECT count() FROM person WHERE team_id = %(team_id)s"
-_TEAM_MAPPING_ROWS_SQL = "SELECT count() FROM person_distinct_id2 WHERE team_id = %(team_id)s"
-
-_TEAM_PERSONS_SQL = """
-SELECT toString(id)
-FROM person
-WHERE team_id = %(team_id)s AND cityHash64(id) %% %(modulus)s = 0
-GROUP BY id
-HAVING argMax(is_deleted, version) = 0 AND max(_timestamp) < fromUnixTimestamp(%(before)s)
-ORDER BY cityHash64(id)
-LIMIT %(limit)s
-"""
-
-_TEAM_MAPPINGS_SQL = """
-SELECT distinct_id
-FROM person_distinct_id2
-WHERE team_id = %(team_id)s AND cityHash64(distinct_id) %% %(modulus)s = 0
-GROUP BY distinct_id
-HAVING argMax(is_deleted, version) = 0 AND max(_timestamp) < fromUnixTimestamp(%(before)s)
-ORDER BY cityHash64(distinct_id)
-LIMIT %(limit)s
-"""
-
-
-def _team_check_modulus(rows: int, sample_size: int) -> int:
- return max(1, rows // max(1, sample_size * _TEAM_CHECK_OVERSAMPLE))
-
-
-def _team_check_args(rows_sql: str, *, team_id: int, sample_size: int, before: datetime) -> dict[str, int]:
- [[rows]] = _ch(rows_sql, {"team_id": team_id}, _TEAM_CHECK_SETTINGS)
- return {
- "team_id": team_id,
- "before": int(before.timestamp()),
- "limit": sample_size,
- "modulus": _team_check_modulus(int(rows), sample_size),
- }
-
-
-def check_team(*, team_id: int, sample_size: int, before: datetime) -> TeamCheck:
- """Count how many of a team's old live ClickHouse persons and mappings Postgres still holds live."""
- person_args = _team_check_args(_TEAM_PERSON_ROWS_SQL, team_id=team_id, sample_size=sample_size, before=before)
- mapping_args = _team_check_args(_TEAM_MAPPING_ROWS_SQL, team_id=team_id, sample_size=sample_size, before=before)
- person_uuids = [row[0] for row in _ch(_TEAM_PERSONS_SQL, person_args, _TEAM_CHECK_SETTINGS)]
- distinct_ids = [row[0] for row in _ch(_TEAM_MAPPINGS_SQL, mapping_args, _TEAM_CHECK_SETTINGS)]
- live_persons = _live_person_versions(team_id, person_uuids, "person_divergence_team_check") if person_uuids else {}
- live_mappings = (
- personhog_call(
- "person_divergence_team_check",
- lambda: _batched_get_persons_by_distinct_ids(
- team_id,
- distinct_ids,
- "person_divergence_team_check",
- deduplicate_by_person=False,
- read_options=_VERSION_ONLY_READ_OPTIONS,
- ),
- )
- if distinct_ids
- else []
- )
- return TeamCheck(
- team_id=team_id,
- persons_sampled=len(person_uuids),
- persons_live_in_postgres=len(live_persons),
- distinct_ids_sampled=len(distinct_ids),
- distinct_ids_live_in_postgres=len({r.distinct_id for r in live_mappings}),
- )
-
-
-@frozen
-class _ChPersonState:
- visible_rows: int
- visible_max_version: int
- visible_winner_deleted: bool
- # Includes lightweight-deleted rows, so a republish lands above every row the sweep masked.
- max_version: int
-
-
-@frozen
-class _ChMappingState:
- visible_rows: int
- visible_max_version: int
- visible_winner_deleted: bool
- visible_winner_person_uuid: str
- max_version: int
-
-
-@frozen
-class _MappingPlan:
- distinct_id: str
- kind: MappingDivergenceKind | None
- pg_version: int
- ch_max_version: int | None
- target_version: int
-
-
-@frozen
-class _PersonPlan:
- team_id: int
- person_uuid: str
- person: Person | None
- kind: PersonDivergenceKind | None
- ch_max_version: int | None
- target_version: int | None
- mappings: list[_MappingPlan]
- too_many_distinct_ids: bool = False
-
-
-_PERSON_STATE_SQL = """
-SELECT
- toString(id),
- countIf(_row_exists),
- maxIf(version, _row_exists),
- argMaxIf(is_deleted, version, _row_exists),
- max(version)
-FROM person
-WHERE team_id = %(team_id)s AND id IN %(person_uuids)s
-GROUP BY id
-"""
-
-_PERSON_WINNER_PROPERTIES_SQL = """
-SELECT argMaxIf(properties, version, _row_exists)
-FROM person
-WHERE team_id = %(team_id)s AND id = %(person_uuid)s
-"""
-
-_MAPPING_STATE_SQL = """
-SELECT
- distinct_id,
- countIf(_row_exists),
- maxIf(version, _row_exists),
- argMaxIf(is_deleted, version, _row_exists),
- toString(argMaxIf(person_id, version, _row_exists)),
- max(version)
-FROM person_distinct_id2
-WHERE team_id = %(team_id)s AND distinct_id IN %(distinct_ids)s
-GROUP BY distinct_id
-"""
-
-
-def _person_kind(pg_version: int, state: _ChPersonState | None) -> PersonDivergenceKind | None:
- # Any winner that differs from Postgres counts, whatever the versions: a raise whose publish
- # never landed leaves ClickHouse below Postgres, and a rerun must still finish it.
- if state is None:
- return "absent"
- if state.visible_rows == 0:
- return "swept"
- if state.visible_winner_deleted:
- return "hidden"
- if state.visible_max_version > pg_version:
- return "stale"
- if state.visible_max_version < pg_version:
- return "behind"
- return None
-
-
-def _mapping_kind(person_uuid: str, pg_version: int, state: _ChMappingState | None) -> MappingDivergenceKind | None:
- if state is None or state.visible_rows == 0:
- return "absent"
- if state.visible_winner_deleted:
- return "hidden"
- if state.visible_winner_person_uuid != person_uuid:
- return "other_person"
- # The owner matches, but a later move of this mapping would be written below the ClickHouse winner and lost.
- if state.visible_max_version > pg_version:
- return "stale"
- return None
-
-
-def _target_version(pg_version: int, ch_max_version: int | None) -> int:
- # One above ClickHouse, not 100 above: a later Postgres write or tombstone takes the next version
- # and must outrank this row.
- return pg_version if ch_max_version is None else max(pg_version, ch_max_version + 1)
-
-
-def _ch_person_states(team_id: int, person_uuids: list[str]) -> dict[str, _ChPersonState]:
- rows = _ch(_PERSON_STATE_SQL, {"team_id": team_id, "person_uuids": person_uuids}, _REPAIR_READ_SETTINGS)
- return {
- person_uuid: _ChPersonState(
- visible_rows=int(visible_rows),
- visible_max_version=int(visible_max_version),
- visible_winner_deleted=bool(visible_winner_deleted),
- max_version=int(max_version),
- )
- for person_uuid, visible_rows, visible_max_version, visible_winner_deleted, max_version in rows
- }
-
-
-def _ch_mapping_states(team_id: int, distinct_ids: list[str]) -> dict[str, _ChMappingState]:
- states: dict[str, _ChMappingState] = {}
- for chunk in _chunks(distinct_ids, _MAPPING_QUERY_CHUNK_SIZE):
- rows = _ch(_MAPPING_STATE_SQL, {"team_id": team_id, "distinct_ids": list(chunk)}, _REPAIR_READ_SETTINGS)
- for (
- distinct_id,
- visible_rows,
- visible_max_version,
- visible_winner_deleted,
- visible_winner_person_uuid,
- max_version,
- ) in rows:
- states[distinct_id] = _ChMappingState(
- visible_rows=int(visible_rows),
- visible_max_version=int(visible_max_version),
- visible_winner_deleted=bool(visible_winner_deleted),
- visible_winner_person_uuid=visible_winner_person_uuid,
- max_version=int(max_version),
- )
- return states
-
-
-def _plan_chunk(team_id: int, person_uuids: Sequence[str], *, only_distinct_id: str | None = None) -> list[_PersonPlan]:
- live = {str(p.uuid): p for p in get_persons_by_uuids(team_id, list(person_uuids), distinct_id_limit=0)}
- person_states = _ch_person_states(team_id, list(live)) if live else {}
- distinct_ids_by_person = (
- personhog_call(
- "person_divergence_repair_distinct_ids",
- lambda: _batched_get_distinct_ids_for_persons(
- team_id, [p.pk for p in live.values()], limit_per_person=_MAX_REPAIR_DISTINCT_IDS_PER_PERSON + 1
- ),
- )
- if live
- else {}
- )
- too_many: set[int] = set()
- all_distinct_ids: list[str] = []
- for person_id, dids in list(distinct_ids_by_person.items()):
- over_cap = len(dids) > _MAX_REPAIR_DISTINCT_IDS_PER_PERSON
- if only_distinct_id is not None:
- dids = [d for d in dids if d.id == only_distinct_id]
- distinct_ids_by_person[person_id] = dids
- # The capped read can miss the requested id, which must then be reported rather than dropped.
- over_cap = over_cap and not dids
- if over_cap:
- too_many.add(person_id)
- else:
- all_distinct_ids.extend(d.id for d in dids)
- mapping_states = _ch_mapping_states(team_id, all_distinct_ids) if all_distinct_ids else {}
-
- plans: list[_PersonPlan] = []
- for person_uuid in person_uuids:
- person = live.get(person_uuid)
- if person is None:
- plans.append(
- _PersonPlan(
- team_id=team_id,
- person_uuid=person_uuid,
- person=None,
- kind=None,
- ch_max_version=None,
- target_version=None,
- mappings=[],
- )
- )
- continue
- pg_version = int(person.version or 0)
- state = person_states.get(person_uuid)
- ch_max_version = state.max_version if state is not None else None
- person_distinct_ids = distinct_ids_by_person.get(person.pk, [])
- too_many_distinct_ids = person.pk in too_many
- mappings = []
- for mapping in [] if too_many_distinct_ids else person_distinct_ids:
- mapping_state = mapping_states.get(mapping.id)
- mapping_ch_max = mapping_state.max_version if mapping_state is not None else None
- mappings.append(
- _MappingPlan(
- distinct_id=mapping.id,
- kind=_mapping_kind(person_uuid, mapping.version, mapping_state),
- pg_version=mapping.version,
- ch_max_version=mapping_ch_max,
- target_version=_target_version(mapping.version, mapping_ch_max),
- )
- )
- plans.append(
- _PersonPlan(
- team_id=team_id,
- person_uuid=person_uuid,
- person=person,
- kind=_person_kind(pg_version, state),
- ch_max_version=ch_max_version,
- target_version=_target_version(pg_version, ch_max_version),
- mappings=mappings,
- too_many_distinct_ids=too_many_distinct_ids,
- )
- )
- return plans
-
-
-def _raise_person_version_floor(team_id: int, person_id: int, min_version: int) -> None:
- personhog_call(
- "person_divergence_set_person_version_floor",
- lambda: require_personhog_client().set_person_version_floor(
- SetPersonVersionFloorRequest(team_id=team_id, person_id=person_id, min_version=min_version)
- ),
- )
-
-
-def _raise_mapping_version_floor(team_id: int, distinct_id: str, min_version: int) -> str | None:
- """Raise the mapping's version on the primary and return the uuid of the person it maps to there."""
- response = personhog_call(
- "person_divergence_set_person_distinct_id_version_floor",
- lambda: require_personhog_client().set_person_distinct_id_version_floor(
- SetPersonDistinctIdVersionFloorRequest(team_id=team_id, distinct_id=distinct_id, min_version=min_version)
- ),
- )
- if not response.HasField("person"):
- return None
- return str(UUID(response.person.uuid))
-
-
-def _primary_mapping_versions(team_id: int, person_id: int) -> dict[str, int]:
- response = personhog_call(
- "person_divergence_confirm_mapping_versions",
- lambda: require_personhog_client().get_distinct_ids_for_persons(
- GetDistinctIdsForPersonsRequest(
- team_id=team_id,
- person_ids=[person_id],
- read_options=ReadOptions(consistency=CONSISTENCY_LEVEL_STRONG),
- limit_per_person=_MAX_REPAIR_DISTINCT_IDS_PER_PERSON + 1,
- )
- ),
- )
- return {d.distinct_id: int(d.version or 0) for pd in response.person_distinct_ids for d in pd.distinct_ids}
-
-
-def _person_action(plan: _PersonPlan, outcome: RepairOutcome) -> RepairAction:
- return RepairAction(
- team_id=plan.team_id,
- person_uuid=plan.person_uuid,
- distinct_id=None,
- kind=plan.kind,
- pg_version=int(plan.person.version or 0) if plan.person is not None else None,
- ch_max_version=plan.ch_max_version,
- target_version=plan.target_version if plan.kind is not None else None,
- outcome=outcome,
- )
-
-
-def _too_many_distinct_ids_action(plan: _PersonPlan) -> RepairAction:
- return RepairAction(
- team_id=plan.team_id,
- person_uuid=plan.person_uuid,
- distinct_id=None,
- kind=None,
- pg_version=None,
- ch_max_version=None,
- target_version=None,
- outcome="skipped_too_many_distinct_ids",
- )
-
-
-def _mapping_action(plan: _PersonPlan, mapping: _MappingPlan, outcome: RepairOutcome) -> RepairAction:
- return RepairAction(
- team_id=plan.team_id,
- person_uuid=plan.person_uuid,
- distinct_id=mapping.distinct_id,
- kind=mapping.kind,
- pg_version=mapping.pg_version,
- ch_max_version=mapping.ch_max_version,
- target_version=mapping.target_version if mapping.kind is not None else None,
- outcome=outcome,
- )
-
-
-def _publish_person(team_id: int, person: Person) -> ProduceResult:
- row = _person_row(
- team_id=team_id,
- uuid=str(person.uuid),
- version=int(person.version or 0),
- properties=person.properties,
- is_identified=person.is_identified,
- is_deleted=False,
- created_at=person.created_at,
- last_seen_at=person.last_seen_at,
- )
- # _person_row fills a missing last_seen_at with the current hour, but ingestion publishes null for it.
- if person.last_seen_at is None:
- row["last_seen_at"] = None
- return ClickhouseProducer().produce(topic=KAFKA_PERSON, sql=INSERT_PERSON_SQL, data=row)
-
-
-def _reread_person(team_id: int, person_uuid: str, target_version: int | None) -> Person | None:
- """Read the person from the replica again while it shows a version below ``target_version``."""
- found = get_persons_by_uuids(team_id, [person_uuid], distinct_id_limit=0)
- reread = found[0] if found else None
- for delay in _REREAD_BACKOFF_SECONDS:
- if reread is None or target_version is None or int(reread.version or 0) >= target_version:
- break
- _sleep(delay)
- found = get_persons_by_uuids(team_id, [person_uuid], distinct_id_limit=0)
- reread = found[0] if found else None
- return reread
-
-
-def _clickhouse_only_properties(plan: _PersonPlan) -> dict[str, Any]:
- """The properties of the ClickHouse winner whose keys the Postgres person lacks."""
- assert plan.person is not None
- rows = _ch(
- _PERSON_WINNER_PROPERTIES_SQL,
- {"team_id": plan.team_id, "person_uuid": plan.person_uuid},
- _REPAIR_READ_SETTINGS,
- )
- ch_properties = json.loads(rows[0][0]) if rows and rows[0][0] else {}
- pg_properties = plan.person.properties or {}
- return {key: value for key, value in ch_properties.items() if key not in pg_properties}
-
-
-def _queue_stale_merge(plan: _PersonPlan, distinct_id: str, missing: dict[str, Any]) -> RepairOutcome | None:
- """Send the ClickHouse-only properties to ingestion as $set_once, or return why they were not sent."""
- assert plan.person is not None
- # Every member can read the $set event, and event reads apply event property restrictions only, so a team with
- # any property rule could expose a person property that some member is denied.
- if team_has_property_access_rules(team_id=plan.team_id):
- return "skipped_stale_merge_restricted"
- if len(json.dumps({**missing, **(plan.person.properties or {})})) > _STALE_MERGE_MAX_BYTES:
- return "skipped_stale_merge_too_large"
- try:
- token = Team.objects.only("api_token").get(id=plan.team_id).api_token
- capture_internal(
- token=token,
- event_name="$set",
- event_source="person_divergence_reset",
- distinct_id=distinct_id,
- timestamp=datetime.now(UTC),
- properties={"$set_once": missing},
- process_person_profile=True,
- ).raise_for_status()
- except Exception:
- return "skipped_stale_merge_failed"
- return None
-
-
-def _is_transient(exc: BaseException) -> bool:
- return isinstance(exc, (*CH_TRANSIENT_ERRORS, ClickHouseQueryTimeOut)) or is_transient_rpc_error(
- exc, codes=_TRANSIENT_RPC_CODES
- )
-
-
-def _retry_transient(fn: Callable[[], _T], *, what: str, log: Callable[[str], None]) -> _T:
- """Run ``fn`` again after a transient ClickHouse or personhog error, then re-raise once the retries run out.
-
- Repeating a step is safe because every repair write is idempotent: a version raise never lowers a
- version, and ClickHouse keeps one row per version.
- """
- for delay in _TRANSIENT_RETRY_BACKOFF_SECONDS:
- try:
- return fn()
- except Exception as exc:
- if not _is_transient(exc):
- raise
- log(f"{what}: {type(exc).__name__}, retrying in {delay:g} s")
- _sleep(delay)
- return fn()
-
-
-def _execute_plan(
- plan: _PersonPlan,
- *,
- apply: bool,
- include_stale: bool,
- before_write: Callable[[], None],
- published: Callable[[ProduceResult], None],
- stale_merge_distinct_id: str | None = None,
-) -> list[RepairAction]:
- person = plan.person
- if person is None:
- return [_person_action(plan, "skipped_not_live")]
- if plan.kind == "stale" and not include_stale:
- refusal: RepairOutcome | None = "skipped_stale"
- if apply and stale_merge_distinct_id is not None:
- missing = _clickhouse_only_properties(plan)
- # A later reset that finds nothing missing raises Postgres. Capture accepting the $set does not mean
- # ingestion applies it, and a raise without the merged properties lets the next update overwrite them.
- refusal = (
- (_queue_stale_merge(plan, stale_merge_distinct_id, missing) or "merge_queued") if missing else None
- )
- if refusal is not None:
- return [
- _person_action(plan, refusal),
- *(
- _mapping_action(plan, m, "skipped_not_divergent" if m.kind is None else "skipped_stale")
- for m in plan.mappings
- ),
- *([_too_many_distinct_ids_action(plan)] if plan.too_many_distinct_ids else []),
- ]
-
- divergent_mappings = [m for m in plan.mappings if m.kind is not None]
- mapping_actions = [_mapping_action(plan, m, "skipped_not_divergent") for m in plan.mappings if m.kind is None]
- if plan.too_many_distinct_ids:
- mapping_actions.append(_too_many_distinct_ids_action(plan))
- if not apply:
- person_outcome: RepairOutcome = "would_repair" if plan.kind is not None else "skipped_not_divergent"
- return [
- _person_action(plan, person_outcome),
- *mapping_actions,
- *(_mapping_action(plan, m, "would_repair") for m in divergent_mappings),
- ]
-
- pg_version = int(person.version or 0)
- if plan.kind is not None and plan.target_version is not None and plan.target_version > pg_version:
- before_write()
- _raise_person_version_floor(plan.team_id, person.pk, plan.target_version)
-
- # SetPersonDistinctIdVersionFloor runs on the primary, so its reply is the owner check: the replica list
- # that produced this mapping can lag a merge that moved it to another person.
- owned: list[_MappingPlan] = []
- for mapping in divergent_mappings:
- before_write()
- owner = _raise_mapping_version_floor(plan.team_id, mapping.distinct_id, mapping.target_version)
- if owner is None:
- mapping_actions.append(_mapping_action(plan, mapping, "skipped_mapping_gone"))
- elif owner != plan.person_uuid:
- mapping_actions.append(_mapping_action(plan, mapping, "skipped_owner_changed"))
- else:
- owned.append(mapping)
-
- # Re-read after the raise so the published properties are the ones Postgres holds at the
- # published version, including any ingestion update that landed after the first read.
- reread: Person | None = None
- if plan.kind is not None:
- reread = _reread_person(plan.team_id, plan.person_uuid, plan.target_version)
-
- if (plan.kind is not None or owned) and _tombstoned_uuids(plan.team_id, [plan.person_uuid]):
- person_outcome = "skipped_tombstoned" if plan.kind is not None else "skipped_not_divergent"
- return [
- _person_action(plan, person_outcome),
- *mapping_actions,
- *(_mapping_action(plan, m, "skipped_tombstoned") for m in owned),
- ]
-
- if plan.kind is None:
- person_outcome = "skipped_not_divergent"
- elif reread is None:
- person_outcome = "skipped_not_live"
- elif plan.target_version is None or int(reread.version or 0) < plan.target_version:
- # The replica has not caught up with the raise, so its properties may predate the raised version.
- person_outcome = "skipped_reread_lagging"
- else:
- published(_publish_person(plan.team_id, reread))
- person_outcome = "repaired"
-
- # The SetPersonDistinctIdVersionFloor reply does not say whether it wrote, and a personhog-replica build
- # without NULL-version handling leaves a NULL version unchanged, so publish only the mappings whose
- # version on the primary reached the target.
- stored_versions = _primary_mapping_versions(plan.team_id, person.pk) if owned else {}
- for mapping in owned:
- stored_version = stored_versions.get(mapping.distinct_id)
- if stored_version is None:
- # The primary no longer lists the mapping under this person: it moved or was tombstoned after the raise.
- mapping_actions.append(_mapping_action(plan, mapping, "skipped_mapping_gone"))
- elif stored_version < mapping.target_version:
- mapping_actions.append(_mapping_action(plan, mapping, "skipped_reread_lagging"))
- else:
- published(
- create_person_distinct_id(
- team_id=plan.team_id,
- distinct_id=mapping.distinct_id,
- person_id=plan.person_uuid,
- version=stored_version,
- is_deleted=False,
- )
- )
- mapping_actions.append(_mapping_action(plan, mapping, "repaired"))
- return [_person_action(plan, person_outcome), *mapping_actions]
-
-
-@frozen(frozen=False)
-class _WritePacer:
- """Space writes at least 1 / max_per_second apart, so idle time never buys a later burst."""
-
- max_per_second: float | None
- next_write_at: float = 0.0
-
- def before_write(self) -> None:
- if self.max_per_second is None:
- return
- now = time.monotonic()
- if now < self.next_write_at:
- time.sleep(self.next_write_at - now)
- now = self.next_write_at
- self.next_write_at = now + 1 / self.max_per_second
-
-
-@frozen(frozen=False)
-class _Deliveries:
- """Kafka produce results of the published rows, so a failed delivery counts as undelivered."""
-
- pending: list[ProduceResult] = field(default_factory=list)
- failed: int = 0
-
- def track(self, result: ProduceResult) -> None:
- self.pending.append(result)
- if len(self.pending) >= _DELIVERY_PRUNE_EVERY:
- self._prune()
-
- def undelivered(self, timeout: float) -> int:
- _flush_person_producers(timeout)
- self._prune()
- return self.failed + len(self.pending)
-
- def _prune(self) -> None:
- waiting: list[ProduceResult] = []
- for result in self.pending:
- if not result.done():
- waiting.append(result)
- continue
- try:
- result.get(timeout=0)
- except Exception:
- self.failed += 1
- self.pending = waiting
-
-
-@frozen(frozen=False)
-class _OutcomeCounts:
- person: Counter[RepairOutcome] = field(default_factory=Counter)
- mapping: Counter[RepairOutcome] = field(default_factory=Counter)
-
- def add(self, action: RepairAction) -> None:
- is_person_row = action.distinct_id is None and action.outcome != "skipped_too_many_distinct_ids"
- (self.person if is_person_row else self.mapping)[action.outcome] += 1
-
- def summary(self, *, applied: bool, persons: int, undelivered: int) -> RepairSummary:
- return RepairSummary(
- applied=applied,
- persons=persons,
- person_outcomes=dict(self.person),
- mapping_outcomes=dict(self.mapping),
- undelivered=undelivered,
- )
-
-
-def repair_persons(
- targets: Sequence[PersonRef],
- *,
- apply: bool,
- include_stale: bool = False,
- max_writes_per_second: float | None = None,
- on_action: Callable[[RepairAction], None],
- log: Callable[[str], None],
-) -> RepairSummary:
- """Republish the Postgres state of each target person and its mappings where ClickHouse disagrees.
-
- A rerun is safe: each Postgres write only raises a version, and a person whose publish never landed
- still counts as divergent. ``max_writes_per_second`` paces each Postgres write, one per divergent
- person or distinct id.
-
- ``include_stale`` also repairs stale persons and their mappings. That replaces their ClickHouse properties
- with the Postgres ones for good, so never use it on a team waiting for a restore from ClickHouse.
- """
- if max_writes_per_second is not None and max_writes_per_second <= 0:
- raise ValueError("max_writes_per_second must be above 0")
- by_team: dict[int, list[str]] = defaultdict(list)
- for target in dict.fromkeys(targets):
- by_team[target.team_id].append(target.person_uuid)
-
- outcomes = _OutcomeCounts()
- processed = 0
- undelivered = 0
- pacer = _WritePacer(max_per_second=max_writes_per_second)
- deliveries = _Deliveries()
- existing_teams = set(Team.objects.filter(id__in=list(by_team)).values_list("id", flat=True))
- try:
- for team_id, person_uuids in sorted(by_team.items()):
- if team_id not in existing_teams:
- # A deleted team's ClickHouse rows go with the team, so a repair would only republish what the deletion removed.
- for person_uuid in person_uuids:
- action = RepairAction(
- team_id=team_id,
- person_uuid=person_uuid,
- distinct_id=None,
- kind=None,
- pg_version=None,
- ch_max_version=None,
- target_version=None,
- outcome="skipped_team_gone",
- )
- outcomes.add(action)
- on_action(action)
- processed += 1
- log(f"team {team_id}: no longer exists, {len(person_uuids)} persons skipped")
- continue
- for chunk in _chunks(person_uuids, _REPAIR_CHUNK_SIZE):
- plans = _retry_transient(
- partial(_plan_chunk, team_id, chunk), what=f"team {team_id}: planning", log=log
- )
- for plan in plans:
- # Only the attempt that succeeds returns actions, so a retried person is counted once.
- actions = _retry_transient(
- partial(
- _execute_plan,
- plan,
- apply=apply,
- include_stale=include_stale,
- before_write=pacer.before_write,
- published=deliveries.track,
- ),
- what=f"team {team_id} person {plan.person_uuid}",
- log=log,
- )
- for action in actions:
- outcomes.add(action)
- on_action(action)
- processed += 1
- log(f"team {team_id}: {len(person_uuids)} persons, {processed} processed in total")
- finally:
- if apply:
- undelivered = deliveries.undelivered(_FLUSH_TIMEOUT_SECONDS)
- if undelivered:
- log(f"{undelivered} ClickHouse messages were not delivered; rerun the repair for the same input")
- return outcomes.summary(applied=apply, persons=processed, undelivered=undelivered)
-
-
-def repair_distinct_id(team_id: int, distinct_id: str, *, delivery_timeout_seconds: float) -> RepairSummary | None:
- """Repair the live person that owns ``distinct_id``, and that one mapping, where ClickHouse disagrees.
-
- It always applies. For a stale person whose ClickHouse winner holds properties Postgres lacks, it sends them
- to ingestion as $set_once, so newer Postgres values win, and writes nothing else. A later reset, once
- ingestion applied them, raises Postgres above ClickHouse and republishes. It leaves the person stale when the
- team has property access rules.
- """
- owner = get_person_by_distinct_id(team_id, distinct_id, distinct_id_limit=0)
- if owner is None:
- return None
- outcomes = _OutcomeCounts()
- deliveries = _Deliveries()
- undelivered = 0
- try:
- for plan in _plan_chunk(team_id, [str(owner.uuid)], only_distinct_id=distinct_id):
- for action in _execute_plan(
- plan,
- apply=True,
- include_stale=False,
- before_write=lambda: None,
- published=deliveries.track,
- stale_merge_distinct_id=distinct_id,
- ):
- outcomes.add(action)
- finally:
- undelivered = deliveries.undelivered(delivery_timeout_seconds)
- return outcomes.summary(applied=True, persons=1, undelivered=undelivered)
diff --git a/posthog/models/person/divergence/__init__.py b/posthog/models/person/divergence/__init__.py
new file mode 100644
index 000000000000..8389a43e5d29
--- /dev/null
+++ b/posthog/models/person/divergence/__init__.py
@@ -0,0 +1 @@
+"""Scans for persons whose ClickHouse rows disagree with Postgres, and repairs them. One module per concern."""
diff --git a/posthog/models/person/divergence/datatypes.py b/posthog/models/person/divergence/datatypes.py
new file mode 100644
index 000000000000..39ed46e31874
--- /dev/null
+++ b/posthog/models/person/divergence/datatypes.py
@@ -0,0 +1,121 @@
+"""Result types and literals shared by the divergence scans and the repair."""
+
+from __future__ import annotations
+
+from typing import Literal
+from uuid import UUID
+
+from posthog.dataclasses import frozen
+
+PersonDivergenceKind = Literal["hidden", "swept", "stale", "behind", "absent"]
+
+MappingDivergenceKind = Literal["hidden", "other_person", "stale", "absent"]
+
+SampleClassification = Literal["equal", "pg_below_ch", "pg_above_ch", "team_gone", "pg_tombstone", "pg_absent"]
+
+SampleEra = Literal["before_cutoff", "since_cutoff", "any"]
+
+RepairOutcome = Literal[
+ "would_repair",
+ "repaired",
+ "skipped_not_divergent",
+ "skipped_not_live",
+ "skipped_team_gone",
+ "skipped_tombstoned",
+ "skipped_owner_changed",
+ "skipped_mapping_gone",
+ "skipped_reread_lagging",
+ "skipped_stale",
+ "skipped_stale_merge_failed",
+ "skipped_stale_merge_too_large",
+ "skipped_stale_merge_restricted",
+ "merge_queued",
+ "skipped_too_many_distinct_ids",
+]
+
+
+@frozen
+class DivergentPerson:
+ team_id: int
+ person_uuid: str
+ kind: PersonDivergenceKind
+ ch_max_version: int
+ pg_version: int
+
+
+@frozen
+class ScanSummary:
+ candidates: int
+ divergent: int
+ skipped_team_ids: list[int]
+
+
+@frozen
+class SampledPerson:
+ team_id: int
+ person_uuid: str
+ classification: SampleClassification
+ era: SampleEra
+ ch_max_version: int
+ pg_version: int | None
+
+
+@frozen
+class SampleBucket:
+ classification: SampleClassification
+ era: SampleEra
+
+
+@frozen
+class SampleSummary:
+ sampled: int
+ counts: dict[SampleBucket, int]
+ skipped_team_ids: list[int]
+
+
+@frozen
+class TeamCheck:
+ team_id: int
+ persons_sampled: int
+ persons_live_in_postgres: int
+ distinct_ids_sampled: int
+ distinct_ids_live_in_postgres: int
+
+
+@frozen
+class PersonRef:
+ team_id: int
+ person_uuid: str
+
+ def __post_init__(self) -> None:
+ # Lookups key on the canonical form that ClickHouse toString(id) and the persons reads return.
+ if str(UUID(self.person_uuid)) != self.person_uuid:
+ raise ValueError(f"person_uuid must be a canonical lowercase UUID: {self.person_uuid!r}")
+
+
+@frozen
+class RepairAction:
+ """One person row (``distinct_id`` is None) or one of its mappings, with what the repair did to it.
+
+ A person with too many distinct ids to repair gets one extra row with no ``distinct_id`` and the
+ outcome ``skipped_too_many_distinct_ids``, which stands for all of its mappings.
+ """
+
+ team_id: int
+ person_uuid: str
+ distinct_id: str | None
+ kind: PersonDivergenceKind | MappingDivergenceKind | None
+ pg_version: int | None
+ ch_max_version: int | None
+ target_version: int | None
+ outcome: RepairOutcome
+
+
+@frozen
+class RepairSummary:
+ applied: bool
+ persons: int
+ person_outcomes: dict[RepairOutcome, int]
+ mapping_outcomes: dict[RepairOutcome, int]
+ # Published rows that Kafka did not confirm: the delivery failed, or it was still queued at the deadline.
+ undelivered: int
diff --git a/posthog/models/person/divergence/fleet_scans.py b/posthog/models/person/divergence/fleet_scans.py
new file mode 100644
index 000000000000..f0b7259b714e
--- /dev/null
+++ b/posthog/models/person/divergence/fleet_scans.py
@@ -0,0 +1,358 @@
+"""Fleet-wide divergence scans: hidden, swept, stale and the uniform sample."""
+
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from collections.abc import Callable
+from datetime import datetime
+from typing import Any
+
+from posthog.exceptions import ClickHouseQueryMemoryLimitExceeded, ClickHouseQueryTimeOut
+from posthog.models.team import Team
+
+from . import datatypes, lookups
+
+# The legacy delete path wrote ClickHouse tombstones at version + 100, so those rows sit at 100 or above.
+LEGACY_TOMBSTONE_MIN_VERSION = 100
+
+HIDDEN_TEAM_STEP = 10_000
+
+SWEPT_TEAM_STEP = 10_000
+
+STALE_TEAM_STEP = 5_000
+
+SAMPLE_TEAM_STEP = 10_000
+
+_HIDDEN_SETTINGS = {"max_execution_time": 900, "max_memory_usage": 32_000_000_000}
+
+_SWEPT_SETTINGS = {"apply_deleted_mask": 0, "max_execution_time": 1800, "max_memory_usage": 64_000_000_000}
+
+_STALE_SETTINGS = {
+ "max_execution_time": 1800,
+ "max_memory_usage": 48_000_000_000,
+ "max_bytes_before_external_group_by": 20_000_000_000,
+}
+
+_SAMPLE_SETTINGS = {"max_execution_time": 3600, "max_memory_usage": 32_000_000_000}
+
+
+def _resolve_max_team_id(max_team_id: int | None) -> int:
+ if max_team_id is not None:
+ return max_team_id
+ [[highest]] = lookups._ch("SELECT max(team_id) FROM person", {}, _HIDDEN_SETTINGS)
+ return int(highest or 0) + 1
+
+
+def _scan_team_ranges(
+ query: Callable[[int, int], list[Any]],
+ *,
+ min_team_id: int,
+ max_team_id: int,
+ team_step: int,
+ on_rows: Callable[[list[Any]], None],
+ log: Callable[[str], None],
+) -> list[int]:
+ """Return each team that runs out of memory or time even when scanned alone, so the caller can rerun it with higher limits."""
+ skipped: list[int] = []
+
+ def scan(lo: int, hi: int) -> None:
+ try:
+ rows = query(lo, hi)
+ except (ClickHouseQueryMemoryLimitExceeded, ClickHouseQueryTimeOut) as exc:
+ reason = "out of memory" if isinstance(exc, ClickHouseQueryMemoryLimitExceeded) else "timed out"
+ if hi - lo == 1:
+ log(f"team {lo}: {reason}, skipped")
+ skipped.append(lo)
+ return
+ mid = (lo + hi) // 2
+ log(f"teams [{lo}, {hi}): {reason}, bisecting")
+ scan(lo, mid)
+ scan(mid, hi)
+ return
+ on_rows(rows)
+ log(f"teams [{lo}, {hi}): {len(rows)} ClickHouse candidates")
+
+ for lo in range(min_team_id, max_team_id, team_step):
+ scan(lo, min(lo + team_step, max_team_id))
+ return skipped
+
+
+def _scan_divergent_persons(
+ *,
+ sql: str,
+ settings: dict[str, int],
+ extra_args: dict[str, Any],
+ kind: datatypes.PersonDivergenceKind,
+ divergent: Callable[[int, int], bool],
+ min_team_id: int,
+ max_team_id: int | None,
+ team_step: int,
+ on_found: Callable[[datatypes.DivergentPerson], None],
+ log: Callable[[str], None],
+) -> datatypes.ScanSummary:
+ candidates = 0
+ found = 0
+
+ def on_rows(rows: list[Any]) -> None:
+ nonlocal candidates, found
+ by_team: dict[int, dict[str, int]] = defaultdict(dict)
+ for team_id, person_uuid, ch_max_version in rows:
+ by_team[int(team_id)][person_uuid] = int(ch_max_version)
+ for team_id, ch_versions in sorted(by_team.items()):
+ candidates += len(ch_versions)
+ pg_versions = lookups._live_person_versions(team_id, list(ch_versions), f"person_divergence_scan_{kind}")
+ for person_uuid, pg_version in pg_versions.items():
+ ch_max_version = ch_versions[person_uuid]
+ if not divergent(pg_version, ch_max_version):
+ continue
+ on_found(
+ datatypes.DivergentPerson(
+ team_id=team_id,
+ person_uuid=person_uuid,
+ kind=kind,
+ ch_max_version=ch_max_version,
+ pg_version=pg_version,
+ )
+ )
+ found += 1
+
+ skipped = _scan_team_ranges(
+ lambda lo, hi: lookups._ch(sql, {"min_team_id": lo, "max_team_id": hi, **extra_args}, settings),
+ min_team_id=min_team_id,
+ max_team_id=_resolve_max_team_id(max_team_id),
+ team_step=team_step,
+ on_rows=on_rows,
+ log=log,
+ )
+ return datatypes.ScanSummary(candidates=candidates, divergent=found, skipped_team_ids=skipped)
+
+
+_HIDDEN_SQL = """
+SELECT team_id, toString(id), max(version) AS max_version
+FROM person
+WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s
+ AND (team_id, id) IN (
+ SELECT team_id, id FROM person
+ WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s
+ AND is_deleted = 1 AND version >= %(min_tombstone_version)s
+ )
+GROUP BY team_id, id
+HAVING argMax(is_deleted, version) = 1 AND max_version >= %(min_tombstone_version)s
+"""
+
+
+def scan_hidden_persons(
+ *,
+ min_team_id: int = 0,
+ max_team_id: int | None = None,
+ team_step: int = HIDDEN_TEAM_STEP,
+ on_found: Callable[[datatypes.DivergentPerson], None],
+ log: Callable[[str], None],
+) -> datatypes.ScanSummary:
+ """Persons live in Postgres whose newest ClickHouse row is a legacy (version 100 or above) tombstone.
+
+ Run it, then repair what it finds, right before every ClickHouse deletion sweep while such
+ tombstones remain: the sweep deletes every ClickHouse row of these persons.
+ """
+ return _scan_divergent_persons(
+ sql=_HIDDEN_SQL,
+ settings=_HIDDEN_SETTINGS,
+ extra_args={"min_tombstone_version": LEGACY_TOMBSTONE_MIN_VERSION},
+ kind="hidden",
+ divergent=lambda _pg_version, _ch_max_version: True,
+ min_team_id=min_team_id,
+ max_team_id=max_team_id,
+ team_step=team_step,
+ on_found=on_found,
+ log=log,
+ )
+
+
+# The ClickHouse deletion sweep deletes every row of a person whose highest-version row is a tombstone.
+# That includes a live row written after the tombstone at a lower version, so the person stays live in
+# Postgres with no visible ClickHouse row.
+_SWEPT_SQL = """
+SELECT team_id, toString(id), max(version) AS max_version
+FROM person
+WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s
+ AND (team_id, id) IN (
+ SELECT team_id, id FROM person
+ WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s
+ AND NOT _row_exists AND is_deleted = 1 AND version >= %(min_tombstone_version)s
+ )
+GROUP BY team_id, id
+HAVING countIf(_row_exists) = 0
+ AND maxIf(_timestamp, is_deleted = 0) > maxIf(_timestamp, is_deleted = 1 AND version >= %(min_tombstone_version)s)
+"""
+
+
+def scan_swept_persons(
+ *,
+ min_team_id: int = 0,
+ max_team_id: int | None = None,
+ team_step: int = SWEPT_TEAM_STEP,
+ on_found: Callable[[datatypes.DivergentPerson], None],
+ log: Callable[[str], None],
+) -> datatypes.ScanSummary:
+ """Live Postgres persons with every ClickHouse row deleted, including a live row written after a legacy tombstone.
+
+ Run it soon after a ClickHouse deletion sweep: once merges remove the deleted rows, no scan finds the person.
+ """
+ return _scan_divergent_persons(
+ sql=_SWEPT_SQL,
+ settings=_SWEPT_SETTINGS,
+ extra_args={"min_tombstone_version": LEGACY_TOMBSTONE_MIN_VERSION},
+ kind="swept",
+ divergent=lambda _pg_version, _ch_max_version: True,
+ min_team_id=min_team_id,
+ max_team_id=max_team_id,
+ team_step=team_step,
+ on_found=on_found,
+ log=log,
+ )
+
+
+# A row with a lower version that arrives well after the winning row means Postgres restarted the
+# person below ClickHouse, so every later update loses to the old winner.
+_STALE_SQL = """
+SELECT team_id, toString(id), max(version) AS max_version
+FROM person
+WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s
+ AND (team_id, id) IN (
+ SELECT team_id, id FROM person
+ WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s
+ AND _timestamp > now() - toIntervalDay(%(window_days)s)
+ )
+GROUP BY team_id, id
+HAVING argMax(is_deleted, version) = 0
+ AND argMax(version, _timestamp) < max_version
+ AND max(_timestamp) > argMax(_timestamp, version) + INTERVAL 1 HOUR
+"""
+
+
+def scan_stale_persons(
+ *,
+ window_days: int,
+ min_team_id: int = 0,
+ max_team_id: int | None = None,
+ team_step: int = STALE_TEAM_STEP,
+ on_found: Callable[[datatypes.DivergentPerson], None],
+ log: Callable[[str], None],
+) -> datatypes.ScanSummary:
+ """Persons whose live ClickHouse winner outranks Postgres, found through a late lower-version row."""
+ return _scan_divergent_persons(
+ sql=_STALE_SQL,
+ settings=_STALE_SETTINGS,
+ extra_args={"window_days": window_days},
+ kind="stale",
+ divergent=lambda pg_version, ch_max_version: pg_version < ch_max_version,
+ min_team_id=min_team_id,
+ max_team_id=max_team_id,
+ team_step=team_step,
+ on_found=on_found,
+ log=log,
+ )
+
+
+_SAMPLE_SQL = """
+SELECT team_id, toString(id), max(version) AS max_version, toUnixTimestamp(argMax(_timestamp, version)) AS written_at
+FROM person
+WHERE cityHash64(id) %% %(modulus)s = %(residue)s AND team_id >= %(min_team_id)s AND team_id < %(max_team_id)s
+GROUP BY team_id, id
+HAVING argMax(is_deleted, version) = 0
+ AND (%(written_within_days)s = 0 OR argMax(_timestamp, version) >= now() - toIntervalDay(%(written_within_days)s))
+"""
+
+
+def _classify_sampled(
+ pg_version: int | None, ch_max_version: int, *, team_exists: bool, tombstoned: bool
+) -> datatypes.SampleClassification:
+ if not team_exists:
+ return "team_gone"
+ if pg_version is None:
+ return "pg_tombstone" if tombstoned else "pg_absent"
+ if pg_version == ch_max_version:
+ return "equal"
+ return "pg_below_ch" if pg_version < ch_max_version else "pg_above_ch"
+
+
+def scan_sample(
+ *,
+ modulus: int,
+ residue: int,
+ written_within_days: int | None = None,
+ cutoff: datetime | None = None,
+ min_team_id: int = 0,
+ max_team_id: int | None = None,
+ team_step: int = SAMPLE_TEAM_STEP,
+ on_sampled: Callable[[datatypes.SampledPerson], None],
+ log: Callable[[str], None],
+) -> datatypes.SampleSummary:
+ """Classify a uniform sample of live ClickHouse persons against Postgres."""
+ if not 0 <= residue < modulus:
+ raise ValueError("residue must be in [0, modulus)")
+ cutoff_ts = cutoff.timestamp() if cutoff is not None else None
+ counts: Counter[datatypes.SampleBucket] = Counter()
+ sampled_count = 0
+
+ def query(lo: int, hi: int) -> list[Any]:
+ return lookups._ch(
+ _SAMPLE_SQL,
+ {
+ "modulus": modulus,
+ "residue": residue,
+ "written_within_days": written_within_days or 0,
+ "min_team_id": lo,
+ "max_team_id": hi,
+ },
+ _SAMPLE_SETTINGS,
+ )
+
+ def on_rows(rows: list[Any]) -> None:
+ nonlocal sampled_count
+ if not rows:
+ return
+ sampled_count += len(rows)
+ by_team: dict[int, dict[str, tuple[int, datatypes.SampleEra]]] = defaultdict(dict)
+ for team_id, person_uuid, ch_max_version, written_at in rows:
+ era: datatypes.SampleEra = (
+ "any" if cutoff_ts is None else "before_cutoff" if written_at < cutoff_ts else "since_cutoff"
+ )
+ by_team[int(team_id)][person_uuid] = (int(ch_max_version), era)
+
+ existing_teams = set(Team.objects.filter(id__in=list(by_team)).values_list("id", flat=True))
+ for team_id, sampled in sorted(by_team.items()):
+ team_exists = team_id in existing_teams
+ # A deleted team's persons can outlive it in Postgres, so they are team_gone whatever their version.
+ pg_versions = (
+ lookups._live_person_versions(team_id, list(sampled), "person_divergence_sample") if team_exists else {}
+ )
+ missing = [u for u in sampled if u not in pg_versions]
+ tombstoned = lookups._tombstoned_uuids(team_id, missing) if team_exists else set()
+ for person_uuid, (ch_max_version, era) in sampled.items():
+ pg_version = pg_versions.get(person_uuid)
+ classification = _classify_sampled(
+ pg_version, ch_max_version, team_exists=team_exists, tombstoned=person_uuid in tombstoned
+ )
+ counts[datatypes.SampleBucket(classification=classification, era=era)] += 1
+ on_sampled(
+ datatypes.SampledPerson(
+ team_id=team_id,
+ person_uuid=person_uuid,
+ classification=classification,
+ era=era,
+ ch_max_version=ch_max_version,
+ pg_version=pg_version,
+ )
+ )
+
+ skipped = _scan_team_ranges(
+ query,
+ min_team_id=min_team_id,
+ max_team_id=_resolve_max_team_id(max_team_id),
+ team_step=team_step,
+ on_rows=on_rows,
+ log=log,
+ )
+ log(f"sampled {sampled_count} live ClickHouse persons")
+ return datatypes.SampleSummary(sampled=sampled_count, counts=dict(counts), skipped_team_ids=skipped)
diff --git a/posthog/models/person/divergence/lookups.py b/posthog/models/person/divergence/lookups.py
new file mode 100644
index 000000000000..1330db8ca1da
--- /dev/null
+++ b/posthog/models/person/divergence/lookups.py
@@ -0,0 +1,43 @@
+"""ClickHouse and personhog reads shared by the scans and the repair."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any, TypeVar
+from uuid import UUID
+
+from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.workload import Workload
+from posthog.models.person.util import _batched_get_persons_by_uuids, get_person_tombstones
+from posthog.personhog_client.client import personhog_call
+from posthog.personhog_client.proto import ReadOptions
+
+_T = TypeVar("_T")
+
+
+# A scan reads only these fields, which keeps person properties out of the RPC payloads.
+_VERSION_ONLY_READ_OPTIONS = ReadOptions(field_mask=["id", "uuid", "team_id", "version"])
+
+
+def _ch(sql: str, args: dict[str, Any], settings: dict[str, int]) -> list[Any]:
+ return sync_execute(sql, args, settings=settings, workload=Workload.OFFLINE, readonly=True)
+
+
+def _live_person_versions(team_id: int, person_uuids: list[str], operation: str) -> dict[str, int]:
+ persons = personhog_call(
+ operation,
+ lambda: _batched_get_persons_by_uuids(
+ team_id, person_uuids, operation, read_options=_VERSION_ONLY_READ_OPTIONS
+ ),
+ )
+ return {str(UUID(p.uuid)): int(p.version) for p in persons}
+
+
+def _tombstoned_uuids(team_id: int, person_uuids: list[str]) -> set[str]:
+ if not person_uuids:
+ return set()
+ return {str(t.uuid) for t in get_person_tombstones(team_id, [UUID(u) for u in person_uuids])}
+
+
+def _chunks(items: Sequence[_T], size: int) -> list[Sequence[_T]]:
+ return [items[i : i + size] for i in range(0, len(items), size)]
diff --git a/posthog/models/person/divergence/repair_plan.py b/posthog/models/person/divergence/repair_plan.py
new file mode 100644
index 000000000000..f99815a6a0c7
--- /dev/null
+++ b/posthog/models/person/divergence/repair_plan.py
@@ -0,0 +1,235 @@
+"""Reads ClickHouse state for target persons and plans what the repair writes."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from posthog.dataclasses import frozen
+from posthog.models.person import Person
+from posthog.models.person.util import _batched_get_distinct_ids_for_persons, get_persons_by_uuids
+from posthog.personhog_client.client import personhog_call
+
+from . import datatypes, lookups
+
+# Repair reads see lightweight-deleted rows, so a republish lands above every row a sweep masked.
+_REPAIR_READ_SETTINGS = {"apply_deleted_mask": 0, "max_execution_time": 60, "max_memory_usage": 4_000_000_000}
+
+_MAPPING_QUERY_CHUNK_SIZE = 1_000
+
+# A person with more distinct ids keeps its person repair, and its mappings are reported for a separate plan:
+# one read of every mapping can exceed the gRPC message limit, and their paced writes would run for hours.
+_MAX_REPAIR_DISTINCT_IDS_PER_PERSON = 1_000
+
+
+@frozen
+class _ChPersonState:
+ visible_rows: int
+ visible_max_version: int
+ visible_winner_deleted: bool
+ # Includes lightweight-deleted rows, so a republish lands above every row the sweep masked.
+ max_version: int
+
+
+@frozen
+class _ChMappingState:
+ visible_rows: int
+ visible_max_version: int
+ visible_winner_deleted: bool
+ visible_winner_person_uuid: str
+ max_version: int
+
+
+@frozen
+class _MappingPlan:
+ distinct_id: str
+ kind: datatypes.MappingDivergenceKind | None
+ pg_version: int
+ ch_max_version: int | None
+ target_version: int
+
+
+@frozen
+class _PersonPlan:
+ team_id: int
+ person_uuid: str
+ person: Person | None
+ kind: datatypes.PersonDivergenceKind | None
+ ch_max_version: int | None
+ target_version: int | None
+ mappings: list[_MappingPlan]
+ too_many_distinct_ids: bool = False
+
+
+_PERSON_STATE_SQL = """
+SELECT
+ toString(id),
+ countIf(_row_exists),
+ maxIf(version, _row_exists),
+ argMaxIf(is_deleted, version, _row_exists),
+ max(version)
+FROM person
+WHERE team_id = %(team_id)s AND id IN %(person_uuids)s
+GROUP BY id
+"""
+
+_MAPPING_STATE_SQL = """
+SELECT
+ distinct_id,
+ countIf(_row_exists),
+ maxIf(version, _row_exists),
+ argMaxIf(is_deleted, version, _row_exists),
+ toString(argMaxIf(person_id, version, _row_exists)),
+ max(version)
+FROM person_distinct_id2
+WHERE team_id = %(team_id)s AND distinct_id IN %(distinct_ids)s
+GROUP BY distinct_id
+"""
+
+
+def _person_kind(pg_version: int, state: _ChPersonState | None) -> datatypes.PersonDivergenceKind | None:
+ # Any winner that differs from Postgres counts, whatever the versions: a raise whose publish
+ # never landed leaves ClickHouse below Postgres, and a rerun must still finish it.
+ if state is None:
+ return "absent"
+ if state.visible_rows == 0:
+ return "swept"
+ if state.visible_winner_deleted:
+ return "hidden"
+ if state.visible_max_version > pg_version:
+ return "stale"
+ if state.visible_max_version < pg_version:
+ return "behind"
+ return None
+
+
+def _mapping_kind(
+ person_uuid: str, pg_version: int, state: _ChMappingState | None
+) -> datatypes.MappingDivergenceKind | None:
+ if state is None or state.visible_rows == 0:
+ return "absent"
+ if state.visible_winner_deleted:
+ return "hidden"
+ if state.visible_winner_person_uuid != person_uuid:
+ return "other_person"
+ # The owner matches, but a later move of this mapping would be written below the ClickHouse winner and lost.
+ if state.visible_max_version > pg_version:
+ return "stale"
+ return None
+
+
+def _target_version(pg_version: int, ch_max_version: int | None) -> int:
+ # One above ClickHouse, not 100 above: a later Postgres write or tombstone takes the next version
+ # and must outrank this row.
+ return pg_version if ch_max_version is None else max(pg_version, ch_max_version + 1)
+
+
+def _ch_person_states(team_id: int, person_uuids: list[str]) -> dict[str, _ChPersonState]:
+ rows = lookups._ch(_PERSON_STATE_SQL, {"team_id": team_id, "person_uuids": person_uuids}, _REPAIR_READ_SETTINGS)
+ return {
+ person_uuid: _ChPersonState(
+ visible_rows=int(visible_rows),
+ visible_max_version=int(visible_max_version),
+ visible_winner_deleted=bool(visible_winner_deleted),
+ max_version=int(max_version),
+ )
+ for person_uuid, visible_rows, visible_max_version, visible_winner_deleted, max_version in rows
+ }
+
+
+def _ch_mapping_states(team_id: int, distinct_ids: list[str]) -> dict[str, _ChMappingState]:
+ states: dict[str, _ChMappingState] = {}
+ for chunk in lookups._chunks(distinct_ids, _MAPPING_QUERY_CHUNK_SIZE):
+ rows = lookups._ch(_MAPPING_STATE_SQL, {"team_id": team_id, "distinct_ids": list(chunk)}, _REPAIR_READ_SETTINGS)
+ for (
+ distinct_id,
+ visible_rows,
+ visible_max_version,
+ visible_winner_deleted,
+ visible_winner_person_uuid,
+ max_version,
+ ) in rows:
+ states[distinct_id] = _ChMappingState(
+ visible_rows=int(visible_rows),
+ visible_max_version=int(visible_max_version),
+ visible_winner_deleted=bool(visible_winner_deleted),
+ visible_winner_person_uuid=visible_winner_person_uuid,
+ max_version=int(max_version),
+ )
+ return states
+
+
+def _plan_chunk(team_id: int, person_uuids: Sequence[str], *, only_distinct_id: str | None = None) -> list[_PersonPlan]:
+ live = {str(p.uuid): p for p in get_persons_by_uuids(team_id, list(person_uuids), distinct_id_limit=0)}
+ person_states = _ch_person_states(team_id, list(live)) if live else {}
+ distinct_ids_by_person = (
+ personhog_call(
+ "person_divergence_repair_distinct_ids",
+ lambda: _batched_get_distinct_ids_for_persons(
+ team_id, [p.pk for p in live.values()], limit_per_person=_MAX_REPAIR_DISTINCT_IDS_PER_PERSON + 1
+ ),
+ )
+ if live
+ else {}
+ )
+ too_many: set[int] = set()
+ all_distinct_ids: list[str] = []
+ for person_id, dids in list(distinct_ids_by_person.items()):
+ over_cap = len(dids) > _MAX_REPAIR_DISTINCT_IDS_PER_PERSON
+ if only_distinct_id is not None:
+ dids = [d for d in dids if d.id == only_distinct_id]
+ distinct_ids_by_person[person_id] = dids
+ # The capped read can miss the requested id, which must then be reported rather than dropped.
+ over_cap = over_cap and not dids
+ if over_cap:
+ too_many.add(person_id)
+ else:
+ all_distinct_ids.extend(d.id for d in dids)
+ mapping_states = _ch_mapping_states(team_id, all_distinct_ids) if all_distinct_ids else {}
+
+ plans: list[_PersonPlan] = []
+ for person_uuid in person_uuids:
+ person = live.get(person_uuid)
+ if person is None:
+ plans.append(
+ _PersonPlan(
+ team_id=team_id,
+ person_uuid=person_uuid,
+ person=None,
+ kind=None,
+ ch_max_version=None,
+ target_version=None,
+ mappings=[],
+ )
+ )
+ continue
+ pg_version = int(person.version or 0)
+ state = person_states.get(person_uuid)
+ ch_max_version = state.max_version if state is not None else None
+ person_distinct_ids = distinct_ids_by_person.get(person.pk, [])
+ too_many_distinct_ids = person.pk in too_many
+ mappings = []
+ for mapping in [] if too_many_distinct_ids else person_distinct_ids:
+ mapping_state = mapping_states.get(mapping.id)
+ mapping_ch_max = mapping_state.max_version if mapping_state is not None else None
+ mappings.append(
+ _MappingPlan(
+ distinct_id=mapping.id,
+ kind=_mapping_kind(person_uuid, mapping.version, mapping_state),
+ pg_version=mapping.version,
+ ch_max_version=mapping_ch_max,
+ target_version=_target_version(mapping.version, mapping_ch_max),
+ )
+ )
+ plans.append(
+ _PersonPlan(
+ team_id=team_id,
+ person_uuid=person_uuid,
+ person=person,
+ kind=_person_kind(pg_version, state),
+ ch_max_version=ch_max_version,
+ target_version=_target_version(pg_version, ch_max_version),
+ mappings=mappings,
+ too_many_distinct_ids=too_many_distinct_ids,
+ )
+ )
+ return plans
diff --git a/posthog/models/person/divergence/repairs.py b/posthog/models/person/divergence/repairs.py
new file mode 100644
index 000000000000..3db3b3e8215e
--- /dev/null
+++ b/posthog/models/person/divergence/repairs.py
@@ -0,0 +1,484 @@
+"""Executes repair plans: version raises, publishes, pacing, retries and delivery tracking."""
+
+from __future__ import annotations
+
+import time
+from collections import Counter, defaultdict
+from collections.abc import Callable, Sequence
+from dataclasses import field
+from functools import partial
+from typing import TypeVar
+from uuid import UUID
+
+import grpc
+
+from posthog.dataclasses import frozen
+from posthog.errors import CH_TRANSIENT_ERRORS
+from posthog.exceptions import ClickHouseQueryTimeOut
+from posthog.kafka_client.client import ClickhouseProducer, ProduceResult
+from posthog.kafka_client.topics import KAFKA_PERSON
+from posthog.models.person import Person
+from posthog.models.person.sql import INSERT_PERSON_SQL
+from posthog.models.person.util import (
+ _flush_person_producers,
+ _person_row,
+ create_person_distinct_id,
+ get_person_by_distinct_id,
+ get_persons_by_uuids,
+)
+from posthog.models.team import Team
+from posthog.personhog_client.client import personhog_call, require_personhog_client
+from posthog.personhog_client.interceptor import is_transient_rpc_error
+from posthog.personhog_client.proto import (
+ CONSISTENCY_LEVEL_STRONG,
+ GetDistinctIdsForPersonsRequest,
+ ReadOptions,
+ SetPersonDistinctIdVersionFloorRequest,
+ SetPersonVersionFloorRequest,
+)
+
+from . import datatypes, lookups, repair_plan, stale_merge
+
+_T = TypeVar("_T")
+
+
+_REPAIR_CHUNK_SIZE = 100
+
+# The waits between re-reads give the replica about 0.5 s to show a raise, far above its usual lag of a few milliseconds.
+_REREAD_BACKOFF_SECONDS = (0.025, 0.05, 0.1, 0.15, 0.175)
+
+_TRANSIENT_RETRY_BACKOFF_SECONDS = (1.0, 2.0)
+
+_TRANSIENT_RPC_CODES = frozenset(
+ {grpc.StatusCode.INTERNAL, grpc.StatusCode.UNAVAILABLE, grpc.StatusCode.DEADLINE_EXCEEDED}
+)
+
+_FLUSH_TIMEOUT_SECONDS = 5 * 60
+
+# Confirmed produce results are dropped this often, so a long repair does not hold one per published row.
+_DELIVERY_PRUNE_EVERY = 1_000
+
+_sleep = time.sleep
+
+
+def _raise_person_version_floor(team_id: int, person_id: int, min_version: int) -> None:
+ personhog_call(
+ "person_divergence_set_person_version_floor",
+ lambda: require_personhog_client().set_person_version_floor(
+ SetPersonVersionFloorRequest(team_id=team_id, person_id=person_id, min_version=min_version)
+ ),
+ )
+
+
+def _raise_mapping_version_floor(team_id: int, distinct_id: str, min_version: int) -> str | None:
+ """Raise the mapping's version on the primary and return the uuid of the person it maps to there."""
+ response = personhog_call(
+ "person_divergence_set_person_distinct_id_version_floor",
+ lambda: require_personhog_client().set_person_distinct_id_version_floor(
+ SetPersonDistinctIdVersionFloorRequest(team_id=team_id, distinct_id=distinct_id, min_version=min_version)
+ ),
+ )
+ if not response.HasField("person"):
+ return None
+ return str(UUID(response.person.uuid))
+
+
+def _primary_mapping_versions(team_id: int, person_id: int) -> dict[str, int]:
+ response = personhog_call(
+ "person_divergence_confirm_mapping_versions",
+ lambda: require_personhog_client().get_distinct_ids_for_persons(
+ GetDistinctIdsForPersonsRequest(
+ team_id=team_id,
+ person_ids=[person_id],
+ read_options=ReadOptions(consistency=CONSISTENCY_LEVEL_STRONG),
+ limit_per_person=repair_plan._MAX_REPAIR_DISTINCT_IDS_PER_PERSON + 1,
+ )
+ ),
+ )
+ return {d.distinct_id: int(d.version or 0) for pd in response.person_distinct_ids for d in pd.distinct_ids}
+
+
+def _person_action(plan: repair_plan._PersonPlan, outcome: datatypes.RepairOutcome) -> datatypes.RepairAction:
+ return datatypes.RepairAction(
+ team_id=plan.team_id,
+ person_uuid=plan.person_uuid,
+ distinct_id=None,
+ kind=plan.kind,
+ pg_version=int(plan.person.version or 0) if plan.person is not None else None,
+ ch_max_version=plan.ch_max_version,
+ target_version=plan.target_version if plan.kind is not None else None,
+ outcome=outcome,
+ )
+
+
+def _too_many_distinct_ids_action(plan: repair_plan._PersonPlan) -> datatypes.RepairAction:
+ return datatypes.RepairAction(
+ team_id=plan.team_id,
+ person_uuid=plan.person_uuid,
+ distinct_id=None,
+ kind=None,
+ pg_version=None,
+ ch_max_version=None,
+ target_version=None,
+ outcome="skipped_too_many_distinct_ids",
+ )
+
+
+def _mapping_action(
+ plan: repair_plan._PersonPlan, mapping: repair_plan._MappingPlan, outcome: datatypes.RepairOutcome
+) -> datatypes.RepairAction:
+ return datatypes.RepairAction(
+ team_id=plan.team_id,
+ person_uuid=plan.person_uuid,
+ distinct_id=mapping.distinct_id,
+ kind=mapping.kind,
+ pg_version=mapping.pg_version,
+ ch_max_version=mapping.ch_max_version,
+ target_version=mapping.target_version if mapping.kind is not None else None,
+ outcome=outcome,
+ )
+
+
+def _publish_person(team_id: int, person: Person) -> ProduceResult:
+ row = _person_row(
+ team_id=team_id,
+ uuid=str(person.uuid),
+ version=int(person.version or 0),
+ properties=person.properties,
+ is_identified=person.is_identified,
+ is_deleted=False,
+ created_at=person.created_at,
+ last_seen_at=person.last_seen_at,
+ )
+ # _person_row fills a missing last_seen_at with the current hour, but ingestion publishes null for it.
+ if person.last_seen_at is None:
+ row["last_seen_at"] = None
+ return ClickhouseProducer().produce(topic=KAFKA_PERSON, sql=INSERT_PERSON_SQL, data=row)
+
+
+def _reread_person(team_id: int, person_uuid: str, target_version: int | None) -> Person | None:
+ """Read the person from the replica again while it shows a version below ``target_version``."""
+ found = get_persons_by_uuids(team_id, [person_uuid], distinct_id_limit=0)
+ reread = found[0] if found else None
+ for delay in _REREAD_BACKOFF_SECONDS:
+ if reread is None or target_version is None or int(reread.version or 0) >= target_version:
+ break
+ _sleep(delay)
+ found = get_persons_by_uuids(team_id, [person_uuid], distinct_id_limit=0)
+ reread = found[0] if found else None
+ return reread
+
+
+def _is_transient(exc: BaseException) -> bool:
+ return isinstance(exc, (*CH_TRANSIENT_ERRORS, ClickHouseQueryTimeOut)) or is_transient_rpc_error(
+ exc, codes=_TRANSIENT_RPC_CODES
+ )
+
+
+def _retry_transient(fn: Callable[[], _T], *, what: str, log: Callable[[str], None]) -> _T:
+ """Run ``fn`` again after a transient ClickHouse or personhog error, then re-raise once the retries run out.
+
+ Repeating a step is safe because every repair write is idempotent: a version raise never lowers a
+ version, and ClickHouse keeps one row per version.
+ """
+ for delay in _TRANSIENT_RETRY_BACKOFF_SECONDS:
+ try:
+ return fn()
+ except Exception as exc:
+ if not _is_transient(exc):
+ raise
+ log(f"{what}: {type(exc).__name__}, retrying in {delay:g} s")
+ _sleep(delay)
+ return fn()
+
+
+def _execute_plan(
+ plan: repair_plan._PersonPlan,
+ *,
+ apply: bool,
+ include_stale: bool,
+ before_write: Callable[[], None],
+ published: Callable[[ProduceResult], None],
+ stale_merge_distinct_id: str | None = None,
+) -> list[datatypes.RepairAction]:
+ person = plan.person
+ if person is None:
+ return [_person_action(plan, "skipped_not_live")]
+ if plan.kind == "stale" and not include_stale:
+ refusal: datatypes.RepairOutcome | None = "skipped_stale"
+ if apply and stale_merge_distinct_id is not None:
+ missing = stale_merge._clickhouse_only_properties(plan)
+ # A later reset that finds nothing missing raises Postgres. Capture accepting the $set does not mean
+ # ingestion applies it, and a raise without the merged properties lets the next update overwrite them.
+ refusal = (
+ (stale_merge._queue_stale_merge(plan, stale_merge_distinct_id, missing) or "merge_queued")
+ if missing
+ else None
+ )
+ if refusal is not None:
+ return [
+ _person_action(plan, refusal),
+ *(
+ _mapping_action(plan, m, "skipped_not_divergent" if m.kind is None else "skipped_stale")
+ for m in plan.mappings
+ ),
+ *([_too_many_distinct_ids_action(plan)] if plan.too_many_distinct_ids else []),
+ ]
+
+ divergent_mappings = [m for m in plan.mappings if m.kind is not None]
+ mapping_actions = [_mapping_action(plan, m, "skipped_not_divergent") for m in plan.mappings if m.kind is None]
+ if plan.too_many_distinct_ids:
+ mapping_actions.append(_too_many_distinct_ids_action(plan))
+ if not apply:
+ person_outcome: datatypes.RepairOutcome = "would_repair" if plan.kind is not None else "skipped_not_divergent"
+ return [
+ _person_action(plan, person_outcome),
+ *mapping_actions,
+ *(_mapping_action(plan, m, "would_repair") for m in divergent_mappings),
+ ]
+
+ pg_version = int(person.version or 0)
+ if plan.kind is not None and plan.target_version is not None and plan.target_version > pg_version:
+ before_write()
+ _raise_person_version_floor(plan.team_id, person.pk, plan.target_version)
+
+ # SetPersonDistinctIdVersionFloor runs on the primary, so its reply is the owner check: the replica list
+ # that produced this mapping can lag a merge that moved it to another person.
+ owned: list[repair_plan._MappingPlan] = []
+ for mapping in divergent_mappings:
+ before_write()
+ owner = _raise_mapping_version_floor(plan.team_id, mapping.distinct_id, mapping.target_version)
+ if owner is None:
+ mapping_actions.append(_mapping_action(plan, mapping, "skipped_mapping_gone"))
+ elif owner != plan.person_uuid:
+ mapping_actions.append(_mapping_action(plan, mapping, "skipped_owner_changed"))
+ else:
+ owned.append(mapping)
+
+ # Re-read after the raise so the published properties are the ones Postgres holds at the
+ # published version, including any ingestion update that landed after the first read.
+ reread: Person | None = None
+ if plan.kind is not None:
+ reread = _reread_person(plan.team_id, plan.person_uuid, plan.target_version)
+
+ if (plan.kind is not None or owned) and lookups._tombstoned_uuids(plan.team_id, [plan.person_uuid]):
+ person_outcome = "skipped_tombstoned" if plan.kind is not None else "skipped_not_divergent"
+ return [
+ _person_action(plan, person_outcome),
+ *mapping_actions,
+ *(_mapping_action(plan, m, "skipped_tombstoned") for m in owned),
+ ]
+
+ if plan.kind is None:
+ person_outcome = "skipped_not_divergent"
+ elif reread is None:
+ person_outcome = "skipped_not_live"
+ elif plan.target_version is None or int(reread.version or 0) < plan.target_version:
+ # The replica has not caught up with the raise, so its properties may predate the raised version.
+ person_outcome = "skipped_reread_lagging"
+ else:
+ published(_publish_person(plan.team_id, reread))
+ person_outcome = "repaired"
+
+ # The SetPersonDistinctIdVersionFloor reply does not say whether it wrote, and a personhog-replica build
+ # without NULL-version handling leaves a NULL version unchanged, so publish only the mappings whose
+ # version on the primary reached the target.
+ stored_versions = _primary_mapping_versions(plan.team_id, person.pk) if owned else {}
+ for mapping in owned:
+ stored_version = stored_versions.get(mapping.distinct_id)
+ if stored_version is None:
+ # The primary no longer lists the mapping under this person: it moved or was tombstoned after the raise.
+ mapping_actions.append(_mapping_action(plan, mapping, "skipped_mapping_gone"))
+ elif stored_version < mapping.target_version:
+ mapping_actions.append(_mapping_action(plan, mapping, "skipped_reread_lagging"))
+ else:
+ published(
+ create_person_distinct_id(
+ team_id=plan.team_id,
+ distinct_id=mapping.distinct_id,
+ person_id=plan.person_uuid,
+ version=stored_version,
+ is_deleted=False,
+ )
+ )
+ mapping_actions.append(_mapping_action(plan, mapping, "repaired"))
+ return [_person_action(plan, person_outcome), *mapping_actions]
+
+
+@frozen(frozen=False)
+class _WritePacer:
+ """Space writes at least 1 / max_per_second apart, so idle time never buys a later burst."""
+
+ max_per_second: float | None
+ next_write_at: float = 0.0
+
+ def before_write(self) -> None:
+ if self.max_per_second is None:
+ return
+ now = time.monotonic()
+ if now < self.next_write_at:
+ time.sleep(self.next_write_at - now)
+ now = self.next_write_at
+ self.next_write_at = now + 1 / self.max_per_second
+
+
+@frozen(frozen=False)
+class _Deliveries:
+ """Kafka produce results of the published rows, so a failed delivery counts as undelivered."""
+
+ pending: list[ProduceResult] = field(default_factory=list)
+ failed: int = 0
+
+ def track(self, result: ProduceResult) -> None:
+ self.pending.append(result)
+ if len(self.pending) >= _DELIVERY_PRUNE_EVERY:
+ self._prune()
+
+ def undelivered(self, timeout: float) -> int:
+ _flush_person_producers(timeout)
+ self._prune()
+ return self.failed + len(self.pending)
+
+ def _prune(self) -> None:
+ waiting: list[ProduceResult] = []
+ for result in self.pending:
+ if not result.done():
+ waiting.append(result)
+ continue
+ try:
+ result.get(timeout=0)
+ except Exception:
+ self.failed += 1
+ self.pending = waiting
+
+
+@frozen(frozen=False)
+class _OutcomeCounts:
+ person: Counter[datatypes.RepairOutcome] = field(default_factory=Counter)
+ mapping: Counter[datatypes.RepairOutcome] = field(default_factory=Counter)
+
+ def add(self, action: datatypes.RepairAction) -> None:
+ is_person_row = action.distinct_id is None and action.outcome != "skipped_too_many_distinct_ids"
+ (self.person if is_person_row else self.mapping)[action.outcome] += 1
+
+ def summary(self, *, applied: bool, persons: int, undelivered: int) -> datatypes.RepairSummary:
+ return datatypes.RepairSummary(
+ applied=applied,
+ persons=persons,
+ person_outcomes=dict(self.person),
+ mapping_outcomes=dict(self.mapping),
+ undelivered=undelivered,
+ )
+
+
+def repair_persons(
+ targets: Sequence[datatypes.PersonRef],
+ *,
+ apply: bool,
+ include_stale: bool = False,
+ max_writes_per_second: float | None = None,
+ on_action: Callable[[datatypes.RepairAction], None],
+ log: Callable[[str], None],
+) -> datatypes.RepairSummary:
+ """Republish the Postgres state of each target person and its mappings where ClickHouse disagrees.
+
+ A rerun is safe: each Postgres write only raises a version, and a person whose publish never landed
+ still counts as divergent. ``max_writes_per_second`` paces each Postgres write, one per divergent
+ person or distinct id.
+
+ ``include_stale`` also repairs stale persons and their mappings. That replaces their ClickHouse properties
+ with the Postgres ones for good, so never use it on a team waiting for a restore from ClickHouse.
+ """
+ if max_writes_per_second is not None and max_writes_per_second <= 0:
+ raise ValueError("max_writes_per_second must be above 0")
+ by_team: dict[int, list[str]] = defaultdict(list)
+ for target in dict.fromkeys(targets):
+ by_team[target.team_id].append(target.person_uuid)
+
+ outcomes = _OutcomeCounts()
+ processed = 0
+ undelivered = 0
+ pacer = _WritePacer(max_per_second=max_writes_per_second)
+ deliveries = _Deliveries()
+ existing_teams = set(Team.objects.filter(id__in=list(by_team)).values_list("id", flat=True))
+ try:
+ for team_id, person_uuids in sorted(by_team.items()):
+ if team_id not in existing_teams:
+ # A deleted team's ClickHouse rows go with the team, so a repair would only republish what the deletion removed.
+ for person_uuid in person_uuids:
+ action = datatypes.RepairAction(
+ team_id=team_id,
+ person_uuid=person_uuid,
+ distinct_id=None,
+ kind=None,
+ pg_version=None,
+ ch_max_version=None,
+ target_version=None,
+ outcome="skipped_team_gone",
+ )
+ outcomes.add(action)
+ on_action(action)
+ processed += 1
+ log(f"team {team_id}: no longer exists, {len(person_uuids)} persons skipped")
+ continue
+ for chunk in lookups._chunks(person_uuids, _REPAIR_CHUNK_SIZE):
+ plans = _retry_transient(
+ partial(repair_plan._plan_chunk, team_id, chunk), what=f"team {team_id}: planning", log=log
+ )
+ for plan in plans:
+ # Only the attempt that succeeds returns actions, so a retried person is counted once.
+ actions = _retry_transient(
+ partial(
+ _execute_plan,
+ plan,
+ apply=apply,
+ include_stale=include_stale,
+ before_write=pacer.before_write,
+ published=deliveries.track,
+ ),
+ what=f"team {team_id} person {plan.person_uuid}",
+ log=log,
+ )
+ for action in actions:
+ outcomes.add(action)
+ on_action(action)
+ processed += 1
+ log(f"team {team_id}: {len(person_uuids)} persons, {processed} processed in total")
+ finally:
+ if apply:
+ undelivered = deliveries.undelivered(_FLUSH_TIMEOUT_SECONDS)
+ if undelivered:
+ log(f"{undelivered} ClickHouse messages were not delivered; rerun the repair for the same input")
+ return outcomes.summary(applied=apply, persons=processed, undelivered=undelivered)
+
+
+def repair_distinct_id(
+ team_id: int, distinct_id: str, *, delivery_timeout_seconds: float
+) -> datatypes.RepairSummary | None:
+ """Repair the live person that owns ``distinct_id``, and that one mapping, where ClickHouse disagrees.
+
+ It always applies. For a stale person whose ClickHouse winner holds properties Postgres lacks, it sends them
+ to ingestion as $set_once, so newer Postgres values win, and writes nothing else. A later reset, once
+ ingestion applied them, raises Postgres above ClickHouse and republishes. It leaves the person stale when the
+ team has property access rules.
+ """
+ owner = get_person_by_distinct_id(team_id, distinct_id, distinct_id_limit=0)
+ if owner is None:
+ return None
+ outcomes = _OutcomeCounts()
+ deliveries = _Deliveries()
+ undelivered = 0
+ try:
+ for plan in repair_plan._plan_chunk(team_id, [str(owner.uuid)], only_distinct_id=distinct_id):
+ for action in _execute_plan(
+ plan,
+ apply=True,
+ include_stale=False,
+ before_write=lambda: None,
+ published=deliveries.track,
+ stale_merge_distinct_id=distinct_id,
+ ):
+ outcomes.add(action)
+ finally:
+ undelivered = deliveries.undelivered(delivery_timeout_seconds)
+ return outcomes.summary(applied=True, persons=1, undelivered=undelivered)
diff --git a/posthog/models/person/divergence/stale_merge.py b/posthog/models/person/divergence/stale_merge.py
new file mode 100644
index 000000000000..e2e726881173
--- /dev/null
+++ b/posthog/models/person/divergence/stale_merge.py
@@ -0,0 +1,64 @@
+"""Sends a stale person's ClickHouse-only properties to ingestion before Postgres is raised."""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from typing import Any
+
+from posthog.api.capture import capture_internal
+from posthog.models.team import Team
+
+from products.access_control.backend.facade.api import team_has_property_access_rules
+
+from . import datatypes, lookups, repair_plan
+
+# Well under capture's 1,000,000-byte Kafka message limit and ingestion's 655,360-byte person properties limit
+# (trimmed to 512 KiB), so ingestion applies the merge whole.
+_STALE_MERGE_MAX_BYTES = 256 * 1024
+
+_PERSON_WINNER_PROPERTIES_SQL = """
+SELECT argMaxIf(properties, version, _row_exists)
+FROM person
+WHERE team_id = %(team_id)s AND id = %(person_uuid)s
+"""
+
+
+def _clickhouse_only_properties(plan: repair_plan._PersonPlan) -> dict[str, Any]:
+ """The properties of the ClickHouse winner whose keys the Postgres person lacks."""
+ assert plan.person is not None
+ rows = lookups._ch(
+ _PERSON_WINNER_PROPERTIES_SQL,
+ {"team_id": plan.team_id, "person_uuid": plan.person_uuid},
+ repair_plan._REPAIR_READ_SETTINGS,
+ )
+ ch_properties = json.loads(rows[0][0]) if rows and rows[0][0] else {}
+ pg_properties = plan.person.properties or {}
+ return {key: value for key, value in ch_properties.items() if key not in pg_properties}
+
+
+def _queue_stale_merge(
+ plan: repair_plan._PersonPlan, distinct_id: str, missing: dict[str, Any]
+) -> datatypes.RepairOutcome | None:
+ """Send the ClickHouse-only properties to ingestion as $set_once, or return why they were not sent."""
+ assert plan.person is not None
+ # Every member can read the $set event, and event reads apply event property restrictions only, so a team with
+ # any property rule could expose a person property that some member is denied.
+ if team_has_property_access_rules(team_id=plan.team_id):
+ return "skipped_stale_merge_restricted"
+ if len(json.dumps({**missing, **(plan.person.properties or {})})) > _STALE_MERGE_MAX_BYTES:
+ return "skipped_stale_merge_too_large"
+ try:
+ token = Team.objects.only("api_token").get(id=plan.team_id).api_token
+ capture_internal(
+ token=token,
+ event_name="$set",
+ event_source="person_divergence_reset",
+ distinct_id=distinct_id,
+ timestamp=datetime.now(UTC),
+ properties={"$set_once": missing},
+ process_person_profile=True,
+ ).raise_for_status()
+ except Exception:
+ return "skipped_stale_merge_failed"
+ return None
diff --git a/posthog/models/person/divergence/team_check.py b/posthog/models/person/divergence/team_check.py
new file mode 100644
index 000000000000..74411a6f7888
--- /dev/null
+++ b/posthog/models/person/divergence/team_check.py
@@ -0,0 +1,85 @@
+"""Per-team damage gauge: how many old ClickHouse persons and mappings Postgres still holds."""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from posthog.models.person.util import _batched_get_persons_by_distinct_ids
+from posthog.personhog_client.client import personhog_call
+
+from . import datatypes, lookups
+
+_TEAM_CHECK_SETTINGS = {"max_execution_time": 900, "max_memory_usage": 32_000_000_000}
+
+# The hash pre-filter keeps about this many times the sample, so the sample stays full size and uniform.
+_TEAM_CHECK_OVERSAMPLE = 10
+
+_TEAM_PERSON_ROWS_SQL = "SELECT count() FROM person WHERE team_id = %(team_id)s"
+
+_TEAM_MAPPING_ROWS_SQL = "SELECT count() FROM person_distinct_id2 WHERE team_id = %(team_id)s"
+
+_TEAM_PERSONS_SQL = """
+SELECT toString(id)
+FROM person
+WHERE team_id = %(team_id)s AND cityHash64(id) %% %(modulus)s = 0
+GROUP BY id
+HAVING argMax(is_deleted, version) = 0 AND max(_timestamp) < fromUnixTimestamp(%(before)s)
+ORDER BY cityHash64(id)
+LIMIT %(limit)s
+"""
+
+_TEAM_MAPPINGS_SQL = """
+SELECT distinct_id
+FROM person_distinct_id2
+WHERE team_id = %(team_id)s AND cityHash64(distinct_id) %% %(modulus)s = 0
+GROUP BY distinct_id
+HAVING argMax(is_deleted, version) = 0 AND max(_timestamp) < fromUnixTimestamp(%(before)s)
+ORDER BY cityHash64(distinct_id)
+LIMIT %(limit)s
+"""
+
+
+def _team_check_modulus(rows: int, sample_size: int) -> int:
+ return max(1, rows // max(1, sample_size * _TEAM_CHECK_OVERSAMPLE))
+
+
+def _team_check_args(rows_sql: str, *, team_id: int, sample_size: int, before: datetime) -> dict[str, int]:
+ [[rows]] = lookups._ch(rows_sql, {"team_id": team_id}, _TEAM_CHECK_SETTINGS)
+ return {
+ "team_id": team_id,
+ "before": int(before.timestamp()),
+ "limit": sample_size,
+ "modulus": _team_check_modulus(int(rows), sample_size),
+ }
+
+
+def check_team(*, team_id: int, sample_size: int, before: datetime) -> datatypes.TeamCheck:
+ """Count how many of a team's old live ClickHouse persons and mappings Postgres still holds live."""
+ person_args = _team_check_args(_TEAM_PERSON_ROWS_SQL, team_id=team_id, sample_size=sample_size, before=before)
+ mapping_args = _team_check_args(_TEAM_MAPPING_ROWS_SQL, team_id=team_id, sample_size=sample_size, before=before)
+ person_uuids = [row[0] for row in lookups._ch(_TEAM_PERSONS_SQL, person_args, _TEAM_CHECK_SETTINGS)]
+ distinct_ids = [row[0] for row in lookups._ch(_TEAM_MAPPINGS_SQL, mapping_args, _TEAM_CHECK_SETTINGS)]
+ live_persons = (
+ lookups._live_person_versions(team_id, person_uuids, "person_divergence_team_check") if person_uuids else {}
+ )
+ live_mappings = (
+ personhog_call(
+ "person_divergence_team_check",
+ lambda: _batched_get_persons_by_distinct_ids(
+ team_id,
+ distinct_ids,
+ "person_divergence_team_check",
+ deduplicate_by_person=False,
+ read_options=lookups._VERSION_ONLY_READ_OPTIONS,
+ ),
+ )
+ if distinct_ids
+ else []
+ )
+ return datatypes.TeamCheck(
+ team_id=team_id,
+ persons_sampled=len(person_uuids),
+ persons_live_in_postgres=len(live_persons),
+ distinct_ids_sampled=len(distinct_ids),
+ distinct_ids_live_in_postgres=len({r.distinct_id for r in live_mappings}),
+ )
diff --git a/posthog/models/person/test/test_divergence.py b/posthog/models/person/test/test_divergence.py
index 9fc18888c3a4..d250b7e2f1a8 100644
--- a/posthog/models/person/test/test_divergence.py
+++ b/posthog/models/person/test/test_divergence.py
@@ -21,7 +21,7 @@ from posthog.kafka_client.client import ClickhouseProducer, ProduceResult
from posthog.kafka_client.topics import KAFKA_PERSON
from posthog.models import OrganizationMembership, PropertyDefinition, Team, User
from posthog.models.person import Person
-from posthog.models.person.divergence import (
+from posthog.models.person.divergence.datatypes import (
DivergentPerson,
MappingDivergenceKind,
PersonDivergenceKind,
@@ -34,16 +34,15 @@ from posthog.models.person.divergence import (
SampleSummary,
ScanSummary,
TeamCheck,
- _team_check_modulus,
- _WritePacer,
- check_team,
- repair_distinct_id,
- repair_persons,
+)
+from posthog.models.person.divergence.fleet_scans import (
scan_hidden_persons,
scan_sample,
scan_stale_persons,
scan_swept_persons,
)
+from posthog.models.person.divergence.repairs import _WritePacer, repair_distinct_id, repair_persons
+from posthog.models.person.divergence.team_check import _team_check_modulus, check_team
from posthog.models.person.sql import BULK_INSERT_PERSON_DISTINCT_ID2
from posthog.models.person.util import (
create_person as create_person_in_ch,
@@ -515,7 +514,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
self._ch_person_row(person.uuid, 103, deleted=True)
self._ch_mapping_row("throttled-did", person.uuid, 100, deleted=True)
- with patch("posthog.models.person.divergence.time") as clock:
+ with patch("posthog.models.person.divergence.repairs.time") as clock:
clock.monotonic.return_value = 0.0
repair_persons(
[PersonRef(team_id=self.team.pk, person_uuid=str(person.uuid))],
@@ -592,7 +591,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
with (
patch.object(fake, "set_person_version_floor", side_effect=primary_only_raise),
patch.object(fake, "get_persons_by_uuids", side_effect=lagging_replica),
- patch("posthog.models.person.divergence._sleep") as sleep,
+ patch("posthog.models.person.divergence.repairs._sleep") as sleep,
):
_, actions = self._repair(person.uuid)
@@ -616,7 +615,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
person = self._divergent_person("hidden")
failure = _rpc_error(error) if isinstance(error, grpc.StatusCode) else error
- with self._failing_once(step, failure), patch("posthog.models.person.divergence._sleep") as sleep:
+ with self._failing_once(step, failure), patch("posthog.models.person.divergence.repairs._sleep") as sleep:
summary, actions = self._repair(person.uuid)
assert [a.outcome for a in actions] == ["repaired"]
@@ -638,7 +637,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
with (
patch.object(fake, "set_person_version_floor", side_effect=failure) as raise_floor,
- patch("posthog.models.person.divergence._sleep") as sleep,
+ patch("posthog.models.person.divergence.repairs._sleep") as sleep,
):
with self.assertRaises(type(failure)):
self._repair(person.uuid)
@@ -658,7 +657,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
raise failure
return query(*args, **kwargs)
- return patch("posthog.models.person.divergence.sync_execute", side_effect=ch_fails_once)
+ return patch("posthog.models.person.divergence.lookups.sync_execute", side_effect=ch_fails_once)
raise_floor = fake.set_person_version_floor
calls: list[bool] = []
@@ -882,7 +881,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
for distinct_id in distinct_ids:
self._ch_mapping_row(distinct_id, person.uuid, 100, deleted=True)
- with patch("posthog.models.person.divergence._MAX_REPAIR_DISTINCT_IDS_PER_PERSON", cap):
+ with patch("posthog.models.person.divergence.repair_plan._MAX_REPAIR_DISTINCT_IDS_PER_PERSON", cap):
summary = repair_distinct_id(self.team.pk, "target", delivery_timeout_seconds=1)
assert summary is not None
@@ -921,7 +920,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
stored = get_active_fake().stored_person(self.team.pk, str(person.uuid))
assert stored is not None
- with patch("posthog.models.person.divergence.capture_internal") as capture:
+ with patch("posthog.models.person.divergence.stale_merge.capture_internal") as capture:
first = repair_distinct_id(self.team.pk, "target", delivery_timeout_seconds=1)
assert first is not None
@@ -943,7 +942,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
assert self._ch_person(person.uuid) == (0, 10, {**CH_PROPERTIES, **CH_ONLY_PROPERTIES})
stored.properties = json.dumps({**CH_ONLY_PROPERTIES, **PG_PROPERTIES}).encode()
- with patch("posthog.models.person.divergence.capture_internal") as capture:
+ with patch("posthog.models.person.divergence.stale_merge.capture_internal") as capture:
second = repair_distinct_id(self.team.pk, "target", delivery_timeout_seconds=1)
assert second is not None
@@ -956,7 +955,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
def test_reset_republishes_a_stale_person_whose_clickhouse_properties_postgres_already_holds(self) -> None:
person = self._stale_person_with_target(CH_PROPERTIES)
- with patch("posthog.models.person.divergence.capture_internal") as capture:
+ with patch("posthog.models.person.divergence.stale_merge.capture_internal") as capture:
summary = repair_distinct_id(self.team.pk, "target", delivery_timeout_seconds=1)
assert summary is not None
@@ -981,8 +980,8 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
failing.raise_for_status.side_effect = RuntimeError("capture rejected the event")
with (
- patch("posthog.models.person.divergence._STALE_MERGE_MAX_BYTES", max_bytes),
- patch("posthog.models.person.divergence.capture_internal", return_value=failing) as capture,
+ patch("posthog.models.person.divergence.stale_merge._STALE_MERGE_MAX_BYTES", max_bytes),
+ patch("posthog.models.person.divergence.stale_merge.capture_internal", return_value=failing) as capture,
):
summary = repair_distinct_id(self.team.pk, "target", delivery_timeout_seconds=1)
@@ -999,7 +998,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
self._ch_person_row(person.uuid, 103, deleted=True)
self._ch_mapping_row("a", person.uuid, 100, deleted=True)
- with patch("posthog.models.person.divergence._MAX_REPAIR_DISTINCT_IDS_PER_PERSON", 2):
+ with patch("posthog.models.person.divergence.repair_plan._MAX_REPAIR_DISTINCT_IDS_PER_PERSON", 2):
summary, actions = self._repair(person.uuid)
assert actions == [
@@ -1079,7 +1078,7 @@ class TestWritePacer(SimpleTestCase):
sleeps.append(seconds)
clock[0] += seconds
- with patch("posthog.models.person.divergence.time") as time_module:
+ with patch("posthog.models.person.divergence.repairs.time") as time_module:
time_module.monotonic.side_effect = lambda: clock[0]
time_module.sleep.side_effect = sleep
pacer = _WritePacer(max_per_second=2)
@@ -1113,7 +1112,7 @@ class TestScanTeamRanges(SimpleTestCase):
scanned.extend(teams)
return []
- with patch("posthog.models.person.divergence.sync_execute", side_effect=query):
+ with patch("posthog.models.person.divergence.lookups.sync_execute", side_effect=query):
if scan == "stale":
summary: ScanSummary | SampleSummary = scan_stale_persons(
window_days=60,