:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(persons): split the divergence module into a package (#117388)
Eli Reisman01c3b352f2d7
diff --git a/posthog/api/person.py b/posthog/api/person.pyindex 687fe08ee4ce..33a14be5fc84 100644--- a/posthog/api/person.py+++ b/posthog/api/person.py@@ -61,7 +61,7 @@ from posthog.models.person.bulk_delete import (queue_person_recording_deletion,resolve_persons_for_deletion,)-from posthog.models.person.divergence import repair_distinct_id+from posthog.models.person.divergence.repairs import repair_distinct_idfrom posthog.models.person.missing_person import MissingPersonfrom posthog.models.person.util import (get_distinct_ids_for_persons,diff --git a/posthog/api/test/test_person.py b/posthog/api/test/test_person.pyindex 51f0dbcccc83..fdc340bf2b06 100644--- a/posthog/api/test/test_person.py+++ b/posthog/api/test/test_person.py@@ -28,7 +28,8 @@ from parameterized import parameterizedfrom rest_framework import statusfrom rest_framework.exceptions import ValidationError-import posthog.models.person.divergence+import posthog.models.person.divergence.repairs+import posthog.models.person.divergence.stale_mergefrom posthog.api.person import tag_client_query_idfrom posthog.clickhouse.client import sync_executefrom posthog.clickhouse.query_tagging import get_query_tag_value, reset_query_tags@@ -2418,8 +2419,8 @@ class TestPerson(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):assert person_after.version is not None and person_after.version > 105@mock.patch(- f"{posthog.models.person.divergence.__name__}.create_person_distinct_id",- wraps=posthog.models.person.divergence.create_person_distinct_id,+ f"{posthog.models.person.divergence.repairs.__name__}.create_person_distinct_id",+ wraps=posthog.models.person.divergence.repairs.create_person_distinct_id,)@pytest.mark.flaky(reruns=2)def test_reset_person_distinct_id_not_found(self, mocked_ch_call):@@ -2698,7 +2699,7 @@ class TestPersonBatchRestrictedProperties(ClickhouseTestMixin, APIBaseTest):team=self.team, property_definition=restricted, access_level=PropertyAccessLevel.NONE.value)- @mock.patch(f"{posthog.models.person.divergence.__name__}.capture_internal")+ @mock.patch(f"{posthog.models.person.divergence.stale_merge.__name__}.capture_internal")def test_reset_does_not_copy_a_restricted_property_into_an_event(self, capture: mock.MagicMock) -> None:person = create_person(team=self.team, properties={"email": "[email protected]"}, version=5, uuid=str(uuid4()))add_distinct_id(person=person, distinct_id="target", version=0)diff --git a/posthog/management/commands/person_divergence.py b/posthog/management/commands/person_divergence.pyindex c2a9174d9e0a..e201c2a601b5 100644--- a/posthog/management/commands/person_divergence.py+++ b/posthog/management/commands/person_divergence.py@@ -27,24 +27,26 @@ from uuid import UUIDfrom django.core.management.base import BaseCommand, CommandError, CommandParserfrom posthog.clickhouse.query_tagging import Feature, Product, tags_context-from posthog.models.person.divergence import (- HIDDEN_TEAM_STEP,- SAMPLE_TEAM_STEP,- STALE_TEAM_STEP,- SWEPT_TEAM_STEP,+from posthog.models.person.divergence.datatypes import (DivergentPerson,PersonRef,RepairAction,SampledPerson,ScanSummary,TeamCheck,- check_team,- repair_persons,+)+from posthog.models.person.divergence.fleet_scans import (+ HIDDEN_TEAM_STEP,+ SAMPLE_TEAM_STEP,+ STALE_TEAM_STEP,+ SWEPT_TEAM_STEP,scan_hidden_persons,scan_sample,scan_stale_persons,scan_swept_persons,)+from posthog.models.person.divergence.repairs import repair_persons+from posthog.models.person.divergence.team_check import check_team_DIVERGENT_SCANS: dict[str, tuple[Callable[..., ScanSummary], int, str]] = {"hidden": (diff --git a/posthog/models/person/divergence.py b/posthog/models/person/divergence.pydeleted file mode 100644index 7c53d0a848ad..000000000000--- a/posthog/models/person/divergence.py+++ /dev/null@@ -1,1299 +0,0 @@-"""Find and repair persons whose ClickHouse rows disagree with the persons database.--The persons database, read through personhog, is the source of truth. Scans only read. A repair-republishes the Postgres state of a person and its distinct ids to ClickHouse. When ClickHouse-holds a version at or above the Postgres one, the repair first raises the Postgres version above-it, so the published row wins and the next ingestion update still outranks it.-"""--from __future__ import annotations--import json-import time-from collections import Counter, defaultdict-from collections.abc import Callable, Sequence-from dataclasses import field-from datetime import UTC, datetime-from functools import partial-from typing import Any, Literal, TypeVar-from uuid import UUID--import grpc--from posthog.api.capture import capture_internal-from posthog.clickhouse.client import sync_execute-from posthog.clickhouse.workload import Workload-from posthog.dataclasses import frozen-from posthog.errors import CH_TRANSIENT_ERRORS-from posthog.exceptions import ClickHouseQueryMemoryLimitExceeded, ClickHouseQueryTimeOut-from posthog.kafka_client.client import ClickhouseProducer, ProduceResult-from posthog.kafka_client.topics import KAFKA_PERSON-from posthog.models.person import Person-from posthog.models.person.sql import INSERT_PERSON_SQL-from posthog.models.person.util import (- _batched_get_distinct_ids_for_persons,- _batched_get_persons_by_distinct_ids,- _batched_get_persons_by_uuids,- _flush_person_producers,- _person_row,- create_person_distinct_id,- get_person_by_distinct_id,- get_person_tombstones,- get_persons_by_uuids,-)-from posthog.models.team import Team-from posthog.personhog_client.client import personhog_call, require_personhog_client-from posthog.personhog_client.interceptor import is_transient_rpc_error-from posthog.personhog_client.proto import (- CONSISTENCY_LEVEL_STRONG,- GetDistinctIdsForPersonsRequest,- ReadOptions,- SetPersonDistinctIdVersionFloorRequest,- SetPersonVersionFloorRequest,-)--from products.access_control.backend.facade.api import team_has_property_access_rules--PersonDivergenceKind = Literal["hidden", "swept", "stale", "behind", "absent"]-MappingDivergenceKind = Literal["hidden", "other_person", "stale", "absent"]-SampleClassification = Literal["equal", "pg_below_ch", "pg_above_ch", "team_gone", "pg_tombstone", "pg_absent"]-SampleEra = Literal["before_cutoff", "since_cutoff", "any"]-RepairOutcome = Literal[- "would_repair",- "repaired",- "skipped_not_divergent",- "skipped_not_live",- "skipped_team_gone",- "skipped_tombstoned",- "skipped_owner_changed",- "skipped_mapping_gone",- "skipped_reread_lagging",- "skipped_stale",- "skipped_stale_merge_failed",- "skipped_stale_merge_too_large",- "skipped_stale_merge_restricted",- "merge_queued",- "skipped_too_many_distinct_ids",-]--# The legacy delete path wrote ClickHouse tombstones at version + 100, so those rows sit at 100 or above.-LEGACY_TOMBSTONE_MIN_VERSION = 100--HIDDEN_TEAM_STEP = 10_000-SWEPT_TEAM_STEP = 10_000-STALE_TEAM_STEP = 5_000-SAMPLE_TEAM_STEP = 10_000--_HIDDEN_SETTINGS = {"max_execution_time": 900, "max_memory_usage": 32_000_000_000}-_SWEPT_SETTINGS = {"apply_deleted_mask": 0, "max_execution_time": 1800, "max_memory_usage": 64_000_000_000}-_STALE_SETTINGS = {- "max_execution_time": 1800,- "max_memory_usage": 48_000_000_000,- "max_bytes_before_external_group_by": 20_000_000_000,-}-_SAMPLE_SETTINGS = {"max_execution_time": 3600, "max_memory_usage": 32_000_000_000}-_TEAM_CHECK_SETTINGS = {"max_execution_time": 900, "max_memory_usage": 32_000_000_000}-# Repair reads see lightweight-deleted rows, so a republish lands above every row a sweep masked.-_REPAIR_READ_SETTINGS = {"apply_deleted_mask": 0, "max_execution_time": 60, "max_memory_usage": 4_000_000_000}--# A scan reads only these fields, which keeps person properties out of the RPC payloads.-_VERSION_ONLY_READ_OPTIONS = ReadOptions(field_mask=["id", "uuid", "team_id", "version"])--_REPAIR_CHUNK_SIZE = 100-_MAPPING_QUERY_CHUNK_SIZE = 1_000-# A person with more distinct ids keeps its person repair, and its mappings are reported for a separate plan:-# one read of every mapping can exceed the gRPC message limit, and their paced writes would run for hours.-_MAX_REPAIR_DISTINCT_IDS_PER_PERSON = 1_000-# The waits between re-reads give the replica about 0.5 s to show a raise, far above its usual lag of a few milliseconds.-_REREAD_BACKOFF_SECONDS = (0.025, 0.05, 0.1, 0.15, 0.175)-# Well under capture's 1,000,000-byte Kafka message limit and ingestion's 655,360-byte person properties limit-# (trimmed to 512 KiB), so ingestion applies the merge whole.-_STALE_MERGE_MAX_BYTES = 256 * 1024-_TRANSIENT_RETRY_BACKOFF_SECONDS = (1.0, 2.0)-_TRANSIENT_RPC_CODES = frozenset(- {grpc.StatusCode.INTERNAL, grpc.StatusCode.UNAVAILABLE, grpc.StatusCode.DEADLINE_EXCEEDED}-)-_FLUSH_TIMEOUT_SECONDS = 5 * 60-# Confirmed produce results are dropped this often, so a long repair does not hold one per published row.-_DELIVERY_PRUNE_EVERY = 1_000--_T = TypeVar("_T")--_sleep = time.sleep---@frozen-class DivergentPerson:- team_id: int- person_uuid: str- kind: PersonDivergenceKind- ch_max_version: int- pg_version: int---@frozen-class ScanSummary:- candidates: int- divergent: int- skipped_team_ids: list[int]---@frozen-class SampledPerson:- team_id: int- person_uuid: str- classification: SampleClassification- era: SampleEra- ch_max_version: int- pg_version: int | None---@frozen-class SampleBucket:- classification: SampleClassification- era: SampleEra---@frozen-class SampleSummary:- sampled: int- counts: dict[SampleBucket, int]- skipped_team_ids: list[int]---@frozen-class TeamCheck:- team_id: int- persons_sampled: int- persons_live_in_postgres: int- distinct_ids_sampled: int- distinct_ids_live_in_postgres: int---@frozen-class PersonRef:- team_id: int- person_uuid: str-- def __post_init__(self) -> None:- # Lookups key on the canonical form that ClickHouse toString(id) and the persons reads return.- if str(UUID(self.person_uuid)) != self.person_uuid:- raise ValueError(f"person_uuid must be a canonical lowercase UUID: {self.person_uuid!r}")---@frozen-class RepairAction:- """One person row (``distinct_id`` is None) or one of its mappings, with what the repair did to it.-- A person with too many distinct ids to repair gets one extra row with no ``distinct_id`` and the- outcome ``skipped_too_many_distinct_ids``, which stands for all of its mappings.- """-- team_id: int- person_uuid: str- distinct_id: str | None- kind: PersonDivergenceKind | MappingDivergenceKind | None- pg_version: int | None- ch_max_version: int | None- target_version: int | None- outcome: RepairOutcome---@frozen-class RepairSummary:- applied: bool- persons: int- person_outcomes: dict[RepairOutcome, int]- mapping_outcomes: dict[RepairOutcome, int]- # Published rows that Kafka did not confirm: the delivery failed, or it was still queued at the deadline.- undelivered: int---def _ch(sql: str, args: dict[str, Any], settings: dict[str, int]) -> list[Any]:- return sync_execute(sql, args, settings=settings, workload=Workload.OFFLINE, readonly=True)---def _live_person_versions(team_id: int, person_uuids: list[str], operation: str) -> dict[str, int]:- persons = personhog_call(- operation,- lambda: _batched_get_persons_by_uuids(- team_id, person_uuids, operation, read_options=_VERSION_ONLY_READ_OPTIONS- ),- )- return {str(UUID(p.uuid)): int(p.version) for p in persons}---def _tombstoned_uuids(team_id: int, person_uuids: list[str]) -> set[str]:- if not person_uuids:- return set()- return {str(t.uuid) for t in get_person_tombstones(team_id, [UUID(u) for u in person_uuids])}---def _resolve_max_team_id(max_team_id: int | None) -> int:- if max_team_id is not None:- return max_team_id- [[highest]] = _ch("SELECT max(team_id) FROM person", {}, _HIDDEN_SETTINGS)- return int(highest or 0) + 1---def _chunks(items: Sequence[_T], size: int) -> list[Sequence[_T]]:- return [items[i : i + size] for i in range(0, len(items), size)]---def _scan_team_ranges(- query: Callable[[int, int], list[Any]],- *,- min_team_id: int,- max_team_id: int,- team_step: int,- on_rows: Callable[[list[Any]], None],- log: Callable[[str], None],-) -> list[int]:- """Return each team that runs out of memory or time even when scanned alone, so the caller can rerun it with higher limits."""- skipped: list[int] = []-- def scan(lo: int, hi: int) -> None:- try:- rows = query(lo, hi)- except (ClickHouseQueryMemoryLimitExceeded, ClickHouseQueryTimeOut) as exc:- reason = "out of memory" if isinstance(exc, ClickHouseQueryMemoryLimitExceeded) else "timed out"- if hi - lo == 1:- log(f"team {lo}: {reason}, skipped")- skipped.append(lo)- return- mid = (lo + hi) // 2- log(f"teams [{lo}, {hi}): {reason}, bisecting")- scan(lo, mid)- scan(mid, hi)- return- on_rows(rows)- log(f"teams [{lo}, {hi}): {len(rows)} ClickHouse candidates")-- for lo in range(min_team_id, max_team_id, team_step):- scan(lo, min(lo + team_step, max_team_id))- return skipped---def _scan_divergent_persons(- *,- sql: str,- settings: dict[str, int],- extra_args: dict[str, Any],- kind: PersonDivergenceKind,- divergent: Callable[[int, int], bool],- min_team_id: int,- max_team_id: int | None,- team_step: int,- on_found: Callable[[DivergentPerson], None],- log: Callable[[str], None],-) -> ScanSummary:- candidates = 0- found = 0-- def on_rows(rows: list[Any]) -> None:- nonlocal candidates, found- by_team: dict[int, dict[str, int]] = defaultdict(dict)- for team_id, person_uuid, ch_max_version in rows:- by_team[int(team_id)][person_uuid] = int(ch_max_version)- for team_id, ch_versions in sorted(by_team.items()):- candidates += len(ch_versions)- pg_versions = _live_person_versions(team_id, list(ch_versions), f"person_divergence_scan_{kind}")- for person_uuid, pg_version in pg_versions.items():- ch_max_version = ch_versions[person_uuid]- if not divergent(pg_version, ch_max_version):- continue- on_found(- DivergentPerson(- team_id=team_id,- person_uuid=person_uuid,- kind=kind,- ch_max_version=ch_max_version,- pg_version=pg_version,- )- )- found += 1-- skipped = _scan_team_ranges(- lambda lo, hi: _ch(sql, {"min_team_id": lo, "max_team_id": hi, **extra_args}, settings),- min_team_id=min_team_id,- max_team_id=_resolve_max_team_id(max_team_id),- team_step=team_step,- on_rows=on_rows,- log=log,- )- return ScanSummary(candidates=candidates, divergent=found, skipped_team_ids=skipped)---_HIDDEN_SQL = """-SELECT team_id, toString(id), max(version) AS max_version-FROM person-WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s- AND (team_id, id) IN (- SELECT team_id, id FROM person- WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s- AND is_deleted = 1 AND version >= %(min_tombstone_version)s- )-GROUP BY team_id, id-HAVING argMax(is_deleted, version) = 1 AND max_version >= %(min_tombstone_version)s-"""---def scan_hidden_persons(- *,- min_team_id: int = 0,- max_team_id: int | None = None,- team_step: int = HIDDEN_TEAM_STEP,- on_found: Callable[[DivergentPerson], None],- log: Callable[[str], None],-) -> ScanSummary:- """Persons live in Postgres whose newest ClickHouse row is a legacy (version 100 or above) tombstone.-- Run it, then repair what it finds, right before every ClickHouse deletion sweep while such- tombstones remain: the sweep deletes every ClickHouse row of these persons.- """- return _scan_divergent_persons(- sql=_HIDDEN_SQL,- settings=_HIDDEN_SETTINGS,- extra_args={"min_tombstone_version": LEGACY_TOMBSTONE_MIN_VERSION},- kind="hidden",- divergent=lambda _pg_version, _ch_max_version: True,- min_team_id=min_team_id,- max_team_id=max_team_id,- team_step=team_step,- on_found=on_found,- log=log,- )---# The ClickHouse deletion sweep deletes every row of a person whose highest-version row is a tombstone.-# That includes a live row written after the tombstone at a lower version, so the person stays live in-# Postgres with no visible ClickHouse row.-_SWEPT_SQL = """-SELECT team_id, toString(id), max(version) AS max_version-FROM person-WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s- AND (team_id, id) IN (- SELECT team_id, id FROM person- WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s- AND NOT _row_exists AND is_deleted = 1 AND version >= %(min_tombstone_version)s- )-GROUP BY team_id, id-HAVING countIf(_row_exists) = 0- AND maxIf(_timestamp, is_deleted = 0) > maxIf(_timestamp, is_deleted = 1 AND version >= %(min_tombstone_version)s)-"""---def scan_swept_persons(- *,- min_team_id: int = 0,- max_team_id: int | None = None,- team_step: int = SWEPT_TEAM_STEP,- on_found: Callable[[DivergentPerson], None],- log: Callable[[str], None],-) -> ScanSummary:- """Live Postgres persons with every ClickHouse row deleted, including a live row written after a legacy tombstone.-- Run it soon after a ClickHouse deletion sweep: once merges remove the deleted rows, no scan finds the person.- """- return _scan_divergent_persons(- sql=_SWEPT_SQL,- settings=_SWEPT_SETTINGS,- extra_args={"min_tombstone_version": LEGACY_TOMBSTONE_MIN_VERSION},- kind="swept",- divergent=lambda _pg_version, _ch_max_version: True,- min_team_id=min_team_id,- max_team_id=max_team_id,- team_step=team_step,- on_found=on_found,- log=log,- )---# A row with a lower version that arrives well after the winning row means Postgres restarted the-# person below ClickHouse, so every later update loses to the old winner.-_STALE_SQL = """-SELECT team_id, toString(id), max(version) AS max_version-FROM person-WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s- AND (team_id, id) IN (- SELECT team_id, id FROM person- WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s- AND _timestamp > now() - toIntervalDay(%(window_days)s)- )-GROUP BY team_id, id-HAVING argMax(is_deleted, version) = 0- AND argMax(version, _timestamp) < max_version- AND max(_timestamp) > argMax(_timestamp, version) + INTERVAL 1 HOUR-"""---def scan_stale_persons(- *,- window_days: int,- min_team_id: int = 0,- max_team_id: int | None = None,- team_step: int = STALE_TEAM_STEP,- on_found: Callable[[DivergentPerson], None],- log: Callable[[str], None],-) -> ScanSummary:- """Persons whose live ClickHouse winner outranks Postgres, found through a late lower-version row."""- return _scan_divergent_persons(- sql=_STALE_SQL,- settings=_STALE_SETTINGS,- extra_args={"window_days": window_days},- kind="stale",- divergent=lambda pg_version, ch_max_version: pg_version < ch_max_version,- min_team_id=min_team_id,- max_team_id=max_team_id,- team_step=team_step,- on_found=on_found,- log=log,- )---_SAMPLE_SQL = """-SELECT team_id, toString(id), max(version) AS max_version, toUnixTimestamp(argMax(_timestamp, version)) AS written_at-FROM person-WHERE cityHash64(id) %% %(modulus)s = %(residue)s AND team_id >= %(min_team_id)s AND team_id < %(max_team_id)s-GROUP BY team_id, id-HAVING argMax(is_deleted, version) = 0- AND (%(written_within_days)s = 0 OR argMax(_timestamp, version) >= now() - toIntervalDay(%(written_within_days)s))-"""---def _classify_sampled(- pg_version: int | None, ch_max_version: int, *, team_exists: bool, tombstoned: bool-) -> SampleClassification:- if not team_exists:- return "team_gone"- if pg_version is None:- return "pg_tombstone" if tombstoned else "pg_absent"- if pg_version == ch_max_version:- return "equal"- return "pg_below_ch" if pg_version < ch_max_version else "pg_above_ch"---def scan_sample(- *,- modulus: int,- residue: int,- written_within_days: int | None = None,- cutoff: datetime | None = None,- min_team_id: int = 0,- max_team_id: int | None = None,- team_step: int = SAMPLE_TEAM_STEP,- on_sampled: Callable[[SampledPerson], None],- log: Callable[[str], None],-) -> SampleSummary:- """Classify a uniform sample of live ClickHouse persons against Postgres."""- if not 0 <= residue < modulus:- raise ValueError("residue must be in [0, modulus)")- cutoff_ts = cutoff.timestamp() if cutoff is not None else None- counts: Counter[SampleBucket] = Counter()- sampled_count = 0-- def query(lo: int, hi: int) -> list[Any]:- return _ch(- _SAMPLE_SQL,- {- "modulus": modulus,- "residue": residue,- "written_within_days": written_within_days or 0,- "min_team_id": lo,- "max_team_id": hi,- },- _SAMPLE_SETTINGS,- )-- def on_rows(rows: list[Any]) -> None:- nonlocal sampled_count- if not rows:- return- sampled_count += len(rows)- by_team: dict[int, dict[str, tuple[int, SampleEra]]] = defaultdict(dict)- for team_id, person_uuid, ch_max_version, written_at in rows:- era: SampleEra = (- "any" if cutoff_ts is None else "before_cutoff" if written_at < cutoff_ts else "since_cutoff"- )- by_team[int(team_id)][person_uuid] = (int(ch_max_version), era)-- existing_teams = set(Team.objects.filter(id__in=list(by_team)).values_list("id", flat=True))- for team_id, sampled in sorted(by_team.items()):- team_exists = team_id in existing_teams- # A deleted team's persons can outlive it in Postgres, so they are team_gone whatever their version.- pg_versions = (- _live_person_versions(team_id, list(sampled), "person_divergence_sample") if team_exists else {}- )- missing = [u for u in sampled if u not in pg_versions]- tombstoned = _tombstoned_uuids(team_id, missing) if team_exists else set()- for person_uuid, (ch_max_version, era) in sampled.items():- pg_version = pg_versions.get(person_uuid)- classification = _classify_sampled(- pg_version, ch_max_version, team_exists=team_exists, tombstoned=person_uuid in tombstoned- )- counts[SampleBucket(classification=classification, era=era)] += 1- on_sampled(- SampledPerson(- team_id=team_id,- person_uuid=person_uuid,- classification=classification,- era=era,- ch_max_version=ch_max_version,- pg_version=pg_version,- )- )-- skipped = _scan_team_ranges(- query,- min_team_id=min_team_id,- max_team_id=_resolve_max_team_id(max_team_id),- team_step=team_step,- on_rows=on_rows,- log=log,- )- log(f"sampled {sampled_count} live ClickHouse persons")- return SampleSummary(sampled=sampled_count, counts=dict(counts), skipped_team_ids=skipped)---# The hash pre-filter keeps about this many times the sample, so the sample stays full size and uniform.-_TEAM_CHECK_OVERSAMPLE = 10--_TEAM_PERSON_ROWS_SQL = "SELECT count() FROM person WHERE team_id = %(team_id)s"-_TEAM_MAPPING_ROWS_SQL = "SELECT count() FROM person_distinct_id2 WHERE team_id = %(team_id)s"--_TEAM_PERSONS_SQL = """-SELECT toString(id)-FROM person-WHERE team_id = %(team_id)s AND cityHash64(id) %% %(modulus)s = 0-GROUP BY id-HAVING argMax(is_deleted, version) = 0 AND max(_timestamp) < fromUnixTimestamp(%(before)s)-ORDER BY cityHash64(id)-LIMIT %(limit)s-"""--_TEAM_MAPPINGS_SQL = """-SELECT distinct_id-FROM person_distinct_id2-WHERE team_id = %(team_id)s AND cityHash64(distinct_id) %% %(modulus)s = 0-GROUP BY distinct_id-HAVING argMax(is_deleted, version) = 0 AND max(_timestamp) < fromUnixTimestamp(%(before)s)-ORDER BY cityHash64(distinct_id)-LIMIT %(limit)s-"""---def _team_check_modulus(rows: int, sample_size: int) -> int:- return max(1, rows // max(1, sample_size * _TEAM_CHECK_OVERSAMPLE))---def _team_check_args(rows_sql: str, *, team_id: int, sample_size: int, before: datetime) -> dict[str, int]:- [[rows]] = _ch(rows_sql, {"team_id": team_id}, _TEAM_CHECK_SETTINGS)- return {- "team_id": team_id,- "before": int(before.timestamp()),- "limit": sample_size,- "modulus": _team_check_modulus(int(rows), sample_size),- }---def check_team(*, team_id: int, sample_size: int, before: datetime) -> TeamCheck:- """Count how many of a team's old live ClickHouse persons and mappings Postgres still holds live."""- person_args = _team_check_args(_TEAM_PERSON_ROWS_SQL, team_id=team_id, sample_size=sample_size, before=before)- mapping_args = _team_check_args(_TEAM_MAPPING_ROWS_SQL, team_id=team_id, sample_size=sample_size, before=before)- person_uuids = [row[0] for row in _ch(_TEAM_PERSONS_SQL, person_args, _TEAM_CHECK_SETTINGS)]- distinct_ids = [row[0] for row in _ch(_TEAM_MAPPINGS_SQL, mapping_args, _TEAM_CHECK_SETTINGS)]- live_persons = _live_person_versions(team_id, person_uuids, "person_divergence_team_check") if person_uuids else {}- live_mappings = (- personhog_call(- "person_divergence_team_check",- lambda: _batched_get_persons_by_distinct_ids(- team_id,- distinct_ids,- "person_divergence_team_check",- deduplicate_by_person=False,- read_options=_VERSION_ONLY_READ_OPTIONS,- ),- )- if distinct_ids- else []- )- return TeamCheck(- team_id=team_id,- persons_sampled=len(person_uuids),- persons_live_in_postgres=len(live_persons),- distinct_ids_sampled=len(distinct_ids),- distinct_ids_live_in_postgres=len({r.distinct_id for r in live_mappings}),- )---@frozen-class _ChPersonState:- visible_rows: int- visible_max_version: int- visible_winner_deleted: bool- # Includes lightweight-deleted rows, so a republish lands above every row the sweep masked.- max_version: int---@frozen-class _ChMappingState:- visible_rows: int- visible_max_version: int- visible_winner_deleted: bool- visible_winner_person_uuid: str- max_version: int---@frozen-class _MappingPlan:- distinct_id: str- kind: MappingDivergenceKind | None- pg_version: int- ch_max_version: int | None- target_version: int---@frozen-class _PersonPlan:- team_id: int- person_uuid: str- person: Person | None- kind: PersonDivergenceKind | None- ch_max_version: int | None- target_version: int | None- mappings: list[_MappingPlan]- too_many_distinct_ids: bool = False---_PERSON_STATE_SQL = """-SELECT- toString(id),- countIf(_row_exists),- maxIf(version, _row_exists),- argMaxIf(is_deleted, version, _row_exists),- max(version)-FROM person-WHERE team_id = %(team_id)s AND id IN %(person_uuids)s-GROUP BY id-"""--_PERSON_WINNER_PROPERTIES_SQL = """-SELECT argMaxIf(properties, version, _row_exists)-FROM person-WHERE team_id = %(team_id)s AND id = %(person_uuid)s-"""--_MAPPING_STATE_SQL = """-SELECT- distinct_id,- countIf(_row_exists),- maxIf(version, _row_exists),- argMaxIf(is_deleted, version, _row_exists),- toString(argMaxIf(person_id, version, _row_exists)),- max(version)-FROM person_distinct_id2-WHERE team_id = %(team_id)s AND distinct_id IN %(distinct_ids)s-GROUP BY distinct_id-"""---def _person_kind(pg_version: int, state: _ChPersonState | None) -> PersonDivergenceKind | None:- # Any winner that differs from Postgres counts, whatever the versions: a raise whose publish- # never landed leaves ClickHouse below Postgres, and a rerun must still finish it.- if state is None:- return "absent"- if state.visible_rows == 0:- return "swept"- if state.visible_winner_deleted:- return "hidden"- if state.visible_max_version > pg_version:- return "stale"- if state.visible_max_version < pg_version:- return "behind"- return None---def _mapping_kind(person_uuid: str, pg_version: int, state: _ChMappingState | None) -> MappingDivergenceKind | None:- if state is None or state.visible_rows == 0:- return "absent"- if state.visible_winner_deleted:- return "hidden"- if state.visible_winner_person_uuid != person_uuid:- return "other_person"- # The owner matches, but a later move of this mapping would be written below the ClickHouse winner and lost.- if state.visible_max_version > pg_version:- return "stale"- return None---def _target_version(pg_version: int, ch_max_version: int | None) -> int:- # One above ClickHouse, not 100 above: a later Postgres write or tombstone takes the next version- # and must outrank this row.- return pg_version if ch_max_version is None else max(pg_version, ch_max_version + 1)---def _ch_person_states(team_id: int, person_uuids: list[str]) -> dict[str, _ChPersonState]:- rows = _ch(_PERSON_STATE_SQL, {"team_id": team_id, "person_uuids": person_uuids}, _REPAIR_READ_SETTINGS)- return {- person_uuid: _ChPersonState(- visible_rows=int(visible_rows),- visible_max_version=int(visible_max_version),- visible_winner_deleted=bool(visible_winner_deleted),- max_version=int(max_version),- )- for person_uuid, visible_rows, visible_max_version, visible_winner_deleted, max_version in rows- }---def _ch_mapping_states(team_id: int, distinct_ids: list[str]) -> dict[str, _ChMappingState]:- states: dict[str, _ChMappingState] = {}- for chunk in _chunks(distinct_ids, _MAPPING_QUERY_CHUNK_SIZE):- rows = _ch(_MAPPING_STATE_SQL, {"team_id": team_id, "distinct_ids": list(chunk)}, _REPAIR_READ_SETTINGS)- for (- distinct_id,- visible_rows,- visible_max_version,- visible_winner_deleted,- visible_winner_person_uuid,- max_version,- ) in rows:- states[distinct_id] = _ChMappingState(- visible_rows=int(visible_rows),- visible_max_version=int(visible_max_version),- visible_winner_deleted=bool(visible_winner_deleted),- visible_winner_person_uuid=visible_winner_person_uuid,- max_version=int(max_version),- )- return states---def _plan_chunk(team_id: int, person_uuids: Sequence[str], *, only_distinct_id: str | None = None) -> list[_PersonPlan]:- live = {str(p.uuid): p for p in get_persons_by_uuids(team_id, list(person_uuids), distinct_id_limit=0)}- person_states = _ch_person_states(team_id, list(live)) if live else {}- distinct_ids_by_person = (- personhog_call(- "person_divergence_repair_distinct_ids",- lambda: _batched_get_distinct_ids_for_persons(- team_id, [p.pk for p in live.values()], limit_per_person=_MAX_REPAIR_DISTINCT_IDS_PER_PERSON + 1- ),- )- if live- else {}- )- too_many: set[int] = set()- all_distinct_ids: list[str] = []- for person_id, dids in list(distinct_ids_by_person.items()):- over_cap = len(dids) > _MAX_REPAIR_DISTINCT_IDS_PER_PERSON- if only_distinct_id is not None:- dids = [d for d in dids if d.id == only_distinct_id]- distinct_ids_by_person[person_id] = dids- # The capped read can miss the requested id, which must then be reported rather than dropped.- over_cap = over_cap and not dids- if over_cap:- too_many.add(person_id)- else:- all_distinct_ids.extend(d.id for d in dids)- mapping_states = _ch_mapping_states(team_id, all_distinct_ids) if all_distinct_ids else {}-- plans: list[_PersonPlan] = []- for person_uuid in person_uuids:- person = live.get(person_uuid)- if person is None:- plans.append(- _PersonPlan(- team_id=team_id,- person_uuid=person_uuid,- person=None,- kind=None,- ch_max_version=None,- target_version=None,- mappings=[],- )- )- continue- pg_version = int(person.version or 0)- state = person_states.get(person_uuid)- ch_max_version = state.max_version if state is not None else None- person_distinct_ids = distinct_ids_by_person.get(person.pk, [])- too_many_distinct_ids = person.pk in too_many- mappings = []- for mapping in [] if too_many_distinct_ids else person_distinct_ids:- mapping_state = mapping_states.get(mapping.id)- mapping_ch_max = mapping_state.max_version if mapping_state is not None else None- mappings.append(- _MappingPlan(- distinct_id=mapping.id,- kind=_mapping_kind(person_uuid, mapping.version, mapping_state),- pg_version=mapping.version,- ch_max_version=mapping_ch_max,- target_version=_target_version(mapping.version, mapping_ch_max),- )- )- plans.append(- _PersonPlan(- team_id=team_id,- person_uuid=person_uuid,- person=person,- kind=_person_kind(pg_version, state),- ch_max_version=ch_max_version,- target_version=_target_version(pg_version, ch_max_version),- mappings=mappings,- too_many_distinct_ids=too_many_distinct_ids,- )- )- return plans---def _raise_person_version_floor(team_id: int, person_id: int, min_version: int) -> None:- personhog_call(- "person_divergence_set_person_version_floor",- lambda: require_personhog_client().set_person_version_floor(- SetPersonVersionFloorRequest(team_id=team_id, person_id=person_id, min_version=min_version)- ),- )---def _raise_mapping_version_floor(team_id: int, distinct_id: str, min_version: int) -> str | None:- """Raise the mapping's version on the primary and return the uuid of the person it maps to there."""- response = personhog_call(- "person_divergence_set_person_distinct_id_version_floor",- lambda: require_personhog_client().set_person_distinct_id_version_floor(- SetPersonDistinctIdVersionFloorRequest(team_id=team_id, distinct_id=distinct_id, min_version=min_version)- ),- )- if not response.HasField("person"):- return None- return str(UUID(response.person.uuid))---def _primary_mapping_versions(team_id: int, person_id: int) -> dict[str, int]:- response = personhog_call(- "person_divergence_confirm_mapping_versions",- lambda: require_personhog_client().get_distinct_ids_for_persons(- GetDistinctIdsForPersonsRequest(- team_id=team_id,- person_ids=[person_id],- read_options=ReadOptions(consistency=CONSISTENCY_LEVEL_STRONG),- limit_per_person=_MAX_REPAIR_DISTINCT_IDS_PER_PERSON + 1,- )- ),- )- return {d.distinct_id: int(d.version or 0) for pd in response.person_distinct_ids for d in pd.distinct_ids}---def _person_action(plan: _PersonPlan, outcome: RepairOutcome) -> RepairAction:- return RepairAction(- team_id=plan.team_id,- person_uuid=plan.person_uuid,- distinct_id=None,- kind=plan.kind,- pg_version=int(plan.person.version or 0) if plan.person is not None else None,- ch_max_version=plan.ch_max_version,- target_version=plan.target_version if plan.kind is not None else None,- outcome=outcome,- )---def _too_many_distinct_ids_action(plan: _PersonPlan) -> RepairAction:- return RepairAction(- team_id=plan.team_id,- person_uuid=plan.person_uuid,- distinct_id=None,- kind=None,- pg_version=None,- ch_max_version=None,- target_version=None,- outcome="skipped_too_many_distinct_ids",- )---def _mapping_action(plan: _PersonPlan, mapping: _MappingPlan, outcome: RepairOutcome) -> RepairAction:- return RepairAction(- team_id=plan.team_id,- person_uuid=plan.person_uuid,- distinct_id=mapping.distinct_id,- kind=mapping.kind,- pg_version=mapping.pg_version,- ch_max_version=mapping.ch_max_version,- target_version=mapping.target_version if mapping.kind is not None else None,- outcome=outcome,- )---def _publish_person(team_id: int, person: Person) -> ProduceResult:- row = _person_row(- team_id=team_id,- uuid=str(person.uuid),- version=int(person.version or 0),- properties=person.properties,- is_identified=person.is_identified,- is_deleted=False,- created_at=person.created_at,- last_seen_at=person.last_seen_at,- )- # _person_row fills a missing last_seen_at with the current hour, but ingestion publishes null for it.- if person.last_seen_at is None:- row["last_seen_at"] = None- return ClickhouseProducer().produce(topic=KAFKA_PERSON, sql=INSERT_PERSON_SQL, data=row)---def _reread_person(team_id: int, person_uuid: str, target_version: int | None) -> Person | None:- """Read the person from the replica again while it shows a version below ``target_version``."""- found = get_persons_by_uuids(team_id, [person_uuid], distinct_id_limit=0)- reread = found[0] if found else None- for delay in _REREAD_BACKOFF_SECONDS:- if reread is None or target_version is None or int(reread.version or 0) >= target_version:- break- _sleep(delay)- found = get_persons_by_uuids(team_id, [person_uuid], distinct_id_limit=0)- reread = found[0] if found else None- return reread---def _clickhouse_only_properties(plan: _PersonPlan) -> dict[str, Any]:- """The properties of the ClickHouse winner whose keys the Postgres person lacks."""- assert plan.person is not None- rows = _ch(- _PERSON_WINNER_PROPERTIES_SQL,- {"team_id": plan.team_id, "person_uuid": plan.person_uuid},- _REPAIR_READ_SETTINGS,- )- ch_properties = json.loads(rows[0][0]) if rows and rows[0][0] else {}- pg_properties = plan.person.properties or {}- return {key: value for key, value in ch_properties.items() if key not in pg_properties}---def _queue_stale_merge(plan: _PersonPlan, distinct_id: str, missing: dict[str, Any]) -> RepairOutcome | None:- """Send the ClickHouse-only properties to ingestion as $set_once, or return why they were not sent."""- assert plan.person is not None- # Every member can read the $set event, and event reads apply event property restrictions only, so a team with- # any property rule could expose a person property that some member is denied.- if team_has_property_access_rules(team_id=plan.team_id):- return "skipped_stale_merge_restricted"- if len(json.dumps({**missing, **(plan.person.properties or {})})) > _STALE_MERGE_MAX_BYTES:- return "skipped_stale_merge_too_large"- try:- token = Team.objects.only("api_token").get(id=plan.team_id).api_token- capture_internal(- token=token,- event_name="$set",- event_source="person_divergence_reset",- distinct_id=distinct_id,- timestamp=datetime.now(UTC),- properties={"$set_once": missing},- process_person_profile=True,- ).raise_for_status()- except Exception:- return "skipped_stale_merge_failed"- return None---def _is_transient(exc: BaseException) -> bool:- return isinstance(exc, (*CH_TRANSIENT_ERRORS, ClickHouseQueryTimeOut)) or is_transient_rpc_error(- exc, codes=_TRANSIENT_RPC_CODES- )---def _retry_transient(fn: Callable[[], _T], *, what: str, log: Callable[[str], None]) -> _T:- """Run ``fn`` again after a transient ClickHouse or personhog error, then re-raise once the retries run out.-- Repeating a step is safe because every repair write is idempotent: a version raise never lowers a- version, and ClickHouse keeps one row per version.- """- for delay in _TRANSIENT_RETRY_BACKOFF_SECONDS:- try:- return fn()- except Exception as exc:- if not _is_transient(exc):- raise- log(f"{what}: {type(exc).__name__}, retrying in {delay:g} s")- _sleep(delay)- return fn()---def _execute_plan(- plan: _PersonPlan,- *,- apply: bool,- include_stale: bool,- before_write: Callable[[], None],- published: Callable[[ProduceResult], None],- stale_merge_distinct_id: str | None = None,-) -> list[RepairAction]:- person = plan.person- if person is None:- return [_person_action(plan, "skipped_not_live")]- if plan.kind == "stale" and not include_stale:- refusal: RepairOutcome | None = "skipped_stale"- if apply and stale_merge_distinct_id is not None:- missing = _clickhouse_only_properties(plan)- # A later reset that finds nothing missing raises Postgres. Capture accepting the $set does not mean- # ingestion applies it, and a raise without the merged properties lets the next update overwrite them.- refusal = (- (_queue_stale_merge(plan, stale_merge_distinct_id, missing) or "merge_queued") if missing else None- )- if refusal is not None:- return [- _person_action(plan, refusal),- *(- _mapping_action(plan, m, "skipped_not_divergent" if m.kind is None else "skipped_stale")- for m in plan.mappings- ),- *([_too_many_distinct_ids_action(plan)] if plan.too_many_distinct_ids else []),- ]-- divergent_mappings = [m for m in plan.mappings if m.kind is not None]- mapping_actions = [_mapping_action(plan, m, "skipped_not_divergent") for m in plan.mappings if m.kind is None]- if plan.too_many_distinct_ids:- mapping_actions.append(_too_many_distinct_ids_action(plan))- if not apply:- person_outcome: RepairOutcome = "would_repair" if plan.kind is not None else "skipped_not_divergent"- return [- _person_action(plan, person_outcome),- *mapping_actions,- *(_mapping_action(plan, m, "would_repair") for m in divergent_mappings),- ]-- pg_version = int(person.version or 0)- if plan.kind is not None and plan.target_version is not None and plan.target_version > pg_version:- before_write()- _raise_person_version_floor(plan.team_id, person.pk, plan.target_version)-- # SetPersonDistinctIdVersionFloor runs on the primary, so its reply is the owner check: the replica list- # that produced this mapping can lag a merge that moved it to another person.- owned: list[_MappingPlan] = []- for mapping in divergent_mappings:- before_write()- owner = _raise_mapping_version_floor(plan.team_id, mapping.distinct_id, mapping.target_version)- if owner is None:- mapping_actions.append(_mapping_action(plan, mapping, "skipped_mapping_gone"))- elif owner != plan.person_uuid:- mapping_actions.append(_mapping_action(plan, mapping, "skipped_owner_changed"))- else:- owned.append(mapping)-- # Re-read after the raise so the published properties are the ones Postgres holds at the- # published version, including any ingestion update that landed after the first read.- reread: Person | None = None- if plan.kind is not None:- reread = _reread_person(plan.team_id, plan.person_uuid, plan.target_version)-- if (plan.kind is not None or owned) and _tombstoned_uuids(plan.team_id, [plan.person_uuid]):- person_outcome = "skipped_tombstoned" if plan.kind is not None else "skipped_not_divergent"- return [- _person_action(plan, person_outcome),- *mapping_actions,- *(_mapping_action(plan, m, "skipped_tombstoned") for m in owned),- ]-- if plan.kind is None:- person_outcome = "skipped_not_divergent"- elif reread is None:- person_outcome = "skipped_not_live"- elif plan.target_version is None or int(reread.version or 0) < plan.target_version:- # The replica has not caught up with the raise, so its properties may predate the raised version.- person_outcome = "skipped_reread_lagging"- else:- published(_publish_person(plan.team_id, reread))- person_outcome = "repaired"-- # The SetPersonDistinctIdVersionFloor reply does not say whether it wrote, and a personhog-replica build- # without NULL-version handling leaves a NULL version unchanged, so publish only the mappings whose- # version on the primary reached the target.- stored_versions = _primary_mapping_versions(plan.team_id, person.pk) if owned else {}- for mapping in owned:- stored_version = stored_versions.get(mapping.distinct_id)- if stored_version is None:- # The primary no longer lists the mapping under this person: it moved or was tombstoned after the raise.- mapping_actions.append(_mapping_action(plan, mapping, "skipped_mapping_gone"))- elif stored_version < mapping.target_version:- mapping_actions.append(_mapping_action(plan, mapping, "skipped_reread_lagging"))- else:- published(- create_person_distinct_id(- team_id=plan.team_id,- distinct_id=mapping.distinct_id,- person_id=plan.person_uuid,- version=stored_version,- is_deleted=False,- )- )- mapping_actions.append(_mapping_action(plan, mapping, "repaired"))- return [_person_action(plan, person_outcome), *mapping_actions]---@frozen(frozen=False)-class _WritePacer:- """Space writes at least 1 / max_per_second apart, so idle time never buys a later burst."""-- max_per_second: float | None- next_write_at: float = 0.0-- def before_write(self) -> None:- if self.max_per_second is None:- return- now = time.monotonic()- if now < self.next_write_at:- time.sleep(self.next_write_at - now)- now = self.next_write_at- self.next_write_at = now + 1 / self.max_per_second---@frozen(frozen=False)-class _Deliveries:- """Kafka produce results of the published rows, so a failed delivery counts as undelivered."""-- pending: list[ProduceResult] = field(default_factory=list)- failed: int = 0-- def track(self, result: ProduceResult) -> None:- self.pending.append(result)- if len(self.pending) >= _DELIVERY_PRUNE_EVERY:- self._prune()-- def undelivered(self, timeout: float) -> int:- _flush_person_producers(timeout)- self._prune()- return self.failed + len(self.pending)-- def _prune(self) -> None:- waiting: list[ProduceResult] = []- for result in self.pending:- if not result.done():- waiting.append(result)- continue- try:- result.get(timeout=0)- except Exception:- self.failed += 1- self.pending = waiting---@frozen(frozen=False)-class _OutcomeCounts:- person: Counter[RepairOutcome] = field(default_factory=Counter)- mapping: Counter[RepairOutcome] = field(default_factory=Counter)-- def add(self, action: RepairAction) -> None:- is_person_row = action.distinct_id is None and action.outcome != "skipped_too_many_distinct_ids"- (self.person if is_person_row else self.mapping)[action.outcome] += 1-- def summary(self, *, applied: bool, persons: int, undelivered: int) -> RepairSummary:- return RepairSummary(- applied=applied,- persons=persons,- person_outcomes=dict(self.person),- mapping_outcomes=dict(self.mapping),- undelivered=undelivered,- )---def repair_persons(- targets: Sequence[PersonRef],- *,- apply: bool,- include_stale: bool = False,- max_writes_per_second: float | None = None,- on_action: Callable[[RepairAction], None],- log: Callable[[str], None],-) -> RepairSummary:- """Republish the Postgres state of each target person and its mappings where ClickHouse disagrees.-- A rerun is safe: each Postgres write only raises a version, and a person whose publish never landed- still counts as divergent. ``max_writes_per_second`` paces each Postgres write, one per divergent- person or distinct id.-- ``include_stale`` also repairs stale persons and their mappings. That replaces their ClickHouse properties- with the Postgres ones for good, so never use it on a team waiting for a restore from ClickHouse.- """- if max_writes_per_second is not None and max_writes_per_second <= 0:- raise ValueError("max_writes_per_second must be above 0")- by_team: dict[int, list[str]] = defaultdict(list)- for target in dict.fromkeys(targets):- by_team[target.team_id].append(target.person_uuid)-- outcomes = _OutcomeCounts()- processed = 0- undelivered = 0- pacer = _WritePacer(max_per_second=max_writes_per_second)- deliveries = _Deliveries()- existing_teams = set(Team.objects.filter(id__in=list(by_team)).values_list("id", flat=True))- try:- for team_id, person_uuids in sorted(by_team.items()):- if team_id not in existing_teams:- # A deleted team's ClickHouse rows go with the team, so a repair would only republish what the deletion removed.- for person_uuid in person_uuids:- action = RepairAction(- team_id=team_id,- person_uuid=person_uuid,- distinct_id=None,- kind=None,- pg_version=None,- ch_max_version=None,- target_version=None,- outcome="skipped_team_gone",- )- outcomes.add(action)- on_action(action)- processed += 1- log(f"team {team_id}: no longer exists, {len(person_uuids)} persons skipped")- continue- for chunk in _chunks(person_uuids, _REPAIR_CHUNK_SIZE):- plans = _retry_transient(- partial(_plan_chunk, team_id, chunk), what=f"team {team_id}: planning", log=log- )- for plan in plans:- # Only the attempt that succeeds returns actions, so a retried person is counted once.- actions = _retry_transient(- partial(- _execute_plan,- plan,- apply=apply,- include_stale=include_stale,- before_write=pacer.before_write,- published=deliveries.track,- ),- what=f"team {team_id} person {plan.person_uuid}",- log=log,- )- for action in actions:- outcomes.add(action)- on_action(action)- processed += 1- log(f"team {team_id}: {len(person_uuids)} persons, {processed} processed in total")- finally:- if apply:- undelivered = deliveries.undelivered(_FLUSH_TIMEOUT_SECONDS)- if undelivered:- log(f"{undelivered} ClickHouse messages were not delivered; rerun the repair for the same input")- return outcomes.summary(applied=apply, persons=processed, undelivered=undelivered)---def repair_distinct_id(team_id: int, distinct_id: str, *, delivery_timeout_seconds: float) -> RepairSummary | None:- """Repair the live person that owns ``distinct_id``, and that one mapping, where ClickHouse disagrees.-- It always applies. For a stale person whose ClickHouse winner holds properties Postgres lacks, it sends them- to ingestion as $set_once, so newer Postgres values win, and writes nothing else. A later reset, once- ingestion applied them, raises Postgres above ClickHouse and republishes. It leaves the person stale when the- team has property access rules.- """- owner = get_person_by_distinct_id(team_id, distinct_id, distinct_id_limit=0)- if owner is None:- return None- outcomes = _OutcomeCounts()- deliveries = _Deliveries()- undelivered = 0- try:- for plan in _plan_chunk(team_id, [str(owner.uuid)], only_distinct_id=distinct_id):- for action in _execute_plan(- plan,- apply=True,- include_stale=False,- before_write=lambda: None,- published=deliveries.track,- stale_merge_distinct_id=distinct_id,- ):- outcomes.add(action)- finally:- undelivered = deliveries.undelivered(delivery_timeout_seconds)- return outcomes.summary(applied=True, persons=1, undelivered=undelivered)diff --git a/posthog/models/person/divergence/__init__.py b/posthog/models/person/divergence/__init__.pynew file mode 100644index 000000000000..8389a43e5d29--- /dev/null+++ b/posthog/models/person/divergence/__init__.py@@ -0,0 +1 @@+"""Scans for persons whose ClickHouse rows disagree with Postgres, and repairs them. One module per concern."""diff --git a/posthog/models/person/divergence/datatypes.py b/posthog/models/person/divergence/datatypes.pynew file mode 100644index 000000000000..39ed46e31874--- /dev/null+++ b/posthog/models/person/divergence/datatypes.py@@ -0,0 +1,121 @@+"""Result types and literals shared by the divergence scans and the repair."""++from __future__ import annotations++from typing import Literal+from uuid import UUID++from posthog.dataclasses import frozen++PersonDivergenceKind = Literal["hidden", "swept", "stale", "behind", "absent"]++MappingDivergenceKind = Literal["hidden", "other_person", "stale", "absent"]++SampleClassification = Literal["equal", "pg_below_ch", "pg_above_ch", "team_gone", "pg_tombstone", "pg_absent"]++SampleEra = Literal["before_cutoff", "since_cutoff", "any"]++RepairOutcome = Literal[+ "would_repair",+ "repaired",+ "skipped_not_divergent",+ "skipped_not_live",+ "skipped_team_gone",+ "skipped_tombstoned",+ "skipped_owner_changed",+ "skipped_mapping_gone",+ "skipped_reread_lagging",+ "skipped_stale",+ "skipped_stale_merge_failed",+ "skipped_stale_merge_too_large",+ "skipped_stale_merge_restricted",+ "merge_queued",+ "skipped_too_many_distinct_ids",+]+++@frozen+class DivergentPerson:+ team_id: int+ person_uuid: str+ kind: PersonDivergenceKind+ ch_max_version: int+ pg_version: int+++@frozen+class ScanSummary:+ candidates: int+ divergent: int+ skipped_team_ids: list[int]+++@frozen+class SampledPerson:+ team_id: int+ person_uuid: str+ classification: SampleClassification+ era: SampleEra+ ch_max_version: int+ pg_version: int | None+++@frozen+class SampleBucket:+ classification: SampleClassification+ era: SampleEra+++@frozen+class SampleSummary:+ sampled: int+ counts: dict[SampleBucket, int]+ skipped_team_ids: list[int]+++@frozen+class TeamCheck:+ team_id: int+ persons_sampled: int+ persons_live_in_postgres: int+ distinct_ids_sampled: int+ distinct_ids_live_in_postgres: int+++@frozen+class PersonRef:+ team_id: int+ person_uuid: str++ def __post_init__(self) -> None:+ # Lookups key on the canonical form that ClickHouse toString(id) and the persons reads return.+ if str(UUID(self.person_uuid)) != self.person_uuid:+ raise ValueError(f"person_uuid must be a canonical lowercase UUID: {self.person_uuid!r}")+++@frozen+class RepairAction:+ """One person row (``distinct_id`` is None) or one of its mappings, with what the repair did to it.++ A person with too many distinct ids to repair gets one extra row with no ``distinct_id`` and the+ outcome ``skipped_too_many_distinct_ids``, which stands for all of its mappings.+ """++ team_id: int+ person_uuid: str+ distinct_id: str | None+ kind: PersonDivergenceKind | MappingDivergenceKind | None+ pg_version: int | None+ ch_max_version: int | None+ target_version: int | None+ outcome: RepairOutcome+++@frozen+class RepairSummary:+ applied: bool+ persons: int+ person_outcomes: dict[RepairOutcome, int]+ mapping_outcomes: dict[RepairOutcome, int]+ # Published rows that Kafka did not confirm: the delivery failed, or it was still queued at the deadline.+ undelivered: intdiff --git a/posthog/models/person/divergence/fleet_scans.py b/posthog/models/person/divergence/fleet_scans.pynew file mode 100644index 000000000000..f0b7259b714e--- /dev/null+++ b/posthog/models/person/divergence/fleet_scans.py@@ -0,0 +1,358 @@+"""Fleet-wide divergence scans: hidden, swept, stale and the uniform sample."""++from __future__ import annotations++from collections import Counter, defaultdict+from collections.abc import Callable+from datetime import datetime+from typing import Any++from posthog.exceptions import ClickHouseQueryMemoryLimitExceeded, ClickHouseQueryTimeOut+from posthog.models.team import Team++from . import datatypes, lookups++# The legacy delete path wrote ClickHouse tombstones at version + 100, so those rows sit at 100 or above.+LEGACY_TOMBSTONE_MIN_VERSION = 100++HIDDEN_TEAM_STEP = 10_000++SWEPT_TEAM_STEP = 10_000++STALE_TEAM_STEP = 5_000++SAMPLE_TEAM_STEP = 10_000++_HIDDEN_SETTINGS = {"max_execution_time": 900, "max_memory_usage": 32_000_000_000}++_SWEPT_SETTINGS = {"apply_deleted_mask": 0, "max_execution_time": 1800, "max_memory_usage": 64_000_000_000}++_STALE_SETTINGS = {+ "max_execution_time": 1800,+ "max_memory_usage": 48_000_000_000,+ "max_bytes_before_external_group_by": 20_000_000_000,+}++_SAMPLE_SETTINGS = {"max_execution_time": 3600, "max_memory_usage": 32_000_000_000}+++def _resolve_max_team_id(max_team_id: int | None) -> int:+ if max_team_id is not None:+ return max_team_id+ [[highest]] = lookups._ch("SELECT max(team_id) FROM person", {}, _HIDDEN_SETTINGS)+ return int(highest or 0) + 1+++def _scan_team_ranges(+ query: Callable[[int, int], list[Any]],+ *,+ min_team_id: int,+ max_team_id: int,+ team_step: int,+ on_rows: Callable[[list[Any]], None],+ log: Callable[[str], None],+) -> list[int]:+ """Return each team that runs out of memory or time even when scanned alone, so the caller can rerun it with higher limits."""+ skipped: list[int] = []++ def scan(lo: int, hi: int) -> None:+ try:+ rows = query(lo, hi)+ except (ClickHouseQueryMemoryLimitExceeded, ClickHouseQueryTimeOut) as exc:+ reason = "out of memory" if isinstance(exc, ClickHouseQueryMemoryLimitExceeded) else "timed out"+ if hi - lo == 1:+ log(f"team {lo}: {reason}, skipped")+ skipped.append(lo)+ return+ mid = (lo + hi) // 2+ log(f"teams [{lo}, {hi}): {reason}, bisecting")+ scan(lo, mid)+ scan(mid, hi)+ return+ on_rows(rows)+ log(f"teams [{lo}, {hi}): {len(rows)} ClickHouse candidates")++ for lo in range(min_team_id, max_team_id, team_step):+ scan(lo, min(lo + team_step, max_team_id))+ return skipped+++def _scan_divergent_persons(+ *,+ sql: str,+ settings: dict[str, int],+ extra_args: dict[str, Any],+ kind: datatypes.PersonDivergenceKind,+ divergent: Callable[[int, int], bool],+ min_team_id: int,+ max_team_id: int | None,+ team_step: int,+ on_found: Callable[[datatypes.DivergentPerson], None],+ log: Callable[[str], None],+) -> datatypes.ScanSummary:+ candidates = 0+ found = 0++ def on_rows(rows: list[Any]) -> None:+ nonlocal candidates, found+ by_team: dict[int, dict[str, int]] = defaultdict(dict)+ for team_id, person_uuid, ch_max_version in rows:+ by_team[int(team_id)][person_uuid] = int(ch_max_version)+ for team_id, ch_versions in sorted(by_team.items()):+ candidates += len(ch_versions)+ pg_versions = lookups._live_person_versions(team_id, list(ch_versions), f"person_divergence_scan_{kind}")+ for person_uuid, pg_version in pg_versions.items():+ ch_max_version = ch_versions[person_uuid]+ if not divergent(pg_version, ch_max_version):+ continue+ on_found(+ datatypes.DivergentPerson(+ team_id=team_id,+ person_uuid=person_uuid,+ kind=kind,+ ch_max_version=ch_max_version,+ pg_version=pg_version,+ )+ )+ found += 1++ skipped = _scan_team_ranges(+ lambda lo, hi: lookups._ch(sql, {"min_team_id": lo, "max_team_id": hi, **extra_args}, settings),+ min_team_id=min_team_id,+ max_team_id=_resolve_max_team_id(max_team_id),+ team_step=team_step,+ on_rows=on_rows,+ log=log,+ )+ return datatypes.ScanSummary(candidates=candidates, divergent=found, skipped_team_ids=skipped)+++_HIDDEN_SQL = """+SELECT team_id, toString(id), max(version) AS max_version+FROM person+WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s+ AND (team_id, id) IN (+ SELECT team_id, id FROM person+ WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s+ AND is_deleted = 1 AND version >= %(min_tombstone_version)s+ )+GROUP BY team_id, id+HAVING argMax(is_deleted, version) = 1 AND max_version >= %(min_tombstone_version)s+"""+++def scan_hidden_persons(+ *,+ min_team_id: int = 0,+ max_team_id: int | None = None,+ team_step: int = HIDDEN_TEAM_STEP,+ on_found: Callable[[datatypes.DivergentPerson], None],+ log: Callable[[str], None],+) -> datatypes.ScanSummary:+ """Persons live in Postgres whose newest ClickHouse row is a legacy (version 100 or above) tombstone.++ Run it, then repair what it finds, right before every ClickHouse deletion sweep while such+ tombstones remain: the sweep deletes every ClickHouse row of these persons.+ """+ return _scan_divergent_persons(+ sql=_HIDDEN_SQL,+ settings=_HIDDEN_SETTINGS,+ extra_args={"min_tombstone_version": LEGACY_TOMBSTONE_MIN_VERSION},+ kind="hidden",+ divergent=lambda _pg_version, _ch_max_version: True,+ min_team_id=min_team_id,+ max_team_id=max_team_id,+ team_step=team_step,+ on_found=on_found,+ log=log,+ )+++# The ClickHouse deletion sweep deletes every row of a person whose highest-version row is a tombstone.+# That includes a live row written after the tombstone at a lower version, so the person stays live in+# Postgres with no visible ClickHouse row.+_SWEPT_SQL = """+SELECT team_id, toString(id), max(version) AS max_version+FROM person+WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s+ AND (team_id, id) IN (+ SELECT team_id, id FROM person+ WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s+ AND NOT _row_exists AND is_deleted = 1 AND version >= %(min_tombstone_version)s+ )+GROUP BY team_id, id+HAVING countIf(_row_exists) = 0+ AND maxIf(_timestamp, is_deleted = 0) > maxIf(_timestamp, is_deleted = 1 AND version >= %(min_tombstone_version)s)+"""+++def scan_swept_persons(+ *,+ min_team_id: int = 0,+ max_team_id: int | None = None,+ team_step: int = SWEPT_TEAM_STEP,+ on_found: Callable[[datatypes.DivergentPerson], None],+ log: Callable[[str], None],+) -> datatypes.ScanSummary:+ """Live Postgres persons with every ClickHouse row deleted, including a live row written after a legacy tombstone.++ Run it soon after a ClickHouse deletion sweep: once merges remove the deleted rows, no scan finds the person.+ """+ return _scan_divergent_persons(+ sql=_SWEPT_SQL,+ settings=_SWEPT_SETTINGS,+ extra_args={"min_tombstone_version": LEGACY_TOMBSTONE_MIN_VERSION},+ kind="swept",+ divergent=lambda _pg_version, _ch_max_version: True,+ min_team_id=min_team_id,+ max_team_id=max_team_id,+ team_step=team_step,+ on_found=on_found,+ log=log,+ )+++# A row with a lower version that arrives well after the winning row means Postgres restarted the+# person below ClickHouse, so every later update loses to the old winner.+_STALE_SQL = """+SELECT team_id, toString(id), max(version) AS max_version+FROM person+WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s+ AND (team_id, id) IN (+ SELECT team_id, id FROM person+ WHERE team_id >= %(min_team_id)s AND team_id < %(max_team_id)s+ AND _timestamp > now() - toIntervalDay(%(window_days)s)+ )+GROUP BY team_id, id+HAVING argMax(is_deleted, version) = 0+ AND argMax(version, _timestamp) < max_version+ AND max(_timestamp) > argMax(_timestamp, version) + INTERVAL 1 HOUR+"""+++def scan_stale_persons(+ *,+ window_days: int,+ min_team_id: int = 0,+ max_team_id: int | None = None,+ team_step: int = STALE_TEAM_STEP,+ on_found: Callable[[datatypes.DivergentPerson], None],+ log: Callable[[str], None],+) -> datatypes.ScanSummary:+ """Persons whose live ClickHouse winner outranks Postgres, found through a late lower-version row."""+ return _scan_divergent_persons(+ sql=_STALE_SQL,+ settings=_STALE_SETTINGS,+ extra_args={"window_days": window_days},+ kind="stale",+ divergent=lambda pg_version, ch_max_version: pg_version < ch_max_version,+ min_team_id=min_team_id,+ max_team_id=max_team_id,+ team_step=team_step,+ on_found=on_found,+ log=log,+ )+++_SAMPLE_SQL = """+SELECT team_id, toString(id), max(version) AS max_version, toUnixTimestamp(argMax(_timestamp, version)) AS written_at+FROM person+WHERE cityHash64(id) %% %(modulus)s = %(residue)s AND team_id >= %(min_team_id)s AND team_id < %(max_team_id)s+GROUP BY team_id, id+HAVING argMax(is_deleted, version) = 0+ AND (%(written_within_days)s = 0 OR argMax(_timestamp, version) >= now() - toIntervalDay(%(written_within_days)s))+"""+++def _classify_sampled(+ pg_version: int | None, ch_max_version: int, *, team_exists: bool, tombstoned: bool+) -> datatypes.SampleClassification:+ if not team_exists:+ return "team_gone"+ if pg_version is None:+ return "pg_tombstone" if tombstoned else "pg_absent"+ if pg_version == ch_max_version:+ return "equal"+ return "pg_below_ch" if pg_version < ch_max_version else "pg_above_ch"+++def scan_sample(+ *,+ modulus: int,+ residue: int,+ written_within_days: int | None = None,+ cutoff: datetime | None = None,+ min_team_id: int = 0,+ max_team_id: int | None = None,+ team_step: int = SAMPLE_TEAM_STEP,+ on_sampled: Callable[[datatypes.SampledPerson], None],+ log: Callable[[str], None],+) -> datatypes.SampleSummary:+ """Classify a uniform sample of live ClickHouse persons against Postgres."""+ if not 0 <= residue < modulus:+ raise ValueError("residue must be in [0, modulus)")+ cutoff_ts = cutoff.timestamp() if cutoff is not None else None+ counts: Counter[datatypes.SampleBucket] = Counter()+ sampled_count = 0++ def query(lo: int, hi: int) -> list[Any]:+ return lookups._ch(+ _SAMPLE_SQL,+ {+ "modulus": modulus,+ "residue": residue,+ "written_within_days": written_within_days or 0,+ "min_team_id": lo,+ "max_team_id": hi,+ },+ _SAMPLE_SETTINGS,+ )++ def on_rows(rows: list[Any]) -> None:+ nonlocal sampled_count+ if not rows:+ return+ sampled_count += len(rows)+ by_team: dict[int, dict[str, tuple[int, datatypes.SampleEra]]] = defaultdict(dict)+ for team_id, person_uuid, ch_max_version, written_at in rows:+ era: datatypes.SampleEra = (+ "any" if cutoff_ts is None else "before_cutoff" if written_at < cutoff_ts else "since_cutoff"+ )+ by_team[int(team_id)][person_uuid] = (int(ch_max_version), era)++ existing_teams = set(Team.objects.filter(id__in=list(by_team)).values_list("id", flat=True))+ for team_id, sampled in sorted(by_team.items()):+ team_exists = team_id in existing_teams+ # A deleted team's persons can outlive it in Postgres, so they are team_gone whatever their version.+ pg_versions = (+ lookups._live_person_versions(team_id, list(sampled), "person_divergence_sample") if team_exists else {}+ )+ missing = [u for u in sampled if u not in pg_versions]+ tombstoned = lookups._tombstoned_uuids(team_id, missing) if team_exists else set()+ for person_uuid, (ch_max_version, era) in sampled.items():+ pg_version = pg_versions.get(person_uuid)+ classification = _classify_sampled(+ pg_version, ch_max_version, team_exists=team_exists, tombstoned=person_uuid in tombstoned+ )+ counts[datatypes.SampleBucket(classification=classification, era=era)] += 1+ on_sampled(+ datatypes.SampledPerson(+ team_id=team_id,+ person_uuid=person_uuid,+ classification=classification,+ era=era,+ ch_max_version=ch_max_version,+ pg_version=pg_version,+ )+ )++ skipped = _scan_team_ranges(+ query,+ min_team_id=min_team_id,+ max_team_id=_resolve_max_team_id(max_team_id),+ team_step=team_step,+ on_rows=on_rows,+ log=log,+ )+ log(f"sampled {sampled_count} live ClickHouse persons")+ return datatypes.SampleSummary(sampled=sampled_count, counts=dict(counts), skipped_team_ids=skipped)diff --git a/posthog/models/person/divergence/lookups.py b/posthog/models/person/divergence/lookups.pynew file mode 100644index 000000000000..1330db8ca1da--- /dev/null+++ b/posthog/models/person/divergence/lookups.py@@ -0,0 +1,43 @@+"""ClickHouse and personhog reads shared by the scans and the repair."""++from __future__ import annotations++from collections.abc import Sequence+from typing import Any, TypeVar+from uuid import UUID++from posthog.clickhouse.client import sync_execute+from posthog.clickhouse.workload import Workload+from posthog.models.person.util import _batched_get_persons_by_uuids, get_person_tombstones+from posthog.personhog_client.client import personhog_call+from posthog.personhog_client.proto import ReadOptions++_T = TypeVar("_T")+++# A scan reads only these fields, which keeps person properties out of the RPC payloads.+_VERSION_ONLY_READ_OPTIONS = ReadOptions(field_mask=["id", "uuid", "team_id", "version"])+++def _ch(sql: str, args: dict[str, Any], settings: dict[str, int]) -> list[Any]:+ return sync_execute(sql, args, settings=settings, workload=Workload.OFFLINE, readonly=True)+++def _live_person_versions(team_id: int, person_uuids: list[str], operation: str) -> dict[str, int]:+ persons = personhog_call(+ operation,+ lambda: _batched_get_persons_by_uuids(+ team_id, person_uuids, operation, read_options=_VERSION_ONLY_READ_OPTIONS+ ),+ )+ return {str(UUID(p.uuid)): int(p.version) for p in persons}+++def _tombstoned_uuids(team_id: int, person_uuids: list[str]) -> set[str]:+ if not person_uuids:+ return set()+ return {str(t.uuid) for t in get_person_tombstones(team_id, [UUID(u) for u in person_uuids])}+++def _chunks(items: Sequence[_T], size: int) -> list[Sequence[_T]]:+ return [items[i : i + size] for i in range(0, len(items), size)]diff --git a/posthog/models/person/divergence/repair_plan.py b/posthog/models/person/divergence/repair_plan.pynew file mode 100644index 000000000000..f99815a6a0c7--- /dev/null+++ b/posthog/models/person/divergence/repair_plan.py@@ -0,0 +1,235 @@+"""Reads ClickHouse state for target persons and plans what the repair writes."""++from __future__ import annotations++from collections.abc import Sequence++from posthog.dataclasses import frozen+from posthog.models.person import Person+from posthog.models.person.util import _batched_get_distinct_ids_for_persons, get_persons_by_uuids+from posthog.personhog_client.client import personhog_call++from . import datatypes, lookups++# Repair reads see lightweight-deleted rows, so a republish lands above every row a sweep masked.+_REPAIR_READ_SETTINGS = {"apply_deleted_mask": 0, "max_execution_time": 60, "max_memory_usage": 4_000_000_000}++_MAPPING_QUERY_CHUNK_SIZE = 1_000++# A person with more distinct ids keeps its person repair, and its mappings are reported for a separate plan:+# one read of every mapping can exceed the gRPC message limit, and their paced writes would run for hours.+_MAX_REPAIR_DISTINCT_IDS_PER_PERSON = 1_000+++@frozen+class _ChPersonState:+ visible_rows: int+ visible_max_version: int+ visible_winner_deleted: bool+ # Includes lightweight-deleted rows, so a republish lands above every row the sweep masked.+ max_version: int+++@frozen+class _ChMappingState:+ visible_rows: int+ visible_max_version: int+ visible_winner_deleted: bool+ visible_winner_person_uuid: str+ max_version: int+++@frozen+class _MappingPlan:+ distinct_id: str+ kind: datatypes.MappingDivergenceKind | None+ pg_version: int+ ch_max_version: int | None+ target_version: int+++@frozen+class _PersonPlan:+ team_id: int+ person_uuid: str+ person: Person | None+ kind: datatypes.PersonDivergenceKind | None+ ch_max_version: int | None+ target_version: int | None+ mappings: list[_MappingPlan]+ too_many_distinct_ids: bool = False+++_PERSON_STATE_SQL = """+SELECT+ toString(id),+ countIf(_row_exists),+ maxIf(version, _row_exists),+ argMaxIf(is_deleted, version, _row_exists),+ max(version)+FROM person+WHERE team_id = %(team_id)s AND id IN %(person_uuids)s+GROUP BY id+"""++_MAPPING_STATE_SQL = """+SELECT+ distinct_id,+ countIf(_row_exists),+ maxIf(version, _row_exists),+ argMaxIf(is_deleted, version, _row_exists),+ toString(argMaxIf(person_id, version, _row_exists)),+ max(version)+FROM person_distinct_id2+WHERE team_id = %(team_id)s AND distinct_id IN %(distinct_ids)s+GROUP BY distinct_id+"""+++def _person_kind(pg_version: int, state: _ChPersonState | None) -> datatypes.PersonDivergenceKind | None:+ # Any winner that differs from Postgres counts, whatever the versions: a raise whose publish+ # never landed leaves ClickHouse below Postgres, and a rerun must still finish it.+ if state is None:+ return "absent"+ if state.visible_rows == 0:+ return "swept"+ if state.visible_winner_deleted:+ return "hidden"+ if state.visible_max_version > pg_version:+ return "stale"+ if state.visible_max_version < pg_version:+ return "behind"+ return None+++def _mapping_kind(+ person_uuid: str, pg_version: int, state: _ChMappingState | None+) -> datatypes.MappingDivergenceKind | None:+ if state is None or state.visible_rows == 0:+ return "absent"+ if state.visible_winner_deleted:+ return "hidden"+ if state.visible_winner_person_uuid != person_uuid:+ return "other_person"+ # The owner matches, but a later move of this mapping would be written below the ClickHouse winner and lost.+ if state.visible_max_version > pg_version:+ return "stale"+ return None+++def _target_version(pg_version: int, ch_max_version: int | None) -> int:+ # One above ClickHouse, not 100 above: a later Postgres write or tombstone takes the next version+ # and must outrank this row.+ return pg_version if ch_max_version is None else max(pg_version, ch_max_version + 1)+++def _ch_person_states(team_id: int, person_uuids: list[str]) -> dict[str, _ChPersonState]:+ rows = lookups._ch(_PERSON_STATE_SQL, {"team_id": team_id, "person_uuids": person_uuids}, _REPAIR_READ_SETTINGS)+ return {+ person_uuid: _ChPersonState(+ visible_rows=int(visible_rows),+ visible_max_version=int(visible_max_version),+ visible_winner_deleted=bool(visible_winner_deleted),+ max_version=int(max_version),+ )+ for person_uuid, visible_rows, visible_max_version, visible_winner_deleted, max_version in rows+ }+++def _ch_mapping_states(team_id: int, distinct_ids: list[str]) -> dict[str, _ChMappingState]:+ states: dict[str, _ChMappingState] = {}+ for chunk in lookups._chunks(distinct_ids, _MAPPING_QUERY_CHUNK_SIZE):+ rows = lookups._ch(_MAPPING_STATE_SQL, {"team_id": team_id, "distinct_ids": list(chunk)}, _REPAIR_READ_SETTINGS)+ for (+ distinct_id,+ visible_rows,+ visible_max_version,+ visible_winner_deleted,+ visible_winner_person_uuid,+ max_version,+ ) in rows:+ states[distinct_id] = _ChMappingState(+ visible_rows=int(visible_rows),+ visible_max_version=int(visible_max_version),+ visible_winner_deleted=bool(visible_winner_deleted),+ visible_winner_person_uuid=visible_winner_person_uuid,+ max_version=int(max_version),+ )+ return states+++def _plan_chunk(team_id: int, person_uuids: Sequence[str], *, only_distinct_id: str | None = None) -> list[_PersonPlan]:+ live = {str(p.uuid): p for p in get_persons_by_uuids(team_id, list(person_uuids), distinct_id_limit=0)}+ person_states = _ch_person_states(team_id, list(live)) if live else {}+ distinct_ids_by_person = (+ personhog_call(+ "person_divergence_repair_distinct_ids",+ lambda: _batched_get_distinct_ids_for_persons(+ team_id, [p.pk for p in live.values()], limit_per_person=_MAX_REPAIR_DISTINCT_IDS_PER_PERSON + 1+ ),+ )+ if live+ else {}+ )+ too_many: set[int] = set()+ all_distinct_ids: list[str] = []+ for person_id, dids in list(distinct_ids_by_person.items()):+ over_cap = len(dids) > _MAX_REPAIR_DISTINCT_IDS_PER_PERSON+ if only_distinct_id is not None:+ dids = [d for d in dids if d.id == only_distinct_id]+ distinct_ids_by_person[person_id] = dids+ # The capped read can miss the requested id, which must then be reported rather than dropped.+ over_cap = over_cap and not dids+ if over_cap:+ too_many.add(person_id)+ else:+ all_distinct_ids.extend(d.id for d in dids)+ mapping_states = _ch_mapping_states(team_id, all_distinct_ids) if all_distinct_ids else {}++ plans: list[_PersonPlan] = []+ for person_uuid in person_uuids:+ person = live.get(person_uuid)+ if person is None:+ plans.append(+ _PersonPlan(+ team_id=team_id,+ person_uuid=person_uuid,+ person=None,+ kind=None,+ ch_max_version=None,+ target_version=None,+ mappings=[],+ )+ )+ continue+ pg_version = int(person.version or 0)+ state = person_states.get(person_uuid)+ ch_max_version = state.max_version if state is not None else None+ person_distinct_ids = distinct_ids_by_person.get(person.pk, [])+ too_many_distinct_ids = person.pk in too_many+ mappings = []+ for mapping in [] if too_many_distinct_ids else person_distinct_ids:+ mapping_state = mapping_states.get(mapping.id)+ mapping_ch_max = mapping_state.max_version if mapping_state is not None else None+ mappings.append(+ _MappingPlan(+ distinct_id=mapping.id,+ kind=_mapping_kind(person_uuid, mapping.version, mapping_state),+ pg_version=mapping.version,+ ch_max_version=mapping_ch_max,+ target_version=_target_version(mapping.version, mapping_ch_max),+ )+ )+ plans.append(+ _PersonPlan(+ team_id=team_id,+ person_uuid=person_uuid,+ person=person,+ kind=_person_kind(pg_version, state),+ ch_max_version=ch_max_version,+ target_version=_target_version(pg_version, ch_max_version),+ mappings=mappings,+ too_many_distinct_ids=too_many_distinct_ids,+ )+ )+ return plansdiff --git a/posthog/models/person/divergence/repairs.py b/posthog/models/person/divergence/repairs.pynew file mode 100644index 000000000000..3db3b3e8215e--- /dev/null+++ b/posthog/models/person/divergence/repairs.py@@ -0,0 +1,484 @@+"""Executes repair plans: version raises, publishes, pacing, retries and delivery tracking."""++from __future__ import annotations++import time+from collections import Counter, defaultdict+from collections.abc import Callable, Sequence+from dataclasses import field+from functools import partial+from typing import TypeVar+from uuid import UUID++import grpc++from posthog.dataclasses import frozen+from posthog.errors import CH_TRANSIENT_ERRORS+from posthog.exceptions import ClickHouseQueryTimeOut+from posthog.kafka_client.client import ClickhouseProducer, ProduceResult+from posthog.kafka_client.topics import KAFKA_PERSON+from posthog.models.person import Person+from posthog.models.person.sql import INSERT_PERSON_SQL+from posthog.models.person.util import (+ _flush_person_producers,+ _person_row,+ create_person_distinct_id,+ get_person_by_distinct_id,+ get_persons_by_uuids,+)+from posthog.models.team import Team+from posthog.personhog_client.client import personhog_call, require_personhog_client+from posthog.personhog_client.interceptor import is_transient_rpc_error+from posthog.personhog_client.proto import (+ CONSISTENCY_LEVEL_STRONG,+ GetDistinctIdsForPersonsRequest,+ ReadOptions,+ SetPersonDistinctIdVersionFloorRequest,+ SetPersonVersionFloorRequest,+)++from . import datatypes, lookups, repair_plan, stale_merge++_T = TypeVar("_T")+++_REPAIR_CHUNK_SIZE = 100++# The waits between re-reads give the replica about 0.5 s to show a raise, far above its usual lag of a few milliseconds.+_REREAD_BACKOFF_SECONDS = (0.025, 0.05, 0.1, 0.15, 0.175)++_TRANSIENT_RETRY_BACKOFF_SECONDS = (1.0, 2.0)++_TRANSIENT_RPC_CODES = frozenset(+ {grpc.StatusCode.INTERNAL, grpc.StatusCode.UNAVAILABLE, grpc.StatusCode.DEADLINE_EXCEEDED}+)++_FLUSH_TIMEOUT_SECONDS = 5 * 60++# Confirmed produce results are dropped this often, so a long repair does not hold one per published row.+_DELIVERY_PRUNE_EVERY = 1_000++_sleep = time.sleep+++def _raise_person_version_floor(team_id: int, person_id: int, min_version: int) -> None:+ personhog_call(+ "person_divergence_set_person_version_floor",+ lambda: require_personhog_client().set_person_version_floor(+ SetPersonVersionFloorRequest(team_id=team_id, person_id=person_id, min_version=min_version)+ ),+ )+++def _raise_mapping_version_floor(team_id: int, distinct_id: str, min_version: int) -> str | None:+ """Raise the mapping's version on the primary and return the uuid of the person it maps to there."""+ response = personhog_call(+ "person_divergence_set_person_distinct_id_version_floor",+ lambda: require_personhog_client().set_person_distinct_id_version_floor(+ SetPersonDistinctIdVersionFloorRequest(team_id=team_id, distinct_id=distinct_id, min_version=min_version)+ ),+ )+ if not response.HasField("person"):+ return None+ return str(UUID(response.person.uuid))+++def _primary_mapping_versions(team_id: int, person_id: int) -> dict[str, int]:+ response = personhog_call(+ "person_divergence_confirm_mapping_versions",+ lambda: require_personhog_client().get_distinct_ids_for_persons(+ GetDistinctIdsForPersonsRequest(+ team_id=team_id,+ person_ids=[person_id],+ read_options=ReadOptions(consistency=CONSISTENCY_LEVEL_STRONG),+ limit_per_person=repair_plan._MAX_REPAIR_DISTINCT_IDS_PER_PERSON + 1,+ )+ ),+ )+ return {d.distinct_id: int(d.version or 0) for pd in response.person_distinct_ids for d in pd.distinct_ids}+++def _person_action(plan: repair_plan._PersonPlan, outcome: datatypes.RepairOutcome) -> datatypes.RepairAction:+ return datatypes.RepairAction(+ team_id=plan.team_id,+ person_uuid=plan.person_uuid,+ distinct_id=None,+ kind=plan.kind,+ pg_version=int(plan.person.version or 0) if plan.person is not None else None,+ ch_max_version=plan.ch_max_version,+ target_version=plan.target_version if plan.kind is not None else None,+ outcome=outcome,+ )+++def _too_many_distinct_ids_action(plan: repair_plan._PersonPlan) -> datatypes.RepairAction:+ return datatypes.RepairAction(+ team_id=plan.team_id,+ person_uuid=plan.person_uuid,+ distinct_id=None,+ kind=None,+ pg_version=None,+ ch_max_version=None,+ target_version=None,+ outcome="skipped_too_many_distinct_ids",+ )+++def _mapping_action(+ plan: repair_plan._PersonPlan, mapping: repair_plan._MappingPlan, outcome: datatypes.RepairOutcome+) -> datatypes.RepairAction:+ return datatypes.RepairAction(+ team_id=plan.team_id,+ person_uuid=plan.person_uuid,+ distinct_id=mapping.distinct_id,+ kind=mapping.kind,+ pg_version=mapping.pg_version,+ ch_max_version=mapping.ch_max_version,+ target_version=mapping.target_version if mapping.kind is not None else None,+ outcome=outcome,+ )+++def _publish_person(team_id: int, person: Person) -> ProduceResult:+ row = _person_row(+ team_id=team_id,+ uuid=str(person.uuid),+ version=int(person.version or 0),+ properties=person.properties,+ is_identified=person.is_identified,+ is_deleted=False,+ created_at=person.created_at,+ last_seen_at=person.last_seen_at,+ )+ # _person_row fills a missing last_seen_at with the current hour, but ingestion publishes null for it.+ if person.last_seen_at is None:+ row["last_seen_at"] = None+ return ClickhouseProducer().produce(topic=KAFKA_PERSON, sql=INSERT_PERSON_SQL, data=row)+++def _reread_person(team_id: int, person_uuid: str, target_version: int | None) -> Person | None:+ """Read the person from the replica again while it shows a version below ``target_version``."""+ found = get_persons_by_uuids(team_id, [person_uuid], distinct_id_limit=0)+ reread = found[0] if found else None+ for delay in _REREAD_BACKOFF_SECONDS:+ if reread is None or target_version is None or int(reread.version or 0) >= target_version:+ break+ _sleep(delay)+ found = get_persons_by_uuids(team_id, [person_uuid], distinct_id_limit=0)+ reread = found[0] if found else None+ return reread+++def _is_transient(exc: BaseException) -> bool:+ return isinstance(exc, (*CH_TRANSIENT_ERRORS, ClickHouseQueryTimeOut)) or is_transient_rpc_error(+ exc, codes=_TRANSIENT_RPC_CODES+ )+++def _retry_transient(fn: Callable[[], _T], *, what: str, log: Callable[[str], None]) -> _T:+ """Run ``fn`` again after a transient ClickHouse or personhog error, then re-raise once the retries run out.++ Repeating a step is safe because every repair write is idempotent: a version raise never lowers a+ version, and ClickHouse keeps one row per version.+ """+ for delay in _TRANSIENT_RETRY_BACKOFF_SECONDS:+ try:+ return fn()+ except Exception as exc:+ if not _is_transient(exc):+ raise+ log(f"{what}: {type(exc).__name__}, retrying in {delay:g} s")+ _sleep(delay)+ return fn()+++def _execute_plan(+ plan: repair_plan._PersonPlan,+ *,+ apply: bool,+ include_stale: bool,+ before_write: Callable[[], None],+ published: Callable[[ProduceResult], None],+ stale_merge_distinct_id: str | None = None,+) -> list[datatypes.RepairAction]:+ person = plan.person+ if person is None:+ return [_person_action(plan, "skipped_not_live")]+ if plan.kind == "stale" and not include_stale:+ refusal: datatypes.RepairOutcome | None = "skipped_stale"+ if apply and stale_merge_distinct_id is not None:+ missing = stale_merge._clickhouse_only_properties(plan)+ # A later reset that finds nothing missing raises Postgres. Capture accepting the $set does not mean+ # ingestion applies it, and a raise without the merged properties lets the next update overwrite them.+ refusal = (+ (stale_merge._queue_stale_merge(plan, stale_merge_distinct_id, missing) or "merge_queued")+ if missing+ else None+ )+ if refusal is not None:+ return [+ _person_action(plan, refusal),+ *(+ _mapping_action(plan, m, "skipped_not_divergent" if m.kind is None else "skipped_stale")+ for m in plan.mappings+ ),+ *([_too_many_distinct_ids_action(plan)] if plan.too_many_distinct_ids else []),+ ]++ divergent_mappings = [m for m in plan.mappings if m.kind is not None]+ mapping_actions = [_mapping_action(plan, m, "skipped_not_divergent") for m in plan.mappings if m.kind is None]+ if plan.too_many_distinct_ids:+ mapping_actions.append(_too_many_distinct_ids_action(plan))+ if not apply:+ person_outcome: datatypes.RepairOutcome = "would_repair" if plan.kind is not None else "skipped_not_divergent"+ return [+ _person_action(plan, person_outcome),+ *mapping_actions,+ *(_mapping_action(plan, m, "would_repair") for m in divergent_mappings),+ ]++ pg_version = int(person.version or 0)+ if plan.kind is not None and plan.target_version is not None and plan.target_version > pg_version:+ before_write()+ _raise_person_version_floor(plan.team_id, person.pk, plan.target_version)++ # SetPersonDistinctIdVersionFloor runs on the primary, so its reply is the owner check: the replica list+ # that produced this mapping can lag a merge that moved it to another person.+ owned: list[repair_plan._MappingPlan] = []+ for mapping in divergent_mappings:+ before_write()+ owner = _raise_mapping_version_floor(plan.team_id, mapping.distinct_id, mapping.target_version)+ if owner is None:+ mapping_actions.append(_mapping_action(plan, mapping, "skipped_mapping_gone"))+ elif owner != plan.person_uuid:+ mapping_actions.append(_mapping_action(plan, mapping, "skipped_owner_changed"))+ else:+ owned.append(mapping)++ # Re-read after the raise so the published properties are the ones Postgres holds at the+ # published version, including any ingestion update that landed after the first read.+ reread: Person | None = None+ if plan.kind is not None:+ reread = _reread_person(plan.team_id, plan.person_uuid, plan.target_version)++ if (plan.kind is not None or owned) and lookups._tombstoned_uuids(plan.team_id, [plan.person_uuid]):+ person_outcome = "skipped_tombstoned" if plan.kind is not None else "skipped_not_divergent"+ return [+ _person_action(plan, person_outcome),+ *mapping_actions,+ *(_mapping_action(plan, m, "skipped_tombstoned") for m in owned),+ ]++ if plan.kind is None:+ person_outcome = "skipped_not_divergent"+ elif reread is None:+ person_outcome = "skipped_not_live"+ elif plan.target_version is None or int(reread.version or 0) < plan.target_version:+ # The replica has not caught up with the raise, so its properties may predate the raised version.+ person_outcome = "skipped_reread_lagging"+ else:+ published(_publish_person(plan.team_id, reread))+ person_outcome = "repaired"++ # The SetPersonDistinctIdVersionFloor reply does not say whether it wrote, and a personhog-replica build+ # without NULL-version handling leaves a NULL version unchanged, so publish only the mappings whose+ # version on the primary reached the target.+ stored_versions = _primary_mapping_versions(plan.team_id, person.pk) if owned else {}+ for mapping in owned:+ stored_version = stored_versions.get(mapping.distinct_id)+ if stored_version is None:+ # The primary no longer lists the mapping under this person: it moved or was tombstoned after the raise.+ mapping_actions.append(_mapping_action(plan, mapping, "skipped_mapping_gone"))+ elif stored_version < mapping.target_version:+ mapping_actions.append(_mapping_action(plan, mapping, "skipped_reread_lagging"))+ else:+ published(+ create_person_distinct_id(+ team_id=plan.team_id,+ distinct_id=mapping.distinct_id,+ person_id=plan.person_uuid,+ version=stored_version,+ is_deleted=False,+ )+ )+ mapping_actions.append(_mapping_action(plan, mapping, "repaired"))+ return [_person_action(plan, person_outcome), *mapping_actions]+++@frozen(frozen=False)+class _WritePacer:+ """Space writes at least 1 / max_per_second apart, so idle time never buys a later burst."""++ max_per_second: float | None+ next_write_at: float = 0.0++ def before_write(self) -> None:+ if self.max_per_second is None:+ return+ now = time.monotonic()+ if now < self.next_write_at:+ time.sleep(self.next_write_at - now)+ now = self.next_write_at+ self.next_write_at = now + 1 / self.max_per_second+++@frozen(frozen=False)+class _Deliveries:+ """Kafka produce results of the published rows, so a failed delivery counts as undelivered."""++ pending: list[ProduceResult] = field(default_factory=list)+ failed: int = 0++ def track(self, result: ProduceResult) -> None:+ self.pending.append(result)+ if len(self.pending) >= _DELIVERY_PRUNE_EVERY:+ self._prune()++ def undelivered(self, timeout: float) -> int:+ _flush_person_producers(timeout)+ self._prune()+ return self.failed + len(self.pending)++ def _prune(self) -> None:+ waiting: list[ProduceResult] = []+ for result in self.pending:+ if not result.done():+ waiting.append(result)+ continue+ try:+ result.get(timeout=0)+ except Exception:+ self.failed += 1+ self.pending = waiting+++@frozen(frozen=False)+class _OutcomeCounts:+ person: Counter[datatypes.RepairOutcome] = field(default_factory=Counter)+ mapping: Counter[datatypes.RepairOutcome] = field(default_factory=Counter)++ def add(self, action: datatypes.RepairAction) -> None:+ is_person_row = action.distinct_id is None and action.outcome != "skipped_too_many_distinct_ids"+ (self.person if is_person_row else self.mapping)[action.outcome] += 1++ def summary(self, *, applied: bool, persons: int, undelivered: int) -> datatypes.RepairSummary:+ return datatypes.RepairSummary(+ applied=applied,+ persons=persons,+ person_outcomes=dict(self.person),+ mapping_outcomes=dict(self.mapping),+ undelivered=undelivered,+ )+++def repair_persons(+ targets: Sequence[datatypes.PersonRef],+ *,+ apply: bool,+ include_stale: bool = False,+ max_writes_per_second: float | None = None,+ on_action: Callable[[datatypes.RepairAction], None],+ log: Callable[[str], None],+) -> datatypes.RepairSummary:+ """Republish the Postgres state of each target person and its mappings where ClickHouse disagrees.++ A rerun is safe: each Postgres write only raises a version, and a person whose publish never landed+ still counts as divergent. ``max_writes_per_second`` paces each Postgres write, one per divergent+ person or distinct id.++ ``include_stale`` also repairs stale persons and their mappings. That replaces their ClickHouse properties+ with the Postgres ones for good, so never use it on a team waiting for a restore from ClickHouse.+ """+ if max_writes_per_second is not None and max_writes_per_second <= 0:+ raise ValueError("max_writes_per_second must be above 0")+ by_team: dict[int, list[str]] = defaultdict(list)+ for target in dict.fromkeys(targets):+ by_team[target.team_id].append(target.person_uuid)++ outcomes = _OutcomeCounts()+ processed = 0+ undelivered = 0+ pacer = _WritePacer(max_per_second=max_writes_per_second)+ deliveries = _Deliveries()+ existing_teams = set(Team.objects.filter(id__in=list(by_team)).values_list("id", flat=True))+ try:+ for team_id, person_uuids in sorted(by_team.items()):+ if team_id not in existing_teams:+ # A deleted team's ClickHouse rows go with the team, so a repair would only republish what the deletion removed.+ for person_uuid in person_uuids:+ action = datatypes.RepairAction(+ team_id=team_id,+ person_uuid=person_uuid,+ distinct_id=None,+ kind=None,+ pg_version=None,+ ch_max_version=None,+ target_version=None,+ outcome="skipped_team_gone",+ )+ outcomes.add(action)+ on_action(action)+ processed += 1+ log(f"team {team_id}: no longer exists, {len(person_uuids)} persons skipped")+ continue+ for chunk in lookups._chunks(person_uuids, _REPAIR_CHUNK_SIZE):+ plans = _retry_transient(+ partial(repair_plan._plan_chunk, team_id, chunk), what=f"team {team_id}: planning", log=log+ )+ for plan in plans:+ # Only the attempt that succeeds returns actions, so a retried person is counted once.+ actions = _retry_transient(+ partial(+ _execute_plan,+ plan,+ apply=apply,+ include_stale=include_stale,+ before_write=pacer.before_write,+ published=deliveries.track,+ ),+ what=f"team {team_id} person {plan.person_uuid}",+ log=log,+ )+ for action in actions:+ outcomes.add(action)+ on_action(action)+ processed += 1+ log(f"team {team_id}: {len(person_uuids)} persons, {processed} processed in total")+ finally:+ if apply:+ undelivered = deliveries.undelivered(_FLUSH_TIMEOUT_SECONDS)+ if undelivered:+ log(f"{undelivered} ClickHouse messages were not delivered; rerun the repair for the same input")+ return outcomes.summary(applied=apply, persons=processed, undelivered=undelivered)+++def repair_distinct_id(+ team_id: int, distinct_id: str, *, delivery_timeout_seconds: float+) -> datatypes.RepairSummary | None:+ """Repair the live person that owns ``distinct_id``, and that one mapping, where ClickHouse disagrees.++ It always applies. For a stale person whose ClickHouse winner holds properties Postgres lacks, it sends them+ to ingestion as $set_once, so newer Postgres values win, and writes nothing else. A later reset, once+ ingestion applied them, raises Postgres above ClickHouse and republishes. It leaves the person stale when the+ team has property access rules.+ """+ owner = get_person_by_distinct_id(team_id, distinct_id, distinct_id_limit=0)+ if owner is None:+ return None+ outcomes = _OutcomeCounts()+ deliveries = _Deliveries()+ undelivered = 0+ try:+ for plan in repair_plan._plan_chunk(team_id, [str(owner.uuid)], only_distinct_id=distinct_id):+ for action in _execute_plan(+ plan,+ apply=True,+ include_stale=False,+ before_write=lambda: None,+ published=deliveries.track,+ stale_merge_distinct_id=distinct_id,+ ):+ outcomes.add(action)+ finally:+ undelivered = deliveries.undelivered(delivery_timeout_seconds)+ return outcomes.summary(applied=True, persons=1, undelivered=undelivered)diff --git a/posthog/models/person/divergence/stale_merge.py b/posthog/models/person/divergence/stale_merge.pynew file mode 100644index 000000000000..e2e726881173--- /dev/null+++ b/posthog/models/person/divergence/stale_merge.py@@ -0,0 +1,64 @@+"""Sends a stale person's ClickHouse-only properties to ingestion before Postgres is raised."""++from __future__ import annotations++import json+from datetime import UTC, datetime+from typing import Any++from posthog.api.capture import capture_internal+from posthog.models.team import Team++from products.access_control.backend.facade.api import team_has_property_access_rules++from . import datatypes, lookups, repair_plan++# Well under capture's 1,000,000-byte Kafka message limit and ingestion's 655,360-byte person properties limit+# (trimmed to 512 KiB), so ingestion applies the merge whole.+_STALE_MERGE_MAX_BYTES = 256 * 1024++_PERSON_WINNER_PROPERTIES_SQL = """+SELECT argMaxIf(properties, version, _row_exists)+FROM person+WHERE team_id = %(team_id)s AND id = %(person_uuid)s+"""+++def _clickhouse_only_properties(plan: repair_plan._PersonPlan) -> dict[str, Any]:+ """The properties of the ClickHouse winner whose keys the Postgres person lacks."""+ assert plan.person is not None+ rows = lookups._ch(+ _PERSON_WINNER_PROPERTIES_SQL,+ {"team_id": plan.team_id, "person_uuid": plan.person_uuid},+ repair_plan._REPAIR_READ_SETTINGS,+ )+ ch_properties = json.loads(rows[0][0]) if rows and rows[0][0] else {}+ pg_properties = plan.person.properties or {}+ return {key: value for key, value in ch_properties.items() if key not in pg_properties}+++def _queue_stale_merge(+ plan: repair_plan._PersonPlan, distinct_id: str, missing: dict[str, Any]+) -> datatypes.RepairOutcome | None:+ """Send the ClickHouse-only properties to ingestion as $set_once, or return why they were not sent."""+ assert plan.person is not None+ # Every member can read the $set event, and event reads apply event property restrictions only, so a team with+ # any property rule could expose a person property that some member is denied.+ if team_has_property_access_rules(team_id=plan.team_id):+ return "skipped_stale_merge_restricted"+ if len(json.dumps({**missing, **(plan.person.properties or {})})) > _STALE_MERGE_MAX_BYTES:+ return "skipped_stale_merge_too_large"+ try:+ token = Team.objects.only("api_token").get(id=plan.team_id).api_token+ capture_internal(+ token=token,+ event_name="$set",+ event_source="person_divergence_reset",+ distinct_id=distinct_id,+ timestamp=datetime.now(UTC),+ properties={"$set_once": missing},+ process_person_profile=True,+ ).raise_for_status()+ except Exception:+ return "skipped_stale_merge_failed"+ return Nonediff --git a/posthog/models/person/divergence/team_check.py b/posthog/models/person/divergence/team_check.pynew file mode 100644index 000000000000..74411a6f7888--- /dev/null+++ b/posthog/models/person/divergence/team_check.py@@ -0,0 +1,85 @@+"""Per-team damage gauge: how many old ClickHouse persons and mappings Postgres still holds."""++from __future__ import annotations++from datetime import datetime++from posthog.models.person.util import _batched_get_persons_by_distinct_ids+from posthog.personhog_client.client import personhog_call++from . import datatypes, lookups++_TEAM_CHECK_SETTINGS = {"max_execution_time": 900, "max_memory_usage": 32_000_000_000}++# The hash pre-filter keeps about this many times the sample, so the sample stays full size and uniform.+_TEAM_CHECK_OVERSAMPLE = 10++_TEAM_PERSON_ROWS_SQL = "SELECT count() FROM person WHERE team_id = %(team_id)s"++_TEAM_MAPPING_ROWS_SQL = "SELECT count() FROM person_distinct_id2 WHERE team_id = %(team_id)s"++_TEAM_PERSONS_SQL = """+SELECT toString(id)+FROM person+WHERE team_id = %(team_id)s AND cityHash64(id) %% %(modulus)s = 0+GROUP BY id+HAVING argMax(is_deleted, version) = 0 AND max(_timestamp) < fromUnixTimestamp(%(before)s)+ORDER BY cityHash64(id)+LIMIT %(limit)s+"""++_TEAM_MAPPINGS_SQL = """+SELECT distinct_id+FROM person_distinct_id2+WHERE team_id = %(team_id)s AND cityHash64(distinct_id) %% %(modulus)s = 0+GROUP BY distinct_id+HAVING argMax(is_deleted, version) = 0 AND max(_timestamp) < fromUnixTimestamp(%(before)s)+ORDER BY cityHash64(distinct_id)+LIMIT %(limit)s+"""+++def _team_check_modulus(rows: int, sample_size: int) -> int:+ return max(1, rows // max(1, sample_size * _TEAM_CHECK_OVERSAMPLE))+++def _team_check_args(rows_sql: str, *, team_id: int, sample_size: int, before: datetime) -> dict[str, int]:+ [[rows]] = lookups._ch(rows_sql, {"team_id": team_id}, _TEAM_CHECK_SETTINGS)+ return {+ "team_id": team_id,+ "before": int(before.timestamp()),+ "limit": sample_size,+ "modulus": _team_check_modulus(int(rows), sample_size),+ }+++def check_team(*, team_id: int, sample_size: int, before: datetime) -> datatypes.TeamCheck:+ """Count how many of a team's old live ClickHouse persons and mappings Postgres still holds live."""+ person_args = _team_check_args(_TEAM_PERSON_ROWS_SQL, team_id=team_id, sample_size=sample_size, before=before)+ mapping_args = _team_check_args(_TEAM_MAPPING_ROWS_SQL, team_id=team_id, sample_size=sample_size, before=before)+ person_uuids = [row[0] for row in lookups._ch(_TEAM_PERSONS_SQL, person_args, _TEAM_CHECK_SETTINGS)]+ distinct_ids = [row[0] for row in lookups._ch(_TEAM_MAPPINGS_SQL, mapping_args, _TEAM_CHECK_SETTINGS)]+ live_persons = (+ lookups._live_person_versions(team_id, person_uuids, "person_divergence_team_check") if person_uuids else {}+ )+ live_mappings = (+ personhog_call(+ "person_divergence_team_check",+ lambda: _batched_get_persons_by_distinct_ids(+ team_id,+ distinct_ids,+ "person_divergence_team_check",+ deduplicate_by_person=False,+ read_options=lookups._VERSION_ONLY_READ_OPTIONS,+ ),+ )+ if distinct_ids+ else []+ )+ return datatypes.TeamCheck(+ team_id=team_id,+ persons_sampled=len(person_uuids),+ persons_live_in_postgres=len(live_persons),+ distinct_ids_sampled=len(distinct_ids),+ distinct_ids_live_in_postgres=len({r.distinct_id for r in live_mappings}),+ )diff --git a/posthog/models/person/test/test_divergence.py b/posthog/models/person/test/test_divergence.pyindex 9fc18888c3a4..d250b7e2f1a8 100644--- a/posthog/models/person/test/test_divergence.py+++ b/posthog/models/person/test/test_divergence.py@@ -21,7 +21,7 @@ from posthog.kafka_client.client import ClickhouseProducer, ProduceResultfrom posthog.kafka_client.topics import KAFKA_PERSONfrom posthog.models import OrganizationMembership, PropertyDefinition, Team, Userfrom posthog.models.person import Person-from posthog.models.person.divergence import (+from posthog.models.person.divergence.datatypes import (DivergentPerson,MappingDivergenceKind,PersonDivergenceKind,@@ -34,16 +34,15 @@ from posthog.models.person.divergence import (SampleSummary,ScanSummary,TeamCheck,- _team_check_modulus,- _WritePacer,- check_team,- repair_distinct_id,- repair_persons,+)+from posthog.models.person.divergence.fleet_scans import (scan_hidden_persons,scan_sample,scan_stale_persons,scan_swept_persons,)+from posthog.models.person.divergence.repairs import _WritePacer, repair_distinct_id, repair_persons+from posthog.models.person.divergence.team_check import _team_check_modulus, check_teamfrom posthog.models.person.sql import BULK_INSERT_PERSON_DISTINCT_ID2from posthog.models.person.util import (create_person as create_person_in_ch,@@ -515,7 +514,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):self._ch_person_row(person.uuid, 103, deleted=True)self._ch_mapping_row("throttled-did", person.uuid, 100, deleted=True)- with patch("posthog.models.person.divergence.time") as clock:+ with patch("posthog.models.person.divergence.repairs.time") as clock:clock.monotonic.return_value = 0.0repair_persons([PersonRef(team_id=self.team.pk, person_uuid=str(person.uuid))],@@ -592,7 +591,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):with (patch.object(fake, "set_person_version_floor", side_effect=primary_only_raise),patch.object(fake, "get_persons_by_uuids", side_effect=lagging_replica),- patch("posthog.models.person.divergence._sleep") as sleep,+ patch("posthog.models.person.divergence.repairs._sleep") as sleep,):_, actions = self._repair(person.uuid)@@ -616,7 +615,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):person = self._divergent_person("hidden")failure = _rpc_error(error) if isinstance(error, grpc.StatusCode) else error- with self._failing_once(step, failure), patch("posthog.models.person.divergence._sleep") as sleep:+ with self._failing_once(step, failure), patch("posthog.models.person.divergence.repairs._sleep") as sleep:summary, actions = self._repair(person.uuid)assert [a.outcome for a in actions] == ["repaired"]@@ -638,7 +637,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):with (patch.object(fake, "set_person_version_floor", side_effect=failure) as raise_floor,- patch("posthog.models.person.divergence._sleep") as sleep,+ patch("posthog.models.person.divergence.repairs._sleep") as sleep,):with self.assertRaises(type(failure)):self._repair(person.uuid)@@ -658,7 +657,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):raise failurereturn query(*args, **kwargs)- return patch("posthog.models.person.divergence.sync_execute", side_effect=ch_fails_once)+ return patch("posthog.models.person.divergence.lookups.sync_execute", side_effect=ch_fails_once)raise_floor = fake.set_person_version_floorcalls: list[bool] = []@@ -882,7 +881,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):for distinct_id in distinct_ids:self._ch_mapping_row(distinct_id, person.uuid, 100, deleted=True)- with patch("posthog.models.person.divergence._MAX_REPAIR_DISTINCT_IDS_PER_PERSON", cap):+ with patch("posthog.models.person.divergence.repair_plan._MAX_REPAIR_DISTINCT_IDS_PER_PERSON", cap):summary = repair_distinct_id(self.team.pk, "target", delivery_timeout_seconds=1)assert summary is not None@@ -921,7 +920,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):stored = get_active_fake().stored_person(self.team.pk, str(person.uuid))assert stored is not None- with patch("posthog.models.person.divergence.capture_internal") as capture:+ with patch("posthog.models.person.divergence.stale_merge.capture_internal") as capture:first = repair_distinct_id(self.team.pk, "target", delivery_timeout_seconds=1)assert first is not None@@ -943,7 +942,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):assert self._ch_person(person.uuid) == (0, 10, {**CH_PROPERTIES, **CH_ONLY_PROPERTIES})stored.properties = json.dumps({**CH_ONLY_PROPERTIES, **PG_PROPERTIES}).encode()- with patch("posthog.models.person.divergence.capture_internal") as capture:+ with patch("posthog.models.person.divergence.stale_merge.capture_internal") as capture:second = repair_distinct_id(self.team.pk, "target", delivery_timeout_seconds=1)assert second is not None@@ -956,7 +955,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):def test_reset_republishes_a_stale_person_whose_clickhouse_properties_postgres_already_holds(self) -> None:person = self._stale_person_with_target(CH_PROPERTIES)- with patch("posthog.models.person.divergence.capture_internal") as capture:+ with patch("posthog.models.person.divergence.stale_merge.capture_internal") as capture:summary = repair_distinct_id(self.team.pk, "target", delivery_timeout_seconds=1)assert summary is not None@@ -981,8 +980,8 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):failing.raise_for_status.side_effect = RuntimeError("capture rejected the event")with (- patch("posthog.models.person.divergence._STALE_MERGE_MAX_BYTES", max_bytes),- patch("posthog.models.person.divergence.capture_internal", return_value=failing) as capture,+ patch("posthog.models.person.divergence.stale_merge._STALE_MERGE_MAX_BYTES", max_bytes),+ patch("posthog.models.person.divergence.stale_merge.capture_internal", return_value=failing) as capture,):summary = repair_distinct_id(self.team.pk, "target", delivery_timeout_seconds=1)@@ -999,7 +998,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):self._ch_person_row(person.uuid, 103, deleted=True)self._ch_mapping_row("a", person.uuid, 100, deleted=True)- with patch("posthog.models.person.divergence._MAX_REPAIR_DISTINCT_IDS_PER_PERSON", 2):+ with patch("posthog.models.person.divergence.repair_plan._MAX_REPAIR_DISTINCT_IDS_PER_PERSON", 2):summary, actions = self._repair(person.uuid)assert actions == [@@ -1079,7 +1078,7 @@ class TestWritePacer(SimpleTestCase):sleeps.append(seconds)clock[0] += seconds- with patch("posthog.models.person.divergence.time") as time_module:+ with patch("posthog.models.person.divergence.repairs.time") as time_module:time_module.monotonic.side_effect = lambda: clock[0]time_module.sleep.side_effect = sleeppacer = _WritePacer(max_per_second=2)@@ -1113,7 +1112,7 @@ class TestScanTeamRanges(SimpleTestCase):scanned.extend(teams)return []- with patch("posthog.models.person.divergence.sync_execute", side_effect=query):+ with patch("posthog.models.person.divergence.lookups.sync_execute", side_effect=query):if scan == "stale":summary: ScanSummary | SampleSummary = scan_stale_persons(window_days=60,