Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(persons): add restore and merge repair verbs (#117445)

Eli Reisman4a01dbed8a6b
diff --git a/posthog/models/person/divergence/restore_verbs.py b/posthog/models/person/divergence/restore_verbs.py
new file mode 100644
index 000000000000..fe7d34212b03
--- /dev/null
+++ b/posthog/models/person/divergence/restore_verbs.py
@@ -0,0 +1,461 @@
+"""Repair verbs that rebuild Postgres from ClickHouse: restore a lost person with its distinct ids, or merge two
+live persons through ingestion."""
+
+from __future__ import annotations
+
+import json
+import time
+from collections import defaultdict
+from collections.abc import Sequence
+from datetime import UTC, datetime
+from functools import partial
+from typing import TYPE_CHECKING, Any
+
+from posthog.api.capture import capture_internal
+from posthog.dataclasses import frozen
+from posthog.kafka_client.client import ClickhouseProducer
+from posthog.kafka_client.topics import KAFKA_PERSON
+from posthog.models.person.sql import INSERT_PERSON_SQL
+from posthog.models.person.util import (
+ _person_row,
+ create_person_distinct_id,
+ get_distinct_id_rows,
+ get_persons_by_uuids,
+)
+from posthog.models.team import Team
+from posthog.personhog_client.client import personhog_call, require_personhog_client
+from posthog.personhog_client.proto import (
+ DISTINCT_ID_WRITE_ACTION_INSERT,
+ DISTINCT_ID_WRITE_ACTION_RAISE,
+ DISTINCT_ID_WRITE_OUTCOME_APPLIED,
+ DISTINCT_ID_WRITE_OUTCOME_PERSON_NOT_LIVE,
+ DISTINCT_ID_WRITE_OUTCOME_UNCHANGED,
+ RESTORE_OUTCOME_EXISTS_LIVE,
+ RESTORE_OUTCOME_INSERTED,
+ RESTORE_OUTCOME_NO_DISTINCT_ID,
+ DistinctIdWrite,
+ RestoredDistinctId,
+ RestoredPerson,
+ RestorePersonsRequest,
+ WriteDistinctIdsRequest,
+)
+
+from . import lookups, observe, repairs
+
+if TYPE_CHECKING:
+ from .verbs import VerbContext
+
+_LIVE = "is_deleted = 0 AND _row_exists"
+_PAYLOAD_SQL = f"""
+SELECT
+ toString(id),
+ max(version),
+ countIf({_LIVE}),
+ maxIf(version, {_LIVE}),
+ maxIfOrNull(version, is_deleted = 1 AND _row_exists),
+ argMaxIf(properties, version, {_LIVE}),
+ argMaxIf(is_identified, version, {_LIVE}),
+ argMaxIf(created_at, version, {_LIVE})
+FROM person
+WHERE team_id = %(team_id)s AND id IN %(person_uuids)s
+GROUP BY id
+"""
+
+
+@frozen
+class _Payload:
+ max_version: int
+ properties: dict[str, Any]
+ is_identified: bool
+ created_at: datetime
+
+
+def _live_payloads(team_id: int, person_uuids: Sequence[str]) -> dict[str, _Payload]:
+ """The ClickHouse winner of each person that ClickHouse still shows live, with what a restore copies."""
+ payloads: dict[str, _Payload] = {}
+ for chunk in lookups._chunks(list(person_uuids), 1_000):
+ rows = lookups._ch(_PAYLOAD_SQL, {"team_id": team_id, "person_uuids": list(chunk)}, observe._READ_SETTINGS)
+ for person_uuid, max_version, live_rows, live_max, tomb_max, properties, is_identified, created_at in rows:
+ # A tie counts as visible, as in the classifier: ClickHouse may return either row.
+ if not live_rows or (tomb_max is not None and int(tomb_max) > int(live_max)):
+ continue
+ payloads[str(person_uuid)] = _Payload(
+ max_version=int(max_version),
+ properties=json.loads(properties) if properties else {},
+ is_identified=bool(is_identified),
+ created_at=created_at if created_at.tzinfo else created_at.replace(tzinfo=UTC),
+ )
+ return payloads
+
+
+def _publish_person(team_id: int, person_uuid: str, payload: _Payload, version: int) -> Any:
+ row = _person_row(
+ team_id=team_id,
+ uuid=person_uuid,
+ version=version,
+ properties=payload.properties,
+ is_identified=payload.is_identified,
+ is_deleted=False,
+ created_at=payload.created_at,
+ last_seen_at=None,
+ )
+ row["last_seen_at"] = None
+ return ClickhouseProducer().produce(topic=KAFKA_PERSON, sql=INSERT_PERSON_SQL, data=row)
+
+
+def _mapping_write(row: dict[str, Any], target: str, ch_max: int | None) -> DistinctIdWrite:
+ """Insert where Postgres has no row; otherwise raise the row Postgres already points at the target, while it
+ holds the reported version."""
+ pg = row["evidence"]["pg"]
+ min_version = max(int(pg["version"]) if pg else 0, (ch_max + 1) if ch_max is not None else 0)
+ if pg is None:
+ return DistinctIdWrite(
+ distinct_id=row["key"], action=DISTINCT_ID_WRITE_ACTION_INSERT, person_uuid=target, min_version=min_version
+ )
+ return DistinctIdWrite(
+ distinct_id=row["key"],
+ action=DISTINCT_ID_WRITE_ACTION_RAISE,
+ person_uuid=target,
+ min_version=min_version,
+ expected_version=int(pg["version"]),
+ )
+
+
+def _write_mappings(context: VerbContext, rows: list[dict[str, Any]], targets: dict[str, str], done: str) -> list[str]:
+ """Point each row's mapping at its target person: insert where Postgres has no row, raise where Postgres
+ already names the target. Publishes what Postgres stores and returns the distinct ids that now point there."""
+ moved: list[str] = []
+ for batch in lookups._chunks(rows, repairs._WRITE_BATCH_SIZE):
+ ch = observe.ch_mapping_states(context.team_id, [r["key"] for r in batch])
+ writes = []
+ for row in batch:
+ state = ch.get(row["key"])
+ writes.append(_mapping_write(row, targets[row["key"]], state.max_version if state else None))
+ by_key = {row["key"]: row for row in batch}
+ if not context.apply:
+ current = {r.distinct_id: r for r in get_distinct_id_rows(context.team_id, list(by_key), strong=True)}
+ for write in writes:
+ now = current.get(write.distinct_id)
+ expected = (
+ now is None
+ if write.action == DISTINCT_ID_WRITE_ACTION_INSERT
+ else now is not None
+ and not now.is_deleted
+ and now.person_uuid == write.person_uuid
+ and now.version == write.expected_version
+ )
+ context.action(
+ by_key[write.distinct_id],
+ "would_repair" if expected else "skipped_changed",
+ target_person=write.person_uuid,
+ target_version=write.min_version,
+ )
+ continue
+ context.pacer.before_write()
+ response = personhog_call(
+ "person_consistency_write_distinct_ids",
+ partial(
+ require_personhog_client().write_distinct_ids,
+ WriteDistinctIdsRequest(team_id=context.team_id, writes=writes),
+ ),
+ )
+ wanted = {w.distinct_id: w for w in writes}
+ for result in response.results:
+ write = wanted[result.distinct_id]
+ stored = result.row if result.HasField("row") else None
+ in_place = (
+ stored is not None
+ and not stored.is_deleted
+ and stored.person_uuid == write.person_uuid
+ and stored.version >= write.min_version
+ )
+ if stored is not None and (
+ result.outcome in (DISTINCT_ID_WRITE_OUTCOME_APPLIED, DISTINCT_ID_WRITE_OUTCOME_UNCHANGED) or in_place
+ ):
+ context.deliveries.track(
+ create_person_distinct_id(
+ context.team_id, result.distinct_id, write.person_uuid, version=int(stored.version)
+ ),
+ repairs.PublishedKey(kind="mapping", key=result.distinct_id),
+ )
+ outcome = done if result.outcome == DISTINCT_ID_WRITE_OUTCOME_APPLIED else f"already_{done}"
+ context.action(
+ by_key[result.distinct_id], outcome, target_person=write.person_uuid, version=int(stored.version)
+ )
+ moved.append(result.distinct_id)
+ elif result.outcome == DISTINCT_ID_WRITE_OUTCOME_PERSON_NOT_LIVE:
+ context.action(by_key[result.distinct_id], "skipped_person_not_live", target_person=write.person_uuid)
+ else:
+ context.action(by_key[result.distinct_id], "skipped_changed", target_person=write.person_uuid)
+ return moved
+
+
+def restore(rows: list[dict[str, Any]], context: VerbContext) -> None:
+ """Insert lost persons into Postgres from their ClickHouse winner, one version above ClickHouse, together with
+ their lost distinct ids, and publish both. A person or mapping that exists again in Postgres is left alone."""
+ mapping_rows = [r for r in rows if r["kind"] == "mapping"]
+ person_rows = [r for r in rows if r["kind"] == "person"]
+ outcome = _restore_persons(person_rows, mapping_rows, context)
+ rest = [r for r in mapping_rows if r["key"] not in outcome.handled_ids]
+ _write_mappings(context, rest, {r["key"]: r["evidence"]["ch"]["winner_person_uuid"] for r in rest}, "restored")
+
+
+@frozen
+class _RestoreOutcome:
+ restored: dict[str, list[str]]
+ handled_ids: frozenset[str]
+
+
+def _restore_persons(
+ rows: list[dict[str, Any]], mapping_rows: list[dict[str, Any]], context: VerbContext
+) -> _RestoreOutcome:
+ """Restore each person with the lost distinct ids ClickHouse gives it, in one personhog call per batch.
+ A person none of whose ids can be restored is not inserted, so nothing ends up live and unreachable."""
+ ids_by_owner: dict[str, list[dict[str, Any]]] = defaultdict(list)
+ for row in mapping_rows:
+ if row["evidence"]["pg"] is None:
+ ids_by_owner[row["evidence"]["ch"]["winner_person_uuid"]].append(row)
+ restored: dict[str, list[str]] = {}
+ handled: set[str] = set()
+ for batch in lookups._chunks(rows, repairs._WRITE_BATCH_SIZE):
+ by_uuid = {row["key"]: row for row in batch}
+ payloads = _live_payloads(context.team_id, list(by_uuid))
+ in_postgres = observe.pg_person_states(context.team_id, list(by_uuid))
+ candidates: list[str] = []
+ for person_uuid, row in by_uuid.items():
+ if person_uuid in in_postgres:
+ context.action(
+ row,
+ "skipped_exists_live" if not in_postgres[person_uuid].is_deleted else "skipped_exists_tombstoned",
+ )
+ elif person_uuid not in payloads:
+ context.action(row, "skipped_not_divergent")
+ elif not ids_by_owner.get(person_uuid):
+ # Restored alone, the person would have no distinct id, so nothing could reach it.
+ context.action(row, "skipped_no_distinct_id")
+ else:
+ candidates.append(person_uuid)
+ if not candidates:
+ continue
+ id_rows = {u: ids_by_owner[u] for u in candidates}
+ handled.update(r["key"] for u in candidates for r in id_rows[u])
+ if not context.apply:
+ for person_uuid in candidates:
+ for row in [by_uuid[person_uuid], *id_rows[person_uuid]]:
+ context.action(row, "would_repair", target_version=payloads[person_uuid].max_version + 1)
+ continue
+ ch = observe.ch_mapping_states(context.team_id, [r["key"] for u in candidates for r in id_rows[u]])
+ versions = {key: (state.max_version + 1) for key, state in ch.items()}
+ context.pacer.before_write()
+ request = RestorePersonsRequest(
+ team_id=context.team_id,
+ persons=[
+ RestoredPerson(
+ person_uuid=u,
+ properties=json.dumps(payloads[u].properties).encode(),
+ created_at_ms=int(payloads[u].created_at.timestamp() * 1000),
+ is_identified=payloads[u].is_identified,
+ version=payloads[u].max_version + 1,
+ distinct_ids=[
+ RestoredDistinctId(distinct_id=r["key"], version=versions.get(r["key"], 0)) for r in id_rows[u]
+ ],
+ )
+ for u in candidates
+ ],
+ )
+ response = personhog_call(
+ "person_consistency_restore_persons", partial(require_personhog_client().restore_persons, request)
+ )
+ for result in response.results:
+ row = by_uuid[result.person_uuid]
+ ids = list(result.restored_distinct_ids)
+ if result.outcome == RESTORE_OUTCOME_INSERTED:
+ # A person whose publish does not land is recorded undelivered; it then exists in Postgres, so the
+ # next scan reports it as behind ClickHouse, and republish publishes it.
+ context.deliveries.track(
+ _publish_person(
+ context.team_id, result.person_uuid, payloads[result.person_uuid], int(result.version)
+ ),
+ repairs.PublishedKey(kind="person", key=result.person_uuid),
+ )
+ for distinct_id in ids:
+ context.deliveries.track(
+ create_person_distinct_id(
+ context.team_id, distinct_id, result.person_uuid, version=versions.get(distinct_id, 0)
+ ),
+ repairs.PublishedKey(kind="mapping", key=distinct_id),
+ )
+ restored[result.person_uuid] = ids
+ context.action(row, "restored", version=int(result.version))
+ elif result.outcome == RESTORE_OUTCOME_NO_DISTINCT_ID:
+ context.action(row, "skipped_no_distinct_id")
+ else:
+ live = result.outcome == RESTORE_OUTCOME_EXISTS_LIVE
+ context.action(row, "skipped_exists_live" if live else "skipped_exists_tombstoned")
+ for id_row in id_rows[result.person_uuid]:
+ if id_row["key"] in ids:
+ context.action(
+ id_row, "restored", target_person=result.person_uuid, version=versions.get(id_row["key"], 0)
+ )
+ else:
+ # Claimed since the scan, or its person was not inserted: the next scan names its new state.
+ context.action(id_row, "skipped_changed", target_person=result.person_uuid)
+ return _RestoreOutcome(restored=restored, handled_ids=frozenset(handled))
+
+
+@frozen
+class _Pair:
+ # The old person whose ids moved, and the person that holds them now.
+ old: str
+ new: str
+
+
+def _pairs(rows: list[dict[str, Any]], verb: str) -> dict[_Pair, list[dict[str, Any]]]:
+ pairs: defaultdict[_Pair, list[dict[str, Any]]] = defaultdict(list)
+ for row in rows:
+ group = row.get("group") or ""
+ old, _, new = group.removeprefix(f"{verb}:").partition(">")
+ if not group.startswith(f"{verb}:") or not old or not new:
+ continue
+ pairs[_Pair(old=old, new=new)].append(row)
+ return pairs
+
+
+def _person_processing_off(team_id: int) -> bool:
+ return bool(Team.objects.only("person_processing_opt_out").get(id=team_id).person_processing_opt_out)
+
+
+def _report_merge(
+ context: VerbContext, old: str, new: str, group: str, rows: list[dict[str, Any]], outcome: str
+) -> None:
+ for row in rows:
+ if row["kind"] == "person":
+ context.action(row, outcome, survivor=new)
+ else:
+ context.action(row, "awaiting_merge" if outcome == "merge_sent" else outcome, target_person=new)
+ if not any(r["kind"] == "person" and r["key"] == old for r in rows):
+ context.emit_extra(kind="person", key=old, group=group, outcome=outcome, survivor=new)
+
+
+def _owned_on_primary(team_id: int, owners: dict[str, str]) -> bool:
+ """Whether each distinct id maps to the given person on the primary, so the event merges the intended pair."""
+ rows = {r.distinct_id: r for r in get_distinct_id_rows(team_id, list(owners), strong=True)}
+ return all(
+ (row := rows.get(d)) is not None and not row.is_deleted and row.person_uuid == u for d, u in owners.items()
+ )
+
+
+# Ingestion applies a merge within seconds, so a merge still pending after this wait is left for the next scan.
+_MERGE_CONFIRM_SECONDS = 60.0
+_MERGE_POLL_MAX_SECONDS = 8.0
+_sleep = time.sleep
+_monotonic = time.monotonic
+
+
+@frozen
+class _PendingMerge:
+ old: str
+ new: str
+ group: str
+ rows: list[dict[str, Any]]
+
+
+def _confirm_merges(context: VerbContext, pending: list[_PendingMerge], *, done: str) -> None:
+ """Wait for ingestion to merge each old person away, then point its diverged ids at the survivor.
+
+ A merge that has not landed by the deadline leaves its ids diverged, so the next scan finds the group again.
+ """
+ deadline = _monotonic() + _MERGE_CONFIRM_SECONDS
+ wait = 1.0
+ while pending:
+ _sleep(wait)
+ wait = min(wait * 2, _MERGE_POLL_MAX_SECONDS)
+ states = observe.pg_person_states(context.team_id, [p.old for p in pending])
+ landed = [p for p in pending if p.old not in states or states[p.old].is_deleted]
+ for merge in landed:
+ diverged = [r for r in merge.rows if r["kind"] == "mapping" and r["flavor"] == "diverged"]
+ _write_mappings(context, diverged, {r["key"]: merge.new for r in diverged}, "moved")
+ for row in merge.rows:
+ if row["kind"] == "person":
+ context.action(row, done, survivor=merge.new)
+ if not any(r["kind"] == "person" and r["key"] == merge.old for r in merge.rows):
+ context.emit_extra(kind="person", key=merge.old, group=merge.group, outcome=done, survivor=merge.new)
+ pending = [p for p in pending if p not in landed]
+ if _monotonic() >= deadline:
+ break
+ for merge in pending:
+ _report_merge(context, merge.old, merge.new, merge.group, merge.rows, "merge_sent")
+
+
+def _send_merge(context: VerbContext, *, distinct_id: str, alias: str) -> str:
+ """Send $merge_dangerously on B's distinct id, so B survives and ingestion merges the alias's person into it."""
+ context.pacer.before_write()
+ token = Team.objects.only("api_token").get(id=context.team_id).api_token
+ try:
+ capture_internal(
+ token=token,
+ event_name="$merge_dangerously",
+ event_source=f"person_consistency_{context.verb}",
+ distinct_id=distinct_id,
+ timestamp=datetime.now(UTC),
+ properties={"alias": alias},
+ process_person_profile=True,
+ ).raise_for_status()
+ except Exception:
+ return "skipped_merge_failed"
+ return "merge_sent"
+
+
+def _any_live_distinct_id(team_id: int, person_uuid: str) -> str | None:
+ persons = get_persons_by_uuids(team_id, [person_uuid], distinct_id_limit=0)
+ if not persons:
+ return None
+ dids = lookups._live_distinct_ids_for_persons(team_id, [persons[0].pk]).get(persons[0].pk) or []
+ return dids[0][0] if dids else None
+
+
+def merge(rows: list[dict[str, Any]], context: VerbContext) -> None:
+ """Merge a live old person A into B, which holds A's ids in Postgres, with $merge_dangerously on a B id.
+
+ The diverged ids move to B once ingestion merged A.
+ """
+ for row in rows:
+ if not (row.get("group") or "").startswith("merge:"):
+ context.action(row, "skipped_no_group")
+ if _person_processing_off(context.team_id):
+ for row in rows:
+ context.action(row, "skipped_person_processing_off")
+ return
+ pending: list[_PendingMerge] = []
+ for pair, group_rows in _pairs(rows, "merge").items():
+ old, new = pair.old, pair.new
+ states = observe.pg_person_states(context.team_id, [old, new])
+ if any(u not in states or states[u].is_deleted for u in (old, new)):
+ for row in group_rows:
+ context.action(row, "skipped_changed")
+ continue
+ alias = _any_live_distinct_id(context.team_id, old)
+ target = next(
+ (
+ r["key"]
+ for r in group_rows
+ if r["kind"] == "mapping" and (r["evidence"]["pg"] or {}).get("person_uuid") == new
+ ),
+ None,
+ )
+ if alias is None or target is None or not _owned_on_primary(context.team_id, {alias: old, target: new}):
+ for row in group_rows:
+ context.action(row, "skipped_no_alias")
+ continue
+ if not context.apply:
+ for row in group_rows:
+ context.action(row, "would_repair", target_person=new)
+ continue
+ outcome = repairs._retry_transient(
+ partial(_send_merge, context, distinct_id=target, alias=alias), what=f"merge {old}>{new}", log=context.log
+ )
+ if outcome == "merge_sent":
+ pending.append(_PendingMerge(old=old, new=new, group=group_rows[0]["group"], rows=group_rows))
+ else:
+ _report_merge(context, old, new, group_rows[0]["group"], group_rows, outcome)
+ _confirm_merges(context, pending, done="merged")
diff --git a/posthog/models/person/divergence/verbs.py b/posthog/models/person/divergence/verbs.py
index 55c79ef27f81..75272009c514 100644
--- a/posthog/models/person/divergence/verbs.py
+++ b/posthog/models/person/divergence/verbs.py
@@ -25,10 +25,22 @@ from posthog.models.person.deletion import (
from posthog.models.person.util import create_person_distinct_id, get_distinct_id_rows, get_persons_by_uuids
from posthog.personhog_client.proto import DistinctIdRow
-from . import datatypes, lookups, observe, repairs, team_scan
+from . import datatypes, lookups, observe, repairs, restore_verbs, team_scan
# A key whose last recorded outcome is one of these needs nothing more, so a resumed run skips it.
-DONE_OUTCOMES = frozenset({"repaired", "tombstoned", "republished", "buried"})
+DONE_OUTCOMES = frozenset(
+ {
+ "repaired",
+ "tombstoned",
+ "republished",
+ "buried",
+ "restored",
+ "already_restored",
+ "moved",
+ "already_moved",
+ "merged",
+ }
+)
class VerbError(Exception):
@@ -77,6 +89,14 @@ class VerbContext:
)
)
+ def emit_extra(self, *, kind: str, key: str, group: str | None, outcome: str, **detail: Any) -> None:
+ """Record an action on a key the report has no row for, such as the old person of a merge."""
+ self.emit(
+ VerbAction(
+ verb=self.verb, team_id=self.team_id, kind=kind, key=key, group=group, outcome=outcome, detail=detail
+ )
+ )
+
VerbFn = Callable[[list[dict[str, Any]], VerbContext], None]
@@ -396,4 +416,6 @@ VERBS: dict[str, VerbFn] = {
"republish": republish,
"merge_stale": merge_stale,
"bury": bury,
+ "restore": restore_verbs.restore,
+ "merge": restore_verbs.merge,
}
diff --git a/posthog/models/person/test/test_divergence_restore_verbs.py b/posthog/models/person/test/test_divergence_restore_verbs.py
new file mode 100644
index 000000000000..88ba7ba5e816
--- /dev/null
+++ b/posthog/models/person/test/test_divergence_restore_verbs.py
@@ -0,0 +1,138 @@
+import json
+import shutil
+import tempfile
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+from posthog.test.base import BaseTest, ClickhouseTestMixin
+from unittest.mock import patch
+
+from posthog.models.person.divergence import observe, restore_verbs, team_scan, verbs
+from posthog.models.person.test.divergence_fixtures import DivergenceFixturesMixin
+from posthog.models.person.util import tombstone_persons_in_postgres
+from posthog.personhog_client.fake_client import get_active_fake
+
+_CAPTURE = "posthog.models.person.divergence.restore_verbs.capture_internal"
+
+
+class TestRestoreVerbs(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ for name, value in (("_sleep", lambda _: None), ("_MERGE_CONFIRM_SECONDS", 0.0)):
+ patcher = patch.object(restore_verbs, name, value)
+ patcher.start()
+ self.addCleanup(patcher.stop)
+ self.root = Path(tempfile.mkdtemp())
+ self.addCleanup(shutil.rmtree, self.root)
+ self.scans = 0
+
+ def _scan(self, *, purged_at: datetime | None = None) -> tuple[Path, dict[str, tuple[Any, ...]]]:
+ self.scans += 1
+ scan_dir = self.root / f"scan-{self.scans}"
+ team_scan.create_plan(scan_dir, self.team.pk, purged_at=purged_at)
+ team_scan.run_chunk(scan_dir, 0, resume=False)
+ team_scan.run_postgres(scan_dir, resume=False)
+ team_scan.finalize(scan_dir)
+ rows = [json.loads(line) for line in (scan_dir / team_scan.REPORT_FILE).read_text().splitlines()]
+ return scan_dir, {r["key"]: (r["flavor"], r["verb"], r["group"]) for r in rows}
+
+ def _run(self, scan_dir: Path, verb: str, *, apply: bool) -> dict[str, str]:
+ actions_path = scan_dir / f"{verb}-{'apply' if apply else 'dry'}.jsonl"
+ verbs.run_verb(scan_dir, verb, apply=apply, actions_path=actions_path)
+ return {a["key"]: a["outcome"] for a in map(json.loads, actions_path.read_text().splitlines())}
+
+ def _live_person(self, distinct_id: str, properties: dict | None = None):
+ person = self._pg_person(version=1, distinct_ids=[distinct_id])
+ self._ch_person(str(person.uuid), 1, properties or {"email": "[email protected]"})
+ self._ch_mapping(distinct_id, str(person.uuid), 0)
+ return person
+
+ def test_restore_rebuilds_a_lost_person_and_its_distinct_ids_from_clickhouse(self):
+ lost = str(uuid4())
+ self._ch_person(lost, 4, {"email": "[email protected]"})
+ self._ch_mapping("r-1", lost, 2)
+ self._ch_mapping("r-2", lost, 0)
+ purged_at = datetime.now(UTC) + timedelta(hours=1)
+ scan_dir, before = self._scan(purged_at=purged_at)
+ group = f"restore:{lost}"
+ assert before == {
+ lost: ("ghost", "restore", group),
+ "r-1": ("ghost", "restore", group),
+ "r-2": ("ghost", "restore", group),
+ }
+
+ assert self._run(scan_dir, "restore", apply=False) == dict.fromkeys(before, "would_repair")
+ assert self._scan(purged_at=purged_at)[1] == before
+
+ assert self._run(scan_dir, "restore", apply=True) == dict.fromkeys(before, "restored")
+
+ fake = get_active_fake()
+ assert fake is not None
+ stored = fake.stored_person(self.team.pk, lost)
+ assert stored is not None
+ assert (stored.version, json.loads(stored.properties)) == (5, {"email": "[email protected]"})
+ assert self._scan(purged_at=purged_at)[1] == {}
+
+ def test_merge_points_the_shared_ids_at_the_survivor_once_ingestion_merged(self):
+ old = self._live_person("a-own")
+ survivor = self._live_person("shared")
+ self._ch_mapping("shared", str(old.uuid), 5)
+ scan_dir, _ = self._scan()
+
+ def ingestion_merges(_: float) -> None:
+ tombstone_persons_in_postgres(self.team.pk, [old.uuid])
+
+ with patch(_CAPTURE), patch.object(restore_verbs, "_sleep", side_effect=ingestion_merges):
+ outcomes = self._run(scan_dir, "merge", apply=True)
+
+ assert outcomes == {"shared": "moved", str(old.uuid): "merged"}
+ shared = observe.ch_mapping_states(self.team.pk, ["shared"])["shared"]
+ assert (shared.winner_person_uuid, shared.winner_version > 5) == (str(survivor.uuid), True)
+
+ def test_merge_asks_ingestion_to_merge_and_leaves_the_shared_ids_until_it_did(self):
+ old = self._live_person("a-own")
+ survivor = self._live_person("shared")
+ self._ch_mapping("shared", str(old.uuid), 5)
+ scan_dir, before = self._scan()
+ assert before == {"shared": ("diverged", "merge", f"merge:{old.uuid}>{survivor.uuid}")}
+
+ with patch(_CAPTURE) as capture:
+ outcomes = self._run(scan_dir, "merge", apply=True)
+
+ assert outcomes == {"shared": "awaiting_merge", str(old.uuid): "merge_sent"}
+ assert capture.call_args.kwargs["event_name"] == "$merge_dangerously"
+ assert capture.call_args.kwargs["distinct_id"] == "shared"
+ assert capture.call_args.kwargs["properties"] == {"alias": "a-own"}
+ assert self._scan()[1] == before
+
+ def test_restore_leaves_a_lost_person_out_when_its_only_id_was_claimed_after_the_scan(self):
+ lost = str(uuid4())
+ self._ch_person(lost, 4, {"email": "[email protected]"})
+ self._ch_mapping("claimed", lost, 2)
+ purged_at = datetime.now(UTC) + timedelta(hours=1)
+ scan_dir, before = self._scan(purged_at=purged_at)
+ assert {key: verb for key, (_, verb, _) in before.items()} == {lost: "restore", "claimed": "restore"}
+ self._pg_person(version=0, distinct_ids=["claimed"])
+
+ outcomes = self._run(scan_dir, "restore", apply=True)
+
+ assert outcomes == {lost: "skipped_no_distinct_id", "claimed": "skipped_changed"}
+ fake = get_active_fake()
+ assert fake is not None
+ assert fake.stored_person(self.team.pk, lost) is None
+
+ def test_fold_and_merge_skip_a_team_with_person_processing_off(self):
+ old = self._live_person("a-own")
+ self._live_person("shared")
+ self._ch_mapping("shared", str(old.uuid), 5)
+ self.team.person_processing_opt_out = True
+ self.team.save()
+ scan_dir, _ = self._scan()
+
+ with patch(_CAPTURE) as capture:
+ outcomes = self._run(scan_dir, "merge", apply=True)
+
+ capture.assert_not_called()
+ assert outcomes == {"shared": "skipped_person_processing_off"}