Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(persons): add republish, merge_stale and bury repair verbs (#117442)

Eli Reismana436e822f30e
diff --git a/posthog/management/commands/person_consistency.py b/posthog/management/commands/person_consistency.py
index 8a3c34c4e9f7..ae43f196c55a 100644
--- a/posthog/management/commands/person_consistency.py
+++ b/posthog/management/commands/person_consistency.py
@@ -10,8 +10,16 @@ Usage:
python manage.py person_consistency scan --dir /tmp/scan-2 --phase postgres
python manage.py person_consistency finalize --dir /tmp/scan-2
-Read-only. A run that stops resumes with --resume. The report (report.jsonl) holds one row per key that
-is not ok, with the raw keys a repair needs; summary.json holds counts and hashed group names, and names no key.
+ # Repair the rows of one verb: a dry run first, then a canary, then the rest.
+ python manage.py person_consistency repair --dir /tmp/scan-2 --verb republish
+ python manage.py person_consistency repair --dir /tmp/scan-2 --verb republish --apply --limit 10
+ python manage.py person_consistency repair --dir /tmp/scan-2 --verb republish --apply --resume <actions file>
+
+Scans are read-only. A scan that stops resumes with --resume. The report (report.jsonl) holds one row per
+key that is not ok, with the raw keys a repair needs; summary.json holds counts and hashed group names, and names no key.
+A repair writes nothing without --apply, and records one action per key in an actions file.
+Rescan after every repair run. The rescan confirms the fix, and finds any ClickHouse write lost when a run was
+killed before it recorded its outcomes.
"""
import time
@@ -34,6 +42,7 @@ from posthog.models.person.divergence.team_scan import (
run_chunk,
run_postgres,
)
+from posthog.models.person.divergence.verbs import VERBS, VerbError, run_verb
def _parse_utc(value: str) -> datetime:
@@ -60,7 +69,10 @@ def _id_range(value: str) -> IdRange:
class Command(BaseCommand):
- help = "Read-only. Classify every person and mapping of one team and write a report grouped by repair verb."
+ help = (
+ "Classify every person and mapping of one team into a report grouped by repair verb, and repair them. "
+ "Scans are read-only; repair writes only with --apply."
+ )
def add_arguments(self, parser: CommandParser) -> None:
actions = parser.add_subparsers(dest="action", required=True, metavar="action")
@@ -102,6 +114,34 @@ class Command(BaseCommand):
help="Pause between pages, to limit load on ClickHouse and personhog (default: %(default)s).",
)
+ repair = actions.add_parser(
+ "repair",
+ help="DRY RUN unless --apply is passed. Repair the report rows that name one verb.",
+ )
+ self._add_dir(repair)
+ repair.add_argument("--verb", choices=sorted(VERBS), required=True, help="The repair to run.")
+ repair.add_argument("--group", action="append", default=[], help="Only rows of this group. Repeatable.")
+ repair.add_argument("--key", action="append", default=[], help="Only rows with this key. Repeatable.")
+ repair.add_argument("--limit", type=_positive_int, default=None, help="At most this many rows, for a canary.")
+ repair.add_argument("--apply", action="store_true", help="Write. Without it the repair only reports.")
+ repair.add_argument(
+ "--max-calls-per-second",
+ type=float,
+ default=20.0,
+ help="Personhog write calls per second (default: %(default)s).",
+ )
+ repair.add_argument(
+ "--resume",
+ type=Path,
+ default=None,
+ help="An actions file of an earlier run: append to it and skip the keys it records as done.",
+ )
+ repair.add_argument(
+ "--force",
+ action="store_true",
+ help="bury: tombstone new persons even past the share limit that protects a team needing a restore.",
+ )
+
final = actions.add_parser("finalize", help="Join every run's rows into report.jsonl and summary.json.")
self._add_dir(final)
final.add_argument(
@@ -152,9 +192,11 @@ class Command(BaseCommand):
self._plan(options, chunks=options["chunks"])
elif options["action"] == "scan":
self._scan(options)
+ elif options["action"] == "repair":
+ self._repair(options)
else:
self._finalize(options["dir"], partial=options["partial"])
- except ScanError as exc:
+ except (ScanError, VerbError) as exc:
raise CommandError(str(exc)) from exc
def _plan(self, options: dict[str, Any], *, chunks: int) -> None:
@@ -208,3 +250,32 @@ class Command(BaseCommand):
f"report: {rows} rows, {summary['groups']['count']} groups, complete {summary['complete']}; "
f"by verb {verbs or 'none'}; see {scan_dir / SUMMARY_FILE}"
)
+
+ def _repair(self, options: dict[str, Any]) -> None:
+ scan_dir: Path = options["dir"]
+ if not options["max_calls_per_second"] > 0:
+ raise CommandError("--max-calls-per-second must be above 0")
+ mode = "apply" if options["apply"] else "dry-run"
+ actions_path = options["resume"] or scan_dir / (
+ f"actions-{options['verb']}-{mode}-{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}.jsonl"
+ )
+ counts = run_verb(
+ scan_dir,
+ options["verb"],
+ apply=options["apply"],
+ actions_path=actions_path,
+ resume=options["resume"] is not None,
+ groups=options["group"],
+ keys=options["key"],
+ limit=options["limit"],
+ max_calls_per_second=options["max_calls_per_second"],
+ force=options["force"],
+ log=self._log,
+ )
+ outcomes = ", ".join(f"{outcome}={n}" for outcome, n in sorted(counts.items())) or "nothing to do"
+ label = "APPLY" if options["apply"] else "DRY RUN"
+ self._log(f"{options['verb']} {label}: {outcomes}; actions in {actions_path}")
+ if not options["apply"]:
+ self._log("Dry run: nothing was written. Pass --apply to write.")
+ if counts.get("undelivered"):
+ raise CommandError("Some ClickHouse messages were not delivered; rerun with --resume")
diff --git a/posthog/models/person/deletion.py b/posthog/models/person/deletion.py
index b1dfb458169d..8e3020f2698c 100644
--- a/posthog/models/person/deletion.py
+++ b/posthog/models/person/deletion.py
@@ -1,4 +1,5 @@
import datetime as dt
+from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Optional
from uuid import UUID
@@ -119,13 +120,19 @@ def find_orphaned_ch_persons(team_id: int, uuids: Optional[list[str]] = None) ->
def tombstone_orphaned_ch_persons(
- team_id: int, orphans: list[OrphanedPerson], *, dry_run: bool = True
+ team_id: int,
+ orphans: list[OrphanedPerson],
+ *,
+ dry_run: bool = True,
+ on_outcome: Optional[Callable[[str, str], None]] = None,
) -> OrphanRepairResult:
"""Tombstone orphaned persons in the persons DB one version above ClickHouse, then publish those tombstones.
Each ClickHouse tombstone carries the exact stored version, so a later revival lands above it and a rerun
republishes the same versions. Distinct ids get no tombstone of their own, because the ClickHouse deletion
sweep removes every live mapping of a person it deletes.
+
+ Once applied, ``on_outcome`` receives each orphan's uuid with "tombstoned", "republished" or "skipped_live".
"""
result = OrphanRepairResult(orphaned_person_uuids=sorted(o.uuid for o in orphans), dry_run=dry_run)
if not orphans:
@@ -153,12 +160,16 @@ def tombstone_orphaned_ch_persons(
uuid = str(floor.uuid)
if floor.outcome == VersionFloorOutcome.LIVE:
result.skipped_live_persons += 1
+ if on_outcome is not None:
+ on_outcome(uuid, "skipped_live")
continue
stored = stored_by_uuid.get(uuid)
if stored is None:
result.tombstoned_persons += 1
else:
result.republished_persons += 1
+ if on_outcome is not None:
+ on_outcome(uuid, "tombstoned" if stored is None else "republished")
tombstone = PersonTombstone(
uuid=floor.uuid, version=floor.version, distinct_ids=stored.distinct_ids if stored else []
)
diff --git a/posthog/models/person/divergence/datatypes.py b/posthog/models/person/divergence/datatypes.py
index 39ed46e31874..e1f639456de2 100644
--- a/posthog/models/person/divergence/datatypes.py
+++ b/posthog/models/person/divergence/datatypes.py
@@ -7,7 +7,8 @@ from uuid import UUID
from posthog.dataclasses import frozen
-PersonDivergenceKind = Literal["hidden", "swept", "stale", "behind", "absent"]
+# "equal": the versions match but the ClickHouse winner holds other content (a tie or a blanked person).
+PersonDivergenceKind = Literal["hidden", "swept", "stale", "behind", "absent", "equal"]
MappingDivergenceKind = Literal["hidden", "other_person", "stale", "absent"]
diff --git a/posthog/models/person/divergence/repair_plan.py b/posthog/models/person/divergence/repair_plan.py
index f99815a6a0c7..c57aae404179 100644
--- a/posthog/models/person/divergence/repair_plan.py
+++ b/posthog/models/person/divergence/repair_plan.py
@@ -2,7 +2,7 @@
from __future__ import annotations
-from collections.abc import Sequence
+from collections.abc import Collection, Sequence
from posthog.dataclasses import frozen
from posthog.models.person import Person
@@ -158,7 +158,15 @@ def _ch_mapping_states(team_id: int, distinct_ids: list[str]) -> dict[str, _ChMa
return states
-def _plan_chunk(team_id: int, person_uuids: Sequence[str], *, only_distinct_id: str | None = None) -> list[_PersonPlan]:
+def _plan_chunk(
+ team_id: int,
+ person_uuids: Sequence[str],
+ *,
+ only_distinct_id: str | None = None,
+ republish_equal: Collection[str] = (),
+) -> list[_PersonPlan]:
+ """`republish_equal` names persons to republish even when ClickHouse holds their Postgres version, because
+ a scan found other content at that version."""
live = {str(p.uuid): p for p in get_persons_by_uuids(team_id, list(person_uuids), distinct_id_limit=0)}
person_states = _ch_person_states(team_id, list(live)) if live else {}
distinct_ids_by_person = (
@@ -225,7 +233,8 @@ def _plan_chunk(team_id: int, person_uuids: Sequence[str], *, only_distinct_id:
team_id=team_id,
person_uuid=person_uuid,
person=person,
- kind=_person_kind(pg_version, state),
+ kind=_person_kind(pg_version, state)
+ or ("equal" if person_uuid in republish_equal and state is not None and state.visible_rows else None),
ch_max_version=ch_max_version,
target_version=_target_version(pg_version, ch_max_version),
mappings=mappings,
diff --git a/posthog/models/person/divergence/repairs.py b/posthog/models/person/divergence/repairs.py
index e0069543818b..0b90de6d6279 100644
--- a/posthog/models/person/divergence/repairs.py
+++ b/posthog/models/person/divergence/repairs.py
@@ -4,10 +4,11 @@ from __future__ import annotations
import time
from collections import Counter, defaultdict
-from collections.abc import Callable, Sequence
+from collections.abc import Callable, Collection, Mapping, Sequence
from dataclasses import field
from functools import partial
from typing import TypeVar
+from uuid import UUID
import grpc
@@ -35,6 +36,7 @@ from posthog.personhog_client.proto import (
DistinctIdWriteResult,
PersonVersionFloor,
RaisePersonVersionFloorsRequest,
+ SetPersonDistinctIdVersionFloorRequest,
WriteDistinctIdsRequest,
WriteDistinctIdsResponse,
)
@@ -121,6 +123,22 @@ def _raise_mapping_version_floors(
return results
+def _raise_mapping_version_floor(team_id: int, distinct_id: str, min_version: int) -> str | None:
+ """Raise the mapping's version on the primary, live or tombstoned, and return the uuid of its person there.
+
+ Bury uses it for tombstoned mappings, which the batched raise leaves alone.
+ """
+ response = personhog_call(
+ "person_divergence_set_person_distinct_id_version_floor",
+ lambda: require_personhog_client().set_person_distinct_id_version_floor(
+ SetPersonDistinctIdVersionFloorRequest(team_id=team_id, distinct_id=distinct_id, min_version=min_version)
+ ),
+ )
+ if not response.HasField("person"):
+ return None
+ return str(UUID(response.person.uuid))
+
+
def _person_action(plan: repair_plan._PersonPlan, outcome: datatypes.RepairOutcome) -> datatypes.RepairAction:
return datatypes.RepairAction(
team_id=plan.team_id,
@@ -245,18 +263,23 @@ def _execute_plan(
apply: bool,
include_stale: bool,
before_write: Callable[[], None],
- published: Callable[[ProduceResult], None],
+ published: Callable[[ProduceResult, PublishedKey], None],
stale_merge_distinct_id: str | None = None,
) -> list[datatypes.RepairAction]:
person = plan.person
if person is None:
return [_person_action(plan, "skipped_not_live")]
if plan.kind == "stale" and not include_stale:
- refusal: datatypes.RepairOutcome | None = "skipped_stale"
+ refusal: datatypes.RepairOutcome | None = (
+ "would_repair" if not apply and stale_merge_distinct_id is not None else "skipped_stale"
+ )
if apply and stale_merge_distinct_id is not None:
missing = stale_merge._clickhouse_only_properties(plan)
# A later reset that finds nothing missing raises Postgres. Capture accepting the $set does not mean
# ingestion applies it, and a raise without the merged properties lets the next update overwrite them.
+ if missing:
+ # The $set_once event lands on ingestion like any write, so it is paced like one.
+ before_write()
refusal = (
(stale_merge._queue_stale_merge(plan, stale_merge_distinct_id, missing) or "merge_queued")
if missing
@@ -333,7 +356,7 @@ def _execute_plan(
# The replica has not caught up with the raise, so its properties may predate the raised version.
person_outcome = "skipped_reread_lagging"
else:
- published(_publish_person(plan.team_id, reread))
+ published(_publish_person(plan.team_id, reread), PublishedKey(kind="person", key=plan.person_uuid))
person_outcome = "repaired"
# The write reply holds the row the primary stored, at or above the target, so it is the version to publish.
@@ -345,12 +368,19 @@ def _execute_plan(
person_id=plan.person_uuid,
version=stored_versions[mapping.distinct_id],
is_deleted=False,
- )
+ ),
+ PublishedKey(kind="mapping", key=mapping.distinct_id),
)
mapping_actions.append(_mapping_action(plan, mapping, "repaired"))
return [_person_action(plan, person_outcome), *mapping_actions]
+@frozen
+class PublishedKey:
+ kind: str
+ key: str
+
+
@frozen(frozen=False)
class _WritePacer:
"""Space writes at least 1 / max_per_second apart, so idle time never buys a later burst."""
@@ -370,13 +400,17 @@ class _WritePacer:
@frozen(frozen=False)
class _Deliveries:
- """Kafka produce results of the published rows, so a failed delivery counts as undelivered."""
+ """Kafka produce results of the published rows, so a failed delivery counts as undelivered.
+
+ A row tracked with a key names that key in `undelivered_keys` when its delivery fails.
+ """
- pending: list[ProduceResult] = field(default_factory=list)
+ pending: list[tuple[ProduceResult, PublishedKey | None]] = field(default_factory=list)
failed: int = 0
+ failed_keys: list[PublishedKey] = field(default_factory=list)
- def track(self, result: ProduceResult) -> None:
- self.pending.append(result)
+ def track(self, result: ProduceResult, key: PublishedKey | None = None) -> None:
+ self.pending.append((result, key))
if len(self.pending) >= _DELIVERY_PRUNE_EVERY:
self._prune()
@@ -385,16 +419,22 @@ class _Deliveries:
self._prune()
return self.failed + len(self.pending)
+ def undelivered_keys(self) -> list[PublishedKey]:
+ """The keys whose rows failed or are still waiting, as of the last `undelivered` call."""
+ return [*self.failed_keys, *(key for _, key in self.pending if key is not None)]
+
def _prune(self) -> None:
- waiting: list[ProduceResult] = []
- for result in self.pending:
+ waiting: list[tuple[ProduceResult, PublishedKey | None]] = []
+ for result, key in self.pending:
if not result.done():
- waiting.append(result)
+ waiting.append((result, key))
continue
try:
result.get(timeout=0)
except Exception:
self.failed += 1
+ if key is not None:
+ self.failed_keys.append(key)
self.pending = waiting
@@ -425,6 +465,9 @@ def repair_persons(
max_writes_per_second: float | None = None,
on_action: Callable[[datatypes.RepairAction], None],
log: Callable[[str], None],
+ republish_equal: Collection[str] = (),
+ stale_merge_distinct_ids: Mapping[str, str] | None = None,
+ deliveries: _Deliveries | None = None,
) -> datatypes.RepairSummary:
"""Republish the Postgres state of each target person and its mappings where ClickHouse disagrees.
@@ -434,6 +477,11 @@ def repair_persons(
``include_stale`` also repairs stale persons and their mappings. That replaces their ClickHouse properties
with the Postgres ones for good, so never use it on a team waiting for a restore from ClickHouse.
+
+ ``republish_equal`` names persons to republish although ClickHouse holds their Postgres version.
+ ``stale_merge_distinct_ids`` maps a stale person to the distinct id that sends its ClickHouse-only properties
+ to ingestion; a rerun after ingestion applied them raises and republishes the person.
+ ``deliveries``, when given, collects the publishes; the caller flushes it, and the summary counts none.
"""
if max_writes_per_second is not None and max_writes_per_second <= 0:
raise ValueError("max_writes_per_second must be above 0")
@@ -445,7 +493,9 @@ def repair_persons(
processed = 0
undelivered = 0
pacer = _WritePacer(max_per_second=max_writes_per_second)
- deliveries = _Deliveries()
+ # A caller that passes its own tracker flushes it, and learns which keys did not land.
+ flush_here = deliveries is None
+ deliveries = deliveries if deliveries is not None else _Deliveries()
existing_teams = set(Team.objects.filter(id__in=list(by_team)).values_list("id", flat=True))
try:
for team_id, person_uuids in sorted(by_team.items()):
@@ -469,7 +519,9 @@ def repair_persons(
continue
for chunk in lookups._chunks(person_uuids, _REPAIR_CHUNK_SIZE):
plans = _retry_transient(
- partial(repair_plan._plan_chunk, team_id, chunk), what=f"team {team_id}: planning", log=log
+ partial(repair_plan._plan_chunk, team_id, chunk, republish_equal=frozenset(republish_equal)),
+ what=f"team {team_id}: planning",
+ log=log,
)
for plan in plans:
# Only the attempt that succeeds returns actions, so a retried person is counted once.
@@ -481,6 +533,7 @@ def repair_persons(
include_stale=include_stale,
before_write=pacer.before_write,
published=deliveries.track,
+ stale_merge_distinct_id=(stale_merge_distinct_ids or {}).get(plan.person_uuid),
),
what=f"team {team_id} person {plan.person_uuid}",
log=log,
@@ -491,7 +544,7 @@ def repair_persons(
processed += 1
log(f"team {team_id}: {len(person_uuids)} persons, {processed} processed in total")
finally:
- if apply:
+ if apply and flush_here:
undelivered = deliveries.undelivered(_FLUSH_TIMEOUT_SECONDS)
if undelivered:
log(f"{undelivered} ClickHouse messages were not delivered; rerun the repair for the same input")
diff --git a/posthog/models/person/divergence/team_scan.py b/posthog/models/person/divergence/team_scan.py
index ab368456eeb2..ab72e142d99b 100644
--- a/posthog/models/person/divergence/team_scan.py
+++ b/posthog/models/person/divergence/team_scan.py
@@ -529,7 +529,7 @@ def run_chunk(
files = chunk_files(scan_dir, chunk)
cursor = load_cursor(files.cursor)
if cursor is not None and not resume:
- raise ScanError(f"{files.cursor} exists; pass resume to continue it")
+ raise ScanError(f"{files.cursor} exists; pass --resume to continue it")
cursor = cursor or Cursor(phase=CH_PHASES[0])
writer = _RunWriter(files, cursor)
try:
@@ -621,7 +621,7 @@ def run_postgres(
files = postgres_files(scan_dir, id_range)
cursor = load_cursor(files.cursor)
if cursor is not None and not resume:
- raise ScanError(f"{files.cursor} exists; pass resume to continue it")
+ raise ScanError(f"{files.cursor} exists; pass --resume to continue it")
cursor = cursor or Cursor(phase="postgres", after=str(id_range.start - 1 if id_range else 0))
before = id_range.end if id_range else None
writer = _RunWriter(files, cursor)
diff --git a/posthog/models/person/divergence/verbs.py b/posthog/models/person/divergence/verbs.py
new file mode 100644
index 000000000000..55c79ef27f81
--- /dev/null
+++ b/posthog/models/person/divergence/verbs.py
@@ -0,0 +1,399 @@
+"""Repair verbs. Each takes the report rows of a team scan that name it, re-checks every key against the current
+state, and writes only what still applies. Every verb is a dry run unless applied, and every write only raises a
+version or acts on the exact state it read, so a rerun repeats nothing."""
+
+from __future__ import annotations
+
+import json
+import signal
+import threading
+import dataclasses
+from collections import Counter
+from collections.abc import Callable, Iterable, Sequence
+from datetime import UTC, datetime
+from functools import partial
+from pathlib import Path
+from typing import Any
+
+from posthog.dataclasses import frozen
+from posthog.models.person.deletion import (
+ count_live_ch_persons,
+ find_orphaned_ch_persons,
+ orphan_share_refusal,
+ tombstone_orphaned_ch_persons,
+)
+from posthog.models.person.util import create_person_distinct_id, get_distinct_id_rows, get_persons_by_uuids
+from posthog.personhog_client.proto import DistinctIdRow
+
+from . import datatypes, lookups, observe, repairs, team_scan
+
+# A key whose last recorded outcome is one of these needs nothing more, so a resumed run skips it.
+DONE_OUTCOMES = frozenset({"repaired", "tombstoned", "republished", "buried"})
+
+
+class VerbError(Exception):
+ pass
+
+
+@frozen
+class RowKey:
+ kind: str
+ key: str
+
+
+@frozen
+class VerbAction:
+ verb: str
+ team_id: int
+ kind: str
+ key: str
+ group: str | None
+ outcome: str
+ detail: dict[str, Any]
+
+
+@frozen(frozen=False)
+class VerbContext:
+ verb: str
+ team_id: int
+ apply: bool
+ force: bool
+ max_calls_per_second: float | None
+ log: Callable[[str], None]
+ emit: Callable[[VerbAction], None]
+ pacer: repairs._WritePacer
+ deliveries: repairs._Deliveries
+
+ def action(self, row: dict[str, Any], outcome: str, **detail: Any) -> None:
+ self.emit(
+ VerbAction(
+ verb=self.verb,
+ team_id=self.team_id,
+ kind=row["kind"],
+ key=row["key"],
+ group=row.get("group"),
+ outcome=outcome,
+ detail=detail,
+ )
+ )
+
+
+VerbFn = Callable[[list[dict[str, Any]], VerbContext], None]
+
+
+def load_rows(
+ scan_dir: Path,
+ verb: str,
+ *,
+ groups: Sequence[str] = (),
+ keys: Sequence[str] = (),
+ limit: int | None = None,
+ skip: Iterable[RowKey] = (),
+) -> list[dict[str, Any]]:
+ """The report rows that name `verb`, optionally only those in `groups` or with `keys`, at most `limit`."""
+ path = scan_dir / team_scan.REPORT_FILE
+ if not path.exists():
+ raise VerbError(f"{path} does not exist; finalize the scan first")
+ wanted_groups, wanted_keys, skipped = set(groups), set(keys), set(skip)
+ rows: list[dict[str, Any]] = []
+ with path.open() as handle:
+ for line in handle:
+ row = json.loads(line)
+ if row["verb"] != verb or RowKey(kind=row["kind"], key=row["key"]) in skipped:
+ continue
+ if wanted_groups and row["group"] not in wanted_groups:
+ continue
+ if wanted_keys and row["key"] not in wanted_keys:
+ continue
+ rows.append(row)
+ if limit is not None and len(rows) >= limit:
+ break
+ return rows
+
+
+def done_keys(actions_path: Path) -> set[RowKey]:
+ """Keys whose last recorded outcome in an actions file needs nothing more."""
+ last: dict[RowKey, str] = {}
+ if actions_path.exists():
+ with actions_path.open() as handle:
+ for line in handle:
+ action = json.loads(line)
+ last[RowKey(kind=action["kind"], key=action["key"])] = action["outcome"]
+ return {key for key, outcome in last.items() if outcome in DONE_OUTCOMES}
+
+
+def run_verb(
+ scan_dir: Path,
+ verb: str,
+ *,
+ apply: bool,
+ actions_path: Path,
+ resume: bool = False,
+ groups: Sequence[str] = (),
+ keys: Sequence[str] = (),
+ limit: int | None = None,
+ max_calls_per_second: float | None = None,
+ force: bool = False,
+ log: Callable[[str], None] = lambda _: None,
+) -> Counter[str]:
+ """Run one verb over the scan's report rows and append one action per key to `actions_path`.
+
+ With `resume`, keys that `actions_path` already records as done are skipped. Returns counts by outcome.
+ """
+ if verb not in VERBS:
+ raise VerbError(f"unknown verb {verb}; choose one of {', '.join(sorted(VERBS))}")
+ if actions_path.exists() and not resume:
+ raise VerbError(f"{actions_path} exists; pass --resume to continue it")
+ plan = team_scan.load_plan(scan_dir)
+ if not plan.team_exists:
+ raise VerbError(f"team {plan.team_id} does not exist; there is nothing to repair")
+ rows = load_rows(scan_dir, verb, groups=groups, keys=keys, limit=limit, skip=done_keys(actions_path))
+ counts: Counter[str] = Counter()
+ with actions_path.open("a") as out:
+
+ def emit(action: VerbAction) -> None:
+ out.write(json.dumps({**_asdict(action), "applied": apply, "at": datetime.now(UTC).isoformat()}) + "\n")
+ out.flush()
+ counts[action.outcome] += 1
+
+ context = VerbContext(
+ verb=verb,
+ team_id=plan.team_id,
+ apply=apply,
+ force=force,
+ max_calls_per_second=max_calls_per_second,
+ log=log,
+ emit=emit,
+ pacer=repairs._WritePacer(max_per_second=max_calls_per_second),
+ deliveries=repairs._Deliveries(),
+ )
+ log(f"{verb}: {len(rows)} rows, {'APPLY' if apply else 'DRY RUN'}")
+ terminated = False
+
+ def on_sigterm(_signum: int, _frame: Any) -> None:
+ nonlocal terminated
+ terminated = True
+ raise SystemExit(128 + signal.SIGTERM)
+
+ restore_sigterm = _handle_sigterm(on_sigterm)
+ try:
+ if rows:
+ VERBS[verb](rows, context)
+ finally:
+ restore_sigterm()
+ if apply:
+ undelivered = context.deliveries.undelivered(
+ _TERMINATION_FLUSH_SECONDS if terminated else repairs._FLUSH_TIMEOUT_SECONDS
+ )
+ if undelivered:
+ # A key whose publish did not land gets a last action that is not done, so resume retries it.
+ by_key = {RowKey(kind=r["kind"], key=r["key"]): r for r in rows}
+ for key in context.deliveries.undelivered_keys():
+ row = by_key.get(RowKey(kind=key.kind, key=key.key)) or {
+ "kind": key.kind,
+ "key": key.key,
+ "group": None,
+ }
+ context.action(row, "undelivered")
+ counts["undelivered"] += undelivered - len(context.deliveries.undelivered_keys())
+ log(f"{verb}: {undelivered} ClickHouse messages were not delivered; rerun with --resume")
+ return counts
+
+
+# Kubernetes sends SIGKILL a grace period after SIGTERM, so the final flush must finish well inside it.
+_TERMINATION_FLUSH_SECONDS = 20.0
+
+
+def _handle_sigterm(handler: Callable[[int, Any], None]) -> Callable[[], None]:
+ """Run `handler` on SIGTERM until the returned callable restores the previous handler.
+
+ A toolbox pod that is deleted or redeployed gets SIGTERM, which by default ends the process without running
+ `finally`, so its undelivered keys would be recorded as done. Signals only reach the main thread.
+ """
+ if threading.current_thread() is not threading.main_thread():
+ return lambda: None
+ previous = signal.signal(signal.SIGTERM, handler)
+
+ def restore() -> None:
+ signal.signal(signal.SIGTERM, previous)
+
+ return restore
+
+
+def _collect_outcome(sink: list[tuple[str, str]], person_uuid: str, outcome: str) -> None:
+ sink.append((person_uuid, outcome))
+
+
+def _asdict(action: VerbAction) -> dict[str, Any]:
+ return dataclasses.asdict(action)
+
+
+def _from_repair_action(
+ context: VerbContext, rows_by_key: dict[str, dict[str, Any]], a: datatypes.RepairAction
+) -> None:
+ key = a.distinct_id or a.person_uuid
+ row = rows_by_key.get(key) or {"kind": "mapping" if a.distinct_id else "person", "key": key, "group": None}
+ context.action(
+ row,
+ a.outcome,
+ person_uuid=a.person_uuid,
+ divergence=a.kind,
+ pg_version=a.pg_version,
+ ch_max_version=a.ch_max_version,
+ target_version=a.target_version,
+ )
+
+
+def _republish_persons(
+ rows: list[dict[str, Any]],
+ context: VerbContext,
+ person_uuids: list[str],
+ *,
+ republish_equal: Iterable[str] = (),
+ stale_merge_distinct_ids: dict[str, str] | None = None,
+) -> None:
+ rows_by_key = {row["key"]: row for row in rows}
+ repairs.repair_persons(
+ [datatypes.PersonRef(team_id=context.team_id, person_uuid=u) for u in dict.fromkeys(person_uuids)],
+ apply=context.apply,
+ max_writes_per_second=context.max_calls_per_second,
+ on_action=lambda a: _from_repair_action(context, rows_by_key, a),
+ log=context.log,
+ republish_equal=frozenset(republish_equal),
+ stale_merge_distinct_ids=stale_merge_distinct_ids,
+ deliveries=context.deliveries,
+ )
+
+
+def republish(rows: list[dict[str, Any]], context: VerbContext) -> None:
+ """Raise Postgres above ClickHouse and publish the Postgres state: hidden, swept, behind, missing, tied and
+ blanked persons, and stale, hidden, missing or diverged mappings, through their Postgres owner."""
+ person_uuids: list[str] = []
+ for row in rows:
+ if row["kind"] == "person":
+ person_uuids.append(row["key"])
+ continue
+ owner = (row["evidence"]["pg"] or {}).get("person_uuid")
+ if owner is None:
+ context.action(row, "skipped_no_owner")
+ else:
+ person_uuids.append(owner)
+ equal = [r["key"] for r in rows if r["kind"] == "person" and r["flavor"] in ("tie", "blanked")]
+ _republish_persons(rows, context, person_uuids, republish_equal=equal)
+
+
+def merge_stale(rows: list[dict[str, Any]], context: VerbContext) -> None:
+ """Send each stale person's ClickHouse-only properties to ingestion as $set_once.
+
+ Run it again once ingestion applied them: a person with nothing left to send is then raised and republished.
+ """
+ person_rows = [r for r in rows if r["kind"] == "person"]
+ persons = get_persons_by_uuids(context.team_id, [r["key"] for r in person_rows], distinct_id_limit=0)
+ pk_by_uuid = {str(p.uuid): p.pk for p in persons}
+ distinct_ids = lookups._live_distinct_ids_for_persons(context.team_id, list(pk_by_uuid.values()))
+ merge_ids: dict[str, str] = {}
+ for row in person_rows:
+ dids = distinct_ids.get(pk_by_uuid.get(row["key"], -1)) or []
+ if dids:
+ merge_ids[row["key"]] = dids[0][0]
+ else:
+ context.action(row, "skipped_no_distinct_id")
+ _republish_persons(rows, context, list(merge_ids), stale_merge_distinct_ids=merge_ids)
+
+
+def bury(rows: list[dict[str, Any]], context: VerbContext) -> None:
+ """Tombstone persons and mappings that Postgres deleted but ClickHouse still shows, one version above
+ ClickHouse in both stores."""
+ person_rows = [r for r in rows if r["kind"] == "person"]
+ if person_rows:
+ _bury_persons(person_rows, context)
+ for row in rows:
+ if row["kind"] == "mapping":
+ repairs._retry_transient(partial(_bury_mapping, row, context), what="bury mapping", log=context.log)
+
+
+def _bury_persons(rows: list[dict[str, Any]], context: VerbContext) -> None:
+ by_uuid = {row["key"]: row for row in rows}
+ orphans = find_orphaned_ch_persons(context.team_id, list(by_uuid))
+ orphaned = {o.uuid for o in orphans}
+ for person_uuid, row in by_uuid.items():
+ if person_uuid not in orphaned:
+ # Live in Postgres, or no longer live in ClickHouse.
+ context.action(row, "skipped_not_divergent")
+ if not orphans:
+ return
+ if not context.apply:
+ for orphan in orphans:
+ context.action(by_uuid[orphan.uuid], "would_repair", ch_max_version=orphan.ch_max_version)
+ return
+ planned = tombstone_orphaned_ch_persons(context.team_id, orphans, dry_run=True)
+ if planned.tombstoned_persons and not context.force:
+ refusal = orphan_share_refusal(
+ context.team_id, planned.tombstoned_persons, count_live_ch_persons(context.team_id)
+ )
+ if refusal is not None:
+ context.log(refusal)
+ for orphan in orphans:
+ context.action(by_uuid[orphan.uuid], "skipped_share_limit")
+ return
+ # The orphan repair reports an outcome before it publishes and raises when a publish fails, so outcomes are
+ # recorded only once every publish landed; otherwise each key is recorded as undelivered for resume to retry.
+ for chunk in lookups._chunks(orphans, repairs._WRITE_BATCH_SIZE):
+ outcomes: list[tuple[str, str]] = []
+ context.pacer.before_write()
+ try:
+ tombstone_orphaned_ch_persons(
+ context.team_id,
+ list(chunk),
+ dry_run=False,
+ on_outcome=partial(_collect_outcome, outcomes),
+ )
+ except Exception:
+ for person_uuid, _ in outcomes:
+ context.action(by_uuid[person_uuid], "undelivered")
+ raise
+ for person_uuid, outcome in outcomes:
+ context.action(by_uuid[person_uuid], outcome)
+
+
+def _strong_row(team_id: int, distinct_id: str) -> DistinctIdRow | None:
+ rows = get_distinct_id_rows(team_id, [distinct_id], strong=True)
+ return rows[0] if rows else None
+
+
+def _bury_mapping(row: dict[str, Any], context: VerbContext) -> None:
+ team_id, distinct_id = context.team_id, row["key"]
+ reported = row["evidence"]["pg"]
+ current = _strong_row(team_id, distinct_id)
+ if current is None or not current.is_deleted or current.person_id != reported["person_id"]:
+ context.action(row, "skipped_changed")
+ return
+ ch = observe.ch_mapping_states(team_id, [distinct_id]).get(distinct_id)
+ if ch is None or ch.visible_rows == 0 or ch.winner_deleted:
+ context.action(row, "skipped_not_divergent")
+ return
+ target = max(int(current.version), ch.max_version + 1)
+ if not context.apply:
+ context.action(row, "would_repair", target_version=target)
+ return
+ if current.version < target:
+ context.pacer.before_write()
+ repairs._raise_mapping_version_floor(team_id, distinct_id, target)
+ confirmed = _strong_row(team_id, distinct_id)
+ if confirmed is None or not confirmed.is_deleted or confirmed.person_id != current.person_id:
+ # An attach revived the row between the reads; ingestion publishes the live row above the raise.
+ context.action(row, "skipped_revived")
+ return
+ owner = confirmed.person_uuid if confirmed.HasField("person_uuid") else ch.winner_person_uuid
+ context.deliveries.track(
+ create_person_distinct_id(team_id, distinct_id, owner, version=int(confirmed.version), is_deleted=True),
+ repairs.PublishedKey(kind="mapping", key=distinct_id),
+ )
+ context.action(row, "buried", version=int(confirmed.version))
+
+
+VERBS: dict[str, VerbFn] = {
+ "republish": republish,
+ "merge_stale": merge_stale,
+ "bury": bury,
+}
diff --git a/posthog/models/person/test/test_divergence_team_scan.py b/posthog/models/person/test/test_divergence_team_scan.py
index f33d8ea23746..617cbdd9642f 100644
--- a/posthog/models/person/test/test_divergence_team_scan.py
+++ b/posthog/models/person/test/test_divergence_team_scan.py
@@ -122,7 +122,7 @@ class TestTeamScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
for distinct_id in ("moved", "lost", "x1", "unseen-1", "stray-1"):
assert distinct_id not in out.getvalue()
- with pytest.raises(CommandError, match="pass resume"):
+ with pytest.raises(CommandError, match="pass --resume"):
call_command("person_consistency", "scan", "--dir", str(scan_dir), stdout=StringIO())
def test_a_deleted_team_reports_its_clickhouse_leftovers_without_repairs(self):
diff --git a/posthog/models/person/test/test_divergence_verbs.py b/posthog/models/person/test/test_divergence_verbs.py
new file mode 100644
index 000000000000..a36a6cb9f50b
--- /dev/null
+++ b/posthog/models/person/test/test_divergence_verbs.py
@@ -0,0 +1,311 @@
+import os
+import json
+import shutil
+import signal
+import tempfile
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+import pytest
+from posthog.test.base import BaseTest, ClickhouseTestMixin
+from unittest.mock import patch
+
+from confluent_kafka import KafkaError
+from parameterized import parameterized
+
+from posthog.kafka_client.client import ClickhouseProducer, ProduceResult
+from posthog.kafka_client.topics import KAFKA_PERSON
+from posthog.models.person.divergence import repairs, team_scan, verbs
+from posthog.models.person.test.divergence_fixtures import DivergenceFixturesMixin
+from posthog.models.person.util import tombstone_persons_in_postgres
+from posthog.personhog_client.fake_client import get_active_fake
+
+
+class _Stop(Exception):
+ pass
+
+
+class TestRepairVerbs(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ self.root = Path(tempfile.mkdtemp())
+ self.addCleanup(shutil.rmtree, self.root)
+ self.scans = 0
+
+ def _scan(self) -> tuple[Path, dict[str, tuple[str, str | None]]]:
+ self.scans += 1
+ scan_dir = self.root / f"scan-{self.scans}"
+ team_scan.create_plan(scan_dir, self.team.pk)
+ team_scan.run_chunk(scan_dir, 0, resume=False)
+ team_scan.run_postgres(scan_dir, resume=False)
+ team_scan.finalize(scan_dir)
+ rows = [json.loads(line) for line in (scan_dir / team_scan.REPORT_FILE).read_text().splitlines()]
+ return scan_dir, {r["key"]: (r["flavor"], r["verb"]) for r in rows}
+
+ def _run(self, scan_dir: Path, verb: str, *, apply: bool, name: str = "actions", **kwargs: Any) -> list[dict]:
+ actions_path = scan_dir / f"{name}-{verb}.jsonl"
+ verbs.run_verb(scan_dir, verb, apply=apply, actions_path=actions_path, **kwargs)
+ return [json.loads(line) for line in actions_path.read_text().splitlines()]
+
+ def _mapped_person(self, version: int):
+ distinct_id = f"did-{uuid4()}"
+ person = self._pg_person(version=version, distinct_ids=[distinct_id])
+ self._ch_mapping(distinct_id, str(person.uuid), 0)
+ return person
+
+ def _person_with_ch(self, version: int, distinct_ids: list[str] | None = None):
+ person = self._pg_person(version=version, distinct_ids=distinct_ids)
+ self._ch_person(str(person.uuid), version, {"email": "[email protected]"})
+ for distinct_id in distinct_ids or []:
+ self._ch_mapping(distinct_id, str(person.uuid), 0)
+ return person
+
+ def test_republish_reports_in_a_dry_run_and_clears_every_row_when_applied(self):
+ hidden = self._mapped_person(1)
+ self._ch_person(str(hidden.uuid), 1, {"email": "[email protected]"})
+ self._ch_person(str(hidden.uuid), 100, {}, deleted=True)
+ blanked = self._mapped_person(105)
+ self._ch_person(str(blanked.uuid), 1, {"email": "[email protected]"})
+ self._ch_person(str(blanked.uuid), 105, {})
+ tied = self._mapped_person(2)
+ self._ch_person(str(tied.uuid), 2, {"email": "[email protected]"})
+ self._ch_person(str(tied.uuid), 2, {"email": "[email protected]"})
+ owner = self._person_with_ch(1, ["m-stale"])
+ self._ch_mapping("m-stale", str(owner.uuid), 5)
+ scan_dir, before = self._scan()
+ expected = {
+ str(hidden.uuid): ("hidden", "republish"),
+ str(blanked.uuid): ("blanked", "republish"),
+ str(tied.uuid): ("tie", "republish"),
+ "m-stale": ("stale", "republish"),
+ }
+ assert before == expected
+
+ dry = self._run(scan_dir, "republish", apply=False, name="dry")
+
+ assert {a["key"]: a["outcome"] for a in dry if a["outcome"] != "skipped_not_divergent"} == dict.fromkeys(
+ expected, "would_repair"
+ )
+ assert self._scan()[1] == expected
+
+ applied = self._run(scan_dir, "republish", apply=True)
+
+ assert {a["key"]: a["outcome"] for a in applied if a["outcome"] != "skipped_not_divergent"} == dict.fromkeys(
+ expected, "repaired"
+ )
+ assert self._scan()[1] == {}
+
+ def test_resume_skips_keys_an_earlier_run_finished(self):
+ first, second = self._mapped_person(1), self._mapped_person(1)
+ for person in (first, second):
+ self._ch_person(str(person.uuid), 1, {"email": "[email protected]"})
+ self._ch_person(str(person.uuid), 100, {}, deleted=True)
+ scan_dir, _ = self._scan()
+
+ canary = self._run(scan_dir, "republish", apply=True, limit=1)
+ rest = self._run(scan_dir, "republish", apply=True, resume=True)
+
+ canary_persons = [(a["key"], a["outcome"]) for a in canary if a["kind"] == "person"]
+ assert len(canary_persons) == 1 and canary_persons[0][1] == "repaired"
+ resumed = [a["key"] for a in rest[len(canary) :] if a["kind"] == "person"]
+ assert resumed == [k for k in (str(first.uuid), str(second.uuid)) if k != canary_persons[0][0]]
+ assert self._scan()[1] == {}
+
+ def test_bury_tombstones_what_postgres_deleted_and_leaves_strays_to_ingestion(self):
+ deleted = self._pg_person(version=2)
+ tombstone_persons_in_postgres(self.team.pk, [deleted.uuid])
+ self._ch_person(str(deleted.uuid), 7, {"email": "[email protected]"})
+ fake = get_active_fake()
+ assert fake is not None
+ holder = fake.add_person(
+ team_id=self.team.pk,
+ person_id=990_001,
+ uuid=str(uuid4()),
+ version=0,
+ distinct_ids=["gone", "unpublished", "holder-1"],
+ distinct_id_versions={"gone": 3, "unpublished": 3},
+ tombstoned_distinct_ids=["gone", "unpublished"],
+ )
+ self._ch_person(holder.uuid, 0, {})
+ self._ch_mapping("gone", holder.uuid, 5)
+ self._ch_mapping("unpublished", holder.uuid, 1)
+ self._ch_mapping("holder-1", holder.uuid, 0)
+ stray_owner = fake.add_person(
+ team_id=self.team.pk,
+ person_id=990_002,
+ uuid=str(uuid4()),
+ version=3,
+ is_deleted=True,
+ distinct_ids=["stray-1"],
+ )
+ self._ch_mapping("stray-1", stray_owner.uuid, 0)
+ scan_dir, before = self._scan()
+ assert before == {
+ str(deleted.uuid): ("deleted_visible", "bury"),
+ "gone": ("deleted_visible", "bury"),
+ "unpublished": ("deleted_visible", "bury"),
+ "stray-1": ("stray", None),
+ }
+
+ buried = self._run(scan_dir, "bury", apply=True)
+
+ assert {a["key"]: a["outcome"] for a in buried} == {
+ str(deleted.uuid): "republished",
+ "gone": "buried",
+ "unpublished": "buried",
+ }
+ assert self._scan()[1] == {"stray-1": ("stray", None)}
+
+ def _hidden_person_with_stale_mapping(self) -> str:
+ person = self._pg_person(version=1, distinct_ids=["h-1"])
+ self._ch_person(str(person.uuid), 1, {"email": "[email protected]"})
+ self._ch_person(str(person.uuid), 100, {}, deleted=True)
+ self._ch_mapping("h-1", str(person.uuid), 5)
+ return str(person.uuid)
+
+ def _tombstoned_mapping_visible_in_ch(self) -> str:
+ fake = get_active_fake()
+ assert fake is not None
+ holder = fake.add_person(
+ team_id=self.team.pk,
+ person_id=990_003,
+ uuid=str(uuid4()),
+ version=0,
+ distinct_ids=["gone", "holder-1"],
+ distinct_id_versions={"gone": 3},
+ tombstoned_distinct_ids=["gone"],
+ )
+ self._ch_person(holder.uuid, 0, {})
+ self._ch_mapping("gone", holder.uuid, 5)
+ self._ch_mapping("holder-1", holder.uuid, 0)
+ return "gone"
+
+ @parameterized.expand(
+ [
+ ("republish_after_person_raise", "republish", "repairs._raise_person_version_floor"),
+ ("republish_after_mapping_raise", "republish", "repairs._raise_mapping_version_floors"),
+ ("republish_after_person_publish", "republish", "repairs._publish_person"),
+ ("republish_after_mapping_publish", "republish", "repairs.create_person_distinct_id"),
+ ("bury_after_mapping_raise", "bury", "repairs._raise_mapping_version_floor"),
+ ("bury_after_mapping_publish", "bury", "verbs.create_person_distinct_id"),
+ ]
+ )
+ def test_a_run_stopped_after_any_step_converges_on_rerun(self, _name: str, verb: str, step: str):
+ if verb == "republish":
+ self._hidden_person_with_stale_mapping()
+ else:
+ self._tombstoned_mapping_visible_in_ch()
+ scan_dir, before = self._scan()
+ assert {v for _, v in before.values()} == {verb}
+ module, name = step.split(".")
+ target = f"posthog.models.person.divergence.{module}.{name}"
+ original = getattr({"repairs": repairs, "verbs": verbs}[module], name)
+
+ def step_then_stop(*args: Any, **kwargs: Any) -> Any:
+ original(*args, **kwargs)
+ raise _Stop
+
+ with patch(target, side_effect=step_then_stop), pytest.raises(_Stop):
+ self._run(scan_dir, verb, apply=True)
+ self._run(scan_dir, verb, apply=True, resume=True)
+
+ assert self._scan()[1] == {}
+
+ # Republish counts a failed delivery after the run; the orphan repair behind bury raises on it.
+ @parameterized.expand([("republish", False), ("bury", True)])
+ def test_a_publish_that_did_not_land_is_retried_on_resume(self, verb: str, raises: bool):
+ if verb == "republish":
+ key = self._hidden_person_with_stale_mapping()
+ else:
+ deleted = self._pg_person(version=2)
+ tombstone_persons_in_postgres(self.team.pk, [deleted.uuid])
+ self._ch_person(str(deleted.uuid), 7, {"email": "[email protected]"})
+ key = str(deleted.uuid)
+ scan_dir, _ = self._scan()
+ failed = ProduceResult(topic=KAFKA_PERSON)
+ failed.set_result(KafkaError(-192, "Local: Message timed out"), None)
+ actions_path = scan_dir / f"{verb}.jsonl"
+
+ with patch.object(ClickhouseProducer, "produce", return_value=failed):
+ if raises:
+ with pytest.raises(Exception):
+ verbs.run_verb(scan_dir, verb, apply=True, actions_path=actions_path)
+ else:
+ counts = verbs.run_verb(scan_dir, verb, apply=True, actions_path=actions_path)
+ assert counts["undelivered"] > 0
+ last = {}
+ for action in map(json.loads, actions_path.read_text().splitlines()):
+ last[action["key"]] = action["outcome"]
+ assert last[key] == "undelivered"
+
+ verbs.run_verb(scan_dir, verb, apply=True, actions_path=actions_path, resume=True)
+
+ assert self._scan()[1] == {}
+
+ def test_a_run_stopped_by_sigterm_records_its_undelivered_keys_for_resume(self):
+ key = self._hidden_person_with_stale_mapping()
+ scan_dir, _ = self._scan()
+ failed = ProduceResult(topic=KAFKA_PERSON)
+ failed.set_result(KafkaError(-192, "Local: Message timed out"), None)
+ actions_path = scan_dir / "republish.jsonl"
+ republish = verbs.VERBS["republish"]
+
+ def republish_then_sigterm(rows: list[dict], context: verbs.VerbContext) -> None:
+ republish(rows, context)
+ os.kill(os.getpid(), signal.SIGTERM)
+
+ with (
+ patch.object(ClickhouseProducer, "produce", return_value=failed),
+ patch.dict(verbs.VERBS, {"republish": republish_then_sigterm}),
+ pytest.raises(SystemExit),
+ ):
+ verbs.run_verb(scan_dir, "republish", apply=True, actions_path=actions_path)
+
+ last = {a["key"]: a["outcome"] for a in map(json.loads, actions_path.read_text().splitlines())}
+ assert last[key] == "undelivered"
+ assert signal.getsignal(signal.SIGTERM) is signal.SIG_DFL
+
+ def test_bury_paces_each_tombstone_call(self):
+ for version in (2, 3):
+ deleted = self._pg_person(version=version)
+ tombstone_persons_in_postgres(self.team.pk, [deleted.uuid])
+ self._ch_person(str(deleted.uuid), 7, {"email": "[email protected]"})
+ scan_dir, _ = self._scan()
+
+ with (
+ patch.object(repairs, "_WRITE_BATCH_SIZE", 1),
+ patch.object(repairs._WritePacer, "before_write", autospec=True) as before_write,
+ ):
+ outcomes = self._run(scan_dir, "bury", apply=True)
+
+ assert [a["outcome"] for a in outcomes] == ["republished", "republished"]
+ assert before_write.call_count == 2
+
+ def test_merge_stale_sends_clickhouse_only_properties_then_republishes_once_ingestion_applied_them(self):
+ stale = self._pg_person(version=1, distinct_ids=["s-1"])
+ self._ch_person(str(stale.uuid), 40, {"email": "[email protected]", "plan": "pro"})
+ self._ch_mapping("s-1", str(stale.uuid), 0)
+ scan_dir, before = self._scan()
+ assert before == {str(stale.uuid): ("stale", "merge_stale")}
+
+ with patch("posthog.models.person.divergence.stale_merge.capture_internal") as capture:
+ first = self._run(scan_dir, "merge_stale", apply=True)
+
+ assert [(a["key"], a["outcome"]) for a in first if a["kind"] == "person"] == [(str(stale.uuid), "merge_queued")]
+ assert capture.call_args.kwargs["distinct_id"] == "s-1"
+ assert capture.call_args.kwargs["properties"] == {"$set_once": {"plan": "pro"}}
+
+ fake = get_active_fake()
+ assert fake is not None
+ stored = fake.stored_person(self.team.pk, str(stale.uuid))
+ assert stored is not None
+ stored.properties = json.dumps({"email": "[email protected]", "plan": "pro"}).encode()
+ with patch("posthog.models.person.divergence.stale_merge.capture_internal") as capture:
+ second = self._run(scan_dir, "merge_stale", apply=True, resume=True)
+
+ capture.assert_not_called()
+ assert [(a["key"], a["outcome"]) for a in second[len(first) :] if a["kind"] == "person"] == [
+ (str(stale.uuid), "repaired")
+ ]
+ assert self._scan()[1] == {}