:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(persons): add the Postgres phase, repair groups and command to the team scan (#118359)
Eli Reismancc9ba2d71e02
diff --git a/posthog/management/commands/person_consistency.py b/posthog/management/commands/person_consistency.pynew file mode 100644index 000000000000..8a3c34c4e9f7--- /dev/null+++ b/posthog/management/commands/person_consistency.py@@ -0,0 +1,210 @@+"""Classify every person and mapping of one team, and write a report grouped by flavor and repair verb.++Usage:+ # One run: plan, both ClickHouse phases, the Postgres phase, and the report.+ python manage.py person_consistency scan --team-id 2 --dir /tmp/scan-2++ # A large team, split into chunks that run in parallel:+ python manage.py person_consistency plan --team-id 2 --dir /tmp/scan-2 --chunks 4+ python manage.py person_consistency scan --dir /tmp/scan-2 --chunk 0 # and 1, 2, 3+ python manage.py person_consistency scan --dir /tmp/scan-2 --phase postgres+ python manage.py person_consistency finalize --dir /tmp/scan-2++Read-only. A run that stops resumes with --resume. The report (report.jsonl) holds one row per key that+is not ok, with the raw keys a repair needs; summary.json holds counts and hashed group names, and names no key.+"""++import time+import argparse+from datetime import UTC, datetime+from pathlib import Path+from typing import Any++from django.core.management.base import BaseCommand, CommandError, CommandParser++from posthog.clickhouse.query_tagging import Feature, Product, tags_context+from posthog.models.person.divergence.team_scan import (+ PLAN_FILE,+ SUMMARY_FILE,+ IdRange,+ ScanError,+ create_plan,+ finalize,+ load_plan,+ run_chunk,+ run_postgres,+)+++def _parse_utc(value: str) -> datetime:+ parsed = datetime.fromisoformat(value)+ return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC)+++def _positive_int(value: str) -> int:+ number = int(value)+ if number < 1:+ raise argparse.ArgumentTypeError(f"{value} must be 1 or more")+ return number+++def _id_range(value: str) -> IdRange:+ start, _, end = value.partition(":")+ try:+ bounds = IdRange(start=int(start), end=int(end))+ except ValueError as exc:+ raise argparse.ArgumentTypeError(f"{value} is not START:END") from exc+ if not 1 <= bounds.start < bounds.end:+ raise argparse.ArgumentTypeError(f"{value} must have 1 <= START < END")+ return bounds+++class Command(BaseCommand):+ help = "Read-only. Classify every person and mapping of one team and write a report grouped by repair verb."++ def add_arguments(self, parser: CommandParser) -> None:+ actions = parser.add_subparsers(dest="action", required=True, metavar="action")++ plan = actions.add_parser("plan", help="Create a scan directory and split the team into chunks.")+ self._add_dir(plan)+ self._add_plan_options(plan, team_required=True)+ plan.add_argument(+ "--chunks",+ type=_positive_int,+ default=1,+ help="ClickHouse key ranges that can run in parallel (default: %(default)s).",+ )++ scan = actions.add_parser(+ "scan",+ help="Run a scan. Without --chunk and --phase it runs every phase and writes the report.",+ )+ self._add_dir(scan)+ self._add_plan_options(scan, team_required=False)+ scan.add_argument("--chunk", type=int, default=None, help="Run only this chunk's ClickHouse phases.")+ scan.add_argument(+ "--phase",+ choices=["all", "clickhouse", "postgres"],+ default="all",+ help="clickhouse runs the chunks; postgres runs after every chunk finished (default: %(default)s).",+ )+ scan.add_argument(+ "--pg-id-range",+ type=_id_range,+ default=None,+ help="START:END. Limit the Postgres phase to person ids in [START, END), to split it across runs.",+ )+ scan.add_argument("--resume", action="store_true", help="Continue runs that already started.")+ scan.add_argument(+ "--pause-seconds",+ type=float,+ default=0.2,+ help="Pause between pages, to limit load on ClickHouse and personhog (default: %(default)s).",+ )++ final = actions.add_parser("finalize", help="Join every run's rows into report.jsonl and summary.json.")+ self._add_dir(final)+ final.add_argument(+ "--partial", action="store_true", help="Write the report even if some runs have not finished."+ )++ @staticmethod+ def _add_dir(parser: CommandParser) -> None:+ parser.add_argument("--dir", type=Path, required=True, help="The scan directory.")++ @staticmethod+ def _add_plan_options(parser: CommandParser, *, team_required: bool) -> None:+ parser.add_argument(+ "--team-id",+ type=int,+ required=team_required,+ default=None,+ help="Team to scan." + ("" if team_required else " Creates a one-chunk plan when --dir has none."),+ )+ parser.add_argument(+ "--purged-at",+ type=_parse_utc,+ default=None,+ help="When the team's Postgres persons were purged; dates each ghost as after_purge or historical.",+ )+ parser.add_argument(+ "--ch-page-rows",+ type=_positive_int,+ default=100_000,+ help="ClickHouse rows per page. Each page also reads about one granule per part, which larger pages "+ "amortize (default: %(default)s).",+ )+ parser.add_argument(+ "--pg-page-size",+ type=_positive_int,+ default=5_000,+ help="Postgres persons per page, at most 10000 (default: %(default)s).",+ )++ def _log(self, message: str) -> None:+ self.stdout.write(f"{time.strftime('%H:%M:%S')} {message}")+ self.stdout.flush()++ def handle(self, *args: Any, **options: Any) -> None:+ with tags_context(product=Product.INTERNAL, feature=Feature.MANAGEMENT_COMMAND):+ try:+ if options["action"] == "plan":+ self._plan(options, chunks=options["chunks"])+ elif options["action"] == "scan":+ self._scan(options)+ else:+ self._finalize(options["dir"], partial=options["partial"])+ except ScanError as exc:+ raise CommandError(str(exc)) from exc++ def _plan(self, options: dict[str, Any], *, chunks: int) -> None:+ if options["pg_page_size"] > 10_000:+ raise CommandError("--pg-page-size must be at most 10000")+ plan = create_plan(+ options["dir"],+ options["team_id"],+ chunks=chunks,+ purged_at=options["purged_at"],+ ch_page_rows=options["ch_page_rows"],+ pg_page_size=options["pg_page_size"],+ )+ self._log(f"plan: team {plan.team_id}, {len(plan.chunks)} chunks, team exists {plan.team_exists}")++ def _scan(self, options: dict[str, Any]) -> None:+ scan_dir: Path = options["dir"]+ chunk, phase = options["chunk"], options["phase"]+ if chunk is not None and phase == "postgres":+ raise CommandError("--chunk selects ClickHouse phases; drop it to run the Postgres phase")+ if options["pg_id_range"] is not None and phase != "postgres":+ raise CommandError("--pg-id-range applies to --phase postgres only")+ if not (scan_dir / PLAN_FILE).exists():+ if options["team_id"] is None:+ raise CommandError(f"{scan_dir} has no plan; pass --team-id to create one, or run plan first")+ self._plan(options, chunks=1)+ plan = load_plan(scan_dir)+ if options["team_id"] is not None and options["team_id"] != plan.team_id:+ raise CommandError(f"{scan_dir} holds a scan of team {plan.team_id}, not {options['team_id']}")+ run_options = {"resume": options["resume"], "pause_seconds": options["pause_seconds"], "log": self._log}++ if phase in ("all", "clickhouse"):+ for index in [chunk] if chunk is not None else range(len(plan.chunks)):+ run_chunk(scan_dir, index, **run_options)+ if phase == "postgres" or (phase == "all" and chunk is None):+ if plan.team_exists:+ run_postgres(scan_dir, id_range=options["pg_id_range"], **run_options)+ else:+ self._log(f"postgres: team {plan.team_id} does not exist, skipped")+ if phase == "all" and chunk is None:+ self._finalize(scan_dir, partial=False)++ def _finalize(self, scan_dir: Path, *, partial: bool) -> None:+ summary = finalize(scan_dir, partial=partial)+ rows = sum(entry["rows"] for entry in summary["reported"])+ verbs = ", ".join(+ f"{verb}={counts.get('person_rows', 0)}p/{counts.get('mapping_rows', 0)}m"+ for verb, counts in summary["verbs"].items()+ )+ self._log(+ f"report: {rows} rows, {summary['groups']['count']} groups, complete {summary['complete']}; "+ f"by verb {verbs or 'none'}; see {scan_dir / SUMMARY_FILE}"+ )diff --git a/posthog/models/person/divergence/team_scan.py b/posthog/models/person/divergence/team_scan.pyindex 124a2d41e742..ab368456eeb2 100644--- a/posthog/models/person/divergence/team_scan.py+++ b/posthog/models/person/divergence/team_scan.py@@ -1,9 +1,9 @@-"""Classify every person and mapping of one team, in ClickHouse primary key order, into resumable chunk files.+"""Classify every person and mapping of one team, in ClickHouse primary key order, and write a resumable report.-Each chunk reads persons by (team_id, id) and mappings by (team_id, distinct_id), compares them with Postgres,-and writes a hash of every key it classified for a later Postgres pass. A page is recorded in the cursor only-after its rows are written, so a stopped run resumes at the first unrecorded page. Rows hold raw keys, so the-scan directory is private to its owner.+Each chunk reads persons by (team_id, id) and mappings by (team_id, distinct_id) and compares them with Postgres.+A Postgres pass then lists persons by id and reports only what ClickHouse has no row for, so it runs after every+chunk. A page is recorded in the cursor only after its rows are written, so a stopped run resumes at the first+unrecorded page. Rows hold raw keys, so the scan directory is private to its owner; the summary carries none."""from __future__ import annotations@@ -15,22 +15,33 @@ import timeimport fcntlimport hashlibimport dataclasses-from collections import Counter-from collections.abc import Callable, Sequence+from collections import Counter, defaultdict+from collections.abc import Callable, Iterator, Sequencefrom datetime import UTC, datetimefrom pathlib import Pathfrom typing import Any, Literal+from uuid import UUIDimport numpy as npfrom posthog.dataclasses import frozenfrom posthog.models.person.divergence import lookups, observe-from posthog.models.person.divergence.flavors import Verdict, classify_mapping, classify_person+from posthog.models.person.divergence.flavors import (+ PERSON_VERBS,+ PgMapping,+ PgPerson,+ Verdict,+ classify_mapping,+ classify_person,+)from posthog.models.person.divergence.inspection import ghost_era, hash_distinct_id+from posthog.models.person.util import list_persons_for_teamfrom posthog.models.team import TeamPLAN_VERSION = 1PLAN_FILE = "plan.json"+REPORT_FILE = "report.jsonl"+SUMMARY_FILE = "summary.json"KeyKind = Literal["person", "mapping"]ChPhase = Literal["persons", "mappings"]@@ -96,6 +107,13 @@ class KeyBounds:upper: str | None+@frozen+class IdRange:+ # Postgres person ids in [start, end).+ start: int+ end: int++@frozenclass PageResult:rows: list[dict[str, Any]]@@ -175,6 +193,16 @@ def chunk_files(scan_dir: Path, chunk: int) -> RunFiles:)+def postgres_files(scan_dir: Path, id_range: IdRange | None) -> RunFiles:+ name = "postgres" if id_range is None else f"postgres-{id_range.start}-{id_range.end}"+ return RunFiles(+ rows=scan_dir / f"{name}.rows.jsonl",+ cursor=scan_dir / f"{name}.cursor.json",+ person_keys=None,+ mapping_keys=None,+ )++def key_hash(key: str) -> int:"""A 64-bit hash of a uuid or distinct id, for the key sets the Postgres phase checks against."""return int.from_bytes(hashlib.blake2b(key.encode(), digest_size=8).digest(), "little")@@ -543,3 +571,387 @@ def _first_key(phase: ChPhase) -> str:def _next_phase(phase: ChPhase) -> str:return "mappings" if phase == "persons" else DONE+++def _load_key_set(paths: Sequence[Path]) -> np.ndarray:+ arrays = [np.fromfile(p, dtype="<u8") for p in paths if p.exists()]+ if not arrays:+ return np.empty(0, dtype="<u8")+ # Chunk key files never share keys, so an in-place sort is enough and peaks at half of np.unique's memory.+ keys = np.concatenate(arrays)+ keys.sort()+ return keys+++def _contains(key_set: np.ndarray, keys: Sequence[int]) -> list[bool]:+ """Membership in a sorted key set, in O(len(keys) log len(key_set))."""+ if not keys or not len(key_set):+ return [False] * len(keys)+ wanted = np.asarray(keys, dtype="<u8")+ positions = np.minimum(np.searchsorted(key_set, wanted), len(key_set) - 1)+ return (key_set[positions] == wanted).tolist()+++def _require_chunks_done(scan_dir: Path, plan: ScanPlan) -> list[RunFiles]:+ files = [chunk_files(scan_dir, i) for i in range(len(plan.chunks))]+ unfinished = [i for i, f in enumerate(files) if (c := load_cursor(f.cursor)) is None or c.phase != DONE]+ if unfinished:+ raise ScanError(f"chunks {unfinished} have not finished their ClickHouse phases")+ return files+++def run_postgres(+ scan_dir: Path,+ *,+ resume: bool,+ id_range: IdRange | None = None,+ pause_seconds: float = 0.0,+ log: Callable[[str], None] = lambda _: None,+) -> Cursor:+ """List Postgres persons by id and report the persons and mappings no ClickHouse phase saw.++ `id_range` limits the run to Postgres person ids in [start, end), so several runs can split the team.+ """+ plan = load_plan(scan_dir)+ if not plan.team_exists:+ raise ScanError(f"team {plan.team_id} does not exist in Postgres; there is nothing to list")+ chunks = _require_chunks_done(scan_dir, plan)+ person_keys = _load_key_set([f.person_keys for f in chunks if f.person_keys is not None])+ mapping_keys = _load_key_set([f.mapping_keys for f in chunks if f.mapping_keys is not None])+ files = postgres_files(scan_dir, id_range)+ cursor = load_cursor(files.cursor)+ if cursor is not None and not resume:+ raise ScanError(f"{files.cursor} exists; pass resume to continue it")+ cursor = cursor or Cursor(phase="postgres", after=str(id_range.start - 1 if id_range else 0))+ before = id_range.end if id_range else None+ writer = _RunWriter(files, cursor)+ try:+ while cursor.phase != DONE:+ persons, next_after = list_persons_for_team(+ plan.team_id, int(cursor.after or 0), before_person_id=before, limit=plan.pg_page_size+ )+ if not persons:+ writer.finish_phase(DONE)+ break+ rows, counts = _classify_postgres_page(plan, persons, person_keys, mapping_keys)+ writer.commit(rows, counts=counts, after=str(persons[-1].id), phase="postgres")+ log(f"postgres: page {cursor.pages}, {len(persons)} persons, {len(rows)} reported")+ if next_after is None:+ writer.finish_phase(DONE)+ elif pause_seconds:+ time.sleep(pause_seconds)+ finally:+ writer.close()+ return cursor+++def _classify_postgres_page(+ plan: ScanPlan, persons: Sequence[Any], person_keys: np.ndarray, mapping_keys: np.ndarray+) -> tuple[list[dict[str, Any]], Counter[str]]:+ observed_at = _now()+ team_id = plan.team_id+ rows: list[dict[str, Any]] = []+ counts: Counter[str] = Counter()+ uuids = [str(UUID(p.uuid)) for p in persons]+ seen = _contains(person_keys, [key_hash(u) for u in uuids])++ def pg_person(p: Any) -> PgPerson:+ return PgPerson(+ id=int(p.id), version=int(p.version), is_deleted=p.is_deleted, has_live_mappings=p.has_live_distinct_ids+ )++ unseen_live = [(p, u) for p, u, s in zip(persons, uuids, seen) if not s and not p.is_deleted]+ # The key sets decide which keys to read; ClickHouse itself decides whether a row exists.+ present = observe.ch_person_states(team_id, [u for _, u in unseen_live]) if unseen_live else {}+ for p, person_uuid in unseen_live:+ if person_uuid in present:+ counts["postgres/key_set_miss/person"] += 1+ continue+ verdict = classify_person(pg_person(p), None)+ counts[_count_key("person", verdict)] += 1+ rows.append(+ _row(+ kind="person",+ team_id=team_id,+ key=person_uuid,+ verdict=verdict,+ phase="postgres",+ observed_at=observed_at,+ pg=pg_person(p),+ ch=None,+ )+ )+ for p, person_uuid, s in zip(persons, uuids, seen):+ counts[f"postgres/person/{'deleted' if p.is_deleted else 'live'}"] += 1+ if s and not p.is_deleted and not p.has_live_distinct_ids:+ verdict = Verdict(flavor="no_live_mappings", sub_flavor=None, verb=PERSON_VERBS[("no_live_mappings", None)])+ counts[_count_key("person", verdict)] += 1+ rows.append(+ _row(+ kind="person",+ team_id=team_id,+ key=person_uuid,+ verdict=verdict,+ phase="postgres",+ observed_at=observed_at,+ pg=pg_person(p),+ ch=None,+ )+ )++ owners = {int(p.id): (p, u) for p, u in zip(persons, uuids) if p.has_live_distinct_ids}+ by_person = lookups._live_distinct_ids_for_persons(team_id, sorted(owners))+ mappings = [(owners[pid], did, version) for pid, dids in by_person.items() for did, version in dids]+ counts["postgres/mapping/live"] += len(mappings)+ mapping_seen = _contains(mapping_keys, [key_hash(did) for _, did, _ in mappings])+ unseen = [m for m, s in zip(mappings, mapping_seen) if not s]+ present_mappings = observe.ch_mapping_states(team_id, [did for _, did, _ in unseen]) if unseen else {}+ for (p, person_uuid), distinct_id, version in unseen:+ if distinct_id in present_mappings:+ counts["postgres/key_set_miss/mapping"] += 1+ continue+ pg_row = PgMapping(+ person_id=int(p.id),+ person_uuid=person_uuid,+ version=int(version),+ is_deleted=False,+ person_is_deleted=p.is_deleted,+ )+ verdict = classify_mapping(pg_row, None)+ counts[_count_key("mapping", verdict)] += 1+ rows.append(+ _row(+ kind="mapping",+ team_id=team_id,+ key=distinct_id,+ verdict=verdict,+ phase="postgres",+ observed_at=observed_at,+ pg=pg_row,+ ch=None,+ )+ )+ return rows, counts+++@frozen+class ScanRuns:+ chunks: list[RunFiles]+ postgres: list[RunFiles]+++def _run_files(scan_dir: Path, plan: ScanPlan) -> ScanRuns:+ chunks = [chunk_files(scan_dir, i) for i in range(len(plan.chunks))]+ postgres = [+ RunFiles(+ rows=p.with_name(p.name.replace(".cursor.json", ".rows.jsonl")),+ cursor=p,+ person_keys=None,+ mapping_keys=None,+ )+ for p in sorted(scan_dir.glob("postgres*.cursor.json"))+ ]+ return ScanRuns(chunks=chunks, postgres=postgres)+++def _iter_rows(runs: Sequence[RunFiles]) -> Iterator[dict[str, Any]]:+ """Each run's rows up to its cursor: bytes after it are a page still being written, or one a resume redoes."""+ for run in runs:+ cursor = load_cursor(run.cursor)+ if cursor is None or not run.rows.exists():+ continue+ read = 0+ # Binary, so the count is in bytes like the cursor's offset; recorded offsets fall at line ends.+ with run.rows.open("rb") as handle:+ for line in handle:+ read += len(line)+ if read > cursor.rows_bytes:+ break+ yield json.loads(line)+++@frozen+class _Pair:+ old: str+ new: str+++class _Groups:+ """Repair groups that span keys: a diverged mapping names its old owner (the ClickHouse one) and its+ new owner (the Postgres one). A pair is safe to repair as a unit only when neither person takes part+ in any other pair."""++ def __init__(self) -> None:+ self.new_by_old: defaultdict[str, set[str]] = defaultdict(set)+ self.old_by_new: defaultdict[str, set[str]] = defaultdict(set)+ self.person_verbs: dict[str, str | None] = {}+ self.primary_person_rows: set[str] = set()+ self.no_live_mapping_persons: set[str] = set()+ self.ghost_mapping_owners: set[str] = set()++ def observe(self, row: dict[str, Any]) -> None:+ if row["kind"] == "mapping" and row["flavor"] == "diverged" and row["sub_flavor"] in ("a_absent", "a_live"):+ old = row["evidence"]["ch"]["winner_person_uuid"]+ new = row["evidence"]["pg"]["person_uuid"]+ self.new_by_old[old].add(new)+ self.old_by_new[new].add(old)+ elif row["kind"] == "mapping" and row["flavor"] == "ghost" and row["sub_flavor"] == "owner_ghost":+ self.ghost_mapping_owners.add(row["evidence"]["ch"]["winner_person_uuid"])+ elif row["kind"] == "person":+ if row["flavor"] == "no_live_mappings":+ self.no_live_mapping_persons.add(row["key"])+ else:+ self.primary_person_rows.add(row["key"])+ if row["flavor"] == "ghost":+ self.person_verbs[row["key"]] = row["verb"]++ def pair_for_old(self, old: str) -> _Pair | None:+ news = self.new_by_old.get(old)+ if not news or len(news) != 1:+ return None+ (new,) = news+ return _Pair(old=old, new=new) if self.old_by_new[new] == {old} else None++ def ambiguous(self, old: str) -> bool:+ return old in self.new_by_old and self.pair_for_old(old) is None++ def fold_mode(self, old: str) -> str:+ # A full fold restores the old person with its ghost mappings, then merges it with an event.+ # Without a ghost mapping there is no distinct id to merge from, so a light fold moves the data instead.+ return "full" if old in self.ghost_mapping_owners else "light"+++def _regroup(row: dict[str, Any], groups: _Groups) -> dict[str, Any] | None:+ """Apply the cross-key decisions to one row. Returns None for a row merged into another."""+ kind, flavor, sub = row["kind"], row["flavor"], row["sub_flavor"]+ if kind == "person":+ key = row["key"]+ if flavor == "no_live_mappings" and key in groups.primary_person_rows:+ return None+ if flavor != "no_live_mappings" and key in groups.no_live_mapping_persons:+ row["also"] = ["no_live_mappings"]+ if flavor == "ghost":+ pair = groups.pair_for_old(key)+ if pair is not None:+ row["verb"], row["group"] = "fold", f"fold:{pair.old}>{pair.new}"+ row["fold_mode"] = groups.fold_mode(pair.old)+ elif groups.ambiguous(key):+ row["verb"], row["group"] = None, f"ambiguous:{key}"+ elif row["verb"] == "restore":+ row["group"] = f"restore:{key}"+ return row++ if flavor == "diverged" and sub in ("a_absent", "a_live"):+ old = row["evidence"]["ch"]["winner_person_uuid"]+ pair = groups.pair_for_old(old)+ if pair is None:+ new = row["evidence"]["pg"]["person_uuid"]+ row["sub_flavor"], row["verb"] = "ambiguous", None+ row["group"] = f"ambiguous:{new}" if len(groups.old_by_new[new]) > 1 else f"ambiguous:{old}"+ else:+ row["group"] = f"{row['verb']}:{pair.old}>{pair.new}"+ if row["verb"] == "fold":+ row["fold_mode"] = groups.fold_mode(pair.old)+ elif flavor == "ghost":+ owner = row["evidence"]["ch"]["winner_person_uuid"]+ if sub == "owner_ghost":+ pair = groups.pair_for_old(owner)+ if pair is not None:+ row["verb"], row["group"] = "fold", f"fold:{pair.old}>{pair.new}"+ row["fold_mode"] = groups.fold_mode(pair.old)+ elif groups.ambiguous(owner):+ row["verb"], row["group"] = None, f"ambiguous:{owner}"+ else:+ # A ghost mapping follows its ghost owner: restored with it, or left with it.+ row["verb"] = groups.person_verbs.get(owner)+ if row["verb"] == "restore":+ row["group"] = f"restore:{owner}"+ return row+++def _summary_group_name(team_id: int, group: str) -> str:+ # Group names embed person uuids; the summary keeps the verb and a hash, which report rows can be matched by.+ return f"{group.split(':', 1)[0]}:{hash_distinct_id(team_id, group)}"+++def finalize(scan_dir: Path, *, partial: bool = False) -> dict[str, Any]:+ """Join every run's rows into the report, apply the cross-key groups, and write the summary."""+ plan = load_plan(scan_dir)+ runs_found = _run_files(scan_dir, plan)+ chunks, postgres = runs_found.chunks, runs_found.postgres+ cursors = {str(f.cursor.name): load_cursor(f.cursor) for f in [*chunks, *postgres]}+ unfinished = sorted(name for name, c in cursors.items() if c is None or c.phase != DONE)+ if plan.team_exists and not postgres:+ unfinished.append("postgres.cursor.json")+ complete = not unfinished+ if not complete and not partial:+ raise ScanError(f"unfinished runs: {unfinished}; finish them, or finalize a partial report")++ runs = [*chunks, *postgres]+ groups = _Groups()+ for row in _iter_rows(runs):+ groups.observe(row)++ reported: Counter[tuple[str, str, str | None, str | None]] = Counter()+ group_sizes: defaultdict[str, Counter[str]] = defaultdict(Counter)+ group_verbs: dict[str, str | None] = {}+ fold_modes: dict[str, str] = {}+ tmp = scan_dir / (REPORT_FILE + ".tmp")+ with tmp.open("w") as out:+ for row in _iter_rows(runs):+ final = _regroup(row, groups)+ if final is None:+ continue+ out.write(json.dumps(final, sort_keys=True) + "\n")+ reported[(final["kind"], final["flavor"], final["sub_flavor"], final["verb"])] += 1+ if final["group"] is not None:+ group_sizes[final["group"]][final["kind"]] += 1+ group_verbs[final["group"]] = final["verb"]+ if "fold_mode" in final:+ fold_modes[final["group"]] = final["fold_mode"]+ os.replace(tmp, scan_dir / REPORT_FILE)++ classified: Counter[str] = Counter()+ for cursor in cursors.values():+ if cursor is not None:+ classified.update(cursor.counts)+ verbs: defaultdict[str, Counter[str]] = defaultdict(Counter)+ for (kind, _, _, verb), n in reported.items():+ verbs[verb or "none"][f"{kind}_rows"] += n+ for verb in group_verbs.values():+ verbs[verb or "none"]["groups"] += 1+ largest = sorted(group_sizes.items(), key=lambda item: -sum(item[1].values()))[:25]+ summary = {+ "team_id": plan.team_id,+ "team_exists": plan.team_exists,+ "complete": complete,+ "unfinished_runs": unfinished,+ "finalized_at": _now(),+ # Chunk bounds are sampled keys, so the summary keeps only how many chunks the plan has.+ "plan": {**{k: v for k, v in plan.to_json().items() if k != "chunks"}, "chunks": len(plan.chunks)},+ "report": REPORT_FILE,+ # Postgres runs split by id range cover the team only if their ranges do; check them here.+ "postgres_runs": [f.cursor.name.removesuffix(".cursor.json") for f in postgres],+ "classified": dict(sorted(classified.items())),+ "reported": [+ {"kind": kind, "flavor": flavor, "sub_flavor": sub, "verb": verb, "rows": n}+ for (kind, flavor, sub, verb), n in sorted(reported.items(), key=lambda item: (-item[1], str(item[0])))+ ],+ "verbs": {verb: dict(c) for verb, c in sorted(verbs.items())},+ "groups": {+ "count": len(group_sizes),+ "fold_modes": dict(sorted(Counter(fold_modes.values()).items())),+ "largest": [+ {+ "group": _summary_group_name(plan.team_id, group),+ "verb": group_verbs[group],+ "fold_mode": fold_modes.get(group),+ "person_rows": c["person"],+ "mapping_rows": c["mapping"],+ }+ for group, c in largest+ ],+ },+ }+ _write_json_atomic(scan_dir / SUMMARY_FILE, summary)+ return summarydiff --git a/posthog/models/person/test/test_divergence_team_scan.py b/posthog/models/person/test/test_divergence_team_scan.pyindex d5607c0e599e..f33d8ea23746 100644--- a/posthog/models/person/test/test_divergence_team_scan.py+++ b/posthog/models/person/test/test_divergence_team_scan.py@@ -2,6 +2,7 @@ import osimport jsonimport shutilimport tempfile+from io import StringIOfrom pathlib import Pathfrom typing import Anyfrom uuid import uuid4@@ -10,66 +11,119 @@ import pytestfrom posthog.test.base import BaseTest, ClickhouseTestMixinfrom unittest.mock import patch+from django.core.management import call_command+from django.core.management.base import CommandError+from django.test import SimpleTestCase++from parameterized import parameterized+from posthog.models.person.divergence import team_scan+from posthog.models.person.divergence.inspection import hash_distinct_idfrom posthog.models.person.test.divergence_fixtures import DivergenceFixturesMixinfrom posthog.models.person.util import tombstone_persons_in_postgres+from posthog.personhog_client.fake_client import get_active_fake-def _chunk_rows(scan_dir: Path, chunks: int) -> dict[str, tuple[Any, ...]]:- rows = []- for chunk in range(chunks):- path = team_scan.chunk_files(scan_dir, chunk).rows- rows += [json.loads(line) for line in path.read_text().splitlines()]- found = {r["key"]: (r["flavor"], r["sub_flavor"], r["verb"]) for r in rows}- assert len(found) == len(rows), "a key was reported twice"- return found+def _report(scan_dir: Path) -> dict[str, tuple[Any, ...]]:+ rows = [json.loads(line) for line in (scan_dir / team_scan.REPORT_FILE).read_text().splitlines()]+ report = {r["key"]: (r["flavor"], r["sub_flavor"], r["verb"], r["group"], tuple(r["also"])) for r in rows}+ assert len(report) == len(rows), "a key was reported twice"+ return reportclass _Stop(Exception):pass-class TestChunkScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):+class TestTeamScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):def setUp(self) -> None:super().setUp()self.scan_root = Path(tempfile.mkdtemp())self.addCleanup(shutil.rmtree, self.scan_root)def _build_team(self) -> dict[str, str]:- """One key per flavor the ClickHouse phases must find."""+ """One key per flavor the scan must find, from each of its three phases."""fine = self._pg_person(version=1, distinct_ids=["fine-1"])self._ch_person(str(fine.uuid), 1, {"email": "[email protected]"})self._ch_mapping("fine-1", str(fine.uuid), 0)+ # Stale, and with no mapping, so the Postgres phase adds no_live_mappings to its row.stale = self._pg_person(version=1)self._ch_person(str(stale.uuid), 40, {"email": "[email protected]"})deleted = self._pg_person(version=2)tombstone_persons_in_postgres(self.team.pk, [deleted.uuid])self._ch_person(str(deleted.uuid), 7, {"email": "[email protected]"})+ # Ghost A's only ids moved to B in Postgres: fold A into B.new_owner = self._pg_person(version=1, distinct_ids=["moved"])self._ch_person(str(new_owner.uuid), 1, {"email": "[email protected]"})ghost = str(uuid4())self._ch_person(ghost, 3, {"email": "[email protected]"})self._ch_mapping("moved", ghost, 5)self._ch_mapping("lost", ghost, 5)- return {"stale": str(stale.uuid), "deleted": str(deleted.uuid), "ghost": ghost}+ # C holds ids of two different old persons: no single repair is safe.+ holder = self._pg_person(version=1, distinct_ids=["x1", "x2"])+ self._ch_person(str(holder.uuid), 1, {"email": "[email protected]"})+ self._ch_mapping("x1", str(uuid4()), 5)+ self._ch_mapping("x2", str(uuid4()), 5)+ # Only the Postgres phase can see these: ClickHouse has no row for them.+ unseen = self._pg_person(version=1, distinct_ids=["unseen-1"])+ lonely = self._pg_person(version=1)+ self._ch_person(str(lonely.uuid), 1, {"email": "[email protected]"})+ fake = get_active_fake()+ assert fake is not None+ fake.add_person(+ team_id=self.team.pk,+ person_id=990_001,+ uuid=str(uuid4()),+ version=3,+ is_deleted=True,+ distinct_ids=["stray-1"],+ )+ return {+ "stale": str(stale.uuid),+ "deleted": str(deleted.uuid),+ "ghost": ghost,+ "new_owner": str(new_owner.uuid),+ "holder": str(holder.uuid),+ "unseen": str(unseen.uuid),+ "lonely": str(lonely.uuid),+ }- def test_a_chunk_reports_each_clickhouse_flavor_with_its_verb(self):+ def test_scan_reports_every_flavor_with_its_verb_and_group(self):keys = self._build_team()scan_dir = self.scan_root / "scan"- team_scan.create_plan(scan_dir, self.team.pk)+ out = StringIO()- cursor = team_scan.run_chunk(scan_dir, 0, resume=False)+ call_command("person_consistency", "scan", "--team-id", str(self.team.pk), "--dir", str(scan_dir), stdout=out)- assert cursor.phase == team_scan.DONE- assert _chunk_rows(scan_dir, 1) == {- keys["stale"]: ("stale", None, "merge_stale"),- keys["deleted"]: ("deleted_visible", None, "bury"),- keys["ghost"]: ("ghost", "historical", None),- "moved": ("diverged", "a_absent", "fold"),- "lost": ("ghost", "owner_ghost", "restore"),+ fold = f"fold:{keys['ghost']}>{keys['new_owner']}"+ assert _report(scan_dir) == {+ keys["stale"]: ("stale", None, "merge_stale", None, ("no_live_mappings",)),+ keys["deleted"]: ("deleted_visible", None, "bury", None, ()),+ keys["ghost"]: ("ghost", "historical", "fold", fold, ()),+ "moved": ("diverged", "a_absent", "fold", fold, ()),+ "lost": ("ghost", "owner_ghost", "fold", fold, ()),+ "x1": ("diverged", "ambiguous", None, f"ambiguous:{keys['holder']}", ()),+ "x2": ("diverged", "ambiguous", None, f"ambiguous:{keys['holder']}", ()),+ keys["unseen"]: ("missing_in_ch", None, "republish", None, ()),+ "unseen-1": ("missing_in_ch", None, "republish", None, ()),+ "stray-1": ("stray", None, None, None, ()),+ keys["lonely"]: ("no_live_mappings", None, None, None, ()),}- with pytest.raises(team_scan.ScanError, match="pass resume"):- team_scan.run_chunk(scan_dir, 0, resume=False)+ summary = json.loads((scan_dir / team_scan.SUMMARY_FILE).read_text())+ assert summary["complete"] is True+ assert [g["group"] for g in summary["groups"]["largest"] if g["verb"] == "fold"] == [+ f"fold:{hash_distinct_id(self.team.pk, fold)}"+ ]+ assert keys["ghost"] not in json.dumps(summary)+ assert summary["verbs"]["fold"] == {"person_rows": 1, "mapping_rows": 2, "groups": 1}+ assert summary["groups"]["fold_modes"] == {"full": 1}+ assert summary["classified"]["person/ok/-"] == 4+ # Distinct ids stay in the report file; the command output names no key.+ for distinct_id in ("moved", "lost", "x1", "unseen-1", "stray-1"):+ assert distinct_id not in out.getvalue()++ with pytest.raises(CommandError, match="pass resume"):+ call_command("person_consistency", "scan", "--dir", str(scan_dir), stdout=StringIO())def test_a_deleted_team_reports_its_clickhouse_leftovers_without_repairs(self):keys = self._build_team()@@ -78,13 +132,14 @@ class TestChunkScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):team_filter.return_value.exists.return_value = Falseteam_scan.create_plan(scan_dir, self.team.pk)- team_scan.run_chunk(scan_dir, 0, resume=False)+ call_command("person_consistency", "scan", "--dir", str(scan_dir), stdout=StringIO())- rows = _chunk_rows(scan_dir, 1)- # Five persons and three mappings, all ClickHouse-only once Postgres is not read.- assert len(rows) == 8- assert {keys["stale"], keys["deleted"], keys["ghost"], "moved", "lost", "fine-1"} <= rows.keys()- assert set(rows.values()) == {("ghost", "team_gone", None)}+ report = _report(scan_dir)+ assert {keys["stale"], keys["deleted"], keys["ghost"], "moved", "lost"} <= report.keys()+ assert set(report.values()) == {("ghost", "team_gone", None, None, ())}+ summary = json.loads((scan_dir / team_scan.SUMMARY_FILE).read_text())+ assert set(summary["verbs"]) == {"none"}+ assert {key for key in summary["classified"] if key.startswith("mapping/ghost/")} == {"mapping/ghost/team_gone"}def test_a_second_run_of_a_running_chunk_is_refused(self):self._build_team()@@ -103,14 +158,16 @@ class TestChunkScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):assert len(refusals) == 1 and cursor.phase == team_scan.DONEassert team_scan.run_chunk(scan_dir, 0, resume=True).phase == team_scan.DONE- def test_chunked_paged_and_interrupted_scan_gives_the_same_rows_as_one_run(self):+ def test_chunked_paged_and_interrupted_scan_gives_the_same_report_as_one_run(self):self._build_team()whole = self.scan_root / "whole"team_scan.create_plan(whole, self.team.pk)team_scan.run_chunk(whole, 0, resume=False)+ team_scan.run_postgres(whole, resume=False)+ team_scan.finalize(whole)split = self.scan_root / "split"- plan = team_scan.create_plan(split, self.team.pk, chunks=3, ch_page_rows=2)+ plan = team_scan.create_plan(split, self.team.pk, chunks=3, ch_page_rows=2, pg_page_size=2)assert len(plan.chunks) == 3def stop_after_first_page(message: str) -> None:@@ -125,11 +182,20 @@ class TestChunkScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):assert path is not Nonewith path.open("ab") as handle:handle.write(b'{"kind": "person", "key": "partial"')+ # A partial report reads each run only up to its cursor, so the half-written line is left out.+ assert team_scan.finalize(split, partial=True)["complete"] is False+ assert "partial" not in _report(split)for chunk in range(3):team_scan.run_chunk(split, chunk, resume=True)+ team_scan.run_postgres(split, resume=False)+ summary = team_scan.finalize(split)- assert _chunk_rows(split, 3) == _chunk_rows(whole, 1)- assert len(_chunk_rows(whole, 1)) == 5+ assert _report(split) == _report(whole)+ # The chunk bounds are sampled keys, which the summary must not carry.+ assert summary["plan"]["chunks"] == 3+ bounds = [b for c in plan.chunks for b in (c.person_from, c.person_to, c.mapping_from, c.mapping_to) if b]+ assert bounds and not any(b in json.dumps(summary) for b in bounds)+ assert len(_report(whole)) == 11def test_the_scan_directory_is_private_to_its_owner(self):scan_dir = self.scan_root / "private"@@ -158,3 +224,83 @@ class TestChunkScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):with pytest.raises(team_scan.ScanError, match="readable by other users"):team_scan.run_chunk(scan_dir, 0, resume=False)scan_dir.chmod(0o700)+++def _mapping(old: str, new: str, sub: str = "a_absent", key: str | None = None) -> dict[str, Any]:+ verb = "fold" if sub == "a_absent" else "merge"+ return {+ "kind": "mapping",+ "key": key or f"{old}-{new}",+ "flavor": "diverged",+ "sub_flavor": sub,+ "verb": verb,+ "group": None,+ "also": [],+ "evidence": {"pg": {"person_uuid": new}, "ch": {"winner_person_uuid": old}},+ }+++def _ghost_mapping(owner: str) -> dict[str, Any]:+ return {+ "kind": "mapping",+ "key": f"ghost-{owner}",+ "flavor": "ghost",+ "sub_flavor": "owner_ghost",+ "verb": "restore",+ "group": None,+ "also": [],+ "evidence": {"pg": None, "ch": {"winner_person_uuid": owner}},+ }+++def _ghost_person(key: str, verb: str | None) -> dict[str, Any]:+ return {+ "kind": "person",+ "key": key,+ "flavor": "ghost",+ "sub_flavor": "after_purge",+ "verb": verb,+ "group": None,+ "also": [],+ }+++class TestRegroup(SimpleTestCase):+ @parameterized.expand(+ [+ # A's ids went to two persons: folding A would copy it into both.+ (+ "old_owner_split",+ [_mapping("a", "b1"), _mapping("a", "b2")],+ {"a-b1": (None, "ambiguous:a", None), "a-b2": (None, "ambiguous:a", None)},+ ),+ ("live_pair_merges", [_mapping("a", "b", sub="a_live")], {"a-b": ("merge", "merge:a>b", None)}),+ # Every id of A moved to B, so no ghost id is left to merge from.+ ("absent_pair_without_ghost_ids_folds_light", [_mapping("a", "b")], {"a-b": ("fold", "fold:a>b", "light")}),+ (+ "absent_pair_with_a_ghost_id_folds_full",+ [_mapping("a", "b"), _ghost_mapping("a")],+ {"a-b": ("fold", "fold:a>b", "full"), "ghost-a": ("fold", "fold:a>b", "full")},+ ),+ (+ "ghost_mapping_restores_with_its_owner",+ [_ghost_person("a", "restore"), _ghost_mapping("a")],+ {"a": ("restore", "restore:a", None), "ghost-a": ("restore", "restore:a", None)},+ ),+ (+ "ghost_mapping_stays_with_an_unrepaired_owner",+ [_ghost_person("a", None), _ghost_mapping("a")],+ {"a": (None, None, None), "ghost-a": (None, None, None)},+ ),+ # ClickHouse holds the owner's tombstone, so no person row asks to restore it.+ ("ghost_mapping_without_an_owner_row_stays", [_ghost_mapping("a")], {"ghost-a": (None, None, None)}),+ ]+ )+ def test_cross_key_groups(self, _name: str, rows: list[dict[str, Any]], expected: dict[str, tuple[Any, Any, Any]]):+ groups = team_scan._Groups()+ for row in rows:+ groups.observe(row)++ final = [team_scan._regroup(dict(row), groups) for row in rows]++ assert {r["key"]: (r["verb"], r["group"], r.get("fold_mode")) for r in final if r is not None} == expected