Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(persons): add the Postgres phase, repair groups and command to the team scan (#118359)

Eli Reismancc9ba2d71e02
diff --git a/posthog/management/commands/person_consistency.py b/posthog/management/commands/person_consistency.py
new file mode 100644
index 000000000000..8a3c34c4e9f7
--- /dev/null
+++ b/posthog/management/commands/person_consistency.py
@@ -0,0 +1,210 @@
+"""Classify every person and mapping of one team, and write a report grouped by flavor and repair verb.
+
+Usage:
+ # One run: plan, both ClickHouse phases, the Postgres phase, and the report.
+ python manage.py person_consistency scan --team-id 2 --dir /tmp/scan-2
+
+ # A large team, split into chunks that run in parallel:
+ python manage.py person_consistency plan --team-id 2 --dir /tmp/scan-2 --chunks 4
+ python manage.py person_consistency scan --dir /tmp/scan-2 --chunk 0 # and 1, 2, 3
+ python manage.py person_consistency scan --dir /tmp/scan-2 --phase postgres
+ python manage.py person_consistency finalize --dir /tmp/scan-2
+
+Read-only. A run that stops resumes with --resume. The report (report.jsonl) holds one row per key that
+is not ok, with the raw keys a repair needs; summary.json holds counts and hashed group names, and names no key.
+"""
+
+import time
+import argparse
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandError, CommandParser
+
+from posthog.clickhouse.query_tagging import Feature, Product, tags_context
+from posthog.models.person.divergence.team_scan import (
+ PLAN_FILE,
+ SUMMARY_FILE,
+ IdRange,
+ ScanError,
+ create_plan,
+ finalize,
+ load_plan,
+ run_chunk,
+ run_postgres,
+)
+
+
+def _parse_utc(value: str) -> datetime:
+ parsed = datetime.fromisoformat(value)
+ return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC)
+
+
+def _positive_int(value: str) -> int:
+ number = int(value)
+ if number < 1:
+ raise argparse.ArgumentTypeError(f"{value} must be 1 or more")
+ return number
+
+
+def _id_range(value: str) -> IdRange:
+ start, _, end = value.partition(":")
+ try:
+ bounds = IdRange(start=int(start), end=int(end))
+ except ValueError as exc:
+ raise argparse.ArgumentTypeError(f"{value} is not START:END") from exc
+ if not 1 <= bounds.start < bounds.end:
+ raise argparse.ArgumentTypeError(f"{value} must have 1 <= START < END")
+ return bounds
+
+
+class Command(BaseCommand):
+ help = "Read-only. Classify every person and mapping of one team and write a report grouped by repair verb."
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ actions = parser.add_subparsers(dest="action", required=True, metavar="action")
+
+ plan = actions.add_parser("plan", help="Create a scan directory and split the team into chunks.")
+ self._add_dir(plan)
+ self._add_plan_options(plan, team_required=True)
+ plan.add_argument(
+ "--chunks",
+ type=_positive_int,
+ default=1,
+ help="ClickHouse key ranges that can run in parallel (default: %(default)s).",
+ )
+
+ scan = actions.add_parser(
+ "scan",
+ help="Run a scan. Without --chunk and --phase it runs every phase and writes the report.",
+ )
+ self._add_dir(scan)
+ self._add_plan_options(scan, team_required=False)
+ scan.add_argument("--chunk", type=int, default=None, help="Run only this chunk's ClickHouse phases.")
+ scan.add_argument(
+ "--phase",
+ choices=["all", "clickhouse", "postgres"],
+ default="all",
+ help="clickhouse runs the chunks; postgres runs after every chunk finished (default: %(default)s).",
+ )
+ scan.add_argument(
+ "--pg-id-range",
+ type=_id_range,
+ default=None,
+ help="START:END. Limit the Postgres phase to person ids in [START, END), to split it across runs.",
+ )
+ scan.add_argument("--resume", action="store_true", help="Continue runs that already started.")
+ scan.add_argument(
+ "--pause-seconds",
+ type=float,
+ default=0.2,
+ help="Pause between pages, to limit load on ClickHouse and personhog (default: %(default)s).",
+ )
+
+ final = actions.add_parser("finalize", help="Join every run's rows into report.jsonl and summary.json.")
+ self._add_dir(final)
+ final.add_argument(
+ "--partial", action="store_true", help="Write the report even if some runs have not finished."
+ )
+
+ @staticmethod
+ def _add_dir(parser: CommandParser) -> None:
+ parser.add_argument("--dir", type=Path, required=True, help="The scan directory.")
+
+ @staticmethod
+ def _add_plan_options(parser: CommandParser, *, team_required: bool) -> None:
+ parser.add_argument(
+ "--team-id",
+ type=int,
+ required=team_required,
+ default=None,
+ help="Team to scan." + ("" if team_required else " Creates a one-chunk plan when --dir has none."),
+ )
+ parser.add_argument(
+ "--purged-at",
+ type=_parse_utc,
+ default=None,
+ help="When the team's Postgres persons were purged; dates each ghost as after_purge or historical.",
+ )
+ parser.add_argument(
+ "--ch-page-rows",
+ type=_positive_int,
+ default=100_000,
+ help="ClickHouse rows per page. Each page also reads about one granule per part, which larger pages "
+ "amortize (default: %(default)s).",
+ )
+ parser.add_argument(
+ "--pg-page-size",
+ type=_positive_int,
+ default=5_000,
+ help="Postgres persons per page, at most 10000 (default: %(default)s).",
+ )
+
+ def _log(self, message: str) -> None:
+ self.stdout.write(f"{time.strftime('%H:%M:%S')} {message}")
+ self.stdout.flush()
+
+ def handle(self, *args: Any, **options: Any) -> None:
+ with tags_context(product=Product.INTERNAL, feature=Feature.MANAGEMENT_COMMAND):
+ try:
+ if options["action"] == "plan":
+ self._plan(options, chunks=options["chunks"])
+ elif options["action"] == "scan":
+ self._scan(options)
+ else:
+ self._finalize(options["dir"], partial=options["partial"])
+ except ScanError as exc:
+ raise CommandError(str(exc)) from exc
+
+ def _plan(self, options: dict[str, Any], *, chunks: int) -> None:
+ if options["pg_page_size"] > 10_000:
+ raise CommandError("--pg-page-size must be at most 10000")
+ plan = create_plan(
+ options["dir"],
+ options["team_id"],
+ chunks=chunks,
+ purged_at=options["purged_at"],
+ ch_page_rows=options["ch_page_rows"],
+ pg_page_size=options["pg_page_size"],
+ )
+ self._log(f"plan: team {plan.team_id}, {len(plan.chunks)} chunks, team exists {plan.team_exists}")
+
+ def _scan(self, options: dict[str, Any]) -> None:
+ scan_dir: Path = options["dir"]
+ chunk, phase = options["chunk"], options["phase"]
+ if chunk is not None and phase == "postgres":
+ raise CommandError("--chunk selects ClickHouse phases; drop it to run the Postgres phase")
+ if options["pg_id_range"] is not None and phase != "postgres":
+ raise CommandError("--pg-id-range applies to --phase postgres only")
+ if not (scan_dir / PLAN_FILE).exists():
+ if options["team_id"] is None:
+ raise CommandError(f"{scan_dir} has no plan; pass --team-id to create one, or run plan first")
+ self._plan(options, chunks=1)
+ plan = load_plan(scan_dir)
+ if options["team_id"] is not None and options["team_id"] != plan.team_id:
+ raise CommandError(f"{scan_dir} holds a scan of team {plan.team_id}, not {options['team_id']}")
+ run_options = {"resume": options["resume"], "pause_seconds": options["pause_seconds"], "log": self._log}
+
+ if phase in ("all", "clickhouse"):
+ for index in [chunk] if chunk is not None else range(len(plan.chunks)):
+ run_chunk(scan_dir, index, **run_options)
+ if phase == "postgres" or (phase == "all" and chunk is None):
+ if plan.team_exists:
+ run_postgres(scan_dir, id_range=options["pg_id_range"], **run_options)
+ else:
+ self._log(f"postgres: team {plan.team_id} does not exist, skipped")
+ if phase == "all" and chunk is None:
+ self._finalize(scan_dir, partial=False)
+
+ def _finalize(self, scan_dir: Path, *, partial: bool) -> None:
+ summary = finalize(scan_dir, partial=partial)
+ rows = sum(entry["rows"] for entry in summary["reported"])
+ verbs = ", ".join(
+ f"{verb}={counts.get('person_rows', 0)}p/{counts.get('mapping_rows', 0)}m"
+ for verb, counts in summary["verbs"].items()
+ )
+ self._log(
+ f"report: {rows} rows, {summary['groups']['count']} groups, complete {summary['complete']}; "
+ f"by verb {verbs or 'none'}; see {scan_dir / SUMMARY_FILE}"
+ )
diff --git a/posthog/models/person/divergence/team_scan.py b/posthog/models/person/divergence/team_scan.py
index 124a2d41e742..ab368456eeb2 100644
--- a/posthog/models/person/divergence/team_scan.py
+++ b/posthog/models/person/divergence/team_scan.py
@@ -1,9 +1,9 @@
-"""Classify every person and mapping of one team, in ClickHouse primary key order, into resumable chunk files.
+"""Classify every person and mapping of one team, in ClickHouse primary key order, and write a resumable report.
-Each chunk reads persons by (team_id, id) and mappings by (team_id, distinct_id), compares them with Postgres,
-and writes a hash of every key it classified for a later Postgres pass. A page is recorded in the cursor only
-after its rows are written, so a stopped run resumes at the first unrecorded page. Rows hold raw keys, so the
-scan directory is private to its owner.
+Each chunk reads persons by (team_id, id) and mappings by (team_id, distinct_id) and compares them with Postgres.
+A Postgres pass then lists persons by id and reports only what ClickHouse has no row for, so it runs after every
+chunk. A page is recorded in the cursor only after its rows are written, so a stopped run resumes at the first
+unrecorded page. Rows hold raw keys, so the scan directory is private to its owner; the summary carries none.
"""
from __future__ import annotations
@@ -15,22 +15,33 @@ import time
import fcntl
import hashlib
import dataclasses
-from collections import Counter
-from collections.abc import Callable, Sequence
+from collections import Counter, defaultdict
+from collections.abc import Callable, Iterator, Sequence
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Literal
+from uuid import UUID
import numpy as np
from posthog.dataclasses import frozen
from posthog.models.person.divergence import lookups, observe
-from posthog.models.person.divergence.flavors import Verdict, classify_mapping, classify_person
+from posthog.models.person.divergence.flavors import (
+ PERSON_VERBS,
+ PgMapping,
+ PgPerson,
+ Verdict,
+ classify_mapping,
+ classify_person,
+)
from posthog.models.person.divergence.inspection import ghost_era, hash_distinct_id
+from posthog.models.person.util import list_persons_for_team
from posthog.models.team import Team
PLAN_VERSION = 1
PLAN_FILE = "plan.json"
+REPORT_FILE = "report.jsonl"
+SUMMARY_FILE = "summary.json"
KeyKind = Literal["person", "mapping"]
ChPhase = Literal["persons", "mappings"]
@@ -96,6 +107,13 @@ class KeyBounds:
upper: str | None
+@frozen
+class IdRange:
+ # Postgres person ids in [start, end).
+ start: int
+ end: int
+
+
@frozen
class PageResult:
rows: list[dict[str, Any]]
@@ -175,6 +193,16 @@ def chunk_files(scan_dir: Path, chunk: int) -> RunFiles:
)
+def postgres_files(scan_dir: Path, id_range: IdRange | None) -> RunFiles:
+ name = "postgres" if id_range is None else f"postgres-{id_range.start}-{id_range.end}"
+ return RunFiles(
+ rows=scan_dir / f"{name}.rows.jsonl",
+ cursor=scan_dir / f"{name}.cursor.json",
+ person_keys=None,
+ mapping_keys=None,
+ )
+
+
def key_hash(key: str) -> int:
"""A 64-bit hash of a uuid or distinct id, for the key sets the Postgres phase checks against."""
return int.from_bytes(hashlib.blake2b(key.encode(), digest_size=8).digest(), "little")
@@ -543,3 +571,387 @@ def _first_key(phase: ChPhase) -> str:
def _next_phase(phase: ChPhase) -> str:
return "mappings" if phase == "persons" else DONE
+
+
+def _load_key_set(paths: Sequence[Path]) -> np.ndarray:
+ arrays = [np.fromfile(p, dtype="<u8") for p in paths if p.exists()]
+ if not arrays:
+ return np.empty(0, dtype="<u8")
+ # Chunk key files never share keys, so an in-place sort is enough and peaks at half of np.unique's memory.
+ keys = np.concatenate(arrays)
+ keys.sort()
+ return keys
+
+
+def _contains(key_set: np.ndarray, keys: Sequence[int]) -> list[bool]:
+ """Membership in a sorted key set, in O(len(keys) log len(key_set))."""
+ if not keys or not len(key_set):
+ return [False] * len(keys)
+ wanted = np.asarray(keys, dtype="<u8")
+ positions = np.minimum(np.searchsorted(key_set, wanted), len(key_set) - 1)
+ return (key_set[positions] == wanted).tolist()
+
+
+def _require_chunks_done(scan_dir: Path, plan: ScanPlan) -> list[RunFiles]:
+ files = [chunk_files(scan_dir, i) for i in range(len(plan.chunks))]
+ unfinished = [i for i, f in enumerate(files) if (c := load_cursor(f.cursor)) is None or c.phase != DONE]
+ if unfinished:
+ raise ScanError(f"chunks {unfinished} have not finished their ClickHouse phases")
+ return files
+
+
+def run_postgres(
+ scan_dir: Path,
+ *,
+ resume: bool,
+ id_range: IdRange | None = None,
+ pause_seconds: float = 0.0,
+ log: Callable[[str], None] = lambda _: None,
+) -> Cursor:
+ """List Postgres persons by id and report the persons and mappings no ClickHouse phase saw.
+
+ `id_range` limits the run to Postgres person ids in [start, end), so several runs can split the team.
+ """
+ plan = load_plan(scan_dir)
+ if not plan.team_exists:
+ raise ScanError(f"team {plan.team_id} does not exist in Postgres; there is nothing to list")
+ chunks = _require_chunks_done(scan_dir, plan)
+ person_keys = _load_key_set([f.person_keys for f in chunks if f.person_keys is not None])
+ mapping_keys = _load_key_set([f.mapping_keys for f in chunks if f.mapping_keys is not None])
+ files = postgres_files(scan_dir, id_range)
+ cursor = load_cursor(files.cursor)
+ if cursor is not None and not resume:
+ raise ScanError(f"{files.cursor} exists; pass resume to continue it")
+ cursor = cursor or Cursor(phase="postgres", after=str(id_range.start - 1 if id_range else 0))
+ before = id_range.end if id_range else None
+ writer = _RunWriter(files, cursor)
+ try:
+ while cursor.phase != DONE:
+ persons, next_after = list_persons_for_team(
+ plan.team_id, int(cursor.after or 0), before_person_id=before, limit=plan.pg_page_size
+ )
+ if not persons:
+ writer.finish_phase(DONE)
+ break
+ rows, counts = _classify_postgres_page(plan, persons, person_keys, mapping_keys)
+ writer.commit(rows, counts=counts, after=str(persons[-1].id), phase="postgres")
+ log(f"postgres: page {cursor.pages}, {len(persons)} persons, {len(rows)} reported")
+ if next_after is None:
+ writer.finish_phase(DONE)
+ elif pause_seconds:
+ time.sleep(pause_seconds)
+ finally:
+ writer.close()
+ return cursor
+
+
+def _classify_postgres_page(
+ plan: ScanPlan, persons: Sequence[Any], person_keys: np.ndarray, mapping_keys: np.ndarray
+) -> tuple[list[dict[str, Any]], Counter[str]]:
+ observed_at = _now()
+ team_id = plan.team_id
+ rows: list[dict[str, Any]] = []
+ counts: Counter[str] = Counter()
+ uuids = [str(UUID(p.uuid)) for p in persons]
+ seen = _contains(person_keys, [key_hash(u) for u in uuids])
+
+ def pg_person(p: Any) -> PgPerson:
+ return PgPerson(
+ id=int(p.id), version=int(p.version), is_deleted=p.is_deleted, has_live_mappings=p.has_live_distinct_ids
+ )
+
+ unseen_live = [(p, u) for p, u, s in zip(persons, uuids, seen) if not s and not p.is_deleted]
+ # The key sets decide which keys to read; ClickHouse itself decides whether a row exists.
+ present = observe.ch_person_states(team_id, [u for _, u in unseen_live]) if unseen_live else {}
+ for p, person_uuid in unseen_live:
+ if person_uuid in present:
+ counts["postgres/key_set_miss/person"] += 1
+ continue
+ verdict = classify_person(pg_person(p), None)
+ counts[_count_key("person", verdict)] += 1
+ rows.append(
+ _row(
+ kind="person",
+ team_id=team_id,
+ key=person_uuid,
+ verdict=verdict,
+ phase="postgres",
+ observed_at=observed_at,
+ pg=pg_person(p),
+ ch=None,
+ )
+ )
+ for p, person_uuid, s in zip(persons, uuids, seen):
+ counts[f"postgres/person/{'deleted' if p.is_deleted else 'live'}"] += 1
+ if s and not p.is_deleted and not p.has_live_distinct_ids:
+ verdict = Verdict(flavor="no_live_mappings", sub_flavor=None, verb=PERSON_VERBS[("no_live_mappings", None)])
+ counts[_count_key("person", verdict)] += 1
+ rows.append(
+ _row(
+ kind="person",
+ team_id=team_id,
+ key=person_uuid,
+ verdict=verdict,
+ phase="postgres",
+ observed_at=observed_at,
+ pg=pg_person(p),
+ ch=None,
+ )
+ )
+
+ owners = {int(p.id): (p, u) for p, u in zip(persons, uuids) if p.has_live_distinct_ids}
+ by_person = lookups._live_distinct_ids_for_persons(team_id, sorted(owners))
+ mappings = [(owners[pid], did, version) for pid, dids in by_person.items() for did, version in dids]
+ counts["postgres/mapping/live"] += len(mappings)
+ mapping_seen = _contains(mapping_keys, [key_hash(did) for _, did, _ in mappings])
+ unseen = [m for m, s in zip(mappings, mapping_seen) if not s]
+ present_mappings = observe.ch_mapping_states(team_id, [did for _, did, _ in unseen]) if unseen else {}
+ for (p, person_uuid), distinct_id, version in unseen:
+ if distinct_id in present_mappings:
+ counts["postgres/key_set_miss/mapping"] += 1
+ continue
+ pg_row = PgMapping(
+ person_id=int(p.id),
+ person_uuid=person_uuid,
+ version=int(version),
+ is_deleted=False,
+ person_is_deleted=p.is_deleted,
+ )
+ verdict = classify_mapping(pg_row, None)
+ counts[_count_key("mapping", verdict)] += 1
+ rows.append(
+ _row(
+ kind="mapping",
+ team_id=team_id,
+ key=distinct_id,
+ verdict=verdict,
+ phase="postgres",
+ observed_at=observed_at,
+ pg=pg_row,
+ ch=None,
+ )
+ )
+ return rows, counts
+
+
+@frozen
+class ScanRuns:
+ chunks: list[RunFiles]
+ postgres: list[RunFiles]
+
+
+def _run_files(scan_dir: Path, plan: ScanPlan) -> ScanRuns:
+ chunks = [chunk_files(scan_dir, i) for i in range(len(plan.chunks))]
+ postgres = [
+ RunFiles(
+ rows=p.with_name(p.name.replace(".cursor.json", ".rows.jsonl")),
+ cursor=p,
+ person_keys=None,
+ mapping_keys=None,
+ )
+ for p in sorted(scan_dir.glob("postgres*.cursor.json"))
+ ]
+ return ScanRuns(chunks=chunks, postgres=postgres)
+
+
+def _iter_rows(runs: Sequence[RunFiles]) -> Iterator[dict[str, Any]]:
+ """Each run's rows up to its cursor: bytes after it are a page still being written, or one a resume redoes."""
+ for run in runs:
+ cursor = load_cursor(run.cursor)
+ if cursor is None or not run.rows.exists():
+ continue
+ read = 0
+ # Binary, so the count is in bytes like the cursor's offset; recorded offsets fall at line ends.
+ with run.rows.open("rb") as handle:
+ for line in handle:
+ read += len(line)
+ if read > cursor.rows_bytes:
+ break
+ yield json.loads(line)
+
+
+@frozen
+class _Pair:
+ old: str
+ new: str
+
+
+class _Groups:
+ """Repair groups that span keys: a diverged mapping names its old owner (the ClickHouse one) and its
+ new owner (the Postgres one). A pair is safe to repair as a unit only when neither person takes part
+ in any other pair."""
+
+ def __init__(self) -> None:
+ self.new_by_old: defaultdict[str, set[str]] = defaultdict(set)
+ self.old_by_new: defaultdict[str, set[str]] = defaultdict(set)
+ self.person_verbs: dict[str, str | None] = {}
+ self.primary_person_rows: set[str] = set()
+ self.no_live_mapping_persons: set[str] = set()
+ self.ghost_mapping_owners: set[str] = set()
+
+ def observe(self, row: dict[str, Any]) -> None:
+ if row["kind"] == "mapping" and row["flavor"] == "diverged" and row["sub_flavor"] in ("a_absent", "a_live"):
+ old = row["evidence"]["ch"]["winner_person_uuid"]
+ new = row["evidence"]["pg"]["person_uuid"]
+ self.new_by_old[old].add(new)
+ self.old_by_new[new].add(old)
+ elif row["kind"] == "mapping" and row["flavor"] == "ghost" and row["sub_flavor"] == "owner_ghost":
+ self.ghost_mapping_owners.add(row["evidence"]["ch"]["winner_person_uuid"])
+ elif row["kind"] == "person":
+ if row["flavor"] == "no_live_mappings":
+ self.no_live_mapping_persons.add(row["key"])
+ else:
+ self.primary_person_rows.add(row["key"])
+ if row["flavor"] == "ghost":
+ self.person_verbs[row["key"]] = row["verb"]
+
+ def pair_for_old(self, old: str) -> _Pair | None:
+ news = self.new_by_old.get(old)
+ if not news or len(news) != 1:
+ return None
+ (new,) = news
+ return _Pair(old=old, new=new) if self.old_by_new[new] == {old} else None
+
+ def ambiguous(self, old: str) -> bool:
+ return old in self.new_by_old and self.pair_for_old(old) is None
+
+ def fold_mode(self, old: str) -> str:
+ # A full fold restores the old person with its ghost mappings, then merges it with an event.
+ # Without a ghost mapping there is no distinct id to merge from, so a light fold moves the data instead.
+ return "full" if old in self.ghost_mapping_owners else "light"
+
+
+def _regroup(row: dict[str, Any], groups: _Groups) -> dict[str, Any] | None:
+ """Apply the cross-key decisions to one row. Returns None for a row merged into another."""
+ kind, flavor, sub = row["kind"], row["flavor"], row["sub_flavor"]
+ if kind == "person":
+ key = row["key"]
+ if flavor == "no_live_mappings" and key in groups.primary_person_rows:
+ return None
+ if flavor != "no_live_mappings" and key in groups.no_live_mapping_persons:
+ row["also"] = ["no_live_mappings"]
+ if flavor == "ghost":
+ pair = groups.pair_for_old(key)
+ if pair is not None:
+ row["verb"], row["group"] = "fold", f"fold:{pair.old}>{pair.new}"
+ row["fold_mode"] = groups.fold_mode(pair.old)
+ elif groups.ambiguous(key):
+ row["verb"], row["group"] = None, f"ambiguous:{key}"
+ elif row["verb"] == "restore":
+ row["group"] = f"restore:{key}"
+ return row
+
+ if flavor == "diverged" and sub in ("a_absent", "a_live"):
+ old = row["evidence"]["ch"]["winner_person_uuid"]
+ pair = groups.pair_for_old(old)
+ if pair is None:
+ new = row["evidence"]["pg"]["person_uuid"]
+ row["sub_flavor"], row["verb"] = "ambiguous", None
+ row["group"] = f"ambiguous:{new}" if len(groups.old_by_new[new]) > 1 else f"ambiguous:{old}"
+ else:
+ row["group"] = f"{row['verb']}:{pair.old}>{pair.new}"
+ if row["verb"] == "fold":
+ row["fold_mode"] = groups.fold_mode(pair.old)
+ elif flavor == "ghost":
+ owner = row["evidence"]["ch"]["winner_person_uuid"]
+ if sub == "owner_ghost":
+ pair = groups.pair_for_old(owner)
+ if pair is not None:
+ row["verb"], row["group"] = "fold", f"fold:{pair.old}>{pair.new}"
+ row["fold_mode"] = groups.fold_mode(pair.old)
+ elif groups.ambiguous(owner):
+ row["verb"], row["group"] = None, f"ambiguous:{owner}"
+ else:
+ # A ghost mapping follows its ghost owner: restored with it, or left with it.
+ row["verb"] = groups.person_verbs.get(owner)
+ if row["verb"] == "restore":
+ row["group"] = f"restore:{owner}"
+ return row
+
+
+def _summary_group_name(team_id: int, group: str) -> str:
+ # Group names embed person uuids; the summary keeps the verb and a hash, which report rows can be matched by.
+ return f"{group.split(':', 1)[0]}:{hash_distinct_id(team_id, group)}"
+
+
+def finalize(scan_dir: Path, *, partial: bool = False) -> dict[str, Any]:
+ """Join every run's rows into the report, apply the cross-key groups, and write the summary."""
+ plan = load_plan(scan_dir)
+ runs_found = _run_files(scan_dir, plan)
+ chunks, postgres = runs_found.chunks, runs_found.postgres
+ cursors = {str(f.cursor.name): load_cursor(f.cursor) for f in [*chunks, *postgres]}
+ unfinished = sorted(name for name, c in cursors.items() if c is None or c.phase != DONE)
+ if plan.team_exists and not postgres:
+ unfinished.append("postgres.cursor.json")
+ complete = not unfinished
+ if not complete and not partial:
+ raise ScanError(f"unfinished runs: {unfinished}; finish them, or finalize a partial report")
+
+ runs = [*chunks, *postgres]
+ groups = _Groups()
+ for row in _iter_rows(runs):
+ groups.observe(row)
+
+ reported: Counter[tuple[str, str, str | None, str | None]] = Counter()
+ group_sizes: defaultdict[str, Counter[str]] = defaultdict(Counter)
+ group_verbs: dict[str, str | None] = {}
+ fold_modes: dict[str, str] = {}
+ tmp = scan_dir / (REPORT_FILE + ".tmp")
+ with tmp.open("w") as out:
+ for row in _iter_rows(runs):
+ final = _regroup(row, groups)
+ if final is None:
+ continue
+ out.write(json.dumps(final, sort_keys=True) + "\n")
+ reported[(final["kind"], final["flavor"], final["sub_flavor"], final["verb"])] += 1
+ if final["group"] is not None:
+ group_sizes[final["group"]][final["kind"]] += 1
+ group_verbs[final["group"]] = final["verb"]
+ if "fold_mode" in final:
+ fold_modes[final["group"]] = final["fold_mode"]
+ os.replace(tmp, scan_dir / REPORT_FILE)
+
+ classified: Counter[str] = Counter()
+ for cursor in cursors.values():
+ if cursor is not None:
+ classified.update(cursor.counts)
+ verbs: defaultdict[str, Counter[str]] = defaultdict(Counter)
+ for (kind, _, _, verb), n in reported.items():
+ verbs[verb or "none"][f"{kind}_rows"] += n
+ for verb in group_verbs.values():
+ verbs[verb or "none"]["groups"] += 1
+ largest = sorted(group_sizes.items(), key=lambda item: -sum(item[1].values()))[:25]
+ summary = {
+ "team_id": plan.team_id,
+ "team_exists": plan.team_exists,
+ "complete": complete,
+ "unfinished_runs": unfinished,
+ "finalized_at": _now(),
+ # Chunk bounds are sampled keys, so the summary keeps only how many chunks the plan has.
+ "plan": {**{k: v for k, v in plan.to_json().items() if k != "chunks"}, "chunks": len(plan.chunks)},
+ "report": REPORT_FILE,
+ # Postgres runs split by id range cover the team only if their ranges do; check them here.
+ "postgres_runs": [f.cursor.name.removesuffix(".cursor.json") for f in postgres],
+ "classified": dict(sorted(classified.items())),
+ "reported": [
+ {"kind": kind, "flavor": flavor, "sub_flavor": sub, "verb": verb, "rows": n}
+ for (kind, flavor, sub, verb), n in sorted(reported.items(), key=lambda item: (-item[1], str(item[0])))
+ ],
+ "verbs": {verb: dict(c) for verb, c in sorted(verbs.items())},
+ "groups": {
+ "count": len(group_sizes),
+ "fold_modes": dict(sorted(Counter(fold_modes.values()).items())),
+ "largest": [
+ {
+ "group": _summary_group_name(plan.team_id, group),
+ "verb": group_verbs[group],
+ "fold_mode": fold_modes.get(group),
+ "person_rows": c["person"],
+ "mapping_rows": c["mapping"],
+ }
+ for group, c in largest
+ ],
+ },
+ }
+ _write_json_atomic(scan_dir / SUMMARY_FILE, summary)
+ return summary
diff --git a/posthog/models/person/test/test_divergence_team_scan.py b/posthog/models/person/test/test_divergence_team_scan.py
index d5607c0e599e..f33d8ea23746 100644
--- a/posthog/models/person/test/test_divergence_team_scan.py
+++ b/posthog/models/person/test/test_divergence_team_scan.py
@@ -2,6 +2,7 @@ import os
import json
import shutil
import tempfile
+from io import StringIO
from pathlib import Path
from typing import Any
from uuid import uuid4
@@ -10,66 +11,119 @@ import pytest
from posthog.test.base import BaseTest, ClickhouseTestMixin
from unittest.mock import patch
+from django.core.management import call_command
+from django.core.management.base import CommandError
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
from posthog.models.person.divergence import team_scan
+from posthog.models.person.divergence.inspection import hash_distinct_id
from posthog.models.person.test.divergence_fixtures import DivergenceFixturesMixin
from posthog.models.person.util import tombstone_persons_in_postgres
+from posthog.personhog_client.fake_client import get_active_fake
-def _chunk_rows(scan_dir: Path, chunks: int) -> dict[str, tuple[Any, ...]]:
- rows = []
- for chunk in range(chunks):
- path = team_scan.chunk_files(scan_dir, chunk).rows
- rows += [json.loads(line) for line in path.read_text().splitlines()]
- found = {r["key"]: (r["flavor"], r["sub_flavor"], r["verb"]) for r in rows}
- assert len(found) == len(rows), "a key was reported twice"
- return found
+def _report(scan_dir: Path) -> dict[str, tuple[Any, ...]]:
+ rows = [json.loads(line) for line in (scan_dir / team_scan.REPORT_FILE).read_text().splitlines()]
+ report = {r["key"]: (r["flavor"], r["sub_flavor"], r["verb"], r["group"], tuple(r["also"])) for r in rows}
+ assert len(report) == len(rows), "a key was reported twice"
+ return report
class _Stop(Exception):
pass
-class TestChunkScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
+class TestTeamScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
def setUp(self) -> None:
super().setUp()
self.scan_root = Path(tempfile.mkdtemp())
self.addCleanup(shutil.rmtree, self.scan_root)
def _build_team(self) -> dict[str, str]:
- """One key per flavor the ClickHouse phases must find."""
+ """One key per flavor the scan must find, from each of its three phases."""
fine = self._pg_person(version=1, distinct_ids=["fine-1"])
self._ch_person(str(fine.uuid), 1, {"email": "[email protected]"})
self._ch_mapping("fine-1", str(fine.uuid), 0)
+ # Stale, and with no mapping, so the Postgres phase adds no_live_mappings to its row.
stale = self._pg_person(version=1)
self._ch_person(str(stale.uuid), 40, {"email": "[email protected]"})
deleted = self._pg_person(version=2)
tombstone_persons_in_postgres(self.team.pk, [deleted.uuid])
self._ch_person(str(deleted.uuid), 7, {"email": "[email protected]"})
+ # Ghost A's only ids moved to B in Postgres: fold A into B.
new_owner = self._pg_person(version=1, distinct_ids=["moved"])
self._ch_person(str(new_owner.uuid), 1, {"email": "[email protected]"})
ghost = str(uuid4())
self._ch_person(ghost, 3, {"email": "[email protected]"})
self._ch_mapping("moved", ghost, 5)
self._ch_mapping("lost", ghost, 5)
- return {"stale": str(stale.uuid), "deleted": str(deleted.uuid), "ghost": ghost}
+ # C holds ids of two different old persons: no single repair is safe.
+ holder = self._pg_person(version=1, distinct_ids=["x1", "x2"])
+ self._ch_person(str(holder.uuid), 1, {"email": "[email protected]"})
+ self._ch_mapping("x1", str(uuid4()), 5)
+ self._ch_mapping("x2", str(uuid4()), 5)
+ # Only the Postgres phase can see these: ClickHouse has no row for them.
+ unseen = self._pg_person(version=1, distinct_ids=["unseen-1"])
+ lonely = self._pg_person(version=1)
+ self._ch_person(str(lonely.uuid), 1, {"email": "[email protected]"})
+ fake = get_active_fake()
+ assert fake is not None
+ fake.add_person(
+ team_id=self.team.pk,
+ person_id=990_001,
+ uuid=str(uuid4()),
+ version=3,
+ is_deleted=True,
+ distinct_ids=["stray-1"],
+ )
+ return {
+ "stale": str(stale.uuid),
+ "deleted": str(deleted.uuid),
+ "ghost": ghost,
+ "new_owner": str(new_owner.uuid),
+ "holder": str(holder.uuid),
+ "unseen": str(unseen.uuid),
+ "lonely": str(lonely.uuid),
+ }
- def test_a_chunk_reports_each_clickhouse_flavor_with_its_verb(self):
+ def test_scan_reports_every_flavor_with_its_verb_and_group(self):
keys = self._build_team()
scan_dir = self.scan_root / "scan"
- team_scan.create_plan(scan_dir, self.team.pk)
+ out = StringIO()
- cursor = team_scan.run_chunk(scan_dir, 0, resume=False)
+ call_command("person_consistency", "scan", "--team-id", str(self.team.pk), "--dir", str(scan_dir), stdout=out)
- assert cursor.phase == team_scan.DONE
- assert _chunk_rows(scan_dir, 1) == {
- keys["stale"]: ("stale", None, "merge_stale"),
- keys["deleted"]: ("deleted_visible", None, "bury"),
- keys["ghost"]: ("ghost", "historical", None),
- "moved": ("diverged", "a_absent", "fold"),
- "lost": ("ghost", "owner_ghost", "restore"),
+ fold = f"fold:{keys['ghost']}>{keys['new_owner']}"
+ assert _report(scan_dir) == {
+ keys["stale"]: ("stale", None, "merge_stale", None, ("no_live_mappings",)),
+ keys["deleted"]: ("deleted_visible", None, "bury", None, ()),
+ keys["ghost"]: ("ghost", "historical", "fold", fold, ()),
+ "moved": ("diverged", "a_absent", "fold", fold, ()),
+ "lost": ("ghost", "owner_ghost", "fold", fold, ()),
+ "x1": ("diverged", "ambiguous", None, f"ambiguous:{keys['holder']}", ()),
+ "x2": ("diverged", "ambiguous", None, f"ambiguous:{keys['holder']}", ()),
+ keys["unseen"]: ("missing_in_ch", None, "republish", None, ()),
+ "unseen-1": ("missing_in_ch", None, "republish", None, ()),
+ "stray-1": ("stray", None, None, None, ()),
+ keys["lonely"]: ("no_live_mappings", None, None, None, ()),
}
- with pytest.raises(team_scan.ScanError, match="pass resume"):
- team_scan.run_chunk(scan_dir, 0, resume=False)
+ summary = json.loads((scan_dir / team_scan.SUMMARY_FILE).read_text())
+ assert summary["complete"] is True
+ assert [g["group"] for g in summary["groups"]["largest"] if g["verb"] == "fold"] == [
+ f"fold:{hash_distinct_id(self.team.pk, fold)}"
+ ]
+ assert keys["ghost"] not in json.dumps(summary)
+ assert summary["verbs"]["fold"] == {"person_rows": 1, "mapping_rows": 2, "groups": 1}
+ assert summary["groups"]["fold_modes"] == {"full": 1}
+ assert summary["classified"]["person/ok/-"] == 4
+ # Distinct ids stay in the report file; the command output names no key.
+ for distinct_id in ("moved", "lost", "x1", "unseen-1", "stray-1"):
+ assert distinct_id not in out.getvalue()
+
+ with pytest.raises(CommandError, match="pass resume"):
+ call_command("person_consistency", "scan", "--dir", str(scan_dir), stdout=StringIO())
def test_a_deleted_team_reports_its_clickhouse_leftovers_without_repairs(self):
keys = self._build_team()
@@ -78,13 +132,14 @@ class TestChunkScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
team_filter.return_value.exists.return_value = False
team_scan.create_plan(scan_dir, self.team.pk)
- team_scan.run_chunk(scan_dir, 0, resume=False)
+ call_command("person_consistency", "scan", "--dir", str(scan_dir), stdout=StringIO())
- rows = _chunk_rows(scan_dir, 1)
- # Five persons and three mappings, all ClickHouse-only once Postgres is not read.
- assert len(rows) == 8
- assert {keys["stale"], keys["deleted"], keys["ghost"], "moved", "lost", "fine-1"} <= rows.keys()
- assert set(rows.values()) == {("ghost", "team_gone", None)}
+ report = _report(scan_dir)
+ assert {keys["stale"], keys["deleted"], keys["ghost"], "moved", "lost"} <= report.keys()
+ assert set(report.values()) == {("ghost", "team_gone", None, None, ())}
+ summary = json.loads((scan_dir / team_scan.SUMMARY_FILE).read_text())
+ assert set(summary["verbs"]) == {"none"}
+ assert {key for key in summary["classified"] if key.startswith("mapping/ghost/")} == {"mapping/ghost/team_gone"}
def test_a_second_run_of_a_running_chunk_is_refused(self):
self._build_team()
@@ -103,14 +158,16 @@ class TestChunkScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
assert len(refusals) == 1 and cursor.phase == team_scan.DONE
assert team_scan.run_chunk(scan_dir, 0, resume=True).phase == team_scan.DONE
- def test_chunked_paged_and_interrupted_scan_gives_the_same_rows_as_one_run(self):
+ def test_chunked_paged_and_interrupted_scan_gives_the_same_report_as_one_run(self):
self._build_team()
whole = self.scan_root / "whole"
team_scan.create_plan(whole, self.team.pk)
team_scan.run_chunk(whole, 0, resume=False)
+ team_scan.run_postgres(whole, resume=False)
+ team_scan.finalize(whole)
split = self.scan_root / "split"
- plan = team_scan.create_plan(split, self.team.pk, chunks=3, ch_page_rows=2)
+ plan = team_scan.create_plan(split, self.team.pk, chunks=3, ch_page_rows=2, pg_page_size=2)
assert len(plan.chunks) == 3
def stop_after_first_page(message: str) -> None:
@@ -125,11 +182,20 @@ class TestChunkScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
assert path is not None
with path.open("ab") as handle:
handle.write(b'{"kind": "person", "key": "partial"')
+ # A partial report reads each run only up to its cursor, so the half-written line is left out.
+ assert team_scan.finalize(split, partial=True)["complete"] is False
+ assert "partial" not in _report(split)
for chunk in range(3):
team_scan.run_chunk(split, chunk, resume=True)
+ team_scan.run_postgres(split, resume=False)
+ summary = team_scan.finalize(split)
- assert _chunk_rows(split, 3) == _chunk_rows(whole, 1)
- assert len(_chunk_rows(whole, 1)) == 5
+ assert _report(split) == _report(whole)
+ # The chunk bounds are sampled keys, which the summary must not carry.
+ assert summary["plan"]["chunks"] == 3
+ bounds = [b for c in plan.chunks for b in (c.person_from, c.person_to, c.mapping_from, c.mapping_to) if b]
+ assert bounds and not any(b in json.dumps(summary) for b in bounds)
+ assert len(_report(whole)) == 11
def test_the_scan_directory_is_private_to_its_owner(self):
scan_dir = self.scan_root / "private"
@@ -158,3 +224,83 @@ class TestChunkScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
with pytest.raises(team_scan.ScanError, match="readable by other users"):
team_scan.run_chunk(scan_dir, 0, resume=False)
scan_dir.chmod(0o700)
+
+
+def _mapping(old: str, new: str, sub: str = "a_absent", key: str | None = None) -> dict[str, Any]:
+ verb = "fold" if sub == "a_absent" else "merge"
+ return {
+ "kind": "mapping",
+ "key": key or f"{old}-{new}",
+ "flavor": "diverged",
+ "sub_flavor": sub,
+ "verb": verb,
+ "group": None,
+ "also": [],
+ "evidence": {"pg": {"person_uuid": new}, "ch": {"winner_person_uuid": old}},
+ }
+
+
+def _ghost_mapping(owner: str) -> dict[str, Any]:
+ return {
+ "kind": "mapping",
+ "key": f"ghost-{owner}",
+ "flavor": "ghost",
+ "sub_flavor": "owner_ghost",
+ "verb": "restore",
+ "group": None,
+ "also": [],
+ "evidence": {"pg": None, "ch": {"winner_person_uuid": owner}},
+ }
+
+
+def _ghost_person(key: str, verb: str | None) -> dict[str, Any]:
+ return {
+ "kind": "person",
+ "key": key,
+ "flavor": "ghost",
+ "sub_flavor": "after_purge",
+ "verb": verb,
+ "group": None,
+ "also": [],
+ }
+
+
+class TestRegroup(SimpleTestCase):
+ @parameterized.expand(
+ [
+ # A's ids went to two persons: folding A would copy it into both.
+ (
+ "old_owner_split",
+ [_mapping("a", "b1"), _mapping("a", "b2")],
+ {"a-b1": (None, "ambiguous:a", None), "a-b2": (None, "ambiguous:a", None)},
+ ),
+ ("live_pair_merges", [_mapping("a", "b", sub="a_live")], {"a-b": ("merge", "merge:a>b", None)}),
+ # Every id of A moved to B, so no ghost id is left to merge from.
+ ("absent_pair_without_ghost_ids_folds_light", [_mapping("a", "b")], {"a-b": ("fold", "fold:a>b", "light")}),
+ (
+ "absent_pair_with_a_ghost_id_folds_full",
+ [_mapping("a", "b"), _ghost_mapping("a")],
+ {"a-b": ("fold", "fold:a>b", "full"), "ghost-a": ("fold", "fold:a>b", "full")},
+ ),
+ (
+ "ghost_mapping_restores_with_its_owner",
+ [_ghost_person("a", "restore"), _ghost_mapping("a")],
+ {"a": ("restore", "restore:a", None), "ghost-a": ("restore", "restore:a", None)},
+ ),
+ (
+ "ghost_mapping_stays_with_an_unrepaired_owner",
+ [_ghost_person("a", None), _ghost_mapping("a")],
+ {"a": (None, None, None), "ghost-a": (None, None, None)},
+ ),
+ # ClickHouse holds the owner's tombstone, so no person row asks to restore it.
+ ("ghost_mapping_without_an_owner_row_stays", [_ghost_mapping("a")], {"ghost-a": (None, None, None)}),
+ ]
+ )
+ def test_cross_key_groups(self, _name: str, rows: list[dict[str, Any]], expected: dict[str, tuple[Any, Any, Any]]):
+ groups = team_scan._Groups()
+ for row in rows:
+ groups.observe(row)
+
+ final = [team_scan._regroup(dict(row), groups) for row in rows]
+
+ assert {r["key"]: (r["verb"], r["group"], r.get("fold_mode")) for r in final if r is not None} == expected