:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(persons): add the ClickHouse phases of the person consistency team scan (#117440)
Eli Reisman3d3e6e96ece1
diff --git a/posthog/models/person/divergence/lookups.py b/posthog/models/person/divergence/lookups.pyindex 1a42bac4ecd5..92e79aa2a6a6 100644--- a/posthog/models/person/divergence/lookups.py+++ b/posthog/models/person/divergence/lookups.py@@ -8,7 +8,11 @@ from uuid import UUIDfrom posthog.clickhouse.client import sync_executefrom posthog.clickhouse.workload import Workload-from posthog.models.person.util import _batched_get_persons_by_uuids, get_person_tombstones+from posthog.models.person.util import (+ _batched_get_distinct_ids_for_persons,+ _batched_get_persons_by_uuids,+ get_person_tombstones,+)from posthog.personhog_client.client import personhog_callfrom posthog.personhog_client.proto import ReadOptions@@ -47,3 +51,13 @@ def _tombstoned_uuids(team_id: int, person_uuids: list[str]) -> set[str]:def _chunks(items: Sequence[_T], size: int) -> list[Sequence[_T]]:return [items[i : i + size] for i in range(0, len(items), size)]+++def _live_distinct_ids_for_persons(team_id: int, person_ids: list[int]) -> dict[int, list[tuple[str, int]]]:+ """Each person's live distinct ids with their versions."""+ if not person_ids:+ return {}+ by_person = personhog_call(+ "get_distinct_ids_for_persons", lambda: _batched_get_distinct_ids_for_persons(team_id, person_ids)+ )+ return {person_id: [(d.id, d.version) for d in dids] for person_id, dids in by_person.items()}diff --git a/posthog/models/person/divergence/observe.py b/posthog/models/person/divergence/observe.pyindex 16daded93149..0696131eca46 100644--- a/posthog/models/person/divergence/observe.py+++ b/posthog/models/person/divergence/observe.py@@ -17,7 +17,7 @@ _READ_SETTINGS = {"apply_deleted_mask": 0, "max_execution_time": 120, "max_memor_CH_CHUNK = 1_000_OPERATION = "person_consistency_observe"-_CH_PERSONS_SQL = """+_CH_PERSONS_SELECT = """SELECTtoString(id),max(version),@@ -30,11 +30,11 @@ SELECT),maxIf(toUnixTimestamp(_timestamp), _row_exists)FROM person-WHERE team_id = %(team_id)s AND id IN %(person_uuids)s+WHERE team_id = %(team_id)s AND {keys}GROUP BY id"""-_CH_MAPPINGS_SQL = """+_CH_MAPPINGS_SELECT = """SELECTdistinct_id,max(version),@@ -43,10 +43,26 @@ SELECTmaxIf(version, _row_exists),argMaxIf(is_deleted, version, _row_exists)FROM person_distinct_id2-WHERE team_id = %(team_id)s AND distinct_id IN %(distinct_ids)s+WHERE team_id = %(team_id)s AND {keys}GROUP BY distinct_id"""+# A range starts after `after`, or at it on a scan's first page, and ends at `last` inclusive.+_CH_PERSONS_SQL = _CH_PERSONS_SELECT.replace("{keys}", "id IN %(person_uuids)s")+_CH_PERSONS_RANGE_SQL = {+ first: _CH_PERSONS_SELECT.replace(+ "{keys}", f"id {'>=' if first else '>'} toUUID(%(after)s) AND id <= toUUID(%(last)s)"+ )+ for first in (True, False)+}+_CH_MAPPINGS_SQL = _CH_MAPPINGS_SELECT.replace("{keys}", "distinct_id IN %(distinct_ids)s")+_CH_MAPPINGS_RANGE_SQL = {+ first: _CH_MAPPINGS_SELECT.replace(+ "{keys}", f"distinct_id {'>=' if first else '>'} %(after)s AND distinct_id <= %(last)s"+ )+ for first in (True, False)+}+def ch_person_from_row(row: Sequence[Any]) -> tuple[str, ChPerson]:person_uuid, max_version, visible_rows, live_max, tomb_max, visible, newest_written_at = row@@ -75,21 +91,37 @@ def ch_person_states(team_id: int, person_uuids: Sequence[str]) -> dict[str, ChPreturn states+def ch_person_states_in_range(team_id: int, after: str, last: str, *, first: bool) -> dict[str, ChPerson]:+ rows = lookups._ch(_CH_PERSONS_RANGE_SQL[first], {"team_id": team_id, "after": after, "last": last}, _READ_SETTINGS)+ return dict(ch_person_from_row(row) for row in rows)+++def ch_mapping_from_row(row: Sequence[Any]) -> tuple[str, ChMapping]:+ distinct_id, max_version, visible_rows, owner, winner_version, winner_deleted = row+ return distinct_id, ChMapping(+ max_version=int(max_version),+ visible_rows=int(visible_rows),+ winner_person_uuid=str(owner),+ winner_version=int(winner_version),+ winner_deleted=bool(winner_deleted),+ )++def ch_mapping_states(team_id: int, distinct_ids: Sequence[str]) -> dict[str, ChMapping]:states: dict[str, ChMapping] = {}for chunk in lookups._chunks(list(distinct_ids), _CH_CHUNK):rows = lookups._ch(_CH_MAPPINGS_SQL, {"team_id": team_id, "distinct_ids": list(chunk)}, _READ_SETTINGS)- for distinct_id, max_version, visible_rows, owner, winner_version, winner_deleted in rows:- states[distinct_id] = ChMapping(- max_version=int(max_version),- visible_rows=int(visible_rows),- winner_person_uuid=str(owner),- winner_version=int(winner_version),- winner_deleted=bool(winner_deleted),- )+ states.update(ch_mapping_from_row(row) for row in rows)return states+def ch_mapping_states_in_range(team_id: int, after: str, last: str, *, first: bool) -> dict[str, ChMapping]:+ rows = lookups._ch(+ _CH_MAPPINGS_RANGE_SQL[first], {"team_id": team_id, "after": after, "last": last}, _READ_SETTINGS+ )+ return dict(ch_mapping_from_row(row) for row in rows)++def pg_person_states(team_id: int, person_uuids: Sequence[str]) -> dict[str, PgPerson]:"""Live and tombstoned Postgres persons by uuid; absent uuids are left out."""uuids = list(person_uuids)diff --git a/posthog/models/person/divergence/team_scan.py b/posthog/models/person/divergence/team_scan.pynew file mode 100644index 000000000000..124a2d41e742--- /dev/null+++ b/posthog/models/person/divergence/team_scan.py@@ -0,0 +1,545 @@+"""Classify every person and mapping of one team, in ClickHouse primary key order, into resumable chunk files.++Each chunk reads persons by (team_id, id) and mappings by (team_id, distinct_id), compares them with Postgres,+and writes a hash of every key it classified for a later Postgres pass. A page is recorded in the cursor only+after its rows are written, so a stopped run resumes at the first unrecorded page. Rows hold raw keys, so the+scan directory is private to its owner.+"""++from __future__ import annotations++import os+import json+import stat+import time+import fcntl+import hashlib+import dataclasses+from collections import Counter+from collections.abc import Callable, Sequence+from datetime import UTC, datetime+from pathlib import Path+from typing import Any, Literal++import numpy as np++from posthog.dataclasses import frozen+from posthog.models.person.divergence import lookups, observe+from posthog.models.person.divergence.flavors import Verdict, classify_mapping, classify_person+from posthog.models.person.divergence.inspection import ghost_era, hash_distinct_id+from posthog.models.team import Team++PLAN_VERSION = 1+PLAN_FILE = "plan.json"++KeyKind = Literal["person", "mapping"]+ChPhase = Literal["persons", "mappings"]+CH_PHASES: tuple[ChPhase, ...] = ("persons", "mappings")+DONE = "done"++# The smallest uuid and distinct id. A chunk's first page starts at its lower bound inclusive.+_FIRST_PERSON_KEY = "00000000-0000-0000-0000-000000000000"+_FIRST_MAPPING_KEY = ""++_READ_SETTINGS = {"apply_deleted_mask": 0, "max_execution_time": 120}+_SAMPLE_PER_CHUNK = 200++_PERSON_PAGE_END = """+SELECT count(), toString(max(id)) FROM (+ SELECT id FROM person+ WHERE team_id = %(team_id)s AND id {op} toUUID(%(after)s){upper}+ ORDER BY id LIMIT %(rows)s+)+"""+_MAPPING_PAGE_END = """+SELECT count(), max(distinct_id) FROM (+ SELECT distinct_id FROM person_distinct_id2+ WHERE team_id = %(team_id)s AND distinct_id {op} %(after)s{upper}+ ORDER BY distinct_id LIMIT %(rows)s+)+"""+# Keyed by (phase, first page, bounded above). The page query reads in primary key order and stops at+# the row limit, and returns only the last key, so a page never sends keys back to the client.+_PAGE_END_SQL = {+ (phase, first, bounded): template.replace("{op}", ">=" if first else ">").replace(+ "{upper}", f" AND {column} < {cast}" if bounded else ""+ )+ for phase, template, column, cast in (+ ("persons", _PERSON_PAGE_END, "id", "toUUID(%(upper)s)"),+ ("mappings", _MAPPING_PAGE_END, "distinct_id", "%(upper)s"),+ )+ for first in (True, False)+ for bounded in (True, False)+}++# arraySort orders the sample with ClickHouse's own comparison, which the page queries also use.+_SAMPLE_SQL = {+ "persons": """+ SELECT arrayMap(x -> toString(x), arraySort(groupArraySample(%(size)s, 1)(id)))+ FROM person WHERE team_id = %(team_id)s+ """,+ "mappings": """+ SELECT arraySort(groupArraySample(%(size)s, 1)(distinct_id))+ FROM person_distinct_id2 WHERE team_id = %(team_id)s+ """,+}+++class ScanError(Exception):+ pass+++@frozen+class KeyBounds:+ # Inclusive lower and exclusive upper bound; None is open.+ lower: str | None+ upper: str | None+++@frozen+class PageResult:+ rows: list[dict[str, Any]]+ # Hashes of every key the page classified, for the Postgres phase's key sets.+ keys: list[int]+ counts: Counter[str]+++@frozen+class ChunkBounds:+ # Inclusive lower and exclusive upper bound per ClickHouse phase; None is open.+ person_from: str | None+ person_to: str | None+ mapping_from: str | None+ mapping_to: str | None++ def bounds(self, phase: ChPhase) -> KeyBounds:+ if phase == "persons":+ return KeyBounds(lower=self.person_from, upper=self.person_to)+ return KeyBounds(lower=self.mapping_from, upper=self.mapping_to)+++@frozen+class ScanPlan:+ version: int+ team_id: int+ team_exists: bool+ created_at: str+ # When the team's Postgres persons were purged, if ever; dates ghosts.+ purged_at: str | None+ chunks: tuple[ChunkBounds, ...]+ ch_page_rows: int+ pg_page_size: int++ def purged_at_datetime(self) -> datetime | None:+ return None if self.purged_at is None else datetime.fromisoformat(self.purged_at)++ def to_json(self) -> dict[str, Any]:+ return dataclasses.asdict(self)++ @classmethod+ def from_json(cls, data: dict[str, Any]) -> ScanPlan:+ if data.get("version") != PLAN_VERSION:+ raise ScanError(f"plan version {data.get('version')} is not {PLAN_VERSION}")+ return cls(**{**data, "chunks": tuple(ChunkBounds(**c) for c in data["chunks"])})++[email protected](frozen=False)+class Cursor:+ # A ClickHouse phase name, "postgres", or "done".+ phase: str+ # The last key the phase finished; None before its first page.+ after: str | None = None+ rows_bytes: int = 0+ person_keys_bytes: int = 0+ mapping_keys_bytes: int = 0+ pages: int = 0+ # Classified keys by "kind/flavor/sub_flavor", including the ok ones the rows file leaves out.+ counts: dict[str, int] = dataclasses.field(default_factory=dict)+ updated_at: str | None = None+++@frozen+class RunFiles:+ rows: Path+ cursor: Path+ person_keys: Path | None+ mapping_keys: Path | None+++def chunk_files(scan_dir: Path, chunk: int) -> RunFiles:+ return RunFiles(+ rows=scan_dir / f"chunk-{chunk}.rows.jsonl",+ cursor=scan_dir / f"chunk-{chunk}.cursor.json",+ person_keys=scan_dir / f"chunk-{chunk}.person-keys.u64",+ mapping_keys=scan_dir / f"chunk-{chunk}.mapping-keys.u64",+ )+++def key_hash(key: str) -> int:+ """A 64-bit hash of a uuid or distinct id, for the key sets the Postgres phase checks against."""+ return int.from_bytes(hashlib.blake2b(key.encode(), digest_size=8).digest(), "little")+++def _private_dir(scan_dir: Path, *, create: bool = False) -> None:+ """Check that the scan directory is a real directory that only this user owns and can read, because its rows+ hold raw customer keys. With `create`, a missing directory is created that way first."""+ if create and not os.path.lexists(scan_dir):+ scan_dir.mkdir(mode=0o700, parents=True)+ info = scan_dir.lstat()+ if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode):+ raise ScanError(f"{scan_dir} is not a directory; choose a new directory for the scan")+ if info.st_uid != os.geteuid():+ raise ScanError(f"{scan_dir} is owned by another user; choose a new directory for the scan")+ if info.st_mode & 0o077:+ raise ScanError(f"{scan_dir} is readable by other users; choose a new directory for the scan")+++def _now() -> str:+ return datetime.now(UTC).isoformat()+++def _write_json_atomic(path: Path, data: dict[str, Any]) -> None:+ tmp = path.with_suffix(path.suffix + ".tmp")+ tmp.write_text(json.dumps(data, indent=2, sort_keys=True))+ os.replace(tmp, path)+++def load_plan(scan_dir: Path) -> ScanPlan:+ path = scan_dir / PLAN_FILE+ if not path.exists():+ raise ScanError(f"{path} does not exist; create a plan first")+ # Every run and verb loads the plan, so each re-checks the directory before it writes or reads rows.+ _private_dir(scan_dir)+ return ScanPlan.from_json(json.loads(path.read_text()))+++def load_cursor(path: Path) -> Cursor | None:+ return Cursor(**json.loads(path.read_text())) if path.exists() else None+++def _sample_bounds(team_id: int, phase: ChPhase, chunks: int) -> list[str]:+ rows = lookups._ch(_SAMPLE_SQL[phase], {"team_id": team_id, "size": chunks * _SAMPLE_PER_CHUNK}, _READ_SETTINGS)+ sample = list(rows[0][0]) if rows else []+ bounds: list[str] = []+ for k in range(1, chunks):+ if not sample:+ break+ candidate = sample[k * len(sample) // chunks]+ if not bounds or candidate != bounds[-1]:+ bounds.append(candidate)+ return bounds+++def _phase_edges(bounds: list[str], chunks: int, first_key: str) -> list[KeyBounds]:+ points: list[str | None] = [None, *bounds, None]+ edges = [KeyBounds(lower=points[i], upper=points[i + 1]) for i in range(len(points) - 1)]+ # A small team can yield fewer distinct bounds than chunks. The extra chunks get an empty range.+ empty = bounds[-1] if bounds else first_key+ return edges + [KeyBounds(lower=empty, upper=empty)] * (chunks - len(edges))+++def _chunks_from_bounds(person_bounds: list[str], mapping_bounds: list[str], chunks: int) -> tuple[ChunkBounds, ...]:+ persons = _phase_edges(person_bounds, chunks, _FIRST_PERSON_KEY)+ mappings = _phase_edges(mapping_bounds, chunks, _FIRST_MAPPING_KEY)+ return tuple(+ ChunkBounds(person_from=p.lower, person_to=p.upper, mapping_from=m.lower, mapping_to=m.upper)+ for p, m in zip(persons, mappings)+ )+++def create_plan(+ scan_dir: Path,+ team_id: int,+ *,+ chunks: int = 1,+ purged_at: datetime | None = None,+ ch_page_rows: int = 100_000,+ pg_page_size: int = 5_000,+) -> ScanPlan:+ """Write a scan plan. Chunk bounds come from one sample of the team's keys, taken once, so every+ chunk run of the scan shares the same bounds."""+ if (scan_dir / PLAN_FILE).exists():+ raise ScanError(f"{scan_dir / PLAN_FILE} already exists")+ _private_dir(scan_dir, create=True)+ if chunks > 1:+ bounds = _chunks_from_bounds(+ _sample_bounds(team_id, "persons", chunks), _sample_bounds(team_id, "mappings", chunks), chunks+ )+ else:+ bounds = (ChunkBounds(person_from=None, person_to=None, mapping_from=None, mapping_to=None),)+ plan = ScanPlan(+ version=PLAN_VERSION,+ team_id=team_id,+ team_exists=Team.objects.filter(id=team_id).exists(),+ created_at=_now(),+ purged_at=None if purged_at is None else purged_at.isoformat(),+ chunks=bounds,+ ch_page_rows=ch_page_rows,+ pg_page_size=pg_page_size,+ )+ _write_json_atomic(scan_dir / PLAN_FILE, plan.to_json())+ return plan+++class _RunWriter:+ """Appends a page's rows and key hashes, then records the page in the cursor.++ Opening it truncates every file to the offsets the cursor recorded, which drops a page that was+ written but not recorded before a run stopped.+ """++ def __init__(self, files: RunFiles, cursor: Cursor) -> None:+ self.files = files+ self.cursor = cursor+ self._handles: dict[str, Any] = {}+ # A second run would truncate the files under this one. The cursor file is replaced on every page, so the+ # lock needs its own file; the OS releases it when the process dies.+ self._lock = files.cursor.with_suffix(".lock").open("w")+ try:+ fcntl.flock(self._lock, fcntl.LOCK_EX | fcntl.LOCK_NB)+ except BlockingIOError:+ self._lock.close()+ raise ScanError(f"{files.cursor.name} is in use by another run") from None+ on_disk = load_cursor(files.cursor)+ if on_disk is not None and on_disk != cursor:+ self._lock.close()+ raise ScanError(f"{files.cursor.name} changed while this run started; run it again")+ for name, path, offset in (+ ("rows", files.rows, cursor.rows_bytes),+ ("person_keys", files.person_keys, cursor.person_keys_bytes),+ ("mapping_keys", files.mapping_keys, cursor.mapping_keys_bytes),+ ):+ if path is None:+ continue+ path.touch(exist_ok=True)+ os.truncate(path, offset)+ self._handles[name] = path.open("ab")++ def commit(+ self,+ rows: Sequence[dict[str, Any]],+ *,+ person_keys: Sequence[int] = (),+ mapping_keys: Sequence[int] = (),+ counts: Counter[str],+ after: str | None,+ phase: str,+ ) -> None:+ self._handles["rows"].write(b"".join(json.dumps(r, sort_keys=True).encode() + b"\n" for r in rows))+ for name, keys in (("person_keys", person_keys), ("mapping_keys", mapping_keys)):+ if keys:+ np.asarray(keys, dtype="<u8").tofile(self._handles[name])+ for handle in self._handles.values():+ handle.flush()+ os.fsync(handle.fileno())+ cursor = self.cursor+ cursor.rows_bytes = self._handles["rows"].tell()+ if "person_keys" in self._handles:+ cursor.person_keys_bytes = self._handles["person_keys"].tell()+ cursor.mapping_keys_bytes = self._handles["mapping_keys"].tell()+ for key, n in counts.items():+ cursor.counts[key] = cursor.counts.get(key, 0) + n+ cursor.after = after+ cursor.phase = phase+ cursor.pages += 1+ cursor.updated_at = _now()+ _write_json_atomic(self.files.cursor, dataclasses.asdict(cursor))++ def finish_phase(self, next_phase: str) -> None:+ self.cursor.phase = next_phase+ self.cursor.after = None+ self.cursor.updated_at = _now()+ _write_json_atomic(self.files.cursor, dataclasses.asdict(self.cursor))++ def close(self) -> None:+ for handle in self._handles.values():+ handle.close()+ self._lock.close()+++def _state(state: Any) -> dict[str, Any] | None:+ return None if state is None else dataclasses.asdict(state)+++def _row(+ *,+ kind: KeyKind,+ team_id: int,+ key: str,+ verdict: Verdict,+ phase: str,+ observed_at: str,+ pg: Any,+ ch: Any,+ ch_owner_pg_state: str | None = None,+) -> dict[str, Any]:+ return {+ "kind": kind,+ "team_id": team_id,+ "key": key,+ # Safe to share: distinct ids are customer data, their hash is not.+ "key_hash": hash_distinct_id(team_id, key) if kind == "mapping" else None,+ "flavor": verdict.flavor,+ "sub_flavor": verdict.sub_flavor,+ "verb": verdict.verb,+ "group": None,+ "also": [],+ "phase": phase,+ "observed_at": observed_at,+ "evidence": {"pg": _state(pg), "ch": _state(ch), "ch_owner_pg_state": ch_owner_pg_state},+ }+++def _count_key(kind: str, verdict: Verdict) -> str:+ return f"{kind}/{verdict.flavor}/{verdict.sub_flavor or '-'}"+++def _reported(verdict: Verdict) -> bool:+ return verdict.flavor not in ("ok", "ok_deleted")+++def _page_end(plan: ScanPlan, phase: ChPhase, after: str, first: bool, upper: str | None) -> tuple[int, str | None]:+ sql = _PAGE_END_SQL[(phase, first, upper is not None)]+ args: dict[str, Any] = {"team_id": plan.team_id, "after": after, "rows": plan.ch_page_rows}+ if upper is not None:+ args["upper"] = upper+ rows = lookups._ch(sql, args, _READ_SETTINGS)+ count = int(rows[0][0]) if rows else 0+ return count, (str(rows[0][1]) if count else None)+++def _classify_persons_page(plan: ScanPlan, after: str, last: str, first: bool) -> PageResult:+ observed_at = _now()+ ch = observe.ch_person_states_in_range(plan.team_id, after, last, first=first)+ uuids = list(ch)+ pg = observe.pg_person_states(plan.team_id, uuids) if plan.team_exists and uuids else {}+ purged_at = plan.purged_at_datetime()+ rows: list[dict[str, Any]] = []+ counts: Counter[str] = Counter()+ for person_uuid, ch_state in ch.items():+ pg_state = pg.get(person_uuid)+ verdict = classify_person(+ pg_state,+ ch_state,+ team_exists=plan.team_exists,+ ghost_era=ghost_era(ch_state.newest_written_at, purged_at),+ )+ counts[_count_key("person", verdict)] += 1+ if _reported(verdict):+ rows.append(+ _row(+ kind="person",+ team_id=plan.team_id,+ key=person_uuid,+ verdict=verdict,+ phase="persons",+ observed_at=observed_at,+ pg=pg_state,+ ch=ch_state,+ )+ )+ return PageResult(rows=rows, keys=[key_hash(u) for u in uuids], counts=counts)+++def _classify_mappings_page(plan: ScanPlan, after: str, last: str, first: bool) -> PageResult:+ observed_at = _now()+ ch = observe.ch_mapping_states_in_range(plan.team_id, after, last, first=first)+ distinct_ids = list(ch)+ pg = observe.pg_mapping_rows(plan.team_id, distinct_ids) if plan.team_exists and distinct_ids else {}+ # Only ghost and diverged mappings need their ClickHouse owner's Postgres state.+ needs_owner = {+ distinct_id+ for distinct_id, state in ch.items()+ if pg.get(distinct_id) is None or pg[distinct_id].person_uuid != state.winner_person_uuid+ }+ owners = sorted({ch[distinct_id].winner_person_uuid for distinct_id in needs_owner})+ owner_states = observe.pg_person_states(plan.team_id, owners) if plan.team_exists and owners else {}+ rows: list[dict[str, Any]] = []+ counts: Counter[str] = Counter()+ for distinct_id, ch_state in ch.items():+ pg_row = pg.get(distinct_id)+ owner_state = (+ observe.person_pg_state(owner_states, ch_state.winner_person_uuid) if distinct_id in needs_owner else None+ )+ verdict = classify_mapping(pg_row, ch_state, ch_owner_pg_state=owner_state, team_exists=plan.team_exists)+ counts[_count_key("mapping", verdict)] += 1+ if _reported(verdict):+ rows.append(+ _row(+ kind="mapping",+ team_id=plan.team_id,+ key=distinct_id,+ verdict=verdict,+ phase="mappings",+ observed_at=observed_at,+ pg=pg_row,+ ch=ch_state,+ ch_owner_pg_state=owner_state,+ )+ )+ return PageResult(rows=rows, keys=[key_hash(d) for d in distinct_ids], counts=counts)+++_PAGE_CLASSIFIERS: dict[ChPhase, Callable[..., PageResult]] = {+ "persons": _classify_persons_page,+ "mappings": _classify_mappings_page,+}+++def run_chunk(+ scan_dir: Path,+ chunk: int,+ *,+ resume: bool,+ pause_seconds: float = 0.0,+ log: Callable[[str], None] = lambda _: None,+) -> Cursor:+ """Run both ClickHouse phases of one chunk to the end, or resume them from the cursor."""+ plan = load_plan(scan_dir)+ if not 0 <= chunk < len(plan.chunks):+ raise ScanError(f"chunk {chunk} is not in the plan's {len(plan.chunks)} chunks")+ files = chunk_files(scan_dir, chunk)+ cursor = load_cursor(files.cursor)+ if cursor is not None and not resume:+ raise ScanError(f"{files.cursor} exists; pass resume to continue it")+ cursor = cursor or Cursor(phase=CH_PHASES[0])+ writer = _RunWriter(files, cursor)+ try:+ while cursor.phase != DONE:+ phase: ChPhase = "persons" if cursor.phase == "persons" else "mappings"+ bounds = plan.chunks[chunk].bounds(phase)+ lower, upper = bounds.lower, bounds.upper+ first = cursor.after is None+ after = cursor.after if cursor.after is not None else (lower if lower is not None else _first_key(phase))+ count, last = _page_end(plan, phase, after, first, upper)+ if last is None:+ writer.finish_phase(_next_phase(phase))+ log(f"chunk {chunk} {phase}: done")+ continue+ page = _PAGE_CLASSIFIERS[phase](plan, after, last, first)+ rows, keys, counts = page.rows, page.keys, page.counts+ writer.commit(+ rows,+ person_keys=keys if phase == "persons" else (),+ mapping_keys=keys if phase == "mappings" else (),+ counts=counts,+ after=last,+ phase=phase,+ )+ log(f"chunk {chunk} {phase}: page {cursor.pages}, {len(keys)} keys, {len(rows)} reported")+ if count < plan.ch_page_rows:+ writer.finish_phase(_next_phase(phase))+ log(f"chunk {chunk} {phase}: done")+ elif pause_seconds:+ time.sleep(pause_seconds)+ finally:+ writer.close()+ return cursor+++def _first_key(phase: ChPhase) -> str:+ return _FIRST_PERSON_KEY if phase == "persons" else _FIRST_MAPPING_KEY+++def _next_phase(phase: ChPhase) -> str:+ return "mappings" if phase == "persons" else DONEdiff --git a/posthog/models/person/test/divergence_fixtures.py b/posthog/models/person/test/divergence_fixtures.pynew file mode 100644index 000000000000..d1101336b4f6--- /dev/null+++ b/posthog/models/person/test/divergence_fixtures.py@@ -0,0 +1,61 @@+from typing import TYPE_CHECKING++from django.utils.timezone import now++from posthog.clickhouse.client import sync_execute+from posthog.models.person.sql import BULK_INSERT_PERSON_DISTINCT_ID2+from posthog.models.person.util import create_person as create_person_in_ch+from posthog.models.signals import mute_selected_signals+from posthog.test.persons import add_distinct_id, create_person++if TYPE_CHECKING:+ from posthog.models.team import Team+++class DivergenceFixturesMixin:+ """Writes Postgres (personhog fake) and ClickHouse person state independently, so a test can build any+ disagreement between them. Needs ClickhouseTestMixin and BaseTest."""++ team: "Team"++ def setUp(self) -> None:+ super().setUp() # type: ignore[misc]+ # A background merge collapses the duplicate and older rows the tie and blanked shapes are made of.+ for table in ("person", "person_distinct_id2"):+ sync_execute(f"SYSTEM STOP MERGES {table}")+ self.addCleanup(sync_execute, f"SYSTEM START MERGES {table}") # type: ignore[attr-defined]++ def _pg_person(self, *, version: int, distinct_ids: list[str] | None = None):+ with mute_selected_signals():+ person = create_person(team=self.team, version=version, properties={"email": "[email protected]"})+ for distinct_id in distinct_ids or []:+ add_distinct_id(person=person, distinct_id=distinct_id, version=0)+ return person++ def _ch_person(self, person_uuid: str, version: int, properties: dict, *, deleted: bool = False) -> None:+ create_person_in_ch(+ team_id=self.team.pk,+ uuid=person_uuid,+ version=version,+ is_deleted=deleted,+ properties=properties,+ timestamp=now(),+ )++ def _ch_mapping(self, distinct_id: str, person_uuid: str, version: int, *, team_id: int | None = None) -> None:+ sync_execute(+ BULK_INSERT_PERSON_DISTINCT_ID2,+ [+ {+ "distinct_id": distinct_id,+ "person_id": person_uuid,+ "team_id": team_id or self.team.pk,+ "is_deleted": 0,+ "version": version,+ "_timestamp": now().replace(tzinfo=None),+ "_offset": 0,+ "_partition": 0,+ }+ ],+ flush=False,+ )diff --git a/posthog/models/person/test/test_divergence_inspection.py b/posthog/models/person/test/test_divergence_inspection.pyindex 901e11589c19..a5027d9633dd 100644--- a/posthog/models/person/test/test_divergence_inspection.py+++ b/posthog/models/person/test/test_divergence_inspection.py@@ -5,64 +5,15 @@ from uuid import uuid4from posthog.test.base import BaseTest, ClickhouseTestMixinfrom django.core.management import call_command-from django.utils.timezone import now-from posthog.clickhouse.client import sync_executefrom posthog.models.activity_logging.activity_log import ActivityLog, Detailfrom posthog.models.person.divergence.inspection import hash_distinct_id, inspect_mappings, inspect_persons-from posthog.models.person.sql import BULK_INSERT_PERSON_DISTINCT_ID2-from posthog.models.person.util import (- create_person as create_person_in_ch,- tombstone_persons_in_postgres,-)-from posthog.models.signals import mute_selected_signals+from posthog.models.person.test.divergence_fixtures import DivergenceFixturesMixin+from posthog.models.person.util import tombstone_persons_in_postgresfrom posthog.personhog_client.fake_client import get_active_fake-from posthog.test.persons import add_distinct_id, create_person---class TestDivergenceInspection(ClickhouseTestMixin, BaseTest):- def setUp(self) -> None:- super().setUp()- # A background merge collapses the duplicate and older rows the tie and blanked shapes are made of.- for table in ("person", "person_distinct_id2"):- sync_execute(f"SYSTEM STOP MERGES {table}")- self.addCleanup(sync_execute, f"SYSTEM START MERGES {table}")-- def _pg_person(self, *, version: int, distinct_ids: list[str] | None = None):- with mute_selected_signals():- person = create_person(team=self.team, version=version, properties={"email": "[email protected]"})- for distinct_id in distinct_ids or []:- add_distinct_id(person=person, distinct_id=distinct_id, version=0)- return person-- def _ch_person(self, person_uuid: str, version: int, properties: dict, *, deleted: bool = False) -> None:- create_person_in_ch(- team_id=self.team.pk,- uuid=person_uuid,- version=version,- is_deleted=deleted,- properties=properties,- timestamp=now(),- )- def _ch_mapping(self, distinct_id: str, person_uuid: str, version: int, *, team_id: int | None = None) -> None:- sync_execute(- BULK_INSERT_PERSON_DISTINCT_ID2,- [- {- "distinct_id": distinct_id,- "person_id": person_uuid,- "team_id": team_id or self.team.pk,- "is_deleted": 0,- "version": version,- "_timestamp": now().replace(tzinfo=None),- "_offset": 0,- "_partition": 0,- }- ],- flush=False,- )+class TestDivergenceInspection(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):def _log_person_activity(self, activity: str, *, item_id: int, name: str | None) -> None:ActivityLog.objects.create(organization_id=self.organization.id,diff --git a/posthog/models/person/test/test_divergence_team_scan.py b/posthog/models/person/test/test_divergence_team_scan.pynew file mode 100644index 000000000000..d5607c0e599e--- /dev/null+++ b/posthog/models/person/test/test_divergence_team_scan.py@@ -0,0 +1,160 @@+import os+import json+import shutil+import tempfile+from pathlib import Path+from typing import Any+from uuid import uuid4++import pytest+from posthog.test.base import BaseTest, ClickhouseTestMixin+from unittest.mock import patch++from posthog.models.person.divergence import team_scan+from posthog.models.person.test.divergence_fixtures import DivergenceFixturesMixin+from posthog.models.person.util import tombstone_persons_in_postgres+++def _chunk_rows(scan_dir: Path, chunks: int) -> dict[str, tuple[Any, ...]]:+ rows = []+ for chunk in range(chunks):+ path = team_scan.chunk_files(scan_dir, chunk).rows+ rows += [json.loads(line) for line in path.read_text().splitlines()]+ found = {r["key"]: (r["flavor"], r["sub_flavor"], r["verb"]) for r in rows}+ assert len(found) == len(rows), "a key was reported twice"+ return found+++class _Stop(Exception):+ pass+++class TestChunkScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):+ def setUp(self) -> None:+ super().setUp()+ self.scan_root = Path(tempfile.mkdtemp())+ self.addCleanup(shutil.rmtree, self.scan_root)++ def _build_team(self) -> dict[str, str]:+ """One key per flavor the ClickHouse phases must find."""+ fine = self._pg_person(version=1, distinct_ids=["fine-1"])+ self._ch_person(str(fine.uuid), 1, {"email": "[email protected]"})+ self._ch_mapping("fine-1", str(fine.uuid), 0)+ stale = self._pg_person(version=1)+ self._ch_person(str(stale.uuid), 40, {"email": "[email protected]"})+ deleted = self._pg_person(version=2)+ tombstone_persons_in_postgres(self.team.pk, [deleted.uuid])+ self._ch_person(str(deleted.uuid), 7, {"email": "[email protected]"})+ new_owner = self._pg_person(version=1, distinct_ids=["moved"])+ self._ch_person(str(new_owner.uuid), 1, {"email": "[email protected]"})+ ghost = str(uuid4())+ self._ch_person(ghost, 3, {"email": "[email protected]"})+ self._ch_mapping("moved", ghost, 5)+ self._ch_mapping("lost", ghost, 5)+ return {"stale": str(stale.uuid), "deleted": str(deleted.uuid), "ghost": ghost}++ def test_a_chunk_reports_each_clickhouse_flavor_with_its_verb(self):+ keys = self._build_team()+ scan_dir = self.scan_root / "scan"+ team_scan.create_plan(scan_dir, self.team.pk)++ cursor = team_scan.run_chunk(scan_dir, 0, resume=False)++ assert cursor.phase == team_scan.DONE+ assert _chunk_rows(scan_dir, 1) == {+ keys["stale"]: ("stale", None, "merge_stale"),+ keys["deleted"]: ("deleted_visible", None, "bury"),+ keys["ghost"]: ("ghost", "historical", None),+ "moved": ("diverged", "a_absent", "fold"),+ "lost": ("ghost", "owner_ghost", "restore"),+ }+ with pytest.raises(team_scan.ScanError, match="pass resume"):+ team_scan.run_chunk(scan_dir, 0, resume=False)++ def test_a_deleted_team_reports_its_clickhouse_leftovers_without_repairs(self):+ keys = self._build_team()+ scan_dir = self.scan_root / "scan"+ with patch.object(team_scan.Team.objects, "filter") as team_filter:+ team_filter.return_value.exists.return_value = False+ team_scan.create_plan(scan_dir, self.team.pk)++ team_scan.run_chunk(scan_dir, 0, resume=False)++ rows = _chunk_rows(scan_dir, 1)+ # Five persons and three mappings, all ClickHouse-only once Postgres is not read.+ assert len(rows) == 8+ assert {keys["stale"], keys["deleted"], keys["ghost"], "moved", "lost", "fine-1"} <= rows.keys()+ assert set(rows.values()) == {("ghost", "team_gone", None)}++ def test_a_second_run_of_a_running_chunk_is_refused(self):+ self._build_team()+ scan_dir = self.scan_root / "scan"+ team_scan.create_plan(scan_dir, self.team.pk, ch_page_rows=2)+ refusals: list[str] = []++ def start_a_second_run(message: str) -> None:+ if "page" in message and not refusals:+ with pytest.raises(team_scan.ScanError, match="in use by another run") as refused:+ team_scan.run_chunk(scan_dir, 0, resume=True)+ refusals.append(str(refused.value))++ cursor = team_scan.run_chunk(scan_dir, 0, resume=False, log=start_a_second_run)++ assert len(refusals) == 1 and cursor.phase == team_scan.DONE+ assert team_scan.run_chunk(scan_dir, 0, resume=True).phase == team_scan.DONE++ def test_chunked_paged_and_interrupted_scan_gives_the_same_rows_as_one_run(self):+ self._build_team()+ whole = self.scan_root / "whole"+ team_scan.create_plan(whole, self.team.pk)+ team_scan.run_chunk(whole, 0, resume=False)++ split = self.scan_root / "split"+ plan = team_scan.create_plan(split, self.team.pk, chunks=3, ch_page_rows=2)+ assert len(plan.chunks) == 3++ def stop_after_first_page(message: str) -> None:+ if "page" in message:+ raise _Stop++ with pytest.raises(_Stop):+ team_scan.run_chunk(split, 0, resume=False, log=stop_after_first_page)+ # Bytes written after the last recorded page, as by a run that stopped mid-write.+ files = team_scan.chunk_files(split, 0)+ for path in (files.rows, files.person_keys, files.mapping_keys):+ assert path is not None+ with path.open("ab") as handle:+ handle.write(b'{"kind": "person", "key": "partial"')+ for chunk in range(3):+ team_scan.run_chunk(split, chunk, resume=True)++ assert _chunk_rows(split, 3) == _chunk_rows(whole, 1)+ assert len(_chunk_rows(whole, 1)) == 5++ def test_the_scan_directory_is_private_to_its_owner(self):+ scan_dir = self.scan_root / "private"+ team_scan.create_plan(scan_dir, self.team.pk)+ assert scan_dir.stat().st_mode & 0o077 == 0++ shared = self.scan_root / "shared"+ shared.mkdir(mode=0o755)+ shared.chmod(0o755)+ with pytest.raises(team_scan.ScanError, match="readable by other users"):+ team_scan.create_plan(shared, self.team.pk)++ target = self.scan_root / "target"+ target.mkdir(mode=0o700)+ linked = self.scan_root / "linked"+ linked.symlink_to(target)+ with pytest.raises(team_scan.ScanError, match="not a directory"):+ team_scan.create_plan(linked, self.team.pk)++ with patch("posthog.models.person.divergence.team_scan.os.geteuid", return_value=os.geteuid() + 1):+ with pytest.raises(team_scan.ScanError, match="owned by another user"):+ team_scan.run_chunk(scan_dir, 0, resume=False)++ # A directory opened up after its plan was written is refused when the scan resumes.+ scan_dir.chmod(0o755)+ with pytest.raises(team_scan.ScanError, match="readable by other users"):+ team_scan.run_chunk(scan_dir, 0, resume=False)+ scan_dir.chmod(0o700)