Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(persons): add the ClickHouse phases of the person consistency team scan (#117440)

Eli Reisman3d3e6e96ece1
diff --git a/posthog/models/person/divergence/lookups.py b/posthog/models/person/divergence/lookups.py
index 1a42bac4ecd5..92e79aa2a6a6 100644
--- a/posthog/models/person/divergence/lookups.py
+++ b/posthog/models/person/divergence/lookups.py
@@ -8,7 +8,11 @@ from uuid import UUID
from posthog.clickhouse.client import sync_execute
from posthog.clickhouse.workload import Workload
-from posthog.models.person.util import _batched_get_persons_by_uuids, get_person_tombstones
+from posthog.models.person.util import (
+ _batched_get_distinct_ids_for_persons,
+ _batched_get_persons_by_uuids,
+ get_person_tombstones,
+)
from posthog.personhog_client.client import personhog_call
from posthog.personhog_client.proto import ReadOptions
@@ -47,3 +51,13 @@ def _tombstoned_uuids(team_id: int, person_uuids: list[str]) -> set[str]:
def _chunks(items: Sequence[_T], size: int) -> list[Sequence[_T]]:
return [items[i : i + size] for i in range(0, len(items), size)]
+
+
+def _live_distinct_ids_for_persons(team_id: int, person_ids: list[int]) -> dict[int, list[tuple[str, int]]]:
+ """Each person's live distinct ids with their versions."""
+ if not person_ids:
+ return {}
+ by_person = personhog_call(
+ "get_distinct_ids_for_persons", lambda: _batched_get_distinct_ids_for_persons(team_id, person_ids)
+ )
+ return {person_id: [(d.id, d.version) for d in dids] for person_id, dids in by_person.items()}
diff --git a/posthog/models/person/divergence/observe.py b/posthog/models/person/divergence/observe.py
index 16daded93149..0696131eca46 100644
--- a/posthog/models/person/divergence/observe.py
+++ b/posthog/models/person/divergence/observe.py
@@ -17,7 +17,7 @@ _READ_SETTINGS = {"apply_deleted_mask": 0, "max_execution_time": 120, "max_memor
_CH_CHUNK = 1_000
_OPERATION = "person_consistency_observe"
-_CH_PERSONS_SQL = """
+_CH_PERSONS_SELECT = """
SELECT
toString(id),
max(version),
@@ -30,11 +30,11 @@ SELECT
),
maxIf(toUnixTimestamp(_timestamp), _row_exists)
FROM person
-WHERE team_id = %(team_id)s AND id IN %(person_uuids)s
+WHERE team_id = %(team_id)s AND {keys}
GROUP BY id
"""
-_CH_MAPPINGS_SQL = """
+_CH_MAPPINGS_SELECT = """
SELECT
distinct_id,
max(version),
@@ -43,10 +43,26 @@ SELECT
maxIf(version, _row_exists),
argMaxIf(is_deleted, version, _row_exists)
FROM person_distinct_id2
-WHERE team_id = %(team_id)s AND distinct_id IN %(distinct_ids)s
+WHERE team_id = %(team_id)s AND {keys}
GROUP BY distinct_id
"""
+# A range starts after `after`, or at it on a scan's first page, and ends at `last` inclusive.
+_CH_PERSONS_SQL = _CH_PERSONS_SELECT.replace("{keys}", "id IN %(person_uuids)s")
+_CH_PERSONS_RANGE_SQL = {
+ first: _CH_PERSONS_SELECT.replace(
+ "{keys}", f"id {'>=' if first else '>'} toUUID(%(after)s) AND id <= toUUID(%(last)s)"
+ )
+ for first in (True, False)
+}
+_CH_MAPPINGS_SQL = _CH_MAPPINGS_SELECT.replace("{keys}", "distinct_id IN %(distinct_ids)s")
+_CH_MAPPINGS_RANGE_SQL = {
+ first: _CH_MAPPINGS_SELECT.replace(
+ "{keys}", f"distinct_id {'>=' if first else '>'} %(after)s AND distinct_id <= %(last)s"
+ )
+ for first in (True, False)
+}
+
def ch_person_from_row(row: Sequence[Any]) -> tuple[str, ChPerson]:
person_uuid, max_version, visible_rows, live_max, tomb_max, visible, newest_written_at = row
@@ -75,21 +91,37 @@ def ch_person_states(team_id: int, person_uuids: Sequence[str]) -> dict[str, ChP
return states
+def ch_person_states_in_range(team_id: int, after: str, last: str, *, first: bool) -> dict[str, ChPerson]:
+ rows = lookups._ch(_CH_PERSONS_RANGE_SQL[first], {"team_id": team_id, "after": after, "last": last}, _READ_SETTINGS)
+ return dict(ch_person_from_row(row) for row in rows)
+
+
+def ch_mapping_from_row(row: Sequence[Any]) -> tuple[str, ChMapping]:
+ distinct_id, max_version, visible_rows, owner, winner_version, winner_deleted = row
+ return distinct_id, ChMapping(
+ max_version=int(max_version),
+ visible_rows=int(visible_rows),
+ winner_person_uuid=str(owner),
+ winner_version=int(winner_version),
+ winner_deleted=bool(winner_deleted),
+ )
+
+
def ch_mapping_states(team_id: int, distinct_ids: Sequence[str]) -> dict[str, ChMapping]:
states: dict[str, ChMapping] = {}
for chunk in lookups._chunks(list(distinct_ids), _CH_CHUNK):
rows = lookups._ch(_CH_MAPPINGS_SQL, {"team_id": team_id, "distinct_ids": list(chunk)}, _READ_SETTINGS)
- for distinct_id, max_version, visible_rows, owner, winner_version, winner_deleted in rows:
- states[distinct_id] = ChMapping(
- max_version=int(max_version),
- visible_rows=int(visible_rows),
- winner_person_uuid=str(owner),
- winner_version=int(winner_version),
- winner_deleted=bool(winner_deleted),
- )
+ states.update(ch_mapping_from_row(row) for row in rows)
return states
+def ch_mapping_states_in_range(team_id: int, after: str, last: str, *, first: bool) -> dict[str, ChMapping]:
+ rows = lookups._ch(
+ _CH_MAPPINGS_RANGE_SQL[first], {"team_id": team_id, "after": after, "last": last}, _READ_SETTINGS
+ )
+ return dict(ch_mapping_from_row(row) for row in rows)
+
+
def pg_person_states(team_id: int, person_uuids: Sequence[str]) -> dict[str, PgPerson]:
"""Live and tombstoned Postgres persons by uuid; absent uuids are left out."""
uuids = list(person_uuids)
diff --git a/posthog/models/person/divergence/team_scan.py b/posthog/models/person/divergence/team_scan.py
new file mode 100644
index 000000000000..124a2d41e742
--- /dev/null
+++ b/posthog/models/person/divergence/team_scan.py
@@ -0,0 +1,545 @@
+"""Classify every person and mapping of one team, in ClickHouse primary key order, into resumable chunk files.
+
+Each chunk reads persons by (team_id, id) and mappings by (team_id, distinct_id), compares them with Postgres,
+and writes a hash of every key it classified for a later Postgres pass. A page is recorded in the cursor only
+after its rows are written, so a stopped run resumes at the first unrecorded page. Rows hold raw keys, so the
+scan directory is private to its owner.
+"""
+
+from __future__ import annotations
+
+import os
+import json
+import stat
+import time
+import fcntl
+import hashlib
+import dataclasses
+from collections import Counter
+from collections.abc import Callable, Sequence
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, Literal
+
+import numpy as np
+
+from posthog.dataclasses import frozen
+from posthog.models.person.divergence import lookups, observe
+from posthog.models.person.divergence.flavors import Verdict, classify_mapping, classify_person
+from posthog.models.person.divergence.inspection import ghost_era, hash_distinct_id
+from posthog.models.team import Team
+
+PLAN_VERSION = 1
+PLAN_FILE = "plan.json"
+
+KeyKind = Literal["person", "mapping"]
+ChPhase = Literal["persons", "mappings"]
+CH_PHASES: tuple[ChPhase, ...] = ("persons", "mappings")
+DONE = "done"
+
+# The smallest uuid and distinct id. A chunk's first page starts at its lower bound inclusive.
+_FIRST_PERSON_KEY = "00000000-0000-0000-0000-000000000000"
+_FIRST_MAPPING_KEY = ""
+
+_READ_SETTINGS = {"apply_deleted_mask": 0, "max_execution_time": 120}
+_SAMPLE_PER_CHUNK = 200
+
+_PERSON_PAGE_END = """
+SELECT count(), toString(max(id)) FROM (
+ SELECT id FROM person
+ WHERE team_id = %(team_id)s AND id {op} toUUID(%(after)s){upper}
+ ORDER BY id LIMIT %(rows)s
+)
+"""
+_MAPPING_PAGE_END = """
+SELECT count(), max(distinct_id) FROM (
+ SELECT distinct_id FROM person_distinct_id2
+ WHERE team_id = %(team_id)s AND distinct_id {op} %(after)s{upper}
+ ORDER BY distinct_id LIMIT %(rows)s
+)
+"""
+# Keyed by (phase, first page, bounded above). The page query reads in primary key order and stops at
+# the row limit, and returns only the last key, so a page never sends keys back to the client.
+_PAGE_END_SQL = {
+ (phase, first, bounded): template.replace("{op}", ">=" if first else ">").replace(
+ "{upper}", f" AND {column} < {cast}" if bounded else ""
+ )
+ for phase, template, column, cast in (
+ ("persons", _PERSON_PAGE_END, "id", "toUUID(%(upper)s)"),
+ ("mappings", _MAPPING_PAGE_END, "distinct_id", "%(upper)s"),
+ )
+ for first in (True, False)
+ for bounded in (True, False)
+}
+
+# arraySort orders the sample with ClickHouse's own comparison, which the page queries also use.
+_SAMPLE_SQL = {
+ "persons": """
+ SELECT arrayMap(x -> toString(x), arraySort(groupArraySample(%(size)s, 1)(id)))
+ FROM person WHERE team_id = %(team_id)s
+ """,
+ "mappings": """
+ SELECT arraySort(groupArraySample(%(size)s, 1)(distinct_id))
+ FROM person_distinct_id2 WHERE team_id = %(team_id)s
+ """,
+}
+
+
+class ScanError(Exception):
+ pass
+
+
+@frozen
+class KeyBounds:
+ # Inclusive lower and exclusive upper bound; None is open.
+ lower: str | None
+ upper: str | None
+
+
+@frozen
+class PageResult:
+ rows: list[dict[str, Any]]
+ # Hashes of every key the page classified, for the Postgres phase's key sets.
+ keys: list[int]
+ counts: Counter[str]
+
+
+@frozen
+class ChunkBounds:
+ # Inclusive lower and exclusive upper bound per ClickHouse phase; None is open.
+ person_from: str | None
+ person_to: str | None
+ mapping_from: str | None
+ mapping_to: str | None
+
+ def bounds(self, phase: ChPhase) -> KeyBounds:
+ if phase == "persons":
+ return KeyBounds(lower=self.person_from, upper=self.person_to)
+ return KeyBounds(lower=self.mapping_from, upper=self.mapping_to)
+
+
+@frozen
+class ScanPlan:
+ version: int
+ team_id: int
+ team_exists: bool
+ created_at: str
+ # When the team's Postgres persons were purged, if ever; dates ghosts.
+ purged_at: str | None
+ chunks: tuple[ChunkBounds, ...]
+ ch_page_rows: int
+ pg_page_size: int
+
+ def purged_at_datetime(self) -> datetime | None:
+ return None if self.purged_at is None else datetime.fromisoformat(self.purged_at)
+
+ def to_json(self) -> dict[str, Any]:
+ return dataclasses.asdict(self)
+
+ @classmethod
+ def from_json(cls, data: dict[str, Any]) -> ScanPlan:
+ if data.get("version") != PLAN_VERSION:
+ raise ScanError(f"plan version {data.get('version')} is not {PLAN_VERSION}")
+ return cls(**{**data, "chunks": tuple(ChunkBounds(**c) for c in data["chunks"])})
+
+
[email protected](frozen=False)
+class Cursor:
+ # A ClickHouse phase name, "postgres", or "done".
+ phase: str
+ # The last key the phase finished; None before its first page.
+ after: str | None = None
+ rows_bytes: int = 0
+ person_keys_bytes: int = 0
+ mapping_keys_bytes: int = 0
+ pages: int = 0
+ # Classified keys by "kind/flavor/sub_flavor", including the ok ones the rows file leaves out.
+ counts: dict[str, int] = dataclasses.field(default_factory=dict)
+ updated_at: str | None = None
+
+
+@frozen
+class RunFiles:
+ rows: Path
+ cursor: Path
+ person_keys: Path | None
+ mapping_keys: Path | None
+
+
+def chunk_files(scan_dir: Path, chunk: int) -> RunFiles:
+ return RunFiles(
+ rows=scan_dir / f"chunk-{chunk}.rows.jsonl",
+ cursor=scan_dir / f"chunk-{chunk}.cursor.json",
+ person_keys=scan_dir / f"chunk-{chunk}.person-keys.u64",
+ mapping_keys=scan_dir / f"chunk-{chunk}.mapping-keys.u64",
+ )
+
+
+def key_hash(key: str) -> int:
+ """A 64-bit hash of a uuid or distinct id, for the key sets the Postgres phase checks against."""
+ return int.from_bytes(hashlib.blake2b(key.encode(), digest_size=8).digest(), "little")
+
+
+def _private_dir(scan_dir: Path, *, create: bool = False) -> None:
+ """Check that the scan directory is a real directory that only this user owns and can read, because its rows
+ hold raw customer keys. With `create`, a missing directory is created that way first."""
+ if create and not os.path.lexists(scan_dir):
+ scan_dir.mkdir(mode=0o700, parents=True)
+ info = scan_dir.lstat()
+ if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode):
+ raise ScanError(f"{scan_dir} is not a directory; choose a new directory for the scan")
+ if info.st_uid != os.geteuid():
+ raise ScanError(f"{scan_dir} is owned by another user; choose a new directory for the scan")
+ if info.st_mode & 0o077:
+ raise ScanError(f"{scan_dir} is readable by other users; choose a new directory for the scan")
+
+
+def _now() -> str:
+ return datetime.now(UTC).isoformat()
+
+
+def _write_json_atomic(path: Path, data: dict[str, Any]) -> None:
+ tmp = path.with_suffix(path.suffix + ".tmp")
+ tmp.write_text(json.dumps(data, indent=2, sort_keys=True))
+ os.replace(tmp, path)
+
+
+def load_plan(scan_dir: Path) -> ScanPlan:
+ path = scan_dir / PLAN_FILE
+ if not path.exists():
+ raise ScanError(f"{path} does not exist; create a plan first")
+ # Every run and verb loads the plan, so each re-checks the directory before it writes or reads rows.
+ _private_dir(scan_dir)
+ return ScanPlan.from_json(json.loads(path.read_text()))
+
+
+def load_cursor(path: Path) -> Cursor | None:
+ return Cursor(**json.loads(path.read_text())) if path.exists() else None
+
+
+def _sample_bounds(team_id: int, phase: ChPhase, chunks: int) -> list[str]:
+ rows = lookups._ch(_SAMPLE_SQL[phase], {"team_id": team_id, "size": chunks * _SAMPLE_PER_CHUNK}, _READ_SETTINGS)
+ sample = list(rows[0][0]) if rows else []
+ bounds: list[str] = []
+ for k in range(1, chunks):
+ if not sample:
+ break
+ candidate = sample[k * len(sample) // chunks]
+ if not bounds or candidate != bounds[-1]:
+ bounds.append(candidate)
+ return bounds
+
+
+def _phase_edges(bounds: list[str], chunks: int, first_key: str) -> list[KeyBounds]:
+ points: list[str | None] = [None, *bounds, None]
+ edges = [KeyBounds(lower=points[i], upper=points[i + 1]) for i in range(len(points) - 1)]
+ # A small team can yield fewer distinct bounds than chunks. The extra chunks get an empty range.
+ empty = bounds[-1] if bounds else first_key
+ return edges + [KeyBounds(lower=empty, upper=empty)] * (chunks - len(edges))
+
+
+def _chunks_from_bounds(person_bounds: list[str], mapping_bounds: list[str], chunks: int) -> tuple[ChunkBounds, ...]:
+ persons = _phase_edges(person_bounds, chunks, _FIRST_PERSON_KEY)
+ mappings = _phase_edges(mapping_bounds, chunks, _FIRST_MAPPING_KEY)
+ return tuple(
+ ChunkBounds(person_from=p.lower, person_to=p.upper, mapping_from=m.lower, mapping_to=m.upper)
+ for p, m in zip(persons, mappings)
+ )
+
+
+def create_plan(
+ scan_dir: Path,
+ team_id: int,
+ *,
+ chunks: int = 1,
+ purged_at: datetime | None = None,
+ ch_page_rows: int = 100_000,
+ pg_page_size: int = 5_000,
+) -> ScanPlan:
+ """Write a scan plan. Chunk bounds come from one sample of the team's keys, taken once, so every
+ chunk run of the scan shares the same bounds."""
+ if (scan_dir / PLAN_FILE).exists():
+ raise ScanError(f"{scan_dir / PLAN_FILE} already exists")
+ _private_dir(scan_dir, create=True)
+ if chunks > 1:
+ bounds = _chunks_from_bounds(
+ _sample_bounds(team_id, "persons", chunks), _sample_bounds(team_id, "mappings", chunks), chunks
+ )
+ else:
+ bounds = (ChunkBounds(person_from=None, person_to=None, mapping_from=None, mapping_to=None),)
+ plan = ScanPlan(
+ version=PLAN_VERSION,
+ team_id=team_id,
+ team_exists=Team.objects.filter(id=team_id).exists(),
+ created_at=_now(),
+ purged_at=None if purged_at is None else purged_at.isoformat(),
+ chunks=bounds,
+ ch_page_rows=ch_page_rows,
+ pg_page_size=pg_page_size,
+ )
+ _write_json_atomic(scan_dir / PLAN_FILE, plan.to_json())
+ return plan
+
+
+class _RunWriter:
+ """Appends a page's rows and key hashes, then records the page in the cursor.
+
+ Opening it truncates every file to the offsets the cursor recorded, which drops a page that was
+ written but not recorded before a run stopped.
+ """
+
+ def __init__(self, files: RunFiles, cursor: Cursor) -> None:
+ self.files = files
+ self.cursor = cursor
+ self._handles: dict[str, Any] = {}
+ # A second run would truncate the files under this one. The cursor file is replaced on every page, so the
+ # lock needs its own file; the OS releases it when the process dies.
+ self._lock = files.cursor.with_suffix(".lock").open("w")
+ try:
+ fcntl.flock(self._lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except BlockingIOError:
+ self._lock.close()
+ raise ScanError(f"{files.cursor.name} is in use by another run") from None
+ on_disk = load_cursor(files.cursor)
+ if on_disk is not None and on_disk != cursor:
+ self._lock.close()
+ raise ScanError(f"{files.cursor.name} changed while this run started; run it again")
+ for name, path, offset in (
+ ("rows", files.rows, cursor.rows_bytes),
+ ("person_keys", files.person_keys, cursor.person_keys_bytes),
+ ("mapping_keys", files.mapping_keys, cursor.mapping_keys_bytes),
+ ):
+ if path is None:
+ continue
+ path.touch(exist_ok=True)
+ os.truncate(path, offset)
+ self._handles[name] = path.open("ab")
+
+ def commit(
+ self,
+ rows: Sequence[dict[str, Any]],
+ *,
+ person_keys: Sequence[int] = (),
+ mapping_keys: Sequence[int] = (),
+ counts: Counter[str],
+ after: str | None,
+ phase: str,
+ ) -> None:
+ self._handles["rows"].write(b"".join(json.dumps(r, sort_keys=True).encode() + b"\n" for r in rows))
+ for name, keys in (("person_keys", person_keys), ("mapping_keys", mapping_keys)):
+ if keys:
+ np.asarray(keys, dtype="<u8").tofile(self._handles[name])
+ for handle in self._handles.values():
+ handle.flush()
+ os.fsync(handle.fileno())
+ cursor = self.cursor
+ cursor.rows_bytes = self._handles["rows"].tell()
+ if "person_keys" in self._handles:
+ cursor.person_keys_bytes = self._handles["person_keys"].tell()
+ cursor.mapping_keys_bytes = self._handles["mapping_keys"].tell()
+ for key, n in counts.items():
+ cursor.counts[key] = cursor.counts.get(key, 0) + n
+ cursor.after = after
+ cursor.phase = phase
+ cursor.pages += 1
+ cursor.updated_at = _now()
+ _write_json_atomic(self.files.cursor, dataclasses.asdict(cursor))
+
+ def finish_phase(self, next_phase: str) -> None:
+ self.cursor.phase = next_phase
+ self.cursor.after = None
+ self.cursor.updated_at = _now()
+ _write_json_atomic(self.files.cursor, dataclasses.asdict(self.cursor))
+
+ def close(self) -> None:
+ for handle in self._handles.values():
+ handle.close()
+ self._lock.close()
+
+
+def _state(state: Any) -> dict[str, Any] | None:
+ return None if state is None else dataclasses.asdict(state)
+
+
+def _row(
+ *,
+ kind: KeyKind,
+ team_id: int,
+ key: str,
+ verdict: Verdict,
+ phase: str,
+ observed_at: str,
+ pg: Any,
+ ch: Any,
+ ch_owner_pg_state: str | None = None,
+) -> dict[str, Any]:
+ return {
+ "kind": kind,
+ "team_id": team_id,
+ "key": key,
+ # Safe to share: distinct ids are customer data, their hash is not.
+ "key_hash": hash_distinct_id(team_id, key) if kind == "mapping" else None,
+ "flavor": verdict.flavor,
+ "sub_flavor": verdict.sub_flavor,
+ "verb": verdict.verb,
+ "group": None,
+ "also": [],
+ "phase": phase,
+ "observed_at": observed_at,
+ "evidence": {"pg": _state(pg), "ch": _state(ch), "ch_owner_pg_state": ch_owner_pg_state},
+ }
+
+
+def _count_key(kind: str, verdict: Verdict) -> str:
+ return f"{kind}/{verdict.flavor}/{verdict.sub_flavor or '-'}"
+
+
+def _reported(verdict: Verdict) -> bool:
+ return verdict.flavor not in ("ok", "ok_deleted")
+
+
+def _page_end(plan: ScanPlan, phase: ChPhase, after: str, first: bool, upper: str | None) -> tuple[int, str | None]:
+ sql = _PAGE_END_SQL[(phase, first, upper is not None)]
+ args: dict[str, Any] = {"team_id": plan.team_id, "after": after, "rows": plan.ch_page_rows}
+ if upper is not None:
+ args["upper"] = upper
+ rows = lookups._ch(sql, args, _READ_SETTINGS)
+ count = int(rows[0][0]) if rows else 0
+ return count, (str(rows[0][1]) if count else None)
+
+
+def _classify_persons_page(plan: ScanPlan, after: str, last: str, first: bool) -> PageResult:
+ observed_at = _now()
+ ch = observe.ch_person_states_in_range(plan.team_id, after, last, first=first)
+ uuids = list(ch)
+ pg = observe.pg_person_states(plan.team_id, uuids) if plan.team_exists and uuids else {}
+ purged_at = plan.purged_at_datetime()
+ rows: list[dict[str, Any]] = []
+ counts: Counter[str] = Counter()
+ for person_uuid, ch_state in ch.items():
+ pg_state = pg.get(person_uuid)
+ verdict = classify_person(
+ pg_state,
+ ch_state,
+ team_exists=plan.team_exists,
+ ghost_era=ghost_era(ch_state.newest_written_at, purged_at),
+ )
+ counts[_count_key("person", verdict)] += 1
+ if _reported(verdict):
+ rows.append(
+ _row(
+ kind="person",
+ team_id=plan.team_id,
+ key=person_uuid,
+ verdict=verdict,
+ phase="persons",
+ observed_at=observed_at,
+ pg=pg_state,
+ ch=ch_state,
+ )
+ )
+ return PageResult(rows=rows, keys=[key_hash(u) for u in uuids], counts=counts)
+
+
+def _classify_mappings_page(plan: ScanPlan, after: str, last: str, first: bool) -> PageResult:
+ observed_at = _now()
+ ch = observe.ch_mapping_states_in_range(plan.team_id, after, last, first=first)
+ distinct_ids = list(ch)
+ pg = observe.pg_mapping_rows(plan.team_id, distinct_ids) if plan.team_exists and distinct_ids else {}
+ # Only ghost and diverged mappings need their ClickHouse owner's Postgres state.
+ needs_owner = {
+ distinct_id
+ for distinct_id, state in ch.items()
+ if pg.get(distinct_id) is None or pg[distinct_id].person_uuid != state.winner_person_uuid
+ }
+ owners = sorted({ch[distinct_id].winner_person_uuid for distinct_id in needs_owner})
+ owner_states = observe.pg_person_states(plan.team_id, owners) if plan.team_exists and owners else {}
+ rows: list[dict[str, Any]] = []
+ counts: Counter[str] = Counter()
+ for distinct_id, ch_state in ch.items():
+ pg_row = pg.get(distinct_id)
+ owner_state = (
+ observe.person_pg_state(owner_states, ch_state.winner_person_uuid) if distinct_id in needs_owner else None
+ )
+ verdict = classify_mapping(pg_row, ch_state, ch_owner_pg_state=owner_state, team_exists=plan.team_exists)
+ counts[_count_key("mapping", verdict)] += 1
+ if _reported(verdict):
+ rows.append(
+ _row(
+ kind="mapping",
+ team_id=plan.team_id,
+ key=distinct_id,
+ verdict=verdict,
+ phase="mappings",
+ observed_at=observed_at,
+ pg=pg_row,
+ ch=ch_state,
+ ch_owner_pg_state=owner_state,
+ )
+ )
+ return PageResult(rows=rows, keys=[key_hash(d) for d in distinct_ids], counts=counts)
+
+
+_PAGE_CLASSIFIERS: dict[ChPhase, Callable[..., PageResult]] = {
+ "persons": _classify_persons_page,
+ "mappings": _classify_mappings_page,
+}
+
+
+def run_chunk(
+ scan_dir: Path,
+ chunk: int,
+ *,
+ resume: bool,
+ pause_seconds: float = 0.0,
+ log: Callable[[str], None] = lambda _: None,
+) -> Cursor:
+ """Run both ClickHouse phases of one chunk to the end, or resume them from the cursor."""
+ plan = load_plan(scan_dir)
+ if not 0 <= chunk < len(plan.chunks):
+ raise ScanError(f"chunk {chunk} is not in the plan's {len(plan.chunks)} chunks")
+ files = chunk_files(scan_dir, chunk)
+ cursor = load_cursor(files.cursor)
+ if cursor is not None and not resume:
+ raise ScanError(f"{files.cursor} exists; pass resume to continue it")
+ cursor = cursor or Cursor(phase=CH_PHASES[0])
+ writer = _RunWriter(files, cursor)
+ try:
+ while cursor.phase != DONE:
+ phase: ChPhase = "persons" if cursor.phase == "persons" else "mappings"
+ bounds = plan.chunks[chunk].bounds(phase)
+ lower, upper = bounds.lower, bounds.upper
+ first = cursor.after is None
+ after = cursor.after if cursor.after is not None else (lower if lower is not None else _first_key(phase))
+ count, last = _page_end(plan, phase, after, first, upper)
+ if last is None:
+ writer.finish_phase(_next_phase(phase))
+ log(f"chunk {chunk} {phase}: done")
+ continue
+ page = _PAGE_CLASSIFIERS[phase](plan, after, last, first)
+ rows, keys, counts = page.rows, page.keys, page.counts
+ writer.commit(
+ rows,
+ person_keys=keys if phase == "persons" else (),
+ mapping_keys=keys if phase == "mappings" else (),
+ counts=counts,
+ after=last,
+ phase=phase,
+ )
+ log(f"chunk {chunk} {phase}: page {cursor.pages}, {len(keys)} keys, {len(rows)} reported")
+ if count < plan.ch_page_rows:
+ writer.finish_phase(_next_phase(phase))
+ log(f"chunk {chunk} {phase}: done")
+ elif pause_seconds:
+ time.sleep(pause_seconds)
+ finally:
+ writer.close()
+ return cursor
+
+
+def _first_key(phase: ChPhase) -> str:
+ return _FIRST_PERSON_KEY if phase == "persons" else _FIRST_MAPPING_KEY
+
+
+def _next_phase(phase: ChPhase) -> str:
+ return "mappings" if phase == "persons" else DONE
diff --git a/posthog/models/person/test/divergence_fixtures.py b/posthog/models/person/test/divergence_fixtures.py
new file mode 100644
index 000000000000..d1101336b4f6
--- /dev/null
+++ b/posthog/models/person/test/divergence_fixtures.py
@@ -0,0 +1,61 @@
+from typing import TYPE_CHECKING
+
+from django.utils.timezone import now
+
+from posthog.clickhouse.client import sync_execute
+from posthog.models.person.sql import BULK_INSERT_PERSON_DISTINCT_ID2
+from posthog.models.person.util import create_person as create_person_in_ch
+from posthog.models.signals import mute_selected_signals
+from posthog.test.persons import add_distinct_id, create_person
+
+if TYPE_CHECKING:
+ from posthog.models.team import Team
+
+
+class DivergenceFixturesMixin:
+ """Writes Postgres (personhog fake) and ClickHouse person state independently, so a test can build any
+ disagreement between them. Needs ClickhouseTestMixin and BaseTest."""
+
+ team: "Team"
+
+ def setUp(self) -> None:
+ super().setUp() # type: ignore[misc]
+ # A background merge collapses the duplicate and older rows the tie and blanked shapes are made of.
+ for table in ("person", "person_distinct_id2"):
+ sync_execute(f"SYSTEM STOP MERGES {table}")
+ self.addCleanup(sync_execute, f"SYSTEM START MERGES {table}") # type: ignore[attr-defined]
+
+ def _pg_person(self, *, version: int, distinct_ids: list[str] | None = None):
+ with mute_selected_signals():
+ person = create_person(team=self.team, version=version, properties={"email": "[email protected]"})
+ for distinct_id in distinct_ids or []:
+ add_distinct_id(person=person, distinct_id=distinct_id, version=0)
+ return person
+
+ def _ch_person(self, person_uuid: str, version: int, properties: dict, *, deleted: bool = False) -> None:
+ create_person_in_ch(
+ team_id=self.team.pk,
+ uuid=person_uuid,
+ version=version,
+ is_deleted=deleted,
+ properties=properties,
+ timestamp=now(),
+ )
+
+ def _ch_mapping(self, distinct_id: str, person_uuid: str, version: int, *, team_id: int | None = None) -> None:
+ sync_execute(
+ BULK_INSERT_PERSON_DISTINCT_ID2,
+ [
+ {
+ "distinct_id": distinct_id,
+ "person_id": person_uuid,
+ "team_id": team_id or self.team.pk,
+ "is_deleted": 0,
+ "version": version,
+ "_timestamp": now().replace(tzinfo=None),
+ "_offset": 0,
+ "_partition": 0,
+ }
+ ],
+ flush=False,
+ )
diff --git a/posthog/models/person/test/test_divergence_inspection.py b/posthog/models/person/test/test_divergence_inspection.py
index 901e11589c19..a5027d9633dd 100644
--- a/posthog/models/person/test/test_divergence_inspection.py
+++ b/posthog/models/person/test/test_divergence_inspection.py
@@ -5,64 +5,15 @@ from uuid import uuid4
from posthog.test.base import BaseTest, ClickhouseTestMixin
from django.core.management import call_command
-from django.utils.timezone import now
-from posthog.clickhouse.client import sync_execute
from posthog.models.activity_logging.activity_log import ActivityLog, Detail
from posthog.models.person.divergence.inspection import hash_distinct_id, inspect_mappings, inspect_persons
-from posthog.models.person.sql import BULK_INSERT_PERSON_DISTINCT_ID2
-from posthog.models.person.util import (
- create_person as create_person_in_ch,
- tombstone_persons_in_postgres,
-)
-from posthog.models.signals import mute_selected_signals
+from posthog.models.person.test.divergence_fixtures import DivergenceFixturesMixin
+from posthog.models.person.util import tombstone_persons_in_postgres
from posthog.personhog_client.fake_client import get_active_fake
-from posthog.test.persons import add_distinct_id, create_person
-
-
-class TestDivergenceInspection(ClickhouseTestMixin, BaseTest):
- def setUp(self) -> None:
- super().setUp()
- # A background merge collapses the duplicate and older rows the tie and blanked shapes are made of.
- for table in ("person", "person_distinct_id2"):
- sync_execute(f"SYSTEM STOP MERGES {table}")
- self.addCleanup(sync_execute, f"SYSTEM START MERGES {table}")
-
- def _pg_person(self, *, version: int, distinct_ids: list[str] | None = None):
- with mute_selected_signals():
- person = create_person(team=self.team, version=version, properties={"email": "[email protected]"})
- for distinct_id in distinct_ids or []:
- add_distinct_id(person=person, distinct_id=distinct_id, version=0)
- return person
-
- def _ch_person(self, person_uuid: str, version: int, properties: dict, *, deleted: bool = False) -> None:
- create_person_in_ch(
- team_id=self.team.pk,
- uuid=person_uuid,
- version=version,
- is_deleted=deleted,
- properties=properties,
- timestamp=now(),
- )
- def _ch_mapping(self, distinct_id: str, person_uuid: str, version: int, *, team_id: int | None = None) -> None:
- sync_execute(
- BULK_INSERT_PERSON_DISTINCT_ID2,
- [
- {
- "distinct_id": distinct_id,
- "person_id": person_uuid,
- "team_id": team_id or self.team.pk,
- "is_deleted": 0,
- "version": version,
- "_timestamp": now().replace(tzinfo=None),
- "_offset": 0,
- "_partition": 0,
- }
- ],
- flush=False,
- )
+class TestDivergenceInspection(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
def _log_person_activity(self, activity: str, *, item_id: int, name: str | None) -> None:
ActivityLog.objects.create(
organization_id=self.organization.id,
diff --git a/posthog/models/person/test/test_divergence_team_scan.py b/posthog/models/person/test/test_divergence_team_scan.py
new file mode 100644
index 000000000000..d5607c0e599e
--- /dev/null
+++ b/posthog/models/person/test/test_divergence_team_scan.py
@@ -0,0 +1,160 @@
+import os
+import json
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+import pytest
+from posthog.test.base import BaseTest, ClickhouseTestMixin
+from unittest.mock import patch
+
+from posthog.models.person.divergence import team_scan
+from posthog.models.person.test.divergence_fixtures import DivergenceFixturesMixin
+from posthog.models.person.util import tombstone_persons_in_postgres
+
+
+def _chunk_rows(scan_dir: Path, chunks: int) -> dict[str, tuple[Any, ...]]:
+ rows = []
+ for chunk in range(chunks):
+ path = team_scan.chunk_files(scan_dir, chunk).rows
+ rows += [json.loads(line) for line in path.read_text().splitlines()]
+ found = {r["key"]: (r["flavor"], r["sub_flavor"], r["verb"]) for r in rows}
+ assert len(found) == len(rows), "a key was reported twice"
+ return found
+
+
+class _Stop(Exception):
+ pass
+
+
+class TestChunkScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ self.scan_root = Path(tempfile.mkdtemp())
+ self.addCleanup(shutil.rmtree, self.scan_root)
+
+ def _build_team(self) -> dict[str, str]:
+ """One key per flavor the ClickHouse phases must find."""
+ fine = self._pg_person(version=1, distinct_ids=["fine-1"])
+ self._ch_person(str(fine.uuid), 1, {"email": "[email protected]"})
+ self._ch_mapping("fine-1", str(fine.uuid), 0)
+ stale = self._pg_person(version=1)
+ self._ch_person(str(stale.uuid), 40, {"email": "[email protected]"})
+ deleted = self._pg_person(version=2)
+ tombstone_persons_in_postgres(self.team.pk, [deleted.uuid])
+ self._ch_person(str(deleted.uuid), 7, {"email": "[email protected]"})
+ new_owner = self._pg_person(version=1, distinct_ids=["moved"])
+ self._ch_person(str(new_owner.uuid), 1, {"email": "[email protected]"})
+ ghost = str(uuid4())
+ self._ch_person(ghost, 3, {"email": "[email protected]"})
+ self._ch_mapping("moved", ghost, 5)
+ self._ch_mapping("lost", ghost, 5)
+ return {"stale": str(stale.uuid), "deleted": str(deleted.uuid), "ghost": ghost}
+
+ def test_a_chunk_reports_each_clickhouse_flavor_with_its_verb(self):
+ keys = self._build_team()
+ scan_dir = self.scan_root / "scan"
+ team_scan.create_plan(scan_dir, self.team.pk)
+
+ cursor = team_scan.run_chunk(scan_dir, 0, resume=False)
+
+ assert cursor.phase == team_scan.DONE
+ assert _chunk_rows(scan_dir, 1) == {
+ keys["stale"]: ("stale", None, "merge_stale"),
+ keys["deleted"]: ("deleted_visible", None, "bury"),
+ keys["ghost"]: ("ghost", "historical", None),
+ "moved": ("diverged", "a_absent", "fold"),
+ "lost": ("ghost", "owner_ghost", "restore"),
+ }
+ with pytest.raises(team_scan.ScanError, match="pass resume"):
+ team_scan.run_chunk(scan_dir, 0, resume=False)
+
+ def test_a_deleted_team_reports_its_clickhouse_leftovers_without_repairs(self):
+ keys = self._build_team()
+ scan_dir = self.scan_root / "scan"
+ with patch.object(team_scan.Team.objects, "filter") as team_filter:
+ team_filter.return_value.exists.return_value = False
+ team_scan.create_plan(scan_dir, self.team.pk)
+
+ team_scan.run_chunk(scan_dir, 0, resume=False)
+
+ rows = _chunk_rows(scan_dir, 1)
+ # Five persons and three mappings, all ClickHouse-only once Postgres is not read.
+ assert len(rows) == 8
+ assert {keys["stale"], keys["deleted"], keys["ghost"], "moved", "lost", "fine-1"} <= rows.keys()
+ assert set(rows.values()) == {("ghost", "team_gone", None)}
+
+ def test_a_second_run_of_a_running_chunk_is_refused(self):
+ self._build_team()
+ scan_dir = self.scan_root / "scan"
+ team_scan.create_plan(scan_dir, self.team.pk, ch_page_rows=2)
+ refusals: list[str] = []
+
+ def start_a_second_run(message: str) -> None:
+ if "page" in message and not refusals:
+ with pytest.raises(team_scan.ScanError, match="in use by another run") as refused:
+ team_scan.run_chunk(scan_dir, 0, resume=True)
+ refusals.append(str(refused.value))
+
+ cursor = team_scan.run_chunk(scan_dir, 0, resume=False, log=start_a_second_run)
+
+ assert len(refusals) == 1 and cursor.phase == team_scan.DONE
+ assert team_scan.run_chunk(scan_dir, 0, resume=True).phase == team_scan.DONE
+
+ def test_chunked_paged_and_interrupted_scan_gives_the_same_rows_as_one_run(self):
+ self._build_team()
+ whole = self.scan_root / "whole"
+ team_scan.create_plan(whole, self.team.pk)
+ team_scan.run_chunk(whole, 0, resume=False)
+
+ split = self.scan_root / "split"
+ plan = team_scan.create_plan(split, self.team.pk, chunks=3, ch_page_rows=2)
+ assert len(plan.chunks) == 3
+
+ def stop_after_first_page(message: str) -> None:
+ if "page" in message:
+ raise _Stop
+
+ with pytest.raises(_Stop):
+ team_scan.run_chunk(split, 0, resume=False, log=stop_after_first_page)
+ # Bytes written after the last recorded page, as by a run that stopped mid-write.
+ files = team_scan.chunk_files(split, 0)
+ for path in (files.rows, files.person_keys, files.mapping_keys):
+ assert path is not None
+ with path.open("ab") as handle:
+ handle.write(b'{"kind": "person", "key": "partial"')
+ for chunk in range(3):
+ team_scan.run_chunk(split, chunk, resume=True)
+
+ assert _chunk_rows(split, 3) == _chunk_rows(whole, 1)
+ assert len(_chunk_rows(whole, 1)) == 5
+
+ def test_the_scan_directory_is_private_to_its_owner(self):
+ scan_dir = self.scan_root / "private"
+ team_scan.create_plan(scan_dir, self.team.pk)
+ assert scan_dir.stat().st_mode & 0o077 == 0
+
+ shared = self.scan_root / "shared"
+ shared.mkdir(mode=0o755)
+ shared.chmod(0o755)
+ with pytest.raises(team_scan.ScanError, match="readable by other users"):
+ team_scan.create_plan(shared, self.team.pk)
+
+ target = self.scan_root / "target"
+ target.mkdir(mode=0o700)
+ linked = self.scan_root / "linked"
+ linked.symlink_to(target)
+ with pytest.raises(team_scan.ScanError, match="not a directory"):
+ team_scan.create_plan(linked, self.team.pk)
+
+ with patch("posthog.models.person.divergence.team_scan.os.geteuid", return_value=os.geteuid() + 1):
+ with pytest.raises(team_scan.ScanError, match="owned by another user"):
+ team_scan.run_chunk(scan_dir, 0, resume=False)
+
+ # A directory opened up after its plan was written is refused when the scan resumes.
+ scan_dir.chmod(0o755)
+ with pytest.raises(team_scan.ScanError, match="readable by other users"):
+ team_scan.run_chunk(scan_dir, 0, resume=False)
+ scan_dir.chmod(0o700)