Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(persons): add the person divergence classifier (#117415)

Eli Reisman80ed0b0ab0e6
diff --git a/posthog/models/person/divergence/flavors.py b/posthog/models/person/divergence/flavors.py
new file mode 100644
index 000000000000..dbea6aab5d75
--- /dev/null
+++ b/posthog/models/person/divergence/flavors.py
@@ -0,0 +1,215 @@
+"""Classify one person or mapping by comparing its Postgres and ClickHouse state.
+
+Pure functions over small state records, so every combination of states is testable without a database.
+Each flavor names what is wrong; `PERSON_VERBS` and `MAPPING_VERBS` name the repair that fixes it, or None
+when the state is accepted as it is.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from posthog.dataclasses import frozen
+
+PersonFlavor = Literal[
+ "ok",
+ "ok_deleted",
+ "ghost",
+ "deleted_visible",
+ "hidden",
+ "swept",
+ "stale",
+ "behind",
+ "missing_in_ch",
+ "tie",
+ "blanked",
+ "no_live_mappings",
+]
+
+MappingFlavor = Literal[
+ "ok",
+ "ok_deleted",
+ "ghost",
+ "deleted_visible",
+ "diverged",
+ "stale",
+ "hidden",
+ "missing_in_ch",
+ "stray",
+]
+
+RepairVerb = Literal["republish", "merge_stale", "bury", "restore", "fold", "merge"]
+
+# Keyed by (flavor, sub_flavor). A ghost's or diverged mapping's sub-flavor decides between restore,
+# fold and merge. None means the state is accepted as it is.
+PERSON_VERBS: dict[tuple[str, str | None], RepairVerb | None] = {
+ # Ghosts of deleted teams are historical stock that no query reads; cleaning them up is out of scope.
+ ("ghost", "team_gone"): None,
+ ("ghost", "after_purge"): "restore",
+ ("ghost", "historical"): None,
+ ("deleted_visible", None): "bury",
+ ("hidden", None): "republish",
+ ("swept", None): "republish",
+ ("stale", None): "merge_stale",
+ ("behind", None): "republish",
+ ("missing_in_ch", None): "republish",
+ ("tie", None): "republish",
+ ("blanked", None): "republish",
+ ("no_live_mappings", None): None,
+}
+
+MAPPING_VERBS: dict[tuple[str, str | None], RepairVerb | None] = {
+ ("ghost", "owner_ghost"): "restore",
+ ("ghost", "owner_live"): "restore",
+ ("ghost", "owner_deleted"): None,
+ # Like their owners, ghosts of a deleted team are historical stock left as they are.
+ ("ghost", "team_gone"): None,
+ ("deleted_visible", None): "bury",
+ ("diverged", "a_absent"): "fold",
+ ("diverged", "a_live"): "merge",
+ ("diverged", "a_deleted"): "republish",
+ # The owning person holds mappings of more than one old person, so no single repair is safe.
+ ("diverged", "ambiguous"): None,
+ ("diverged", "a_unknown"): None,
+ ("stale", None): "republish",
+ ("hidden", None): "republish",
+ ("missing_in_ch", None): "republish",
+ # Ingestion tombstones and re-attaches a stray on the distinct id's next event, so it is reported only.
+ ("stray", None): None,
+}
+
+
+@frozen
+class PgPerson:
+ # None when the source of the state did not report it (tombstone lookups do not).
+ id: int | None
+ version: int
+ is_deleted: bool
+ # None when the source of the state did not report it (lookups by uuid do not).
+ has_live_mappings: bool | None = None
+
+
+@frozen
+class ChPerson:
+ # Over every row, including lightweight-deleted ones, so a raise lands above all of them.
+ max_version: int
+ # Over visible rows only.
+ visible_rows: int
+ live_max_version: int | None
+ tombstone_max_version: int | None
+ # Visible rows at the visible maximum version, and how many distinct payloads they carry.
+ rows_at_max: int
+ payloads_at_max: int
+ # The visible winner is empty and unidentified while an earlier visible row had properties.
+ winner_blanked: bool
+ # Unix seconds of the newest visible row; dates a ghost against a known purge.
+ newest_written_at: float | None = None
+
+
+@frozen
+class PgMapping:
+ person_id: int
+ person_uuid: str | None
+ version: int
+ is_deleted: bool
+ person_is_deleted: bool | None
+
+
+@frozen
+class ChMapping:
+ max_version: int
+ visible_rows: int
+ winner_person_uuid: str
+ winner_version: int
+ winner_deleted: bool
+
+
+@frozen
+class Verdict:
+ flavor: str
+ sub_flavor: str | None
+ verb: RepairVerb | None
+
+
+def _verdict(kind: str, flavor: str, sub_flavor: str | None = None) -> Verdict:
+ verbs = PERSON_VERBS if kind == "person" else MAPPING_VERBS
+ return Verdict(flavor=flavor, sub_flavor=sub_flavor, verb=verbs.get((flavor, sub_flavor)))
+
+
+def classify_person(
+ pg: PgPerson | None, ch: ChPerson | None, *, team_exists: bool = True, ghost_era: str = "historical"
+) -> Verdict:
+ """The flavor of one person. `ghost_era` is the caller's dating of a ghost: `after_purge` when its
+ newest ClickHouse row predates a known Postgres purge of the team, else `historical`."""
+ if pg is None:
+ if ch is None or ch.visible_rows == 0 or ch.live_max_version is None:
+ return _verdict("person", "ok_deleted")
+ if ch.tombstone_max_version is not None and ch.tombstone_max_version >= ch.live_max_version:
+ return _verdict("person", "ok_deleted")
+ return _verdict("person", "ghost", "team_gone" if not team_exists else ghost_era)
+
+ if pg.is_deleted:
+ # A visible live row at or above every visible tombstone keeps the person visible; ties count,
+ # because ClickHouse picks either row at an equal version.
+ if (
+ ch is not None
+ and ch.live_max_version is not None
+ and (ch.tombstone_max_version is None or ch.live_max_version >= ch.tombstone_max_version)
+ ):
+ return _verdict("person", "deleted_visible")
+ return _verdict("person", "ok_deleted")
+
+ if ch is None:
+ return _verdict("person", "missing_in_ch")
+ if ch.visible_rows == 0:
+ return _verdict("person", "swept")
+ visible_max = max(v for v in (ch.live_max_version, ch.tombstone_max_version) if v is not None)
+ if ch.tombstone_max_version is not None and ch.tombstone_max_version == visible_max:
+ return _verdict("person", "hidden")
+ if visible_max > pg.version:
+ return _verdict("person", "stale")
+ if visible_max < pg.version:
+ return _verdict("person", "behind")
+ if ch.rows_at_max > 1 and ch.payloads_at_max > 1:
+ return _verdict("person", "tie")
+ if ch.winner_blanked:
+ return _verdict("person", "blanked")
+ if pg.has_live_mappings is False:
+ return _verdict("person", "no_live_mappings")
+ return _verdict("person", "ok")
+
+
+def classify_mapping(
+ pg: PgMapping | None, ch: ChMapping | None, *, ch_owner_pg_state: str | None = None, team_exists: bool = True
+) -> Verdict:
+ """The flavor of one mapping. `ch_owner_pg_state` is the Postgres state of ClickHouse's winning owner
+ (`live`, `deleted` or `absent`), needed only for ghost and diverged mappings."""
+ ch_live = ch is not None and ch.visible_rows > 0 and not ch.winner_deleted
+
+ if pg is None:
+ if not ch_live:
+ return _verdict("mapping", "ok_deleted")
+ if not team_exists:
+ return _verdict("mapping", "ghost", "team_gone")
+ sub = {"absent": "owner_ghost", "live": "owner_live", "deleted": "owner_deleted"}.get(
+ ch_owner_pg_state or "absent", "owner_ghost"
+ )
+ return _verdict("mapping", "ghost", sub)
+
+ if pg.is_deleted:
+ # A live winner at any version still resolves the id, so the tombstone has not reached ClickHouse.
+ return _verdict("mapping", "deleted_visible" if ch_live else "ok_deleted")
+
+ # None is an orphan row whose person is gone; ingestion re-attaches it like any other stray.
+ if pg.person_is_deleted is not False:
+ return _verdict("mapping", "stray")
+ if ch is None or ch.visible_rows == 0:
+ return _verdict("mapping", "missing_in_ch")
+ if ch.winner_deleted:
+ return _verdict("mapping", "hidden")
+ if ch.winner_person_uuid != pg.person_uuid:
+ sub = {"absent": "a_absent", "live": "a_live", "deleted": "a_deleted"}.get(ch_owner_pg_state or "", "a_unknown")
+ return _verdict("mapping", "diverged", sub)
+ if ch.winner_version > pg.version:
+ return _verdict("mapping", "stale")
+ return _verdict("mapping", "ok")
diff --git a/posthog/models/person/test/test_divergence_flavors.py b/posthog/models/person/test/test_divergence_flavors.py
new file mode 100644
index 000000000000..d2a84602f1d4
--- /dev/null
+++ b/posthog/models/person/test/test_divergence_flavors.py
@@ -0,0 +1,217 @@
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from posthog.models.person.divergence.flavors import (
+ ChMapping,
+ ChPerson,
+ PgMapping,
+ PgPerson,
+ RepairVerb,
+ Verdict,
+ classify_mapping,
+ classify_person,
+)
+
+
+def _v(flavor: str, sub_flavor: str | None, verb: RepairVerb | None) -> Verdict:
+ return Verdict(flavor=flavor, sub_flavor=sub_flavor, verb=verb)
+
+
+def _ch(
+ *,
+ live: int | None = None,
+ tomb: int | None = None,
+ visible_rows: int = 1,
+ rows_at_max: int = 1,
+ payloads_at_max: int = 1,
+ blanked: bool = False,
+ masked_max: int | None = None,
+) -> ChPerson:
+ visible = [v for v in (live, tomb) if v is not None]
+ return ChPerson(
+ max_version=masked_max if masked_max is not None else max(visible, default=0),
+ visible_rows=visible_rows,
+ live_max_version=live,
+ tombstone_max_version=tomb,
+ rows_at_max=rows_at_max,
+ payloads_at_max=payloads_at_max,
+ winner_blanked=blanked,
+ )
+
+
+def _pg(version: int, *, deleted: bool = False, has_live_mappings: bool | None = None) -> PgPerson:
+ return PgPerson(id=1, version=version, is_deleted=deleted, has_live_mappings=has_live_mappings)
+
+
+class TestClassifyPerson(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("absent_everywhere", None, None, True, _v("ok_deleted", None, None)),
+ ("ch_only_tombstoned", None, _ch(tomb=3), True, _v("ok_deleted", None, None)),
+ ("ch_only_swept", None, _ch(live=3, visible_rows=0), True, _v("ok_deleted", None, None)),
+ ("ghost_historical", None, _ch(live=3), True, _v("ghost", "historical", None)),
+ ("ghost_of_deleted_team", None, _ch(live=3), False, _v("ghost", "team_gone", None)),
+ (
+ "deleted_and_ch_newer",
+ _pg(2, deleted=True),
+ _ch(live=5, tomb=2),
+ True,
+ _v("deleted_visible", None, "bury"),
+ ),
+ (
+ "deleted_and_ch_tie",
+ _pg(4, deleted=True),
+ _ch(live=4, tomb=4),
+ True,
+ _v("deleted_visible", None, "bury"),
+ ),
+ (
+ "deleted_never_published",
+ _pg(4, deleted=True),
+ _ch(live=3),
+ True,
+ _v("deleted_visible", None, "bury"),
+ ),
+ (
+ "deleted_and_published",
+ _pg(4, deleted=True),
+ _ch(live=3, tomb=4),
+ True,
+ _v("ok_deleted", None, None),
+ ),
+ ("deleted_and_swept", _pg(4, deleted=True), None, True, _v("ok_deleted", None, None)),
+ ("live_missing_in_ch", _pg(2), None, True, _v("missing_in_ch", None, "republish")),
+ (
+ "live_and_swept",
+ _pg(2),
+ _ch(live=2, visible_rows=0, masked_max=105),
+ True,
+ _v("swept", None, "republish"),
+ ),
+ ("live_hidden_by_tombstone", _pg(2), _ch(live=2, tomb=102), True, _v("hidden", None, "republish")),
+ ("live_and_stale", _pg(2), _ch(live=40), True, _v("stale", None, "merge_stale")),
+ ("live_and_behind", _pg(9), _ch(live=4), True, _v("behind", None, "republish")),
+ (
+ "live_tie",
+ _pg(4),
+ _ch(live=4, rows_at_max=2, payloads_at_max=2),
+ True,
+ _v("tie", None, "republish"),
+ ),
+ (
+ "live_duplicate_row_is_not_a_tie",
+ _pg(4),
+ _ch(live=4, rows_at_max=2, payloads_at_max=1),
+ True,
+ _v("ok", None, None),
+ ),
+ ("live_blanked", _pg(105), _ch(live=105, blanked=True), True, _v("blanked", None, "republish")),
+ (
+ "live_without_mappings",
+ _pg(2, has_live_mappings=False),
+ _ch(live=2),
+ True,
+ _v("no_live_mappings", None, None),
+ ),
+ ("live_and_equal", _pg(2, has_live_mappings=True), _ch(live=2), True, _v("ok", None, None)),
+ ]
+ )
+ def test_classifies(self, _name, pg, ch, team_exists, expected):
+ assert classify_person(pg, ch, team_exists=team_exists) == expected
+
+ def test_ghost_after_a_purge_is_restored(self):
+ assert classify_person(None, _ch(live=3), ghost_era="after_purge") == _v("ghost", "after_purge", "restore")
+
+
+def _pg_map(
+ *, version: int = 0, deleted: bool = False, owner: str | None = "b", owner_deleted: bool | None = False
+) -> PgMapping:
+ return PgMapping(
+ person_id=1, person_uuid=owner, version=version, is_deleted=deleted, person_is_deleted=owner_deleted
+ )
+
+
+def _ch_map(*, owner: str = "b", version: int = 0, deleted: bool = False, visible_rows: int = 1) -> ChMapping:
+ return ChMapping(
+ max_version=version,
+ visible_rows=visible_rows,
+ winner_person_uuid=owner,
+ winner_version=version,
+ winner_deleted=deleted,
+ )
+
+
+class TestClassifyMapping(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("absent_everywhere", None, None, None, _v("ok_deleted", None, None)),
+ ("ch_only_tombstoned", None, _ch_map(deleted=True), None, _v("ok_deleted", None, None)),
+ ("ghost_of_a_ghost_person", None, _ch_map(owner="a"), "absent", _v("ghost", "owner_ghost", "restore")),
+ ("ghost_of_a_live_person", None, _ch_map(owner="a"), "live", _v("ghost", "owner_live", "restore")),
+ ("ghost_of_a_deleted_person", None, _ch_map(owner="a"), "deleted", _v("ghost", "owner_deleted", None)),
+ (
+ "deleted_and_ch_live_above",
+ _pg_map(version=2, deleted=True),
+ _ch_map(version=3),
+ None,
+ _v("deleted_visible", None, "bury"),
+ ),
+ (
+ "deleted_and_never_published",
+ _pg_map(version=2, deleted=True),
+ _ch_map(version=1),
+ None,
+ _v("deleted_visible", None, "bury"),
+ ),
+ (
+ "deleted_and_published",
+ _pg_map(version=2, deleted=True),
+ _ch_map(version=2, deleted=True),
+ None,
+ _v("ok_deleted", None, None),
+ ),
+ (
+ "stray_on_deleted_person",
+ _pg_map(owner_deleted=True),
+ _ch_map(),
+ None,
+ _v("stray", None, None),
+ ),
+ (
+ "orphan_with_ch_on_another_person",
+ _pg_map(owner=None, owner_deleted=None),
+ _ch_map(owner="a"),
+ "absent",
+ _v("stray", None, None),
+ ),
+ ("live_missing_in_ch", _pg_map(), None, None, _v("missing_in_ch", None, "republish")),
+ ("live_and_swept", _pg_map(), _ch_map(visible_rows=0), None, _v("missing_in_ch", None, "republish")),
+ ("live_hidden", _pg_map(), _ch_map(deleted=True), None, _v("hidden", None, "republish")),
+ (
+ "diverged_old_owner_absent",
+ _pg_map(),
+ _ch_map(owner="a"),
+ "absent",
+ _v("diverged", "a_absent", "fold"),
+ ),
+ ("diverged_old_owner_live", _pg_map(), _ch_map(owner="a"), "live", _v("diverged", "a_live", "merge")),
+ (
+ "diverged_old_owner_deleted",
+ _pg_map(),
+ _ch_map(owner="a"),
+ "deleted",
+ _v("diverged", "a_deleted", "republish"),
+ ),
+ ("diverged_old_owner_unknown", _pg_map(), _ch_map(owner="a"), None, _v("diverged", "a_unknown", None)),
+ ("stale_same_owner", _pg_map(version=1), _ch_map(version=5), None, _v("stale", None, "republish")),
+ ("ok", _pg_map(version=5), _ch_map(version=5), None, _v("ok", None, None)),
+ ]
+ )
+ def test_classifies(self, _name, pg, ch, owner_state, expected):
+ assert classify_mapping(pg, ch, ch_owner_pg_state=owner_state) == expected
+
+ def test_ghost_of_a_deleted_team_is_left_alone(self):
+ assert classify_mapping(None, _ch_map(owner="a"), ch_owner_pg_state="absent", team_exists=False) == _v(
+ "ghost", "team_gone", None
+ )
diff --git a/posthog/personhog_client/proto/__init__.py b/posthog/personhog_client/proto/__init__.py
index 5a34e46d4c46..d90b4cbb70d3 100644
--- a/posthog/personhog_client/proto/__init__.py
+++ b/posthog/personhog_client/proto/__init__.py
@@ -109,9 +109,11 @@ from personhog.types.v1.person_pb2 import (
PersonVersionFloor,
EnsurePersonVersionFloorsRequest,
EnsurePersonVersionFloorsResponse,
+ DistinctIdRow,
GetDistinctIdRowsRequest,
GetDistinctIdRowsResponse,
ListPersonsForTeamRequest,
+ PersonListEntry,
ListPersonsForTeamResponse,
RaisePersonVersionFloorsRequest,
RaisePersonVersionFloorsResponse,