:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(persons): add the person divergence classifier (#117415)
Eli Reisman80ed0b0ab0e6
diff --git a/posthog/models/person/divergence/flavors.py b/posthog/models/person/divergence/flavors.pynew file mode 100644index 000000000000..dbea6aab5d75--- /dev/null+++ b/posthog/models/person/divergence/flavors.py@@ -0,0 +1,215 @@+"""Classify one person or mapping by comparing its Postgres and ClickHouse state.++Pure functions over small state records, so every combination of states is testable without a database.+Each flavor names what is wrong; `PERSON_VERBS` and `MAPPING_VERBS` name the repair that fixes it, or None+when the state is accepted as it is.+"""++from __future__ import annotations++from typing import Literal++from posthog.dataclasses import frozen++PersonFlavor = Literal[+ "ok",+ "ok_deleted",+ "ghost",+ "deleted_visible",+ "hidden",+ "swept",+ "stale",+ "behind",+ "missing_in_ch",+ "tie",+ "blanked",+ "no_live_mappings",+]++MappingFlavor = Literal[+ "ok",+ "ok_deleted",+ "ghost",+ "deleted_visible",+ "diverged",+ "stale",+ "hidden",+ "missing_in_ch",+ "stray",+]++RepairVerb = Literal["republish", "merge_stale", "bury", "restore", "fold", "merge"]++# Keyed by (flavor, sub_flavor). A ghost's or diverged mapping's sub-flavor decides between restore,+# fold and merge. None means the state is accepted as it is.+PERSON_VERBS: dict[tuple[str, str | None], RepairVerb | None] = {+ # Ghosts of deleted teams are historical stock that no query reads; cleaning them up is out of scope.+ ("ghost", "team_gone"): None,+ ("ghost", "after_purge"): "restore",+ ("ghost", "historical"): None,+ ("deleted_visible", None): "bury",+ ("hidden", None): "republish",+ ("swept", None): "republish",+ ("stale", None): "merge_stale",+ ("behind", None): "republish",+ ("missing_in_ch", None): "republish",+ ("tie", None): "republish",+ ("blanked", None): "republish",+ ("no_live_mappings", None): None,+}++MAPPING_VERBS: dict[tuple[str, str | None], RepairVerb | None] = {+ ("ghost", "owner_ghost"): "restore",+ ("ghost", "owner_live"): "restore",+ ("ghost", "owner_deleted"): None,+ # Like their owners, ghosts of a deleted team are historical stock left as they are.+ ("ghost", "team_gone"): None,+ ("deleted_visible", None): "bury",+ ("diverged", "a_absent"): "fold",+ ("diverged", "a_live"): "merge",+ ("diverged", "a_deleted"): "republish",+ # The owning person holds mappings of more than one old person, so no single repair is safe.+ ("diverged", "ambiguous"): None,+ ("diverged", "a_unknown"): None,+ ("stale", None): "republish",+ ("hidden", None): "republish",+ ("missing_in_ch", None): "republish",+ # Ingestion tombstones and re-attaches a stray on the distinct id's next event, so it is reported only.+ ("stray", None): None,+}+++@frozen+class PgPerson:+ # None when the source of the state did not report it (tombstone lookups do not).+ id: int | None+ version: int+ is_deleted: bool+ # None when the source of the state did not report it (lookups by uuid do not).+ has_live_mappings: bool | None = None+++@frozen+class ChPerson:+ # Over every row, including lightweight-deleted ones, so a raise lands above all of them.+ max_version: int+ # Over visible rows only.+ visible_rows: int+ live_max_version: int | None+ tombstone_max_version: int | None+ # Visible rows at the visible maximum version, and how many distinct payloads they carry.+ rows_at_max: int+ payloads_at_max: int+ # The visible winner is empty and unidentified while an earlier visible row had properties.+ winner_blanked: bool+ # Unix seconds of the newest visible row; dates a ghost against a known purge.+ newest_written_at: float | None = None+++@frozen+class PgMapping:+ person_id: int+ person_uuid: str | None+ version: int+ is_deleted: bool+ person_is_deleted: bool | None+++@frozen+class ChMapping:+ max_version: int+ visible_rows: int+ winner_person_uuid: str+ winner_version: int+ winner_deleted: bool+++@frozen+class Verdict:+ flavor: str+ sub_flavor: str | None+ verb: RepairVerb | None+++def _verdict(kind: str, flavor: str, sub_flavor: str | None = None) -> Verdict:+ verbs = PERSON_VERBS if kind == "person" else MAPPING_VERBS+ return Verdict(flavor=flavor, sub_flavor=sub_flavor, verb=verbs.get((flavor, sub_flavor)))+++def classify_person(+ pg: PgPerson | None, ch: ChPerson | None, *, team_exists: bool = True, ghost_era: str = "historical"+) -> Verdict:+ """The flavor of one person. `ghost_era` is the caller's dating of a ghost: `after_purge` when its+ newest ClickHouse row predates a known Postgres purge of the team, else `historical`."""+ if pg is None:+ if ch is None or ch.visible_rows == 0 or ch.live_max_version is None:+ return _verdict("person", "ok_deleted")+ if ch.tombstone_max_version is not None and ch.tombstone_max_version >= ch.live_max_version:+ return _verdict("person", "ok_deleted")+ return _verdict("person", "ghost", "team_gone" if not team_exists else ghost_era)++ if pg.is_deleted:+ # A visible live row at or above every visible tombstone keeps the person visible; ties count,+ # because ClickHouse picks either row at an equal version.+ if (+ ch is not None+ and ch.live_max_version is not None+ and (ch.tombstone_max_version is None or ch.live_max_version >= ch.tombstone_max_version)+ ):+ return _verdict("person", "deleted_visible")+ return _verdict("person", "ok_deleted")++ if ch is None:+ return _verdict("person", "missing_in_ch")+ if ch.visible_rows == 0:+ return _verdict("person", "swept")+ visible_max = max(v for v in (ch.live_max_version, ch.tombstone_max_version) if v is not None)+ if ch.tombstone_max_version is not None and ch.tombstone_max_version == visible_max:+ return _verdict("person", "hidden")+ if visible_max > pg.version:+ return _verdict("person", "stale")+ if visible_max < pg.version:+ return _verdict("person", "behind")+ if ch.rows_at_max > 1 and ch.payloads_at_max > 1:+ return _verdict("person", "tie")+ if ch.winner_blanked:+ return _verdict("person", "blanked")+ if pg.has_live_mappings is False:+ return _verdict("person", "no_live_mappings")+ return _verdict("person", "ok")+++def classify_mapping(+ pg: PgMapping | None, ch: ChMapping | None, *, ch_owner_pg_state: str | None = None, team_exists: bool = True+) -> Verdict:+ """The flavor of one mapping. `ch_owner_pg_state` is the Postgres state of ClickHouse's winning owner+ (`live`, `deleted` or `absent`), needed only for ghost and diverged mappings."""+ ch_live = ch is not None and ch.visible_rows > 0 and not ch.winner_deleted++ if pg is None:+ if not ch_live:+ return _verdict("mapping", "ok_deleted")+ if not team_exists:+ return _verdict("mapping", "ghost", "team_gone")+ sub = {"absent": "owner_ghost", "live": "owner_live", "deleted": "owner_deleted"}.get(+ ch_owner_pg_state or "absent", "owner_ghost"+ )+ return _verdict("mapping", "ghost", sub)++ if pg.is_deleted:+ # A live winner at any version still resolves the id, so the tombstone has not reached ClickHouse.+ return _verdict("mapping", "deleted_visible" if ch_live else "ok_deleted")++ # None is an orphan row whose person is gone; ingestion re-attaches it like any other stray.+ if pg.person_is_deleted is not False:+ return _verdict("mapping", "stray")+ if ch is None or ch.visible_rows == 0:+ return _verdict("mapping", "missing_in_ch")+ if ch.winner_deleted:+ return _verdict("mapping", "hidden")+ if ch.winner_person_uuid != pg.person_uuid:+ sub = {"absent": "a_absent", "live": "a_live", "deleted": "a_deleted"}.get(ch_owner_pg_state or "", "a_unknown")+ return _verdict("mapping", "diverged", sub)+ if ch.winner_version > pg.version:+ return _verdict("mapping", "stale")+ return _verdict("mapping", "ok")diff --git a/posthog/models/person/test/test_divergence_flavors.py b/posthog/models/person/test/test_divergence_flavors.pynew file mode 100644index 000000000000..d2a84602f1d4--- /dev/null+++ b/posthog/models/person/test/test_divergence_flavors.py@@ -0,0 +1,217 @@+from django.test import SimpleTestCase++from parameterized import parameterized++from posthog.models.person.divergence.flavors import (+ ChMapping,+ ChPerson,+ PgMapping,+ PgPerson,+ RepairVerb,+ Verdict,+ classify_mapping,+ classify_person,+)+++def _v(flavor: str, sub_flavor: str | None, verb: RepairVerb | None) -> Verdict:+ return Verdict(flavor=flavor, sub_flavor=sub_flavor, verb=verb)+++def _ch(+ *,+ live: int | None = None,+ tomb: int | None = None,+ visible_rows: int = 1,+ rows_at_max: int = 1,+ payloads_at_max: int = 1,+ blanked: bool = False,+ masked_max: int | None = None,+) -> ChPerson:+ visible = [v for v in (live, tomb) if v is not None]+ return ChPerson(+ max_version=masked_max if masked_max is not None else max(visible, default=0),+ visible_rows=visible_rows,+ live_max_version=live,+ tombstone_max_version=tomb,+ rows_at_max=rows_at_max,+ payloads_at_max=payloads_at_max,+ winner_blanked=blanked,+ )+++def _pg(version: int, *, deleted: bool = False, has_live_mappings: bool | None = None) -> PgPerson:+ return PgPerson(id=1, version=version, is_deleted=deleted, has_live_mappings=has_live_mappings)+++class TestClassifyPerson(SimpleTestCase):+ @parameterized.expand(+ [+ ("absent_everywhere", None, None, True, _v("ok_deleted", None, None)),+ ("ch_only_tombstoned", None, _ch(tomb=3), True, _v("ok_deleted", None, None)),+ ("ch_only_swept", None, _ch(live=3, visible_rows=0), True, _v("ok_deleted", None, None)),+ ("ghost_historical", None, _ch(live=3), True, _v("ghost", "historical", None)),+ ("ghost_of_deleted_team", None, _ch(live=3), False, _v("ghost", "team_gone", None)),+ (+ "deleted_and_ch_newer",+ _pg(2, deleted=True),+ _ch(live=5, tomb=2),+ True,+ _v("deleted_visible", None, "bury"),+ ),+ (+ "deleted_and_ch_tie",+ _pg(4, deleted=True),+ _ch(live=4, tomb=4),+ True,+ _v("deleted_visible", None, "bury"),+ ),+ (+ "deleted_never_published",+ _pg(4, deleted=True),+ _ch(live=3),+ True,+ _v("deleted_visible", None, "bury"),+ ),+ (+ "deleted_and_published",+ _pg(4, deleted=True),+ _ch(live=3, tomb=4),+ True,+ _v("ok_deleted", None, None),+ ),+ ("deleted_and_swept", _pg(4, deleted=True), None, True, _v("ok_deleted", None, None)),+ ("live_missing_in_ch", _pg(2), None, True, _v("missing_in_ch", None, "republish")),+ (+ "live_and_swept",+ _pg(2),+ _ch(live=2, visible_rows=0, masked_max=105),+ True,+ _v("swept", None, "republish"),+ ),+ ("live_hidden_by_tombstone", _pg(2), _ch(live=2, tomb=102), True, _v("hidden", None, "republish")),+ ("live_and_stale", _pg(2), _ch(live=40), True, _v("stale", None, "merge_stale")),+ ("live_and_behind", _pg(9), _ch(live=4), True, _v("behind", None, "republish")),+ (+ "live_tie",+ _pg(4),+ _ch(live=4, rows_at_max=2, payloads_at_max=2),+ True,+ _v("tie", None, "republish"),+ ),+ (+ "live_duplicate_row_is_not_a_tie",+ _pg(4),+ _ch(live=4, rows_at_max=2, payloads_at_max=1),+ True,+ _v("ok", None, None),+ ),+ ("live_blanked", _pg(105), _ch(live=105, blanked=True), True, _v("blanked", None, "republish")),+ (+ "live_without_mappings",+ _pg(2, has_live_mappings=False),+ _ch(live=2),+ True,+ _v("no_live_mappings", None, None),+ ),+ ("live_and_equal", _pg(2, has_live_mappings=True), _ch(live=2), True, _v("ok", None, None)),+ ]+ )+ def test_classifies(self, _name, pg, ch, team_exists, expected):+ assert classify_person(pg, ch, team_exists=team_exists) == expected++ def test_ghost_after_a_purge_is_restored(self):+ assert classify_person(None, _ch(live=3), ghost_era="after_purge") == _v("ghost", "after_purge", "restore")+++def _pg_map(+ *, version: int = 0, deleted: bool = False, owner: str | None = "b", owner_deleted: bool | None = False+) -> PgMapping:+ return PgMapping(+ person_id=1, person_uuid=owner, version=version, is_deleted=deleted, person_is_deleted=owner_deleted+ )+++def _ch_map(*, owner: str = "b", version: int = 0, deleted: bool = False, visible_rows: int = 1) -> ChMapping:+ return ChMapping(+ max_version=version,+ visible_rows=visible_rows,+ winner_person_uuid=owner,+ winner_version=version,+ winner_deleted=deleted,+ )+++class TestClassifyMapping(SimpleTestCase):+ @parameterized.expand(+ [+ ("absent_everywhere", None, None, None, _v("ok_deleted", None, None)),+ ("ch_only_tombstoned", None, _ch_map(deleted=True), None, _v("ok_deleted", None, None)),+ ("ghost_of_a_ghost_person", None, _ch_map(owner="a"), "absent", _v("ghost", "owner_ghost", "restore")),+ ("ghost_of_a_live_person", None, _ch_map(owner="a"), "live", _v("ghost", "owner_live", "restore")),+ ("ghost_of_a_deleted_person", None, _ch_map(owner="a"), "deleted", _v("ghost", "owner_deleted", None)),+ (+ "deleted_and_ch_live_above",+ _pg_map(version=2, deleted=True),+ _ch_map(version=3),+ None,+ _v("deleted_visible", None, "bury"),+ ),+ (+ "deleted_and_never_published",+ _pg_map(version=2, deleted=True),+ _ch_map(version=1),+ None,+ _v("deleted_visible", None, "bury"),+ ),+ (+ "deleted_and_published",+ _pg_map(version=2, deleted=True),+ _ch_map(version=2, deleted=True),+ None,+ _v("ok_deleted", None, None),+ ),+ (+ "stray_on_deleted_person",+ _pg_map(owner_deleted=True),+ _ch_map(),+ None,+ _v("stray", None, None),+ ),+ (+ "orphan_with_ch_on_another_person",+ _pg_map(owner=None, owner_deleted=None),+ _ch_map(owner="a"),+ "absent",+ _v("stray", None, None),+ ),+ ("live_missing_in_ch", _pg_map(), None, None, _v("missing_in_ch", None, "republish")),+ ("live_and_swept", _pg_map(), _ch_map(visible_rows=0), None, _v("missing_in_ch", None, "republish")),+ ("live_hidden", _pg_map(), _ch_map(deleted=True), None, _v("hidden", None, "republish")),+ (+ "diverged_old_owner_absent",+ _pg_map(),+ _ch_map(owner="a"),+ "absent",+ _v("diverged", "a_absent", "fold"),+ ),+ ("diverged_old_owner_live", _pg_map(), _ch_map(owner="a"), "live", _v("diverged", "a_live", "merge")),+ (+ "diverged_old_owner_deleted",+ _pg_map(),+ _ch_map(owner="a"),+ "deleted",+ _v("diverged", "a_deleted", "republish"),+ ),+ ("diverged_old_owner_unknown", _pg_map(), _ch_map(owner="a"), None, _v("diverged", "a_unknown", None)),+ ("stale_same_owner", _pg_map(version=1), _ch_map(version=5), None, _v("stale", None, "republish")),+ ("ok", _pg_map(version=5), _ch_map(version=5), None, _v("ok", None, None)),+ ]+ )+ def test_classifies(self, _name, pg, ch, owner_state, expected):+ assert classify_mapping(pg, ch, ch_owner_pg_state=owner_state) == expected++ def test_ghost_of_a_deleted_team_is_left_alone(self):+ assert classify_mapping(None, _ch_map(owner="a"), ch_owner_pg_state="absent", team_exists=False) == _v(+ "ghost", "team_gone", None+ )diff --git a/posthog/personhog_client/proto/__init__.py b/posthog/personhog_client/proto/__init__.pyindex 5a34e46d4c46..d90b4cbb70d3 100644--- a/posthog/personhog_client/proto/__init__.py+++ b/posthog/personhog_client/proto/__init__.py@@ -109,9 +109,11 @@ from personhog.types.v1.person_pb2 import (PersonVersionFloor,EnsurePersonVersionFloorsRequest,EnsurePersonVersionFloorsResponse,+ DistinctIdRow,GetDistinctIdRowsRequest,GetDistinctIdRowsResponse,ListPersonsForTeamRequest,+ PersonListEntry,ListPersonsForTeamResponse,RaisePersonVersionFloorsRequest,RaisePersonVersionFloorsResponse,