Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(persons): add person_inspect for explicit persons and distinct ids (#118358)

Eli Reisman81d4ff0b3593
diff --git a/posthog/management/commands/person_inspect.py b/posthog/management/commands/person_inspect.py
new file mode 100644
index 000000000000..70f1b301f879
--- /dev/null
+++ b/posthog/management/commands/person_inspect.py
@@ -0,0 +1,55 @@
+"""Show the Postgres and ClickHouse state of explicit persons and distinct ids, and their flavor.
+
+Usage:
+ python manage.py person_inspect --team-id 2 --person-uuid <uuid> --distinct-id <id>
+
+Read-only. Prints one JSON object per key. Distinct ids are hashed unless --show-distinct-ids is passed.
+"""
+
+import json
+import dataclasses
+from datetime import UTC, datetime
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandError, CommandParser
+
+from posthog.clickhouse.query_tagging import Feature, Product, tags_context
+from posthog.models.person.divergence.inspection import hash_distinct_id, inspect_mappings, inspect_persons
+
+
+def _parse_utc(value: str) -> datetime:
+ parsed = datetime.fromisoformat(value)
+ return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC)
+
+
+class Command(BaseCommand):
+ help = "Read-only. Show the Postgres and ClickHouse state and the flavor of explicit persons and distinct ids."
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ parser.add_argument("--team-id", type=int, required=True)
+ parser.add_argument("--person-uuid", action="append", default=[], help="Person to inspect. Repeatable.")
+ parser.add_argument("--distinct-id", action="append", default=[], help="Distinct id to inspect. Repeatable.")
+ parser.add_argument(
+ "--purged-at",
+ type=_parse_utc,
+ default=None,
+ help="When the team's Postgres persons were purged; dates a ghost as after_purge or historical.",
+ )
+ parser.add_argument(
+ "--show-distinct-ids", action="store_true", help="Print distinct ids as they are instead of hashed."
+ )
+
+ def handle(self, *args: Any, **options: Any) -> None:
+ if not options["person_uuid"] and not options["distinct_id"]:
+ raise CommandError("Pass at least one --person-uuid or --distinct-id")
+ team_id = options["team_id"]
+ with tags_context(product=Product.INTERNAL, feature=Feature.MANAGEMENT_COMMAND):
+ inspections = [
+ *inspect_persons(team_id, options["person_uuid"], purged_at=options["purged_at"]),
+ *inspect_mappings(team_id, options["distinct_id"]),
+ ]
+ for inspection in inspections:
+ row = dataclasses.asdict(inspection)
+ if inspection.kind == "mapping" and not options["show_distinct_ids"]:
+ row["key"] = hash_distinct_id(team_id, inspection.key)
+ self.stdout.write(json.dumps(row, sort_keys=True))
diff --git a/posthog/models/person/divergence/inspection.py b/posthog/models/person/divergence/inspection.py
new file mode 100644
index 000000000000..3e5feaf63c5c
--- /dev/null
+++ b/posthog/models/person/divergence/inspection.py
@@ -0,0 +1,211 @@
+"""Classify explicit persons and mappings of one team, with the evidence each verdict rests on."""
+
+from __future__ import annotations
+
+import hashlib
+import dataclasses
+from collections.abc import Sequence
+from datetime import datetime
+from typing import Any, Literal
+from uuid import UUID
+
+from django.db.models import Q
+
+from posthog.dataclasses import frozen
+from posthog.models.activity_logging.activity_log import ActivityLog
+from posthog.models.person.divergence import lookups, observe
+from posthog.models.person.divergence.flavors import classify_mapping, classify_person
+from posthog.models.person.util import list_person_tombstone_queue
+from posthog.models.team import Team
+
+KeyKind = Literal["person", "mapping"]
+
+
+@frozen
+class Inspection:
+ kind: KeyKind
+ team_id: int
+ key: str
+ flavor: str
+ sub_flavor: str | None
+ verb: str | None
+ pg: dict[str, Any] | None
+ ch: dict[str, Any] | None
+ # For mappings: the owner ClickHouse names and that owner's Postgres state.
+ ch_owner_pg_state: str | None = None
+ # Every ClickHouse row of the key, lightweight-deleted ones included, oldest version first.
+ ch_history: list[dict[str, Any]] | None = None
+ activity: list[dict[str, Any]] | None = None
+ tombstone_queue: dict[str, Any] | None = None
+
+
+def hash_distinct_id(team_id: int, distinct_id: str) -> str:
+ return hashlib.sha256(f"{team_id}:{distinct_id}".encode()).hexdigest()[:16]
+
+
+def ghost_era(newest_written_at: float | None, purged_at: datetime | None) -> str:
+ if purged_at is not None and newest_written_at is not None and newest_written_at < purged_at.timestamp():
+ return "after_purge"
+ return "historical"
+
+
+def _asdict(state: Any) -> dict[str, Any] | None:
+ return None if state is None else dataclasses.asdict(state)
+
+
+_HISTORY_LIMIT = 50
+_ACTIVITY_LIMIT = 20
+
+_PERSON_HISTORY_SQL = """
+SELECT toString(id), version, is_deleted, is_identified, length(properties), toString(_timestamp), _row_exists
+FROM person
+WHERE team_id = %(team_id)s AND id IN %(keys)s
+ORDER BY id, version, _timestamp
+LIMIT %(limit)s BY id
+"""
+
+_MAPPING_HISTORY_SQL = """
+SELECT distinct_id, toString(person_id), version, is_deleted, toString(_timestamp), _row_exists
+FROM person_distinct_id2
+WHERE team_id = %(team_id)s AND distinct_id IN %(keys)s
+ORDER BY distinct_id, version, _timestamp
+LIMIT %(limit)s BY distinct_id
+"""
+
+
+def _person_history(team_id: int, person_uuids: Sequence[str]) -> dict[str, list[dict[str, Any]]]:
+ history: dict[str, list[dict[str, Any]]] = {u: [] for u in person_uuids}
+ if not person_uuids:
+ return history
+ rows = lookups._ch(
+ _PERSON_HISTORY_SQL,
+ {"team_id": team_id, "keys": list(person_uuids), "limit": _HISTORY_LIMIT},
+ observe._READ_SETTINGS,
+ )
+ for person_uuid, version, is_deleted, is_identified, properties_bytes, written_at, row_exists in rows:
+ history.setdefault(person_uuid, []).append(
+ {
+ "version": int(version),
+ "is_deleted": bool(is_deleted),
+ "is_identified": bool(is_identified),
+ "properties_bytes": int(properties_bytes),
+ "written_at": written_at,
+ "visible": bool(row_exists),
+ }
+ )
+ return history
+
+
+def _mapping_history(team_id: int, distinct_ids: Sequence[str]) -> dict[str, list[dict[str, Any]]]:
+ history: dict[str, list[dict[str, Any]]] = {d: [] for d in distinct_ids}
+ if not distinct_ids:
+ return history
+ rows = lookups._ch(
+ _MAPPING_HISTORY_SQL,
+ {"team_id": team_id, "keys": list(distinct_ids), "limit": _HISTORY_LIMIT},
+ observe._READ_SETTINGS,
+ )
+ for distinct_id, person_uuid, version, is_deleted, written_at, row_exists in rows:
+ history.setdefault(distinct_id, []).append(
+ {
+ "person_uuid": person_uuid,
+ "version": int(version),
+ "is_deleted": bool(is_deleted),
+ "written_at": written_at,
+ "visible": bool(row_exists),
+ }
+ )
+ return history
+
+
+def _activity(team_id: int, person_uuid: str, person_id: int | None) -> list[dict[str, Any]]:
+ # Deletes and splits name the person by uuid in detail.name, which the GIN index on detail serves.
+ # Property edits carry only item_id, and a tombstoned or purged person has no known pk.
+ names_person = Q(detail__contains={"name": person_uuid})
+ if person_id is not None:
+ names_person |= Q(item_id=str(person_id))
+ entries = (
+ ActivityLog.objects.filter(names_person, team_id=team_id, scope="Person")
+ .order_by("-created_at")
+ .values("activity", "created_at", "user_id", "was_impersonated", "is_system")[:_ACTIVITY_LIMIT]
+ )
+ return [{**entry, "created_at": entry["created_at"].isoformat()} for entry in entries]
+
+
+def _tombstone_queue_entry(team_id: int, person_uuid: str) -> dict[str, Any] | None:
+ # The queue is keyset-paged after (team, uuid), so start just below this uuid to read exactly its entry.
+ before = UUID(int=UUID(person_uuid).int - 1) if UUID(person_uuid).int else None
+ entries = list_person_tombstone_queue((team_id, before) if before else None, 1, team_id=team_id)
+ entry = next((e for e in entries if str(e.person_uuid) == person_uuid), None)
+ if entry is None:
+ return None
+ return {"person_version": entry.person_version, "tombstoned_at_ms": entry.tombstoned_at_ms}
+
+
+def inspect_persons(
+ team_id: int, person_uuids: Sequence[str], *, purged_at: datetime | None = None
+) -> list[Inspection]:
+ # Every store keys persons by the canonical lowercase form, so other spellings would read as absent.
+ uuids = list(dict.fromkeys(str(UUID(u)) for u in person_uuids))
+ team_exists = Team.objects.filter(id=team_id).exists()
+ pg = observe.pg_person_states(team_id, uuids) if team_exists else {}
+ ch = observe.ch_person_states(team_id, uuids)
+ history = _person_history(team_id, uuids)
+ results = []
+ for person_uuid in uuids:
+ pg_state = pg.get(person_uuid)
+ ch_state = ch.get(person_uuid)
+ verdict = classify_person(
+ pg_state,
+ ch_state,
+ team_exists=team_exists,
+ ghost_era=ghost_era(ch_state.newest_written_at if ch_state else None, purged_at),
+ )
+ results.append(
+ Inspection(
+ kind="person",
+ team_id=team_id,
+ key=person_uuid,
+ flavor=verdict.flavor,
+ sub_flavor=verdict.sub_flavor,
+ verb=verdict.verb,
+ pg=_asdict(pg_state),
+ ch=_asdict(ch_state),
+ ch_history=history[person_uuid],
+ activity=_activity(team_id, person_uuid, pg_state.id if pg_state else None),
+ tombstone_queue=_tombstone_queue_entry(team_id, person_uuid) if team_exists else None,
+ )
+ )
+ return results
+
+
+def inspect_mappings(team_id: int, distinct_ids: Sequence[str]) -> list[Inspection]:
+ dids = list(dict.fromkeys(distinct_ids))
+ team_exists = Team.objects.filter(id=team_id).exists()
+ pg = observe.pg_mapping_rows(team_id, dids) if team_exists else {}
+ ch = observe.ch_mapping_states(team_id, dids)
+ owners = sorted({c.winner_person_uuid for c in ch.values()})
+ owner_states = observe.pg_person_states(team_id, owners) if team_exists and owners else {}
+ history = _mapping_history(team_id, dids)
+ results = []
+ for distinct_id in dids:
+ ch_state = ch.get(distinct_id)
+ owner_state = observe.person_pg_state(owner_states, ch_state.winner_person_uuid) if ch_state else None
+ verdict = classify_mapping(
+ pg.get(distinct_id), ch_state, ch_owner_pg_state=owner_state, team_exists=team_exists
+ )
+ results.append(
+ Inspection(
+ kind="mapping",
+ team_id=team_id,
+ key=distinct_id,
+ flavor=verdict.flavor,
+ sub_flavor=verdict.sub_flavor,
+ verb=verdict.verb,
+ pg=_asdict(pg.get(distinct_id)),
+ ch=_asdict(ch_state),
+ ch_owner_pg_state=owner_state,
+ ch_history=history[distinct_id],
+ )
+ )
+ return results
diff --git a/posthog/models/person/divergence/lookups.py b/posthog/models/person/divergence/lookups.py
index 1330db8ca1da..1a42bac4ecd5 100644
--- a/posthog/models/person/divergence/lookups.py
+++ b/posthog/models/person/divergence/lookups.py
@@ -3,7 +3,7 @@
from __future__ import annotations
from collections.abc import Sequence
-from typing import Any, TypeVar
+from typing import TYPE_CHECKING, Any, TypeVar
from uuid import UUID
from posthog.clickhouse.client import sync_execute
@@ -12,6 +12,9 @@ from posthog.models.person.util import _batched_get_persons_by_uuids, get_person
from posthog.personhog_client.client import personhog_call
from posthog.personhog_client.proto import ReadOptions
+if TYPE_CHECKING:
+ from personhog.types.v1 import person_pb2
+
_T = TypeVar("_T")
@@ -23,14 +26,17 @@ def _ch(sql: str, args: dict[str, Any], settings: dict[str, int]) -> list[Any]:
return sync_execute(sql, args, settings=settings, workload=Workload.OFFLINE, readonly=True)
-def _live_person_versions(team_id: int, person_uuids: list[str], operation: str) -> dict[str, int]:
- persons = personhog_call(
+def _live_persons(team_id: int, person_uuids: list[str], operation: str) -> list[person_pb2.Person]:
+ return personhog_call(
operation,
lambda: _batched_get_persons_by_uuids(
team_id, person_uuids, operation, read_options=_VERSION_ONLY_READ_OPTIONS
),
)
- return {str(UUID(p.uuid)): int(p.version) for p in persons}
+
+
+def _live_person_versions(team_id: int, person_uuids: list[str], operation: str) -> dict[str, int]:
+ return {str(UUID(p.uuid)): int(p.version) for p in _live_persons(team_id, person_uuids, operation)}
def _tombstoned_uuids(team_id: int, person_uuids: list[str]) -> set[str]:
diff --git a/posthog/models/person/divergence/observe.py b/posthog/models/person/divergence/observe.py
new file mode 100644
index 000000000000..16daded93149
--- /dev/null
+++ b/posthog/models/person/divergence/observe.py
@@ -0,0 +1,125 @@
+"""Read one batch of keys from both stores into the classifier's state records."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any, Literal
+from uuid import UUID
+
+from posthog.models.person.divergence import lookups
+from posthog.models.person.divergence.flavors import ChMapping, ChPerson, PgMapping, PgPerson
+from posthog.models.person.util import get_distinct_id_rows, get_person_tombstones
+
+PersonPgState = Literal["live", "deleted", "absent"]
+
+# apply_deleted_mask = 0 shows lightweight-deleted rows, so a raise lands above every row a sweep masked.
+_READ_SETTINGS = {"apply_deleted_mask": 0, "max_execution_time": 120, "max_memory_usage": 8_000_000_000}
+_CH_CHUNK = 1_000
+_OPERATION = "person_consistency_observe"
+
+_CH_PERSONS_SQL = """
+SELECT
+ toString(id),
+ max(version),
+ countIf(_row_exists),
+ maxIfOrNull(version, _row_exists AND is_deleted = 0),
+ maxIfOrNull(version, _row_exists AND is_deleted = 1),
+ groupArrayIf(
+ (version, cityHash64(properties, is_identified, is_deleted), properties = '{}' AND is_identified = 0),
+ _row_exists
+ ),
+ maxIf(toUnixTimestamp(_timestamp), _row_exists)
+FROM person
+WHERE team_id = %(team_id)s AND id IN %(person_uuids)s
+GROUP BY id
+"""
+
+_CH_MAPPINGS_SQL = """
+SELECT
+ distinct_id,
+ max(version),
+ countIf(_row_exists),
+ toString(argMaxIf(person_id, version, _row_exists)),
+ maxIf(version, _row_exists),
+ argMaxIf(is_deleted, version, _row_exists)
+FROM person_distinct_id2
+WHERE team_id = %(team_id)s AND distinct_id IN %(distinct_ids)s
+GROUP BY distinct_id
+"""
+
+
+def ch_person_from_row(row: Sequence[Any]) -> tuple[str, ChPerson]:
+ person_uuid, max_version, visible_rows, live_max, tomb_max, visible, newest_written_at = row
+ visible_max = max((int(v) for v, _, _ in visible), default=None)
+ at_max = [(payload, blank) for v, payload, blank in visible if visible_max is not None and int(v) == visible_max]
+ earlier_had_properties = any(
+ not blank for v, _, blank in visible if visible_max is not None and int(v) < visible_max
+ )
+ return str(person_uuid), ChPerson(
+ max_version=int(max_version),
+ visible_rows=int(visible_rows),
+ live_max_version=None if live_max is None else int(live_max),
+ tombstone_max_version=None if tomb_max is None else int(tomb_max),
+ rows_at_max=len(at_max),
+ payloads_at_max=len({payload for payload, _ in at_max}),
+ winner_blanked=bool(at_max) and all(blank for _, blank in at_max) and earlier_had_properties,
+ newest_written_at=float(newest_written_at) if int(visible_rows) else None,
+ )
+
+
+def ch_person_states(team_id: int, person_uuids: Sequence[str]) -> dict[str, ChPerson]:
+ states: dict[str, ChPerson] = {}
+ for chunk in lookups._chunks(list(person_uuids), _CH_CHUNK):
+ rows = lookups._ch(_CH_PERSONS_SQL, {"team_id": team_id, "person_uuids": list(chunk)}, _READ_SETTINGS)
+ states.update(ch_person_from_row(row) for row in rows)
+ return states
+
+
+def ch_mapping_states(team_id: int, distinct_ids: Sequence[str]) -> dict[str, ChMapping]:
+ states: dict[str, ChMapping] = {}
+ for chunk in lookups._chunks(list(distinct_ids), _CH_CHUNK):
+ rows = lookups._ch(_CH_MAPPINGS_SQL, {"team_id": team_id, "distinct_ids": list(chunk)}, _READ_SETTINGS)
+ for distinct_id, max_version, visible_rows, owner, winner_version, winner_deleted in rows:
+ states[distinct_id] = ChMapping(
+ max_version=int(max_version),
+ visible_rows=int(visible_rows),
+ winner_person_uuid=str(owner),
+ winner_version=int(winner_version),
+ winner_deleted=bool(winner_deleted),
+ )
+ return states
+
+
+def pg_person_states(team_id: int, person_uuids: Sequence[str]) -> dict[str, PgPerson]:
+ """Live and tombstoned Postgres persons by uuid; absent uuids are left out."""
+ uuids = list(person_uuids)
+ live = {
+ str(UUID(p.uuid)): PgPerson(id=int(p.id), version=int(p.version), is_deleted=False)
+ for p in lookups._live_persons(team_id, uuids, _OPERATION)
+ }
+ missing = [UUID(u) for u in uuids if u not in live]
+ tombstoned = {
+ str(t.uuid): PgPerson(id=None, version=int(t.version), is_deleted=True)
+ for t in (get_person_tombstones(team_id, missing) if missing else [])
+ }
+ return {**tombstoned, **live}
+
+
+def pg_mapping_rows(team_id: int, distinct_ids: Sequence[str]) -> dict[str, PgMapping]:
+ return {
+ row.distinct_id: PgMapping(
+ person_id=int(row.person_id),
+ person_uuid=row.person_uuid if row.HasField("person_uuid") else None,
+ version=int(row.version),
+ is_deleted=row.is_deleted,
+ person_is_deleted=row.person_is_deleted if row.HasField("person_is_deleted") else None,
+ )
+ for row in get_distinct_id_rows(team_id, list(distinct_ids))
+ }
+
+
+def person_pg_state(states: dict[str, PgPerson], person_uuid: str) -> PersonPgState:
+ state = states.get(person_uuid)
+ if state is None:
+ return "absent"
+ return "deleted" if state.is_deleted else "live"
diff --git a/posthog/models/person/test/test_divergence_inspection.py b/posthog/models/person/test/test_divergence_inspection.py
new file mode 100644
index 000000000000..901e11589c19
--- /dev/null
+++ b/posthog/models/person/test/test_divergence_inspection.py
@@ -0,0 +1,173 @@
+import json
+from io import StringIO
+from uuid import uuid4
+
+from posthog.test.base import BaseTest, ClickhouseTestMixin
+
+from django.core.management import call_command
+from django.utils.timezone import now
+
+from posthog.clickhouse.client import sync_execute
+from posthog.models.activity_logging.activity_log import ActivityLog, Detail
+from posthog.models.person.divergence.inspection import hash_distinct_id, inspect_mappings, inspect_persons
+from posthog.models.person.sql import BULK_INSERT_PERSON_DISTINCT_ID2
+from posthog.models.person.util import (
+ create_person as create_person_in_ch,
+ tombstone_persons_in_postgres,
+)
+from posthog.models.signals import mute_selected_signals
+from posthog.personhog_client.fake_client import get_active_fake
+from posthog.test.persons import add_distinct_id, create_person
+
+
+class TestDivergenceInspection(ClickhouseTestMixin, BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ # A background merge collapses the duplicate and older rows the tie and blanked shapes are made of.
+ for table in ("person", "person_distinct_id2"):
+ sync_execute(f"SYSTEM STOP MERGES {table}")
+ self.addCleanup(sync_execute, f"SYSTEM START MERGES {table}")
+
+ def _pg_person(self, *, version: int, distinct_ids: list[str] | None = None):
+ with mute_selected_signals():
+ person = create_person(team=self.team, version=version, properties={"email": "[email protected]"})
+ for distinct_id in distinct_ids or []:
+ add_distinct_id(person=person, distinct_id=distinct_id, version=0)
+ return person
+
+ def _ch_person(self, person_uuid: str, version: int, properties: dict, *, deleted: bool = False) -> None:
+ create_person_in_ch(
+ team_id=self.team.pk,
+ uuid=person_uuid,
+ version=version,
+ is_deleted=deleted,
+ properties=properties,
+ timestamp=now(),
+ )
+
+ def _ch_mapping(self, distinct_id: str, person_uuid: str, version: int, *, team_id: int | None = None) -> None:
+ sync_execute(
+ BULK_INSERT_PERSON_DISTINCT_ID2,
+ [
+ {
+ "distinct_id": distinct_id,
+ "person_id": person_uuid,
+ "team_id": team_id or self.team.pk,
+ "is_deleted": 0,
+ "version": version,
+ "_timestamp": now().replace(tzinfo=None),
+ "_offset": 0,
+ "_partition": 0,
+ }
+ ],
+ flush=False,
+ )
+
+ def _log_person_activity(self, activity: str, *, item_id: int, name: str | None) -> None:
+ ActivityLog.objects.create(
+ organization_id=self.organization.id,
+ team_id=self.team.pk,
+ item_id=str(item_id),
+ scope="Person",
+ activity=activity,
+ detail=Detail(name=name),
+ )
+
+ def _flavors(self, inspections) -> dict[str, tuple[str, str | None, str | None]]:
+ return {i.key: (i.flavor, i.sub_flavor, i.verb) for i in inspections}
+
+ def test_inspect_persons_reads_both_stores_into_each_flavor(self):
+ tie = self._pg_person(version=2)
+ self._ch_person(str(tie.uuid), 2, {"email": "[email protected]"})
+ self._ch_person(str(tie.uuid), 2, {"email": "[email protected]"})
+ blanked = self._pg_person(version=105)
+ self._ch_person(str(blanked.uuid), 1, {"email": "[email protected]"})
+ self._ch_person(str(blanked.uuid), 105, {})
+ deleted = self._pg_person(version=2)
+ tombstone_persons_in_postgres(self.team.pk, [deleted.uuid])
+ self._ch_person(str(deleted.uuid), 7, {"email": "[email protected]"})
+ stale = self._pg_person(version=1)
+ self._ch_person(str(stale.uuid), 40, {"email": "[email protected]"})
+ ghost = str(uuid4())
+ self._ch_person(ghost, 3, {"email": "[email protected]"})
+
+ self._log_person_activity("deleted", item_id=deleted.pk, name=str(deleted.uuid))
+ self._log_person_activity("updated", item_id=stale.pk, name=None)
+ self._log_person_activity("updated", item_id=tie.pk, name=None)
+
+ inspections = inspect_persons(
+ self.team.pk, [str(p.uuid) for p in (tie, blanked, deleted)] + [str(stale.uuid).upper(), ghost]
+ )
+ by_key = {i.key: i for i in inspections}
+ flavors = self._flavors(inspections)
+
+ assert [(r["version"], r["properties_bytes"] > 2) for r in by_key[str(blanked.uuid)].ch_history or []] == [
+ (1, True),
+ (105, False),
+ ]
+ assert [a["activity"] for a in by_key[str(deleted.uuid)].activity or []] == ["deleted"]
+ assert [a["activity"] for a in by_key[str(stale.uuid)].activity or []] == ["updated"]
+ assert by_key[ghost].activity == []
+ queued = by_key[str(deleted.uuid)].tombstone_queue
+ assert queued is not None and queued["person_version"] == 3 and queued["tombstoned_at_ms"] > 0
+ assert by_key[str(stale.uuid)].tombstone_queue is None
+ assert flavors == {
+ str(tie.uuid): ("tie", None, "republish"),
+ str(blanked.uuid): ("blanked", None, "republish"),
+ str(deleted.uuid): ("deleted_visible", None, "bury"),
+ str(stale.uuid): ("stale", None, "merge_stale"),
+ ghost: ("ghost", "historical", None),
+ }
+
+ def test_inspect_mappings_dates_owners_and_finds_strays(self):
+ self._pg_person(version=0, distinct_ids=["moved"])
+ old_owner = str(uuid4())
+ self._ch_mapping("moved", old_owner, 5)
+ fake = get_active_fake()
+ assert fake is not None
+ deleted = fake.add_person(
+ team_id=self.team.pk,
+ person_id=990_001,
+ uuid=str(uuid4()),
+ version=3,
+ is_deleted=True,
+ distinct_ids=["stray"],
+ )
+ self._ch_mapping("stray", deleted.uuid, 0)
+
+ flavors = self._flavors(inspect_mappings(self.team.pk, ["moved", "stray"]))
+
+ assert flavors == {
+ "moved": ("diverged", "a_absent", "fold"),
+ "stray": ("stray", None, None),
+ }
+
+ def test_inspect_mappings_leaves_ghosts_of_a_deleted_team_alone(self):
+ gone_team_id = self.team.pk + 10_000
+ self._ch_mapping("gone", str(uuid4()), 4, team_id=gone_team_id)
+
+ assert self._flavors(inspect_mappings(gone_team_id, ["gone"])) == {"gone": ("ghost", "team_gone", None)}
+
+ def test_command_prints_one_json_row_per_key_with_hashed_distinct_ids(self):
+ person = self._pg_person(version=2, distinct_ids=["secret-did-7f3a"])
+ self._ch_person(str(person.uuid), 2, {"email": "[email protected]"})
+ self._ch_mapping("secret-did-7f3a", str(person.uuid), 0)
+ out = StringIO()
+
+ call_command(
+ "person_inspect",
+ "--team-id",
+ str(self.team.pk),
+ "--person-uuid",
+ str(person.uuid),
+ "--distinct-id",
+ "secret-did-7f3a",
+ stdout=out,
+ )
+
+ rows = [json.loads(line) for line in out.getvalue().splitlines()]
+ assert [(r["kind"], r["key"], r["flavor"]) for r in rows] == [
+ ("person", str(person.uuid), "ok"),
+ ("mapping", hash_distinct_id(self.team.pk, "secret-did-7f3a"), "ok"),
+ ]
+ assert "secret-did-7f3a" not in out.getvalue()
diff --git a/posthog/models/person/util.py b/posthog/models/person/util.py
index 31e9e020034d..202f404379f3 100644
--- a/posthog/models/person/util.py
+++ b/posthog/models/person/util.py
@@ -39,11 +39,14 @@ from posthog.personhog_client.converters import proto_person_to_model
from posthog.personhog_client.interceptor import is_transient_rpc_error
from posthog.personhog_client.metrics import PERSONHOG_TEAM_MISMATCH_TOTAL, get_client_name
from posthog.personhog_client.proto import (
+ CONSISTENCY_LEVEL_STRONG,
AckedPersonTombstone,
AckPersonTombstonesRequest,
DeletePersonsMode,
DeletePersonsRequest,
+ DistinctIdRow,
EnsurePersonVersionFloorsRequest,
+ GetDistinctIdRowsRequest,
GetDistinctIdsForPersonRequest,
GetDistinctIdsForPersonsRequest,
GetPersonByDistinctIdRequest,
@@ -52,7 +55,9 @@ from posthog.personhog_client.proto import (
GetPersonsByDistinctIdsInTeamRequest,
GetPersonsByUuidsRequest,
GetPersonTombstonesRequest,
+ ListPersonsForTeamRequest,
ListPersonTombstoneQueueRequest,
+ PersonListEntry,
PersonVersionFloor as PersonVersionFloorProto,
ReadOptions,
SetPersonDistinctIdVersionFloorRequest,
@@ -774,6 +779,40 @@ def get_person_tombstones(team_id: int, person_uuids: list[UUID]) -> list[Person
return personhog_call("get_person_tombstones", personhog_fn)
+def get_distinct_id_rows(team_id: int, distinct_ids: list[str], *, strong: bool = False) -> list[DistinctIdRow]:
+ """Mapping rows, live or tombstoned, with their owner's state. `strong` reads the primary."""
+
+ def personhog_fn() -> list[DistinctIdRow]:
+ read_options = ReadOptions(consistency=CONSISTENCY_LEVEL_STRONG) if strong else None
+ rows: list[DistinctIdRow] = []
+ for i in range(0, len(distinct_ids), 1000):
+ response = _get_client().get_distinct_id_rows(
+ GetDistinctIdRowsRequest(
+ team_id=team_id, distinct_ids=distinct_ids[i : i + 1000], read_options=read_options
+ )
+ )
+ rows.extend(response.rows)
+ return rows
+
+ return personhog_call("get_distinct_id_rows", personhog_fn)
+
+
+def list_persons_for_team(
+ team_id: int, after_person_id: int, *, before_person_id: int | None = None, limit: int = 1000
+) -> tuple[list[PersonListEntry], int | None]:
+ """One id-ordered page of a team's persons, live and tombstoned, and the cursor for the next page."""
+
+ def personhog_fn() -> tuple[list[PersonListEntry], int | None]:
+ request = ListPersonsForTeamRequest(team_id=team_id, after_person_id=after_person_id, limit=limit)
+ if before_person_id is not None:
+ request.before_person_id = before_person_id
+ response = _get_client().list_persons_for_team(request)
+ next_after = response.next_after_person_id if response.HasField("next_after_person_id") else None
+ return list(response.persons), next_after
+
+ return personhog_call("list_persons_for_team", personhog_fn)
+
+
def list_person_tombstone_queue(
after: tuple[int, UUID] | None, limit: int, team_id: int | None = None
) -> list[QueuedPersonTombstone]: