:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(persons): add person_inspect for explicit persons and distinct ids (#118358)
Eli Reisman81d4ff0b3593
diff --git a/posthog/management/commands/person_inspect.py b/posthog/management/commands/person_inspect.pynew file mode 100644index 000000000000..70f1b301f879--- /dev/null+++ b/posthog/management/commands/person_inspect.py@@ -0,0 +1,55 @@+"""Show the Postgres and ClickHouse state of explicit persons and distinct ids, and their flavor.++Usage:+ python manage.py person_inspect --team-id 2 --person-uuid <uuid> --distinct-id <id>++Read-only. Prints one JSON object per key. Distinct ids are hashed unless --show-distinct-ids is passed.+"""++import json+import dataclasses+from datetime import UTC, datetime+from typing import Any++from django.core.management.base import BaseCommand, CommandError, CommandParser++from posthog.clickhouse.query_tagging import Feature, Product, tags_context+from posthog.models.person.divergence.inspection import hash_distinct_id, inspect_mappings, inspect_persons+++def _parse_utc(value: str) -> datetime:+ parsed = datetime.fromisoformat(value)+ return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC)+++class Command(BaseCommand):+ help = "Read-only. Show the Postgres and ClickHouse state and the flavor of explicit persons and distinct ids."++ def add_arguments(self, parser: CommandParser) -> None:+ parser.add_argument("--team-id", type=int, required=True)+ parser.add_argument("--person-uuid", action="append", default=[], help="Person to inspect. Repeatable.")+ parser.add_argument("--distinct-id", action="append", default=[], help="Distinct id to inspect. Repeatable.")+ parser.add_argument(+ "--purged-at",+ type=_parse_utc,+ default=None,+ help="When the team's Postgres persons were purged; dates a ghost as after_purge or historical.",+ )+ parser.add_argument(+ "--show-distinct-ids", action="store_true", help="Print distinct ids as they are instead of hashed."+ )++ def handle(self, *args: Any, **options: Any) -> None:+ if not options["person_uuid"] and not options["distinct_id"]:+ raise CommandError("Pass at least one --person-uuid or --distinct-id")+ team_id = options["team_id"]+ with tags_context(product=Product.INTERNAL, feature=Feature.MANAGEMENT_COMMAND):+ inspections = [+ *inspect_persons(team_id, options["person_uuid"], purged_at=options["purged_at"]),+ *inspect_mappings(team_id, options["distinct_id"]),+ ]+ for inspection in inspections:+ row = dataclasses.asdict(inspection)+ if inspection.kind == "mapping" and not options["show_distinct_ids"]:+ row["key"] = hash_distinct_id(team_id, inspection.key)+ self.stdout.write(json.dumps(row, sort_keys=True))diff --git a/posthog/models/person/divergence/inspection.py b/posthog/models/person/divergence/inspection.pynew file mode 100644index 000000000000..3e5feaf63c5c--- /dev/null+++ b/posthog/models/person/divergence/inspection.py@@ -0,0 +1,211 @@+"""Classify explicit persons and mappings of one team, with the evidence each verdict rests on."""++from __future__ import annotations++import hashlib+import dataclasses+from collections.abc import Sequence+from datetime import datetime+from typing import Any, Literal+from uuid import UUID++from django.db.models import Q++from posthog.dataclasses import frozen+from posthog.models.activity_logging.activity_log import ActivityLog+from posthog.models.person.divergence import lookups, observe+from posthog.models.person.divergence.flavors import classify_mapping, classify_person+from posthog.models.person.util import list_person_tombstone_queue+from posthog.models.team import Team++KeyKind = Literal["person", "mapping"]+++@frozen+class Inspection:+ kind: KeyKind+ team_id: int+ key: str+ flavor: str+ sub_flavor: str | None+ verb: str | None+ pg: dict[str, Any] | None+ ch: dict[str, Any] | None+ # For mappings: the owner ClickHouse names and that owner's Postgres state.+ ch_owner_pg_state: str | None = None+ # Every ClickHouse row of the key, lightweight-deleted ones included, oldest version first.+ ch_history: list[dict[str, Any]] | None = None+ activity: list[dict[str, Any]] | None = None+ tombstone_queue: dict[str, Any] | None = None+++def hash_distinct_id(team_id: int, distinct_id: str) -> str:+ return hashlib.sha256(f"{team_id}:{distinct_id}".encode()).hexdigest()[:16]+++def ghost_era(newest_written_at: float | None, purged_at: datetime | None) -> str:+ if purged_at is not None and newest_written_at is not None and newest_written_at < purged_at.timestamp():+ return "after_purge"+ return "historical"+++def _asdict(state: Any) -> dict[str, Any] | None:+ return None if state is None else dataclasses.asdict(state)+++_HISTORY_LIMIT = 50+_ACTIVITY_LIMIT = 20++_PERSON_HISTORY_SQL = """+SELECT toString(id), version, is_deleted, is_identified, length(properties), toString(_timestamp), _row_exists+FROM person+WHERE team_id = %(team_id)s AND id IN %(keys)s+ORDER BY id, version, _timestamp+LIMIT %(limit)s BY id+"""++_MAPPING_HISTORY_SQL = """+SELECT distinct_id, toString(person_id), version, is_deleted, toString(_timestamp), _row_exists+FROM person_distinct_id2+WHERE team_id = %(team_id)s AND distinct_id IN %(keys)s+ORDER BY distinct_id, version, _timestamp+LIMIT %(limit)s BY distinct_id+"""+++def _person_history(team_id: int, person_uuids: Sequence[str]) -> dict[str, list[dict[str, Any]]]:+ history: dict[str, list[dict[str, Any]]] = {u: [] for u in person_uuids}+ if not person_uuids:+ return history+ rows = lookups._ch(+ _PERSON_HISTORY_SQL,+ {"team_id": team_id, "keys": list(person_uuids), "limit": _HISTORY_LIMIT},+ observe._READ_SETTINGS,+ )+ for person_uuid, version, is_deleted, is_identified, properties_bytes, written_at, row_exists in rows:+ history.setdefault(person_uuid, []).append(+ {+ "version": int(version),+ "is_deleted": bool(is_deleted),+ "is_identified": bool(is_identified),+ "properties_bytes": int(properties_bytes),+ "written_at": written_at,+ "visible": bool(row_exists),+ }+ )+ return history+++def _mapping_history(team_id: int, distinct_ids: Sequence[str]) -> dict[str, list[dict[str, Any]]]:+ history: dict[str, list[dict[str, Any]]] = {d: [] for d in distinct_ids}+ if not distinct_ids:+ return history+ rows = lookups._ch(+ _MAPPING_HISTORY_SQL,+ {"team_id": team_id, "keys": list(distinct_ids), "limit": _HISTORY_LIMIT},+ observe._READ_SETTINGS,+ )+ for distinct_id, person_uuid, version, is_deleted, written_at, row_exists in rows:+ history.setdefault(distinct_id, []).append(+ {+ "person_uuid": person_uuid,+ "version": int(version),+ "is_deleted": bool(is_deleted),+ "written_at": written_at,+ "visible": bool(row_exists),+ }+ )+ return history+++def _activity(team_id: int, person_uuid: str, person_id: int | None) -> list[dict[str, Any]]:+ # Deletes and splits name the person by uuid in detail.name, which the GIN index on detail serves.+ # Property edits carry only item_id, and a tombstoned or purged person has no known pk.+ names_person = Q(detail__contains={"name": person_uuid})+ if person_id is not None:+ names_person |= Q(item_id=str(person_id))+ entries = (+ ActivityLog.objects.filter(names_person, team_id=team_id, scope="Person")+ .order_by("-created_at")+ .values("activity", "created_at", "user_id", "was_impersonated", "is_system")[:_ACTIVITY_LIMIT]+ )+ return [{**entry, "created_at": entry["created_at"].isoformat()} for entry in entries]+++def _tombstone_queue_entry(team_id: int, person_uuid: str) -> dict[str, Any] | None:+ # The queue is keyset-paged after (team, uuid), so start just below this uuid to read exactly its entry.+ before = UUID(int=UUID(person_uuid).int - 1) if UUID(person_uuid).int else None+ entries = list_person_tombstone_queue((team_id, before) if before else None, 1, team_id=team_id)+ entry = next((e for e in entries if str(e.person_uuid) == person_uuid), None)+ if entry is None:+ return None+ return {"person_version": entry.person_version, "tombstoned_at_ms": entry.tombstoned_at_ms}+++def inspect_persons(+ team_id: int, person_uuids: Sequence[str], *, purged_at: datetime | None = None+) -> list[Inspection]:+ # Every store keys persons by the canonical lowercase form, so other spellings would read as absent.+ uuids = list(dict.fromkeys(str(UUID(u)) for u in person_uuids))+ team_exists = Team.objects.filter(id=team_id).exists()+ pg = observe.pg_person_states(team_id, uuids) if team_exists else {}+ ch = observe.ch_person_states(team_id, uuids)+ history = _person_history(team_id, uuids)+ results = []+ for person_uuid in uuids:+ pg_state = pg.get(person_uuid)+ ch_state = ch.get(person_uuid)+ verdict = classify_person(+ pg_state,+ ch_state,+ team_exists=team_exists,+ ghost_era=ghost_era(ch_state.newest_written_at if ch_state else None, purged_at),+ )+ results.append(+ Inspection(+ kind="person",+ team_id=team_id,+ key=person_uuid,+ flavor=verdict.flavor,+ sub_flavor=verdict.sub_flavor,+ verb=verdict.verb,+ pg=_asdict(pg_state),+ ch=_asdict(ch_state),+ ch_history=history[person_uuid],+ activity=_activity(team_id, person_uuid, pg_state.id if pg_state else None),+ tombstone_queue=_tombstone_queue_entry(team_id, person_uuid) if team_exists else None,+ )+ )+ return results+++def inspect_mappings(team_id: int, distinct_ids: Sequence[str]) -> list[Inspection]:+ dids = list(dict.fromkeys(distinct_ids))+ team_exists = Team.objects.filter(id=team_id).exists()+ pg = observe.pg_mapping_rows(team_id, dids) if team_exists else {}+ ch = observe.ch_mapping_states(team_id, dids)+ owners = sorted({c.winner_person_uuid for c in ch.values()})+ owner_states = observe.pg_person_states(team_id, owners) if team_exists and owners else {}+ history = _mapping_history(team_id, dids)+ results = []+ for distinct_id in dids:+ ch_state = ch.get(distinct_id)+ owner_state = observe.person_pg_state(owner_states, ch_state.winner_person_uuid) if ch_state else None+ verdict = classify_mapping(+ pg.get(distinct_id), ch_state, ch_owner_pg_state=owner_state, team_exists=team_exists+ )+ results.append(+ Inspection(+ kind="mapping",+ team_id=team_id,+ key=distinct_id,+ flavor=verdict.flavor,+ sub_flavor=verdict.sub_flavor,+ verb=verdict.verb,+ pg=_asdict(pg.get(distinct_id)),+ ch=_asdict(ch_state),+ ch_owner_pg_state=owner_state,+ ch_history=history[distinct_id],+ )+ )+ return resultsdiff --git a/posthog/models/person/divergence/lookups.py b/posthog/models/person/divergence/lookups.pyindex 1330db8ca1da..1a42bac4ecd5 100644--- a/posthog/models/person/divergence/lookups.py+++ b/posthog/models/person/divergence/lookups.py@@ -3,7 +3,7 @@from __future__ import annotationsfrom collections.abc import Sequence-from typing import Any, TypeVar+from typing import TYPE_CHECKING, Any, TypeVarfrom uuid import UUIDfrom posthog.clickhouse.client import sync_execute@@ -12,6 +12,9 @@ from posthog.models.person.util import _batched_get_persons_by_uuids, get_personfrom posthog.personhog_client.client import personhog_callfrom posthog.personhog_client.proto import ReadOptions+if TYPE_CHECKING:+ from personhog.types.v1 import person_pb2+_T = TypeVar("_T")@@ -23,14 +26,17 @@ def _ch(sql: str, args: dict[str, Any], settings: dict[str, int]) -> list[Any]:return sync_execute(sql, args, settings=settings, workload=Workload.OFFLINE, readonly=True)-def _live_person_versions(team_id: int, person_uuids: list[str], operation: str) -> dict[str, int]:- persons = personhog_call(+def _live_persons(team_id: int, person_uuids: list[str], operation: str) -> list[person_pb2.Person]:+ return personhog_call(operation,lambda: _batched_get_persons_by_uuids(team_id, person_uuids, operation, read_options=_VERSION_ONLY_READ_OPTIONS),)- return {str(UUID(p.uuid)): int(p.version) for p in persons}+++def _live_person_versions(team_id: int, person_uuids: list[str], operation: str) -> dict[str, int]:+ return {str(UUID(p.uuid)): int(p.version) for p in _live_persons(team_id, person_uuids, operation)}def _tombstoned_uuids(team_id: int, person_uuids: list[str]) -> set[str]:diff --git a/posthog/models/person/divergence/observe.py b/posthog/models/person/divergence/observe.pynew file mode 100644index 000000000000..16daded93149--- /dev/null+++ b/posthog/models/person/divergence/observe.py@@ -0,0 +1,125 @@+"""Read one batch of keys from both stores into the classifier's state records."""++from __future__ import annotations++from collections.abc import Sequence+from typing import Any, Literal+from uuid import UUID++from posthog.models.person.divergence import lookups+from posthog.models.person.divergence.flavors import ChMapping, ChPerson, PgMapping, PgPerson+from posthog.models.person.util import get_distinct_id_rows, get_person_tombstones++PersonPgState = Literal["live", "deleted", "absent"]++# apply_deleted_mask = 0 shows lightweight-deleted rows, so a raise lands above every row a sweep masked.+_READ_SETTINGS = {"apply_deleted_mask": 0, "max_execution_time": 120, "max_memory_usage": 8_000_000_000}+_CH_CHUNK = 1_000+_OPERATION = "person_consistency_observe"++_CH_PERSONS_SQL = """+SELECT+ toString(id),+ max(version),+ countIf(_row_exists),+ maxIfOrNull(version, _row_exists AND is_deleted = 0),+ maxIfOrNull(version, _row_exists AND is_deleted = 1),+ groupArrayIf(+ (version, cityHash64(properties, is_identified, is_deleted), properties = '{}' AND is_identified = 0),+ _row_exists+ ),+ maxIf(toUnixTimestamp(_timestamp), _row_exists)+FROM person+WHERE team_id = %(team_id)s AND id IN %(person_uuids)s+GROUP BY id+"""++_CH_MAPPINGS_SQL = """+SELECT+ distinct_id,+ max(version),+ countIf(_row_exists),+ toString(argMaxIf(person_id, version, _row_exists)),+ maxIf(version, _row_exists),+ argMaxIf(is_deleted, version, _row_exists)+FROM person_distinct_id2+WHERE team_id = %(team_id)s AND distinct_id IN %(distinct_ids)s+GROUP BY distinct_id+"""+++def ch_person_from_row(row: Sequence[Any]) -> tuple[str, ChPerson]:+ person_uuid, max_version, visible_rows, live_max, tomb_max, visible, newest_written_at = row+ visible_max = max((int(v) for v, _, _ in visible), default=None)+ at_max = [(payload, blank) for v, payload, blank in visible if visible_max is not None and int(v) == visible_max]+ earlier_had_properties = any(+ not blank for v, _, blank in visible if visible_max is not None and int(v) < visible_max+ )+ return str(person_uuid), ChPerson(+ max_version=int(max_version),+ visible_rows=int(visible_rows),+ live_max_version=None if live_max is None else int(live_max),+ tombstone_max_version=None if tomb_max is None else int(tomb_max),+ rows_at_max=len(at_max),+ payloads_at_max=len({payload for payload, _ in at_max}),+ winner_blanked=bool(at_max) and all(blank for _, blank in at_max) and earlier_had_properties,+ newest_written_at=float(newest_written_at) if int(visible_rows) else None,+ )+++def ch_person_states(team_id: int, person_uuids: Sequence[str]) -> dict[str, ChPerson]:+ states: dict[str, ChPerson] = {}+ for chunk in lookups._chunks(list(person_uuids), _CH_CHUNK):+ rows = lookups._ch(_CH_PERSONS_SQL, {"team_id": team_id, "person_uuids": list(chunk)}, _READ_SETTINGS)+ states.update(ch_person_from_row(row) for row in rows)+ return states+++def ch_mapping_states(team_id: int, distinct_ids: Sequence[str]) -> dict[str, ChMapping]:+ states: dict[str, ChMapping] = {}+ for chunk in lookups._chunks(list(distinct_ids), _CH_CHUNK):+ rows = lookups._ch(_CH_MAPPINGS_SQL, {"team_id": team_id, "distinct_ids": list(chunk)}, _READ_SETTINGS)+ for distinct_id, max_version, visible_rows, owner, winner_version, winner_deleted in rows:+ states[distinct_id] = ChMapping(+ max_version=int(max_version),+ visible_rows=int(visible_rows),+ winner_person_uuid=str(owner),+ winner_version=int(winner_version),+ winner_deleted=bool(winner_deleted),+ )+ return states+++def pg_person_states(team_id: int, person_uuids: Sequence[str]) -> dict[str, PgPerson]:+ """Live and tombstoned Postgres persons by uuid; absent uuids are left out."""+ uuids = list(person_uuids)+ live = {+ str(UUID(p.uuid)): PgPerson(id=int(p.id), version=int(p.version), is_deleted=False)+ for p in lookups._live_persons(team_id, uuids, _OPERATION)+ }+ missing = [UUID(u) for u in uuids if u not in live]+ tombstoned = {+ str(t.uuid): PgPerson(id=None, version=int(t.version), is_deleted=True)+ for t in (get_person_tombstones(team_id, missing) if missing else [])+ }+ return {**tombstoned, **live}+++def pg_mapping_rows(team_id: int, distinct_ids: Sequence[str]) -> dict[str, PgMapping]:+ return {+ row.distinct_id: PgMapping(+ person_id=int(row.person_id),+ person_uuid=row.person_uuid if row.HasField("person_uuid") else None,+ version=int(row.version),+ is_deleted=row.is_deleted,+ person_is_deleted=row.person_is_deleted if row.HasField("person_is_deleted") else None,+ )+ for row in get_distinct_id_rows(team_id, list(distinct_ids))+ }+++def person_pg_state(states: dict[str, PgPerson], person_uuid: str) -> PersonPgState:+ state = states.get(person_uuid)+ if state is None:+ return "absent"+ return "deleted" if state.is_deleted else "live"diff --git a/posthog/models/person/test/test_divergence_inspection.py b/posthog/models/person/test/test_divergence_inspection.pynew file mode 100644index 000000000000..901e11589c19--- /dev/null+++ b/posthog/models/person/test/test_divergence_inspection.py@@ -0,0 +1,173 @@+import json+from io import StringIO+from uuid import uuid4++from posthog.test.base import BaseTest, ClickhouseTestMixin++from django.core.management import call_command+from django.utils.timezone import now++from posthog.clickhouse.client import sync_execute+from posthog.models.activity_logging.activity_log import ActivityLog, Detail+from posthog.models.person.divergence.inspection import hash_distinct_id, inspect_mappings, inspect_persons+from posthog.models.person.sql import BULK_INSERT_PERSON_DISTINCT_ID2+from posthog.models.person.util import (+ create_person as create_person_in_ch,+ tombstone_persons_in_postgres,+)+from posthog.models.signals import mute_selected_signals+from posthog.personhog_client.fake_client import get_active_fake+from posthog.test.persons import add_distinct_id, create_person+++class TestDivergenceInspection(ClickhouseTestMixin, BaseTest):+ def setUp(self) -> None:+ super().setUp()+ # A background merge collapses the duplicate and older rows the tie and blanked shapes are made of.+ for table in ("person", "person_distinct_id2"):+ sync_execute(f"SYSTEM STOP MERGES {table}")+ self.addCleanup(sync_execute, f"SYSTEM START MERGES {table}")++ def _pg_person(self, *, version: int, distinct_ids: list[str] | None = None):+ with mute_selected_signals():+ person = create_person(team=self.team, version=version, properties={"email": "[email protected]"})+ for distinct_id in distinct_ids or []:+ add_distinct_id(person=person, distinct_id=distinct_id, version=0)+ return person++ def _ch_person(self, person_uuid: str, version: int, properties: dict, *, deleted: bool = False) -> None:+ create_person_in_ch(+ team_id=self.team.pk,+ uuid=person_uuid,+ version=version,+ is_deleted=deleted,+ properties=properties,+ timestamp=now(),+ )++ def _ch_mapping(self, distinct_id: str, person_uuid: str, version: int, *, team_id: int | None = None) -> None:+ sync_execute(+ BULK_INSERT_PERSON_DISTINCT_ID2,+ [+ {+ "distinct_id": distinct_id,+ "person_id": person_uuid,+ "team_id": team_id or self.team.pk,+ "is_deleted": 0,+ "version": version,+ "_timestamp": now().replace(tzinfo=None),+ "_offset": 0,+ "_partition": 0,+ }+ ],+ flush=False,+ )++ def _log_person_activity(self, activity: str, *, item_id: int, name: str | None) -> None:+ ActivityLog.objects.create(+ organization_id=self.organization.id,+ team_id=self.team.pk,+ item_id=str(item_id),+ scope="Person",+ activity=activity,+ detail=Detail(name=name),+ )++ def _flavors(self, inspections) -> dict[str, tuple[str, str | None, str | None]]:+ return {i.key: (i.flavor, i.sub_flavor, i.verb) for i in inspections}++ def test_inspect_persons_reads_both_stores_into_each_flavor(self):+ tie = self._pg_person(version=2)+ self._ch_person(str(tie.uuid), 2, {"email": "[email protected]"})+ self._ch_person(str(tie.uuid), 2, {"email": "[email protected]"})+ blanked = self._pg_person(version=105)+ self._ch_person(str(blanked.uuid), 1, {"email": "[email protected]"})+ self._ch_person(str(blanked.uuid), 105, {})+ deleted = self._pg_person(version=2)+ tombstone_persons_in_postgres(self.team.pk, [deleted.uuid])+ self._ch_person(str(deleted.uuid), 7, {"email": "[email protected]"})+ stale = self._pg_person(version=1)+ self._ch_person(str(stale.uuid), 40, {"email": "[email protected]"})+ ghost = str(uuid4())+ self._ch_person(ghost, 3, {"email": "[email protected]"})++ self._log_person_activity("deleted", item_id=deleted.pk, name=str(deleted.uuid))+ self._log_person_activity("updated", item_id=stale.pk, name=None)+ self._log_person_activity("updated", item_id=tie.pk, name=None)++ inspections = inspect_persons(+ self.team.pk, [str(p.uuid) for p in (tie, blanked, deleted)] + [str(stale.uuid).upper(), ghost]+ )+ by_key = {i.key: i for i in inspections}+ flavors = self._flavors(inspections)++ assert [(r["version"], r["properties_bytes"] > 2) for r in by_key[str(blanked.uuid)].ch_history or []] == [+ (1, True),+ (105, False),+ ]+ assert [a["activity"] for a in by_key[str(deleted.uuid)].activity or []] == ["deleted"]+ assert [a["activity"] for a in by_key[str(stale.uuid)].activity or []] == ["updated"]+ assert by_key[ghost].activity == []+ queued = by_key[str(deleted.uuid)].tombstone_queue+ assert queued is not None and queued["person_version"] == 3 and queued["tombstoned_at_ms"] > 0+ assert by_key[str(stale.uuid)].tombstone_queue is None+ assert flavors == {+ str(tie.uuid): ("tie", None, "republish"),+ str(blanked.uuid): ("blanked", None, "republish"),+ str(deleted.uuid): ("deleted_visible", None, "bury"),+ str(stale.uuid): ("stale", None, "merge_stale"),+ ghost: ("ghost", "historical", None),+ }++ def test_inspect_mappings_dates_owners_and_finds_strays(self):+ self._pg_person(version=0, distinct_ids=["moved"])+ old_owner = str(uuid4())+ self._ch_mapping("moved", old_owner, 5)+ fake = get_active_fake()+ assert fake is not None+ deleted = fake.add_person(+ team_id=self.team.pk,+ person_id=990_001,+ uuid=str(uuid4()),+ version=3,+ is_deleted=True,+ distinct_ids=["stray"],+ )+ self._ch_mapping("stray", deleted.uuid, 0)++ flavors = self._flavors(inspect_mappings(self.team.pk, ["moved", "stray"]))++ assert flavors == {+ "moved": ("diverged", "a_absent", "fold"),+ "stray": ("stray", None, None),+ }++ def test_inspect_mappings_leaves_ghosts_of_a_deleted_team_alone(self):+ gone_team_id = self.team.pk + 10_000+ self._ch_mapping("gone", str(uuid4()), 4, team_id=gone_team_id)++ assert self._flavors(inspect_mappings(gone_team_id, ["gone"])) == {"gone": ("ghost", "team_gone", None)}++ def test_command_prints_one_json_row_per_key_with_hashed_distinct_ids(self):+ person = self._pg_person(version=2, distinct_ids=["secret-did-7f3a"])+ self._ch_person(str(person.uuid), 2, {"email": "[email protected]"})+ self._ch_mapping("secret-did-7f3a", str(person.uuid), 0)+ out = StringIO()++ call_command(+ "person_inspect",+ "--team-id",+ str(self.team.pk),+ "--person-uuid",+ str(person.uuid),+ "--distinct-id",+ "secret-did-7f3a",+ stdout=out,+ )++ rows = [json.loads(line) for line in out.getvalue().splitlines()]+ assert [(r["kind"], r["key"], r["flavor"]) for r in rows] == [+ ("person", str(person.uuid), "ok"),+ ("mapping", hash_distinct_id(self.team.pk, "secret-did-7f3a"), "ok"),+ ]+ assert "secret-did-7f3a" not in out.getvalue()diff --git a/posthog/models/person/util.py b/posthog/models/person/util.pyindex 31e9e020034d..202f404379f3 100644--- a/posthog/models/person/util.py+++ b/posthog/models/person/util.py@@ -39,11 +39,14 @@ from posthog.personhog_client.converters import proto_person_to_modelfrom posthog.personhog_client.interceptor import is_transient_rpc_errorfrom posthog.personhog_client.metrics import PERSONHOG_TEAM_MISMATCH_TOTAL, get_client_namefrom posthog.personhog_client.proto import (+ CONSISTENCY_LEVEL_STRONG,AckedPersonTombstone,AckPersonTombstonesRequest,DeletePersonsMode,DeletePersonsRequest,+ DistinctIdRow,EnsurePersonVersionFloorsRequest,+ GetDistinctIdRowsRequest,GetDistinctIdsForPersonRequest,GetDistinctIdsForPersonsRequest,GetPersonByDistinctIdRequest,@@ -52,7 +55,9 @@ from posthog.personhog_client.proto import (GetPersonsByDistinctIdsInTeamRequest,GetPersonsByUuidsRequest,GetPersonTombstonesRequest,+ ListPersonsForTeamRequest,ListPersonTombstoneQueueRequest,+ PersonListEntry,PersonVersionFloor as PersonVersionFloorProto,ReadOptions,SetPersonDistinctIdVersionFloorRequest,@@ -774,6 +779,40 @@ def get_person_tombstones(team_id: int, person_uuids: list[UUID]) -> list[Personreturn personhog_call("get_person_tombstones", personhog_fn)+def get_distinct_id_rows(team_id: int, distinct_ids: list[str], *, strong: bool = False) -> list[DistinctIdRow]:+ """Mapping rows, live or tombstoned, with their owner's state. `strong` reads the primary."""++ def personhog_fn() -> list[DistinctIdRow]:+ read_options = ReadOptions(consistency=CONSISTENCY_LEVEL_STRONG) if strong else None+ rows: list[DistinctIdRow] = []+ for i in range(0, len(distinct_ids), 1000):+ response = _get_client().get_distinct_id_rows(+ GetDistinctIdRowsRequest(+ team_id=team_id, distinct_ids=distinct_ids[i : i + 1000], read_options=read_options+ )+ )+ rows.extend(response.rows)+ return rows++ return personhog_call("get_distinct_id_rows", personhog_fn)+++def list_persons_for_team(+ team_id: int, after_person_id: int, *, before_person_id: int | None = None, limit: int = 1000+) -> tuple[list[PersonListEntry], int | None]:+ """One id-ordered page of a team's persons, live and tombstoned, and the cursor for the next page."""++ def personhog_fn() -> tuple[list[PersonListEntry], int | None]:+ request = ListPersonsForTeamRequest(team_id=team_id, after_person_id=after_person_id, limit=limit)+ if before_person_id is not None:+ request.before_person_id = before_person_id+ response = _get_client().list_persons_for_team(request)+ next_after = response.next_after_person_id if response.HasField("next_after_person_id") else None+ return list(response.persons), next_after++ return personhog_call("list_persons_for_team", personhog_fn)++def list_person_tombstone_queue(after: tuple[int, UUID] | None, limit: int, team_id: int | None = None) -> list[QueuedPersonTombstone]: