:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(persons): detach strays and split distinct ids with person_consistency (#126688)
Eli Reismanb6e9c8e8c64b
diff --git a/posthog/dags/clickhouse_cleanup.py b/posthog/dags/clickhouse_cleanup.pyindex 5657cc688ab2..bc40d9b20529 100644--- a/posthog/dags/clickhouse_cleanup.py+++ b/posthog/dags/clickhouse_cleanup.py@@ -54,12 +54,11 @@ from posthog.dags.person_tombstone_queue import publish_queue_gauges, resolve_pefrom posthog.dataclasses import frozenfrom posthog.metrics import pushed_metrics_registryfrom posthog.models.async_deletion.delete_cohorts import sweep_cohort_deletions+from posthog.models.person.divergence.cleanup_queue import PG_CLEANUP_QUEUE_TABLEfrom posthog.models.person.sql import PERSON_DISTINCT_ID2_TABLE, PERSONS_TABLElogger = dagster.get_dagster_logger(__name__)-PG_CLEANUP_QUEUE_TABLE = "person_pg_cleanup_queue"-# How many queued persons travel per ClickHouse read and per Postgres upsert transaction. Bounds# the op's memory to one page however large the snapshot is.PERSIST_PAGE_SIZE = 50_000diff --git a/posthog/dags/detach_distinct_id.py b/posthog/dags/detach_distinct_id.pydeleted file mode 100644index 93f4498c86bc..000000000000--- a/posthog/dags/detach_distinct_id.py+++ /dev/null@@ -1,338 +0,0 @@-"""Dagster job to detach a distinct_id from its person.--Three cleanup phases, all required:- 1. Postgres: tombstone the posthog_persondistinctid row (stops future ingestion lookups).- The tombstone version is above both the row's own version and the highest version ClickHouse- holds for the distinct id, so the ClickHouse tombstone and override win, and a later re-add- revives the row above them instead of starting at version 0 underneath them.- 2. Kafka: publish is_deleted to person_distinct_id2 at that exact version (stops ClickHouse lookups).- 3. Override: insert into person_distinct_id_overrides so the HogQL query layer- immediately re-attributes historical events whose person_id was baked in at- ingestion time (Person-on-Events). squash_person_overrides later makes it- permanent and removes the override row.--When no ``override_person_id`` is supplied a random dummy UUID is used, effectively-orphaning the events (the dummy person has no record in the person table).--Typical use: removing a sentinel ``$posthog_cookieless`` distinct_id that was-erroneously associated with a real person due to a cookieless-ingestion bug.--Example Dagster launchpad config (run with dry_run: true first!)::-- ops:- detach_distinct_id_op:- config:- team_id: 12345- distinct_id: "$posthog_cookieless"- expected_person_id: "5e00024e-cb68-59f6-821f-6150fcffc431"- dry_run: true-"""--import json-import uuid-import shlex-from datetime import datetime--import dagster-import pydantic-import psycopg2.extensions--from posthog.clickhouse.client import sync_execute-from posthog.dags.common import JobOwners-from posthog.kafka_client.client import _KafkaProducer-from posthog.kafka_client.topics import KAFKA_PERSON_DISTINCT_ID--KAFKA_DELIVERY_TIMEOUT_SECONDS = 30---class DetachDistinctIdConfig(dagster.Config):- """Configuration for the detach distinct_id job."""-- team_id: int = pydantic.Field(description="Team ID that owns the distinct_id")- distinct_id: str = pydantic.Field(description="The distinct_id to detach (e.g. $posthog_cookieless)")- expected_person_id: str = pydantic.Field(- description="UUID of the person we expect the distinct_id to belong to (safety check)"- )- override_person_id: str | None = pydantic.Field(- default=None,- description=(- "If set, insert a person_distinct_id_overrides row pointing historical events "- "at this person UUID. If not set, a random dummy UUID is generated so events "- "become orphaned. The scheduled squash job will later rewrite the embedded "- "person_id on events and delete the override."- ),- )- dry_run: bool = pydantic.Field(- default=True,- description="If true, log what would happen without making changes",- )---def _lookup_distinct_id(- cursor: psycopg2.extensions.cursor,- team_id: int,- distinct_id: str,-) -> dict | None:- """Return the posthog_persondistinctid row + person uuid, or None."""- cursor.execute(- """- SELECT pdi.id, pdi.version, pdi.person_id, p.uuid- FROM posthog_persondistinctid pdi- JOIN posthog_person p ON p.id = pdi.person_id AND p.team_id = pdi.team_id- WHERE pdi.team_id = %s AND pdi.distinct_id = %s- AND pdi.is_deleted = false AND p.is_deleted = false- """,- [team_id, distinct_id],- )- row = cursor.fetchone()- if row is None:- return None- # Handle both dict rows (RealDictCursor) and tuple rows (regular cursor)- if isinstance(row, dict):- return {- "pdi_id": row["id"],- "version": row["version"],- "person_pk": row["person_id"],- "person_uuid": str(row["uuid"]),- }- return {- "pdi_id": row[0],- "version": row[1],- "person_pk": row[2],- "person_uuid": str(row[3]),- }---def _count_other_distinct_ids(- cursor: psycopg2.extensions.cursor,- team_id: int,- person_pk: int,- exclude_pdi_id: int,-) -> int:- """Count how many *other* distinct_ids the person has."""- cursor.execute(- """- SELECT COUNT(*) AS count- FROM posthog_persondistinctid- WHERE team_id = %s AND person_id = %s AND id != %s AND is_deleted = false- """,- [team_id, person_pk, exclude_pdi_id],- )- row = cursor.fetchone()- return row["count"] if isinstance(row, dict) else row[0]---def _clickhouse_max_version(team_id: int, distinct_id: str) -> int:- """Return the highest ClickHouse version of the distinct id across the mapping and override tables, or 0."""- rows = sync_execute(- """- SELECT greatest(- (SELECT max(version) FROM person_distinct_id2- WHERE team_id = %(team_id)s AND distinct_id = %(distinct_id)s),- (SELECT max(version) FROM person_distinct_id_overrides- WHERE team_id = %(team_id)s AND distinct_id = %(distinct_id)s)- )- """,- {"team_id": team_id, "distinct_id": distinct_id},- )- return int(rows[0][0])---def _tombstone_distinct_id_row(- cursor: psycopg2.extensions.cursor,- pdi_id: int,- person_pk: int,- min_version: int,-) -> int:- """Tombstone the row at ``min_version`` or above while it still belongs to ``person_pk``; return its version."""- cursor.execute(- """- UPDATE posthog_persondistinctid- SET is_deleted = true, version = GREATEST(COALESCE(version, 0) + 1, %s)- WHERE id = %s AND person_id = %s AND is_deleted = false- RETURNING version- """,- [min_version, pdi_id, person_pk],- )- row = cursor.fetchone()- if row is None:- raise RuntimeError(- f"posthog_persondistinctid id={pdi_id} disappeared or moved off person {person_pk} between lookup and tombstone"- )- return row["version"] if isinstance(row, dict) else row[0]---def _publish_deletion_to_kafka(- producer: _KafkaProducer,- team_id: int,- distinct_id: str,- person_uuid: str,- version: int,-) -> None:- """Publish an is_deleted message so ClickHouse person_distinct_id2 drops the mapping.-- ``version`` is the one the Postgres tombstone carries, so a revived mapping lands above it.- """- result = producer.produce(- topic=KAFKA_PERSON_DISTINCT_ID,- data={- "distinct_id": distinct_id,- "person_id": person_uuid,- "team_id": team_id,- "version": version,- "is_deleted": 1,- },- )- producer.flush(KAFKA_DELIVERY_TIMEOUT_SECONDS)- # flush() counts only messages still queued. A failed delivery leaves the queue, so only its result shows it.- result.get(timeout=0)---def _undelivered_tombstone_message(team_id: int, distinct_id: str, override_person_uuid: str, version: int) -> str:- insert_override = (- "from posthog.dags.detach_distinct_id import _insert_ch_override; "- f"_insert_ch_override({team_id}, {json.dumps(distinct_id)}, {json.dumps(override_person_uuid)}, {version + 1})"- )- return (- f"Postgres tombstoned distinct_id={distinct_id!r} at version {version}, but Kafka did not deliver the "- "ClickHouse tombstone, so the override was not inserted. A rerun of this job cannot find the row. "- "Recover with:\n"- f" 1. python manage.py sync_persons_to_clickhouse --person-distinct-id --deletes --live-run --team-id {team_id}\n"- f" 2. python manage.py shell -c {shlex.quote(insert_override)}"- )---def _insert_ch_override(- team_id: int,- distinct_id: str,- override_person_uuid: str,- version: int,-) -> None:- """Insert a person_distinct_id_overrides row so the squash job re-attributes historical events.-- Follows the same pattern as insert_override_batch in fix_person_id_overrides.py.- """- sync_execute(- """- INSERT INTO person_distinct_id_overrides- (team_id, distinct_id, person_id, is_deleted, version, _timestamp, _offset, _partition)- VALUES- """,- [(team_id, distinct_id, override_person_uuid, 0, version, datetime.now(), 0, 0)],- )---def detach_distinct_id_op(- context: dagster.OpExecutionContext,- config: DetachDistinctIdConfig,- persons_database: dagster.ResourceParam[psycopg2.extensions.connection],- kafka_producer: dagster.ResourceParam[_KafkaProducer],-) -> None:- """Detach a distinct_id from its person in Postgres and ClickHouse."""- log = context.log-- log.info(f"team_id={config.team_id} distinct_id={config.distinct_id!r}")- log.info(f"expected_person_id={config.expected_person_id}")- log.info(f"dry_run={config.dry_run}")-- # --- 1. Validate in Postgres ---- with persons_database.cursor() as cursor:- cursor.execute("SET application_name = 'detach_distinct_id'")- cursor.execute("SET lock_timeout = '5s'")- cursor.execute("SET statement_timeout = '30s'")-- info = _lookup_distinct_id(cursor, config.team_id, config.distinct_id)- if info is None:- raise dagster.Failure(f"distinct_id={config.distinct_id!r} not found for team_id={config.team_id}")-- log.info(- f"Found: pdi_id={info['pdi_id']} version={info['version']} "- f"person_uuid={info['person_uuid']} person_pk={info['person_pk']}"- )-- if info["person_uuid"] != config.expected_person_id:- raise dagster.Failure(- f"Person mismatch: distinct_id belongs to {info['person_uuid']}, expected {config.expected_person_id}"- )-- other_count = _count_other_distinct_ids(cursor, config.team_id, info["person_pk"], info["pdi_id"])- if other_count == 0:- raise dagster.Failure(- f"Cannot detach: this is the person's only distinct_id. "- f"Detaching would orphan person {info['person_uuid']}"- )- log.info(f"Person has {other_count} other distinct_id(s) — safe to detach")-- # --- 2. Resolve override target ---- override_target = config.override_person_id or str(uuid.uuid4())- log.info(f"Override target person_id={override_target}")-- # --- 3. Tombstone in Postgres ---- if config.dry_run:- log.info("[DRY RUN] Would tombstone posthog_persondistinctid row")- log.info("[DRY RUN] Would publish Kafka deletion to person_distinct_id2")- log.info(f"[DRY RUN] Would insert person_distinct_id_overrides -> {override_target}")- persons_database.rollback()- return-- ch_max_version = _clickhouse_max_version(config.team_id, config.distinct_id)- version = _tombstone_distinct_id_row(- cursor, info["pdi_id"], person_pk=info["person_pk"], min_version=ch_max_version + 1- )- persons_database.commit()- log.info(- f"Tombstoned posthog_persondistinctid id={info['pdi_id']} "- f"(version={version}, ClickHouse max version={ch_max_version})"- )-- # --- 4. Sync deletion to ClickHouse via Kafka ---- try:- _publish_deletion_to_kafka(- kafka_producer,- team_id=config.team_id,- distinct_id=config.distinct_id,- person_uuid=info["person_uuid"],- version=version,- )- except Exception as e:- raise dagster.Failure(- _undelivered_tombstone_message(config.team_id, config.distinct_id, override_target, version)- ) from e- log.info(f"Published deletion to {KAFKA_PERSON_DISTINCT_ID} (version={version}, is_deleted=1)")-- # --- 5. Override: fix person_id baked into historical events (see module docstring) ---- _insert_ch_override(- team_id=config.team_id,- distinct_id=config.distinct_id,- override_person_uuid=override_target,- version=version + 1,- )- log.info(f"Inserted person_distinct_id_overrides: {config.distinct_id!r} -> {override_target}")-- # --- 6. Verification hints ---- log.info("--- Verification queries (run manually) ---")- escaped_did = json.dumps(config.distinct_id)- log.info(- f"Postgres: SELECT * FROM posthog_persondistinctid "- f"WHERE team_id = {config.team_id} AND distinct_id = {escaped_did}"- )- log.info(- f"ClickHouse (pdi2): SELECT distinct_id, argMax(person_id, version), "- f"argMax(is_deleted, version), max(version) "- f"FROM person_distinct_id2 "- f"WHERE team_id = {config.team_id} AND distinct_id = {escaped_did} "- f"GROUP BY distinct_id"- )- log.info(- f"ClickHouse (overrides): SELECT distinct_id, argMax(person_id, version), max(version) "- f"FROM person_distinct_id_overrides "- f"WHERE team_id = {config.team_id} AND distinct_id = {escaped_did} "- f"GROUP BY distinct_id"- )--[email protected](tags={"owner": JobOwners.TEAM_INGESTION.value})-def detach_distinct_id_job():- """Job to detach a distinct_id from its person in Postgres and ClickHouse."""- detach_distinct_id_op()diff --git a/posthog/dags/locations/ingestion.py b/posthog/dags/locations/ingestion.pyindex fd4a03b2294e..13fb3456f155 100644--- a/posthog/dags/locations/ingestion.py+++ b/posthog/dags/locations/ingestion.py@@ -1,6 +1,6 @@import dagster-from posthog.dags import detach_distinct_id, distinct_id_usage, ingestion_assets, person_property_reconciliation+from posthog.dags import distinct_id_usage, ingestion_assets, person_property_reconciliationfrom . import loggers, resources@@ -9,7 +9,6 @@ defs = dagster.Definitions(ingestion_assets.postgres_env_check,],jobs=[- detach_distinct_id.detach_distinct_id_job,distinct_id_usage.distinct_id_usage_monitoring,person_property_reconciliation.person_property_reconciliation_job,],diff --git a/posthog/dags/owners.yaml b/posthog/dags/owners.yamlindex ab44c8654ff1..65a0f1eec679 100644--- a/posthog/dags/owners.yaml+++ b/posthog/dags/owners.yaml@@ -2,7 +2,6 @@ version: 1owners: []rules:- match:- - '/detach_distinct_id.py'- '/fix_person_id_overrides.py'- '/person_property_reconciliation.py'- '/tests/test_person_property_reconciliation.py'diff --git a/posthog/dags/tests/test_detach_distinct_id.py b/posthog/dags/tests/test_detach_distinct_id.pydeleted file mode 100644index f4102bcb4f5a..000000000000--- a/posthog/dags/tests/test_detach_distinct_id.py+++ /dev/null@@ -1,618 +0,0 @@-import uuid as uuid_module-import contextlib-from datetime import datetime-from uuid import UUID--import pytest-from unittest.mock import MagicMock, patch--from confluent_kafka import KafkaError, KafkaException--from posthog.clickhouse.client import sync_execute-from posthog.dags.detach_distinct_id import (- _count_other_distinct_ids,- _insert_ch_override,- _lookup_distinct_id,- _publish_deletion_to_kafka,- _tombstone_distinct_id_row,- detach_distinct_id_job,-)-from posthog.kafka_client.topics import KAFKA_PERSON_DISTINCT_ID-from posthog.models.person.sql import (- TRUNCATE_PERSON_DISTINCT_ID2_TABLE_SQL,- TRUNCATE_PERSON_DISTINCT_ID_OVERRIDES_TABLE_SQL,-)-from posthog.models.person.util import create_person_distinct_id-from posthog.persons_db import persons_db_connection--PERSON_UUID = "5e00024e-cb68-59f6-821f-6150fcffc431"-PERSON_PK = 42-PDI_ID = 99-PDI_VERSION = 3-TEAM_ID = 7-DUMMY_OVERRIDE_UUID = "00000000-0000-0000-0000-000000000001"--# The integration class writes persons to the persons DB directly, so it must run against the real-# persons DB rather than the personhog fake.-pytestmark = pytest.mark.persons_db_direct---def _make_cursor(fetchone_values: list | None = None) -> MagicMock:- cursor = MagicMock()- if fetchone_values is not None:- cursor.fetchone.side_effect = fetchone_values- return cursor---def _override_insert_rows(mock_sync_execute: MagicMock) -> list:- [insert] = [c for c in mock_sync_execute.call_args_list if "INSERT INTO person_distinct_id_overrides" in c.args[0]]- return insert.args[1]---class TestLookupDistinctId:- def test_returns_row_when_found_tuple(self):- cursor = _make_cursor(fetchone_values=[(PDI_ID, PDI_VERSION, PERSON_PK, UUID(PERSON_UUID))])-- result = _lookup_distinct_id(cursor, TEAM_ID, "$posthog_cookieless")-- assert result is not None- assert result["pdi_id"] == PDI_ID- assert result["version"] == PDI_VERSION- assert result["person_pk"] == PERSON_PK- assert result["person_uuid"] == PERSON_UUID-- def test_returns_row_when_found_dict(self):- cursor = _make_cursor(- fetchone_values=[{"id": PDI_ID, "version": PDI_VERSION, "person_id": PERSON_PK, "uuid": UUID(PERSON_UUID)}]- )-- result = _lookup_distinct_id(cursor, TEAM_ID, "$posthog_cookieless")-- assert result is not None- assert result["pdi_id"] == PDI_ID- assert result["version"] == PDI_VERSION- assert result["person_pk"] == PERSON_PK- assert result["person_uuid"] == PERSON_UUID-- def test_returns_none_when_not_found(self):- cursor = _make_cursor(fetchone_values=[None])-- result = _lookup_distinct_id(cursor, TEAM_ID, "nonexistent")-- assert result is None---class TestCountOtherDistinctIds:- def test_returns_count_tuple(self):- cursor = _make_cursor(fetchone_values=[(5,)])-- count = _count_other_distinct_ids(cursor, TEAM_ID, PERSON_PK, PDI_ID)-- assert count == 5-- def test_returns_count_dict(self):- cursor = _make_cursor(fetchone_values=[{"count": 5}])-- count = _count_other_distinct_ids(cursor, TEAM_ID, PERSON_PK, PDI_ID)-- assert count == 5-- def test_returns_zero_when_none(self):- cursor = _make_cursor(fetchone_values=[(0,)])-- count = _count_other_distinct_ids(cursor, TEAM_ID, PERSON_PK, PDI_ID)-- assert count == 0---class TestTombstoneDistinctIdRow:- @pytest.mark.parametrize("row", [(PDI_VERSION + 1,), {"version": PDI_VERSION + 1}])- def test_tombstones_the_row_and_returns_the_stamped_version(self, row):- cursor = _make_cursor(fetchone_values=[row])-- version = _tombstone_distinct_id_row(cursor, PDI_ID, person_pk=PERSON_PK, min_version=1)-- assert version == PDI_VERSION + 1- sql = cursor.execute.call_args.args[0]- assert "UPDATE posthog_persondistinctid" in sql- assert "is_deleted = true" in sql- assert "RETURNING version" in sql- assert "DELETE" not in sql-- def test_raises_if_row_disappears(self):- cursor = _make_cursor(fetchone_values=[None])-- with pytest.raises(RuntimeError, match="disappeared"):- _tombstone_distinct_id_row(cursor, PDI_ID, person_pk=PERSON_PK, min_version=1)---class TestPublishDeletionToKafka:- def test_publishes_at_the_tombstone_version(self):- producer = MagicMock()-- _publish_deletion_to_kafka(producer, TEAM_ID, "$posthog_cookieless", PERSON_UUID, PDI_VERSION + 1)-- producer.produce.assert_called_once_with(- topic=KAFKA_PERSON_DISTINCT_ID,- data={- "distinct_id": "$posthog_cookieless",- "person_id": PERSON_UUID,- "team_id": TEAM_ID,- "version": PDI_VERSION + 1,- "is_deleted": 1,- },- )- producer.flush.assert_called_once()---class TestInsertChOverride:- @patch("posthog.dags.detach_distinct_id.sync_execute")- def test_inserts_override_row(self, mock_sync_execute):- _insert_ch_override(TEAM_ID, "$posthog_cookieless", DUMMY_OVERRIDE_UUID, PDI_VERSION + 1)-- mock_sync_execute.assert_called_once()- sql = mock_sync_execute.call_args.args[0]- assert "person_distinct_id_overrides" in sql-- rows = mock_sync_execute.call_args.args[1]- assert len(rows) == 1- row = rows[0]- assert row[0] == TEAM_ID- assert row[1] == "$posthog_cookieless"- assert row[2] == DUMMY_OVERRIDE_UUID- assert row[3] == 0 # is_deleted- assert row[4] == PDI_VERSION + 1 # version---class TestDetachDistinctIdJob:- """Run the full Dagster job with mock Postgres (psycopg2), Kafka, and ClickHouse- (sync_execute). Verifies the three cleanup phases only fire when all safety checks pass."""-- def _make_connection(self, lookup_row, other_count, tombstone_version=PDI_VERSION + 1):- """Build a mock psycopg2 connection with a cursor that responds to the- queries issued by detach_distinct_id_op in order."""- conn = MagicMock(spec=["cursor", "commit", "rollback"])- cursor = MagicMock()-- # The op issues fetchone calls in this order:- # 1. _lookup_distinct_id- # 2. _count_other_distinct_ids- # 3. _tombstone_distinct_id_row UPDATE ... RETURNING (only when not dry_run)- fetchone_sequence = [lookup_row, (other_count,)]- if tombstone_version is not None:- fetchone_sequence.append((tombstone_version,))-- cursor.fetchone.side_effect = fetchone_sequence- cursor.__enter__ = MagicMock(return_value=cursor)- cursor.__exit__ = MagicMock(return_value=False)- conn.cursor.return_value = cursor- return conn, cursor-- def _run_config(self, *, dry_run: bool = True, override_person_id: str | None = None) -> dict:- config: dict = {- "team_id": TEAM_ID,- "distinct_id": "$posthog_cookieless",- "expected_person_id": PERSON_UUID,- "dry_run": dry_run,- }- if override_person_id is not None:- config["override_person_id"] = override_person_id- return {"ops": {"detach_distinct_id_op": {"config": config}}}-- @patch("posthog.dags.detach_distinct_id.sync_execute")- def test_dry_run_does_not_mutate(self, mock_sync_execute):- lookup_row = (PDI_ID, PDI_VERSION, PERSON_PK, UUID(PERSON_UUID))- conn, cursor = self._make_connection(lookup_row, other_count=2)- producer = MagicMock()-- result = detach_distinct_id_job.execute_in_process(- run_config=self._run_config(dry_run=True),- resources={"persons_database": conn, "kafka_producer": producer},- )-- assert result.success- conn.commit.assert_not_called()- conn.rollback.assert_called_once()- producer.produce.assert_not_called()- mock_sync_execute.assert_not_called()-- @patch("posthog.dags.detach_distinct_id.sync_execute")- @patch("posthog.dags.detach_distinct_id.uuid.uuid4", return_value=UUID(DUMMY_OVERRIDE_UUID))- def test_detaches_publishes_kafka_and_inserts_override(self, _mock_uuid4, mock_sync_execute):- mock_sync_execute.return_value = [(0,)]- lookup_row = (PDI_ID, PDI_VERSION, PERSON_PK, UUID(PERSON_UUID))- conn, cursor = self._make_connection(lookup_row, other_count=2)- producer = MagicMock()-- result = detach_distinct_id_job.execute_in_process(- run_config=self._run_config(dry_run=False),- resources={"persons_database": conn, "kafka_producer": producer},- )-- assert result.success- conn.commit.assert_called_once()-- # Kafka deletion- producer.produce.assert_called_once()- kafka_data = producer.produce.call_args.kwargs["data"]- assert kafka_data["distinct_id"] == "$posthog_cookieless"- assert kafka_data["person_id"] == PERSON_UUID- assert kafka_data["is_deleted"] == 1- assert kafka_data["version"] == PDI_VERSION + 1- producer.flush.assert_called_once()-- # ClickHouse override- override_rows = _override_insert_rows(mock_sync_execute)- assert override_rows[0][0] == TEAM_ID- assert override_rows[0][1] == "$posthog_cookieless"- assert override_rows[0][2] == DUMMY_OVERRIDE_UUID- assert override_rows[0][4] == PDI_VERSION + 2-- @patch("posthog.dags.detach_distinct_id.sync_execute")- def test_a_failed_kafka_delivery_fails_the_job_before_the_override(self, mock_sync_execute):- mock_sync_execute.return_value = [(0,)]- lookup_row = (PDI_ID, PDI_VERSION, PERSON_PK, UUID(PERSON_UUID))- conn, _cursor = self._make_connection(lookup_row, other_count=2)- producer = MagicMock()- producer.produce.return_value.get.side_effect = KafkaException(KafkaError(KafkaError._MSG_TIMED_OUT)) # type: ignore[attr-defined]-- result = detach_distinct_id_job.execute_in_process(- run_config=self._run_config(dry_run=False),- resources={"persons_database": conn, "kafka_producer": producer},- raise_on_error=False,- )-- assert not result.success- conn.commit.assert_called_once()- assert not [- c for c in mock_sync_execute.call_args_list if "INSERT INTO person_distinct_id_overrides" in c.args[0]- ]-- @patch("posthog.dags.detach_distinct_id.sync_execute")- def test_uses_explicit_override_person_id(self, mock_sync_execute):- mock_sync_execute.return_value = [(0,)]- explicit_uuid = "11111111-2222-3333-4444-555555555555"- lookup_row = (PDI_ID, PDI_VERSION, PERSON_PK, UUID(PERSON_UUID))- conn, cursor = self._make_connection(lookup_row, other_count=2)- producer = MagicMock()-- result = detach_distinct_id_job.execute_in_process(- run_config=self._run_config(dry_run=False, override_person_id=explicit_uuid),- resources={"persons_database": conn, "kafka_producer": producer},- )-- assert result.success- override_rows = _override_insert_rows(mock_sync_execute)- assert override_rows[0][2] == explicit_uuid-- @patch("posthog.dags.detach_distinct_id.sync_execute")- def test_fails_when_distinct_id_not_found(self, mock_sync_execute):- conn, cursor = self._make_connection(lookup_row=None, other_count=0, tombstone_version=None)- cursor.fetchone.side_effect = [None]- producer = MagicMock()-- result = detach_distinct_id_job.execute_in_process(- run_config=self._run_config(dry_run=False),- resources={"persons_database": conn, "kafka_producer": producer},- raise_on_error=False,- )-- assert not result.success- conn.commit.assert_not_called()- producer.produce.assert_not_called()- mock_sync_execute.assert_not_called()-- @patch("posthog.dags.detach_distinct_id.sync_execute")- def test_fails_when_person_id_mismatch(self, mock_sync_execute):- wrong_person = UUID("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee")- lookup_row = (PDI_ID, PDI_VERSION, PERSON_PK, wrong_person)- conn, cursor = self._make_connection(lookup_row, other_count=2)- producer = MagicMock()-- result = detach_distinct_id_job.execute_in_process(- run_config=self._run_config(dry_run=False),- resources={"persons_database": conn, "kafka_producer": producer},- raise_on_error=False,- )-- assert not result.success- conn.commit.assert_not_called()- producer.produce.assert_not_called()- mock_sync_execute.assert_not_called()-- @patch("posthog.dags.detach_distinct_id.sync_execute")- def test_fails_when_only_distinct_id(self, mock_sync_execute):- lookup_row = (PDI_ID, PDI_VERSION, PERSON_PK, UUID(PERSON_UUID))- conn, cursor = self._make_connection(lookup_row, other_count=0)- producer = MagicMock()-- result = detach_distinct_id_job.execute_in_process(- run_config=self._run_config(dry_run=False),- resources={"persons_database": conn, "kafka_producer": producer},- raise_on_error=False,- )-- assert not result.success- conn.commit.assert_not_called()- producer.produce.assert_not_called()- mock_sync_execute.assert_not_called()--[email protected]_db(transaction=True)-class TestDetachDistinctIdIntegration:- """Integration tests using a real Django Postgres connection.-- Exercises the full Dagster job against real DB rows so cursor return types- (tuple vs dict) are tested against production-like conditions.- Kafka stays mocked. ClickHouse is mocked except in the live-run test, which needs the real version reads.- """-- @pytest.fixture- def organization(self):- from posthog.models import Organization-- return Organization.objects.create(name="Detach Test Org")-- @pytest.fixture- def team(self, organization):- from posthog.models import Team-- return Team.objects.create(organization=organization, name="Detach Test Team")-- @classmethod- def _pdi_state(cls, pdi_id: int) -> tuple[bool, int] | None:- with cls._get_persons_conn() as conn, conn.cursor() as cursor:- cursor.execute("SELECT is_deleted, version FROM posthog_persondistinctid WHERE id = %s", [pdi_id])- row = cursor.fetchone()- return None if row is None else (row[0], row[1])-- @pytest.fixture- def person_with_two_distinct_ids(self, team):- from posthog.models.utils import UUIDT-- person_uuid = UUIDT()- with self._get_persons_conn() as conn, conn.cursor() as cursor:- cursor.execute(- "INSERT INTO posthog_person (team_id, uuid, properties, is_identified, version, created_at, properties_last_updated_at, properties_last_operation) "- "VALUES (%s, %s, '{}', false, 1, NOW(), '{}', '{}') RETURNING id",- [team.id, str(person_uuid)],- )- person_id = cursor.fetchone()[0]- cursor.execute(- "INSERT INTO posthog_persondistinctid (team_id, person_id, distinct_id, version) "- "VALUES (%s, %s, %s, %s) RETURNING id",- [team.id, person_id, "keep_this", 0],- )- pdi_keep_id = cursor.fetchone()[0]- cursor.execute(- "INSERT INTO posthog_persondistinctid (team_id, person_id, distinct_id, version) "- "VALUES (%s, %s, %s, %s) RETURNING id",- [team.id, person_id, "$posthog_cookieless", 3],- )- pdi_detach_id = cursor.fetchone()[0]-- from types import SimpleNamespace-- person = SimpleNamespace(id=person_id, uuid=person_uuid, version=1)- pdi_keep = SimpleNamespace(id=pdi_keep_id, distinct_id="keep_this", version=0)- pdi_detach = SimpleNamespace(id=pdi_detach_id, distinct_id="$posthog_cookieless", version=3)- return person, pdi_keep, pdi_detach-- @staticmethod- def _run_config(- team_id: int,- person_uuid: str,- *,- dry_run: bool,- distinct_id: str = "$posthog_cookieless",- override_person_id: str | None = None,- ) -> dict:- config: dict = {- "team_id": team_id,- "distinct_id": distinct_id,- "expected_person_id": person_uuid,- "dry_run": dry_run,- }- if override_person_id is not None:- config["override_person_id"] = override_person_id- return {"ops": {"detach_distinct_id_op": {"config": config}}}-- @staticmethod- @contextlib.contextmanager- def _get_persons_conn():- # autocommit=False: the dagster job consumes this connection as the `persons_database`- # resource and drives its own commit()/rollback() (dry runs roll back), so it must be- # transactional rather than autocommit. The cursor-only callers commit on block exit.- with persons_db_connection(writer=True) as conn:- yield conn-- @patch("posthog.dags.detach_distinct_id.sync_execute")- def test_dry_run_does_not_tombstone(self, mock_sync_execute, team, person_with_two_distinct_ids):- person, _pdi_keep, pdi_detach = person_with_two_distinct_ids- producer = MagicMock()-- with self._get_persons_conn() as conn:- result = detach_distinct_id_job.execute_in_process(- run_config=self._run_config(team.id, str(person.uuid), dry_run=True),- resources={"persons_database": conn, "kafka_producer": producer},- )-- assert result.success- assert self._pdi_state(pdi_detach.id) == (False, pdi_detach.version)- producer.produce.assert_not_called()- mock_sync_execute.assert_not_called()-- @pytest.mark.parametrize(- "clickhouse_table, expected_version",- [- # (table holding a version-20 row for the distinct id, version the tombstone gets)- (None, 4),- ("person_distinct_id2", 21),- ("person_distinct_id_overrides", 21),- ],- )- def test_live_run_tombstones_above_postgres_and_clickhouse(- self, clickhouse_table, expected_version, team, person_with_two_distinct_ids- ):- person, pdi_keep, pdi_detach = person_with_two_distinct_ids- override_target = str(uuid_module.uuid4())- sync_execute(TRUNCATE_PERSON_DISTINCT_ID2_TABLE_SQL)- sync_execute(TRUNCATE_PERSON_DISTINCT_ID_OVERRIDES_TABLE_SQL())- if clickhouse_table == "person_distinct_id2":- create_person_distinct_id(team.id, pdi_detach.distinct_id, str(person.uuid), version=20)- elif clickhouse_table == "person_distinct_id_overrides":- _insert_ch_override(team.id, pdi_detach.distinct_id, str(uuid_module.uuid4()), version=20)- producer = MagicMock()-- with self._get_persons_conn() as conn:- result = detach_distinct_id_job.execute_in_process(- run_config=self._run_config(- team.id, str(person.uuid), dry_run=False, override_person_id=override_target- ),- resources={"persons_database": conn, "kafka_producer": producer},- )-- assert result.success- assert self._pdi_state(pdi_detach.id) == (True, expected_version)- assert self._pdi_state(pdi_keep.id) == (False, pdi_keep.version)- producer.produce.assert_called_once_with(- topic=KAFKA_PERSON_DISTINCT_ID,- data={- "distinct_id": pdi_detach.distinct_id,- "person_id": str(person.uuid),- "team_id": team.id,- "version": expected_version,- "is_deleted": 1,- },- )- # The overrides MV consumes the same topic, so the published tombstone also lands in the overrides- # table, after the direct insert.- tombstone = producer.produce.call_args.kwargs["data"]- sync_execute(- """- INSERT INTO person_distinct_id_overrides- (team_id, distinct_id, person_id, is_deleted, version, _timestamp, _offset, _partition)- VALUES- """,- [- (- tombstone["team_id"],- tombstone["distinct_id"],- tombstone["person_id"],- tombstone["is_deleted"],- tombstone["version"],- datetime.now(),- 0,- 0,- )- ],- )- override = sync_execute(- """- SELECT person_id, is_deleted, version FROM person_distinct_id_overrides FINAL- WHERE team_id = %(team_id)s AND distinct_id = %(distinct_id)s- """,- {"team_id": team.id, "distinct_id": pdi_detach.distinct_id},- )- assert override == [(UUID(override_target), 0, expected_version + 1)]-- def test_fails_without_tombstoning_when_the_distinct_id_moves_after_the_lookup(- self, team, person_with_two_distinct_ids- ):- person, _pdi_keep, pdi_detach = person_with_two_distinct_ids- with self._get_persons_conn() as conn, conn.cursor() as cursor:- cursor.execute(- "INSERT INTO posthog_person (team_id, uuid, properties, is_identified, version, created_at, properties_last_updated_at, properties_last_operation) "- "VALUES (%s, %s, '{}', false, 0, NOW(), '{}', '{}') RETURNING id",- [team.id, str(uuid_module.uuid4())],- )- other_person_id = cursor.fetchone()[0]-- def move_then_read(team_id: int, distinct_id: str) -> int:- with self._get_persons_conn() as conn, conn.cursor() as cursor:- cursor.execute(- "UPDATE posthog_persondistinctid SET person_id = %s WHERE id = %s", [other_person_id, pdi_detach.id]- )- return 0-- producer = MagicMock()- with (- patch("posthog.dags.detach_distinct_id._clickhouse_max_version", side_effect=move_then_read),- self._get_persons_conn() as conn,- ):- result = detach_distinct_id_job.execute_in_process(- run_config=self._run_config(team.id, str(person.uuid), dry_run=False),- resources={"persons_database": conn, "kafka_producer": producer},- raise_on_error=False,- )-- assert not result.success- assert self._pdi_state(pdi_detach.id) == (False, pdi_detach.version)- producer.produce.assert_not_called()-- @patch("posthog.dags.detach_distinct_id.sync_execute")- def test_fails_on_person_id_mismatch(self, mock_sync_execute, team, person_with_two_distinct_ids):- person, _, _ = person_with_two_distinct_ids- wrong_uuid = str(uuid_module.uuid4())- producer = MagicMock()-- with self._get_persons_conn() as conn:- result = detach_distinct_id_job.execute_in_process(- run_config=self._run_config(team.id, wrong_uuid, dry_run=False),- resources={"persons_database": conn, "kafka_producer": producer},- raise_on_error=False,- )-- assert not result.success- producer.produce.assert_not_called()- mock_sync_execute.assert_not_called()-- @patch("posthog.dags.detach_distinct_id.sync_execute")- def test_fails_when_only_distinct_id(self, mock_sync_execute, team):- from types import SimpleNamespace-- from posthog.models.utils import UUIDT-- person_uuid = UUIDT()- with self._get_persons_conn() as conn, conn.cursor() as cursor:- cursor.execute(- "INSERT INTO posthog_person (team_id, uuid, properties, is_identified, version, created_at, properties_last_updated_at, properties_last_operation) "- "VALUES (%s, %s, '{}', false, 1, NOW(), '{}', '{}') RETURNING id",- [team.id, str(person_uuid)],- )- person_id = cursor.fetchone()[0]- cursor.execute(- "INSERT INTO posthog_persondistinctid (team_id, person_id, distinct_id, version) "- "VALUES (%s, %s, %s, %s)",- [team.id, person_id, "$posthog_cookieless", 3],- )- person = SimpleNamespace(id=person_id, uuid=person_uuid, version=1)- producer = MagicMock()-- with self._get_persons_conn() as conn:- result = detach_distinct_id_job.execute_in_process(- run_config=self._run_config(team.id, str(person.uuid), dry_run=False),- resources={"persons_database": conn, "kafka_producer": producer},- raise_on_error=False,- )-- assert not result.success- producer.produce.assert_not_called()- mock_sync_execute.assert_not_called()-- @patch("posthog.dags.detach_distinct_id.sync_execute")- def test_fails_when_distinct_id_not_found(self, mock_sync_execute, team):- producer = MagicMock()-- with self._get_persons_conn() as conn:- result = detach_distinct_id_job.execute_in_process(- run_config=self._run_config(- team.id, str(uuid_module.uuid4()), dry_run=False, distinct_id="nonexistent"- ),- resources={"persons_database": conn, "kafka_producer": producer},- raise_on_error=False,- )-- assert not result.success- producer.produce.assert_not_called()- mock_sync_execute.assert_not_called()diff --git a/posthog/management/commands/person_consistency.py b/posthog/management/commands/person_consistency.pyindex 7849516e9010..563bb868434b 100644--- a/posthog/management/commands/person_consistency.py+++ b/posthog/management/commands/person_consistency.py@@ -15,6 +15,13 @@ Usage:python manage.py person_consistency repair --dir /tmp/scan-2 --verb republish --apply --limit 10python manage.py person_consistency repair --dir /tmp/scan-2 --verb republish --apply --resume <actions file>+ # Move distinct ids off a live person onto persons of their own (no scan needed):+ python manage.py person_consistency split --team-id 2 --person-uuid <uuid> --distinct-id <id> --apply++ # Republish named persons, or the owners of named distinct ids, from Postgres (no scan needed):+ python manage.py person_consistency republish --team-id 2 --distinct-id <id> --apply+ python manage.py person_consistency republish --team-id 2 --distinct-ids-file <file> --apply+Scans are read-only. A scan that stops resumes with --resume. The report (report.jsonl) holds one row perkey that is not ok, with the raw keys a repair needs; summary.json holds counts and hashed group names, and names no key.A repair writes nothing without --apply, and records one action per key in an actions file.@@ -27,10 +34,13 @@ import argparsefrom datetime import UTC, datetimefrom pathlib import Pathfrom typing import Any+from uuid import UUIDfrom django.core.management.base import BaseCommand, CommandError, CommandParserfrom posthog.clickhouse.query_tagging import Feature, Product, tags_context+from posthog.models.person.divergence.detach import SplitError, split_distinct_ids+from posthog.models.person.divergence.repairs import DEFAULT_MAX_WRITES_PER_SECOND, republish_namedfrom posthog.models.person.divergence.team_scan import (PLAN_FILE,SUMMARY_FILE,@@ -44,6 +54,8 @@ from posthog.models.person.divergence.team_scan import ()from posthog.models.person.divergence.verbs import VERBS, VerbError, run_verb+_SHOWN_KEYS = 20+def _parse_utc(value: str) -> datetime:parsed = datetime.fromisoformat(value)@@ -134,7 +146,7 @@ class Command(BaseCommand):repair.add_argument("--max-calls-per-second",type=float,- default=20.0,+ default=DEFAULT_MAX_WRITES_PER_SECOND,help="Personhog write calls per second (default: %(default)s).",)repair.add_argument(@@ -149,6 +161,35 @@ class Command(BaseCommand):help="bury: tombstone new persons even past the share limit that protects a team needing a restore.",)+ split = actions.add_parser(+ "split",+ help="DRY RUN unless --apply. Move distinct ids off a live person onto persons of their own.",+ )+ split.add_argument("--team-id", type=int, required=True, help="Team of the person.")+ split.add_argument("--person-uuid", required=True, help="The live person that owns the distinct ids.")+ split.add_argument(+ "--distinct-id", action="append", required=True, help="Distinct id to move off the person. Repeatable."+ )+ split.add_argument("--apply", action="store_true", help="Split. Without it the command only checks.")++ republish = actions.add_parser(+ "republish",+ help="DRY RUN unless --apply. Republish named persons, or the owners of named distinct ids, where "+ "ClickHouse disagrees with Postgres.",+ )+ republish.add_argument("--team-id", type=int, required=True, help="Team of the persons.")+ republish.add_argument("--person-uuid", action="append", default=[], help="Person to republish. Repeatable.")+ republish.add_argument(+ "--distinct-id", action="append", default=[], help="Republish the person that owns it. Repeatable."+ )+ republish.add_argument(+ "--distinct-ids-file",+ type=Path,+ default=None,+ help="A file of distinct ids, one per line, read like --distinct-id.",+ )+ republish.add_argument("--apply", action="store_true", help="Write. Without it the command only reports.")+final = actions.add_parser("finalize", help="Join every run's rows into report.jsonl and summary.json.")self._add_dir(final)final.add_argument(@@ -201,11 +242,53 @@ class Command(BaseCommand):self._scan(options)elif options["action"] == "repair":self._repair(options)+ elif options["action"] == "split":+ split_distinct_ids(+ options["team_id"],+ options["person_uuid"],+ options["distinct_id"],+ apply=options["apply"],+ log=self._log,+ )+ elif options["action"] == "republish":+ self._republish(options)else:self._finalize(options["dir"], partial=options["partial"])- except (ScanError, VerbError) as exc:+ except (ScanError, VerbError, SplitError) as exc:raise CommandError(str(exc)) from exc+ def _republish(self, options: dict[str, Any]) -> None:+ distinct_ids = list(options["distinct_id"])+ if options["distinct_ids_file"] is not None:+ # Read as bytes, because text mode turns a carriage return into a newline; only a newline ends an id.+ text = options["distinct_ids_file"].read_bytes().decode()+ distinct_ids.extend(line for line in text.split("\n") if line)+ if not options["person_uuid"] and not distinct_ids:+ raise CommandError("name at least one --person-uuid, --distinct-id or --distinct-ids-file")+ try:+ person_uuids = [str(UUID(uuid)) for uuid in options["person_uuid"]]+ except ValueError as exc:+ raise CommandError(f"--person-uuid must be a uuid: {exc}") from exc+ apply = options["apply"]+ result = republish_named(+ options["team_id"], person_uuids=person_uuids, distinct_ids=distinct_ids, apply=apply, log=self._log+ )+ summary = result.summary+ self._log(+ f"republish {'APPLY' if apply else 'DRY RUN'}: {summary.persons} persons, "+ f"person outcomes {summary.person_outcomes}, mapping outcomes {summary.mapping_outcomes}, "+ f"undelivered {summary.undelivered}"+ )+ if not apply:+ self._log("Dry run: nothing was written. Pass --apply to write.")+ elif result.unconfirmed:+ shown = ", ".join(result.unconfirmed[:_SHOWN_KEYS])+ more = f" and {len(result.unconfirmed) - _SHOWN_KEYS} more" if len(result.unconfirmed) > _SHOWN_KEYS else ""+ raise CommandError(+ f"ClickHouse may still disagree with Postgres for {len(result.unconfirmed)} named keys: "+ f"{shown}{more}. Check them with person_inspect."+ )+def _plan(self, options: dict[str, Any], *, chunks: int) -> None:if options["pg_page_size"] > 10_000:raise CommandError("--pg-page-size must be at most 10000")diff --git a/posthog/models/person/divergence/cleanup_queue.py b/posthog/models/person/divergence/cleanup_queue.pynew file mode 100644index 000000000000..f1e16f65693f--- /dev/null+++ b/posthog/models/person/divergence/cleanup_queue.py@@ -0,0 +1,47 @@+"""The persons Postgres cleanup queue, as the repair tools see it.++The cleanup drain (`posthog/dags/person_pg_cleanup_drain.py`) owns this table and hard-deletes the persons queued in+it. A person that still owns a live distinct id is stamped `blocked_at` and skipped for a retry window. The repair+tools only clear `blocked_at`, once a repair removed what blocked the person, so the next drain run retries it.++The queue is not person data that personhog serves, so it is read and written here directly, as the drain does.+"""++from collections.abc import Sequence+from typing import Any++from posthog.persons_db import persons_db_connection++PG_CLEANUP_QUEUE_TABLE = "person_pg_cleanup_queue"+++def clear_blocked(team_id: int, person_uuids: Sequence[str]) -> int:+ """Clear `blocked_at` on the queued persons, and return how many queue rows changed."""+ if not person_uuids:+ return 0+ with persons_db_connection(writer=True) as conn, conn.cursor() as cursor:+ cursor.execute(+ f"UPDATE {PG_CLEANUP_QUEUE_TABLE} SET blocked_at = NULL WHERE team_id = %s AND person_uuid = ANY(%s::uuid[])",+ [team_id, list(person_uuids)],+ )+ return cursor.rowcount+++def queue_entries(team_id: int, person_uuids: Sequence[str]) -> dict[str, dict[str, Any]]:+ """Each queued person's row by uuid: when the sweep queued it, the version bound, and when the drain blocked it."""+ if not person_uuids:+ return {}+ with persons_db_connection(writer=False) as conn, conn.cursor() as cursor:+ cursor.execute(+ f"SELECT person_uuid::text, deleted_at, max_version, blocked_at FROM {PG_CLEANUP_QUEUE_TABLE} "+ "WHERE team_id = %s AND person_uuid = ANY(%s::uuid[])",+ [team_id, list(person_uuids)],+ )+ return {+ person_uuid: {+ "deleted_at": deleted_at.isoformat(),+ "max_version": max_version,+ "blocked_at": blocked_at.isoformat() if blocked_at else None,+ }+ for person_uuid, deleted_at, max_version, blocked_at in cursor.fetchall()+ }diff --git a/posthog/models/person/divergence/datatypes.py b/posthog/models/person/divergence/datatypes.pyindex dfb82c96edaa..5a3a19508f02 100644--- a/posthog/models/person/divergence/datatypes.py+++ b/posthog/models/person/divergence/datatypes.py@@ -121,3 +121,10 @@ class RepairSummary:mapping_outcomes: dict[RepairOutcome, int]# Published rows that Kafka did not confirm: the delivery failed, or it was still queued at the deadline.undelivered: int+++@frozen+class RepublishResult:+ summary: RepairSummary+ # Named persons and distinct ids that ClickHouse may still disagree on: not repaired, or not delivered.+ unconfirmed: tuple[str, ...]diff --git a/posthog/models/person/divergence/detach.py b/posthog/models/person/divergence/detach.pynew file mode 100644index 000000000000..8c13faf042fb--- /dev/null+++ b/posthog/models/person/divergence/detach.py@@ -0,0 +1,286 @@+"""Take a distinct id off a person. `detach` tombstones strays from a scan report; `split_distinct_ids` moves+distinct ids off a live person onto persons of their own."""++from __future__ import annotations++import time+import shlex+import tempfile+from collections.abc import Callable, Sequence+from functools import partial+from typing import TYPE_CHECKING, Any+from uuid import UUID++from posthog.kafka_client.routing import flush_all_producers+from posthog.models.person.util import _batched_get_distinct_ids_for_persons, get_distinct_id_rows, get_persons_by_uuids+from posthog.personhog_client.client import personhog_call, require_personhog_client+from posthog.personhog_client.proto import (+ DISTINCT_ID_WRITE_ACTION_TOMBSTONE,+ DISTINCT_ID_WRITE_OUTCOME_APPLIED,+ DISTINCT_ID_WRITE_OUTCOME_PERSON_NOT_LIVE,+ DISTINCT_ID_WRITE_OUTCOME_UNCHANGED,+ DistinctIdRow,+ DistinctIdWrite,+ WriteDistinctIdsRequest,+)++from . import cleanup_queue, lookups, observe, repairs+from .flavors import PgMapping, classify_mapping++if TYPE_CHECKING:+ from .verbs import VerbContext++# How long the split waits for its own ClickHouse messages to go out before it republishes and checks them.+_SPLIT_FLUSH_SECONDS = 60.0++# A republish command for more distinct ids than this reads them from a file, so the command stays short.+_INLINE_DISTINCT_IDS = 10++# The pause before the resend tries unconfirmed ids again, for a replica that had not caught up with the split.+_RESEND_RETRY_SECONDS = 1.0++_sleep = time.sleep+++class SplitError(Exception):+ pass+++def detach(rows: list[dict[str, Any]], context: VerbContext) -> None:+ """Tombstone strays in Postgres: distinct ids live there on a tombstoned person.++ It writes nothing to ClickHouse. The cleanup drain hard-deletes a person's Postgres rows once its distinct ids are+ tombstoned, so a ClickHouse tombstone that outlived that delete would hide a later re-use of the distinct id. A+ stray is tombstoned only when the classifier, re-run on fresh ClickHouse reads just before the write, still names+ it `detach`: ClickHouse holds nothing for it, or the deletion sweep will remove its rows together with the owner+ before the drain runs. Any other stray is skipped under the sub-flavor it now has.++ The tombstone lands above every version ClickHouse holds for the distinct id, so if the distinct id is captured+ again, ingestion revives the row one version higher and ClickHouse shows the revival.+ """+ for batch in lookups._chunks(rows, repairs._WRITE_BATCH_SIZE):+ _detach_batch(list(batch), context)+++def _detach_batch(rows: list[dict[str, Any]], context: VerbContext) -> None:+ team_id = context.team_id+ by_key = {row["key"]: row for row in rows}+ reported = {row["key"]: PgMapping(**row["evidence"]["pg"]) for row in rows}+ ch = observe.ch_mapping_states(team_id, list(by_key))+ owner_ch = observe.owner_ch_states(team_id, reported, ch)+ writes: list[DistinctIdWrite] = []+ for key, row in by_key.items():+ pg, state = reported[key], ch.get(key)+ verdict = classify_mapping(pg, state, owner_ch=owner_ch.get(key))+ if verdict.verb != "detach":+ context.action(row, f"skipped_{verdict.sub_flavor or verdict.flavor}")+ continue+ writes.append(+ DistinctIdWrite(+ distinct_id=key,+ action=DISTINCT_ID_WRITE_ACTION_TOMBSTONE,+ person_uuid=pg.person_uuid,+ min_version=max(pg.version + 1, (state.max_version + 1) if state else 0),+ expected_version=pg.version,+ )+ )+ if not writes:+ return+ if not context.apply:+ current = {+ r.distinct_id: r for r in get_distinct_id_rows(team_id, [w.distinct_id for w in writes], strong=True)+ }+ for write in writes:+ context.action(+ by_key[write.distinct_id],+ _predicted_outcome(current.get(write.distinct_id), write),+ target_version=write.min_version,+ )+ return++ context.pacer.before_write()+ response = repairs._retry_transient(+ partial(+ personhog_call,+ "person_consistency_detach",+ partial(+ require_personhog_client().write_distinct_ids,+ WriteDistinctIdsRequest(team_id=team_id, writes=writes),+ ),+ ),+ what="detach",+ log=context.log,+ )+ detached_owners: set[int] = set()+ for result in response.results:+ row = by_key[result.distinct_id]+ if result.outcome in (DISTINCT_ID_WRITE_OUTCOME_APPLIED, DISTINCT_ID_WRITE_OUTCOME_UNCHANGED):+ detached_owners.add(reported[result.distinct_id].person_id)+ outcome = "detached" if result.outcome == DISTINCT_ID_WRITE_OUTCOME_APPLIED else "already_detached"+ context.action(row, outcome, version=int(result.row.version) if result.HasField("row") else None)+ elif result.outcome == DISTINCT_ID_WRITE_OUTCOME_PERSON_NOT_LIVE:+ context.action(row, "skipped_person_gone")+ else:+ context.action(row, "skipped_changed")+ _release_drained_owners(team_id, detached_owners, by_key, reported)+++def _predicted_outcome(row: DistinctIdRow | None, write: DistinctIdWrite) -> str:+ """What applying `write` would report, from the Postgres primary's row, by the RPC's rules."""+ if row is None or not row.HasField("person_is_deleted"):+ return "skipped_person_gone"+ if row.person_uuid != write.person_uuid or not row.person_is_deleted:+ return "skipped_changed"+ if row.is_deleted:+ return "already_detached" if row.version >= write.min_version else "skipped_changed"+ return "would_detach" if row.version == write.expected_version else "skipped_changed"+++def _release_drained_owners(+ team_id: int, owner_ids: set[int], by_key: dict[str, dict[str, Any]], reported: dict[str, PgMapping]+) -> None:+ """Clear `blocked_at` for owners left with no live distinct id, so the next drain run retries them.++ An owner that still has a live distinct id stays blocked: the drain would block it again. The live distinct ids+ come from the replica, so a lagging replica only leaves an owner blocked until the drain's own retry.+ """+ if not owner_ids:+ return+ live = _live_distinct_ids(team_id, sorted(owner_ids), limit=1)+ uuids = {mapping.person_id: mapping.person_uuid for key in by_key if (mapping := reported[key]).person_uuid}+ released = sorted(uuid for owner in owner_ids if not live.get(owner) and (uuid := uuids[owner]) is not None)+ cleanup_queue.clear_blocked(team_id, released)+++def _live_distinct_ids(team_id: int, person_ids: list[int], *, limit: int) -> dict[int, list[str]]:+ by_person = personhog_call(+ "get_distinct_ids_for_persons",+ lambda: _batched_get_distinct_ids_for_persons(team_id, person_ids, limit_per_person=limit),+ )+ return {person_id: [d.id for d in dids] for person_id, dids in by_person.items()}+++def split_distinct_ids(+ team_id: int,+ person_uuid: str,+ distinct_ids: Sequence[str],+ *,+ apply: bool,+ log: Callable[[str], None],+) -> list[str]:+ """Move `distinct_ids` off a live person onto persons of their own, and return the distinct ids split.++ `Person.split_person` does the split through personhog and refuses one that would leave the person without a+ distinct id; the dry run applies the same check. A distinct id that is not live on this person stops the whole+ split, so a mistyped id never splits the rest alone. Every applied split republishes the moved persons from+ Postgres and checks the result, and raises with the command that finishes the job when ClickHouse may still+ show a moved id on this person.+ """+ ids = list(dict.fromkeys(distinct_ids))+ if not ids:+ raise SplitError("name at least one --distinct-id")+ try:+ person_uuid = str(UUID(person_uuid))+ except ValueError as exc:+ raise SplitError(f"{person_uuid!r} is not a uuid") from exc+ persons = get_persons_by_uuids(team_id, [person_uuid], distinct_id_limit=0)+ if not persons:+ raise SplitError(f"person {person_uuid} is not live in team {team_id}")+ person = persons[0]+ rows = {r.distinct_id: r for r in get_distinct_id_rows(team_id, ids, strong=True)}+ elsewhere = sorted(+ d for d in ids if (row := rows.get(d)) is None or row.is_deleted or row.person_uuid != person_uuid+ )+ if elsewhere:+ raise SplitError(f"not live on person {person_uuid}: {', '.join(elsewhere)}")+ # One more than the request names: if every returned id is in the request, none would remain.+ remaining = _live_distinct_ids(team_id, [int(person.pk)], limit=len(ids) + 1).get(int(person.pk), [])+ if all(distinct_id in ids for distinct_id in remaining):+ raise SplitError(f"the split would leave person {person_uuid} with no distinct id")+ if not apply:+ log(f"split: would move {len(ids)} distinct ids off person {person_uuid}")+ return []+ failure: Exception | None = None+ try:+ person.split_person(None, None, distinct_ids_to_split=ids)+ except Exception as exc:+ # A failure after the first batch leaves the earlier batches split in Postgres, so they are resent too.+ failure = exc+ finally:+ # The split publishes to ClickHouse without waiting, and a command that exits first can lose those messages.+ flush_all_producers(_SPLIT_FLUSH_SECONDS)+ moved: list[str] = []+ try:+ moved = _moved_ids(team_id, person_uuid, ids)+ unconfirmed = _resend_split(team_id, moved, log=log)+ except Exception as exc:+ log(f"split: sending the split to ClickHouse again failed: {exc!r}")+ raise SplitError(_unconfirmed_message(team_id, person_uuid, moved or ids, failure)) from (failure or exc)+ if unconfirmed:+ raise SplitError(_unconfirmed_message(team_id, person_uuid, unconfirmed, failure)) from failure+ if failure is not None:+ raise SplitError(+ f"split moved {len(moved)} of {len(ids)} distinct ids off person {person_uuid}, then failed: {failure}"+ ) from failure+ if len(moved) < len(ids):+ raise SplitError(+ f"split moved {len(moved)} of {len(ids)} distinct ids off person {person_uuid}; check the rest with "+ "person_inspect"+ )+ log(f"split: moved {len(moved)} distinct ids off person {person_uuid}")+ return moved+++def _moved_ids(team_id: int, person_uuid: str, distinct_ids: list[str]) -> list[str]:+ rows = get_distinct_id_rows(team_id, distinct_ids, strong=True)+ return [r.distinct_id for r in rows if not r.is_deleted and r.person_uuid and r.person_uuid != person_uuid]+++def _resend_split(team_id: int, moved: list[str], *, log: Callable[[str], None]) -> list[str]:+ """Republish the persons the split moved distinct ids to, and return the moved ids still unconfirmed.++ Kafka's flush counts only queued messages, not failed deliveries, so every split is resent and checked. Without+ this, ClickHouse can keep the moved ids on the old person, and a scan suggests merging the two persons back.+ """+ if not moved:+ return []+ unconfirmed = list(repairs.republish_named(team_id, distinct_ids=moved, apply=True, log=log).unconfirmed)+ if unconfirmed:+ _sleep(_RESEND_RETRY_SECONDS)+ unconfirmed = list(repairs.republish_named(team_id, distinct_ids=unconfirmed, apply=True, log=log).unconfirmed)+ log(f"split: sent {len(moved)} moved distinct ids and their persons again, {len(unconfirmed)} unconfirmed")+ return unconfirmed+++def _unconfirmed_message(team_id: int, person_uuid: str, distinct_ids: Sequence[str], failure: Exception | None) -> str:+ # The command prints only this message, not the exception it chains, so the split's own failure goes in it.+ failed = f" The split also failed: {failure}." if failure is not None else ""+ return (+ f"ClickHouse may still show {len(distinct_ids)} split distinct ids on person {person_uuid}.{failed} Do not "+ f"run a merge a scan suggests for them. Run: {_republish_command(team_id, distinct_ids)}"+ )+++def _republish_command(team_id: int, distinct_ids: Sequence[str]) -> str:+ ids_file = _write_ids_file(team_id, distinct_ids) if len(distinct_ids) > _INLINE_DISTINCT_IDS else None+ if ids_file is not None:+ flags = f"--distinct-ids-file {shlex.quote(ids_file)}"+ else:+ flags = " ".join(f"--distinct-id {shlex.quote(d)}" for d in distinct_ids)+ return f"python manage.py person_consistency republish --team-id {team_id} {flags} --apply"+++def _write_ids_file(team_id: int, distinct_ids: Sequence[str]) -> str | None:+ """Write one distinct id per line to a local file and return its path, or None when the ids must go inline.++ An id that is empty or holds a newline would not read back whole, and a failed write must not replace the+ error that prints the command.+ """+ if any(not d or "\n" in d for d in distinct_ids):+ return None+ try:+ with tempfile.NamedTemporaryFile("w", prefix=f"split-{team_id}-", suffix=".txt", delete=False) as ids_file:+ ids_file.write("".join(f"{d}\n" for d in distinct_ids))+ except OSError:+ return None+ return ids_file.namediff --git a/posthog/models/person/divergence/flavors.py b/posthog/models/person/divergence/flavors.pyindex dbea6aab5d75..582b2d60ba59 100644--- a/posthog/models/person/divergence/flavors.py+++ b/posthog/models/person/divergence/flavors.py@@ -36,9 +36,15 @@ MappingFlavor = Literal["hidden","missing_in_ch","stray",+ "awaiting_sweep",]-RepairVerb = Literal["republish", "merge_stale", "bury", "restore", "fold", "merge"]+# Where a person tombstoned in Postgres stands in ClickHouse, by the rule the deletion sweep uses to take a person:+# `pending` when its newest visible row is a tombstone, `visible` when a live row is newest or ties it, `absent` when+# it has no visible row (already swept, or never published).+OwnerChState = Literal["pending", "visible", "absent"]++RepairVerb = Literal["republish", "merge_stale", "bury", "restore", "fold", "merge", "detach"]# Keyed by (flavor, sub_flavor). A ghost's or diverged mapping's sub-flavor decides between restore,# fold and merge. None means the state is accepted as it is.@@ -74,8 +80,26 @@ MAPPING_VERBS: dict[tuple[str, str | None], RepairVerb | None] = {("stale", None): "republish",("hidden", None): "republish",("missing_in_ch", None): "republish",- # Ingestion tombstones and re-attaches a stray on the distinct id's next event, so it is reported only.- ("stray", None): None,+ # Ingestion also repairs a stray on the distinct id's next event, but a distinct id that sees no more events+ # keeps it, and the stray blocks the cleanup drain from hard-deleting its person. `detach` tombstones it in+ # Postgres only, which is safe only while ClickHouse holds nothing for it or the deletion sweep will remove what+ # it holds; the sub-flavors below are strays where neither is true, so they are reported for a person to fix.+ ("stray", None): "detach",+ # Its person row is gone, so there is no owner to name; ingestion re-attaches it on the next event.+ ("stray", "orphan"): None,+ # The owner's ClickHouse tombstone never landed, so no sweep takes it: `bury` the owner, then rescan.+ ("stray", "owner_visible"): None,+ # ClickHouse resolves the distinct id to a different person than Postgres does.+ ("stray", "ch_owner_differs"): None,+ # The owner is gone from ClickHouse but the distinct id still has rows there, which no sweep will remove.+ ("stray", "rows_outlive_owner"): None,+ # The caller did not read the owner's ClickHouse state, so whether a sweep will clean up is unknown.+ ("stray", "owner_unchecked"): None,+ # Rows at ClickHouse's top version name different owners, so the sweep may resolve another one.+ ("stray", "ch_tie"): None,+ # Tombstoned in Postgres; ClickHouse still resolves it to its tombstoned owner, which the next sweep removes+ # together with that owner. A ClickHouse tombstone published now could outlive the drain's hard delete.+ ("awaiting_sweep", None): None,}@@ -122,6 +146,9 @@ class ChMapping:winner_person_uuid: strwinner_version: intwinner_deleted: bool+ # Visible rows at the top version disagree on the owner or on is_deleted, so which one ClickHouse and the+ # deletion sweep treat as the winner is not fixed.+ winner_tied: bool = False@frozen@@ -179,11 +206,41 @@ def classify_person(return _verdict("person", "ok")+def owner_ch_state(state: ChPerson | None) -> OwnerChState:+ """Where a person stands in ClickHouse, by the sweep's rule: it takes a person whose newest visible row is a+ tombstone. A tie between a live and a tombstone row at the top version counts as visible, because which one+ ClickHouse treats as newest is not fixed."""+ if state is None or state.visible_rows == 0:+ return "absent"+ if state.tombstone_max_version is not None and (+ state.live_max_version is None or state.tombstone_max_version > state.live_max_version+ ):+ return "pending"+ return "visible"+++def needs_owner_ch_state(pg: PgMapping | None, ch: ChMapping | None) -> bool:+ """Whether classifying this mapping depends on its Postgres owner's ClickHouse state."""+ return (+ pg is not None+ and pg.person_is_deleted is True+ and pg.person_uuid is not None+ and ch is not None+ and ch.visible_rows > 0+ )++def classify_mapping(- pg: PgMapping | None, ch: ChMapping | None, *, ch_owner_pg_state: str | None = None, team_exists: bool = True+ pg: PgMapping | None,+ ch: ChMapping | None,+ *,+ ch_owner_pg_state: str | None = None,+ team_exists: bool = True,+ owner_ch: OwnerChState | None = None,) -> Verdict:"""The flavor of one mapping. `ch_owner_pg_state` is the Postgres state of ClickHouse's winning owner- (`live`, `deleted` or `absent`), needed only for ghost and diverged mappings."""+ (`live`, `deleted` or `absent`), needed only for ghost and diverged mappings. `owner_ch` is the ClickHouse+ state of the Postgres owner, needed only when `needs_owner_ch_state` holds."""ch_live = ch is not None and ch.visible_rows > 0 and not ch.winner_deletedif pg is None:@@ -197,12 +254,25 @@ def classify_mapping(return _verdict("mapping", "ghost", sub)if pg.is_deleted:+ if (+ ch_live+ and ch is not None+ and pg.person_is_deleted+ and ch.winner_person_uuid == pg.person_uuid+ and not ch.winner_tied+ and owner_ch == "pending"+ # Postgres must hold the tombstone above every ClickHouse version, so a returning distinct id revives+ # above them and ClickHouse shows it before the sweep removes the old rows.+ and pg.version > ch.max_version+ ):+ return _verdict("mapping", "awaiting_sweep")# A live winner at any version still resolves the id, so the tombstone has not reached ClickHouse.return _verdict("mapping", "deleted_visible" if ch_live else "ok_deleted")- # None is an orphan row whose person is gone; ingestion re-attaches it like any other stray.- if pg.person_is_deleted is not False:- return _verdict("mapping", "stray")+ if pg.person_is_deleted is None:+ return _verdict("mapping", "stray", "orphan")+ if pg.person_is_deleted:+ return _verdict("mapping", "stray", _stray_sub_flavor(pg, ch, owner_ch))if ch is None or ch.visible_rows == 0:return _verdict("mapping", "missing_in_ch")if ch.winner_deleted:@@ -213,3 +283,19 @@ def classify_mapping(if ch.winner_version > pg.version:return _verdict("mapping", "stale")return _verdict("mapping", "ok")+++def _stray_sub_flavor(pg: PgMapping, ch: ChMapping | None, owner_ch: OwnerChState | None) -> str | None:+ """None when tombstoning the stray in Postgres alone is safe: the cleanup drain hard-deletes its Postgres row only+ after the deletion sweep removed the owner, and that sweep removes every ClickHouse row of a key whose newest row+ names the owner, live or tombstone. So it is safe when ClickHouse shows nothing, or when its newest row names the+ owner and the owner still waits for the sweep."""+ if ch is None or ch.visible_rows == 0:+ return None+ if ch.winner_tied:+ return "ch_tie"+ if ch.winner_person_uuid != pg.person_uuid:+ return "ch_owner_differs"+ if owner_ch is None:+ return "owner_unchecked"+ return {"pending": None, "visible": "owner_visible", "absent": "rows_outlive_owner"}[owner_ch]diff --git a/posthog/models/person/divergence/inspection.py b/posthog/models/person/divergence/inspection.pyindex 3e5feaf63c5c..95869b26a07a 100644--- a/posthog/models/person/divergence/inspection.py+++ b/posthog/models/person/divergence/inspection.py@@ -186,13 +186,18 @@ def inspect_mappings(team_id: int, distinct_ids: Sequence[str]) -> list[Inspectich = observe.ch_mapping_states(team_id, dids)owners = sorted({c.winner_person_uuid for c in ch.values()})owner_states = observe.pg_person_states(team_id, owners) if team_exists and owners else {}+ owner_ch = observe.owner_ch_states(team_id, pg, ch)history = _mapping_history(team_id, dids)results = []for distinct_id in dids:ch_state = ch.get(distinct_id)owner_state = observe.person_pg_state(owner_states, ch_state.winner_person_uuid) if ch_state else Noneverdict = classify_mapping(- pg.get(distinct_id), ch_state, ch_owner_pg_state=owner_state, team_exists=team_exists+ pg.get(distinct_id),+ ch_state,+ ch_owner_pg_state=owner_state,+ team_exists=team_exists,+ owner_ch=owner_ch.get(distinct_id),)results.append(Inspection(diff --git a/posthog/models/person/divergence/observe.py b/posthog/models/person/divergence/observe.pyindex 0696131eca46..1bc677696d94 100644--- a/posthog/models/person/divergence/observe.py+++ b/posthog/models/person/divergence/observe.py@@ -6,8 +6,16 @@ from collections.abc import Sequencefrom typing import Any, Literalfrom uuid import UUID-from posthog.models.person.divergence import lookups-from posthog.models.person.divergence.flavors import ChMapping, ChPerson, PgMapping, PgPerson+from posthog.models.person.divergence import cleanup_queue, lookups+from posthog.models.person.divergence.flavors import (+ ChMapping,+ ChPerson,+ OwnerChState,+ PgMapping,+ PgPerson,+ needs_owner_ch_state,+ owner_ch_state,+)from posthog.models.person.util import get_distinct_id_rows, get_person_tombstonesPersonPgState = Literal["live", "deleted", "absent"]@@ -41,7 +49,8 @@ SELECTcountIf(_row_exists),toString(argMaxIf(person_id, version, _row_exists)),maxIf(version, _row_exists),- argMaxIf(is_deleted, version, _row_exists)+ argMaxIf(is_deleted, version, _row_exists),+ groupUniqArrayIf((version, toString(person_id), is_deleted), _row_exists)FROM person_distinct_id2WHERE team_id = %(team_id)s AND {keys}GROUP BY distinct_id@@ -97,13 +106,17 @@ def ch_person_states_in_range(team_id: int, after: str, last: str, *, first: boodef ch_mapping_from_row(row: Sequence[Any]) -> tuple[str, ChMapping]:- distinct_id, max_version, visible_rows, owner, winner_version, winner_deleted = row+ distinct_id, max_version, visible_rows, owner, winner_version, winner_deleted, visible = row+ at_max = {+ (str(person), bool(deleted)) for version, person, deleted in visible if int(version) == int(winner_version)+ }return distinct_id, ChMapping(max_version=int(max_version),visible_rows=int(visible_rows),winner_person_uuid=str(owner),winner_version=int(winner_version),winner_deleted=bool(winner_deleted),+ winner_tied=len(at_max) > 1,)@@ -115,6 +128,35 @@ def ch_mapping_states(team_id: int, distinct_ids: Sequence[str]) -> dict[str, Chreturn states+def owner_ch_states(team_id: int, pg: dict[str, PgMapping], ch: dict[str, ChMapping]) -> dict[str, OwnerChState]:+ """The ClickHouse state of each mapping's Postgres owner, read only for the mappings whose classification needs it."""+ owners = {+ distinct_id: str(UUID(pg_row.person_uuid))+ for distinct_id, ch_row in ch.items()+ if needs_owner_ch_state(pg_row := pg.get(distinct_id), ch_row) and pg_row is not None and pg_row.person_uuid+ }+ if not owners:+ return {}+ uuids = sorted(set(owners.values()))+ persons = ch_person_states(team_id, uuids)+ queued = cleanup_queue.queue_entries(team_id, uuids)+ return {+ distinct_id: "absent"+ if _swept(persons.get(person_uuid), queued.get(person_uuid))+ else owner_ch_state(persons.get(person_uuid))+ for distinct_id, person_uuid in owners.items()+ }+++def _swept(state: ChPerson | None, queued: dict[str, Any] | None) -> bool:+ """The sweep queues a person after it removed the person's ClickHouse rows up to the queued bound, so rows at or+ below that bound that show up again (a replayed Kafka batch) do not make the person pending. Rows above the bound+ are a later tombstone the next sweep takes, and a row without a bound is never drained."""+ if queued is None or queued["max_version"] is None:+ return False+ return state is None or state.max_version <= queued["max_version"]++def ch_mapping_states_in_range(team_id: int, after: str, last: str, *, first: bool) -> dict[str, ChMapping]:rows = lookups._ch(_CH_MAPPINGS_RANGE_SQL[first], {"team_id": team_id, "after": after, "last": last}, _READ_SETTINGSdiff --git a/posthog/models/person/divergence/repairs.py b/posthog/models/person/divergence/repairs.pyindex b7ccf539e7d9..a21a965afb6e 100644--- a/posthog/models/person/divergence/repairs.py+++ b/posthog/models/person/divergence/repairs.py@@ -5,7 +5,7 @@ from __future__ import annotationsimport timefrom collections import Counter, defaultdictfrom collections.abc import Callable, Collection, Mapping, Sequence-from dataclasses import field+from dataclasses import field, replacefrom functools import partialfrom typing import TypeVarfrom uuid import UUID@@ -24,6 +24,7 @@ from posthog.models.person.util import (_flush_person_producers,_person_row,create_person_distinct_id,+ get_distinct_id_rows,get_persons_by_uuids,raise_person_version,)@@ -69,6 +70,11 @@ _TRANSIENT_RPC_CODES = frozenset(_FLUSH_TIMEOUT_SECONDS = 5 * 60+DEFAULT_MAX_WRITES_PER_SECOND = 20.0++# Outcomes that leave ClickHouse agreeing with Postgres for the key, once the publish is delivered.+_RESENT = frozenset({"repaired", "skipped_not_divergent"})+# Confirmed produce results are dropped this often, so a long repair does not hold one per published row._DELIVERY_PRUNE_EVERY = 1_000@@ -519,3 +525,69 @@ def repair_persons(if undelivered:log(f"{undelivered} ClickHouse messages were not delivered; rerun the repair for the same input")return outcomes.summary(applied=apply, persons=processed, undelivered=undelivered)+++def _landed(action: datatypes.RepairAction, lost: set[tuple[str, str]]) -> bool:+ key = ("person", action.person_uuid) if action.distinct_id is None else ("mapping", action.distinct_id)+ return action.outcome in _RESENT and key not in lost+++def republish_named(+ team_id: int,+ *,+ person_uuids: Sequence[str] = (),+ distinct_ids: Sequence[str] = (),+ apply: bool,+ max_writes_per_second: float | None = DEFAULT_MAX_WRITES_PER_SECOND,+ log: Callable[[str], None],+) -> datatypes.RepublishResult:+ """Republish named persons, and the live owners of named distinct ids, where ClickHouse disagrees.++ A scan routes a mapping that ClickHouse still gives to another live person to `merge`, which is wrong when+ Postgres moved it on purpose and only the publish was lost, as after a split. This republishes from Postgres.+ The owners are read on the primary, so a distinct id moved moments ago resolves to its new person.++ A named person is confirmed only when it and every mapping it owns were repaired or already agreed, and every+ row published for them was delivered. A named distinct id needs the same for itself and its owner's row. A+ distinct id that is not live in Postgres, or a person with too many distinct ids to repair, stays unconfirmed.+ """+ named_persons = list(dict.fromkeys(person_uuids))+ named_ids = list(dict.fromkeys(distinct_ids))+ rows = get_distinct_id_rows(team_id, named_ids, strong=True) if named_ids else []+ owner_of = {row.distinct_id: row.person_uuid for row in rows if not row.is_deleted and row.person_uuid}+ if len(owner_of) < len(named_ids):+ log(+ f"republish: {len(named_ids) - len(owner_of)} distinct ids are not live in Postgres, so nothing republishes them"+ )+ actions: list[datatypes.RepairAction] = []+ deliveries = _Deliveries()+ undelivered = 0+ try:+ summary = repair_persons(+ [+ datatypes.PersonRef(team_id=team_id, person_uuid=uuid)+ for uuid in dict.fromkeys([*named_persons, *owner_of.values()])+ ],+ apply=apply,+ max_writes_per_second=max_writes_per_second,+ on_action=actions.append,+ log=log,+ deliveries=deliveries,+ )+ finally:+ if apply:+ undelivered = deliveries.undelivered(_FLUSH_TIMEOUT_SECONDS)+ lost = {(key.kind, key.key) for key in deliveries.undelivered_keys()}+ person_rows_ok = {a.person_uuid for a in actions if a.distinct_id is None and _landed(a, lost)}+ persons_with_misses = {a.person_uuid for a in actions if not _landed(a, lost)}+ # A replica that lags a move can still list the distinct id under its old person, so only the owner counts.+ ids_ok = {+ a.distinct_id+ for a in actions+ if a.distinct_id is not None and a.person_uuid == owner_of.get(a.distinct_id) and _landed(a, lost)+ }+ unconfirmed = (+ *(uuid for uuid in named_persons if uuid not in person_rows_ok or uuid in persons_with_misses),+ *(d for d in named_ids if d not in ids_ok or owner_of[d] not in person_rows_ok),+ )+ return datatypes.RepublishResult(summary=replace(summary, undelivered=undelivered), unconfirmed=unconfirmed)diff --git a/posthog/models/person/divergence/team_scan.py b/posthog/models/person/divergence/team_scan.pyindex ab72e142d99b..d028ebe77601 100644--- a/posthog/models/person/divergence/team_scan.py+++ b/posthog/models/person/divergence/team_scan.py@@ -482,6 +482,7 @@ def _classify_mappings_page(plan: ScanPlan, after: str, last: str, first: bool)}owners = sorted({ch[distinct_id].winner_person_uuid for distinct_id in needs_owner})owner_states = observe.pg_person_states(plan.team_id, owners) if plan.team_exists and owners else {}+ owner_ch = observe.owner_ch_states(plan.team_id, pg, ch)rows: list[dict[str, Any]] = []counts: Counter[str] = Counter()for distinct_id, ch_state in ch.items():@@ -489,7 +490,13 @@ def _classify_mappings_page(plan: ScanPlan, after: str, last: str, first: bool)owner_state = (observe.person_pg_state(owner_states, ch_state.winner_person_uuid) if distinct_id in needs_owner else None)- verdict = classify_mapping(pg_row, ch_state, ch_owner_pg_state=owner_state, team_exists=plan.team_exists)+ verdict = classify_mapping(+ pg_row,+ ch_state,+ ch_owner_pg_state=owner_state,+ team_exists=plan.team_exists,+ owner_ch=owner_ch.get(distinct_id),+ )counts[_count_key("mapping", verdict)] += 1if _reported(verdict):rows.append(diff --git a/posthog/models/person/divergence/verbs.py b/posthog/models/person/divergence/verbs.pyindex 691e44236e37..77f8af19c4b5 100644--- a/posthog/models/person/divergence/verbs.py+++ b/posthog/models/person/divergence/verbs.py@@ -25,7 +25,8 @@ from posthog.models.person.deletion import (from posthog.models.person.util import create_person_distinct_id, get_distinct_id_rows, get_persons_by_uuidsfrom posthog.personhog_client.proto import DistinctIdRow-from . import datatypes, lookups, observe, repairs, restore_verbs, team_scan+from . import datatypes, detach, lookups, observe, repairs, restore_verbs, team_scan+from .flavors import PgMapping, classify_mapping# A key whose last recorded outcome is one of these needs nothing more, so a resumed run skips it.DONE_OUTCOMES = frozenset(@@ -40,6 +41,9 @@ DONE_OUTCOMES = frozenset("already_moved","folded","merged",+ "detached",+ "already_detached",+ "raised_for_sweep",})@@ -433,7 +437,35 @@ def _bury_mapping(row: dict[str, Any], context: VerbContext) -> None:if ch is None or ch.visible_rows == 0 or ch.winner_deleted:context.action(row, "skipped_not_divergent")return+ # A report written before the owner became pending would still name bury, but the sweep removes these rows+ # with the owner, and a tombstone published mid-sweep could outlive the drain's hard delete.+ pg_now = PgMapping(+ person_id=int(current.person_id),+ person_uuid=current.person_uuid if current.HasField("person_uuid") else None,+ version=int(current.version),+ is_deleted=current.is_deleted,+ person_is_deleted=current.person_is_deleted if current.HasField("person_is_deleted") else None,+ )+ owner_ch = observe.owner_ch_states(team_id, {distinct_id: pg_now}, {distinct_id: ch}).get(distinct_id)+ if classify_mapping(pg_now, ch, owner_ch=owner_ch).flavor == "awaiting_sweep":+ context.action(row, "skipped_awaiting_sweep")+ returntarget = max(int(current.version), ch.max_version + 1)+ if owner_ch == "pending" and ch.winner_person_uuid == pg_now.person_uuid and not ch.winner_tied:+ # The sweep removes these rows with the owner, so raising Postgres above ClickHouse is the whole repair: a+ # returning distinct id then revives above ClickHouse, and no tombstone is published that a running sweep+ # could leave behind.+ if not context.apply:+ context.action(row, "would_raise_for_sweep", target_version=target)+ return+ context.pacer.before_write()+ repairs._raise_mapping_version_floor(team_id, distinct_id, target)+ confirmed = _strong_row(team_id, distinct_id)+ if confirmed is None or not confirmed.is_deleted or confirmed.person_id != current.person_id:+ context.action(row, "skipped_revived")+ return+ context.action(row, "raised_for_sweep", version=int(confirmed.version))+ returnif not context.apply:context.action(row, "would_repair", target_version=target)return@@ -460,4 +492,5 @@ VERBS: dict[str, VerbFn] = {"restore": restore_verbs.restore,"fold": restore_verbs.fold,"merge": restore_verbs.merge,+ "detach": detach.detach,}diff --git a/posthog/models/person/test/test_divergence.py b/posthog/models/person/test/test_divergence.pyindex 6b208722d9d9..eff234635979 100644--- a/posthog/models/person/test/test_divergence.py+++ b/posthog/models/person/test/test_divergence.py@@ -796,10 +796,14 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):stored = fake.stored_person(self.team.pk, str(person.uuid))assert stored is not Nonereplica_view.CopyFrom(stored)+ replica_ids = fake.get_distinct_ids_for_persons(+ person_pb2.GetDistinctIdsForPersonsRequest(team_id=self.team.pk, person_ids=[stored.id])+ )tombstone_persons_in_postgres(self.team.pk, [person.uuid])- with patch.object(- fake, "get_persons_by_uuids", return_value=person_pb2.PersonsResponse(persons=[replica_view])+ with (+ patch.object(fake, "get_persons_by_uuids", return_value=person_pb2.PersonsResponse(persons=[replica_view])),+ patch.object(fake, "get_distinct_ids_for_persons", return_value=replica_ids),):_, actions = self._repair(person.uuid)diff --git a/posthog/models/person/test/test_divergence_flavors.py b/posthog/models/person/test/test_divergence_flavors.pyindex d2a84602f1d4..9a2d18349db4 100644--- a/posthog/models/person/test/test_divergence_flavors.py+++ b/posthog/models/person/test/test_divergence_flavors.py@@ -11,6 +11,7 @@ from posthog.models.person.divergence.flavors import (Verdict,classify_mapping,classify_person,+ owner_ch_state,)@@ -132,13 +133,16 @@ def _pg_map()-def _ch_map(*, owner: str = "b", version: int = 0, deleted: bool = False, visible_rows: int = 1) -> ChMapping:+def _ch_map(+ *, owner: str = "b", version: int = 0, deleted: bool = False, visible_rows: int = 1, tied: bool = False+) -> ChMapping:return ChMapping(max_version=version,visible_rows=visible_rows,winner_person_uuid=owner,winner_version=version,winner_deleted=deleted,+ winner_tied=tied,)@@ -171,19 +175,12 @@ class TestClassifyMapping(SimpleTestCase):None,_v("ok_deleted", None, None),),- (- "stray_on_deleted_person",- _pg_map(owner_deleted=True),- _ch_map(),- None,- _v("stray", None, None),- ),("orphan_with_ch_on_another_person",_pg_map(owner=None, owner_deleted=None),_ch_map(owner="a"),"absent",- _v("stray", None, None),+ _v("stray", "orphan", None),),("live_missing_in_ch", _pg_map(), None, None, _v("missing_in_ch", None, "republish")),("live_and_swept", _pg_map(), _ch_map(visible_rows=0), None, _v("missing_in_ch", None, "republish")),@@ -211,6 +208,133 @@ class TestClassifyMapping(SimpleTestCase):def test_classifies(self, _name, pg, ch, owner_state, expected):assert classify_mapping(pg, ch, ch_owner_pg_state=owner_state) == expected+ @parameterized.expand(+ [+ # (name, Postgres mapping, ClickHouse mapping, owner's ClickHouse state, expected)+ ("stray_nothing_in_ch", _pg_map(owner_deleted=True), None, None, _v("stray", None, "detach")),+ (+ "stray_ch_rows_all_lightweight_deleted",+ _pg_map(owner_deleted=True),+ _ch_map(visible_rows=0),+ None,+ _v("stray", None, "detach"),+ ),+ (+ "stray_live_on_pending_owner",+ _pg_map(owner_deleted=True),+ _ch_map(),+ "pending",+ _v("stray", None, "detach"),+ ),+ (+ "stray_tombstone_naming_pending_owner",+ _pg_map(owner_deleted=True),+ _ch_map(deleted=True),+ "pending",+ _v("stray", None, "detach"),+ ),+ (+ "stray_owner_visible",+ _pg_map(owner_deleted=True),+ _ch_map(),+ "visible",+ _v("stray", "owner_visible", None),+ ),+ (+ "stray_rows_outlive_owner",+ _pg_map(owner_deleted=True),+ _ch_map(),+ "absent",+ _v("stray", "rows_outlive_owner", None),+ ),+ (+ "stray_live_on_another_person",+ _pg_map(owner_deleted=True),+ _ch_map(owner="a"),+ "pending",+ _v("stray", "ch_owner_differs", None),+ ),+ (+ "stray_tombstone_naming_another_person",+ _pg_map(owner_deleted=True),+ _ch_map(owner="a", deleted=True),+ "pending",+ _v("stray", "ch_owner_differs", None),+ ),+ (+ "stray_owner_unchecked",+ _pg_map(owner_deleted=True),+ _ch_map(),+ None,+ _v("stray", "owner_unchecked", None),+ ),+ (+ "stray_ch_top_version_tied",+ _pg_map(owner_deleted=True),+ _ch_map(tied=True),+ "pending",+ _v("stray", "ch_tie", None),+ ),+ (+ "tombstoned_on_pending_owner_awaits_the_sweep",+ _pg_map(version=6, deleted=True, owner_deleted=True),+ _ch_map(version=5),+ "pending",+ _v("awaiting_sweep", None, None),+ ),+ (+ "tombstoned_not_above_clickhouse_is_buried",+ _pg_map(version=5, deleted=True, owner_deleted=True),+ _ch_map(version=5),+ "pending",+ _v("deleted_visible", None, "bury"),+ ),+ (+ "tombstoned_with_tied_clickhouse_rows_is_buried",+ _pg_map(version=6, deleted=True, owner_deleted=True),+ _ch_map(version=5, tied=True),+ "pending",+ _v("deleted_visible", None, "bury"),+ ),+ (+ "tombstoned_on_visible_owner_is_buried",+ _pg_map(version=6, deleted=True, owner_deleted=True),+ _ch_map(version=5),+ "visible",+ _v("deleted_visible", None, "bury"),+ ),+ (+ "tombstoned_on_live_owner_is_buried",+ _pg_map(deleted=True),+ _ch_map(),+ None,+ _v("deleted_visible", None, "bury"),+ ),+ (+ "tombstoned_ch_names_another_person_is_buried",+ _pg_map(deleted=True, owner_deleted=True),+ _ch_map(owner="a"),+ "pending",+ _v("deleted_visible", None, "bury"),+ ),+ ]+ )+ def test_classifies_by_where_the_owner_stands_in_clickhouse(self, _name, pg, ch, owner_ch, expected):+ assert classify_mapping(pg, ch, owner_ch=owner_ch) == expected++ @parameterized.expand(+ [+ ("no_rows", None, "absent"),+ ("only_lightweight_deleted_rows", _ch(tomb=4, visible_rows=0), "absent"),+ ("tombstone_newest", _ch(live=3, tomb=4), "pending"),+ ("only_a_tombstone", _ch(tomb=2), "pending"),+ ("live_newest", _ch(live=5, tomb=4), "visible"),+ ("live_and_tombstone_tie", _ch(live=4, tomb=4), "visible"),+ ]+ )+ def test_owner_ch_state_follows_the_sweeps_rule(self, _name, state, expected):+ assert owner_ch_state(state) == expected+def test_ghost_of_a_deleted_team_is_left_alone(self):assert classify_mapping(None, _ch_map(owner="a"), ch_owner_pg_state="absent", team_exists=False) == _v("ghost", "team_gone", Nonediff --git a/posthog/models/person/test/test_divergence_inspection.py b/posthog/models/person/test/test_divergence_inspection.pyindex a5027d9633dd..2159bf21e5d7 100644--- a/posthog/models/person/test/test_divergence_inspection.py+++ b/posthog/models/person/test/test_divergence_inspection.py@@ -84,13 +84,14 @@ class TestDivergenceInspection(DivergenceFixturesMixin, ClickhouseTestMixin, Basis_deleted=True,distinct_ids=["stray"],)+ self._ch_person(deleted.uuid, 3, {}, deleted=True)self._ch_mapping("stray", deleted.uuid, 0)flavors = self._flavors(inspect_mappings(self.team.pk, ["moved", "stray"]))assert flavors == {"moved": ("diverged", "a_absent", "fold"),- "stray": ("stray", None, None),+ "stray": ("stray", None, "detach"),}def test_inspect_mappings_leaves_ghosts_of_a_deleted_team_alone(self):diff --git a/posthog/models/person/test/test_divergence_split.py b/posthog/models/person/test/test_divergence_split.pynew file mode 100644index 000000000000..3e79d5aba199--- /dev/null+++ b/posthog/models/person/test/test_divergence_split.py@@ -0,0 +1,317 @@+import shlex+from io import StringIO+from pathlib import Path+from typing import Any++from posthog.test.base import BaseTest, ClickhouseTestMixin+from unittest.mock import patch++from django.core.management import CommandError, call_command+from django.utils.timezone import now++from confluent_kafka import KafkaError+from parameterized import parameterized++from posthog.clickhouse.client import sync_execute+from posthog.kafka_client.client import ClickhouseProducer, ProduceResult+from posthog.kafka_client.topics import KAFKA_PERSON, KAFKA_PERSON_DISTINCT_ID+from posthog.models.person import Person+from posthog.models.person.divergence import observe+from posthog.models.person.divergence.detach import SplitError, split_distinct_ids+from posthog.models.person.sql import BULK_INSERT_PERSON_DISTINCT_ID2+from posthog.models.person.util import (+ DistinctIdForPerson,+ _batched_get_distinct_ids_for_persons,+ create_person as create_person_in_ch,+ get_distinct_id_rows,+ get_persons_by_uuids,+)+from posthog.models.signals import mute_selected_signals+from posthog.personhog_client.fake_client import fake_personhog_client+from posthog.test.persons import create_person++PERSON_UUID = "00000000-0000-0000-0000-0000000000a1"+++def _insert_ch_mapping(team_id: int, distinct_id: str, person_uuid: str) -> None:+ sync_execute(+ BULK_INSERT_PERSON_DISTINCT_ID2,+ [+ {+ "distinct_id": distinct_id,+ "person_id": person_uuid,+ "team_id": team_id,+ "is_deleted": 0,+ "version": 0,+ "_timestamp": now().replace(tzinfo=None),+ "_offset": 0,+ "_partition": 0,+ }+ ],+ flush=False,+ )+++def _failed_produce() -> ProduceResult:+ failed = ProduceResult(topic=KAFKA_PERSON)+ failed.set_result(KafkaError(-192, "Local: Message timed out"), None)+ return failed+++@patch("posthog.models.person.util.create_person")+@patch("posthog.models.person.util.create_person_distinct_id")+class TestSplitDistinctIds(BaseTest):+ @parameterized.expand(+ [+ ("applies", ["id2", "id3"], True, ["id2", "id3"]),+ ("dry_run_writes_nothing", ["id2"], False, []),+ ("a_mistyped_id_stops_the_whole_split", ["id2", "typo"], True, SplitError),+ ("an_id_of_another_person_stops_the_whole_split", ["id2", "other-1"], True, SplitError),+ ("a_split_leaving_no_id_is_refused", ["id1", "id2", "id3"], True, SplitError),+ ("a_dry_run_leaving_no_id_is_refused", ["id1", "id2", "id3"], False, SplitError),+ ]+ )+ def test_split_moves_only_ids_live_on_the_person(+ self, _mock_create_pdi, _mock_create_person, _name, distinct_ids, apply, expected+ ):+ with fake_personhog_client() as fake:+ fake.add_person(team_id=self.team.id, person_id=501, uuid=PERSON_UUID, distinct_ids=["id1", "id2", "id3"])+ fake.add_person(+ team_id=self.team.id,+ person_id=502,+ uuid="00000000-0000-0000-0000-0000000000a2",+ distinct_ids=["other-1"],+ )++ if expected is SplitError:+ with self.assertRaises(SplitError):+ split_distinct_ids(self.team.id, PERSON_UUID, distinct_ids, apply=apply, log=lambda _: None)+ fake.assert_not_called("split_person")+ return++ assert (+ split_distinct_ids(self.team.id, PERSON_UUID, distinct_ids, apply=apply, log=lambda _: None) == expected+ )+ if apply:+ assert set(fake.assert_called("split_person", times=1)[0].request.distinct_ids_to_split) == set(+ expected+ )+ else:+ fake.assert_not_called("split_person")+ owners = {d: fake._persons_by_distinct_id[(self.team.id, d)].id for d in ["id1", "id2", "id3"]}+ assert owners["id1"] == 501+ assert all((owners[d] != 501) == (d in expected) for d in ["id2", "id3"])++ def test_split_accepts_any_uuid_spelling_and_moves_each_id_once(self, *_mocks):+ with fake_personhog_client() as fake:+ fake.add_person(team_id=self.team.id, person_id=501, uuid=PERSON_UUID, distinct_ids=["id1", "id2"])++ moved = split_distinct_ids(+ self.team.id, PERSON_UUID.upper(), ["id2", "id2"], apply=True, log=lambda _: None+ )++ assert moved == ["id2"]+ assert list(fake.assert_called("split_person", times=1)[0].request.distinct_ids_to_split) == ["id2"]++ def test_a_split_that_moves_fewer_ids_than_asked_says_so(self, *_mocks):+ with (+ fake_personhog_client() as fake,+ patch("posthog.models.person.person.Person._split_distinct_ids_batch", return_value=[]),+ ):+ fake.add_person(team_id=self.team.id, person_id=501, uuid=PERSON_UUID, distinct_ids=["id1", "id2"])++ with self.assertRaisesRegex(SplitError, "moved 0 of 1 distinct ids"):+ split_distinct_ids(self.team.id, PERSON_UUID, ["id2"], apply=True, log=lambda _: None)++ def test_a_failed_split_still_waits_for_the_messages_it_already_sent(self, *_mocks):+ with (+ fake_personhog_client() as fake,+ patch("posthog.models.person.person.Person.split_person", side_effect=RuntimeError("batch 2 failed")),+ patch("posthog.models.person.divergence.detach.flush_all_producers", return_value=0) as flush,+ ):+ fake.add_person(team_id=self.team.id, person_id=501, uuid=PERSON_UUID, distinct_ids=["id1", "id2"])++ with self.assertRaisesRegex(SplitError, "batch 2 failed"):+ split_distinct_ids(self.team.id, PERSON_UUID, ["id2"], apply=True, log=lambda _: None)++ flush.assert_called_once()+++class TestSplitLostInClickHouse(ClickhouseTestMixin, BaseTest):+ DISTINCT_IDS = ["id1", "id2", "id3", " padded id ", " ", "carriage\rreturn", "two\nlines"]++ def setUp(self) -> None:+ super().setUp()+ with mute_selected_signals():+ self.person = create_person(team=self.team, distinct_ids=self.DISTINCT_IDS)+ create_person_in_ch(team_id=self.team.pk, uuid=str(self.person.uuid), version=0, timestamp=now())+ for distinct_id in self.DISTINCT_IDS:+ _insert_ch_mapping(self.team.pk, distinct_id, str(self.person.uuid))+ sleep = patch("posthog.models.person.divergence.detach._sleep")+ self.sleep = sleep.start()+ self.addCleanup(sleep.stop)++ def _split_losing_its_messages(self, distinct_ids: list[str]) -> list[str]:+ # The split's own publishes never reach Kafka, so Kafka has nothing queued to report undelivered.+ with (+ patch("posthog.models.person.util.create_person"),+ patch("posthog.models.person.util.create_person_distinct_id"),+ ):+ return split_distinct_ids(self.team.pk, str(self.person.uuid), distinct_ids, apply=True, log=lambda _: None)++ def _assert_clickhouse_agrees_with_postgres(self, distinct_id: str) -> None:+ owner = get_distinct_id_rows(self.team.pk, [distinct_id], strong=True)[0].person_uuid+ assert owner != str(self.person.uuid)+ assert observe.ch_mapping_states(self.team.pk, [distinct_id])[distinct_id].winner_person_uuid == owner+ assert observe.ch_person_states(self.team.pk, [owner])[owner].live_max_version is not None++ def _assert_clickhouse_shows_old_owner(self, distinct_id: str) -> None:+ winner = observe.ch_mapping_states(self.team.pk, [distinct_id])[distinct_id].winner_person_uuid+ assert winner == str(self.person.uuid)++ def test_a_split_clickhouse_missed_is_sent_again_so_no_scan_suggests_merging_it_back(self) -> None:+ assert self._split_losing_its_messages(["id2"]) == ["id2"]++ self._assert_clickhouse_agrees_with_postgres("id2")+ self._assert_clickhouse_shows_old_owner("id1")+ self.sleep.assert_not_called()++ def test_a_replica_behind_the_split_is_read_again_before_the_split_reports_it(self) -> None:+ reads: list[list[str]] = []++ def lagging_replica(team_id: int, person_uuids: list[str], **kwargs: Any) -> list[Person]:+ reads.append(person_uuids)+ return [] if len(reads) == 1 else get_persons_by_uuids(team_id, person_uuids, **kwargs)++ with patch("posthog.models.person.divergence.repair_plan.get_persons_by_uuids", side_effect=lagging_replica):+ assert self._split_losing_its_messages(["id2"]) == ["id2"]++ self.sleep.assert_called_once()+ self._assert_clickhouse_agrees_with_postgres("id2")++ def _split_failing_partway(self) -> None:+ with (+ patch("posthog.models.person.person.PERSONHOG_SPLIT_BATCH_SIZE", 1),+ patch(+ "posthog.models.person.person.Person._publish_split_to_kafka",+ side_effect=[None, RuntimeError("Kafka is down")],+ ),+ ):+ split_distinct_ids(self.team.pk, str(self.person.uuid), ["id2", "id3"], apply=True, log=lambda _: None)++ def test_a_split_that_fails_partway_sends_what_it_moved(self) -> None:+ with self.assertRaisesRegex(SplitError, "moved 2 of 2 distinct ids .* then failed: Kafka is down"):+ self._split_failing_partway()++ self._assert_clickhouse_agrees_with_postgres("id2")+ self._assert_clickhouse_agrees_with_postgres("id3")++ def test_a_split_that_fails_partway_and_cannot_be_sent_again_reports_both(self) -> None:+ with (+ patch.object(ClickhouseProducer, "produce", return_value=_failed_produce()),+ self.assertRaisesRegex(SplitError, "may still show 2 .* also failed: Kafka is down. Do not run a merge"),+ ):+ self._split_failing_partway()++ @parameterized.expand(+ [+ ("listed_inline", ["id2"], 10, False),+ ("read_from_a_file", ["id2", " padded id ", " ", "carriage\rreturn"], 0, True),+ ("a_distinct_id_with_a_newline_stays_inline", ["two\nlines"], 0, False),+ ]+ )+ def test_a_split_that_cannot_be_sent_again_names_the_command_that_finishes_it(+ self, _name, distinct_ids, inline_ids, from_file+ ) -> None:+ with (+ patch("posthog.models.person.divergence.detach._INLINE_DISTINCT_IDS", inline_ids),+ patch.object(ClickhouseProducer, "produce", return_value=_failed_produce()),+ self.assertRaisesRegex(SplitError, "Do not run a merge") as raised,+ ):+ self._split_losing_its_messages(distinct_ids)++ command = shlex.split(str(raised.exception).split("Run: python manage.py ")[1])+ assert ("--distinct-ids-file" in command) == from_file+ if from_file:+ self.addCleanup(Path(command[command.index("--distinct-ids-file") + 1]).unlink)+ assert command[-1] == "--apply"+ call_command(*command[:-1], stdout=StringIO())+ for distinct_id in distinct_ids:+ self._assert_clickhouse_shows_old_owner(distinct_id)+ call_command(*command, stdout=StringIO())+ for distinct_id in distinct_ids:+ self._assert_clickhouse_agrees_with_postgres(distinct_id)+++class TestRepublishCommand(ClickhouseTestMixin, BaseTest):+ def setUp(self) -> None:+ super().setUp()+ with mute_selected_signals():+ self.person = create_person(team=self.team, distinct_ids=["id1", "id2"])++ def _republish(self, *args: str) -> None:+ call_command(+ "person_consistency", "republish", "--team-id", str(self.team.pk), *args, "--apply", stdout=StringIO()+ )++ def test_a_person_uuid_in_any_spelling_is_republished(self) -> None:+ self._republish("--person-uuid", str(self.person.uuid).upper())++ assert (+ observe.ch_person_states(self.team.pk, [str(self.person.uuid)])[str(self.person.uuid)].live_max_version+ is not None+ )++ @parameterized.expand(+ [+ ("nothing_named", [], "name at least one"),+ ("a_person_uuid_that_is_not_a_uuid", ["--person-uuid", "not-a-uuid"], "must be a uuid"),+ ("a_distinct_id_not_live_in_postgres", ["--distinct-id", "ghost"], "may still disagree .* ghost"),+ ]+ )+ def test_republish_fails_when_it_cannot_confirm_a_named_key(self, _name, args, error) -> None:+ with self.assertRaisesRegex(CommandError, error):+ self._republish(*args)++ def test_republish_fails_for_a_distinct_id_whose_owner_has_too_many_distinct_ids(self) -> None:+ with (+ patch("posthog.models.person.divergence.repair_plan._MAX_REPAIR_DISTINCT_IDS_PER_PERSON", 1),+ self.assertRaisesRegex(CommandError, "may still disagree .* id2"),+ ):+ self._republish("--distinct-id", "id2")++ assert observe.ch_mapping_states(self.team.pk, ["id2"]).get("id2") is None++ def test_republish_fails_for_a_named_person_whose_mapping_was_not_delivered(self) -> None:+ def produce(*args: Any, **kwargs: Any) -> ProduceResult:+ if kwargs["topic"] == KAFKA_PERSON_DISTINCT_ID:+ return _failed_produce()+ delivered = ProduceResult(topic=kwargs["topic"])+ delivered.set_result(None, None)+ return delivered++ with (+ patch.object(ClickhouseProducer, "produce", side_effect=produce),+ self.assertRaisesRegex(CommandError, f"may still disagree .* {self.person.uuid}"),+ ):+ self._republish("--person-uuid", str(self.person.uuid))++ def test_a_replica_listing_a_distinct_id_under_its_old_person_does_not_confirm_it(self) -> None:+ with mute_selected_signals():+ new_owner = create_person(team=self.team, distinct_ids=["moved"])+ _insert_ch_mapping(self.team.pk, "moved", str(self.person.uuid))++ def lagging_replica(team_id: int, person_ids: list[int], **kwargs: Any) -> dict[int, list[DistinctIdForPerson]]:+ by_person = _batched_get_distinct_ids_for_persons(team_id, person_ids, **kwargs)+ by_person[new_owner.pk] = [d for d in by_person.get(new_owner.pk, []) if d.id != "moved"]+ by_person.setdefault(self.person.pk, []).append(DistinctIdForPerson(id="moved", version=0))+ return by_person++ with (+ patch(+ "posthog.models.person.divergence.repair_plan._batched_get_distinct_ids_for_persons",+ side_effect=lagging_replica,+ ),+ self.assertRaisesRegex(CommandError, "may still disagree with Postgres for 1 named keys: moved\\."),+ ):+ self._republish("--person-uuid", str(self.person.uuid), "--distinct-id", "moved")diff --git a/posthog/models/person/test/test_divergence_team_scan.py b/posthog/models/person/test/test_divergence_team_scan.pyindex 617cbdd9642f..37eb7ff24c92 100644--- a/posthog/models/person/test/test_divergence_team_scan.py+++ b/posthog/models/person/test/test_divergence_team_scan.py@@ -106,7 +106,7 @@ class TestTeamScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):"x2": ("diverged", "ambiguous", None, f"ambiguous:{keys['holder']}", ()),keys["unseen"]: ("missing_in_ch", None, "republish", None, ()),"unseen-1": ("missing_in_ch", None, "republish", None, ()),- "stray-1": ("stray", None, None, None, ()),+ "stray-1": ("stray", None, "detach", None, ()),keys["lonely"]: ("no_live_mappings", None, None, None, ()),}summary = json.loads((scan_dir / team_scan.SUMMARY_FILE).read_text())diff --git a/posthog/models/person/test/test_divergence_verbs.py b/posthog/models/person/test/test_divergence_verbs.pyindex 7dfbccbf0aa8..2e77cfe66980 100644--- a/posthog/models/person/test/test_divergence_verbs.py+++ b/posthog/models/person/test/test_divergence_verbs.py@@ -17,9 +17,11 @@ from parameterized import parameterizedfrom posthog.kafka_client.client import ClickhouseProducer, ProduceResultfrom posthog.kafka_client.topics import KAFKA_PERSONfrom posthog.models.person.divergence import repairs, team_scan, verbs+from posthog.models.person.divergence.cleanup_queue import PG_CLEANUP_QUEUE_TABLEfrom posthog.models.person.test.divergence_fixtures import DivergenceFixturesMixinfrom posthog.models.person.util import tombstone_persons_in_postgresfrom posthog.personhog_client.fake_client import get_active_fake+from posthog.persons_db import persons_db_connectionclass _Stop(Exception):@@ -112,7 +114,7 @@ class TestRepairVerbs(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):assert resumed == [k for k in (str(first.uuid), str(second.uuid)) if k != canary_persons[0][0]]assert self._scan()[1] == {}- def test_bury_tombstones_what_postgres_deleted_and_leaves_strays_to_ingestion(self):+ def test_bury_and_detach_tombstone_what_postgres_deleted(self):deleted = self._pg_person(version=2)tombstone_persons_in_postgres(self.team.pk, [deleted.uuid])self._ch_person(str(deleted.uuid), 7, {"email": "[email protected]"})@@ -131,31 +133,166 @@ class TestRepairVerbs(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):self._ch_mapping("gone", holder.uuid, 5)self._ch_mapping("unpublished", holder.uuid, 1)self._ch_mapping("holder-1", holder.uuid, 0)- stray_owner = fake.add_person(- team_id=self.team.pk,- person_id=990_002,- uuid=str(uuid4()),- version=3,- is_deleted=True,- distinct_ids=["stray-1"],- )- self._ch_mapping("stray-1", stray_owner.uuid, 0)++ def stray_owner(person_id: int, *distinct_ids: str) -> str:+ return fake.add_person(+ team_id=self.team.pk,+ person_id=person_id,+ uuid=str(uuid4()),+ version=3,+ is_deleted=True,+ distinct_ids=list(distinct_ids),+ ).uuid++ # No ClickHouse rows for the strays.+ nothing_in_ch = stray_owner(990_010, "s-none", "s-later-diverged")+ # ClickHouse resolves the stray to its owner, which still waits for the sweep, at a version above Postgres.+ pending = stray_owner(990_011, "s-pending")+ self._ch_person(pending, 3, {}, deleted=True)+ self._ch_mapping("s-pending", pending, 5)+ # The owner's tombstone never reached ClickHouse.+ visible = stray_owner(990_012, "s-owner-visible")+ self._ch_person(visible, 2, {"email": "[email protected]"})+ self._ch_mapping("s-owner-visible", visible, 0)+ # ClickHouse resolves the stray to another person.+ differs = stray_owner(990_013, "s-differs")+ self._ch_person(differs, 3, {}, deleted=True)+ self._ch_mapping("s-differs", holder.uuid, 4)+ # The owner is gone from ClickHouse, but the stray still has rows there.+ outlived = stray_owner(990_014, "s-outlived")+ self._ch_mapping("s-outlived", outlived, 0)+ # A sweep already queued the owner, so it is swept even though its ClickHouse rows show up again.+ requeued = stray_owner(990_015, "s-requeued")+ self._ch_person(requeued, 3, {}, deleted=True)+ self._ch_mapping("s-requeued", requeued, 0)+ # Rows at ClickHouse's top version name two owners.+ tied = stray_owner(990_016, "s-tied")+ self._ch_person(tied, 3, {}, deleted=True)+ self._ch_mapping("s-tied", tied, 2)+ self._ch_mapping("s-tied", holder.uuid, 2)+ # Its only stray has no ClickHouse rows, so detaching it leaves the owner nothing live.+ released = stray_owner(990_017, "s-released")+ # Queued by an earlier sweep, then tombstoned again above that bound: the next sweep takes it.+ retombstoned = stray_owner(990_018, "s-retombstoned")+ self._ch_person(retombstoned, 5, {}, deleted=True)+ self._ch_mapping("s-retombstoned", retombstoned, 0)+ self._queue_person(retombstoned, max_version=3)+ self._queue_person(nothing_in_ch, blocked=True)+ self._queue_person(requeued, blocked=True)+ self._queue_person(released, blocked=True)+scan_dir, before = self._scan()assert before == {str(deleted.uuid): ("deleted_visible", "bury"),"gone": ("deleted_visible", "bury"),"unpublished": ("deleted_visible", "bury"),- "stray-1": ("stray", None),+ visible: ("deleted_visible", "bury"),+ "s-none": ("stray", "detach"),+ "s-later-diverged": ("stray", "detach"),+ "s-pending": ("stray", "detach"),+ "s-released": ("stray", "detach"),+ "s-retombstoned": ("stray", "detach"),+ "s-owner-visible": ("stray", None),+ "s-differs": ("stray", None),+ "s-outlived": ("stray", None),+ "s-requeued": ("stray", None),+ "s-tied": ("stray", None),}+ assert self._sub_flavors(scan_dir, "s-owner-visible", "s-differs", "s-outlived", "s-requeued", "s-tied") == [+ "owner_visible",+ "ch_owner_differs",+ "rows_outlive_owner",+ "rows_outlive_owner",+ "ch_tie",+ ]buried = self._run(scan_dir, "bury", apply=True)+ # ClickHouse changes after the scan: detach re-checks before it writes.+ self._ch_mapping("s-later-diverged", holder.uuid, 2)+ planned = self._run(scan_dir, "detach", apply=False, name="dry")+ detached = self._run(scan_dir, "detach", apply=True)assert {a["key"]: a["outcome"] for a in buried} == {str(deleted.uuid): "republished","gone": "buried","unpublished": "buried",+ visible: "republished",+ }+ assert {a["key"]: a["outcome"] for a in planned} == {+ "s-none": "would_detach",+ "s-later-diverged": "skipped_ch_owner_differs",+ "s-pending": "would_detach",+ "s-released": "would_detach",+ "s-retombstoned": "would_detach",+ }+ assert {a["key"]: (a["outcome"], a["detail"].get("version")) for a in detached} == {+ "s-none": ("detached", 1),+ "s-later-diverged": ("skipped_ch_owner_differs", None),+ "s-pending": ("detached", 6),+ "s-released": ("detached", 1),+ "s-retombstoned": ("detached", 1),+ }+ # Released only once the owner has no live distinct id left: s-later-diverged still holds its owner.+ assert not self._queue_blocked(released)+ assert self._queue_blocked(nothing_in_ch)+ assert self._queue_blocked(requeued)++ rescan_dir, after = self._scan()+ assert after == {+ "s-later-diverged": ("stray", None),+ "s-pending": ("awaiting_sweep", None),+ "s-retombstoned": ("awaiting_sweep", None),+ "s-owner-visible": ("stray", "detach"),+ "s-differs": ("stray", None),+ "s-outlived": ("stray", None),+ "s-requeued": ("stray", None),+ "s-tied": ("stray", None),}- assert self._scan()[1] == {"stray-1": ("stray", None)}++ assert [(a["key"], a["outcome"]) for a in self._run(rescan_dir, "detach", apply=True)] == [+ ("s-owner-visible", "detached")+ ]+ assert self._scan()[1]["s-owner-visible"] == ("awaiting_sweep", None)++ # A report from before the owner was pending still names bury; bury re-checks and leaves it to the sweep.+ report = rescan_dir / team_scan.REPORT_FILE+ stale = [json.loads(line) for line in report.read_text().splitlines()]+ report.write_text(+ "".join(json.dumps({**r, "verb": "bury"} if r["key"] == "s-pending" else r) + "\n" for r in stale)+ )+ assert [(a["key"], a["outcome"]) for a in self._run(rescan_dir, "bury", apply=True)] == [+ ("s-pending", "skipped_awaiting_sweep")+ ]++ def test_bury_only_raises_postgres_for_a_mapping_the_sweep_removes_with_its_owner(self):+ fake = get_active_fake()+ assert fake is not None+ owner = fake.add_person(+ team_id=self.team.pk,+ person_id=990_020,+ uuid=str(uuid4()),+ version=3,+ is_deleted=True,+ distinct_ids=["t-below"],+ distinct_id_versions={"t-below": 2},+ tombstoned_distinct_ids=["t-below"],+ ).uuid+ self._ch_person(owner, 3, {}, deleted=True)+ self._ch_mapping("t-below", owner, 4)+ scan_dir, before = self._scan()+ assert before == {"t-below": ("deleted_visible", "bury")}++ planned = self._run(scan_dir, "bury", apply=False, name="dry")+ raised = self._run(scan_dir, "bury", apply=True)++ assert [(a["outcome"], a["detail"]["target_version"]) for a in planned] == [("would_raise_for_sweep", 5)]+ assert [(a["outcome"], a["detail"]["version"]) for a in raised] == [("raised_for_sweep", 5)]+ # Still live in ClickHouse, so no tombstone was published; Postgres is now above it.+ assert self._scan()[1] == {"t-below": ("awaiting_sweep", None)}++ def _sub_flavors(self, scan_dir: Path, *keys: str) -> list[str | None]:+ rows = {r["key"]: r for r in map(json.loads, (scan_dir / team_scan.REPORT_FILE).read_text().splitlines())}+ return [rows[key]["sub_flavor"] for key in keys]def _hidden_person_with_stale_mapping(self) -> str:person = self._pg_person(version=1, distinct_ids=["h-1"])@@ -243,6 +380,24 @@ class TestRepairVerbs(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):assert self._scan()[1] == {}+ def _queue_person(self, person_uuid: str, *, blocked: bool = False, max_version: int = 3) -> None:+ with persons_db_connection(writer=True) as conn, conn.cursor() as cursor:+ cursor.execute(+ f"INSERT INTO {PG_CLEANUP_QUEUE_TABLE} (team_id, person_uuid, deleted_at, max_version, blocked_at) "+ f"VALUES (%s, %s, now(), %s, {'now()' if blocked else 'NULL'})",+ [self.team.pk, person_uuid, max_version],+ )++ def _queue_blocked(self, person_uuid: str) -> bool:+ with persons_db_connection(writer=False) as conn, conn.cursor() as cursor:+ cursor.execute(+ f"SELECT blocked_at IS NOT NULL FROM {PG_CLEANUP_QUEUE_TABLE} WHERE team_id = %s AND person_uuid = %s",+ [self.team.pk, person_uuid],+ )+ row = cursor.fetchone()+ assert row is not None+ return row[0]+def test_a_run_stopped_by_sigterm_records_its_undelivered_keys_for_resume(self):key = self._hidden_person_with_stale_mapping()scan_dir, _ = self._scan()diff --git a/posthog/personhog_client/fake_client.py b/posthog/personhog_client/fake_client.pyindex 4c2a596b5842..51323769b0a6 100644--- a/posthog/personhog_client/fake_client.py+++ b/posthog/personhog_client/fake_client.py@@ -194,6 +194,12 @@ class FakePersonHogClient:if uuid:self._persons_by_uuid[(team_id, uuid)] = personfor did in distinct_ids or []:+ previous = self._persons_by_distinct_id.get((team_id, did))+ if previous is not None and previous.id != person_id:+ # The replica keeps one row per distinct id, so the previous owner no longer lists it.+ self._distinct_ids[(team_id, previous.id)] = [+ d for d in self._distinct_ids.get((team_id, previous.id), []) if d.distinct_id != did+ ]self._persons_by_distinct_id[(team_id, did)] = person# A re-added mapping is live again, as a revival upsert makes it in the replica.self._tombstoned_distinct_ids.discard((team_id, did))@@ -359,7 +365,7 @@ class FakePersonHogClient:self, request: person_pb2.GetDistinctIdsForPersonRequest) -> person_pb2.GetDistinctIdsForPersonResponse:self.calls.append(_Call("get_distinct_ids_for_person", request))- dids = list(self._distinct_ids.get((request.team_id, request.person_id), []))+ dids = self._live_distinct_ids(request.team_id, request.person_id)limit = request.limit if request.HasField("limit") and request.limit > 0 else Nonehas_cursor = request.HasField("cursor_id")cursor_id = request.cursor_id if has_cursor else None@@ -386,6 +392,14 @@ class FakePersonHogClient:next_cursor_id=next_cursor_id,)+ def _live_distinct_ids(self, team_id: int, person_id: int) -> list[person_pb2.DistinctIdWithVersion]:+ # The replica reads only rows with is_deleted = false, so a tombstoned mapping is never listed.+ return [+ d+ for d in self._distinct_ids.get((team_id, person_id), [])+ if (team_id, d.distinct_id) not in self._tombstoned_distinct_ids+ ]+def get_distinct_ids_for_persons(self, request: person_pb2.GetDistinctIdsForPersonsRequest) -> person_pb2.GetDistinctIdsForPersonsResponse:@@ -395,7 +409,7 @@ class FakePersonHogClient:)results = []for pid in request.person_ids:- dids = self._distinct_ids.get((request.team_id, pid), [])+ dids = self._live_distinct_ids(request.team_id, pid)if limit is not None:dids = _order_identified_first(dids)[:limit]results.append(person_pb2.PersonDistinctIds(person_id=pid, distinct_ids=dids))