Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(persons): detach strays and split distinct ids with person_consistency (#126688)

Eli Reismanb6e9c8e8c64b
diff --git a/posthog/dags/clickhouse_cleanup.py b/posthog/dags/clickhouse_cleanup.py
index 5657cc688ab2..bc40d9b20529 100644
--- a/posthog/dags/clickhouse_cleanup.py
+++ b/posthog/dags/clickhouse_cleanup.py
@@ -54,12 +54,11 @@ from posthog.dags.person_tombstone_queue import publish_queue_gauges, resolve_pe
from posthog.dataclasses import frozen
from posthog.metrics import pushed_metrics_registry
from posthog.models.async_deletion.delete_cohorts import sweep_cohort_deletions
+from posthog.models.person.divergence.cleanup_queue import PG_CLEANUP_QUEUE_TABLE
from posthog.models.person.sql import PERSON_DISTINCT_ID2_TABLE, PERSONS_TABLE
logger = dagster.get_dagster_logger(__name__)
-PG_CLEANUP_QUEUE_TABLE = "person_pg_cleanup_queue"
-
# How many queued persons travel per ClickHouse read and per Postgres upsert transaction. Bounds
# the op's memory to one page however large the snapshot is.
PERSIST_PAGE_SIZE = 50_000
diff --git a/posthog/dags/detach_distinct_id.py b/posthog/dags/detach_distinct_id.py
deleted file mode 100644
index 93f4498c86bc..000000000000
--- a/posthog/dags/detach_distinct_id.py
+++ /dev/null
@@ -1,338 +0,0 @@
-"""Dagster job to detach a distinct_id from its person.
-
-Three cleanup phases, all required:
- 1. Postgres: tombstone the posthog_persondistinctid row (stops future ingestion lookups).
- The tombstone version is above both the row's own version and the highest version ClickHouse
- holds for the distinct id, so the ClickHouse tombstone and override win, and a later re-add
- revives the row above them instead of starting at version 0 underneath them.
- 2. Kafka: publish is_deleted to person_distinct_id2 at that exact version (stops ClickHouse lookups).
- 3. Override: insert into person_distinct_id_overrides so the HogQL query layer
- immediately re-attributes historical events whose person_id was baked in at
- ingestion time (Person-on-Events). squash_person_overrides later makes it
- permanent and removes the override row.
-
-When no ``override_person_id`` is supplied a random dummy UUID is used, effectively
-orphaning the events (the dummy person has no record in the person table).
-
-Typical use: removing a sentinel ``$posthog_cookieless`` distinct_id that was
-erroneously associated with a real person due to a cookieless-ingestion bug.
-
-Example Dagster launchpad config (run with dry_run: true first!)::
-
- ops:
- detach_distinct_id_op:
- config:
- team_id: 12345
- distinct_id: "$posthog_cookieless"
- expected_person_id: "5e00024e-cb68-59f6-821f-6150fcffc431"
- dry_run: true
-"""
-
-import json
-import uuid
-import shlex
-from datetime import datetime
-
-import dagster
-import pydantic
-import psycopg2.extensions
-
-from posthog.clickhouse.client import sync_execute
-from posthog.dags.common import JobOwners
-from posthog.kafka_client.client import _KafkaProducer
-from posthog.kafka_client.topics import KAFKA_PERSON_DISTINCT_ID
-
-KAFKA_DELIVERY_TIMEOUT_SECONDS = 30
-
-
-class DetachDistinctIdConfig(dagster.Config):
- """Configuration for the detach distinct_id job."""
-
- team_id: int = pydantic.Field(description="Team ID that owns the distinct_id")
- distinct_id: str = pydantic.Field(description="The distinct_id to detach (e.g. $posthog_cookieless)")
- expected_person_id: str = pydantic.Field(
- description="UUID of the person we expect the distinct_id to belong to (safety check)"
- )
- override_person_id: str | None = pydantic.Field(
- default=None,
- description=(
- "If set, insert a person_distinct_id_overrides row pointing historical events "
- "at this person UUID. If not set, a random dummy UUID is generated so events "
- "become orphaned. The scheduled squash job will later rewrite the embedded "
- "person_id on events and delete the override."
- ),
- )
- dry_run: bool = pydantic.Field(
- default=True,
- description="If true, log what would happen without making changes",
- )
-
-
-def _lookup_distinct_id(
- cursor: psycopg2.extensions.cursor,
- team_id: int,
- distinct_id: str,
-) -> dict | None:
- """Return the posthog_persondistinctid row + person uuid, or None."""
- cursor.execute(
- """
- SELECT pdi.id, pdi.version, pdi.person_id, p.uuid
- FROM posthog_persondistinctid pdi
- JOIN posthog_person p ON p.id = pdi.person_id AND p.team_id = pdi.team_id
- WHERE pdi.team_id = %s AND pdi.distinct_id = %s
- AND pdi.is_deleted = false AND p.is_deleted = false
- """,
- [team_id, distinct_id],
- )
- row = cursor.fetchone()
- if row is None:
- return None
- # Handle both dict rows (RealDictCursor) and tuple rows (regular cursor)
- if isinstance(row, dict):
- return {
- "pdi_id": row["id"],
- "version": row["version"],
- "person_pk": row["person_id"],
- "person_uuid": str(row["uuid"]),
- }
- return {
- "pdi_id": row[0],
- "version": row[1],
- "person_pk": row[2],
- "person_uuid": str(row[3]),
- }
-
-
-def _count_other_distinct_ids(
- cursor: psycopg2.extensions.cursor,
- team_id: int,
- person_pk: int,
- exclude_pdi_id: int,
-) -> int:
- """Count how many *other* distinct_ids the person has."""
- cursor.execute(
- """
- SELECT COUNT(*) AS count
- FROM posthog_persondistinctid
- WHERE team_id = %s AND person_id = %s AND id != %s AND is_deleted = false
- """,
- [team_id, person_pk, exclude_pdi_id],
- )
- row = cursor.fetchone()
- return row["count"] if isinstance(row, dict) else row[0]
-
-
-def _clickhouse_max_version(team_id: int, distinct_id: str) -> int:
- """Return the highest ClickHouse version of the distinct id across the mapping and override tables, or 0."""
- rows = sync_execute(
- """
- SELECT greatest(
- (SELECT max(version) FROM person_distinct_id2
- WHERE team_id = %(team_id)s AND distinct_id = %(distinct_id)s),
- (SELECT max(version) FROM person_distinct_id_overrides
- WHERE team_id = %(team_id)s AND distinct_id = %(distinct_id)s)
- )
- """,
- {"team_id": team_id, "distinct_id": distinct_id},
- )
- return int(rows[0][0])
-
-
-def _tombstone_distinct_id_row(
- cursor: psycopg2.extensions.cursor,
- pdi_id: int,
- person_pk: int,
- min_version: int,
-) -> int:
- """Tombstone the row at ``min_version`` or above while it still belongs to ``person_pk``; return its version."""
- cursor.execute(
- """
- UPDATE posthog_persondistinctid
- SET is_deleted = true, version = GREATEST(COALESCE(version, 0) + 1, %s)
- WHERE id = %s AND person_id = %s AND is_deleted = false
- RETURNING version
- """,
- [min_version, pdi_id, person_pk],
- )
- row = cursor.fetchone()
- if row is None:
- raise RuntimeError(
- f"posthog_persondistinctid id={pdi_id} disappeared or moved off person {person_pk} between lookup and tombstone"
- )
- return row["version"] if isinstance(row, dict) else row[0]
-
-
-def _publish_deletion_to_kafka(
- producer: _KafkaProducer,
- team_id: int,
- distinct_id: str,
- person_uuid: str,
- version: int,
-) -> None:
- """Publish an is_deleted message so ClickHouse person_distinct_id2 drops the mapping.
-
- ``version`` is the one the Postgres tombstone carries, so a revived mapping lands above it.
- """
- result = producer.produce(
- topic=KAFKA_PERSON_DISTINCT_ID,
- data={
- "distinct_id": distinct_id,
- "person_id": person_uuid,
- "team_id": team_id,
- "version": version,
- "is_deleted": 1,
- },
- )
- producer.flush(KAFKA_DELIVERY_TIMEOUT_SECONDS)
- # flush() counts only messages still queued. A failed delivery leaves the queue, so only its result shows it.
- result.get(timeout=0)
-
-
-def _undelivered_tombstone_message(team_id: int, distinct_id: str, override_person_uuid: str, version: int) -> str:
- insert_override = (
- "from posthog.dags.detach_distinct_id import _insert_ch_override; "
- f"_insert_ch_override({team_id}, {json.dumps(distinct_id)}, {json.dumps(override_person_uuid)}, {version + 1})"
- )
- return (
- f"Postgres tombstoned distinct_id={distinct_id!r} at version {version}, but Kafka did not deliver the "
- "ClickHouse tombstone, so the override was not inserted. A rerun of this job cannot find the row. "
- "Recover with:\n"
- f" 1. python manage.py sync_persons_to_clickhouse --person-distinct-id --deletes --live-run --team-id {team_id}\n"
- f" 2. python manage.py shell -c {shlex.quote(insert_override)}"
- )
-
-
-def _insert_ch_override(
- team_id: int,
- distinct_id: str,
- override_person_uuid: str,
- version: int,
-) -> None:
- """Insert a person_distinct_id_overrides row so the squash job re-attributes historical events.
-
- Follows the same pattern as insert_override_batch in fix_person_id_overrides.py.
- """
- sync_execute(
- """
- INSERT INTO person_distinct_id_overrides
- (team_id, distinct_id, person_id, is_deleted, version, _timestamp, _offset, _partition)
- VALUES
- """,
- [(team_id, distinct_id, override_person_uuid, 0, version, datetime.now(), 0, 0)],
- )
-
-
-def detach_distinct_id_op(
- context: dagster.OpExecutionContext,
- config: DetachDistinctIdConfig,
- persons_database: dagster.ResourceParam[psycopg2.extensions.connection],
- kafka_producer: dagster.ResourceParam[_KafkaProducer],
-) -> None:
- """Detach a distinct_id from its person in Postgres and ClickHouse."""
- log = context.log
-
- log.info(f"team_id={config.team_id} distinct_id={config.distinct_id!r}")
- log.info(f"expected_person_id={config.expected_person_id}")
- log.info(f"dry_run={config.dry_run}")
-
- # --- 1. Validate in Postgres ---
- with persons_database.cursor() as cursor:
- cursor.execute("SET application_name = 'detach_distinct_id'")
- cursor.execute("SET lock_timeout = '5s'")
- cursor.execute("SET statement_timeout = '30s'")
-
- info = _lookup_distinct_id(cursor, config.team_id, config.distinct_id)
- if info is None:
- raise dagster.Failure(f"distinct_id={config.distinct_id!r} not found for team_id={config.team_id}")
-
- log.info(
- f"Found: pdi_id={info['pdi_id']} version={info['version']} "
- f"person_uuid={info['person_uuid']} person_pk={info['person_pk']}"
- )
-
- if info["person_uuid"] != config.expected_person_id:
- raise dagster.Failure(
- f"Person mismatch: distinct_id belongs to {info['person_uuid']}, expected {config.expected_person_id}"
- )
-
- other_count = _count_other_distinct_ids(cursor, config.team_id, info["person_pk"], info["pdi_id"])
- if other_count == 0:
- raise dagster.Failure(
- f"Cannot detach: this is the person's only distinct_id. "
- f"Detaching would orphan person {info['person_uuid']}"
- )
- log.info(f"Person has {other_count} other distinct_id(s) — safe to detach")
-
- # --- 2. Resolve override target ---
- override_target = config.override_person_id or str(uuid.uuid4())
- log.info(f"Override target person_id={override_target}")
-
- # --- 3. Tombstone in Postgres ---
- if config.dry_run:
- log.info("[DRY RUN] Would tombstone posthog_persondistinctid row")
- log.info("[DRY RUN] Would publish Kafka deletion to person_distinct_id2")
- log.info(f"[DRY RUN] Would insert person_distinct_id_overrides -> {override_target}")
- persons_database.rollback()
- return
-
- ch_max_version = _clickhouse_max_version(config.team_id, config.distinct_id)
- version = _tombstone_distinct_id_row(
- cursor, info["pdi_id"], person_pk=info["person_pk"], min_version=ch_max_version + 1
- )
- persons_database.commit()
- log.info(
- f"Tombstoned posthog_persondistinctid id={info['pdi_id']} "
- f"(version={version}, ClickHouse max version={ch_max_version})"
- )
-
- # --- 4. Sync deletion to ClickHouse via Kafka ---
- try:
- _publish_deletion_to_kafka(
- kafka_producer,
- team_id=config.team_id,
- distinct_id=config.distinct_id,
- person_uuid=info["person_uuid"],
- version=version,
- )
- except Exception as e:
- raise dagster.Failure(
- _undelivered_tombstone_message(config.team_id, config.distinct_id, override_target, version)
- ) from e
- log.info(f"Published deletion to {KAFKA_PERSON_DISTINCT_ID} (version={version}, is_deleted=1)")
-
- # --- 5. Override: fix person_id baked into historical events (see module docstring) ---
- _insert_ch_override(
- team_id=config.team_id,
- distinct_id=config.distinct_id,
- override_person_uuid=override_target,
- version=version + 1,
- )
- log.info(f"Inserted person_distinct_id_overrides: {config.distinct_id!r} -> {override_target}")
-
- # --- 6. Verification hints ---
- log.info("--- Verification queries (run manually) ---")
- escaped_did = json.dumps(config.distinct_id)
- log.info(
- f"Postgres: SELECT * FROM posthog_persondistinctid "
- f"WHERE team_id = {config.team_id} AND distinct_id = {escaped_did}"
- )
- log.info(
- f"ClickHouse (pdi2): SELECT distinct_id, argMax(person_id, version), "
- f"argMax(is_deleted, version), max(version) "
- f"FROM person_distinct_id2 "
- f"WHERE team_id = {config.team_id} AND distinct_id = {escaped_did} "
- f"GROUP BY distinct_id"
- )
- log.info(
- f"ClickHouse (overrides): SELECT distinct_id, argMax(person_id, version), max(version) "
- f"FROM person_distinct_id_overrides "
- f"WHERE team_id = {config.team_id} AND distinct_id = {escaped_did} "
- f"GROUP BY distinct_id"
- )
-
-
[email protected](tags={"owner": JobOwners.TEAM_INGESTION.value})
-def detach_distinct_id_job():
- """Job to detach a distinct_id from its person in Postgres and ClickHouse."""
- detach_distinct_id_op()
diff --git a/posthog/dags/locations/ingestion.py b/posthog/dags/locations/ingestion.py
index fd4a03b2294e..13fb3456f155 100644
--- a/posthog/dags/locations/ingestion.py
+++ b/posthog/dags/locations/ingestion.py
@@ -1,6 +1,6 @@
import dagster
-from posthog.dags import detach_distinct_id, distinct_id_usage, ingestion_assets, person_property_reconciliation
+from posthog.dags import distinct_id_usage, ingestion_assets, person_property_reconciliation
from . import loggers, resources
@@ -9,7 +9,6 @@ defs = dagster.Definitions(
ingestion_assets.postgres_env_check,
],
jobs=[
- detach_distinct_id.detach_distinct_id_job,
distinct_id_usage.distinct_id_usage_monitoring,
person_property_reconciliation.person_property_reconciliation_job,
],
diff --git a/posthog/dags/owners.yaml b/posthog/dags/owners.yaml
index ab44c8654ff1..65a0f1eec679 100644
--- a/posthog/dags/owners.yaml
+++ b/posthog/dags/owners.yaml
@@ -2,7 +2,6 @@ version: 1
owners: []
rules:
- match:
- - '/detach_distinct_id.py'
- '/fix_person_id_overrides.py'
- '/person_property_reconciliation.py'
- '/tests/test_person_property_reconciliation.py'
diff --git a/posthog/dags/tests/test_detach_distinct_id.py b/posthog/dags/tests/test_detach_distinct_id.py
deleted file mode 100644
index f4102bcb4f5a..000000000000
--- a/posthog/dags/tests/test_detach_distinct_id.py
+++ /dev/null
@@ -1,618 +0,0 @@
-import uuid as uuid_module
-import contextlib
-from datetime import datetime
-from uuid import UUID
-
-import pytest
-from unittest.mock import MagicMock, patch
-
-from confluent_kafka import KafkaError, KafkaException
-
-from posthog.clickhouse.client import sync_execute
-from posthog.dags.detach_distinct_id import (
- _count_other_distinct_ids,
- _insert_ch_override,
- _lookup_distinct_id,
- _publish_deletion_to_kafka,
- _tombstone_distinct_id_row,
- detach_distinct_id_job,
-)
-from posthog.kafka_client.topics import KAFKA_PERSON_DISTINCT_ID
-from posthog.models.person.sql import (
- TRUNCATE_PERSON_DISTINCT_ID2_TABLE_SQL,
- TRUNCATE_PERSON_DISTINCT_ID_OVERRIDES_TABLE_SQL,
-)
-from posthog.models.person.util import create_person_distinct_id
-from posthog.persons_db import persons_db_connection
-
-PERSON_UUID = "5e00024e-cb68-59f6-821f-6150fcffc431"
-PERSON_PK = 42
-PDI_ID = 99
-PDI_VERSION = 3
-TEAM_ID = 7
-DUMMY_OVERRIDE_UUID = "00000000-0000-0000-0000-000000000001"
-
-# The integration class writes persons to the persons DB directly, so it must run against the real
-# persons DB rather than the personhog fake.
-pytestmark = pytest.mark.persons_db_direct
-
-
-def _make_cursor(fetchone_values: list | None = None) -> MagicMock:
- cursor = MagicMock()
- if fetchone_values is not None:
- cursor.fetchone.side_effect = fetchone_values
- return cursor
-
-
-def _override_insert_rows(mock_sync_execute: MagicMock) -> list:
- [insert] = [c for c in mock_sync_execute.call_args_list if "INSERT INTO person_distinct_id_overrides" in c.args[0]]
- return insert.args[1]
-
-
-class TestLookupDistinctId:
- def test_returns_row_when_found_tuple(self):
- cursor = _make_cursor(fetchone_values=[(PDI_ID, PDI_VERSION, PERSON_PK, UUID(PERSON_UUID))])
-
- result = _lookup_distinct_id(cursor, TEAM_ID, "$posthog_cookieless")
-
- assert result is not None
- assert result["pdi_id"] == PDI_ID
- assert result["version"] == PDI_VERSION
- assert result["person_pk"] == PERSON_PK
- assert result["person_uuid"] == PERSON_UUID
-
- def test_returns_row_when_found_dict(self):
- cursor = _make_cursor(
- fetchone_values=[{"id": PDI_ID, "version": PDI_VERSION, "person_id": PERSON_PK, "uuid": UUID(PERSON_UUID)}]
- )
-
- result = _lookup_distinct_id(cursor, TEAM_ID, "$posthog_cookieless")
-
- assert result is not None
- assert result["pdi_id"] == PDI_ID
- assert result["version"] == PDI_VERSION
- assert result["person_pk"] == PERSON_PK
- assert result["person_uuid"] == PERSON_UUID
-
- def test_returns_none_when_not_found(self):
- cursor = _make_cursor(fetchone_values=[None])
-
- result = _lookup_distinct_id(cursor, TEAM_ID, "nonexistent")
-
- assert result is None
-
-
-class TestCountOtherDistinctIds:
- def test_returns_count_tuple(self):
- cursor = _make_cursor(fetchone_values=[(5,)])
-
- count = _count_other_distinct_ids(cursor, TEAM_ID, PERSON_PK, PDI_ID)
-
- assert count == 5
-
- def test_returns_count_dict(self):
- cursor = _make_cursor(fetchone_values=[{"count": 5}])
-
- count = _count_other_distinct_ids(cursor, TEAM_ID, PERSON_PK, PDI_ID)
-
- assert count == 5
-
- def test_returns_zero_when_none(self):
- cursor = _make_cursor(fetchone_values=[(0,)])
-
- count = _count_other_distinct_ids(cursor, TEAM_ID, PERSON_PK, PDI_ID)
-
- assert count == 0
-
-
-class TestTombstoneDistinctIdRow:
- @pytest.mark.parametrize("row", [(PDI_VERSION + 1,), {"version": PDI_VERSION + 1}])
- def test_tombstones_the_row_and_returns_the_stamped_version(self, row):
- cursor = _make_cursor(fetchone_values=[row])
-
- version = _tombstone_distinct_id_row(cursor, PDI_ID, person_pk=PERSON_PK, min_version=1)
-
- assert version == PDI_VERSION + 1
- sql = cursor.execute.call_args.args[0]
- assert "UPDATE posthog_persondistinctid" in sql
- assert "is_deleted = true" in sql
- assert "RETURNING version" in sql
- assert "DELETE" not in sql
-
- def test_raises_if_row_disappears(self):
- cursor = _make_cursor(fetchone_values=[None])
-
- with pytest.raises(RuntimeError, match="disappeared"):
- _tombstone_distinct_id_row(cursor, PDI_ID, person_pk=PERSON_PK, min_version=1)
-
-
-class TestPublishDeletionToKafka:
- def test_publishes_at_the_tombstone_version(self):
- producer = MagicMock()
-
- _publish_deletion_to_kafka(producer, TEAM_ID, "$posthog_cookieless", PERSON_UUID, PDI_VERSION + 1)
-
- producer.produce.assert_called_once_with(
- topic=KAFKA_PERSON_DISTINCT_ID,
- data={
- "distinct_id": "$posthog_cookieless",
- "person_id": PERSON_UUID,
- "team_id": TEAM_ID,
- "version": PDI_VERSION + 1,
- "is_deleted": 1,
- },
- )
- producer.flush.assert_called_once()
-
-
-class TestInsertChOverride:
- @patch("posthog.dags.detach_distinct_id.sync_execute")
- def test_inserts_override_row(self, mock_sync_execute):
- _insert_ch_override(TEAM_ID, "$posthog_cookieless", DUMMY_OVERRIDE_UUID, PDI_VERSION + 1)
-
- mock_sync_execute.assert_called_once()
- sql = mock_sync_execute.call_args.args[0]
- assert "person_distinct_id_overrides" in sql
-
- rows = mock_sync_execute.call_args.args[1]
- assert len(rows) == 1
- row = rows[0]
- assert row[0] == TEAM_ID
- assert row[1] == "$posthog_cookieless"
- assert row[2] == DUMMY_OVERRIDE_UUID
- assert row[3] == 0 # is_deleted
- assert row[4] == PDI_VERSION + 1 # version
-
-
-class TestDetachDistinctIdJob:
- """Run the full Dagster job with mock Postgres (psycopg2), Kafka, and ClickHouse
- (sync_execute). Verifies the three cleanup phases only fire when all safety checks pass."""
-
- def _make_connection(self, lookup_row, other_count, tombstone_version=PDI_VERSION + 1):
- """Build a mock psycopg2 connection with a cursor that responds to the
- queries issued by detach_distinct_id_op in order."""
- conn = MagicMock(spec=["cursor", "commit", "rollback"])
- cursor = MagicMock()
-
- # The op issues fetchone calls in this order:
- # 1. _lookup_distinct_id
- # 2. _count_other_distinct_ids
- # 3. _tombstone_distinct_id_row UPDATE ... RETURNING (only when not dry_run)
- fetchone_sequence = [lookup_row, (other_count,)]
- if tombstone_version is not None:
- fetchone_sequence.append((tombstone_version,))
-
- cursor.fetchone.side_effect = fetchone_sequence
- cursor.__enter__ = MagicMock(return_value=cursor)
- cursor.__exit__ = MagicMock(return_value=False)
- conn.cursor.return_value = cursor
- return conn, cursor
-
- def _run_config(self, *, dry_run: bool = True, override_person_id: str | None = None) -> dict:
- config: dict = {
- "team_id": TEAM_ID,
- "distinct_id": "$posthog_cookieless",
- "expected_person_id": PERSON_UUID,
- "dry_run": dry_run,
- }
- if override_person_id is not None:
- config["override_person_id"] = override_person_id
- return {"ops": {"detach_distinct_id_op": {"config": config}}}
-
- @patch("posthog.dags.detach_distinct_id.sync_execute")
- def test_dry_run_does_not_mutate(self, mock_sync_execute):
- lookup_row = (PDI_ID, PDI_VERSION, PERSON_PK, UUID(PERSON_UUID))
- conn, cursor = self._make_connection(lookup_row, other_count=2)
- producer = MagicMock()
-
- result = detach_distinct_id_job.execute_in_process(
- run_config=self._run_config(dry_run=True),
- resources={"persons_database": conn, "kafka_producer": producer},
- )
-
- assert result.success
- conn.commit.assert_not_called()
- conn.rollback.assert_called_once()
- producer.produce.assert_not_called()
- mock_sync_execute.assert_not_called()
-
- @patch("posthog.dags.detach_distinct_id.sync_execute")
- @patch("posthog.dags.detach_distinct_id.uuid.uuid4", return_value=UUID(DUMMY_OVERRIDE_UUID))
- def test_detaches_publishes_kafka_and_inserts_override(self, _mock_uuid4, mock_sync_execute):
- mock_sync_execute.return_value = [(0,)]
- lookup_row = (PDI_ID, PDI_VERSION, PERSON_PK, UUID(PERSON_UUID))
- conn, cursor = self._make_connection(lookup_row, other_count=2)
- producer = MagicMock()
-
- result = detach_distinct_id_job.execute_in_process(
- run_config=self._run_config(dry_run=False),
- resources={"persons_database": conn, "kafka_producer": producer},
- )
-
- assert result.success
- conn.commit.assert_called_once()
-
- # Kafka deletion
- producer.produce.assert_called_once()
- kafka_data = producer.produce.call_args.kwargs["data"]
- assert kafka_data["distinct_id"] == "$posthog_cookieless"
- assert kafka_data["person_id"] == PERSON_UUID
- assert kafka_data["is_deleted"] == 1
- assert kafka_data["version"] == PDI_VERSION + 1
- producer.flush.assert_called_once()
-
- # ClickHouse override
- override_rows = _override_insert_rows(mock_sync_execute)
- assert override_rows[0][0] == TEAM_ID
- assert override_rows[0][1] == "$posthog_cookieless"
- assert override_rows[0][2] == DUMMY_OVERRIDE_UUID
- assert override_rows[0][4] == PDI_VERSION + 2
-
- @patch("posthog.dags.detach_distinct_id.sync_execute")
- def test_a_failed_kafka_delivery_fails_the_job_before_the_override(self, mock_sync_execute):
- mock_sync_execute.return_value = [(0,)]
- lookup_row = (PDI_ID, PDI_VERSION, PERSON_PK, UUID(PERSON_UUID))
- conn, _cursor = self._make_connection(lookup_row, other_count=2)
- producer = MagicMock()
- producer.produce.return_value.get.side_effect = KafkaException(KafkaError(KafkaError._MSG_TIMED_OUT)) # type: ignore[attr-defined]
-
- result = detach_distinct_id_job.execute_in_process(
- run_config=self._run_config(dry_run=False),
- resources={"persons_database": conn, "kafka_producer": producer},
- raise_on_error=False,
- )
-
- assert not result.success
- conn.commit.assert_called_once()
- assert not [
- c for c in mock_sync_execute.call_args_list if "INSERT INTO person_distinct_id_overrides" in c.args[0]
- ]
-
- @patch("posthog.dags.detach_distinct_id.sync_execute")
- def test_uses_explicit_override_person_id(self, mock_sync_execute):
- mock_sync_execute.return_value = [(0,)]
- explicit_uuid = "11111111-2222-3333-4444-555555555555"
- lookup_row = (PDI_ID, PDI_VERSION, PERSON_PK, UUID(PERSON_UUID))
- conn, cursor = self._make_connection(lookup_row, other_count=2)
- producer = MagicMock()
-
- result = detach_distinct_id_job.execute_in_process(
- run_config=self._run_config(dry_run=False, override_person_id=explicit_uuid),
- resources={"persons_database": conn, "kafka_producer": producer},
- )
-
- assert result.success
- override_rows = _override_insert_rows(mock_sync_execute)
- assert override_rows[0][2] == explicit_uuid
-
- @patch("posthog.dags.detach_distinct_id.sync_execute")
- def test_fails_when_distinct_id_not_found(self, mock_sync_execute):
- conn, cursor = self._make_connection(lookup_row=None, other_count=0, tombstone_version=None)
- cursor.fetchone.side_effect = [None]
- producer = MagicMock()
-
- result = detach_distinct_id_job.execute_in_process(
- run_config=self._run_config(dry_run=False),
- resources={"persons_database": conn, "kafka_producer": producer},
- raise_on_error=False,
- )
-
- assert not result.success
- conn.commit.assert_not_called()
- producer.produce.assert_not_called()
- mock_sync_execute.assert_not_called()
-
- @patch("posthog.dags.detach_distinct_id.sync_execute")
- def test_fails_when_person_id_mismatch(self, mock_sync_execute):
- wrong_person = UUID("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee")
- lookup_row = (PDI_ID, PDI_VERSION, PERSON_PK, wrong_person)
- conn, cursor = self._make_connection(lookup_row, other_count=2)
- producer = MagicMock()
-
- result = detach_distinct_id_job.execute_in_process(
- run_config=self._run_config(dry_run=False),
- resources={"persons_database": conn, "kafka_producer": producer},
- raise_on_error=False,
- )
-
- assert not result.success
- conn.commit.assert_not_called()
- producer.produce.assert_not_called()
- mock_sync_execute.assert_not_called()
-
- @patch("posthog.dags.detach_distinct_id.sync_execute")
- def test_fails_when_only_distinct_id(self, mock_sync_execute):
- lookup_row = (PDI_ID, PDI_VERSION, PERSON_PK, UUID(PERSON_UUID))
- conn, cursor = self._make_connection(lookup_row, other_count=0)
- producer = MagicMock()
-
- result = detach_distinct_id_job.execute_in_process(
- run_config=self._run_config(dry_run=False),
- resources={"persons_database": conn, "kafka_producer": producer},
- raise_on_error=False,
- )
-
- assert not result.success
- conn.commit.assert_not_called()
- producer.produce.assert_not_called()
- mock_sync_execute.assert_not_called()
-
-
[email protected]_db(transaction=True)
-class TestDetachDistinctIdIntegration:
- """Integration tests using a real Django Postgres connection.
-
- Exercises the full Dagster job against real DB rows so cursor return types
- (tuple vs dict) are tested against production-like conditions.
- Kafka stays mocked. ClickHouse is mocked except in the live-run test, which needs the real version reads.
- """
-
- @pytest.fixture
- def organization(self):
- from posthog.models import Organization
-
- return Organization.objects.create(name="Detach Test Org")
-
- @pytest.fixture
- def team(self, organization):
- from posthog.models import Team
-
- return Team.objects.create(organization=organization, name="Detach Test Team")
-
- @classmethod
- def _pdi_state(cls, pdi_id: int) -> tuple[bool, int] | None:
- with cls._get_persons_conn() as conn, conn.cursor() as cursor:
- cursor.execute("SELECT is_deleted, version FROM posthog_persondistinctid WHERE id = %s", [pdi_id])
- row = cursor.fetchone()
- return None if row is None else (row[0], row[1])
-
- @pytest.fixture
- def person_with_two_distinct_ids(self, team):
- from posthog.models.utils import UUIDT
-
- person_uuid = UUIDT()
- with self._get_persons_conn() as conn, conn.cursor() as cursor:
- cursor.execute(
- "INSERT INTO posthog_person (team_id, uuid, properties, is_identified, version, created_at, properties_last_updated_at, properties_last_operation) "
- "VALUES (%s, %s, '{}', false, 1, NOW(), '{}', '{}') RETURNING id",
- [team.id, str(person_uuid)],
- )
- person_id = cursor.fetchone()[0]
- cursor.execute(
- "INSERT INTO posthog_persondistinctid (team_id, person_id, distinct_id, version) "
- "VALUES (%s, %s, %s, %s) RETURNING id",
- [team.id, person_id, "keep_this", 0],
- )
- pdi_keep_id = cursor.fetchone()[0]
- cursor.execute(
- "INSERT INTO posthog_persondistinctid (team_id, person_id, distinct_id, version) "
- "VALUES (%s, %s, %s, %s) RETURNING id",
- [team.id, person_id, "$posthog_cookieless", 3],
- )
- pdi_detach_id = cursor.fetchone()[0]
-
- from types import SimpleNamespace
-
- person = SimpleNamespace(id=person_id, uuid=person_uuid, version=1)
- pdi_keep = SimpleNamespace(id=pdi_keep_id, distinct_id="keep_this", version=0)
- pdi_detach = SimpleNamespace(id=pdi_detach_id, distinct_id="$posthog_cookieless", version=3)
- return person, pdi_keep, pdi_detach
-
- @staticmethod
- def _run_config(
- team_id: int,
- person_uuid: str,
- *,
- dry_run: bool,
- distinct_id: str = "$posthog_cookieless",
- override_person_id: str | None = None,
- ) -> dict:
- config: dict = {
- "team_id": team_id,
- "distinct_id": distinct_id,
- "expected_person_id": person_uuid,
- "dry_run": dry_run,
- }
- if override_person_id is not None:
- config["override_person_id"] = override_person_id
- return {"ops": {"detach_distinct_id_op": {"config": config}}}
-
- @staticmethod
- @contextlib.contextmanager
- def _get_persons_conn():
- # autocommit=False: the dagster job consumes this connection as the `persons_database`
- # resource and drives its own commit()/rollback() (dry runs roll back), so it must be
- # transactional rather than autocommit. The cursor-only callers commit on block exit.
- with persons_db_connection(writer=True) as conn:
- yield conn
-
- @patch("posthog.dags.detach_distinct_id.sync_execute")
- def test_dry_run_does_not_tombstone(self, mock_sync_execute, team, person_with_two_distinct_ids):
- person, _pdi_keep, pdi_detach = person_with_two_distinct_ids
- producer = MagicMock()
-
- with self._get_persons_conn() as conn:
- result = detach_distinct_id_job.execute_in_process(
- run_config=self._run_config(team.id, str(person.uuid), dry_run=True),
- resources={"persons_database": conn, "kafka_producer": producer},
- )
-
- assert result.success
- assert self._pdi_state(pdi_detach.id) == (False, pdi_detach.version)
- producer.produce.assert_not_called()
- mock_sync_execute.assert_not_called()
-
- @pytest.mark.parametrize(
- "clickhouse_table, expected_version",
- [
- # (table holding a version-20 row for the distinct id, version the tombstone gets)
- (None, 4),
- ("person_distinct_id2", 21),
- ("person_distinct_id_overrides", 21),
- ],
- )
- def test_live_run_tombstones_above_postgres_and_clickhouse(
- self, clickhouse_table, expected_version, team, person_with_two_distinct_ids
- ):
- person, pdi_keep, pdi_detach = person_with_two_distinct_ids
- override_target = str(uuid_module.uuid4())
- sync_execute(TRUNCATE_PERSON_DISTINCT_ID2_TABLE_SQL)
- sync_execute(TRUNCATE_PERSON_DISTINCT_ID_OVERRIDES_TABLE_SQL())
- if clickhouse_table == "person_distinct_id2":
- create_person_distinct_id(team.id, pdi_detach.distinct_id, str(person.uuid), version=20)
- elif clickhouse_table == "person_distinct_id_overrides":
- _insert_ch_override(team.id, pdi_detach.distinct_id, str(uuid_module.uuid4()), version=20)
- producer = MagicMock()
-
- with self._get_persons_conn() as conn:
- result = detach_distinct_id_job.execute_in_process(
- run_config=self._run_config(
- team.id, str(person.uuid), dry_run=False, override_person_id=override_target
- ),
- resources={"persons_database": conn, "kafka_producer": producer},
- )
-
- assert result.success
- assert self._pdi_state(pdi_detach.id) == (True, expected_version)
- assert self._pdi_state(pdi_keep.id) == (False, pdi_keep.version)
- producer.produce.assert_called_once_with(
- topic=KAFKA_PERSON_DISTINCT_ID,
- data={
- "distinct_id": pdi_detach.distinct_id,
- "person_id": str(person.uuid),
- "team_id": team.id,
- "version": expected_version,
- "is_deleted": 1,
- },
- )
- # The overrides MV consumes the same topic, so the published tombstone also lands in the overrides
- # table, after the direct insert.
- tombstone = producer.produce.call_args.kwargs["data"]
- sync_execute(
- """
- INSERT INTO person_distinct_id_overrides
- (team_id, distinct_id, person_id, is_deleted, version, _timestamp, _offset, _partition)
- VALUES
- """,
- [
- (
- tombstone["team_id"],
- tombstone["distinct_id"],
- tombstone["person_id"],
- tombstone["is_deleted"],
- tombstone["version"],
- datetime.now(),
- 0,
- 0,
- )
- ],
- )
- override = sync_execute(
- """
- SELECT person_id, is_deleted, version FROM person_distinct_id_overrides FINAL
- WHERE team_id = %(team_id)s AND distinct_id = %(distinct_id)s
- """,
- {"team_id": team.id, "distinct_id": pdi_detach.distinct_id},
- )
- assert override == [(UUID(override_target), 0, expected_version + 1)]
-
- def test_fails_without_tombstoning_when_the_distinct_id_moves_after_the_lookup(
- self, team, person_with_two_distinct_ids
- ):
- person, _pdi_keep, pdi_detach = person_with_two_distinct_ids
- with self._get_persons_conn() as conn, conn.cursor() as cursor:
- cursor.execute(
- "INSERT INTO posthog_person (team_id, uuid, properties, is_identified, version, created_at, properties_last_updated_at, properties_last_operation) "
- "VALUES (%s, %s, '{}', false, 0, NOW(), '{}', '{}') RETURNING id",
- [team.id, str(uuid_module.uuid4())],
- )
- other_person_id = cursor.fetchone()[0]
-
- def move_then_read(team_id: int, distinct_id: str) -> int:
- with self._get_persons_conn() as conn, conn.cursor() as cursor:
- cursor.execute(
- "UPDATE posthog_persondistinctid SET person_id = %s WHERE id = %s", [other_person_id, pdi_detach.id]
- )
- return 0
-
- producer = MagicMock()
- with (
- patch("posthog.dags.detach_distinct_id._clickhouse_max_version", side_effect=move_then_read),
- self._get_persons_conn() as conn,
- ):
- result = detach_distinct_id_job.execute_in_process(
- run_config=self._run_config(team.id, str(person.uuid), dry_run=False),
- resources={"persons_database": conn, "kafka_producer": producer},
- raise_on_error=False,
- )
-
- assert not result.success
- assert self._pdi_state(pdi_detach.id) == (False, pdi_detach.version)
- producer.produce.assert_not_called()
-
- @patch("posthog.dags.detach_distinct_id.sync_execute")
- def test_fails_on_person_id_mismatch(self, mock_sync_execute, team, person_with_two_distinct_ids):
- person, _, _ = person_with_two_distinct_ids
- wrong_uuid = str(uuid_module.uuid4())
- producer = MagicMock()
-
- with self._get_persons_conn() as conn:
- result = detach_distinct_id_job.execute_in_process(
- run_config=self._run_config(team.id, wrong_uuid, dry_run=False),
- resources={"persons_database": conn, "kafka_producer": producer},
- raise_on_error=False,
- )
-
- assert not result.success
- producer.produce.assert_not_called()
- mock_sync_execute.assert_not_called()
-
- @patch("posthog.dags.detach_distinct_id.sync_execute")
- def test_fails_when_only_distinct_id(self, mock_sync_execute, team):
- from types import SimpleNamespace
-
- from posthog.models.utils import UUIDT
-
- person_uuid = UUIDT()
- with self._get_persons_conn() as conn, conn.cursor() as cursor:
- cursor.execute(
- "INSERT INTO posthog_person (team_id, uuid, properties, is_identified, version, created_at, properties_last_updated_at, properties_last_operation) "
- "VALUES (%s, %s, '{}', false, 1, NOW(), '{}', '{}') RETURNING id",
- [team.id, str(person_uuid)],
- )
- person_id = cursor.fetchone()[0]
- cursor.execute(
- "INSERT INTO posthog_persondistinctid (team_id, person_id, distinct_id, version) "
- "VALUES (%s, %s, %s, %s)",
- [team.id, person_id, "$posthog_cookieless", 3],
- )
- person = SimpleNamespace(id=person_id, uuid=person_uuid, version=1)
- producer = MagicMock()
-
- with self._get_persons_conn() as conn:
- result = detach_distinct_id_job.execute_in_process(
- run_config=self._run_config(team.id, str(person.uuid), dry_run=False),
- resources={"persons_database": conn, "kafka_producer": producer},
- raise_on_error=False,
- )
-
- assert not result.success
- producer.produce.assert_not_called()
- mock_sync_execute.assert_not_called()
-
- @patch("posthog.dags.detach_distinct_id.sync_execute")
- def test_fails_when_distinct_id_not_found(self, mock_sync_execute, team):
- producer = MagicMock()
-
- with self._get_persons_conn() as conn:
- result = detach_distinct_id_job.execute_in_process(
- run_config=self._run_config(
- team.id, str(uuid_module.uuid4()), dry_run=False, distinct_id="nonexistent"
- ),
- resources={"persons_database": conn, "kafka_producer": producer},
- raise_on_error=False,
- )
-
- assert not result.success
- producer.produce.assert_not_called()
- mock_sync_execute.assert_not_called()
diff --git a/posthog/management/commands/person_consistency.py b/posthog/management/commands/person_consistency.py
index 7849516e9010..563bb868434b 100644
--- a/posthog/management/commands/person_consistency.py
+++ b/posthog/management/commands/person_consistency.py
@@ -15,6 +15,13 @@ Usage:
python manage.py person_consistency repair --dir /tmp/scan-2 --verb republish --apply --limit 10
python manage.py person_consistency repair --dir /tmp/scan-2 --verb republish --apply --resume <actions file>
+ # Move distinct ids off a live person onto persons of their own (no scan needed):
+ python manage.py person_consistency split --team-id 2 --person-uuid <uuid> --distinct-id <id> --apply
+
+ # Republish named persons, or the owners of named distinct ids, from Postgres (no scan needed):
+ python manage.py person_consistency republish --team-id 2 --distinct-id <id> --apply
+ python manage.py person_consistency republish --team-id 2 --distinct-ids-file <file> --apply
+
Scans are read-only. A scan that stops resumes with --resume. The report (report.jsonl) holds one row per
key that is not ok, with the raw keys a repair needs; summary.json holds counts and hashed group names, and names no key.
A repair writes nothing without --apply, and records one action per key in an actions file.
@@ -27,10 +34,13 @@ import argparse
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
+from uuid import UUID
from django.core.management.base import BaseCommand, CommandError, CommandParser
from posthog.clickhouse.query_tagging import Feature, Product, tags_context
+from posthog.models.person.divergence.detach import SplitError, split_distinct_ids
+from posthog.models.person.divergence.repairs import DEFAULT_MAX_WRITES_PER_SECOND, republish_named
from posthog.models.person.divergence.team_scan import (
PLAN_FILE,
SUMMARY_FILE,
@@ -44,6 +54,8 @@ from posthog.models.person.divergence.team_scan import (
)
from posthog.models.person.divergence.verbs import VERBS, VerbError, run_verb
+_SHOWN_KEYS = 20
+
def _parse_utc(value: str) -> datetime:
parsed = datetime.fromisoformat(value)
@@ -134,7 +146,7 @@ class Command(BaseCommand):
repair.add_argument(
"--max-calls-per-second",
type=float,
- default=20.0,
+ default=DEFAULT_MAX_WRITES_PER_SECOND,
help="Personhog write calls per second (default: %(default)s).",
)
repair.add_argument(
@@ -149,6 +161,35 @@ class Command(BaseCommand):
help="bury: tombstone new persons even past the share limit that protects a team needing a restore.",
)
+ split = actions.add_parser(
+ "split",
+ help="DRY RUN unless --apply. Move distinct ids off a live person onto persons of their own.",
+ )
+ split.add_argument("--team-id", type=int, required=True, help="Team of the person.")
+ split.add_argument("--person-uuid", required=True, help="The live person that owns the distinct ids.")
+ split.add_argument(
+ "--distinct-id", action="append", required=True, help="Distinct id to move off the person. Repeatable."
+ )
+ split.add_argument("--apply", action="store_true", help="Split. Without it the command only checks.")
+
+ republish = actions.add_parser(
+ "republish",
+ help="DRY RUN unless --apply. Republish named persons, or the owners of named distinct ids, where "
+ "ClickHouse disagrees with Postgres.",
+ )
+ republish.add_argument("--team-id", type=int, required=True, help="Team of the persons.")
+ republish.add_argument("--person-uuid", action="append", default=[], help="Person to republish. Repeatable.")
+ republish.add_argument(
+ "--distinct-id", action="append", default=[], help="Republish the person that owns it. Repeatable."
+ )
+ republish.add_argument(
+ "--distinct-ids-file",
+ type=Path,
+ default=None,
+ help="A file of distinct ids, one per line, read like --distinct-id.",
+ )
+ republish.add_argument("--apply", action="store_true", help="Write. Without it the command only reports.")
+
final = actions.add_parser("finalize", help="Join every run's rows into report.jsonl and summary.json.")
self._add_dir(final)
final.add_argument(
@@ -201,11 +242,53 @@ class Command(BaseCommand):
self._scan(options)
elif options["action"] == "repair":
self._repair(options)
+ elif options["action"] == "split":
+ split_distinct_ids(
+ options["team_id"],
+ options["person_uuid"],
+ options["distinct_id"],
+ apply=options["apply"],
+ log=self._log,
+ )
+ elif options["action"] == "republish":
+ self._republish(options)
else:
self._finalize(options["dir"], partial=options["partial"])
- except (ScanError, VerbError) as exc:
+ except (ScanError, VerbError, SplitError) as exc:
raise CommandError(str(exc)) from exc
+ def _republish(self, options: dict[str, Any]) -> None:
+ distinct_ids = list(options["distinct_id"])
+ if options["distinct_ids_file"] is not None:
+ # Read as bytes, because text mode turns a carriage return into a newline; only a newline ends an id.
+ text = options["distinct_ids_file"].read_bytes().decode()
+ distinct_ids.extend(line for line in text.split("\n") if line)
+ if not options["person_uuid"] and not distinct_ids:
+ raise CommandError("name at least one --person-uuid, --distinct-id or --distinct-ids-file")
+ try:
+ person_uuids = [str(UUID(uuid)) for uuid in options["person_uuid"]]
+ except ValueError as exc:
+ raise CommandError(f"--person-uuid must be a uuid: {exc}") from exc
+ apply = options["apply"]
+ result = republish_named(
+ options["team_id"], person_uuids=person_uuids, distinct_ids=distinct_ids, apply=apply, log=self._log
+ )
+ summary = result.summary
+ self._log(
+ f"republish {'APPLY' if apply else 'DRY RUN'}: {summary.persons} persons, "
+ f"person outcomes {summary.person_outcomes}, mapping outcomes {summary.mapping_outcomes}, "
+ f"undelivered {summary.undelivered}"
+ )
+ if not apply:
+ self._log("Dry run: nothing was written. Pass --apply to write.")
+ elif result.unconfirmed:
+ shown = ", ".join(result.unconfirmed[:_SHOWN_KEYS])
+ more = f" and {len(result.unconfirmed) - _SHOWN_KEYS} more" if len(result.unconfirmed) > _SHOWN_KEYS else ""
+ raise CommandError(
+ f"ClickHouse may still disagree with Postgres for {len(result.unconfirmed)} named keys: "
+ f"{shown}{more}. Check them with person_inspect."
+ )
+
def _plan(self, options: dict[str, Any], *, chunks: int) -> None:
if options["pg_page_size"] > 10_000:
raise CommandError("--pg-page-size must be at most 10000")
diff --git a/posthog/models/person/divergence/cleanup_queue.py b/posthog/models/person/divergence/cleanup_queue.py
new file mode 100644
index 000000000000..f1e16f65693f
--- /dev/null
+++ b/posthog/models/person/divergence/cleanup_queue.py
@@ -0,0 +1,47 @@
+"""The persons Postgres cleanup queue, as the repair tools see it.
+
+The cleanup drain (`posthog/dags/person_pg_cleanup_drain.py`) owns this table and hard-deletes the persons queued in
+it. A person that still owns a live distinct id is stamped `blocked_at` and skipped for a retry window. The repair
+tools only clear `blocked_at`, once a repair removed what blocked the person, so the next drain run retries it.
+
+The queue is not person data that personhog serves, so it is read and written here directly, as the drain does.
+"""
+
+from collections.abc import Sequence
+from typing import Any
+
+from posthog.persons_db import persons_db_connection
+
+PG_CLEANUP_QUEUE_TABLE = "person_pg_cleanup_queue"
+
+
+def clear_blocked(team_id: int, person_uuids: Sequence[str]) -> int:
+ """Clear `blocked_at` on the queued persons, and return how many queue rows changed."""
+ if not person_uuids:
+ return 0
+ with persons_db_connection(writer=True) as conn, conn.cursor() as cursor:
+ cursor.execute(
+ f"UPDATE {PG_CLEANUP_QUEUE_TABLE} SET blocked_at = NULL WHERE team_id = %s AND person_uuid = ANY(%s::uuid[])",
+ [team_id, list(person_uuids)],
+ )
+ return cursor.rowcount
+
+
+def queue_entries(team_id: int, person_uuids: Sequence[str]) -> dict[str, dict[str, Any]]:
+ """Each queued person's row by uuid: when the sweep queued it, the version bound, and when the drain blocked it."""
+ if not person_uuids:
+ return {}
+ with persons_db_connection(writer=False) as conn, conn.cursor() as cursor:
+ cursor.execute(
+ f"SELECT person_uuid::text, deleted_at, max_version, blocked_at FROM {PG_CLEANUP_QUEUE_TABLE} "
+ "WHERE team_id = %s AND person_uuid = ANY(%s::uuid[])",
+ [team_id, list(person_uuids)],
+ )
+ return {
+ person_uuid: {
+ "deleted_at": deleted_at.isoformat(),
+ "max_version": max_version,
+ "blocked_at": blocked_at.isoformat() if blocked_at else None,
+ }
+ for person_uuid, deleted_at, max_version, blocked_at in cursor.fetchall()
+ }
diff --git a/posthog/models/person/divergence/datatypes.py b/posthog/models/person/divergence/datatypes.py
index dfb82c96edaa..5a3a19508f02 100644
--- a/posthog/models/person/divergence/datatypes.py
+++ b/posthog/models/person/divergence/datatypes.py
@@ -121,3 +121,10 @@ class RepairSummary:
mapping_outcomes: dict[RepairOutcome, int]
# Published rows that Kafka did not confirm: the delivery failed, or it was still queued at the deadline.
undelivered: int
+
+
+@frozen
+class RepublishResult:
+ summary: RepairSummary
+ # Named persons and distinct ids that ClickHouse may still disagree on: not repaired, or not delivered.
+ unconfirmed: tuple[str, ...]
diff --git a/posthog/models/person/divergence/detach.py b/posthog/models/person/divergence/detach.py
new file mode 100644
index 000000000000..8c13faf042fb
--- /dev/null
+++ b/posthog/models/person/divergence/detach.py
@@ -0,0 +1,286 @@
+"""Take a distinct id off a person. `detach` tombstones strays from a scan report; `split_distinct_ids` moves
+distinct ids off a live person onto persons of their own."""
+
+from __future__ import annotations
+
+import time
+import shlex
+import tempfile
+from collections.abc import Callable, Sequence
+from functools import partial
+from typing import TYPE_CHECKING, Any
+from uuid import UUID
+
+from posthog.kafka_client.routing import flush_all_producers
+from posthog.models.person.util import _batched_get_distinct_ids_for_persons, get_distinct_id_rows, get_persons_by_uuids
+from posthog.personhog_client.client import personhog_call, require_personhog_client
+from posthog.personhog_client.proto import (
+ DISTINCT_ID_WRITE_ACTION_TOMBSTONE,
+ DISTINCT_ID_WRITE_OUTCOME_APPLIED,
+ DISTINCT_ID_WRITE_OUTCOME_PERSON_NOT_LIVE,
+ DISTINCT_ID_WRITE_OUTCOME_UNCHANGED,
+ DistinctIdRow,
+ DistinctIdWrite,
+ WriteDistinctIdsRequest,
+)
+
+from . import cleanup_queue, lookups, observe, repairs
+from .flavors import PgMapping, classify_mapping
+
+if TYPE_CHECKING:
+ from .verbs import VerbContext
+
+# How long the split waits for its own ClickHouse messages to go out before it republishes and checks them.
+_SPLIT_FLUSH_SECONDS = 60.0
+
+# A republish command for more distinct ids than this reads them from a file, so the command stays short.
+_INLINE_DISTINCT_IDS = 10
+
+# The pause before the resend tries unconfirmed ids again, for a replica that had not caught up with the split.
+_RESEND_RETRY_SECONDS = 1.0
+
+_sleep = time.sleep
+
+
+class SplitError(Exception):
+ pass
+
+
+def detach(rows: list[dict[str, Any]], context: VerbContext) -> None:
+ """Tombstone strays in Postgres: distinct ids live there on a tombstoned person.
+
+ It writes nothing to ClickHouse. The cleanup drain hard-deletes a person's Postgres rows once its distinct ids are
+ tombstoned, so a ClickHouse tombstone that outlived that delete would hide a later re-use of the distinct id. A
+ stray is tombstoned only when the classifier, re-run on fresh ClickHouse reads just before the write, still names
+ it `detach`: ClickHouse holds nothing for it, or the deletion sweep will remove its rows together with the owner
+ before the drain runs. Any other stray is skipped under the sub-flavor it now has.
+
+ The tombstone lands above every version ClickHouse holds for the distinct id, so if the distinct id is captured
+ again, ingestion revives the row one version higher and ClickHouse shows the revival.
+ """
+ for batch in lookups._chunks(rows, repairs._WRITE_BATCH_SIZE):
+ _detach_batch(list(batch), context)
+
+
+def _detach_batch(rows: list[dict[str, Any]], context: VerbContext) -> None:
+ team_id = context.team_id
+ by_key = {row["key"]: row for row in rows}
+ reported = {row["key"]: PgMapping(**row["evidence"]["pg"]) for row in rows}
+ ch = observe.ch_mapping_states(team_id, list(by_key))
+ owner_ch = observe.owner_ch_states(team_id, reported, ch)
+ writes: list[DistinctIdWrite] = []
+ for key, row in by_key.items():
+ pg, state = reported[key], ch.get(key)
+ verdict = classify_mapping(pg, state, owner_ch=owner_ch.get(key))
+ if verdict.verb != "detach":
+ context.action(row, f"skipped_{verdict.sub_flavor or verdict.flavor}")
+ continue
+ writes.append(
+ DistinctIdWrite(
+ distinct_id=key,
+ action=DISTINCT_ID_WRITE_ACTION_TOMBSTONE,
+ person_uuid=pg.person_uuid,
+ min_version=max(pg.version + 1, (state.max_version + 1) if state else 0),
+ expected_version=pg.version,
+ )
+ )
+ if not writes:
+ return
+ if not context.apply:
+ current = {
+ r.distinct_id: r for r in get_distinct_id_rows(team_id, [w.distinct_id for w in writes], strong=True)
+ }
+ for write in writes:
+ context.action(
+ by_key[write.distinct_id],
+ _predicted_outcome(current.get(write.distinct_id), write),
+ target_version=write.min_version,
+ )
+ return
+
+ context.pacer.before_write()
+ response = repairs._retry_transient(
+ partial(
+ personhog_call,
+ "person_consistency_detach",
+ partial(
+ require_personhog_client().write_distinct_ids,
+ WriteDistinctIdsRequest(team_id=team_id, writes=writes),
+ ),
+ ),
+ what="detach",
+ log=context.log,
+ )
+ detached_owners: set[int] = set()
+ for result in response.results:
+ row = by_key[result.distinct_id]
+ if result.outcome in (DISTINCT_ID_WRITE_OUTCOME_APPLIED, DISTINCT_ID_WRITE_OUTCOME_UNCHANGED):
+ detached_owners.add(reported[result.distinct_id].person_id)
+ outcome = "detached" if result.outcome == DISTINCT_ID_WRITE_OUTCOME_APPLIED else "already_detached"
+ context.action(row, outcome, version=int(result.row.version) if result.HasField("row") else None)
+ elif result.outcome == DISTINCT_ID_WRITE_OUTCOME_PERSON_NOT_LIVE:
+ context.action(row, "skipped_person_gone")
+ else:
+ context.action(row, "skipped_changed")
+ _release_drained_owners(team_id, detached_owners, by_key, reported)
+
+
+def _predicted_outcome(row: DistinctIdRow | None, write: DistinctIdWrite) -> str:
+ """What applying `write` would report, from the Postgres primary's row, by the RPC's rules."""
+ if row is None or not row.HasField("person_is_deleted"):
+ return "skipped_person_gone"
+ if row.person_uuid != write.person_uuid or not row.person_is_deleted:
+ return "skipped_changed"
+ if row.is_deleted:
+ return "already_detached" if row.version >= write.min_version else "skipped_changed"
+ return "would_detach" if row.version == write.expected_version else "skipped_changed"
+
+
+def _release_drained_owners(
+ team_id: int, owner_ids: set[int], by_key: dict[str, dict[str, Any]], reported: dict[str, PgMapping]
+) -> None:
+ """Clear `blocked_at` for owners left with no live distinct id, so the next drain run retries them.
+
+ An owner that still has a live distinct id stays blocked: the drain would block it again. The live distinct ids
+ come from the replica, so a lagging replica only leaves an owner blocked until the drain's own retry.
+ """
+ if not owner_ids:
+ return
+ live = _live_distinct_ids(team_id, sorted(owner_ids), limit=1)
+ uuids = {mapping.person_id: mapping.person_uuid for key in by_key if (mapping := reported[key]).person_uuid}
+ released = sorted(uuid for owner in owner_ids if not live.get(owner) and (uuid := uuids[owner]) is not None)
+ cleanup_queue.clear_blocked(team_id, released)
+
+
+def _live_distinct_ids(team_id: int, person_ids: list[int], *, limit: int) -> dict[int, list[str]]:
+ by_person = personhog_call(
+ "get_distinct_ids_for_persons",
+ lambda: _batched_get_distinct_ids_for_persons(team_id, person_ids, limit_per_person=limit),
+ )
+ return {person_id: [d.id for d in dids] for person_id, dids in by_person.items()}
+
+
+def split_distinct_ids(
+ team_id: int,
+ person_uuid: str,
+ distinct_ids: Sequence[str],
+ *,
+ apply: bool,
+ log: Callable[[str], None],
+) -> list[str]:
+ """Move `distinct_ids` off a live person onto persons of their own, and return the distinct ids split.
+
+ `Person.split_person` does the split through personhog and refuses one that would leave the person without a
+ distinct id; the dry run applies the same check. A distinct id that is not live on this person stops the whole
+ split, so a mistyped id never splits the rest alone. Every applied split republishes the moved persons from
+ Postgres and checks the result, and raises with the command that finishes the job when ClickHouse may still
+ show a moved id on this person.
+ """
+ ids = list(dict.fromkeys(distinct_ids))
+ if not ids:
+ raise SplitError("name at least one --distinct-id")
+ try:
+ person_uuid = str(UUID(person_uuid))
+ except ValueError as exc:
+ raise SplitError(f"{person_uuid!r} is not a uuid") from exc
+ persons = get_persons_by_uuids(team_id, [person_uuid], distinct_id_limit=0)
+ if not persons:
+ raise SplitError(f"person {person_uuid} is not live in team {team_id}")
+ person = persons[0]
+ rows = {r.distinct_id: r for r in get_distinct_id_rows(team_id, ids, strong=True)}
+ elsewhere = sorted(
+ d for d in ids if (row := rows.get(d)) is None or row.is_deleted or row.person_uuid != person_uuid
+ )
+ if elsewhere:
+ raise SplitError(f"not live on person {person_uuid}: {', '.join(elsewhere)}")
+ # One more than the request names: if every returned id is in the request, none would remain.
+ remaining = _live_distinct_ids(team_id, [int(person.pk)], limit=len(ids) + 1).get(int(person.pk), [])
+ if all(distinct_id in ids for distinct_id in remaining):
+ raise SplitError(f"the split would leave person {person_uuid} with no distinct id")
+ if not apply:
+ log(f"split: would move {len(ids)} distinct ids off person {person_uuid}")
+ return []
+ failure: Exception | None = None
+ try:
+ person.split_person(None, None, distinct_ids_to_split=ids)
+ except Exception as exc:
+ # A failure after the first batch leaves the earlier batches split in Postgres, so they are resent too.
+ failure = exc
+ finally:
+ # The split publishes to ClickHouse without waiting, and a command that exits first can lose those messages.
+ flush_all_producers(_SPLIT_FLUSH_SECONDS)
+ moved: list[str] = []
+ try:
+ moved = _moved_ids(team_id, person_uuid, ids)
+ unconfirmed = _resend_split(team_id, moved, log=log)
+ except Exception as exc:
+ log(f"split: sending the split to ClickHouse again failed: {exc!r}")
+ raise SplitError(_unconfirmed_message(team_id, person_uuid, moved or ids, failure)) from (failure or exc)
+ if unconfirmed:
+ raise SplitError(_unconfirmed_message(team_id, person_uuid, unconfirmed, failure)) from failure
+ if failure is not None:
+ raise SplitError(
+ f"split moved {len(moved)} of {len(ids)} distinct ids off person {person_uuid}, then failed: {failure}"
+ ) from failure
+ if len(moved) < len(ids):
+ raise SplitError(
+ f"split moved {len(moved)} of {len(ids)} distinct ids off person {person_uuid}; check the rest with "
+ "person_inspect"
+ )
+ log(f"split: moved {len(moved)} distinct ids off person {person_uuid}")
+ return moved
+
+
+def _moved_ids(team_id: int, person_uuid: str, distinct_ids: list[str]) -> list[str]:
+ rows = get_distinct_id_rows(team_id, distinct_ids, strong=True)
+ return [r.distinct_id for r in rows if not r.is_deleted and r.person_uuid and r.person_uuid != person_uuid]
+
+
+def _resend_split(team_id: int, moved: list[str], *, log: Callable[[str], None]) -> list[str]:
+ """Republish the persons the split moved distinct ids to, and return the moved ids still unconfirmed.
+
+ Kafka's flush counts only queued messages, not failed deliveries, so every split is resent and checked. Without
+ this, ClickHouse can keep the moved ids on the old person, and a scan suggests merging the two persons back.
+ """
+ if not moved:
+ return []
+ unconfirmed = list(repairs.republish_named(team_id, distinct_ids=moved, apply=True, log=log).unconfirmed)
+ if unconfirmed:
+ _sleep(_RESEND_RETRY_SECONDS)
+ unconfirmed = list(repairs.republish_named(team_id, distinct_ids=unconfirmed, apply=True, log=log).unconfirmed)
+ log(f"split: sent {len(moved)} moved distinct ids and their persons again, {len(unconfirmed)} unconfirmed")
+ return unconfirmed
+
+
+def _unconfirmed_message(team_id: int, person_uuid: str, distinct_ids: Sequence[str], failure: Exception | None) -> str:
+ # The command prints only this message, not the exception it chains, so the split's own failure goes in it.
+ failed = f" The split also failed: {failure}." if failure is not None else ""
+ return (
+ f"ClickHouse may still show {len(distinct_ids)} split distinct ids on person {person_uuid}.{failed} Do not "
+ f"run a merge a scan suggests for them. Run: {_republish_command(team_id, distinct_ids)}"
+ )
+
+
+def _republish_command(team_id: int, distinct_ids: Sequence[str]) -> str:
+ ids_file = _write_ids_file(team_id, distinct_ids) if len(distinct_ids) > _INLINE_DISTINCT_IDS else None
+ if ids_file is not None:
+ flags = f"--distinct-ids-file {shlex.quote(ids_file)}"
+ else:
+ flags = " ".join(f"--distinct-id {shlex.quote(d)}" for d in distinct_ids)
+ return f"python manage.py person_consistency republish --team-id {team_id} {flags} --apply"
+
+
+def _write_ids_file(team_id: int, distinct_ids: Sequence[str]) -> str | None:
+ """Write one distinct id per line to a local file and return its path, or None when the ids must go inline.
+
+ An id that is empty or holds a newline would not read back whole, and a failed write must not replace the
+ error that prints the command.
+ """
+ if any(not d or "\n" in d for d in distinct_ids):
+ return None
+ try:
+ with tempfile.NamedTemporaryFile("w", prefix=f"split-{team_id}-", suffix=".txt", delete=False) as ids_file:
+ ids_file.write("".join(f"{d}\n" for d in distinct_ids))
+ except OSError:
+ return None
+ return ids_file.name
diff --git a/posthog/models/person/divergence/flavors.py b/posthog/models/person/divergence/flavors.py
index dbea6aab5d75..582b2d60ba59 100644
--- a/posthog/models/person/divergence/flavors.py
+++ b/posthog/models/person/divergence/flavors.py
@@ -36,9 +36,15 @@ MappingFlavor = Literal[
"hidden",
"missing_in_ch",
"stray",
+ "awaiting_sweep",
]
-RepairVerb = Literal["republish", "merge_stale", "bury", "restore", "fold", "merge"]
+# Where a person tombstoned in Postgres stands in ClickHouse, by the rule the deletion sweep uses to take a person:
+# `pending` when its newest visible row is a tombstone, `visible` when a live row is newest or ties it, `absent` when
+# it has no visible row (already swept, or never published).
+OwnerChState = Literal["pending", "visible", "absent"]
+
+RepairVerb = Literal["republish", "merge_stale", "bury", "restore", "fold", "merge", "detach"]
# Keyed by (flavor, sub_flavor). A ghost's or diverged mapping's sub-flavor decides between restore,
# fold and merge. None means the state is accepted as it is.
@@ -74,8 +80,26 @@ MAPPING_VERBS: dict[tuple[str, str | None], RepairVerb | None] = {
("stale", None): "republish",
("hidden", None): "republish",
("missing_in_ch", None): "republish",
- # Ingestion tombstones and re-attaches a stray on the distinct id's next event, so it is reported only.
- ("stray", None): None,
+ # Ingestion also repairs a stray on the distinct id's next event, but a distinct id that sees no more events
+ # keeps it, and the stray blocks the cleanup drain from hard-deleting its person. `detach` tombstones it in
+ # Postgres only, which is safe only while ClickHouse holds nothing for it or the deletion sweep will remove what
+ # it holds; the sub-flavors below are strays where neither is true, so they are reported for a person to fix.
+ ("stray", None): "detach",
+ # Its person row is gone, so there is no owner to name; ingestion re-attaches it on the next event.
+ ("stray", "orphan"): None,
+ # The owner's ClickHouse tombstone never landed, so no sweep takes it: `bury` the owner, then rescan.
+ ("stray", "owner_visible"): None,
+ # ClickHouse resolves the distinct id to a different person than Postgres does.
+ ("stray", "ch_owner_differs"): None,
+ # The owner is gone from ClickHouse but the distinct id still has rows there, which no sweep will remove.
+ ("stray", "rows_outlive_owner"): None,
+ # The caller did not read the owner's ClickHouse state, so whether a sweep will clean up is unknown.
+ ("stray", "owner_unchecked"): None,
+ # Rows at ClickHouse's top version name different owners, so the sweep may resolve another one.
+ ("stray", "ch_tie"): None,
+ # Tombstoned in Postgres; ClickHouse still resolves it to its tombstoned owner, which the next sweep removes
+ # together with that owner. A ClickHouse tombstone published now could outlive the drain's hard delete.
+ ("awaiting_sweep", None): None,
}
@@ -122,6 +146,9 @@ class ChMapping:
winner_person_uuid: str
winner_version: int
winner_deleted: bool
+ # Visible rows at the top version disagree on the owner or on is_deleted, so which one ClickHouse and the
+ # deletion sweep treat as the winner is not fixed.
+ winner_tied: bool = False
@frozen
@@ -179,11 +206,41 @@ def classify_person(
return _verdict("person", "ok")
+def owner_ch_state(state: ChPerson | None) -> OwnerChState:
+ """Where a person stands in ClickHouse, by the sweep's rule: it takes a person whose newest visible row is a
+ tombstone. A tie between a live and a tombstone row at the top version counts as visible, because which one
+ ClickHouse treats as newest is not fixed."""
+ if state is None or state.visible_rows == 0:
+ return "absent"
+ if state.tombstone_max_version is not None and (
+ state.live_max_version is None or state.tombstone_max_version > state.live_max_version
+ ):
+ return "pending"
+ return "visible"
+
+
+def needs_owner_ch_state(pg: PgMapping | None, ch: ChMapping | None) -> bool:
+ """Whether classifying this mapping depends on its Postgres owner's ClickHouse state."""
+ return (
+ pg is not None
+ and pg.person_is_deleted is True
+ and pg.person_uuid is not None
+ and ch is not None
+ and ch.visible_rows > 0
+ )
+
+
def classify_mapping(
- pg: PgMapping | None, ch: ChMapping | None, *, ch_owner_pg_state: str | None = None, team_exists: bool = True
+ pg: PgMapping | None,
+ ch: ChMapping | None,
+ *,
+ ch_owner_pg_state: str | None = None,
+ team_exists: bool = True,
+ owner_ch: OwnerChState | None = None,
) -> Verdict:
"""The flavor of one mapping. `ch_owner_pg_state` is the Postgres state of ClickHouse's winning owner
- (`live`, `deleted` or `absent`), needed only for ghost and diverged mappings."""
+ (`live`, `deleted` or `absent`), needed only for ghost and diverged mappings. `owner_ch` is the ClickHouse
+ state of the Postgres owner, needed only when `needs_owner_ch_state` holds."""
ch_live = ch is not None and ch.visible_rows > 0 and not ch.winner_deleted
if pg is None:
@@ -197,12 +254,25 @@ def classify_mapping(
return _verdict("mapping", "ghost", sub)
if pg.is_deleted:
+ if (
+ ch_live
+ and ch is not None
+ and pg.person_is_deleted
+ and ch.winner_person_uuid == pg.person_uuid
+ and not ch.winner_tied
+ and owner_ch == "pending"
+ # Postgres must hold the tombstone above every ClickHouse version, so a returning distinct id revives
+ # above them and ClickHouse shows it before the sweep removes the old rows.
+ and pg.version > ch.max_version
+ ):
+ return _verdict("mapping", "awaiting_sweep")
# A live winner at any version still resolves the id, so the tombstone has not reached ClickHouse.
return _verdict("mapping", "deleted_visible" if ch_live else "ok_deleted")
- # None is an orphan row whose person is gone; ingestion re-attaches it like any other stray.
- if pg.person_is_deleted is not False:
- return _verdict("mapping", "stray")
+ if pg.person_is_deleted is None:
+ return _verdict("mapping", "stray", "orphan")
+ if pg.person_is_deleted:
+ return _verdict("mapping", "stray", _stray_sub_flavor(pg, ch, owner_ch))
if ch is None or ch.visible_rows == 0:
return _verdict("mapping", "missing_in_ch")
if ch.winner_deleted:
@@ -213,3 +283,19 @@ def classify_mapping(
if ch.winner_version > pg.version:
return _verdict("mapping", "stale")
return _verdict("mapping", "ok")
+
+
+def _stray_sub_flavor(pg: PgMapping, ch: ChMapping | None, owner_ch: OwnerChState | None) -> str | None:
+ """None when tombstoning the stray in Postgres alone is safe: the cleanup drain hard-deletes its Postgres row only
+ after the deletion sweep removed the owner, and that sweep removes every ClickHouse row of a key whose newest row
+ names the owner, live or tombstone. So it is safe when ClickHouse shows nothing, or when its newest row names the
+ owner and the owner still waits for the sweep."""
+ if ch is None or ch.visible_rows == 0:
+ return None
+ if ch.winner_tied:
+ return "ch_tie"
+ if ch.winner_person_uuid != pg.person_uuid:
+ return "ch_owner_differs"
+ if owner_ch is None:
+ return "owner_unchecked"
+ return {"pending": None, "visible": "owner_visible", "absent": "rows_outlive_owner"}[owner_ch]
diff --git a/posthog/models/person/divergence/inspection.py b/posthog/models/person/divergence/inspection.py
index 3e5feaf63c5c..95869b26a07a 100644
--- a/posthog/models/person/divergence/inspection.py
+++ b/posthog/models/person/divergence/inspection.py
@@ -186,13 +186,18 @@ def inspect_mappings(team_id: int, distinct_ids: Sequence[str]) -> list[Inspecti
ch = observe.ch_mapping_states(team_id, dids)
owners = sorted({c.winner_person_uuid for c in ch.values()})
owner_states = observe.pg_person_states(team_id, owners) if team_exists and owners else {}
+ owner_ch = observe.owner_ch_states(team_id, pg, ch)
history = _mapping_history(team_id, dids)
results = []
for distinct_id in dids:
ch_state = ch.get(distinct_id)
owner_state = observe.person_pg_state(owner_states, ch_state.winner_person_uuid) if ch_state else None
verdict = classify_mapping(
- pg.get(distinct_id), ch_state, ch_owner_pg_state=owner_state, team_exists=team_exists
+ pg.get(distinct_id),
+ ch_state,
+ ch_owner_pg_state=owner_state,
+ team_exists=team_exists,
+ owner_ch=owner_ch.get(distinct_id),
)
results.append(
Inspection(
diff --git a/posthog/models/person/divergence/observe.py b/posthog/models/person/divergence/observe.py
index 0696131eca46..1bc677696d94 100644
--- a/posthog/models/person/divergence/observe.py
+++ b/posthog/models/person/divergence/observe.py
@@ -6,8 +6,16 @@ from collections.abc import Sequence
from typing import Any, Literal
from uuid import UUID
-from posthog.models.person.divergence import lookups
-from posthog.models.person.divergence.flavors import ChMapping, ChPerson, PgMapping, PgPerson
+from posthog.models.person.divergence import cleanup_queue, lookups
+from posthog.models.person.divergence.flavors import (
+ ChMapping,
+ ChPerson,
+ OwnerChState,
+ PgMapping,
+ PgPerson,
+ needs_owner_ch_state,
+ owner_ch_state,
+)
from posthog.models.person.util import get_distinct_id_rows, get_person_tombstones
PersonPgState = Literal["live", "deleted", "absent"]
@@ -41,7 +49,8 @@ SELECT
countIf(_row_exists),
toString(argMaxIf(person_id, version, _row_exists)),
maxIf(version, _row_exists),
- argMaxIf(is_deleted, version, _row_exists)
+ argMaxIf(is_deleted, version, _row_exists),
+ groupUniqArrayIf((version, toString(person_id), is_deleted), _row_exists)
FROM person_distinct_id2
WHERE team_id = %(team_id)s AND {keys}
GROUP BY distinct_id
@@ -97,13 +106,17 @@ def ch_person_states_in_range(team_id: int, after: str, last: str, *, first: boo
def ch_mapping_from_row(row: Sequence[Any]) -> tuple[str, ChMapping]:
- distinct_id, max_version, visible_rows, owner, winner_version, winner_deleted = row
+ distinct_id, max_version, visible_rows, owner, winner_version, winner_deleted, visible = row
+ at_max = {
+ (str(person), bool(deleted)) for version, person, deleted in visible if int(version) == int(winner_version)
+ }
return distinct_id, ChMapping(
max_version=int(max_version),
visible_rows=int(visible_rows),
winner_person_uuid=str(owner),
winner_version=int(winner_version),
winner_deleted=bool(winner_deleted),
+ winner_tied=len(at_max) > 1,
)
@@ -115,6 +128,35 @@ def ch_mapping_states(team_id: int, distinct_ids: Sequence[str]) -> dict[str, Ch
return states
+def owner_ch_states(team_id: int, pg: dict[str, PgMapping], ch: dict[str, ChMapping]) -> dict[str, OwnerChState]:
+ """The ClickHouse state of each mapping's Postgres owner, read only for the mappings whose classification needs it."""
+ owners = {
+ distinct_id: str(UUID(pg_row.person_uuid))
+ for distinct_id, ch_row in ch.items()
+ if needs_owner_ch_state(pg_row := pg.get(distinct_id), ch_row) and pg_row is not None and pg_row.person_uuid
+ }
+ if not owners:
+ return {}
+ uuids = sorted(set(owners.values()))
+ persons = ch_person_states(team_id, uuids)
+ queued = cleanup_queue.queue_entries(team_id, uuids)
+ return {
+ distinct_id: "absent"
+ if _swept(persons.get(person_uuid), queued.get(person_uuid))
+ else owner_ch_state(persons.get(person_uuid))
+ for distinct_id, person_uuid in owners.items()
+ }
+
+
+def _swept(state: ChPerson | None, queued: dict[str, Any] | None) -> bool:
+ """The sweep queues a person after it removed the person's ClickHouse rows up to the queued bound, so rows at or
+ below that bound that show up again (a replayed Kafka batch) do not make the person pending. Rows above the bound
+ are a later tombstone the next sweep takes, and a row without a bound is never drained."""
+ if queued is None or queued["max_version"] is None:
+ return False
+ return state is None or state.max_version <= queued["max_version"]
+
+
def ch_mapping_states_in_range(team_id: int, after: str, last: str, *, first: bool) -> dict[str, ChMapping]:
rows = lookups._ch(
_CH_MAPPINGS_RANGE_SQL[first], {"team_id": team_id, "after": after, "last": last}, _READ_SETTINGS
diff --git a/posthog/models/person/divergence/repairs.py b/posthog/models/person/divergence/repairs.py
index b7ccf539e7d9..a21a965afb6e 100644
--- a/posthog/models/person/divergence/repairs.py
+++ b/posthog/models/person/divergence/repairs.py
@@ -5,7 +5,7 @@ from __future__ import annotations
import time
from collections import Counter, defaultdict
from collections.abc import Callable, Collection, Mapping, Sequence
-from dataclasses import field
+from dataclasses import field, replace
from functools import partial
from typing import TypeVar
from uuid import UUID
@@ -24,6 +24,7 @@ from posthog.models.person.util import (
_flush_person_producers,
_person_row,
create_person_distinct_id,
+ get_distinct_id_rows,
get_persons_by_uuids,
raise_person_version,
)
@@ -69,6 +70,11 @@ _TRANSIENT_RPC_CODES = frozenset(
_FLUSH_TIMEOUT_SECONDS = 5 * 60
+DEFAULT_MAX_WRITES_PER_SECOND = 20.0
+
+# Outcomes that leave ClickHouse agreeing with Postgres for the key, once the publish is delivered.
+_RESENT = frozenset({"repaired", "skipped_not_divergent"})
+
# Confirmed produce results are dropped this often, so a long repair does not hold one per published row.
_DELIVERY_PRUNE_EVERY = 1_000
@@ -519,3 +525,69 @@ def repair_persons(
if undelivered:
log(f"{undelivered} ClickHouse messages were not delivered; rerun the repair for the same input")
return outcomes.summary(applied=apply, persons=processed, undelivered=undelivered)
+
+
+def _landed(action: datatypes.RepairAction, lost: set[tuple[str, str]]) -> bool:
+ key = ("person", action.person_uuid) if action.distinct_id is None else ("mapping", action.distinct_id)
+ return action.outcome in _RESENT and key not in lost
+
+
+def republish_named(
+ team_id: int,
+ *,
+ person_uuids: Sequence[str] = (),
+ distinct_ids: Sequence[str] = (),
+ apply: bool,
+ max_writes_per_second: float | None = DEFAULT_MAX_WRITES_PER_SECOND,
+ log: Callable[[str], None],
+) -> datatypes.RepublishResult:
+ """Republish named persons, and the live owners of named distinct ids, where ClickHouse disagrees.
+
+ A scan routes a mapping that ClickHouse still gives to another live person to `merge`, which is wrong when
+ Postgres moved it on purpose and only the publish was lost, as after a split. This republishes from Postgres.
+ The owners are read on the primary, so a distinct id moved moments ago resolves to its new person.
+
+ A named person is confirmed only when it and every mapping it owns were repaired or already agreed, and every
+ row published for them was delivered. A named distinct id needs the same for itself and its owner's row. A
+ distinct id that is not live in Postgres, or a person with too many distinct ids to repair, stays unconfirmed.
+ """
+ named_persons = list(dict.fromkeys(person_uuids))
+ named_ids = list(dict.fromkeys(distinct_ids))
+ rows = get_distinct_id_rows(team_id, named_ids, strong=True) if named_ids else []
+ owner_of = {row.distinct_id: row.person_uuid for row in rows if not row.is_deleted and row.person_uuid}
+ if len(owner_of) < len(named_ids):
+ log(
+ f"republish: {len(named_ids) - len(owner_of)} distinct ids are not live in Postgres, so nothing republishes them"
+ )
+ actions: list[datatypes.RepairAction] = []
+ deliveries = _Deliveries()
+ undelivered = 0
+ try:
+ summary = repair_persons(
+ [
+ datatypes.PersonRef(team_id=team_id, person_uuid=uuid)
+ for uuid in dict.fromkeys([*named_persons, *owner_of.values()])
+ ],
+ apply=apply,
+ max_writes_per_second=max_writes_per_second,
+ on_action=actions.append,
+ log=log,
+ deliveries=deliveries,
+ )
+ finally:
+ if apply:
+ undelivered = deliveries.undelivered(_FLUSH_TIMEOUT_SECONDS)
+ lost = {(key.kind, key.key) for key in deliveries.undelivered_keys()}
+ person_rows_ok = {a.person_uuid for a in actions if a.distinct_id is None and _landed(a, lost)}
+ persons_with_misses = {a.person_uuid for a in actions if not _landed(a, lost)}
+ # A replica that lags a move can still list the distinct id under its old person, so only the owner counts.
+ ids_ok = {
+ a.distinct_id
+ for a in actions
+ if a.distinct_id is not None and a.person_uuid == owner_of.get(a.distinct_id) and _landed(a, lost)
+ }
+ unconfirmed = (
+ *(uuid for uuid in named_persons if uuid not in person_rows_ok or uuid in persons_with_misses),
+ *(d for d in named_ids if d not in ids_ok or owner_of[d] not in person_rows_ok),
+ )
+ return datatypes.RepublishResult(summary=replace(summary, undelivered=undelivered), unconfirmed=unconfirmed)
diff --git a/posthog/models/person/divergence/team_scan.py b/posthog/models/person/divergence/team_scan.py
index ab72e142d99b..d028ebe77601 100644
--- a/posthog/models/person/divergence/team_scan.py
+++ b/posthog/models/person/divergence/team_scan.py
@@ -482,6 +482,7 @@ def _classify_mappings_page(plan: ScanPlan, after: str, last: str, first: bool)
}
owners = sorted({ch[distinct_id].winner_person_uuid for distinct_id in needs_owner})
owner_states = observe.pg_person_states(plan.team_id, owners) if plan.team_exists and owners else {}
+ owner_ch = observe.owner_ch_states(plan.team_id, pg, ch)
rows: list[dict[str, Any]] = []
counts: Counter[str] = Counter()
for distinct_id, ch_state in ch.items():
@@ -489,7 +490,13 @@ def _classify_mappings_page(plan: ScanPlan, after: str, last: str, first: bool)
owner_state = (
observe.person_pg_state(owner_states, ch_state.winner_person_uuid) if distinct_id in needs_owner else None
)
- verdict = classify_mapping(pg_row, ch_state, ch_owner_pg_state=owner_state, team_exists=plan.team_exists)
+ verdict = classify_mapping(
+ pg_row,
+ ch_state,
+ ch_owner_pg_state=owner_state,
+ team_exists=plan.team_exists,
+ owner_ch=owner_ch.get(distinct_id),
+ )
counts[_count_key("mapping", verdict)] += 1
if _reported(verdict):
rows.append(
diff --git a/posthog/models/person/divergence/verbs.py b/posthog/models/person/divergence/verbs.py
index 691e44236e37..77f8af19c4b5 100644
--- a/posthog/models/person/divergence/verbs.py
+++ b/posthog/models/person/divergence/verbs.py
@@ -25,7 +25,8 @@ from posthog.models.person.deletion import (
from posthog.models.person.util import create_person_distinct_id, get_distinct_id_rows, get_persons_by_uuids
from posthog.personhog_client.proto import DistinctIdRow
-from . import datatypes, lookups, observe, repairs, restore_verbs, team_scan
+from . import datatypes, detach, lookups, observe, repairs, restore_verbs, team_scan
+from .flavors import PgMapping, classify_mapping
# A key whose last recorded outcome is one of these needs nothing more, so a resumed run skips it.
DONE_OUTCOMES = frozenset(
@@ -40,6 +41,9 @@ DONE_OUTCOMES = frozenset(
"already_moved",
"folded",
"merged",
+ "detached",
+ "already_detached",
+ "raised_for_sweep",
}
)
@@ -433,7 +437,35 @@ def _bury_mapping(row: dict[str, Any], context: VerbContext) -> None:
if ch is None or ch.visible_rows == 0 or ch.winner_deleted:
context.action(row, "skipped_not_divergent")
return
+ # A report written before the owner became pending would still name bury, but the sweep removes these rows
+ # with the owner, and a tombstone published mid-sweep could outlive the drain's hard delete.
+ pg_now = PgMapping(
+ person_id=int(current.person_id),
+ person_uuid=current.person_uuid if current.HasField("person_uuid") else None,
+ version=int(current.version),
+ is_deleted=current.is_deleted,
+ person_is_deleted=current.person_is_deleted if current.HasField("person_is_deleted") else None,
+ )
+ owner_ch = observe.owner_ch_states(team_id, {distinct_id: pg_now}, {distinct_id: ch}).get(distinct_id)
+ if classify_mapping(pg_now, ch, owner_ch=owner_ch).flavor == "awaiting_sweep":
+ context.action(row, "skipped_awaiting_sweep")
+ return
target = max(int(current.version), ch.max_version + 1)
+ if owner_ch == "pending" and ch.winner_person_uuid == pg_now.person_uuid and not ch.winner_tied:
+ # The sweep removes these rows with the owner, so raising Postgres above ClickHouse is the whole repair: a
+ # returning distinct id then revives above ClickHouse, and no tombstone is published that a running sweep
+ # could leave behind.
+ if not context.apply:
+ context.action(row, "would_raise_for_sweep", target_version=target)
+ return
+ context.pacer.before_write()
+ repairs._raise_mapping_version_floor(team_id, distinct_id, target)
+ confirmed = _strong_row(team_id, distinct_id)
+ if confirmed is None or not confirmed.is_deleted or confirmed.person_id != current.person_id:
+ context.action(row, "skipped_revived")
+ return
+ context.action(row, "raised_for_sweep", version=int(confirmed.version))
+ return
if not context.apply:
context.action(row, "would_repair", target_version=target)
return
@@ -460,4 +492,5 @@ VERBS: dict[str, VerbFn] = {
"restore": restore_verbs.restore,
"fold": restore_verbs.fold,
"merge": restore_verbs.merge,
+ "detach": detach.detach,
}
diff --git a/posthog/models/person/test/test_divergence.py b/posthog/models/person/test/test_divergence.py
index 6b208722d9d9..eff234635979 100644
--- a/posthog/models/person/test/test_divergence.py
+++ b/posthog/models/person/test/test_divergence.py
@@ -796,10 +796,14 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
stored = fake.stored_person(self.team.pk, str(person.uuid))
assert stored is not None
replica_view.CopyFrom(stored)
+ replica_ids = fake.get_distinct_ids_for_persons(
+ person_pb2.GetDistinctIdsForPersonsRequest(team_id=self.team.pk, person_ids=[stored.id])
+ )
tombstone_persons_in_postgres(self.team.pk, [person.uuid])
- with patch.object(
- fake, "get_persons_by_uuids", return_value=person_pb2.PersonsResponse(persons=[replica_view])
+ with (
+ patch.object(fake, "get_persons_by_uuids", return_value=person_pb2.PersonsResponse(persons=[replica_view])),
+ patch.object(fake, "get_distinct_ids_for_persons", return_value=replica_ids),
):
_, actions = self._repair(person.uuid)
diff --git a/posthog/models/person/test/test_divergence_flavors.py b/posthog/models/person/test/test_divergence_flavors.py
index d2a84602f1d4..9a2d18349db4 100644
--- a/posthog/models/person/test/test_divergence_flavors.py
+++ b/posthog/models/person/test/test_divergence_flavors.py
@@ -11,6 +11,7 @@ from posthog.models.person.divergence.flavors import (
Verdict,
classify_mapping,
classify_person,
+ owner_ch_state,
)
@@ -132,13 +133,16 @@ def _pg_map(
)
-def _ch_map(*, owner: str = "b", version: int = 0, deleted: bool = False, visible_rows: int = 1) -> ChMapping:
+def _ch_map(
+ *, owner: str = "b", version: int = 0, deleted: bool = False, visible_rows: int = 1, tied: bool = False
+) -> ChMapping:
return ChMapping(
max_version=version,
visible_rows=visible_rows,
winner_person_uuid=owner,
winner_version=version,
winner_deleted=deleted,
+ winner_tied=tied,
)
@@ -171,19 +175,12 @@ class TestClassifyMapping(SimpleTestCase):
None,
_v("ok_deleted", None, None),
),
- (
- "stray_on_deleted_person",
- _pg_map(owner_deleted=True),
- _ch_map(),
- None,
- _v("stray", None, None),
- ),
(
"orphan_with_ch_on_another_person",
_pg_map(owner=None, owner_deleted=None),
_ch_map(owner="a"),
"absent",
- _v("stray", None, None),
+ _v("stray", "orphan", None),
),
("live_missing_in_ch", _pg_map(), None, None, _v("missing_in_ch", None, "republish")),
("live_and_swept", _pg_map(), _ch_map(visible_rows=0), None, _v("missing_in_ch", None, "republish")),
@@ -211,6 +208,133 @@ class TestClassifyMapping(SimpleTestCase):
def test_classifies(self, _name, pg, ch, owner_state, expected):
assert classify_mapping(pg, ch, ch_owner_pg_state=owner_state) == expected
+ @parameterized.expand(
+ [
+ # (name, Postgres mapping, ClickHouse mapping, owner's ClickHouse state, expected)
+ ("stray_nothing_in_ch", _pg_map(owner_deleted=True), None, None, _v("stray", None, "detach")),
+ (
+ "stray_ch_rows_all_lightweight_deleted",
+ _pg_map(owner_deleted=True),
+ _ch_map(visible_rows=0),
+ None,
+ _v("stray", None, "detach"),
+ ),
+ (
+ "stray_live_on_pending_owner",
+ _pg_map(owner_deleted=True),
+ _ch_map(),
+ "pending",
+ _v("stray", None, "detach"),
+ ),
+ (
+ "stray_tombstone_naming_pending_owner",
+ _pg_map(owner_deleted=True),
+ _ch_map(deleted=True),
+ "pending",
+ _v("stray", None, "detach"),
+ ),
+ (
+ "stray_owner_visible",
+ _pg_map(owner_deleted=True),
+ _ch_map(),
+ "visible",
+ _v("stray", "owner_visible", None),
+ ),
+ (
+ "stray_rows_outlive_owner",
+ _pg_map(owner_deleted=True),
+ _ch_map(),
+ "absent",
+ _v("stray", "rows_outlive_owner", None),
+ ),
+ (
+ "stray_live_on_another_person",
+ _pg_map(owner_deleted=True),
+ _ch_map(owner="a"),
+ "pending",
+ _v("stray", "ch_owner_differs", None),
+ ),
+ (
+ "stray_tombstone_naming_another_person",
+ _pg_map(owner_deleted=True),
+ _ch_map(owner="a", deleted=True),
+ "pending",
+ _v("stray", "ch_owner_differs", None),
+ ),
+ (
+ "stray_owner_unchecked",
+ _pg_map(owner_deleted=True),
+ _ch_map(),
+ None,
+ _v("stray", "owner_unchecked", None),
+ ),
+ (
+ "stray_ch_top_version_tied",
+ _pg_map(owner_deleted=True),
+ _ch_map(tied=True),
+ "pending",
+ _v("stray", "ch_tie", None),
+ ),
+ (
+ "tombstoned_on_pending_owner_awaits_the_sweep",
+ _pg_map(version=6, deleted=True, owner_deleted=True),
+ _ch_map(version=5),
+ "pending",
+ _v("awaiting_sweep", None, None),
+ ),
+ (
+ "tombstoned_not_above_clickhouse_is_buried",
+ _pg_map(version=5, deleted=True, owner_deleted=True),
+ _ch_map(version=5),
+ "pending",
+ _v("deleted_visible", None, "bury"),
+ ),
+ (
+ "tombstoned_with_tied_clickhouse_rows_is_buried",
+ _pg_map(version=6, deleted=True, owner_deleted=True),
+ _ch_map(version=5, tied=True),
+ "pending",
+ _v("deleted_visible", None, "bury"),
+ ),
+ (
+ "tombstoned_on_visible_owner_is_buried",
+ _pg_map(version=6, deleted=True, owner_deleted=True),
+ _ch_map(version=5),
+ "visible",
+ _v("deleted_visible", None, "bury"),
+ ),
+ (
+ "tombstoned_on_live_owner_is_buried",
+ _pg_map(deleted=True),
+ _ch_map(),
+ None,
+ _v("deleted_visible", None, "bury"),
+ ),
+ (
+ "tombstoned_ch_names_another_person_is_buried",
+ _pg_map(deleted=True, owner_deleted=True),
+ _ch_map(owner="a"),
+ "pending",
+ _v("deleted_visible", None, "bury"),
+ ),
+ ]
+ )
+ def test_classifies_by_where_the_owner_stands_in_clickhouse(self, _name, pg, ch, owner_ch, expected):
+ assert classify_mapping(pg, ch, owner_ch=owner_ch) == expected
+
+ @parameterized.expand(
+ [
+ ("no_rows", None, "absent"),
+ ("only_lightweight_deleted_rows", _ch(tomb=4, visible_rows=0), "absent"),
+ ("tombstone_newest", _ch(live=3, tomb=4), "pending"),
+ ("only_a_tombstone", _ch(tomb=2), "pending"),
+ ("live_newest", _ch(live=5, tomb=4), "visible"),
+ ("live_and_tombstone_tie", _ch(live=4, tomb=4), "visible"),
+ ]
+ )
+ def test_owner_ch_state_follows_the_sweeps_rule(self, _name, state, expected):
+ assert owner_ch_state(state) == expected
+
def test_ghost_of_a_deleted_team_is_left_alone(self):
assert classify_mapping(None, _ch_map(owner="a"), ch_owner_pg_state="absent", team_exists=False) == _v(
"ghost", "team_gone", None
diff --git a/posthog/models/person/test/test_divergence_inspection.py b/posthog/models/person/test/test_divergence_inspection.py
index a5027d9633dd..2159bf21e5d7 100644
--- a/posthog/models/person/test/test_divergence_inspection.py
+++ b/posthog/models/person/test/test_divergence_inspection.py
@@ -84,13 +84,14 @@ class TestDivergenceInspection(DivergenceFixturesMixin, ClickhouseTestMixin, Bas
is_deleted=True,
distinct_ids=["stray"],
)
+ self._ch_person(deleted.uuid, 3, {}, deleted=True)
self._ch_mapping("stray", deleted.uuid, 0)
flavors = self._flavors(inspect_mappings(self.team.pk, ["moved", "stray"]))
assert flavors == {
"moved": ("diverged", "a_absent", "fold"),
- "stray": ("stray", None, None),
+ "stray": ("stray", None, "detach"),
}
def test_inspect_mappings_leaves_ghosts_of_a_deleted_team_alone(self):
diff --git a/posthog/models/person/test/test_divergence_split.py b/posthog/models/person/test/test_divergence_split.py
new file mode 100644
index 000000000000..3e79d5aba199
--- /dev/null
+++ b/posthog/models/person/test/test_divergence_split.py
@@ -0,0 +1,317 @@
+import shlex
+from io import StringIO
+from pathlib import Path
+from typing import Any
+
+from posthog.test.base import BaseTest, ClickhouseTestMixin
+from unittest.mock import patch
+
+from django.core.management import CommandError, call_command
+from django.utils.timezone import now
+
+from confluent_kafka import KafkaError
+from parameterized import parameterized
+
+from posthog.clickhouse.client import sync_execute
+from posthog.kafka_client.client import ClickhouseProducer, ProduceResult
+from posthog.kafka_client.topics import KAFKA_PERSON, KAFKA_PERSON_DISTINCT_ID
+from posthog.models.person import Person
+from posthog.models.person.divergence import observe
+from posthog.models.person.divergence.detach import SplitError, split_distinct_ids
+from posthog.models.person.sql import BULK_INSERT_PERSON_DISTINCT_ID2
+from posthog.models.person.util import (
+ DistinctIdForPerson,
+ _batched_get_distinct_ids_for_persons,
+ create_person as create_person_in_ch,
+ get_distinct_id_rows,
+ get_persons_by_uuids,
+)
+from posthog.models.signals import mute_selected_signals
+from posthog.personhog_client.fake_client import fake_personhog_client
+from posthog.test.persons import create_person
+
+PERSON_UUID = "00000000-0000-0000-0000-0000000000a1"
+
+
+def _insert_ch_mapping(team_id: int, distinct_id: str, person_uuid: str) -> None:
+ sync_execute(
+ BULK_INSERT_PERSON_DISTINCT_ID2,
+ [
+ {
+ "distinct_id": distinct_id,
+ "person_id": person_uuid,
+ "team_id": team_id,
+ "is_deleted": 0,
+ "version": 0,
+ "_timestamp": now().replace(tzinfo=None),
+ "_offset": 0,
+ "_partition": 0,
+ }
+ ],
+ flush=False,
+ )
+
+
+def _failed_produce() -> ProduceResult:
+ failed = ProduceResult(topic=KAFKA_PERSON)
+ failed.set_result(KafkaError(-192, "Local: Message timed out"), None)
+ return failed
+
+
+@patch("posthog.models.person.util.create_person")
+@patch("posthog.models.person.util.create_person_distinct_id")
+class TestSplitDistinctIds(BaseTest):
+ @parameterized.expand(
+ [
+ ("applies", ["id2", "id3"], True, ["id2", "id3"]),
+ ("dry_run_writes_nothing", ["id2"], False, []),
+ ("a_mistyped_id_stops_the_whole_split", ["id2", "typo"], True, SplitError),
+ ("an_id_of_another_person_stops_the_whole_split", ["id2", "other-1"], True, SplitError),
+ ("a_split_leaving_no_id_is_refused", ["id1", "id2", "id3"], True, SplitError),
+ ("a_dry_run_leaving_no_id_is_refused", ["id1", "id2", "id3"], False, SplitError),
+ ]
+ )
+ def test_split_moves_only_ids_live_on_the_person(
+ self, _mock_create_pdi, _mock_create_person, _name, distinct_ids, apply, expected
+ ):
+ with fake_personhog_client() as fake:
+ fake.add_person(team_id=self.team.id, person_id=501, uuid=PERSON_UUID, distinct_ids=["id1", "id2", "id3"])
+ fake.add_person(
+ team_id=self.team.id,
+ person_id=502,
+ uuid="00000000-0000-0000-0000-0000000000a2",
+ distinct_ids=["other-1"],
+ )
+
+ if expected is SplitError:
+ with self.assertRaises(SplitError):
+ split_distinct_ids(self.team.id, PERSON_UUID, distinct_ids, apply=apply, log=lambda _: None)
+ fake.assert_not_called("split_person")
+ return
+
+ assert (
+ split_distinct_ids(self.team.id, PERSON_UUID, distinct_ids, apply=apply, log=lambda _: None) == expected
+ )
+ if apply:
+ assert set(fake.assert_called("split_person", times=1)[0].request.distinct_ids_to_split) == set(
+ expected
+ )
+ else:
+ fake.assert_not_called("split_person")
+ owners = {d: fake._persons_by_distinct_id[(self.team.id, d)].id for d in ["id1", "id2", "id3"]}
+ assert owners["id1"] == 501
+ assert all((owners[d] != 501) == (d in expected) for d in ["id2", "id3"])
+
+ def test_split_accepts_any_uuid_spelling_and_moves_each_id_once(self, *_mocks):
+ with fake_personhog_client() as fake:
+ fake.add_person(team_id=self.team.id, person_id=501, uuid=PERSON_UUID, distinct_ids=["id1", "id2"])
+
+ moved = split_distinct_ids(
+ self.team.id, PERSON_UUID.upper(), ["id2", "id2"], apply=True, log=lambda _: None
+ )
+
+ assert moved == ["id2"]
+ assert list(fake.assert_called("split_person", times=1)[0].request.distinct_ids_to_split) == ["id2"]
+
+ def test_a_split_that_moves_fewer_ids_than_asked_says_so(self, *_mocks):
+ with (
+ fake_personhog_client() as fake,
+ patch("posthog.models.person.person.Person._split_distinct_ids_batch", return_value=[]),
+ ):
+ fake.add_person(team_id=self.team.id, person_id=501, uuid=PERSON_UUID, distinct_ids=["id1", "id2"])
+
+ with self.assertRaisesRegex(SplitError, "moved 0 of 1 distinct ids"):
+ split_distinct_ids(self.team.id, PERSON_UUID, ["id2"], apply=True, log=lambda _: None)
+
+ def test_a_failed_split_still_waits_for_the_messages_it_already_sent(self, *_mocks):
+ with (
+ fake_personhog_client() as fake,
+ patch("posthog.models.person.person.Person.split_person", side_effect=RuntimeError("batch 2 failed")),
+ patch("posthog.models.person.divergence.detach.flush_all_producers", return_value=0) as flush,
+ ):
+ fake.add_person(team_id=self.team.id, person_id=501, uuid=PERSON_UUID, distinct_ids=["id1", "id2"])
+
+ with self.assertRaisesRegex(SplitError, "batch 2 failed"):
+ split_distinct_ids(self.team.id, PERSON_UUID, ["id2"], apply=True, log=lambda _: None)
+
+ flush.assert_called_once()
+
+
+class TestSplitLostInClickHouse(ClickhouseTestMixin, BaseTest):
+ DISTINCT_IDS = ["id1", "id2", "id3", " padded id ", " ", "carriage\rreturn", "two\nlines"]
+
+ def setUp(self) -> None:
+ super().setUp()
+ with mute_selected_signals():
+ self.person = create_person(team=self.team, distinct_ids=self.DISTINCT_IDS)
+ create_person_in_ch(team_id=self.team.pk, uuid=str(self.person.uuid), version=0, timestamp=now())
+ for distinct_id in self.DISTINCT_IDS:
+ _insert_ch_mapping(self.team.pk, distinct_id, str(self.person.uuid))
+ sleep = patch("posthog.models.person.divergence.detach._sleep")
+ self.sleep = sleep.start()
+ self.addCleanup(sleep.stop)
+
+ def _split_losing_its_messages(self, distinct_ids: list[str]) -> list[str]:
+ # The split's own publishes never reach Kafka, so Kafka has nothing queued to report undelivered.
+ with (
+ patch("posthog.models.person.util.create_person"),
+ patch("posthog.models.person.util.create_person_distinct_id"),
+ ):
+ return split_distinct_ids(self.team.pk, str(self.person.uuid), distinct_ids, apply=True, log=lambda _: None)
+
+ def _assert_clickhouse_agrees_with_postgres(self, distinct_id: str) -> None:
+ owner = get_distinct_id_rows(self.team.pk, [distinct_id], strong=True)[0].person_uuid
+ assert owner != str(self.person.uuid)
+ assert observe.ch_mapping_states(self.team.pk, [distinct_id])[distinct_id].winner_person_uuid == owner
+ assert observe.ch_person_states(self.team.pk, [owner])[owner].live_max_version is not None
+
+ def _assert_clickhouse_shows_old_owner(self, distinct_id: str) -> None:
+ winner = observe.ch_mapping_states(self.team.pk, [distinct_id])[distinct_id].winner_person_uuid
+ assert winner == str(self.person.uuid)
+
+ def test_a_split_clickhouse_missed_is_sent_again_so_no_scan_suggests_merging_it_back(self) -> None:
+ assert self._split_losing_its_messages(["id2"]) == ["id2"]
+
+ self._assert_clickhouse_agrees_with_postgres("id2")
+ self._assert_clickhouse_shows_old_owner("id1")
+ self.sleep.assert_not_called()
+
+ def test_a_replica_behind_the_split_is_read_again_before_the_split_reports_it(self) -> None:
+ reads: list[list[str]] = []
+
+ def lagging_replica(team_id: int, person_uuids: list[str], **kwargs: Any) -> list[Person]:
+ reads.append(person_uuids)
+ return [] if len(reads) == 1 else get_persons_by_uuids(team_id, person_uuids, **kwargs)
+
+ with patch("posthog.models.person.divergence.repair_plan.get_persons_by_uuids", side_effect=lagging_replica):
+ assert self._split_losing_its_messages(["id2"]) == ["id2"]
+
+ self.sleep.assert_called_once()
+ self._assert_clickhouse_agrees_with_postgres("id2")
+
+ def _split_failing_partway(self) -> None:
+ with (
+ patch("posthog.models.person.person.PERSONHOG_SPLIT_BATCH_SIZE", 1),
+ patch(
+ "posthog.models.person.person.Person._publish_split_to_kafka",
+ side_effect=[None, RuntimeError("Kafka is down")],
+ ),
+ ):
+ split_distinct_ids(self.team.pk, str(self.person.uuid), ["id2", "id3"], apply=True, log=lambda _: None)
+
+ def test_a_split_that_fails_partway_sends_what_it_moved(self) -> None:
+ with self.assertRaisesRegex(SplitError, "moved 2 of 2 distinct ids .* then failed: Kafka is down"):
+ self._split_failing_partway()
+
+ self._assert_clickhouse_agrees_with_postgres("id2")
+ self._assert_clickhouse_agrees_with_postgres("id3")
+
+ def test_a_split_that_fails_partway_and_cannot_be_sent_again_reports_both(self) -> None:
+ with (
+ patch.object(ClickhouseProducer, "produce", return_value=_failed_produce()),
+ self.assertRaisesRegex(SplitError, "may still show 2 .* also failed: Kafka is down. Do not run a merge"),
+ ):
+ self._split_failing_partway()
+
+ @parameterized.expand(
+ [
+ ("listed_inline", ["id2"], 10, False),
+ ("read_from_a_file", ["id2", " padded id ", " ", "carriage\rreturn"], 0, True),
+ ("a_distinct_id_with_a_newline_stays_inline", ["two\nlines"], 0, False),
+ ]
+ )
+ def test_a_split_that_cannot_be_sent_again_names_the_command_that_finishes_it(
+ self, _name, distinct_ids, inline_ids, from_file
+ ) -> None:
+ with (
+ patch("posthog.models.person.divergence.detach._INLINE_DISTINCT_IDS", inline_ids),
+ patch.object(ClickhouseProducer, "produce", return_value=_failed_produce()),
+ self.assertRaisesRegex(SplitError, "Do not run a merge") as raised,
+ ):
+ self._split_losing_its_messages(distinct_ids)
+
+ command = shlex.split(str(raised.exception).split("Run: python manage.py ")[1])
+ assert ("--distinct-ids-file" in command) == from_file
+ if from_file:
+ self.addCleanup(Path(command[command.index("--distinct-ids-file") + 1]).unlink)
+ assert command[-1] == "--apply"
+ call_command(*command[:-1], stdout=StringIO())
+ for distinct_id in distinct_ids:
+ self._assert_clickhouse_shows_old_owner(distinct_id)
+ call_command(*command, stdout=StringIO())
+ for distinct_id in distinct_ids:
+ self._assert_clickhouse_agrees_with_postgres(distinct_id)
+
+
+class TestRepublishCommand(ClickhouseTestMixin, BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ with mute_selected_signals():
+ self.person = create_person(team=self.team, distinct_ids=["id1", "id2"])
+
+ def _republish(self, *args: str) -> None:
+ call_command(
+ "person_consistency", "republish", "--team-id", str(self.team.pk), *args, "--apply", stdout=StringIO()
+ )
+
+ def test_a_person_uuid_in_any_spelling_is_republished(self) -> None:
+ self._republish("--person-uuid", str(self.person.uuid).upper())
+
+ assert (
+ observe.ch_person_states(self.team.pk, [str(self.person.uuid)])[str(self.person.uuid)].live_max_version
+ is not None
+ )
+
+ @parameterized.expand(
+ [
+ ("nothing_named", [], "name at least one"),
+ ("a_person_uuid_that_is_not_a_uuid", ["--person-uuid", "not-a-uuid"], "must be a uuid"),
+ ("a_distinct_id_not_live_in_postgres", ["--distinct-id", "ghost"], "may still disagree .* ghost"),
+ ]
+ )
+ def test_republish_fails_when_it_cannot_confirm_a_named_key(self, _name, args, error) -> None:
+ with self.assertRaisesRegex(CommandError, error):
+ self._republish(*args)
+
+ def test_republish_fails_for_a_distinct_id_whose_owner_has_too_many_distinct_ids(self) -> None:
+ with (
+ patch("posthog.models.person.divergence.repair_plan._MAX_REPAIR_DISTINCT_IDS_PER_PERSON", 1),
+ self.assertRaisesRegex(CommandError, "may still disagree .* id2"),
+ ):
+ self._republish("--distinct-id", "id2")
+
+ assert observe.ch_mapping_states(self.team.pk, ["id2"]).get("id2") is None
+
+ def test_republish_fails_for_a_named_person_whose_mapping_was_not_delivered(self) -> None:
+ def produce(*args: Any, **kwargs: Any) -> ProduceResult:
+ if kwargs["topic"] == KAFKA_PERSON_DISTINCT_ID:
+ return _failed_produce()
+ delivered = ProduceResult(topic=kwargs["topic"])
+ delivered.set_result(None, None)
+ return delivered
+
+ with (
+ patch.object(ClickhouseProducer, "produce", side_effect=produce),
+ self.assertRaisesRegex(CommandError, f"may still disagree .* {self.person.uuid}"),
+ ):
+ self._republish("--person-uuid", str(self.person.uuid))
+
+ def test_a_replica_listing_a_distinct_id_under_its_old_person_does_not_confirm_it(self) -> None:
+ with mute_selected_signals():
+ new_owner = create_person(team=self.team, distinct_ids=["moved"])
+ _insert_ch_mapping(self.team.pk, "moved", str(self.person.uuid))
+
+ def lagging_replica(team_id: int, person_ids: list[int], **kwargs: Any) -> dict[int, list[DistinctIdForPerson]]:
+ by_person = _batched_get_distinct_ids_for_persons(team_id, person_ids, **kwargs)
+ by_person[new_owner.pk] = [d for d in by_person.get(new_owner.pk, []) if d.id != "moved"]
+ by_person.setdefault(self.person.pk, []).append(DistinctIdForPerson(id="moved", version=0))
+ return by_person
+
+ with (
+ patch(
+ "posthog.models.person.divergence.repair_plan._batched_get_distinct_ids_for_persons",
+ side_effect=lagging_replica,
+ ),
+ self.assertRaisesRegex(CommandError, "may still disagree with Postgres for 1 named keys: moved\\."),
+ ):
+ self._republish("--person-uuid", str(self.person.uuid), "--distinct-id", "moved")
diff --git a/posthog/models/person/test/test_divergence_team_scan.py b/posthog/models/person/test/test_divergence_team_scan.py
index 617cbdd9642f..37eb7ff24c92 100644
--- a/posthog/models/person/test/test_divergence_team_scan.py
+++ b/posthog/models/person/test/test_divergence_team_scan.py
@@ -106,7 +106,7 @@ class TestTeamScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
"x2": ("diverged", "ambiguous", None, f"ambiguous:{keys['holder']}", ()),
keys["unseen"]: ("missing_in_ch", None, "republish", None, ()),
"unseen-1": ("missing_in_ch", None, "republish", None, ()),
- "stray-1": ("stray", None, None, None, ()),
+ "stray-1": ("stray", None, "detach", None, ()),
keys["lonely"]: ("no_live_mappings", None, None, None, ()),
}
summary = json.loads((scan_dir / team_scan.SUMMARY_FILE).read_text())
diff --git a/posthog/models/person/test/test_divergence_verbs.py b/posthog/models/person/test/test_divergence_verbs.py
index 7dfbccbf0aa8..2e77cfe66980 100644
--- a/posthog/models/person/test/test_divergence_verbs.py
+++ b/posthog/models/person/test/test_divergence_verbs.py
@@ -17,9 +17,11 @@ from parameterized import parameterized
from posthog.kafka_client.client import ClickhouseProducer, ProduceResult
from posthog.kafka_client.topics import KAFKA_PERSON
from posthog.models.person.divergence import repairs, team_scan, verbs
+from posthog.models.person.divergence.cleanup_queue import PG_CLEANUP_QUEUE_TABLE
from posthog.models.person.test.divergence_fixtures import DivergenceFixturesMixin
from posthog.models.person.util import tombstone_persons_in_postgres
from posthog.personhog_client.fake_client import get_active_fake
+from posthog.persons_db import persons_db_connection
class _Stop(Exception):
@@ -112,7 +114,7 @@ class TestRepairVerbs(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
assert resumed == [k for k in (str(first.uuid), str(second.uuid)) if k != canary_persons[0][0]]
assert self._scan()[1] == {}
- def test_bury_tombstones_what_postgres_deleted_and_leaves_strays_to_ingestion(self):
+ def test_bury_and_detach_tombstone_what_postgres_deleted(self):
deleted = self._pg_person(version=2)
tombstone_persons_in_postgres(self.team.pk, [deleted.uuid])
self._ch_person(str(deleted.uuid), 7, {"email": "[email protected]"})
@@ -131,31 +133,166 @@ class TestRepairVerbs(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
self._ch_mapping("gone", holder.uuid, 5)
self._ch_mapping("unpublished", holder.uuid, 1)
self._ch_mapping("holder-1", holder.uuid, 0)
- stray_owner = fake.add_person(
- team_id=self.team.pk,
- person_id=990_002,
- uuid=str(uuid4()),
- version=3,
- is_deleted=True,
- distinct_ids=["stray-1"],
- )
- self._ch_mapping("stray-1", stray_owner.uuid, 0)
+
+ def stray_owner(person_id: int, *distinct_ids: str) -> str:
+ return fake.add_person(
+ team_id=self.team.pk,
+ person_id=person_id,
+ uuid=str(uuid4()),
+ version=3,
+ is_deleted=True,
+ distinct_ids=list(distinct_ids),
+ ).uuid
+
+ # No ClickHouse rows for the strays.
+ nothing_in_ch = stray_owner(990_010, "s-none", "s-later-diverged")
+ # ClickHouse resolves the stray to its owner, which still waits for the sweep, at a version above Postgres.
+ pending = stray_owner(990_011, "s-pending")
+ self._ch_person(pending, 3, {}, deleted=True)
+ self._ch_mapping("s-pending", pending, 5)
+ # The owner's tombstone never reached ClickHouse.
+ visible = stray_owner(990_012, "s-owner-visible")
+ self._ch_person(visible, 2, {"email": "[email protected]"})
+ self._ch_mapping("s-owner-visible", visible, 0)
+ # ClickHouse resolves the stray to another person.
+ differs = stray_owner(990_013, "s-differs")
+ self._ch_person(differs, 3, {}, deleted=True)
+ self._ch_mapping("s-differs", holder.uuid, 4)
+ # The owner is gone from ClickHouse, but the stray still has rows there.
+ outlived = stray_owner(990_014, "s-outlived")
+ self._ch_mapping("s-outlived", outlived, 0)
+ # A sweep already queued the owner, so it is swept even though its ClickHouse rows show up again.
+ requeued = stray_owner(990_015, "s-requeued")
+ self._ch_person(requeued, 3, {}, deleted=True)
+ self._ch_mapping("s-requeued", requeued, 0)
+ # Rows at ClickHouse's top version name two owners.
+ tied = stray_owner(990_016, "s-tied")
+ self._ch_person(tied, 3, {}, deleted=True)
+ self._ch_mapping("s-tied", tied, 2)
+ self._ch_mapping("s-tied", holder.uuid, 2)
+ # Its only stray has no ClickHouse rows, so detaching it leaves the owner nothing live.
+ released = stray_owner(990_017, "s-released")
+ # Queued by an earlier sweep, then tombstoned again above that bound: the next sweep takes it.
+ retombstoned = stray_owner(990_018, "s-retombstoned")
+ self._ch_person(retombstoned, 5, {}, deleted=True)
+ self._ch_mapping("s-retombstoned", retombstoned, 0)
+ self._queue_person(retombstoned, max_version=3)
+ self._queue_person(nothing_in_ch, blocked=True)
+ self._queue_person(requeued, blocked=True)
+ self._queue_person(released, blocked=True)
+
scan_dir, before = self._scan()
assert before == {
str(deleted.uuid): ("deleted_visible", "bury"),
"gone": ("deleted_visible", "bury"),
"unpublished": ("deleted_visible", "bury"),
- "stray-1": ("stray", None),
+ visible: ("deleted_visible", "bury"),
+ "s-none": ("stray", "detach"),
+ "s-later-diverged": ("stray", "detach"),
+ "s-pending": ("stray", "detach"),
+ "s-released": ("stray", "detach"),
+ "s-retombstoned": ("stray", "detach"),
+ "s-owner-visible": ("stray", None),
+ "s-differs": ("stray", None),
+ "s-outlived": ("stray", None),
+ "s-requeued": ("stray", None),
+ "s-tied": ("stray", None),
}
+ assert self._sub_flavors(scan_dir, "s-owner-visible", "s-differs", "s-outlived", "s-requeued", "s-tied") == [
+ "owner_visible",
+ "ch_owner_differs",
+ "rows_outlive_owner",
+ "rows_outlive_owner",
+ "ch_tie",
+ ]
buried = self._run(scan_dir, "bury", apply=True)
+ # ClickHouse changes after the scan: detach re-checks before it writes.
+ self._ch_mapping("s-later-diverged", holder.uuid, 2)
+ planned = self._run(scan_dir, "detach", apply=False, name="dry")
+ detached = self._run(scan_dir, "detach", apply=True)
assert {a["key"]: a["outcome"] for a in buried} == {
str(deleted.uuid): "republished",
"gone": "buried",
"unpublished": "buried",
+ visible: "republished",
+ }
+ assert {a["key"]: a["outcome"] for a in planned} == {
+ "s-none": "would_detach",
+ "s-later-diverged": "skipped_ch_owner_differs",
+ "s-pending": "would_detach",
+ "s-released": "would_detach",
+ "s-retombstoned": "would_detach",
+ }
+ assert {a["key"]: (a["outcome"], a["detail"].get("version")) for a in detached} == {
+ "s-none": ("detached", 1),
+ "s-later-diverged": ("skipped_ch_owner_differs", None),
+ "s-pending": ("detached", 6),
+ "s-released": ("detached", 1),
+ "s-retombstoned": ("detached", 1),
+ }
+ # Released only once the owner has no live distinct id left: s-later-diverged still holds its owner.
+ assert not self._queue_blocked(released)
+ assert self._queue_blocked(nothing_in_ch)
+ assert self._queue_blocked(requeued)
+
+ rescan_dir, after = self._scan()
+ assert after == {
+ "s-later-diverged": ("stray", None),
+ "s-pending": ("awaiting_sweep", None),
+ "s-retombstoned": ("awaiting_sweep", None),
+ "s-owner-visible": ("stray", "detach"),
+ "s-differs": ("stray", None),
+ "s-outlived": ("stray", None),
+ "s-requeued": ("stray", None),
+ "s-tied": ("stray", None),
}
- assert self._scan()[1] == {"stray-1": ("stray", None)}
+
+ assert [(a["key"], a["outcome"]) for a in self._run(rescan_dir, "detach", apply=True)] == [
+ ("s-owner-visible", "detached")
+ ]
+ assert self._scan()[1]["s-owner-visible"] == ("awaiting_sweep", None)
+
+ # A report from before the owner was pending still names bury; bury re-checks and leaves it to the sweep.
+ report = rescan_dir / team_scan.REPORT_FILE
+ stale = [json.loads(line) for line in report.read_text().splitlines()]
+ report.write_text(
+ "".join(json.dumps({**r, "verb": "bury"} if r["key"] == "s-pending" else r) + "\n" for r in stale)
+ )
+ assert [(a["key"], a["outcome"]) for a in self._run(rescan_dir, "bury", apply=True)] == [
+ ("s-pending", "skipped_awaiting_sweep")
+ ]
+
+ def test_bury_only_raises_postgres_for_a_mapping_the_sweep_removes_with_its_owner(self):
+ fake = get_active_fake()
+ assert fake is not None
+ owner = fake.add_person(
+ team_id=self.team.pk,
+ person_id=990_020,
+ uuid=str(uuid4()),
+ version=3,
+ is_deleted=True,
+ distinct_ids=["t-below"],
+ distinct_id_versions={"t-below": 2},
+ tombstoned_distinct_ids=["t-below"],
+ ).uuid
+ self._ch_person(owner, 3, {}, deleted=True)
+ self._ch_mapping("t-below", owner, 4)
+ scan_dir, before = self._scan()
+ assert before == {"t-below": ("deleted_visible", "bury")}
+
+ planned = self._run(scan_dir, "bury", apply=False, name="dry")
+ raised = self._run(scan_dir, "bury", apply=True)
+
+ assert [(a["outcome"], a["detail"]["target_version"]) for a in planned] == [("would_raise_for_sweep", 5)]
+ assert [(a["outcome"], a["detail"]["version"]) for a in raised] == [("raised_for_sweep", 5)]
+ # Still live in ClickHouse, so no tombstone was published; Postgres is now above it.
+ assert self._scan()[1] == {"t-below": ("awaiting_sweep", None)}
+
+ def _sub_flavors(self, scan_dir: Path, *keys: str) -> list[str | None]:
+ rows = {r["key"]: r for r in map(json.loads, (scan_dir / team_scan.REPORT_FILE).read_text().splitlines())}
+ return [rows[key]["sub_flavor"] for key in keys]
def _hidden_person_with_stale_mapping(self) -> str:
person = self._pg_person(version=1, distinct_ids=["h-1"])
@@ -243,6 +380,24 @@ class TestRepairVerbs(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
assert self._scan()[1] == {}
+ def _queue_person(self, person_uuid: str, *, blocked: bool = False, max_version: int = 3) -> None:
+ with persons_db_connection(writer=True) as conn, conn.cursor() as cursor:
+ cursor.execute(
+ f"INSERT INTO {PG_CLEANUP_QUEUE_TABLE} (team_id, person_uuid, deleted_at, max_version, blocked_at) "
+ f"VALUES (%s, %s, now(), %s, {'now()' if blocked else 'NULL'})",
+ [self.team.pk, person_uuid, max_version],
+ )
+
+ def _queue_blocked(self, person_uuid: str) -> bool:
+ with persons_db_connection(writer=False) as conn, conn.cursor() as cursor:
+ cursor.execute(
+ f"SELECT blocked_at IS NOT NULL FROM {PG_CLEANUP_QUEUE_TABLE} WHERE team_id = %s AND person_uuid = %s",
+ [self.team.pk, person_uuid],
+ )
+ row = cursor.fetchone()
+ assert row is not None
+ return row[0]
+
def test_a_run_stopped_by_sigterm_records_its_undelivered_keys_for_resume(self):
key = self._hidden_person_with_stale_mapping()
scan_dir, _ = self._scan()
diff --git a/posthog/personhog_client/fake_client.py b/posthog/personhog_client/fake_client.py
index 4c2a596b5842..51323769b0a6 100644
--- a/posthog/personhog_client/fake_client.py
+++ b/posthog/personhog_client/fake_client.py
@@ -194,6 +194,12 @@ class FakePersonHogClient:
if uuid:
self._persons_by_uuid[(team_id, uuid)] = person
for did in distinct_ids or []:
+ previous = self._persons_by_distinct_id.get((team_id, did))
+ if previous is not None and previous.id != person_id:
+ # The replica keeps one row per distinct id, so the previous owner no longer lists it.
+ self._distinct_ids[(team_id, previous.id)] = [
+ d for d in self._distinct_ids.get((team_id, previous.id), []) if d.distinct_id != did
+ ]
self._persons_by_distinct_id[(team_id, did)] = person
# A re-added mapping is live again, as a revival upsert makes it in the replica.
self._tombstoned_distinct_ids.discard((team_id, did))
@@ -359,7 +365,7 @@ class FakePersonHogClient:
self, request: person_pb2.GetDistinctIdsForPersonRequest
) -> person_pb2.GetDistinctIdsForPersonResponse:
self.calls.append(_Call("get_distinct_ids_for_person", request))
- dids = list(self._distinct_ids.get((request.team_id, request.person_id), []))
+ dids = self._live_distinct_ids(request.team_id, request.person_id)
limit = request.limit if request.HasField("limit") and request.limit > 0 else None
has_cursor = request.HasField("cursor_id")
cursor_id = request.cursor_id if has_cursor else None
@@ -386,6 +392,14 @@ class FakePersonHogClient:
next_cursor_id=next_cursor_id,
)
+ def _live_distinct_ids(self, team_id: int, person_id: int) -> list[person_pb2.DistinctIdWithVersion]:
+ # The replica reads only rows with is_deleted = false, so a tombstoned mapping is never listed.
+ return [
+ d
+ for d in self._distinct_ids.get((team_id, person_id), [])
+ if (team_id, d.distinct_id) not in self._tombstoned_distinct_ids
+ ]
+
def get_distinct_ids_for_persons(
self, request: person_pb2.GetDistinctIdsForPersonsRequest
) -> person_pb2.GetDistinctIdsForPersonsResponse:
@@ -395,7 +409,7 @@ class FakePersonHogClient:
)
results = []
for pid in request.person_ids:
- dids = self._distinct_ids.get((request.team_id, pid), [])
+ dids = self._live_distinct_ids(request.team_id, pid)
if limit is not None:
dids = _order_identified_first(dids)[:limit]
results.append(person_pb2.PersonDistinctIds(person_id=pid, distinct_ids=dids))