:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(persons): send person deletes to the lifecycle saga behind a flag (#126004)
Nick Best889ec9efe8b7
diff --git a/posthog/api/person.py b/posthog/api/person.pyindex 3fbe1f7b4037..927e68d27167 100644--- a/posthog/api/person.py+++ b/posthog/api/person.py@@ -59,6 +59,7 @@ from posthog.models.activity_logging.activity_log import Change, Detail, load_acfrom posthog.models.activity_logging.activity_page import activity_page_response, parse_activity_page_paramsfrom posthog.models.async_deletion import AsyncDeletion, DeletionTypefrom posthog.models.person.bulk_delete import (+ DeletionRun,delete_persons_profile,queue_person_event_deletion,queue_person_recording_deletion,@@ -1173,6 +1174,7 @@ class PersonViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):persons_deleted = 0errors: builtins.list[dict[str, str]] = []if not keep_person:+ # This path holds a web worker, so each saga call gets the request-path deadline.result = delete_persons_profile(self.team_id,persons,@@ -1180,6 +1182,7 @@ class PersonViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):request=request,organization_id=self.organization.id,queue_ai_training_deletion=False,+ run=DeletionRun.fresh(rpc_timeout=settings.PERSON_DELETE_LIFECYCLE_REQUEST_TIMEOUT_MS / 1000),)persons_deleted = result.deleted_counterrors = [diff --git a/posthog/api/test/test_person.py b/posthog/api/test/test_person.pyindex 59b62fce97b5..d298a10ea07d 100644--- a/posthog/api/test/test_person.py+++ b/posthog/api/test/test_person.py@@ -45,7 +45,7 @@ from posthog.models.person.util import ()from posthog.models.team.extensions import get_or_create_team_extensionfrom posthog.models.team.team_revenue_analytics_config import TeamRevenueAnalyticsConfig-from posthog.personhog_client.fake_client import fake_personhog_client+from posthog.personhog_client.fake_client import fake_personhog_client, get_active_fakefrom posthog.test.persons import add_distinct_id, create_person, delete_personfrom products.access_control.backend.models.property_access_control import PropertyAccessControl@@ -759,6 +759,21 @@ class TestPerson(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):self.assertEqual(data["code"], "person_deletion_failed")self.assertIn("tombstone_postgres", data["detail"])+ @parameterized.expand([("deleted", False, status.HTTP_202_ACCEPTED), ("held_by_another_op", True, 503)])+ def test_delete_person_through_the_lifecycle_saga_with_the_request_deadline(self, _name, held, expected):+ person = _create_person(team=self.team, distinct_ids=["person_1"], immediate=True)+ fake = get_active_fake()+ if held:+ fake.hold_lifecycle_mark(team_id=self.team.pk, person_id=person.pk)++ with self.settings(PERSONHOG_WRITE_PATH="leader", PERSON_DELETE_LIFECYCLE_REQUEST_TIMEOUT_MS=4_000):+ response = self.client.delete(f"/api/person/{person.uuid}/")++ self.assertEqual(response.status_code, expected)+ [call] = [call for call in fake.calls if call.method == "lifecycle.delete_persons"]+ self.assertEqual((list(call.request.person_ids), call.timeout), ([person.pk], 4.0))+ self.assertEqual(get_person_by_uuid(self.team.pk, str(person.uuid)) is None, not held)+@parameterized.expand([("publish_failed", PersonDeletionStep.PUBLISH_CLICKHOUSE_TOMBSTONE, 1, status.HTTP_202_ACCEPTED),diff --git a/posthog/dags/data_deletion_requests.py b/posthog/dags/data_deletion_requests.pyindex 83ae1f462609..69dcf0bc7b03 100644--- a/posthog/dags/data_deletion_requests.py+++ b/posthog/dags/data_deletion_requests.py@@ -77,6 +77,7 @@ from posthog.models.deletion_targets import ()from posthog.models.event.sql import json_property_presence_exprfrom posthog.models.person.bulk_delete import (+ DeletionRun,PersonDeletionStep,delete_persons_profile,queue_person_recording_deletion,@@ -2152,7 +2153,7 @@ def delete_person_profiles_op(context.log.info("No persons resolved; nothing to delete")return person_removal- result = delete_persons_profile(person_removal.team_id, persons, actor=None)+ result = delete_persons_profile(person_removal.team_id, persons, actor=None, run=DeletionRun(run_id=context.run_id))metadata: dict[str, dagster.MetadataValue] = {"deleted_count": dagster.MetadataValue.int(result.deleted_count),"errors": dagster.MetadataValue.int(len(result.errors)),diff --git a/posthog/dags/tests/test_data_deletion_requests.py b/posthog/dags/tests/test_data_deletion_requests.pyindex c4999f50338a..68d4ca7e373a 100644--- a/posthog/dags/tests/test_data_deletion_requests.py+++ b/posthog/dags/tests/test_data_deletion_requests.py@@ -92,7 +92,12 @@ from posthog.models.flag_evaluations.sql import (FLAG_EVALUATIONS_SOURCE_EVENT,FLAG_EVALUATIONS_TTL_DAYS,)-from posthog.models.person.bulk_delete import PersonDeletionFailure, PersonDeletionStep, PersonProfileDeletionResult+from posthog.models.person.bulk_delete import (+ DeletionRun,+ PersonDeletionFailure,+ PersonDeletionStep,+ PersonProfileDeletionResult,+)from posthog.test.persons import create_personTEAM_ID = 99999@@ -3285,10 +3290,13 @@ def test_delete_person_profiles_op_calls_helper_when_enabled():drop_events=False,drop_recordings=False,)+ op_context = build_op_context()with patch("posthog.dags.data_deletion_requests.delete_persons_profile") as deleter:deleter.return_value = PersonProfileDeletionResult(deleted_count=1)- delete_person_profiles_op(build_op_context(), ctx)+ delete_person_profiles_op(op_context, ctx)deleter.assert_called_once()+ # The Dagster run id keeps a retried run on the saga op it started.+ assert deleter.call_args.kwargs["run"] == DeletionRun(run_id=op_context.run_id)@pytest.mark.django_dbdiff --git a/posthog/models/person/bulk_delete.py b/posthog/models/person/bulk_delete.pyindex 7d594fc7c83a..9e291c21163a 100644--- a/posthog/models/person/bulk_delete.py+++ b/posthog/models/person/bulk_delete.py@@ -27,7 +27,9 @@ from posthog.models.activity_logging.activity_log import (from posthog.models.async_deletion import AsyncDeletion, DeletionTypefrom posthog.models.person import Personfrom posthog.models.person.util import (+ LIFECYCLE_DELETE_PERSONS_PER_RPC,DistinctIdForPerson,+ PersonDeletionConflict,PersonTombstone,PersonTombstonePublication,_fetch_persons_by_distinct_ids_via_personhog,@@ -35,6 +37,7 @@ from posthog.models.person.util import (_paginated_get_distinct_ids_for_person,get_person_tombstones,tombstone_persons_in_postgres,+ tombstone_persons_via_lifecycle,)from posthog.models.user import Userfrom posthog.temporal.common.client import sync_connect@@ -175,6 +178,25 @@ def resolve_persons_for_deletion(return personhog_call("resolve_persons_for_deletion", _fetch, caller_tag="persons/deletion-resolve")+@frozen+class DeletionRun:+ """One deletion request as the lifecycle saga sees it, shared by every retry of that request.++ ``run_id`` goes into each saga op_id, so a retry of the same chunk attaches to the op it+ started instead of being refused or starting a second one. Callers with a retry loop pass+ the id that loop keeps (the Celery task id, the Temporal workflow run id, the Dagster run id);+ a caller without one takes a fresh id. ``rpc_timeout`` bounds each saga call for the request+ path; None keeps the lifecycle channel's default deadline.+ """++ run_id: str+ rpc_timeout: float | None = None++ @classmethod+ def fresh(cls, *, rpc_timeout: float | None = None) -> "DeletionRun":+ return cls(run_id=str(uuid_lib.uuid4()), rpc_timeout=rpc_timeout)++def delete_persons_profile(team_id: int,persons: builtins.list[Person],@@ -183,6 +205,7 @@ def delete_persons_profile(request=None,organization_id=None,queue_ai_training_deletion: bool = True,+ run: DeletionRun | None = None,) -> PersonProfileDeletionResult:"""Tombstone the persons in Postgres and publish their ClickHouse tombstones.@@ -200,6 +223,7 @@ def delete_persons_profile(actor=actor,was_impersonated=is_impersonated(request),organization_id=organization_id,+ run=run,)_observe_person_outcomes("sync", result)return result@@ -209,7 +233,9 @@ class PersonTombstoneFailed(Exception):"""The Postgres tombstone failed for some persons, so they are still live and a retry must resolve them again."""-def tombstone_and_publish_persons(team_id: int, persons: builtins.list[Person]) -> int:+def tombstone_and_publish_persons(+ team_id: int, persons: builtins.list[Person], *, run: DeletionRun | None = None+) -> int:"""Raise PersonTombstoneFailed when a Postgres tombstone fails, because that person is still live.A failed ClickHouse publish does not raise: the ClickHouse deletion sweep republishes from the tombstone queue.@@ -219,6 +245,7 @@ def tombstone_and_publish_persons(team_id: int, persons: builtins.list[Person])persons,actor=None,queue_ai_training_deletion=False,+ run=run,)if result.retryable_errors:first = next(f for f in result.failures if f.step not in STEPS_AFTER_DELETION)@@ -254,17 +281,20 @@ DELETION_LOG_DEDUPE_WINDOW = timedelta(hours=24)@frozen(slots=False) # The activity log encoder writes a context out through __dict__.class PersonDeletionActivityContext(ActivityContextBase):# The version of the Postgres tombstone. A retry reports the same version, and a later deletion of the revived- # person a higher one, so the version tells a retried deletion from a new one.- person_version: int+ # person a higher one, so the version tells a retried deletion from a new one. None when the lifecycle saga+ # answered from an op recorded without one.+ person_version: int | None@frozenclass _DeletedPerson:person: Person- tombstone_version: int+ tombstone_version: int | None-def tombstone_and_publish_persons_by_uuids(team_id: int, person_uuids: builtins.list[str]) -> int:+def tombstone_and_publish_persons_by_uuids(+ team_id: int, person_uuids: builtins.list[str], *, run: DeletionRun | None = None+) -> int:"""Like tombstone_and_publish_persons, but read distinct IDs in batches, page heavy persons, and bound each RPC."""from posthog.personhog_client.client import personhog_call@@ -293,15 +323,15 @@ def tombstone_and_publish_persons_by_uuids(team_id: int, person_uuids: builtins.batch.append(person)batch_distinct_id_count += len(person._distinct_ids or [])if batch_distinct_id_count >= QUEUED_DELETION_DISTINCT_IDS_PER_BATCH:- tombstoned += _tombstone_batch_and_release(team_id, batch)+ tombstoned += _tombstone_batch_and_release(team_id, batch, run)batch, batch_distinct_id_count = [], 0if batch:- tombstoned += _tombstone_batch_and_release(team_id, batch)+ tombstoned += _tombstone_batch_and_release(team_id, batch, run)return tombstoned-def _tombstone_batch_and_release(team_id: int, persons: builtins.list[Person]) -> int:- tombstoned = tombstone_and_publish_persons(team_id, persons)+def _tombstone_batch_and_release(team_id: int, persons: builtins.list[Person], run: DeletionRun | None) -> int:+ tombstoned = tombstone_and_publish_persons(team_id, persons, run=run)# The resolved persons outlive their batch, so drop their ID strings to hold one batch of them at a time.for person in persons:person._distinct_ids = None@@ -315,6 +345,7 @@ class _QueuedDeletionOptions:actor: User | Nonewas_impersonated: boolorganization_id: uuid_lib.UUID | None+ run: DeletionRundef process_queued_person_deletion(@@ -327,6 +358,7 @@ def process_queued_person_deletion(was_impersonated: bool,organization_id: uuid_lib.UUID | None,unmatched_distinct_ids: builtins.list[str] | None = None,+ run_id: str | None = None,) -> PersonProfileDeletionResult:"""Run every distinct-ID-dependent deletion step for ``person_uuids`` from a background task.@@ -342,6 +374,8 @@ def process_queued_person_deletion(Every failure is recorded against the step it happened in. The profile delete only runswhen the steps before it succeeded, because it removes the distinct IDs a retry of thosesteps would need.++ ``run_id`` is the task id, which Celery keeps across retries and redeliveries of the chunk."""from posthog.personhog_client.client import personhog_call@@ -351,6 +385,7 @@ def process_queued_person_deletion(actor=actor,was_impersonated=was_impersonated,organization_id=organization_id,+ run=DeletionRun(run_id=run_id) if run_id else DeletionRun.fresh(),)failures: builtins.list[PersonDeletionFailure] = []requested = [uuid_lib.UUID(u) for u in person_uuids]@@ -523,6 +558,7 @@ def _run_queued_deletion_steps(actor=options.actor,was_impersonated=options.was_impersonated,organization_id=options.organization_id,+ run=options.run,)failures.extend(result.failures)return result.deleted_count@@ -535,6 +571,7 @@ def _tombstone_and_delete_persons(actor: User | None,was_impersonated: bool,organization_id: uuid_lib.UUID | None,+ run: DeletionRun | None,) -> PersonProfileDeletionResult:"""Tombstone each person in Postgres, publish the ClickHouse tombstones, then log the deletions.@@ -546,7 +583,7 @@ def _tombstone_and_delete_persons(raising would hide a completed deletion behind an error."""failures: builtins.list[PersonDeletionFailure] = []- deleted = _tombstone_persons_at_exact_versions(team_id, persons, failures)+ deleted = _tombstone_persons_at_exact_versions(team_id, persons, failures, run or DeletionRun.fresh())if organization_id is not None and deleted:try:@@ -601,7 +638,7 @@ def _logged_deletion_versions(team_id: int, person_ids: builtins.list[int]) -> dreturn versions-def _deletion_already_logged(logged_versions: set[int | None], tombstone_version: int) -> bool:+def _deletion_already_logged(logged_versions: set[int | None], tombstone_version: int | None) -> bool:return tombstone_version in logged_versions or None in logged_versions@@ -609,18 +646,28 @@ def _tombstone_persons_at_exact_versions(team_id: int,persons: builtins.list[Person],failures: builtins.list[PersonDeletionFailure],+ run: DeletionRun,) -> builtins.list[_DeletedPerson]:"""Tombstone Postgres first, then publish ClickHouse tombstones at the versions it wrote.Postgres goes first because only the replica knows the versions. A person whose Postgrestombstone landed is deleted, so it is counted and logged even when the ClickHouse publishfails; that failure gets its own step and the weekly deletion sweep republishes it.++ On the leader write path the identity server's fenced saga does the whole job: it tells+ the person's leader before the row changes, and its publish outbox and the person changelog+ carry the ClickHouse rows, so nothing is published from here. The replica path must not+ serve a delete once writes go through the leader: the replica never tells the leader, so+ the leader's cached person outlives the tombstone and its next write revives the person."""+ if settings.PERSONHOG_WRITE_PATH == "leader":+ return _tombstone_persons_via_lifecycle(team_id, persons, failures, run)+deleted: builtins.list[_DeletedPerson] = []person_by_uuid = {person.uuid: person for person in persons}publication = PersonTombstonePublication(team_id=team_id)- for batch in _batches_by_distinct_id_count(persons):+ for batch in _batches_by_distinct_id_count(persons, max_persons=None):uuids = [person.uuid for person in batch]try:tombstones = tombstone_persons_in_postgres(team_id, uuids)@@ -649,6 +696,74 @@ def _tombstone_persons_at_exact_versions(return deleted+def _tombstone_persons_via_lifecycle(+ team_id: int,+ persons: builtins.list[Person],+ failures: builtins.list[PersonDeletionFailure],+ run: DeletionRun,+) -> builtins.list[_DeletedPerson]:+ """Run the saga per batch; a person another live op holds is a retryable failure for the next attempt.++ A call that raises after the saga tombstoned the rows still counts them: the replica shows the+ tombstone, and the identity sweeper re-drives the op to completion, which is what writes the+ outbox pointer. Nothing here publishes to ClickHouse.++ Every person here was live when the caller resolved it, so a NOT_FOUND means it was tombstoned+ between that read and the saga's claim: by an earlier attempt's op that finished after its call+ failed, or by another deleter. Its tombstone is read back so the delete is counted and logged.+ """+ person_by_uuid = {person.uuid: person for person in persons}+ deleted: builtins.list[_DeletedPerson] = []+ for batch in _batches_by_distinct_id_count(persons, max_persons=LIFECYCLE_DELETE_PERSONS_PER_RPC):+ try:+ outcome = tombstone_persons_via_lifecycle(+ team_id, {person.pk: person.uuid for person in batch}, run_id=run.run_id, timeout=run.rpc_timeout+ )+ except Exception as exc:+ committed = _committed_tombstones(team_id, [person.uuid for person in batch], exc, failures)+ deleted.extend(+ _DeletedPerson(person=person_by_uuid[tombstone.uuid], tombstone_version=tombstone.version)+ for tombstone in committed+ )+ continue+ if outcome.conflicts:+ _record_step_failure(+ failures,+ step=PersonDeletionStep.TOMBSTONE_POSTGRES,+ team_id=team_id,+ exc=PersonDeletionConflict("another lifecycle operation holds the person; it stays live for a retry"),+ person_uuids=outcome.conflicts,+ )+ deleted.extend(+ _DeletedPerson(person=person_by_uuid[uuid], tombstone_version=version)+ for uuid, version in outcome.deleted.items()+ )+ deleted.extend(+ _DeletedPerson(person=person_by_uuid[tombstone.uuid], tombstone_version=tombstone.version)+ for tombstone in _tombstones_behind_not_found(team_id, outcome.not_found)+ )+ return deleted+++def _tombstones_behind_not_found(+ team_id: int, person_uuids: builtins.list[uuid_lib.UUID]+) -> builtins.list[PersonTombstone]:+ """The tombstones of persons the saga did not find. A failed read drops the count rather than the run:+ the persons are deleted either way, and a failure here would retry into the same NOT_FOUND."""+ if not person_uuids:+ return []+ try:+ return get_person_tombstones(team_id, person_uuids)+ except Exception:+ logger.warning(+ "person_deletion.not_found_tombstone_check_failed",+ team_id=team_id,+ person_count=len(person_uuids),+ exc_info=True,+ )+ return []++class PersonTombstoneStateUnknown(Exception):"""The tombstone RPC failed, and so did the check for what it committed before failing."""@@ -688,18 +803,20 @@ def _committed_tombstones(return stored-def _batches_by_distinct_id_count(persons: builtins.list[Person]) -> Iterator[builtins.list[Person]]:+def _batches_by_distinct_id_count(+ persons: builtins.list[Person], *, max_persons: int | None = None+) -> Iterator[builtins.list[Person]]:"""Group persons so one RPC carries at most TOMBSTONE_DISTINCT_IDS_PER_RPC distinct IDs.Every caller resolves persons with their distinct IDs loaded, and Person.distinct_ids raiseswhen they are not, so a person never counts as narrower than it is. A single person wider than- the budget goes alone.+ the budget goes alone. ``max_persons`` also caps the persons in a batch, for the saga's id limit."""batch: builtins.list[Person] = []count = 0for person in persons:width = len(person.distinct_ids)- if batch and count + width > TOMBSTONE_DISTINCT_IDS_PER_RPC:+ if batch and (count + width > TOMBSTONE_DISTINCT_IDS_PER_RPC or len(batch) == max_persons):yield batchbatch, count = [], 0batch.append(person)diff --git a/posthog/models/person/test/test_bulk_delete_helpers.py b/posthog/models/person/test/test_bulk_delete_helpers.pyindex d6b64eae09cf..12a30ef51b1b 100644--- a/posthog/models/person/test/test_bulk_delete_helpers.py+++ b/posthog/models/person/test/test_bulk_delete_helpers.py@@ -5,6 +5,7 @@ from uuid import uuid4from posthog.test.base import BaseTestfrom unittest.mock import AsyncMock, MagicMock, patch+from django.test import override_settingsfrom django.utils import timezonefrom confluent_kafka import KafkaError@@ -16,7 +17,9 @@ from posthog.models.activity_logging.activity_log import ActivityLogfrom posthog.models.async_deletion import AsyncDeletion, DeletionTypefrom posthog.models.person import Personfrom posthog.models.person.bulk_delete import (+ DeletionRun,PersonDeletionStep,+ PersonTombstoneFailed,_start_recording_workflows,delete_persons_profile,process_queued_person_deletion,@@ -28,6 +31,7 @@ from posthog.models.person.bulk_delete import (from posthog.models.person.util import (TOMBSTONE_DELIVERY_TIMEOUT_SECONDS,PersonTombstone,+ lifecycle_delete_op_id,tombstone_persons_in_postgres,)from posthog.personhog_client.fake_client import get_active_fake@@ -286,6 +290,139 @@ class DeletePersonsProfileTests(BaseTest):assert get_active_fake().tombstone_queue == {}+@override_settings(PERSONHOG_WRITE_PATH="leader")+class LifecycleDeletePersonsProfileTests(BaseTest):+ def _saga_calls(self):+ return [call for call in get_active_fake().calls if call.method == "lifecycle.delete_persons"]++ def test_runs_the_saga_with_the_sorted_ids_and_leaves_the_clickhouse_rows_to_the_outbox(self):+ p1 = create_person(team=self.team, distinct_ids=["a"], properties={})+ p2 = create_person(team=self.team, distinct_ids=["b", "c"], properties={})+ deleted_before = _sample("posthog_person_deletion_lifecycle_outcomes_total", outcome="deleted")+ with _no_publish() as publish:+ result = delete_persons_profile(+ self.team.pk,+ [p2, p1],+ actor=self.user,+ organization_id=self.organization.id,+ run=DeletionRun(run_id="run-1", rpc_timeout=7.5),+ )++ assert result.deleted_count == 2+ assert result.failures == []+ [call] = self._saga_calls()+ assert list(call.request.person_ids) == sorted([p1.pk, p2.pk])+ assert call.request.op_id == str(lifecycle_delete_op_id("run-1", self.team.pk, [p2.pk, p1.pk]))+ assert call.timeout == 7.5+ publish.assert_not_called()+ assert {call.method for call in get_active_fake().calls} & {+ "delete_persons",+ "ack_person_tombstones",+ "get_person_tombstones",+ } == set()+ logged = ActivityLog.objects.filter(team_id=self.team.pk, scope="Person", activity="deleted")+ assert {(log.item_id, (log.detail or {})["context"]["person_version"]) for log in logged} == {+ (str(p1.pk), 1),+ (str(p2.pk), 1),+ }+ assert _sample("posthog_person_deletion_lifecycle_outcomes_total", outcome="deleted") == deleted_before + 2++ def test_a_retry_of_the_run_attaches_to_its_op_and_a_new_run_starts_another(self):+ p = create_person(team=self.team, distinct_ids=["a"], properties={})+ with _no_publish():+ counts = [+ delete_persons_profile(self.team.pk, [p], actor=self.user, run=DeletionRun(run_id=run_id)).deleted_count+ for run_id in ("run-1", "run-1", "run-2")+ ]++ # The retry is answered from the op's record; the new run starts an op that finds no live person.+ assert counts == [1, 1, 1]+ ops = [call.request.op_id for call in self._saga_calls()]+ assert ops[0] == ops[1] != ops[2]+ assert [call.method for call in get_active_fake().calls if call.method == "get_person_tombstones"] == [+ "get_person_tombstones"+ ]+ assert get_active_fake()._persons_by_uuid[(self.team.pk, str(p.uuid))].version == 1++ def test_a_person_held_by_another_op_stays_live_and_fails_the_tombstone_step_until_the_next_attempt(self):+ p = create_person(team=self.team, distinct_ids=["a"], properties={})+ get_active_fake().hold_lifecycle_mark(team_id=self.team.pk, person_id=p.pk)+ with _no_publish() as publish:+ result = delete_persons_profile(+ self.team.pk,+ [p],+ actor=self.user,+ organization_id=self.organization.id,+ run=DeletionRun(run_id="task-1:0"),+ )+ with self.assertRaises(PersonTombstoneFailed) as raised:+ tombstone_and_publish_persons(self.team.pk, [p])++ assert result.deleted_count == 0+ assert [(f.step, f.person_uuid) for f in result.failures] == [(PersonDeletionStep.TOMBSTONE_POSTGRES, p.uuid)]+ assert result.failures[0].error.startswith("PersonDeletionConflict")+ assert result.retryable_errors == [p.uuid]+ assert "PersonDeletionConflict" in str(raised.exception)+ publish.assert_not_called()+ assert not ActivityLog.objects.filter(team_id=self.team.pk, scope="Person", item_id=str(p.pk)).exists()+ assert get_active_fake()._persons_by_uuid[(self.team.pk, str(p.uuid))].is_deleted is False++ get_active_fake().release_lifecycle_mark(team_id=self.team.pk, person_id=p.pk)+ with _no_publish():+ replayed = delete_persons_profile(self.team.pk, [p], actor=self.user, run=DeletionRun(run_id="task-1:0"))+ retried = delete_persons_profile(self.team.pk, [p], actor=self.user, run=DeletionRun(run_id="task-1:1"))+ # The first op recorded the conflict, so only the next attempt's op id reaches the released person.+ assert (replayed.retryable_errors, retried.deleted_count) == ([p.uuid], 1)++ def test_a_deleted_person_without_a_version_is_still_counted_and_logged(self):+ p = create_person(team=self.team, distinct_ids=["a"], properties={})+ get_active_fake().omit_lifecycle_person_version(team_id=self.team.pk, person_id=p.pk)+ with _no_publish():+ result = delete_persons_profile(self.team.pk, [p], actor=self.user, organization_id=self.organization.id)++ assert (result.deleted_count, result.failures) == (1, [])+ [log] = ActivityLog.objects.filter(team_id=self.team.pk, scope="Person", item_id=str(p.pk))+ assert (log.detail or {})["context"]["person_version"] is None+ assert get_active_fake()._persons_by_uuid[(self.team.pk, str(p.uuid))].is_deleted is True++ def test_a_person_tombstoned_before_the_saga_claimed_it_is_counted_from_its_tombstone(self):+ p = create_person(team=self.team, distinct_ids=["a"], properties={})+ tombstone_persons_in_postgres(self.team.pk, [p.uuid])+ ghost = Person(id=10**9, uuid=uuid4(), team=self.team)+ ghost._distinct_ids = []+ with _no_publish() as publish:+ result = delete_persons_profile(+ self.team.pk, [p, ghost], actor=self.user, organization_id=self.organization.id+ )++ assert (result.deleted_count, result.failures) == (1, [])+ publish.assert_not_called()+ [lookup] = [call for call in get_active_fake().calls if call.method == "get_person_tombstones"]+ assert set(lookup.request.person_uuids) == {str(p.uuid), str(ghost.uuid)}+ [log] = ActivityLog.objects.filter(team_id=self.team.pk, scope="Person", activity="deleted")+ assert (log.item_id, (log.detail or {})["context"]["person_version"]) == (str(p.pk), 1)++ def test_a_failed_tombstone_lookup_for_an_unfound_person_drops_the_count_but_not_the_run(self):+ p = create_person(team=self.team, distinct_ids=["a"], properties={})+ tombstone_persons_in_postgres(self.team.pk, [p.uuid])+ with (+ _no_publish(),+ patch("posthog.models.person.bulk_delete.get_person_tombstones", side_effect=RuntimeError("replica down")),+ ):+ result = delete_persons_profile(self.team.pk, [p], actor=self.user, organization_id=self.organization.id)++ assert (result.deleted_count, result.failures) == (0, [])+ assert not ActivityLog.objects.filter(team_id=self.team.pk, scope="Person", item_id=str(p.pk)).exists()++ def test_splits_a_batch_at_the_saga_person_cap(self):+ persons = [create_person(team=self.team, distinct_ids=[f"d{i}"], properties={}) for i in range(3)]+ with patch("posthog.models.person.bulk_delete.LIFECYCLE_DELETE_PERSONS_PER_RPC", 2), _no_publish():+ result = delete_persons_profile(self.team.pk, persons, actor=self.user)++ assert result.deleted_count == 3+ assert [len(call.request.person_ids) for call in self._saga_calls()] == [2, 1]++class TombstoneAndPublishPersonsTests(BaseTest):def test_a_failed_clickhouse_publish_does_not_raise_and_stays_queued_for_the_sweep(self):p = create_person(team=self.team, distinct_ids=["a"], properties={})diff --git a/posthog/models/person/util.py b/posthog/models/person/util.pyindex 9e47ab183d09..2bc249f1e810 100644--- a/posthog/models/person/util.py+++ b/posthog/models/person/util.py@@ -4,13 +4,13 @@ import jsonimport timeimport datetimeimport contextvars-from collections.abc import Callable, Sequence+from collections.abc import Callable, Mapping, Sequencefrom concurrent.futures import ThreadPoolExecutorfrom dataclasses import dataclass, fieldfrom enum import StrEnumfrom functools import partialfrom typing import TYPE_CHECKING, Any, Optional, TypeVar, Union-from uuid import UUID+from uuid import UUID, uuid5from zoneinfo import ZoneInfofrom django.conf import settings@@ -40,8 +40,12 @@ from posthog.personhog_client.interceptor import is_transient_rpc_error, semantifrom posthog.personhog_client.metrics import PERSONHOG_TEAM_MISMATCH_TOTAL, get_client_namefrom posthog.personhog_client.proto import (CONSISTENCY_LEVEL_STRONG,+ DELETE_PERSON_OUTCOME_DELETED,+ DELETE_PERSON_OUTCOME_NOT_FOUND,+ DELETE_PERSON_OUTCOME_SKIPPED_CONFLICT,AckedPersonTombstone,AckPersonTombstonesRequest,+ DeletePersonOutcome,DeletePersonsMode,DeletePersonsRequest,DistinctIdRow,@@ -55,6 +59,8 @@ from posthog.personhog_client.proto import (GetPersonsByDistinctIdsInTeamRequest,GetPersonsByUuidsRequest,GetPersonTombstonesRequest,+ LifecycleDeletePersonResult,+ LifecycleDeletePersonsRequest,ListPersonsForTeamRequest,ListPersonTombstoneQueueRequest,PersonListEntry,@@ -759,6 +765,87 @@ def tombstone_persons_in_postgres(team_id: int, person_uuids: list[UUID]) -> lisreturn personhog_call("tombstone_persons", personhog_fn)+# The identity server's cap on person ids per lifecycle DeletePersons request.+LIFECYCLE_DELETE_PERSONS_PER_RPC = 250++# Namespace for the saga op_id. Changing it makes a retry of an in-flight deletion start a second op.+_LIFECYCLE_DELETE_OP_NAMESPACE = UUID("4b4d2d2e-5a3e-4c1f-9d4a-7c0b2f4e8a61")++PERSON_DELETION_LIFECYCLE_OUTCOMES_COUNTER = Counter(+ "posthog_person_deletion_lifecycle_outcomes_total",+ "Per-person outcomes the lifecycle delete saga reported to a Django deletion.",+ labelnames=["outcome"],+)+++class PersonDeletionConflict(Exception):+ """Another live lifecycle op holds the person, so the saga skipped it; it is still live and a retry can delete it."""+++@frozen+class LifecycleDeleteOutcome:+ """What one saga call left behind: the version each deleted person's tombstone committed at, and the persons it could not claim.++ A deleted person has no version when the op was recorded by a build that did not store one.+ ``not_found`` holds the persons that had no live row when the saga claimed them.+ """++ deleted: dict[UUID, int | None]+ conflicts: list[UUID]+ not_found: list[UUID]+++def lifecycle_delete_op_id(run_id: str, team_id: int, person_ids: Sequence[int]) -> UUID:+ """The saga op_id for one chunk of a deletion run.++ The engine attaches a repeated op_id to its op, and a completed op answers from its record without+ touching the persons. ``run_id`` keeps that attach to retries of the same request: the same ids+ deleted again later, after a revival, carry another run_id and so start a new op.+ """+ return uuid5(_LIFECYCLE_DELETE_OP_NAMESPACE, f"{run_id}|{team_id}|{','.join(str(i) for i in sorted(person_ids))}")+++def tombstone_persons_via_lifecycle(+ team_id: int, person_uuids_by_id: Mapping[int, UUID], *, run_id: str, timeout: float | None = None+) -> LifecycleDeleteOutcome:+ """Delete up to LIFECYCLE_DELETE_PERSONS_PER_RPC persons through the identity server's fenced saga.++ The saga answers with an outcome per id and, for a deleted person, the version its tombstone+ committed at. Nothing is produced to ClickHouse from here: the saga's publish outbox carries the+ distinct-id tombstones and the person changelog carries the person row. NOT_FOUND means no live+ row existed when the saga claimed the id; the caller decides whether a tombstone explains it.+ """+ person_ids = sorted(person_uuids_by_id)+ request = LifecycleDeletePersonsRequest(+ team_id=team_id, person_ids=person_ids, op_id=str(lifecycle_delete_op_id(run_id, team_id, person_ids))+ )++ def personhog_fn() -> dict[int, LifecycleDeletePersonResult]:+ response = _get_client().lifecycle.delete_persons(request, timeout=timeout)+ return {result.person_id: result for result in response.results}++ results = personhog_call("lifecycle_delete_persons", personhog_fn, caller_tag="persons/deletion-lifecycle")++ conflicts: list[UUID] = []+ not_found: list[UUID] = []+ deleted: dict[UUID, int | None] = {}+ for person_id in person_ids:+ uuid = person_uuids_by_id[person_id]+ result = results.get(person_id)+ # An id the response left out never had a live row to claim.+ outcome = result.outcome if result is not None else DELETE_PERSON_OUTCOME_NOT_FOUND+ PERSON_DELETION_LIFECYCLE_OUTCOMES_COUNTER.labels(+ outcome=DeletePersonOutcome.Name(outcome).removeprefix("DELETE_PERSON_OUTCOME_").lower()+ ).inc()+ if outcome == DELETE_PERSON_OUTCOME_SKIPPED_CONFLICT:+ conflicts.append(uuid)+ elif outcome == DELETE_PERSON_OUTCOME_DELETED and result is not None:+ deleted[uuid] = int(result.person_version) if result.HasField("person_version") else None+ elif outcome == DELETE_PERSON_OUTCOME_NOT_FOUND:+ not_found.append(uuid)+ return LifecycleDeleteOutcome(deleted=deleted, conflicts=conflicts, not_found=not_found)++@frozenclass QueuedPersonTombstone:team_id: intdiff --git a/posthog/personhog_client/README.md b/posthog/personhog_client/README.mdindex bdc3c7bfb562..3a22a66476d3 100644--- a/posthog/personhog_client/README.md+++ b/posthog/personhog_client/README.md@@ -83,6 +83,8 @@ The call waits for the saga to complete, so its deadline is `PERSONHOG_LIFECYCLEEvery request carries a caller-generated `op_id`. Retry a transient failure with the same `op_id` and the identical `person_ids` list; the retry attaches to the existing operation and returns its recorded outcome.A `FAILED_PRECONDITION` with a reason from `semantic_refusal_reason()` (in `interceptor.py`) is final: never retry it under the same `op_id`. The identity server's own reason is `op_id_reused` (the `op_id` belongs to a different request; reissue under a fresh one). The leader's reasons (`release-unverified`, `uuid-mismatch`, `no-lifecycle-db`) pass through the saga unchanged and mean the operation cannot finish as issued; surface them rather than looping.The request and response types are re-exported from `posthog.personhog_client.proto` as `LifecycleDeletePersonsRequest` and `LifecycleDeletePersonsResponse`, because the facade's delete messages share their names.+Django's person deletes reach it through `tombstone_persons_via_lifecycle` in `posthog/models/person/util.py` when `PERSONHOG_WRITE_PATH` is `leader`; on `replica` they stay on the facade's TOMBSTONE delete. The leader path must be set before ingestion moves to the leader, because a facade tombstone never reaches the leader's cache and the next leader write revives the person. The op_id is `uuid5(run id, team, sorted ids)`, where the run id is the caller's run plus its attempt number (the Celery task id and retry count, the Temporal workflow run id and activity attempt, the Dagster run id), so a redelivery inside one attempt attaches to its op and each scheduled retry starts a new one; a new op is needed because the server records an all-conflict op as completed and replays it.+Django produces nothing to ClickHouse on this path: the saga's publish outbox carries the distinct-id tombstones and the person changelog carries the person row.**Version floors:**`ensure_person_version_floors` (raise each tombstone to a minimum version, inserting a tombstone where no row exists, and leave live rows unchanged; a lost insert race fails the whole request with `FAILED_PRECONDITION`).diff --git a/posthog/personhog_client/fake_client.py b/posthog/personhog_client/fake_client.pyindex 70bf43ddb0eb..6cb4a76ef271 100644--- a/posthog/personhog_client/fake_client.py+++ b/posthog/personhog_client/fake_client.py@@ -21,7 +21,6 @@ import jsonimport timefrom collections.abc import Sequencefrom contextlib import contextmanager-from dataclasses import dataclassfrom typing import Anyfrom uuid import UUID, uuid4@@ -31,16 +30,19 @@ import grpcfrom personhog.lifecycle.v1 import lifecycle_pb2from personhog.types.v1 import cohort_pb2, feature_flag_pb2, group_pb2, person_pb2+from posthog.dataclasses import frozenfrom posthog.models.person.missing_person import uuidFromDistinctIdfrom posthog.personhog_client.interceptor import SEMANTIC_REFUSAL_METADATA_KEYfrom posthog.utils import is_anonymous_id-@dataclass+@frozen(kw_only=False)class _Call:method: strrequest: Anyresponse: Any = None+ # The per-call deadline the caller passed, for the lifecycle sagas; None means the channel default.+ timeout: float | None = Noneclass _FakeRpcError(grpc.RpcError):@@ -101,7 +103,7 @@ class _FakeLifecycleClient:def delete_persons(self, request: lifecycle_pb2.DeletePersonsRequest, timeout: float | None = None) -> lifecycle_pb2.DeletePersonsResponse:- return self._fake._lifecycle_delete_persons(request)+ return self._fake._lifecycle_delete_persons(request, timeout)class FakePersonHogClient:@@ -135,6 +137,7 @@ class FakePersonHogClient:self._lifecycle_ops: dict[str, tuple[int, tuple[int, ...], lifecycle_pb2.DeletePersonsResponse]] = {}# (team_id, person_id) held by another live lifecycle opself._lifecycle_marks: set[tuple[int, int]] = set()+ self._lifecycle_versionless: set[tuple[int, int]] = set()# keyed by project_id -> list of GroupTypeMappingself._group_type_mappings_by_project: dict[int, list[group_pb2.GroupTypeMapping]] = {}@@ -259,6 +262,13 @@ class FakePersonHogClient:"""Make a person look held by another live lifecycle op, so a delete reports SKIPPED_CONFLICT."""self._lifecycle_marks.add((team_id, person_id))+ def release_lifecycle_mark(self, *, team_id: int, person_id: int) -> None:+ self._lifecycle_marks.discard((team_id, person_id))++ def omit_lifecycle_person_version(self, *, team_id: int, person_id: int) -> None:+ """Answer a delete of this person like an op recorded by a build that stored no tombstone version."""+ self._lifecycle_versionless.add((team_id, person_id))+def add_cohort_membership(self, *, person_id: int, cohort_id: int, is_member: bool = True) -> None:self._cohort_memberships.setdefault(person_id, []).append(cohort_pb2.CohortMembership(cohort_id=cohort_id, is_member=is_member)@@ -757,9 +767,9 @@ class FakePersonHogClient:# ── Lifecycle sagas ───────────────────────────────────────────────def _lifecycle_delete_persons(- self, request: lifecycle_pb2.DeletePersonsRequest+ self, request: lifecycle_pb2.DeletePersonsRequest, timeout: float | None = None) -> lifecycle_pb2.DeletePersonsResponse:- self.calls.append(_Call("lifecycle.delete_persons", request))+ self.calls.append(_Call("lifecycle.delete_persons", request, timeout=timeout))self._validate_lifecycle_delete(request)frozen = (request.team_id, tuple(request.person_ids))@@ -778,6 +788,7 @@ class FakePersonHogClient:# One decision per distinct person, so a duplicated id reports the same outcome twice.outcomes: dict[int, lifecycle_pb2.DeletePersonOutcome] = {}+ versions: dict[int, int] = {}for person_id in request.person_ids:if person_id in outcomes:continue@@ -788,7 +799,7 @@ class FakePersonHogClient:outcomes[person_id] = lifecycle_pb2.DELETE_PERSON_OUTCOME_SKIPPED_CONFLICTelse:# No publish-queue row: the death document and the outbox carry the ClickHouse tombstones.- self._tombstone_person(request.team_id, person, person_pb2.TombstonedPerson())+ versions[person_id] = self._tombstone_person(request.team_id, person, person_pb2.TombstonedPerson())self._cohort_memberships.pop(person.id, None)for key in [key for key in self._cohort_members if key[1] == person.id]:del self._cohort_members[key]@@ -796,7 +807,9 @@ class FakePersonHogClient:response = lifecycle_pb2.DeletePersonsResponse(op_id=request.op_id)for person_id in request.person_ids:- response.results.add(person_id=person_id, outcome=outcomes[person_id])+ result = response.results.add(person_id=person_id, outcome=outcomes[person_id])+ if person_id in versions and (request.team_id, person_id) not in self._lifecycle_versionless:+ result.person_version = versions[person_id]self._lifecycle_ops[request.op_id] = (frozen[0], frozen[1], response)copy = lifecycle_pb2.DeletePersonsResponse()copy.CopyFrom(response)diff --git a/posthog/settings/personhog.py b/posthog/settings/personhog.pyindex be48b049642e..7429d49fe5e1 100644--- a/posthog/settings/personhog.py+++ b/posthog/settings/personhog.py@@ -9,6 +9,14 @@ PERSONHOG_LIFECYCLE_ADDR = get_from_env("PERSONHOG_LIFECYCLE_ADDR", "")# UNAVAILABLE itself; a shorter client deadline would abandon calls the server was about to answer.PERSONHOG_LIFECYCLE_TIMEOUT_MS = get_from_env("PERSONHOG_LIFECYCLE_TIMEOUT_MS", 45_000, type_cast=int)+# Deadline per saga call on the request path (the person API), which holds a web worker while the saga+# runs and retries. Background callers keep PERSONHOG_LIFECYCLE_TIMEOUT_MS. Keep it above the identity+# server's lifecycle_execute_timeout_secs (30 s): a shorter deadline makes the API report a failure for a+# slow saga that still commits the delete, so the person is gone with no response or activity log for it.+PERSON_DELETE_LIFECYCLE_REQUEST_TIMEOUT_MS = get_from_env(+ "PERSON_DELETE_LIFECYCLE_REQUEST_TIMEOUT_MS", 35_000, type_cast=int+)+# gRPC channel optionsPERSONHOG_KEEPALIVE_TIME_MS = get_from_env("PERSONHOG_KEEPALIVE_TIME_MS", 30_000, type_cast=int)PERSONHOG_KEEPALIVE_TIMEOUT_MS = get_from_env("PERSONHOG_KEEPALIVE_TIMEOUT_MS", 5_000, type_cast=int)diff --git a/posthog/tasks/delete_persons.py b/posthog/tasks/delete_persons.pyindex 3fdac976f837..5e546d8dabe7 100644--- a/posthog/tasks/delete_persons.py+++ b/posthog/tasks/delete_persons.py@@ -157,6 +157,8 @@ def delete_persons_async(was_impersonated=was_impersonated,organization_id=uuid_lib.UUID(organization_id) if organization_id else None,unmatched_distinct_ids=unmatched_distinct_ids,+ # A retry under the first attempt's op id replays its recorded conflicts instead of trying again.+ run_id=f"{self.request.id}:{retries}",)except Exception as exc:# Nothing per person was recorded, so the same arguments go round again.diff --git a/posthog/tasks/test/test_delete_persons.py b/posthog/tasks/test/test_delete_persons.pyindex 92a818e5a5a9..3aee1c51a773 100644--- a/posthog/tasks/test/test_delete_persons.py+++ b/posthog/tasks/test/test_delete_persons.py@@ -100,6 +100,24 @@ class TestDeletePersonsAsync(SimpleTestCase):unmatched_distinct_ids=["ghost"],)+ def test_passes_the_task_id_and_attempt_as_the_deletion_run(self) -> None:+ delete_persons_async.push_request(id="task-1", retries=2)+ self.addCleanup(delete_persons_async.pop_request)+ with patch(+ "posthog.tasks.delete_persons.process_queued_person_deletion",+ return_value=PersonProfileDeletionResult(deleted_count=2),+ ) as process:+ delete_persons_async.run(+ team_id=1,+ person_uuids=["a"],+ delete_profile=True,+ delete_recordings=False,+ actor_id=None,+ organization_id=None,+ was_impersonated=False,+ )+ assert process.call_args.kwargs["run_id"] == "task-1:2"+def test_retries_only_the_failed_persons(self) -> None:failed = uuid4()result = PersonProfileDeletionResult(diff --git a/posthog/temporal/delete_persons/delete_persons_workflow.py b/posthog/temporal/delete_persons/delete_persons_workflow.pyindex 1ec7158ebf02..0367ddad868d 100644--- a/posthog/temporal/delete_persons/delete_persons_workflow.py+++ b/posthog/temporal/delete_persons/delete_persons_workflow.py@@ -26,12 +26,13 @@ def _chunked(items: list, size: int) -> typing.Iterator[list]:yield chunk-def _delete_specific_persons_via_personhog(team_id: int, person_ids: list[int]) -> int:+def _delete_specific_persons_via_personhog(team_id: int, person_ids: list[int], run_id: str | None = None) -> int:"""Tombstone specific persons by id via personhog and publish their ClickHouse tombstones.A hard delete would leave live ClickHouse rows above the version 0 a re-created person starts at.+ ``run_id`` is the workflow run, which every retry of the activity shares."""- from posthog.models.person.bulk_delete import tombstone_and_publish_persons_by_uuids+ from posthog.models.person.bulk_delete import DeletionRun, tombstone_and_publish_persons_by_uuidsfrom posthog.personhog_client.caller_tag import personhog_caller_tagfrom posthog.personhog_client.client import get_personhog_clientfrom posthog.personhog_client.proto import GetPersonsRequest@@ -48,7 +49,9 @@ def _delete_specific_persons_via_personhog(team_id: int, person_ids: list[int])if not uuids:return 0- return tombstone_and_publish_persons_by_uuids(team_id, uuids)+ return tombstone_and_publish_persons_by_uuids(+ team_id, uuids, run=DeletionRun(run_id=run_id) if run_id else None+ )def _require_team_deleted(team_id: int) -> None:@@ -155,7 +158,14 @@ async def delete_persons_activity(inputs: DeletePersonsActivityInputs) -> tuple[id_slice = inputs.person_ids[start : start + inputs.batch_size]if not id_slice:return 0, False- deleted = await asyncio.to_thread(_delete_specific_persons_via_personhog, inputs.team_id, id_slice)+ info = temporalio.activity.info()+ deleted = await asyncio.to_thread(+ _delete_specific_persons_via_personhog,+ inputs.team_id,+ id_slice,+ # A retry under the first attempt's op id replays its recorded conflicts instead of trying again.+ f"{info.workflow_run_id}:{info.attempt}",+ )should_continue = start + inputs.batch_size < len(inputs.person_ids)else:# Whole team: delete up to batch_size and keep going until a batch is short.diff --git a/posthog/temporal/tests/delete_persons/test_delete_persons_workflow.py b/posthog/temporal/tests/delete_persons/test_delete_persons_workflow.pyindex 43daf042e328..d09b8f2a5798 100644--- a/posthog/temporal/tests/delete_persons/test_delete_persons_workflow.py+++ b/posthog/temporal/tests/delete_persons/test_delete_persons_workflow.py@@ -3,12 +3,15 @@ import uuid as uuid_libimport pytestfrom unittest.mock import patch+from django.test import override_settings+from asgiref.sync import sync_to_asyncfrom temporalio.exceptions import ApplicationErrorfrom posthog.models.async_deletion import AsyncDeletion, DeletionTypefrom posthog.models.person import Person, bulk_deletefrom posthog.models.person.bulk_delete import PersonTombstoneFailed+from posthog.models.person.util import lifecycle_delete_op_idfrom posthog.personhog_client.fake_client import fake_personhog_clientfrom posthog.personhog_client.proto import DeletePersonsModefrom posthog.temporal.delete_persons.delete_persons_workflow import (@@ -50,12 +53,29 @@ class TestDeletePersonsActivity:assert [(d.id, d.version) for d in tombstone.distinct_ids] == [("d1", 1)]assert fake.tombstone_queue == {}+ async def test_by_ids_runs_the_saga_under_the_workflow_run_and_attempt(self, activity_environment):+ person_uuid = str(uuid_lib.uuid4())+ with fake_personhog_client() as fake, _no_publish(), override_settings(PERSONHOG_WRITE_PATH="leader"):+ fake.add_person(team_id=1, person_id=10, uuid=person_uuid, version=3, distinct_ids=["d1"])++ deleted, _ = await activity_environment.run(+ delete_persons_activity,+ DeletePersonsActivityInputs(team_id=1, person_ids=[10], batch_size=1000),+ )++ assert deleted == 1+ [call] = [call for call in fake.calls if call.method == "lifecycle.delete_persons"]+ info = activity_environment.info+ run_id = f"{info.workflow_run_id}:{info.attempt}"+ assert call.request.op_id == str(lifecycle_delete_op_id(run_id, 1, [10]))+ assert fake._persons_by_uuid[(1, person_uuid)].is_deleted is True+async def test_by_ids_reads_distinct_ids_in_one_batch_and_pages_only_heavy_persons(self, activity_environment):counted: dict[int, int] = {}- def record_batch(team_id: int, persons: list[Person]) -> int:+ def record_batch(team_id: int, persons: list[Person], *, run: bulk_delete.DeletionRun | None = None) -> int:counted.update({person.pk: len(person.distinct_ids) for person in persons})- return real_tombstone(team_id, persons)+ return real_tombstone(team_id, persons, run=run)real_tombstone = bulk_delete.tombstone_and_publish_personswith (