:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(persons): raise divergence repairs through marked personhog writes (#118360)
Eli Reismana4d893a7b0ef
diff --git a/posthog/api/person.py b/posthog/api/person.pyindex ddc9b572c669..983f47096324 100644--- a/posthog/api/person.py+++ b/posthog/api/person.py@@ -15,6 +15,7 @@ from drf_spectacular.types import OpenApiTypesfrom drf_spectacular.utils import (OpenApiExample,OpenApiParameter,+ OpenApiResponse,extend_schema,extend_schema_field,extend_schema_serializer,@@ -63,7 +64,7 @@ from posthog.models.person.bulk_delete import (queue_person_recording_deletion,resolve_persons_for_deletion,)-from posthog.models.person.divergence.repairs import repair_distinct_id+from posthog.models.person.divergence.repairs import is_person_busy, is_retryable, repair_distinct_idfrom posthog.models.person.missing_person import MissingPersonfrom posthog.models.person.util import (get_distinct_ids_for_persons,@@ -1852,6 +1853,11 @@ class PersonViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):@extend_schema(description="Fix a distinct_id that stays hidden after its person was deleted and created again. Does nothing if no live person uses this distinct_id. In that case, send a new event for it instead.",+ responses={+ 202: OpenApiResponse(description="The fix was applied, or nothing needed fixing."),+ 409: OpenApiResponse(description="The person is being updated by another operation. Try again shortly."),+ 503: OpenApiResponse(description="Person data is temporarily unavailable. Try again shortly."),+ },)@action(methods=["POST"], detail=False, required_scopes=["person:write"])def reset_person_distinct_id(self, request: request.Request, *args: Any, **kwargs: Any) -> response.Response:@@ -1859,9 +1865,22 @@ class PersonViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):if not distinct_id or not isinstance(distinct_id, str):raise ValidationError(detail="distinct_id is required")- summary = repair_distinct_id(- self.team_id, distinct_id, delivery_timeout_seconds=RESET_DISTINCT_ID_DELIVERY_TIMEOUT_SECONDS- )+ try:+ summary = repair_distinct_id(+ self.team_id, distinct_id, delivery_timeout_seconds=RESET_DISTINCT_ID_DELIVERY_TIMEOUT_SECONDS+ )+ except Exception as error:+ if is_person_busy(error):+ return response.Response(+ {"detail": "This person is being updated right now. Try again in a few seconds."},+ status=409,+ )+ if is_retryable(error):+ return response.Response(+ {"detail": "Person data is temporarily unavailable. Try again in a minute."},+ status=503,+ )+ raiseif summary is not None:logger.info("person_reset_distinct_id",diff --git a/posthog/api/test/test_person.py b/posthog/api/test/test_person.pyindex fdc340bf2b06..ffe82ca2c92b 100644--- a/posthog/api/test/test_person.py+++ b/posthog/api/test/test_person.py@@ -23,6 +23,7 @@ from unittest import mockfrom django.test import SimpleTestCasefrom django.utils import timezone+import grpcfrom clickhouse_driver.errors import ServerExceptionfrom parameterized import parameterizedfrom rest_framework import status@@ -2418,6 +2419,36 @@ class TestPerson(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):assert person_after is not Noneassert person_after.version is not None and person_after.version > 105+ @parameterized.expand(+ [+ ("mark_held_by_another_operation", grpc.StatusCode.FAILED_PRECONDITION, (), 409),+ ("row_lock_timed_out", grpc.StatusCode.DEADLINE_EXCEEDED, (("x-db-timeout", "lock_timeout"),), 409),+ ("statement_timed_out", grpc.StatusCode.DEADLINE_EXCEEDED, (("x-db-timeout", "statement_timeout"),), 503),+ ("personhog_deadline", grpc.StatusCode.DEADLINE_EXCEEDED, (), 503),+ ("personhog_unavailable", grpc.StatusCode.UNAVAILABLE, (), 503),+ ]+ )+ def test_reset_person_distinct_id_asks_for_a_retry_on_a_busy_person_or_an_unavailable_service(+ self, _name: str, code: grpc.StatusCode, trailing: tuple[tuple[str, str], ...], status: int+ ) -> None:+ failure = grpc.RpcError()+ failure.code = mock.MagicMock(return_value=code)+ failure.trailing_metadata = mock.MagicMock(return_value=trailing)++ with mock.patch("posthog.api.person.repair_distinct_id", side_effect=failure):+ response = self.client.post(+ f"/api/projects/{self.team.pk}/persons/reset_person_distinct_id/", {"distinct_id": "busy"}+ )++ assert response.status_code == status+ assert (+ response.json()["detail"]+ == {+ 409: "This person is being updated right now. Try again in a few seconds.",+ 503: "Person data is temporarily unavailable. Try again in a minute.",+ }[status]+ )+@mock.patch(f"{posthog.models.person.divergence.repairs.__name__}.create_person_distinct_id",wraps=posthog.models.person.divergence.repairs.create_person_distinct_id,diff --git a/posthog/models/person/divergence/repairs.py b/posthog/models/person/divergence/repairs.pyindex 3db3b3e8215e..e0069543818b 100644--- a/posthog/models/person/divergence/repairs.py+++ b/posthog/models/person/divergence/repairs.py@@ -8,7 +8,6 @@ from collections.abc import Callable, Sequencefrom dataclasses import fieldfrom functools import partialfrom typing import TypeVar-from uuid import UUIDimport grpc@@ -30,11 +29,14 @@ from posthog.models.team import Teamfrom posthog.personhog_client.client import personhog_call, require_personhog_clientfrom posthog.personhog_client.interceptor import is_transient_rpc_errorfrom posthog.personhog_client.proto import (- CONSISTENCY_LEVEL_STRONG,- GetDistinctIdsForPersonsRequest,- ReadOptions,- SetPersonDistinctIdVersionFloorRequest,- SetPersonVersionFloorRequest,+ DISTINCT_ID_WRITE_ACTION_RAISE,+ DISTINCT_ID_WRITE_OUTCOME_PERSON_NOT_LIVE,+ DistinctIdWrite,+ DistinctIdWriteResult,+ PersonVersionFloor,+ RaisePersonVersionFloorsRequest,+ WriteDistinctIdsRequest,+ WriteDistinctIdsResponse,)from . import datatypes, lookups, repair_plan, stale_merge@@ -44,15 +46,27 @@ _T = TypeVar("_T")_REPAIR_CHUNK_SIZE = 100+# The most rows one personhog repair write takes.+_WRITE_BATCH_SIZE = 250+# The waits between re-reads give the replica about 0.5 s to show a raise, far above its usual lag of a few milliseconds._REREAD_BACKOFF_SECONDS = (0.025, 0.05, 0.1, 0.15, 0.175)_TRANSIENT_RETRY_BACKOFF_SECONDS = (1.0, 2.0)+# FAILED_PRECONDITION is transient for the repair writes: personhog returns it only for a person whose lifecycle+# mark another operation holds, or for a row that changed between its read and its lock, and both clear on a retry._TRANSIENT_RPC_CODES = frozenset(- {grpc.StatusCode.INTERNAL, grpc.StatusCode.UNAVAILABLE, grpc.StatusCode.DEADLINE_EXCEEDED}+ {+ grpc.StatusCode.INTERNAL,+ grpc.StatusCode.UNAVAILABLE,+ grpc.StatusCode.DEADLINE_EXCEEDED,+ grpc.StatusCode.FAILED_PRECONDITION,+ })+_DB_TIMEOUT_METADATA_KEY = "x-db-timeout"+_FLUSH_TIMEOUT_SECONDS = 5 * 60# Confirmed produce results are dropped this often, so a long repair does not hold one per published row.@@ -61,41 +75,50 @@ _DELIVERY_PRUNE_EVERY = 1_000_sleep = time.sleep-def _raise_person_version_floor(team_id: int, person_id: int, min_version: int) -> None:+def _raise_person_version_floor(team_id: int, person_uuid: str, min_version: int) -> None:+ # Claims the person's lifecycle mark, so the raise waits out an in-flight merge or delete of the person.personhog_call(- "person_divergence_set_person_version_floor",- lambda: require_personhog_client().set_person_version_floor(- SetPersonVersionFloorRequest(team_id=team_id, person_id=person_id, min_version=min_version)+ "person_divergence_raise_person_version_floors",+ lambda: require_personhog_client().raise_person_version_floors(+ RaisePersonVersionFloorsRequest(+ team_id=team_id, floors=[PersonVersionFloor(person_uuid=person_uuid, min_version=min_version)]+ )),)-def _raise_mapping_version_floor(team_id: int, distinct_id: str, min_version: int) -> str | None:- """Raise the mapping's version on the primary and return the uuid of the person it maps to there."""- response = personhog_call(- "person_divergence_set_person_distinct_id_version_floor",- lambda: require_personhog_client().set_person_distinct_id_version_floor(- SetPersonDistinctIdVersionFloorRequest(team_id=team_id, distinct_id=distinct_id, min_version=min_version)- ),- )- if not response.HasField("person"):- return None- return str(UUID(response.person.uuid))---def _primary_mapping_versions(team_id: int, person_id: int) -> dict[str, int]:- response = personhog_call(- "person_divergence_confirm_mapping_versions",- lambda: require_personhog_client().get_distinct_ids_for_persons(- GetDistinctIdsForPersonsRequest(- team_id=team_id,- person_ids=[person_id],- read_options=ReadOptions(consistency=CONSISTENCY_LEVEL_STRONG),- limit_per_person=repair_plan._MAX_REPAIR_DISTINCT_IDS_PER_PERSON + 1,- )- ),- )- return {d.distinct_id: int(d.version or 0) for pd in response.person_distinct_ids for d in pd.distinct_ids}+def _write_distinct_ids(request: WriteDistinctIdsRequest) -> WriteDistinctIdsResponse:+ return require_personhog_client().write_distinct_ids(request)+++def _raise_mapping_version_floors(+ team_id: int, person_uuid: str, mappings: Sequence[repair_plan._MappingPlan], before_write: Callable[[], None]+) -> dict[str, DistinctIdWriteResult]:+ """Raise each live mapping of the person to its target version on the primary, in batches.++ A mapping that moved to another person, or was tombstoned, is left alone and reported with its current row.+ """+ results: dict[str, DistinctIdWriteResult] = {}+ for batch in lookups._chunks(list(mappings), _WRITE_BATCH_SIZE):+ before_write()+ request = WriteDistinctIdsRequest(+ team_id=team_id,+ writes=[+ DistinctIdWrite(+ distinct_id=m.distinct_id,+ action=DISTINCT_ID_WRITE_ACTION_RAISE,+ person_uuid=person_uuid,+ min_version=m.target_version,+ )+ for m in batch+ ],+ )+ response = personhog_call(+ "person_divergence_raise_mapping_versions",+ partial(_write_distinct_ids, request),+ )+ results.update({r.distinct_id: r for r in response.results})+ return resultsdef _person_action(plan: repair_plan._PersonPlan, outcome: datatypes.RepairOutcome) -> datatypes.RepairAction:@@ -175,6 +198,30 @@ def _is_transient(exc: BaseException) -> bool:)+def is_retryable(exc: BaseException) -> bool:+ """Whether a repair failed on a transient error that a later attempt can get past: a busy person, or+ personhog or ClickHouse briefly unavailable."""+ return _is_transient(exc)+++def is_person_busy(exc: BaseException) -> bool:+ """Whether another operation, such as a merge or delete, held the person: its lifecycle mark was claimed, or+ a wait for its row lock timed out. Other transient errors mean a service was unavailable."""+ if is_transient_rpc_error(exc, codes=frozenset({grpc.StatusCode.FAILED_PRECONDITION})):+ return True+ current: BaseException | None = exc+ while current is not None:+ if isinstance(current, grpc.RpcError):+ code = getattr(current, "code", None)+ trailing = getattr(current, "trailing_metadata", None)+ if not callable(code) or code() != grpc.StatusCode.DEADLINE_EXCEEDED or not callable(trailing):+ return False+ # personhog-replica names the timeout that ended the call; only a lock wait means the person was busy.+ return any(key == _DB_TIMEOUT_METADATA_KEY and value == "lock_timeout" for key, value in trailing() or ())+ current = current.__cause__+ return False++def _retry_transient(fn: Callable[[], _T], *, what: str, log: Callable[[str], None]) -> _T:"""Run ``fn`` again after a transient ClickHouse or personhog error, then re-raise once the retries run out.@@ -240,20 +287,29 @@ def _execute_plan(pg_version = int(person.version or 0)if plan.kind is not None and plan.target_version is not None and plan.target_version > pg_version:before_write()- _raise_person_version_floor(plan.team_id, person.pk, plan.target_version)-- # SetPersonDistinctIdVersionFloor runs on the primary, so its reply is the owner check: the replica list- # that produced this mapping can lag a merge that moved it to another person.+ _raise_person_version_floor(plan.team_id, plan.person_uuid, plan.target_version)++ # The raise runs on the primary, so its reply is the owner check: the replica list that produced this+ # mapping can lag a merge that moved it to another person.+ written = (+ _raise_mapping_version_floors(plan.team_id, plan.person_uuid, divergent_mappings, before_write)+ if divergent_mappings+ else {}+ )owned: list[repair_plan._MappingPlan] = []+ stored_versions: dict[str, int] = {}for mapping in divergent_mappings:- before_write()- owner = _raise_mapping_version_floor(plan.team_id, mapping.distinct_id, mapping.target_version)- if owner is None:+ result = written.get(mapping.distinct_id)+ row = result.row if result is not None and result.HasField("row") else None+ if result is not None and result.outcome == DISTINCT_ID_WRITE_OUTCOME_PERSON_NOT_LIVE:+ mapping_actions.append(_mapping_action(plan, mapping, "skipped_tombstoned"))+ elif row is None or row.is_deleted:mapping_actions.append(_mapping_action(plan, mapping, "skipped_mapping_gone"))- elif owner != plan.person_uuid:+ elif row.person_uuid != plan.person_uuid:mapping_actions.append(_mapping_action(plan, mapping, "skipped_owner_changed"))else:owned.append(mapping)+ stored_versions[mapping.distinct_id] = int(row.version)# Re-read after the raise so the published properties are the ones Postgres holds at the# published version, including any ingestion update that landed after the first read.@@ -280,28 +336,18 @@ def _execute_plan(published(_publish_person(plan.team_id, reread))person_outcome = "repaired"- # The SetPersonDistinctIdVersionFloor reply does not say whether it wrote, and a personhog-replica build- # without NULL-version handling leaves a NULL version unchanged, so publish only the mappings whose- # version on the primary reached the target.- stored_versions = _primary_mapping_versions(plan.team_id, person.pk) if owned else {}+ # The write reply holds the row the primary stored, at or above the target, so it is the version to publish.for mapping in owned:- stored_version = stored_versions.get(mapping.distinct_id)- if stored_version is None:- # The primary no longer lists the mapping under this person: it moved or was tombstoned after the raise.- mapping_actions.append(_mapping_action(plan, mapping, "skipped_mapping_gone"))- elif stored_version < mapping.target_version:- mapping_actions.append(_mapping_action(plan, mapping, "skipped_reread_lagging"))- else:- published(- create_person_distinct_id(- team_id=plan.team_id,- distinct_id=mapping.distinct_id,- person_id=plan.person_uuid,- version=stored_version,- is_deleted=False,- )+ published(+ create_person_distinct_id(+ team_id=plan.team_id,+ distinct_id=mapping.distinct_id,+ person_id=plan.person_uuid,+ version=stored_versions[mapping.distinct_id],+ is_deleted=False,)- mapping_actions.append(_mapping_action(plan, mapping, "repaired"))+ )+ mapping_actions.append(_mapping_action(plan, mapping, "repaired"))return [_person_action(plan, person_outcome), *mapping_actions]diff --git a/posthog/models/person/test/test_divergence.py b/posthog/models/person/test/test_divergence.pyindex d250b7e2f1a8..dee1221217bd 100644--- a/posthog/models/person/test/test_divergence.py+++ b/posthog/models/person/test/test_divergence.py@@ -50,7 +50,6 @@ from posthog.models.person.util import ()from posthog.models.signals import mute_selected_signalsfrom posthog.personhog_client.fake_client import get_active_fake-from posthog.personhog_client.proto import CONSISTENCY_LEVEL_STRONGfrom posthog.test.persons import add_distinct_id, create_personfrom products.access_control.backend.models.property_access_control import PropertyAccessControl@@ -502,8 +501,8 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):assert self._ch_person(person.uuid) == (0, 104, PG_PROPERTIES)assert self._ch_mapping("hidden-did") == (str(person.uuid), 0, 101)else:- get_active_fake().assert_not_called("set_person_version_floor")- get_active_fake().assert_not_called("set_person_distinct_id_version_floor")+ get_active_fake().assert_not_called("raise_person_version_floors")+ get_active_fake().assert_not_called("write_distinct_ids")assert (self._pg_version(person), self._pg_mapping_versions(person)) == (3, {"hidden-did": 0})assert self._ch_person(person.uuid) == (1, 103, CH_PROPERTIES)assert self._ch_mapping("hidden-did") == (str(person.uuid), 1, 100)@@ -529,19 +528,19 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):def test_publishes_the_properties_postgres_holds_after_the_raise(self) -> None:person = self._divergent_person("hidden")fake = get_active_fake()- raise_floor = fake.set_person_version_floor+ raise_floor = fake.raise_person_version_floorsupdated = {"email": "[email protected]"}def ingestion_update_then_raise(- request: person_pb2.SetPersonVersionFloorRequest, timeout: float | None = None- ) -> person_pb2.SetPersonVersionFloorResponse:+ request: person_pb2.RaisePersonVersionFloorsRequest, timeout: float | None = None+ ) -> person_pb2.RaisePersonVersionFloorsResponse:stored = fake.stored_person(self.team.pk, str(person.uuid))assert stored is not Nonestored.properties = json.dumps(updated).encode()stored.version += 1return raise_floor(request, timeout)- with patch.object(fake, "set_person_version_floor", side_effect=ingestion_update_then_raise):+ with patch.object(fake, "raise_person_version_floors", side_effect=ingestion_update_then_raise):self._repair(person.uuid)assert self._ch_person(person.uuid) == (0, 104, updated)@@ -573,10 +572,15 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):rereads = 0def primary_only_raise(- request: person_pb2.SetPersonVersionFloorRequest, timeout: float | None = None- ) -> person_pb2.SetPersonVersionFloorResponse:- raised_to.append(request.min_version)- return person_pb2.SetPersonVersionFloorResponse(updated=True)+ request: person_pb2.RaisePersonVersionFloorsRequest, timeout: float | None = None+ ) -> person_pb2.RaisePersonVersionFloorsResponse:+ (floor,) = request.floors+ raised_to.append(floor.min_version)+ response = person_pb2.RaisePersonVersionFloorsResponse()+ response.results.add(+ person_uuid=floor.person_uuid, outcome=person_pb2.RAISE_OUTCOME_RAISED, version=floor.min_version+ )+ return responsedef lagging_replica(request: person_pb2.GetPersonsByUuidsRequest) -> person_pb2.PersonsResponse:nonlocal rereads@@ -589,7 +593,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):return read(request)with (- patch.object(fake, "set_person_version_floor", side_effect=primary_only_raise),+ patch.object(fake, "raise_person_version_floors", side_effect=primary_only_raise),patch.object(fake, "get_persons_by_uuids", side_effect=lagging_replica),patch("posthog.models.person.divergence.repairs._sleep") as sleep,):@@ -607,6 +611,8 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):("clickhouse_timeout_while_planning", "planning", ClickHouseQueryTimeOut()),("personhog_internal_error_on_the_raise", "raise", grpc.StatusCode.INTERNAL),("personhog_unavailable_on_the_raise", "raise", grpc.StatusCode.UNAVAILABLE),+ # A merge or delete holding the person's lifecycle mark clears once that operation commits.+ ("personhog_mark_held_on_the_raise", "raise", grpc.StatusCode.FAILED_PRECONDITION),])def test_retries_a_transient_error_and_counts_the_person_once(@@ -636,7 +642,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):fake = get_active_fake()with (- patch.object(fake, "set_person_version_floor", side_effect=failure) as raise_floor,+ patch.object(fake, "raise_person_version_floors", side_effect=failure) as raise_floor,patch("posthog.models.person.divergence.repairs._sleep") as sleep,):with self.assertRaises(type(failure)):@@ -658,18 +664,18 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):return query(*args, **kwargs)return patch("posthog.models.person.divergence.lookups.sync_execute", side_effect=ch_fails_once)- raise_floor = fake.set_person_version_floor+ raise_floor = fake.raise_person_version_floorscalls: list[bool] = []def raise_fails_once(- request: person_pb2.SetPersonVersionFloorRequest, timeout: float | None = None- ) -> person_pb2.SetPersonVersionFloorResponse:+ request: person_pb2.RaisePersonVersionFloorsRequest, timeout: float | None = None+ ) -> person_pb2.RaisePersonVersionFloorsResponse:if not calls:calls.append(True)raise failurereturn raise_floor(request, timeout)- return patch.object(fake, "set_person_version_floor", side_effect=raise_fails_once)+ return patch.object(fake, "raise_person_version_floors", side_effect=raise_fails_once)def _skipped_person(self, case: str) -> UUID:if case == "in_sync":@@ -709,8 +715,8 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):_, actions = self._repair(person_uuid)assert [a.outcome for a in actions] == outcomes- get_active_fake().assert_not_called("set_person_version_floor")- get_active_fake().assert_not_called("set_person_distinct_id_version_floor")+ get_active_fake().assert_not_called("raise_person_version_floors")+ get_active_fake().assert_not_called("write_distinct_ids")assert self._ch_person(person_uuid) == beforedef test_skips_every_person_of_a_team_that_no_longer_exists(self) -> None:@@ -731,7 +737,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):assert [a.outcome for a in actions] == ["skipped_team_gone"]assert summary.person_outcomes == {"skipped_team_gone": 1}- get_active_fake().assert_not_called("set_person_version_floor")+ get_active_fake().assert_not_called("raise_person_version_floors")get_active_fake().assert_not_called("get_persons_by_uuids")def test_skips_a_person_the_primary_tombstoned_while_the_replica_still_shows_it_live(self) -> None:@@ -800,54 +806,37 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):target_version=target_version,),]- floors = get_active_fake().assert_called("set_person_distinct_id_version_floor", times=1)- assert floors[0].request.distinct_id == "divergent"- get_active_fake().assert_not_called("set_person_version_floor")+ floors = get_active_fake().assert_called("write_distinct_ids", times=1)+ assert [(w.distinct_id, w.min_version) for w in floors[0].request.writes] == [("divergent", target_version)]+ get_active_fake().assert_not_called("raise_person_version_floors")assert self._pg_mapping_versions(person)["divergent"] == target_versionassert self._ch_mapping("divergent") == (str(person.uuid), 0, target_version)- @parameterized.expand(- [- # A personhog-replica build without NULL-version handling leaves the stored version unchanged.- ("floor_not_applied", None, "skipped_reread_lagging", 2, (1, 100)),- # A concurrent write took the mapping past the target, so the publish carries the stored version.- ("stored_past_the_target", 5, "repaired", 106, (0, 106)),- ]- )- def test_publishes_a_mapping_only_at_the_version_the_primary_holds(- self,- _name: str,- extra_versions: int | None,- outcome: RepairOutcome,- stored_version: int,- ch_deleted_and_version: tuple[int, int],- ) -> None:+ def test_publishes_a_mapping_at_the_version_the_primary_holds_after_the_raise(self) -> None:person = self._pg_person(version=3, distinct_ids={"divergent": 2})self._ch_person_row(person.uuid, 3)self._ch_mapping_row("divergent", person.uuid, 100, deleted=True)fake = get_active_fake()- raise_floor = fake.set_person_distinct_id_version_floor-- def raise_floor_differently(- request: person_pb2.SetPersonDistinctIdVersionFloorRequest, timeout: float | None = None- ) -> person_pb2.SetPersonDistinctIdVersionFloorResponse:- min_version = 0 if extra_versions is None else request.min_version + extra_versions- return raise_floor(- person_pb2.SetPersonDistinctIdVersionFloorRequest(- team_id=request.team_id, distinct_id=request.distinct_id, min_version=min_version- ),- timeout,- )-- with patch.object(fake, "set_person_distinct_id_version_floor", side_effect=raise_floor_differently):+ write = fake.write_distinct_ids++ def concurrent_write_past_the_target(+ request: person_pb2.WriteDistinctIdsRequest, timeout: float | None = None+ ) -> person_pb2.WriteDistinctIdsResponse:+ moved_on = person_pb2.WriteDistinctIdsRequest()+ moved_on.CopyFrom(request)+ for w in moved_on.writes:+ w.min_version += 5+ return write(moved_on, timeout)++ with patch.object(fake, "write_distinct_ids", side_effect=concurrent_write_past_the_target):_, actions = self._repair(person.uuid)assert [(a.distinct_id, a.outcome, a.target_version) for a in actions] == [(None, "skipped_not_divergent", None),- ("divergent", outcome, 101),+ ("divergent", "repaired", 101),]- assert self._pg_mapping_versions(person)["divergent"] == stored_version- assert self._ch_mapping("divergent") == (str(person.uuid), *ch_deleted_and_version)+ assert self._pg_mapping_versions(person)["divergent"] == 106+ assert self._ch_mapping("divergent") == (str(person.uuid), 0, 106)@parameterized.expand([@@ -936,8 +925,8 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):"properties": {"$set_once": CH_ONLY_PROPERTIES},"process_person_profile": True,}- get_active_fake().assert_not_called("set_person_version_floor")- get_active_fake().assert_not_called("set_person_distinct_id_version_floor")+ get_active_fake().assert_not_called("raise_person_version_floors")+ get_active_fake().assert_not_called("write_distinct_ids")assert self._pg_version(person) == 5assert self._ch_person(person.uuid) == (0, 10, {**CH_PROPERTIES, **CH_ONLY_PROPERTIES})@@ -988,8 +977,8 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):assert summary is not Noneassert (summary.person_outcomes, summary.mapping_outcomes) == ({person_outcome: 1}, {"skipped_stale": 1})assert capture.called == capture_called- get_active_fake().assert_not_called("set_person_version_floor")- get_active_fake().assert_not_called("set_person_distinct_id_version_floor")+ get_active_fake().assert_not_called("raise_person_version_floors")+ get_active_fake().assert_not_called("write_distinct_ids")assert self._pg_version(person) == 5assert self._ch_person(person.uuid) == (0, 10, {**CH_PROPERTIES, **CH_ONLY_PROPERTIES})@@ -1011,7 +1000,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):)reads = get_active_fake().assert_called("get_distinct_ids_for_persons", times=1)assert reads[0].request.limit_per_person == 3- get_active_fake().assert_not_called("set_person_distinct_id_version_floor")+ get_active_fake().assert_not_called("write_distinct_ids")assert self._ch_person(person.uuid) == (0, 104, PG_PROPERTIES)assert self._ch_mapping("a") == (str(person.uuid), 1, 100)@@ -1032,21 +1021,23 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):]assert self._ch_mapping("moved") == (str(person.uuid), 1, 100)- def test_leaves_a_mapping_unpublished_once_the_primary_no_longer_lists_it_after_the_raise(self) -> None:+ def test_leaves_a_mapping_unpublished_once_the_primary_tombstoned_it(self) -> None:person = self._pg_person(version=3, distinct_ids={"gone": 0})self._ch_person_row(person.uuid, 3)self._ch_mapping_row("gone", person.uuid, 100, deleted=True)fake = get_active_fake()- list_distinct_ids = fake.get_distinct_ids_for_persons-- def primary_lost_the_mapping(- request: person_pb2.GetDistinctIdsForPersonsRequest,- ) -> person_pb2.GetDistinctIdsForPersonsResponse:- if request.read_options.consistency == CONSISTENCY_LEVEL_STRONG:- return person_pb2.GetDistinctIdsForPersonsResponse()- return list_distinct_ids(request)-- with patch.object(fake, "get_distinct_ids_for_persons", side_effect=primary_lost_the_mapping):+ write = fake.write_distinct_ids++ def primary_tombstoned_the_mapping(+ request: person_pb2.WriteDistinctIdsRequest, timeout: float | None = None+ ) -> person_pb2.WriteDistinctIdsResponse:+ response = write(request, timeout)+ for result in response.results:+ result.outcome = person_pb2.DISTINCT_ID_WRITE_OUTCOME_PRECONDITION_FAILED+ result.row.is_deleted = True+ return response++ with patch.object(fake, "write_distinct_ids", side_effect=primary_tombstoned_the_mapping):_, actions = self._repair(person.uuid)assert [(a.distinct_id, a.outcome) for a in actions] == [diff --git a/posthog/personhog_client/proto/__init__.py b/posthog/personhog_client/proto/__init__.pyindex c1cb545feeeb..5a34e46d4c46 100644--- a/posthog/personhog_client/proto/__init__.py+++ b/posthog/personhog_client/proto/__init__.py@@ -121,5 +121,17 @@ from personhog.types.v1.person_pb2 import (RESTORE_OUTCOME_NO_DISTINCT_ID,WriteDistinctIdsRequest,WriteDistinctIdsResponse,+ DistinctIdWrite,+ DistinctIdWriteResult,+ RestoredPerson,+ DISTINCT_ID_WRITE_ACTION_INSERT,+ DISTINCT_ID_WRITE_ACTION_RAISE,+ DISTINCT_ID_WRITE_OUTCOME_APPLIED,+ DISTINCT_ID_WRITE_OUTCOME_UNCHANGED,+ DISTINCT_ID_WRITE_OUTCOME_PRECONDITION_FAILED,+ DISTINCT_ID_WRITE_OUTCOME_PERSON_NOT_LIVE,+ RESTORE_OUTCOME_INSERTED,+ RESTORE_OUTCOME_EXISTS_LIVE,+ RESTORE_OUTCOME_EXISTS_TOMBSTONED,PersonVersionFloorResult,)