Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(persons): raise divergence repairs through marked personhog writes (#118360)

Eli Reismana4d893a7b0ef
diff --git a/posthog/api/person.py b/posthog/api/person.py
index ddc9b572c669..983f47096324 100644
--- a/posthog/api/person.py
+++ b/posthog/api/person.py
@@ -15,6 +15,7 @@ from drf_spectacular.types import OpenApiTypes
from drf_spectacular.utils import (
OpenApiExample,
OpenApiParameter,
+ OpenApiResponse,
extend_schema,
extend_schema_field,
extend_schema_serializer,
@@ -63,7 +64,7 @@ from posthog.models.person.bulk_delete import (
queue_person_recording_deletion,
resolve_persons_for_deletion,
)
-from posthog.models.person.divergence.repairs import repair_distinct_id
+from posthog.models.person.divergence.repairs import is_person_busy, is_retryable, repair_distinct_id
from posthog.models.person.missing_person import MissingPerson
from posthog.models.person.util import (
get_distinct_ids_for_persons,
@@ -1852,6 +1853,11 @@ class PersonViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):
@extend_schema(
description="Fix a distinct_id that stays hidden after its person was deleted and created again. Does nothing if no live person uses this distinct_id. In that case, send a new event for it instead.",
+ responses={
+ 202: OpenApiResponse(description="The fix was applied, or nothing needed fixing."),
+ 409: OpenApiResponse(description="The person is being updated by another operation. Try again shortly."),
+ 503: OpenApiResponse(description="Person data is temporarily unavailable. Try again shortly."),
+ },
)
@action(methods=["POST"], detail=False, required_scopes=["person:write"])
def reset_person_distinct_id(self, request: request.Request, *args: Any, **kwargs: Any) -> response.Response:
@@ -1859,9 +1865,22 @@ class PersonViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):
if not distinct_id or not isinstance(distinct_id, str):
raise ValidationError(detail="distinct_id is required")
- summary = repair_distinct_id(
- self.team_id, distinct_id, delivery_timeout_seconds=RESET_DISTINCT_ID_DELIVERY_TIMEOUT_SECONDS
- )
+ try:
+ summary = repair_distinct_id(
+ self.team_id, distinct_id, delivery_timeout_seconds=RESET_DISTINCT_ID_DELIVERY_TIMEOUT_SECONDS
+ )
+ except Exception as error:
+ if is_person_busy(error):
+ return response.Response(
+ {"detail": "This person is being updated right now. Try again in a few seconds."},
+ status=409,
+ )
+ if is_retryable(error):
+ return response.Response(
+ {"detail": "Person data is temporarily unavailable. Try again in a minute."},
+ status=503,
+ )
+ raise
if summary is not None:
logger.info(
"person_reset_distinct_id",
diff --git a/posthog/api/test/test_person.py b/posthog/api/test/test_person.py
index fdc340bf2b06..ffe82ca2c92b 100644
--- a/posthog/api/test/test_person.py
+++ b/posthog/api/test/test_person.py
@@ -23,6 +23,7 @@ from unittest import mock
from django.test import SimpleTestCase
from django.utils import timezone
+import grpc
from clickhouse_driver.errors import ServerException
from parameterized import parameterized
from rest_framework import status
@@ -2418,6 +2419,36 @@ class TestPerson(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):
assert person_after is not None
assert person_after.version is not None and person_after.version > 105
+ @parameterized.expand(
+ [
+ ("mark_held_by_another_operation", grpc.StatusCode.FAILED_PRECONDITION, (), 409),
+ ("row_lock_timed_out", grpc.StatusCode.DEADLINE_EXCEEDED, (("x-db-timeout", "lock_timeout"),), 409),
+ ("statement_timed_out", grpc.StatusCode.DEADLINE_EXCEEDED, (("x-db-timeout", "statement_timeout"),), 503),
+ ("personhog_deadline", grpc.StatusCode.DEADLINE_EXCEEDED, (), 503),
+ ("personhog_unavailable", grpc.StatusCode.UNAVAILABLE, (), 503),
+ ]
+ )
+ def test_reset_person_distinct_id_asks_for_a_retry_on_a_busy_person_or_an_unavailable_service(
+ self, _name: str, code: grpc.StatusCode, trailing: tuple[tuple[str, str], ...], status: int
+ ) -> None:
+ failure = grpc.RpcError()
+ failure.code = mock.MagicMock(return_value=code)
+ failure.trailing_metadata = mock.MagicMock(return_value=trailing)
+
+ with mock.patch("posthog.api.person.repair_distinct_id", side_effect=failure):
+ response = self.client.post(
+ f"/api/projects/{self.team.pk}/persons/reset_person_distinct_id/", {"distinct_id": "busy"}
+ )
+
+ assert response.status_code == status
+ assert (
+ response.json()["detail"]
+ == {
+ 409: "This person is being updated right now. Try again in a few seconds.",
+ 503: "Person data is temporarily unavailable. Try again in a minute.",
+ }[status]
+ )
+
@mock.patch(
f"{posthog.models.person.divergence.repairs.__name__}.create_person_distinct_id",
wraps=posthog.models.person.divergence.repairs.create_person_distinct_id,
diff --git a/posthog/models/person/divergence/repairs.py b/posthog/models/person/divergence/repairs.py
index 3db3b3e8215e..e0069543818b 100644
--- a/posthog/models/person/divergence/repairs.py
+++ b/posthog/models/person/divergence/repairs.py
@@ -8,7 +8,6 @@ from collections.abc import Callable, Sequence
from dataclasses import field
from functools import partial
from typing import TypeVar
-from uuid import UUID
import grpc
@@ -30,11 +29,14 @@ from posthog.models.team import Team
from posthog.personhog_client.client import personhog_call, require_personhog_client
from posthog.personhog_client.interceptor import is_transient_rpc_error
from posthog.personhog_client.proto import (
- CONSISTENCY_LEVEL_STRONG,
- GetDistinctIdsForPersonsRequest,
- ReadOptions,
- SetPersonDistinctIdVersionFloorRequest,
- SetPersonVersionFloorRequest,
+ DISTINCT_ID_WRITE_ACTION_RAISE,
+ DISTINCT_ID_WRITE_OUTCOME_PERSON_NOT_LIVE,
+ DistinctIdWrite,
+ DistinctIdWriteResult,
+ PersonVersionFloor,
+ RaisePersonVersionFloorsRequest,
+ WriteDistinctIdsRequest,
+ WriteDistinctIdsResponse,
)
from . import datatypes, lookups, repair_plan, stale_merge
@@ -44,15 +46,27 @@ _T = TypeVar("_T")
_REPAIR_CHUNK_SIZE = 100
+# The most rows one personhog repair write takes.
+_WRITE_BATCH_SIZE = 250
+
# The waits between re-reads give the replica about 0.5 s to show a raise, far above its usual lag of a few milliseconds.
_REREAD_BACKOFF_SECONDS = (0.025, 0.05, 0.1, 0.15, 0.175)
_TRANSIENT_RETRY_BACKOFF_SECONDS = (1.0, 2.0)
+# FAILED_PRECONDITION is transient for the repair writes: personhog returns it only for a person whose lifecycle
+# mark another operation holds, or for a row that changed between its read and its lock, and both clear on a retry.
_TRANSIENT_RPC_CODES = frozenset(
- {grpc.StatusCode.INTERNAL, grpc.StatusCode.UNAVAILABLE, grpc.StatusCode.DEADLINE_EXCEEDED}
+ {
+ grpc.StatusCode.INTERNAL,
+ grpc.StatusCode.UNAVAILABLE,
+ grpc.StatusCode.DEADLINE_EXCEEDED,
+ grpc.StatusCode.FAILED_PRECONDITION,
+ }
)
+_DB_TIMEOUT_METADATA_KEY = "x-db-timeout"
+
_FLUSH_TIMEOUT_SECONDS = 5 * 60
# Confirmed produce results are dropped this often, so a long repair does not hold one per published row.
@@ -61,41 +75,50 @@ _DELIVERY_PRUNE_EVERY = 1_000
_sleep = time.sleep
-def _raise_person_version_floor(team_id: int, person_id: int, min_version: int) -> None:
+def _raise_person_version_floor(team_id: int, person_uuid: str, min_version: int) -> None:
+ # Claims the person's lifecycle mark, so the raise waits out an in-flight merge or delete of the person.
personhog_call(
- "person_divergence_set_person_version_floor",
- lambda: require_personhog_client().set_person_version_floor(
- SetPersonVersionFloorRequest(team_id=team_id, person_id=person_id, min_version=min_version)
+ "person_divergence_raise_person_version_floors",
+ lambda: require_personhog_client().raise_person_version_floors(
+ RaisePersonVersionFloorsRequest(
+ team_id=team_id, floors=[PersonVersionFloor(person_uuid=person_uuid, min_version=min_version)]
+ )
),
)
-def _raise_mapping_version_floor(team_id: int, distinct_id: str, min_version: int) -> str | None:
- """Raise the mapping's version on the primary and return the uuid of the person it maps to there."""
- response = personhog_call(
- "person_divergence_set_person_distinct_id_version_floor",
- lambda: require_personhog_client().set_person_distinct_id_version_floor(
- SetPersonDistinctIdVersionFloorRequest(team_id=team_id, distinct_id=distinct_id, min_version=min_version)
- ),
- )
- if not response.HasField("person"):
- return None
- return str(UUID(response.person.uuid))
-
-
-def _primary_mapping_versions(team_id: int, person_id: int) -> dict[str, int]:
- response = personhog_call(
- "person_divergence_confirm_mapping_versions",
- lambda: require_personhog_client().get_distinct_ids_for_persons(
- GetDistinctIdsForPersonsRequest(
- team_id=team_id,
- person_ids=[person_id],
- read_options=ReadOptions(consistency=CONSISTENCY_LEVEL_STRONG),
- limit_per_person=repair_plan._MAX_REPAIR_DISTINCT_IDS_PER_PERSON + 1,
- )
- ),
- )
- return {d.distinct_id: int(d.version or 0) for pd in response.person_distinct_ids for d in pd.distinct_ids}
+def _write_distinct_ids(request: WriteDistinctIdsRequest) -> WriteDistinctIdsResponse:
+ return require_personhog_client().write_distinct_ids(request)
+
+
+def _raise_mapping_version_floors(
+ team_id: int, person_uuid: str, mappings: Sequence[repair_plan._MappingPlan], before_write: Callable[[], None]
+) -> dict[str, DistinctIdWriteResult]:
+ """Raise each live mapping of the person to its target version on the primary, in batches.
+
+ A mapping that moved to another person, or was tombstoned, is left alone and reported with its current row.
+ """
+ results: dict[str, DistinctIdWriteResult] = {}
+ for batch in lookups._chunks(list(mappings), _WRITE_BATCH_SIZE):
+ before_write()
+ request = WriteDistinctIdsRequest(
+ team_id=team_id,
+ writes=[
+ DistinctIdWrite(
+ distinct_id=m.distinct_id,
+ action=DISTINCT_ID_WRITE_ACTION_RAISE,
+ person_uuid=person_uuid,
+ min_version=m.target_version,
+ )
+ for m in batch
+ ],
+ )
+ response = personhog_call(
+ "person_divergence_raise_mapping_versions",
+ partial(_write_distinct_ids, request),
+ )
+ results.update({r.distinct_id: r for r in response.results})
+ return results
def _person_action(plan: repair_plan._PersonPlan, outcome: datatypes.RepairOutcome) -> datatypes.RepairAction:
@@ -175,6 +198,30 @@ def _is_transient(exc: BaseException) -> bool:
)
+def is_retryable(exc: BaseException) -> bool:
+ """Whether a repair failed on a transient error that a later attempt can get past: a busy person, or
+ personhog or ClickHouse briefly unavailable."""
+ return _is_transient(exc)
+
+
+def is_person_busy(exc: BaseException) -> bool:
+ """Whether another operation, such as a merge or delete, held the person: its lifecycle mark was claimed, or
+ a wait for its row lock timed out. Other transient errors mean a service was unavailable."""
+ if is_transient_rpc_error(exc, codes=frozenset({grpc.StatusCode.FAILED_PRECONDITION})):
+ return True
+ current: BaseException | None = exc
+ while current is not None:
+ if isinstance(current, grpc.RpcError):
+ code = getattr(current, "code", None)
+ trailing = getattr(current, "trailing_metadata", None)
+ if not callable(code) or code() != grpc.StatusCode.DEADLINE_EXCEEDED or not callable(trailing):
+ return False
+ # personhog-replica names the timeout that ended the call; only a lock wait means the person was busy.
+ return any(key == _DB_TIMEOUT_METADATA_KEY and value == "lock_timeout" for key, value in trailing() or ())
+ current = current.__cause__
+ return False
+
+
def _retry_transient(fn: Callable[[], _T], *, what: str, log: Callable[[str], None]) -> _T:
"""Run ``fn`` again after a transient ClickHouse or personhog error, then re-raise once the retries run out.
@@ -240,20 +287,29 @@ def _execute_plan(
pg_version = int(person.version or 0)
if plan.kind is not None and plan.target_version is not None and plan.target_version > pg_version:
before_write()
- _raise_person_version_floor(plan.team_id, person.pk, plan.target_version)
-
- # SetPersonDistinctIdVersionFloor runs on the primary, so its reply is the owner check: the replica list
- # that produced this mapping can lag a merge that moved it to another person.
+ _raise_person_version_floor(plan.team_id, plan.person_uuid, plan.target_version)
+
+ # The raise runs on the primary, so its reply is the owner check: the replica list that produced this
+ # mapping can lag a merge that moved it to another person.
+ written = (
+ _raise_mapping_version_floors(plan.team_id, plan.person_uuid, divergent_mappings, before_write)
+ if divergent_mappings
+ else {}
+ )
owned: list[repair_plan._MappingPlan] = []
+ stored_versions: dict[str, int] = {}
for mapping in divergent_mappings:
- before_write()
- owner = _raise_mapping_version_floor(plan.team_id, mapping.distinct_id, mapping.target_version)
- if owner is None:
+ result = written.get(mapping.distinct_id)
+ row = result.row if result is not None and result.HasField("row") else None
+ if result is not None and result.outcome == DISTINCT_ID_WRITE_OUTCOME_PERSON_NOT_LIVE:
+ mapping_actions.append(_mapping_action(plan, mapping, "skipped_tombstoned"))
+ elif row is None or row.is_deleted:
mapping_actions.append(_mapping_action(plan, mapping, "skipped_mapping_gone"))
- elif owner != plan.person_uuid:
+ elif row.person_uuid != plan.person_uuid:
mapping_actions.append(_mapping_action(plan, mapping, "skipped_owner_changed"))
else:
owned.append(mapping)
+ stored_versions[mapping.distinct_id] = int(row.version)
# Re-read after the raise so the published properties are the ones Postgres holds at the
# published version, including any ingestion update that landed after the first read.
@@ -280,28 +336,18 @@ def _execute_plan(
published(_publish_person(plan.team_id, reread))
person_outcome = "repaired"
- # The SetPersonDistinctIdVersionFloor reply does not say whether it wrote, and a personhog-replica build
- # without NULL-version handling leaves a NULL version unchanged, so publish only the mappings whose
- # version on the primary reached the target.
- stored_versions = _primary_mapping_versions(plan.team_id, person.pk) if owned else {}
+ # The write reply holds the row the primary stored, at or above the target, so it is the version to publish.
for mapping in owned:
- stored_version = stored_versions.get(mapping.distinct_id)
- if stored_version is None:
- # The primary no longer lists the mapping under this person: it moved or was tombstoned after the raise.
- mapping_actions.append(_mapping_action(plan, mapping, "skipped_mapping_gone"))
- elif stored_version < mapping.target_version:
- mapping_actions.append(_mapping_action(plan, mapping, "skipped_reread_lagging"))
- else:
- published(
- create_person_distinct_id(
- team_id=plan.team_id,
- distinct_id=mapping.distinct_id,
- person_id=plan.person_uuid,
- version=stored_version,
- is_deleted=False,
- )
+ published(
+ create_person_distinct_id(
+ team_id=plan.team_id,
+ distinct_id=mapping.distinct_id,
+ person_id=plan.person_uuid,
+ version=stored_versions[mapping.distinct_id],
+ is_deleted=False,
)
- mapping_actions.append(_mapping_action(plan, mapping, "repaired"))
+ )
+ mapping_actions.append(_mapping_action(plan, mapping, "repaired"))
return [_person_action(plan, person_outcome), *mapping_actions]
diff --git a/posthog/models/person/test/test_divergence.py b/posthog/models/person/test/test_divergence.py
index d250b7e2f1a8..dee1221217bd 100644
--- a/posthog/models/person/test/test_divergence.py
+++ b/posthog/models/person/test/test_divergence.py
@@ -50,7 +50,6 @@ from posthog.models.person.util import (
)
from posthog.models.signals import mute_selected_signals
from posthog.personhog_client.fake_client import get_active_fake
-from posthog.personhog_client.proto import CONSISTENCY_LEVEL_STRONG
from posthog.test.persons import add_distinct_id, create_person
from products.access_control.backend.models.property_access_control import PropertyAccessControl
@@ -502,8 +501,8 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
assert self._ch_person(person.uuid) == (0, 104, PG_PROPERTIES)
assert self._ch_mapping("hidden-did") == (str(person.uuid), 0, 101)
else:
- get_active_fake().assert_not_called("set_person_version_floor")
- get_active_fake().assert_not_called("set_person_distinct_id_version_floor")
+ get_active_fake().assert_not_called("raise_person_version_floors")
+ get_active_fake().assert_not_called("write_distinct_ids")
assert (self._pg_version(person), self._pg_mapping_versions(person)) == (3, {"hidden-did": 0})
assert self._ch_person(person.uuid) == (1, 103, CH_PROPERTIES)
assert self._ch_mapping("hidden-did") == (str(person.uuid), 1, 100)
@@ -529,19 +528,19 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
def test_publishes_the_properties_postgres_holds_after_the_raise(self) -> None:
person = self._divergent_person("hidden")
fake = get_active_fake()
- raise_floor = fake.set_person_version_floor
+ raise_floor = fake.raise_person_version_floors
updated = {"email": "[email protected]"}
def ingestion_update_then_raise(
- request: person_pb2.SetPersonVersionFloorRequest, timeout: float | None = None
- ) -> person_pb2.SetPersonVersionFloorResponse:
+ request: person_pb2.RaisePersonVersionFloorsRequest, timeout: float | None = None
+ ) -> person_pb2.RaisePersonVersionFloorsResponse:
stored = fake.stored_person(self.team.pk, str(person.uuid))
assert stored is not None
stored.properties = json.dumps(updated).encode()
stored.version += 1
return raise_floor(request, timeout)
- with patch.object(fake, "set_person_version_floor", side_effect=ingestion_update_then_raise):
+ with patch.object(fake, "raise_person_version_floors", side_effect=ingestion_update_then_raise):
self._repair(person.uuid)
assert self._ch_person(person.uuid) == (0, 104, updated)
@@ -573,10 +572,15 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
rereads = 0
def primary_only_raise(
- request: person_pb2.SetPersonVersionFloorRequest, timeout: float | None = None
- ) -> person_pb2.SetPersonVersionFloorResponse:
- raised_to.append(request.min_version)
- return person_pb2.SetPersonVersionFloorResponse(updated=True)
+ request: person_pb2.RaisePersonVersionFloorsRequest, timeout: float | None = None
+ ) -> person_pb2.RaisePersonVersionFloorsResponse:
+ (floor,) = request.floors
+ raised_to.append(floor.min_version)
+ response = person_pb2.RaisePersonVersionFloorsResponse()
+ response.results.add(
+ person_uuid=floor.person_uuid, outcome=person_pb2.RAISE_OUTCOME_RAISED, version=floor.min_version
+ )
+ return response
def lagging_replica(request: person_pb2.GetPersonsByUuidsRequest) -> person_pb2.PersonsResponse:
nonlocal rereads
@@ -589,7 +593,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
return read(request)
with (
- patch.object(fake, "set_person_version_floor", side_effect=primary_only_raise),
+ patch.object(fake, "raise_person_version_floors", side_effect=primary_only_raise),
patch.object(fake, "get_persons_by_uuids", side_effect=lagging_replica),
patch("posthog.models.person.divergence.repairs._sleep") as sleep,
):
@@ -607,6 +611,8 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
("clickhouse_timeout_while_planning", "planning", ClickHouseQueryTimeOut()),
("personhog_internal_error_on_the_raise", "raise", grpc.StatusCode.INTERNAL),
("personhog_unavailable_on_the_raise", "raise", grpc.StatusCode.UNAVAILABLE),
+ # A merge or delete holding the person's lifecycle mark clears once that operation commits.
+ ("personhog_mark_held_on_the_raise", "raise", grpc.StatusCode.FAILED_PRECONDITION),
]
)
def test_retries_a_transient_error_and_counts_the_person_once(
@@ -636,7 +642,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
fake = get_active_fake()
with (
- patch.object(fake, "set_person_version_floor", side_effect=failure) as raise_floor,
+ patch.object(fake, "raise_person_version_floors", side_effect=failure) as raise_floor,
patch("posthog.models.person.divergence.repairs._sleep") as sleep,
):
with self.assertRaises(type(failure)):
@@ -658,18 +664,18 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
return query(*args, **kwargs)
return patch("posthog.models.person.divergence.lookups.sync_execute", side_effect=ch_fails_once)
- raise_floor = fake.set_person_version_floor
+ raise_floor = fake.raise_person_version_floors
calls: list[bool] = []
def raise_fails_once(
- request: person_pb2.SetPersonVersionFloorRequest, timeout: float | None = None
- ) -> person_pb2.SetPersonVersionFloorResponse:
+ request: person_pb2.RaisePersonVersionFloorsRequest, timeout: float | None = None
+ ) -> person_pb2.RaisePersonVersionFloorsResponse:
if not calls:
calls.append(True)
raise failure
return raise_floor(request, timeout)
- return patch.object(fake, "set_person_version_floor", side_effect=raise_fails_once)
+ return patch.object(fake, "raise_person_version_floors", side_effect=raise_fails_once)
def _skipped_person(self, case: str) -> UUID:
if case == "in_sync":
@@ -709,8 +715,8 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
_, actions = self._repair(person_uuid)
assert [a.outcome for a in actions] == outcomes
- get_active_fake().assert_not_called("set_person_version_floor")
- get_active_fake().assert_not_called("set_person_distinct_id_version_floor")
+ get_active_fake().assert_not_called("raise_person_version_floors")
+ get_active_fake().assert_not_called("write_distinct_ids")
assert self._ch_person(person_uuid) == before
def test_skips_every_person_of_a_team_that_no_longer_exists(self) -> None:
@@ -731,7 +737,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
assert [a.outcome for a in actions] == ["skipped_team_gone"]
assert summary.person_outcomes == {"skipped_team_gone": 1}
- get_active_fake().assert_not_called("set_person_version_floor")
+ get_active_fake().assert_not_called("raise_person_version_floors")
get_active_fake().assert_not_called("get_persons_by_uuids")
def test_skips_a_person_the_primary_tombstoned_while_the_replica_still_shows_it_live(self) -> None:
@@ -800,54 +806,37 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
target_version=target_version,
),
]
- floors = get_active_fake().assert_called("set_person_distinct_id_version_floor", times=1)
- assert floors[0].request.distinct_id == "divergent"
- get_active_fake().assert_not_called("set_person_version_floor")
+ floors = get_active_fake().assert_called("write_distinct_ids", times=1)
+ assert [(w.distinct_id, w.min_version) for w in floors[0].request.writes] == [("divergent", target_version)]
+ get_active_fake().assert_not_called("raise_person_version_floors")
assert self._pg_mapping_versions(person)["divergent"] == target_version
assert self._ch_mapping("divergent") == (str(person.uuid), 0, target_version)
- @parameterized.expand(
- [
- # A personhog-replica build without NULL-version handling leaves the stored version unchanged.
- ("floor_not_applied", None, "skipped_reread_lagging", 2, (1, 100)),
- # A concurrent write took the mapping past the target, so the publish carries the stored version.
- ("stored_past_the_target", 5, "repaired", 106, (0, 106)),
- ]
- )
- def test_publishes_a_mapping_only_at_the_version_the_primary_holds(
- self,
- _name: str,
- extra_versions: int | None,
- outcome: RepairOutcome,
- stored_version: int,
- ch_deleted_and_version: tuple[int, int],
- ) -> None:
+ def test_publishes_a_mapping_at_the_version_the_primary_holds_after_the_raise(self) -> None:
person = self._pg_person(version=3, distinct_ids={"divergent": 2})
self._ch_person_row(person.uuid, 3)
self._ch_mapping_row("divergent", person.uuid, 100, deleted=True)
fake = get_active_fake()
- raise_floor = fake.set_person_distinct_id_version_floor
-
- def raise_floor_differently(
- request: person_pb2.SetPersonDistinctIdVersionFloorRequest, timeout: float | None = None
- ) -> person_pb2.SetPersonDistinctIdVersionFloorResponse:
- min_version = 0 if extra_versions is None else request.min_version + extra_versions
- return raise_floor(
- person_pb2.SetPersonDistinctIdVersionFloorRequest(
- team_id=request.team_id, distinct_id=request.distinct_id, min_version=min_version
- ),
- timeout,
- )
-
- with patch.object(fake, "set_person_distinct_id_version_floor", side_effect=raise_floor_differently):
+ write = fake.write_distinct_ids
+
+ def concurrent_write_past_the_target(
+ request: person_pb2.WriteDistinctIdsRequest, timeout: float | None = None
+ ) -> person_pb2.WriteDistinctIdsResponse:
+ moved_on = person_pb2.WriteDistinctIdsRequest()
+ moved_on.CopyFrom(request)
+ for w in moved_on.writes:
+ w.min_version += 5
+ return write(moved_on, timeout)
+
+ with patch.object(fake, "write_distinct_ids", side_effect=concurrent_write_past_the_target):
_, actions = self._repair(person.uuid)
assert [(a.distinct_id, a.outcome, a.target_version) for a in actions] == [
(None, "skipped_not_divergent", None),
- ("divergent", outcome, 101),
+ ("divergent", "repaired", 101),
]
- assert self._pg_mapping_versions(person)["divergent"] == stored_version
- assert self._ch_mapping("divergent") == (str(person.uuid), *ch_deleted_and_version)
+ assert self._pg_mapping_versions(person)["divergent"] == 106
+ assert self._ch_mapping("divergent") == (str(person.uuid), 0, 106)
@parameterized.expand(
[
@@ -936,8 +925,8 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
"properties": {"$set_once": CH_ONLY_PROPERTIES},
"process_person_profile": True,
}
- get_active_fake().assert_not_called("set_person_version_floor")
- get_active_fake().assert_not_called("set_person_distinct_id_version_floor")
+ get_active_fake().assert_not_called("raise_person_version_floors")
+ get_active_fake().assert_not_called("write_distinct_ids")
assert self._pg_version(person) == 5
assert self._ch_person(person.uuid) == (0, 10, {**CH_PROPERTIES, **CH_ONLY_PROPERTIES})
@@ -988,8 +977,8 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
assert summary is not None
assert (summary.person_outcomes, summary.mapping_outcomes) == ({person_outcome: 1}, {"skipped_stale": 1})
assert capture.called == capture_called
- get_active_fake().assert_not_called("set_person_version_floor")
- get_active_fake().assert_not_called("set_person_distinct_id_version_floor")
+ get_active_fake().assert_not_called("raise_person_version_floors")
+ get_active_fake().assert_not_called("write_distinct_ids")
assert self._pg_version(person) == 5
assert self._ch_person(person.uuid) == (0, 10, {**CH_PROPERTIES, **CH_ONLY_PROPERTIES})
@@ -1011,7 +1000,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
)
reads = get_active_fake().assert_called("get_distinct_ids_for_persons", times=1)
assert reads[0].request.limit_per_person == 3
- get_active_fake().assert_not_called("set_person_distinct_id_version_floor")
+ get_active_fake().assert_not_called("write_distinct_ids")
assert self._ch_person(person.uuid) == (0, 104, PG_PROPERTIES)
assert self._ch_mapping("a") == (str(person.uuid), 1, 100)
@@ -1032,21 +1021,23 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
]
assert self._ch_mapping("moved") == (str(person.uuid), 1, 100)
- def test_leaves_a_mapping_unpublished_once_the_primary_no_longer_lists_it_after_the_raise(self) -> None:
+ def test_leaves_a_mapping_unpublished_once_the_primary_tombstoned_it(self) -> None:
person = self._pg_person(version=3, distinct_ids={"gone": 0})
self._ch_person_row(person.uuid, 3)
self._ch_mapping_row("gone", person.uuid, 100, deleted=True)
fake = get_active_fake()
- list_distinct_ids = fake.get_distinct_ids_for_persons
-
- def primary_lost_the_mapping(
- request: person_pb2.GetDistinctIdsForPersonsRequest,
- ) -> person_pb2.GetDistinctIdsForPersonsResponse:
- if request.read_options.consistency == CONSISTENCY_LEVEL_STRONG:
- return person_pb2.GetDistinctIdsForPersonsResponse()
- return list_distinct_ids(request)
-
- with patch.object(fake, "get_distinct_ids_for_persons", side_effect=primary_lost_the_mapping):
+ write = fake.write_distinct_ids
+
+ def primary_tombstoned_the_mapping(
+ request: person_pb2.WriteDistinctIdsRequest, timeout: float | None = None
+ ) -> person_pb2.WriteDistinctIdsResponse:
+ response = write(request, timeout)
+ for result in response.results:
+ result.outcome = person_pb2.DISTINCT_ID_WRITE_OUTCOME_PRECONDITION_FAILED
+ result.row.is_deleted = True
+ return response
+
+ with patch.object(fake, "write_distinct_ids", side_effect=primary_tombstoned_the_mapping):
_, actions = self._repair(person.uuid)
assert [(a.distinct_id, a.outcome) for a in actions] == [
diff --git a/posthog/personhog_client/proto/__init__.py b/posthog/personhog_client/proto/__init__.py
index c1cb545feeeb..5a34e46d4c46 100644
--- a/posthog/personhog_client/proto/__init__.py
+++ b/posthog/personhog_client/proto/__init__.py
@@ -121,5 +121,17 @@ from personhog.types.v1.person_pb2 import (
RESTORE_OUTCOME_NO_DISTINCT_ID,
WriteDistinctIdsRequest,
WriteDistinctIdsResponse,
+ DistinctIdWrite,
+ DistinctIdWriteResult,
+ RestoredPerson,
+ DISTINCT_ID_WRITE_ACTION_INSERT,
+ DISTINCT_ID_WRITE_ACTION_RAISE,
+ DISTINCT_ID_WRITE_OUTCOME_APPLIED,
+ DISTINCT_ID_WRITE_OUTCOME_UNCHANGED,
+ DISTINCT_ID_WRITE_OUTCOME_PRECONDITION_FAILED,
+ DISTINCT_ID_WRITE_OUTCOME_PERSON_NOT_LIVE,
+ RESTORE_OUTCOME_INSERTED,
+ RESTORE_OUTCOME_EXISTS_LIVE,
+ RESTORE_OUTCOME_EXISTS_TOMBSTONED,
PersonVersionFloorResult,
)