:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(personhog): raise repaired persons through the leader (#125788)
Zach Brown325a6cb1edfc
diff --git a/posthog/models/person/divergence/datatypes.py b/posthog/models/person/divergence/datatypes.pyindex e1f639456de2..dfb82c96edaa 100644--- a/posthog/models/person/divergence/datatypes.py+++ b/posthog/models/person/divergence/datatypes.py@@ -26,6 +26,7 @@ RepairOutcome = Literal["skipped_owner_changed","skipped_mapping_gone","skipped_reread_lagging",+ "skipped_raise_refused","skipped_stale","skipped_stale_merge_failed","skipped_stale_merge_too_large",diff --git a/posthog/models/person/divergence/repairs.py b/posthog/models/person/divergence/repairs.pyindex 0b90de6d6279..b094078de243 100644--- a/posthog/models/person/divergence/repairs.py+++ b/posthog/models/person/divergence/repairs.py@@ -20,11 +20,13 @@ from posthog.kafka_client.topics import KAFKA_PERSONfrom posthog.models.person import Personfrom posthog.models.person.sql import INSERT_PERSON_SQLfrom posthog.models.person.util import (+ PersonRaiseRefused,_flush_person_producers,_person_row,create_person_distinct_id,get_person_by_distinct_id,get_persons_by_uuids,+ raise_person_version,)from posthog.models.team import Teamfrom posthog.personhog_client.client import personhog_call, require_personhog_client@@ -34,8 +36,6 @@ from posthog.personhog_client.proto import (DISTINCT_ID_WRITE_OUTCOME_PERSON_NOT_LIVE,DistinctIdWrite,DistinctIdWriteResult,- PersonVersionFloor,- RaisePersonVersionFloorsRequest,SetPersonDistinctIdVersionFloorRequest,WriteDistinctIdsRequest,WriteDistinctIdsResponse,@@ -77,18 +77,6 @@ _DELIVERY_PRUNE_EVERY = 1_000_sleep = time.sleep-def _raise_person_version_floor(team_id: int, person_uuid: str, min_version: int) -> None:- # Claims the person's lifecycle mark, so the raise waits out an in-flight merge or delete of the person.- personhog_call(- "person_divergence_raise_person_version_floors",- lambda: require_personhog_client().raise_person_version_floors(- RaisePersonVersionFloorsRequest(- team_id=team_id, floors=[PersonVersionFloor(person_uuid=person_uuid, min_version=min_version)]- )- ),- )--def _write_distinct_ids(request: WriteDistinctIdsRequest) -> WriteDistinctIdsResponse:return require_personhog_client().write_distinct_ids(request)@@ -308,9 +296,17 @@ def _execute_plan(]pg_version = int(person.version or 0)+ raised: Person | None = Noneif plan.kind is not None and plan.target_version is not None and plan.target_version > pg_version:before_write()- _raise_person_version_floor(plan.team_id, plan.person_uuid, plan.target_version)+ try:+ raised = raise_person_version(plan.team_id, person, plan.target_version)+ except PersonRaiseRefused:+ return [+ _person_action(plan, "skipped_raise_refused"),+ *mapping_actions,+ *(_mapping_action(plan, m, "skipped_raise_refused") for m in divergent_mappings),+ ]# The raise runs on the primary, so its reply is the owner check: the replica list that produced this# mapping can lag a merge that moved it to another person.@@ -334,11 +330,11 @@ def _execute_plan(owned.append(mapping)stored_versions[mapping.distinct_id] = int(row.version)- # Re-read after the raise so the published properties are the ones Postgres holds at the- # published version, including any ingestion update that landed after the first read.+ # Without the person as written, re-read after the raise so the published properties are the ones Postgres+ # holds at the published version, including any ingestion update that landed after the first read.reread: Person | None = Noneif plan.kind is not None:- reread = _reread_person(plan.team_id, plan.person_uuid, plan.target_version)+ reread = raised or _reread_person(plan.team_id, plan.person_uuid, plan.target_version)if (plan.kind is not None or owned) and lookups._tombstoned_uuids(plan.team_id, [plan.person_uuid]):person_outcome = "skipped_tombstoned" if plan.kind is not None else "skipped_not_divergent"diff --git a/posthog/models/person/test/test_divergence.py b/posthog/models/person/test/test_divergence.pyindex c1ddcd45bf60..4eedd00638f0 100644--- a/posthog/models/person/test/test_divergence.py+++ b/posthog/models/person/test/test_divergence.py@@ -6,7 +6,7 @@ from uuid import UUID, uuid4from posthog.test.base import BaseTest, ClickhouseTestMixinfrom unittest.mock import MagicMock, patch-from django.test import SimpleTestCase+from django.test import SimpleTestCase, override_settingsfrom django.utils.timezone import nowimport grpc@@ -502,6 +502,58 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):assert self._ch_person(person.uuid) == (1, 103, CH_PROPERTIES)assert self._ch_mapping("hidden-did") == (str(person.uuid), 1, 100)+ @override_settings(PERSONHOG_WRITE_PATH="leader")+ def test_raises_through_the_leader_and_publishes_the_person_it_wrote(self) -> None:+ person = self._pg_person(version=3, distinct_ids={"hidden-did": 0})+ self._ch_person_row(person.uuid, 103, deleted=True)+ self._ch_mapping_row("hidden-did", person.uuid, 100, deleted=True)++ _, actions = self._repair(person.uuid)++ fake = get_active_fake()+ fake.assert_not_called("raise_person_version_floors")+ assert [c.request for c in fake.calls if c.method == "raise_person_version"] == [+ person_pb2.RaisePersonVersionRequest(team_id=self.team.pk, person_id=person.pk, min_version=104)+ ]+ assert [a.outcome for a in actions] == ["repaired", "repaired"]+ assert self._ch_person(person.uuid) == (0, 104, PG_PROPERTIES)+ assert self._ch_mapping("hidden-did") == (str(person.uuid), 0, 101)++ @override_settings(PERSONHOG_WRITE_PATH="leader")+ def test_skips_a_person_deleted_before_the_leader_raise(self) -> None:+ person = self._pg_person(version=3, distinct_ids={"hidden-did": 0})+ self._ch_person_row(person.uuid, 103, deleted=True)+ self._ch_mapping_row("hidden-did", person.uuid, 100, deleted=True)+ fake = get_active_fake()+ raise_on_leader = fake.raise_person_version++ def deleted_then_raise(+ request: person_pb2.RaisePersonVersionRequest, timeout: float | None = None+ ) -> person_pb2.RaisePersonVersionResponse:+ tombstone_persons_in_postgres(self.team.pk, [person.uuid])+ return raise_on_leader(request, timeout)++ with patch.object(fake, "raise_person_version", side_effect=deleted_then_raise):+ _, actions = self._repair(person.uuid)++ assert [a.outcome for a in actions] == ["skipped_tombstoned", "skipped_tombstoned"]+ assert self._ch_person(person.uuid) == (1, 103, CH_PROPERTIES)++ @override_settings(PERSONHOG_WRITE_PATH="leader")+ def test_skips_a_person_the_leader_refuses_as_too_large(self) -> None:+ person = self._pg_person(version=3, distinct_ids={"hidden-did": 0})+ self._ch_person_row(person.uuid, 103, deleted=True)+ self._ch_mapping_row("hidden-did", person.uuid, 100, deleted=True)+ refusal = rpc_error(grpc.StatusCode.FAILED_PRECONDITION)+ refusal.trailing_metadata = MagicMock(return_value=(("x-semantic-refusal", "person-too-large"),))+ fake = get_active_fake()++ with patch.object(fake, "raise_person_version", side_effect=refusal):+ _, actions = self._repair(person.uuid)++ assert [a.outcome for a in actions] == ["skipped_raise_refused", "skipped_raise_refused"]+ fake.assert_not_called("write_distinct_ids")+@parameterized.expand([("dry_run", False, []), ("apply", True, [1.0])])def test_throttles_on_each_divergent_row_written(self, _name: str, apply: bool, sleeps: list[float]) -> None:person = self._pg_person(version=3, distinct_ids={"throttled-did": 0})diff --git a/posthog/models/person/test/test_divergence_verbs.py b/posthog/models/person/test/test_divergence_verbs.pyindex a36a6cb9f50b..7dfbccbf0aa8 100644--- a/posthog/models/person/test/test_divergence_verbs.py+++ b/posthog/models/person/test/test_divergence_verbs.py@@ -183,7 +183,7 @@ class TestRepairVerbs(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):@parameterized.expand([- ("republish_after_person_raise", "republish", "repairs._raise_person_version_floor"),+ ("republish_after_person_raise", "republish", "repairs.raise_person_version"),("republish_after_mapping_raise", "republish", "repairs._raise_mapping_version_floors"),("republish_after_person_publish", "republish", "repairs._publish_person"),("republish_after_mapping_publish", "republish", "repairs.create_person_distinct_id"),diff --git a/posthog/models/person/util.py b/posthog/models/person/util.pyindex 202f404379f3..9e47ab183d09 100644--- a/posthog/models/person/util.py+++ b/posthog/models/person/util.py@@ -36,7 +36,7 @@ from posthog.models.person.sql import (from posthog.models.utils import UUIDTfrom posthog.personhog_client.client import personhog_call, require_personhog_clientfrom posthog.personhog_client.converters import proto_person_to_model-from posthog.personhog_client.interceptor import is_transient_rpc_error+from posthog.personhog_client.interceptor import is_transient_rpc_error, semantic_refusal_reasonfrom posthog.personhog_client.metrics import PERSONHOG_TEAM_MISMATCH_TOTAL, get_client_namefrom posthog.personhog_client.proto import (CONSISTENCY_LEVEL_STRONG,@@ -59,6 +59,8 @@ from posthog.personhog_client.proto import (ListPersonTombstoneQueueRequest,PersonListEntry,PersonVersionFloor as PersonVersionFloorProto,+ RaisePersonVersionFloorsRequest,+ RaisePersonVersionRequest,ReadOptions,SetPersonDistinctIdVersionFloorRequest,VersionFloorOutcome as VersionFloorOutcomeProto,@@ -1061,6 +1063,41 @@ def ensure_person_version_floors(team_id: int, floors: Sequence[PersonVersionFloreturn personhog_call("ensure_person_version_floors", personhog_fn)+class PersonRaiseRefused(Exception):+ """The leader refused the raise because the person's protected properties alone exceed the size limit."""+++def raise_person_version(team_id: int, person: Person, min_version: int) -> Person | None:+ """Raise a live person's version to at least ``min_version`` on the active write path.++ Returns the person as written when the path reports it. Otherwise returns None, and the caller reads the person.+ """+ if settings.PERSONHOG_WRITE_PATH == "leader":+ return personhog_call(+ "raise_person_version", partial(_raise_person_version_on_leader, team_id, person, min_version)+ )+ # Claims the person's lifecycle mark, so the raise waits out an in-flight merge or delete of the person.+ request = RaisePersonVersionFloorsRequest(+ team_id=team_id, floors=[PersonVersionFloorProto(person_uuid=str(person.uuid), min_version=min_version)]+ )+ personhog_call("raise_person_version_floors", lambda: _get_client().raise_person_version_floors(request))+ return None+++def _raise_person_version_on_leader(team_id: int, person: Person, min_version: int) -> Person | None:+ request = RaisePersonVersionRequest(team_id=team_id, person_id=person.pk, min_version=min_version)+ try:+ response = _get_client().raise_person_version(request)+ except grpc.RpcError as exc:+ # The leader holds the person as deleted.+ if callable(getattr(exc, "code", None)) and exc.code() == grpc.StatusCode.NOT_FOUND:+ return None+ if semantic_refusal_reason(exc) == "person-too-large":+ raise PersonRaiseRefused(f"person {person.uuid} is too large to raise") from exc+ raise+ return proto_person_to_model(response.person)++def set_distinct_id_version_floor(team_id: int, distinct_id: str, min_version: int) -> None:"""Raise the Postgres version of the distinct id's row to at least ``min_version``, live or tombstoned.diff --git a/posthog/personhog_client/client.py b/posthog/personhog_client/client.pyindex c837681c82c9..6df14f45f0e2 100644--- a/posthog/personhog_client/client.py+++ b/posthog/personhog_client/client.py@@ -95,6 +95,8 @@ from posthog.personhog_client.proto import (PersonsResponse,RaisePersonVersionFloorsRequest,RaisePersonVersionFloorsResponse,+ RaisePersonVersionRequest,+ RaisePersonVersionResponse,RestorePersonsRequest,RestorePersonsResponse,SetPersonDistinctIdVersionFloorRequest,@@ -269,6 +271,7 @@ class PersonHogClient:self._lifecycle_channel: grpc.Channel | None = Noneself._lifecycle_state_monitor: _ChannelStateMonitor | None = Nonelifecycle_stub: PersonHogLifecycleStub | None = None+ self._leader_stub: PersonHogServiceStub | None = Noneif lifecycle_addr:lifecycle_channel = grpc.insecure_channel(lifecycle_addr, options=options)self._lifecycle_channel = grpc.intercept_channel(@@ -280,6 +283,8 @@ class PersonHogClient:# The channel-state gauge is keyed by client_name alone; two channels under one name would fight over it.self._lifecycle_state_monitor = _ChannelStateMonitor(lifecycle_channel, f"{client_name}-lifecycle")lifecycle_stub = PersonHogLifecycleStub(self._lifecycle_channel)+ # The same router reaches the leader, which the facade router cannot.+ self._leader_stub = PersonHogServiceStub(self._lifecycle_channel)self.lifecycle = PersonHogLifecycleClient(lifecycle_stub, lifecycle_timeout_ms / 1000.0)def close(self) -> None:@@ -359,6 +364,18 @@ class PersonHogClient:) -> RaisePersonVersionFloorsResponse:return self._stub.RaisePersonVersionFloors(request, timeout=timeout or self._timeout)+ def raise_person_version(+ self, request: RaisePersonVersionRequest, timeout: float | None = None+ ) -> RaisePersonVersionResponse:+ if self._leader_stub is None:+ raise PersonHogLifecycleUnavailable("personhog leader path not configured (PERSONHOG_LIFECYCLE_ADDR)")+ # The router routes leader calls by these headers, never by the request body.+ return self._leader_stub.RaisePersonVersion(+ request,+ timeout=timeout or self._timeout,+ metadata=[("x-team-id", str(request.team_id)), ("x-person-id", str(request.person_id))],+ )+def restore_persons(self, request: RestorePersonsRequest, timeout: float | None = None) -> RestorePersonsResponse:return self._stub.RestorePersons(request, timeout=timeout or self._timeout)diff --git a/posthog/personhog_client/fake_client.py b/posthog/personhog_client/fake_client.pyindex 509103c305d5..70bf43ddb0eb 100644--- a/posthog/personhog_client/fake_client.py+++ b/posthog/personhog_client/fake_client.py@@ -1201,6 +1201,25 @@ class FakePersonHogClient:response.results.add(person_uuid=floor.person_uuid, outcome=outcome, version=version)return response+ def raise_person_version(+ self, request: person_pb2.RaisePersonVersionRequest, timeout: float | None = None+ ) -> person_pb2.RaisePersonVersionResponse:+ self.calls.append(_Call("raise_person_version", request))+ person = self._live(self._persons_by_id.get((request.team_id, request.person_id)))+ if person is None:+ import grpc++ error = grpc.RpcError()+ error.code = lambda: grpc.StatusCode.NOT_FOUND+ raise error+ # The stored row stays as it is, because Postgres trails the leader until its writer applies the raise.+ written = person_pb2.Person()+ written.CopyFrom(person)+ raised = person.version < request.min_version+ if raised:+ written.version = request.min_version+ return person_pb2.RaisePersonVersionResponse(person=written, raised=raised)+def restore_persons(self, request: person_pb2.RestorePersonsRequest, timeout: float | None = None) -> person_pb2.RestorePersonsResponse:diff --git a/posthog/personhog_client/proto/__init__.py b/posthog/personhog_client/proto/__init__.pyindex 8c03166fbd28..e172c520b319 100644--- a/posthog/personhog_client/proto/__init__.py+++ b/posthog/personhog_client/proto/__init__.py@@ -130,6 +130,8 @@ from personhog.types.v1.person_pb2 import (ListPersonsForTeamResponse,RaisePersonVersionFloorsRequest,RaisePersonVersionFloorsResponse,+ RaisePersonVersionRequest,+ RaisePersonVersionResponse,RestorePersonsRequest,RestorePersonsResponse,RestoredDistinctId,diff --git a/posthog/personhog_client/test_leader_routing.py b/posthog/personhog_client/test_leader_routing.pynew file mode 100644index 000000000000..6089be57603b--- /dev/null+++ b/posthog/personhog_client/test_leader_routing.py@@ -0,0 +1,18 @@+from unittest.mock import MagicMock++from django.test import SimpleTestCase++from posthog.personhog_client.client import PersonHogClient+from posthog.personhog_client.proto import RaisePersonVersionRequest+++class TestLeaderRouting(SimpleTestCase):+ def test_raise_person_version_goes_to_the_leader_router_with_its_routing_key(self) -> None:+ client = PersonHogClient("localhost:1", lifecycle_addr="localhost:2")+ self.addCleanup(client.close)+ stub = MagicMock()+ client._leader_stub = stub++ client.raise_person_version(RaisePersonVersionRequest(team_id=7, person_id=42, min_version=5))++ assert stub.RaisePersonVersion.call_args.kwargs["metadata"] == [("x-team-id", "7"), ("x-person-id", "42")]diff --git a/posthog/settings/personhog.py b/posthog/settings/personhog.pyindex 8672900ee2aa..be48b049642e 100644--- a/posthog/settings/personhog.py+++ b/posthog/settings/personhog.py@@ -19,6 +19,11 @@ PERSONHOG_MAX_SEND_MESSAGE_LENGTH = get_from_env("PERSONHOG_MAX_SEND_MESSAGE_LENPERSONHOG_MAX_RECV_MESSAGE_LENGTH = get_from_env("PERSONHOG_MAX_RECV_MESSAGE_LENGTH", 128 * 1024 * 1024, type_cast=int)PERSONHOG_CLIENT_IDLE_TIMEOUT_MS = get_from_env("PERSONHOG_CLIENT_IDLE_TIMEOUT_MS", 0, type_cast=int)+# Where person writes go: "replica" writes Postgres directly, "leader" goes through the personhog leader.+PERSONHOG_WRITE_PATH = get_from_env("PERSONHOG_WRITE_PATH", "replica")+if PERSONHOG_WRITE_PATH not in ("replica", "leader"):+ raise ValueError(f"PERSONHOG_WRITE_PATH must be 'replica' or 'leader', got {PERSONHOG_WRITE_PATH!r}")+# Retry settings for transient gRPC errorsPERSONHOG_MAX_RETRIES = get_from_env("PERSONHOG_MAX_RETRIES", 2, type_cast=int)PERSONHOG_INITIAL_BACKOFF_MS = get_from_env("PERSONHOG_INITIAL_BACKOFF_MS", 50, type_cast=int)