Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(personhog): raise repaired persons through the leader (#125788)

Zach Brown325a6cb1edfc
diff --git a/posthog/models/person/divergence/datatypes.py b/posthog/models/person/divergence/datatypes.py
index e1f639456de2..dfb82c96edaa 100644
--- a/posthog/models/person/divergence/datatypes.py
+++ b/posthog/models/person/divergence/datatypes.py
@@ -26,6 +26,7 @@ RepairOutcome = Literal[
"skipped_owner_changed",
"skipped_mapping_gone",
"skipped_reread_lagging",
+ "skipped_raise_refused",
"skipped_stale",
"skipped_stale_merge_failed",
"skipped_stale_merge_too_large",
diff --git a/posthog/models/person/divergence/repairs.py b/posthog/models/person/divergence/repairs.py
index 0b90de6d6279..b094078de243 100644
--- a/posthog/models/person/divergence/repairs.py
+++ b/posthog/models/person/divergence/repairs.py
@@ -20,11 +20,13 @@ from posthog.kafka_client.topics import KAFKA_PERSON
from posthog.models.person import Person
from posthog.models.person.sql import INSERT_PERSON_SQL
from posthog.models.person.util import (
+ PersonRaiseRefused,
_flush_person_producers,
_person_row,
create_person_distinct_id,
get_person_by_distinct_id,
get_persons_by_uuids,
+ raise_person_version,
)
from posthog.models.team import Team
from posthog.personhog_client.client import personhog_call, require_personhog_client
@@ -34,8 +36,6 @@ from posthog.personhog_client.proto import (
DISTINCT_ID_WRITE_OUTCOME_PERSON_NOT_LIVE,
DistinctIdWrite,
DistinctIdWriteResult,
- PersonVersionFloor,
- RaisePersonVersionFloorsRequest,
SetPersonDistinctIdVersionFloorRequest,
WriteDistinctIdsRequest,
WriteDistinctIdsResponse,
@@ -77,18 +77,6 @@ _DELIVERY_PRUNE_EVERY = 1_000
_sleep = time.sleep
-def _raise_person_version_floor(team_id: int, person_uuid: str, min_version: int) -> None:
- # Claims the person's lifecycle mark, so the raise waits out an in-flight merge or delete of the person.
- personhog_call(
- "person_divergence_raise_person_version_floors",
- lambda: require_personhog_client().raise_person_version_floors(
- RaisePersonVersionFloorsRequest(
- team_id=team_id, floors=[PersonVersionFloor(person_uuid=person_uuid, min_version=min_version)]
- )
- ),
- )
-
-
def _write_distinct_ids(request: WriteDistinctIdsRequest) -> WriteDistinctIdsResponse:
return require_personhog_client().write_distinct_ids(request)
@@ -308,9 +296,17 @@ def _execute_plan(
]
pg_version = int(person.version or 0)
+ raised: Person | None = None
if plan.kind is not None and plan.target_version is not None and plan.target_version > pg_version:
before_write()
- _raise_person_version_floor(plan.team_id, plan.person_uuid, plan.target_version)
+ try:
+ raised = raise_person_version(plan.team_id, person, plan.target_version)
+ except PersonRaiseRefused:
+ return [
+ _person_action(plan, "skipped_raise_refused"),
+ *mapping_actions,
+ *(_mapping_action(plan, m, "skipped_raise_refused") for m in divergent_mappings),
+ ]
# The raise runs on the primary, so its reply is the owner check: the replica list that produced this
# mapping can lag a merge that moved it to another person.
@@ -334,11 +330,11 @@ def _execute_plan(
owned.append(mapping)
stored_versions[mapping.distinct_id] = int(row.version)
- # Re-read after the raise so the published properties are the ones Postgres holds at the
- # published version, including any ingestion update that landed after the first read.
+ # Without the person as written, re-read after the raise so the published properties are the ones Postgres
+ # holds at the published version, including any ingestion update that landed after the first read.
reread: Person | None = None
if plan.kind is not None:
- reread = _reread_person(plan.team_id, plan.person_uuid, plan.target_version)
+ reread = raised or _reread_person(plan.team_id, plan.person_uuid, plan.target_version)
if (plan.kind is not None or owned) and lookups._tombstoned_uuids(plan.team_id, [plan.person_uuid]):
person_outcome = "skipped_tombstoned" if plan.kind is not None else "skipped_not_divergent"
diff --git a/posthog/models/person/test/test_divergence.py b/posthog/models/person/test/test_divergence.py
index c1ddcd45bf60..4eedd00638f0 100644
--- a/posthog/models/person/test/test_divergence.py
+++ b/posthog/models/person/test/test_divergence.py
@@ -6,7 +6,7 @@ from uuid import UUID, uuid4
from posthog.test.base import BaseTest, ClickhouseTestMixin
from unittest.mock import MagicMock, patch
-from django.test import SimpleTestCase
+from django.test import SimpleTestCase, override_settings
from django.utils.timezone import now
import grpc
@@ -502,6 +502,58 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
assert self._ch_person(person.uuid) == (1, 103, CH_PROPERTIES)
assert self._ch_mapping("hidden-did") == (str(person.uuid), 1, 100)
+ @override_settings(PERSONHOG_WRITE_PATH="leader")
+ def test_raises_through_the_leader_and_publishes_the_person_it_wrote(self) -> None:
+ person = self._pg_person(version=3, distinct_ids={"hidden-did": 0})
+ self._ch_person_row(person.uuid, 103, deleted=True)
+ self._ch_mapping_row("hidden-did", person.uuid, 100, deleted=True)
+
+ _, actions = self._repair(person.uuid)
+
+ fake = get_active_fake()
+ fake.assert_not_called("raise_person_version_floors")
+ assert [c.request for c in fake.calls if c.method == "raise_person_version"] == [
+ person_pb2.RaisePersonVersionRequest(team_id=self.team.pk, person_id=person.pk, min_version=104)
+ ]
+ assert [a.outcome for a in actions] == ["repaired", "repaired"]
+ assert self._ch_person(person.uuid) == (0, 104, PG_PROPERTIES)
+ assert self._ch_mapping("hidden-did") == (str(person.uuid), 0, 101)
+
+ @override_settings(PERSONHOG_WRITE_PATH="leader")
+ def test_skips_a_person_deleted_before_the_leader_raise(self) -> None:
+ person = self._pg_person(version=3, distinct_ids={"hidden-did": 0})
+ self._ch_person_row(person.uuid, 103, deleted=True)
+ self._ch_mapping_row("hidden-did", person.uuid, 100, deleted=True)
+ fake = get_active_fake()
+ raise_on_leader = fake.raise_person_version
+
+ def deleted_then_raise(
+ request: person_pb2.RaisePersonVersionRequest, timeout: float | None = None
+ ) -> person_pb2.RaisePersonVersionResponse:
+ tombstone_persons_in_postgres(self.team.pk, [person.uuid])
+ return raise_on_leader(request, timeout)
+
+ with patch.object(fake, "raise_person_version", side_effect=deleted_then_raise):
+ _, actions = self._repair(person.uuid)
+
+ assert [a.outcome for a in actions] == ["skipped_tombstoned", "skipped_tombstoned"]
+ assert self._ch_person(person.uuid) == (1, 103, CH_PROPERTIES)
+
+ @override_settings(PERSONHOG_WRITE_PATH="leader")
+ def test_skips_a_person_the_leader_refuses_as_too_large(self) -> None:
+ person = self._pg_person(version=3, distinct_ids={"hidden-did": 0})
+ self._ch_person_row(person.uuid, 103, deleted=True)
+ self._ch_mapping_row("hidden-did", person.uuid, 100, deleted=True)
+ refusal = rpc_error(grpc.StatusCode.FAILED_PRECONDITION)
+ refusal.trailing_metadata = MagicMock(return_value=(("x-semantic-refusal", "person-too-large"),))
+ fake = get_active_fake()
+
+ with patch.object(fake, "raise_person_version", side_effect=refusal):
+ _, actions = self._repair(person.uuid)
+
+ assert [a.outcome for a in actions] == ["skipped_raise_refused", "skipped_raise_refused"]
+ fake.assert_not_called("write_distinct_ids")
+
@parameterized.expand([("dry_run", False, []), ("apply", True, [1.0])])
def test_throttles_on_each_divergent_row_written(self, _name: str, apply: bool, sleeps: list[float]) -> None:
person = self._pg_person(version=3, distinct_ids={"throttled-did": 0})
diff --git a/posthog/models/person/test/test_divergence_verbs.py b/posthog/models/person/test/test_divergence_verbs.py
index a36a6cb9f50b..7dfbccbf0aa8 100644
--- a/posthog/models/person/test/test_divergence_verbs.py
+++ b/posthog/models/person/test/test_divergence_verbs.py
@@ -183,7 +183,7 @@ class TestRepairVerbs(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
@parameterized.expand(
[
- ("republish_after_person_raise", "republish", "repairs._raise_person_version_floor"),
+ ("republish_after_person_raise", "republish", "repairs.raise_person_version"),
("republish_after_mapping_raise", "republish", "repairs._raise_mapping_version_floors"),
("republish_after_person_publish", "republish", "repairs._publish_person"),
("republish_after_mapping_publish", "republish", "repairs.create_person_distinct_id"),
diff --git a/posthog/models/person/util.py b/posthog/models/person/util.py
index 202f404379f3..9e47ab183d09 100644
--- a/posthog/models/person/util.py
+++ b/posthog/models/person/util.py
@@ -36,7 +36,7 @@ from posthog.models.person.sql import (
from posthog.models.utils import UUIDT
from posthog.personhog_client.client import personhog_call, require_personhog_client
from posthog.personhog_client.converters import proto_person_to_model
-from posthog.personhog_client.interceptor import is_transient_rpc_error
+from posthog.personhog_client.interceptor import is_transient_rpc_error, semantic_refusal_reason
from posthog.personhog_client.metrics import PERSONHOG_TEAM_MISMATCH_TOTAL, get_client_name
from posthog.personhog_client.proto import (
CONSISTENCY_LEVEL_STRONG,
@@ -59,6 +59,8 @@ from posthog.personhog_client.proto import (
ListPersonTombstoneQueueRequest,
PersonListEntry,
PersonVersionFloor as PersonVersionFloorProto,
+ RaisePersonVersionFloorsRequest,
+ RaisePersonVersionRequest,
ReadOptions,
SetPersonDistinctIdVersionFloorRequest,
VersionFloorOutcome as VersionFloorOutcomeProto,
@@ -1061,6 +1063,41 @@ def ensure_person_version_floors(team_id: int, floors: Sequence[PersonVersionFlo
return personhog_call("ensure_person_version_floors", personhog_fn)
+class PersonRaiseRefused(Exception):
+ """The leader refused the raise because the person's protected properties alone exceed the size limit."""
+
+
+def raise_person_version(team_id: int, person: Person, min_version: int) -> Person | None:
+ """Raise a live person's version to at least ``min_version`` on the active write path.
+
+ Returns the person as written when the path reports it. Otherwise returns None, and the caller reads the person.
+ """
+ if settings.PERSONHOG_WRITE_PATH == "leader":
+ return personhog_call(
+ "raise_person_version", partial(_raise_person_version_on_leader, team_id, person, min_version)
+ )
+ # Claims the person's lifecycle mark, so the raise waits out an in-flight merge or delete of the person.
+ request = RaisePersonVersionFloorsRequest(
+ team_id=team_id, floors=[PersonVersionFloorProto(person_uuid=str(person.uuid), min_version=min_version)]
+ )
+ personhog_call("raise_person_version_floors", lambda: _get_client().raise_person_version_floors(request))
+ return None
+
+
+def _raise_person_version_on_leader(team_id: int, person: Person, min_version: int) -> Person | None:
+ request = RaisePersonVersionRequest(team_id=team_id, person_id=person.pk, min_version=min_version)
+ try:
+ response = _get_client().raise_person_version(request)
+ except grpc.RpcError as exc:
+ # The leader holds the person as deleted.
+ if callable(getattr(exc, "code", None)) and exc.code() == grpc.StatusCode.NOT_FOUND:
+ return None
+ if semantic_refusal_reason(exc) == "person-too-large":
+ raise PersonRaiseRefused(f"person {person.uuid} is too large to raise") from exc
+ raise
+ return proto_person_to_model(response.person)
+
+
def set_distinct_id_version_floor(team_id: int, distinct_id: str, min_version: int) -> None:
"""Raise the Postgres version of the distinct id's row to at least ``min_version``, live or tombstoned.
diff --git a/posthog/personhog_client/client.py b/posthog/personhog_client/client.py
index c837681c82c9..6df14f45f0e2 100644
--- a/posthog/personhog_client/client.py
+++ b/posthog/personhog_client/client.py
@@ -95,6 +95,8 @@ from posthog.personhog_client.proto import (
PersonsResponse,
RaisePersonVersionFloorsRequest,
RaisePersonVersionFloorsResponse,
+ RaisePersonVersionRequest,
+ RaisePersonVersionResponse,
RestorePersonsRequest,
RestorePersonsResponse,
SetPersonDistinctIdVersionFloorRequest,
@@ -269,6 +271,7 @@ class PersonHogClient:
self._lifecycle_channel: grpc.Channel | None = None
self._lifecycle_state_monitor: _ChannelStateMonitor | None = None
lifecycle_stub: PersonHogLifecycleStub | None = None
+ self._leader_stub: PersonHogServiceStub | None = None
if lifecycle_addr:
lifecycle_channel = grpc.insecure_channel(lifecycle_addr, options=options)
self._lifecycle_channel = grpc.intercept_channel(
@@ -280,6 +283,8 @@ class PersonHogClient:
# The channel-state gauge is keyed by client_name alone; two channels under one name would fight over it.
self._lifecycle_state_monitor = _ChannelStateMonitor(lifecycle_channel, f"{client_name}-lifecycle")
lifecycle_stub = PersonHogLifecycleStub(self._lifecycle_channel)
+ # The same router reaches the leader, which the facade router cannot.
+ self._leader_stub = PersonHogServiceStub(self._lifecycle_channel)
self.lifecycle = PersonHogLifecycleClient(lifecycle_stub, lifecycle_timeout_ms / 1000.0)
def close(self) -> None:
@@ -359,6 +364,18 @@ class PersonHogClient:
) -> RaisePersonVersionFloorsResponse:
return self._stub.RaisePersonVersionFloors(request, timeout=timeout or self._timeout)
+ def raise_person_version(
+ self, request: RaisePersonVersionRequest, timeout: float | None = None
+ ) -> RaisePersonVersionResponse:
+ if self._leader_stub is None:
+ raise PersonHogLifecycleUnavailable("personhog leader path not configured (PERSONHOG_LIFECYCLE_ADDR)")
+ # The router routes leader calls by these headers, never by the request body.
+ return self._leader_stub.RaisePersonVersion(
+ request,
+ timeout=timeout or self._timeout,
+ metadata=[("x-team-id", str(request.team_id)), ("x-person-id", str(request.person_id))],
+ )
+
def restore_persons(self, request: RestorePersonsRequest, timeout: float | None = None) -> RestorePersonsResponse:
return self._stub.RestorePersons(request, timeout=timeout or self._timeout)
diff --git a/posthog/personhog_client/fake_client.py b/posthog/personhog_client/fake_client.py
index 509103c305d5..70bf43ddb0eb 100644
--- a/posthog/personhog_client/fake_client.py
+++ b/posthog/personhog_client/fake_client.py
@@ -1201,6 +1201,25 @@ class FakePersonHogClient:
response.results.add(person_uuid=floor.person_uuid, outcome=outcome, version=version)
return response
+ def raise_person_version(
+ self, request: person_pb2.RaisePersonVersionRequest, timeout: float | None = None
+ ) -> person_pb2.RaisePersonVersionResponse:
+ self.calls.append(_Call("raise_person_version", request))
+ person = self._live(self._persons_by_id.get((request.team_id, request.person_id)))
+ if person is None:
+ import grpc
+
+ error = grpc.RpcError()
+ error.code = lambda: grpc.StatusCode.NOT_FOUND
+ raise error
+ # The stored row stays as it is, because Postgres trails the leader until its writer applies the raise.
+ written = person_pb2.Person()
+ written.CopyFrom(person)
+ raised = person.version < request.min_version
+ if raised:
+ written.version = request.min_version
+ return person_pb2.RaisePersonVersionResponse(person=written, raised=raised)
+
def restore_persons(
self, request: person_pb2.RestorePersonsRequest, timeout: float | None = None
) -> person_pb2.RestorePersonsResponse:
diff --git a/posthog/personhog_client/proto/__init__.py b/posthog/personhog_client/proto/__init__.py
index 8c03166fbd28..e172c520b319 100644
--- a/posthog/personhog_client/proto/__init__.py
+++ b/posthog/personhog_client/proto/__init__.py
@@ -130,6 +130,8 @@ from personhog.types.v1.person_pb2 import (
ListPersonsForTeamResponse,
RaisePersonVersionFloorsRequest,
RaisePersonVersionFloorsResponse,
+ RaisePersonVersionRequest,
+ RaisePersonVersionResponse,
RestorePersonsRequest,
RestorePersonsResponse,
RestoredDistinctId,
diff --git a/posthog/personhog_client/test_leader_routing.py b/posthog/personhog_client/test_leader_routing.py
new file mode 100644
index 000000000000..6089be57603b
--- /dev/null
+++ b/posthog/personhog_client/test_leader_routing.py
@@ -0,0 +1,18 @@
+from unittest.mock import MagicMock
+
+from django.test import SimpleTestCase
+
+from posthog.personhog_client.client import PersonHogClient
+from posthog.personhog_client.proto import RaisePersonVersionRequest
+
+
+class TestLeaderRouting(SimpleTestCase):
+ def test_raise_person_version_goes_to_the_leader_router_with_its_routing_key(self) -> None:
+ client = PersonHogClient("localhost:1", lifecycle_addr="localhost:2")
+ self.addCleanup(client.close)
+ stub = MagicMock()
+ client._leader_stub = stub
+
+ client.raise_person_version(RaisePersonVersionRequest(team_id=7, person_id=42, min_version=5))
+
+ assert stub.RaisePersonVersion.call_args.kwargs["metadata"] == [("x-team-id", "7"), ("x-person-id", "42")]
diff --git a/posthog/settings/personhog.py b/posthog/settings/personhog.py
index 8672900ee2aa..be48b049642e 100644
--- a/posthog/settings/personhog.py
+++ b/posthog/settings/personhog.py
@@ -19,6 +19,11 @@ PERSONHOG_MAX_SEND_MESSAGE_LENGTH = get_from_env("PERSONHOG_MAX_SEND_MESSAGE_LEN
PERSONHOG_MAX_RECV_MESSAGE_LENGTH = get_from_env("PERSONHOG_MAX_RECV_MESSAGE_LENGTH", 128 * 1024 * 1024, type_cast=int)
PERSONHOG_CLIENT_IDLE_TIMEOUT_MS = get_from_env("PERSONHOG_CLIENT_IDLE_TIMEOUT_MS", 0, type_cast=int)
+# Where person writes go: "replica" writes Postgres directly, "leader" goes through the personhog leader.
+PERSONHOG_WRITE_PATH = get_from_env("PERSONHOG_WRITE_PATH", "replica")
+if PERSONHOG_WRITE_PATH not in ("replica", "leader"):
+ raise ValueError(f"PERSONHOG_WRITE_PATH must be 'replica' or 'leader', got {PERSONHOG_WRITE_PATH!r}")
+
# Retry settings for transient gRPC errors
PERSONHOG_MAX_RETRIES = get_from_env("PERSONHOG_MAX_RETRIES", 2, type_cast=int)
PERSONHOG_INITIAL_BACKOFF_MS = get_from_env("PERSONHOG_INITIAL_BACKOFF_MS", 50, type_cast=int)