:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(persons): keep restore requests under the replica distinct id cap (#123747)
Eli Reismanfd0bfde00cf3
diff --git a/posthog/models/person/divergence/restore_verbs.py b/posthog/models/person/divergence/restore_verbs.pyindex bbfedc943bec..3dad7dad270c 100644--- a/posthog/models/person/divergence/restore_verbs.py+++ b/posthog/models/person/divergence/restore_verbs.py@@ -6,7 +6,7 @@ from __future__ import annotationsimport jsonimport timefrom collections import defaultdict-from collections.abc import Sequence+from collections.abc import Iterator, Sequencefrom datetime import UTC, datetimefrom functools import partialfrom typing import TYPE_CHECKING, Any@@ -47,6 +47,8 @@ if TYPE_CHECKING:from .verbs import VerbContext_LIVE = "is_deleted = 0 AND _row_exists"+# The replica rejects a RestorePersons request that carries more distinct ids than this, across all its persons.+RESTORE_MAX_DISTINCT_IDS = 1_000_PAYLOAD_SQL = f"""SELECTtoString(id),@@ -153,12 +155,17 @@ def _write_mappings(context: VerbContext, rows: list[dict[str, Any]], targets: d)continuecontext.pacer.before_write()- response = personhog_call(- "person_consistency_write_distinct_ids",+ response = repairs._retry_transient(partial(- require_personhog_client().write_distinct_ids,- WriteDistinctIdsRequest(team_id=context.team_id, writes=writes),+ personhog_call,+ "person_consistency_write_distinct_ids",+ partial(+ require_personhog_client().write_distinct_ids,+ WriteDistinctIdsRequest(team_id=context.team_id, writes=writes),+ ),),+ what="write distinct ids",+ log=context.log,)wanted = {w.distinct_id: w for w in writes}for result in response.results:@@ -201,6 +208,23 @@ def restore(rows: list[dict[str, Any]], context: VerbContext) -> None:_write_mappings(context, rest, {r["key"]: r["evidence"]["ch"]["winner_person_uuid"] for r in rest}, "restored")+def _batches_by_distinct_id_count(+ person_uuids: list[str], id_rows: dict[str, list[dict[str, Any]]]+) -> Iterator[list[str]]:+ """Group persons so one RestorePersons request carries at most RESTORE_MAX_DISTINCT_IDS distinct ids."""+ batch: list[str] = []+ count = 0+ for person_uuid in person_uuids:+ width = len(id_rows[person_uuid])+ if batch and count + width > RESTORE_MAX_DISTINCT_IDS:+ yield batch+ batch, count = [], 0+ batch.append(person_uuid)+ count += width+ if batch:+ yield batch++@frozenclass _RestoreOutcome:restored: dict[str, list[str]]@@ -234,6 +258,8 @@ def _restore_persons(elif not ids_by_owner.get(person_uuid):# Restored alone, the person would have no distinct id, so nothing could reach it.context.action(row, "skipped_no_distinct_id")+ elif len(ids_by_owner[person_uuid]) > RESTORE_MAX_DISTINCT_IDS:+ context.action(row, "skipped_too_many_distinct_ids")else:candidates.append(person_uuid)if not candidates:@@ -247,60 +273,70 @@ def _restore_persons(continuech = observe.ch_mapping_states(context.team_id, [r["key"] for u in candidates for r in id_rows[u]])versions = {key: (state.max_version + 1) for key, state in ch.items()}- context.pacer.before_write()- request = RestorePersonsRequest(- team_id=context.team_id,- persons=[- RestoredPerson(- person_uuid=u,- properties=json.dumps(payloads[u].properties).encode(),- created_at_ms=int(payloads[u].created_at.timestamp() * 1000),- is_identified=payloads[u].is_identified,- version=payloads[u].max_version + 1,- distinct_ids=[- RestoredDistinctId(distinct_id=r["key"], version=versions.get(r["key"], 0)) for r in id_rows[u]- ],- )- for u in candidates- ],- )- response = personhog_call(- "person_consistency_restore_persons", partial(require_personhog_client().restore_persons, request)- )- for result in response.results:- row = by_uuid[result.person_uuid]- ids = list(result.restored_distinct_ids)- if result.outcome == RESTORE_OUTCOME_INSERTED:- # A person whose publish does not land is recorded undelivered; it then exists in Postgres, so the- # next scan reports it as behind ClickHouse, and republish publishes it.- context.deliveries.track(- _publish_person(- context.team_id, result.person_uuid, payloads[result.person_uuid], int(result.version)- ),- repairs.PublishedKey(kind="person", key=result.person_uuid),- )- for distinct_id in ids:+ for request_uuids in _batches_by_distinct_id_count(candidates, id_rows):+ context.pacer.before_write()+ request = RestorePersonsRequest(+ team_id=context.team_id,+ persons=[+ RestoredPerson(+ person_uuid=u,+ properties=json.dumps(payloads[u].properties).encode(),+ created_at_ms=int(payloads[u].created_at.timestamp() * 1000),+ is_identified=payloads[u].is_identified,+ version=payloads[u].max_version + 1,+ distinct_ids=[+ RestoredDistinctId(distinct_id=r["key"], version=versions.get(r["key"], 0))+ for r in id_rows[u]+ ],+ )+ for u in request_uuids+ ],+ )+ # A retry after a lost response finds these persons live and reports them skipped, not restored.+ # They then sit one version above ClickHouse, so the next scan sends them to republish.+ response = repairs._retry_transient(+ partial(+ personhog_call,+ "person_consistency_restore_persons",+ partial(require_personhog_client().restore_persons, request),+ ),+ what="restore persons",+ log=context.log,+ )+ for result in response.results:+ row = by_uuid[result.person_uuid]+ ids = list(result.restored_distinct_ids)+ if result.outcome == RESTORE_OUTCOME_INSERTED:+ # A person whose publish does not land is recorded undelivered; it then exists in Postgres, so the+ # next scan reports it as behind ClickHouse, and republish publishes it.context.deliveries.track(- create_person_distinct_id(- context.team_id, distinct_id, result.person_uuid, version=versions.get(distinct_id, 0)+ _publish_person(+ context.team_id, result.person_uuid, payloads[result.person_uuid], int(result.version)),- repairs.PublishedKey(kind="mapping", key=distinct_id),- )- restored[result.person_uuid] = ids- context.action(row, "restored", version=int(result.version))- elif result.outcome == RESTORE_OUTCOME_NO_DISTINCT_ID:- context.action(row, "skipped_no_distinct_id")- else:- live = result.outcome == RESTORE_OUTCOME_EXISTS_LIVE- context.action(row, "skipped_exists_live" if live else "skipped_exists_tombstoned")- for id_row in id_rows[result.person_uuid]:- if id_row["key"] in ids:- context.action(- id_row, "restored", target_person=result.person_uuid, version=versions.get(id_row["key"], 0)+ repairs.PublishedKey(kind="person", key=result.person_uuid),)+ for distinct_id in ids:+ context.deliveries.track(+ create_person_distinct_id(+ context.team_id, distinct_id, result.person_uuid, version=versions.get(distinct_id, 0)+ ),+ repairs.PublishedKey(kind="mapping", key=distinct_id),+ )+ restored[result.person_uuid] = ids+ context.action(row, "restored", version=int(result.version))+ elif result.outcome == RESTORE_OUTCOME_NO_DISTINCT_ID:+ context.action(row, "skipped_no_distinct_id")else:- # Claimed since the scan, or its person was not inserted: the next scan names its new state.- context.action(id_row, "skipped_changed", target_person=result.person_uuid)+ live = result.outcome == RESTORE_OUTCOME_EXISTS_LIVE+ context.action(row, "skipped_exists_live" if live else "skipped_exists_tombstoned")+ for id_row in id_rows[result.person_uuid]:+ if id_row["key"] in ids:+ context.action(+ id_row, "restored", target_person=result.person_uuid, version=versions.get(id_row["key"], 0)+ )+ else:+ # Claimed since the scan, or its person was not inserted: the next scan names its new state.+ context.action(id_row, "skipped_changed", target_person=result.person_uuid)return _RestoreOutcome(restored=restored, handled_ids=frozenset(handled))diff --git a/posthog/models/person/test/divergence_fixtures.py b/posthog/models/person/test/divergence_fixtures.pyindex 80dc9ef73074..c2d8e80d590e 100644--- a/posthog/models/person/test/divergence_fixtures.py+++ b/posthog/models/person/test/divergence_fixtures.py@@ -1,7 +1,11 @@from typing import TYPE_CHECKING+from unittest.mock import MagicMock+from django.utils.timezone import now+import grpc+from posthog.clickhouse.client import sync_executefrom posthog.models.person.sql import BULK_INSERT_PERSON_DISTINCT_ID2from posthog.models.person.util import create_person as create_person_in_ch@@ -12,6 +16,12 @@ if TYPE_CHECKING:from posthog.models.team import Team+def rpc_error(code: grpc.StatusCode) -> grpc.RpcError:+ error = grpc.RpcError()+ error.code = MagicMock(return_value=code)+ return error++class DivergenceFixturesMixin:"""Writes Postgres (personhog fake) and ClickHouse person state independently, so a test can build anydisagreement between them. Needs ClickhouseTestMixin and BaseTest."""diff --git a/posthog/models/person/test/test_divergence.py b/posthog/models/person/test/test_divergence.pyindex dee1221217bd..c1ddcd45bf60 100644--- a/posthog/models/person/test/test_divergence.py+++ b/posthog/models/person/test/test_divergence.py@@ -44,6 +44,7 @@ from posthog.models.person.divergence.fleet_scans import (from posthog.models.person.divergence.repairs import _WritePacer, repair_distinct_id, repair_personsfrom posthog.models.person.divergence.team_check import _team_check_modulus, check_teamfrom posthog.models.person.sql import BULK_INSERT_PERSON_DISTINCT_ID2+from posthog.models.person.test.divergence_fixtures import rpc_errorfrom posthog.models.person.util import (create_person as create_person_in_ch,tombstone_persons_in_postgres,@@ -64,12 +65,6 @@ _PERSON_COLUMNS = ()-def _rpc_error(code: grpc.StatusCode) -> grpc.RpcError:- error = grpc.RpcError()- error.code = MagicMock(return_value=code)- return error--def _utc_naive(hours_ago: float) -> datetime:return (now() - timedelta(hours=hours_ago)).astimezone(UTC).replace(tzinfo=None)@@ -619,7 +614,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):self, _name: str, step: str, error: Exception | grpc.StatusCode) -> None:person = self._divergent_person("hidden")- failure = _rpc_error(error) if isinstance(error, grpc.StatusCode) else error+ failure = rpc_error(error) if isinstance(error, grpc.StatusCode) else errorwith self._failing_once(step, failure), patch("posthog.models.person.divergence.repairs._sleep") as sleep:summary, actions = self._repair(person.uuid)@@ -632,7 +627,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):@parameterized.expand([("non_transient_error_is_not_retried", ValueError("bad request"), 1, []),- ("transient_error_on_every_attempt_stops_after_three", _rpc_error(grpc.StatusCode.INTERNAL), 3, [1.0, 2.0]),+ ("transient_error_on_every_attempt_stops_after_three", rpc_error(grpc.StatusCode.INTERNAL), 3, [1.0, 2.0]),])def test_stops_the_run_on_an_error_that_does_not_clear(diff --git a/posthog/models/person/test/test_divergence_restore_verbs.py b/posthog/models/person/test/test_divergence_restore_verbs.pyindex 2f4e84a2f97b..13bed3c13e4c 100644--- a/posthog/models/person/test/test_divergence_restore_verbs.py+++ b/posthog/models/person/test/test_divergence_restore_verbs.py@@ -10,11 +10,13 @@ import pytestfrom posthog.test.base import BaseTest, ClickhouseTestMixinfrom unittest.mock import patch+import grpcfrom parameterized import parameterized-from posthog.models.person.divergence import observe, restore_verbs, team_scan, verbs-from posthog.models.person.test.divergence_fixtures import DivergenceFixturesMixin+from posthog.models.person.divergence import observe, repairs, restore_verbs, team_scan, verbs+from posthog.models.person.test.divergence_fixtures import DivergenceFixturesMixin, rpc_errorfrom posthog.models.person.util import get_distinct_id_rows, tombstone_persons_in_postgres+from posthog.personhog_client import fake_clientfrom posthog.personhog_client.fake_client import get_active_fake_CAPTURE = "posthog.models.person.divergence.restore_verbs.capture_internal"@@ -85,6 +87,62 @@ class TestRestoreVerbs(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):assert (stored.version, json.loads(stored.properties)) == (5, {"email": "[email protected]"})assert self._scan(purged_at=purged_at)[1] == {}+ def test_restore_splits_requests_at_the_distinct_id_cap_and_skips_a_person_over_it(self):+ widths = {str(uuid4()): width for width in (2, 1, 2, 4)}+ for person_uuid, width in widths.items():+ self._ch_person(person_uuid, 4, {"email": "[email protected]"})+ for i in range(width):+ self._ch_mapping(f"{person_uuid}-{i}", person_uuid, 0)+ purged_at = datetime.now(UTC) + timedelta(hours=1)+ scan_dir, before = self._scan(purged_at=purged_at)+ over_cap = list(widths)[-1]+ over_cap_ids = {f"{over_cap}-{i}" for i in range(4)}++ with (+ patch.object(restore_verbs, "RESTORE_MAX_DISTINCT_IDS", 3),+ patch.object(fake_client, "RESTORE_MAX_DISTINCT_IDS", 3),+ ):+ outcomes = self._run(scan_dir, "restore", apply=True)++ fake = get_active_fake()+ assert fake is not None+ sent = [c.request for c in fake.calls if c.method == "restore_persons"]+ assert sorted(sum(len(p.distinct_ids) for p in request.persons) for request in sent) == [2, 3]+ assert outcomes == {+ **dict.fromkeys(set(before) - over_cap_ids - {over_cap}, "restored"),+ over_cap: "skipped_too_many_distinct_ids",+ **dict.fromkeys(over_cap_ids, "skipped_person_not_live"),+ }+ assert fake.stored_person(self.team.pk, over_cap) is None++ @parameterized.expand([("restore_persons",), ("write_distinct_ids",)])+ def test_restore_retries_a_personhog_write_that_failed_transiently(self, method: str):+ if method == "restore_persons":+ lost = str(uuid4())+ self._ch_person(lost, 4, {"email": "[email protected]"})+ self._ch_mapping("lost-id", lost, 2)+ else:+ owner = self._live_person("own")+ self._ch_mapping("lost-id", str(owner.uuid), 1)+ purged_at = datetime.now(UTC) + timedelta(hours=1)+ scan_dir, before = self._scan(purged_at=purged_at)+ fake = get_active_fake()+ assert fake is not None+ write = getattr(fake, method)+ failures = [rpc_error(grpc.StatusCode.UNAVAILABLE)]++ def fails_once(*args: Any, **kwargs: Any) -> Any:+ if failures:+ raise failures.pop()+ return write(*args, **kwargs)++ with patch.object(fake, method, side_effect=fails_once) as patched, patch.object(repairs, "_sleep"):+ outcomes = self._run(scan_dir, "restore", apply=True)++ assert patched.call_count == 2+ assert outcomes == dict.fromkeys(before, "restored")+ assert self._scan(purged_at=purged_at)[1] == {}+def _lost_person_split_to(self, survivor_ids: list[str], *, identified: bool, ghost_ids: list[str]) -> tuple[str, Any]:diff --git a/posthog/personhog_client/fake_client.py b/posthog/personhog_client/fake_client.pyindex f9abbf26326a..b5a8e148658c 100644--- a/posthog/personhog_client/fake_client.py+++ b/posthog/personhog_client/fake_client.py@@ -53,6 +53,9 @@ DELETE_TOMBSTONED_DEFAULT_ROWS = 1000# The replica's cap on keys per version floor request.VERSION_RPC_MAX_KEYS = 250+# The replica's cap on distinct ids across all persons in one restore request.+RESTORE_MAX_DISTINCT_IDS = 1000+class FakePersonHogClient:"""In-memory fake that implements the same interface as PersonHogClient.@@ -1084,6 +1087,8 @@ class FakePersonHogClient:if not all(isinstance(props, dict) for props in parsed_properties):raise ValueError("properties must be a JSON object")listed = [d for p in request.persons for d in p.distinct_ids]+ if len(listed) > RESTORE_MAX_DISTINCT_IDS:+ raise ValueError(f"At most {RESTORE_MAX_DISTINCT_IDS} distinct ids per restore, got {len(listed)}")if len({d.distinct_id for d in listed}) != len(listed) or any(not d.distinct_id or d.version < 0 for d in listed):