Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(persons): keep restore requests under the replica distinct id cap (#123747)

Eli Reismanfd0bfde00cf3
diff --git a/posthog/models/person/divergence/restore_verbs.py b/posthog/models/person/divergence/restore_verbs.py
index bbfedc943bec..3dad7dad270c 100644
--- a/posthog/models/person/divergence/restore_verbs.py
+++ b/posthog/models/person/divergence/restore_verbs.py
@@ -6,7 +6,7 @@ from __future__ import annotations
import json
import time
from collections import defaultdict
-from collections.abc import Sequence
+from collections.abc import Iterator, Sequence
from datetime import UTC, datetime
from functools import partial
from typing import TYPE_CHECKING, Any
@@ -47,6 +47,8 @@ if TYPE_CHECKING:
from .verbs import VerbContext
_LIVE = "is_deleted = 0 AND _row_exists"
+# The replica rejects a RestorePersons request that carries more distinct ids than this, across all its persons.
+RESTORE_MAX_DISTINCT_IDS = 1_000
_PAYLOAD_SQL = f"""
SELECT
toString(id),
@@ -153,12 +155,17 @@ def _write_mappings(context: VerbContext, rows: list[dict[str, Any]], targets: d
)
continue
context.pacer.before_write()
- response = personhog_call(
- "person_consistency_write_distinct_ids",
+ response = repairs._retry_transient(
partial(
- require_personhog_client().write_distinct_ids,
- WriteDistinctIdsRequest(team_id=context.team_id, writes=writes),
+ personhog_call,
+ "person_consistency_write_distinct_ids",
+ partial(
+ require_personhog_client().write_distinct_ids,
+ WriteDistinctIdsRequest(team_id=context.team_id, writes=writes),
+ ),
),
+ what="write distinct ids",
+ log=context.log,
)
wanted = {w.distinct_id: w for w in writes}
for result in response.results:
@@ -201,6 +208,23 @@ def restore(rows: list[dict[str, Any]], context: VerbContext) -> None:
_write_mappings(context, rest, {r["key"]: r["evidence"]["ch"]["winner_person_uuid"] for r in rest}, "restored")
+def _batches_by_distinct_id_count(
+ person_uuids: list[str], id_rows: dict[str, list[dict[str, Any]]]
+) -> Iterator[list[str]]:
+ """Group persons so one RestorePersons request carries at most RESTORE_MAX_DISTINCT_IDS distinct ids."""
+ batch: list[str] = []
+ count = 0
+ for person_uuid in person_uuids:
+ width = len(id_rows[person_uuid])
+ if batch and count + width > RESTORE_MAX_DISTINCT_IDS:
+ yield batch
+ batch, count = [], 0
+ batch.append(person_uuid)
+ count += width
+ if batch:
+ yield batch
+
+
@frozen
class _RestoreOutcome:
restored: dict[str, list[str]]
@@ -234,6 +258,8 @@ def _restore_persons(
elif not ids_by_owner.get(person_uuid):
# Restored alone, the person would have no distinct id, so nothing could reach it.
context.action(row, "skipped_no_distinct_id")
+ elif len(ids_by_owner[person_uuid]) > RESTORE_MAX_DISTINCT_IDS:
+ context.action(row, "skipped_too_many_distinct_ids")
else:
candidates.append(person_uuid)
if not candidates:
@@ -247,60 +273,70 @@ def _restore_persons(
continue
ch = observe.ch_mapping_states(context.team_id, [r["key"] for u in candidates for r in id_rows[u]])
versions = {key: (state.max_version + 1) for key, state in ch.items()}
- context.pacer.before_write()
- request = RestorePersonsRequest(
- team_id=context.team_id,
- persons=[
- RestoredPerson(
- person_uuid=u,
- properties=json.dumps(payloads[u].properties).encode(),
- created_at_ms=int(payloads[u].created_at.timestamp() * 1000),
- is_identified=payloads[u].is_identified,
- version=payloads[u].max_version + 1,
- distinct_ids=[
- RestoredDistinctId(distinct_id=r["key"], version=versions.get(r["key"], 0)) for r in id_rows[u]
- ],
- )
- for u in candidates
- ],
- )
- response = personhog_call(
- "person_consistency_restore_persons", partial(require_personhog_client().restore_persons, request)
- )
- for result in response.results:
- row = by_uuid[result.person_uuid]
- ids = list(result.restored_distinct_ids)
- if result.outcome == RESTORE_OUTCOME_INSERTED:
- # A person whose publish does not land is recorded undelivered; it then exists in Postgres, so the
- # next scan reports it as behind ClickHouse, and republish publishes it.
- context.deliveries.track(
- _publish_person(
- context.team_id, result.person_uuid, payloads[result.person_uuid], int(result.version)
- ),
- repairs.PublishedKey(kind="person", key=result.person_uuid),
- )
- for distinct_id in ids:
+ for request_uuids in _batches_by_distinct_id_count(candidates, id_rows):
+ context.pacer.before_write()
+ request = RestorePersonsRequest(
+ team_id=context.team_id,
+ persons=[
+ RestoredPerson(
+ person_uuid=u,
+ properties=json.dumps(payloads[u].properties).encode(),
+ created_at_ms=int(payloads[u].created_at.timestamp() * 1000),
+ is_identified=payloads[u].is_identified,
+ version=payloads[u].max_version + 1,
+ distinct_ids=[
+ RestoredDistinctId(distinct_id=r["key"], version=versions.get(r["key"], 0))
+ for r in id_rows[u]
+ ],
+ )
+ for u in request_uuids
+ ],
+ )
+ # A retry after a lost response finds these persons live and reports them skipped, not restored.
+ # They then sit one version above ClickHouse, so the next scan sends them to republish.
+ response = repairs._retry_transient(
+ partial(
+ personhog_call,
+ "person_consistency_restore_persons",
+ partial(require_personhog_client().restore_persons, request),
+ ),
+ what="restore persons",
+ log=context.log,
+ )
+ for result in response.results:
+ row = by_uuid[result.person_uuid]
+ ids = list(result.restored_distinct_ids)
+ if result.outcome == RESTORE_OUTCOME_INSERTED:
+ # A person whose publish does not land is recorded undelivered; it then exists in Postgres, so the
+ # next scan reports it as behind ClickHouse, and republish publishes it.
context.deliveries.track(
- create_person_distinct_id(
- context.team_id, distinct_id, result.person_uuid, version=versions.get(distinct_id, 0)
+ _publish_person(
+ context.team_id, result.person_uuid, payloads[result.person_uuid], int(result.version)
),
- repairs.PublishedKey(kind="mapping", key=distinct_id),
- )
- restored[result.person_uuid] = ids
- context.action(row, "restored", version=int(result.version))
- elif result.outcome == RESTORE_OUTCOME_NO_DISTINCT_ID:
- context.action(row, "skipped_no_distinct_id")
- else:
- live = result.outcome == RESTORE_OUTCOME_EXISTS_LIVE
- context.action(row, "skipped_exists_live" if live else "skipped_exists_tombstoned")
- for id_row in id_rows[result.person_uuid]:
- if id_row["key"] in ids:
- context.action(
- id_row, "restored", target_person=result.person_uuid, version=versions.get(id_row["key"], 0)
+ repairs.PublishedKey(kind="person", key=result.person_uuid),
)
+ for distinct_id in ids:
+ context.deliveries.track(
+ create_person_distinct_id(
+ context.team_id, distinct_id, result.person_uuid, version=versions.get(distinct_id, 0)
+ ),
+ repairs.PublishedKey(kind="mapping", key=distinct_id),
+ )
+ restored[result.person_uuid] = ids
+ context.action(row, "restored", version=int(result.version))
+ elif result.outcome == RESTORE_OUTCOME_NO_DISTINCT_ID:
+ context.action(row, "skipped_no_distinct_id")
else:
- # Claimed since the scan, or its person was not inserted: the next scan names its new state.
- context.action(id_row, "skipped_changed", target_person=result.person_uuid)
+ live = result.outcome == RESTORE_OUTCOME_EXISTS_LIVE
+ context.action(row, "skipped_exists_live" if live else "skipped_exists_tombstoned")
+ for id_row in id_rows[result.person_uuid]:
+ if id_row["key"] in ids:
+ context.action(
+ id_row, "restored", target_person=result.person_uuid, version=versions.get(id_row["key"], 0)
+ )
+ else:
+ # Claimed since the scan, or its person was not inserted: the next scan names its new state.
+ context.action(id_row, "skipped_changed", target_person=result.person_uuid)
return _RestoreOutcome(restored=restored, handled_ids=frozenset(handled))
diff --git a/posthog/models/person/test/divergence_fixtures.py b/posthog/models/person/test/divergence_fixtures.py
index 80dc9ef73074..c2d8e80d590e 100644
--- a/posthog/models/person/test/divergence_fixtures.py
+++ b/posthog/models/person/test/divergence_fixtures.py
@@ -1,7 +1,11 @@
from typing import TYPE_CHECKING
+from unittest.mock import MagicMock
+
from django.utils.timezone import now
+import grpc
+
from posthog.clickhouse.client import sync_execute
from posthog.models.person.sql import BULK_INSERT_PERSON_DISTINCT_ID2
from posthog.models.person.util import create_person as create_person_in_ch
@@ -12,6 +16,12 @@ if TYPE_CHECKING:
from posthog.models.team import Team
+def rpc_error(code: grpc.StatusCode) -> grpc.RpcError:
+ error = grpc.RpcError()
+ error.code = MagicMock(return_value=code)
+ return error
+
+
class DivergenceFixturesMixin:
"""Writes Postgres (personhog fake) and ClickHouse person state independently, so a test can build any
disagreement between them. Needs ClickhouseTestMixin and BaseTest."""
diff --git a/posthog/models/person/test/test_divergence.py b/posthog/models/person/test/test_divergence.py
index dee1221217bd..c1ddcd45bf60 100644
--- a/posthog/models/person/test/test_divergence.py
+++ b/posthog/models/person/test/test_divergence.py
@@ -44,6 +44,7 @@ from posthog.models.person.divergence.fleet_scans import (
from posthog.models.person.divergence.repairs import _WritePacer, repair_distinct_id, repair_persons
from posthog.models.person.divergence.team_check import _team_check_modulus, check_team
from posthog.models.person.sql import BULK_INSERT_PERSON_DISTINCT_ID2
+from posthog.models.person.test.divergence_fixtures import rpc_error
from posthog.models.person.util import (
create_person as create_person_in_ch,
tombstone_persons_in_postgres,
@@ -64,12 +65,6 @@ _PERSON_COLUMNS = (
)
-def _rpc_error(code: grpc.StatusCode) -> grpc.RpcError:
- error = grpc.RpcError()
- error.code = MagicMock(return_value=code)
- return error
-
-
def _utc_naive(hours_ago: float) -> datetime:
return (now() - timedelta(hours=hours_ago)).astimezone(UTC).replace(tzinfo=None)
@@ -619,7 +614,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
self, _name: str, step: str, error: Exception | grpc.StatusCode
) -> None:
person = self._divergent_person("hidden")
- failure = _rpc_error(error) if isinstance(error, grpc.StatusCode) else error
+ failure = rpc_error(error) if isinstance(error, grpc.StatusCode) else error
with self._failing_once(step, failure), patch("posthog.models.person.divergence.repairs._sleep") as sleep:
summary, actions = self._repair(person.uuid)
@@ -632,7 +627,7 @@ class TestPersonDivergence(ClickhouseTestMixin, BaseTest):
@parameterized.expand(
[
("non_transient_error_is_not_retried", ValueError("bad request"), 1, []),
- ("transient_error_on_every_attempt_stops_after_three", _rpc_error(grpc.StatusCode.INTERNAL), 3, [1.0, 2.0]),
+ ("transient_error_on_every_attempt_stops_after_three", rpc_error(grpc.StatusCode.INTERNAL), 3, [1.0, 2.0]),
]
)
def test_stops_the_run_on_an_error_that_does_not_clear(
diff --git a/posthog/models/person/test/test_divergence_restore_verbs.py b/posthog/models/person/test/test_divergence_restore_verbs.py
index 2f4e84a2f97b..13bed3c13e4c 100644
--- a/posthog/models/person/test/test_divergence_restore_verbs.py
+++ b/posthog/models/person/test/test_divergence_restore_verbs.py
@@ -10,11 +10,13 @@ import pytest
from posthog.test.base import BaseTest, ClickhouseTestMixin
from unittest.mock import patch
+import grpc
from parameterized import parameterized
-from posthog.models.person.divergence import observe, restore_verbs, team_scan, verbs
-from posthog.models.person.test.divergence_fixtures import DivergenceFixturesMixin
+from posthog.models.person.divergence import observe, repairs, restore_verbs, team_scan, verbs
+from posthog.models.person.test.divergence_fixtures import DivergenceFixturesMixin, rpc_error
from posthog.models.person.util import get_distinct_id_rows, tombstone_persons_in_postgres
+from posthog.personhog_client import fake_client
from posthog.personhog_client.fake_client import get_active_fake
_CAPTURE = "posthog.models.person.divergence.restore_verbs.capture_internal"
@@ -85,6 +87,62 @@ class TestRestoreVerbs(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):
assert (stored.version, json.loads(stored.properties)) == (5, {"email": "[email protected]"})
assert self._scan(purged_at=purged_at)[1] == {}
+ def test_restore_splits_requests_at_the_distinct_id_cap_and_skips_a_person_over_it(self):
+ widths = {str(uuid4()): width for width in (2, 1, 2, 4)}
+ for person_uuid, width in widths.items():
+ self._ch_person(person_uuid, 4, {"email": "[email protected]"})
+ for i in range(width):
+ self._ch_mapping(f"{person_uuid}-{i}", person_uuid, 0)
+ purged_at = datetime.now(UTC) + timedelta(hours=1)
+ scan_dir, before = self._scan(purged_at=purged_at)
+ over_cap = list(widths)[-1]
+ over_cap_ids = {f"{over_cap}-{i}" for i in range(4)}
+
+ with (
+ patch.object(restore_verbs, "RESTORE_MAX_DISTINCT_IDS", 3),
+ patch.object(fake_client, "RESTORE_MAX_DISTINCT_IDS", 3),
+ ):
+ outcomes = self._run(scan_dir, "restore", apply=True)
+
+ fake = get_active_fake()
+ assert fake is not None
+ sent = [c.request for c in fake.calls if c.method == "restore_persons"]
+ assert sorted(sum(len(p.distinct_ids) for p in request.persons) for request in sent) == [2, 3]
+ assert outcomes == {
+ **dict.fromkeys(set(before) - over_cap_ids - {over_cap}, "restored"),
+ over_cap: "skipped_too_many_distinct_ids",
+ **dict.fromkeys(over_cap_ids, "skipped_person_not_live"),
+ }
+ assert fake.stored_person(self.team.pk, over_cap) is None
+
+ @parameterized.expand([("restore_persons",), ("write_distinct_ids",)])
+ def test_restore_retries_a_personhog_write_that_failed_transiently(self, method: str):
+ if method == "restore_persons":
+ lost = str(uuid4())
+ self._ch_person(lost, 4, {"email": "[email protected]"})
+ self._ch_mapping("lost-id", lost, 2)
+ else:
+ owner = self._live_person("own")
+ self._ch_mapping("lost-id", str(owner.uuid), 1)
+ purged_at = datetime.now(UTC) + timedelta(hours=1)
+ scan_dir, before = self._scan(purged_at=purged_at)
+ fake = get_active_fake()
+ assert fake is not None
+ write = getattr(fake, method)
+ failures = [rpc_error(grpc.StatusCode.UNAVAILABLE)]
+
+ def fails_once(*args: Any, **kwargs: Any) -> Any:
+ if failures:
+ raise failures.pop()
+ return write(*args, **kwargs)
+
+ with patch.object(fake, method, side_effect=fails_once) as patched, patch.object(repairs, "_sleep"):
+ outcomes = self._run(scan_dir, "restore", apply=True)
+
+ assert patched.call_count == 2
+ assert outcomes == dict.fromkeys(before, "restored")
+ assert self._scan(purged_at=purged_at)[1] == {}
+
def _lost_person_split_to(
self, survivor_ids: list[str], *, identified: bool, ghost_ids: list[str]
) -> tuple[str, Any]:
diff --git a/posthog/personhog_client/fake_client.py b/posthog/personhog_client/fake_client.py
index f9abbf26326a..b5a8e148658c 100644
--- a/posthog/personhog_client/fake_client.py
+++ b/posthog/personhog_client/fake_client.py
@@ -53,6 +53,9 @@ DELETE_TOMBSTONED_DEFAULT_ROWS = 1000
# The replica's cap on keys per version floor request.
VERSION_RPC_MAX_KEYS = 250
+# The replica's cap on distinct ids across all persons in one restore request.
+RESTORE_MAX_DISTINCT_IDS = 1000
+
class FakePersonHogClient:
"""In-memory fake that implements the same interface as PersonHogClient.
@@ -1084,6 +1087,8 @@ class FakePersonHogClient:
if not all(isinstance(props, dict) for props in parsed_properties):
raise ValueError("properties must be a JSON object")
listed = [d for p in request.persons for d in p.distinct_ids]
+ if len(listed) > RESTORE_MAX_DISTINCT_IDS:
+ raise ValueError(f"At most {RESTORE_MAX_DISTINCT_IDS} distinct ids per restore, got {len(listed)}")
if len({d.distinct_id for d in listed}) != len(listed) or any(
not d.distinct_id or d.version < 0 for d in listed
):