:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(persons): add republish, merge_stale and bury repair verbs (#117442)
Eli Reismana436e822f30e
diff --git a/posthog/management/commands/person_consistency.py b/posthog/management/commands/person_consistency.pyindex 8a3c34c4e9f7..ae43f196c55a 100644--- a/posthog/management/commands/person_consistency.py+++ b/posthog/management/commands/person_consistency.py@@ -10,8 +10,16 @@ Usage:python manage.py person_consistency scan --dir /tmp/scan-2 --phase postgrespython manage.py person_consistency finalize --dir /tmp/scan-2-Read-only. A run that stops resumes with --resume. The report (report.jsonl) holds one row per key that-is not ok, with the raw keys a repair needs; summary.json holds counts and hashed group names, and names no key.+ # Repair the rows of one verb: a dry run first, then a canary, then the rest.+ python manage.py person_consistency repair --dir /tmp/scan-2 --verb republish+ python manage.py person_consistency repair --dir /tmp/scan-2 --verb republish --apply --limit 10+ python manage.py person_consistency repair --dir /tmp/scan-2 --verb republish --apply --resume <actions file>++Scans are read-only. A scan that stops resumes with --resume. The report (report.jsonl) holds one row per+key that is not ok, with the raw keys a repair needs; summary.json holds counts and hashed group names, and names no key.+A repair writes nothing without --apply, and records one action per key in an actions file.+Rescan after every repair run. The rescan confirms the fix, and finds any ClickHouse write lost when a run was+killed before it recorded its outcomes."""import time@@ -34,6 +42,7 @@ from posthog.models.person.divergence.team_scan import (run_chunk,run_postgres,)+from posthog.models.person.divergence.verbs import VERBS, VerbError, run_verbdef _parse_utc(value: str) -> datetime:@@ -60,7 +69,10 @@ def _id_range(value: str) -> IdRange:class Command(BaseCommand):- help = "Read-only. Classify every person and mapping of one team and write a report grouped by repair verb."+ help = (+ "Classify every person and mapping of one team into a report grouped by repair verb, and repair them. "+ "Scans are read-only; repair writes only with --apply."+ )def add_arguments(self, parser: CommandParser) -> None:actions = parser.add_subparsers(dest="action", required=True, metavar="action")@@ -102,6 +114,34 @@ class Command(BaseCommand):help="Pause between pages, to limit load on ClickHouse and personhog (default: %(default)s).",)+ repair = actions.add_parser(+ "repair",+ help="DRY RUN unless --apply is passed. Repair the report rows that name one verb.",+ )+ self._add_dir(repair)+ repair.add_argument("--verb", choices=sorted(VERBS), required=True, help="The repair to run.")+ repair.add_argument("--group", action="append", default=[], help="Only rows of this group. Repeatable.")+ repair.add_argument("--key", action="append", default=[], help="Only rows with this key. Repeatable.")+ repair.add_argument("--limit", type=_positive_int, default=None, help="At most this many rows, for a canary.")+ repair.add_argument("--apply", action="store_true", help="Write. Without it the repair only reports.")+ repair.add_argument(+ "--max-calls-per-second",+ type=float,+ default=20.0,+ help="Personhog write calls per second (default: %(default)s).",+ )+ repair.add_argument(+ "--resume",+ type=Path,+ default=None,+ help="An actions file of an earlier run: append to it and skip the keys it records as done.",+ )+ repair.add_argument(+ "--force",+ action="store_true",+ help="bury: tombstone new persons even past the share limit that protects a team needing a restore.",+ )+final = actions.add_parser("finalize", help="Join every run's rows into report.jsonl and summary.json.")self._add_dir(final)final.add_argument(@@ -152,9 +192,11 @@ class Command(BaseCommand):self._plan(options, chunks=options["chunks"])elif options["action"] == "scan":self._scan(options)+ elif options["action"] == "repair":+ self._repair(options)else:self._finalize(options["dir"], partial=options["partial"])- except ScanError as exc:+ except (ScanError, VerbError) as exc:raise CommandError(str(exc)) from excdef _plan(self, options: dict[str, Any], *, chunks: int) -> None:@@ -208,3 +250,32 @@ class Command(BaseCommand):f"report: {rows} rows, {summary['groups']['count']} groups, complete {summary['complete']}; "f"by verb {verbs or 'none'}; see {scan_dir / SUMMARY_FILE}")++ def _repair(self, options: dict[str, Any]) -> None:+ scan_dir: Path = options["dir"]+ if not options["max_calls_per_second"] > 0:+ raise CommandError("--max-calls-per-second must be above 0")+ mode = "apply" if options["apply"] else "dry-run"+ actions_path = options["resume"] or scan_dir / (+ f"actions-{options['verb']}-{mode}-{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}.jsonl"+ )+ counts = run_verb(+ scan_dir,+ options["verb"],+ apply=options["apply"],+ actions_path=actions_path,+ resume=options["resume"] is not None,+ groups=options["group"],+ keys=options["key"],+ limit=options["limit"],+ max_calls_per_second=options["max_calls_per_second"],+ force=options["force"],+ log=self._log,+ )+ outcomes = ", ".join(f"{outcome}={n}" for outcome, n in sorted(counts.items())) or "nothing to do"+ label = "APPLY" if options["apply"] else "DRY RUN"+ self._log(f"{options['verb']} {label}: {outcomes}; actions in {actions_path}")+ if not options["apply"]:+ self._log("Dry run: nothing was written. Pass --apply to write.")+ if counts.get("undelivered"):+ raise CommandError("Some ClickHouse messages were not delivered; rerun with --resume")diff --git a/posthog/models/person/deletion.py b/posthog/models/person/deletion.pyindex b1dfb458169d..8e3020f2698c 100644--- a/posthog/models/person/deletion.py+++ b/posthog/models/person/deletion.py@@ -1,4 +1,5 @@import datetime as dt+from collections.abc import Callablefrom dataclasses import dataclass, fieldfrom typing import Optionalfrom uuid import UUID@@ -119,13 +120,19 @@ def find_orphaned_ch_persons(team_id: int, uuids: Optional[list[str]] = None) ->def tombstone_orphaned_ch_persons(- team_id: int, orphans: list[OrphanedPerson], *, dry_run: bool = True+ team_id: int,+ orphans: list[OrphanedPerson],+ *,+ dry_run: bool = True,+ on_outcome: Optional[Callable[[str, str], None]] = None,) -> OrphanRepairResult:"""Tombstone orphaned persons in the persons DB one version above ClickHouse, then publish those tombstones.Each ClickHouse tombstone carries the exact stored version, so a later revival lands above it and a rerunrepublishes the same versions. Distinct ids get no tombstone of their own, because the ClickHouse deletionsweep removes every live mapping of a person it deletes.++ Once applied, ``on_outcome`` receives each orphan's uuid with "tombstoned", "republished" or "skipped_live"."""result = OrphanRepairResult(orphaned_person_uuids=sorted(o.uuid for o in orphans), dry_run=dry_run)if not orphans:@@ -153,12 +160,16 @@ def tombstone_orphaned_ch_persons(uuid = str(floor.uuid)if floor.outcome == VersionFloorOutcome.LIVE:result.skipped_live_persons += 1+ if on_outcome is not None:+ on_outcome(uuid, "skipped_live")continuestored = stored_by_uuid.get(uuid)if stored is None:result.tombstoned_persons += 1else:result.republished_persons += 1+ if on_outcome is not None:+ on_outcome(uuid, "tombstoned" if stored is None else "republished")tombstone = PersonTombstone(uuid=floor.uuid, version=floor.version, distinct_ids=stored.distinct_ids if stored else [])diff --git a/posthog/models/person/divergence/datatypes.py b/posthog/models/person/divergence/datatypes.pyindex 39ed46e31874..e1f639456de2 100644--- a/posthog/models/person/divergence/datatypes.py+++ b/posthog/models/person/divergence/datatypes.py@@ -7,7 +7,8 @@ from uuid import UUIDfrom posthog.dataclasses import frozen-PersonDivergenceKind = Literal["hidden", "swept", "stale", "behind", "absent"]+# "equal": the versions match but the ClickHouse winner holds other content (a tie or a blanked person).+PersonDivergenceKind = Literal["hidden", "swept", "stale", "behind", "absent", "equal"]MappingDivergenceKind = Literal["hidden", "other_person", "stale", "absent"]diff --git a/posthog/models/person/divergence/repair_plan.py b/posthog/models/person/divergence/repair_plan.pyindex f99815a6a0c7..c57aae404179 100644--- a/posthog/models/person/divergence/repair_plan.py+++ b/posthog/models/person/divergence/repair_plan.py@@ -2,7 +2,7 @@from __future__ import annotations-from collections.abc import Sequence+from collections.abc import Collection, Sequencefrom posthog.dataclasses import frozenfrom posthog.models.person import Person@@ -158,7 +158,15 @@ def _ch_mapping_states(team_id: int, distinct_ids: list[str]) -> dict[str, _ChMareturn states-def _plan_chunk(team_id: int, person_uuids: Sequence[str], *, only_distinct_id: str | None = None) -> list[_PersonPlan]:+def _plan_chunk(+ team_id: int,+ person_uuids: Sequence[str],+ *,+ only_distinct_id: str | None = None,+ republish_equal: Collection[str] = (),+) -> list[_PersonPlan]:+ """`republish_equal` names persons to republish even when ClickHouse holds their Postgres version, because+ a scan found other content at that version."""live = {str(p.uuid): p for p in get_persons_by_uuids(team_id, list(person_uuids), distinct_id_limit=0)}person_states = _ch_person_states(team_id, list(live)) if live else {}distinct_ids_by_person = (@@ -225,7 +233,8 @@ def _plan_chunk(team_id: int, person_uuids: Sequence[str], *, only_distinct_id:team_id=team_id,person_uuid=person_uuid,person=person,- kind=_person_kind(pg_version, state),+ kind=_person_kind(pg_version, state)+ or ("equal" if person_uuid in republish_equal and state is not None and state.visible_rows else None),ch_max_version=ch_max_version,target_version=_target_version(pg_version, ch_max_version),mappings=mappings,diff --git a/posthog/models/person/divergence/repairs.py b/posthog/models/person/divergence/repairs.pyindex e0069543818b..0b90de6d6279 100644--- a/posthog/models/person/divergence/repairs.py+++ b/posthog/models/person/divergence/repairs.py@@ -4,10 +4,11 @@ from __future__ import annotationsimport timefrom collections import Counter, defaultdict-from collections.abc import Callable, Sequence+from collections.abc import Callable, Collection, Mapping, Sequencefrom dataclasses import fieldfrom functools import partialfrom typing import TypeVar+from uuid import UUIDimport grpc@@ -35,6 +36,7 @@ from posthog.personhog_client.proto import (DistinctIdWriteResult,PersonVersionFloor,RaisePersonVersionFloorsRequest,+ SetPersonDistinctIdVersionFloorRequest,WriteDistinctIdsRequest,WriteDistinctIdsResponse,)@@ -121,6 +123,22 @@ def _raise_mapping_version_floors(return results+def _raise_mapping_version_floor(team_id: int, distinct_id: str, min_version: int) -> str | None:+ """Raise the mapping's version on the primary, live or tombstoned, and return the uuid of its person there.++ Bury uses it for tombstoned mappings, which the batched raise leaves alone.+ """+ response = personhog_call(+ "person_divergence_set_person_distinct_id_version_floor",+ lambda: require_personhog_client().set_person_distinct_id_version_floor(+ SetPersonDistinctIdVersionFloorRequest(team_id=team_id, distinct_id=distinct_id, min_version=min_version)+ ),+ )+ if not response.HasField("person"):+ return None+ return str(UUID(response.person.uuid))++def _person_action(plan: repair_plan._PersonPlan, outcome: datatypes.RepairOutcome) -> datatypes.RepairAction:return datatypes.RepairAction(team_id=plan.team_id,@@ -245,18 +263,23 @@ def _execute_plan(apply: bool,include_stale: bool,before_write: Callable[[], None],- published: Callable[[ProduceResult], None],+ published: Callable[[ProduceResult, PublishedKey], None],stale_merge_distinct_id: str | None = None,) -> list[datatypes.RepairAction]:person = plan.personif person is None:return [_person_action(plan, "skipped_not_live")]if plan.kind == "stale" and not include_stale:- refusal: datatypes.RepairOutcome | None = "skipped_stale"+ refusal: datatypes.RepairOutcome | None = (+ "would_repair" if not apply and stale_merge_distinct_id is not None else "skipped_stale"+ )if apply and stale_merge_distinct_id is not None:missing = stale_merge._clickhouse_only_properties(plan)# A later reset that finds nothing missing raises Postgres. Capture accepting the $set does not mean# ingestion applies it, and a raise without the merged properties lets the next update overwrite them.+ if missing:+ # The $set_once event lands on ingestion like any write, so it is paced like one.+ before_write()refusal = ((stale_merge._queue_stale_merge(plan, stale_merge_distinct_id, missing) or "merge_queued")if missing@@ -333,7 +356,7 @@ def _execute_plan(# The replica has not caught up with the raise, so its properties may predate the raised version.person_outcome = "skipped_reread_lagging"else:- published(_publish_person(plan.team_id, reread))+ published(_publish_person(plan.team_id, reread), PublishedKey(kind="person", key=plan.person_uuid))person_outcome = "repaired"# The write reply holds the row the primary stored, at or above the target, so it is the version to publish.@@ -345,12 +368,19 @@ def _execute_plan(person_id=plan.person_uuid,version=stored_versions[mapping.distinct_id],is_deleted=False,- )+ ),+ PublishedKey(kind="mapping", key=mapping.distinct_id),)mapping_actions.append(_mapping_action(plan, mapping, "repaired"))return [_person_action(plan, person_outcome), *mapping_actions]+@frozen+class PublishedKey:+ kind: str+ key: str++@frozen(frozen=False)class _WritePacer:"""Space writes at least 1 / max_per_second apart, so idle time never buys a later burst."""@@ -370,13 +400,17 @@ class _WritePacer:@frozen(frozen=False)class _Deliveries:- """Kafka produce results of the published rows, so a failed delivery counts as undelivered."""+ """Kafka produce results of the published rows, so a failed delivery counts as undelivered.++ A row tracked with a key names that key in `undelivered_keys` when its delivery fails.+ """- pending: list[ProduceResult] = field(default_factory=list)+ pending: list[tuple[ProduceResult, PublishedKey | None]] = field(default_factory=list)failed: int = 0+ failed_keys: list[PublishedKey] = field(default_factory=list)- def track(self, result: ProduceResult) -> None:- self.pending.append(result)+ def track(self, result: ProduceResult, key: PublishedKey | None = None) -> None:+ self.pending.append((result, key))if len(self.pending) >= _DELIVERY_PRUNE_EVERY:self._prune()@@ -385,16 +419,22 @@ class _Deliveries:self._prune()return self.failed + len(self.pending)+ def undelivered_keys(self) -> list[PublishedKey]:+ """The keys whose rows failed or are still waiting, as of the last `undelivered` call."""+ return [*self.failed_keys, *(key for _, key in self.pending if key is not None)]+def _prune(self) -> None:- waiting: list[ProduceResult] = []- for result in self.pending:+ waiting: list[tuple[ProduceResult, PublishedKey | None]] = []+ for result, key in self.pending:if not result.done():- waiting.append(result)+ waiting.append((result, key))continuetry:result.get(timeout=0)except Exception:self.failed += 1+ if key is not None:+ self.failed_keys.append(key)self.pending = waiting@@ -425,6 +465,9 @@ def repair_persons(max_writes_per_second: float | None = None,on_action: Callable[[datatypes.RepairAction], None],log: Callable[[str], None],+ republish_equal: Collection[str] = (),+ stale_merge_distinct_ids: Mapping[str, str] | None = None,+ deliveries: _Deliveries | None = None,) -> datatypes.RepairSummary:"""Republish the Postgres state of each target person and its mappings where ClickHouse disagrees.@@ -434,6 +477,11 @@ def repair_persons(``include_stale`` also repairs stale persons and their mappings. That replaces their ClickHouse propertieswith the Postgres ones for good, so never use it on a team waiting for a restore from ClickHouse.++ ``republish_equal`` names persons to republish although ClickHouse holds their Postgres version.+ ``stale_merge_distinct_ids`` maps a stale person to the distinct id that sends its ClickHouse-only properties+ to ingestion; a rerun after ingestion applied them raises and republishes the person.+ ``deliveries``, when given, collects the publishes; the caller flushes it, and the summary counts none."""if max_writes_per_second is not None and max_writes_per_second <= 0:raise ValueError("max_writes_per_second must be above 0")@@ -445,7 +493,9 @@ def repair_persons(processed = 0undelivered = 0pacer = _WritePacer(max_per_second=max_writes_per_second)- deliveries = _Deliveries()+ # A caller that passes its own tracker flushes it, and learns which keys did not land.+ flush_here = deliveries is None+ deliveries = deliveries if deliveries is not None else _Deliveries()existing_teams = set(Team.objects.filter(id__in=list(by_team)).values_list("id", flat=True))try:for team_id, person_uuids in sorted(by_team.items()):@@ -469,7 +519,9 @@ def repair_persons(continuefor chunk in lookups._chunks(person_uuids, _REPAIR_CHUNK_SIZE):plans = _retry_transient(- partial(repair_plan._plan_chunk, team_id, chunk), what=f"team {team_id}: planning", log=log+ partial(repair_plan._plan_chunk, team_id, chunk, republish_equal=frozenset(republish_equal)),+ what=f"team {team_id}: planning",+ log=log,)for plan in plans:# Only the attempt that succeeds returns actions, so a retried person is counted once.@@ -481,6 +533,7 @@ def repair_persons(include_stale=include_stale,before_write=pacer.before_write,published=deliveries.track,+ stale_merge_distinct_id=(stale_merge_distinct_ids or {}).get(plan.person_uuid),),what=f"team {team_id} person {plan.person_uuid}",log=log,@@ -491,7 +544,7 @@ def repair_persons(processed += 1log(f"team {team_id}: {len(person_uuids)} persons, {processed} processed in total")finally:- if apply:+ if apply and flush_here:undelivered = deliveries.undelivered(_FLUSH_TIMEOUT_SECONDS)if undelivered:log(f"{undelivered} ClickHouse messages were not delivered; rerun the repair for the same input")diff --git a/posthog/models/person/divergence/team_scan.py b/posthog/models/person/divergence/team_scan.pyindex ab368456eeb2..ab72e142d99b 100644--- a/posthog/models/person/divergence/team_scan.py+++ b/posthog/models/person/divergence/team_scan.py@@ -529,7 +529,7 @@ def run_chunk(files = chunk_files(scan_dir, chunk)cursor = load_cursor(files.cursor)if cursor is not None and not resume:- raise ScanError(f"{files.cursor} exists; pass resume to continue it")+ raise ScanError(f"{files.cursor} exists; pass --resume to continue it")cursor = cursor or Cursor(phase=CH_PHASES[0])writer = _RunWriter(files, cursor)try:@@ -621,7 +621,7 @@ def run_postgres(files = postgres_files(scan_dir, id_range)cursor = load_cursor(files.cursor)if cursor is not None and not resume:- raise ScanError(f"{files.cursor} exists; pass resume to continue it")+ raise ScanError(f"{files.cursor} exists; pass --resume to continue it")cursor = cursor or Cursor(phase="postgres", after=str(id_range.start - 1 if id_range else 0))before = id_range.end if id_range else Nonewriter = _RunWriter(files, cursor)diff --git a/posthog/models/person/divergence/verbs.py b/posthog/models/person/divergence/verbs.pynew file mode 100644index 000000000000..55c79ef27f81--- /dev/null+++ b/posthog/models/person/divergence/verbs.py@@ -0,0 +1,399 @@+"""Repair verbs. Each takes the report rows of a team scan that name it, re-checks every key against the current+state, and writes only what still applies. Every verb is a dry run unless applied, and every write only raises a+version or acts on the exact state it read, so a rerun repeats nothing."""++from __future__ import annotations++import json+import signal+import threading+import dataclasses+from collections import Counter+from collections.abc import Callable, Iterable, Sequence+from datetime import UTC, datetime+from functools import partial+from pathlib import Path+from typing import Any++from posthog.dataclasses import frozen+from posthog.models.person.deletion import (+ count_live_ch_persons,+ find_orphaned_ch_persons,+ orphan_share_refusal,+ tombstone_orphaned_ch_persons,+)+from posthog.models.person.util import create_person_distinct_id, get_distinct_id_rows, get_persons_by_uuids+from posthog.personhog_client.proto import DistinctIdRow++from . import datatypes, lookups, observe, repairs, team_scan++# A key whose last recorded outcome is one of these needs nothing more, so a resumed run skips it.+DONE_OUTCOMES = frozenset({"repaired", "tombstoned", "republished", "buried"})+++class VerbError(Exception):+ pass+++@frozen+class RowKey:+ kind: str+ key: str+++@frozen+class VerbAction:+ verb: str+ team_id: int+ kind: str+ key: str+ group: str | None+ outcome: str+ detail: dict[str, Any]+++@frozen(frozen=False)+class VerbContext:+ verb: str+ team_id: int+ apply: bool+ force: bool+ max_calls_per_second: float | None+ log: Callable[[str], None]+ emit: Callable[[VerbAction], None]+ pacer: repairs._WritePacer+ deliveries: repairs._Deliveries++ def action(self, row: dict[str, Any], outcome: str, **detail: Any) -> None:+ self.emit(+ VerbAction(+ verb=self.verb,+ team_id=self.team_id,+ kind=row["kind"],+ key=row["key"],+ group=row.get("group"),+ outcome=outcome,+ detail=detail,+ )+ )+++VerbFn = Callable[[list[dict[str, Any]], VerbContext], None]+++def load_rows(+ scan_dir: Path,+ verb: str,+ *,+ groups: Sequence[str] = (),+ keys: Sequence[str] = (),+ limit: int | None = None,+ skip: Iterable[RowKey] = (),+) -> list[dict[str, Any]]:+ """The report rows that name `verb`, optionally only those in `groups` or with `keys`, at most `limit`."""+ path = scan_dir / team_scan.REPORT_FILE+ if not path.exists():+ raise VerbError(f"{path} does not exist; finalize the scan first")+ wanted_groups, wanted_keys, skipped = set(groups), set(keys), set(skip)+ rows: list[dict[str, Any]] = []+ with path.open() as handle:+ for line in handle:+ row = json.loads(line)+ if row["verb"] != verb or RowKey(kind=row["kind"], key=row["key"]) in skipped:+ continue+ if wanted_groups and row["group"] not in wanted_groups:+ continue+ if wanted_keys and row["key"] not in wanted_keys:+ continue+ rows.append(row)+ if limit is not None and len(rows) >= limit:+ break+ return rows+++def done_keys(actions_path: Path) -> set[RowKey]:+ """Keys whose last recorded outcome in an actions file needs nothing more."""+ last: dict[RowKey, str] = {}+ if actions_path.exists():+ with actions_path.open() as handle:+ for line in handle:+ action = json.loads(line)+ last[RowKey(kind=action["kind"], key=action["key"])] = action["outcome"]+ return {key for key, outcome in last.items() if outcome in DONE_OUTCOMES}+++def run_verb(+ scan_dir: Path,+ verb: str,+ *,+ apply: bool,+ actions_path: Path,+ resume: bool = False,+ groups: Sequence[str] = (),+ keys: Sequence[str] = (),+ limit: int | None = None,+ max_calls_per_second: float | None = None,+ force: bool = False,+ log: Callable[[str], None] = lambda _: None,+) -> Counter[str]:+ """Run one verb over the scan's report rows and append one action per key to `actions_path`.++ With `resume`, keys that `actions_path` already records as done are skipped. Returns counts by outcome.+ """+ if verb not in VERBS:+ raise VerbError(f"unknown verb {verb}; choose one of {', '.join(sorted(VERBS))}")+ if actions_path.exists() and not resume:+ raise VerbError(f"{actions_path} exists; pass --resume to continue it")+ plan = team_scan.load_plan(scan_dir)+ if not plan.team_exists:+ raise VerbError(f"team {plan.team_id} does not exist; there is nothing to repair")+ rows = load_rows(scan_dir, verb, groups=groups, keys=keys, limit=limit, skip=done_keys(actions_path))+ counts: Counter[str] = Counter()+ with actions_path.open("a") as out:++ def emit(action: VerbAction) -> None:+ out.write(json.dumps({**_asdict(action), "applied": apply, "at": datetime.now(UTC).isoformat()}) + "\n")+ out.flush()+ counts[action.outcome] += 1++ context = VerbContext(+ verb=verb,+ team_id=plan.team_id,+ apply=apply,+ force=force,+ max_calls_per_second=max_calls_per_second,+ log=log,+ emit=emit,+ pacer=repairs._WritePacer(max_per_second=max_calls_per_second),+ deliveries=repairs._Deliveries(),+ )+ log(f"{verb}: {len(rows)} rows, {'APPLY' if apply else 'DRY RUN'}")+ terminated = False++ def on_sigterm(_signum: int, _frame: Any) -> None:+ nonlocal terminated+ terminated = True+ raise SystemExit(128 + signal.SIGTERM)++ restore_sigterm = _handle_sigterm(on_sigterm)+ try:+ if rows:+ VERBS[verb](rows, context)+ finally:+ restore_sigterm()+ if apply:+ undelivered = context.deliveries.undelivered(+ _TERMINATION_FLUSH_SECONDS if terminated else repairs._FLUSH_TIMEOUT_SECONDS+ )+ if undelivered:+ # A key whose publish did not land gets a last action that is not done, so resume retries it.+ by_key = {RowKey(kind=r["kind"], key=r["key"]): r for r in rows}+ for key in context.deliveries.undelivered_keys():+ row = by_key.get(RowKey(kind=key.kind, key=key.key)) or {+ "kind": key.kind,+ "key": key.key,+ "group": None,+ }+ context.action(row, "undelivered")+ counts["undelivered"] += undelivered - len(context.deliveries.undelivered_keys())+ log(f"{verb}: {undelivered} ClickHouse messages were not delivered; rerun with --resume")+ return counts+++# Kubernetes sends SIGKILL a grace period after SIGTERM, so the final flush must finish well inside it.+_TERMINATION_FLUSH_SECONDS = 20.0+++def _handle_sigterm(handler: Callable[[int, Any], None]) -> Callable[[], None]:+ """Run `handler` on SIGTERM until the returned callable restores the previous handler.++ A toolbox pod that is deleted or redeployed gets SIGTERM, which by default ends the process without running+ `finally`, so its undelivered keys would be recorded as done. Signals only reach the main thread.+ """+ if threading.current_thread() is not threading.main_thread():+ return lambda: None+ previous = signal.signal(signal.SIGTERM, handler)++ def restore() -> None:+ signal.signal(signal.SIGTERM, previous)++ return restore+++def _collect_outcome(sink: list[tuple[str, str]], person_uuid: str, outcome: str) -> None:+ sink.append((person_uuid, outcome))+++def _asdict(action: VerbAction) -> dict[str, Any]:+ return dataclasses.asdict(action)+++def _from_repair_action(+ context: VerbContext, rows_by_key: dict[str, dict[str, Any]], a: datatypes.RepairAction+) -> None:+ key = a.distinct_id or a.person_uuid+ row = rows_by_key.get(key) or {"kind": "mapping" if a.distinct_id else "person", "key": key, "group": None}+ context.action(+ row,+ a.outcome,+ person_uuid=a.person_uuid,+ divergence=a.kind,+ pg_version=a.pg_version,+ ch_max_version=a.ch_max_version,+ target_version=a.target_version,+ )+++def _republish_persons(+ rows: list[dict[str, Any]],+ context: VerbContext,+ person_uuids: list[str],+ *,+ republish_equal: Iterable[str] = (),+ stale_merge_distinct_ids: dict[str, str] | None = None,+) -> None:+ rows_by_key = {row["key"]: row for row in rows}+ repairs.repair_persons(+ [datatypes.PersonRef(team_id=context.team_id, person_uuid=u) for u in dict.fromkeys(person_uuids)],+ apply=context.apply,+ max_writes_per_second=context.max_calls_per_second,+ on_action=lambda a: _from_repair_action(context, rows_by_key, a),+ log=context.log,+ republish_equal=frozenset(republish_equal),+ stale_merge_distinct_ids=stale_merge_distinct_ids,+ deliveries=context.deliveries,+ )+++def republish(rows: list[dict[str, Any]], context: VerbContext) -> None:+ """Raise Postgres above ClickHouse and publish the Postgres state: hidden, swept, behind, missing, tied and+ blanked persons, and stale, hidden, missing or diverged mappings, through their Postgres owner."""+ person_uuids: list[str] = []+ for row in rows:+ if row["kind"] == "person":+ person_uuids.append(row["key"])+ continue+ owner = (row["evidence"]["pg"] or {}).get("person_uuid")+ if owner is None:+ context.action(row, "skipped_no_owner")+ else:+ person_uuids.append(owner)+ equal = [r["key"] for r in rows if r["kind"] == "person" and r["flavor"] in ("tie", "blanked")]+ _republish_persons(rows, context, person_uuids, republish_equal=equal)+++def merge_stale(rows: list[dict[str, Any]], context: VerbContext) -> None:+ """Send each stale person's ClickHouse-only properties to ingestion as $set_once.++ Run it again once ingestion applied them: a person with nothing left to send is then raised and republished.+ """+ person_rows = [r for r in rows if r["kind"] == "person"]+ persons = get_persons_by_uuids(context.team_id, [r["key"] for r in person_rows], distinct_id_limit=0)+ pk_by_uuid = {str(p.uuid): p.pk for p in persons}+ distinct_ids = lookups._live_distinct_ids_for_persons(context.team_id, list(pk_by_uuid.values()))+ merge_ids: dict[str, str] = {}+ for row in person_rows:+ dids = distinct_ids.get(pk_by_uuid.get(row["key"], -1)) or []+ if dids:+ merge_ids[row["key"]] = dids[0][0]+ else:+ context.action(row, "skipped_no_distinct_id")+ _republish_persons(rows, context, list(merge_ids), stale_merge_distinct_ids=merge_ids)+++def bury(rows: list[dict[str, Any]], context: VerbContext) -> None:+ """Tombstone persons and mappings that Postgres deleted but ClickHouse still shows, one version above+ ClickHouse in both stores."""+ person_rows = [r for r in rows if r["kind"] == "person"]+ if person_rows:+ _bury_persons(person_rows, context)+ for row in rows:+ if row["kind"] == "mapping":+ repairs._retry_transient(partial(_bury_mapping, row, context), what="bury mapping", log=context.log)+++def _bury_persons(rows: list[dict[str, Any]], context: VerbContext) -> None:+ by_uuid = {row["key"]: row for row in rows}+ orphans = find_orphaned_ch_persons(context.team_id, list(by_uuid))+ orphaned = {o.uuid for o in orphans}+ for person_uuid, row in by_uuid.items():+ if person_uuid not in orphaned:+ # Live in Postgres, or no longer live in ClickHouse.+ context.action(row, "skipped_not_divergent")+ if not orphans:+ return+ if not context.apply:+ for orphan in orphans:+ context.action(by_uuid[orphan.uuid], "would_repair", ch_max_version=orphan.ch_max_version)+ return+ planned = tombstone_orphaned_ch_persons(context.team_id, orphans, dry_run=True)+ if planned.tombstoned_persons and not context.force:+ refusal = orphan_share_refusal(+ context.team_id, planned.tombstoned_persons, count_live_ch_persons(context.team_id)+ )+ if refusal is not None:+ context.log(refusal)+ for orphan in orphans:+ context.action(by_uuid[orphan.uuid], "skipped_share_limit")+ return+ # The orphan repair reports an outcome before it publishes and raises when a publish fails, so outcomes are+ # recorded only once every publish landed; otherwise each key is recorded as undelivered for resume to retry.+ for chunk in lookups._chunks(orphans, repairs._WRITE_BATCH_SIZE):+ outcomes: list[tuple[str, str]] = []+ context.pacer.before_write()+ try:+ tombstone_orphaned_ch_persons(+ context.team_id,+ list(chunk),+ dry_run=False,+ on_outcome=partial(_collect_outcome, outcomes),+ )+ except Exception:+ for person_uuid, _ in outcomes:+ context.action(by_uuid[person_uuid], "undelivered")+ raise+ for person_uuid, outcome in outcomes:+ context.action(by_uuid[person_uuid], outcome)+++def _strong_row(team_id: int, distinct_id: str) -> DistinctIdRow | None:+ rows = get_distinct_id_rows(team_id, [distinct_id], strong=True)+ return rows[0] if rows else None+++def _bury_mapping(row: dict[str, Any], context: VerbContext) -> None:+ team_id, distinct_id = context.team_id, row["key"]+ reported = row["evidence"]["pg"]+ current = _strong_row(team_id, distinct_id)+ if current is None or not current.is_deleted or current.person_id != reported["person_id"]:+ context.action(row, "skipped_changed")+ return+ ch = observe.ch_mapping_states(team_id, [distinct_id]).get(distinct_id)+ if ch is None or ch.visible_rows == 0 or ch.winner_deleted:+ context.action(row, "skipped_not_divergent")+ return+ target = max(int(current.version), ch.max_version + 1)+ if not context.apply:+ context.action(row, "would_repair", target_version=target)+ return+ if current.version < target:+ context.pacer.before_write()+ repairs._raise_mapping_version_floor(team_id, distinct_id, target)+ confirmed = _strong_row(team_id, distinct_id)+ if confirmed is None or not confirmed.is_deleted or confirmed.person_id != current.person_id:+ # An attach revived the row between the reads; ingestion publishes the live row above the raise.+ context.action(row, "skipped_revived")+ return+ owner = confirmed.person_uuid if confirmed.HasField("person_uuid") else ch.winner_person_uuid+ context.deliveries.track(+ create_person_distinct_id(team_id, distinct_id, owner, version=int(confirmed.version), is_deleted=True),+ repairs.PublishedKey(kind="mapping", key=distinct_id),+ )+ context.action(row, "buried", version=int(confirmed.version))+++VERBS: dict[str, VerbFn] = {+ "republish": republish,+ "merge_stale": merge_stale,+ "bury": bury,+}diff --git a/posthog/models/person/test/test_divergence_team_scan.py b/posthog/models/person/test/test_divergence_team_scan.pyindex f33d8ea23746..617cbdd9642f 100644--- a/posthog/models/person/test/test_divergence_team_scan.py+++ b/posthog/models/person/test/test_divergence_team_scan.py@@ -122,7 +122,7 @@ class TestTeamScan(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):for distinct_id in ("moved", "lost", "x1", "unseen-1", "stray-1"):assert distinct_id not in out.getvalue()- with pytest.raises(CommandError, match="pass resume"):+ with pytest.raises(CommandError, match="pass --resume"):call_command("person_consistency", "scan", "--dir", str(scan_dir), stdout=StringIO())def test_a_deleted_team_reports_its_clickhouse_leftovers_without_repairs(self):diff --git a/posthog/models/person/test/test_divergence_verbs.py b/posthog/models/person/test/test_divergence_verbs.pynew file mode 100644index 000000000000..a36a6cb9f50b--- /dev/null+++ b/posthog/models/person/test/test_divergence_verbs.py@@ -0,0 +1,311 @@+import os+import json+import shutil+import signal+import tempfile+from pathlib import Path+from typing import Any+from uuid import uuid4++import pytest+from posthog.test.base import BaseTest, ClickhouseTestMixin+from unittest.mock import patch++from confluent_kafka import KafkaError+from parameterized import parameterized++from posthog.kafka_client.client import ClickhouseProducer, ProduceResult+from posthog.kafka_client.topics import KAFKA_PERSON+from posthog.models.person.divergence import repairs, team_scan, verbs+from posthog.models.person.test.divergence_fixtures import DivergenceFixturesMixin+from posthog.models.person.util import tombstone_persons_in_postgres+from posthog.personhog_client.fake_client import get_active_fake+++class _Stop(Exception):+ pass+++class TestRepairVerbs(DivergenceFixturesMixin, ClickhouseTestMixin, BaseTest):+ def setUp(self) -> None:+ super().setUp()+ self.root = Path(tempfile.mkdtemp())+ self.addCleanup(shutil.rmtree, self.root)+ self.scans = 0++ def _scan(self) -> tuple[Path, dict[str, tuple[str, str | None]]]:+ self.scans += 1+ scan_dir = self.root / f"scan-{self.scans}"+ team_scan.create_plan(scan_dir, self.team.pk)+ team_scan.run_chunk(scan_dir, 0, resume=False)+ team_scan.run_postgres(scan_dir, resume=False)+ team_scan.finalize(scan_dir)+ rows = [json.loads(line) for line in (scan_dir / team_scan.REPORT_FILE).read_text().splitlines()]+ return scan_dir, {r["key"]: (r["flavor"], r["verb"]) for r in rows}++ def _run(self, scan_dir: Path, verb: str, *, apply: bool, name: str = "actions", **kwargs: Any) -> list[dict]:+ actions_path = scan_dir / f"{name}-{verb}.jsonl"+ verbs.run_verb(scan_dir, verb, apply=apply, actions_path=actions_path, **kwargs)+ return [json.loads(line) for line in actions_path.read_text().splitlines()]++ def _mapped_person(self, version: int):+ distinct_id = f"did-{uuid4()}"+ person = self._pg_person(version=version, distinct_ids=[distinct_id])+ self._ch_mapping(distinct_id, str(person.uuid), 0)+ return person++ def _person_with_ch(self, version: int, distinct_ids: list[str] | None = None):+ person = self._pg_person(version=version, distinct_ids=distinct_ids)+ self._ch_person(str(person.uuid), version, {"email": "[email protected]"})+ for distinct_id in distinct_ids or []:+ self._ch_mapping(distinct_id, str(person.uuid), 0)+ return person++ def test_republish_reports_in_a_dry_run_and_clears_every_row_when_applied(self):+ hidden = self._mapped_person(1)+ self._ch_person(str(hidden.uuid), 1, {"email": "[email protected]"})+ self._ch_person(str(hidden.uuid), 100, {}, deleted=True)+ blanked = self._mapped_person(105)+ self._ch_person(str(blanked.uuid), 1, {"email": "[email protected]"})+ self._ch_person(str(blanked.uuid), 105, {})+ tied = self._mapped_person(2)+ self._ch_person(str(tied.uuid), 2, {"email": "[email protected]"})+ self._ch_person(str(tied.uuid), 2, {"email": "[email protected]"})+ owner = self._person_with_ch(1, ["m-stale"])+ self._ch_mapping("m-stale", str(owner.uuid), 5)+ scan_dir, before = self._scan()+ expected = {+ str(hidden.uuid): ("hidden", "republish"),+ str(blanked.uuid): ("blanked", "republish"),+ str(tied.uuid): ("tie", "republish"),+ "m-stale": ("stale", "republish"),+ }+ assert before == expected++ dry = self._run(scan_dir, "republish", apply=False, name="dry")++ assert {a["key"]: a["outcome"] for a in dry if a["outcome"] != "skipped_not_divergent"} == dict.fromkeys(+ expected, "would_repair"+ )+ assert self._scan()[1] == expected++ applied = self._run(scan_dir, "republish", apply=True)++ assert {a["key"]: a["outcome"] for a in applied if a["outcome"] != "skipped_not_divergent"} == dict.fromkeys(+ expected, "repaired"+ )+ assert self._scan()[1] == {}++ def test_resume_skips_keys_an_earlier_run_finished(self):+ first, second = self._mapped_person(1), self._mapped_person(1)+ for person in (first, second):+ self._ch_person(str(person.uuid), 1, {"email": "[email protected]"})+ self._ch_person(str(person.uuid), 100, {}, deleted=True)+ scan_dir, _ = self._scan()++ canary = self._run(scan_dir, "republish", apply=True, limit=1)+ rest = self._run(scan_dir, "republish", apply=True, resume=True)++ canary_persons = [(a["key"], a["outcome"]) for a in canary if a["kind"] == "person"]+ assert len(canary_persons) == 1 and canary_persons[0][1] == "repaired"+ resumed = [a["key"] for a in rest[len(canary) :] if a["kind"] == "person"]+ assert resumed == [k for k in (str(first.uuid), str(second.uuid)) if k != canary_persons[0][0]]+ assert self._scan()[1] == {}++ def test_bury_tombstones_what_postgres_deleted_and_leaves_strays_to_ingestion(self):+ deleted = self._pg_person(version=2)+ tombstone_persons_in_postgres(self.team.pk, [deleted.uuid])+ self._ch_person(str(deleted.uuid), 7, {"email": "[email protected]"})+ fake = get_active_fake()+ assert fake is not None+ holder = fake.add_person(+ team_id=self.team.pk,+ person_id=990_001,+ uuid=str(uuid4()),+ version=0,+ distinct_ids=["gone", "unpublished", "holder-1"],+ distinct_id_versions={"gone": 3, "unpublished": 3},+ tombstoned_distinct_ids=["gone", "unpublished"],+ )+ self._ch_person(holder.uuid, 0, {})+ self._ch_mapping("gone", holder.uuid, 5)+ self._ch_mapping("unpublished", holder.uuid, 1)+ self._ch_mapping("holder-1", holder.uuid, 0)+ stray_owner = fake.add_person(+ team_id=self.team.pk,+ person_id=990_002,+ uuid=str(uuid4()),+ version=3,+ is_deleted=True,+ distinct_ids=["stray-1"],+ )+ self._ch_mapping("stray-1", stray_owner.uuid, 0)+ scan_dir, before = self._scan()+ assert before == {+ str(deleted.uuid): ("deleted_visible", "bury"),+ "gone": ("deleted_visible", "bury"),+ "unpublished": ("deleted_visible", "bury"),+ "stray-1": ("stray", None),+ }++ buried = self._run(scan_dir, "bury", apply=True)++ assert {a["key"]: a["outcome"] for a in buried} == {+ str(deleted.uuid): "republished",+ "gone": "buried",+ "unpublished": "buried",+ }+ assert self._scan()[1] == {"stray-1": ("stray", None)}++ def _hidden_person_with_stale_mapping(self) -> str:+ person = self._pg_person(version=1, distinct_ids=["h-1"])+ self._ch_person(str(person.uuid), 1, {"email": "[email protected]"})+ self._ch_person(str(person.uuid), 100, {}, deleted=True)+ self._ch_mapping("h-1", str(person.uuid), 5)+ return str(person.uuid)++ def _tombstoned_mapping_visible_in_ch(self) -> str:+ fake = get_active_fake()+ assert fake is not None+ holder = fake.add_person(+ team_id=self.team.pk,+ person_id=990_003,+ uuid=str(uuid4()),+ version=0,+ distinct_ids=["gone", "holder-1"],+ distinct_id_versions={"gone": 3},+ tombstoned_distinct_ids=["gone"],+ )+ self._ch_person(holder.uuid, 0, {})+ self._ch_mapping("gone", holder.uuid, 5)+ self._ch_mapping("holder-1", holder.uuid, 0)+ return "gone"++ @parameterized.expand(+ [+ ("republish_after_person_raise", "republish", "repairs._raise_person_version_floor"),+ ("republish_after_mapping_raise", "republish", "repairs._raise_mapping_version_floors"),+ ("republish_after_person_publish", "republish", "repairs._publish_person"),+ ("republish_after_mapping_publish", "republish", "repairs.create_person_distinct_id"),+ ("bury_after_mapping_raise", "bury", "repairs._raise_mapping_version_floor"),+ ("bury_after_mapping_publish", "bury", "verbs.create_person_distinct_id"),+ ]+ )+ def test_a_run_stopped_after_any_step_converges_on_rerun(self, _name: str, verb: str, step: str):+ if verb == "republish":+ self._hidden_person_with_stale_mapping()+ else:+ self._tombstoned_mapping_visible_in_ch()+ scan_dir, before = self._scan()+ assert {v for _, v in before.values()} == {verb}+ module, name = step.split(".")+ target = f"posthog.models.person.divergence.{module}.{name}"+ original = getattr({"repairs": repairs, "verbs": verbs}[module], name)++ def step_then_stop(*args: Any, **kwargs: Any) -> Any:+ original(*args, **kwargs)+ raise _Stop++ with patch(target, side_effect=step_then_stop), pytest.raises(_Stop):+ self._run(scan_dir, verb, apply=True)+ self._run(scan_dir, verb, apply=True, resume=True)++ assert self._scan()[1] == {}++ # Republish counts a failed delivery after the run; the orphan repair behind bury raises on it.+ @parameterized.expand([("republish", False), ("bury", True)])+ def test_a_publish_that_did_not_land_is_retried_on_resume(self, verb: str, raises: bool):+ if verb == "republish":+ key = self._hidden_person_with_stale_mapping()+ else:+ deleted = self._pg_person(version=2)+ tombstone_persons_in_postgres(self.team.pk, [deleted.uuid])+ self._ch_person(str(deleted.uuid), 7, {"email": "[email protected]"})+ key = str(deleted.uuid)+ scan_dir, _ = self._scan()+ failed = ProduceResult(topic=KAFKA_PERSON)+ failed.set_result(KafkaError(-192, "Local: Message timed out"), None)+ actions_path = scan_dir / f"{verb}.jsonl"++ with patch.object(ClickhouseProducer, "produce", return_value=failed):+ if raises:+ with pytest.raises(Exception):+ verbs.run_verb(scan_dir, verb, apply=True, actions_path=actions_path)+ else:+ counts = verbs.run_verb(scan_dir, verb, apply=True, actions_path=actions_path)+ assert counts["undelivered"] > 0+ last = {}+ for action in map(json.loads, actions_path.read_text().splitlines()):+ last[action["key"]] = action["outcome"]+ assert last[key] == "undelivered"++ verbs.run_verb(scan_dir, verb, apply=True, actions_path=actions_path, resume=True)++ assert self._scan()[1] == {}++ def test_a_run_stopped_by_sigterm_records_its_undelivered_keys_for_resume(self):+ key = self._hidden_person_with_stale_mapping()+ scan_dir, _ = self._scan()+ failed = ProduceResult(topic=KAFKA_PERSON)+ failed.set_result(KafkaError(-192, "Local: Message timed out"), None)+ actions_path = scan_dir / "republish.jsonl"+ republish = verbs.VERBS["republish"]++ def republish_then_sigterm(rows: list[dict], context: verbs.VerbContext) -> None:+ republish(rows, context)+ os.kill(os.getpid(), signal.SIGTERM)++ with (+ patch.object(ClickhouseProducer, "produce", return_value=failed),+ patch.dict(verbs.VERBS, {"republish": republish_then_sigterm}),+ pytest.raises(SystemExit),+ ):+ verbs.run_verb(scan_dir, "republish", apply=True, actions_path=actions_path)++ last = {a["key"]: a["outcome"] for a in map(json.loads, actions_path.read_text().splitlines())}+ assert last[key] == "undelivered"+ assert signal.getsignal(signal.SIGTERM) is signal.SIG_DFL++ def test_bury_paces_each_tombstone_call(self):+ for version in (2, 3):+ deleted = self._pg_person(version=version)+ tombstone_persons_in_postgres(self.team.pk, [deleted.uuid])+ self._ch_person(str(deleted.uuid), 7, {"email": "[email protected]"})+ scan_dir, _ = self._scan()++ with (+ patch.object(repairs, "_WRITE_BATCH_SIZE", 1),+ patch.object(repairs._WritePacer, "before_write", autospec=True) as before_write,+ ):+ outcomes = self._run(scan_dir, "bury", apply=True)++ assert [a["outcome"] for a in outcomes] == ["republished", "republished"]+ assert before_write.call_count == 2++ def test_merge_stale_sends_clickhouse_only_properties_then_republishes_once_ingestion_applied_them(self):+ stale = self._pg_person(version=1, distinct_ids=["s-1"])+ self._ch_person(str(stale.uuid), 40, {"email": "[email protected]", "plan": "pro"})+ self._ch_mapping("s-1", str(stale.uuid), 0)+ scan_dir, before = self._scan()+ assert before == {str(stale.uuid): ("stale", "merge_stale")}++ with patch("posthog.models.person.divergence.stale_merge.capture_internal") as capture:+ first = self._run(scan_dir, "merge_stale", apply=True)++ assert [(a["key"], a["outcome"]) for a in first if a["kind"] == "person"] == [(str(stale.uuid), "merge_queued")]+ assert capture.call_args.kwargs["distinct_id"] == "s-1"+ assert capture.call_args.kwargs["properties"] == {"$set_once": {"plan": "pro"}}++ fake = get_active_fake()+ assert fake is not None+ stored = fake.stored_person(self.team.pk, str(stale.uuid))+ assert stored is not None+ stored.properties = json.dumps({"email": "[email protected]", "plan": "pro"}).encode()+ with patch("posthog.models.person.divergence.stale_merge.capture_internal") as capture:+ second = self._run(scan_dir, "merge_stale", apply=True, resume=True)++ capture.assert_not_called()+ assert [(a["key"], a["outcome"]) for a in second[len(first) :] if a["kind"] == "person"] == [+ (str(stale.uuid), "repaired")+ ]+ assert self._scan()[1] == {}