:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(flags): add a job that repairs person_id on imported flag_evaluations rows (#119898)
Phil Haack0ba4963868e9
diff --git a/posthog/dags/flag_evaluations_backfill.py b/posthog/dags/flag_evaluations_backfill.pyindex 3d668a8cfca9..ca8a6661d874 100644--- a/posthog/dags/flag_evaluations_backfill.py+++ b/posthog/dags/flag_evaluations_backfill.py@@ -52,7 +52,7 @@ _FINISHED_RUN_STATUSES = (# it does for rows from Kafka. inserted_at is left out so its DEFAULT stamps the event timestamp:# that keeps every copied row inside the deletion sweep's `inserted_at <= request.created_at` bound,# and it keeps copied rows out of the consumer-lag query below. _partition and _offset are left out-# so that both are 0 on a copied row. _COPIED_ROW_FILTER relies on that.+# so that both are 0 on a copied row. COPIED_ROW_FILTER relies on that._COPIED_COLUMNS = ("uuid","event",@@ -77,7 +77,7 @@ _COPIED_VALUES = ", ".join(# A row from Kafka also has inserted_at = timestamp when its timestamp has no sub-second part and# falls in the second that Kafka received it. Kafka gives offset 0 only to the first message in a# partition, so the only Kafka row this filter can match is the first message in partition 0.-_COPIED_ROW_FILTER = "_partition = 0 AND _offset = 0 AND inserted_at = timestamp"+COPIED_ROW_FILTER = "_partition = 0 AND _offset = 0 AND inserted_at = timestamp"# A copied row's inserted_at is its timestamp, so the inserted_at != timestamp filter limits this# query to rows that arrived through the Kafka path.@@ -235,11 +235,13 @@ def earliest_backfill_day(today: date) -> date:return today - timedelta(days=_EARLIEST_START_DAYS_AGO)-def resolve_backfill_days(config: FlagEvaluationsBackfillConfig, *, today: date) -> tuple[date, ...]:+def resolve_backfill_days(+ *, start_date: str | None = None, end_date: str | None = None, today: date+) -> tuple[date, ...]:latest_end = today - timedelta(days=1)earliest_start = earliest_backfill_day(today)- start = date.fromisoformat(config.start_date) if config.start_date else earliest_start- end = date.fromisoformat(config.end_date) if config.end_date else latest_end+ start = date.fromisoformat(start_date) if start_date else earliest_start+ end = date.fromisoformat(end_date) if end_date else latest_endif start < earliest_start:raise dagster.Failure(description=f"start_date {start} is before {earliest_start}. "@@ -303,7 +305,7 @@ def plan_flag_evaluations_backfill(context: dagster.OpExecutionContext, config: FlagEvaluationsBackfillConfig) -> BackfillPlan:"""Resolve the window once, so a shard re-executed from the UI later copies the same days."""- days = resolve_backfill_days(config, today=datetime.now(UTC).date())+ days = resolve_backfill_days(start_date=config.start_date, end_date=config.end_date, today=datetime.now(UTC).date())context.log.info(f"Backfilling {len(days)} day(s), newest first, from {days[0]} back to {days[-1]}")return BackfillPlan(days=days, config=config)@@ -434,39 +436,10 @@ class ShardBackfill:)def wait_for_blocking_runs(self) -> BlockingRunCheck:- while True:- # The watermark comes before the scan. A run created during the scan then counts as created- # after it. Run storage can record a creation time to the whole second. The watermark- # therefore reaches one second back, and the post-copy check ignores runs that finished then.- since = datetime.now(UTC) - timedelta(seconds=1)- blockers = describe_runs(self.instance, BLOCKING_JOB_NAMES, exclude_run_id=self.run_id)- if not blockers:- break- self.log.info(- f"Waiting {self.config.blocking_run_poll_seconds}s for these runs to finish: {'; '.join(blockers)}"- )- time.sleep(self.config.blocking_run_poll_seconds)- finished = describe_runs(- self.instance,- BLOCKING_JOB_NAMES,- statuses=_FINISHED_RUN_STATUSES,- created_after=since,- exclude_run_id=self.run_id,- )- return BlockingRunCheck(since=since, finished_runs=frozenset(finished))+ return wait_for_blocking_runs(self.instance, self.run_id, self.log, self.config.blocking_run_poll_seconds)def check_no_blocking_run_started(self, check: BlockingRunCheck, *, day: date) -> None:- started = [- run- for run in describe_runs(- self.instance,- BLOCKING_JOB_NAMES,- statuses=(*EXECUTING_RUN_STATUSES, *_FINISHED_RUN_STATUSES),- created_after=check.since,- exclude_run_id=self.run_id,- )- if run not in check.finished_runs- ]+ started = blocking_runs_started_since(self.instance, check, self.run_id)if started:# A rerun for the same teams does not copy back the rows that earlier runs copied for other teams.team_filter = (@@ -477,7 +450,7 @@ class ShardBackfill:"The copy can hold rows or person_ids that the run removed from sharded_flag_evaluations. "f"After that run finishes, delete the rows this job copied for {day} on shard {self.shard_num} "f"(`DELETE FROM {FLAG_EVALUATIONS_DATA_TABLE} WHERE toDate(timestamp) = '{day}' "- f"AND {_COPIED_ROW_FILTER}{team_filter}`), then run the backfill again for the same teams."+ f"AND {COPIED_ROW_FILTER}{team_filter}`), then run the backfill again for the same teams.")def wait_for_disk_headroom(self) -> None:@@ -502,7 +475,7 @@ class ShardBackfill:"""# Every replica of the shard, including offline ones, stores a copy of each inserted part.disks_by_host = self.cluster.map_hosts_in_shard_by_role(- self.shard_num, self._tagged(_read_policy_disks), node_role=self.node_role+ self.shard_num, tagged(self.query_tags, _read_policy_disks), node_role=self.node_role).result()if not disks_by_host:raise dagster.Failure(description=f"No replica of shard {self.shard_num} reported its disks.")@@ -568,30 +541,78 @@ class ShardBackfill:)if self.config.dry_run:return result[0][0]- # sync_execute returns the written row count for an INSERT that wrote rows. It returns- # an empty result for an INSERT that wrote none.- return result if isinstance(result, int) else 0+ return inserted_rows(result)return sum(self._on_copy_host(partial(run_chunk, chunk)) for chunk in range(self.config.team_id_chunks))def _first_row(self, query: str, client: Client) -> tuple:return sync_execute(query, workload=self.workload, sync_client=client)[0]- def _tagged(self, fn: Callable[[Client], T]) -> Callable[[Client], T]:- # The cluster runs fn on a pool thread. The query tags do not cross into that thread.- def tagged(client: Client) -> T:- with tags_context(- kind="dagster", dagster=self.query_tags, product=ProductKey.FEATURE_FLAGS, feature=Feature.BACKFILL- ):- return fn(client)+ def _on_copy_host(self, fn: Callable[[Client], T]) -> T:+ return on_shard_host(self.cluster, self.shard_num, fn, self.query_tags, self.workload, self.node_role)+++def wait_for_blocking_runs(+ instance: dagster.DagsterInstance, run_id: str, log: dagster.DagsterLogManager, poll_seconds: int+) -> BlockingRunCheck:+ """Wait until no blocking run is active, then return the watermark for blocking_runs_started_since."""+ while True:+ # The watermark comes before the scan. A run created during the scan then counts as created+ # after it. Run storage can record a creation time to the whole second. The watermark+ # therefore reaches one second back, and blocking_runs_started_since ignores runs that finished then.+ since = datetime.now(UTC) - timedelta(seconds=1)+ blockers = describe_runs(instance, BLOCKING_JOB_NAMES, exclude_run_id=run_id)+ if not blockers:+ break+ log.info(f"Waiting {poll_seconds}s for these runs to finish: {'; '.join(blockers)}")+ time.sleep(poll_seconds)+ finished = describe_runs(+ instance, BLOCKING_JOB_NAMES, statuses=_FINISHED_RUN_STATUSES, created_after=since, exclude_run_id=run_id+ )+ return BlockingRunCheck(since=since, finished_runs=frozenset(finished))- return tagged- def _on_copy_host(self, fn: Callable[[Client], T]) -> T:- result = self.cluster.map_any_host_in_shards_by_role(- {self.shard_num: self._tagged(fn)}, workload=self.workload, node_role=self.node_role- ).result()- return next(iter(result.values()))+def blocking_runs_started_since(instance: dagster.DagsterInstance, check: BlockingRunCheck, run_id: str) -> list[str]:+ statuses = (*EXECUTING_RUN_STATUSES, *_FINISHED_RUN_STATUSES)+ runs = describe_runs(+ instance, BLOCKING_JOB_NAMES, statuses=statuses, created_after=check.since, exclude_run_id=run_id+ )+ return [run for run in runs if run not in check.finished_runs]+++def shard_workload() -> tuple[Workload, NodeRole]:+ return (Workload.OFFLINE, NodeRole.DATA) if is_cloud() else (Workload.DEFAULT, NodeRole.ALL)+++def tagged(query_tags: DagsterTags, fn: Callable[[Client], T]) -> Callable[[Client], T]:+ # The cluster runs fn on a pool thread. The query tags do not cross into that thread.+ def run(client: Client) -> T:+ with tags_context(+ kind="dagster", dagster=query_tags, product=ProductKey.FEATURE_FLAGS, feature=Feature.BACKFILL+ ):+ return fn(client)++ return run+++def on_shard_host(+ cluster: ClickhouseCluster,+ shard_num: int,+ fn: Callable[[Client], T],+ query_tags: DagsterTags,+ workload: Workload,+ node_role: NodeRole,+) -> T:+ result = cluster.map_any_host_in_shards_by_role(+ {shard_num: tagged(query_tags, fn)}, workload=workload, node_role=node_role+ ).result()+ return next(iter(result.values()))+++def inserted_rows(result: Any) -> int:+ # sync_execute returns the written row count for an INSERT that wrote rows. It returns+ # an empty result for an INSERT that wrote none.+ return result if isinstance(result, int) else 0def _read_policy_disks(client: Client) -> list[PolicyDisk]:@@ -608,7 +629,7 @@ def backfill_flag_evaluations_shard(shard_num: int,plan: BackfillPlan,) -> int:- workload, node_role = (Workload.OFFLINE, NodeRole.DATA) if is_cloud() else (Workload.DEFAULT, NodeRole.ALL)+ workload, node_role = shard_workload()totals = ShardBackfill(cluster=cluster,shard_num=shard_num,@@ -637,7 +658,7 @@ def flag_evaluations_backfill_job() -> None:Safe to re-run over the same window: the anti-join inserts only rows that are missing.A re-run does not rewrite a stored row. To copy a day again, first delete the rows that an earlier- run copied, which match _COPIED_ROW_FILTER. Delete a row that the Kafka path wrote only when events+ run copied, which match COPIED_ROW_FILTER. Delete a row that the Kafka path wrote only when eventsalso holds the call. Events does not hold the calls of an organization on FLAG_EVALUATIONS_ONLY."""plan = plan_flag_evaluations_backfill()diff --git a/posthog/dags/flag_evaluations_person_id_repair.py b/posthog/dags/flag_evaluations_person_id_repair.pynew file mode 100644index 000000000000..2d69a1acf872--- /dev/null+++ b/posthog/dags/flag_evaluations_person_id_repair.py@@ -0,0 +1,584 @@+"""Rewrite the person_id of imported flag_evaluations rows that missed a person_id rewrite.++A bulk copy into flag_evaluations can read events before a squash rewrites person_id there and+attach its rows after the squash has swept flag_evaluations. The squash then deletes the overrides+it applied, so those rows keep the merged-away person_id and no override is left for reads to+correct. This one-off job copies the current person_id from each row's events copy, through the+squash's dictionary-and-mutation lifecycle.+"""++import uuid+from collections.abc import Callable, Iterator+from datetime import UTC, date, datetime, timedelta+from functools import partial+from typing import Any++import dagster+import pydantic+from clickhouse_driver import Client++from posthog import settings+from posthog.clickhouse.client import sync_execute+from posthog.clickhouse.cluster import AlterTableMutationRunner, ClickhouseCluster+from posthog.dags.common import dagster_tags+from posthog.dags.common.dictionaries import Dictionary+from posthog.dags.common.staged_dictionary import (+ StagedDictionary,+ create_on_every_cluster,+ load_and_verify_on_every_cluster,+)+from posthog.dags.flag_evaluations_backfill import (+ COPIED_ROW_FILTER,+ GIB,+ OWNER_TAG,+ BlockingRunCheck,+ blocking_runs_started_since,+ inserted_rows,+ on_shard_host,+ resolve_backfill_days,+ shard_workload,+ wait_for_blocking_runs,+)+from posthog.dataclasses import frozen+from posthog.models.deletion_targets import FLAG_EVALUATIONS, dispatchable_here+from posthog.models.event.sql import EVENTS_DATA_TABLE+from posthog.models.flag_evaluations.sql import FLAG_EVALUATIONS_DATA_TABLE, FLAG_EVALUATIONS_SOURCE_EVENT++_TEAM_FILTER = " AND team_id IN %(team_ids)s"++_CLEANUP = "Drop this run's table and dictionary with cleanup_orphaned_flag_evaluations_person_id_repair."+++class FlagEvaluationsPersonIdRepairConfig(dagster.Config):+ start_date: str | None = pydantic.Field(+ default=None,+ description="First day to repair (YYYY-MM-DD, UTC, inclusive). Same default and bounds as the flag_evaluations backfill.",+ )+ end_date: str | None = pydantic.Field(+ default=None,+ description="Day after the last day to repair (YYYY-MM-DD, UTC, exclusive). Same default and bounds as the flag_evaluations backfill.",+ )+ team_ids: list[int] | None = pydantic.Field(+ default=None, description="Repair only these teams. Leave empty to repair every team."+ )+ dry_run: bool = pydantic.Field(+ default=False,+ description="Build the repair table and report what it holds, then drop it without changing flag_evaluations.",+ )+ blocking_run_poll_seconds: int = pydantic.Field(+ default=5 * 60,+ description="How often to check again while a squash, deletes or data deletion request run is active.",+ )+ build_max_execution_time_seconds: int = pydantic.Field(+ default=30 * 60, description="ClickHouse max_execution_time for each shard's insert of one day."+ )+ build_max_memory_usage_bytes: int = pydantic.Field(+ default=32 * GIB, description="ClickHouse max_memory_usage for each shard's insert of one day."+ )+ dictionary_shards: int = pydantic.Field(default=16, description="SHARDS of the dictionary's hashed layout.")+ dictionary_max_execution_time: int = pydantic.Field(+ default=0, description="Seconds the dictionary may take to load, or 0 for no limit."+ )+ dictionary_max_memory_usage: int = pydantic.Field(+ default=0, description="Bytes the dictionary may use while it loads, or 0 for no limit."+ )+++@frozen+class RepairPlan:+ days: tuple[date, ...]+ config: FlagEvaluationsPersonIdRepairConfig+ blocking_run_check: BlockingRunCheck+++@frozen+class ShardRepairCounts:+ calls: int+ distinct_ids: int+ keys: int+++@frozen+class RepairTableCheck:+ conflicts: int+ chains: int+ shards: dict[int, ShardRepairCounts]+++@frozen+class PersonIdRepairTable:+ """Maps (team_id, distinct_id, stale_person_id) to the person_id that the row's events copy holds.++ A distinct_id that merged twice can hold two stale ids, so the stale id is part of the key.+ A key that maps to two person_ids stays as two rows, which is how the check finds it.+ """++ id: uuid.UUID++ @property+ def name(self) -> str:+ # The dictionary reads its source as dagster_dict_reader. That user can read only the tables+ # that infra grants it. The grant for the squash's snapshot tables matches this prefix.+ # The prefix also matches the snapshot glob in posthog/clickhouse/hcl/exclude.hcl.+ return f"person_distinct_id_overrides_snapshot_flag_evaluations_repair_{self.id.hex}"++ @property+ def qualified_name(self) -> str:+ return f"{settings.CLICKHOUSE_DATABASE}.{self.name}"++ def create(self, client: Client) -> None:+ client.execute(+ f"""+ CREATE TABLE IF NOT EXISTS {self.qualified_name} (+ team_id Int64,+ distinct_id String,+ stale_person_id UUID,+ person_id UUID,+ shard_num UInt32,+ day Date,+ calls UInt64+ )+ ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/{self.qualified_name}', '{{replica}}-{{shard}}')+ ORDER BY (team_id, distinct_id, stale_person_id, person_id, shard_num, day)+ """+ )++ def exists(self, client: Client) -> bool:+ [[count]] = client.execute(+ "SELECT count() FROM system.tables WHERE database = %(database)s AND name = %(name)s",+ {"database": settings.CLICKHOUSE_DATABASE, "name": self.name},+ )+ return count > 0++ def drop(self, client: Client) -> None:+ client.execute(f"DROP TABLE IF EXISTS {self.qualified_name} SYNC")++ def sync(self, client: Client) -> None:+ client.execute(f"SYSTEM SYNC REPLICA {self.qualified_name} STRICT")++ def stale_keys_query(self, *, filter_team_ids: bool) -> str:+ team_filter = _TEAM_FILTER if filter_team_ids else ""+ # sharded_events and sharded_flag_evaluations share a shard key. A row's events copy is+ # therefore on its shard.+ return f"""+INSERT INTO {self.qualified_name} (team_id, distinct_id, stale_person_id, person_id, shard_num, day, calls)+SELECT f.team_id, f.distinct_id, f.person_id, e.person_id, %(shard_num)s, toDate(%(day_start)s), count()+FROM (+ SELECT team_id, uuid, distinct_id, person_id+ FROM {FLAG_EVALUATIONS_DATA_TABLE}+ WHERE timestamp >= %(day_start)s AND timestamp < %(day_end)s AND {COPIED_ROW_FILTER}{team_filter}+) AS f+INNER JOIN (+ SELECT team_id, uuid, distinct_id, person_id+ FROM {EVENTS_DATA_TABLE()}+ WHERE event = %(event)s AND timestamp >= %(day_start)s AND timestamp < %(day_end)s{team_filter}+) AS e ON f.team_id = e.team_id AND f.uuid = e.uuid AND f.distinct_id = e.distinct_id+WHERE f.person_id != e.person_id+GROUP BY f.team_id, f.distinct_id, f.person_id, e.person_id+"""++ def agreeing_calls_query(self) -> str:+ # A stale key's calls on any day may still agree with their events copies. This records+ # each such key with its own id as a second target, so the check fails it instead of the+ # mutation rewriting the agreeing calls. Both sides read only the shard's stale keys.+ stale_keys = (+ f"SELECT team_id, distinct_id, stale_person_id FROM {self.qualified_name} "+ "WHERE shard_num = %(shard_num)s AND person_id != stale_person_id"+ )+ return f"""+INSERT INTO {self.qualified_name} (team_id, distinct_id, stale_person_id, person_id, shard_num, day, calls)+SELECT f.team_id, f.distinct_id, f.person_id, f.person_id, %(shard_num)s, toDate(%(day_start)s), count()+FROM (+ SELECT team_id, uuid, distinct_id, person_id+ FROM {FLAG_EVALUATIONS_DATA_TABLE}+ WHERE timestamp >= %(day_start)s AND timestamp < %(day_end)s AND {COPIED_ROW_FILTER}+ AND (team_id, distinct_id, person_id) IN ({stale_keys})+) AS f+INNER JOIN (+ SELECT team_id, uuid, distinct_id, person_id+ FROM {EVENTS_DATA_TABLE()}+ WHERE event = %(event)s AND timestamp >= %(day_start)s AND timestamp < %(day_end)s+ AND (team_id, distinct_id) IN (SELECT team_id, distinct_id FROM ({stale_keys}))+) AS e ON f.team_id = e.team_id AND f.uuid = e.uuid AND f.distinct_id = e.distinct_id+WHERE f.person_id = e.person_id+GROUP BY f.team_id, f.distinct_id, f.person_id+"""+++@frozen+class PersonIdRepairDictionary(Dictionary):+ source: PersonIdRepairTable++ @property+ def name(self) -> str:+ return f"{self.source.name}_dictionary"++ @property+ def schema(self) -> str:+ return "team_id Int64, distinct_id String, stale_person_id UUID, person_id UUID"++ @property+ def primary_key(self) -> str:+ return "team_id, distinct_id, stale_person_id"++ @property+ def query(self) -> str:+ # The check proves one person_id per key before this loads. any() is therefore exact.+ return (+ f"SELECT team_id, distinct_id, stale_person_id, any(person_id) AS person_id "+ f"FROM {self.source.qualified_name} GROUP BY team_id, distinct_id, stale_person_id"+ )++ def staged(self) -> StagedDictionary:+ return StagedDictionary(+ key=f"{self.name}.parquet",+ columns="team_id, distinct_id, stale_person_id, person_id",+ structure=self.schema,+ )++ def update_command(self, *, filter_team_ids: bool) -> str:+ # The key includes the row's current person_id. A row matches only while it still holds the+ # stale id. A rerun therefore changes nothing. A row that a later squash moved stays where it is.+ # The build checked only the rows inside the window, so the mutation stays inside it too.+ team_filter = _TEAM_FILTER if filter_team_ids else ""+ return (+ "UPDATE person_id = dictGet(%(name)s, 'person_id', (team_id, distinct_id, person_id)) "+ "WHERE timestamp >= %(window_start)s AND timestamp < %(window_end)s "+ f"AND {COPIED_ROW_FILTER} AND dictHas(%(name)s, (team_id, distinct_id, person_id)){team_filter}"+ )++[email protected](tags=OWNER_TAG)+def plan_person_id_repair(+ context: dagster.OpExecutionContext,+ cluster: dagster.ResourceParam[ClickhouseCluster],+ config: FlagEvaluationsPersonIdRepairConfig,+) -> RepairPlan:+ """Resolve the days, then wait until no squash, deletes or data deletion request run is active.++ The build reads person_id from events, so the watermark taken here covers it as well as the mutation.+ """+ if not dispatchable_here(cluster, FLAG_EVALUATIONS):+ raise dagster.Failure(description=f"{FLAG_EVALUATIONS_DATA_TABLE} does not exist on this cluster.")+ days = resolve_backfill_days(start_date=config.start_date, end_date=config.end_date, today=datetime.now(UTC).date())+ check = wait_for_blocking_runs(context.instance, context.run_id, context.log, config.blocking_run_poll_seconds)+ context.log.info(f"Repairing {len(days)} day(s), newest first, from {days[0]} back to {days[-1]}")+ return RepairPlan(days=days, config=config, blocking_run_check=check)++[email protected](tags=OWNER_TAG)+def create_repair_table(+ context: dagster.OpExecutionContext,+ cluster: dagster.ResourceParam[ClickhouseCluster],+ plan: RepairPlan,+) -> PersonIdRepairTable:+ table = PersonIdRepairTable(id=uuid.UUID(context.run.run_id))+ cluster.map_all_hosts(table.create).result()+ return table++[email protected](out=dagster.DynamicOut(int), tags=OWNER_TAG)+def get_repair_shards(+ cluster: dagster.ResourceParam[ClickhouseCluster], table: PersonIdRepairTable+) -> Iterator[dagster.DynamicOutput[int]]:+ """Fan out one build op per shard, so a failed shard can be re-executed on its own from the UI."""+ for shard_num in sorted(cluster.shards):+ yield dagster.DynamicOutput(shard_num, mapping_key=f"shard_{shard_num}")++[email protected](tags=OWNER_TAG)+def build_repair_shard(+ context: dagster.OpExecutionContext,+ cluster: dagster.ResourceParam[ClickhouseCluster],+ shard_num: int,+ plan: RepairPlan,+ table: PersonIdRepairTable,+) -> int:+ """Insert this shard's stale keys one day per query, then the keys whose calls also agree with events.++ A rerun inserts the same rows, which the table collapses.+ """+ config = plan.config+ workload, node_role = shard_workload()+ stale_keys_query = table.stale_keys_query(filter_team_ids=bool(config.team_ids))+ agreeing_calls_query = table.agreeing_calls_query()+ query_settings = {+ "max_execution_time": config.build_max_execution_time_seconds,+ "max_memory_usage": config.build_max_memory_usage_bytes,+ }++ def day_args(day: date) -> dict[str, Any]:+ return {+ "shard_num": shard_num,+ "event": FLAG_EVALUATIONS_SOURCE_EVENT,+ "day_start": f"{day:%Y-%m-%d} 00:00:00",+ "day_end": f"{day + timedelta(days=1):%Y-%m-%d} 00:00:00",+ "team_ids": tuple(config.team_ids or ()),+ }++ def insert(query: str, day: date, client: Client) -> int:+ return inserted_rows(+ sync_execute(query, day_args(day), settings=query_settings, workload=workload, sync_client=client)+ )++ def insert_agreeing_calls(client: Client) -> int:+ # The stale keys came from inserts on any replica. The sync makes this host read all of them.+ table.sync(client)+ return sum(insert(agreeing_calls_query, day, client) for day in plan.days)++ def on_this_shard(fn: Callable[[Client], int]) -> int:+ return on_shard_host(cluster, shard_num, fn, dagster_tags(context), workload, node_role)++ stale_keys = 0+ for day in plan.days:+ keys = on_this_shard(partial(insert, stale_keys_query, day))+ stale_keys += keys+ context.log.info(f"Shard {shard_num}, {day}: {keys} stale key(s)")+ agreeing_keys = on_this_shard(insert_agreeing_calls)+ context.log.info(f"Shard {shard_num}: {agreeing_keys} stale key-day(s) with calls that agree with events")+ context.add_output_metadata(+ {+ "shard": dagster.MetadataValue.int(shard_num),+ "keys": dagster.MetadataValue.int(stale_keys),+ "agreeing": dagster.MetadataValue.int(agreeing_keys),+ }+ )+ return stale_keys++[email protected](tags=OWNER_TAG)+def check_repair_table(+ context: dagster.OpExecutionContext,+ cluster: dagster.ResourceParam[ClickhouseCluster],+ table: PersonIdRepairTable,+ built_shards: list[int],+) -> PersonIdRepairTable:+ """Fail unless each key maps to one person_id that is not itself a stale key, and report what the table holds.++ The report counts the calls the build joined. The mutation changes fewer if a squash moves a row first.+ """+ cluster.map_all_hosts(table.sync).result()++ def read(client: Client) -> RepairTableCheck:+ [[conflicts]] = client.execute(+ f"""+ SELECT count() FROM (+ SELECT 1 FROM {table.qualified_name}+ GROUP BY team_id, distinct_id, stale_person_id+ HAVING uniqExact(person_id) > 1+ )+ """+ )+ # A target that is also a stale key would move a row once per run instead of once in total.+ # A key that maps to its own id comes from the agreeing-calls pass and already counts as a conflict.+ [[chains]] = client.execute(+ f"""+ SELECT count() FROM (+ SELECT DISTINCT team_id, distinct_id, person_id FROM {table.qualified_name}+ WHERE person_id != stale_person_id+ )+ WHERE (team_id, distinct_id, person_id) IN (+ SELECT team_id, distinct_id, stale_person_id FROM {table.qualified_name}+ )+ """+ )+ shards = {+ shard: ShardRepairCounts(calls=calls, distinct_ids=distinct_ids, keys=keys)+ for shard, calls, distinct_ids, keys in client.execute(+ f"""+ SELECT shard_num, sum(calls), uniqExact(team_id, distinct_id), uniqExact(team_id, distinct_id, stale_person_id)+ FROM {table.qualified_name} FINAL+ GROUP BY shard_num+ """+ )+ }+ return RepairTableCheck(conflicts=conflicts, chains=chains, shards=shards)++ check = cluster.any_host(read).result()+ report = {+ f"shard_{shard}": {"calls": counts.calls, "distinct_ids": counts.distinct_ids, "keys": counts.keys}+ for shard, counts in sorted(check.shards.items())+ }+ context.add_output_metadata(+ {+ **{+ total: dagster.MetadataValue.int(sum(counts[total] for counts in report.values()))+ for total in ("calls", "distinct_ids", "keys")+ },+ "per_shard": dagster.MetadataValue.json(report),+ "conflicts": dagster.MetadataValue.int(check.conflicts),+ "chains": dagster.MetadataValue.int(check.chains),+ }+ )+ if check.conflicts or check.chains:+ raise dagster.Failure(+ description=f"{table.qualified_name} has {check.conflicts} key(s) that map to more than one person_id and "+ f"{check.chains} target(s) that are also stale keys. Inspect the table. {_CLEANUP}"+ )+ return table++[email protected](tags=OWNER_TAG)+def create_repair_dictionary(+ context: dagster.OpExecutionContext,+ cluster: dagster.ResourceParam[ClickhouseCluster],+ plan: RepairPlan,+ table: PersonIdRepairTable,+) -> PersonIdRepairDictionary:+ dictionary = PersonIdRepairDictionary(source=table)+ create_on_every_cluster(+ context,+ [cluster],+ dictionary,+ shards=plan.config.dictionary_shards,+ max_execution_time=plan.config.dictionary_max_execution_time,+ max_memory_usage=plan.config.dictionary_max_memory_usage,+ )+ return dictionary++[email protected](tags=OWNER_TAG)+def load_and_verify_repair_dictionary(+ context: dagster.OpExecutionContext,+ cluster: dagster.ResourceParam[ClickhouseCluster],+ dictionary: PersonIdRepairDictionary,+) -> PersonIdRepairDictionary:+ load_and_verify_on_every_cluster([cluster], dictionary)++ def read_size(client: Client) -> dict[str, int]:+ [[elements, bytes_allocated]] = client.execute(+ "SELECT element_count, bytes_allocated FROM system.dictionaries WHERE database = %(database)s AND name = %(name)s",+ {"database": settings.CLICKHOUSE_DATABASE, "name": dictionary.name},+ )+ return {"elements": elements, "bytes_allocated": bytes_allocated}++ context.add_output_metadata(+ {name: dagster.MetadataValue.int(value) for name, value in cluster.any_host(read_size).result().items()}+ )+ return dictionary++[email protected](tags=OWNER_TAG)+def run_repair_mutation(+ context: dagster.OpExecutionContext,+ cluster: dagster.ResourceParam[ClickhouseCluster],+ plan: RepairPlan,+ dictionary: PersonIdRepairDictionary,+) -> PersonIdRepairDictionary:+ """Rewrite the stale rows on every shard.++ Fail before the mutation if a blocking run started since the plan's watermark, and again after it.+ """+ if plan.config.dry_run:+ context.log.info("Dry run: leaving flag_evaluations unchanged")+ return dictionary++ def blocking_runs_started() -> str:+ return "; ".join(blocking_runs_started_since(context.instance, plan.blocking_run_check, context.run_id))++ if started := blocking_runs_started():+ cluster.map_all_hosts(dictionary.drop).result()+ cluster.map_all_hosts(dictionary.source.drop).result()+ raise dagster.Failure(+ description=f"{started} started after the repair read person_id from events. "+ f"{FLAG_EVALUATIONS_DATA_TABLE} is unchanged, and this run dropped its table and dictionary. "+ "Run the repair again after that run finishes."+ )++ team_ids = plan.config.team_ids+ AlterTableMutationRunner(+ table=FLAG_EVALUATIONS_DATA_TABLE,+ commands={dictionary.update_command(filter_team_ids=bool(team_ids))},+ parameters={+ "name": dictionary.qualified_name,+ "team_ids": tuple(team_ids or ()),+ "window_start": f"{min(plan.days):%Y-%m-%d} 00:00:00",+ "window_end": f"{max(plan.days) + timedelta(days=1):%Y-%m-%d} 00:00:00",+ },+ ).run_on_shards(cluster)++ if started := blocking_runs_started():+ raise dagster.Failure(+ description=f"{started} started during the repair mutation. A deletion that reached "+ f"{FLAG_EVALUATIONS_DATA_TABLE} before the mutation did not match rows that still held a stale "+ f"person_id, and the mutation then gave them the deleted person's id. Check what that run deleted. {_CLEANUP}"+ )+ return dictionary++[email protected](tags=OWNER_TAG)+def drop_repair_dictionary(+ cluster: dagster.ResourceParam[ClickhouseCluster], dictionary: PersonIdRepairDictionary+) -> PersonIdRepairTable:+ cluster.map_all_hosts(dictionary.drop).result()+ return dictionary.source++[email protected](tags=OWNER_TAG)+def drop_repair_table(cluster: dagster.ResourceParam[ClickhouseCluster], table: PersonIdRepairTable) -> None:+ cluster.map_all_hosts(table.drop).result()++[email protected](tags=OWNER_TAG)+def flag_evaluations_person_id_repair_job() -> None:+ """Give imported flag_evaluations rows the person_id of their events copy, where the two differ.++ Only rows that match the backfill's copied-row filter and still hold the stale id change, so a+ rerun changes nothing.+ """+ plan = plan_person_id_repair()+ table = create_repair_table(plan)+ built_shards = get_repair_shards(table).map(lambda shard_num: build_repair_shard(shard_num, plan, table))+ checked_table = check_repair_table(table, built_shards.collect())+ dictionary = load_and_verify_repair_dictionary(create_repair_dictionary(plan, checked_table))+ drop_repair_table(drop_repair_dictionary(run_repair_mutation(plan, dictionary)))+++class RepairRunConfig(dagster.Config):+ run_id: str = pydantic.Field(description="The ID of the repair run whose table and dictionary to drop.")++[email protected](tags=OWNER_TAG)+def get_repair_dictionary_for_run(config: RepairRunConfig) -> PersonIdRepairDictionary:+ """A handle to a repair run's dictionary. It does not check that the dictionary or its table exists."""+ return PersonIdRepairDictionary(source=PersonIdRepairTable(id=uuid.UUID(config.run_id)))++[email protected](tags=OWNER_TAG)+def check_no_repair_mutation_running(+ cluster: dagster.ResourceParam[ClickhouseCluster], dictionary: PersonIdRepairDictionary+) -> PersonIdRepairDictionary:+ """Fail while a mutation that reads this dictionary is unfinished on any host.++ A stopped Dagster run leaves its ClickHouse mutation running. Dropping the dictionary under it+ fails the mutation's remaining parts, and the failed mutation then blocks later mutations on the table.+ """++ def running(client: Client) -> int:+ [[count]] = client.execute(+ """+ SELECT count() FROM system.mutations+ WHERE database = %(database)s AND table = %(table)s AND NOT is_done AND NOT is_killed+ AND position(command, %(name)s) > 0+ """,+ {+ "database": settings.CLICKHOUSE_DATABASE,+ "table": FLAG_EVALUATIONS_DATA_TABLE,+ "name": dictionary.qualified_name,+ },+ )+ return count++ hosts = [host.connection_info.host for host, count in cluster.map_all_hosts(running).result().items() if count]+ if hosts:+ raise dagster.Failure(+ description=f"A mutation that reads {dictionary.qualified_name} is unfinished on {', '.join(hosts)}. "+ "Wait for it to finish, or stop it with KILL MUTATION, then run the cleanup again."+ )+ return dictionary++[email protected](tags=OWNER_TAG)+def cleanup_orphaned_flag_evaluations_person_id_repair() -> None:+ """Drop the table and dictionary that a failed repair run left, once its mutation has finished or been killed."""+ drop_repair_table(drop_repair_dictionary(check_no_repair_mutation_running(get_repair_dictionary_for_run())))diff --git a/posthog/dags/locations/clickhouse.py b/posthog/dags/locations/clickhouse.pyindex 400c2f9a69b7..d88c7c8871d1 100644--- a/posthog/dags/locations/clickhouse.py+++ b/posthog/dags/locations/clickhouse.py@@ -12,6 +12,7 @@ from posthog.dags import (export_query_log_archive_to_s3,fix_person_id_overrides,flag_evaluations_backfill,+ flag_evaluations_person_id_repair,part_breaker,person_overrides,person_pg_cleanup_drain,@@ -42,6 +43,8 @@ defs = dagster.Definitions(backfill_materialized_column.backfill_materialized_column,fix_person_id_overrides.fix_person_id_overrides_job,flag_evaluations_backfill.flag_evaluations_backfill_job,+ flag_evaluations_person_id_repair.cleanup_orphaned_flag_evaluations_person_id_repair,+ flag_evaluations_person_id_repair.flag_evaluations_person_id_repair_job,person_overrides.cleanup_orphaned_person_overrides_snapshot,person_overrides.squash_person_overrides,person_pg_cleanup_drain.person_pg_cleanup_drain_job,diff --git a/posthog/dags/tests/test_flag_evaluations_backfill.py b/posthog/dags/tests/test_flag_evaluations_backfill.pyindex 3ffb708e61a6..5cc4d887952a 100644--- a/posthog/dags/tests/test_flag_evaluations_backfill.py+++ b/posthog/dags/tests/test_flag_evaluations_backfill.py@@ -808,7 +808,7 @@ def test_backfill_waits_while_clickhouse_moves_parts_off_a_full_disk()def test_resolve_backfill_days_rejects_an_unsafe_window(overrides: dict[str, Any]) -> None:with pytest.raises(dagster.Failure):- resolve_backfill_days(FlagEvaluationsBackfillConfig(**overrides), today=date(2026, 3, 10))+ resolve_backfill_days(**overrides, today=date(2026, 3, 10))@pytest.mark.parametrize(@@ -823,6 +823,6 @@ def test_resolve_backfill_days_rejects_an_unsafe_window(overrides: dict[str, Anydef test_resolve_backfill_days_lists_each_day_newest_first(overrides: dict[str, Any], newest: date, oldest: date, count: int) -> None:- days = resolve_backfill_days(FlagEvaluationsBackfillConfig(**overrides), today=date(2026, 3, 10))+ days = resolve_backfill_days(**overrides, today=date(2026, 3, 10))assert (days[0], days[-1], len(days)) == (newest, oldest, count)diff --git a/posthog/dags/tests/test_flag_evaluations_person_id_repair.py b/posthog/dags/tests/test_flag_evaluations_person_id_repair.pynew file mode 100644index 000000000000..9bcd35b6632e--- /dev/null+++ b/posthog/dags/tests/test_flag_evaluations_person_id_repair.py@@ -0,0 +1,294 @@+from collections import Counter+from collections.abc import Sequence+from datetime import UTC, datetime, timedelta+from functools import partial+from typing import Any+from uuid import NAMESPACE_URL, UUID, uuid4, uuid5++import pytest+from unittest.mock import patch++import dagster++from posthog.clickhouse.cluster import (+ AlterTableMutationRunner,+ ClickhouseCluster,+ NodeRole,+ Query,+ wait_for_mutations_on_shards,+)+from posthog.dags import flag_evaluations_person_id_repair+from posthog.dags.deletes import deletes_job+from posthog.dags.flag_evaluations_person_id_repair import (+ FlagEvaluationsPersonIdRepairConfig,+ PersonIdRepairDictionary,+ PersonIdRepairTable,+ RepairRunConfig,+ cleanup_orphaned_flag_evaluations_person_id_repair,+ flag_evaluations_person_id_repair_job,+)+from posthog.dags.tests.conftest import insert_flag_evaluations+from posthog.dataclasses import frozen+from posthog.models.event.sql import EVENTS_DATA_TABLE+from posthog.models.flag_evaluations.sql import FLAG_EVALUATIONS_DATA_TABLE, FLAG_EVALUATIONS_SOURCE_EVENT++TEAM_ID = 1+DISTINCT_ID = "repair-user"+PERSON_A, PERSON_B, PERSON_C, PERSON_D = (UUID(int=n) for n in range(1, 5))+++@frozen+class FlagCall:+ label: str+ stored_person_id: UUID+ events_person_id: UUID+ imported: bool = True+ age: timedelta = timedelta(days=3)++ @property+ def uuid(self) -> UUID:+ return uuid5(NAMESPACE_URL, self.label)++ def timestamp(self, now: datetime) -> datetime:+ return now - self.age+++STALE = FlagCall(label="stale", stored_person_id=PERSON_A, events_person_id=PERSON_B)+KAFKA_PATH = FlagCall(label="kafka_path", stored_person_id=PERSON_A, events_person_id=PERSON_B, imported=False)+MOVED_BY_LATER_SQUASH = FlagCall(label="moved_by_later_squash", stored_person_id=PERSON_C, events_person_id=PERSON_C)+CALLS = (STALE, KAFKA_PATH, MOVED_BY_LATER_SQUASH)++# A second events copy of STALE's call would share its sort key. A merge would then collapse the two copies.+# A separate call gives the key (TEAM_ID, DISTINCT_ID, PERSON_A) a second events person_id without that risk.+CONFLICTING_CALL = FlagCall(label="conflicting_call", stored_person_id=PERSON_A, events_person_id=PERSON_D)+# A day other than STALE's, because the build validates each day on its own.+AGREEING_CALL = FlagCall(+ label="agreeing_call", stored_person_id=PERSON_A, events_person_id=PERSON_A, age=timedelta(days=4)+)+OUTSIDE_WINDOW = FlagCall(+ label="outside_window", stored_person_id=PERSON_A, events_person_id=PERSON_C, age=timedelta(days=7)+)+# With STALE this maps A to B and B to C, so B is both a target and a stale key.+CHAINED_CALL = FlagCall(label="chained_call", stored_person_id=PERSON_B, events_person_id=PERSON_C)++WINDOW_START_DAYS_AGO, WINDOW_END_DAYS_AGO = 5, 1+++def seed(cluster: ClickhouseCluster, now: datetime, calls: Sequence[FlagCall]) -> None:+ imported_rows = [+ (TEAM_ID, DISTINCT_ID, call.stored_person_id, call.uuid, call.timestamp(now)) for call in calls if call.imported+ ]+ cluster.any_host(partial(insert_flag_evaluations, imported_rows)).result()++ # A row from Kafka arrives after its event, so its inserted_at is later than its timestamp.+ # Its offset is 1 because the imported-row filter treats offset 0 in partition 0 as an import.+ kafka_rows = []+ for call in calls:+ if not call.imported:+ kafka_time = call.timestamp(now) + timedelta(seconds=1)+ kafka_rows.append(+ (+ TEAM_ID,+ DISTINCT_ID,+ call.stored_person_id,+ call.uuid,+ call.timestamp(now),+ kafka_time,+ kafka_time,+ 0,+ 1,+ )+ )+ cluster.any_host(+ Query(+ """INSERT INTO writable_flag_evaluations+ (team_id, distinct_id, person_id, uuid, timestamp, inserted_at, _timestamp, _partition, _offset)+ VALUES""",+ kafka_rows,+ )+ ).result()++ events_rows = [+ (+ call.uuid,+ FLAG_EVALUATIONS_SOURCE_EVENT,+ "{}",+ call.timestamp(now),+ TEAM_ID,+ DISTINCT_ID,+ call.timestamp(now),+ call.events_person_id,+ "{}",+ call.timestamp(now),+ "full",+ call.timestamp(now),+ )+ for call in calls+ ]+ cluster.any_host_by_role(+ Query(+ f"""INSERT INTO {EVENTS_DATA_TABLE()}+ (uuid, event, properties, timestamp, team_id, distinct_id, created_at, person_id,+ person_properties, person_created_at, person_mode, _timestamp)+ VALUES""",+ events_rows,+ ),+ NodeRole.DATA,+ ).result()+++def stored_person_ids(cluster: ClickhouseCluster, calls: Sequence[FlagCall]) -> Counter[tuple[str, UUID]]:+ labels = {call.uuid: call.label for call in calls}+ rows = cluster.any_host_by_role(Query("SELECT uuid, person_id FROM flag_evaluations"), NodeRole.DATA).result()+ return Counter((labels.get(uuid, str(uuid)), person_id) for uuid, person_id in rows)+++def days_before(now: datetime, days: int) -> str:+ return (now - timedelta(days=days)).date().isoformat()+++def run_repair(+ cluster: ClickhouseCluster, now: datetime, instance: dagster.DagsterInstance | None = None, **overrides: Any+) -> dagster.ExecuteInProcessResult:+ config = FlagEvaluationsPersonIdRepairConfig(+ start_date=days_before(now, WINDOW_START_DAYS_AGO), end_date=days_before(now, WINDOW_END_DAYS_AGO), **overrides+ )+ return flag_evaluations_person_id_repair_job.execute_in_process(+ run_config=dagster.RunConfig(ops={"plan_person_id_repair": config}),+ resources={"cluster": cluster},+ instance=instance or dagster.DagsterInstance.ephemeral(),+ raise_on_error=False,+ )+++def run_cleanup(cluster: ClickhouseCluster, run_id: UUID) -> dagster.ExecuteInProcessResult:+ return cleanup_orphaned_flag_evaluations_person_id_repair.execute_in_process(+ run_config=dagster.RunConfig(ops={"get_repair_dictionary_for_run": RepairRunConfig(run_id=str(run_id))}),+ resources={"cluster": cluster},+ instance=dagster.DagsterInstance.ephemeral(),+ raise_on_error=False,+ )+++def on_any_host(cluster: ClickhouseCluster, fn: Any) -> bool:+ return any(cluster.map_all_hosts(fn).result().values())+++def drop_repair_resources(cluster: ClickhouseCluster, run_id: UUID) -> None:+ dictionary = PersonIdRepairDictionary(source=PersonIdRepairTable(id=run_id))+ cluster.map_all_hosts(dictionary.drop).result()+ cluster.map_all_hosts(dictionary.source.drop).result()+++ "dry_run, extra_calls, runs, succeeds, rewritten",+ [+ # The second run must change nothing the first one left.+ pytest.param(False, (), 2, True, {STALE.label: PERSON_B}, id="repair"),+ pytest.param(True, (), 1, True, {}, id="dry_run"),+ pytest.param(False, (CONFLICTING_CALL,), 1, False, {}, id="conflicting_events_copies"),+ pytest.param(False, (AGREEING_CALL,), 1, False, {}, id="agreeing_events_copy"),+ pytest.param(False, (CHAINED_CALL,), 1, False, {}, id="chained_targets"),+ pytest.param(False, (OUTSIDE_WINDOW,), 1, True, {STALE.label: PERSON_B}, id="outside_window"),+ ],+)+def test_repair_rewrites_only_imported_rows_that_still_hold_the_stale_id(+ cluster: ClickhouseCluster,+ dry_run: bool,+ extra_calls: tuple[FlagCall, ...],+ runs: int,+ succeeds: bool,+ rewritten: dict[str, UUID],+) -> None:+ # nosemgrep: test-datetime-now-without-freeze (the rows and both window bounds are offsets from this one reading, so passing UTC midnight changes no assertion)+ now = datetime.now(UTC)+ calls = (*CALLS, *extra_calls)+ seed(cluster, now, calls)++ results = [run_repair(cluster, now, dry_run=dry_run) for _ in range(runs)]++ run_ids = [UUID(result.dagster_run.run_id) for result in results]+ try:+ assert [result.success for result in results] == [succeeds] * runs+ expected = {call.label: call.stored_person_id for call in calls} | rewritten+ assert stored_person_ids(cluster, calls) == Counter(expected.items())+ for run_id in run_ids:+ table = PersonIdRepairTable(id=run_id)+ dictionary = PersonIdRepairDictionary(source=table)+ assert on_any_host(cluster, table.exists) is not succeeds+ if succeeds:+ assert not on_any_host(cluster, dictionary.exists)+ finally:+ for run_id in run_ids:+ drop_repair_resources(cluster, run_id)+++def test_repair_stops_before_the_mutation_when_a_deletes_run_starts_during_the_build(+ cluster: ClickhouseCluster,+) -> None:+ # nosemgrep: test-datetime-now-without-freeze (the rows and both window bounds are offsets from this one reading, so passing UTC midnight changes no assertion)+ now = datetime.now(UTC)+ seed(cluster, now, CALLS)+ instance = dagster.DagsterInstance.ephemeral()+ on_shard_host = flag_evaluations_person_id_repair.on_shard_host++ def start_a_deletes_run(*args: Any, **kwargs: Any) -> int:+ instance.create_run_for_job(job_def=deletes_job, status=dagster.DagsterRunStatus.STARTED)+ return on_shard_host(*args, **kwargs)++ with patch.object(flag_evaluations_person_id_repair, "on_shard_host", side_effect=start_a_deletes_run):+ result = run_repair(cluster, now, instance=instance)++ run_id = UUID(result.dagster_run.run_id)+ try:+ [failure] = result.get_step_failure_events()+ assert failure.step_failure_data.error is not None+ assert "started after the repair read person_id" in failure.step_failure_data.error.message+ assert stored_person_ids(cluster, CALLS) == Counter((call.label, call.stored_person_id) for call in CALLS)+ assert not on_any_host(cluster, PersonIdRepairTable(id=run_id).exists)+ assert not on_any_host(cluster, PersonIdRepairDictionary(source=PersonIdRepairTable(id=run_id)).exists)+ finally:+ drop_repair_resources(cluster, run_id)+++def test_cleanup_refuses_while_a_repair_mutation_is_unfinished(cluster: ClickhouseCluster) -> None:+ # nosemgrep: test-datetime-now-without-freeze (the row and the mutation's window are offsets from this one reading, so passing UTC midnight changes no assertion)+ now = datetime.now(UTC)+ seed(cluster, now, (STALE,))+ table = PersonIdRepairTable(id=uuid4())+ dictionary = PersonIdRepairDictionary(source=table)+ runner = AlterTableMutationRunner(+ table=FLAG_EVALUATIONS_DATA_TABLE,+ commands={dictionary.update_command(filter_team_ids=False)},+ parameters={+ "name": dictionary.qualified_name,+ "window_start": f"{days_before(now, WINDOW_START_DAYS_AGO)} 00:00:00",+ "window_end": f"{days_before(now, WINDOW_END_DAYS_AGO)} 00:00:00",+ },+ # The test client otherwise waits for the mutation, which the stopped merges hold open.+ settings={"mutations_sync": 0},+ )+ # Stopped merges also stop mutations, which leaves this one unfinished on every replica.+ start_merges = Query(f"SYSTEM START MERGES {FLAG_EVALUATIONS_DATA_TABLE}")+ try:+ cluster.map_all_hosts(table.create).result()+ cluster.map_all_hosts(partial(dictionary.create, shards=1, max_execution_time=0, max_memory_usage=0)).result()+ cluster.map_all_hosts(dictionary.load).result()+ cluster.map_all_hosts(Query(f"SYSTEM STOP MERGES {FLAG_EVALUATIONS_DATA_TABLE}")).result()+ mutations = runner.enqueue_on_shards(cluster)++ assert not run_cleanup(cluster, table.id).success+ assert on_any_host(cluster, dictionary.exists)++ cluster.map_all_hosts(start_merges).result()+ wait_for_mutations_on_shards(cluster, mutations)++ assert run_cleanup(cluster, table.id).success+ assert not on_any_host(cluster, dictionary.exists)+ assert not on_any_host(cluster, table.exists)+ finally:+ cluster.map_all_hosts(start_merges).result()+ drop_repair_resources(cluster, table.id)