Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(flags): add a job that repairs person_id on imported flag_evaluations rows (#119898)

Phil Haack0ba4963868e9
diff --git a/posthog/dags/flag_evaluations_backfill.py b/posthog/dags/flag_evaluations_backfill.py
index 3d668a8cfca9..ca8a6661d874 100644
--- a/posthog/dags/flag_evaluations_backfill.py
+++ b/posthog/dags/flag_evaluations_backfill.py
@@ -52,7 +52,7 @@ _FINISHED_RUN_STATUSES = (
# it does for rows from Kafka. inserted_at is left out so its DEFAULT stamps the event timestamp:
# that keeps every copied row inside the deletion sweep's `inserted_at <= request.created_at` bound,
# and it keeps copied rows out of the consumer-lag query below. _partition and _offset are left out
-# so that both are 0 on a copied row. _COPIED_ROW_FILTER relies on that.
+# so that both are 0 on a copied row. COPIED_ROW_FILTER relies on that.
_COPIED_COLUMNS = (
"uuid",
"event",
@@ -77,7 +77,7 @@ _COPIED_VALUES = ", ".join(
# A row from Kafka also has inserted_at = timestamp when its timestamp has no sub-second part and
# falls in the second that Kafka received it. Kafka gives offset 0 only to the first message in a
# partition, so the only Kafka row this filter can match is the first message in partition 0.
-_COPIED_ROW_FILTER = "_partition = 0 AND _offset = 0 AND inserted_at = timestamp"
+COPIED_ROW_FILTER = "_partition = 0 AND _offset = 0 AND inserted_at = timestamp"
# A copied row's inserted_at is its timestamp, so the inserted_at != timestamp filter limits this
# query to rows that arrived through the Kafka path.
@@ -235,11 +235,13 @@ def earliest_backfill_day(today: date) -> date:
return today - timedelta(days=_EARLIEST_START_DAYS_AGO)
-def resolve_backfill_days(config: FlagEvaluationsBackfillConfig, *, today: date) -> tuple[date, ...]:
+def resolve_backfill_days(
+ *, start_date: str | None = None, end_date: str | None = None, today: date
+) -> tuple[date, ...]:
latest_end = today - timedelta(days=1)
earliest_start = earliest_backfill_day(today)
- start = date.fromisoformat(config.start_date) if config.start_date else earliest_start
- end = date.fromisoformat(config.end_date) if config.end_date else latest_end
+ start = date.fromisoformat(start_date) if start_date else earliest_start
+ end = date.fromisoformat(end_date) if end_date else latest_end
if start < earliest_start:
raise dagster.Failure(
description=f"start_date {start} is before {earliest_start}. "
@@ -303,7 +305,7 @@ def plan_flag_evaluations_backfill(
context: dagster.OpExecutionContext, config: FlagEvaluationsBackfillConfig
) -> BackfillPlan:
"""Resolve the window once, so a shard re-executed from the UI later copies the same days."""
- days = resolve_backfill_days(config, today=datetime.now(UTC).date())
+ days = resolve_backfill_days(start_date=config.start_date, end_date=config.end_date, today=datetime.now(UTC).date())
context.log.info(f"Backfilling {len(days)} day(s), newest first, from {days[0]} back to {days[-1]}")
return BackfillPlan(days=days, config=config)
@@ -434,39 +436,10 @@ class ShardBackfill:
)
def wait_for_blocking_runs(self) -> BlockingRunCheck:
- while True:
- # The watermark comes before the scan. A run created during the scan then counts as created
- # after it. Run storage can record a creation time to the whole second. The watermark
- # therefore reaches one second back, and the post-copy check ignores runs that finished then.
- since = datetime.now(UTC) - timedelta(seconds=1)
- blockers = describe_runs(self.instance, BLOCKING_JOB_NAMES, exclude_run_id=self.run_id)
- if not blockers:
- break
- self.log.info(
- f"Waiting {self.config.blocking_run_poll_seconds}s for these runs to finish: {'; '.join(blockers)}"
- )
- time.sleep(self.config.blocking_run_poll_seconds)
- finished = describe_runs(
- self.instance,
- BLOCKING_JOB_NAMES,
- statuses=_FINISHED_RUN_STATUSES,
- created_after=since,
- exclude_run_id=self.run_id,
- )
- return BlockingRunCheck(since=since, finished_runs=frozenset(finished))
+ return wait_for_blocking_runs(self.instance, self.run_id, self.log, self.config.blocking_run_poll_seconds)
def check_no_blocking_run_started(self, check: BlockingRunCheck, *, day: date) -> None:
- started = [
- run
- for run in describe_runs(
- self.instance,
- BLOCKING_JOB_NAMES,
- statuses=(*EXECUTING_RUN_STATUSES, *_FINISHED_RUN_STATUSES),
- created_after=check.since,
- exclude_run_id=self.run_id,
- )
- if run not in check.finished_runs
- ]
+ started = blocking_runs_started_since(self.instance, check, self.run_id)
if started:
# A rerun for the same teams does not copy back the rows that earlier runs copied for other teams.
team_filter = (
@@ -477,7 +450,7 @@ class ShardBackfill:
"The copy can hold rows or person_ids that the run removed from sharded_flag_evaluations. "
f"After that run finishes, delete the rows this job copied for {day} on shard {self.shard_num} "
f"(`DELETE FROM {FLAG_EVALUATIONS_DATA_TABLE} WHERE toDate(timestamp) = '{day}' "
- f"AND {_COPIED_ROW_FILTER}{team_filter}`), then run the backfill again for the same teams."
+ f"AND {COPIED_ROW_FILTER}{team_filter}`), then run the backfill again for the same teams."
)
def wait_for_disk_headroom(self) -> None:
@@ -502,7 +475,7 @@ class ShardBackfill:
"""
# Every replica of the shard, including offline ones, stores a copy of each inserted part.
disks_by_host = self.cluster.map_hosts_in_shard_by_role(
- self.shard_num, self._tagged(_read_policy_disks), node_role=self.node_role
+ self.shard_num, tagged(self.query_tags, _read_policy_disks), node_role=self.node_role
).result()
if not disks_by_host:
raise dagster.Failure(description=f"No replica of shard {self.shard_num} reported its disks.")
@@ -568,30 +541,78 @@ class ShardBackfill:
)
if self.config.dry_run:
return result[0][0]
- # sync_execute returns the written row count for an INSERT that wrote rows. It returns
- # an empty result for an INSERT that wrote none.
- return result if isinstance(result, int) else 0
+ return inserted_rows(result)
return sum(self._on_copy_host(partial(run_chunk, chunk)) for chunk in range(self.config.team_id_chunks))
def _first_row(self, query: str, client: Client) -> tuple:
return sync_execute(query, workload=self.workload, sync_client=client)[0]
- def _tagged(self, fn: Callable[[Client], T]) -> Callable[[Client], T]:
- # The cluster runs fn on a pool thread. The query tags do not cross into that thread.
- def tagged(client: Client) -> T:
- with tags_context(
- kind="dagster", dagster=self.query_tags, product=ProductKey.FEATURE_FLAGS, feature=Feature.BACKFILL
- ):
- return fn(client)
+ def _on_copy_host(self, fn: Callable[[Client], T]) -> T:
+ return on_shard_host(self.cluster, self.shard_num, fn, self.query_tags, self.workload, self.node_role)
+
+
+def wait_for_blocking_runs(
+ instance: dagster.DagsterInstance, run_id: str, log: dagster.DagsterLogManager, poll_seconds: int
+) -> BlockingRunCheck:
+ """Wait until no blocking run is active, then return the watermark for blocking_runs_started_since."""
+ while True:
+ # The watermark comes before the scan. A run created during the scan then counts as created
+ # after it. Run storage can record a creation time to the whole second. The watermark
+ # therefore reaches one second back, and blocking_runs_started_since ignores runs that finished then.
+ since = datetime.now(UTC) - timedelta(seconds=1)
+ blockers = describe_runs(instance, BLOCKING_JOB_NAMES, exclude_run_id=run_id)
+ if not blockers:
+ break
+ log.info(f"Waiting {poll_seconds}s for these runs to finish: {'; '.join(blockers)}")
+ time.sleep(poll_seconds)
+ finished = describe_runs(
+ instance, BLOCKING_JOB_NAMES, statuses=_FINISHED_RUN_STATUSES, created_after=since, exclude_run_id=run_id
+ )
+ return BlockingRunCheck(since=since, finished_runs=frozenset(finished))
- return tagged
- def _on_copy_host(self, fn: Callable[[Client], T]) -> T:
- result = self.cluster.map_any_host_in_shards_by_role(
- {self.shard_num: self._tagged(fn)}, workload=self.workload, node_role=self.node_role
- ).result()
- return next(iter(result.values()))
+def blocking_runs_started_since(instance: dagster.DagsterInstance, check: BlockingRunCheck, run_id: str) -> list[str]:
+ statuses = (*EXECUTING_RUN_STATUSES, *_FINISHED_RUN_STATUSES)
+ runs = describe_runs(
+ instance, BLOCKING_JOB_NAMES, statuses=statuses, created_after=check.since, exclude_run_id=run_id
+ )
+ return [run for run in runs if run not in check.finished_runs]
+
+
+def shard_workload() -> tuple[Workload, NodeRole]:
+ return (Workload.OFFLINE, NodeRole.DATA) if is_cloud() else (Workload.DEFAULT, NodeRole.ALL)
+
+
+def tagged(query_tags: DagsterTags, fn: Callable[[Client], T]) -> Callable[[Client], T]:
+ # The cluster runs fn on a pool thread. The query tags do not cross into that thread.
+ def run(client: Client) -> T:
+ with tags_context(
+ kind="dagster", dagster=query_tags, product=ProductKey.FEATURE_FLAGS, feature=Feature.BACKFILL
+ ):
+ return fn(client)
+
+ return run
+
+
+def on_shard_host(
+ cluster: ClickhouseCluster,
+ shard_num: int,
+ fn: Callable[[Client], T],
+ query_tags: DagsterTags,
+ workload: Workload,
+ node_role: NodeRole,
+) -> T:
+ result = cluster.map_any_host_in_shards_by_role(
+ {shard_num: tagged(query_tags, fn)}, workload=workload, node_role=node_role
+ ).result()
+ return next(iter(result.values()))
+
+
+def inserted_rows(result: Any) -> int:
+ # sync_execute returns the written row count for an INSERT that wrote rows. It returns
+ # an empty result for an INSERT that wrote none.
+ return result if isinstance(result, int) else 0
def _read_policy_disks(client: Client) -> list[PolicyDisk]:
@@ -608,7 +629,7 @@ def backfill_flag_evaluations_shard(
shard_num: int,
plan: BackfillPlan,
) -> int:
- workload, node_role = (Workload.OFFLINE, NodeRole.DATA) if is_cloud() else (Workload.DEFAULT, NodeRole.ALL)
+ workload, node_role = shard_workload()
totals = ShardBackfill(
cluster=cluster,
shard_num=shard_num,
@@ -637,7 +658,7 @@ def flag_evaluations_backfill_job() -> None:
Safe to re-run over the same window: the anti-join inserts only rows that are missing.
A re-run does not rewrite a stored row. To copy a day again, first delete the rows that an earlier
- run copied, which match _COPIED_ROW_FILTER. Delete a row that the Kafka path wrote only when events
+ run copied, which match COPIED_ROW_FILTER. Delete a row that the Kafka path wrote only when events
also holds the call. Events does not hold the calls of an organization on FLAG_EVALUATIONS_ONLY.
"""
plan = plan_flag_evaluations_backfill()
diff --git a/posthog/dags/flag_evaluations_person_id_repair.py b/posthog/dags/flag_evaluations_person_id_repair.py
new file mode 100644
index 000000000000..2d69a1acf872
--- /dev/null
+++ b/posthog/dags/flag_evaluations_person_id_repair.py
@@ -0,0 +1,584 @@
+"""Rewrite the person_id of imported flag_evaluations rows that missed a person_id rewrite.
+
+A bulk copy into flag_evaluations can read events before a squash rewrites person_id there and
+attach its rows after the squash has swept flag_evaluations. The squash then deletes the overrides
+it applied, so those rows keep the merged-away person_id and no override is left for reads to
+correct. This one-off job copies the current person_id from each row's events copy, through the
+squash's dictionary-and-mutation lifecycle.
+"""
+
+import uuid
+from collections.abc import Callable, Iterator
+from datetime import UTC, date, datetime, timedelta
+from functools import partial
+from typing import Any
+
+import dagster
+import pydantic
+from clickhouse_driver import Client
+
+from posthog import settings
+from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.cluster import AlterTableMutationRunner, ClickhouseCluster
+from posthog.dags.common import dagster_tags
+from posthog.dags.common.dictionaries import Dictionary
+from posthog.dags.common.staged_dictionary import (
+ StagedDictionary,
+ create_on_every_cluster,
+ load_and_verify_on_every_cluster,
+)
+from posthog.dags.flag_evaluations_backfill import (
+ COPIED_ROW_FILTER,
+ GIB,
+ OWNER_TAG,
+ BlockingRunCheck,
+ blocking_runs_started_since,
+ inserted_rows,
+ on_shard_host,
+ resolve_backfill_days,
+ shard_workload,
+ wait_for_blocking_runs,
+)
+from posthog.dataclasses import frozen
+from posthog.models.deletion_targets import FLAG_EVALUATIONS, dispatchable_here
+from posthog.models.event.sql import EVENTS_DATA_TABLE
+from posthog.models.flag_evaluations.sql import FLAG_EVALUATIONS_DATA_TABLE, FLAG_EVALUATIONS_SOURCE_EVENT
+
+_TEAM_FILTER = " AND team_id IN %(team_ids)s"
+
+_CLEANUP = "Drop this run's table and dictionary with cleanup_orphaned_flag_evaluations_person_id_repair."
+
+
+class FlagEvaluationsPersonIdRepairConfig(dagster.Config):
+ start_date: str | None = pydantic.Field(
+ default=None,
+ description="First day to repair (YYYY-MM-DD, UTC, inclusive). Same default and bounds as the flag_evaluations backfill.",
+ )
+ end_date: str | None = pydantic.Field(
+ default=None,
+ description="Day after the last day to repair (YYYY-MM-DD, UTC, exclusive). Same default and bounds as the flag_evaluations backfill.",
+ )
+ team_ids: list[int] | None = pydantic.Field(
+ default=None, description="Repair only these teams. Leave empty to repair every team."
+ )
+ dry_run: bool = pydantic.Field(
+ default=False,
+ description="Build the repair table and report what it holds, then drop it without changing flag_evaluations.",
+ )
+ blocking_run_poll_seconds: int = pydantic.Field(
+ default=5 * 60,
+ description="How often to check again while a squash, deletes or data deletion request run is active.",
+ )
+ build_max_execution_time_seconds: int = pydantic.Field(
+ default=30 * 60, description="ClickHouse max_execution_time for each shard's insert of one day."
+ )
+ build_max_memory_usage_bytes: int = pydantic.Field(
+ default=32 * GIB, description="ClickHouse max_memory_usage for each shard's insert of one day."
+ )
+ dictionary_shards: int = pydantic.Field(default=16, description="SHARDS of the dictionary's hashed layout.")
+ dictionary_max_execution_time: int = pydantic.Field(
+ default=0, description="Seconds the dictionary may take to load, or 0 for no limit."
+ )
+ dictionary_max_memory_usage: int = pydantic.Field(
+ default=0, description="Bytes the dictionary may use while it loads, or 0 for no limit."
+ )
+
+
+@frozen
+class RepairPlan:
+ days: tuple[date, ...]
+ config: FlagEvaluationsPersonIdRepairConfig
+ blocking_run_check: BlockingRunCheck
+
+
+@frozen
+class ShardRepairCounts:
+ calls: int
+ distinct_ids: int
+ keys: int
+
+
+@frozen
+class RepairTableCheck:
+ conflicts: int
+ chains: int
+ shards: dict[int, ShardRepairCounts]
+
+
+@frozen
+class PersonIdRepairTable:
+ """Maps (team_id, distinct_id, stale_person_id) to the person_id that the row's events copy holds.
+
+ A distinct_id that merged twice can hold two stale ids, so the stale id is part of the key.
+ A key that maps to two person_ids stays as two rows, which is how the check finds it.
+ """
+
+ id: uuid.UUID
+
+ @property
+ def name(self) -> str:
+ # The dictionary reads its source as dagster_dict_reader. That user can read only the tables
+ # that infra grants it. The grant for the squash's snapshot tables matches this prefix.
+ # The prefix also matches the snapshot glob in posthog/clickhouse/hcl/exclude.hcl.
+ return f"person_distinct_id_overrides_snapshot_flag_evaluations_repair_{self.id.hex}"
+
+ @property
+ def qualified_name(self) -> str:
+ return f"{settings.CLICKHOUSE_DATABASE}.{self.name}"
+
+ def create(self, client: Client) -> None:
+ client.execute(
+ f"""
+ CREATE TABLE IF NOT EXISTS {self.qualified_name} (
+ team_id Int64,
+ distinct_id String,
+ stale_person_id UUID,
+ person_id UUID,
+ shard_num UInt32,
+ day Date,
+ calls UInt64
+ )
+ ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/noshard/{self.qualified_name}', '{{replica}}-{{shard}}')
+ ORDER BY (team_id, distinct_id, stale_person_id, person_id, shard_num, day)
+ """
+ )
+
+ def exists(self, client: Client) -> bool:
+ [[count]] = client.execute(
+ "SELECT count() FROM system.tables WHERE database = %(database)s AND name = %(name)s",
+ {"database": settings.CLICKHOUSE_DATABASE, "name": self.name},
+ )
+ return count > 0
+
+ def drop(self, client: Client) -> None:
+ client.execute(f"DROP TABLE IF EXISTS {self.qualified_name} SYNC")
+
+ def sync(self, client: Client) -> None:
+ client.execute(f"SYSTEM SYNC REPLICA {self.qualified_name} STRICT")
+
+ def stale_keys_query(self, *, filter_team_ids: bool) -> str:
+ team_filter = _TEAM_FILTER if filter_team_ids else ""
+ # sharded_events and sharded_flag_evaluations share a shard key. A row's events copy is
+ # therefore on its shard.
+ return f"""
+INSERT INTO {self.qualified_name} (team_id, distinct_id, stale_person_id, person_id, shard_num, day, calls)
+SELECT f.team_id, f.distinct_id, f.person_id, e.person_id, %(shard_num)s, toDate(%(day_start)s), count()
+FROM (
+ SELECT team_id, uuid, distinct_id, person_id
+ FROM {FLAG_EVALUATIONS_DATA_TABLE}
+ WHERE timestamp >= %(day_start)s AND timestamp < %(day_end)s AND {COPIED_ROW_FILTER}{team_filter}
+) AS f
+INNER JOIN (
+ SELECT team_id, uuid, distinct_id, person_id
+ FROM {EVENTS_DATA_TABLE()}
+ WHERE event = %(event)s AND timestamp >= %(day_start)s AND timestamp < %(day_end)s{team_filter}
+) AS e ON f.team_id = e.team_id AND f.uuid = e.uuid AND f.distinct_id = e.distinct_id
+WHERE f.person_id != e.person_id
+GROUP BY f.team_id, f.distinct_id, f.person_id, e.person_id
+"""
+
+ def agreeing_calls_query(self) -> str:
+ # A stale key's calls on any day may still agree with their events copies. This records
+ # each such key with its own id as a second target, so the check fails it instead of the
+ # mutation rewriting the agreeing calls. Both sides read only the shard's stale keys.
+ stale_keys = (
+ f"SELECT team_id, distinct_id, stale_person_id FROM {self.qualified_name} "
+ "WHERE shard_num = %(shard_num)s AND person_id != stale_person_id"
+ )
+ return f"""
+INSERT INTO {self.qualified_name} (team_id, distinct_id, stale_person_id, person_id, shard_num, day, calls)
+SELECT f.team_id, f.distinct_id, f.person_id, f.person_id, %(shard_num)s, toDate(%(day_start)s), count()
+FROM (
+ SELECT team_id, uuid, distinct_id, person_id
+ FROM {FLAG_EVALUATIONS_DATA_TABLE}
+ WHERE timestamp >= %(day_start)s AND timestamp < %(day_end)s AND {COPIED_ROW_FILTER}
+ AND (team_id, distinct_id, person_id) IN ({stale_keys})
+) AS f
+INNER JOIN (
+ SELECT team_id, uuid, distinct_id, person_id
+ FROM {EVENTS_DATA_TABLE()}
+ WHERE event = %(event)s AND timestamp >= %(day_start)s AND timestamp < %(day_end)s
+ AND (team_id, distinct_id) IN (SELECT team_id, distinct_id FROM ({stale_keys}))
+) AS e ON f.team_id = e.team_id AND f.uuid = e.uuid AND f.distinct_id = e.distinct_id
+WHERE f.person_id = e.person_id
+GROUP BY f.team_id, f.distinct_id, f.person_id
+"""
+
+
+@frozen
+class PersonIdRepairDictionary(Dictionary):
+ source: PersonIdRepairTable
+
+ @property
+ def name(self) -> str:
+ return f"{self.source.name}_dictionary"
+
+ @property
+ def schema(self) -> str:
+ return "team_id Int64, distinct_id String, stale_person_id UUID, person_id UUID"
+
+ @property
+ def primary_key(self) -> str:
+ return "team_id, distinct_id, stale_person_id"
+
+ @property
+ def query(self) -> str:
+ # The check proves one person_id per key before this loads. any() is therefore exact.
+ return (
+ f"SELECT team_id, distinct_id, stale_person_id, any(person_id) AS person_id "
+ f"FROM {self.source.qualified_name} GROUP BY team_id, distinct_id, stale_person_id"
+ )
+
+ def staged(self) -> StagedDictionary:
+ return StagedDictionary(
+ key=f"{self.name}.parquet",
+ columns="team_id, distinct_id, stale_person_id, person_id",
+ structure=self.schema,
+ )
+
+ def update_command(self, *, filter_team_ids: bool) -> str:
+ # The key includes the row's current person_id. A row matches only while it still holds the
+ # stale id. A rerun therefore changes nothing. A row that a later squash moved stays where it is.
+ # The build checked only the rows inside the window, so the mutation stays inside it too.
+ team_filter = _TEAM_FILTER if filter_team_ids else ""
+ return (
+ "UPDATE person_id = dictGet(%(name)s, 'person_id', (team_id, distinct_id, person_id)) "
+ "WHERE timestamp >= %(window_start)s AND timestamp < %(window_end)s "
+ f"AND {COPIED_ROW_FILTER} AND dictHas(%(name)s, (team_id, distinct_id, person_id)){team_filter}"
+ )
+
+
[email protected](tags=OWNER_TAG)
+def plan_person_id_repair(
+ context: dagster.OpExecutionContext,
+ cluster: dagster.ResourceParam[ClickhouseCluster],
+ config: FlagEvaluationsPersonIdRepairConfig,
+) -> RepairPlan:
+ """Resolve the days, then wait until no squash, deletes or data deletion request run is active.
+
+ The build reads person_id from events, so the watermark taken here covers it as well as the mutation.
+ """
+ if not dispatchable_here(cluster, FLAG_EVALUATIONS):
+ raise dagster.Failure(description=f"{FLAG_EVALUATIONS_DATA_TABLE} does not exist on this cluster.")
+ days = resolve_backfill_days(start_date=config.start_date, end_date=config.end_date, today=datetime.now(UTC).date())
+ check = wait_for_blocking_runs(context.instance, context.run_id, context.log, config.blocking_run_poll_seconds)
+ context.log.info(f"Repairing {len(days)} day(s), newest first, from {days[0]} back to {days[-1]}")
+ return RepairPlan(days=days, config=config, blocking_run_check=check)
+
+
[email protected](tags=OWNER_TAG)
+def create_repair_table(
+ context: dagster.OpExecutionContext,
+ cluster: dagster.ResourceParam[ClickhouseCluster],
+ plan: RepairPlan,
+) -> PersonIdRepairTable:
+ table = PersonIdRepairTable(id=uuid.UUID(context.run.run_id))
+ cluster.map_all_hosts(table.create).result()
+ return table
+
+
[email protected](out=dagster.DynamicOut(int), tags=OWNER_TAG)
+def get_repair_shards(
+ cluster: dagster.ResourceParam[ClickhouseCluster], table: PersonIdRepairTable
+) -> Iterator[dagster.DynamicOutput[int]]:
+ """Fan out one build op per shard, so a failed shard can be re-executed on its own from the UI."""
+ for shard_num in sorted(cluster.shards):
+ yield dagster.DynamicOutput(shard_num, mapping_key=f"shard_{shard_num}")
+
+
[email protected](tags=OWNER_TAG)
+def build_repair_shard(
+ context: dagster.OpExecutionContext,
+ cluster: dagster.ResourceParam[ClickhouseCluster],
+ shard_num: int,
+ plan: RepairPlan,
+ table: PersonIdRepairTable,
+) -> int:
+ """Insert this shard's stale keys one day per query, then the keys whose calls also agree with events.
+
+ A rerun inserts the same rows, which the table collapses.
+ """
+ config = plan.config
+ workload, node_role = shard_workload()
+ stale_keys_query = table.stale_keys_query(filter_team_ids=bool(config.team_ids))
+ agreeing_calls_query = table.agreeing_calls_query()
+ query_settings = {
+ "max_execution_time": config.build_max_execution_time_seconds,
+ "max_memory_usage": config.build_max_memory_usage_bytes,
+ }
+
+ def day_args(day: date) -> dict[str, Any]:
+ return {
+ "shard_num": shard_num,
+ "event": FLAG_EVALUATIONS_SOURCE_EVENT,
+ "day_start": f"{day:%Y-%m-%d} 00:00:00",
+ "day_end": f"{day + timedelta(days=1):%Y-%m-%d} 00:00:00",
+ "team_ids": tuple(config.team_ids or ()),
+ }
+
+ def insert(query: str, day: date, client: Client) -> int:
+ return inserted_rows(
+ sync_execute(query, day_args(day), settings=query_settings, workload=workload, sync_client=client)
+ )
+
+ def insert_agreeing_calls(client: Client) -> int:
+ # The stale keys came from inserts on any replica. The sync makes this host read all of them.
+ table.sync(client)
+ return sum(insert(agreeing_calls_query, day, client) for day in plan.days)
+
+ def on_this_shard(fn: Callable[[Client], int]) -> int:
+ return on_shard_host(cluster, shard_num, fn, dagster_tags(context), workload, node_role)
+
+ stale_keys = 0
+ for day in plan.days:
+ keys = on_this_shard(partial(insert, stale_keys_query, day))
+ stale_keys += keys
+ context.log.info(f"Shard {shard_num}, {day}: {keys} stale key(s)")
+ agreeing_keys = on_this_shard(insert_agreeing_calls)
+ context.log.info(f"Shard {shard_num}: {agreeing_keys} stale key-day(s) with calls that agree with events")
+ context.add_output_metadata(
+ {
+ "shard": dagster.MetadataValue.int(shard_num),
+ "keys": dagster.MetadataValue.int(stale_keys),
+ "agreeing": dagster.MetadataValue.int(agreeing_keys),
+ }
+ )
+ return stale_keys
+
+
[email protected](tags=OWNER_TAG)
+def check_repair_table(
+ context: dagster.OpExecutionContext,
+ cluster: dagster.ResourceParam[ClickhouseCluster],
+ table: PersonIdRepairTable,
+ built_shards: list[int],
+) -> PersonIdRepairTable:
+ """Fail unless each key maps to one person_id that is not itself a stale key, and report what the table holds.
+
+ The report counts the calls the build joined. The mutation changes fewer if a squash moves a row first.
+ """
+ cluster.map_all_hosts(table.sync).result()
+
+ def read(client: Client) -> RepairTableCheck:
+ [[conflicts]] = client.execute(
+ f"""
+ SELECT count() FROM (
+ SELECT 1 FROM {table.qualified_name}
+ GROUP BY team_id, distinct_id, stale_person_id
+ HAVING uniqExact(person_id) > 1
+ )
+ """
+ )
+ # A target that is also a stale key would move a row once per run instead of once in total.
+ # A key that maps to its own id comes from the agreeing-calls pass and already counts as a conflict.
+ [[chains]] = client.execute(
+ f"""
+ SELECT count() FROM (
+ SELECT DISTINCT team_id, distinct_id, person_id FROM {table.qualified_name}
+ WHERE person_id != stale_person_id
+ )
+ WHERE (team_id, distinct_id, person_id) IN (
+ SELECT team_id, distinct_id, stale_person_id FROM {table.qualified_name}
+ )
+ """
+ )
+ shards = {
+ shard: ShardRepairCounts(calls=calls, distinct_ids=distinct_ids, keys=keys)
+ for shard, calls, distinct_ids, keys in client.execute(
+ f"""
+ SELECT shard_num, sum(calls), uniqExact(team_id, distinct_id), uniqExact(team_id, distinct_id, stale_person_id)
+ FROM {table.qualified_name} FINAL
+ GROUP BY shard_num
+ """
+ )
+ }
+ return RepairTableCheck(conflicts=conflicts, chains=chains, shards=shards)
+
+ check = cluster.any_host(read).result()
+ report = {
+ f"shard_{shard}": {"calls": counts.calls, "distinct_ids": counts.distinct_ids, "keys": counts.keys}
+ for shard, counts in sorted(check.shards.items())
+ }
+ context.add_output_metadata(
+ {
+ **{
+ total: dagster.MetadataValue.int(sum(counts[total] for counts in report.values()))
+ for total in ("calls", "distinct_ids", "keys")
+ },
+ "per_shard": dagster.MetadataValue.json(report),
+ "conflicts": dagster.MetadataValue.int(check.conflicts),
+ "chains": dagster.MetadataValue.int(check.chains),
+ }
+ )
+ if check.conflicts or check.chains:
+ raise dagster.Failure(
+ description=f"{table.qualified_name} has {check.conflicts} key(s) that map to more than one person_id and "
+ f"{check.chains} target(s) that are also stale keys. Inspect the table. {_CLEANUP}"
+ )
+ return table
+
+
[email protected](tags=OWNER_TAG)
+def create_repair_dictionary(
+ context: dagster.OpExecutionContext,
+ cluster: dagster.ResourceParam[ClickhouseCluster],
+ plan: RepairPlan,
+ table: PersonIdRepairTable,
+) -> PersonIdRepairDictionary:
+ dictionary = PersonIdRepairDictionary(source=table)
+ create_on_every_cluster(
+ context,
+ [cluster],
+ dictionary,
+ shards=plan.config.dictionary_shards,
+ max_execution_time=plan.config.dictionary_max_execution_time,
+ max_memory_usage=plan.config.dictionary_max_memory_usage,
+ )
+ return dictionary
+
+
[email protected](tags=OWNER_TAG)
+def load_and_verify_repair_dictionary(
+ context: dagster.OpExecutionContext,
+ cluster: dagster.ResourceParam[ClickhouseCluster],
+ dictionary: PersonIdRepairDictionary,
+) -> PersonIdRepairDictionary:
+ load_and_verify_on_every_cluster([cluster], dictionary)
+
+ def read_size(client: Client) -> dict[str, int]:
+ [[elements, bytes_allocated]] = client.execute(
+ "SELECT element_count, bytes_allocated FROM system.dictionaries WHERE database = %(database)s AND name = %(name)s",
+ {"database": settings.CLICKHOUSE_DATABASE, "name": dictionary.name},
+ )
+ return {"elements": elements, "bytes_allocated": bytes_allocated}
+
+ context.add_output_metadata(
+ {name: dagster.MetadataValue.int(value) for name, value in cluster.any_host(read_size).result().items()}
+ )
+ return dictionary
+
+
[email protected](tags=OWNER_TAG)
+def run_repair_mutation(
+ context: dagster.OpExecutionContext,
+ cluster: dagster.ResourceParam[ClickhouseCluster],
+ plan: RepairPlan,
+ dictionary: PersonIdRepairDictionary,
+) -> PersonIdRepairDictionary:
+ """Rewrite the stale rows on every shard.
+
+ Fail before the mutation if a blocking run started since the plan's watermark, and again after it.
+ """
+ if plan.config.dry_run:
+ context.log.info("Dry run: leaving flag_evaluations unchanged")
+ return dictionary
+
+ def blocking_runs_started() -> str:
+ return "; ".join(blocking_runs_started_since(context.instance, plan.blocking_run_check, context.run_id))
+
+ if started := blocking_runs_started():
+ cluster.map_all_hosts(dictionary.drop).result()
+ cluster.map_all_hosts(dictionary.source.drop).result()
+ raise dagster.Failure(
+ description=f"{started} started after the repair read person_id from events. "
+ f"{FLAG_EVALUATIONS_DATA_TABLE} is unchanged, and this run dropped its table and dictionary. "
+ "Run the repair again after that run finishes."
+ )
+
+ team_ids = plan.config.team_ids
+ AlterTableMutationRunner(
+ table=FLAG_EVALUATIONS_DATA_TABLE,
+ commands={dictionary.update_command(filter_team_ids=bool(team_ids))},
+ parameters={
+ "name": dictionary.qualified_name,
+ "team_ids": tuple(team_ids or ()),
+ "window_start": f"{min(plan.days):%Y-%m-%d} 00:00:00",
+ "window_end": f"{max(plan.days) + timedelta(days=1):%Y-%m-%d} 00:00:00",
+ },
+ ).run_on_shards(cluster)
+
+ if started := blocking_runs_started():
+ raise dagster.Failure(
+ description=f"{started} started during the repair mutation. A deletion that reached "
+ f"{FLAG_EVALUATIONS_DATA_TABLE} before the mutation did not match rows that still held a stale "
+ f"person_id, and the mutation then gave them the deleted person's id. Check what that run deleted. {_CLEANUP}"
+ )
+ return dictionary
+
+
[email protected](tags=OWNER_TAG)
+def drop_repair_dictionary(
+ cluster: dagster.ResourceParam[ClickhouseCluster], dictionary: PersonIdRepairDictionary
+) -> PersonIdRepairTable:
+ cluster.map_all_hosts(dictionary.drop).result()
+ return dictionary.source
+
+
[email protected](tags=OWNER_TAG)
+def drop_repair_table(cluster: dagster.ResourceParam[ClickhouseCluster], table: PersonIdRepairTable) -> None:
+ cluster.map_all_hosts(table.drop).result()
+
+
[email protected](tags=OWNER_TAG)
+def flag_evaluations_person_id_repair_job() -> None:
+ """Give imported flag_evaluations rows the person_id of their events copy, where the two differ.
+
+ Only rows that match the backfill's copied-row filter and still hold the stale id change, so a
+ rerun changes nothing.
+ """
+ plan = plan_person_id_repair()
+ table = create_repair_table(plan)
+ built_shards = get_repair_shards(table).map(lambda shard_num: build_repair_shard(shard_num, plan, table))
+ checked_table = check_repair_table(table, built_shards.collect())
+ dictionary = load_and_verify_repair_dictionary(create_repair_dictionary(plan, checked_table))
+ drop_repair_table(drop_repair_dictionary(run_repair_mutation(plan, dictionary)))
+
+
+class RepairRunConfig(dagster.Config):
+ run_id: str = pydantic.Field(description="The ID of the repair run whose table and dictionary to drop.")
+
+
[email protected](tags=OWNER_TAG)
+def get_repair_dictionary_for_run(config: RepairRunConfig) -> PersonIdRepairDictionary:
+ """A handle to a repair run's dictionary. It does not check that the dictionary or its table exists."""
+ return PersonIdRepairDictionary(source=PersonIdRepairTable(id=uuid.UUID(config.run_id)))
+
+
[email protected](tags=OWNER_TAG)
+def check_no_repair_mutation_running(
+ cluster: dagster.ResourceParam[ClickhouseCluster], dictionary: PersonIdRepairDictionary
+) -> PersonIdRepairDictionary:
+ """Fail while a mutation that reads this dictionary is unfinished on any host.
+
+ A stopped Dagster run leaves its ClickHouse mutation running. Dropping the dictionary under it
+ fails the mutation's remaining parts, and the failed mutation then blocks later mutations on the table.
+ """
+
+ def running(client: Client) -> int:
+ [[count]] = client.execute(
+ """
+ SELECT count() FROM system.mutations
+ WHERE database = %(database)s AND table = %(table)s AND NOT is_done AND NOT is_killed
+ AND position(command, %(name)s) > 0
+ """,
+ {
+ "database": settings.CLICKHOUSE_DATABASE,
+ "table": FLAG_EVALUATIONS_DATA_TABLE,
+ "name": dictionary.qualified_name,
+ },
+ )
+ return count
+
+ hosts = [host.connection_info.host for host, count in cluster.map_all_hosts(running).result().items() if count]
+ if hosts:
+ raise dagster.Failure(
+ description=f"A mutation that reads {dictionary.qualified_name} is unfinished on {', '.join(hosts)}. "
+ "Wait for it to finish, or stop it with KILL MUTATION, then run the cleanup again."
+ )
+ return dictionary
+
+
[email protected](tags=OWNER_TAG)
+def cleanup_orphaned_flag_evaluations_person_id_repair() -> None:
+ """Drop the table and dictionary that a failed repair run left, once its mutation has finished or been killed."""
+ drop_repair_table(drop_repair_dictionary(check_no_repair_mutation_running(get_repair_dictionary_for_run())))
diff --git a/posthog/dags/locations/clickhouse.py b/posthog/dags/locations/clickhouse.py
index 400c2f9a69b7..d88c7c8871d1 100644
--- a/posthog/dags/locations/clickhouse.py
+++ b/posthog/dags/locations/clickhouse.py
@@ -12,6 +12,7 @@ from posthog.dags import (
export_query_log_archive_to_s3,
fix_person_id_overrides,
flag_evaluations_backfill,
+ flag_evaluations_person_id_repair,
part_breaker,
person_overrides,
person_pg_cleanup_drain,
@@ -42,6 +43,8 @@ defs = dagster.Definitions(
backfill_materialized_column.backfill_materialized_column,
fix_person_id_overrides.fix_person_id_overrides_job,
flag_evaluations_backfill.flag_evaluations_backfill_job,
+ flag_evaluations_person_id_repair.cleanup_orphaned_flag_evaluations_person_id_repair,
+ flag_evaluations_person_id_repair.flag_evaluations_person_id_repair_job,
person_overrides.cleanup_orphaned_person_overrides_snapshot,
person_overrides.squash_person_overrides,
person_pg_cleanup_drain.person_pg_cleanup_drain_job,
diff --git a/posthog/dags/tests/test_flag_evaluations_backfill.py b/posthog/dags/tests/test_flag_evaluations_backfill.py
index 3ffb708e61a6..5cc4d887952a 100644
--- a/posthog/dags/tests/test_flag_evaluations_backfill.py
+++ b/posthog/dags/tests/test_flag_evaluations_backfill.py
@@ -808,7 +808,7 @@ def test_backfill_waits_while_clickhouse_moves_parts_off_a_full_disk(
)
def test_resolve_backfill_days_rejects_an_unsafe_window(overrides: dict[str, Any]) -> None:
with pytest.raises(dagster.Failure):
- resolve_backfill_days(FlagEvaluationsBackfillConfig(**overrides), today=date(2026, 3, 10))
+ resolve_backfill_days(**overrides, today=date(2026, 3, 10))
@pytest.mark.parametrize(
@@ -823,6 +823,6 @@ def test_resolve_backfill_days_rejects_an_unsafe_window(overrides: dict[str, Any
def test_resolve_backfill_days_lists_each_day_newest_first(
overrides: dict[str, Any], newest: date, oldest: date, count: int
) -> None:
- days = resolve_backfill_days(FlagEvaluationsBackfillConfig(**overrides), today=date(2026, 3, 10))
+ days = resolve_backfill_days(**overrides, today=date(2026, 3, 10))
assert (days[0], days[-1], len(days)) == (newest, oldest, count)
diff --git a/posthog/dags/tests/test_flag_evaluations_person_id_repair.py b/posthog/dags/tests/test_flag_evaluations_person_id_repair.py
new file mode 100644
index 000000000000..9bcd35b6632e
--- /dev/null
+++ b/posthog/dags/tests/test_flag_evaluations_person_id_repair.py
@@ -0,0 +1,294 @@
+from collections import Counter
+from collections.abc import Sequence
+from datetime import UTC, datetime, timedelta
+from functools import partial
+from typing import Any
+from uuid import NAMESPACE_URL, UUID, uuid4, uuid5
+
+import pytest
+from unittest.mock import patch
+
+import dagster
+
+from posthog.clickhouse.cluster import (
+ AlterTableMutationRunner,
+ ClickhouseCluster,
+ NodeRole,
+ Query,
+ wait_for_mutations_on_shards,
+)
+from posthog.dags import flag_evaluations_person_id_repair
+from posthog.dags.deletes import deletes_job
+from posthog.dags.flag_evaluations_person_id_repair import (
+ FlagEvaluationsPersonIdRepairConfig,
+ PersonIdRepairDictionary,
+ PersonIdRepairTable,
+ RepairRunConfig,
+ cleanup_orphaned_flag_evaluations_person_id_repair,
+ flag_evaluations_person_id_repair_job,
+)
+from posthog.dags.tests.conftest import insert_flag_evaluations
+from posthog.dataclasses import frozen
+from posthog.models.event.sql import EVENTS_DATA_TABLE
+from posthog.models.flag_evaluations.sql import FLAG_EVALUATIONS_DATA_TABLE, FLAG_EVALUATIONS_SOURCE_EVENT
+
+TEAM_ID = 1
+DISTINCT_ID = "repair-user"
+PERSON_A, PERSON_B, PERSON_C, PERSON_D = (UUID(int=n) for n in range(1, 5))
+
+
+@frozen
+class FlagCall:
+ label: str
+ stored_person_id: UUID
+ events_person_id: UUID
+ imported: bool = True
+ age: timedelta = timedelta(days=3)
+
+ @property
+ def uuid(self) -> UUID:
+ return uuid5(NAMESPACE_URL, self.label)
+
+ def timestamp(self, now: datetime) -> datetime:
+ return now - self.age
+
+
+STALE = FlagCall(label="stale", stored_person_id=PERSON_A, events_person_id=PERSON_B)
+KAFKA_PATH = FlagCall(label="kafka_path", stored_person_id=PERSON_A, events_person_id=PERSON_B, imported=False)
+MOVED_BY_LATER_SQUASH = FlagCall(label="moved_by_later_squash", stored_person_id=PERSON_C, events_person_id=PERSON_C)
+CALLS = (STALE, KAFKA_PATH, MOVED_BY_LATER_SQUASH)
+
+# A second events copy of STALE's call would share its sort key. A merge would then collapse the two copies.
+# A separate call gives the key (TEAM_ID, DISTINCT_ID, PERSON_A) a second events person_id without that risk.
+CONFLICTING_CALL = FlagCall(label="conflicting_call", stored_person_id=PERSON_A, events_person_id=PERSON_D)
+# A day other than STALE's, because the build validates each day on its own.
+AGREEING_CALL = FlagCall(
+ label="agreeing_call", stored_person_id=PERSON_A, events_person_id=PERSON_A, age=timedelta(days=4)
+)
+OUTSIDE_WINDOW = FlagCall(
+ label="outside_window", stored_person_id=PERSON_A, events_person_id=PERSON_C, age=timedelta(days=7)
+)
+# With STALE this maps A to B and B to C, so B is both a target and a stale key.
+CHAINED_CALL = FlagCall(label="chained_call", stored_person_id=PERSON_B, events_person_id=PERSON_C)
+
+WINDOW_START_DAYS_AGO, WINDOW_END_DAYS_AGO = 5, 1
+
+
+def seed(cluster: ClickhouseCluster, now: datetime, calls: Sequence[FlagCall]) -> None:
+ imported_rows = [
+ (TEAM_ID, DISTINCT_ID, call.stored_person_id, call.uuid, call.timestamp(now)) for call in calls if call.imported
+ ]
+ cluster.any_host(partial(insert_flag_evaluations, imported_rows)).result()
+
+ # A row from Kafka arrives after its event, so its inserted_at is later than its timestamp.
+ # Its offset is 1 because the imported-row filter treats offset 0 in partition 0 as an import.
+ kafka_rows = []
+ for call in calls:
+ if not call.imported:
+ kafka_time = call.timestamp(now) + timedelta(seconds=1)
+ kafka_rows.append(
+ (
+ TEAM_ID,
+ DISTINCT_ID,
+ call.stored_person_id,
+ call.uuid,
+ call.timestamp(now),
+ kafka_time,
+ kafka_time,
+ 0,
+ 1,
+ )
+ )
+ cluster.any_host(
+ Query(
+ """INSERT INTO writable_flag_evaluations
+ (team_id, distinct_id, person_id, uuid, timestamp, inserted_at, _timestamp, _partition, _offset)
+ VALUES""",
+ kafka_rows,
+ )
+ ).result()
+
+ events_rows = [
+ (
+ call.uuid,
+ FLAG_EVALUATIONS_SOURCE_EVENT,
+ "{}",
+ call.timestamp(now),
+ TEAM_ID,
+ DISTINCT_ID,
+ call.timestamp(now),
+ call.events_person_id,
+ "{}",
+ call.timestamp(now),
+ "full",
+ call.timestamp(now),
+ )
+ for call in calls
+ ]
+ cluster.any_host_by_role(
+ Query(
+ f"""INSERT INTO {EVENTS_DATA_TABLE()}
+ (uuid, event, properties, timestamp, team_id, distinct_id, created_at, person_id,
+ person_properties, person_created_at, person_mode, _timestamp)
+ VALUES""",
+ events_rows,
+ ),
+ NodeRole.DATA,
+ ).result()
+
+
+def stored_person_ids(cluster: ClickhouseCluster, calls: Sequence[FlagCall]) -> Counter[tuple[str, UUID]]:
+ labels = {call.uuid: call.label for call in calls}
+ rows = cluster.any_host_by_role(Query("SELECT uuid, person_id FROM flag_evaluations"), NodeRole.DATA).result()
+ return Counter((labels.get(uuid, str(uuid)), person_id) for uuid, person_id in rows)
+
+
+def days_before(now: datetime, days: int) -> str:
+ return (now - timedelta(days=days)).date().isoformat()
+
+
+def run_repair(
+ cluster: ClickhouseCluster, now: datetime, instance: dagster.DagsterInstance | None = None, **overrides: Any
+) -> dagster.ExecuteInProcessResult:
+ config = FlagEvaluationsPersonIdRepairConfig(
+ start_date=days_before(now, WINDOW_START_DAYS_AGO), end_date=days_before(now, WINDOW_END_DAYS_AGO), **overrides
+ )
+ return flag_evaluations_person_id_repair_job.execute_in_process(
+ run_config=dagster.RunConfig(ops={"plan_person_id_repair": config}),
+ resources={"cluster": cluster},
+ instance=instance or dagster.DagsterInstance.ephemeral(),
+ raise_on_error=False,
+ )
+
+
+def run_cleanup(cluster: ClickhouseCluster, run_id: UUID) -> dagster.ExecuteInProcessResult:
+ return cleanup_orphaned_flag_evaluations_person_id_repair.execute_in_process(
+ run_config=dagster.RunConfig(ops={"get_repair_dictionary_for_run": RepairRunConfig(run_id=str(run_id))}),
+ resources={"cluster": cluster},
+ instance=dagster.DagsterInstance.ephemeral(),
+ raise_on_error=False,
+ )
+
+
+def on_any_host(cluster: ClickhouseCluster, fn: Any) -> bool:
+ return any(cluster.map_all_hosts(fn).result().values())
+
+
+def drop_repair_resources(cluster: ClickhouseCluster, run_id: UUID) -> None:
+ dictionary = PersonIdRepairDictionary(source=PersonIdRepairTable(id=run_id))
+ cluster.map_all_hosts(dictionary.drop).result()
+ cluster.map_all_hosts(dictionary.source.drop).result()
+
+
+ "dry_run, extra_calls, runs, succeeds, rewritten",
+ [
+ # The second run must change nothing the first one left.
+ pytest.param(False, (), 2, True, {STALE.label: PERSON_B}, id="repair"),
+ pytest.param(True, (), 1, True, {}, id="dry_run"),
+ pytest.param(False, (CONFLICTING_CALL,), 1, False, {}, id="conflicting_events_copies"),
+ pytest.param(False, (AGREEING_CALL,), 1, False, {}, id="agreeing_events_copy"),
+ pytest.param(False, (CHAINED_CALL,), 1, False, {}, id="chained_targets"),
+ pytest.param(False, (OUTSIDE_WINDOW,), 1, True, {STALE.label: PERSON_B}, id="outside_window"),
+ ],
+)
+def test_repair_rewrites_only_imported_rows_that_still_hold_the_stale_id(
+ cluster: ClickhouseCluster,
+ dry_run: bool,
+ extra_calls: tuple[FlagCall, ...],
+ runs: int,
+ succeeds: bool,
+ rewritten: dict[str, UUID],
+) -> None:
+ # nosemgrep: test-datetime-now-without-freeze (the rows and both window bounds are offsets from this one reading, so passing UTC midnight changes no assertion)
+ now = datetime.now(UTC)
+ calls = (*CALLS, *extra_calls)
+ seed(cluster, now, calls)
+
+ results = [run_repair(cluster, now, dry_run=dry_run) for _ in range(runs)]
+
+ run_ids = [UUID(result.dagster_run.run_id) for result in results]
+ try:
+ assert [result.success for result in results] == [succeeds] * runs
+ expected = {call.label: call.stored_person_id for call in calls} | rewritten
+ assert stored_person_ids(cluster, calls) == Counter(expected.items())
+ for run_id in run_ids:
+ table = PersonIdRepairTable(id=run_id)
+ dictionary = PersonIdRepairDictionary(source=table)
+ assert on_any_host(cluster, table.exists) is not succeeds
+ if succeeds:
+ assert not on_any_host(cluster, dictionary.exists)
+ finally:
+ for run_id in run_ids:
+ drop_repair_resources(cluster, run_id)
+
+
+def test_repair_stops_before_the_mutation_when_a_deletes_run_starts_during_the_build(
+ cluster: ClickhouseCluster,
+) -> None:
+ # nosemgrep: test-datetime-now-without-freeze (the rows and both window bounds are offsets from this one reading, so passing UTC midnight changes no assertion)
+ now = datetime.now(UTC)
+ seed(cluster, now, CALLS)
+ instance = dagster.DagsterInstance.ephemeral()
+ on_shard_host = flag_evaluations_person_id_repair.on_shard_host
+
+ def start_a_deletes_run(*args: Any, **kwargs: Any) -> int:
+ instance.create_run_for_job(job_def=deletes_job, status=dagster.DagsterRunStatus.STARTED)
+ return on_shard_host(*args, **kwargs)
+
+ with patch.object(flag_evaluations_person_id_repair, "on_shard_host", side_effect=start_a_deletes_run):
+ result = run_repair(cluster, now, instance=instance)
+
+ run_id = UUID(result.dagster_run.run_id)
+ try:
+ [failure] = result.get_step_failure_events()
+ assert failure.step_failure_data.error is not None
+ assert "started after the repair read person_id" in failure.step_failure_data.error.message
+ assert stored_person_ids(cluster, CALLS) == Counter((call.label, call.stored_person_id) for call in CALLS)
+ assert not on_any_host(cluster, PersonIdRepairTable(id=run_id).exists)
+ assert not on_any_host(cluster, PersonIdRepairDictionary(source=PersonIdRepairTable(id=run_id)).exists)
+ finally:
+ drop_repair_resources(cluster, run_id)
+
+
+def test_cleanup_refuses_while_a_repair_mutation_is_unfinished(cluster: ClickhouseCluster) -> None:
+ # nosemgrep: test-datetime-now-without-freeze (the row and the mutation's window are offsets from this one reading, so passing UTC midnight changes no assertion)
+ now = datetime.now(UTC)
+ seed(cluster, now, (STALE,))
+ table = PersonIdRepairTable(id=uuid4())
+ dictionary = PersonIdRepairDictionary(source=table)
+ runner = AlterTableMutationRunner(
+ table=FLAG_EVALUATIONS_DATA_TABLE,
+ commands={dictionary.update_command(filter_team_ids=False)},
+ parameters={
+ "name": dictionary.qualified_name,
+ "window_start": f"{days_before(now, WINDOW_START_DAYS_AGO)} 00:00:00",
+ "window_end": f"{days_before(now, WINDOW_END_DAYS_AGO)} 00:00:00",
+ },
+ # The test client otherwise waits for the mutation, which the stopped merges hold open.
+ settings={"mutations_sync": 0},
+ )
+ # Stopped merges also stop mutations, which leaves this one unfinished on every replica.
+ start_merges = Query(f"SYSTEM START MERGES {FLAG_EVALUATIONS_DATA_TABLE}")
+ try:
+ cluster.map_all_hosts(table.create).result()
+ cluster.map_all_hosts(partial(dictionary.create, shards=1, max_execution_time=0, max_memory_usage=0)).result()
+ cluster.map_all_hosts(dictionary.load).result()
+ cluster.map_all_hosts(Query(f"SYSTEM STOP MERGES {FLAG_EVALUATIONS_DATA_TABLE}")).result()
+ mutations = runner.enqueue_on_shards(cluster)
+
+ assert not run_cleanup(cluster, table.id).success
+ assert on_any_host(cluster, dictionary.exists)
+
+ cluster.map_all_hosts(start_merges).result()
+ wait_for_mutations_on_shards(cluster, mutations)
+
+ assert run_cleanup(cluster, table.id).success
+ assert not on_any_host(cluster, dictionary.exists)
+ assert not on_any_host(cluster, table.exists)
+ finally:
+ cluster.map_all_hosts(start_merges).result()
+ drop_repair_resources(cluster, table.id)