:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(dags): apply the deleted mask on every shard in parallel (#130850)
Paweł Szczur94c72de91cd4
diff --git a/posthog/dags/apply_deleted_mask.py b/posthog/dags/apply_deleted_mask.pyindex 4bbec4066a32..56d6b5baefec 100644--- a/posthog/dags/apply_deleted_mask.py+++ b/posthog/dags/apply_deleted_mask.py@@ -1,4 +1,4 @@-from collections.abc import Callable, Sequence+from collections.abc import Callable, Mapping, Sequencefrom django.conf import settings@@ -18,7 +18,7 @@ class FindDeletedMaskPartitionsConfig(dagster.Config):default=0.1,gt=0,le=1,- description="Apply the deleted mask on a shard's partition when at least this share of its rows is deleted.",+ description="Apply the deleted mask on a month, on every shard, when at least this share of its rows is deleted on any shard.",)partitions: list[str] = pydantic.Field(default_factory=list,@@ -35,8 +35,9 @@ class ApplyDeletedMaskConfig(dagster.Config):@frozenclass DeletedMaskPartition:+ """A month to rewrite on every shard, with its row counts summed over all shards."""+partition: str- shard: intphysical_rows: intdeleted_rows: int@@ -46,6 +47,41 @@ class PartitionRowCounts:physical_rows: intlive_rows: int+ @property+ def deleted_rows(self) -> int:+ return self.physical_rows - self.live_rows++ @property+ def deleted_ratio(self) -> float:+ return self.deleted_rows / self.physical_rows if self.physical_rows else 0.0+++def _partitions_due(+ counts_by_shard: Mapping[int, Mapping[str, PartitionRowCounts]], min_deleted_ratio: float+) -> list[DeletedMaskPartition]:+ """The months where at least one shard reaches the threshold, most deleted rows first."""+ totals: dict[str, PartitionRowCounts] = {}+ due: set[str] = set()+ for counts in counts_by_shard.values():+ for partition, rows in counts.items():+ total = totals.get(partition, PartitionRowCounts(physical_rows=0, live_rows=0))+ totals[partition] = PartitionRowCounts(+ physical_rows=total.physical_rows + rows.physical_rows, live_rows=total.live_rows + rows.live_rows+ )+ if rows.deleted_ratio >= min_deleted_ratio:+ due.add(partition)++ found = [+ DeletedMaskPartition(+ partition=partition,+ physical_rows=totals[partition].physical_rows,+ deleted_rows=totals[partition].deleted_rows,+ )+ for partition in due+ ]+ found.sort(key=lambda item: item.deleted_rows, reverse=True)+ return found+def _partition_filter(partitions: Sequence[str], column: str) -> str:return f"AND {column} IN %(partitions)s" if partitions else ""@@ -101,40 +137,31 @@ def _find_deleted_mask_partitions_op(name: str, target: DeletionTarget) -> dagstconfig: FindDeletedMaskPartitionsConfig,cluster: dagster.ResourceParam[ClickhouseCluster],) -> list[DeletedMaskPartition]:- """Find each month and shard of one events table whose deleted rows reach the threshold, most deleted rows first."""+ """Find each month of one events table whose deleted rows reach the threshold on any shard, most deleted rows first."""placement = placement_for(cluster, target)if placement is None:context.log.info(f"{target.data_table} is not on any cluster this deployment reaches")return []- found: list[DeletedMaskPartition] = []- report: dict[str, dict[int, float]] = {}+ counts_by_shard: dict[int, dict[str, PartitionRowCounts]] = {}per_host = placement.cluster.map_one_host_per_shard(_deleted_rows_by_partition(placement, config.partitions)).result()for host, counts in per_host.items():assert host.shard_num is not None+ counts_by_shard[host.shard_num] = counts++ report: dict[str, dict[int, float]] = {}+ for shard, counts in counts_by_shard.items():for partition, rows in counts.items():- if not rows.physical_rows:- continue- deleted_rows = rows.physical_rows - rows.live_rows- ratio = deleted_rows / rows.physical_rows- report.setdefault(partition, {})[host.shard_num] = round(ratio, 4)- if ratio >= config.min_deleted_ratio:- found.append(- DeletedMaskPartition(- partition=partition,- shard=host.shard_num,- physical_rows=rows.physical_rows,- deleted_rows=deleted_rows,- )- )-- found.sort(key=lambda item: item.deleted_rows, reverse=True)+ if rows.physical_rows:+ report.setdefault(partition, {})[shard] = round(rows.deleted_ratio, 4)++ found = _partitions_due(counts_by_shard, config.min_deleted_ratio)for item in found:context.log.info(- f"{target.data_table} {item.partition} shard {item.shard}: {item.deleted_rows} of "- f"{item.physical_rows} rows deleted ({item.deleted_rows / item.physical_rows:.1%})"+ f"{target.data_table} {item.partition}: {item.deleted_rows} of {item.physical_rows} rows deleted "+ f"on all shards, by shard {report[item.partition]}")if not found:context.log.info(@@ -160,7 +187,7 @@ def _apply_deleted_mask_op(name: str, target: DeletionTarget) -> dagster.OpDefincluster: dagster.ResourceParam[ClickhouseCluster],partitions: list[DeletedMaskPartition],) -> None:- """Rewrite each found month on its shard without its deleted rows, one mutation at a time."""+ """Rewrite each found month without its deleted rows, on all shards at once and one month at a time."""if config.dry_run or not partitions:context.log.info(f"Not rewriting {len(partitions)} {target.data_table} partition(s), dry run: {config.dry_run}"@@ -175,7 +202,7 @@ def _apply_deleted_mask_op(name: str, target: DeletionTarget) -> dagster.OpDefinfor idx, item in enumerate(partitions, 1):context.log.info(- f"{target.data_table} {item.partition} shard {item.shard}: removing {item.deleted_rows} rows "+ f"{target.data_table} {item.partition}: removing {item.deleted_rows} rows on all shards "f"({idx}/{len(partitions)})")commands = [f"APPLY DELETED MASK IN PARTITION '{item.partition}'"]@@ -184,9 +211,10 @@ def _apply_deleted_mask_op(name: str, target: DeletionTarget) -> dagster.OpDefin# applied. One ALTER with both commands applies the mask first, so they run in turn.commands.insert(0, f"APPLY PATCHES IN PARTITION '{item.partition}'")for command in commands:- # Each rewrite is heavy, so the next one starts only after every replica finishes this one.+ # Every shard runs the rewrite at the same time, because each shard rewrites only its own+ # disks. The next rewrite starts only after every replica of every shard finishes this one.AlterTableMutationRunner(target.data_table, commands={command}, reuse_since=reuse_floor).run_on_shards(- placement.cluster, [item.shard]+ placement.cluster)context.add_output_metadata({"partitions_processed": dagster.MetadataValue.int(len(partitions))})diff --git a/posthog/dags/tests/test_apply_deleted_mask.py b/posthog/dags/tests/test_apply_deleted_mask.pyindex 80e3d5477e1b..6a3504c6260d 100644--- a/posthog/dags/tests/test_apply_deleted_mask.py+++ b/posthog/dags/tests/test_apply_deleted_mask.py@@ -8,7 +8,12 @@ from django.conf import settingsfrom clickhouse_driver import Clientfrom posthog.clickhouse.cluster import ClickhouseCluster, LightweightDeleteMutationRunner, Query-from posthog.dags.apply_deleted_mask import DeletedMaskPartition, apply_deleted_mask_job+from posthog.dags.apply_deleted_mask import (+ DeletedMaskPartition,+ PartitionRowCounts,+ _partitions_due,+ apply_deleted_mask_job,+)from posthog.models.deletion_targets import EVENTS_TARGETS, resolve_placementsfrom posthog.models.event.sql import EVENTS_DATA_TABLE, EVENTS_JSON_DATA_TABLE@@ -124,3 +129,20 @@ def _find_and_apply(cluster: ClickhouseCluster) -> None:}for table, partition in before:assert after[(table, partition)] == before[(table, partition)] - removed[table][partition]+++def test_a_month_due_on_one_shard_is_found_with_the_rows_of_every_shard() -> None:+ counts_by_shard = {+ 1: {+ MOSTLY_DELETED: PartitionRowCounts(physical_rows=100, live_rows=50),+ FEW_DELETED: PartitionRowCounts(physical_rows=100, live_rows=99),+ },+ 2: {+ MOSTLY_DELETED: PartitionRowCounts(physical_rows=100, live_rows=100),+ FEW_DELETED: PartitionRowCounts(physical_rows=100, live_rows=95),+ },+ }++ assert _partitions_due(counts_by_shard, min_deleted_ratio=0.1) == [+ DeletedMaskPartition(partition=MOSTLY_DELETED, physical_rows=200, deleted_rows=50)+ ]