:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(dags): add job that applies the deleted mask on heavily deleted partitions (#123422)
Paweł Szczur2520435add1b
diff --git a/posthog/dags/apply_deleted_mask.py b/posthog/dags/apply_deleted_mask.pynew file mode 100644index 000000000000..4bbec4066a32--- /dev/null+++ b/posthog/dags/apply_deleted_mask.py@@ -0,0 +1,217 @@+from collections.abc import Callable, Sequence++from django.conf import settings++import dagster+import pydantic+from clickhouse_driver.client import Client++from posthog.clickhouse.cluster import AlterTableMutationRunner, ClickhouseCluster+from posthog.dags.common import JobOwners+from posthog.dags.deletes import _mutation_reuse_floor, ensure_no_concurrent_deletes_run+from posthog.dataclasses import frozen+from posthog.models.deletion_targets import EVENTS, EVENTS_JSON, DeletionTarget, TargetPlacement, placement_for+++class FindDeletedMaskPartitionsConfig(dagster.Config):+ min_deleted_ratio: float = pydantic.Field(+ default=0.1,+ gt=0,+ le=1,+ description="Apply the deleted mask on a shard's partition when at least this share of its rows is deleted.",+ )+ partitions: list[str] = pydantic.Field(+ default_factory=list,+ description="Only check these months, as YYYYMM (e.g. ['202412']). Empty means every month.",+ )+++class ApplyDeletedMaskConfig(dagster.Config):+ dry_run: bool = pydantic.Field(+ default=False,+ description="Only log the partitions this run would rewrite.",+ )+++@frozen+class DeletedMaskPartition:+ partition: str+ shard: int+ physical_rows: int+ deleted_rows: int+++@frozen+class PartitionRowCounts:+ physical_rows: int+ live_rows: int+++def _partition_filter(partitions: Sequence[str], column: str) -> str:+ return f"AND {column} IN %(partitions)s" if partitions else ""+++def _deleted_rows_by_partition(+ placement: TargetPlacement, partitions: Sequence[str]+) -> Callable[[Client], dict[str, PartitionRowCounts]]:+ """A callable that returns the row counts of each partition of the table on one host."""+ parameters = {+ "database": settings.CLICKHOUSE_DATABASE,+ "table": placement.target.data_table,+ "partitions": list(partitions),+ }++ def count(client: Client) -> dict[str, PartitionRowCounts]:+ # Patch parts sit in partitions of their own and hold no rows of the table, so only the+ # data parts count as physical rows. count() applies both the deleted mask and patch parts.+ physical = client.execute(+ f"""+ SELECT partition_id, sum(rows)+ FROM system.parts+ WHERE database = %(database)s AND table = %(table)s AND active+ AND NOT startsWith(partition_id, 'patch-')+ {_partition_filter(partitions, "partition_id")}+ GROUP BY partition_id+ """,+ parameters,+ )+ live = dict(+ client.execute(+ f"""+ SELECT _partition_id, count()+ FROM {settings.CLICKHOUSE_DATABASE}.{placement.target.data_table}+ WHERE 1 {_partition_filter(partitions, "_partition_id")}+ GROUP BY _partition_id+ """,+ parameters,+ )+ )+ return {+ partition_id: PartitionRowCounts(physical_rows=rows, live_rows=live.get(partition_id, 0))+ for partition_id, rows in physical+ }++ return count+++def _find_deleted_mask_partitions_op(name: str, target: DeletionTarget) -> dagster.OpDefinition:+ @dagster.op(name=f"find_deleted_mask_partitions_{name}", ins={"start_after": dagster.In(dagster.Nothing)})+ def find(+ context: dagster.OpExecutionContext,+ config: FindDeletedMaskPartitionsConfig,+ cluster: dagster.ResourceParam[ClickhouseCluster],+ ) -> list[DeletedMaskPartition]:+ """Find each month and shard of one events table whose deleted rows reach the threshold, most deleted rows first."""+ placement = placement_for(cluster, target)+ if placement is None:+ context.log.info(f"{target.data_table} is not on any cluster this deployment reaches")+ return []++ found: list[DeletedMaskPartition] = []+ report: dict[str, dict[int, float]] = {}+ per_host = placement.cluster.map_one_host_per_shard(+ _deleted_rows_by_partition(placement, config.partitions)+ ).result()+ for host, counts in per_host.items():+ assert host.shard_num is not None+ for partition, rows in counts.items():+ if not rows.physical_rows:+ continue+ deleted_rows = rows.physical_rows - rows.live_rows+ ratio = deleted_rows / rows.physical_rows+ report.setdefault(partition, {})[host.shard_num] = round(ratio, 4)+ if ratio >= config.min_deleted_ratio:+ found.append(+ DeletedMaskPartition(+ partition=partition,+ shard=host.shard_num,+ physical_rows=rows.physical_rows,+ deleted_rows=deleted_rows,+ )+ )++ found.sort(key=lambda item: item.deleted_rows, reverse=True)+ for item in found:+ context.log.info(+ f"{target.data_table} {item.partition} shard {item.shard}: {item.deleted_rows} of "+ f"{item.physical_rows} rows deleted ({item.deleted_rows / item.physical_rows:.1%})"+ )+ if not found:+ context.log.info(+ f"No {target.data_table} partition has {config.min_deleted_ratio:.0%} or more of its rows deleted"+ )++ context.add_output_metadata(+ {+ "deleted_ratio_by_partition_shard": dagster.MetadataValue.json(report),+ "partitions": dagster.MetadataValue.int(len(found)),+ }+ )+ return found++ return find+++def _apply_deleted_mask_op(name: str, target: DeletionTarget) -> dagster.OpDefinition:+ @dagster.op(name=f"apply_deleted_mask_{name}")+ def apply(+ context: dagster.OpExecutionContext,+ config: ApplyDeletedMaskConfig,+ cluster: dagster.ResourceParam[ClickhouseCluster],+ partitions: list[DeletedMaskPartition],+ ) -> None:+ """Rewrite each found month on its shard without its deleted rows, one mutation at a time."""+ if config.dry_run or not partitions:+ context.log.info(+ f"Not rewriting {len(partitions)} {target.data_table} partition(s), dry run: {config.dry_run}"+ )+ return++ placement = placement_for(cluster, target)+ assert placement is not None+ # The commands name only the month, so a mask an earlier run applied matches them too.+ # Adopting that finished mutation would rewrite nothing.+ reuse_floor = _mutation_reuse_floor(cluster)++ for idx, item in enumerate(partitions, 1):+ context.log.info(+ f"{target.data_table} {item.partition} shard {item.shard}: removing {item.deleted_rows} rows "+ f"({idx}/{len(partitions)})"+ )+ commands = [f"APPLY DELETED MASK IN PARTITION '{item.partition}'"]+ if target.uses_patch_parts:+ # Rows deleted through a patch part are not in the deleted mask until the patch is+ # applied. One ALTER with both commands applies the mask first, so they run in turn.+ commands.insert(0, f"APPLY PATCHES IN PARTITION '{item.partition}'")+ for command in commands:+ # Each rewrite is heavy, so the next one starts only after every replica finishes this one.+ AlterTableMutationRunner(target.data_table, commands={command}, reuse_since=reuse_floor).run_on_shards(+ placement.cluster, [item.shard]+ )++ context.add_output_metadata({"partitions_processed": dagster.MetadataValue.int(len(partitions))})++ return apply+++# Each table sits on its own cluster with its own hardware, so each gets its own threshold, its own+# dry run switch, and its own queue of rewrites. The two queues run side by side.+_TABLE_OPS = {+ name: (_find_deleted_mask_partitions_op(name, target), _apply_deleted_mask_op(name, target))+ for name, target in (("events", EVENTS), ("events_json", EVENTS_JSON))+}+++ tags={+ "owner": JobOwners.TEAM_CLICKHOUSE.value,+ # Shares the run-queue limit with deletes_job, the old events cleanup and squash, because+ # each rewrite is a mutation on the same events tables.+ "deletes_job_concurrency": "v1",+ }+)+def apply_deleted_mask_job():+ """Find events table months with many deleted rows, then remove those rows from disk, most deleted rows first."""+ guard = ensure_no_concurrent_deletes_run()+ for find, apply in _TABLE_OPS.values():+ apply(find(start_after=guard))diff --git a/posthog/dags/locations/clickhouse.py b/posthog/dags/locations/clickhouse.pyindex 2695a2f81861..b65d5d788cde 100644--- a/posthog/dags/locations/clickhouse.py+++ b/posthog/dags/locations/clickhouse.py@@ -2,6 +2,7 @@ import dagsterfrom posthog.dags import (add_index_to_materialized_column,+ apply_deleted_mask,backfill_materialized_column,ch_examples,clickhouse_cleanup,@@ -34,6 +35,7 @@ defs = dagster.Definitions(],jobs=[add_index_to_materialized_column.add_index_to_materialized_column,+ apply_deleted_mask.apply_deleted_mask_job,clickhouse_cleanup.clickhouse_deletion_sweep_job,create_materialized_column.create_materialized_column,drop_materialized_column.drop_materialized_column,diff --git a/posthog/dags/tests/test_apply_deleted_mask.py b/posthog/dags/tests/test_apply_deleted_mask.pynew file mode 100644index 000000000000..80e3d5477e1b--- /dev/null+++ b/posthog/dags/tests/test_apply_deleted_mask.py@@ -0,0 +1,126 @@+from datetime import datetime+from uuid import UUID++import pytest++from django.conf import settings++from clickhouse_driver import Client++from posthog.clickhouse.cluster import ClickhouseCluster, LightweightDeleteMutationRunner, Query+from posthog.dags.apply_deleted_mask import DeletedMaskPartition, apply_deleted_mask_job+from posthog.models.deletion_targets import EVENTS_TARGETS, resolve_placements+from posthog.models.event.sql import EVENTS_DATA_TABLE, EVENTS_JSON_DATA_TABLE++TEAM_ID = 4242+MOSTLY_DELETED = "202412"+FEW_DELETED = "202501"+MOST_DELETED = "202502"+# The events_json threshold sits between the 50% and 80% months, so it rewrites only one of them.+MIN_DELETED_RATIO = {"events": 0.1, "events_json": 0.6}+++def _insert_events(cluster: ClickhouseCluster) -> None:+ rows = [+ (TEAM_ID, f"user_{i}", UUID(int=i), datetime(2024, 12, 10) if i < 100 else datetime(2025, i // 100, 10))+ for i in range(300)+ ]++ def insert(client: Client) -> None:+ client.execute("INSERT INTO writable_events (team_id, distinct_id, uuid, timestamp) VALUES", rows)+ client.execute(f"INSERT INTO {EVENTS_JSON_DATA_TABLE} (team_id, distinct_id, uuid, timestamp) VALUES", rows)++ cluster.any_host(insert).result()+++def _delete(cluster: ClickhouseCluster, partition: str, below_uuid: int) -> None:+ for placement in resolve_placements(cluster, EVENTS_TARGETS):+ LightweightDeleteMutationRunner(+ table=placement.target.data_table,+ predicate="team_id = %(team_id)s AND uuid < %(below)s",+ parameters={"team_id": TEAM_ID, "below": UUID(int=below_uuid)},+ partition=partition,+ settings={"lightweight_deletes_sync": 2},+ patch_parts=placement.target.uses_patch_parts,+ ).run_on_shards(placement.cluster)+++def _physical_rows(cluster: ClickhouseCluster) -> dict[tuple[str, str], int]:+ counts: dict[tuple[str, str], int] = {}+ for placement in resolve_placements(cluster, EVENTS_TARGETS):++ def count(client: Client, table: str = placement.target.data_table) -> list[tuple[str, int]]:+ return client.execute(+ """+ SELECT partition_id, sum(rows) FROM system.parts+ WHERE database = %(database)s AND table = %(table)s AND active AND partition_id IN %(partitions)s+ GROUP BY partition_id+ """,+ {+ "database": settings.CLICKHOUSE_DATABASE,+ "table": table,+ "partitions": [MOSTLY_DELETED, FEW_DELETED, MOST_DELETED],+ },+ )++ for rows in placement.cluster.map_one_host_per_shard(count).result().values():+ for partition, physical in rows:+ key = (placement.target.data_table, partition)+ counts[key] = counts.get(key, 0) + physical+ return counts+++def _run(cluster: ClickhouseCluster, dry_run: bool) -> dict[str, list[DeletedMaskPartition]]:+ partitions = [MOSTLY_DELETED, FEW_DELETED, MOST_DELETED]+ result = apply_deleted_mask_job.execute_in_process(+ run_config={+ "ops": {+ **{+ f"find_deleted_mask_partitions_{name}": {+ "config": {"partitions": partitions, "min_deleted_ratio": ratio}+ }+ for name, ratio in MIN_DELETED_RATIO.items()+ },+ **{f"apply_deleted_mask_{name}": {"config": {"dry_run": dry_run}} for name in MIN_DELETED_RATIO},+ }+ },+ resources={"cluster": cluster},+ )+ return {name: result.output_for_node(f"find_deleted_mask_partitions_{name}") for name in MIN_DELETED_RATIO}+++def test_found_partitions_lose_their_deleted_rows_and_others_stay_untouched(cluster: ClickhouseCluster):+ try:+ _find_and_apply(cluster)+ finally:+ cluster.map_one_host_per_shard(Query(f"TRUNCATE TABLE {EVENTS_JSON_DATA_TABLE}")).result()+++def _find_and_apply(cluster: ClickhouseCluster) -> None:+ _insert_events(cluster)+ _delete(cluster, MOSTLY_DELETED, below_uuid=50)+ _delete(cluster, FEW_DELETED, below_uuid=101)+ _delete(cluster, MOST_DELETED, below_uuid=280)+ before = _physical_rows(cluster)++ _run(cluster, dry_run=True)+ assert _physical_rows(cluster) == before++ found = _run(cluster, dry_run=False)+ assert {name: {item.partition for item in items} for name, items in found.items()} == {+ "events": {MOSTLY_DELETED, MOST_DELETED},+ "events_json": {MOST_DELETED},+ }+ deleted_rows = [item.deleted_rows for item in found["events"]]+ assert deleted_rows == sorted(deleted_rows, reverse=True)++ after = _physical_rows(cluster)+ # sharded_events_json deletes through patch parts, which the mask only removes after APPLY PATCHES.+ assert {table for table, _ in before} == {EVENTS_DATA_TABLE(), EVENTS_JSON_DATA_TABLE}+ removed = {+ EVENTS_DATA_TABLE(): {MOSTLY_DELETED: 50, FEW_DELETED: 0, MOST_DELETED: 80},+ EVENTS_JSON_DATA_TABLE: {MOSTLY_DELETED: 0, FEW_DELETED: 0, MOST_DELETED: 80},+ }+ for table, partition in before:+ assert after[(table, partition)] == before[(table, partition)] - removed[table][partition]