Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(dags): add job that applies the deleted mask on heavily deleted partitions (#123422)

Paweł Szczur2520435add1b
diff --git a/posthog/dags/apply_deleted_mask.py b/posthog/dags/apply_deleted_mask.py
new file mode 100644
index 000000000000..4bbec4066a32
--- /dev/null
+++ b/posthog/dags/apply_deleted_mask.py
@@ -0,0 +1,217 @@
+from collections.abc import Callable, Sequence
+
+from django.conf import settings
+
+import dagster
+import pydantic
+from clickhouse_driver.client import Client
+
+from posthog.clickhouse.cluster import AlterTableMutationRunner, ClickhouseCluster
+from posthog.dags.common import JobOwners
+from posthog.dags.deletes import _mutation_reuse_floor, ensure_no_concurrent_deletes_run
+from posthog.dataclasses import frozen
+from posthog.models.deletion_targets import EVENTS, EVENTS_JSON, DeletionTarget, TargetPlacement, placement_for
+
+
+class FindDeletedMaskPartitionsConfig(dagster.Config):
+ min_deleted_ratio: float = pydantic.Field(
+ default=0.1,
+ gt=0,
+ le=1,
+ description="Apply the deleted mask on a shard's partition when at least this share of its rows is deleted.",
+ )
+ partitions: list[str] = pydantic.Field(
+ default_factory=list,
+ description="Only check these months, as YYYYMM (e.g. ['202412']). Empty means every month.",
+ )
+
+
+class ApplyDeletedMaskConfig(dagster.Config):
+ dry_run: bool = pydantic.Field(
+ default=False,
+ description="Only log the partitions this run would rewrite.",
+ )
+
+
+@frozen
+class DeletedMaskPartition:
+ partition: str
+ shard: int
+ physical_rows: int
+ deleted_rows: int
+
+
+@frozen
+class PartitionRowCounts:
+ physical_rows: int
+ live_rows: int
+
+
+def _partition_filter(partitions: Sequence[str], column: str) -> str:
+ return f"AND {column} IN %(partitions)s" if partitions else ""
+
+
+def _deleted_rows_by_partition(
+ placement: TargetPlacement, partitions: Sequence[str]
+) -> Callable[[Client], dict[str, PartitionRowCounts]]:
+ """A callable that returns the row counts of each partition of the table on one host."""
+ parameters = {
+ "database": settings.CLICKHOUSE_DATABASE,
+ "table": placement.target.data_table,
+ "partitions": list(partitions),
+ }
+
+ def count(client: Client) -> dict[str, PartitionRowCounts]:
+ # Patch parts sit in partitions of their own and hold no rows of the table, so only the
+ # data parts count as physical rows. count() applies both the deleted mask and patch parts.
+ physical = client.execute(
+ f"""
+ SELECT partition_id, sum(rows)
+ FROM system.parts
+ WHERE database = %(database)s AND table = %(table)s AND active
+ AND NOT startsWith(partition_id, 'patch-')
+ {_partition_filter(partitions, "partition_id")}
+ GROUP BY partition_id
+ """,
+ parameters,
+ )
+ live = dict(
+ client.execute(
+ f"""
+ SELECT _partition_id, count()
+ FROM {settings.CLICKHOUSE_DATABASE}.{placement.target.data_table}
+ WHERE 1 {_partition_filter(partitions, "_partition_id")}
+ GROUP BY _partition_id
+ """,
+ parameters,
+ )
+ )
+ return {
+ partition_id: PartitionRowCounts(physical_rows=rows, live_rows=live.get(partition_id, 0))
+ for partition_id, rows in physical
+ }
+
+ return count
+
+
+def _find_deleted_mask_partitions_op(name: str, target: DeletionTarget) -> dagster.OpDefinition:
+ @dagster.op(name=f"find_deleted_mask_partitions_{name}", ins={"start_after": dagster.In(dagster.Nothing)})
+ def find(
+ context: dagster.OpExecutionContext,
+ config: FindDeletedMaskPartitionsConfig,
+ cluster: dagster.ResourceParam[ClickhouseCluster],
+ ) -> list[DeletedMaskPartition]:
+ """Find each month and shard of one events table whose deleted rows reach the threshold, most deleted rows first."""
+ placement = placement_for(cluster, target)
+ if placement is None:
+ context.log.info(f"{target.data_table} is not on any cluster this deployment reaches")
+ return []
+
+ found: list[DeletedMaskPartition] = []
+ report: dict[str, dict[int, float]] = {}
+ per_host = placement.cluster.map_one_host_per_shard(
+ _deleted_rows_by_partition(placement, config.partitions)
+ ).result()
+ for host, counts in per_host.items():
+ assert host.shard_num is not None
+ for partition, rows in counts.items():
+ if not rows.physical_rows:
+ continue
+ deleted_rows = rows.physical_rows - rows.live_rows
+ ratio = deleted_rows / rows.physical_rows
+ report.setdefault(partition, {})[host.shard_num] = round(ratio, 4)
+ if ratio >= config.min_deleted_ratio:
+ found.append(
+ DeletedMaskPartition(
+ partition=partition,
+ shard=host.shard_num,
+ physical_rows=rows.physical_rows,
+ deleted_rows=deleted_rows,
+ )
+ )
+
+ found.sort(key=lambda item: item.deleted_rows, reverse=True)
+ for item in found:
+ context.log.info(
+ f"{target.data_table} {item.partition} shard {item.shard}: {item.deleted_rows} of "
+ f"{item.physical_rows} rows deleted ({item.deleted_rows / item.physical_rows:.1%})"
+ )
+ if not found:
+ context.log.info(
+ f"No {target.data_table} partition has {config.min_deleted_ratio:.0%} or more of its rows deleted"
+ )
+
+ context.add_output_metadata(
+ {
+ "deleted_ratio_by_partition_shard": dagster.MetadataValue.json(report),
+ "partitions": dagster.MetadataValue.int(len(found)),
+ }
+ )
+ return found
+
+ return find
+
+
+def _apply_deleted_mask_op(name: str, target: DeletionTarget) -> dagster.OpDefinition:
+ @dagster.op(name=f"apply_deleted_mask_{name}")
+ def apply(
+ context: dagster.OpExecutionContext,
+ config: ApplyDeletedMaskConfig,
+ cluster: dagster.ResourceParam[ClickhouseCluster],
+ partitions: list[DeletedMaskPartition],
+ ) -> None:
+ """Rewrite each found month on its shard without its deleted rows, one mutation at a time."""
+ if config.dry_run or not partitions:
+ context.log.info(
+ f"Not rewriting {len(partitions)} {target.data_table} partition(s), dry run: {config.dry_run}"
+ )
+ return
+
+ placement = placement_for(cluster, target)
+ assert placement is not None
+ # The commands name only the month, so a mask an earlier run applied matches them too.
+ # Adopting that finished mutation would rewrite nothing.
+ reuse_floor = _mutation_reuse_floor(cluster)
+
+ for idx, item in enumerate(partitions, 1):
+ context.log.info(
+ f"{target.data_table} {item.partition} shard {item.shard}: removing {item.deleted_rows} rows "
+ f"({idx}/{len(partitions)})"
+ )
+ commands = [f"APPLY DELETED MASK IN PARTITION '{item.partition}'"]
+ if target.uses_patch_parts:
+ # Rows deleted through a patch part are not in the deleted mask until the patch is
+ # applied. One ALTER with both commands applies the mask first, so they run in turn.
+ commands.insert(0, f"APPLY PATCHES IN PARTITION '{item.partition}'")
+ for command in commands:
+ # Each rewrite is heavy, so the next one starts only after every replica finishes this one.
+ AlterTableMutationRunner(target.data_table, commands={command}, reuse_since=reuse_floor).run_on_shards(
+ placement.cluster, [item.shard]
+ )
+
+ context.add_output_metadata({"partitions_processed": dagster.MetadataValue.int(len(partitions))})
+
+ return apply
+
+
+# Each table sits on its own cluster with its own hardware, so each gets its own threshold, its own
+# dry run switch, and its own queue of rewrites. The two queues run side by side.
+_TABLE_OPS = {
+ name: (_find_deleted_mask_partitions_op(name, target), _apply_deleted_mask_op(name, target))
+ for name, target in (("events", EVENTS), ("events_json", EVENTS_JSON))
+}
+
+
+ tags={
+ "owner": JobOwners.TEAM_CLICKHOUSE.value,
+ # Shares the run-queue limit with deletes_job, the old events cleanup and squash, because
+ # each rewrite is a mutation on the same events tables.
+ "deletes_job_concurrency": "v1",
+ }
+)
+def apply_deleted_mask_job():
+ """Find events table months with many deleted rows, then remove those rows from disk, most deleted rows first."""
+ guard = ensure_no_concurrent_deletes_run()
+ for find, apply in _TABLE_OPS.values():
+ apply(find(start_after=guard))
diff --git a/posthog/dags/locations/clickhouse.py b/posthog/dags/locations/clickhouse.py
index 2695a2f81861..b65d5d788cde 100644
--- a/posthog/dags/locations/clickhouse.py
+++ b/posthog/dags/locations/clickhouse.py
@@ -2,6 +2,7 @@ import dagster
from posthog.dags import (
add_index_to_materialized_column,
+ apply_deleted_mask,
backfill_materialized_column,
ch_examples,
clickhouse_cleanup,
@@ -34,6 +35,7 @@ defs = dagster.Definitions(
],
jobs=[
add_index_to_materialized_column.add_index_to_materialized_column,
+ apply_deleted_mask.apply_deleted_mask_job,
clickhouse_cleanup.clickhouse_deletion_sweep_job,
create_materialized_column.create_materialized_column,
drop_materialized_column.drop_materialized_column,
diff --git a/posthog/dags/tests/test_apply_deleted_mask.py b/posthog/dags/tests/test_apply_deleted_mask.py
new file mode 100644
index 000000000000..80e3d5477e1b
--- /dev/null
+++ b/posthog/dags/tests/test_apply_deleted_mask.py
@@ -0,0 +1,126 @@
+from datetime import datetime
+from uuid import UUID
+
+import pytest
+
+from django.conf import settings
+
+from clickhouse_driver import Client
+
+from posthog.clickhouse.cluster import ClickhouseCluster, LightweightDeleteMutationRunner, Query
+from posthog.dags.apply_deleted_mask import DeletedMaskPartition, apply_deleted_mask_job
+from posthog.models.deletion_targets import EVENTS_TARGETS, resolve_placements
+from posthog.models.event.sql import EVENTS_DATA_TABLE, EVENTS_JSON_DATA_TABLE
+
+TEAM_ID = 4242
+MOSTLY_DELETED = "202412"
+FEW_DELETED = "202501"
+MOST_DELETED = "202502"
+# The events_json threshold sits between the 50% and 80% months, so it rewrites only one of them.
+MIN_DELETED_RATIO = {"events": 0.1, "events_json": 0.6}
+
+
+def _insert_events(cluster: ClickhouseCluster) -> None:
+ rows = [
+ (TEAM_ID, f"user_{i}", UUID(int=i), datetime(2024, 12, 10) if i < 100 else datetime(2025, i // 100, 10))
+ for i in range(300)
+ ]
+
+ def insert(client: Client) -> None:
+ client.execute("INSERT INTO writable_events (team_id, distinct_id, uuid, timestamp) VALUES", rows)
+ client.execute(f"INSERT INTO {EVENTS_JSON_DATA_TABLE} (team_id, distinct_id, uuid, timestamp) VALUES", rows)
+
+ cluster.any_host(insert).result()
+
+
+def _delete(cluster: ClickhouseCluster, partition: str, below_uuid: int) -> None:
+ for placement in resolve_placements(cluster, EVENTS_TARGETS):
+ LightweightDeleteMutationRunner(
+ table=placement.target.data_table,
+ predicate="team_id = %(team_id)s AND uuid < %(below)s",
+ parameters={"team_id": TEAM_ID, "below": UUID(int=below_uuid)},
+ partition=partition,
+ settings={"lightweight_deletes_sync": 2},
+ patch_parts=placement.target.uses_patch_parts,
+ ).run_on_shards(placement.cluster)
+
+
+def _physical_rows(cluster: ClickhouseCluster) -> dict[tuple[str, str], int]:
+ counts: dict[tuple[str, str], int] = {}
+ for placement in resolve_placements(cluster, EVENTS_TARGETS):
+
+ def count(client: Client, table: str = placement.target.data_table) -> list[tuple[str, int]]:
+ return client.execute(
+ """
+ SELECT partition_id, sum(rows) FROM system.parts
+ WHERE database = %(database)s AND table = %(table)s AND active AND partition_id IN %(partitions)s
+ GROUP BY partition_id
+ """,
+ {
+ "database": settings.CLICKHOUSE_DATABASE,
+ "table": table,
+ "partitions": [MOSTLY_DELETED, FEW_DELETED, MOST_DELETED],
+ },
+ )
+
+ for rows in placement.cluster.map_one_host_per_shard(count).result().values():
+ for partition, physical in rows:
+ key = (placement.target.data_table, partition)
+ counts[key] = counts.get(key, 0) + physical
+ return counts
+
+
+def _run(cluster: ClickhouseCluster, dry_run: bool) -> dict[str, list[DeletedMaskPartition]]:
+ partitions = [MOSTLY_DELETED, FEW_DELETED, MOST_DELETED]
+ result = apply_deleted_mask_job.execute_in_process(
+ run_config={
+ "ops": {
+ **{
+ f"find_deleted_mask_partitions_{name}": {
+ "config": {"partitions": partitions, "min_deleted_ratio": ratio}
+ }
+ for name, ratio in MIN_DELETED_RATIO.items()
+ },
+ **{f"apply_deleted_mask_{name}": {"config": {"dry_run": dry_run}} for name in MIN_DELETED_RATIO},
+ }
+ },
+ resources={"cluster": cluster},
+ )
+ return {name: result.output_for_node(f"find_deleted_mask_partitions_{name}") for name in MIN_DELETED_RATIO}
+
+
+def test_found_partitions_lose_their_deleted_rows_and_others_stay_untouched(cluster: ClickhouseCluster):
+ try:
+ _find_and_apply(cluster)
+ finally:
+ cluster.map_one_host_per_shard(Query(f"TRUNCATE TABLE {EVENTS_JSON_DATA_TABLE}")).result()
+
+
+def _find_and_apply(cluster: ClickhouseCluster) -> None:
+ _insert_events(cluster)
+ _delete(cluster, MOSTLY_DELETED, below_uuid=50)
+ _delete(cluster, FEW_DELETED, below_uuid=101)
+ _delete(cluster, MOST_DELETED, below_uuid=280)
+ before = _physical_rows(cluster)
+
+ _run(cluster, dry_run=True)
+ assert _physical_rows(cluster) == before
+
+ found = _run(cluster, dry_run=False)
+ assert {name: {item.partition for item in items} for name, items in found.items()} == {
+ "events": {MOSTLY_DELETED, MOST_DELETED},
+ "events_json": {MOST_DELETED},
+ }
+ deleted_rows = [item.deleted_rows for item in found["events"]]
+ assert deleted_rows == sorted(deleted_rows, reverse=True)
+
+ after = _physical_rows(cluster)
+ # sharded_events_json deletes through patch parts, which the mask only removes after APPLY PATCHES.
+ assert {table for table, _ in before} == {EVENTS_DATA_TABLE(), EVENTS_JSON_DATA_TABLE}
+ removed = {
+ EVENTS_DATA_TABLE(): {MOSTLY_DELETED: 50, FEW_DELETED: 0, MOST_DELETED: 80},
+ EVENTS_JSON_DATA_TABLE: {MOSTLY_DELETED: 0, FEW_DELETED: 0, MOST_DELETED: 80},
+ }
+ for table, partition in before:
+ assert after[(table, partition)] == before[(table, partition)] - removed[table][partition]