Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(dags): keep the old events cleanup and deletes_job from running at once (#121462)

Paweł Szczur9531f6aac3fa
diff --git a/docs/internal/clickhouse-deletion-coverage.md b/docs/internal/clickhouse-deletion-coverage.md
index 97d9bfbe8546..62cfe06ed746 100644
--- a/docs/internal/clickhouse-deletion-coverage.md
+++ b/docs/internal/clickhouse-deletion-coverage.md
@@ -332,7 +332,9 @@ Staff set the retention in Django admin, on the team and organization pages:
- `cleanup_old_events_by_partition` deletes each planned month for the teams due in that month only. A team with 13 months and a team with 18 months share the older months, and the 13-month team also has months of its own.
A manual run can narrow the plan with `team_ids` and `partitions`. A listed team with no retention is ignored.
-`monthly_old_events_cleanup_schedule` starts one run on the 1st of each month, in every region, and skips when no retention is set. It is stopped by default. The job shares the `deletes_job` run queue, so it does not mutate the same partitions at the same time.
+`monthly_old_events_cleanup_schedule` starts one run on the 1st of each month, in every region, and skips when no retention is set. It is stopped by default.
+The job runs `ensure_no_concurrent_deletes_run` before it plans, and that guard also runs in `deletes_job`. Each job fails at the start while a `deletes_job`, cleanup or squash run is executing, so the two never mutate the events tables at the same time.
+Both jobs carry the `deletes_job_concurrency` tag, so with a run-queue limit of 1 on that tag in the Dagster deployment settings, a second run queues instead of failing the guard.
`sharded_events_json` keeps events dated before 2020, because it stores them in its 202001 partition and the delete addresses the event's own month.
## Adding a table
diff --git a/posthog/dags/deletes.py b/posthog/dags/deletes.py
index d2da943804ce..bc2656061828 100644
--- a/posthog/dags/deletes.py
+++ b/posthog/dags/deletes.py
@@ -395,11 +395,13 @@ class AdhocEventDeletesDictionary(Dictionary):
@dagster.op(out=dagster.Out(dagster.Nothing))
def ensure_no_concurrent_deletes_run(context: dagster.OpExecutionContext) -> None:
- """Fail this run when another run of the same job, or any squash run, is executing.
+ """Fail this run when a deletes run, an old events cleanup run, or a squash run is executing.
Concurrent deletes runs share their dictionary names, so each run's delete mutations read
whichever contents the other run loaded last, and mark_deletions_verified then claims
- deletions the sweep may not have performed. Any other executing run blocks, deliberately
+ deletions the sweep may not have performed. The old events cleanup mutates the same events
+ tables, which reject new mutations while others are unfinished, so a cleanup run and a
+ deletes run must not overlap either. Any other executing run blocks, deliberately
without an election: ranking by creation time lets a run that was queued early and started
late outrank a run already past this guard, and two runs then proceed together. Two racing
starts can both fail here, which is safe and visible; the run-queue limit named on the job's
@@ -413,14 +415,19 @@ def ensure_no_concurrent_deletes_run(context: dagster.OpExecutionContext) -> Non
# A queued run has done nothing yet. Its own guard sees this run once it starts.
blockers = describe_runs(
context.instance,
- (context.job_name, squash_person_overrides.name),
+ (deletes_job.name, monthly_old_events_cleanup_job.name, squash_person_overrides.name),
statuses=EXECUTING_RUN_STATUSES,
exclude_run_id=context.run_id,
)
if blockers:
+ restart = (
+ "start a new run through manual_deletes_job"
+ if context.job_name == deletes_job.name
+ else f"start a new {context.job_name} run"
+ )
raise dagster.Failure(
description="This run yields to: " + "; ".join(blockers) + ". "
- "Wait for them to finish or cancel them, then start a new run through manual_deletes_job."
+ f"Wait for them to finish or cancel them, then {restart}."
)
@@ -1309,13 +1316,14 @@ def ensure_deletes_job_can_start(context: dagster.OpExecutionContext) -> None:
A concurrent deletes_job run is the collision the in-job guard exists for; failing here
reports it before a run is even launched. A squash run means the weekly chain is already in
motion and will launch deletes_job itself on success, so starting one by hand now would race
- it. Another manual trigger means someone else already asked for a run.
+ it. An old events cleanup run would make the launched run fail its in-job guard. Another
+ manual trigger means someone else already asked for a run.
"""
# Unlike the in-job guard, queued and not-started runs count here. This check asks whether
# launching another run would collide, not which of two started runs came first.
blockers = describe_runs(
context.instance,
- (deletes_job.name, squash_person_overrides.name, context.job_name),
+ (deletes_job.name, monthly_old_events_cleanup_job.name, squash_person_overrides.name, context.job_name),
exclude_run_id=context.run_id,
)
if blockers:
@@ -1330,8 +1338,8 @@ def ensure_deletes_job_can_start(context: dagster.OpExecutionContext) -> None:
def manual_deletes_job():
"""Start a deletes_job run the way the weekly chain does.
- Launch this instead of deletes_job itself. It checks that no deletes_job or squash run is in
- flight, and its success makes run_deletes_after_manual_trigger request a real deletes_job run
+ Launch this instead of deletes_job itself. It checks that no deletes_job, old events cleanup
+ or squash run is in flight, and its success makes run_deletes_after_manual_trigger request a real deletes_job run
with DELETES_RUN_CONFIG. Launching deletes_job directly skips both: the launchpad defaults
carry none of that config, and nothing checks the weekly chain before the in-job guard fails
the run mid-flight.
@@ -1353,7 +1361,7 @@ def run_deletes_after_manual_trigger(context: dagster.RunStatusSensorContext) ->
return dagster.RunRequest(run_key=context.dagster_run.run_id, run_config=DELETES_RUN_CONFIG)
[email protected](ins={"start_after": dagster.In(dagster.Nothing)})
def plan_old_events_cleanup(
context: dagster.OpExecutionContext,
config: MonthlyCleanupConfig,
@@ -1494,7 +1502,7 @@ def cleanup_old_events_by_partition(
)
def monthly_old_events_cleanup_job():
"""Delete events past each team's events retention set in Django admin, a whole month at a time."""
- cleanup_old_events_by_partition(plan_old_events_cleanup())
+ cleanup_old_events_by_partition(plan_old_events_cleanup(start_after=ensure_no_concurrent_deletes_run()))
@dagster.schedule(
diff --git a/posthog/dags/tests/test_deletes.py b/posthog/dags/tests/test_deletes.py
index fd08ee043807..9ef3fd6ffc2c 100644
--- a/posthog/dags/tests/test_deletes.py
+++ b/posthog/dags/tests/test_deletes.py
@@ -1321,18 +1321,23 @@ def test_the_survivor_count_retries_before_reporting_unknown(failing_attempts: i
@pytest.mark.parametrize(
- "blocking_job",
+ "guarded_job,blocking_job",
[
- deletes_job,
+ (deletes_job, deletes_job),
# A squash run blocks whatever its age: manual_deletes_job's preflight can go stale
# before the sensor launches the run, so the guard checks squash again at run start.
- squash_person_overrides,
+ (deletes_job, squash_person_overrides),
+ (deletes_job, monthly_old_events_cleanup_job),
+ (monthly_old_events_cleanup_job, deletes_job),
+ (monthly_old_events_cleanup_job, squash_person_overrides),
],
)
-def test_the_deletes_guard_yields_to_an_executing_deletes_or_squash_run(blocking_job: dagster.JobDefinition):
- assert "ensure_no_concurrent_deletes_run" in deletes_job.graph.node_names()
+def test_the_deletes_guard_yields_to_an_executing_run_that_mutates_events(
+ guarded_job: dagster.JobDefinition, blocking_job: dagster.JobDefinition
+):
+ assert "ensure_no_concurrent_deletes_run" in guarded_job.graph.node_names()
- @dagster.job(name=deletes_job.name)
+ @dagster.job(name=guarded_job.name)
def guard_only_job() -> None:
ensure_no_concurrent_deletes_run()
@@ -1351,6 +1356,7 @@ def test_the_deletes_guard_yields_to_an_executing_deletes_or_squash_run(blocking
# A run that has not started yet blocks here even though the in-job guard ignores it: it
# is a collision about to happen, and the manual trigger has launched nothing yet.
(squash_person_overrides, dagster.DagsterRunStatus.NOT_STARTED),
+ (monthly_old_events_cleanup_job, dagster.DagsterRunStatus.STARTED),
],
)
def test_the_manual_trigger_refuses_while_the_weekly_chain_is_active(