:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(dags): keep run log rows correct when the last step writes first (#122664)
Paweł Szczur10b4abbb6429
diff --git a/posthog/admin/admins/async_deletion_admin.py b/posthog/admin/admins/async_deletion_admin.pyindex 07eac179b32e..f4b014a379a3 100644--- a/posthog/admin/admins/async_deletion_admin.py+++ b/posthog/admin/admins/async_deletion_admin.py@@ -45,7 +45,11 @@ class AsyncDeletionAdmin(admin.ModelAdmin):# A subquery keeps the watermark lookup to the one changelist query instead of one per row.latest_watermark = (DeletionRunLog.objects.filter(- job=DeletionRunJob.DELETES, status=DeletionRunStatus.SUCCEEDED, limited_to_team_id__isnull=True+ job=DeletionRunJob.DELETES,+ status=DeletionRunStatus.SUCCEEDED,+ limited_to_team_id__isnull=True,+ # A run that skipped the watermark step has no watermark and must not hide the last known one.+ watermark__isnull=False,).order_by("-started_at").values("watermark")[:1]diff --git a/posthog/admin/test_async_deletion_admin.py b/posthog/admin/test_async_deletion_admin.pyindex d00a6838ac3d..27706efe11a1 100644--- a/posthog/admin/test_async_deletion_admin.py+++ b/posthog/admin/test_async_deletion_admin.py@@ -1,11 +1,14 @@from datetime import UTC, datetime-from django.test import SimpleTestCase+import pytest++from django.contrib.admin.sites import AdminSite+from django.test import RequestFactory, SimpleTestCasefrom parameterized import parameterized-from posthog.admin.admins.async_deletion_admin import deletion_status-from posthog.models.async_deletion import AsyncDeletion, DeletionType+from posthog.admin.admins.async_deletion_admin import AsyncDeletionAdmin, deletion_status+from posthog.models.async_deletion import AsyncDeletion, DeletionRunJob, DeletionRunLog, DeletionRunStatus, DeletionTypeWATERMARK = datetime(2026, 10, 15, 22, 0, tzinfo=UTC)BEFORE = datetime(2026, 10, 14, 12, 0, tzinfo=UTC)@@ -49,3 +52,25 @@ class TestDeletionStatus(SimpleTestCase):deletion_type=deletion_type, team_id=1, key="k", created_at=created_at, delete_verified_at=verified_at)assert deletion_status(deletion, watermark) == expected+++def test_a_run_without_a_watermark_does_not_hide_the_last_known_watermark(team) -> None:+ deletion = AsyncDeletion.objects.create(deletion_type=DeletionType.Person, team_id=team.pk, key="k")+ AsyncDeletion.objects.filter(pk=deletion.pk).update(created_at=BEFORE)+ DeletionRunLog.record(+ "full",+ DeletionRunJob.DELETES,+ None,+ started_at=WATERMARK,+ watermark=WATERMARK,+ status=DeletionRunStatus.SUCCEEDED,+ )+ DeletionRunLog.record(+ "ends_only", DeletionRunJob.DELETES, None, started_at=AFTER, status=DeletionRunStatus.SUCCEEDED+ )++ admin = AsyncDeletionAdmin(AsyncDeletion, AdminSite())+ row = admin.get_queryset(RequestFactory().get("/")).get(pk=deletion.pk)++ assert admin.status(row) == "Queued for the next deletes_job run"diff --git a/posthog/dags/common/deletion_run_log.py b/posthog/dags/common/deletion_run_log.pyindex 5f2348cb8f48..9f087b9bcc19 100644--- a/posthog/dags/common/deletion_run_log.py+++ b/posthog/dags/common/deletion_run_log.py@@ -1,3 +1,4 @@+from datetime import UTC, datetimefrom typing import Anyfrom django.utils import timezone@@ -14,13 +15,16 @@ _FINAL_STATUSES = {}-def _parent_run_id(instance: dagster.DagsterInstance, run_id: str) -> str | None:- run = instance.get_run_by_id(run_id)- return run.parent_run_id if run else None+def _record(instance: dagster.DagsterInstance, run_id: str, job: DeletionRunJob, **fields: Any) -> None:+ # Any step can make the first write, so the start time comes from Dagster and not from the write.+ record = instance.get_run_record_by_id(run_id)+ started_at = datetime.fromtimestamp(record.start_time, UTC) if record and record.start_time else None+ parent_run_id = record.dagster_run.parent_run_id if record else None+ DeletionRunLog.record(run_id, job, parent_run_id, started_at=started_at, **fields)def record_run(context: dagster.OpExecutionContext, job: DeletionRunJob, **fields: Any) -> None:- DeletionRunLog.record(context.run_id, job, context.run.parent_run_id, **fields)+ _record(context.instance, context.run_id, job, **fields)def record_run_succeeded(context: dagster.OpExecutionContext, job: DeletionRunJob) -> None:@@ -28,10 +32,10 @@ def record_run_succeeded(context: dagster.OpExecutionContext, job: DeletionRunJodef record_run_failed(context: dagster.HookContext, job: DeletionRunJob) -> None:- DeletionRunLog.record(+ _record(+ context.instance,context.run_id,job,- _parent_run_id(context.instance, context.run_id),status=DeletionRunStatus.FAILED,finished_at=timezone.now(),error=f"{context.op.name}: {context.op_exception}",diff --git a/posthog/dags/tests/test_deletion_run_log.py b/posthog/dags/tests/test_deletion_run_log.pyindex a2e14f05e24c..3485de2a2e5e 100644--- a/posthog/dags/tests/test_deletion_run_log.py+++ b/posthog/dags/tests/test_deletion_run_log.py@@ -6,10 +6,11 @@ from unittest.mock import Mockimport dagsterfrom parameterized import parameterized-from posthog.dags.common.deletion_run_log import reconcile_running_logs+from posthog.dags.common.deletion_run_log import reconcile_running_logs, record_run, record_run_succeededfrom posthog.models.async_deletion import DeletionRunJob, DeletionRunLog, DeletionRunStatusWATERMARK = datetime(2026, 10, 15, 22, 0, tzinfo=UTC)+RUN_STARTED = datetime(2026, 10, 17, 22, 0, tzinfo=UTC)@pytest.mark.django_db@@ -54,3 +55,29 @@ def test_reconcile_closes_rows_whose_run_ended(log = DeletionRunLog.objects.get(dagster_run_id="run")assert log.status == expectedassert (log.finished_at is None) == (expected == DeletionRunStatus.RUNNING)+++def _op_context(run_id: str) -> Mock:+ context = Mock(run_id=run_id)+ context.instance.get_run_record_by_id.return_value = Mock(+ start_time=RUN_STARTED.timestamp(), dagster_run=Mock(parent_run_id=None)+ )+ return context+++def test_a_row_first_written_by_the_last_step_starts_when_the_dagster_run_started() -> None:+ record_run_succeeded(_op_context("ends_only"), DeletionRunJob.DELETES)++ log = DeletionRunLog.objects.get(dagster_run_id="ends_only")+ assert (log.started_at, log.status) == (RUN_STARTED, DeletionRunStatus.SUCCEEDED)+ assert log.finished_at is not None and log.finished_at > RUN_STARTED+++def test_a_later_write_keeps_the_start_time_of_the_row() -> None:+ DeletionRunLog.record("run", DeletionRunJob.DELETES, None, started_at=WATERMARK)++ record_run(_op_context("run"), DeletionRunJob.DELETES, watermark=WATERMARK)++ assert DeletionRunLog.objects.get(dagster_run_id="run").started_at == WATERMARKdiff --git a/posthog/models/async_deletion/deletion_run_log.py b/posthog/models/async_deletion/deletion_run_log.pyindex 239735b32398..5f0e63ecaf23 100644--- a/posthog/models/async_deletion/deletion_run_log.py+++ b/posthog/models/async_deletion/deletion_run_log.py@@ -1,3 +1,4 @@+from datetime import datetimefrom typing import Anyfrom django.db import models@@ -66,8 +67,18 @@ class DeletionRunLog(models.Model):indexes = [models.Index(fields=["job", "status", "-started_at"], name="deletion_run_log_latest")]@classmethod- def record(cls, dagster_run_id: str, job: DeletionRunJob, parent_run_id: str | None, **fields: Any) -> None:- """Create the run's row on its first write and update it on every later one."""+ def record(+ cls,+ dagster_run_id: str,+ job: DeletionRunJob,+ parent_run_id: str | None,+ started_at: datetime | None = None,+ **fields: Any,+ ) -> None:+ """Create the run's row on its first write and update it on every later one.++ started_at is only used when this write creates the row, because the first write can come from any step.+ """inherited = (cls.objects.filter(dagster_run_id=parent_run_id).values(*cls.INHERITED_FIELDS).first()if parent_run_id@@ -76,5 +87,11 @@ class DeletionRunLog(models.Model):cls.objects.update_or_create(dagster_run_id=dagster_run_id,defaults={"job": job, **fields},- create_defaults={**(inherited or {}), "job": job, "parent_run_id": parent_run_id, **fields},+ create_defaults={+ **(inherited or {}),+ "job": job,+ "parent_run_id": parent_run_id,+ "started_at": started_at or timezone.now(),+ **fields,+ },)