Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(dags): keep run log rows correct when the last step writes first (#122664)

Paweł Szczur10b4abbb6429
diff --git a/posthog/admin/admins/async_deletion_admin.py b/posthog/admin/admins/async_deletion_admin.py
index 07eac179b32e..f4b014a379a3 100644
--- a/posthog/admin/admins/async_deletion_admin.py
+++ b/posthog/admin/admins/async_deletion_admin.py
@@ -45,7 +45,11 @@ class AsyncDeletionAdmin(admin.ModelAdmin):
# A subquery keeps the watermark lookup to the one changelist query instead of one per row.
latest_watermark = (
DeletionRunLog.objects.filter(
- job=DeletionRunJob.DELETES, status=DeletionRunStatus.SUCCEEDED, limited_to_team_id__isnull=True
+ job=DeletionRunJob.DELETES,
+ status=DeletionRunStatus.SUCCEEDED,
+ limited_to_team_id__isnull=True,
+ # A run that skipped the watermark step has no watermark and must not hide the last known one.
+ watermark__isnull=False,
)
.order_by("-started_at")
.values("watermark")[:1]
diff --git a/posthog/admin/test_async_deletion_admin.py b/posthog/admin/test_async_deletion_admin.py
index d00a6838ac3d..27706efe11a1 100644
--- a/posthog/admin/test_async_deletion_admin.py
+++ b/posthog/admin/test_async_deletion_admin.py
@@ -1,11 +1,14 @@
from datetime import UTC, datetime
-from django.test import SimpleTestCase
+import pytest
+
+from django.contrib.admin.sites import AdminSite
+from django.test import RequestFactory, SimpleTestCase
from parameterized import parameterized
-from posthog.admin.admins.async_deletion_admin import deletion_status
-from posthog.models.async_deletion import AsyncDeletion, DeletionType
+from posthog.admin.admins.async_deletion_admin import AsyncDeletionAdmin, deletion_status
+from posthog.models.async_deletion import AsyncDeletion, DeletionRunJob, DeletionRunLog, DeletionRunStatus, DeletionType
WATERMARK = datetime(2026, 10, 15, 22, 0, tzinfo=UTC)
BEFORE = datetime(2026, 10, 14, 12, 0, tzinfo=UTC)
@@ -49,3 +52,25 @@ class TestDeletionStatus(SimpleTestCase):
deletion_type=deletion_type, team_id=1, key="k", created_at=created_at, delete_verified_at=verified_at
)
assert deletion_status(deletion, watermark) == expected
+
+
+def test_a_run_without_a_watermark_does_not_hide_the_last_known_watermark(team) -> None:
+ deletion = AsyncDeletion.objects.create(deletion_type=DeletionType.Person, team_id=team.pk, key="k")
+ AsyncDeletion.objects.filter(pk=deletion.pk).update(created_at=BEFORE)
+ DeletionRunLog.record(
+ "full",
+ DeletionRunJob.DELETES,
+ None,
+ started_at=WATERMARK,
+ watermark=WATERMARK,
+ status=DeletionRunStatus.SUCCEEDED,
+ )
+ DeletionRunLog.record(
+ "ends_only", DeletionRunJob.DELETES, None, started_at=AFTER, status=DeletionRunStatus.SUCCEEDED
+ )
+
+ admin = AsyncDeletionAdmin(AsyncDeletion, AdminSite())
+ row = admin.get_queryset(RequestFactory().get("/")).get(pk=deletion.pk)
+
+ assert admin.status(row) == "Queued for the next deletes_job run"
diff --git a/posthog/dags/common/deletion_run_log.py b/posthog/dags/common/deletion_run_log.py
index 5f2348cb8f48..9f087b9bcc19 100644
--- a/posthog/dags/common/deletion_run_log.py
+++ b/posthog/dags/common/deletion_run_log.py
@@ -1,3 +1,4 @@
+from datetime import UTC, datetime
from typing import Any
from django.utils import timezone
@@ -14,13 +15,16 @@ _FINAL_STATUSES = {
}
-def _parent_run_id(instance: dagster.DagsterInstance, run_id: str) -> str | None:
- run = instance.get_run_by_id(run_id)
- return run.parent_run_id if run else None
+def _record(instance: dagster.DagsterInstance, run_id: str, job: DeletionRunJob, **fields: Any) -> None:
+ # Any step can make the first write, so the start time comes from Dagster and not from the write.
+ record = instance.get_run_record_by_id(run_id)
+ started_at = datetime.fromtimestamp(record.start_time, UTC) if record and record.start_time else None
+ parent_run_id = record.dagster_run.parent_run_id if record else None
+ DeletionRunLog.record(run_id, job, parent_run_id, started_at=started_at, **fields)
def record_run(context: dagster.OpExecutionContext, job: DeletionRunJob, **fields: Any) -> None:
- DeletionRunLog.record(context.run_id, job, context.run.parent_run_id, **fields)
+ _record(context.instance, context.run_id, job, **fields)
def record_run_succeeded(context: dagster.OpExecutionContext, job: DeletionRunJob) -> None:
@@ -28,10 +32,10 @@ def record_run_succeeded(context: dagster.OpExecutionContext, job: DeletionRunJo
def record_run_failed(context: dagster.HookContext, job: DeletionRunJob) -> None:
- DeletionRunLog.record(
+ _record(
+ context.instance,
context.run_id,
job,
- _parent_run_id(context.instance, context.run_id),
status=DeletionRunStatus.FAILED,
finished_at=timezone.now(),
error=f"{context.op.name}: {context.op_exception}",
diff --git a/posthog/dags/tests/test_deletion_run_log.py b/posthog/dags/tests/test_deletion_run_log.py
index a2e14f05e24c..3485de2a2e5e 100644
--- a/posthog/dags/tests/test_deletion_run_log.py
+++ b/posthog/dags/tests/test_deletion_run_log.py
@@ -6,10 +6,11 @@ from unittest.mock import Mock
import dagster
from parameterized import parameterized
-from posthog.dags.common.deletion_run_log import reconcile_running_logs
+from posthog.dags.common.deletion_run_log import reconcile_running_logs, record_run, record_run_succeeded
from posthog.models.async_deletion import DeletionRunJob, DeletionRunLog, DeletionRunStatus
WATERMARK = datetime(2026, 10, 15, 22, 0, tzinfo=UTC)
+RUN_STARTED = datetime(2026, 10, 17, 22, 0, tzinfo=UTC)
@pytest.mark.django_db
@@ -54,3 +55,29 @@ def test_reconcile_closes_rows_whose_run_ended(
log = DeletionRunLog.objects.get(dagster_run_id="run")
assert log.status == expected
assert (log.finished_at is None) == (expected == DeletionRunStatus.RUNNING)
+
+
+def _op_context(run_id: str) -> Mock:
+ context = Mock(run_id=run_id)
+ context.instance.get_run_record_by_id.return_value = Mock(
+ start_time=RUN_STARTED.timestamp(), dagster_run=Mock(parent_run_id=None)
+ )
+ return context
+
+
+def test_a_row_first_written_by_the_last_step_starts_when_the_dagster_run_started() -> None:
+ record_run_succeeded(_op_context("ends_only"), DeletionRunJob.DELETES)
+
+ log = DeletionRunLog.objects.get(dagster_run_id="ends_only")
+ assert (log.started_at, log.status) == (RUN_STARTED, DeletionRunStatus.SUCCEEDED)
+ assert log.finished_at is not None and log.finished_at > RUN_STARTED
+
+
+def test_a_later_write_keeps_the_start_time_of_the_row() -> None:
+ DeletionRunLog.record("run", DeletionRunJob.DELETES, None, started_at=WATERMARK)
+
+ record_run(_op_context("run"), DeletionRunJob.DELETES, watermark=WATERMARK)
+
+ assert DeletionRunLog.objects.get(dagster_run_id="run").started_at == WATERMARK
diff --git a/posthog/models/async_deletion/deletion_run_log.py b/posthog/models/async_deletion/deletion_run_log.py
index 239735b32398..5f0e63ecaf23 100644
--- a/posthog/models/async_deletion/deletion_run_log.py
+++ b/posthog/models/async_deletion/deletion_run_log.py
@@ -1,3 +1,4 @@
+from datetime import datetime
from typing import Any
from django.db import models
@@ -66,8 +67,18 @@ class DeletionRunLog(models.Model):
indexes = [models.Index(fields=["job", "status", "-started_at"], name="deletion_run_log_latest")]
@classmethod
- def record(cls, dagster_run_id: str, job: DeletionRunJob, parent_run_id: str | None, **fields: Any) -> None:
- """Create the run's row on its first write and update it on every later one."""
+ def record(
+ cls,
+ dagster_run_id: str,
+ job: DeletionRunJob,
+ parent_run_id: str | None,
+ started_at: datetime | None = None,
+ **fields: Any,
+ ) -> None:
+ """Create the run's row on its first write and update it on every later one.
+
+ started_at is only used when this write creates the row, because the first write can come from any step.
+ """
inherited = (
cls.objects.filter(dagster_run_id=parent_run_id).values(*cls.INHERITED_FIELDS).first()
if parent_run_id
@@ -76,5 +87,11 @@ class DeletionRunLog(models.Model):
cls.objects.update_or_create(
dagster_run_id=dagster_run_id,
defaults={"job": job, **fields},
- create_defaults={**(inherited or {}), "job": job, "parent_run_id": parent_run_id, **fields},
+ create_defaults={
+ **(inherited or {}),
+ "job": job,
+ "parent_run_id": parent_run_id,
+ "started_at": started_at or timezone.now(),
+ **fields,
+ },
)