Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(alerts): delete a shadow platform copy with its logs or insight alert (#129755)

Jimmy Zhu8b954840fa6c
diff --git a/posthog/test/repo_invariants/setup_receivers_baseline.txt b/posthog/test/repo_invariants/setup_receivers_baseline.txt
index ec89e0469410..e7bf8d9d67cd 100644
--- a/posthog/test/repo_invariants/setup_receivers_baseline.txt
+++ b/posthog/test/repo_invariants/setup_receivers_baseline.txt
@@ -4,8 +4,6 @@
# A receiver disappearing from this set means it connects in NO process (silent audit/cache loss):
# restore its AppConfig.ready() wiring instead of deleting the line.
experiment_changed:ee.vercel.receivers.sync_experiment_experimentation_item
-model_activity:posthog.models.activity_logging.signal_handlers.handle_workload_trust_policy_change
-model_activity:posthog.models.activity_logging.signal_handlers.handle_workload_trust_policy_request_change
model_activity:posthog.models.activity_logging.signal_handlers.handle_identity_provider_config_change
model_activity:posthog.models.activity_logging.signal_handlers.handle_oauth_application_scopes_change
model_activity:posthog.models.activity_logging.signal_handlers.handle_organization_change
@@ -16,6 +14,8 @@ model_activity:posthog.models.activity_logging.signal_handlers.handle_personal_a
model_activity:posthog.models.activity_logging.signal_handlers.handle_project_secret_api_key_change
model_activity:posthog.models.activity_logging.signal_handlers.handle_tag_change
model_activity:posthog.models.activity_logging.signal_handlers.handle_tagged_item_change
+model_activity:posthog.models.activity_logging.signal_handlers.handle_workload_trust_policy_change
+model_activity:posthog.models.activity_logging.signal_handlers.handle_workload_trust_policy_request_change
model_activity:posthog.models.activity_logging.signal_handlers.log_user_change_activity
model_activity:products.access_control.backend.activity_logging.handle_access_control_change
model_activity:products.actions.backend.activity_logging.handle_action_change
@@ -302,6 +302,7 @@ pre_delete:posthog.storage.team_access_cache_signal_handlers.clear_project_secre
pre_delete:posthog.storage.team_llm_gateway_policy_signal_handlers._clear_cache_on_delete
pre_delete:posthog.storage.team_metadata_signal_handlers.clear_team_metadata_cache_on_delete
pre_delete:products.alerts.backend.activity_logging.cleanup_alert_hog_functions
+pre_delete:products.alerts.backend.activity_logging.delete_platform_shadow_copy
pre_delete:products.alerts.backend.activity_logging.handle_alert_subscription_delete
pre_delete:products.billing_alerts.backend.team_lifecycle.rehome_billing_alerts_before_team_delete
pre_delete:products.customer_analytics.backend.signals.delete_private_account_views_for_deleted_user
diff --git a/products/alerts/backend/activity_logging.py b/products/alerts/backend/activity_logging.py
index c708d2b80047..1ffcc9dc7d2d 100644
--- a/products/alerts/backend/activity_logging.py
+++ b/products/alerts/backend/activity_logging.py
@@ -134,6 +134,16 @@ def cleanup_alert_hog_functions(sender, instance: AlertConfiguration, **kwargs)
hog_function.save()
+@receiver(pre_delete, sender=AlertConfiguration)
+def delete_platform_shadow_copy(sender, instance: AlertConfiguration, **kwargs) -> None:
+ # A copy the platform owns is left to `delete_moved_alert_copy`, which ends its firings first.
+ if instance.platform_owned:
+ return
+ from products.alerts.backend.platform_ownership import delete_shadow_alert_copy
+
+ delete_shadow_alert_copy(team_id=instance.team_id, alert_id=instance.id)
+
+
@receiver(pre_delete, sender=AlertSubscription)
def handle_alert_subscription_delete(sender, instance, **kwargs) -> None:
from posthog.models.activity_logging.model_activity import get_current_user, get_was_impersonated
diff --git a/products/alerts/backend/management/commands/delete_orphaned_platform_insight_alert_copies.py b/products/alerts/backend/management/commands/delete_orphaned_platform_insight_alert_copies.py
new file mode 100644
index 000000000000..59e637ead775
--- /dev/null
+++ b/products/alerts/backend/management/commands/delete_orphaned_platform_insight_alert_copies.py
@@ -0,0 +1,15 @@
+from collections.abc import Collection
+from uuid import UUID
+
+from products.alerts.backend.models.alert import AlertConfiguration
+from products.alerts_platform.backend.facade.contracts import SourceKind
+from products.alerts_platform.backend.facade.orphan_cleanup_command import PlatformOrphanCleanupCommand
+
+
+class Command(PlatformOrphanCleanupCommand):
+ help = "Delete the alerts platform's shadow copies of insight alerts that no longer exist."
+ source_kind = SourceKind.INSIGHT
+
+ def existing_source_ids(self, ids: Collection[UUID]) -> Collection[UUID]:
+ # nosemgrep: idor-lookup-without-team
+ return set(AlertConfiguration.objects.filter(id__in=ids).values_list("id", flat=True))
diff --git a/products/alerts/backend/platform_ownership.py b/products/alerts/backend/platform_ownership.py
index 0c6a9067909e..cd6892a4b425 100644
--- a/products/alerts/backend/platform_ownership.py
+++ b/products/alerts/backend/platform_ownership.py
@@ -12,6 +12,8 @@ from dataclasses import replace
from datetime import datetime
from uuid import UUID
+from django.db import transaction
+
from products.alerts.backend.evaluation.validation import validate_alert_config
from products.alerts.backend.facade.contracts import PlatformCannotOwn
from products.alerts.backend.insight_alert_state_machine import apply_disable, apply_enable, apply_threshold_change
@@ -25,7 +27,7 @@ from products.alerts.backend.platform_cutover import (
in_an_environment,
)
from products.alerts.backend.platform_source_cycle import platform_upsert_of
-from products.alerts_platform.backend.facade.api import close_owned_firings_now, delete_owned_copy, update_owned_copy
+from products.alerts_platform.backend.facade.api import close_owned_firings_now, delete_copy, update_owned_copy
from products.alerts_platform.backend.facade.contracts import (
SOURCE_CONDITION_KEY,
InvalidPlatformAlertEdit,
@@ -108,7 +110,22 @@ def delete_moved_alert_copy(*, team_id: int, alert_id: UUID) -> None:
"""
alert = _locked(team_id, alert_id)
if alert is not None and alert.platform_owned:
- delete_owned_copy(team_id=team_id, legacy_configuration_id=alert_id)
+ delete_copy(team_id=team_id, legacy_configuration_id=alert_id)
+
+
+def delete_shadow_alert_copy(*, team_id: int, alert_id: UUID) -> None:
+ """Deletes the copy of an alert insight owns once the alert's delete commits, by any route: the
+ API, an insight sweep, or a cascade.
+
+ After the commit, so each copy is locked in a transaction of its own. A sweep or a cascade
+ deletes several alerts in one transaction, and locking their copies there in deletion order can
+ deadlock with `record_outcomes`, which locks them in id order. A copy whose delete fails is left
+ for `delete_orphaned_platform_insight_alert_copies`.
+ """
+ transaction.on_commit(
+ lambda: delete_copy(team_id=team_id, legacy_configuration_id=alert_id, owner=PlatformAlertOwner.SOURCE),
+ robust=True,
+ )
def _locked(team_id: int, alert_id: UUID) -> AlertConfiguration | None:
diff --git a/products/alerts/backend/test/test_platform_alert_backfill.py b/products/alerts/backend/test/test_platform_alert_backfill.py
index 017b4f378aaa..b03aabe3e274 100644
--- a/products/alerts/backend/test/test_platform_alert_backfill.py
+++ b/products/alerts/backend/test/test_platform_alert_backfill.py
@@ -5,6 +5,7 @@ from uuid import UUID
from posthog.test.base import APIBaseTest
from django.core.management import call_command
+from django.db.models.signals import pre_delete
from parameterized import parameterized
@@ -13,6 +14,7 @@ from posthog.schema import ChartDisplayType, EventsNode, IntervalType, TrendsFil
from posthog.models.scoping import team_scope
from posthog.schema_enums import AlertCalculationInterval
+from products.alerts.backend.activity_logging import delete_platform_shadow_copy
from products.alerts.backend.models.alert import AlertConfiguration, Threshold
from products.alerts.backend.platform_alert_backfill import (
backfill_platform_insight_alert_configurations,
@@ -155,3 +157,36 @@ class TestPlatformInsightAlertBackfill(APIBaseTest):
copy = self._copies().get(created.id)
assert (copy.name if copy else None) == copy_name
+
+ @parameterized.expand([("with_the_alert",), ("with_its_insight",)])
+ def test_deleting_an_alert_deletes_its_copy_and_no_other(self, route: str) -> None:
+ deleted = self._alert()
+ kept = self._alert()
+ backfill_platform_insight_alert_configurations(team_id=self.team.id)
+
+ with self.captureOnCommitCallbacks(execute=True):
+ if route == "with_the_alert":
+ deleted.delete()
+ else:
+ deleted.insight.delete()
+
+ assert set(self._copies()) == {kept.id}
+
+ def test_the_cleanup_command_deletes_only_copies_whose_alert_is_gone(self) -> None:
+ orphaned = self._alert()
+ orphaned_id = orphaned.id
+ kept = self._alert()
+ backfill_platform_insight_alert_configurations(team_id=self.team.id)
+ pre_delete.disconnect(delete_platform_shadow_copy, sender=AlertConfiguration)
+ try:
+ orphaned.delete()
+ finally:
+ pre_delete.connect(delete_platform_shadow_copy, sender=AlertConfiguration)
+
+ dry_run = StringIO()
+ call_command("delete_orphaned_platform_insight_alert_copies", dry_run=True, stdout=dry_run)
+ assert set(self._copies()) == {orphaned_id, kept.id}
+ call_command("delete_orphaned_platform_insight_alert_copies", stdout=StringIO())
+
+ assert "Would delete 1 " in dry_run.getvalue()
+ assert set(self._copies()) == {kept.id}
diff --git a/products/alerts_platform/backend/facade/api.py b/products/alerts_platform/backend/facade/api.py
index d1429e5fa1b0..ae05609cac6e 100644
--- a/products/alerts_platform/backend/facade/api.py
+++ b/products/alerts_platform/backend/facade/api.py
@@ -36,7 +36,7 @@ from products.alerts_platform.backend.facade.contracts import (
StoredAlertDestination,
VerdictChange,
)
-from products.alerts_platform.backend.facade.enums import HoldRelease, HoldReleasedBy
+from products.alerts_platform.backend.facade.enums import HoldRelease, HoldReleasedBy, PlatformAlertOwner
from products.alerts_platform.backend.logic import (
alert_destinations,
delivery_holds,
@@ -380,8 +380,23 @@ def delete_configuration(*, team_id: int, source_kinds: Sequence[str], configura
)
-def delete_owned_copy(*, team_id: int, legacy_configuration_id: UUID) -> None:
- platform_writes.delete_owned_copy(team_id=team_id, legacy_configuration_id=legacy_configuration_id)
+def delete_copy(*, team_id: int, legacy_configuration_id: UUID, owner: PlatformAlertOwner | None = None) -> None:
+ """Deletes the copy of a source alert with the source row, in its transaction or after it
+ commits. With `owner`, only a copy that stack owns."""
+ platform_writes.delete_copy(team_id=team_id, legacy_configuration_id=legacy_configuration_id, owner=owner)
+
+
+def delete_orphaned_shadow_copies(
+ *,
+ source_kind: SourceKind,
+ existing_source_ids: Callable[[Collection[UUID]], Collection[UUID]],
+ dry_run: bool,
+) -> int:
+ """Deletes the shadow copies of `source_kind` whose source row is gone, and returns the count.
+ `existing_source_ids` returns which of the given source row ids still exist."""
+ return platform_writes.delete_orphaned_shadow_copies(
+ source_kind=source_kind, existing_source_ids=existing_source_ids, dry_run=dry_run
+ )
def close_owned_firings_now(*, team_id: int, legacy_configuration_id: UUID) -> None:
diff --git a/products/alerts_platform/backend/facade/orphan_cleanup_command.py b/products/alerts_platform/backend/facade/orphan_cleanup_command.py
new file mode 100644
index 000000000000..b2aeb9b50322
--- /dev/null
+++ b/products/alerts_platform/backend/facade/orphan_cleanup_command.py
@@ -0,0 +1,31 @@
+"""The command-line surface a source shares for deleting shadow copies whose source row is gone.
+
+Each source keeps its own command module, so `manage.py` finds it by name, and subclasses this to
+say which of its rows still exist.
+"""
+
+from collections.abc import Collection
+from typing import Any
+from uuid import UUID
+
+from django.core.management.base import BaseCommand
+
+from products.alerts_platform.backend.facade.api import delete_orphaned_shadow_copies
+from products.alerts_platform.backend.facade.contracts import SourceKind
+
+
+class PlatformOrphanCleanupCommand(BaseCommand):
+ source_kind: SourceKind
+
+ def existing_source_ids(self, ids: Collection[UUID]) -> Collection[UUID]:
+ raise NotImplementedError
+
+ def add_arguments(self, parser: Any) -> None:
+ parser.add_argument("--dry-run", action="store_true", help="Count the orphaned copies and delete nothing.")
+
+ def handle(self, *args: Any, **options: Any) -> None:
+ count = delete_orphaned_shadow_copies(
+ source_kind=self.source_kind, existing_source_ids=self.existing_source_ids, dry_run=options["dry_run"]
+ )
+ verb = "Would delete" if options["dry_run"] else "Deleted"
+ self.stdout.write(f"{verb} {count} {self.source_kind.value} shadow copies with no source row")
diff --git a/products/alerts_platform/backend/logic/platform_writes.py b/products/alerts_platform/backend/logic/platform_writes.py
index bc9b8f2e43f3..db6811fe028c 100644
--- a/products/alerts_platform/backend/logic/platform_writes.py
+++ b/products/alerts_platform/backend/logic/platform_writes.py
@@ -4,7 +4,7 @@ A configuration its source owns is a copy of the source's own row, which the sou
and the next check reads back. Writing the copy here would fork it, so those are refused.
"""
-from collections.abc import Callable, Sequence
+from collections.abc import Callable, Collection, Sequence
from dataclasses import replace
from datetime import datetime, timedelta
from typing import Final
@@ -373,13 +373,61 @@ def _delete_native(team_id: int, configuration_id: UUID, native: NativeAlertLife
)
-def delete_owned_copy(*, team_id: int, legacy_configuration_id: UUID) -> None:
- """Deletes the copy of a source alert the platform owns, with its instances and threads."""
- _delete_each(
- PlatformAlertConfiguration.objects.for_team(team_id).filter(
- legacy_configuration_id=legacy_configuration_id, owner=PlatformAlertOwner.PLATFORM
- )
+def delete_copy(*, team_id: int, legacy_configuration_id: UUID, owner: PlatformAlertOwner | None = None) -> None:
+ """Deletes the copy of a source alert, with its instances and threads. With `owner`, only a copy
+ that stack owns.
+
+ Call it with the source row's delete, in its transaction or after it commits. The copy has no
+ foreign key to that row, so nothing else removes it.
+ """
+ copies = PlatformAlertConfiguration.objects.for_team(team_id).filter(
+ legacy_configuration_id=legacy_configuration_id
+ )
+ _delete_each(copies if owner is None else copies.filter(owner=owner))
+
+
+def delete_orphaned_shadow_copies(
+ *,
+ source_kind: SourceKind,
+ existing_source_ids: Callable[[Collection[UUID]], Collection[UUID]],
+ dry_run: bool,
+ batch_size: int = 500,
+) -> int:
+ """Deletes every shadow copy of `source_kind` whose source row no longer exists, and returns
+ how many it deleted, or would delete with `dry_run`.
+
+ `existing_source_ids` returns which of the given source row ids still exist. A copy is created
+ only from a source row that exists, and a deleted source row cannot come back with the same id,
+ so a copy found without its row stays an orphan.
+ """
+ # Cross-team on purpose: a one-off sweep over every copy of the kind. Each delete is scoped to
+ # the copy's own team. Paged by id rather than with `iterator()`, because PgBouncer disables
+ # server-side cursors and `iterator()` would then load every copy at once.
+ copies = PlatformAlertConfiguration.objects.unscoped().filter(
+ source_kind=source_kind.value, owner=PlatformAlertOwner.SOURCE, legacy_configuration_id__isnull=False
)
+ found = 0
+ after: UUID | None = None
+ while batch := list(
+ (copies if after is None else copies.filter(id__gt=after))
+ .order_by("id")
+ .values_list("id", "team_id", "legacy_configuration_id")[:batch_size]
+ ):
+ after = batch[-1][0]
+ existing = set(existing_source_ids([legacy_id for _, _, legacy_id in batch if legacy_id is not None]))
+ orphans = [
+ (configuration_id, team_id) for configuration_id, team_id, legacy_id in batch if legacy_id not in existing
+ ]
+ found += len(orphans)
+ if dry_run:
+ continue
+ for configuration_id, team_id in orphans:
+ _delete_each(
+ PlatformAlertConfiguration.objects.for_team(team_id, canonical=True).filter(
+ id=configuration_id, owner=PlatformAlertOwner.SOURCE
+ )
+ )
+ return found
def _delete_each(
diff --git a/products/logs/backend/alert_deletion.py b/products/logs/backend/alert_deletion.py
index 851a17b5b35d..ea73044f3dd0 100644
--- a/products/logs/backend/alert_deletion.py
+++ b/products/logs/backend/alert_deletion.py
@@ -12,7 +12,7 @@ from products.logs.backend.platform_ownership import before_delete, on_deleted
def delete_alert(*, team_id: int, alert_id: UUID, only_platform_owned: bool = False) -> bool:
- """Deletes the alert, its destinations and any platform copy. False when it was already gone.
+ """Deletes the alert, its destinations and its platform copy, whichever stack owns it. False when it was already gone.
With `only_platform_owned`, raises `PlatformAlertNotEditable` when the alert is not, or stops
being, owned by the platform, such as after a rollback hands it back mid-delete.
@@ -34,8 +34,7 @@ def delete_alert(*, team_id: int, alert_id: UUID, only_platform_owned: bool = Fa
soft_delete_all_alert_destinations(
team_id=locked.team_id, alert_id=str(locked.id), allowed_event_ids=LOGS_ALERT_EVENT_IDS
)
- if locked.platform_owned:
- on_deleted(team_id=locked.team_id, alert_id=locked.id)
+ on_deleted(team_id=locked.team_id, alert_id=locked.id)
locked.delete()
return True
diff --git a/products/logs/backend/management/commands/delete_orphaned_platform_logs_alert_copies.py b/products/logs/backend/management/commands/delete_orphaned_platform_logs_alert_copies.py
new file mode 100644
index 000000000000..6623e8026a1f
--- /dev/null
+++ b/products/logs/backend/management/commands/delete_orphaned_platform_logs_alert_copies.py
@@ -0,0 +1,15 @@
+from collections.abc import Collection
+from uuid import UUID
+
+from products.alerts_platform.backend.facade.contracts import SourceKind
+from products.alerts_platform.backend.facade.orphan_cleanup_command import PlatformOrphanCleanupCommand
+from products.logs.backend.models import LogsAlertConfiguration
+
+
+class Command(PlatformOrphanCleanupCommand):
+ help = "Delete the alerts platform's shadow copies of logs alerts that no longer exist."
+ source_kind = SourceKind.LOGS
+
+ def existing_source_ids(self, ids: Collection[UUID]) -> Collection[UUID]:
+ # nosemgrep: idor-lookup-without-team
+ return set(LogsAlertConfiguration.objects.filter(id__in=ids).values_list("id", flat=True))
diff --git a/products/logs/backend/platform_ownership.py b/products/logs/backend/platform_ownership.py
index 8e94f76d21b9..6e72a1155007 100644
--- a/products/logs/backend/platform_ownership.py
+++ b/products/logs/backend/platform_ownership.py
@@ -13,7 +13,7 @@ from uuid import UUID
from django.utils import timezone
-from products.alerts_platform.backend.facade.api import close_owned_firings_now, delete_owned_copy, update_owned_copy
+from products.alerts_platform.backend.facade.api import close_owned_firings_now, delete_copy, update_owned_copy
from products.alerts_platform.backend.facade.contracts import (
SOURCE_CONDITION_KEY,
Grouping,
@@ -103,7 +103,7 @@ def on_updated(alert: LogsAlertConfiguration, *, snooze_changed: bool, grouping:
def on_deleted(*, team_id: int, alert_id: UUID) -> None:
- delete_owned_copy(team_id=team_id, legacy_configuration_id=alert_id)
+ delete_copy(team_id=team_id, legacy_configuration_id=alert_id)
def _platform_change_of(
diff --git a/products/logs/backend/test/test_alerts_api.py b/products/logs/backend/test/test_alerts_api.py
index c6e9f47e212a..d06a0a1e9aac 100644
--- a/products/logs/backend/test/test_alerts_api.py
+++ b/products/logs/backend/test/test_alerts_api.py
@@ -487,6 +487,22 @@ class TestLogsAlertAPI(APIBaseTest):
copy = next((view for view in copies if str(view.legacy_configuration_id) == alert_id), None)
assert (copy.enabled if copy else None) is copy_enabled
+ def test_deleting_an_alert_logs_owns_deletes_its_copy_and_no_other(self) -> None:
+ self._keep_the_team_on_logs()
+ deleted = self._create_via_api(name="Deleted")
+ kept = self._create_via_api(name="Kept")
+ backfill_platform_alert_configurations(team_id=self.team.id)
+
+ response = self.client.delete(f"{self.base_url}{deleted['id']}/")
+
+ assert response.status_code == status.HTTP_204_NO_CONTENT
+ with team_scope(self.team.id):
+ copies = list_configurations(
+ team_id=self.team.id, source_kinds=[SourceKind.LOGS.value], limit=10, offset=0
+ ).configurations
+ copied = {str(view.legacy_configuration_id) for view in copies}
+ assert kept["id"] in copied and deleted["id"] not in copied
+
def test_a_copy_an_operator_retired_by_name_stays_off_when_its_alert_comes_back_on(self) -> None:
self._keep_the_team_on_logs()
self._create_via_api(name="Already on the platform")