Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): close the incident when an edit ends a platform-owned firing (#118317)

Jimmy Zhua3d17e22fd70
diff --git a/products/alerts_platform/backend/facade/api.py b/products/alerts_platform/backend/facade/api.py
index b255d1e3e9ea..06f967efdce8 100644
--- a/products/alerts_platform/backend/facade/api.py
+++ b/products/alerts_platform/backend/facade/api.py
@@ -173,3 +173,9 @@ def update_owned_copy(
def delete_owned_copy(*, team_id: int, legacy_configuration_id: UUID) -> None:
platform_writes.delete_owned_copy(team_id=team_id, legacy_configuration_id=legacy_configuration_id)
+
+
+def close_owned_firings_now(*, team_id: int, legacy_configuration_id: UUID) -> None:
+ """Ends the firings of a copy the platform owns and sends their incident resolves before returning,
+ for a delete that is about to remove the destinations."""
+ platform_writes.close_owned_firings_now(team_id=team_id, legacy_configuration_id=legacy_configuration_id)
diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.py
index 4b3ddce5800f..b265729cf9b8 100644
--- a/products/alerts_platform/backend/facade/contracts.py
+++ b/products/alerts_platform/backend/facade/contracts.py
@@ -875,6 +875,10 @@ class SourceConfigSpec:
model: type[BaseModel]
validate: Callable[[PlatformAlertConfigurationEdit], None]
mirror: SourceEditMirror | None = None
+ # Where the source's incident destinations listen for a close, and the alert's page in the
+ # source's product. A control-plane close needs both to address its resolve.
+ incident_event_ids: Mapping[str, str] | None = None
+ alert_url: Callable[[int, UUID], str] | None = None
SourceConfigLoader = Callable[[], SourceConfigSpec]
diff --git a/products/alerts_platform/backend/facade/grouping.py b/products/alerts_platform/backend/facade/grouping.py
index a88fbeab50e4..b8dac220a56e 100644
--- a/products/alerts_platform/backend/facade/grouping.py
+++ b/products/alerts_platform/backend/facade/grouping.py
@@ -123,6 +123,11 @@ def delivery_parts(outcome: PlatformAlertOutcome, incident_grouping_keys: Collec
)
if outcome.failure is not None and outcome.failure.kind != AlertEventKind.CHECK:
transitions += 1
+ return parts_for(transitions)
+
+
+def parts_for(transitions: int) -> int:
+ """How many delivery activities this many transitions need."""
return max(1, math.ceil(transitions / MAX_TRANSITIONS_PER_DELIVERY))
diff --git a/products/alerts_platform/backend/facade/testing.py b/products/alerts_platform/backend/facade/testing.py
index 2811ebdae122..6256c48a49c3 100644
--- a/products/alerts_platform/backend/facade/testing.py
+++ b/products/alerts_platform/backend/facade/testing.py
@@ -14,6 +14,7 @@ from typing import Any
from uuid import UUID
from products.alerts_platform.backend.facade.contracts import (
+ DeliveryPath,
PlatformAlertSnapshot,
PlatformConfigurationSnapshot,
SourceCorrespondence,
@@ -21,7 +22,7 @@ from products.alerts_platform.backend.facade.contracts import (
)
from products.alerts_platform.backend.logic.platform_alert_events import PlatformAlertEventRow, insert_events
from products.alerts_platform.backend.logic.platform_reads import instance_view
-from products.alerts_platform.backend.models import PlatformAlert, PlatformAlertConfiguration
+from products.alerts_platform.backend.models import PlatformAlert, PlatformAlertConfiguration, PlatformAlertDeliveryPath
def _configuration_snapshot(row: PlatformAlertConfiguration) -> PlatformConfigurationSnapshot:
@@ -70,14 +71,27 @@ def count_still_due(configuration_ids: Iterable[UUID], *, at: datetime) -> int:
return PlatformAlertConfiguration.objects.filter(id__in=list(configuration_ids), next_check_at__lte=at).count()
-def start_firing(configuration_id: UUID, *, team_id: int, at: datetime) -> None:
- """Put a configuration's instance inside a firing, as live delivery can leave one."""
+def start_firing(
+ configuration_id: UUID, *, team_id: int, at: datetime, delivered_via: DeliveryPath | None = None
+) -> None:
+ """Put a configuration's instance inside a firing, as live delivery can leave one.
+
+ With `delivered_via`, the firing's first delivery took that path, so its resolve takes it too.
+ """
PlatformAlert.objects.update_or_create(
team_id=team_id,
configuration_id=configuration_id,
grouping_key="",
defaults={"state": "firing", "firing_started_at": at},
)
+ if delivered_via is not None:
+ PlatformAlertDeliveryPath.objects.for_team(team_id).create(
+ team_id=team_id,
+ configuration_id=str(configuration_id),
+ grouping_key="",
+ episode_started_at=at,
+ path=delivered_via.value,
+ )
def set_check_status(configuration_id: UUID, check_status: str) -> None:
diff --git a/products/alerts_platform/backend/logic/control_plane.py b/products/alerts_platform/backend/logic/control_plane.py
new file mode 100644
index 000000000000..76034af1e0ba
--- /dev/null
+++ b/products/alerts_platform/backend/logic/control_plane.py
@@ -0,0 +1,149 @@
+"""Ends the firings a control-plane change leaves behind, and closes their incidents.
+
+Disabling an alert, snoozing it, changing what it checks or deleting it ends any firing it is in, as
+on the logs stack. An incident manager needs one resolve for every trigger, and no later check would
+send it: discovery skips a disabled or snoozed alert, and a deleted one is gone. So the change writes
+the end of each firing to history and sends the resolve itself. Only an incident destination acts on
+it, so no message goes out, as the logs stack sends only its incident close.
+"""
+
+from datetime import datetime
+from uuid import UUID
+
+from django.db import transaction
+
+from posthog.dataclasses import frozen
+
+from products.alerts_platform.backend.facade.contracts import (
+ AlertDeliveryRequest,
+ AlertEventKind,
+ IncidentAction,
+ PlatformAlertConfigurationEdit,
+ SourceKind,
+)
+from products.alerts_platform.backend.facade.enums import PlatformAlertState
+from products.alerts_platform.backend.facade.grouping import parts_for
+from products.alerts_platform.backend.logic import source_configs
+from products.alerts_platform.backend.logic.platform_alert_events import PlatformAlertEventRow
+from products.alerts_platform.backend.logic.platform_lifecycle import end_firing_instances, event_row, record_history
+from products.alerts_platform.backend.models import PlatformAlertConfiguration
+
+# The settings whose change makes a firing meaningless, as a threshold or filter change does on the
+# logs stack. A rename or a quiet-hours change leaves it running.
+_DECIDING_SETTINGS = ("source_config", "evaluation_periods", "datapoints_to_alarm")
+
+
+def ending_reason(
+ current: PlatformAlertConfigurationEdit, edit: PlatformAlertConfigurationEdit, *, regrouped: bool, snoozing: bool
+) -> str | None:
+ """Why an edit ends the alert's firings, or None. In the logs stack's order: a disable wins over a
+ snooze, which wins over a change to what the alert checks."""
+ if current.enabled and not edit.enabled:
+ return "disabled"
+ if snoozing:
+ return "snoozed"
+ if regrouped:
+ return "regrouped"
+ if any(getattr(current, field) != getattr(edit, field) for field in _DECIDING_SETTINGS):
+ return "config_changed"
+ return None
+
+
+@frozen
+class PendingClose:
+ """The history rows and the resolve that ending an alert's firings owes, sent once they stand."""
+
+ team_id: int
+ source_kind: str
+ configuration_id: UUID
+ legacy_configuration_id: UUID | None
+ rows: tuple[PlatformAlertEventRow, ...]
+
+
+def end_firings(configuration: PlatformAlertConfiguration, *, reason: str, now: datetime) -> PendingClose | None:
+ """Moves every firing instance of a configuration locked by the caller to not firing.
+
+ Call it before the change is saved, so the history rows state the settings the firing ran under.
+ None when nothing was firing.
+ """
+ firing = end_firing_instances(configuration, now=now)
+ if not firing:
+ return None
+ # Unique per change, because a delivery addresses its rows by this key.
+ evaluation_key = f"control:{reason}:{now.isoformat()}"
+ rows = tuple(
+ event_row(
+ configuration,
+ alert=alert,
+ evaluation_key=evaluation_key,
+ kind=AlertEventKind.CHECK.value,
+ previous_state=PlatformAlertState.FIRING.value,
+ state=PlatformAlertState.NOT_FIRING.value,
+ # The episode the trigger named, which is the incident a resolve has to name.
+ episode_started_at=alert.firing_started_at,
+ consecutive_failures=configuration.consecutive_failures,
+ now=now,
+ )
+ for alert in firing
+ )
+ return PendingClose(
+ team_id=configuration.team_id,
+ source_kind=configuration.source_kind,
+ configuration_id=configuration.id,
+ legacy_configuration_id=configuration.legacy_configuration_id,
+ rows=rows,
+ )
+
+
+def announce_after_commit(pending: PendingClose | None) -> None:
+ """Writes the history and starts the resolve's delivery once the change commits."""
+ if pending is not None:
+ transaction.on_commit(lambda: _announce(pending, inline=False), robust=True)
+
+
+def announce_now(pending: PendingClose | None) -> None:
+ """Writes the history and sends the resolve before returning, for a change about to remove the
+ destinations a later delivery would look up."""
+ if pending is not None:
+ _announce(pending, inline=True)
+
+
+def _announce(pending: PendingClose, *, inline: bool) -> None:
+ # A delivery reads its rows back from history, so the rows go first.
+ record_history(pending.team_id, pending.rows)
+ delivery = _resolve_delivery(pending)
+ if delivery is None:
+ return
+ # Imported here because this module loads in processes that never send, and the sender loads the
+ # Temporal client and every transport.
+ from products.alerts_platform.backend.temporal.web_start import ( # noqa: PLC0415 - keeps Temporal and the transports off the import path
+ deliver,
+ deliver_inline,
+ )
+
+ if inline:
+ deliver_inline(delivery)
+ else:
+ deliver(delivery)
+
+
+def _resolve_delivery(pending: PendingClose) -> AlertDeliveryRequest | None:
+ spec = source_configs.spec(SourceKind(pending.source_kind))
+ if spec is None or spec.incident_event_ids is None or pending.legacy_configuration_id is None:
+ return None
+ grouping_keys = [row.grouping_key for row in pending.rows]
+ return AlertDeliveryRequest(
+ source=SourceKind(pending.source_kind),
+ team_id=pending.team_id,
+ configuration_id=str(pending.configuration_id),
+ evaluation_key=pending.rows[0].evaluation_key,
+ # The id the source's destinations are matched on, as on every check's delivery.
+ destination_alert_id=str(pending.legacy_configuration_id),
+ event_ids_by_kind={},
+ incident_actions=dict.fromkeys(grouping_keys, IncidentAction.RESOLVE),
+ sends_messages=False,
+ event_ids_by_incident_action=dict(spec.incident_event_ids),
+ source_alert_url=spec.alert_url(pending.team_id, pending.legacy_configuration_id) if spec.alert_url else None,
+ owned_by_platform=True,
+ parts=parts_for(len(grouping_keys)),
+ )
diff --git a/products/alerts_platform/backend/logic/platform_lifecycle.py b/products/alerts_platform/backend/logic/platform_lifecycle.py
index 8f8d2f8aa905..5a342a4167d5 100644
--- a/products/alerts_platform/backend/logic/platform_lifecycle.py
+++ b/products/alerts_platform/backend/logic/platform_lifecycle.py
@@ -358,15 +358,11 @@ def _event_row(
now: datetime,
) -> PlatformAlertEventRow:
group = verdict if isinstance(verdict, GroupOutcome) else None
- return PlatformAlertEventRow(
- team_id=configuration.team_id,
- source_kind=configuration.source_kind,
- configuration_id=configuration.id,
- alert_id=alert.id if alert else CONFIGURATION_ROW_ALERT_ID,
- grouping_key=alert.grouping_key if alert else "",
+ return event_row(
+ configuration,
+ alert=alert,
evaluation_key=outcome.evaluation_key,
kind=verdict.kind.value,
- alert_name=configuration.name,
previous_state=previous_state,
state=verdict.new_state,
# The whole episode, ended or not. A resolve names the firing it closed, which is what a
@@ -374,17 +370,57 @@ def _event_row(
episode_started_at=verdict.firing_episode.started_at if verdict.firing_episode else None,
value=group.value if group else None,
labels=group.labels if group else {},
- condition_snapshot=_condition_snapshot(configuration),
- source_config_snapshot=configuration.source_config,
query_duration_ms=outcome.query_duration_ms,
error_message=outcome.error_message,
consecutive_failures=outcome.consecutive_failures,
muted_notification=verdict.muted_notification,
+ now=now,
+ )
+
+
+def event_row(
+ configuration: PlatformAlertConfiguration,
+ *,
+ alert: PlatformAlert | None,
+ evaluation_key: str,
+ kind: str,
+ previous_state: str,
+ state: str,
+ episode_started_at: datetime | None,
+ consecutive_failures: int,
+ now: datetime,
+ value: float | None = None,
+ labels: dict[str, str] | None = None,
+ query_duration_ms: int | None = None,
+ error_message: str | None = None,
+ muted_notification: str = "",
+) -> PlatformAlertEventRow:
+ """One history row for a configuration, with the snapshots every row carries."""
+ return PlatformAlertEventRow(
+ team_id=configuration.team_id,
+ source_kind=configuration.source_kind,
+ configuration_id=configuration.id,
+ alert_id=alert.id if alert else CONFIGURATION_ROW_ALERT_ID,
+ grouping_key=alert.grouping_key if alert else "",
+ evaluation_key=evaluation_key,
+ kind=kind,
+ alert_name=configuration.name,
+ previous_state=previous_state,
+ state=state,
+ episode_started_at=episode_started_at,
+ value=value,
+ labels=labels or {},
+ condition_snapshot=_condition_snapshot(configuration),
+ source_config_snapshot=configuration.source_config,
+ query_duration_ms=query_duration_ms,
+ error_message=error_message,
+ consecutive_failures=consecutive_failures,
+ muted_notification=muted_notification,
occurred_at=now,
)
-def _record_history(team_id: int, rows: Sequence[PlatformAlertEventRow]) -> None:
+def record_history(team_id: int, rows: Sequence[PlatformAlertEventRow]) -> None:
recorded = insert_events(team_id, rows)
if recorded < len(rows):
# One batch holds one source's checks.
@@ -564,7 +600,7 @@ def record_outcomes(team_id: int, outcomes: Sequence[PlatformAlertOutcome], now:
)
# `on_commit` rather than a statement after the block, so a caller that wraps this in its
# own `atomic()` cannot leave history for state its rollback removed.
- transaction.on_commit(lambda: _record_history(team_id, rows))
+ transaction.on_commit(lambda: record_history(team_id, rows))
return len(configurations)
@@ -707,6 +743,43 @@ def write_edit(
configuration.save(update_fields=update_fields)
+def end_firing_instances(configuration: PlatformAlertConfiguration, *, now: datetime) -> list[PlatformAlert]:
+ """Moves every firing instance of a configuration locked by the caller to not firing, and returns
+ them as they were.
+
+ The next check moves past a fence, so a check already running finds the row no longer due and
+ records nothing. Otherwise it would write back the firing this just ended, with no incident open.
+ """
+ instances = PlatformAlert.objects.for_team(configuration.team_id, canonical=True).filter(
+ configuration=configuration, state=PlatformAlertState.FIRING
+ )
+ firing = list(instances)
+ if firing:
+ instances.update(state=PlatformAlertState.NOT_FIRING, firing_started_at=None)
+ configuration.next_check_at = max(configuration.next_check_at or now, now + _REVIVE_FENCE)
+ configuration.save(update_fields=["next_check_at"])
+ return firing
+
+
+def write_snooze(configuration: PlatformAlertConfiguration, snooze_until: datetime | None, *, now: datetime) -> None:
+ """Sets or clears a configuration's mute from a source's API.
+
+ A snooze defers the next check to its end, as the logs stack runs no check while snoozed. A mute
+ alone holds only messages, and a breach that went on would open a new incident during the snooze.
+ """
+ next_check_at = configuration.next_check_at or now
+ current = configuration.snooze_until
+ if current is not None and current > now:
+ # The stored due time is the current snooze's end, which the new snooze replaces. A fence an
+ # edit set earlier in this transaction is at most this far out, so it holds.
+ next_check_at = min(next_check_at, now + _REVIVE_FENCE)
+ if snooze_until is not None and snooze_until > now:
+ next_check_at = max(next_check_at, snooze_until)
+ configuration.snooze_until = snooze_until
+ configuration.next_check_at = next_check_at
+ configuration.save(update_fields=["snooze_until", "next_check_at", "updated_at"])
+
+
def upsert_if_running(upsert: PlatformAlertUpsert, *, revive: bool = False) -> None:
"""Copies one configuration in when its team already runs that source here.
diff --git a/products/alerts_platform/backend/logic/platform_writes.py b/products/alerts_platform/backend/logic/platform_writes.py
index 079d6775f0dd..cca495a1464c 100644
--- a/products/alerts_platform/backend/logic/platform_writes.py
+++ b/products/alerts_platform/backend/logic/platform_writes.py
@@ -28,7 +28,7 @@ from products.alerts_platform.backend.facade.contracts import (
)
from products.alerts_platform.backend.facade.enums import PlatformAlertOwner
from products.alerts_platform.backend.facade.scheduling import validate_and_normalize_schedule_restriction
-from products.alerts_platform.backend.logic import platform_lifecycle, platform_reads, source_configs
+from products.alerts_platform.backend.logic import control_plane, platform_lifecycle, platform_reads, source_configs
from products.alerts_platform.backend.models import PlatformAlertConfiguration
@@ -116,9 +116,10 @@ def update_configuration(
configuration = readable.select_for_update().first()
if configuration is None:
return None
- edit = _apply(configuration, change)
+ edit, pending = _apply(configuration, change, now=django_timezone.now())
if write_source is not None:
write_source(edit)
+ control_plane.announce_after_commit(pending)
return platform_reads.get_configuration(
team_id=team_id, source_kinds=source_kinds, configuration_id=configuration_id
@@ -140,10 +141,29 @@ def update_owned_copy(
configuration = _locked_owned_copy(team_id, legacy_configuration_id)
if configuration is None:
return
- _apply(configuration, change)
+ now = django_timezone.now()
+ # A snooze ends the firing, as on the logs stack. A firing after it opens a new incident.
+ snoozing = snooze_changed and snooze_until is not None and snooze_until > now
+ _, pending = _apply(configuration, change, now=now, snoozing=snoozing)
if snooze_changed:
- configuration.snooze_until = snooze_until
- configuration.save(update_fields=["snooze_until", "updated_at"])
+ platform_lifecycle.write_snooze(configuration, snooze_until, now=now)
+ control_plane.announce_after_commit(pending)
+
+
+def close_owned_firings_now(*, team_id: int, legacy_configuration_id: UUID) -> None:
+ """Ends the firings of a copy the platform owns and sends their resolves before returning.
+
+ For a delete, which removes the destinations a delivery after commit would look up. Call it
+ outside the delete's transaction, so the history it reads and the state it moved have committed.
+ """
+ with transaction.atomic():
+ configuration = _locked_owned_copy(team_id, legacy_configuration_id)
+ pending = (
+ None
+ if configuration is None
+ else control_plane.end_firings(configuration, reason="deleted", now=django_timezone.now())
+ )
+ control_plane.announce_now(pending)
def delete_owned_copy(*, team_id: int, legacy_configuration_id: UUID) -> None:
@@ -178,8 +198,12 @@ def _locked_owned_copy(team_id: int, legacy_configuration_id: UUID) -> PlatformA
def _apply(
configuration: PlatformAlertConfiguration,
change: Callable[[PlatformAlertConfigurationEdit], PlatformAlertConfigurationEdit],
-) -> PlatformAlertConfigurationEdit:
- """Validates and writes an edit onto a configuration locked by the caller."""
+ *,
+ now: datetime,
+ snoozing: bool = False,
+) -> tuple[PlatformAlertConfigurationEdit, control_plane.PendingClose | None]:
+ """Validates and writes an edit onto a configuration locked by the caller, and ends the firings it
+ makes meaningless."""
if configuration.owner != PlatformAlertOwner.PLATFORM:
raise PlatformAlertNotEditable(
f"This alert is managed by the {configuration.source_kind} product. Edit it there."
@@ -187,7 +211,10 @@ def _apply(
current = edit_of(configuration)
edit = _validated(SourceKind(configuration.source_kind), current, change(current))
- platform_lifecycle.write_edit(
- configuration, edit, regrouped=not edit.grouping.splits_like(current.grouping), now=django_timezone.now()
- )
- return edit
+ regrouped = not edit.grouping.splits_like(current.grouping)
+ ending = control_plane.ending_reason(current, edit, regrouped=regrouped, snoozing=snoozing)
+ # Before the instances change, so the history rows name the firings as they ran.
+ pending = control_plane.end_firings(configuration, reason=ending, now=now) if ending else None
+
+ platform_lifecycle.write_edit(configuration, edit, regrouped=regrouped, now=now)
+ return edit, pending
diff --git a/products/alerts_platform/backend/temporal/web_start.py b/products/alerts_platform/backend/temporal/web_start.py
new file mode 100644
index 000000000000..010973c4a4da
--- /dev/null
+++ b/products/alerts_platform/backend/temporal/web_start.py
@@ -0,0 +1,59 @@
+"""Sends a delivery from a web request, outside any workflow.
+
+Kept out of the workflow modules, which Temporal loads into its sandbox, because this connects a
+client.
+"""
+
+from dataclasses import replace
+
+from django.conf import settings
+
+from asgiref.sync import async_to_sync
+from temporalio.common import WorkflowIDConflictPolicy
+
+from posthog.exceptions_capture import capture_exception
+from posthog.temporal.common.client import async_connect
+
+from products.alerts_platform.backend.delivery.evaluation import deliver_evaluation
+from products.alerts_platform.backend.facade.contracts import AlertDeliveryRequest
+from products.alerts_platform.backend.temporal.workflows import (
+ AlertsPlatformDeliverPreviewWorkflow,
+ delivery_execution_timeout,
+ delivery_workflow_id,
+)
+
+
+def deliver(delivery: AlertDeliveryRequest) -> None:
+ """Starts the delivery workflow, so the delivery gets its retries and its wait on a busy thread.
+
+ Sends inline when Temporal cannot be reached, because a caller that ends a firing is the only one
+ that will ever send its resolve.
+ """
+ try:
+ async_to_sync(_start)(delivery)
+ except Exception as error:
+ capture_exception(error, {"configuration_id": delivery.configuration_id})
+ deliver_inline(delivery)
+
+
+def deliver_inline(delivery: AlertDeliveryRequest) -> None:
+ """Sends every part before returning, for a caller about to remove the destinations."""
+ for part in range(delivery.parts):
+ try:
+ deliver_evaluation(replace(delivery, part=part))
+ except Exception as error:
+ # The caller's change goes ahead. A send that failed leaves the incident open, which
+ # error tracking shows.
+ capture_exception(error, {"configuration_id": delivery.configuration_id})
+
+
+async def _start(delivery: AlertDeliveryRequest) -> None:
+ client = await async_connect()
+ await client.start_workflow(
+ AlertsPlatformDeliverPreviewWorkflow.run,
+ delivery,
+ id=delivery_workflow_id(delivery),
+ task_queue=settings.ALERTS_PLATFORM_DELIVERY_TASK_QUEUE,
+ id_conflict_policy=WorkflowIDConflictPolicy.USE_EXISTING,
+ execution_timeout=delivery_execution_timeout(delivery),
+ )
diff --git a/products/alerts_platform/backend/temporal/workflows.py b/products/alerts_platform/backend/temporal/workflows.py
index 59f2a29ad345..89814b23fb7b 100644
--- a/products/alerts_platform/backend/temporal/workflows.py
+++ b/products/alerts_platform/backend/temporal/workflows.py
@@ -205,6 +205,16 @@ class AlertsPlatformDeliverPreviewWorkflow(PostHogWorkflow):
await workflow.sleep(BUSY_RETRY_DELAY)
+def delivery_workflow_id(delivery: AlertDeliveryRequest) -> str:
+ """The id of an evaluation's delivery. A re-run of the same occasion reuses it, so one delivery
+ runs whichever caller starts it."""
+ return f"alerts-deliver-preview-{delivery.configuration_id}:{delivery.evaluation_key}"
+
+
+def delivery_execution_timeout(delivery: AlertDeliveryRequest) -> dt.timedelta:
+ return DELIVERY_EXECUTION_TIMEOUT * max(1, delivery.parts)
+
+
async def start_deliveries(deliveries: Sequence[AlertDeliveryRequest]) -> int:
"""Starts one delivery workflow per request, from inside a source's evaluation workflow.
@@ -221,10 +231,10 @@ async def start_deliveries(deliveries: Sequence[AlertDeliveryRequest]) -> int:
workflow.start_child_workflow(
AlertsPlatformDeliverPreviewWorkflow.run,
delivery,
- id=f"alerts-deliver-preview-{delivery.configuration_id}:{delivery.evaluation_key}",
+ id=delivery_workflow_id(delivery),
task_queue=settings.ALERTS_PLATFORM_DELIVERY_TASK_QUEUE,
parent_close_policy=workflow.ParentClosePolicy.ABANDON,
- execution_timeout=DELIVERY_EXECUTION_TIMEOUT * max(1, delivery.parts),
+ execution_timeout=delivery_execution_timeout(delivery),
)
for delivery in deliveries
),
diff --git a/products/logs/backend/alert_source_cycle.py b/products/logs/backend/alert_source_cycle.py
index ebfbc76fcf52..f8f1519aedea 100644
--- a/products/logs/backend/alert_source_cycle.py
+++ b/products/logs/backend/alert_source_cycle.py
@@ -32,7 +32,6 @@ from posthog.dataclasses import frozen
from posthog.exceptions import ClickHouseEstimatedQueryExecutionTimeTooLong, ClickHouseQueryTimeOut
from posthog.models import Team
from posthog.temporal.alerts.admission import hold_evaluation_slot, release_evaluation_slot
-from posthog.utils import absolute_uri
from products.alerts.backend.facade.destinations import configured_destination_template_ids
from products.alerts_platform.backend.facade.api import due_check_ids, due_checks, refresh_settings, slot_of
@@ -99,7 +98,7 @@ from products.logs.backend.alert_group_columns import grouping_problem
from products.logs.backend.alert_lookback import rolling_check_lookback_minutes
from products.logs.backend.models import LogsAlertConfiguration
from products.logs.backend.platform_alert_backfill import SETTINGS_FIELDS, platform_upsert_of
-from products.logs.backend.platform_source_config import filters_of
+from products.logs.backend.platform_source_config import filters_of, logs_alert_url
# Private to the production activity. Reimplementing either would let this path drift from
# what the logs stack evaluates. Promoting them to a shared home is the deeper fix.
@@ -462,11 +461,11 @@ def _request(
evaluation_key=recorded.evaluation_key,
destination_alert_id=destination_alert_id,
event_ids_by_kind=_EVENT_IDS_BY_KIND,
- event_ids_by_incident_action=_EVENT_IDS_BY_INCIDENT_ACTION,
+ event_ids_by_incident_action=EVENT_IDS_BY_INCIDENT_ACTION,
incident_actions=actions,
sends_messages=sends_messages,
# The logs alert page, as the HogFunction messages linked. An alert with no logs row has none.
- source_alert_url=absolute_uri(f"/project/{check.team_id}/logs/alerts/{check.legacy_configuration_id}")
+ source_alert_url=logs_alert_url(check.team_id, check.legacy_configuration_id)
if check.legacy_configuration_id is not None
else None,
owned_by_platform=check.owned_by_platform,
@@ -485,7 +484,7 @@ def _has_incident_destination(team_id: int, destination_alert_id: str) -> bool:
configured_destination_template_ids(
team_id=team_id,
alert_id=destination_alert_id,
- allowed_event_ids=tuple(_EVENT_IDS_BY_INCIDENT_ACTION.values()),
+ allowed_event_ids=tuple(EVENT_IDS_BY_INCIDENT_ACTION.values()),
)
)
@@ -499,7 +498,7 @@ _EVENT_IDS_BY_KIND: Final[dict[str, str]] = {
}
# Which event id an incident manager destination filters on for each incident action.
-_EVENT_IDS_BY_INCIDENT_ACTION: Final[dict[str, str]] = {
+EVENT_IDS_BY_INCIDENT_ACTION: Final[dict[str, str]] = {
spec.incident_action.value: spec.event_id for spec in EVENT_KIND_CONFIG.values() if spec.incident_action
}
diff --git a/products/logs/backend/facade/platform_ownership.py b/products/logs/backend/facade/platform_ownership.py
index 54732c9e947b..8539006648b4 100644
--- a/products/logs/backend/facade/platform_ownership.py
+++ b/products/logs/backend/facade/platform_ownership.py
@@ -9,7 +9,7 @@ from uuid import UUID
from products.logs.backend import platform_ownership
from products.logs.backend.models import LogsAlertConfiguration
-__all__ = ["on_created", "on_deleted", "on_updated"]
+__all__ = ["before_delete", "on_created", "on_deleted", "on_updated"]
def _alert(team_id: int, alert_id: UUID) -> LogsAlertConfiguration | None:
@@ -35,3 +35,11 @@ def on_updated(*, team_id: int, alert_id: UUID, snooze_changed: bool) -> None:
def on_deleted(*, team_id: int, alert_id: UUID) -> None:
platform_ownership.on_deleted(team_id=team_id, alert_id=alert_id)
+
+
+def before_delete(*, team_id: int, alert_id: UUID) -> None:
+ """Closes the incident of a firing alert while its destinations still exist, on whichever stack
+ owns it."""
+ alert = _alert(team_id, alert_id)
+ if alert is not None:
+ platform_ownership.before_delete(alert)
diff --git a/products/logs/backend/platform_alert_wording.py b/products/logs/backend/platform_alert_wording.py
index 80fd7722c38c..663ed304f624 100644
--- a/products/logs/backend/platform_alert_wording.py
+++ b/products/logs/backend/platform_alert_wording.py
@@ -23,7 +23,7 @@ from products.alerts_platform.backend.facade.contracts import (
from products.logs.backend.alert_evaluation_key import window_end_of
from products.logs.backend.alert_group_columns import GROUP_COLUMNS
from products.logs.backend.logs_url_params import build_logs_url_params, has_filter_values
-from products.logs.backend.platform_source_config import filters_of
+from products.logs.backend.platform_source_config import filters_of, logs_alert_url
# A resolve reports the count that ended the firing, so it is not a breach. A held check, which
# only moves a paging incident, can be either, so it gets a neutral label.
@@ -158,7 +158,7 @@ def legacy_logs_webhook_body(
transition = in_utc(transition)
occurred_at = transition.occurred_at
filters = filters_of(transition.source_config)
- alert_url = absolute_uri(f"/project/{project_id}/logs/alerts/{alert_id}")
+ alert_url = logs_alert_url(project_id, alert_id)
data: dict[str, Any] = {
"alert_id": alert_id,
"alert_name": alert_name,
diff --git a/products/logs/backend/platform_ownership.py b/products/logs/backend/platform_ownership.py
index 3bb6d8525a27..ff75a2fcc565 100644
--- a/products/logs/backend/platform_ownership.py
+++ b/products/logs/backend/platform_ownership.py
@@ -12,13 +12,14 @@ from uuid import UUID
from django.utils import timezone
-from products.alerts_platform.backend.facade.api import delete_owned_copy, update_owned_copy
+from products.alerts_platform.backend.facade.api import close_owned_firings_now, delete_owned_copy, update_owned_copy
from products.alerts_platform.backend.facade.contracts import (
SOURCE_CONDITION_KEY,
PlatformAlertConfigurationEdit,
PlatformAlertNotEditable,
)
from products.logs.backend.alert_state_machine import apply_enabled
+from products.logs.backend.facade.api import close_incident_before_delete
from products.logs.backend.models import LogsAlertConfiguration
from products.logs.backend.platform_alert_backfill import CONDITION_FIELDS, SETTINGS_FIELDS, platform_upsert_of
from products.logs.backend.platform_cutover import hand_over, has_begun_cutover, in_an_environment
@@ -95,3 +96,12 @@ def _write_logs_row(alert: LogsAlertConfiguration, edit: PlatformAlertConfigurat
"updated_at",
]
)
+
+
+def before_delete(alert: LogsAlertConfiguration) -> None:
+ """Closes the incident of a firing alert while its destinations still exist, on whichever stack
+ owns it."""
+ if alert.platform_owned:
+ close_owned_firings_now(team_id=alert.team_id, legacy_configuration_id=alert.id)
+ else:
+ close_incident_before_delete(team_id=alert.team_id, alert_id=str(alert.id), state=alert.state)
diff --git a/products/logs/backend/platform_source_config.py b/products/logs/backend/platform_source_config.py
index b1ac05bbd552..86ebcf4f08e5 100644
--- a/products/logs/backend/platform_source_config.py
+++ b/products/logs/backend/platform_source_config.py
@@ -5,6 +5,8 @@ Imported from `ready()` through the message wording, so it stays light: no model
from typing import Any, Final, Literal, get_args
+from posthog.utils import absolute_uri
+
from products.alerts_platform.backend.facade.contracts import SOURCE_CONDITION_KEY, SourceConfigSpec
FILTERS_KEY: Final = "filters"
@@ -43,3 +45,8 @@ def load_logs_source_config() -> SourceConfigSpec:
def has_any_filter(filters: dict[str, Any]) -> bool:
return any(filters.get(key) for key in LOGS_FILTER_KEYS)
+
+
+def logs_alert_url(team_id: int, alert_id: object) -> str:
+ """The alert's page in the logs product, which messages and incidents link to."""
+ return absolute_uri(f"/project/{team_id}/logs/alerts/{alert_id}")
diff --git a/products/logs/backend/platform_source_config_schema.py b/products/logs/backend/platform_source_config_schema.py
index 06b55c7ccdfe..120282035ca0 100644
--- a/products/logs/backend/platform_source_config_schema.py
+++ b/products/logs/backend/platform_source_config_schema.py
@@ -17,9 +17,10 @@ from products.alerts_platform.backend.facade.contracts import (
SourceConfigSpec,
)
from products.logs.backend.alert_group_columns import grouping_problem
+from products.logs.backend.alert_source_cycle import EVENT_IDS_BY_INCIDENT_ACTION
from products.logs.backend.models import LogsAlertConfiguration
from products.logs.backend.platform_ownership import mirror_platform_edit
-from products.logs.backend.platform_source_config import WindowMinutes, has_any_filter
+from products.logs.backend.platform_source_config import WindowMinutes, has_any_filter, logs_alert_url
class LogsPlatformAlertCondition(BaseModel):
@@ -64,5 +65,9 @@ def validate_logs_edit(edit: PlatformAlertConfigurationEdit) -> None:
LOGS_SOURCE_CONFIG = SourceConfigSpec(
- model=LogsPlatformAlertSourceConfig, validate=validate_logs_edit, mirror=mirror_platform_edit
+ model=LogsPlatformAlertSourceConfig,
+ validate=validate_logs_edit,
+ mirror=mirror_platform_edit,
+ incident_event_ids=EVENT_IDS_BY_INCIDENT_ACTION,
+ alert_url=logs_alert_url,
)
diff --git a/products/logs/backend/presentation/views/alerts_api.py b/products/logs/backend/presentation/views/alerts_api.py
index 8063199d957e..2d0cb6291f9d 100644
--- a/products/logs/backend/presentation/views/alerts_api.py
+++ b/products/logs/backend/presentation/views/alerts_api.py
@@ -87,14 +87,13 @@ from products.logs.backend.alert_state_machine import (
)
from products.logs.backend.facade.api import (
ALLOWED_WINDOW_MINUTES,
- close_incident_before_delete,
close_incident_on_commit,
follow_logs_alert,
has_any_filter,
logs_state_of_platform,
next_allowed_check_at,
)
-from products.logs.backend.facade.platform_ownership import on_created, on_deleted, on_updated
+from products.logs.backend.facade.platform_ownership import before_delete, on_created, on_deleted, on_updated
from products.logs.backend.models import MAX_EVALUATION_PERIODS, LogsAlertConfiguration, LogsAlertEvent
MAX_ALERTS_PER_TEAM = 20
@@ -1644,7 +1643,7 @@ class LogsAlertViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet):
self._track("updated", serializer.save())
def perform_destroy(self, instance: LogsAlertConfiguration) -> None:
- close_incident_before_delete(team_id=instance.team_id, alert_id=str(instance.id), state=instance.state)
+ before_delete(team_id=instance.team_id, alert_id=instance.id)
with transaction.atomic():
locked_instance = (
LogsAlertConfiguration.objects.select_for_update()
diff --git a/products/logs/backend/test/test_alerts_api.py b/products/logs/backend/test/test_alerts_api.py
index 5961dd53a2c4..eb21fb13ac08 100644
--- a/products/logs/backend/test/test_alerts_api.py
+++ b/products/logs/backend/test/test_alerts_api.py
@@ -7,7 +7,7 @@ from uuid import UUID, uuid4
import pytest
import time_machine
from posthog.test.base import APIBaseTest, ClickhouseTestMixin
-from unittest.mock import MagicMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
from django.db import OperationalError, connection, transaction
from django.test.utils import CaptureQueriesContext
@@ -25,9 +25,15 @@ from posthog.models.user import User
from products.alerts_platform.backend.facade import testing as platform_testing
from products.alerts_platform.backend.facade.api import get_configuration, list_configurations, upsert_configuration
-from products.alerts_platform.backend.facade.contracts import PlatformConfigurationSnapshot, SourceKind
+from products.alerts_platform.backend.facade.contracts import (
+ DeliveryPath,
+ IncidentAction,
+ PlatformConfigurationSnapshot,
+ SourceKind,
+)
from products.cdp.backend.models.hog_function_template import HogFunctionTemplate
from products.cdp.backend.models.hog_functions.hog_function import HogFunction
+from products.cdp.backend.models.hog_functions.hog_function_revision import HogFunctionRevision
from products.cdp.backend.templates.fixtures import template_pagerduty, template_slack
from products.cdp.backend.templates.hog_function_template import sync_template_to_db
from products.cdp.backend.templates.microsoft_teams.template_microsoft_teams import template as template_microsoft_teams
@@ -65,6 +71,11 @@ def _make_log_row(*, team_id: int, service: str, uuid: str, ts: datetime, body:
}
+# Where a request-side delivery meets Temporal and the transports, which the tests stand in for.
+_WEB_START = "products.alerts_platform.backend.temporal.web_start"
+_NATIVE_PAGERDUTY_DELIVER = "products.alerts_platform.backend.delivery.pagerduty.PagerDutyTransport.deliver"
+
+
class TestLogsAlertAPI(APIBaseTest):
base_url: str
@@ -2458,6 +2469,175 @@ class TestLogsAlertAPI(APIBaseTest):
assert self._copy(created["id"]).check_status == "ok"
+ def _firing_on_the_platform_with_pagerduty(self) -> tuple[dict, PlatformConfigurationSnapshot]:
+ created = self._owned_by_the_platform()
+ self._add_pagerduty_destination(created["id"])
+ copy = self._copy(created["id"])
+ started = datetime(2026, 10, 1, 9, tzinfo=UTC)
+ with team_scope(self.team.id):
+ platform_testing.start_firing(copy.id, team_id=self.team.id, at=started)
+ # Due now, as a copy is while a check of it runs.
+ platform_testing.set_due_at(copy.id, datetime.now(UTC) - timedelta(minutes=1))
+ return created, copy
+
+ @parameterized.expand(
+ [
+ ("logs_disable", "logs", lambda: {"enabled": False}),
+ ("logs_snooze", "logs", lambda: {"snooze_until": (datetime.now(UTC) + timedelta(hours=1)).isoformat()}),
+ ("logs_threshold_change", "logs", lambda: {"threshold_count": 50}),
+ ("logs_unsnooze_threshold_change", "logs", lambda: {"snooze_until": None, "threshold_count": 50}),
+ ("platform_disable", "platform", lambda: {"enabled": False}),
+ ]
+ )
+ def test_an_edit_that_ends_a_platform_firing_resolves_its_incident(
+ self, _name: str, route: str, make_payload: Any
+ ) -> None:
+ created, copy = self._firing_on_the_platform_with_pagerduty()
+ temporal = MagicMock(start_workflow=AsyncMock())
+ url = (
+ f"{self.base_url}{created['id']}/"
+ if route == "logs"
+ else f"/api/projects/{self.team.id}/platform_alerts/{copy.id}/"
+ )
+
+ with (
+ patch("posthog.permissions.posthog_feature_flag_enabled", return_value=True),
+ patch(f"{_WEB_START}.async_connect", AsyncMock(return_value=temporal)),
+ self.captureOnCommitCallbacks(execute=True),
+ ):
+ response = self.client.patch(url, make_payload(), format="json")
+
+ assert response.status_code == status.HTTP_200_OK, response.json()
+ (delivery,) = [call.args[1] for call in temporal.start_workflow.call_args_list]
+ assert (delivery.incident_actions, delivery.sends_messages, delivery.destination_alert_id) == (
+ {"": IncidentAction.RESOLVE},
+ False,
+ created["id"],
+ )
+ with team_scope(self.team.id):
+ instance = platform_testing.alert_for(copy.id)
+ assert instance is not None and (instance.state, instance.firing_started_at) == ("not_firing", None)
+ # A check already running when the edit landed finds the row no longer due, and a snooze runs
+ # no check until it ends.
+ next_check_at = self._copy(created["id"]).next_check_at
+ earliest = datetime.now(UTC) + (timedelta(minutes=59) if _name == "logs_snooze" else timedelta(0))
+ assert next_check_at is not None and next_check_at > earliest
+
+ def _customize_destinations(self) -> None:
+ # A destination edited after it was created keeps sending through its HogFunction.
+ for function in HogFunction.objects.filter(team=self.team, deleted=False):
+ HogFunctionRevision.objects.for_team(self.team.id).create(
+ team=self.team,
+ hog_function=function,
+ version=1,
+ content={
+ "hog": function.hog,
+ "inputs_schema": function.inputs_schema,
+ "inputs": function.inputs,
+ "filters": function.filters,
+ "mappings": function.mappings,
+ "masking": function.masking,
+ },
+ )
+ HogFunction.objects.filter(pk=function.pk).update(version=2, hog=f"// edited\n{function.hog}")
+
+ @parameterized.expand(
+ [
+ ("logs_disable", "logs", lambda: {"enabled": False}, "hog_function"),
+ ("logs_threshold_change", "logs", lambda: {"threshold_count": 50}, "hog_function"),
+ ("platform_disable", "platform", lambda: {"enabled": False}, "hog_function"),
+ ("logs_delete", "logs", None, "hog_function"),
+ # The resolve takes the path its trigger took, though the destination changed since.
+ ("logs_disable_after_a_native_trigger", "logs", lambda: {"enabled": False}, "native"),
+ ]
+ )
+ def test_ending_a_firing_with_a_customized_destination_sends_one_close(
+ self, _name: str, route: str, make_payload: Any, closed_via: str
+ ) -> None:
+ created, copy = self._firing_on_the_platform_with_pagerduty()
+ if closed_via == "native":
+ with team_scope(self.team.id):
+ platform_testing.start_firing(
+ copy.id,
+ team_id=self.team.id,
+ at=datetime(2026, 10, 1, 9, tzinfo=UTC),
+ delivered_via=DeliveryPath.NATIVE,
+ )
+ self._customize_destinations()
+ url = (
+ f"{self.base_url}{created['id']}/"
+ if route == "logs"
+ else f"/api/projects/{self.team.id}/platform_alerts/{copy.id}/"
+ )
+
+ with (
+ patch("posthog.permissions.posthog_feature_flag_enabled", return_value=True),
+ # Sends the resolve inline, so the test sees which path delivered it.
+ patch(f"{_WEB_START}.async_connect", AsyncMock(side_effect=RuntimeError("temporal unreachable"))),
+ patch(_NATIVE_PAGERDUTY_DELIVER, return_value=None) as native,
+ patch("products.logs.backend.platform_alert_legacy_events.emit_alert_internal_events") as emit,
+ self.captureOnCommitCallbacks(execute=True),
+ ):
+ response = (
+ self.client.delete(url)
+ if make_payload is None
+ else self.client.patch(url, make_payload(), format="json")
+ )
+
+ assert response.status_code in (status.HTTP_200_OK, status.HTTP_204_NO_CONTENT), response.content
+ emitted = [
+ (event.event_name, event.properties["alert_id"])
+ for call in emit.call_args_list
+ for event in call.kwargs["events"]
+ ]
+ if closed_via == "native":
+ assert (emitted, native.call_count) == ([], 1)
+ else:
+ assert (emitted, native.call_count) == ([("$logs_alert_incident_closed", created["id"])], 0)
+
+ def test_shortening_a_snooze_on_the_logs_page_brings_the_next_platform_check_forward(self) -> None:
+ created = self._owned_by_the_platform()
+
+ for snooze in (timedelta(hours=24), timedelta(minutes=30)):
+ response = self.client.patch(
+ f"{self.base_url}{created['id']}/",
+ {"snooze_until": (datetime.now(UTC) + snooze).isoformat()},
+ format="json",
+ )
+ assert response.status_code == status.HTTP_200_OK, response.json()
+
+ next_check_at = self._copy(created["id"]).next_check_at
+ assert next_check_at is not None and next_check_at < datetime.now(UTC) + timedelta(minutes=31)
+
+ def test_a_resolve_is_sent_inline_when_temporal_cannot_start_it(self) -> None:
+ created, _copy = self._firing_on_the_platform_with_pagerduty()
+
+ with (
+ patch(f"{_WEB_START}.async_connect", AsyncMock(side_effect=RuntimeError("temporal unreachable"))),
+ patch(f"{_WEB_START}.deliver_evaluation") as deliver_evaluation,
+ self.captureOnCommitCallbacks(execute=True),
+ ):
+ response = self.client.patch(f"{self.base_url}{created['id']}/", {"enabled": False}, format="json")
+
+ assert response.status_code == status.HTTP_200_OK, response.json()
+ (request,) = [call.args[0] for call in deliver_evaluation.call_args_list]
+ assert request.incident_actions == {"": IncidentAction.RESOLVE}
+
+ def test_deleting_a_platform_firing_resolves_its_incident_while_its_destinations_exist(self) -> None:
+ created, _copy = self._firing_on_the_platform_with_pagerduty()
+ destinations_when_sent: list[int] = []
+
+ def deliver(request: Any) -> None:
+ destinations_when_sent.append(HogFunction.objects.filter(team=self.team, deleted=False).count())
+
+ with patch(f"{_WEB_START}.deliver_evaluation", side_effect=deliver) as deliver_evaluation:
+ response = self.client.delete(f"{self.base_url}{created['id']}/")
+
+ assert response.status_code == status.HTTP_204_NO_CONTENT
+ (request,) = [call.args[0] for call in deliver_evaluation.call_args_list]
+ assert request.incident_actions == {"": IncidentAction.RESOLVE}
+ assert destinations_when_sent[0] > 0
+
def test_deleting_an_alert_the_platform_owns_deletes_its_copy(self) -> None:
created = self._owned_by_the_platform()