Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(alerts): notify once per firing episode for ai detector alerts (#130827)

posthog[bot]c2d286bf4daf
diff --git a/docs/internal/alerts-platform-temporal.md b/docs/internal/alerts-platform-temporal.md
index 29fe5b3364b4..5eb877fc769c 100644
--- a/docs/internal/alerts-platform-temporal.md
+++ b/docs/internal/alerts-platform-temporal.md
@@ -536,6 +536,8 @@ It copies every insight alert, on every cadence.
A real-time check gets production's real-time budget: a 7-minute evaluation with one retry inside it.
An AI detector check runs on the AI queue, whose worker holds the model credentials, with production's retries: a check whose model call reaches no verdict is retried, and only its last attempt records the error.
A paused AI detector check, refused for a withdrawn rollout or consent or for no AI credits left, is not retried. It records an errored check and leaves the alert enabled, as production does.
+An AI detector alert runs `AI_DETECTOR_ALERT_POLICY` on both stacks, so it announces a firing once and stays quiet until a clear check ends the firing.
+Production marks each quiet firing check with `firing_not_renotified`, so the investigation safety net and the cutover do not treat it as a held notification.
Every other check gets one 12-minute attempt.
An AI detector check calls the model on both stacks while both run, and each call is billed to the project, so the backfill leaves AI detector alerts out.
Pass `--include-ai-detector` to copy them, with `--team-id` or `--ids-file` for an internal project, until the insight cutover.
diff --git a/posthog/tasks/alerts/investigation_notifications.py b/posthog/tasks/alerts/investigation_notifications.py
index 9a63c82284e3..bca519d6f905 100644
--- a/posthog/tasks/alerts/investigation_notifications.py
+++ b/posthog/tasks/alerts/investigation_notifications.py
@@ -33,6 +33,7 @@ from posthog.tasks.alerts.utils import (
record_alert_delivery,
)
+from products.alerts.backend.insight_alert_state_machine import FIRING_NOT_RENOTIFIED_KEY
from products.alerts.backend.models.alert import AlertCheck, InvestigationStatus
logger = structlog.get_logger(__name__)
@@ -79,6 +80,8 @@ def run_investigation_notification_safety_net() -> int:
)
# A skipped check copies the alert's FIRING state but evaluated nothing, so it has no fire to deliver.
.exclude(triggered_metadata__has_key="skipped_reason")
+ # A firing the policy does not renotify was announced on an earlier check, so this one owes nothing.
+ .exclude(triggered_metadata__has_key=FIRING_NOT_RENOTIFIED_KEY)
.filter(
# Terminal investigation states (DONE / FAILED): the workflow is not coming
# back, so a 5-min grace gets stuck dispatches through quickly. Non-terminal
diff --git a/posthog/tasks/alerts/test/test_investigation_notifications.py b/posthog/tasks/alerts/test/test_investigation_notifications.py
index 471c94901b56..64d88cb64385 100644
--- a/posthog/tasks/alerts/test/test_investigation_notifications.py
+++ b/posthog/tasks/alerts/test/test_investigation_notifications.py
@@ -14,6 +14,7 @@ from posthog.tasks.alerts.investigation_notifications import (
run_investigation_notification_safety_net,
)
+from products.alerts.backend.insight_alert_state_machine import FIRING_NOT_RENOTIFIED_KEY
from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration, InvestigationStatus
from products.alerts_platform.backend.facade.contracts import AlertDelivery
from products.product_analytics.backend.facade.models import Insight
@@ -180,12 +181,20 @@ class TestInvestigationNotificationSafetyNet(APIBaseTest):
assert notified == 0
mock_dispatch.assert_not_called() # type: ignore[attr-defined]
+ @parameterized.expand(
+ [
+ ("evaluation_skipped", {"skipped_reason": "Not enough completed intervals after the evaluation delay."}),
+ ("firing_not_renotified", {FIRING_NOT_RENOTIFIED_KEY: True}),
+ ]
+ )
@patch("posthog.tasks.alerts.investigation_notifications.dispatch_alert_notification")
- def test_skips_evaluation_skipped_check(self, mock_dispatch: object) -> None:
+ def test_skips_firing_check_that_owes_no_notification(
+ self, _name: str, triggered_metadata: dict, mock_dispatch: object
+ ) -> None:
self._make_check(
age_minutes=INVESTIGATION_RUNNING_GRACE_MINUTES + 60,
investigation_status=None,
- triggered_metadata={"skipped_reason": "Not enough completed intervals after the evaluation delay."},
+ triggered_metadata=triggered_metadata,
)
notified = run_investigation_notification_safety_net()
assert notified == 0
diff --git a/posthog/tasks/alerts/utils.py b/posthog/tasks/alerts/utils.py
index d39402cd0104..312d79f264a6 100644
--- a/posthog/tasks/alerts/utils.py
+++ b/posthog/tasks/alerts/utils.py
@@ -36,9 +36,11 @@ from products.alerts.backend.facade.destinations import (
)
from products.alerts.backend.facade.email import alert_email_recipients, send_alert_email
from products.alerts.backend.insight_alert_state_machine import (
+ FIRING_NOT_RENOTIFIED_KEY,
apply_invalid_configuration,
apply_outcome,
evaluate_alert_check,
+ is_firing_not_renotified,
should_notify,
)
from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration
@@ -870,6 +872,9 @@ def add_alert_check(
now=datetime.now(UTC),
)
state_fields = apply_outcome(alert, outcome)
+ triggered_metadata = result.triggered_metadata
+ if is_firing_not_renotified(outcome):
+ triggered_metadata = {**(triggered_metadata or {}), FIRING_NOT_RENOTIFIED_KEY: True}
alert.last_checked_at = datetime.now(UTC)
# Update next_check_at per interval so we don't recheck until the next one is due.
@@ -881,7 +886,7 @@ def add_alert_check(
condition=alert.condition,
targets_notified={},
state=alert.state,
- triggered_metadata=result.triggered_metadata,
+ triggered_metadata=triggered_metadata,
error=error,
anomaly_scores=result.anomaly_scores,
triggered_points=result.triggered_points,
diff --git a/posthog/temporal/alerts/activities.py b/posthog/temporal/alerts/activities.py
index 6c3d834eff65..93cc96071d7a 100644
--- a/posthog/temporal/alerts/activities.py
+++ b/posthog/temporal/alerts/activities.py
@@ -67,6 +67,7 @@ from posthog.temporal.alerts.alert_firing_signal import (
prepare_alert_firing_signal,
)
from posthog.temporal.alerts.investigation import (
+ InvestigationDecision,
carried_verdict_suppresses,
claim_investigation_slot,
decide_investigation,
@@ -769,7 +770,8 @@ async def evaluate_alert(inputs: EvaluateAlertActivityInputs) -> EvaluateAlertRe
new_state=AlertState(alert_check.state),
)
- investigation = decide_investigation(alert, alert_check)
+ # A finished investigation delivers its check's notification, so a check that sends none skips it.
+ investigation = decide_investigation(alert, alert_check) if should_notify else InvestigationDecision()
if investigation.should_investigate and claim_investigation_slot(alert, alert_check):
should_start_investigation = True
should_gate_notification = should_notify and bool(alert.investigation_gates_notifications)
diff --git a/posthog/temporal/tests/test_alerts_activities.py b/posthog/temporal/tests/test_alerts_activities.py
index c80652396b05..34861ef87b82 100644
--- a/posthog/temporal/tests/test_alerts_activities.py
+++ b/posthog/temporal/tests/test_alerts_activities.py
@@ -95,6 +95,7 @@ from products.alerts.backend.facade.api import (
LLMDetectorPausedError,
LLMDetectorUnavailableError,
)
+from products.alerts.backend.insight_alert_state_machine import FIRING_NOT_RENOTIFIED_KEY
from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration, InvestigationStatus, Threshold
from products.alerts_platform.backend.facade.contracts import AlertDelivery
from products.alerts_platform.backend.facade.enums import PlatformAlertOwner
@@ -1174,6 +1175,35 @@ class TestEvaluateAlert:
assert check.notification_sent_at is not None
assert alert.last_notified_at == check.notification_sent_at
+ @pytest.mark.parametrize("already_firing", [False, True])
+ async def test_ai_detector_alert_notifies_and_investigates_only_the_first_firing_check(
+ self, ateam, already_firing: bool
+ ) -> None:
+ alert = await _create_alert(
+ ateam,
+ state=AlertState.FIRING if already_firing else AlertState.NOT_FIRING,
+ detector_config={"type": "llm"},
+ )
+ await sync_to_async(AlertConfiguration.objects.filter(id=alert.id).update)(
+ investigation_agent_enabled=True, investigation_gates_notifications=True
+ )
+
+ with patch(
+ "posthog.temporal.alerts.activities.check_alert_for_insight",
+ return_value=AlertEvaluationResult(value=100.0, breaches=["anomalous point"]),
+ ):
+ result = await ActivityEnvironment().run(
+ evaluate_alert,
+ EvaluateAlertActivityInputs(alert_id=str(alert.id), uses_llm_detector=True, team_id=ateam.id),
+ )
+
+ assert result.new_state == AlertState.FIRING
+ assert result.should_notify is not already_firing
+ assert result.should_start_investigation is not already_firing
+ assert result.should_gate_notification is not already_firing
+ check = await sync_to_async(AlertCheck.objects.get)(pk=result.alert_check_id)
+ assert ((check.triggered_metadata or {}).get(FIRING_NOT_RENOTIFIED_KEY) is True) is already_firing
+
async def test_evaluate_errored_when_permanent_exception(self, alert) -> None:
with patch(
"posthog.temporal.alerts.activities.check_alert_for_insight",
diff --git a/products/alerts/backend/insight_alert_state_machine.py b/products/alerts/backend/insight_alert_state_machine.py
index 01e577d5aff7..79337f124d4d 100644
--- a/products/alerts/backend/insight_alert_state_machine.py
+++ b/products/alerts/backend/insight_alert_state_machine.py
@@ -1,10 +1,12 @@
from __future__ import annotations
+from dataclasses import replace
from datetime import datetime, timedelta
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
from posthog.schema_enums import AlertState as InsightAlertState
+from products.alerts.backend.llm_detector_limits import is_llm_detector_config
from products.alerts_platform.backend.facade.lifecycle import (
AlertCheckOutcome,
AlertPolicy,
@@ -34,6 +36,17 @@ INSIGHT_ALERT_POLICY = AlertPolicy(
errors_set_errored_state=True,
)
+# The AI detector flags the same bad state on check after check, so renotifying sends one message per check.
+AI_DETECTOR_ALERT_POLICY = replace(INSIGHT_ALERT_POLICY, renotify_while_firing=False)
+
+# Tells the investigation safety net and the cutover that an unsent firing check owes no notification.
+FIRING_NOT_RENOTIFIED_KEY = "firing_not_renotified"
+
+
+def insight_alert_policy(detector_config: Any) -> AlertPolicy:
+ """The policy both stacks run for an insight alert with this detector."""
+ return AI_DETECTOR_ALERT_POLICY if is_llm_detector_config(detector_config) else INSIGHT_ALERT_POLICY
+
def insight_snapshot(
state: AlertState,
@@ -79,7 +92,7 @@ def evaluate_alert_check(
snapshot_from_alert(alert),
CheckInput(threshold_breached=threshold_breached, error_message=error_message),
now,
- policy=INSIGHT_ALERT_POLICY,
+ policy=insight_alert_policy(alert.detector_config),
)
@@ -128,3 +141,7 @@ def apply_platform_state(alert: AlertConfiguration, state: InsightAlertState) ->
def should_notify(outcome: AlertCheckOutcome) -> bool:
return outcome.notification != NotificationAction.NONE
+
+
+def is_firing_not_renotified(outcome: AlertCheckOutcome) -> bool:
+ return outcome.new_state == AlertState.FIRING and outcome.error_message is None and not should_notify(outcome)
diff --git a/products/alerts/backend/platform_cutover.py b/products/alerts/backend/platform_cutover.py
index a53946fdd1d1..d3cc769ffca8 100644
--- a/products/alerts/backend/platform_cutover.py
+++ b/products/alerts/backend/platform_cutover.py
@@ -22,7 +22,11 @@ from posthog.temporal.alerts.investigation import IN_FLIGHT_INVESTIGATION_HORIZO
from posthog.temporal.alerts.retry_policy import alert_timeouts
from products.alerts.backend.facade.destinations import choose_delivery_path
-from products.alerts.backend.insight_alert_state_machine import apply_invalid_configuration, apply_platform_state
+from products.alerts.backend.insight_alert_state_machine import (
+ FIRING_NOT_RENOTIFIED_KEY,
+ apply_invalid_configuration,
+ apply_platform_state,
+)
from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration, InvestigationStatus
from products.alerts.backend.platform_source_cycle import DELIVERY_EVENT_IDS, platform_upsert_of
from products.alerts_platform.backend.facade.api import give_back_ownership, take_ownership, upsert_configuration
@@ -132,13 +136,17 @@ def _firing_notification_pending(alert: AlertConfiguration, at: datetime) -> boo
# A check's workflow notifies before its execution timeout ends, so past that the insight stack
# has sent the notification or given up on it.
horizon = alert_timeouts(alert.calculation_interval).workflow_execution
- return AlertCheck.objects.filter(
- alert_configuration=alert,
- created_at__gte=at - horizon,
- state=AlertState.FIRING,
- notification_sent_at__isnull=True,
- notification_suppressed_by_agent=False,
- ).exists()
+ return (
+ AlertCheck.objects.filter(
+ alert_configuration=alert,
+ created_at__gte=at - horizon,
+ state=AlertState.FIRING,
+ notification_sent_at__isnull=True,
+ notification_suppressed_by_agent=False,
+ )
+ .exclude(triggered_metadata__has_key=FIRING_NOT_RENOTIFIED_KEY)
+ .exists()
+ )
def cut_over_team(team_id: int, *, now: datetime | None = None) -> tuple[AlertCutover, ...]:
diff --git a/products/alerts/backend/platform_investigation.py b/products/alerts/backend/platform_investigation.py
index 2feb72c98be6..ced480079bdd 100644
--- a/products/alerts/backend/platform_investigation.py
+++ b/products/alerts/backend/platform_investigation.py
@@ -199,6 +199,9 @@ def with_investigation(check: PlatformAlertCheckInput, evaluation: SourceBatchEv
outcomes=(replace(outcome, groups=(group,), held_delivery=metrics_held),),
deliveries=() if metrics_held else evaluation.deliveries,
)
+ if group.kind != AlertEventKind.FIRING:
+ # A firing check that announces nothing gives the agent no message to gate or follow up on.
+ return evaluation
decision = decide_investigation(
alert,
firing=True,
@@ -219,7 +222,7 @@ def with_investigation(check: PlatformAlertCheckInput, evaluation: SourceBatchEv
outcomes=(replace(outcome, groups=(group,), held_delivery=held),),
deliveries=() if held else evaluation.deliveries,
)
- if group.kind == AlertEventKind.FIRING and carried_verdict_suppresses(alert, decision):
+ if carried_verdict_suppresses(alert, decision):
# As the insight stack: the firing still moves the alert's state and `last_notified_at`.
group = replace(
group,
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index fefc49abf6c7..89967af36524 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -7,7 +7,7 @@ said, and asks for a delivery only for the alerts the platform owns.
Each platform row resolves to its production alert through `legacy_configuration_id`, because
`check_alert_for_insight` reads the insight, the threshold and the condition off that model. The
query runs the way production runs it, and the verdict comes from the shared machine under
-`INSIGHT_ALERT_POLICY`, applied to the platform's own state. `add_alert_check` is the production
+`insight_alert_policy`, applied to the platform's own state. `add_alert_check` is the production
writer, and nothing here calls it.
A check that production skips without a query is skipped here too, so both stacks answer the
@@ -44,7 +44,11 @@ from products.alerts.backend.evaluation.comparator import MAX_BREACH_MESSAGES
from products.alerts.backend.evaluation.contract import AlertDataUnavailableError, AlertExtractionError
from products.alerts.backend.evaluation.dispatcher import PerSeriesEvaluation, check_alert_per_series
from products.alerts.backend.facade.contracts import INSIGHT_EVENT_IDS_BY_INCIDENT_ACTION
-from products.alerts.backend.insight_alert_state_machine import INSIGHT_ALERT_POLICY, insight_snapshot
+from products.alerts.backend.insight_alert_state_machine import (
+ INSIGHT_ALERT_POLICY,
+ insight_alert_policy,
+ insight_snapshot,
+)
from products.alerts.backend.judge.contract import (
LLM_DETECTOR_PAUSED_ERROR_CODE,
LLM_DETECTOR_UNAVAILABLE_ERROR_CODE,
@@ -85,6 +89,7 @@ from products.alerts_platform.backend.facade.grouping import (
)
from products.alerts_platform.backend.facade.lifecycle import (
NOTIFICATION_EVENT_KINDS,
+ AlertPolicy,
AlertSnapshot,
AlertState,
CheckInput,
@@ -616,7 +621,7 @@ def _incident_actions(check: PlatformAlertCheckInput, outcome: PlatformAlertOutc
A failed check moves no instance, so its incident stays open with its firing. Closing it would
leave a re-enabled alert that still breaches firing with no incident.
"""
- policy = INSIGHT_ALERT_POLICY if check.grouping.mode == GroupingMode.SINGLE else GROUPED_INSIGHT_ALERT_POLICY
+ policy = _policy(check) if check.grouping.mode == GroupingMode.SINGLE else GROUPED_INSIGHT_ALERT_POLICY
return {
group.grouping_key: action
for group in outcome.groups
@@ -668,6 +673,11 @@ def _snapshot(
)
+def _policy(check: PlatformAlertCheckInput) -> AlertPolicy:
+ """The policy production runs for the alert, read from the detector the copy stores."""
+ return insight_alert_policy(check.condition.get("detector"))
+
+
def _evaluation_key(check: PlatformAlertCheckInput, cutoff: datetime) -> str:
"""The due slot. An insight check is one scheduled run, not a window, so the slot names it."""
return evaluation_key_for_slot(slot_of(check.next_check_at, cutoff))
@@ -936,6 +946,7 @@ MAX_TOTAL_DETAIL_CHARS: Final = 200_000
# A breakdown value can be muted on its own, which production cannot express, so the platform's
# mute applies: the value is still judged and only its announcement is held. A snooze of the
# production alert never reaches here, because `_decide` records that check without a query.
+# Built on the statistical policy alone, because the AI detector is refused on a breakdown insight.
GROUPED_INSIGHT_ALERT_POLICY: Final = replace(INSIGHT_ALERT_POLICY, mute_gates_notification_only=True)
@@ -1191,7 +1202,7 @@ def _verdict(
context: dict[str, Any] | None = None,
check_detail: dict[str, Any] | None = None,
) -> PlatformAlertOutcome:
- outcome = evaluate_alert_check(snapshot, check_input, now, policy=INSIGHT_ALERT_POLICY)
+ outcome = evaluate_alert_check(snapshot, check_input, now, policy=_policy(check))
source_detail: dict[str, Any] | None
if firing is not None and outcome.notification == NotificationAction.FIRE:
source_detail = _firing_detail(check, *firing, breach_details=breach_details)
@@ -1281,7 +1292,7 @@ def _recorded(
new_state=outcome.new_state.value,
notified=notified,
failed=failed,
- firing_episode=decide_firing_episode(snapshot, outcome, now, policy=INSIGHT_ALERT_POLICY),
+ firing_episode=decide_firing_episode(snapshot, outcome, now, policy=_policy(check)),
value=value,
error_message=error_message,
query_duration_ms=query_duration_ms,
diff --git a/products/alerts/backend/test/test_insight_alert_state_machine.py b/products/alerts/backend/test/test_insight_alert_state_machine.py
index 042eba729cd4..8b50278637f6 100644
--- a/products/alerts/backend/test/test_insight_alert_state_machine.py
+++ b/products/alerts/backend/test/test_insight_alert_state_machine.py
@@ -1,6 +1,8 @@
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
-from typing import cast
+from typing import Any, cast
+
+import pytest
from posthog.schema_enums import AlertState as InsightAlertState
@@ -11,26 +13,76 @@ from products.alerts.backend.insight_alert_state_machine import (
apply_threshold_change,
apply_unsnooze,
evaluate_alert_check,
+ is_firing_not_renotified,
should_notify,
)
from products.alerts.backend.models.alert import AlertConfiguration
from products.alerts_platform.backend.facade.lifecycle import AlertState
NOW = datetime(2026, 7, 27, 12, 0, tzinfo=UTC)
+AI_DETECTOR = {"type": "llm"}
-def alert_with_state(state: InsightAlertState, *, enabled: bool = True) -> AlertConfiguration:
+def alert_with_state(
+ state: InsightAlertState,
+ *,
+ enabled: bool = True,
+ detector_config: dict[str, Any] | None = None,
+ snoozed_until: datetime | None = None,
+) -> AlertConfiguration:
return cast(
AlertConfiguration,
- SimpleNamespace(enabled=enabled, state=state.value, last_notified_at=None, snoozed_until=None),
+ SimpleNamespace(
+ enabled=enabled,
+ state=state.value,
+ last_notified_at=None,
+ snoozed_until=snoozed_until,
+ detector_config=detector_config,
+ ),
)
-def test_repeated_breach_stays_firing_and_notifies() -> None:
+ "detector_config,renotifies",
+ [
+ (None, True),
+ ({"type": "zscore", "threshold": 0.9, "window": 30}, True),
+ (AI_DETECTOR, False),
+ ],
+)
+def test_repeated_breach_stays_firing_and_renotifies_only_for_statistical_alerts(
+ detector_config: dict[str, Any] | None, renotifies: bool
+) -> None:
outcome = evaluate_alert_check(
- alert_with_state(InsightAlertState.FIRING), threshold_breached=True, error_message=None, now=NOW
+ alert_with_state(InsightAlertState.FIRING, detector_config=detector_config),
+ threshold_breached=True,
+ error_message=None,
+ now=NOW,
)
+ assert outcome.new_state == AlertState.FIRING
+ assert should_notify(outcome) is renotifies
+ assert is_firing_not_renotified(outcome) is not renotifies
+
+
+def test_ai_detector_alert_announces_each_firing_episode_once() -> None:
+ alert = alert_with_state(InsightAlertState.NOT_FIRING, detector_config=AI_DETECTOR)
+ announced = []
+ for breached in (True, True, True, False, True, True):
+ outcome = evaluate_alert_check(alert, threshold_breached=breached, error_message=None, now=NOW)
+ alert.state = InsightAlertState[outcome.new_state.name]
+ announced.append(should_notify(outcome))
+
+ assert announced == [True, False, False, False, True, False]
+
+
+def test_ai_detector_alert_announces_a_firing_that_outlasts_its_snooze() -> None:
+ alert = alert_with_state(
+ InsightAlertState.SNOOZED, detector_config=AI_DETECTOR, snoozed_until=NOW - timedelta(minutes=1)
+ )
+
+ outcome = evaluate_alert_check(alert, threshold_breached=True, error_message=None, now=NOW)
+
assert outcome.new_state == AlertState.FIRING
assert should_notify(outcome)
diff --git a/products/alerts/backend/test/test_platform_cutover.py b/products/alerts/backend/test/test_platform_cutover.py
index 19730b414d49..eadc8c271258 100644
--- a/products/alerts/backend/test/test_platform_cutover.py
+++ b/products/alerts/backend/test/test_platform_cutover.py
@@ -10,6 +10,7 @@ from posthog.models import Team
from posthog.models.scoping import team_scope
from posthog.schema_enums import AlertCalculationInterval, AlertState
+from products.alerts.backend.insight_alert_state_machine import FIRING_NOT_RENOTIFIED_KEY
from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration, InvestigationStatus
from products.alerts.backend.platform_cutover import CutoverSkip, RollbackSkip, cut_over_team, roll_back_team
from products.alerts.backend.test.insight_alerts import create_insight_alert
@@ -106,10 +107,24 @@ class TestInsightPlatformCutover(APIBaseTest):
("firing_not_yet_notified", None, None, timedelta(minutes=5), CutoverSkip.NOTIFICATION_PENDING),
("firing_notified", None, NOW, timedelta(minutes=5), None),
("firing_past_its_workflow", None, None, timedelta(minutes=30), None),
+ (
+ "firing_not_renotified",
+ None,
+ None,
+ timedelta(minutes=5),
+ None,
+ {FIRING_NOT_RENOTIFIED_KEY: True},
+ ),
]
)
def test_an_alert_moves_only_once_its_last_firing_settles(
- self, _name: str, status: str | None, sent_at: datetime | None, age: timedelta, reason: CutoverSkip | None
+ self,
+ _name: str,
+ status: str | None,
+ sent_at: datetime | None,
+ age: timedelta,
+ reason: CutoverSkip | None,
+ triggered_metadata: dict | None = None,
) -> None:
alert = self._alert(investigation_agent_enabled=True, detector_config={"type": "zscore"})
check = AlertCheck.objects.create(
@@ -117,6 +132,7 @@ class TestInsightPlatformCutover(APIBaseTest):
state=AlertState.FIRING,
investigation_status=status,
notification_sent_at=sent_at,
+ triggered_metadata=triggered_metadata,
)
AlertCheck.objects.filter(id=check.id).update(created_at=NOW - age)
diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.py
index d2213ad1927c..3d9368686cb2 100644
--- a/products/alerts/backend/test/test_platform_source_cycle.py
+++ b/products/alerts/backend/test/test_platform_source_cycle.py
@@ -974,6 +974,44 @@ class TestPlatformInsightEvaluation(APIBaseTest):
if owner == PlatformAlertOwner.PLATFORM:
assert len(self.deliveries) == (0 if held else 1)
+ @parameterized.expand(
+ [
+ ("a statistical detector renotifies", {"type": "zscore", "threshold": 0.95, "window": 30}, True),
+ ("the ai detector announces the firing once", {"type": "llm"}, False),
+ ]
+ )
+ def test_a_firing_that_stays_breached_renotifies_only_for_a_statistical_detector(
+ self, _name: str, detector_config: dict[str, Any], renotifies: bool
+ ) -> None:
+ alert = self._alert(
+ detector_config=detector_config,
+ investigation_agent_enabled=True,
+ investigation_gates_notifications=True,
+ )
+ configuration = self._copy(alert, owner=PlatformAlertOwner.PLATFORM)
+ uses_llm_detector = detector_config["type"] == "llm"
+ result = AlertEvaluationResult(value=150.0, breaches=["above 100"])
+
+ with patch(f"{_MODULE}.render_alert_insight_chart"):
+ first, _ = self._evaluate(configuration, result=result, uses_llm_detector=uses_llm_detector)
+ assert first is not None
+ assert first.groups[0].kind == AlertEventKind.FIRING
+ assert requested_investigation(first) is not None
+ assert first.held_delivery is not None
+ record_outcomes(self.team.id, (first,), CUTOFF)
+ with team_scope(self.team.id):
+ platform_testing.set_due_at(configuration.id, CUTOFF - timedelta(minutes=1))
+
+ second, _ = self._evaluate(configuration, result=result, uses_llm_detector=uses_llm_detector)
+
+ assert second is not None
+ assert second.groups[0].new_state == "firing"
+ assert (second.groups[0].kind == AlertEventKind.FIRING) is renotifies
+ assert (requested_investigation(second) is not None) is renotifies
+ if not renotifies:
+ assert second.held_delivery is None
+ assert self.deliveries == ()
+
def test_a_firing_past_its_budget_is_muted_by_the_firing_last_verdict(self) -> None:
alert, configuration = self._investigated_alert(investigation_gates_notifications=True)
for index, verdict in enumerate(["true_positive", "true_positive", "false_positive"]):