Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): keep insight alert copies in step with their alerts (#120623)

Jimmy Zhucfa37aa969f5
diff --git a/products/alerts/backend/facade/platform.py b/products/alerts/backend/facade/platform.py
new file mode 100644
index 000000000000..3587efba5ad9
--- /dev/null
+++ b/products/alerts/backend/facade/platform.py
@@ -0,0 +1,21 @@
+"""Insight alerts on the alerts platform, for this product's own API.
+
+Apart from `api.py`, because `posthog.tasks.alerts.utils` imports `api.py`, and the platform
+modules here import `posthog.tasks.alerts.utils`.
+"""
+
+import uuid
+
+from products.alerts.backend.models.alert import AlertConfiguration
+from products.alerts.backend.platform_alert_backfill import follow_insight_alert as follow_alert
+
+
+def follow_insight_alert(*, team_id: int, alert_id: uuid.UUID) -> None:
+ """Copies an insight alert that was just created or turned back on to the platform, if its team
+ runs insight alerts there."""
+ alert = AlertConfiguration.objects.select_related("threshold").filter(team_id=team_id, id=alert_id).first()
+ if alert is not None:
+ follow_alert(alert)
+
+
+__all__ = ["follow_insight_alert"]
diff --git a/products/alerts/backend/max_tools.py b/products/alerts/backend/max_tools.py
index cc22a103d656..4892104fadcf 100644
--- a/products/alerts/backend/max_tools.py
+++ b/products/alerts/backend/max_tools.py
@@ -185,6 +185,14 @@ class UpsertAlertToolArgs(BaseModel):
)
+def _follow_on_commit(alert: AlertConfiguration) -> None:
+ # Deferred to break a cycle: platform_alert_backfill reaches this module back through the alert
+ # workflows and the chat agent.
+ from products.alerts.backend.platform_alert_backfill import follow_insight_alert # noqa: PLC0415
+
+ transaction.on_commit(lambda: follow_insight_alert(alert), robust=True)
+
+
class UpsertAlertTool(MaxTool):
name: str = "upsert_alert"
description: str = UPSERT_ALERT_TOOL_DESCRIPTION
@@ -462,6 +470,8 @@ class UpsertAlertTool(MaxTool):
alert.next_check_at = timezone.now()
update_fields.append("next_check_at")
alert.save(update_fields=update_fields)
+ if enabled_changed and action.enabled:
+ _follow_on_commit(alert)
return None
@staticmethod
@@ -653,5 +663,7 @@ class UpsertAlertTool(MaxTool):
alert_configuration=alert,
created_by=user,
)
+ if alert.enabled:
+ _follow_on_commit(alert)
return alert
diff --git a/products/alerts/backend/platform_alert_backfill.py b/products/alerts/backend/platform_alert_backfill.py
index 205df3847ef2..eb03b2ef3165 100644
--- a/products/alerts/backend/platform_alert_backfill.py
+++ b/products/alerts/backend/platform_alert_backfill.py
@@ -13,28 +13,14 @@ from uuid import UUID
import structlog
from posthog.dataclasses import frozen
-from posthog.schema_enums import AlertCalculationInterval
from products.alerts.backend.models.alert import AlertConfiguration
-from products.alerts.backend.platform_source_cycle import is_evaluated_on_the_platform
-from products.alerts_platform.backend.facade.api import disable_configurations, upsert_configuration
-from products.alerts_platform.backend.facade.contracts import SOURCE_CONDITION_KEY, PlatformAlertUpsert, SourceKind
-from products.alerts_platform.backend.facade.enums import PlatformAlertConfigurationRecurrenceUnit as RecurrenceUnit
-from products.alerts_platform.backend.facade.scheduling import REAL_TIME_CADENCE_MINUTES
+from products.alerts.backend.platform_source_cycle import platform_upsert_of
+from products.alerts_platform.backend.facade.api import disable_configurations, upsert_configuration, upsert_if_running
+from products.alerts_platform.backend.facade.contracts import SourceKind
logger = structlog.get_logger(__name__)
-# Minutes for the real-time, 15-minute and hourly cadences, a calendar unit for the rest. A calendar unit reads its cadence
-# from the unit, so its minutes only feed the shard offset, which an anchored unit ignores.
-_RECURRENCE: dict[str, tuple[int, str | None]] = {
- AlertCalculationInterval.REAL_TIME: (REAL_TIME_CADENCE_MINUTES, None),
- AlertCalculationInterval.EVERY_15_MINUTES: (15, None),
- AlertCalculationInterval.HOURLY: (60, None),
- AlertCalculationInterval.DAILY: (60 * 24, RecurrenceUnit.DAY),
- AlertCalculationInterval.WEEKLY: (60 * 24 * 7, RecurrenceUnit.WEEK),
- AlertCalculationInterval.MONTHLY: (60 * 24 * 30, RecurrenceUnit.MONTH),
-}
-
@frozen
class BackfillCounts:
@@ -44,38 +30,6 @@ class BackfillCounts:
failed: int
-def _upsert(alert: AlertConfiguration) -> PlatformAlertUpsert | None:
- """The copy of one alert, or None when the platform does not evaluate it in parallel."""
- recurrence = _RECURRENCE.get(alert.calculation_interval or "")
- if recurrence is None or not is_evaluated_on_the_platform(alert):
- return None
- check_interval_minutes, recurrence_unit = recurrence
- return PlatformAlertUpsert(
- legacy_configuration_id=alert.id,
- team_id=alert.team_id,
- name=alert.name,
- enabled=alert.enabled,
- source_kind=SourceKind.INSIGHT,
- source_config={
- "insight_id": alert.insight_id,
- SOURCE_CONDITION_KEY: {
- "threshold": alert.threshold.configuration if alert.threshold else None,
- "comparison": alert.condition,
- "detector": alert.detector_config,
- },
- },
- check_interval_minutes=check_interval_minutes,
- recurrence_unit=recurrence_unit,
- anchor_time=alert.schedule_start_time,
- evaluation_periods=1,
- datapoints_to_alarm=1,
- cooldown_minutes=0,
- schedule_restriction=alert.schedule_restriction,
- next_check_at=alert.next_check_at,
- snooze_until=alert.snoozed_until,
- )
-
-
def is_sampled(alert_id: UUID, sample_percent: int) -> bool:
"""Whether an alert falls inside a sample. Decided by its id, so a rerun picks the same alerts
and widening the sample only adds alerts."""
@@ -104,7 +58,7 @@ def backfill_platform_insight_alert_configurations(
skipped = 0
failed = 0
for alert in source.iterator():
- upsert = _upsert(alert) if is_sampled(alert.id, sample_percent) else None
+ upsert = platform_upsert_of(alert) if is_sampled(alert.id, sample_percent) else None
if upsert is None:
skipped += 1
continue
@@ -141,3 +95,14 @@ def disable_platform_insight_alert_configurations(
disabled = disable_configurations(SourceKind.INSIGHT, team_id=team_id, legacy_configuration_ids=alert_ids)
logger.info("platform_insight_alert_backfill.disabled", disabled=disabled, team_id=team_id)
return disabled
+
+
+def follow_insight_alert(alert: AlertConfiguration) -> None:
+ """Copies an insight alert that was just created or turned back on, if its team runs insight on the platform.
+
+ Checks only read enabled copies, so no check would ever see this alert otherwise, and it would
+ stay off the platform until the next backfill.
+ """
+ upsert = platform_upsert_of(alert)
+ if upsert is not None:
+ upsert_if_running(upsert, revive=True)
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index cf4e2390c113..0db5f9e65f35 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -47,18 +47,21 @@ from products.alerts.backend.judge.contract import (
from products.alerts.backend.llm_detector_limits import is_llm_detector_config
from products.alerts.backend.logic.alert_email import INSIGHT_ALERT_ERRORED_EVENT_ID
from products.alerts.backend.models.alert import AlertConfiguration
-from products.alerts_platform.backend.facade.api import due_check_ids, due_checks, slot_of
+from products.alerts_platform.backend.facade.api import due_check_ids, due_checks, refresh_settings, slot_of
from products.alerts_platform.backend.facade.contracts import (
+ SOURCE_CONDITION_KEY,
AlertDeliveryRequest,
AlertEventKind,
GroupingMode,
InstanceCheckState,
PlatformAlertCheckInput,
PlatformAlertOutcome,
+ PlatformAlertUpsert,
SkipReason,
SourceBatchEvaluation,
SourceKind,
)
+from products.alerts_platform.backend.facade.enums import PlatformAlertConfigurationRecurrenceUnit as RecurrenceUnit
from products.alerts_platform.backend.facade.grouping import (
GroupObservation,
bounded_grouping_key,
@@ -87,6 +90,10 @@ from products.alerts_platform.backend.facade.platform_metrics import (
record_scheduler_lag,
safe_record,
)
+from products.alerts_platform.backend.facade.scheduling import (
+ REAL_TIME_CADENCE_MINUTES,
+ validate_and_normalize_schedule_start_time,
+)
from products.cdp.backend.internal_events import LEGACY_INSIGHT_ALERT_EVENT
logger = structlog.get_logger(__name__)
@@ -141,6 +148,62 @@ def is_evaluated_on_the_platform(alert: AlertConfiguration) -> bool:
return alert.calculation_interval in EVALUATED_INTERVALS
+# Minutes for the real-time, 15-minute and hourly cadences, a calendar unit for the rest. A calendar
+# unit reads its cadence from the unit, so its minutes only feed the shard offset, which an anchored
+# unit ignores.
+_RECURRENCE: dict[str, tuple[int, str | None]] = {
+ AlertCalculationInterval.REAL_TIME: (REAL_TIME_CADENCE_MINUTES, None),
+ AlertCalculationInterval.EVERY_15_MINUTES: (15, None),
+ AlertCalculationInterval.HOURLY: (60, None),
+ AlertCalculationInterval.DAILY: (60 * 24, RecurrenceUnit.DAY),
+ AlertCalculationInterval.WEEKLY: (60 * 24 * 7, RecurrenceUnit.WEEK),
+ AlertCalculationInterval.MONTHLY: (60 * 24 * 30, RecurrenceUnit.MONTH),
+}
+
+# What an edit to an insight alert can change on its copy. The check refreshes these, and the due
+# time follows from them, so it is left to the next advance.
+SETTINGS_FIELDS: Final = (
+ "name",
+ "source_config",
+ "schedule_restriction",
+ "check_interval_minutes",
+ "recurrence_unit",
+ "anchor_time",
+)
+
+
+def platform_upsert_of(alert: AlertConfiguration) -> PlatformAlertUpsert | None:
+ """The copy of one alert, or None when the platform does not evaluate it in parallel."""
+ recurrence = _RECURRENCE.get(alert.calculation_interval or "")
+ if recurrence is None or not is_evaluated_on_the_platform(alert):
+ return None
+ check_interval_minutes, recurrence_unit = recurrence
+ return PlatformAlertUpsert(
+ legacy_configuration_id=alert.id,
+ team_id=alert.team_id,
+ name=alert.name,
+ enabled=alert.enabled,
+ source_kind=SourceKind.INSIGHT,
+ source_config={
+ "insight_id": alert.insight_id,
+ SOURCE_CONDITION_KEY: {
+ "threshold": alert.threshold.configuration if alert.threshold else None,
+ "comparison": alert.condition,
+ "detector": alert.detector_config,
+ },
+ },
+ check_interval_minutes=check_interval_minutes,
+ recurrence_unit=recurrence_unit,
+ anchor_time=alert.schedule_start_time,
+ evaluation_periods=1,
+ datapoints_to_alarm=1,
+ cooldown_minutes=0,
+ schedule_restriction=alert.schedule_restriction,
+ next_check_at=alert.next_check_at,
+ snooze_until=alert.snoozed_until,
+ )
+
+
def plan_insight_batch(
team_id: int, slot: str, cutoff: datetime, *, expires_at: float, exclude: Collection[str] = ()
) -> tuple[str, ...]:
@@ -379,6 +442,8 @@ def _decide(
if alert is None:
# The production alert is gone, so this copy can never be compared again.
return _skipped(check, snapshot, now=now, disable=True)
+ # Before any skip, because a skip advances the check by the copy's schedule.
+ check = _following(check, alert)
if not alert.enabled or alert.insight.deleted or not is_evaluated_on_the_platform(alert):
return _skipped(check, snapshot, now=now)
if skip_because_of_weekend(alert) or is_utc_datetime_blocked(alert, now):
@@ -445,6 +510,32 @@ def _decide(
)
+def _following(check: PlatformAlertCheckInput, alert: AlertConfiguration) -> PlatformAlertCheckInput:
+ """The check with its alert's settings as they are now, written back to the copy first.
+
+ An edit to the alert never reaches the copy otherwise, so history and messages would quote an
+ old name or condition, and the copy would keep its old cadence until the next backfill.
+ """
+ upsert = platform_upsert_of(alert)
+ if upsert is None:
+ return check
+ try:
+ # The copy stores the anchor normalized, so the alert's start time is compared in that form.
+ upsert = replace(upsert, anchor_time=validate_and_normalize_schedule_start_time(upsert.anchor_time))
+ except ValueError:
+ # The backfill skips an alert whose start time cannot be read, so the copy keeps its own.
+ return check
+ if all(getattr(check, field) == getattr(upsert, field) for field in SETTINGS_FIELDS):
+ return check
+ try:
+ refresh_settings(check.team_id, check.id, upsert)
+ except Exception:
+ # History records the stored copy, so the check evaluates that copy and stays true to it.
+ logger.exception("Failed to bring a platform insight copy up to date", check_id=str(check.id))
+ return check
+ return replace(check, **{field: getattr(upsert, field) for field in SETTINGS_FIELDS})
+
+
def _broken(
check: PlatformAlertCheckInput, snapshot: AlertSnapshot, *, now: datetime, message: str
) -> PlatformAlertOutcome:
diff --git a/products/alerts/backend/presentation/views/alert.py b/products/alerts/backend/presentation/views/alert.py
index 29c8c61b93e9..02dc52f52496 100644
--- a/products/alerts/backend/presentation/views/alert.py
+++ b/products/alerts/backend/presentation/views/alert.py
@@ -101,6 +101,7 @@ from products.alerts.backend.facade.destinations import (
validate_destination_data,
)
from products.alerts.backend.facade.evaluation import DelayedEvaluationUnavailable, validate_evaluation_delay
+from products.alerts.backend.facade.platform import follow_insight_alert
from products.alerts.backend.insight_alert_state_machine import (
apply_disable,
apply_enable,
@@ -899,6 +900,10 @@ class AlertSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerialize
self.context["request"].user,
analytics_props=get_request_analytics_properties(self.context["request"]),
)
+ if instance.enabled:
+ transaction.on_commit(
+ lambda: follow_insight_alert(team_id=instance.team_id, alert_id=instance.id), robust=True
+ )
return instance
@transaction.atomic
@@ -1023,6 +1028,10 @@ class AlertSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerialize
self.context["request"].user,
analytics_props=get_request_analytics_properties(self.context["request"]),
)
+ if enable_now:
+ transaction.on_commit(
+ lambda: follow_insight_alert(team_id=instance.team_id, alert_id=instance.id), robust=True
+ )
return instance
def validate_schedule_start_time(self, value: str | None) -> str | None:
diff --git a/products/alerts/backend/test/test_platform_alert_backfill.py b/products/alerts/backend/test/test_platform_alert_backfill.py
index c592702aa4fe..0c10f5a8349a 100644
--- a/products/alerts/backend/test/test_platform_alert_backfill.py
+++ b/products/alerts/backend/test/test_platform_alert_backfill.py
@@ -3,6 +3,8 @@ from uuid import UUID
from posthog.test.base import APIBaseTest
+from parameterized import parameterized
+
from posthog.schema import ChartDisplayType, EventsNode, IntervalType, TrendsFilter, TrendsQuery
from posthog.models.scoping import team_scope
@@ -12,6 +14,7 @@ from products.alerts.backend.models.alert import AlertConfiguration, Threshold
from products.alerts.backend.platform_alert_backfill import (
backfill_platform_insight_alert_configurations,
disable_platform_insight_alert_configurations,
+ follow_insight_alert,
)
from products.alerts_platform.backend.facade.api import list_configurations
from products.alerts_platform.backend.facade.contracts import SourceKind
@@ -109,3 +112,14 @@ class TestPlatformInsightAlertBackfill(APIBaseTest):
backfill_platform_insight_alert_configurations(team_id=self.team.id)
assert {view.enabled for view in self._copies().values()} == {True}
+
+ @parameterized.expand([("a_team_on_the_platform", True), ("a_team_off_the_platform", False)])
+ def test_a_created_or_reenabled_alert_follows_its_team_onto_the_platform(self, _name: str, running: bool) -> None:
+ if running:
+ self._alert()
+ backfill_platform_insight_alert_configurations(team_id=self.team.id)
+ created = self._alert()
+
+ follow_insight_alert(created)
+
+ assert (created.id in self._copies()) is running
diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.py
index 5818ced2425d..168675ca3439 100644
--- a/products/alerts/backend/test/test_platform_source_cycle.py
+++ b/products/alerts/backend/test/test_platform_source_cycle.py
@@ -44,7 +44,7 @@ from products.alerts.backend.temporal.platform_evaluate import (
)
from products.alerts.backend.test.insight_alerts import create_insight_alert
from products.alerts_platform.backend.facade import testing as platform_testing
-from products.alerts_platform.backend.facade.api import record_outcomes, slot_of
+from products.alerts_platform.backend.facade.api import list_configurations, record_outcomes, slot_of
from products.alerts_platform.backend.facade.contracts import (
AlertBatchKey,
AlertEventKind,
@@ -167,6 +167,39 @@ class TestPlatformInsightEvaluation(APIBaseTest):
self.deliveries = evaluation.deliveries if evaluation is not None else ()
return (evaluation.outcomes[0] if evaluation is not None else None), query
+ @parameterized.expand(
+ [
+ ("renamed", {"name": "Signups dropped"}, ("Signups dropped", 60 * 24, "day", None)),
+ # A skipped check advances by the copy's schedule, so it must refresh the copy first.
+ (
+ "renamed_while_off",
+ {"name": "Signups dropped", "enabled": False},
+ ("Signups dropped", 60 * 24, "day", None),
+ ),
+ (
+ "weekly",
+ {"calculation_interval": AlertCalculationInterval.WEEKLY.value},
+ ("alert", 60 * 24 * 7, "week", None),
+ ),
+ ("started_later", {"schedule_start_time": "09:30"}, ("alert", 60 * 24, "day", "09:30")),
+ # A stored anchor that cannot advance would fail every outcome recorded beside it.
+ ("unreadable_start_time", {"schedule_start_time": "25:99"}, ("alert", 60 * 24, "day", None)),
+ ]
+ )
+ def test_a_check_brings_its_copy_up_to_date_with_the_alert(
+ self, _name: str, edits: dict[str, Any], expected: tuple[str, int, str, str | None]
+ ) -> None:
+ alert = self._alert(**edits)
+ configuration = self._copy(alert)
+
+ self._evaluate(configuration, result=AlertEvaluationResult(value=50.0, breaches=[]))
+
+ with team_scope(self.team.id):
+ (copy,) = list_configurations(
+ team_id=self.team.id, source_kinds=[SourceKind.INSIGHT.value], limit=10, offset=0
+ ).configurations
+ assert (copy.name, copy.check_interval_minutes, copy.recurrence_unit, copy.anchor_time) == expected
+
@parameterized.expand([("threshold", {}), ("detector", {"detector_config": {"type": "zscore"}})])
def test_a_breach_fires_on_the_platform_and_leaves_the_production_alert_alone(
self, _name: str, alert_overrides: dict[str, Any]
diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.py
index e66a318644c2..ed6ba9b0457a 100644
--- a/products/alerts_platform/backend/facade/contracts.py
+++ b/products/alerts_platform/backend/facade/contracts.py
@@ -265,6 +265,9 @@ class PlatformAlertCheckInput:
owned_by_platform: bool = False
instances: tuple[InstanceCheckState, ...] = ()
grouping: Grouping = field(default_factory=Grouping)
+ # The calendar half of the schedule, so a source can tell when its own cadence has moved.
+ recurrence_unit: str | None = None
+ anchor_time: str | None = None
def admit(self, grouping_keys: Sequence[str], *, now: datetime) -> GroupAdmission:
"""The groups a check may report, from keys in the source's priority order.
diff --git a/products/alerts_platform/backend/logic/platform_lifecycle.py b/products/alerts_platform/backend/logic/platform_lifecycle.py
index 856df7fef803..8aa053f267f0 100644
--- a/products/alerts_platform/backend/logic/platform_lifecycle.py
+++ b/products/alerts_platform/backend/logic/platform_lifecycle.py
@@ -295,6 +295,8 @@ def _check(c: PlatformAlertConfiguration, alerts: Sequence[PlatformAlert]) -> Pl
instances=tuple(_composed(c, alert) for alert in alerts),
grouping=Grouping.from_stored(c.grouping),
owned_by_platform=c.owner == PlatformAlertConfiguration.Owner.PLATFORM,
+ recurrence_unit=c.recurrence_unit,
+ anchor_time=c.anchor_time,
)
@@ -847,12 +849,22 @@ def refresh_settings(team_id: int, configuration_id: UUID, upsert: PlatformAlert
"""Writes a source's current settings onto its copy, so history and messages state what a check
evaluated.
- Settings only. The schedule, `enabled` and the alert rows stay as they are, so an evaluation can
- call this before it decides and a retried attempt still finds the same checks due.
+ Settings only. The due time, `enabled` and the alert rows stay as they are, so an evaluation can
+ call this before it decides and a retried attempt still finds the same checks due. A changed
+ cadence takes effect from the next due time the check advances to.
+
+ Raises `ValueError` on a cadence `upsert_configuration` would refuse, because a stored one that
+ cannot advance fails every outcome of the batch it is recorded in.
"""
+ if upsert.recurrence_unit is not None:
+ to_recurrence_interval(upsert.recurrence_unit)
+ anchor_time = validate_and_normalize_schedule_start_time(upsert.anchor_time)
PlatformAlertConfiguration.objects.for_team(team_id).filter(id=configuration_id).update(
name=upsert.name,
source_config=upsert.source_config,
+ check_interval_minutes=upsert.check_interval_minutes,
+ recurrence_unit=upsert.recurrence_unit,
+ anchor_time=anchor_time,
evaluation_periods=upsert.evaluation_periods,
datapoints_to_alarm=upsert.datapoints_to_alarm,
cooldown_minutes=upsert.cooldown_minutes,