:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(alerts): check insight alert copies at production's minute (#120656)
Jimmy Zhudd57e42f3576
diff --git a/products/alerts/backend/platform_alert_backfill.py b/products/alerts/backend/platform_alert_backfill.pyindex beb8bf345678..205df3847ef2 100644--- a/products/alerts/backend/platform_alert_backfill.py+++ b/products/alerts/backend/platform_alert_backfill.py@@ -66,9 +66,7 @@ def _upsert(alert: AlertConfiguration) -> PlatformAlertUpsert | None:},check_interval_minutes=check_interval_minutes,recurrence_unit=recurrence_unit,- # The hourly cadence has no anchor here: production fires it at the alert's creation- # minute, while sixty minutes on the platform lands on a UTC grid.- anchor_time=alert.schedule_start_time if recurrence_unit is not None else None,+ anchor_time=alert.schedule_start_time,evaluation_periods=1,datapoints_to_alarm=1,cooldown_minutes=0,diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.pyindex f2b964360137..0deccb93f06f 100644--- a/products/alerts/backend/platform_source_cycle.py+++ b/products/alerts/backend/platform_source_cycle.py@@ -283,6 +283,9 @@ def evaluate_insight_check(raiseif outcome is None:return None+ if checks[0].legacy_configuration_id is not None:+ # Production times each check by its own alert's id, so the copy checks at that minute.+ outcome = replace(outcome, schedule_key=str(checks[0].legacy_configuration_id))delivery = _delivery(checks[0], outcome)return SourceBatchEvaluation(outcomes=(outcome,), deliveries=(delivery,) if delivery is not None else ())finally:diff --git a/products/alerts/backend/test/test_platform_alert_backfill.py b/products/alerts/backend/test/test_platform_alert_backfill.pyindex 72cb65e77758..c592702aa4fe 100644--- a/products/alerts/backend/test/test_platform_alert_backfill.py+++ b/products/alerts/backend/test/test_platform_alert_backfill.py@@ -69,7 +69,7 @@ class TestPlatformInsightAlertBackfill(APIBaseTest):assert {key: (v.check_interval_minutes, v.recurrence_unit, v.anchor_time) for key, v in copies.items()} == {real_time.id: (2, None, None),quarter_hourly.id: (15, None, None),- hourly.id: (60, None, None),+ hourly.id: (60, None, "09:30"),daily.id: (60 * 24, "day", "09:30"),detector.id: (60 * 24, "day", None),ai_detector.id: (60 * 24, "day", None),diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.pyindex a73db98c36a5..1fe8f8104d58 100644--- a/products/alerts/backend/test/test_platform_source_cycle.py+++ b/products/alerts/backend/test/test_platform_source_cycle.py@@ -4,6 +4,7 @@ from typing import Anyfrom uuid import UUIDimport pytest+import time_machinefrom posthog.test.base import APIBaseTestfrom unittest.mock import MagicMock, patch@@ -17,7 +18,7 @@ from posthog.exceptions import ClickHouseAtCapacity, ClickHouseClusterMemoryLimifrom posthog.models.scoping import team_scopefrom posthog.redis import get_clientfrom posthog.schema_enums import AlertCalculationInterval-from posthog.tasks.alerts.utils import AlertEvaluationResult+from posthog.tasks.alerts.utils import AlertEvaluationResult, next_check_timefrom posthog.temporal.alerts.admission import inflight_alert_ids, release_evaluation_slotfrom products.alerts.backend.evaluation.contract import AlertExtractionError@@ -72,15 +73,17 @@ class TestPlatformInsightEvaluation(APIBaseTest):def _copy(self, alert: AlertConfiguration | None, **overrides: Any) -> PlatformConfigurationSnapshot:with team_scope(self.team.id):return platform_testing.create_configuration(- team_id=self.team.id,- name="alert",- source_kind=SourceKind.INSIGHT,- source_config={},- check_interval_minutes=60 * 24,- recurrence_unit="day",- next_check_at=CUTOFF - timedelta(minutes=1),- legacy_configuration_id=alert.id if alert else None,- **overrides,+ **{+ "team_id": self.team.id,+ "name": "alert",+ "source_kind": SourceKind.INSIGHT,+ "source_config": {},+ "check_interval_minutes": 60 * 24,+ "recurrence_unit": "day",+ "next_check_at": CUTOFF - timedelta(minutes=1),+ "legacy_configuration_id": alert.id if alert else None,+ **overrides,+ })def _evaluate_per_series(@@ -187,6 +190,51 @@ class TestPlatformInsightEvaluation(APIBaseTest):assert AlertConfiguration.objects.values("state", "next_check_at", "last_checked_at").get(id=alert.id) == beforeassert not AlertCheck.objects.filter(alert_configuration=alert).exists()+ @parameterized.expand(+ [+ ("every_15_minutes", AlertCalculationInterval.EVERY_15_MINUTES, 15, None, None, None),+ ("hourly", AlertCalculationInterval.HOURLY, 60, None, None, None),+ ("daily", AlertCalculationInterval.DAILY, 60 * 24, "day", None, None),+ ("weekly", AlertCalculationInterval.WEEKLY, 60 * 24 * 7, "week", None, None),+ # The next local hour is quiet, so the check moves to the window's end plus its offset.+ ("hourly_quiet_hours", AlertCalculationInterval.HOURLY, 60, None, ("16:00", "17:00"), None),+ ("daily_quiet_hours", AlertCalculationInterval.DAILY, 60 * 24, "day", ("00:30", "02:00"), None),+ ("hourly_start_time", AlertCalculationInterval.HOURLY, 60, None, None, "09:20"),+ ]+ )+ def test_a_copy_is_next_due_when_the_production_alert_is(+ self,+ _name: str,+ interval: AlertCalculationInterval,+ minutes: int,+ unit: str | None,+ quiet: tuple[str, str] | None,+ start_time: str | None,+ ) -> None:+ # A half-hour timezone, where the local hour starts at :30 UTC.+ self.team.timezone = "Asia/Kolkata"+ self.team.save()+ restriction = {"blocked_windows": [{"start": quiet[0], "end": quiet[1]}]} if quiet else None+ alert = self._alert(+ calculation_interval=interval, schedule_restriction=restriction, schedule_start_time=start_time+ )+ configuration = self._copy(+ alert,+ check_interval_minutes=minutes,+ recurrence_unit=unit,+ schedule_restriction=restriction,+ anchor_time=start_time,+ )++ with time_machine.travel(CUTOFF, tick=False):+ outcome, _ = self._evaluate(configuration, result=AlertEvaluationResult(value=1.0, breaches=[]))+ assert outcome is not None+ record_outcomes(self.team.id, (outcome,), CUTOFF)+ expected = next_check_time(alert)++ with team_scope(self.team.id):+ assert platform_testing.configuration(configuration.id).next_check_at == expected+@parameterized.expand([("allowlisted", True, False),diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.pyindex 772f0264d1e0..9e868f61c7e1 100644--- a/products/alerts_platform/backend/facade/contracts.py+++ b/products/alerts_platform/backend/facade/contracts.py@@ -580,6 +580,10 @@ class PlatformAlertOutcome:# A team's checks with the same group are due in one slot, so a source that answers them with# one query can do so. None spreads the configuration by its own id.schedule_group: str | None = None+ # The id the source's own stack times this configuration's checks by. The platform then checks+ # at the minute that stack does, because a metric read at another minute can breach where the+ # source's own check does not. None times the checks by the configuration's own id.+ schedule_key: str | None = None# The grouping the check ran under. An edit can change it while the check is in flight, and the# groups of a stale check are keyed for a grouping the configuration no longer has, so they are# dropped. None when the outcome carries no groups to drop.diff --git a/products/alerts_platform/backend/facade/scheduling.py b/products/alerts_platform/backend/facade/scheduling.pyindex 9050050566a3..31ef8f64e9ff 100644--- a/products/alerts_platform/backend/facade/scheduling.py+++ b/products/alerts_platform/backend/facade/scheduling.py@@ -412,6 +412,14 @@ def to_recurrence_interval(recurrence_unit: str) -> CalendarInterval:raise ValueError(f"Unhandled recurrence unit: {recurrence_unit!r}") from None+# The minute cadences `next_calendar_check_time` also schedules, so a keyed configuration on one of+# them can follow the source's own stack.+_KEYED_MINUTE_CADENCES: Final[dict[int, CalendarInterval]] = {+ EVERY_15_MINUTES_CADENCE_MINUTES: CalendarInterval.EVERY_15_MINUTES,+ 60: CalendarInterval.HOURLY,+}++def advance_schedule(*,current_next_check_at: datetime | None,@@ -422,6 +430,8 @@ def advance_schedule(now: datetime,configuration_id: UUID | str,shard_offset_seconds: int = 0,+ schedule_key: UUID | str | None = None,+ blocked_windows: list[BlockedWindow] | None = None,) -> datetime:"""When a configuration is next due, by whichever recurrence it carries.@@ -433,22 +443,80 @@ def advance_schedule(due in the same minute slot. Spreading those would move a time the user chose. A calendarunit without an anchor has no chosen time, so `next_calendar_check_time` spreads it after theinterval boundary by `configuration_id`, as it does for an insight alert.++ `schedule_key` names the alert the source's own stack times this one by. With it, a 15-minute+ or hourly cadence follows that stack's local interval boundary and offset rather than the UTC+ grid and shard, a calendar unit spreads by the key rather than by `configuration_id`, and a+ check that would land in `blocked_windows` moves out of it the way that stack moves it. Without+ a key, quiet hours never move the schedule: the check runs and only its announcement is held."""if recurrence_unit is None:+ keyed_interval = _KEYED_MINUTE_CADENCES.get(check_interval_minutes)+ if schedule_key is not None and keyed_interval is not None:+ candidate = next_calendar_check_time(+ keyed_interval,+ now=now,+ tz_name=tz_name,+ next_check_at=current_next_check_at,+ alert_id=schedule_key,+ schedule_start_time=anchor_time,+ )+ return snap_out_of_blocked_windows(+ candidate,+ tz_name,+ blocked_windows,+ interval=keyed_interval,+ alert_id=schedule_key,+ anchored=anchor_time is not None,+ )return advance_next_check_at(current_next_check_at,check_interval_minutes,now,shard_offset_seconds=shard_offset_seconds,)- return next_calendar_check_time(- to_recurrence_interval(recurrence_unit),+ interval = to_recurrence_interval(recurrence_unit)+ candidate = next_calendar_check_time(+ interval,now=now,tz_name=tz_name,next_check_at=current_next_check_at,- alert_id=configuration_id,+ alert_id=schedule_key if schedule_key is not None else configuration_id,schedule_start_time=anchor_time,)+ if schedule_key is None:+ return candidate+ return snap_out_of_blocked_windows(+ candidate, tz_name, blocked_windows, interval=interval, alert_id=schedule_key, anchored=anchor_time is not None+ )+++def snap_out_of_blocked_windows(+ candidate_utc: datetime,+ tz_name: str,+ windows: list[BlockedWindow] | None,+ *,+ interval: CalendarInterval,+ alert_id: UUID | str,+ anchored: bool,+) -> datetime:+ """The first unblocked minute at or after `candidate_utc`, the way an insight alert's check moves.++ A snap lands on the end of a quiet window, which is usually on the hour, so every alert behind+ the same window would check at once. An alert without a chosen start time adds its offset to+ spread them again, unless the offset lands back in a window.+ """+ if not windows:+ return candidate_utc+ normalized = candidate_utc.astimezone(UTC).replace(second=0, microsecond=0)+ snapped = scan_next_unblocked_utc(normalized, tz_name, windows)+ if snapped is None:+ # Every minute in the scan is blocked. Try again a day later rather than never.+ return normalized + timedelta(days=1)+ if snapped == normalized or anchored:+ return snapped+ offset_snap = snapped + alert_check_offset(interval, alert_id)+ return snapped if is_utc_datetime_blocked(offset_snap, tz_name, windows) else offset_snap# --- Quiet hours (blocked local time windows) ---diff --git a/products/alerts_platform/backend/logic/platform_lifecycle.py b/products/alerts_platform/backend/logic/platform_lifecycle.pyindex a5c341d15434..856df7fef803 100644--- a/products/alerts_platform/backend/logic/platform_lifecycle.py+++ b/products/alerts_platform/backend/logic/platform_lifecycle.py@@ -45,6 +45,7 @@ from products.alerts_platform.backend.facade.scheduling import (advance_schedule,compute_shard_offset_seconds,compute_team_shard_offset_seconds,+ parse_blocked_windows_tuples,to_recurrence_interval,validate_and_normalize_schedule_start_time,)@@ -614,6 +615,8 @@ def record_outcomes(team_id: int, outcomes: Sequence[PlatformAlertOutcome], now:now=now,configuration_id=configuration.id,shard_offset_seconds=_shard_offset_seconds(configuration, outcome),+ schedule_key=outcome.schedule_key,+ blocked_windows=parse_blocked_windows_tuples(configuration.schedule_restriction),)# Batched, because a grouped check writes a row per group rather than one per configuration.