Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(alerts): check insight alert copies at production's minute (#120656)

Jimmy Zhudd57e42f3576
diff --git a/products/alerts/backend/platform_alert_backfill.py b/products/alerts/backend/platform_alert_backfill.py
index beb8bf345678..205df3847ef2 100644
--- a/products/alerts/backend/platform_alert_backfill.py
+++ b/products/alerts/backend/platform_alert_backfill.py
@@ -66,9 +66,7 @@ def _upsert(alert: AlertConfiguration) -> PlatformAlertUpsert | None:
},
check_interval_minutes=check_interval_minutes,
recurrence_unit=recurrence_unit,
- # The hourly cadence has no anchor here: production fires it at the alert's creation
- # minute, while sixty minutes on the platform lands on a UTC grid.
- anchor_time=alert.schedule_start_time if recurrence_unit is not None else None,
+ anchor_time=alert.schedule_start_time,
evaluation_periods=1,
datapoints_to_alarm=1,
cooldown_minutes=0,
diff --git a/products/alerts/backend/platform_source_cycle.py b/products/alerts/backend/platform_source_cycle.py
index f2b964360137..0deccb93f06f 100644
--- a/products/alerts/backend/platform_source_cycle.py
+++ b/products/alerts/backend/platform_source_cycle.py
@@ -283,6 +283,9 @@ def evaluate_insight_check(
raise
if outcome is None:
return None
+ if checks[0].legacy_configuration_id is not None:
+ # Production times each check by its own alert's id, so the copy checks at that minute.
+ outcome = replace(outcome, schedule_key=str(checks[0].legacy_configuration_id))
delivery = _delivery(checks[0], outcome)
return SourceBatchEvaluation(outcomes=(outcome,), deliveries=(delivery,) if delivery is not None else ())
finally:
diff --git a/products/alerts/backend/test/test_platform_alert_backfill.py b/products/alerts/backend/test/test_platform_alert_backfill.py
index 72cb65e77758..c592702aa4fe 100644
--- a/products/alerts/backend/test/test_platform_alert_backfill.py
+++ b/products/alerts/backend/test/test_platform_alert_backfill.py
@@ -69,7 +69,7 @@ class TestPlatformInsightAlertBackfill(APIBaseTest):
assert {key: (v.check_interval_minutes, v.recurrence_unit, v.anchor_time) for key, v in copies.items()} == {
real_time.id: (2, None, None),
quarter_hourly.id: (15, None, None),
- hourly.id: (60, None, None),
+ hourly.id: (60, None, "09:30"),
daily.id: (60 * 24, "day", "09:30"),
detector.id: (60 * 24, "day", None),
ai_detector.id: (60 * 24, "day", None),
diff --git a/products/alerts/backend/test/test_platform_source_cycle.py b/products/alerts/backend/test/test_platform_source_cycle.py
index a73db98c36a5..1fe8f8104d58 100644
--- a/products/alerts/backend/test/test_platform_source_cycle.py
+++ b/products/alerts/backend/test/test_platform_source_cycle.py
@@ -4,6 +4,7 @@ from typing import Any
from uuid import UUID
import pytest
+import time_machine
from posthog.test.base import APIBaseTest
from unittest.mock import MagicMock, patch
@@ -17,7 +18,7 @@ from posthog.exceptions import ClickHouseAtCapacity, ClickHouseClusterMemoryLimi
from posthog.models.scoping import team_scope
from posthog.redis import get_client
from posthog.schema_enums import AlertCalculationInterval
-from posthog.tasks.alerts.utils import AlertEvaluationResult
+from posthog.tasks.alerts.utils import AlertEvaluationResult, next_check_time
from posthog.temporal.alerts.admission import inflight_alert_ids, release_evaluation_slot
from products.alerts.backend.evaluation.contract import AlertExtractionError
@@ -72,15 +73,17 @@ class TestPlatformInsightEvaluation(APIBaseTest):
def _copy(self, alert: AlertConfiguration | None, **overrides: Any) -> PlatformConfigurationSnapshot:
with team_scope(self.team.id):
return platform_testing.create_configuration(
- team_id=self.team.id,
- name="alert",
- source_kind=SourceKind.INSIGHT,
- source_config={},
- check_interval_minutes=60 * 24,
- recurrence_unit="day",
- next_check_at=CUTOFF - timedelta(minutes=1),
- legacy_configuration_id=alert.id if alert else None,
- **overrides,
+ **{
+ "team_id": self.team.id,
+ "name": "alert",
+ "source_kind": SourceKind.INSIGHT,
+ "source_config": {},
+ "check_interval_minutes": 60 * 24,
+ "recurrence_unit": "day",
+ "next_check_at": CUTOFF - timedelta(minutes=1),
+ "legacy_configuration_id": alert.id if alert else None,
+ **overrides,
+ }
)
def _evaluate_per_series(
@@ -187,6 +190,51 @@ class TestPlatformInsightEvaluation(APIBaseTest):
assert AlertConfiguration.objects.values("state", "next_check_at", "last_checked_at").get(id=alert.id) == before
assert not AlertCheck.objects.filter(alert_configuration=alert).exists()
+ @parameterized.expand(
+ [
+ ("every_15_minutes", AlertCalculationInterval.EVERY_15_MINUTES, 15, None, None, None),
+ ("hourly", AlertCalculationInterval.HOURLY, 60, None, None, None),
+ ("daily", AlertCalculationInterval.DAILY, 60 * 24, "day", None, None),
+ ("weekly", AlertCalculationInterval.WEEKLY, 60 * 24 * 7, "week", None, None),
+ # The next local hour is quiet, so the check moves to the window's end plus its offset.
+ ("hourly_quiet_hours", AlertCalculationInterval.HOURLY, 60, None, ("16:00", "17:00"), None),
+ ("daily_quiet_hours", AlertCalculationInterval.DAILY, 60 * 24, "day", ("00:30", "02:00"), None),
+ ("hourly_start_time", AlertCalculationInterval.HOURLY, 60, None, None, "09:20"),
+ ]
+ )
+ def test_a_copy_is_next_due_when_the_production_alert_is(
+ self,
+ _name: str,
+ interval: AlertCalculationInterval,
+ minutes: int,
+ unit: str | None,
+ quiet: tuple[str, str] | None,
+ start_time: str | None,
+ ) -> None:
+ # A half-hour timezone, where the local hour starts at :30 UTC.
+ self.team.timezone = "Asia/Kolkata"
+ self.team.save()
+ restriction = {"blocked_windows": [{"start": quiet[0], "end": quiet[1]}]} if quiet else None
+ alert = self._alert(
+ calculation_interval=interval, schedule_restriction=restriction, schedule_start_time=start_time
+ )
+ configuration = self._copy(
+ alert,
+ check_interval_minutes=minutes,
+ recurrence_unit=unit,
+ schedule_restriction=restriction,
+ anchor_time=start_time,
+ )
+
+ with time_machine.travel(CUTOFF, tick=False):
+ outcome, _ = self._evaluate(configuration, result=AlertEvaluationResult(value=1.0, breaches=[]))
+ assert outcome is not None
+ record_outcomes(self.team.id, (outcome,), CUTOFF)
+ expected = next_check_time(alert)
+
+ with team_scope(self.team.id):
+ assert platform_testing.configuration(configuration.id).next_check_at == expected
+
@parameterized.expand(
[
("allowlisted", True, False),
diff --git a/products/alerts_platform/backend/facade/contracts.py b/products/alerts_platform/backend/facade/contracts.py
index 772f0264d1e0..9e868f61c7e1 100644
--- a/products/alerts_platform/backend/facade/contracts.py
+++ b/products/alerts_platform/backend/facade/contracts.py
@@ -580,6 +580,10 @@ class PlatformAlertOutcome:
# A team's checks with the same group are due in one slot, so a source that answers them with
# one query can do so. None spreads the configuration by its own id.
schedule_group: str | None = None
+ # The id the source's own stack times this configuration's checks by. The platform then checks
+ # at the minute that stack does, because a metric read at another minute can breach where the
+ # source's own check does not. None times the checks by the configuration's own id.
+ schedule_key: str | None = None
# The grouping the check ran under. An edit can change it while the check is in flight, and the
# groups of a stale check are keyed for a grouping the configuration no longer has, so they are
# dropped. None when the outcome carries no groups to drop.
diff --git a/products/alerts_platform/backend/facade/scheduling.py b/products/alerts_platform/backend/facade/scheduling.py
index 9050050566a3..31ef8f64e9ff 100644
--- a/products/alerts_platform/backend/facade/scheduling.py
+++ b/products/alerts_platform/backend/facade/scheduling.py
@@ -412,6 +412,14 @@ def to_recurrence_interval(recurrence_unit: str) -> CalendarInterval:
raise ValueError(f"Unhandled recurrence unit: {recurrence_unit!r}") from None
+# The minute cadences `next_calendar_check_time` also schedules, so a keyed configuration on one of
+# them can follow the source's own stack.
+_KEYED_MINUTE_CADENCES: Final[dict[int, CalendarInterval]] = {
+ EVERY_15_MINUTES_CADENCE_MINUTES: CalendarInterval.EVERY_15_MINUTES,
+ 60: CalendarInterval.HOURLY,
+}
+
+
def advance_schedule(
*,
current_next_check_at: datetime | None,
@@ -422,6 +430,8 @@ def advance_schedule(
now: datetime,
configuration_id: UUID | str,
shard_offset_seconds: int = 0,
+ schedule_key: UUID | str | None = None,
+ blocked_windows: list[BlockedWindow] | None = None,
) -> datetime:
"""When a configuration is next due, by whichever recurrence it carries.
@@ -433,22 +443,80 @@ def advance_schedule(
due in the same minute slot. Spreading those would move a time the user chose. A calendar
unit without an anchor has no chosen time, so `next_calendar_check_time` spreads it after the
interval boundary by `configuration_id`, as it does for an insight alert.
+
+ `schedule_key` names the alert the source's own stack times this one by. With it, a 15-minute
+ or hourly cadence follows that stack's local interval boundary and offset rather than the UTC
+ grid and shard, a calendar unit spreads by the key rather than by `configuration_id`, and a
+ check that would land in `blocked_windows` moves out of it the way that stack moves it. Without
+ a key, quiet hours never move the schedule: the check runs and only its announcement is held.
"""
if recurrence_unit is None:
+ keyed_interval = _KEYED_MINUTE_CADENCES.get(check_interval_minutes)
+ if schedule_key is not None and keyed_interval is not None:
+ candidate = next_calendar_check_time(
+ keyed_interval,
+ now=now,
+ tz_name=tz_name,
+ next_check_at=current_next_check_at,
+ alert_id=schedule_key,
+ schedule_start_time=anchor_time,
+ )
+ return snap_out_of_blocked_windows(
+ candidate,
+ tz_name,
+ blocked_windows,
+ interval=keyed_interval,
+ alert_id=schedule_key,
+ anchored=anchor_time is not None,
+ )
return advance_next_check_at(
current_next_check_at,
check_interval_minutes,
now,
shard_offset_seconds=shard_offset_seconds,
)
- return next_calendar_check_time(
- to_recurrence_interval(recurrence_unit),
+ interval = to_recurrence_interval(recurrence_unit)
+ candidate = next_calendar_check_time(
+ interval,
now=now,
tz_name=tz_name,
next_check_at=current_next_check_at,
- alert_id=configuration_id,
+ alert_id=schedule_key if schedule_key is not None else configuration_id,
schedule_start_time=anchor_time,
)
+ if schedule_key is None:
+ return candidate
+ return snap_out_of_blocked_windows(
+ candidate, tz_name, blocked_windows, interval=interval, alert_id=schedule_key, anchored=anchor_time is not None
+ )
+
+
+def snap_out_of_blocked_windows(
+ candidate_utc: datetime,
+ tz_name: str,
+ windows: list[BlockedWindow] | None,
+ *,
+ interval: CalendarInterval,
+ alert_id: UUID | str,
+ anchored: bool,
+) -> datetime:
+ """The first unblocked minute at or after `candidate_utc`, the way an insight alert's check moves.
+
+ A snap lands on the end of a quiet window, which is usually on the hour, so every alert behind
+ the same window would check at once. An alert without a chosen start time adds its offset to
+ spread them again, unless the offset lands back in a window.
+ """
+ if not windows:
+ return candidate_utc
+ normalized = candidate_utc.astimezone(UTC).replace(second=0, microsecond=0)
+ snapped = scan_next_unblocked_utc(normalized, tz_name, windows)
+ if snapped is None:
+ # Every minute in the scan is blocked. Try again a day later rather than never.
+ return normalized + timedelta(days=1)
+ if snapped == normalized or anchored:
+ return snapped
+ offset_snap = snapped + alert_check_offset(interval, alert_id)
+ return snapped if is_utc_datetime_blocked(offset_snap, tz_name, windows) else offset_snap
# --- Quiet hours (blocked local time windows) ---
diff --git a/products/alerts_platform/backend/logic/platform_lifecycle.py b/products/alerts_platform/backend/logic/platform_lifecycle.py
index a5c341d15434..856df7fef803 100644
--- a/products/alerts_platform/backend/logic/platform_lifecycle.py
+++ b/products/alerts_platform/backend/logic/platform_lifecycle.py
@@ -45,6 +45,7 @@ from products.alerts_platform.backend.facade.scheduling import (
advance_schedule,
compute_shard_offset_seconds,
compute_team_shard_offset_seconds,
+ parse_blocked_windows_tuples,
to_recurrence_interval,
validate_and_normalize_schedule_start_time,
)
@@ -614,6 +615,8 @@ def record_outcomes(team_id: int, outcomes: Sequence[PlatformAlertOutcome], now:
now=now,
configuration_id=configuration.id,
shard_offset_seconds=_shard_offset_seconds(configuration, outcome),
+ schedule_key=outcome.schedule_key,
+ blocked_windows=parse_blocked_windows_tuples(configuration.schedule_restriction),
)
# Batched, because a grouped check writes a row per group rather than one per configuration.