Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(metrics): evaluate metrics alerts through snuffle (#125092)

Jimmy Zhu4375f9f4fa2c
diff --git a/posthog/api/snuffle_proxy.py b/posthog/api/snuffle_proxy.py
index 536fe000640c..0d7275d0d99f 100644
--- a/posthog/api/snuffle_proxy.py
+++ b/posthog/api/snuffle_proxy.py
@@ -126,6 +126,7 @@ def snuffle_request(
params: dict[str, list[str]] | dict[str, str] | None = None,
data: dict[str, list[str]] | dict[str, str] | None = None,
accept: str = "application/json",
+ timeout: float | None = None,
) -> requests.Response:
"""Send one request to Snuffle as `team_id` and debit the bytes it read from the query budget.
@@ -144,7 +145,7 @@ def snuffle_request(
data=data,
headers=headers,
auth=auth,
- timeout=settings.SNUFFLE_APM_TIMEOUT_SECONDS,
+ timeout=settings.SNUFFLE_APM_TIMEOUT_SECONDS if timeout is None else timeout,
)
bytes_read = _read_bytes_from(upstream)
if bytes_read is not None:
diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.py
index 642190211ae3..50ead805045e 100644
--- a/posthog/management/commands/start_temporal_worker.py
+++ b/posthog/management/commands/start_temporal_worker.py
@@ -275,6 +275,8 @@ from products.messaging.backend.facade.temporal import (
)
from products.metrics.backend.facade.temporal import (
ACTIVITIES as METRICS_ACTIVITIES,
+ METRICS_ALERT_EVALUATION_ACTIVITIES,
+ METRICS_ALERT_EVALUATION_WORKFLOWS,
WORKFLOWS as METRICS_WORKFLOWS,
)
from products.notebooks.backend.facade.temporal import (
@@ -680,12 +682,14 @@ _task_queue_specs = [
+ LOGS_SOURCE_EVALUATION_WORKFLOWS
+ INSIGHT_PLATFORM_EVALUATION_WORKFLOWS
+ BILLING_PLATFORM_EVALUATION_WORKFLOWS
- + WAREHOUSE_ALERT_EVALUATION_WORKFLOWS,
+ + WAREHOUSE_ALERT_EVALUATION_WORKFLOWS
+ + METRICS_ALERT_EVALUATION_WORKFLOWS,
ALERTS_PLATFORM_EVALUATION_ACTIVITIES
+ LOGS_SOURCE_EVALUATION_ACTIVITIES
+ INSIGHT_PLATFORM_EVALUATION_ACTIVITIES
+ BILLING_PLATFORM_EVALUATION_ACTIVITIES
- + WAREHOUSE_ALERT_EVALUATION_ACTIVITIES,
+ + WAREHOUSE_ALERT_EVALUATION_ACTIVITIES
+ + METRICS_ALERT_EVALUATION_ACTIVITIES,
),
(
settings.ALERTS_PLATFORM_DELIVERY_TASK_QUEUE,
diff --git a/posthog/management/commands/test/test_run_async_migrations.py b/posthog/management/commands/test/test_run_async_migrations.py
index 0e1eea4c019e..b51122d503bb 100644
--- a/posthog/management/commands/test/test_run_async_migrations.py
+++ b/posthog/management/commands/test/test_run_async_migrations.py
@@ -84,3 +84,10 @@ def test_complete_noop_migrations(caplog):
@pytest.fixture(autouse=True)
def set_log_level(caplog):
caplog.set_level(logging.INFO)
+
+
[email protected](autouse=True)
+def empty_events_table():
+ # The backfill migrations check every event in ClickHouse, and earlier tests in a shard can
+ # leave events without person data, which fails their postcheck.
+ sync_execute(TRUNCATE_EVENTS_TABLE_SQL())
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py
index d78d8a727f7a..028213a845a8 100644
--- a/posthog/settings/temporal.py
+++ b/posthog/settings/temporal.py
@@ -467,6 +467,15 @@ if ALERTS_PLATFORM_LOGS_MAX_INFLIGHT_BATCHES <= 0:
raise ImproperlyConfigured(
"ALERTS_PLATFORM_LOGS_MAX_INFLIGHT_BATCHES must be a positive integer, or no platform logs check starts"
)
+# The platform's metrics batches allowed to run at once, across every team. A batch runs its Snuffle
+# queries one after another, so this bounds the concurrent metrics alert queries.
+ALERTS_PLATFORM_METRICS_MAX_INFLIGHT_BATCHES: int = get_from_env(
+ "ALERTS_PLATFORM_METRICS_MAX_INFLIGHT_BATCHES", 5, type_cast=int
+)
+if ALERTS_PLATFORM_METRICS_MAX_INFLIGHT_BATCHES <= 0:
+ raise ImproperlyConfigured(
+ "ALERTS_PLATFORM_METRICS_MAX_INFLIGHT_BATCHES must be a positive integer, or no platform metrics check starts"
+ )
# Insight alert checks allowed to run against ClickHouse at once, across every team.
ALERTS_MAX_INFLIGHT_EVALUATIONS: int = get_from_env("ALERTS_MAX_INFLIGHT_EVALUATIONS", 40, type_cast=int)
if ALERTS_MAX_INFLIGHT_EVALUATIONS <= 0:
diff --git a/products/alerts_platform/backend/temporal/sources.py b/products/alerts_platform/backend/temporal/sources.py
index 2a939b56ac12..4d274c1a746f 100644
--- a/products/alerts_platform/backend/temporal/sources.py
+++ b/products/alerts_platform/backend/temporal/sources.py
@@ -75,6 +75,13 @@ SOURCE_BINDINGS: dict[SourceKind, SourceBinding] = {
evaluation_timeout=dt.timedelta(seconds=60),
discovery_limit=DISCOVERY_LIMIT_PER_SOURCE,
),
+ # One PromQL instant query per check through Snuffle. A batch stops starting queries at its budget.
+ SourceKind.METRICS: SourceBinding(
+ workflow="metrics-alert-evaluate",
+ task_queue=settings.ALERTS_PLATFORM_EVALUATION_TASK_QUEUE,
+ evaluation_timeout=dt.timedelta(seconds=120),
+ discovery_limit=DISCOVERY_LIMIT_PER_SOURCE,
+ ),
}
diff --git a/products/metrics/backend/alerts/destinations.py b/products/metrics/backend/alerts/destinations.py
index 32c5845caea0..5e4f0778a94e 100644
--- a/products/metrics/backend/alerts/destinations.py
+++ b/products/metrics/backend/alerts/destinations.py
@@ -7,7 +7,12 @@ find who to tell and words the message itself, so their templates never run.
from dataclasses import replace
from typing import Final, Literal
-from products.alerts_platform.backend.facade.contracts import AlertDestinationAction, EventKindSpec, IncidentAction
+from products.alerts_platform.backend.facade.contracts import (
+ AlertDestinationAction,
+ AlertEventKind,
+ EventKindSpec,
+ IncidentAction,
+)
EventKind = Literal["firing", "resolved", "errored", "broken", "incident_opened", "incident_closed"]
@@ -84,6 +89,12 @@ EVENT_KIND_CONFIG: Final[dict[EventKind, EventKindSpec]] = {
EVENT_IDS: Final[tuple[str, ...]] = tuple(spec.event_id for spec in EVENT_KIND_CONFIG.values())
+# Which event id each announced kind's destinations filter on. Resolved here because the platform
+# imports no source.
+EVENT_IDS_BY_KIND: Final[dict[str, str]] = {
+ key: spec.event_id for key, spec in EVENT_KIND_CONFIG.items() if key in set(AlertEventKind)
+}
+
EVENT_IDS_BY_INCIDENT_ACTION: Final[dict[str, str]] = {
spec.incident_action.value: spec.event_id for spec in EVENT_KIND_CONFIG.values() if spec.incident_action
}
diff --git a/products/metrics/backend/alerts/evaluation.py b/products/metrics/backend/alerts/evaluation.py
new file mode 100644
index 000000000000..d1dc1a647dce
--- /dev/null
+++ b/products/metrics/backend/alerts/evaluation.py
@@ -0,0 +1,479 @@
+"""Evaluation of due metrics alerts on the shared alerts platform.
+
+Each check sends its PromQL expression to Snuffle as one instant query, at the check's due time
+held back to where the team's ingestion has reached. Each series the query returns is compared with the
+condition. Under grouping by result labels, each distinct set of grouping label values is a group.
+Without grouping, the alert has one group that breaches when any series does.
+
+A series the query no longer returns keeps its group's state, and so does a series whose value is
+not a number. Prometheus resolves a missing series, but a team's samples spread over ingestion
+partitions, and the freshness read cannot tell that every partition has caught up. So a missing
+series could be one a lagging partition has not written yet, and resolving it would read lag as
+recovery. The hold is bounded: a group missing for `ABSENT_RESOLVE_AFTER` while the team's
+ingestion is fresh resolves, which no lag that leaves the team fresh explains, and which frees its
+slot under `max_instances`. A watermark covering every partition would let it resolve at once.
+
+This reads and decides. It writes nothing, so a retried attempt reads the same due checks and makes
+the same decisions. The platform records the outcomes in its own activity.
+"""
+
+import json
+import math
+import time
+import operator
+from collections.abc import Callable, Sequence
+from datetime import datetime, timedelta
+from typing import Any, Final
+from uuid import uuid4
+
+from django.conf import settings
+
+import structlog
+from pydantic import ValidationError
+
+from posthog.dataclasses import frozen
+from posthog.models import Team
+from posthog.temporal.alerts.admission import hold_evaluation_slot, release_evaluation_slot
+
+from products.alerts.backend.facade.destinations import configured_destination_template_ids
+from products.alerts_platform.backend.facade.api import due_checks, slot_of
+from products.alerts_platform.backend.facade.contracts import (
+ AlertDeliveryRequest,
+ GroupingMode,
+ InstanceCheckState,
+ PlatformAlertCheckInput,
+ PlatformAlertOutcome,
+ SourceBatchEvaluation,
+ SourceKind,
+)
+from products.alerts_platform.backend.facade.enums import PlatformAlertState
+from products.alerts_platform.backend.facade.grouping import (
+ GroupObservation,
+ bounded_grouping_key,
+ decide_groups,
+ delivery_parts,
+ muted_value,
+ not_breaching,
+)
+from products.alerts_platform.backend.facade.lifecycle import (
+ NOTIFICATION_EVENT_KINDS,
+ PLATFORM_DEFAULT_ALERT_POLICY,
+ AlertSnapshot,
+ AlertState,
+ CheckInput,
+ NotificationAction,
+ decide_firing_episode,
+ decide_incident_action,
+ evaluate_alert_check,
+)
+from products.alerts_platform.backend.facade.payload import MAX_BATCH_PAYLOAD_BYTES, payload_bytes
+from products.alerts_platform.backend.facade.platform_metrics import (
+ increment_checks,
+ increment_checks_deferred,
+ increment_state_transition,
+ record_batch_duration,
+ safe_record,
+)
+from products.alerts_platform.backend.facade.scheduling import is_utc_datetime_blocked, parse_blocked_windows_tuples
+from products.metrics.backend.alerts.destinations import EVENT_IDS_BY_INCIDENT_ACTION, EVENT_IDS_BY_KIND
+from products.metrics.backend.alerts.freshness import evaluation_time, fetch_metrics_freshness
+from products.metrics.backend.alerts.source_config import MetricsPlatformAlertCondition, MetricsPlatformAlertConfig
+from products.metrics.backend.alerts.wording import alert_url
+from products.metrics.backend.promql import PromQLQueryError, snuffle_payload
+
+logger = structlog.get_logger(__name__)
+
+SOURCE: Final = SourceKind.METRICS
+POLICY: Final = PLATFORM_DEFAULT_ALERT_POLICY
+
+# The batch's query time, which `EVALUATE_START_TO_CLOSE` in the workflow sits above. Checks run one
+# after another, and a check the budget has no room for keeps its due time for a later tick.
+BATCH_QUERY_BUDGET_SECONDS: Final = 25
+# No single query may spend the whole budget, or the first slow one starves the rest.
+MAX_QUERY_SECONDS: Final = 10
+# Below this there is no point starting another query.
+MIN_QUERY_SECONDS: Final = 2
+FRESHNESS_QUERY_SECONDS: Final = 5
+INFLIGHT_KEY: Final = "alerts:platform:metrics:batches:inflight"
+# Outlasts the evaluation activity's run, so a slot frees before its batch ends only when a worker
+# died mid-batch.
+BATCH_SLOT_LEASE_SECONDS: Final = 90
+# Ten times the most groups a configuration can hold. A query past it is refused before its series
+# are judged, so one high-cardinality expression cannot exhaust the worker.
+MAX_SERIES: Final = 1000
+# Far past any partition lag that leaves the team fresh, and short enough that groups which stopped
+# reporting cannot hold every instance slot.
+ABSENT_RESOLVE_AFTER: Final = timedelta(hours=1)
+
+_UNEXPECTED_ERROR: Final = "Alert check failed unexpectedly. PostHog has been notified."
+_INVALID_CONFIG: Final = "The alert's configuration is no longer valid. Edit the alert to fix it."
+
+_COMPARE: Final[dict[str, Callable[[float, float], bool]]] = {
+ "gt": operator.gt,
+ "gte": operator.ge,
+ "lt": operator.lt,
+ "lte": operator.le,
+ "eq": operator.eq,
+ "ne": operator.ne,
+}
+
+Decision = tuple[PlatformAlertOutcome, AlertDeliveryRequest | None]
+
+
+@frozen
+class _Sample:
+ labels: dict[str, str]
+ # None for a sample that is not a number, such as the result of 0/0.
+ value: float | None
+
+
+def evaluate_metrics_alert_batch(team_id: int, slot: str, cutoff: datetime) -> SourceBatchEvaluation:
+ """Evaluates one batch key of metrics alerts against the tick's cutoff."""
+ started_at = time.monotonic()
+ checks = due_checks(team_id, SOURCE.value, slot, cutoff)
+ if not checks:
+ return SourceBatchEvaluation(outcomes=(), deliveries=())
+ team = Team.objects.filter(id=team_id).first()
+ if team is None:
+ return SourceBatchEvaluation(outcomes=(), deliveries=())
+
+ # One slot per batch, because a batch runs its queries one after another, so the batches in
+ # flight bound the concurrent metrics queries across the fleet. A batch the pool turns away
+ # records nothing, so its checks stay due and a later tick finds them again.
+ # Unique per execution: a redispatch of a batch whose old query still runs must hold its own slot.
+ member = f"{team_id}:{slot}:{uuid4().hex}"
+ held_until = hold_evaluation_slot(
+ member,
+ limit=settings.ALERTS_PLATFORM_METRICS_MAX_INFLIGHT_BATCHES,
+ lease_seconds=BATCH_SLOT_LEASE_SECONDS,
+ key=INFLIGHT_KEY,
+ )
+ if held_until is None:
+ logger.info("Deferred a platform metrics batch over the pool limit", team_id=team_id, slot=slot)
+ safe_record(increment_checks_deferred, SOURCE.value, len(checks))
+ return SourceBatchEvaluation(outcomes=(), deliveries=(), omitted=len(checks))
+ try:
+ return _evaluate_admitted(team, slot, checks, cutoff=cutoff, started_at=started_at)
+ finally:
+ release_evaluation_slot(member, held_until=held_until, key=INFLIGHT_KEY)
+
+
+def _evaluate_admitted(
+ team: Team, slot: str, checks: Sequence[PlatformAlertCheckInput], *, cutoff: datetime, started_at: float
+) -> SourceBatchEvaluation:
+ freshness = _freshness(team)
+ deadline = started_at + BATCH_QUERY_BUDGET_SECONDS
+ decided: list[Decision] = []
+ for check in checks:
+ remaining = deadline - time.monotonic()
+ if remaining < MIN_QUERY_SECONDS:
+ break
+ at = evaluation_time(check.next_check_at or cutoff, freshness)
+ decided.append(
+ _evaluate(
+ team,
+ check,
+ at=at,
+ now=cutoff,
+ ingestion_fresh=freshness is not None,
+ query_seconds=min(MAX_QUERY_SECONDS, remaining),
+ muted=_in_quiet_hours(check, cutoff, team.timezone),
+ )
+ )
+ evaluation = _collect(decided, team_id=team.id, slot=slot, past_budget=len(checks) - len(decided))
+ safe_record(record_batch_duration, SOURCE.value, int((time.monotonic() - started_at) * 1000))
+ return evaluation
+
+
+def _freshness(team: Team) -> datetime | None:
+ try:
+ return fetch_metrics_freshness(team, max_execution_time=FRESHNESS_QUERY_SECONDS)
+ except Exception:
+ # Without a freshness the batch still runs, at its due time.
+ logger.exception("Failed to fetch the metrics ingestion freshness", team_id=team.id)
+ return None
+
+
+def _in_quiet_hours(check: PlatformAlertCheckInput, now: datetime, tz_name: str) -> bool:
+ try:
+ return is_utc_datetime_blocked(now, tz_name, parse_blocked_windows_tuples(check.schedule_restriction))
+ except Exception:
+ # A restriction that cannot be read must not hold back a breach.
+ logger.exception("Unparseable schedule restriction on a metrics alert", check_id=str(check.id))
+ return False
+
+
+def _evaluate(
+ team: Team,
+ check: PlatformAlertCheckInput,
+ *,
+ at: datetime,
+ now: datetime,
+ ingestion_fresh: bool,
+ query_seconds: float,
+ muted: bool,
+) -> Decision:
+ started_at = time.monotonic()
+ try:
+ config = MetricsPlatformAlertConfig.model_validate(check.source_config)
+ except ValidationError:
+ # Saved through the same model, so this is a config an older version accepted.
+ return _failed(check, _INVALID_CONFIG, transient=False, now=now, muted=muted)
+ try:
+ samples = _query(team, config.query, at=at, timeout_seconds=query_seconds)
+ except PromQLQueryError as error:
+ return _failed(check, str(error), transient=error.transient, now=now, muted=muted)
+ except Exception:
+ logger.exception("Failed to evaluate a metrics alert", check_id=str(check.id))
+ return _failed(check, _UNEXPECTED_ERROR, transient=True, now=now, muted=muted)
+ duration_ms = int((time.monotonic() - started_at) * 1000)
+
+ observations, unknown = _observations(check, config.condition, samples)
+ decision = decide_groups(
+ check,
+ observations,
+ snapshot_of=lambda instance, _prior: _snapshot(check, instance),
+ policy=POLICY,
+ now=now,
+ muted=muted,
+ absent=lambda key: _absent(check, key, unknown=unknown, now=now, ingestion_fresh=ingestion_fresh),
+ )
+ for verdict in decision.verdicts:
+ safe_record(increment_checks, SOURCE.value, verdict.outcome.notification.value)
+ if verdict.snapshot.state != verdict.outcome.new_state:
+ safe_record(
+ increment_state_transition, SOURCE.value, verdict.snapshot.state.value, verdict.outcome.new_state.value
+ )
+ recorded = decision.as_outcome(
+ configuration_id=check.id,
+ evaluation_key=_evaluation_key(check, now),
+ grouping=check.grouping,
+ query_duration_ms=duration_ms,
+ results_cut=False,
+ )
+ moves = {
+ verdict.group.grouping_key: (verdict.snapshot.state, verdict.outcome.new_state) for verdict in decision.verdicts
+ }
+ sends_messages = any(verdict.outcome.notification != NotificationAction.NONE for verdict in decision.verdicts)
+ return recorded, _request(check, recorded, moves, sends_messages=sends_messages)
+
+
+def _query(team: Team, expr: str, *, at: datetime, timeout_seconds: float) -> list[_Sample]:
+ """The series one PromQL instant query returns at `at`."""
+ payload = snuffle_payload(
+ team,
+ "POST",
+ "/api/v1/query",
+ data={"query": expr, "time": str(at.timestamp()), "timeout": f"{int(timeout_seconds)}s"},
+ timeout=timeout_seconds,
+ )
+
+ data = payload.get("data") or {}
+ result_type = data.get("resultType")
+ result: Any = data.get("result")
+ if result_type == "scalar":
+ return [_Sample(labels={}, value=_number(result[1]))] if result else []
+ if result_type != "vector":
+ raise PromQLQueryError(f"A metrics alert query must return an instant vector or a scalar, not a {result_type}.")
+ if len(result or []) > MAX_SERIES:
+ raise PromQLQueryError(
+ f"The query returned more than {MAX_SERIES} series. Aggregate it, for example with sum by (...)."
+ )
+ samples: list[_Sample] = []
+ for series in result or []:
+ # `__name__` stays, so an alert can group by it.
+ labels = {str(key): str(value) for key, value in (series.get("metric") or {}).items()}
+ sample = series.get("value") or [None, None]
+ samples.append(_Sample(labels=labels, value=_number(sample[1])))
+ return samples
+
+
+def _number(raw: Any) -> float | None:
+ try:
+ value = float(raw)
+ except (TypeError, ValueError):
+ return None
+ return value if math.isfinite(value) else None
+
+
+def _breached(condition: MetricsPlatformAlertCondition, value: float | None) -> bool | None:
+ """Whether a sample breaches, or None when its value reaches no verdict."""
+ if condition.kind == "present":
+ return True
+ # The config model requires both for a threshold condition.
+ if value is None or condition.operator is None or condition.threshold is None:
+ return None
+ return _COMPARE[condition.operator](value, condition.threshold)
+
+
+def _absent(
+ check: PlatformAlertCheckInput, key: str, *, unknown: frozenset[str], now: datetime, ingestion_fresh: bool
+) -> GroupObservation | None:
+ """A missing group's verdict: None holds its state, as the module docstring explains, until the hold runs out."""
+ last_seen_at = check.instance(key).last_seen_at
+ if key in unknown or not ingestion_fresh or last_seen_at is None or now - last_seen_at < ABSENT_RESOLVE_AFTER:
+ return None
+ return not_breaching(key)
+
+
+def _observations(
+ check: PlatformAlertCheckInput, condition: MetricsPlatformAlertCondition, samples: Sequence[_Sample]
+) -> tuple[list[GroupObservation], frozenset[str]]:
+ """One observation per sample, leaving out the groups held without a verdict, and those groups' keys.
+
+ A group with a sample that has no number is held, unless another of its samples breaches. A healthy
+ sample beside it does not show that the unknown one recovered.
+ """
+ by_labels = check.grouping.mode == GroupingMode.BY_RESULT_LABELS
+ observations: list[GroupObservation] = []
+ unknown: set[str] = set()
+ for sample in samples:
+ if by_labels:
+ labels = {key: sample.labels.get(key, "") for key in check.grouping.keys}
+ else:
+ labels = {key: value for key, value in sample.labels.items() if key != "__name__"}
+ key = _grouping_key_of(labels) if by_labels else ""
+ breached = _breached(condition, sample.value)
+ if breached is None:
+ unknown.add(key)
+ continue
+ observations.append(
+ GroupObservation(grouping_key=key, current_breached=breached, value=sample.value, labels=labels)
+ )
+ breaching = {observation.grouping_key for observation in observations if observation.current_breached}
+ held = frozenset(unknown - breaching)
+ kept = [observation for observation in observations if observation.grouping_key not in held]
+ # An announced firing the query still returns is reported as still firing, which notifies nothing,
+ # so the platform records it as seen and its absence hold counts from this check. An unannounced
+ # one is left out, because a breach would announce it on a value that reached no verdict.
+ kept.extend(GroupObservation(grouping_key=key, current_breached=True) for key in held if _announced(check, key))
+ return kept, held
+
+
+def _announced(check: PlatformAlertCheckInput, key: str) -> bool:
+ """Whether a group fires and has announced it, the same test the lifecycle machine applies."""
+ instance = check.instance(key)
+ return (
+ instance.state == PlatformAlertState.FIRING.value
+ and instance.firing_started_at is not None
+ and instance.last_notified_at is not None
+ and instance.last_notified_at >= instance.firing_started_at
+ )
+
+
+def _grouping_key_of(labels: dict[str, str]) -> str:
+ """The instance key for one group's label values."""
+ return bounded_grouping_key(json.dumps(labels, sort_keys=True, separators=(",", ":")))
+
+
+def _evaluation_key(check: PlatformAlertCheckInput, now: datetime) -> str:
+ return f"slot:{slot_of(check.next_check_at, now)}"
+
+
+def _snapshot(check: PlatformAlertCheckInput, instance: InstanceCheckState) -> AlertSnapshot:
+ return AlertSnapshot(
+ state=AlertState(instance.state),
+ cooldown=timedelta(minutes=check.cooldown_minutes),
+ last_notified_at=instance.last_notified_at,
+ snooze_until=instance.snooze_until,
+ consecutive_failures=check.consecutive_failures,
+ firing_started_at=instance.firing_started_at,
+ )
+
+
+def _failed(check: PlatformAlertCheckInput, message: str, *, transient: bool, now: datetime, muted: bool) -> Decision:
+ """A check that reached no verdict. The machine counts it toward BROKEN unless it is transient."""
+ snapshot = _snapshot(check, check.instance())
+ outcome = evaluate_alert_check(
+ snapshot,
+ CheckInput(threshold_breached=False, error_message=message, is_transient_error=transient, muted=muted),
+ now,
+ policy=POLICY,
+ )
+ safe_record(increment_checks, SOURCE.value, outcome.notification.value)
+ recorded = PlatformAlertOutcome.ungrouped(
+ configuration_id=check.id,
+ evaluation_key=_evaluation_key(check, now),
+ consecutive_failures=outcome.consecutive_failures,
+ kind=NOTIFICATION_EVENT_KINDS[outcome.notification],
+ new_state=outcome.new_state.value,
+ notified=outcome.update_last_notified_at,
+ failed=True,
+ firing_episode=decide_firing_episode(snapshot, outcome, now, policy=POLICY),
+ muted_notification=muted_value(outcome),
+ error_message=outcome.error_message,
+ disable=outcome.disable,
+ )
+ return recorded, _request(
+ check,
+ recorded,
+ {"": (snapshot.state, outcome.new_state)},
+ sends_messages=outcome.notification != NotificationAction.NONE,
+ )
+
+
+def _request(
+ check: PlatformAlertCheckInput,
+ recorded: PlatformAlertOutcome,
+ moves: dict[str, tuple[AlertState, AlertState]],
+ *,
+ sends_messages: bool,
+) -> AlertDeliveryRequest | None:
+ """The delivery for a recorded outcome, or None when it neither announces nor moves a firing.
+
+ A firing that opens or closes needs a delivery even when cooldown or mute held the message,
+ because a paging destination needs one resolve for every trigger.
+ """
+ destination_alert_id = str(check.destination_alert_id)
+ actions = {
+ key: action
+ for key, (before, after) in moves.items()
+ if (action := decide_incident_action(before, after, policy=POLICY)) is not None
+ }
+ if actions and not configured_destination_template_ids(
+ team_id=check.team_id,
+ alert_id=destination_alert_id,
+ allowed_event_ids=tuple(EVENT_IDS_BY_INCIDENT_ACTION.values()),
+ ):
+ actions = {}
+ if not sends_messages and not actions:
+ return None
+ return AlertDeliveryRequest(
+ source=SOURCE,
+ team_id=check.team_id,
+ configuration_id=str(check.id),
+ evaluation_key=recorded.evaluation_key,
+ destination_alert_id=destination_alert_id,
+ event_ids_by_kind=EVENT_IDS_BY_KIND,
+ event_ids_by_incident_action=EVENT_IDS_BY_INCIDENT_ACTION,
+ incident_actions=actions,
+ sends_messages=sends_messages,
+ source_alert_url=alert_url(check.team_id, check.id),
+ owned_by_platform=check.owned_by_platform,
+ overflowed=recorded.overflowed,
+ overflow_is_lower_bound=recorded.overflow_is_lower_bound,
+ parts=delivery_parts(recorded, actions),
+ )
+
+
+def _collect(decided: Sequence[Decision], *, team_id: int, slot: str, past_budget: int) -> SourceBatchEvaluation:
+ """The batch, held to one payload. A check past the query budget or the payload keeps its due time."""
+ outcomes: list[PlatformAlertOutcome] = []
+ deliveries: list[AlertDeliveryRequest] = []
+ deferred = 0
+ spent = 0
+ for outcome, delivery in decided:
+ size = payload_bytes(outcome) + (payload_bytes(delivery) if delivery is not None else 0)
+ if spent + size > MAX_BATCH_PAYLOAD_BYTES:
+ deferred += 1
+ continue
+ spent += size
+ outcomes.append(outcome)
+ if delivery is not None:
+ deliveries.append(delivery)
+ if past_budget or deferred:
+ logger.warning(
+ "Deferred metrics alert checks", team_id=team_id, slot=slot, past_budget=past_budget, over_payload=deferred
+ )
+ safe_record(increment_checks_deferred, SOURCE.value, past_budget + deferred)
+ return SourceBatchEvaluation(outcomes=tuple(outcomes), deliveries=tuple(deliveries), omitted=past_budget + deferred)
diff --git a/products/metrics/backend/alerts/freshness.py b/products/metrics/backend/alerts/freshness.py
new file mode 100644
index 000000000000..5a55b1ea76d9
--- /dev/null
+++ b/products/metrics/backend/alerts/freshness.py
@@ -0,0 +1,98 @@
+"""How far metrics ingestion has reached for a team, so an alert never reads a window still being written.
+
+A series the pipeline has not written yet is absent from an instant query, and an absent series
+resolves its group. So a consumer lag would read as recovery. The newest point the team's services
+sent shows how far ingestion has reached: a lag delays every service at once.
+"""
+
+import datetime as dt
+from contextlib import AbstractContextManager
+from typing import Final
+from zoneinfo import ZoneInfo
+
+from posthog.schema import HogQLQueryModifiers
+
+from posthog.hogql.constants import HogQLGlobalSettings
+from posthog.hogql.parser import parse_select
+from posthog.hogql.query import HogQLQueryExecutor
+from posthog.hogql.visitor import clone_expr
+
+from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.client.connection import (
+ ClickHouseUser,
+ Workload,
+ connection_creds,
+ get_kwargs_for_client,
+ is_file_backed_user,
+ make_ch_pool,
+)
+from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
+from posthog.models import Team
+
+# The shared pool's socket reads never time out, so a stalled connection would outlive the activity.
+# The margin over the query's own limit lets the server's timeout error arrive first.
+_TRANSPORT_MARGIN_SECONDS: Final = 5
+
+# Reads the `service_freshness_by_hour` projection of `metrics4_samples`, one row per service-hour.
+# The sender sets point times, so a service dated ahead is capped at now rather than dropped, which
+# would let it push live services out and rewind every check. A team over 15 minutes behind gets none.
+_TEAM_FRESHNESS_QUERY: Final = parse_select(
+ """
+ SELECT least(maxOrNull(service_newest), now()) FROM (
+ SELECT service_name, max(arrayMax(timestamp_arr)) AS service_newest
+ FROM posthog.metric_samples
+ WHERE time_bucket >= toStartOfHour(now() - toIntervalHour(1)) AND time_bucket <= now() + toIntervalHour(1)
+ GROUP BY service_name
+ )
+ WHERE service_newest >= now() - toIntervalMinute(15)
+ """
+)
+
+
+def fetch_metrics_freshness(team: Team, *, max_execution_time: int) -> dt.datetime | None:
+ """The newest point the team's ingestion has written, or None when it is more than 15 minutes behind."""
+ tag_queries(product=Product.METRICS, feature=Feature.ALERTING, source="metrics_alert", team_id=str(team.id))
+ sql, context = HogQLQueryExecutor(
+ query_type="metrics_alert_freshness",
+ # The executor edits the node it is given, and this one is shared by every call.
+ query=clone_expr(_TEAM_FRESHNESS_QUERY, True),
+ team=team,
+ workload=Workload.LOGS,
+ modifiers=HogQLQueryModifiers(convertToProjectTimezone=False),
+ settings=HogQLGlobalSettings(max_execution_time=max_execution_time, timeout_overflow_mode="throw"),
+ ).generate_clickhouse_sql()
+ with _bounded_logs_client(max_execution_time + _TRANSPORT_MARGIN_SECONDS) as client:
+ rows = sync_execute(
+ sql, context.values, workload=Workload.LOGS, team_id=team.id, readonly=True, sync_client=client
+ )
+ if not rows or rows[0][0] is None:
+ return None
+ freshness: dt.datetime = rows[0][0]
+ return freshness if freshness.tzinfo is not None else freshness.replace(tzinfo=ZoneInfo("UTC"))
+
+
+def _bounded_logs_client(send_receive_timeout: int) -> AbstractContextManager:
+ """A logs cluster client whose socket reads and writes time out, as the shared pool's do not.
+
+ Mirrors the file-backed branch of `get_pool`, so a token-authenticated user stays token-aware.
+ `make_ch_pool` is cached on its arguments, so every call reuses one pool.
+ """
+ kwargs = get_kwargs_for_client(workload=Workload.LOGS, ch_user=ClickHouseUser.DEFAULT)
+ creds = connection_creds(Workload.LOGS, ClickHouseUser.DEFAULT)
+ if creds is not None and is_file_backed_user(creds, kwargs.get("user")):
+ kwargs.pop("password", None)
+ pool = make_ch_pool(
+ send_receive_timeout=send_receive_timeout, credential_provider=creds.read_password, **kwargs
+ )
+ else:
+ pool = make_ch_pool(send_receive_timeout=send_receive_timeout, **kwargs)
+ return pool.get_client()
+
+
+def evaluation_time(due_at: dt.datetime, freshness: dt.datetime | None) -> dt.datetime:
+ """The instant a check reads: its due time, held back to where ingestion has reached.
+
+ Anchored on the due time rather than the clock, so a retried check sends the same query. Without
+ a freshness the check reads at its due time, and the caller treats a missing series as unknown.
+ """
+ return due_at if freshness is None else min(due_at, freshness)
diff --git a/products/metrics/backend/facade/temporal.py b/products/metrics/backend/facade/temporal.py
index fdd23f9bf047..c222c4e3839a 100644
--- a/products/metrics/backend/facade/temporal.py
+++ b/products/metrics/backend/facade/temporal.py
@@ -2,8 +2,18 @@
activities and schedule."""
from products.metrics.backend.temporal import ACTIVITIES, WORKFLOWS
+from products.metrics.backend.temporal.alert_evaluate import (
+ METRICS_ALERT_EVALUATION_ACTIVITIES,
+ METRICS_ALERT_EVALUATION_WORKFLOWS,
+)
from products.metrics.backend.temporal.schedule import (
create_suggested_dashboards_discovery_schedule as create_suggested_dashboards_discovery_schedule,
)
-__all__ = ["ACTIVITIES", "WORKFLOWS", "create_suggested_dashboards_discovery_schedule"]
+__all__ = [
+ "ACTIVITIES",
+ "METRICS_ALERT_EVALUATION_ACTIVITIES",
+ "METRICS_ALERT_EVALUATION_WORKFLOWS",
+ "WORKFLOWS",
+ "create_suggested_dashboards_discovery_schedule",
+]
diff --git a/products/metrics/backend/promql.py b/products/metrics/backend/promql.py
index 5791a5f8c5cc..590f37c3d06c 100644
--- a/products/metrics/backend/promql.py
+++ b/products/metrics/backend/promql.py
@@ -32,7 +32,14 @@ _SHARED_HISTOGRAM_QUANTILES = re.compile(
class PromQLQueryError(ValueError):
- """A PromQL query that Snuffle rejected or could not run. The message is safe to show."""
+ """A PromQL query that Snuffle rejected or could not run. The message is safe to show.
+
+ `transient` says the backend failed rather than the query, so the same query can succeed later.
+ """
+
+ def __init__(self, message: str, *, transient: bool = False) -> None:
+ super().__init__(message)
+ self.transient = transient
def _sample_value(raw: Any) -> float | None:
@@ -79,25 +86,30 @@ def snuffle_payload(
*,
data: dict[str, str] | None = None,
params: dict[str, list[str]] | None = None,
+ timeout: float | None = None,
) -> dict[str, Any]:
"""Send one Prometheus API request to Snuffle and return its JSON body. Errors are safe to show."""
try:
- response = snuffle_request(team.pk, method, path, data=data, params=params)
+ response = snuffle_request(team.pk, method, path, data=data, params=params, timeout=timeout)
except SnuffleNotConfiguredError as exc:
- raise PromQLQueryError("PromQL queries are not available on this PostHog instance.") from exc
+ # Deployment config, not the query, so an alert waiting on it is not counted as broken.
+ raise PromQLQueryError("PromQL queries are not available on this PostHog instance.", transient=True) from exc
except requests.Timeout as exc:
- raise PromQLQueryError("The PromQL query timed out. Try a shorter date range.") from exc
+ raise PromQLQueryError("The PromQL query timed out. Try a shorter date range.", transient=True) from exc
except requests.RequestException as exc:
- raise PromQLQueryError("The PromQL query backend cannot be reached. Try again later.") from exc
+ raise PromQLQueryError("The PromQL query backend cannot be reached. Try again later.", transient=True) from exc
if response.status_code in (401, 403) or response.status_code >= 500:
- raise PromQLQueryError("The PromQL query backend failed. Try again later.")
+ raise PromQLQueryError("The PromQL query backend failed. Try again later.", transient=True)
+ # Snuffle's rate limit, which passes once the budget refills.
+ if response.status_code == 429:
+ raise PromQLQueryError(_error_message(response), transient=True)
if response.status_code >= 400:
raise PromQLQueryError(_error_message(response))
try:
payload = response.json()
except ValueError as exc:
- raise PromQLQueryError("The PromQL query backend sent a response that cannot be read.") from exc
+ raise PromQLQueryError("The PromQL query backend sent a response that cannot be read.", transient=True) from exc
if not isinstance(payload, dict) or payload.get("status") != "success":
raise PromQLQueryError(_error_message(response))
return payload
diff --git a/products/metrics/backend/temporal/alert_evaluate.py b/products/metrics/backend/temporal/alert_evaluate.py
new file mode 100644
index 000000000000..aace9d9b70fa
--- /dev/null
+++ b/products/metrics/backend/temporal/alert_evaluate.py
@@ -0,0 +1,84 @@
+"""Temporal wiring for metrics alerts on the shared alerts platform.
+
+The evaluation itself lives in `products/metrics/backend/alerts/evaluation.py`, so it stays a plain
+function a test can call without Temporal.
+"""
+
+import datetime as dt
+
+from temporalio import activity, workflow
+from temporalio.common import RetryPolicy
+
+from posthog.temporal.common.base import PostHogWorkflow
+
+with workflow.unsafe.imports_passed_through():
+ from posthog.temporal.common.utils import close_db_connections
+
+ from products.alerts_platform.backend.facade.contracts import (
+ RECORD_OUTCOMES_ACTIVITY,
+ SourceBatchEvaluation,
+ SourceEvaluationInputs,
+ SourceOutcomeInputs,
+ )
+ from products.alerts_platform.backend.facade.payload import outcome_chunks
+ from products.alerts_platform.backend.facade.temporal import start_deliveries
+
+WORKFLOW_NAME = "metrics-alert-evaluate"
+
+# Above `BATCH_QUERY_BUDGET_SECONDS` and `FRESHNESS_QUERY_SECONDS`, so the evaluation stops starting
+# queries before the activity times out.
+EVALUATE_START_TO_CLOSE = dt.timedelta(seconds=40)
+EVALUATE_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=60)
+RECORD_START_TO_CLOSE = dt.timedelta(seconds=8)
+RECORD_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=12)
+
+
+@close_db_connections
+def evaluate_metrics_alerts_activity(inputs: SourceEvaluationInputs) -> SourceBatchEvaluation:
+ """Sync, so the thread this blocks is a slot Temporal is accounting for."""
+ # Imported in the body, because the workflow below makes this module load inside Temporal's
+ # sandbox, which refuses a Django model import.
+ from products.metrics.backend.alerts.evaluation import evaluate_metrics_alert_batch # noqa: PLC0415
+
+ return evaluate_metrics_alert_batch(
+ inputs.batch_key.team_id, inputs.batch_key.slot, dt.datetime.fromisoformat(inputs.cutoff)
+ )
+
+
[email protected](name=WORKFLOW_NAME)
+class MetricsAlertEvaluateWorkflow(PostHogWorkflow):
+ """Evaluates one batch key, records what it decided, and starts the deliveries it asked for."""
+
+ inputs_cls = SourceEvaluationInputs
+
+ @workflow.run
+ async def run(self, inputs: SourceEvaluationInputs) -> int:
+ # One attempt. The evaluation writes nothing, so a lost attempt leaves the checks due and
+ # the next tick reads them again.
+ evaluation = await workflow.execute_activity(
+ evaluate_metrics_alerts_activity,
+ inputs,
+ start_to_close_timeout=EVALUATE_START_TO_CLOSE,
+ schedule_to_close_timeout=EVALUATE_SCHEDULE_TO_CLOSE,
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+ started = 0
+ for chunk in outcome_chunks(evaluation.outcomes):
+ await workflow.execute_activity(
+ RECORD_OUTCOMES_ACTIVITY,
+ SourceOutcomeInputs(team_id=inputs.batch_key.team_id, cutoff=inputs.cutoff, outcomes=chunk),
+ start_to_close_timeout=RECORD_START_TO_CLOSE,
+ schedule_to_close_timeout=RECORD_SCHEDULE_TO_CLOSE,
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+ # Started only after the record, because a delivery reads the history row the record writes.
+ recorded = {str(outcome.configuration_id) for outcome in chunk}
+ started += await start_deliveries(
+ [delivery for delivery in evaluation.deliveries if delivery.configuration_id in recorded]
+ )
+ return started
+
+
+METRICS_ALERT_EVALUATION_WORKFLOWS = [MetricsAlertEvaluateWorkflow]
+METRICS_ALERT_EVALUATION_ACTIVITIES = [evaluate_metrics_alerts_activity]
diff --git a/products/metrics/backend/tests/test_alerts.py b/products/metrics/backend/tests/test_alerts.py
index 84ae58e88f37..0f4c764b8d51 100644
--- a/products/metrics/backend/tests/test_alerts.py
+++ b/products/metrics/backend/tests/test_alerts.py
@@ -1,15 +1,38 @@
+import json
+from datetime import UTC, datetime, timedelta
from typing import Any
+from uuid import UUID
-from posthog.test.base import APIBaseTest
-from unittest.mock import patch
+from posthog.test.base import APIBaseTest, ClickhouseTestMixin
+from unittest.mock import MagicMock, patch
-from django.test import SimpleTestCase
+from django.db import connection
+from django.test import SimpleTestCase, override_settings
+from django.utils import timezone
+import requests
from parameterized import parameterized
from pydantic import ValidationError
from rest_framework import status
+from posthog.models import Team
+from posthog.models.scoping import team_scope
+from posthog.temporal.alerts.admission import hold_evaluation_slot, release_evaluation_slot
+
+from products.alerts_platform.backend.facade import testing as platform_testing
+from products.alerts_platform.backend.facade.api import create_configuration, record_outcomes, slot_of
+from products.alerts_platform.backend.facade.contracts import (
+ AlertEventKind,
+ Grouping,
+ GroupingMode,
+ PlatformAlertConfigurationEdit,
+ PlatformAlertOutcome,
+ SourceKind,
+)
+from products.metrics.backend.alerts.evaluation import ABSENT_RESOLVE_AFTER, INFLIGHT_KEY, evaluate_metrics_alert_batch
+from products.metrics.backend.alerts.freshness import evaluation_time, fetch_metrics_freshness
from products.metrics.backend.alerts.source_config import MetricsPlatformAlertConfig
+from products.metrics.backend.tests._seeder import seed_metric, truncate_metrics_tables
_QUERY = 'sum(kafka_consumer_group_lag{group="example"})'
@@ -61,3 +84,212 @@ class TestMetricsAlertCreate(APIBaseTest):
assert response.status_code == status.HTTP_403_FORBIDDEN, response.json()
assert response.json()["code"] == "feature_flag_required"
+
+
+class TestEvaluationTime(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("ingestion_slightly_behind", timedelta(seconds=-30), timedelta(seconds=-30)),
+ ("ingestion_lagging", timedelta(minutes=-20), timedelta(minutes=-20)),
+ ("ingestion_past_the_due_time", timedelta(seconds=30), timedelta(0)),
+ ("freshness_unknown", None, timedelta(0)),
+ ]
+ )
+ def test_a_check_reads_no_later_than_ingestion_reached(
+ self, _name: str, freshness_offset: timedelta | None, expected_offset: timedelta
+ ) -> None:
+ due_at = datetime(2026, 1, 1, 12, 0, tzinfo=UTC)
+ freshness = None if freshness_offset is None else due_at + freshness_offset
+
+ assert evaluation_time(due_at, freshness) == due_at + expected_offset
+
+
+def _vector(*series: tuple[dict[str, str], str]) -> dict[str, Any]:
+ return {
+ "status": "success",
+ "data": {
+ "resultType": "vector",
+ "result": [{"metric": labels, "value": [0, value]} for labels, value in series],
+ },
+ }
+
+
+def _response(status_code: int, body: dict[str, Any]) -> requests.Response:
+ response = requests.Response()
+ response.status_code = status_code
+ response._content = json.dumps(body).encode()
+ return response
+
+
+class TestMetricsFreshnessRead(ClickhouseTestMixin, APIBaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ truncate_metrics_tables()
+
+ def test_reads_only_the_teams_own_newest_point(self) -> None:
+ # The query compares with ClickHouse's now(), so the points sit just behind the real clock.
+ now = datetime.now(UTC).replace(microsecond=0)
+ other_team = Team.objects.create(organization=self.organization)
+ seed_metric(team_id=self.team.id, metric_name="lag", points=[(now - timedelta(minutes=2), 1.0)])
+ seed_metric(team_id=other_team.id, metric_name="lag", points=[(now - timedelta(seconds=30), 1.0)])
+
+ assert fetch_metrics_freshness(self.team, max_execution_time=5) == now - timedelta(minutes=2)
+
+ def test_points_dated_ahead_cannot_rewind_the_read_to_an_older_service(self) -> None:
+ now = datetime.now(UTC).replace(microsecond=0)
+ seed_metric(
+ team_id=self.team.id, metric_name="lag", service_name="decoy", points=[(now - timedelta(minutes=14), 1.0)]
+ )
+ seed_metric(
+ team_id=self.team.id, metric_name="lag", service_name="live", points=[(now + timedelta(minutes=5), 1.0)]
+ )
+
+ freshness = fetch_metrics_freshness(self.team, max_execution_time=5)
+
+ assert freshness is not None
+ assert now - timedelta(seconds=5) <= freshness <= datetime.now(UTC)
+
+
+class TestMetricsAlertEvaluation(APIBaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ self.cutoff = timezone.now().replace(second=0, microsecond=0) + timedelta(minutes=5)
+ self.snuffle = MagicMock()
+ self.enterContext(patch("products.metrics.backend.promql.snuffle_request", self.snuffle))
+ self.freshness = self.enterContext(
+ patch("products.metrics.backend.alerts.evaluation.fetch_metrics_freshness", return_value=None)
+ )
+ with team_scope(self.team.id):
+ self.configuration_id = create_configuration(
+ team_id=self.team.id,
+ source_kind=SourceKind.METRICS,
+ edit=PlatformAlertConfigurationEdit(
+ name="Consumer lag",
+ enabled=True,
+ source_config={"query": _QUERY, "condition": {"operator": "gt", "threshold": 1000}},
+ grouping=Grouping(mode=GroupingMode.BY_RESULT_LABELS, keys=("group",)),
+ schedule_restriction=None,
+ evaluation_periods=1,
+ datapoints_to_alarm=1,
+ cooldown_minutes=0,
+ ),
+ created_by_id=self.user.id,
+ ).id
+ platform_testing.set_due_at(self.configuration_id, self.due_at)
+
+ @property
+ def due_at(self) -> datetime:
+ return self.cutoff - timedelta(minutes=1)
+
+ def _evaluate(self) -> PlatformAlertOutcome:
+ with team_scope(self.team.id):
+ evaluation = evaluate_metrics_alert_batch(self.team.id, slot_of(self.due_at, self.cutoff), self.cutoff)
+ (outcome,) = evaluation.outcomes
+ return outcome
+
+ def _next_check(self, outcome: PlatformAlertOutcome) -> None:
+ with team_scope(self.team.id):
+ record_outcomes(self.team.id, [outcome], self.cutoff)
+ self.cutoff += timedelta(minutes=5)
+ platform_testing.set_due_at(self.configuration_id, self.due_at)
+
+ def _kinds(self, outcome: PlatformAlertOutcome) -> dict[str, AlertEventKind]:
+ return {
+ json.loads(group.grouping_key)["group"]: group.kind
+ for group in outcome.groups
+ if group.kind != AlertEventKind.CHECK
+ }
+
+ def test_a_missing_series_holds_until_it_has_been_gone_an_hour_with_fresh_ingestion(self) -> None:
+ self.freshness.return_value = self.due_at - timedelta(seconds=30)
+ self.snuffle.return_value = _response(
+ 200, _vector(({"group": "a"}, "2000"), ({"group": "b"}, "3000"), ({"group": "c"}, "10"))
+ )
+ first = self._evaluate()
+
+ assert self._kinds(first) == {"a": AlertEventKind.FIRING, "b": AlertEventKind.FIRING}
+ assert self.snuffle.call_args.kwargs["data"]["time"] == str((self.due_at - timedelta(seconds=30)).timestamp())
+
+ self._next_check(first)
+ self.snuffle.return_value = _response(
+ 200, _vector(({"group": "b"}, "NaN"), ({"group": "b"}, "10"), ({"group": "c"}, "10"))
+ )
+ held = self._evaluate()
+
+ assert self._kinds(held) == {}
+
+ self._next_check(held)
+ self.cutoff += ABSENT_RESOLVE_AFTER
+ with team_scope(self.team.id):
+ platform_testing.set_due_at(self.configuration_id, self.due_at)
+ self.snuffle.return_value = _response(200, _vector(({"group": "b"}, "NaN"), ({"group": "c"}, "10")))
+ self.freshness.return_value = None
+
+ assert self._kinds(self._evaluate()) == {}
+
+ self.freshness.return_value = self.due_at - timedelta(seconds=30)
+ resolved = self._evaluate()
+
+ assert self._kinds(resolved) == {"a": AlertEventKind.RESOLVED}
+
+ self._next_check(resolved)
+ self.snuffle.return_value = _response(200, _vector(({"group": "c"}, "10")))
+ self.freshness.return_value = self.due_at - timedelta(seconds=30)
+
+ assert self._kinds(self._evaluate()) == {}
+
+ def test_a_firing_held_back_by_a_mute_is_not_announced_on_a_value_with_no_number(self) -> None:
+ self.snuffle.return_value = _response(200, _vector(({"group": "a"}, "2000")))
+ first = self._evaluate()
+ self._next_check(first)
+ # The state a mute leaves: firing, with the announcement still owed.
+ with connection.cursor() as cursor:
+ cursor.execute(
+ "UPDATE alerts_platformalert SET last_notified_at = NULL WHERE configuration_id = %s",
+ [str(self.configuration_id)],
+ )
+ self.snuffle.return_value = _response(200, _vector(({"group": "a"}, "NaN")))
+
+ assert self._kinds(self._evaluate()) == {}
+
+ @override_settings(ALERTS_PLATFORM_METRICS_MAX_INFLIGHT_BATCHES=1)
+ def test_a_batch_over_the_pool_limit_stays_due_for_a_later_tick(self) -> None:
+ self.snuffle.return_value = _response(200, _vector(({"group": "a"}, "2000")))
+ slot = slot_of(self.due_at, self.cutoff)
+ held_until = hold_evaluation_slot("other-batch", limit=1, key=INFLIGHT_KEY)
+ assert held_until is not None
+ with team_scope(self.team.id):
+ deferred = evaluate_metrics_alert_batch(self.team.id, slot, self.cutoff)
+ release_evaluation_slot("other-batch", held_until=held_until, key=INFLIGHT_KEY)
+
+ assert (deferred.outcomes, deferred.omitted, self.snuffle.call_count) == ((), 1, 0)
+ assert self._kinds(self._evaluate()) == {"a": AlertEventKind.FIRING}
+ assert self._kinds(self._evaluate()) == {"a": AlertEventKind.FIRING}
+
+ @parameterized.expand(
+ [
+ ("rejected_query_counts_toward_broken", _response(400, {"status": "error", "error": "parse error"}), 1),
+ ("backend_failure_is_transient", _response(503, {"status": "error"}), 0),
+ ("timeout_is_transient", requests.Timeout(), 0),
+ (
+ "too_many_series_counts_toward_broken",
+ _response(200, _vector(*(({"group": str(index)}, "1") for index in range(1001)))),
+ 1,
+ ),
+ ]
+ )
+ def test_a_failed_query_records_a_failure(
+ self, _name: str, result: requests.Response | Exception, consecutive_failures: int
+ ) -> None:
+ if isinstance(result, Exception):
+ self.snuffle.side_effect = result
+ else:
+ self.snuffle.return_value = result
+
+ outcome = self._evaluate()
+
+ assert outcome.failure is not None
+ assert (outcome.configuration_id, outcome.consecutive_failures) == (
+ UUID(str(self.configuration_id)),
+ consecutive_failures,
+ )