:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(metrics): evaluate metrics alerts through snuffle (#125092)
Jimmy Zhu4375f9f4fa2c
diff --git a/posthog/api/snuffle_proxy.py b/posthog/api/snuffle_proxy.pyindex 536fe000640c..0d7275d0d99f 100644--- a/posthog/api/snuffle_proxy.py+++ b/posthog/api/snuffle_proxy.py@@ -126,6 +126,7 @@ def snuffle_request(params: dict[str, list[str]] | dict[str, str] | None = None,data: dict[str, list[str]] | dict[str, str] | None = None,accept: str = "application/json",+ timeout: float | None = None,) -> requests.Response:"""Send one request to Snuffle as `team_id` and debit the bytes it read from the query budget.@@ -144,7 +145,7 @@ def snuffle_request(data=data,headers=headers,auth=auth,- timeout=settings.SNUFFLE_APM_TIMEOUT_SECONDS,+ timeout=settings.SNUFFLE_APM_TIMEOUT_SECONDS if timeout is None else timeout,)bytes_read = _read_bytes_from(upstream)if bytes_read is not None:diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.pyindex 642190211ae3..50ead805045e 100644--- a/posthog/management/commands/start_temporal_worker.py+++ b/posthog/management/commands/start_temporal_worker.py@@ -275,6 +275,8 @@ from products.messaging.backend.facade.temporal import ()from products.metrics.backend.facade.temporal import (ACTIVITIES as METRICS_ACTIVITIES,+ METRICS_ALERT_EVALUATION_ACTIVITIES,+ METRICS_ALERT_EVALUATION_WORKFLOWS,WORKFLOWS as METRICS_WORKFLOWS,)from products.notebooks.backend.facade.temporal import (@@ -680,12 +682,14 @@ _task_queue_specs = [+ LOGS_SOURCE_EVALUATION_WORKFLOWS+ INSIGHT_PLATFORM_EVALUATION_WORKFLOWS+ BILLING_PLATFORM_EVALUATION_WORKFLOWS- + WAREHOUSE_ALERT_EVALUATION_WORKFLOWS,+ + WAREHOUSE_ALERT_EVALUATION_WORKFLOWS+ + METRICS_ALERT_EVALUATION_WORKFLOWS,ALERTS_PLATFORM_EVALUATION_ACTIVITIES+ LOGS_SOURCE_EVALUATION_ACTIVITIES+ INSIGHT_PLATFORM_EVALUATION_ACTIVITIES+ BILLING_PLATFORM_EVALUATION_ACTIVITIES- + WAREHOUSE_ALERT_EVALUATION_ACTIVITIES,+ + WAREHOUSE_ALERT_EVALUATION_ACTIVITIES+ + METRICS_ALERT_EVALUATION_ACTIVITIES,),(settings.ALERTS_PLATFORM_DELIVERY_TASK_QUEUE,diff --git a/posthog/management/commands/test/test_run_async_migrations.py b/posthog/management/commands/test/test_run_async_migrations.pyindex 0e1eea4c019e..b51122d503bb 100644--- a/posthog/management/commands/test/test_run_async_migrations.py+++ b/posthog/management/commands/test/test_run_async_migrations.py@@ -84,3 +84,10 @@ def test_complete_noop_migrations(caplog):@pytest.fixture(autouse=True)def set_log_level(caplog):caplog.set_level(logging.INFO)++[email protected](autouse=True)+def empty_events_table():+ # The backfill migrations check every event in ClickHouse, and earlier tests in a shard can+ # leave events without person data, which fails their postcheck.+ sync_execute(TRUNCATE_EVENTS_TABLE_SQL())diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.pyindex d78d8a727f7a..028213a845a8 100644--- a/posthog/settings/temporal.py+++ b/posthog/settings/temporal.py@@ -467,6 +467,15 @@ if ALERTS_PLATFORM_LOGS_MAX_INFLIGHT_BATCHES <= 0:raise ImproperlyConfigured("ALERTS_PLATFORM_LOGS_MAX_INFLIGHT_BATCHES must be a positive integer, or no platform logs check starts")+# The platform's metrics batches allowed to run at once, across every team. A batch runs its Snuffle+# queries one after another, so this bounds the concurrent metrics alert queries.+ALERTS_PLATFORM_METRICS_MAX_INFLIGHT_BATCHES: int = get_from_env(+ "ALERTS_PLATFORM_METRICS_MAX_INFLIGHT_BATCHES", 5, type_cast=int+)+if ALERTS_PLATFORM_METRICS_MAX_INFLIGHT_BATCHES <= 0:+ raise ImproperlyConfigured(+ "ALERTS_PLATFORM_METRICS_MAX_INFLIGHT_BATCHES must be a positive integer, or no platform metrics check starts"+ )# Insight alert checks allowed to run against ClickHouse at once, across every team.ALERTS_MAX_INFLIGHT_EVALUATIONS: int = get_from_env("ALERTS_MAX_INFLIGHT_EVALUATIONS", 40, type_cast=int)if ALERTS_MAX_INFLIGHT_EVALUATIONS <= 0:diff --git a/products/alerts_platform/backend/temporal/sources.py b/products/alerts_platform/backend/temporal/sources.pyindex 2a939b56ac12..4d274c1a746f 100644--- a/products/alerts_platform/backend/temporal/sources.py+++ b/products/alerts_platform/backend/temporal/sources.py@@ -75,6 +75,13 @@ SOURCE_BINDINGS: dict[SourceKind, SourceBinding] = {evaluation_timeout=dt.timedelta(seconds=60),discovery_limit=DISCOVERY_LIMIT_PER_SOURCE,),+ # One PromQL instant query per check through Snuffle. A batch stops starting queries at its budget.+ SourceKind.METRICS: SourceBinding(+ workflow="metrics-alert-evaluate",+ task_queue=settings.ALERTS_PLATFORM_EVALUATION_TASK_QUEUE,+ evaluation_timeout=dt.timedelta(seconds=120),+ discovery_limit=DISCOVERY_LIMIT_PER_SOURCE,+ ),}diff --git a/products/metrics/backend/alerts/destinations.py b/products/metrics/backend/alerts/destinations.pyindex 32c5845caea0..5e4f0778a94e 100644--- a/products/metrics/backend/alerts/destinations.py+++ b/products/metrics/backend/alerts/destinations.py@@ -7,7 +7,12 @@ find who to tell and words the message itself, so their templates never run.from dataclasses import replacefrom typing import Final, Literal-from products.alerts_platform.backend.facade.contracts import AlertDestinationAction, EventKindSpec, IncidentAction+from products.alerts_platform.backend.facade.contracts import (+ AlertDestinationAction,+ AlertEventKind,+ EventKindSpec,+ IncidentAction,+)EventKind = Literal["firing", "resolved", "errored", "broken", "incident_opened", "incident_closed"]@@ -84,6 +89,12 @@ EVENT_KIND_CONFIG: Final[dict[EventKind, EventKindSpec]] = {EVENT_IDS: Final[tuple[str, ...]] = tuple(spec.event_id for spec in EVENT_KIND_CONFIG.values())+# Which event id each announced kind's destinations filter on. Resolved here because the platform+# imports no source.+EVENT_IDS_BY_KIND: Final[dict[str, str]] = {+ key: spec.event_id for key, spec in EVENT_KIND_CONFIG.items() if key in set(AlertEventKind)+}+EVENT_IDS_BY_INCIDENT_ACTION: Final[dict[str, str]] = {spec.incident_action.value: spec.event_id for spec in EVENT_KIND_CONFIG.values() if spec.incident_action}diff --git a/products/metrics/backend/alerts/evaluation.py b/products/metrics/backend/alerts/evaluation.pynew file mode 100644index 000000000000..d1dc1a647dce--- /dev/null+++ b/products/metrics/backend/alerts/evaluation.py@@ -0,0 +1,479 @@+"""Evaluation of due metrics alerts on the shared alerts platform.++Each check sends its PromQL expression to Snuffle as one instant query, at the check's due time+held back to where the team's ingestion has reached. Each series the query returns is compared with the+condition. Under grouping by result labels, each distinct set of grouping label values is a group.+Without grouping, the alert has one group that breaches when any series does.++A series the query no longer returns keeps its group's state, and so does a series whose value is+not a number. Prometheus resolves a missing series, but a team's samples spread over ingestion+partitions, and the freshness read cannot tell that every partition has caught up. So a missing+series could be one a lagging partition has not written yet, and resolving it would read lag as+recovery. The hold is bounded: a group missing for `ABSENT_RESOLVE_AFTER` while the team's+ingestion is fresh resolves, which no lag that leaves the team fresh explains, and which frees its+slot under `max_instances`. A watermark covering every partition would let it resolve at once.++This reads and decides. It writes nothing, so a retried attempt reads the same due checks and makes+the same decisions. The platform records the outcomes in its own activity.+"""++import json+import math+import time+import operator+from collections.abc import Callable, Sequence+from datetime import datetime, timedelta+from typing import Any, Final+from uuid import uuid4++from django.conf import settings++import structlog+from pydantic import ValidationError++from posthog.dataclasses import frozen+from posthog.models import Team+from posthog.temporal.alerts.admission import hold_evaluation_slot, release_evaluation_slot++from products.alerts.backend.facade.destinations import configured_destination_template_ids+from products.alerts_platform.backend.facade.api import due_checks, slot_of+from products.alerts_platform.backend.facade.contracts import (+ AlertDeliveryRequest,+ GroupingMode,+ InstanceCheckState,+ PlatformAlertCheckInput,+ PlatformAlertOutcome,+ SourceBatchEvaluation,+ SourceKind,+)+from products.alerts_platform.backend.facade.enums import PlatformAlertState+from products.alerts_platform.backend.facade.grouping import (+ GroupObservation,+ bounded_grouping_key,+ decide_groups,+ delivery_parts,+ muted_value,+ not_breaching,+)+from products.alerts_platform.backend.facade.lifecycle import (+ NOTIFICATION_EVENT_KINDS,+ PLATFORM_DEFAULT_ALERT_POLICY,+ AlertSnapshot,+ AlertState,+ CheckInput,+ NotificationAction,+ decide_firing_episode,+ decide_incident_action,+ evaluate_alert_check,+)+from products.alerts_platform.backend.facade.payload import MAX_BATCH_PAYLOAD_BYTES, payload_bytes+from products.alerts_platform.backend.facade.platform_metrics import (+ increment_checks,+ increment_checks_deferred,+ increment_state_transition,+ record_batch_duration,+ safe_record,+)+from products.alerts_platform.backend.facade.scheduling import is_utc_datetime_blocked, parse_blocked_windows_tuples+from products.metrics.backend.alerts.destinations import EVENT_IDS_BY_INCIDENT_ACTION, EVENT_IDS_BY_KIND+from products.metrics.backend.alerts.freshness import evaluation_time, fetch_metrics_freshness+from products.metrics.backend.alerts.source_config import MetricsPlatformAlertCondition, MetricsPlatformAlertConfig+from products.metrics.backend.alerts.wording import alert_url+from products.metrics.backend.promql import PromQLQueryError, snuffle_payload++logger = structlog.get_logger(__name__)++SOURCE: Final = SourceKind.METRICS+POLICY: Final = PLATFORM_DEFAULT_ALERT_POLICY++# The batch's query time, which `EVALUATE_START_TO_CLOSE` in the workflow sits above. Checks run one+# after another, and a check the budget has no room for keeps its due time for a later tick.+BATCH_QUERY_BUDGET_SECONDS: Final = 25+# No single query may spend the whole budget, or the first slow one starves the rest.+MAX_QUERY_SECONDS: Final = 10+# Below this there is no point starting another query.+MIN_QUERY_SECONDS: Final = 2+FRESHNESS_QUERY_SECONDS: Final = 5+INFLIGHT_KEY: Final = "alerts:platform:metrics:batches:inflight"+# Outlasts the evaluation activity's run, so a slot frees before its batch ends only when a worker+# died mid-batch.+BATCH_SLOT_LEASE_SECONDS: Final = 90+# Ten times the most groups a configuration can hold. A query past it is refused before its series+# are judged, so one high-cardinality expression cannot exhaust the worker.+MAX_SERIES: Final = 1000+# Far past any partition lag that leaves the team fresh, and short enough that groups which stopped+# reporting cannot hold every instance slot.+ABSENT_RESOLVE_AFTER: Final = timedelta(hours=1)++_UNEXPECTED_ERROR: Final = "Alert check failed unexpectedly. PostHog has been notified."+_INVALID_CONFIG: Final = "The alert's configuration is no longer valid. Edit the alert to fix it."++_COMPARE: Final[dict[str, Callable[[float, float], bool]]] = {+ "gt": operator.gt,+ "gte": operator.ge,+ "lt": operator.lt,+ "lte": operator.le,+ "eq": operator.eq,+ "ne": operator.ne,+}++Decision = tuple[PlatformAlertOutcome, AlertDeliveryRequest | None]+++@frozen+class _Sample:+ labels: dict[str, str]+ # None for a sample that is not a number, such as the result of 0/0.+ value: float | None+++def evaluate_metrics_alert_batch(team_id: int, slot: str, cutoff: datetime) -> SourceBatchEvaluation:+ """Evaluates one batch key of metrics alerts against the tick's cutoff."""+ started_at = time.monotonic()+ checks = due_checks(team_id, SOURCE.value, slot, cutoff)+ if not checks:+ return SourceBatchEvaluation(outcomes=(), deliveries=())+ team = Team.objects.filter(id=team_id).first()+ if team is None:+ return SourceBatchEvaluation(outcomes=(), deliveries=())++ # One slot per batch, because a batch runs its queries one after another, so the batches in+ # flight bound the concurrent metrics queries across the fleet. A batch the pool turns away+ # records nothing, so its checks stay due and a later tick finds them again.+ # Unique per execution: a redispatch of a batch whose old query still runs must hold its own slot.+ member = f"{team_id}:{slot}:{uuid4().hex}"+ held_until = hold_evaluation_slot(+ member,+ limit=settings.ALERTS_PLATFORM_METRICS_MAX_INFLIGHT_BATCHES,+ lease_seconds=BATCH_SLOT_LEASE_SECONDS,+ key=INFLIGHT_KEY,+ )+ if held_until is None:+ logger.info("Deferred a platform metrics batch over the pool limit", team_id=team_id, slot=slot)+ safe_record(increment_checks_deferred, SOURCE.value, len(checks))+ return SourceBatchEvaluation(outcomes=(), deliveries=(), omitted=len(checks))+ try:+ return _evaluate_admitted(team, slot, checks, cutoff=cutoff, started_at=started_at)+ finally:+ release_evaluation_slot(member, held_until=held_until, key=INFLIGHT_KEY)+++def _evaluate_admitted(+ team: Team, slot: str, checks: Sequence[PlatformAlertCheckInput], *, cutoff: datetime, started_at: float+) -> SourceBatchEvaluation:+ freshness = _freshness(team)+ deadline = started_at + BATCH_QUERY_BUDGET_SECONDS+ decided: list[Decision] = []+ for check in checks:+ remaining = deadline - time.monotonic()+ if remaining < MIN_QUERY_SECONDS:+ break+ at = evaluation_time(check.next_check_at or cutoff, freshness)+ decided.append(+ _evaluate(+ team,+ check,+ at=at,+ now=cutoff,+ ingestion_fresh=freshness is not None,+ query_seconds=min(MAX_QUERY_SECONDS, remaining),+ muted=_in_quiet_hours(check, cutoff, team.timezone),+ )+ )+ evaluation = _collect(decided, team_id=team.id, slot=slot, past_budget=len(checks) - len(decided))+ safe_record(record_batch_duration, SOURCE.value, int((time.monotonic() - started_at) * 1000))+ return evaluation+++def _freshness(team: Team) -> datetime | None:+ try:+ return fetch_metrics_freshness(team, max_execution_time=FRESHNESS_QUERY_SECONDS)+ except Exception:+ # Without a freshness the batch still runs, at its due time.+ logger.exception("Failed to fetch the metrics ingestion freshness", team_id=team.id)+ return None+++def _in_quiet_hours(check: PlatformAlertCheckInput, now: datetime, tz_name: str) -> bool:+ try:+ return is_utc_datetime_blocked(now, tz_name, parse_blocked_windows_tuples(check.schedule_restriction))+ except Exception:+ # A restriction that cannot be read must not hold back a breach.+ logger.exception("Unparseable schedule restriction on a metrics alert", check_id=str(check.id))+ return False+++def _evaluate(+ team: Team,+ check: PlatformAlertCheckInput,+ *,+ at: datetime,+ now: datetime,+ ingestion_fresh: bool,+ query_seconds: float,+ muted: bool,+) -> Decision:+ started_at = time.monotonic()+ try:+ config = MetricsPlatformAlertConfig.model_validate(check.source_config)+ except ValidationError:+ # Saved through the same model, so this is a config an older version accepted.+ return _failed(check, _INVALID_CONFIG, transient=False, now=now, muted=muted)+ try:+ samples = _query(team, config.query, at=at, timeout_seconds=query_seconds)+ except PromQLQueryError as error:+ return _failed(check, str(error), transient=error.transient, now=now, muted=muted)+ except Exception:+ logger.exception("Failed to evaluate a metrics alert", check_id=str(check.id))+ return _failed(check, _UNEXPECTED_ERROR, transient=True, now=now, muted=muted)+ duration_ms = int((time.monotonic() - started_at) * 1000)++ observations, unknown = _observations(check, config.condition, samples)+ decision = decide_groups(+ check,+ observations,+ snapshot_of=lambda instance, _prior: _snapshot(check, instance),+ policy=POLICY,+ now=now,+ muted=muted,+ absent=lambda key: _absent(check, key, unknown=unknown, now=now, ingestion_fresh=ingestion_fresh),+ )+ for verdict in decision.verdicts:+ safe_record(increment_checks, SOURCE.value, verdict.outcome.notification.value)+ if verdict.snapshot.state != verdict.outcome.new_state:+ safe_record(+ increment_state_transition, SOURCE.value, verdict.snapshot.state.value, verdict.outcome.new_state.value+ )+ recorded = decision.as_outcome(+ configuration_id=check.id,+ evaluation_key=_evaluation_key(check, now),+ grouping=check.grouping,+ query_duration_ms=duration_ms,+ results_cut=False,+ )+ moves = {+ verdict.group.grouping_key: (verdict.snapshot.state, verdict.outcome.new_state) for verdict in decision.verdicts+ }+ sends_messages = any(verdict.outcome.notification != NotificationAction.NONE for verdict in decision.verdicts)+ return recorded, _request(check, recorded, moves, sends_messages=sends_messages)+++def _query(team: Team, expr: str, *, at: datetime, timeout_seconds: float) -> list[_Sample]:+ """The series one PromQL instant query returns at `at`."""+ payload = snuffle_payload(+ team,+ "POST",+ "/api/v1/query",+ data={"query": expr, "time": str(at.timestamp()), "timeout": f"{int(timeout_seconds)}s"},+ timeout=timeout_seconds,+ )++ data = payload.get("data") or {}+ result_type = data.get("resultType")+ result: Any = data.get("result")+ if result_type == "scalar":+ return [_Sample(labels={}, value=_number(result[1]))] if result else []+ if result_type != "vector":+ raise PromQLQueryError(f"A metrics alert query must return an instant vector or a scalar, not a {result_type}.")+ if len(result or []) > MAX_SERIES:+ raise PromQLQueryError(+ f"The query returned more than {MAX_SERIES} series. Aggregate it, for example with sum by (...)."+ )+ samples: list[_Sample] = []+ for series in result or []:+ # `__name__` stays, so an alert can group by it.+ labels = {str(key): str(value) for key, value in (series.get("metric") or {}).items()}+ sample = series.get("value") or [None, None]+ samples.append(_Sample(labels=labels, value=_number(sample[1])))+ return samples+++def _number(raw: Any) -> float | None:+ try:+ value = float(raw)+ except (TypeError, ValueError):+ return None+ return value if math.isfinite(value) else None+++def _breached(condition: MetricsPlatformAlertCondition, value: float | None) -> bool | None:+ """Whether a sample breaches, or None when its value reaches no verdict."""+ if condition.kind == "present":+ return True+ # The config model requires both for a threshold condition.+ if value is None or condition.operator is None or condition.threshold is None:+ return None+ return _COMPARE[condition.operator](value, condition.threshold)+++def _absent(+ check: PlatformAlertCheckInput, key: str, *, unknown: frozenset[str], now: datetime, ingestion_fresh: bool+) -> GroupObservation | None:+ """A missing group's verdict: None holds its state, as the module docstring explains, until the hold runs out."""+ last_seen_at = check.instance(key).last_seen_at+ if key in unknown or not ingestion_fresh or last_seen_at is None or now - last_seen_at < ABSENT_RESOLVE_AFTER:+ return None+ return not_breaching(key)+++def _observations(+ check: PlatformAlertCheckInput, condition: MetricsPlatformAlertCondition, samples: Sequence[_Sample]+) -> tuple[list[GroupObservation], frozenset[str]]:+ """One observation per sample, leaving out the groups held without a verdict, and those groups' keys.++ A group with a sample that has no number is held, unless another of its samples breaches. A healthy+ sample beside it does not show that the unknown one recovered.+ """+ by_labels = check.grouping.mode == GroupingMode.BY_RESULT_LABELS+ observations: list[GroupObservation] = []+ unknown: set[str] = set()+ for sample in samples:+ if by_labels:+ labels = {key: sample.labels.get(key, "") for key in check.grouping.keys}+ else:+ labels = {key: value for key, value in sample.labels.items() if key != "__name__"}+ key = _grouping_key_of(labels) if by_labels else ""+ breached = _breached(condition, sample.value)+ if breached is None:+ unknown.add(key)+ continue+ observations.append(+ GroupObservation(grouping_key=key, current_breached=breached, value=sample.value, labels=labels)+ )+ breaching = {observation.grouping_key for observation in observations if observation.current_breached}+ held = frozenset(unknown - breaching)+ kept = [observation for observation in observations if observation.grouping_key not in held]+ # An announced firing the query still returns is reported as still firing, which notifies nothing,+ # so the platform records it as seen and its absence hold counts from this check. An unannounced+ # one is left out, because a breach would announce it on a value that reached no verdict.+ kept.extend(GroupObservation(grouping_key=key, current_breached=True) for key in held if _announced(check, key))+ return kept, held+++def _announced(check: PlatformAlertCheckInput, key: str) -> bool:+ """Whether a group fires and has announced it, the same test the lifecycle machine applies."""+ instance = check.instance(key)+ return (+ instance.state == PlatformAlertState.FIRING.value+ and instance.firing_started_at is not None+ and instance.last_notified_at is not None+ and instance.last_notified_at >= instance.firing_started_at+ )+++def _grouping_key_of(labels: dict[str, str]) -> str:+ """The instance key for one group's label values."""+ return bounded_grouping_key(json.dumps(labels, sort_keys=True, separators=(",", ":")))+++def _evaluation_key(check: PlatformAlertCheckInput, now: datetime) -> str:+ return f"slot:{slot_of(check.next_check_at, now)}"+++def _snapshot(check: PlatformAlertCheckInput, instance: InstanceCheckState) -> AlertSnapshot:+ return AlertSnapshot(+ state=AlertState(instance.state),+ cooldown=timedelta(minutes=check.cooldown_minutes),+ last_notified_at=instance.last_notified_at,+ snooze_until=instance.snooze_until,+ consecutive_failures=check.consecutive_failures,+ firing_started_at=instance.firing_started_at,+ )+++def _failed(check: PlatformAlertCheckInput, message: str, *, transient: bool, now: datetime, muted: bool) -> Decision:+ """A check that reached no verdict. The machine counts it toward BROKEN unless it is transient."""+ snapshot = _snapshot(check, check.instance())+ outcome = evaluate_alert_check(+ snapshot,+ CheckInput(threshold_breached=False, error_message=message, is_transient_error=transient, muted=muted),+ now,+ policy=POLICY,+ )+ safe_record(increment_checks, SOURCE.value, outcome.notification.value)+ recorded = PlatformAlertOutcome.ungrouped(+ configuration_id=check.id,+ evaluation_key=_evaluation_key(check, now),+ consecutive_failures=outcome.consecutive_failures,+ kind=NOTIFICATION_EVENT_KINDS[outcome.notification],+ new_state=outcome.new_state.value,+ notified=outcome.update_last_notified_at,+ failed=True,+ firing_episode=decide_firing_episode(snapshot, outcome, now, policy=POLICY),+ muted_notification=muted_value(outcome),+ error_message=outcome.error_message,+ disable=outcome.disable,+ )+ return recorded, _request(+ check,+ recorded,+ {"": (snapshot.state, outcome.new_state)},+ sends_messages=outcome.notification != NotificationAction.NONE,+ )+++def _request(+ check: PlatformAlertCheckInput,+ recorded: PlatformAlertOutcome,+ moves: dict[str, tuple[AlertState, AlertState]],+ *,+ sends_messages: bool,+) -> AlertDeliveryRequest | None:+ """The delivery for a recorded outcome, or None when it neither announces nor moves a firing.++ A firing that opens or closes needs a delivery even when cooldown or mute held the message,+ because a paging destination needs one resolve for every trigger.+ """+ destination_alert_id = str(check.destination_alert_id)+ actions = {+ key: action+ for key, (before, after) in moves.items()+ if (action := decide_incident_action(before, after, policy=POLICY)) is not None+ }+ if actions and not configured_destination_template_ids(+ team_id=check.team_id,+ alert_id=destination_alert_id,+ allowed_event_ids=tuple(EVENT_IDS_BY_INCIDENT_ACTION.values()),+ ):+ actions = {}+ if not sends_messages and not actions:+ return None+ return AlertDeliveryRequest(+ source=SOURCE,+ team_id=check.team_id,+ configuration_id=str(check.id),+ evaluation_key=recorded.evaluation_key,+ destination_alert_id=destination_alert_id,+ event_ids_by_kind=EVENT_IDS_BY_KIND,+ event_ids_by_incident_action=EVENT_IDS_BY_INCIDENT_ACTION,+ incident_actions=actions,+ sends_messages=sends_messages,+ source_alert_url=alert_url(check.team_id, check.id),+ owned_by_platform=check.owned_by_platform,+ overflowed=recorded.overflowed,+ overflow_is_lower_bound=recorded.overflow_is_lower_bound,+ parts=delivery_parts(recorded, actions),+ )+++def _collect(decided: Sequence[Decision], *, team_id: int, slot: str, past_budget: int) -> SourceBatchEvaluation:+ """The batch, held to one payload. A check past the query budget or the payload keeps its due time."""+ outcomes: list[PlatformAlertOutcome] = []+ deliveries: list[AlertDeliveryRequest] = []+ deferred = 0+ spent = 0+ for outcome, delivery in decided:+ size = payload_bytes(outcome) + (payload_bytes(delivery) if delivery is not None else 0)+ if spent + size > MAX_BATCH_PAYLOAD_BYTES:+ deferred += 1+ continue+ spent += size+ outcomes.append(outcome)+ if delivery is not None:+ deliveries.append(delivery)+ if past_budget or deferred:+ logger.warning(+ "Deferred metrics alert checks", team_id=team_id, slot=slot, past_budget=past_budget, over_payload=deferred+ )+ safe_record(increment_checks_deferred, SOURCE.value, past_budget + deferred)+ return SourceBatchEvaluation(outcomes=tuple(outcomes), deliveries=tuple(deliveries), omitted=past_budget + deferred)diff --git a/products/metrics/backend/alerts/freshness.py b/products/metrics/backend/alerts/freshness.pynew file mode 100644index 000000000000..5a55b1ea76d9--- /dev/null+++ b/products/metrics/backend/alerts/freshness.py@@ -0,0 +1,98 @@+"""How far metrics ingestion has reached for a team, so an alert never reads a window still being written.++A series the pipeline has not written yet is absent from an instant query, and an absent series+resolves its group. So a consumer lag would read as recovery. The newest point the team's services+sent shows how far ingestion has reached: a lag delays every service at once.+"""++import datetime as dt+from contextlib import AbstractContextManager+from typing import Final+from zoneinfo import ZoneInfo++from posthog.schema import HogQLQueryModifiers++from posthog.hogql.constants import HogQLGlobalSettings+from posthog.hogql.parser import parse_select+from posthog.hogql.query import HogQLQueryExecutor+from posthog.hogql.visitor import clone_expr++from posthog.clickhouse.client import sync_execute+from posthog.clickhouse.client.connection import (+ ClickHouseUser,+ Workload,+ connection_creds,+ get_kwargs_for_client,+ is_file_backed_user,+ make_ch_pool,+)+from posthog.clickhouse.query_tagging import Feature, Product, tag_queries+from posthog.models import Team++# The shared pool's socket reads never time out, so a stalled connection would outlive the activity.+# The margin over the query's own limit lets the server's timeout error arrive first.+_TRANSPORT_MARGIN_SECONDS: Final = 5++# Reads the `service_freshness_by_hour` projection of `metrics4_samples`, one row per service-hour.+# The sender sets point times, so a service dated ahead is capped at now rather than dropped, which+# would let it push live services out and rewind every check. A team over 15 minutes behind gets none.+_TEAM_FRESHNESS_QUERY: Final = parse_select(+ """+ SELECT least(maxOrNull(service_newest), now()) FROM (+ SELECT service_name, max(arrayMax(timestamp_arr)) AS service_newest+ FROM posthog.metric_samples+ WHERE time_bucket >= toStartOfHour(now() - toIntervalHour(1)) AND time_bucket <= now() + toIntervalHour(1)+ GROUP BY service_name+ )+ WHERE service_newest >= now() - toIntervalMinute(15)+ """+)+++def fetch_metrics_freshness(team: Team, *, max_execution_time: int) -> dt.datetime | None:+ """The newest point the team's ingestion has written, or None when it is more than 15 minutes behind."""+ tag_queries(product=Product.METRICS, feature=Feature.ALERTING, source="metrics_alert", team_id=str(team.id))+ sql, context = HogQLQueryExecutor(+ query_type="metrics_alert_freshness",+ # The executor edits the node it is given, and this one is shared by every call.+ query=clone_expr(_TEAM_FRESHNESS_QUERY, True),+ team=team,+ workload=Workload.LOGS,+ modifiers=HogQLQueryModifiers(convertToProjectTimezone=False),+ settings=HogQLGlobalSettings(max_execution_time=max_execution_time, timeout_overflow_mode="throw"),+ ).generate_clickhouse_sql()+ with _bounded_logs_client(max_execution_time + _TRANSPORT_MARGIN_SECONDS) as client:+ rows = sync_execute(+ sql, context.values, workload=Workload.LOGS, team_id=team.id, readonly=True, sync_client=client+ )+ if not rows or rows[0][0] is None:+ return None+ freshness: dt.datetime = rows[0][0]+ return freshness if freshness.tzinfo is not None else freshness.replace(tzinfo=ZoneInfo("UTC"))+++def _bounded_logs_client(send_receive_timeout: int) -> AbstractContextManager:+ """A logs cluster client whose socket reads and writes time out, as the shared pool's do not.++ Mirrors the file-backed branch of `get_pool`, so a token-authenticated user stays token-aware.+ `make_ch_pool` is cached on its arguments, so every call reuses one pool.+ """+ kwargs = get_kwargs_for_client(workload=Workload.LOGS, ch_user=ClickHouseUser.DEFAULT)+ creds = connection_creds(Workload.LOGS, ClickHouseUser.DEFAULT)+ if creds is not None and is_file_backed_user(creds, kwargs.get("user")):+ kwargs.pop("password", None)+ pool = make_ch_pool(+ send_receive_timeout=send_receive_timeout, credential_provider=creds.read_password, **kwargs+ )+ else:+ pool = make_ch_pool(send_receive_timeout=send_receive_timeout, **kwargs)+ return pool.get_client()+++def evaluation_time(due_at: dt.datetime, freshness: dt.datetime | None) -> dt.datetime:+ """The instant a check reads: its due time, held back to where ingestion has reached.++ Anchored on the due time rather than the clock, so a retried check sends the same query. Without+ a freshness the check reads at its due time, and the caller treats a missing series as unknown.+ """+ return due_at if freshness is None else min(due_at, freshness)diff --git a/products/metrics/backend/facade/temporal.py b/products/metrics/backend/facade/temporal.pyindex fdd23f9bf047..c222c4e3839a 100644--- a/products/metrics/backend/facade/temporal.py+++ b/products/metrics/backend/facade/temporal.py@@ -2,8 +2,18 @@activities and schedule."""from products.metrics.backend.temporal import ACTIVITIES, WORKFLOWS+from products.metrics.backend.temporal.alert_evaluate import (+ METRICS_ALERT_EVALUATION_ACTIVITIES,+ METRICS_ALERT_EVALUATION_WORKFLOWS,+)from products.metrics.backend.temporal.schedule import (create_suggested_dashboards_discovery_schedule as create_suggested_dashboards_discovery_schedule,)-__all__ = ["ACTIVITIES", "WORKFLOWS", "create_suggested_dashboards_discovery_schedule"]+__all__ = [+ "ACTIVITIES",+ "METRICS_ALERT_EVALUATION_ACTIVITIES",+ "METRICS_ALERT_EVALUATION_WORKFLOWS",+ "WORKFLOWS",+ "create_suggested_dashboards_discovery_schedule",+]diff --git a/products/metrics/backend/promql.py b/products/metrics/backend/promql.pyindex 5791a5f8c5cc..590f37c3d06c 100644--- a/products/metrics/backend/promql.py+++ b/products/metrics/backend/promql.py@@ -32,7 +32,14 @@ _SHARED_HISTOGRAM_QUANTILES = re.compile(class PromQLQueryError(ValueError):- """A PromQL query that Snuffle rejected or could not run. The message is safe to show."""+ """A PromQL query that Snuffle rejected or could not run. The message is safe to show.++ `transient` says the backend failed rather than the query, so the same query can succeed later.+ """++ def __init__(self, message: str, *, transient: bool = False) -> None:+ super().__init__(message)+ self.transient = transientdef _sample_value(raw: Any) -> float | None:@@ -79,25 +86,30 @@ def snuffle_payload(*,data: dict[str, str] | None = None,params: dict[str, list[str]] | None = None,+ timeout: float | None = None,) -> dict[str, Any]:"""Send one Prometheus API request to Snuffle and return its JSON body. Errors are safe to show."""try:- response = snuffle_request(team.pk, method, path, data=data, params=params)+ response = snuffle_request(team.pk, method, path, data=data, params=params, timeout=timeout)except SnuffleNotConfiguredError as exc:- raise PromQLQueryError("PromQL queries are not available on this PostHog instance.") from exc+ # Deployment config, not the query, so an alert waiting on it is not counted as broken.+ raise PromQLQueryError("PromQL queries are not available on this PostHog instance.", transient=True) from excexcept requests.Timeout as exc:- raise PromQLQueryError("The PromQL query timed out. Try a shorter date range.") from exc+ raise PromQLQueryError("The PromQL query timed out. Try a shorter date range.", transient=True) from excexcept requests.RequestException as exc:- raise PromQLQueryError("The PromQL query backend cannot be reached. Try again later.") from exc+ raise PromQLQueryError("The PromQL query backend cannot be reached. Try again later.", transient=True) from excif response.status_code in (401, 403) or response.status_code >= 500:- raise PromQLQueryError("The PromQL query backend failed. Try again later.")+ raise PromQLQueryError("The PromQL query backend failed. Try again later.", transient=True)+ # Snuffle's rate limit, which passes once the budget refills.+ if response.status_code == 429:+ raise PromQLQueryError(_error_message(response), transient=True)if response.status_code >= 400:raise PromQLQueryError(_error_message(response))try:payload = response.json()except ValueError as exc:- raise PromQLQueryError("The PromQL query backend sent a response that cannot be read.") from exc+ raise PromQLQueryError("The PromQL query backend sent a response that cannot be read.", transient=True) from excif not isinstance(payload, dict) or payload.get("status") != "success":raise PromQLQueryError(_error_message(response))return payloaddiff --git a/products/metrics/backend/temporal/alert_evaluate.py b/products/metrics/backend/temporal/alert_evaluate.pynew file mode 100644index 000000000000..aace9d9b70fa--- /dev/null+++ b/products/metrics/backend/temporal/alert_evaluate.py@@ -0,0 +1,84 @@+"""Temporal wiring for metrics alerts on the shared alerts platform.++The evaluation itself lives in `products/metrics/backend/alerts/evaluation.py`, so it stays a plain+function a test can call without Temporal.+"""++import datetime as dt++from temporalio import activity, workflow+from temporalio.common import RetryPolicy++from posthog.temporal.common.base import PostHogWorkflow++with workflow.unsafe.imports_passed_through():+ from posthog.temporal.common.utils import close_db_connections++ from products.alerts_platform.backend.facade.contracts import (+ RECORD_OUTCOMES_ACTIVITY,+ SourceBatchEvaluation,+ SourceEvaluationInputs,+ SourceOutcomeInputs,+ )+ from products.alerts_platform.backend.facade.payload import outcome_chunks+ from products.alerts_platform.backend.facade.temporal import start_deliveries++WORKFLOW_NAME = "metrics-alert-evaluate"++# Above `BATCH_QUERY_BUDGET_SECONDS` and `FRESHNESS_QUERY_SECONDS`, so the evaluation stops starting+# queries before the activity times out.+EVALUATE_START_TO_CLOSE = dt.timedelta(seconds=40)+EVALUATE_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=60)+RECORD_START_TO_CLOSE = dt.timedelta(seconds=8)+RECORD_SCHEDULE_TO_CLOSE = dt.timedelta(seconds=12)+++@close_db_connections+def evaluate_metrics_alerts_activity(inputs: SourceEvaluationInputs) -> SourceBatchEvaluation:+ """Sync, so the thread this blocks is a slot Temporal is accounting for."""+ # Imported in the body, because the workflow below makes this module load inside Temporal's+ # sandbox, which refuses a Django model import.+ from products.metrics.backend.alerts.evaluation import evaluate_metrics_alert_batch # noqa: PLC0415++ return evaluate_metrics_alert_batch(+ inputs.batch_key.team_id, inputs.batch_key.slot, dt.datetime.fromisoformat(inputs.cutoff)+ )++[email protected](name=WORKFLOW_NAME)+class MetricsAlertEvaluateWorkflow(PostHogWorkflow):+ """Evaluates one batch key, records what it decided, and starts the deliveries it asked for."""++ inputs_cls = SourceEvaluationInputs++ @workflow.run+ async def run(self, inputs: SourceEvaluationInputs) -> int:+ # One attempt. The evaluation writes nothing, so a lost attempt leaves the checks due and+ # the next tick reads them again.+ evaluation = await workflow.execute_activity(+ evaluate_metrics_alerts_activity,+ inputs,+ start_to_close_timeout=EVALUATE_START_TO_CLOSE,+ schedule_to_close_timeout=EVALUATE_SCHEDULE_TO_CLOSE,+ retry_policy=RetryPolicy(maximum_attempts=1),+ )+ started = 0+ for chunk in outcome_chunks(evaluation.outcomes):+ await workflow.execute_activity(+ RECORD_OUTCOMES_ACTIVITY,+ SourceOutcomeInputs(team_id=inputs.batch_key.team_id, cutoff=inputs.cutoff, outcomes=chunk),+ start_to_close_timeout=RECORD_START_TO_CLOSE,+ schedule_to_close_timeout=RECORD_SCHEDULE_TO_CLOSE,+ retry_policy=RetryPolicy(maximum_attempts=3),+ )+ # Started only after the record, because a delivery reads the history row the record writes.+ recorded = {str(outcome.configuration_id) for outcome in chunk}+ started += await start_deliveries(+ [delivery for delivery in evaluation.deliveries if delivery.configuration_id in recorded]+ )+ return started+++METRICS_ALERT_EVALUATION_WORKFLOWS = [MetricsAlertEvaluateWorkflow]+METRICS_ALERT_EVALUATION_ACTIVITIES = [evaluate_metrics_alerts_activity]diff --git a/products/metrics/backend/tests/test_alerts.py b/products/metrics/backend/tests/test_alerts.pyindex 84ae58e88f37..0f4c764b8d51 100644--- a/products/metrics/backend/tests/test_alerts.py+++ b/products/metrics/backend/tests/test_alerts.py@@ -1,15 +1,38 @@+import json+from datetime import UTC, datetime, timedeltafrom typing import Any+from uuid import UUID-from posthog.test.base import APIBaseTest-from unittest.mock import patch+from posthog.test.base import APIBaseTest, ClickhouseTestMixin+from unittest.mock import MagicMock, patch-from django.test import SimpleTestCase+from django.db import connection+from django.test import SimpleTestCase, override_settings+from django.utils import timezone+import requestsfrom parameterized import parameterizedfrom pydantic import ValidationErrorfrom rest_framework import status+from posthog.models import Team+from posthog.models.scoping import team_scope+from posthog.temporal.alerts.admission import hold_evaluation_slot, release_evaluation_slot++from products.alerts_platform.backend.facade import testing as platform_testing+from products.alerts_platform.backend.facade.api import create_configuration, record_outcomes, slot_of+from products.alerts_platform.backend.facade.contracts import (+ AlertEventKind,+ Grouping,+ GroupingMode,+ PlatformAlertConfigurationEdit,+ PlatformAlertOutcome,+ SourceKind,+)+from products.metrics.backend.alerts.evaluation import ABSENT_RESOLVE_AFTER, INFLIGHT_KEY, evaluate_metrics_alert_batch+from products.metrics.backend.alerts.freshness import evaluation_time, fetch_metrics_freshnessfrom products.metrics.backend.alerts.source_config import MetricsPlatformAlertConfig+from products.metrics.backend.tests._seeder import seed_metric, truncate_metrics_tables_QUERY = 'sum(kafka_consumer_group_lag{group="example"})'@@ -61,3 +84,212 @@ class TestMetricsAlertCreate(APIBaseTest):assert response.status_code == status.HTTP_403_FORBIDDEN, response.json()assert response.json()["code"] == "feature_flag_required"+++class TestEvaluationTime(SimpleTestCase):+ @parameterized.expand(+ [+ ("ingestion_slightly_behind", timedelta(seconds=-30), timedelta(seconds=-30)),+ ("ingestion_lagging", timedelta(minutes=-20), timedelta(minutes=-20)),+ ("ingestion_past_the_due_time", timedelta(seconds=30), timedelta(0)),+ ("freshness_unknown", None, timedelta(0)),+ ]+ )+ def test_a_check_reads_no_later_than_ingestion_reached(+ self, _name: str, freshness_offset: timedelta | None, expected_offset: timedelta+ ) -> None:+ due_at = datetime(2026, 1, 1, 12, 0, tzinfo=UTC)+ freshness = None if freshness_offset is None else due_at + freshness_offset++ assert evaluation_time(due_at, freshness) == due_at + expected_offset+++def _vector(*series: tuple[dict[str, str], str]) -> dict[str, Any]:+ return {+ "status": "success",+ "data": {+ "resultType": "vector",+ "result": [{"metric": labels, "value": [0, value]} for labels, value in series],+ },+ }+++def _response(status_code: int, body: dict[str, Any]) -> requests.Response:+ response = requests.Response()+ response.status_code = status_code+ response._content = json.dumps(body).encode()+ return response+++class TestMetricsFreshnessRead(ClickhouseTestMixin, APIBaseTest):+ def setUp(self) -> None:+ super().setUp()+ truncate_metrics_tables()++ def test_reads_only_the_teams_own_newest_point(self) -> None:+ # The query compares with ClickHouse's now(), so the points sit just behind the real clock.+ now = datetime.now(UTC).replace(microsecond=0)+ other_team = Team.objects.create(organization=self.organization)+ seed_metric(team_id=self.team.id, metric_name="lag", points=[(now - timedelta(minutes=2), 1.0)])+ seed_metric(team_id=other_team.id, metric_name="lag", points=[(now - timedelta(seconds=30), 1.0)])++ assert fetch_metrics_freshness(self.team, max_execution_time=5) == now - timedelta(minutes=2)++ def test_points_dated_ahead_cannot_rewind_the_read_to_an_older_service(self) -> None:+ now = datetime.now(UTC).replace(microsecond=0)+ seed_metric(+ team_id=self.team.id, metric_name="lag", service_name="decoy", points=[(now - timedelta(minutes=14), 1.0)]+ )+ seed_metric(+ team_id=self.team.id, metric_name="lag", service_name="live", points=[(now + timedelta(minutes=5), 1.0)]+ )++ freshness = fetch_metrics_freshness(self.team, max_execution_time=5)++ assert freshness is not None+ assert now - timedelta(seconds=5) <= freshness <= datetime.now(UTC)+++class TestMetricsAlertEvaluation(APIBaseTest):+ def setUp(self) -> None:+ super().setUp()+ self.cutoff = timezone.now().replace(second=0, microsecond=0) + timedelta(minutes=5)+ self.snuffle = MagicMock()+ self.enterContext(patch("products.metrics.backend.promql.snuffle_request", self.snuffle))+ self.freshness = self.enterContext(+ patch("products.metrics.backend.alerts.evaluation.fetch_metrics_freshness", return_value=None)+ )+ with team_scope(self.team.id):+ self.configuration_id = create_configuration(+ team_id=self.team.id,+ source_kind=SourceKind.METRICS,+ edit=PlatformAlertConfigurationEdit(+ name="Consumer lag",+ enabled=True,+ source_config={"query": _QUERY, "condition": {"operator": "gt", "threshold": 1000}},+ grouping=Grouping(mode=GroupingMode.BY_RESULT_LABELS, keys=("group",)),+ schedule_restriction=None,+ evaluation_periods=1,+ datapoints_to_alarm=1,+ cooldown_minutes=0,+ ),+ created_by_id=self.user.id,+ ).id+ platform_testing.set_due_at(self.configuration_id, self.due_at)++ @property+ def due_at(self) -> datetime:+ return self.cutoff - timedelta(minutes=1)++ def _evaluate(self) -> PlatformAlertOutcome:+ with team_scope(self.team.id):+ evaluation = evaluate_metrics_alert_batch(self.team.id, slot_of(self.due_at, self.cutoff), self.cutoff)+ (outcome,) = evaluation.outcomes+ return outcome++ def _next_check(self, outcome: PlatformAlertOutcome) -> None:+ with team_scope(self.team.id):+ record_outcomes(self.team.id, [outcome], self.cutoff)+ self.cutoff += timedelta(minutes=5)+ platform_testing.set_due_at(self.configuration_id, self.due_at)++ def _kinds(self, outcome: PlatformAlertOutcome) -> dict[str, AlertEventKind]:+ return {+ json.loads(group.grouping_key)["group"]: group.kind+ for group in outcome.groups+ if group.kind != AlertEventKind.CHECK+ }++ def test_a_missing_series_holds_until_it_has_been_gone_an_hour_with_fresh_ingestion(self) -> None:+ self.freshness.return_value = self.due_at - timedelta(seconds=30)+ self.snuffle.return_value = _response(+ 200, _vector(({"group": "a"}, "2000"), ({"group": "b"}, "3000"), ({"group": "c"}, "10"))+ )+ first = self._evaluate()++ assert self._kinds(first) == {"a": AlertEventKind.FIRING, "b": AlertEventKind.FIRING}+ assert self.snuffle.call_args.kwargs["data"]["time"] == str((self.due_at - timedelta(seconds=30)).timestamp())++ self._next_check(first)+ self.snuffle.return_value = _response(+ 200, _vector(({"group": "b"}, "NaN"), ({"group": "b"}, "10"), ({"group": "c"}, "10"))+ )+ held = self._evaluate()++ assert self._kinds(held) == {}++ self._next_check(held)+ self.cutoff += ABSENT_RESOLVE_AFTER+ with team_scope(self.team.id):+ platform_testing.set_due_at(self.configuration_id, self.due_at)+ self.snuffle.return_value = _response(200, _vector(({"group": "b"}, "NaN"), ({"group": "c"}, "10")))+ self.freshness.return_value = None++ assert self._kinds(self._evaluate()) == {}++ self.freshness.return_value = self.due_at - timedelta(seconds=30)+ resolved = self._evaluate()++ assert self._kinds(resolved) == {"a": AlertEventKind.RESOLVED}++ self._next_check(resolved)+ self.snuffle.return_value = _response(200, _vector(({"group": "c"}, "10")))+ self.freshness.return_value = self.due_at - timedelta(seconds=30)++ assert self._kinds(self._evaluate()) == {}++ def test_a_firing_held_back_by_a_mute_is_not_announced_on_a_value_with_no_number(self) -> None:+ self.snuffle.return_value = _response(200, _vector(({"group": "a"}, "2000")))+ first = self._evaluate()+ self._next_check(first)+ # The state a mute leaves: firing, with the announcement still owed.+ with connection.cursor() as cursor:+ cursor.execute(+ "UPDATE alerts_platformalert SET last_notified_at = NULL WHERE configuration_id = %s",+ [str(self.configuration_id)],+ )+ self.snuffle.return_value = _response(200, _vector(({"group": "a"}, "NaN")))++ assert self._kinds(self._evaluate()) == {}++ @override_settings(ALERTS_PLATFORM_METRICS_MAX_INFLIGHT_BATCHES=1)+ def test_a_batch_over_the_pool_limit_stays_due_for_a_later_tick(self) -> None:+ self.snuffle.return_value = _response(200, _vector(({"group": "a"}, "2000")))+ slot = slot_of(self.due_at, self.cutoff)+ held_until = hold_evaluation_slot("other-batch", limit=1, key=INFLIGHT_KEY)+ assert held_until is not None+ with team_scope(self.team.id):+ deferred = evaluate_metrics_alert_batch(self.team.id, slot, self.cutoff)+ release_evaluation_slot("other-batch", held_until=held_until, key=INFLIGHT_KEY)++ assert (deferred.outcomes, deferred.omitted, self.snuffle.call_count) == ((), 1, 0)+ assert self._kinds(self._evaluate()) == {"a": AlertEventKind.FIRING}+ assert self._kinds(self._evaluate()) == {"a": AlertEventKind.FIRING}++ @parameterized.expand(+ [+ ("rejected_query_counts_toward_broken", _response(400, {"status": "error", "error": "parse error"}), 1),+ ("backend_failure_is_transient", _response(503, {"status": "error"}), 0),+ ("timeout_is_transient", requests.Timeout(), 0),+ (+ "too_many_series_counts_toward_broken",+ _response(200, _vector(*(({"group": str(index)}, "1") for index in range(1001)))),+ 1,+ ),+ ]+ )+ def test_a_failed_query_records_a_failure(+ self, _name: str, result: requests.Response | Exception, consecutive_failures: int+ ) -> None:+ if isinstance(result, Exception):+ self.snuffle.side_effect = result+ else:+ self.snuffle.return_value = result++ outcome = self._evaluate()++ assert outcome.failure is not None+ assert (outcome.configuration_id, outcome.consecutive_failures) == (+ UUID(str(self.configuration_id)),+ consecutive_failures,+ )