Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): measure alert firing signals dropped by the cap (#129900)

posthog[bot]daf3c0c89bca
diff --git a/posthog/temporal/alerts/alert_firing_signal.py b/posthog/temporal/alerts/alert_firing_signal.py
index 2fe035915cec..84867990fb75 100644
--- a/posthog/temporal/alerts/alert_firing_signal.py
+++ b/posthog/temporal/alerts/alert_firing_signal.py
@@ -11,12 +11,17 @@ import enum
import asyncio
import structlog
+import posthoganalytics
+from redis.exceptions import RedisError
from posthog.schema import AlertState
from posthog.dataclasses import frozen
+from posthog.event_usage import groups
from posthog.models import Team
+from posthog.redis import get_client
from posthog.temporal.alerts.investigation import investigates
+from posthog.temporal.alerts.metrics import record_alert_firing_signal_outcome
from posthog.token_bucket import BucketUnavailable, Budget, consume, refund
from posthog.utils import absolute_uri
@@ -64,14 +69,28 @@ class AlertFiringSkipReason(enum.StrEnum):
NOT_DISPATCHED = "not_dispatched"
-# Nearly every check fails one of these, so a log line for them would only be noise.
+# Nearly every check fails one of these, so a log line or an outcome count for them would only be noise.
_UNLOGGED_SKIP_REASONS = frozenset({AlertFiringSkipReason.NOT_NEW_EPISODE, AlertFiringSkipReason.INVESTIGATED})
+EMITTED_OUTCOME = "emitted"
+CAP_DROP_EVENT = "alert firing signal skipped"
+_CAP_DROP_REASONS = frozenset({AlertFiringSkipReason.CAP, AlertFiringSkipReason.CAP_UNAVAILABLE})
+# Longer than every attempt of the emit activity together, so a retry finds the outcome of the first attempt.
+_OUTCOME_KEY_TTL_SECONDS = 60 * 60
+
def cap_key(team_id: int) -> str:
return f"signals_alert_firing_rate:{team_id}"
+def _outcome_key(source_id: str) -> str:
+ return f"signals_alert_firing_outcome:{source_id}"
+
+
+def _source_id(alert_id: str, alert_check_id: str) -> str:
+ return f"{alert_id}:{alert_check_id}"
+
+
def alert_firing_skip_reason(
alert: AlertConfiguration, *, previous_state: str, new_state: str
) -> AlertFiringSkipReason | None:
@@ -111,7 +130,8 @@ def prepare_alert_firing_signal(
)
return None
if skip_reason is not None and skip_reason not in _UNLOGGED_SKIP_REASONS:
- _log_skip(str(alert.id), str(alert_check.id), skip_reason)
+ _log_skip(alert.team_id, str(alert.id), str(alert_check.id), skip_reason)
+ _record_outcome(alert.team, str(alert.id), str(alert_check.id), skip_reason)
return signal
@@ -122,7 +142,7 @@ async def dispatch_alert_firing_signal(signal: AlertFiringSignal) -> bool:
default executor, so a call that blocks an executor thread on it can deadlock once every executor
thread waits the same way. A failure never changes the committed check.
"""
- source_id = f"{signal.alert_id}:{signal.alert_check_id}"
+ source_id = _source_id(signal.alert_id, signal.alert_check_id)
try:
async with asyncio.timeout(EMISSION_TIMEOUT_SECONDS):
dispatched = await emit_signal(
@@ -148,15 +168,68 @@ async def dispatch_alert_firing_signal(signal: AlertFiringSignal) -> bool:
dispatched = None
if dispatched:
logger.info("alert_firing.signal_emitted", alert_id=signal.alert_id, alert_check_id=signal.alert_check_id)
+ await asyncio.to_thread(_record_outcome, signal.team, signal.alert_id, signal.alert_check_id, None)
return True
await asyncio.to_thread(refund, cap_key(signal.team.id), CAP_BUDGET)
if dispatched is False:
- _log_skip(signal.alert_id, signal.alert_check_id, AlertFiringSkipReason.NOT_DISPATCHED)
+ reason = AlertFiringSkipReason.NOT_DISPATCHED
+ _log_skip(signal.team.id, signal.alert_id, signal.alert_check_id, reason)
+ await asyncio.to_thread(_record_outcome, signal.team, signal.alert_id, signal.alert_check_id, reason)
return False
-def _log_skip(alert_id: str, alert_check_id: str, reason: AlertFiringSkipReason) -> None:
- logger.info("alert_firing.signal_skipped", alert_id=alert_id, alert_check_id=alert_check_id, reason=reason.value)
+def _log_skip(team_id: int, alert_id: str, alert_check_id: str, reason: AlertFiringSkipReason) -> None:
+ logger.info(
+ "alert_firing.signal_skipped",
+ team_id=team_id,
+ alert_id=alert_id,
+ alert_check_id=alert_check_id,
+ reason=reason.value,
+ )
+
+
+def _record_outcome(team: Team, alert_id: str, alert_check_id: str, skip_reason: AlertFiringSkipReason | None) -> None:
+ """Count the outcome of an episode once, and capture a cap drop for the team. Never raises.
+
+ Call it from a thread. A cancelled attempt records nothing, because its retry decides again.
+ """
+ source_id = _source_id(alert_id, alert_check_id)
+ try:
+ if not _claims_outcome(source_id):
+ return
+ record_alert_firing_signal_outcome(EMITTED_OUTCOME if skip_reason is None else skip_reason.value)
+ if skip_reason is not None and skip_reason in _CAP_DROP_REASONS:
+ posthoganalytics.capture(
+ event=CAP_DROP_EVENT,
+ distinct_id=str(team.uuid),
+ properties={
+ "source_product": SIGNAL_SOURCE_PRODUCT,
+ "source_type": SIGNAL_SOURCE_TYPE,
+ "source_id": source_id,
+ "team_id": team.id,
+ "alert_id": alert_id,
+ "alert_check_id": alert_check_id,
+ "reason": skip_reason.value,
+ },
+ groups=groups(team.organization, team),
+ )
+ except Exception:
+ logger.exception(
+ "alert_firing.signal_outcome_failed", team_id=team.id, alert_id=alert_id, alert_check_id=alert_check_id
+ )
+
+
+def _claims_outcome(source_id: str) -> bool:
+ """Whether this attempt is the first to record an outcome for the episode.
+
+ It uses the emission idempotency key, so an activity retry never counts the episode twice,
+ for example as both `emitted` and `cap`.
+ """
+ try:
+ return bool(get_client().set(_outcome_key(source_id), 1, nx=True, ex=_OUTCOME_KEY_TTL_SECONDS))
+ except RedisError:
+ # Without Redis the cap fails closed as `cap_unavailable`. Counting that drop twice beats losing it.
+ return True
def _resumes_interrupted_episode(alert: AlertConfiguration, alert_check: AlertCheck) -> bool:
diff --git a/posthog/temporal/alerts/metrics.py b/posthog/temporal/alerts/metrics.py
index e2b1aec179ff..ce50cd81e34d 100644
--- a/posthog/temporal/alerts/metrics.py
+++ b/posthog/temporal/alerts/metrics.py
@@ -20,6 +20,18 @@ def record_ai_detector_check_outcome(outcome: AiDetectorCheckOutcome) -> None:
_AI_DETECTOR_CHECK_COUNTER.labels(outcome=outcome).inc()
+# No team label, to bound cardinality. The `alert firing signal skipped` event carries the team for cap drops.
+_ALERT_FIRING_SIGNAL_COUNTER = Counter(
+ "posthog_insight_alerts_firing_signals_total",
+ "Insight alert firing signals that passed the episode gates, by outcome",
+ labelnames=["outcome"],
+)
+
+
+def record_alert_firing_signal_outcome(outcome: str) -> None:
+ _ALERT_FIRING_SIGNAL_COUNTER.labels(outcome=outcome).inc()
+
+
def record_due_insight_alert_metrics(due_count: int, oldest_due_at: datetime | None, polled_at: datetime) -> None:
oldest_due_age_seconds: float = 0.0
if oldest_due_at is not None:
diff --git a/posthog/temporal/tests/test_alerts_alert_firing_signal.py b/posthog/temporal/tests/test_alerts_alert_firing_signal.py
index 526b162a16ee..f04767b87ee9 100644
--- a/posthog/temporal/tests/test_alerts_alert_firing_signal.py
+++ b/posthog/temporal/tests/test_alerts_alert_firing_signal.py
@@ -6,12 +6,20 @@ from unittest.mock import AsyncMock, patch
from asgiref.sync import async_to_sync
from parameterized import parameterized
+from prometheus_client import REGISTRY
from posthog.schema import AlertState
from posthog.redis import TEST_clear_clients
-from posthog.temporal.alerts.alert_firing_signal import dispatch_alert_firing_signal, prepare_alert_firing_signal
-from posthog.token_bucket import Budget, TEST_reset_scripts
+from posthog.temporal.alerts.alert_firing_signal import (
+ CAP_DROP_EVENT,
+ EMITTED_OUTCOME,
+ AlertFiringSkipReason,
+ cap_key,
+ dispatch_alert_firing_signal,
+ prepare_alert_firing_signal,
+)
+from posthog.token_bucket import BucketUnavailable, Budget, TEST_reset_scripts, consume
from products.alerts.backend.models.alert import AlertCheck, AlertConfiguration
from products.product_analytics.backend.facade.models import Insight
@@ -19,6 +27,21 @@ from products.signals.backend.facade.api import validate_signal_input
from products.signals.backend.facade.source_configs import AlertNamePatterns, SignalSourceConfigSnapshot
_MODULE = "posthog.temporal.alerts.alert_firing_signal"
+_ONE_TOKEN = Budget(burst=1, per_hour=1)
+_OUTCOMES = [EMITTED_OUTCOME, *(reason.value for reason in AlertFiringSkipReason)]
+
+
+def _outcome_counts() -> dict[str, float]:
+ return {
+ outcome: REGISTRY.get_sample_value("posthog_insight_alerts_firing_signals_total", {"outcome": outcome}) or 0.0
+ for outcome in _OUTCOMES
+ }
+
+
+def _counted_since(before: dict[str, float]) -> dict[str, float]:
+ return {
+ outcome: count - before[outcome] for outcome, count in _outcome_counts().items() if count != before[outcome]
+ }
def _source(enabled: bool) -> SignalSourceConfigSnapshot:
@@ -192,3 +215,96 @@ class TestEmitAlertFiringSignal(BaseTest):
assert filtered_emitted is False
filtered_emit.assert_not_awaited()
assert passing_emitted is True
+
+ def _spend_the_only_token(self) -> None:
+ consume(cap_key(self.team.id), _ONE_TOKEN)
+
+ @parameterized.expand(
+ [
+ ("emitted", {}, {}, None, EMITTED_OUTCOME),
+ ("not_new_episode_is_not_counted", {"previous_state": AlertState.FIRING}, {}, None, None),
+ (
+ "investigated_is_not_counted",
+ {},
+ {"investigation_agent_enabled": True, "detector_config": {"type": "zscore"}},
+ None,
+ None,
+ ),
+ ("source_disabled", {"source_enabled": False}, {}, None, "source_disabled"),
+ ("scope", {"scope": "investigated_only"}, {}, None, "scope"),
+ ("filtered", {"patterns": AlertNamePatterns(block=("dropped$",))}, {}, None, "filtered"),
+ ("filter_invalid", {"patterns": AlertNamePatterns(invalid=True)}, {}, None, "filter_invalid"),
+ ("cap", {}, {}, "spend", "cap"),
+ ("cap_unavailable", {}, {}, "unavailable", "cap_unavailable"),
+ ("not_dispatched", {"emit": AsyncMock(return_value=False)}, {}, None, "not_dispatched"),
+ ]
+ )
+ def test_records_each_counted_outcome_once(
+ self, _name: str, emit_kwargs: dict, alert_kwargs: dict, cap_state: str | None, expected: str | None
+ ) -> None:
+ alert, check = self._alert_and_check(**alert_kwargs)
+ before = _outcome_counts()
+
+ with (
+ patch(f"{_MODULE}.CAP_BUDGET", _ONE_TOKEN),
+ patch(f"{_MODULE}.posthoganalytics.capture") as capture,
+ patch(f"{_MODULE}.consume", return_value=BucketUnavailable(error="down"))
+ if cap_state == "unavailable"
+ else contextlib.nullcontext(),
+ ):
+ if cap_state == "spend":
+ self._spend_the_only_token()
+ self._emit(alert, check, **emit_kwargs)
+
+ assert _counted_since(before) == ({expected: 1.0} if expected else {})
+ if expected in ("cap", "cap_unavailable"):
+ capture.assert_called_once()
+ assert capture.call_args.kwargs["event"] == CAP_DROP_EVENT
+ assert capture.call_args.kwargs["properties"] == {
+ "source_product": "analytics",
+ "source_type": "alert_firing",
+ "source_id": f"{alert.id}:{check.id}",
+ "team_id": self.team.id,
+ "alert_id": str(alert.id),
+ "alert_check_id": str(check.id),
+ "reason": expected,
+ }
+ else:
+ capture.assert_not_called()
+
+ def test_retried_attempt_does_not_count_the_episode_twice(self) -> None:
+ alert, check = self._alert_and_check()
+ before = _outcome_counts()
+
+ with (
+ patch(f"{_MODULE}.CAP_BUDGET", _ONE_TOKEN),
+ patch(f"{_MODULE}.posthoganalytics.capture") as capture,
+ ):
+ first, _ = self._emit(alert, check)
+ retried, _ = self._emit(alert, check)
+
+ assert (first, retried) == (True, False)
+ assert _counted_since(before) == {EMITTED_OUTCOME: 1.0}
+ capture.assert_not_called()
+
+ @parameterized.expand(
+ [
+ ("failed_capture_on_a_cap_drop", "spend", "posthoganalytics.capture", False),
+ ("failed_counter_on_an_emitted_signal", None, "record_alert_firing_signal_outcome", True),
+ ]
+ )
+ def test_telemetry_failure_does_not_change_the_result(
+ self, _name: str, cap_state: str | None, failing: str, expected: bool
+ ) -> None:
+ alert, check = self._alert_and_check()
+
+ with (
+ patch(f"{_MODULE}.CAP_BUDGET", _ONE_TOKEN),
+ patch(f"{_MODULE}.{failing}", side_effect=RuntimeError("telemetry down")),
+ ):
+ if cap_state == "spend":
+ self._spend_the_only_token()
+ emitted, emit = self._emit(alert, check)
+
+ assert emitted is expected
+ assert emit.await_count == int(expected)
diff --git a/products/signals/ARCHITECTURE.md b/products/signals/ARCHITECTURE.md
index 38cf7ceb3fa9..90204636c896 100644
--- a/products/signals/ARCHITECTURE.md
+++ b/products/signals/ARCHITECTURE.md
@@ -678,7 +678,8 @@ Per-team configuration for which signal sources are enabled.
- The product analytics source (`analytics` / `anomaly_investigation`) accepts `config.alert_scope`: `investigated_only` (the default when the key is absent) or `all_firing_alerts`. The serializer rejects any other value, and rejects the key on every other source (`contracts.ALERT_SCOPE_SOURCE`).
Other products read the scope through `facade.source_configs.get_analytics_alert_scope(team_id=...)`, never the row. The read returns `investigated_only` unless the stored value is `all_firing_alerts` and the `signals-analytics-all-alerts` flag is on for the team's organization. The flag check fails closed.
The read does not check `enabled`: the emitter still gates on `is_source_enabled`.
- With `all_firing_alerts`, insight alert evaluation (`posthog/temporal/alerts/alert_firing_signal.py`) emits `analytics` / `alert_firing` when an alert that the investigation agent does not investigate starts a firing episode. It emits once per episode (source id and idempotency key `<alert id>:<first check id>`), and a failure never changes the check. The evaluation thread runs the gates after the check is committed. The activity awaits the emission on its event loop once that thread returns, because `emit_signal` needs the default executor that the thread runs on. A per-team token bucket caps it, and an unavailable bucket skips the signal. Each skip after a read logs `alert_firing.signal_skipped` with a reason. Alerts the agent investigates keep the verdict gate in both scopes.
+ With `all_firing_alerts`, insight alert evaluation (`posthog/temporal/alerts/alert_firing_signal.py`) emits `analytics` / `alert_firing` when an alert that the investigation agent does not investigate starts a firing episode. It emits once per episode (source id and idempotency key `<alert id>:<first check id>`), and a failure never changes the check. The evaluation thread runs the gates after the check is committed. The activity awaits the emission on its event loop once that thread returns, because `emit_signal` needs the default executor that the thread runs on. A per-team token bucket caps it, and an unavailable bucket skips the signal. Each skip after a read logs `alert_firing.signal_skipped` with the team and a reason. Alerts the agent investigates keep the verdict gate in both scopes.
+ The `posthog_insight_alerts_firing_signals_total` counter records one `outcome` per episode that passes the episode gates (`emitted`, `source_disabled`, `scope`, `filtered`, `filter_invalid`, `cap`, `cap_unavailable`, `not_dispatched`). It has no team label. A drop by the cap (`cap` or `cap_unavailable`) also captures `alert firing signal skipped` with `team_id`, `alert_id` and `reason`, which gives the per-team view for sizing the cap. A Redis key on the source id lets only the first attempt of an episode record, so an activity retry never counts it twice. A cancelled attempt records nothing. A failure to record never changes the check or the signal.
`alert_firing` has no config row of its own: `is_source_enabled` checks the `anomaly_investigation` row for it, so the one product analytics toggle turns both off.
Guidance written on that row filters both `anomaly_investigation` and `alert_firing`, through the direct steering gate.
The same row accepts `config.alert_allow_patterns` and `config.alert_block_patterns`: lists of RE2 patterns (at most `ALERT_PATTERNS_MAX_COUNT`, each at most `ALERT_PATTERN_MAX_LENGTH` characters) matched with search semantics against the alert name and the insight name. When allow patterns are set an alert must match one, and a block match drops the alert even when it also matches an allow pattern. The serializer rejects a pattern that does not compile, and rejects both keys on every other source.
@@ -1209,6 +1210,7 @@ All events use `distinct_id = team.uuid` and `groups(organization, team)`. Per-s
**Lifecycle (in order):**
- `signal_data_source_entered` / `signal_data_source_summarized` / `signal_data_source_filtered` — data-source pipeline only (`pipeline.py`)
+- `alert firing signal skipped` — the per-team cap stopped an `analytics` / `alert_firing` signal before `emit_signal()` (+ `team_id`, `alert_id`, `alert_check_id`, `reason`: `cap` | `cap_unavailable`)
- `signal_emission_started` — `emit_signal()` past validation
- `signal_emitted` — `emit_signal()` after Temporal dispatch succeeds
- `signal_assigned_to_report` — grouping assigned the signal (+ `report_id`, `is_new_report`, `promoted`)