:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): measure alert firing signals dropped by the cap (#129900)
posthog[bot]daf3c0c89bca
diff --git a/posthog/temporal/alerts/alert_firing_signal.py b/posthog/temporal/alerts/alert_firing_signal.pyindex 2fe035915cec..84867990fb75 100644--- a/posthog/temporal/alerts/alert_firing_signal.py+++ b/posthog/temporal/alerts/alert_firing_signal.py@@ -11,12 +11,17 @@ import enumimport asyncioimport structlog+import posthoganalytics+from redis.exceptions import RedisErrorfrom posthog.schema import AlertStatefrom posthog.dataclasses import frozen+from posthog.event_usage import groupsfrom posthog.models import Team+from posthog.redis import get_clientfrom posthog.temporal.alerts.investigation import investigates+from posthog.temporal.alerts.metrics import record_alert_firing_signal_outcomefrom posthog.token_bucket import BucketUnavailable, Budget, consume, refundfrom posthog.utils import absolute_uri@@ -64,14 +69,28 @@ class AlertFiringSkipReason(enum.StrEnum):NOT_DISPATCHED = "not_dispatched"-# Nearly every check fails one of these, so a log line for them would only be noise.+# Nearly every check fails one of these, so a log line or an outcome count for them would only be noise._UNLOGGED_SKIP_REASONS = frozenset({AlertFiringSkipReason.NOT_NEW_EPISODE, AlertFiringSkipReason.INVESTIGATED})+EMITTED_OUTCOME = "emitted"+CAP_DROP_EVENT = "alert firing signal skipped"+_CAP_DROP_REASONS = frozenset({AlertFiringSkipReason.CAP, AlertFiringSkipReason.CAP_UNAVAILABLE})+# Longer than every attempt of the emit activity together, so a retry finds the outcome of the first attempt.+_OUTCOME_KEY_TTL_SECONDS = 60 * 60+def cap_key(team_id: int) -> str:return f"signals_alert_firing_rate:{team_id}"+def _outcome_key(source_id: str) -> str:+ return f"signals_alert_firing_outcome:{source_id}"+++def _source_id(alert_id: str, alert_check_id: str) -> str:+ return f"{alert_id}:{alert_check_id}"++def alert_firing_skip_reason(alert: AlertConfiguration, *, previous_state: str, new_state: str) -> AlertFiringSkipReason | None:@@ -111,7 +130,8 @@ def prepare_alert_firing_signal()return Noneif skip_reason is not None and skip_reason not in _UNLOGGED_SKIP_REASONS:- _log_skip(str(alert.id), str(alert_check.id), skip_reason)+ _log_skip(alert.team_id, str(alert.id), str(alert_check.id), skip_reason)+ _record_outcome(alert.team, str(alert.id), str(alert_check.id), skip_reason)return signal@@ -122,7 +142,7 @@ async def dispatch_alert_firing_signal(signal: AlertFiringSignal) -> bool:default executor, so a call that blocks an executor thread on it can deadlock once every executorthread waits the same way. A failure never changes the committed check."""- source_id = f"{signal.alert_id}:{signal.alert_check_id}"+ source_id = _source_id(signal.alert_id, signal.alert_check_id)try:async with asyncio.timeout(EMISSION_TIMEOUT_SECONDS):dispatched = await emit_signal(@@ -148,15 +168,68 @@ async def dispatch_alert_firing_signal(signal: AlertFiringSignal) -> bool:dispatched = Noneif dispatched:logger.info("alert_firing.signal_emitted", alert_id=signal.alert_id, alert_check_id=signal.alert_check_id)+ await asyncio.to_thread(_record_outcome, signal.team, signal.alert_id, signal.alert_check_id, None)return Trueawait asyncio.to_thread(refund, cap_key(signal.team.id), CAP_BUDGET)if dispatched is False:- _log_skip(signal.alert_id, signal.alert_check_id, AlertFiringSkipReason.NOT_DISPATCHED)+ reason = AlertFiringSkipReason.NOT_DISPATCHED+ _log_skip(signal.team.id, signal.alert_id, signal.alert_check_id, reason)+ await asyncio.to_thread(_record_outcome, signal.team, signal.alert_id, signal.alert_check_id, reason)return False-def _log_skip(alert_id: str, alert_check_id: str, reason: AlertFiringSkipReason) -> None:- logger.info("alert_firing.signal_skipped", alert_id=alert_id, alert_check_id=alert_check_id, reason=reason.value)+def _log_skip(team_id: int, alert_id: str, alert_check_id: str, reason: AlertFiringSkipReason) -> None:+ logger.info(+ "alert_firing.signal_skipped",+ team_id=team_id,+ alert_id=alert_id,+ alert_check_id=alert_check_id,+ reason=reason.value,+ )+++def _record_outcome(team: Team, alert_id: str, alert_check_id: str, skip_reason: AlertFiringSkipReason | None) -> None:+ """Count the outcome of an episode once, and capture a cap drop for the team. Never raises.++ Call it from a thread. A cancelled attempt records nothing, because its retry decides again.+ """+ source_id = _source_id(alert_id, alert_check_id)+ try:+ if not _claims_outcome(source_id):+ return+ record_alert_firing_signal_outcome(EMITTED_OUTCOME if skip_reason is None else skip_reason.value)+ if skip_reason is not None and skip_reason in _CAP_DROP_REASONS:+ posthoganalytics.capture(+ event=CAP_DROP_EVENT,+ distinct_id=str(team.uuid),+ properties={+ "source_product": SIGNAL_SOURCE_PRODUCT,+ "source_type": SIGNAL_SOURCE_TYPE,+ "source_id": source_id,+ "team_id": team.id,+ "alert_id": alert_id,+ "alert_check_id": alert_check_id,+ "reason": skip_reason.value,+ },+ groups=groups(team.organization, team),+ )+ except Exception:+ logger.exception(+ "alert_firing.signal_outcome_failed", team_id=team.id, alert_id=alert_id, alert_check_id=alert_check_id+ )+++def _claims_outcome(source_id: str) -> bool:+ """Whether this attempt is the first to record an outcome for the episode.++ It uses the emission idempotency key, so an activity retry never counts the episode twice,+ for example as both `emitted` and `cap`.+ """+ try:+ return bool(get_client().set(_outcome_key(source_id), 1, nx=True, ex=_OUTCOME_KEY_TTL_SECONDS))+ except RedisError:+ # Without Redis the cap fails closed as `cap_unavailable`. Counting that drop twice beats losing it.+ return Truedef _resumes_interrupted_episode(alert: AlertConfiguration, alert_check: AlertCheck) -> bool:diff --git a/posthog/temporal/alerts/metrics.py b/posthog/temporal/alerts/metrics.pyindex e2b1aec179ff..ce50cd81e34d 100644--- a/posthog/temporal/alerts/metrics.py+++ b/posthog/temporal/alerts/metrics.py@@ -20,6 +20,18 @@ def record_ai_detector_check_outcome(outcome: AiDetectorCheckOutcome) -> None:_AI_DETECTOR_CHECK_COUNTER.labels(outcome=outcome).inc()+# No team label, to bound cardinality. The `alert firing signal skipped` event carries the team for cap drops.+_ALERT_FIRING_SIGNAL_COUNTER = Counter(+ "posthog_insight_alerts_firing_signals_total",+ "Insight alert firing signals that passed the episode gates, by outcome",+ labelnames=["outcome"],+)+++def record_alert_firing_signal_outcome(outcome: str) -> None:+ _ALERT_FIRING_SIGNAL_COUNTER.labels(outcome=outcome).inc()++def record_due_insight_alert_metrics(due_count: int, oldest_due_at: datetime | None, polled_at: datetime) -> None:oldest_due_age_seconds: float = 0.0if oldest_due_at is not None:diff --git a/posthog/temporal/tests/test_alerts_alert_firing_signal.py b/posthog/temporal/tests/test_alerts_alert_firing_signal.pyindex 526b162a16ee..f04767b87ee9 100644--- a/posthog/temporal/tests/test_alerts_alert_firing_signal.py+++ b/posthog/temporal/tests/test_alerts_alert_firing_signal.py@@ -6,12 +6,20 @@ from unittest.mock import AsyncMock, patchfrom asgiref.sync import async_to_syncfrom parameterized import parameterized+from prometheus_client import REGISTRYfrom posthog.schema import AlertStatefrom posthog.redis import TEST_clear_clients-from posthog.temporal.alerts.alert_firing_signal import dispatch_alert_firing_signal, prepare_alert_firing_signal-from posthog.token_bucket import Budget, TEST_reset_scripts+from posthog.temporal.alerts.alert_firing_signal import (+ CAP_DROP_EVENT,+ EMITTED_OUTCOME,+ AlertFiringSkipReason,+ cap_key,+ dispatch_alert_firing_signal,+ prepare_alert_firing_signal,+)+from posthog.token_bucket import BucketUnavailable, Budget, TEST_reset_scripts, consumefrom products.alerts.backend.models.alert import AlertCheck, AlertConfigurationfrom products.product_analytics.backend.facade.models import Insight@@ -19,6 +27,21 @@ from products.signals.backend.facade.api import validate_signal_inputfrom products.signals.backend.facade.source_configs import AlertNamePatterns, SignalSourceConfigSnapshot_MODULE = "posthog.temporal.alerts.alert_firing_signal"+_ONE_TOKEN = Budget(burst=1, per_hour=1)+_OUTCOMES = [EMITTED_OUTCOME, *(reason.value for reason in AlertFiringSkipReason)]+++def _outcome_counts() -> dict[str, float]:+ return {+ outcome: REGISTRY.get_sample_value("posthog_insight_alerts_firing_signals_total", {"outcome": outcome}) or 0.0+ for outcome in _OUTCOMES+ }+++def _counted_since(before: dict[str, float]) -> dict[str, float]:+ return {+ outcome: count - before[outcome] for outcome, count in _outcome_counts().items() if count != before[outcome]+ }def _source(enabled: bool) -> SignalSourceConfigSnapshot:@@ -192,3 +215,96 @@ class TestEmitAlertFiringSignal(BaseTest):assert filtered_emitted is Falsefiltered_emit.assert_not_awaited()assert passing_emitted is True++ def _spend_the_only_token(self) -> None:+ consume(cap_key(self.team.id), _ONE_TOKEN)++ @parameterized.expand(+ [+ ("emitted", {}, {}, None, EMITTED_OUTCOME),+ ("not_new_episode_is_not_counted", {"previous_state": AlertState.FIRING}, {}, None, None),+ (+ "investigated_is_not_counted",+ {},+ {"investigation_agent_enabled": True, "detector_config": {"type": "zscore"}},+ None,+ None,+ ),+ ("source_disabled", {"source_enabled": False}, {}, None, "source_disabled"),+ ("scope", {"scope": "investigated_only"}, {}, None, "scope"),+ ("filtered", {"patterns": AlertNamePatterns(block=("dropped$",))}, {}, None, "filtered"),+ ("filter_invalid", {"patterns": AlertNamePatterns(invalid=True)}, {}, None, "filter_invalid"),+ ("cap", {}, {}, "spend", "cap"),+ ("cap_unavailable", {}, {}, "unavailable", "cap_unavailable"),+ ("not_dispatched", {"emit": AsyncMock(return_value=False)}, {}, None, "not_dispatched"),+ ]+ )+ def test_records_each_counted_outcome_once(+ self, _name: str, emit_kwargs: dict, alert_kwargs: dict, cap_state: str | None, expected: str | None+ ) -> None:+ alert, check = self._alert_and_check(**alert_kwargs)+ before = _outcome_counts()++ with (+ patch(f"{_MODULE}.CAP_BUDGET", _ONE_TOKEN),+ patch(f"{_MODULE}.posthoganalytics.capture") as capture,+ patch(f"{_MODULE}.consume", return_value=BucketUnavailable(error="down"))+ if cap_state == "unavailable"+ else contextlib.nullcontext(),+ ):+ if cap_state == "spend":+ self._spend_the_only_token()+ self._emit(alert, check, **emit_kwargs)++ assert _counted_since(before) == ({expected: 1.0} if expected else {})+ if expected in ("cap", "cap_unavailable"):+ capture.assert_called_once()+ assert capture.call_args.kwargs["event"] == CAP_DROP_EVENT+ assert capture.call_args.kwargs["properties"] == {+ "source_product": "analytics",+ "source_type": "alert_firing",+ "source_id": f"{alert.id}:{check.id}",+ "team_id": self.team.id,+ "alert_id": str(alert.id),+ "alert_check_id": str(check.id),+ "reason": expected,+ }+ else:+ capture.assert_not_called()++ def test_retried_attempt_does_not_count_the_episode_twice(self) -> None:+ alert, check = self._alert_and_check()+ before = _outcome_counts()++ with (+ patch(f"{_MODULE}.CAP_BUDGET", _ONE_TOKEN),+ patch(f"{_MODULE}.posthoganalytics.capture") as capture,+ ):+ first, _ = self._emit(alert, check)+ retried, _ = self._emit(alert, check)++ assert (first, retried) == (True, False)+ assert _counted_since(before) == {EMITTED_OUTCOME: 1.0}+ capture.assert_not_called()++ @parameterized.expand(+ [+ ("failed_capture_on_a_cap_drop", "spend", "posthoganalytics.capture", False),+ ("failed_counter_on_an_emitted_signal", None, "record_alert_firing_signal_outcome", True),+ ]+ )+ def test_telemetry_failure_does_not_change_the_result(+ self, _name: str, cap_state: str | None, failing: str, expected: bool+ ) -> None:+ alert, check = self._alert_and_check()++ with (+ patch(f"{_MODULE}.CAP_BUDGET", _ONE_TOKEN),+ patch(f"{_MODULE}.{failing}", side_effect=RuntimeError("telemetry down")),+ ):+ if cap_state == "spend":+ self._spend_the_only_token()+ emitted, emit = self._emit(alert, check)++ assert emitted is expected+ assert emit.await_count == int(expected)diff --git a/products/signals/ARCHITECTURE.md b/products/signals/ARCHITECTURE.mdindex 38cf7ceb3fa9..90204636c896 100644--- a/products/signals/ARCHITECTURE.md+++ b/products/signals/ARCHITECTURE.md@@ -678,7 +678,8 @@ Per-team configuration for which signal sources are enabled.- The product analytics source (`analytics` / `anomaly_investigation`) accepts `config.alert_scope`: `investigated_only` (the default when the key is absent) or `all_firing_alerts`. The serializer rejects any other value, and rejects the key on every other source (`contracts.ALERT_SCOPE_SOURCE`).Other products read the scope through `facade.source_configs.get_analytics_alert_scope(team_id=...)`, never the row. The read returns `investigated_only` unless the stored value is `all_firing_alerts` and the `signals-analytics-all-alerts` flag is on for the team's organization. The flag check fails closed.The read does not check `enabled`: the emitter still gates on `is_source_enabled`.- With `all_firing_alerts`, insight alert evaluation (`posthog/temporal/alerts/alert_firing_signal.py`) emits `analytics` / `alert_firing` when an alert that the investigation agent does not investigate starts a firing episode. It emits once per episode (source id and idempotency key `<alert id>:<first check id>`), and a failure never changes the check. The evaluation thread runs the gates after the check is committed. The activity awaits the emission on its event loop once that thread returns, because `emit_signal` needs the default executor that the thread runs on. A per-team token bucket caps it, and an unavailable bucket skips the signal. Each skip after a read logs `alert_firing.signal_skipped` with a reason. Alerts the agent investigates keep the verdict gate in both scopes.+ With `all_firing_alerts`, insight alert evaluation (`posthog/temporal/alerts/alert_firing_signal.py`) emits `analytics` / `alert_firing` when an alert that the investigation agent does not investigate starts a firing episode. It emits once per episode (source id and idempotency key `<alert id>:<first check id>`), and a failure never changes the check. The evaluation thread runs the gates after the check is committed. The activity awaits the emission on its event loop once that thread returns, because `emit_signal` needs the default executor that the thread runs on. A per-team token bucket caps it, and an unavailable bucket skips the signal. Each skip after a read logs `alert_firing.signal_skipped` with the team and a reason. Alerts the agent investigates keep the verdict gate in both scopes.+ The `posthog_insight_alerts_firing_signals_total` counter records one `outcome` per episode that passes the episode gates (`emitted`, `source_disabled`, `scope`, `filtered`, `filter_invalid`, `cap`, `cap_unavailable`, `not_dispatched`). It has no team label. A drop by the cap (`cap` or `cap_unavailable`) also captures `alert firing signal skipped` with `team_id`, `alert_id` and `reason`, which gives the per-team view for sizing the cap. A Redis key on the source id lets only the first attempt of an episode record, so an activity retry never counts it twice. A cancelled attempt records nothing. A failure to record never changes the check or the signal.`alert_firing` has no config row of its own: `is_source_enabled` checks the `anomaly_investigation` row for it, so the one product analytics toggle turns both off.Guidance written on that row filters both `anomaly_investigation` and `alert_firing`, through the direct steering gate.The same row accepts `config.alert_allow_patterns` and `config.alert_block_patterns`: lists of RE2 patterns (at most `ALERT_PATTERNS_MAX_COUNT`, each at most `ALERT_PATTERN_MAX_LENGTH` characters) matched with search semantics against the alert name and the insight name. When allow patterns are set an alert must match one, and a block match drops the alert even when it also matches an allow pattern. The serializer rejects a pattern that does not compile, and rejects both keys on every other source.@@ -1209,6 +1210,7 @@ All events use `distinct_id = team.uuid` and `groups(organization, team)`. Per-s**Lifecycle (in order):**- `signal_data_source_entered` / `signal_data_source_summarized` / `signal_data_source_filtered` — data-source pipeline only (`pipeline.py`)+- `alert firing signal skipped` — the per-team cap stopped an `analytics` / `alert_firing` signal before `emit_signal()` (+ `team_id`, `alert_id`, `alert_check_id`, `reason`: `cap` | `cap_unavailable`)- `signal_emission_started` — `emit_signal()` past validation- `signal_emitted` — `emit_signal()` after Temporal dispatch succeeds- `signal_assigned_to_report` — grouping assigned the signal (+ `report_id`, `is_new_report`, `promoted`)