:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(alerts): pause ai detector checks on a gateway credit refusal (#131010)
posthog[bot]90f27dda9fe1
diff --git a/products/alerts/backend/judge/llm.py b/products/alerts/backend/judge/llm.pyindex f998f4f927f9..0d5a93d4bf43 100644--- a/products/alerts/backend/judge/llm.py+++ b/products/alerts/backend/judge/llm.py@@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Anyfrom django.core.cache import cacheimport numpy as np+import anthropicimport structlogfrom posthog.llm.gateway_client import build_ai_gateway_anthropic_client@@ -145,6 +146,16 @@ def _is_over_ai_credit_budget(team: "Team") -> bool:return False+def _is_gateway_credit_refusal(error: Exception) -> bool:+ # The gateway marks its own credit refusals with X-PostHog-Denial. A 402 without that header is the+ # provider blocking the gateway's workspace, which more credits for the organization cannot fix.+ return (+ isinstance(error, anthropic.APIStatusError)+ and error.status_code == 402+ and bool(error.response.headers.get("X-PostHog-Denial"))+ )++def _memoized_verdict(memo_key: str | None) -> LLMDetectionVerdict | None:if not memo_key:return None@@ -266,6 +277,10 @@ class LLMSeriesJudge:)verdict = response.parsed_outputexcept LLM_API_EXCEPTIONS as error:+ # The in-app budget check above fails open and can read a stale budget, so the gateway's+ # refusal is the backstop and pauses the check the same way.+ if _is_gateway_credit_refusal(error):+ raise LLMDetectorPausedError(LLM_DETECTOR_OUT_OF_CREDITS_MESSAGE) from error# Provider credentials, permissions, and model availability are shared across# alerts. Disabling an individual alert cannot fix a provider failure.raise LLMDetectorUnavailableError(f"The AI detector could not reach the model: {error}") from errordiff --git a/products/alerts/backend/test/test_llm_judge.py b/products/alerts/backend/test/test_llm_judge.pyindex 1a3e9f1f645f..f72a8419417b 100644--- a/products/alerts/backend/test/test_llm_judge.py+++ b/products/alerts/backend/test/test_llm_judge.py@@ -433,17 +433,36 @@ def test_an_organization_without_ai_credits_makes_no_model_call(@pytest.mark.parametrize(- "error_type,status_code",- [(anthropic.AuthenticationError, 401), (anthropic.PermissionDeniedError, 403), (anthropic.BadRequestError, 400)],+ "error_type,status_code,headers,expected_error,expected_message",+ [+ (anthropic.AuthenticationError, 401, {}, LLMDetectorUnavailableError, "could not reach the model"),+ (anthropic.PermissionDeniedError, 403, {}, LLMDetectorUnavailableError, "could not reach the model"),+ (anthropic.BadRequestError, 400, {}, LLMDetectorUnavailableError, "could not reach the model"),+ (anthropic.APIStatusError, 402, {}, LLMDetectorUnavailableError, "could not reach the model"),+ (+ anthropic.APIStatusError,+ 402,+ {"X-PostHog-Denial": "insufficient_credits"},+ LLMDetectorPausedError,+ LLM_DETECTOR_OUT_OF_CREDITS_MESSAGE,+ ),+ ],)def test_provider_rejections_do_not_disable_an_alert(- error_type: type[anthropic.APIStatusError], status_code: int+ error_type: type[anthropic.APIStatusError],+ status_code: int,+ headers: dict[str, str],+ expected_error: type[Exception],+ expected_message: str,) -> None:- response = httpx.Response(status_code, request=httpx.Request("POST", "https://example.com/messages"))+ response = httpx.Response(+ status_code, headers=headers, request=httpx.Request("POST", "https://example.com/messages")+ )with _mocked_model(_verdict()) as invoke:invoke.side_effect = error_type("Provider unavailable", response=response, body={})- with pytest.raises(LLMDetectorUnavailableError):+ with pytest.raises(expected_error) as raised:LLMSeriesJudge({"type": "llm"}).judge_latest(SERIES, series=_series(), attribution=_attribution())+ assert expected_message in str(raised.value)@pytest.mark.parametrize(