Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(alerts): pause ai detector checks on a gateway credit refusal (#131010)

posthog[bot]90f27dda9fe1
diff --git a/products/alerts/backend/judge/llm.py b/products/alerts/backend/judge/llm.py
index f998f4f927f9..0d5a93d4bf43 100644
--- a/products/alerts/backend/judge/llm.py
+++ b/products/alerts/backend/judge/llm.py
@@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any
from django.core.cache import cache
import numpy as np
+import anthropic
import structlog
from posthog.llm.gateway_client import build_ai_gateway_anthropic_client
@@ -145,6 +146,16 @@ def _is_over_ai_credit_budget(team: "Team") -> bool:
return False
+def _is_gateway_credit_refusal(error: Exception) -> bool:
+ # The gateway marks its own credit refusals with X-PostHog-Denial. A 402 without that header is the
+ # provider blocking the gateway's workspace, which more credits for the organization cannot fix.
+ return (
+ isinstance(error, anthropic.APIStatusError)
+ and error.status_code == 402
+ and bool(error.response.headers.get("X-PostHog-Denial"))
+ )
+
+
def _memoized_verdict(memo_key: str | None) -> LLMDetectionVerdict | None:
if not memo_key:
return None
@@ -266,6 +277,10 @@ class LLMSeriesJudge:
)
verdict = response.parsed_output
except LLM_API_EXCEPTIONS as error:
+ # The in-app budget check above fails open and can read a stale budget, so the gateway's
+ # refusal is the backstop and pauses the check the same way.
+ if _is_gateway_credit_refusal(error):
+ raise LLMDetectorPausedError(LLM_DETECTOR_OUT_OF_CREDITS_MESSAGE) from error
# Provider credentials, permissions, and model availability are shared across
# alerts. Disabling an individual alert cannot fix a provider failure.
raise LLMDetectorUnavailableError(f"The AI detector could not reach the model: {error}") from error
diff --git a/products/alerts/backend/test/test_llm_judge.py b/products/alerts/backend/test/test_llm_judge.py
index 1a3e9f1f645f..f72a8419417b 100644
--- a/products/alerts/backend/test/test_llm_judge.py
+++ b/products/alerts/backend/test/test_llm_judge.py
@@ -433,17 +433,36 @@ def test_an_organization_without_ai_credits_makes_no_model_call(
@pytest.mark.parametrize(
- "error_type,status_code",
- [(anthropic.AuthenticationError, 401), (anthropic.PermissionDeniedError, 403), (anthropic.BadRequestError, 400)],
+ "error_type,status_code,headers,expected_error,expected_message",
+ [
+ (anthropic.AuthenticationError, 401, {}, LLMDetectorUnavailableError, "could not reach the model"),
+ (anthropic.PermissionDeniedError, 403, {}, LLMDetectorUnavailableError, "could not reach the model"),
+ (anthropic.BadRequestError, 400, {}, LLMDetectorUnavailableError, "could not reach the model"),
+ (anthropic.APIStatusError, 402, {}, LLMDetectorUnavailableError, "could not reach the model"),
+ (
+ anthropic.APIStatusError,
+ 402,
+ {"X-PostHog-Denial": "insufficient_credits"},
+ LLMDetectorPausedError,
+ LLM_DETECTOR_OUT_OF_CREDITS_MESSAGE,
+ ),
+ ],
)
def test_provider_rejections_do_not_disable_an_alert(
- error_type: type[anthropic.APIStatusError], status_code: int
+ error_type: type[anthropic.APIStatusError],
+ status_code: int,
+ headers: dict[str, str],
+ expected_error: type[Exception],
+ expected_message: str,
) -> None:
- response = httpx.Response(status_code, request=httpx.Request("POST", "https://example.com/messages"))
+ response = httpx.Response(
+ status_code, headers=headers, request=httpx.Request("POST", "https://example.com/messages")
+ )
with _mocked_model(_verdict()) as invoke:
invoke.side_effect = error_type("Provider unavailable", response=response, body={})
- with pytest.raises(LLMDetectorUnavailableError):
+ with pytest.raises(expected_error) as raised:
LLMSeriesJudge({"type": "llm"}).judge_latest(SERIES, series=_series(), attribution=_attribution())
+ assert expected_message in str(raised.value)
@pytest.mark.parametrize(