Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): route the AI detector judge through the ai-gateway (#123329)

Jonathan Mieloo477ec286422a
diff --git a/products/alerts/backend/judge/llm.py b/products/alerts/backend/judge/llm.py
index baaaf54aa2f5..46a2e31e09a3 100644
--- a/products/alerts/backend/judge/llm.py
+++ b/products/alerts/backend/judge/llm.py
@@ -19,8 +19,8 @@ from django.core.cache import cache
import numpy as np
import structlog
-import posthoganalytics
+from posthog.llm.gateway_client import build_ai_gateway_anthropic_client
from posthog.tasks.alerts.detector import LLM_DETECTOR_MIN_POINTS, llm_detector_lookback
from products.alerts.backend.judge.contract import (
@@ -194,76 +194,60 @@ class LLMSeriesJudge:
logger.info("alerts.llm_detector.verdict_reused", is_anomaly=memoized.is_anomaly)
return memoized
- # Deferred so importing the evaluation package does not pull langchain into every
- # process that touches an alert.
- from langchain_core.callbacks import BaseCallbackHandler # noqa: PLC0415
- from langchain_core.messages import HumanMessage, SystemMessage # noqa: PLC0415
- from langchain_core.runnables import RunnableConfig # noqa: PLC0415
- from posthoganalytics.ai.langchain.callbacks import CallbackHandler # noqa: PLC0415
-
- from ee.hogai.llm import MaxChatAnthropic # noqa: PLC0415
from ee.hogai.utils.exceptions import LLM_API_EXCEPTIONS # noqa: PLC0415
from ee.hogai.utils.feature_flags import is_privacy_mode_enabled # noqa: PLC0415
instructions_present = bool(series.instructions)
- # No temperature: Haiku 5.5 rejects non-default sampling params with a 400. Disabled
- # thinking is accepted only at effort "high" or below, so do not raise the effort past it.
- model = MaxChatAnthropic(
- model=LLM_DETECTOR_MODEL,
- team=attribution.team,
- user=attribution.user,
- billable=True,
- # The series, its definition, and the author's notes are the whole context; the
- # project/org preamble would only dilute a single-judgment prompt.
- inject_context=False,
- streaming=False,
- disable_streaming=True,
- stream_usage=False,
- # One in-request retry absorbs a transient blip inside the check's own budget;
- # the Temporal evaluate retry is the outer net.
- max_retries=1,
- max_tokens=MAX_OUTPUT_TOKENS,
- thinking={"type": "disabled"},
- default_request_timeout=REQUEST_TIMEOUT_SECONDS,
- posthog_properties={"ai_product": LLM_DETECTOR_AI_PRODUCT},
- ).with_structured_output(LLMDetectionVerdict, include_raw=False)
-
- messages = [
- SystemMessage(content=SYSTEM_PROMPT),
- HumanMessage(content=build_human_message(data=data, context=series, window=window)),
- ]
-
- callbacks: list[BaseCallbackHandler] = []
- if posthoganalytics.default_client is not None:
- callbacks.append(
- CallbackHandler(
- posthoganalytics.default_client,
- distinct_id=str(attribution.team.id),
- trace_id=f"alert-llm-detector-{uuid.uuid4()}",
- properties={"ai_product": LLM_DETECTOR_AI_PRODUCT, "team_id": attribution.team.id},
- privacy_mode=is_privacy_mode_enabled(attribution.team),
- )
- )
-
- if not _model_call_slots.acquire(timeout=MODEL_CALL_SLOT_WAIT_SECONDS):
- raise LLMDetectorUnavailableError(
- f"The AI detector is already running {MAX_CONCURRENT_MODEL_CALLS} model calls on this worker."
- )
try:
- verdict = model.invoke(
- messages,
- config=RunnableConfig(
- callbacks=callbacks, configurable={"is_agent_billable": attribution.is_agent_billable}
- ),
+ client = build_ai_gateway_anthropic_client(
+ ai_product=LLM_DETECTOR_AI_PRODUCT,
+ trace_id=f"alert-llm-detector-{uuid.uuid4()}",
+ team_id=attribution.team.id,
)
- except LLM_API_EXCEPTIONS as error:
- # Provider credentials, permissions, and model availability are shared across
- # alerts. Disabling an individual alert cannot fix a provider failure.
+ except ValueError as error:
raise LLMDetectorUnavailableError(f"The AI detector could not reach the model: {error}") from error
- except Exception as error:
- raise LLMDetectorUnavailableError(f"The AI detector could not read the model response: {error}") from error
- finally:
- _model_call_slots.release()
+
+ extra_headers: dict[str, str] = {}
+ if not attribution.is_agent_billable:
+ extra_headers["X-PostHog-Billable"] = "false"
+ if is_privacy_mode_enabled(attribution.team):
+ extra_headers["X-PostHog-Privacy-Mode"] = "true"
+
+ with client:
+ if not _model_call_slots.acquire(timeout=MODEL_CALL_SLOT_WAIT_SECONDS):
+ raise LLMDetectorUnavailableError(
+ f"The AI detector is already running {MAX_CONCURRENT_MODEL_CALLS} model calls on this worker."
+ )
+ try:
+ # One in-request retry absorbs a transient blip inside the check's own budget;
+ # the Temporal evaluate retry is the outer net.
+ response = client.with_options(max_retries=1, timeout=REQUEST_TIMEOUT_SECONDS).messages.parse(
+ model=LLM_DETECTOR_MODEL,
+ max_tokens=MAX_OUTPUT_TOKENS,
+ # No temperature: Haiku 5.5 rejects non-default sampling params with a 400. Disabled
+ # thinking is accepted only at effort "high" or below, so do not raise the effort past it.
+ thinking={"type": "disabled"},
+ system=SYSTEM_PROMPT,
+ messages=[
+ {
+ "role": "user",
+ "content": build_human_message(data=data, context=series, window=window), # type: ignore[typeddict-item]
+ }
+ ],
+ output_format=LLMDetectionVerdict,
+ extra_headers=extra_headers,
+ )
+ verdict = response.parsed_output
+ except LLM_API_EXCEPTIONS as error:
+ # Provider credentials, permissions, and model availability are shared across
+ # alerts. Disabling an individual alert cannot fix a provider failure.
+ raise LLMDetectorUnavailableError(f"The AI detector could not reach the model: {error}") from error
+ except Exception as error:
+ raise LLMDetectorUnavailableError(
+ f"The AI detector could not read the model response: {error}"
+ ) from error
+ finally:
+ _model_call_slots.release()
if not isinstance(verdict, LLMDetectionVerdict):
raise LLMDetectorUnavailableError(
diff --git a/products/alerts/backend/test/test_llm_judge.py b/products/alerts/backend/test/test_llm_judge.py
index 7cc43826d317..a7c8ed426a95 100644
--- a/products/alerts/backend/test/test_llm_judge.py
+++ b/products/alerts/backend/test/test_llm_judge.py
@@ -276,15 +276,15 @@ _NEXT_SLOT = "b8f3a1e0-0000-0000-0000-000000000001:2026-01-07T00:15:00+00:00"
@contextmanager
-def _mocked_model(verdict: LLMDetectionVerdict) -> Iterator[Any]:
+def _mocked_model(verdict: LLMDetectionVerdict, *, privacy_mode: bool = False) -> Iterator[Any]:
with (
patch("products.alerts.backend.judge.prompt.render_series_chart", return_value=None),
- patch("products.alerts.backend.judge.llm.posthoganalytics.default_client", None),
- patch("ee.hogai.llm.MaxChatAnthropic") as chat,
+ patch("ee.hogai.utils.feature_flags.is_privacy_mode_enabled", return_value=privacy_mode),
+ patch("products.alerts.backend.judge.llm.build_ai_gateway_anthropic_client") as build_client,
):
- invoke = chat.return_value.with_structured_output.return_value.invoke
- invoke.return_value = verdict
- yield invoke
+ parse = build_client.return_value.with_options.return_value.messages.parse
+ parse.return_value = MagicMock(parsed_output=verdict)
+ yield parse
@pytest.fixture
@@ -364,10 +364,20 @@ def test_provider_rejections_do_not_disable_an_alert(
LLMSeriesJudge({"type": "llm"}).judge_latest(SERIES, series=_series(), attribution=_attribution())
[email protected]("is_agent_billable", [True, False])
-def test_model_call_preserves_the_billing_decision(is_agent_billable: bool) -> None:
- with _mocked_model(_verdict()) as invoke:
+ "is_agent_billable,privacy_mode,expected_headers",
+ [
+ (True, False, {}),
+ (False, False, {"X-PostHog-Billable": "false"}),
+ (True, True, {"X-PostHog-Privacy-Mode": "true"}),
+ (False, True, {"X-PostHog-Billable": "false", "X-PostHog-Privacy-Mode": "true"}),
+ ],
+)
+def test_model_call_preserves_the_billing_and_privacy_decisions(
+ is_agent_billable: bool, privacy_mode: bool, expected_headers: dict[str, str]
+) -> None:
+ with _mocked_model(_verdict(), privacy_mode=privacy_mode) as parse:
LLMSeriesJudge({"type": "llm"}).judge_latest(
SERIES, series=_series(), attribution=_attribution(is_agent_billable=is_agent_billable)
)
- assert invoke.call_args.kwargs["config"]["configurable"]["is_agent_billable"] is is_agent_billable
+ assert parse.call_args.kwargs["extra_headers"] == expected_headers