:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(alerts): route the AI detector judge through the ai-gateway (#123329)
Jonathan Mieloo477ec286422a
diff --git a/products/alerts/backend/judge/llm.py b/products/alerts/backend/judge/llm.pyindex baaaf54aa2f5..46a2e31e09a3 100644--- a/products/alerts/backend/judge/llm.py+++ b/products/alerts/backend/judge/llm.py@@ -19,8 +19,8 @@ from django.core.cache import cacheimport numpy as npimport structlog-import posthoganalytics+from posthog.llm.gateway_client import build_ai_gateway_anthropic_clientfrom posthog.tasks.alerts.detector import LLM_DETECTOR_MIN_POINTS, llm_detector_lookbackfrom products.alerts.backend.judge.contract import (@@ -194,76 +194,60 @@ class LLMSeriesJudge:logger.info("alerts.llm_detector.verdict_reused", is_anomaly=memoized.is_anomaly)return memoized- # Deferred so importing the evaluation package does not pull langchain into every- # process that touches an alert.- from langchain_core.callbacks import BaseCallbackHandler # noqa: PLC0415- from langchain_core.messages import HumanMessage, SystemMessage # noqa: PLC0415- from langchain_core.runnables import RunnableConfig # noqa: PLC0415- from posthoganalytics.ai.langchain.callbacks import CallbackHandler # noqa: PLC0415-- from ee.hogai.llm import MaxChatAnthropic # noqa: PLC0415from ee.hogai.utils.exceptions import LLM_API_EXCEPTIONS # noqa: PLC0415from ee.hogai.utils.feature_flags import is_privacy_mode_enabled # noqa: PLC0415instructions_present = bool(series.instructions)- # No temperature: Haiku 5.5 rejects non-default sampling params with a 400. Disabled- # thinking is accepted only at effort "high" or below, so do not raise the effort past it.- model = MaxChatAnthropic(- model=LLM_DETECTOR_MODEL,- team=attribution.team,- user=attribution.user,- billable=True,- # The series, its definition, and the author's notes are the whole context; the- # project/org preamble would only dilute a single-judgment prompt.- inject_context=False,- streaming=False,- disable_streaming=True,- stream_usage=False,- # One in-request retry absorbs a transient blip inside the check's own budget;- # the Temporal evaluate retry is the outer net.- max_retries=1,- max_tokens=MAX_OUTPUT_TOKENS,- thinking={"type": "disabled"},- default_request_timeout=REQUEST_TIMEOUT_SECONDS,- posthog_properties={"ai_product": LLM_DETECTOR_AI_PRODUCT},- ).with_structured_output(LLMDetectionVerdict, include_raw=False)-- messages = [- SystemMessage(content=SYSTEM_PROMPT),- HumanMessage(content=build_human_message(data=data, context=series, window=window)),- ]-- callbacks: list[BaseCallbackHandler] = []- if posthoganalytics.default_client is not None:- callbacks.append(- CallbackHandler(- posthoganalytics.default_client,- distinct_id=str(attribution.team.id),- trace_id=f"alert-llm-detector-{uuid.uuid4()}",- properties={"ai_product": LLM_DETECTOR_AI_PRODUCT, "team_id": attribution.team.id},- privacy_mode=is_privacy_mode_enabled(attribution.team),- )- )-- if not _model_call_slots.acquire(timeout=MODEL_CALL_SLOT_WAIT_SECONDS):- raise LLMDetectorUnavailableError(- f"The AI detector is already running {MAX_CONCURRENT_MODEL_CALLS} model calls on this worker."- )try:- verdict = model.invoke(- messages,- config=RunnableConfig(- callbacks=callbacks, configurable={"is_agent_billable": attribution.is_agent_billable}- ),+ client = build_ai_gateway_anthropic_client(+ ai_product=LLM_DETECTOR_AI_PRODUCT,+ trace_id=f"alert-llm-detector-{uuid.uuid4()}",+ team_id=attribution.team.id,)- except LLM_API_EXCEPTIONS as error:- # Provider credentials, permissions, and model availability are shared across- # alerts. Disabling an individual alert cannot fix a provider failure.+ except ValueError as error:raise LLMDetectorUnavailableError(f"The AI detector could not reach the model: {error}") from error- except Exception as error:- raise LLMDetectorUnavailableError(f"The AI detector could not read the model response: {error}") from error- finally:- _model_call_slots.release()++ extra_headers: dict[str, str] = {}+ if not attribution.is_agent_billable:+ extra_headers["X-PostHog-Billable"] = "false"+ if is_privacy_mode_enabled(attribution.team):+ extra_headers["X-PostHog-Privacy-Mode"] = "true"++ with client:+ if not _model_call_slots.acquire(timeout=MODEL_CALL_SLOT_WAIT_SECONDS):+ raise LLMDetectorUnavailableError(+ f"The AI detector is already running {MAX_CONCURRENT_MODEL_CALLS} model calls on this worker."+ )+ try:+ # One in-request retry absorbs a transient blip inside the check's own budget;+ # the Temporal evaluate retry is the outer net.+ response = client.with_options(max_retries=1, timeout=REQUEST_TIMEOUT_SECONDS).messages.parse(+ model=LLM_DETECTOR_MODEL,+ max_tokens=MAX_OUTPUT_TOKENS,+ # No temperature: Haiku 5.5 rejects non-default sampling params with a 400. Disabled+ # thinking is accepted only at effort "high" or below, so do not raise the effort past it.+ thinking={"type": "disabled"},+ system=SYSTEM_PROMPT,+ messages=[+ {+ "role": "user",+ "content": build_human_message(data=data, context=series, window=window), # type: ignore[typeddict-item]+ }+ ],+ output_format=LLMDetectionVerdict,+ extra_headers=extra_headers,+ )+ verdict = response.parsed_output+ except LLM_API_EXCEPTIONS as error:+ # Provider credentials, permissions, and model availability are shared across+ # alerts. Disabling an individual alert cannot fix a provider failure.+ raise LLMDetectorUnavailableError(f"The AI detector could not reach the model: {error}") from error+ except Exception as error:+ raise LLMDetectorUnavailableError(+ f"The AI detector could not read the model response: {error}"+ ) from error+ finally:+ _model_call_slots.release()if not isinstance(verdict, LLMDetectionVerdict):raise LLMDetectorUnavailableError(diff --git a/products/alerts/backend/test/test_llm_judge.py b/products/alerts/backend/test/test_llm_judge.pyindex 7cc43826d317..a7c8ed426a95 100644--- a/products/alerts/backend/test/test_llm_judge.py+++ b/products/alerts/backend/test/test_llm_judge.py@@ -276,15 +276,15 @@ _NEXT_SLOT = "b8f3a1e0-0000-0000-0000-000000000001:2026-01-07T00:15:00+00:00"@contextmanager-def _mocked_model(verdict: LLMDetectionVerdict) -> Iterator[Any]:+def _mocked_model(verdict: LLMDetectionVerdict, *, privacy_mode: bool = False) -> Iterator[Any]:with (patch("products.alerts.backend.judge.prompt.render_series_chart", return_value=None),- patch("products.alerts.backend.judge.llm.posthoganalytics.default_client", None),- patch("ee.hogai.llm.MaxChatAnthropic") as chat,+ patch("ee.hogai.utils.feature_flags.is_privacy_mode_enabled", return_value=privacy_mode),+ patch("products.alerts.backend.judge.llm.build_ai_gateway_anthropic_client") as build_client,):- invoke = chat.return_value.with_structured_output.return_value.invoke- invoke.return_value = verdict- yield invoke+ parse = build_client.return_value.with_options.return_value.messages.parse+ parse.return_value = MagicMock(parsed_output=verdict)+ yield parse@pytest.fixture@@ -364,10 +364,20 @@ def test_provider_rejections_do_not_disable_an_alert(LLMSeriesJudge({"type": "llm"}).judge_latest(SERIES, series=_series(), attribution=_attribution())[email protected]("is_agent_billable", [True, False])-def test_model_call_preserves_the_billing_decision(is_agent_billable: bool) -> None:- with _mocked_model(_verdict()) as invoke:+ "is_agent_billable,privacy_mode,expected_headers",+ [+ (True, False, {}),+ (False, False, {"X-PostHog-Billable": "false"}),+ (True, True, {"X-PostHog-Privacy-Mode": "true"}),+ (False, True, {"X-PostHog-Billable": "false", "X-PostHog-Privacy-Mode": "true"}),+ ],+)+def test_model_call_preserves_the_billing_and_privacy_decisions(+ is_agent_billable: bool, privacy_mode: bool, expected_headers: dict[str, str]+) -> None:+ with _mocked_model(_verdict(), privacy_mode=privacy_mode) as parse:LLMSeriesJudge({"type": "llm"}).judge_latest(SERIES, series=_series(), attribution=_attribution(is_agent_billable=is_agent_billable))- assert invoke.call_args.kwargs["config"]["configurable"]["is_agent_billable"] is is_agent_billable+ assert parse.call_args.kwargs["extra_headers"] == expected_headers