Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(posthog-ai): route every MaxChatAnthropic call through the Go AI gateway (#127586)

Jonathan Mieloob6a90fd09f3f
diff --git a/ee/hogai/core/agent_modes/executables.py b/ee/hogai/core/agent_modes/executables.py
index c1973f14e00b..b9cd2c8dbf30 100644
--- a/ee/hogai/core/agent_modes/executables.py
+++ b/ee/hogai/core/agent_modes/executables.py
@@ -294,8 +294,8 @@ class AgentExecutable(BaseAgentLoopRootExecutable):
"posthog_properties": self._get_agent_mode_posthog_properties(state),
}
- if ai_gateway := resolve_ai_gateway_config():
- base_model = MaxChatAnthropic.via_ai_gateway(ai_gateway, **model_kwargs)
+ if resolve_ai_gateway_config() is not None:
+ base_model = MaxChatAnthropic(**model_kwargs)
else:
gateway_kwargs = self._get_gateway_kwargs()
base_model = MaxChatAnthropic(**model_kwargs, bypass_proxy=bool(gateway_kwargs), **gateway_kwargs)
diff --git a/ee/hogai/core/runner.py b/ee/hogai/core/runner.py
index 7bfe3ea8045c..1a61fc680e97 100644
--- a/ee/hogai/core/runner.py
+++ b/ee/hogai/core/runner.py
@@ -12,13 +12,11 @@ import structlog
import posthoganalytics
from asgiref.sync import async_to_sync
from langchain_core.callbacks.base import BaseCallbackHandler
-from langchain_core.outputs import LLMResult
from langchain_core.runnables.config import RunnableConfig
from langgraph.errors import GraphInterrupt, GraphRecursionError
from langgraph.graph.state import CompiledStateGraph
from langgraph.types import Command, StreamMode
from opentelemetry import trace
-from posthoganalytics.ai.langchain.callbacks import CallbackHandler, GenerationMetadata
from posthog.schema import (
ApprovalDecisionStatus,
@@ -47,7 +45,7 @@ from products.posthog_ai.backend.models.assistant import Conversation
from ee.hogai.core.ai_event_truncation import ai_event_truncator
from ee.hogai.core.base import BaseAssistantGraph
from ee.hogai.core.stream_processor import AssistantStreamProcessorProtocol
-from ee.hogai.llm import POSTHOG_AI_PRODUCT, is_ai_gateway_served
+from ee.hogai.llm import POSTHOG_AI_PRODUCT, MaxCallbackHandler
from ee.hogai.tool import ApprovalRequest, ClientToolCallRequest, pending_approval_proposal_id
from ee.hogai.utils.exceptions import (
AGENT_RUN_UNHANDLED_ERROR_COUNTER,
@@ -79,26 +77,6 @@ logger = structlog.get_logger(__name__)
_tracer = trace.get_tracer(__name__)
-class MaxCallbackHandler(CallbackHandler):
- """
- Skips $ai_generation for gateway-served calls, which the gateway captures. $ai_trace and $ai_span still emit
- because the AI credits query decides free turns from them.
- """
-
- def _capture_generation(
- self,
- trace_id: Any,
- run_id: UUID,
- run: GenerationMetadata,
- output: LLMResult | BaseException,
- parent_run_id: Optional[UUID] = None,
- include_parent_id: bool = True,
- ):
- if is_ai_gateway_served(output):
- return
- super()._capture_generation(trace_id, run_id, run, output, parent_run_id, include_parent_id)
-
-
class SubagentCallbackHandler(MaxCallbackHandler):
"""
Callback handler for subagents that makes all events appear as children of a parent span.
diff --git a/ee/hogai/llm.py b/ee/hogai/llm.py
index 9160f1638a55..fe4412f5b4d6 100644
--- a/ee/hogai/llm.py
+++ b/ee/hogai/llm.py
@@ -1,4 +1,5 @@
import ssl
+import time
import datetime
from collections.abc import ( # noqa: F401 — Sequence resolves inherited langchain field annotations lazily in this namespace
AsyncIterator,
@@ -7,7 +8,8 @@ from collections.abc import ( # noqa: F401 — Sequence resolves inherited lang
Sequence,
)
from functools import cache, cached_property
-from typing import Any, ClassVar, cast
+from typing import Any, ClassVar, NamedTuple, cast
+from uuid import UUID
from django.conf import settings
@@ -18,22 +20,24 @@ import anthropic
import structlog
from asgiref.sync import sync_to_async
from langchain_anthropic import ChatAnthropic
-from langchain_core.callbacks import AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun
+from langchain_core.callbacks import (
+ AsyncCallbackManagerForLLMRun,
+ BaseCallbackManager,
+ CallbackManagerForLLMRun,
+ Callbacks,
+)
from langchain_core.language_models import BaseChatModel
from langchain_core.messages import BaseMessage, SystemMessage
from langchain_core.outputs import ChatGenerationChunk, ChatResult, LLMResult
from langchain_core.prompts import SystemMessagePromptTemplate
from langchain_core.runnables import ensure_config
from langchain_openai import ChatOpenAI
+from posthoganalytics.ai.langchain.callbacks import CallbackHandler, GenerationMetadata
from prometheus_client import Counter
from pydantic import BaseModel, ConfigDict, Field, model_validator
-from posthog.llm.gateway_client import (
- AIGatewayConfig,
- ai_gateway_headers,
- anthropic_gateway_base_url,
- resolve_ai_gateway_config,
-)
+from posthog.llm.billing import BILLABLE_AI_PRODUCTS
+from posthog.llm.gateway_client import ai_gateway_headers, anthropic_gateway_base_url, resolve_ai_gateway_config
from posthog.models import Team, User
from posthog.settings import CLOUD_DEPLOYMENT
@@ -57,8 +61,23 @@ AI_GATEWAY_TIMEOUT = httpx.Timeout(150.0, connect=10.0, write=15.0, pool=5.0)
"""Above the gateway's 120s stream idle close. Nothing heartbeats while the gateway stalls, so every phase is bounded
to leave the twin time inside the 300s activity heartbeat."""
+AI_GATEWAY_TIMEOUT_COOLDOWN_SECONDS = 300.0
+
AI_GATEWAY_SERVED_KEY = "ai_gateway_served"
+AI_GATEWAY_BILLABLE_PRODUCTS = frozenset(
+ {
+ "posthog_ai",
+ "slack_app",
+ "workflows",
+ "hogql_decide",
+ "posthog_code",
+ "alert_investigation_agent",
+ "alert_llm_detector",
+ "surveys",
+ }
+)
+
PROJECT_ORG_USER_CONTEXT_PROMPT = """
You are currently in project {{{project_name}}}, which is part of the {{{organization_name}}} organization.
The user's name appears to be {{{user_full_name}}} ({{{user_email}}}). Feel free to use their first name when greeting. DO NOT use this name if it appears possibly fake.
@@ -117,6 +136,72 @@ def _is_provider_billing_block(error: anthropic.APIStatusError) -> bool:
return detail.get("type") == "billing_error" or any(sig in message for sig in _PROVIDER_BILLING_SIGNATURES)
+class AIGatewayRoute(NamedTuple):
+ ai_product: str
+ billable: bool
+ trace_id: str | None
+ distinct_id: str | None
+ privacy_mode: bool | None
+ properties: Mapping[str, Any]
+
+
+class MaxCallbackHandler(CallbackHandler):
+ """
+ Skips $ai_generation for gateway-served calls, which the gateway captures. $ai_trace and $ai_span still emit
+ because the AI credits query decides free turns from them.
+ """
+
+ def _capture_generation(
+ self,
+ trace_id: Any,
+ run_id: UUID,
+ run: GenerationMetadata,
+ output: LLMResult | BaseException,
+ parent_run_id: UUID | None = None,
+ include_parent_id: bool = True,
+ ):
+ if is_ai_gateway_served(output):
+ return
+ super()._capture_generation(trace_id, run_id, run, output, parent_run_id, include_parent_id)
+
+ @property
+ def ai_product(self) -> str | None:
+ return (self._properties or {}).get("ai_product")
+
+ @property
+ def trace_id(self) -> str | None:
+ return str(self._trace_id) if self._trace_id is not None else None
+
+ @property
+ def distinct_id(self) -> str | None:
+ return str(self._distinct_id) if self._distinct_id is not None else None
+
+ @property
+ def privacy_mode(self) -> bool:
+ return self._privacy_mode or bool(getattr(self._ph_client, "privacy_mode", False))
+
+ @property
+ def capture_disabled(self) -> bool:
+ return getattr(self._ph_client, "disabled", False) is True
+
+ @property
+ def properties(self) -> Mapping[str, Any]:
+ return self._properties or {}
+
+
+def _posthog_callback_handlers(callbacks: Callbacks) -> list[CallbackHandler]:
+ handlers = callbacks.handlers if isinstance(callbacks, BaseCallbackManager) else callbacks or []
+ return [handler for handler in handlers if isinstance(handler, CallbackHandler)]
+
+
+def _is_unbilled_credit_bucket_denial(error: anthropic.APIStatusError) -> bool:
+ if error.status_code != 402:
+ return False
+ if error.request.headers.get("X-PostHog-Billable") != "false":
+ return False
+ return error.response.headers.get("X-PostHog-Denial", "").startswith("credit_bucket_exhausted")
+
+
def _carries_output(chunk: ChatGenerationChunk) -> bool:
"""Thinking counts: the user sees it stream. An opening metadata chunk does not."""
return bool(chunk.message.content) or bool(getattr(chunk.message, "tool_call_chunks", None))
@@ -144,6 +229,8 @@ def _ai_gateway_fallback_reason(error: Exception) -> str | None:
return "server_error"
if isinstance(error, anthropic.APIStatusError) and _is_provider_billing_block(error):
return "provider_billing"
+ if isinstance(error, anthropic.APIStatusError) and _is_unbilled_credit_bucket_denial(error):
+ return "credit_bucket"
return None
@@ -289,25 +376,51 @@ class MaxChatMixin(BaseModel):
return new_kwargs
- def _routes_via_ai_gateway(self) -> bool:
+ def _ai_gateway_route(self, kwargs: Mapping[str, Any]) -> AIGatewayRoute | None:
"""Other products keep their capture and billing on the direct path."""
if self.ai_gateway_fallback is None:
- return False
+ return None
configurable = ensure_config().get("configurable") or {}
- return (
- configurable.get("ai_product") == POSTHOG_AI_PRODUCT
- and (self.posthog_properties or {}).get("ai_product", POSTHOG_AI_PRODUCT) == POSTHOG_AI_PRODUCT
+ if configurable.get("ai_product") != POSTHOG_AI_PRODUCT:
+ return None
+ if (self.posthog_properties or {}).get("ai_product", POSTHOG_AI_PRODUCT) != POSTHOG_AI_PRODUCT:
+ return None
+ return self._ai_gateway_route_attributed(
+ ai_product=POSTHOG_AI_PRODUCT,
+ billable=kwargs["metadata"]["posthog_properties"]["$ai_billable"],
+ handler=None,
)
- def _with_ai_gateway_headers(self, kwargs: dict[str, Any]) -> dict[str, Any]:
+ def _ai_gateway_route_attributed(
+ self, *, ai_product: str, billable: bool, handler: MaxCallbackHandler | None
+ ) -> AIGatewayRoute:
+ if handler is not None:
+ return AIGatewayRoute(
+ ai_product=ai_product,
+ billable=billable,
+ trace_id=handler.trace_id,
+ distinct_id=handler.distinct_id or self.user.distinct_id,
+ privacy_mode=handler.privacy_mode,
+ properties=handler.properties,
+ )
+ configurable = ensure_config().get("configurable") or {}
+ return AIGatewayRoute(
+ ai_product=ai_product,
+ billable=billable,
+ trace_id=str(configurable["trace_id"]) if configurable.get("trace_id") else None,
+ distinct_id=configurable.get("distinct_id") or self.user.distinct_id,
+ privacy_mode=configurable.get("privacy_mode"),
+ properties={},
+ )
+
+ def _with_ai_gateway_headers(self, kwargs: dict[str, Any], route: AIGatewayRoute) -> dict[str, Any]:
configurable = ensure_config().get("configurable") or {}
posthog_properties = kwargs["metadata"]["posthog_properties"]
- trace_id = str(configurable["trace_id"]) if configurable.get("trace_id") else None
conversation_id = str(configurable["thread_id"]) if configurable.get("thread_id") else None
# The gateway strips `$` keys and stamps ai_product itself; team_id names the customer it bills.
labels = {
key: str(value)
- for key, value in posthog_properties.items()
+ for key, value in {**route.properties, **posthog_properties}.items()
if value is not None and not key.startswith("$") and key != "ai_product"
}
if conversation_id:
@@ -317,18 +430,18 @@ class MaxChatMixin(BaseModel):
labels["ai_support_impersonated"] = "true"
headers = (
ai_gateway_headers(
- ai_product=POSTHOG_AI_PRODUCT,
- trace_id=trace_id,
+ ai_product=route.ai_product,
+ trace_id=route.trace_id,
session_id=conversation_id,
properties=labels,
- distinct_id=configurable.get("distinct_id") or self.user.distinct_id,
+ distinct_id=route.distinct_id,
)
or {}
)
- if not posthog_properties["$ai_billable"]:
+ if not route.billable:
headers["X-PostHog-Billable"] = "false"
# The gateway records prompts and outputs unless asked not to, so an unknown privacy mode redacts too.
- if configurable.get("privacy_mode") is not False:
+ if route.privacy_mode is not False:
headers["X-PostHog-Privacy-Mode"] = "true"
return {**kwargs, "extra_headers": {**(kwargs.get("extra_headers") or {}), **headers}}
@@ -552,8 +665,8 @@ class MaxChatOpenAI(MaxChatMixin, ChatOpenAI):
messages = self._enrich_messages(messages, project_org_user_variables)
kwargs = self._with_posthog_properties(kwargs)
- if self._routes_via_ai_gateway():
- kwargs = self._with_ai_gateway_headers(kwargs)
+ if route := self._ai_gateway_route(kwargs):
+ kwargs = self._with_ai_gateway_headers(kwargs, route)
return super().generate(messages, *args, **kwargs)
@@ -571,8 +684,8 @@ class MaxChatOpenAI(MaxChatMixin, ChatOpenAI):
messages = self._enrich_messages(messages, project_org_user_variables)
kwargs = self._with_posthog_properties(kwargs)
- if self._routes_via_ai_gateway():
- kwargs = self._with_ai_gateway_headers(kwargs)
+ if route := self._ai_gateway_route(kwargs):
+ kwargs = self._with_ai_gateway_headers(kwargs, route)
return await super().agenerate(messages, *args, **kwargs)
@@ -593,24 +706,32 @@ class MaxChatAnthropic(MaxChatMixin, ChatAnthropic):
ai_gateway_fallback: ChatAnthropic | None = Field(default=None, exclude=True)
+ _ai_gateway_timed_out_monotonic: ClassVar[float | None] = None
+
+ @model_validator(mode="before")
@classmethod
- def via_ai_gateway(cls, ai_gateway: AIGatewayConfig, **kwargs: Any) -> "MaxChatAnthropic":
- return cls(
- **{
- **kwargs,
- "anthropic_api_url": anthropic_gateway_base_url(ai_gateway.url),
- "anthropic_api_key": ai_gateway.api_key,
- "bypass_proxy": True,
- # The twin is the retry; an SDK retry doubles the wait before it.
- "max_retries": 0,
- "ai_gateway_fallback": cls(**kwargs),
- }
- )
+ def _via_ai_gateway(cls, data: Any) -> Any:
+ if not isinstance(data, dict) or "ai_gateway_fallback" in data:
+ return data
+ if data.keys() & {"anthropic_api_url", "base_url", "anthropic_api_key", "api_key"}:
+ return data
+ ai_gateway = resolve_ai_gateway_config()
+ if ai_gateway is None:
+ return data
+ return {
+ **data,
+ "anthropic_api_url": anthropic_gateway_base_url(ai_gateway.url),
+ "anthropic_api_key": ai_gateway.api_key,
+ "bypass_proxy": True,
+ # The twin is the retry; an SDK retry doubles the wait before it.
+ "max_retries": 0,
+ "ai_gateway_fallback": cls(**data, ai_gateway_fallback=None),
+ }
@cached_property
def _client_params(self) -> dict[str, Any]:
params = cast(dict[str, Any], ChatAnthropic._client_params.func(self)) # type: ignore[attr-defined]
- if self.ai_gateway_fallback is not None:
+ if self.ai_gateway_fallback is not None and self.default_request_timeout is None:
# langchain-anthropic defaults to no timeout, so a stalled gateway would never fall back.
params["timeout"] = AI_GATEWAY_TIMEOUT
return params
@@ -654,10 +775,62 @@ class MaxChatAnthropic(MaxChatMixin, ChatAnthropic):
return kwargs
def get_num_tokens_from_messages(self, messages: list[BaseMessage], tools: Any = None, **kwargs: Any) -> int:
- # Token counts have no fallback, so a gateway 429 here would fail the turn.
- if self.ai_gateway_fallback is not None:
- return self.ai_gateway_fallback.get_num_tokens_from_messages(messages, tools=tools, **kwargs)
- return super().get_num_tokens_from_messages(messages, tools=tools, **kwargs)
+ fallback = self.ai_gateway_fallback
+ if fallback is None:
+ return super().get_num_tokens_from_messages(messages, tools=tools, **kwargs)
+ if self._ai_gateway_cooling_down():
+ AI_GATEWAY_FALLBACK_COUNTER.labels(reason="timeout_cooldown").inc()
+ return fallback.get_num_tokens_from_messages(messages, tools=tools, **kwargs)
+ try:
+ return super().get_num_tokens_from_messages(
+ messages,
+ tools=tools,
+ **{**kwargs, "extra_headers": {**(kwargs.get("extra_headers") or {}), "X-PostHog-Billable": "false"}},
+ )
+ except Exception as error:
+ reason = _ai_gateway_fallback_reason(error)
+ if reason is None:
+ raise
+ self._record_ai_gateway_fallback(reason, error)
+ return fallback.get_num_tokens_from_messages(messages, tools=tools, **kwargs)
+
+ def _record_ai_gateway_fallback(self, reason: str, error: Exception) -> None:
+ super()._record_ai_gateway_fallback(reason, error)
+ if reason == "timeout":
+ MaxChatAnthropic._ai_gateway_timed_out_monotonic = time.monotonic()
+
+ @staticmethod
+ def _ai_gateway_cooling_down() -> bool:
+ timed_out_monotonic = MaxChatAnthropic._ai_gateway_timed_out_monotonic
+ if timed_out_monotonic is None:
+ return False
+ return time.monotonic() - timed_out_monotonic < AI_GATEWAY_TIMEOUT_COOLDOWN_SECONDS
+
+ def _ai_gateway_route(self, kwargs: Mapping[str, Any]) -> AIGatewayRoute | None:
+ if self.ai_gateway_fallback is None:
+ return None
+ handlers = _posthog_callback_handlers(kwargs.get("callbacks")) + _posthog_callback_handlers(self.callbacks)
+ if not all(isinstance(handler, MaxCallbackHandler) for handler in handlers):
+ return None
+ posthog_properties = kwargs["metadata"]["posthog_properties"]
+ handler = cast(MaxCallbackHandler, handlers[0]) if handlers else None
+ if handler is not None and handler.capture_disabled:
+ return None
+ billable = False
+ if handler is not None:
+ ai_product = handler.ai_product or posthog_properties["ai_product"]
+ if posthog_properties["$ai_billable"]:
+ billable = ai_product in BILLABLE_AI_PRODUCTS
+ else:
+ configurable = ensure_config().get("configurable") or {}
+ ai_product = configurable.get("ai_product") or posthog_properties["ai_product"]
+ if billable:
+ if ai_product not in AI_GATEWAY_BILLABLE_PRODUCTS:
+ return None
+ if self._ai_gateway_cooling_down():
+ AI_GATEWAY_FALLBACK_COUNTER.labels(reason="timeout_cooldown").inc()
+ return None
+ return self._ai_gateway_route_attributed(ai_product=ai_product, billable=billable, handler=handler)
def generate(
self,
@@ -670,6 +843,8 @@ class MaxChatAnthropic(MaxChatMixin, ChatAnthropic):
messages = self._enrich_messages(messages, project_org_user_variables)
kwargs = self._with_posthog_properties(kwargs)
+ if route := self._ai_gateway_route(kwargs):
+ kwargs = self._with_ai_gateway_headers(kwargs, route)
return super().generate(messages, *args, **kwargs)
@@ -684,7 +859,7 @@ class MaxChatAnthropic(MaxChatMixin, ChatAnthropic):
messages = self._enrich_messages(messages, project_org_user_variables)
kwargs = self._with_posthog_properties(kwargs)
- if self._routes_via_ai_gateway():
- kwargs = self._with_ai_gateway_headers(kwargs)
+ if route := self._ai_gateway_route(kwargs):
+ kwargs = self._with_ai_gateway_headers(kwargs, route)
return await super().agenerate(messages, *args, **kwargs)
diff --git a/ee/hogai/test/test_llm.py b/ee/hogai/test/test_llm.py
index 5965e254ba3b..5bccaf16dfdd 100644
--- a/ee/hogai/test/test_llm.py
+++ b/ee/hogai/test/test_llm.py
@@ -5,7 +5,7 @@ from functools import cached_property
from pathlib import Path
from posthog.test.base import BaseTest
-from unittest.mock import patch
+from unittest.mock import Mock, patch
from django.test import SimpleTestCase
@@ -17,20 +17,25 @@ from langchain_anthropic import ChatAnthropic
from langchain_core.messages import AIMessage, AIMessageChunk, BaseMessage, HumanMessage, SystemMessage
from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, Generation, LLMResult
from parameterized import parameterized
+from posthoganalytics.ai.langchain.callbacks import CallbackHandler
from posthog.schema import PersonsOnEventsMode
+from posthog.llm.billing import BILLABLE_AI_PRODUCTS
from posthog.llm.gateway_client import AIGatewayConfig
from posthog.models import Organization, Team, User
from posthog.settings import BASE_DIR
from posthog.temporal.ai.chat_agent import CHAT_AGENT_ACTIVITY_HEARTBEAT_TIMEOUT
from ee.hogai.llm import (
+ AI_GATEWAY_BILLABLE_PRODUCTS,
AI_GATEWAY_FALLBACK_COUNTER,
AI_GATEWAY_SERVED_KEY,
AI_GATEWAY_TIMEOUT,
+ AI_GATEWAY_TIMEOUT_COOLDOWN_SECONDS,
BILLING_SKIPPED_COUNTER,
PROJECT_ORG_USER_CONTEXT_PROMPT,
+ MaxCallbackHandler,
MaxChatAnthropic,
MaxChatOpenAI,
is_ai_gateway_served,
@@ -677,6 +682,9 @@ class TestMaxChatAnthropicAIGateway(BaseTest):
super().setUp()
self.requests: list[httpx.Request] = []
self.gateway_outcome: httpx.Response | Exception = httpx.Response(200, json=_anthropic_message_body("gateway"))
+ cooldown = patch.object(MaxChatAnthropic, "_ai_gateway_timed_out_monotonic", None)
+ cooldown.start()
+ self.addCleanup(cooldown.stop)
def _respond(self, request: httpx.Request) -> httpx.Response:
self.requests.append(request)
@@ -686,12 +694,12 @@ class TestMaxChatAnthropicAIGateway(BaseTest):
raise self.gateway_outcome
return self.gateway_outcome
+ def _gateway_model(self, **kwargs) -> MaxChatAnthropic:
+ with patch("ee.hogai.llm.resolve_ai_gateway_config", return_value=self.gateway):
+ return MaxChatAnthropic(model="claude-sonnet-4-6", user=self.user, team=self.team, **kwargs)
+
def _model(self, **kwargs) -> MaxChatAnthropic:
- model = MaxChatAnthropic.via_ai_gateway(
- self.gateway,
- model="claude-sonnet-4-6",
- user=self.user,
- team=self.team,
+ model = self._gateway_model(
inject_context=False,
billable=True,
posthog_properties={"agent_mode": "sql", "supermode": None},
@@ -711,6 +719,11 @@ class TestMaxChatAnthropicAIGateway(BaseTest):
assert model.ai_gateway_fallback is not None
return httpx.URL(model.ai_gateway_fallback.anthropic_api_url or "").host
+ def _max_handler(self, **kwargs) -> MaxCallbackHandler:
+ return MaxCallbackHandler(
+ Mock(privacy_mode=False), **{"distinct_id": "distinct-1", "trace_id": "trace-1", **kwargs}
+ )
+
def _config(self, **configurable) -> dict:
return {
"configurable": {
@@ -760,8 +773,8 @@ class TestMaxChatAnthropicAIGateway(BaseTest):
return _astream
- def test_via_ai_gateway_targets_the_gateway_and_keeps_a_direct_twin(self):
- model = MaxChatAnthropic.via_ai_gateway(self.gateway, model="claude-sonnet-4-6", user=self.user, team=self.team)
+ def test_gateway_config_targets_the_gateway_and_keeps_a_direct_twin(self):
+ model = self._gateway_model()
twin = model.ai_gateway_fallback
self.assertEqual(model.anthropic_api_url, "https://ai-gateway.test")
@@ -787,7 +800,7 @@ class TestMaxChatAnthropicAIGateway(BaseTest):
model = self._model()
config = self._config(is_agent_billable=is_agent_billable, is_impersonated=is_impersonated)
with patch("ee.hogai.llm.ensure_config", return_value=config):
- result = await model.agenerate([[HumanMessage(content="hello")]])
+ result = await model.agenerate([[HumanMessage(content="hello")]], callbacks=[self._max_handler()])
self.assertEqual(
[str(request.url) for request in self.requests], ["https://ai-gateway.test/v1/messages"]
@@ -823,24 +836,88 @@ class TestMaxChatAnthropicAIGateway(BaseTest):
self.assertEqual(self.requests[0].headers.get("x-posthog-privacy-mode"), redaction_header)
self.assertTrue(is_ai_gateway_served(result))
- async def test_calls_for_other_products_go_direct(self):
- cases: list[tuple[str, dict, dict | None]] = [
- ("mcp conversation", self._config(ai_product="mcp"), None),
- ("outside a posthog_ai run", self._config(ai_product=None), None),
- ("product override", self._config(), {"ai_product": "alert_investigation_agent"}),
+ @parameterized.expand(
+ [
+ ("mcp conversation is unbilled", "mcp", None, True, "mcp", "false"),
+ ("posthog_ai turn is billed", "posthog_ai", None, True, "posthog_ai", None),
+ ("product outside a runner", None, "alert_investigation_agent", True, "alert_investigation_agent", None),
+ (
+ "no handler bills nothing",
+ None,
+ "alert_investigation_agent",
+ False,
+ "alert_investigation_agent",
+ "false",
+ ),
]
- for name, config, posthog_properties in cases:
- with self.subTest(name):
- self.requests.clear()
- model = self._model()
- if posthog_properties:
- model.posthog_properties = posthog_properties
- with patch("ee.hogai.llm.ensure_config", return_value=config):
- result = await model.agenerate([[HumanMessage(content="hello")]])
+ )
+ async def test_routes_every_product_with_its_label_and_current_billing(
+ self, _name, configurable_product, properties_product, with_handler, expected_product, billable_header
+ ):
+ model = self._model()
+ if properties_product:
+ model.posthog_properties = {"ai_product": properties_product}
+ callbacks = [self._max_handler(properties={"ai_product": configurable_product})] if with_handler else None
+ with patch("ee.hogai.llm.ensure_config", return_value=self._config(ai_product=configurable_product)):
+ result = await model.agenerate([[HumanMessage(content="hello")]], callbacks=callbacks)
+
+ self.assertEqual([request.url.host for request in self.requests], ["ai-gateway.test"])
+ headers = self.requests[0].headers
+ self.assertEqual(headers["x-posthog-product"], expected_product)
+ self.assertEqual(headers.get("x-posthog-billable"), billable_header)
+ self.assertTrue(is_ai_gateway_served(result))
- self.assertEqual([request.url.host for request in self.requests], [self._direct_host(model)])
- self.assertFalse(any(header.lower().startswith("x-posthog-") for header in self.requests[0].headers))
- self.assertFalse(is_ai_gateway_served(result))
+ async def test_attached_handler_attribution_wins_over_an_inherited_config(self):
+ model = self._model()
+ model.posthog_properties = {"ai_product": "workflows"}
+ model.billable = False
+ handler = self._max_handler(
+ distinct_id="workflows-user",
+ trace_id="workflows-trace",
+ privacy_mode=True,
+ properties={"ai_product": "workflows", "alert_id": "alert-1"},
+ )
+ with patch("ee.hogai.llm.ensure_config", return_value=self._config(privacy_mode=False)):
+ await model.agenerate([[HumanMessage(content="hello")]], callbacks=[handler])
+
+ headers = self.requests[0].headers
+ self.assertEqual(
+ [
+ headers["x-posthog-product"],
+ headers["x-posthog-trace-id"],
+ headers["x-posthog-distinct-id"],
+ headers.get("x-posthog-privacy-mode"),
+ headers.get("x-posthog-billable"),
+ json.loads(headers["x-posthog-properties"]).get("alert_id"),
+ ],
+ ["workflows", "workflows-trace", "workflows-user", "true", "false", "alert-1"],
+ )
+
+ @parameterized.expand(
+ [
+ ("handler that would capture the generation again", "posthog_ai", "plain"),
+ ("billed product the gateway does not bill", "replay_vision", "max"),
+ ("capture opted out", "posthog_ai", "disabled"),
+ ]
+ )
+ async def test_calls_that_would_change_capture_or_billing_go_direct(self, _name, ai_product, handler_kind):
+ model = self._model()
+ model.posthog_properties = {"ai_product": ai_product}
+ handlers = {
+ "max": lambda: self._max_handler(),
+ "disabled": lambda: MaxCallbackHandler(Mock(privacy_mode=False, disabled=True), distinct_id="d"),
+ "plain": lambda: CallbackHandler(Mock(), distinct_id="d"),
+ }
+ handler = handlers[handler_kind]()
+ with patch("ee.hogai.llm.ensure_config", return_value=self._config(ai_product=None)):
+ result = await model.agenerate([[HumanMessage(content="hello")]], callbacks=[handler])
+
+ self.assertEqual([request.url.host for request in self.requests], [self._direct_host(model)])
+ self.assertFalse(any(header.lower().startswith("x-posthog-") for header in self.requests[0].headers))
+ self.assertFalse(is_ai_gateway_served(result))
+
+ def test_gateway_billable_products_are_billed_today(self):
+ self.assertLessEqual(AI_GATEWAY_BILLABLE_PRODUCTS, BILLABLE_AI_PRODUCTS)
async def test_retryable_gateway_failures_fall_back_to_the_direct_twin(self):
request = httpx.Request("POST", "https://ai-gateway.test/v1/messages")
@@ -853,11 +930,18 @@ class TestMaxChatAnthropicAIGateway(BaseTest):
("server_error", _anthropic_error(529)),
("provider_billing", _anthropic_error(400, "invalid_request_error", _WORKSPACE_LIMIT)),
("provider_billing", _anthropic_error(402, "billing_error", "Payment required")),
+ (
+ "credit_bucket",
+ _anthropic_error(
+ 402, "billing_error", "out", {"X-PostHog-Denial": "credit_bucket_exhausted:ai_credits"}
+ ),
+ ),
]
for reason, outcome in cases:
with self.subTest(reason=reason, outcome=outcome):
self.requests.clear()
self.gateway_outcome = outcome
+ MaxChatAnthropic._ai_gateway_timed_out_monotonic = None
model = self._model()
with (
patch("ee.hogai.llm.ensure_config", return_value=self._config()),
@@ -875,6 +959,36 @@ class TestMaxChatAnthropicAIGateway(BaseTest):
self.assertFalse(is_ai_gateway_served(result))
mock_labels.assert_called_once_with(reason=reason)
+ async def test_a_gateway_timeout_sends_calls_direct_until_the_cooldown_ends(self):
+ self.gateway_outcome = httpx.ReadTimeout("slow", request=httpx.Request("POST", "https://ai-gateway.test"))
+ model = self._model()
+ direct = self._direct_host(model)
+ with patch("ee.hogai.llm.ensure_config", return_value=self._config()):
+ await model.agenerate([[HumanMessage(content="hello")]])
+ await self._model().agenerate([[HumanMessage(content="hello")]])
+ with patch.object(
+ ChatAnthropic,
+ "get_num_tokens_from_messages",
+ lambda counting_model, *args, **kwargs: 3 if counting_model.ai_gateway_fallback else 7,
+ ):
+ tokens = self._model().get_num_tokens_from_messages([HumanMessage(content="hello")])
+ assert MaxChatAnthropic._ai_gateway_timed_out_monotonic is not None
+ MaxChatAnthropic._ai_gateway_timed_out_monotonic -= AI_GATEWAY_TIMEOUT_COOLDOWN_SECONDS
+ self.gateway_outcome = httpx.Response(200, json=_anthropic_message_body("gateway"))
+ result = await self._model().agenerate([[HumanMessage(content="hello")]])
+
+ self.assertEqual(
+ [(request.url.host, request.url.path) for request in self.requests],
+ [
+ ("ai-gateway.test", "/v1/messages"),
+ (direct, "/v1/messages"),
+ (direct, "/v1/messages"),
+ ("ai-gateway.test", "/v1/messages"),
+ ],
+ )
+ self.assertEqual(tokens, 7)
+ self.assertTrue(is_ai_gateway_served(result))
+
async def test_non_retryable_gateway_failures_raise_without_fallback(self):
cases: list[tuple[str, httpx.Response]] = [
("bad request", _anthropic_error(400, "invalid_request_error", "prompt is too long")),
@@ -882,6 +996,12 @@ class TestMaxChatAnthropicAIGateway(BaseTest):
"gateway's own credit denial",
_anthropic_error(402, "billing_error", _WORKSPACE_LIMIT, {"X-PostHog-Denial": "insufficient_credits"}),
),
+ (
+ "credit bucket denial of a billed call",
+ _anthropic_error(
+ 402, "billing_error", "out", {"X-PostHog-Denial": "credit_bucket_exhausted:ai_credits"}
+ ),
+ ),
("forbidden", _anthropic_error(403)),
("not found", _anthropic_error(404)),
]
@@ -895,7 +1015,7 @@ class TestMaxChatAnthropicAIGateway(BaseTest):
patch.object(AI_GATEWAY_FALLBACK_COUNTER, "labels") as mock_labels,
self.assertRaises(Exception),
):
- await model.agenerate([[HumanMessage(content="hello")]])
+ await model.agenerate([[HumanMessage(content="hello")]], callbacks=[self._max_handler()])
self.assertEqual([request.url.host for request in self.requests], ["ai-gateway.test"])
mock_labels.assert_not_called()
@@ -1017,7 +1137,7 @@ class TestMaxChatAnthropicAIGateway(BaseTest):
self.assertTrue(is_ai_gateway_served(result))
def test_gateway_client_has_a_bounded_timeout(self):
- model = MaxChatAnthropic.via_ai_gateway(self.gateway, model="claude-sonnet-4-6", user=self.user, team=self.team)
+ model = self._gateway_model()
twin = model.ai_gateway_fallback
assert twin is not None
@@ -1038,22 +1158,45 @@ class TestMaxChatAnthropicAIGateway(BaseTest):
stalled_gateway_seconds = sum(phase or 0.0 for phase in phases) * (model.max_retries + 1)
self.assertGreaterEqual(CHAT_AGENT_ACTIVITY_HEARTBEAT_TIMEOUT - stalled_gateway_seconds, 120)
- def test_token_counting_uses_the_direct_twin(self):
- model = MaxChatAnthropic.via_ai_gateway(self.gateway, model="claude-sonnet-4-6", user=self.user, team=self.team)
+ def test_gateway_client_keeps_the_callers_timeout(self):
+ model = self._gateway_model(default_request_timeout=30.0)
+
+ self.assertEqual(model._async_client.timeout, 30.0)
+
+ @parameterized.expand(
+ [
+ ("gateway counts", None, 3),
+ ("gateway rate limit falls back to the twin", 429, 7),
+ ("exhausted credits fall back to the twin", 402, 7),
+ ]
+ )
+ def test_token_counting_goes_through_the_gateway(self, _name, gateway_status, expected_tokens):
+ model = self._gateway_model()
def count(counting_model, messages, tools=None, **kwargs):
- return 0 if counting_model.ai_gateway_fallback is not None else 7
+ if counting_model.ai_gateway_fallback is None:
+ return 7
+ if gateway_status is None:
+ return 3
+ request = httpx.Request(
+ "POST", "https://ai-gateway.test/v1/messages/count_tokens", headers=kwargs.get("extra_headers")
+ )
+ response = httpx.Response(
+ gateway_status, request=request, headers={"X-PostHog-Denial": "credit_bucket_exhausted"}
+ )
+ raise anthropic.APIStatusError("denied", response=response, body=None)
with patch.object(ChatAnthropic, "get_num_tokens_from_messages", count):
- self.assertEqual(model.get_num_tokens_from_messages([HumanMessage(content="hello")]), 7)
+ self.assertEqual(model.get_num_tokens_from_messages([HumanMessage(content="hello")]), expected_tokens)
- def test_sync_calls_use_the_direct_twin(self):
+ def test_sync_calls_route_through_the_gateway(self):
model = self._model()
with patch("ee.hogai.llm.ensure_config", return_value=self._config()):
- result = model.generate([[HumanMessage(content="hello")]])
+ result = model.generate([[HumanMessage(content="hello")]], callbacks=[self._max_handler()])
- self.assertEqual([request.url.host for request in self.requests], [self._direct_host(model)])
- self.assertFalse(is_ai_gateway_served(result))
+ self.assertEqual([request.url.host for request in self.requests], ["ai-gateway.test"])
+ self.assertEqual(self.requests[0].headers["x-posthog-product"], "posthog_ai")
+ self.assertTrue(is_ai_gateway_served(result))
def test_is_ai_gateway_served_reads_the_generation_marker(self):
served = LLMResult(
diff --git a/posthog/llm/billing.py b/posthog/llm/billing.py
index 3b936cae0751..3ce7483bf129 100644
--- a/posthog/llm/billing.py
+++ b/posthog/llm/billing.py
@@ -1 +1,20 @@
AI_COST_MARKUP_PERCENT = 0.2
+
+# ai_product values that roll into PostHog AI (Max) billing credits.
+POSTHOG_AI_PRODUCTS = [
+ "posthog_ai",
+ "slack_app",
+ "workflows",
+ "subscriptions",
+ "alert_investigation_agent",
+ "alert_llm_detector",
+ "product_analytics",
+ "surveys",
+ "replay_vision",
+ "hogql_decide",
+]
+
+# ai_product values billed as PostHog Desktop credits.
+POSTHOG_CODE_AI_PRODUCTS = ["posthog_code"]
+
+BILLABLE_AI_PRODUCTS = frozenset(POSTHOG_AI_PRODUCTS + POSTHOG_CODE_AI_PRODUCTS)
diff --git a/posthog/tasks/usage_report.py b/posthog/tasks/usage_report.py
index 2ffb73e4198f..c065548f0997 100644
--- a/posthog/tasks/usage_report.py
+++ b/posthog/tasks/usage_report.py
@@ -33,7 +33,7 @@ from posthog.cloud_utils import get_cached_instance_license
from posthog.constants import AI_EVENT_NAME_PREFIX, MCP_EVENT_NAME_PREFIX, FlagRequestType
from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
-from posthog.llm.billing import AI_COST_MARKUP_PERCENT
+from posthog.llm.billing import AI_COST_MARKUP_PERCENT, POSTHOG_AI_PRODUCTS, POSTHOG_CODE_AI_PRODUCTS
from posthog.logging.timing import timed_log
from posthog.models import OrganizationMembership, User
from posthog.models.ai_events.sql import TABLE_BASE_NAME as AI_EVENTS_TABLE
@@ -1918,23 +1918,7 @@ CLOUD_REGION_TO_URL = {
}
-# ai_product values that roll into PostHog AI (Max) billing credits.
-POSTHOG_AI_PRODUCTS = [
- "posthog_ai",
- "slack_app",
- "workflows",
- "subscriptions",
- "alert_investigation_agent",
- "alert_llm_detector",
- "product_analytics",
- "surveys",
- "replay_vision",
- "hogql_decide",
-]
-
-# ai_product values billed as PostHog Desktop credits.
UNBILLED_TASK_ORIGIN_PRODUCTS = ("task_analysis", "account_audit")
-POSTHOG_CODE_AI_PRODUCTS = ["posthog_code"]
def get_ai_billing_instance_group_type_index(team_id: int) -> int | None:
diff --git a/posthog/temporal/ai/anomaly_investigation/runner.py b/posthog/temporal/ai/anomaly_investigation/runner.py
index f958b1b35d3a..3f9ffc99c351 100644
--- a/posthog/temporal/ai/anomaly_investigation/runner.py
+++ b/posthog/temporal/ai/anomaly_investigation/runner.py
@@ -22,7 +22,6 @@ import posthoganalytics
from langchain_core.callbacks import BaseCallbackHandler
from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage
from langchain_core.runnables import RunnableConfig
-from posthoganalytics.ai.langchain.callbacks import CallbackHandler
from pydantic import BaseModel, ValidationError
from posthog.models import Team, User
@@ -433,6 +432,8 @@ async def run_investigation(
def _build_callbacks(*, team: Team, alert: AlertConfiguration | None) -> list[BaseCallbackHandler]:
callbacks: list[BaseCallbackHandler] = []
+ from ee.hogai.llm import MaxCallbackHandler
+
client = posthoganalytics.default_client
if client is None:
return callbacks
@@ -443,7 +444,7 @@ def _build_callbacks(*, team: Team, alert: AlertConfiguration | None) -> list[Ba
if alert is not None:
properties["alert_id"] = str(alert.id)
callbacks.append(
- CallbackHandler(
+ MaxCallbackHandler(
client,
distinct_id=str(team.id),
trace_id=f"alert-investigation-{uuid.uuid4()}",
diff --git a/posthog/temporal/tests/ai/test_anomaly_investigation_runner.py b/posthog/temporal/tests/ai/test_anomaly_investigation_runner.py
index 1bfd23c4ab2e..648579a2902c 100644
--- a/posthog/temporal/tests/ai/test_anomaly_investigation_runner.py
+++ b/posthog/temporal/tests/ai/test_anomaly_investigation_runner.py
@@ -277,7 +277,7 @@ def test_build_callbacks_tags_ai_product_for_llm_analytics(alert_id, expected_pr
with (
patch("posthog.temporal.ai.anomaly_investigation.runner.posthoganalytics") as mock_module,
- patch("posthog.temporal.ai.anomaly_investigation.runner.CallbackHandler") as mock_handler,
+ patch("ee.hogai.llm.MaxCallbackHandler") as mock_handler,
):
mock_module.default_client = sentinel_client
diff --git a/products/ai_observability/backend/api/personal_spend.py b/products/ai_observability/backend/api/personal_spend.py
index b2ed34c0f453..9f4d6c951a0e 100644
--- a/products/ai_observability/backend/api/personal_spend.py
+++ b/products/ai_observability/backend/api/personal_spend.py
@@ -47,10 +47,10 @@ from posthog.auth import (
WebhookSignatureAuthentication,
)
from posthog.clickhouse.query_tagging import Feature, Product, tags_context
+from posthog.llm.billing import BILLABLE_AI_PRODUCTS
from posthog.models import Team, User
from posthog.permissions import APIScopePermission
from posthog.rate_limit import PersonalSpendBurstThrottle, PersonalSpendDailyThrottle, PersonalSpendSustainedThrottle
-from posthog.tasks.usage_report import POSTHOG_AI_PRODUCTS, POSTHOG_CODE_AI_PRODUCTS
from posthog.utils import relative_date_parse
logger = structlog.get_logger(__name__)
@@ -59,7 +59,6 @@ MAX_PRODUCT_KEY_LENGTH = 64
MAX_DATE_STRING_LENGTH = 32
SUPPORTED_PRODUCTS = frozenset({"posthog_code"})
-BILLABLE_AI_PRODUCTS = frozenset(POSTHOG_AI_PRODUCTS + POSTHOG_CODE_AI_PRODUCTS)
DEFAULT_DATE_FROM = "-30d"
MAX_WINDOW_DAYS = 90
diff --git a/products/workflows/backend/services/data_suggestions/llm.py b/products/workflows/backend/services/data_suggestions/llm.py
index e2292b53198e..5196a6ccfe02 100644
--- a/products/workflows/backend/services/data_suggestions/llm.py
+++ b/products/workflows/backend/services/data_suggestions/llm.py
@@ -5,13 +5,12 @@ from typing import Any, Literal, TypeVar
import structlog
from langchain_core.callbacks import BaseCallbackHandler
from langchain_core.runnables import RunnableConfig
-from posthoganalytics.ai.langchain.callbacks import CallbackHandler
from pydantic import BaseModel, ValidationError
from posthog.exceptions_capture import capture_exception
from posthog.models import Team, User
-from ee.hogai.llm import MaxChatAnthropic
+from ee.hogai.llm import MaxCallbackHandler, MaxChatAnthropic
from ee.hogai.utils.feature_flags import is_privacy_mode_enabled
logger = structlog.get_logger(__name__)
@@ -45,7 +44,7 @@ def invoke_structured(
callbacks: list[BaseCallbackHandler] = []
if analytics_client is not None:
callbacks.append(
- CallbackHandler(
+ MaxCallbackHandler(
analytics_client,
distinct_id=user.distinct_id,
trace_id=f"workflows-{feature}-{uuid.uuid4()}",