:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(posthog-ai): route every MaxChatAnthropic call through the Go AI gateway (#127586)
Jonathan Mieloob6a90fd09f3f
diff --git a/ee/hogai/core/agent_modes/executables.py b/ee/hogai/core/agent_modes/executables.pyindex c1973f14e00b..b9cd2c8dbf30 100644--- a/ee/hogai/core/agent_modes/executables.py+++ b/ee/hogai/core/agent_modes/executables.py@@ -294,8 +294,8 @@ class AgentExecutable(BaseAgentLoopRootExecutable):"posthog_properties": self._get_agent_mode_posthog_properties(state),}- if ai_gateway := resolve_ai_gateway_config():- base_model = MaxChatAnthropic.via_ai_gateway(ai_gateway, **model_kwargs)+ if resolve_ai_gateway_config() is not None:+ base_model = MaxChatAnthropic(**model_kwargs)else:gateway_kwargs = self._get_gateway_kwargs()base_model = MaxChatAnthropic(**model_kwargs, bypass_proxy=bool(gateway_kwargs), **gateway_kwargs)diff --git a/ee/hogai/core/runner.py b/ee/hogai/core/runner.pyindex 7bfe3ea8045c..1a61fc680e97 100644--- a/ee/hogai/core/runner.py+++ b/ee/hogai/core/runner.py@@ -12,13 +12,11 @@ import structlogimport posthoganalyticsfrom asgiref.sync import async_to_syncfrom langchain_core.callbacks.base import BaseCallbackHandler-from langchain_core.outputs import LLMResultfrom langchain_core.runnables.config import RunnableConfigfrom langgraph.errors import GraphInterrupt, GraphRecursionErrorfrom langgraph.graph.state import CompiledStateGraphfrom langgraph.types import Command, StreamModefrom opentelemetry import trace-from posthoganalytics.ai.langchain.callbacks import CallbackHandler, GenerationMetadatafrom posthog.schema import (ApprovalDecisionStatus,@@ -47,7 +45,7 @@ from products.posthog_ai.backend.models.assistant import Conversationfrom ee.hogai.core.ai_event_truncation import ai_event_truncatorfrom ee.hogai.core.base import BaseAssistantGraphfrom ee.hogai.core.stream_processor import AssistantStreamProcessorProtocol-from ee.hogai.llm import POSTHOG_AI_PRODUCT, is_ai_gateway_served+from ee.hogai.llm import POSTHOG_AI_PRODUCT, MaxCallbackHandlerfrom ee.hogai.tool import ApprovalRequest, ClientToolCallRequest, pending_approval_proposal_idfrom ee.hogai.utils.exceptions import (AGENT_RUN_UNHANDLED_ERROR_COUNTER,@@ -79,26 +77,6 @@ logger = structlog.get_logger(__name__)_tracer = trace.get_tracer(__name__)-class MaxCallbackHandler(CallbackHandler):- """- Skips $ai_generation for gateway-served calls, which the gateway captures. $ai_trace and $ai_span still emit- because the AI credits query decides free turns from them.- """-- def _capture_generation(- self,- trace_id: Any,- run_id: UUID,- run: GenerationMetadata,- output: LLMResult | BaseException,- parent_run_id: Optional[UUID] = None,- include_parent_id: bool = True,- ):- if is_ai_gateway_served(output):- return- super()._capture_generation(trace_id, run_id, run, output, parent_run_id, include_parent_id)--class SubagentCallbackHandler(MaxCallbackHandler):"""Callback handler for subagents that makes all events appear as children of a parent span.diff --git a/ee/hogai/llm.py b/ee/hogai/llm.pyindex 9160f1638a55..fe4412f5b4d6 100644--- a/ee/hogai/llm.py+++ b/ee/hogai/llm.py@@ -1,4 +1,5 @@import ssl+import timeimport datetimefrom collections.abc import ( # noqa: F401 — Sequence resolves inherited langchain field annotations lazily in this namespaceAsyncIterator,@@ -7,7 +8,8 @@ from collections.abc import ( # noqa: F401 — Sequence resolves inherited langSequence,)from functools import cache, cached_property-from typing import Any, ClassVar, cast+from typing import Any, ClassVar, NamedTuple, cast+from uuid import UUIDfrom django.conf import settings@@ -18,22 +20,24 @@ import anthropicimport structlogfrom asgiref.sync import sync_to_asyncfrom langchain_anthropic import ChatAnthropic-from langchain_core.callbacks import AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun+from langchain_core.callbacks import (+ AsyncCallbackManagerForLLMRun,+ BaseCallbackManager,+ CallbackManagerForLLMRun,+ Callbacks,+)from langchain_core.language_models import BaseChatModelfrom langchain_core.messages import BaseMessage, SystemMessagefrom langchain_core.outputs import ChatGenerationChunk, ChatResult, LLMResultfrom langchain_core.prompts import SystemMessagePromptTemplatefrom langchain_core.runnables import ensure_configfrom langchain_openai import ChatOpenAI+from posthoganalytics.ai.langchain.callbacks import CallbackHandler, GenerationMetadatafrom prometheus_client import Counterfrom pydantic import BaseModel, ConfigDict, Field, model_validator-from posthog.llm.gateway_client import (- AIGatewayConfig,- ai_gateway_headers,- anthropic_gateway_base_url,- resolve_ai_gateway_config,-)+from posthog.llm.billing import BILLABLE_AI_PRODUCTS+from posthog.llm.gateway_client import ai_gateway_headers, anthropic_gateway_base_url, resolve_ai_gateway_configfrom posthog.models import Team, Userfrom posthog.settings import CLOUD_DEPLOYMENT@@ -57,8 +61,23 @@ AI_GATEWAY_TIMEOUT = httpx.Timeout(150.0, connect=10.0, write=15.0, pool=5.0)"""Above the gateway's 120s stream idle close. Nothing heartbeats while the gateway stalls, so every phase is boundedto leave the twin time inside the 300s activity heartbeat."""+AI_GATEWAY_TIMEOUT_COOLDOWN_SECONDS = 300.0+AI_GATEWAY_SERVED_KEY = "ai_gateway_served"+AI_GATEWAY_BILLABLE_PRODUCTS = frozenset(+ {+ "posthog_ai",+ "slack_app",+ "workflows",+ "hogql_decide",+ "posthog_code",+ "alert_investigation_agent",+ "alert_llm_detector",+ "surveys",+ }+)+PROJECT_ORG_USER_CONTEXT_PROMPT = """You are currently in project {{{project_name}}}, which is part of the {{{organization_name}}} organization.The user's name appears to be {{{user_full_name}}} ({{{user_email}}}). Feel free to use their first name when greeting. DO NOT use this name if it appears possibly fake.@@ -117,6 +136,72 @@ def _is_provider_billing_block(error: anthropic.APIStatusError) -> bool:return detail.get("type") == "billing_error" or any(sig in message for sig in _PROVIDER_BILLING_SIGNATURES)+class AIGatewayRoute(NamedTuple):+ ai_product: str+ billable: bool+ trace_id: str | None+ distinct_id: str | None+ privacy_mode: bool | None+ properties: Mapping[str, Any]+++class MaxCallbackHandler(CallbackHandler):+ """+ Skips $ai_generation for gateway-served calls, which the gateway captures. $ai_trace and $ai_span still emit+ because the AI credits query decides free turns from them.+ """++ def _capture_generation(+ self,+ trace_id: Any,+ run_id: UUID,+ run: GenerationMetadata,+ output: LLMResult | BaseException,+ parent_run_id: UUID | None = None,+ include_parent_id: bool = True,+ ):+ if is_ai_gateway_served(output):+ return+ super()._capture_generation(trace_id, run_id, run, output, parent_run_id, include_parent_id)++ @property+ def ai_product(self) -> str | None:+ return (self._properties or {}).get("ai_product")++ @property+ def trace_id(self) -> str | None:+ return str(self._trace_id) if self._trace_id is not None else None++ @property+ def distinct_id(self) -> str | None:+ return str(self._distinct_id) if self._distinct_id is not None else None++ @property+ def privacy_mode(self) -> bool:+ return self._privacy_mode or bool(getattr(self._ph_client, "privacy_mode", False))++ @property+ def capture_disabled(self) -> bool:+ return getattr(self._ph_client, "disabled", False) is True++ @property+ def properties(self) -> Mapping[str, Any]:+ return self._properties or {}+++def _posthog_callback_handlers(callbacks: Callbacks) -> list[CallbackHandler]:+ handlers = callbacks.handlers if isinstance(callbacks, BaseCallbackManager) else callbacks or []+ return [handler for handler in handlers if isinstance(handler, CallbackHandler)]+++def _is_unbilled_credit_bucket_denial(error: anthropic.APIStatusError) -> bool:+ if error.status_code != 402:+ return False+ if error.request.headers.get("X-PostHog-Billable") != "false":+ return False+ return error.response.headers.get("X-PostHog-Denial", "").startswith("credit_bucket_exhausted")++def _carries_output(chunk: ChatGenerationChunk) -> bool:"""Thinking counts: the user sees it stream. An opening metadata chunk does not."""return bool(chunk.message.content) or bool(getattr(chunk.message, "tool_call_chunks", None))@@ -144,6 +229,8 @@ def _ai_gateway_fallback_reason(error: Exception) -> str | None:return "server_error"if isinstance(error, anthropic.APIStatusError) and _is_provider_billing_block(error):return "provider_billing"+ if isinstance(error, anthropic.APIStatusError) and _is_unbilled_credit_bucket_denial(error):+ return "credit_bucket"return None@@ -289,25 +376,51 @@ class MaxChatMixin(BaseModel):return new_kwargs- def _routes_via_ai_gateway(self) -> bool:+ def _ai_gateway_route(self, kwargs: Mapping[str, Any]) -> AIGatewayRoute | None:"""Other products keep their capture and billing on the direct path."""if self.ai_gateway_fallback is None:- return False+ return Noneconfigurable = ensure_config().get("configurable") or {}- return (- configurable.get("ai_product") == POSTHOG_AI_PRODUCT- and (self.posthog_properties or {}).get("ai_product", POSTHOG_AI_PRODUCT) == POSTHOG_AI_PRODUCT+ if configurable.get("ai_product") != POSTHOG_AI_PRODUCT:+ return None+ if (self.posthog_properties or {}).get("ai_product", POSTHOG_AI_PRODUCT) != POSTHOG_AI_PRODUCT:+ return None+ return self._ai_gateway_route_attributed(+ ai_product=POSTHOG_AI_PRODUCT,+ billable=kwargs["metadata"]["posthog_properties"]["$ai_billable"],+ handler=None,)- def _with_ai_gateway_headers(self, kwargs: dict[str, Any]) -> dict[str, Any]:+ def _ai_gateway_route_attributed(+ self, *, ai_product: str, billable: bool, handler: MaxCallbackHandler | None+ ) -> AIGatewayRoute:+ if handler is not None:+ return AIGatewayRoute(+ ai_product=ai_product,+ billable=billable,+ trace_id=handler.trace_id,+ distinct_id=handler.distinct_id or self.user.distinct_id,+ privacy_mode=handler.privacy_mode,+ properties=handler.properties,+ )+ configurable = ensure_config().get("configurable") or {}+ return AIGatewayRoute(+ ai_product=ai_product,+ billable=billable,+ trace_id=str(configurable["trace_id"]) if configurable.get("trace_id") else None,+ distinct_id=configurable.get("distinct_id") or self.user.distinct_id,+ privacy_mode=configurable.get("privacy_mode"),+ properties={},+ )++ def _with_ai_gateway_headers(self, kwargs: dict[str, Any], route: AIGatewayRoute) -> dict[str, Any]:configurable = ensure_config().get("configurable") or {}posthog_properties = kwargs["metadata"]["posthog_properties"]- trace_id = str(configurable["trace_id"]) if configurable.get("trace_id") else Noneconversation_id = str(configurable["thread_id"]) if configurable.get("thread_id") else None# The gateway strips `$` keys and stamps ai_product itself; team_id names the customer it bills.labels = {key: str(value)- for key, value in posthog_properties.items()+ for key, value in {**route.properties, **posthog_properties}.items()if value is not None and not key.startswith("$") and key != "ai_product"}if conversation_id:@@ -317,18 +430,18 @@ class MaxChatMixin(BaseModel):labels["ai_support_impersonated"] = "true"headers = (ai_gateway_headers(- ai_product=POSTHOG_AI_PRODUCT,- trace_id=trace_id,+ ai_product=route.ai_product,+ trace_id=route.trace_id,session_id=conversation_id,properties=labels,- distinct_id=configurable.get("distinct_id") or self.user.distinct_id,+ distinct_id=route.distinct_id,)or {})- if not posthog_properties["$ai_billable"]:+ if not route.billable:headers["X-PostHog-Billable"] = "false"# The gateway records prompts and outputs unless asked not to, so an unknown privacy mode redacts too.- if configurable.get("privacy_mode") is not False:+ if route.privacy_mode is not False:headers["X-PostHog-Privacy-Mode"] = "true"return {**kwargs, "extra_headers": {**(kwargs.get("extra_headers") or {}), **headers}}@@ -552,8 +665,8 @@ class MaxChatOpenAI(MaxChatMixin, ChatOpenAI):messages = self._enrich_messages(messages, project_org_user_variables)kwargs = self._with_posthog_properties(kwargs)- if self._routes_via_ai_gateway():- kwargs = self._with_ai_gateway_headers(kwargs)+ if route := self._ai_gateway_route(kwargs):+ kwargs = self._with_ai_gateway_headers(kwargs, route)return super().generate(messages, *args, **kwargs)@@ -571,8 +684,8 @@ class MaxChatOpenAI(MaxChatMixin, ChatOpenAI):messages = self._enrich_messages(messages, project_org_user_variables)kwargs = self._with_posthog_properties(kwargs)- if self._routes_via_ai_gateway():- kwargs = self._with_ai_gateway_headers(kwargs)+ if route := self._ai_gateway_route(kwargs):+ kwargs = self._with_ai_gateway_headers(kwargs, route)return await super().agenerate(messages, *args, **kwargs)@@ -593,24 +706,32 @@ class MaxChatAnthropic(MaxChatMixin, ChatAnthropic):ai_gateway_fallback: ChatAnthropic | None = Field(default=None, exclude=True)+ _ai_gateway_timed_out_monotonic: ClassVar[float | None] = None++ @model_validator(mode="before")@classmethod- def via_ai_gateway(cls, ai_gateway: AIGatewayConfig, **kwargs: Any) -> "MaxChatAnthropic":- return cls(- **{- **kwargs,- "anthropic_api_url": anthropic_gateway_base_url(ai_gateway.url),- "anthropic_api_key": ai_gateway.api_key,- "bypass_proxy": True,- # The twin is the retry; an SDK retry doubles the wait before it.- "max_retries": 0,- "ai_gateway_fallback": cls(**kwargs),- }- )+ def _via_ai_gateway(cls, data: Any) -> Any:+ if not isinstance(data, dict) or "ai_gateway_fallback" in data:+ return data+ if data.keys() & {"anthropic_api_url", "base_url", "anthropic_api_key", "api_key"}:+ return data+ ai_gateway = resolve_ai_gateway_config()+ if ai_gateway is None:+ return data+ return {+ **data,+ "anthropic_api_url": anthropic_gateway_base_url(ai_gateway.url),+ "anthropic_api_key": ai_gateway.api_key,+ "bypass_proxy": True,+ # The twin is the retry; an SDK retry doubles the wait before it.+ "max_retries": 0,+ "ai_gateway_fallback": cls(**data, ai_gateway_fallback=None),+ }@cached_propertydef _client_params(self) -> dict[str, Any]:params = cast(dict[str, Any], ChatAnthropic._client_params.func(self)) # type: ignore[attr-defined]- if self.ai_gateway_fallback is not None:+ if self.ai_gateway_fallback is not None and self.default_request_timeout is None:# langchain-anthropic defaults to no timeout, so a stalled gateway would never fall back.params["timeout"] = AI_GATEWAY_TIMEOUTreturn params@@ -654,10 +775,62 @@ class MaxChatAnthropic(MaxChatMixin, ChatAnthropic):return kwargsdef get_num_tokens_from_messages(self, messages: list[BaseMessage], tools: Any = None, **kwargs: Any) -> int:- # Token counts have no fallback, so a gateway 429 here would fail the turn.- if self.ai_gateway_fallback is not None:- return self.ai_gateway_fallback.get_num_tokens_from_messages(messages, tools=tools, **kwargs)- return super().get_num_tokens_from_messages(messages, tools=tools, **kwargs)+ fallback = self.ai_gateway_fallback+ if fallback is None:+ return super().get_num_tokens_from_messages(messages, tools=tools, **kwargs)+ if self._ai_gateway_cooling_down():+ AI_GATEWAY_FALLBACK_COUNTER.labels(reason="timeout_cooldown").inc()+ return fallback.get_num_tokens_from_messages(messages, tools=tools, **kwargs)+ try:+ return super().get_num_tokens_from_messages(+ messages,+ tools=tools,+ **{**kwargs, "extra_headers": {**(kwargs.get("extra_headers") or {}), "X-PostHog-Billable": "false"}},+ )+ except Exception as error:+ reason = _ai_gateway_fallback_reason(error)+ if reason is None:+ raise+ self._record_ai_gateway_fallback(reason, error)+ return fallback.get_num_tokens_from_messages(messages, tools=tools, **kwargs)++ def _record_ai_gateway_fallback(self, reason: str, error: Exception) -> None:+ super()._record_ai_gateway_fallback(reason, error)+ if reason == "timeout":+ MaxChatAnthropic._ai_gateway_timed_out_monotonic = time.monotonic()++ @staticmethod+ def _ai_gateway_cooling_down() -> bool:+ timed_out_monotonic = MaxChatAnthropic._ai_gateway_timed_out_monotonic+ if timed_out_monotonic is None:+ return False+ return time.monotonic() - timed_out_monotonic < AI_GATEWAY_TIMEOUT_COOLDOWN_SECONDS++ def _ai_gateway_route(self, kwargs: Mapping[str, Any]) -> AIGatewayRoute | None:+ if self.ai_gateway_fallback is None:+ return None+ handlers = _posthog_callback_handlers(kwargs.get("callbacks")) + _posthog_callback_handlers(self.callbacks)+ if not all(isinstance(handler, MaxCallbackHandler) for handler in handlers):+ return None+ posthog_properties = kwargs["metadata"]["posthog_properties"]+ handler = cast(MaxCallbackHandler, handlers[0]) if handlers else None+ if handler is not None and handler.capture_disabled:+ return None+ billable = False+ if handler is not None:+ ai_product = handler.ai_product or posthog_properties["ai_product"]+ if posthog_properties["$ai_billable"]:+ billable = ai_product in BILLABLE_AI_PRODUCTS+ else:+ configurable = ensure_config().get("configurable") or {}+ ai_product = configurable.get("ai_product") or posthog_properties["ai_product"]+ if billable:+ if ai_product not in AI_GATEWAY_BILLABLE_PRODUCTS:+ return None+ if self._ai_gateway_cooling_down():+ AI_GATEWAY_FALLBACK_COUNTER.labels(reason="timeout_cooldown").inc()+ return None+ return self._ai_gateway_route_attributed(ai_product=ai_product, billable=billable, handler=handler)def generate(self,@@ -670,6 +843,8 @@ class MaxChatAnthropic(MaxChatMixin, ChatAnthropic):messages = self._enrich_messages(messages, project_org_user_variables)kwargs = self._with_posthog_properties(kwargs)+ if route := self._ai_gateway_route(kwargs):+ kwargs = self._with_ai_gateway_headers(kwargs, route)return super().generate(messages, *args, **kwargs)@@ -684,7 +859,7 @@ class MaxChatAnthropic(MaxChatMixin, ChatAnthropic):messages = self._enrich_messages(messages, project_org_user_variables)kwargs = self._with_posthog_properties(kwargs)- if self._routes_via_ai_gateway():- kwargs = self._with_ai_gateway_headers(kwargs)+ if route := self._ai_gateway_route(kwargs):+ kwargs = self._with_ai_gateway_headers(kwargs, route)return await super().agenerate(messages, *args, **kwargs)diff --git a/ee/hogai/test/test_llm.py b/ee/hogai/test/test_llm.pyindex 5965e254ba3b..5bccaf16dfdd 100644--- a/ee/hogai/test/test_llm.py+++ b/ee/hogai/test/test_llm.py@@ -5,7 +5,7 @@ from functools import cached_propertyfrom pathlib import Pathfrom posthog.test.base import BaseTest-from unittest.mock import patch+from unittest.mock import Mock, patchfrom django.test import SimpleTestCase@@ -17,20 +17,25 @@ from langchain_anthropic import ChatAnthropicfrom langchain_core.messages import AIMessage, AIMessageChunk, BaseMessage, HumanMessage, SystemMessagefrom langchain_core.outputs import ChatGeneration, ChatGenerationChunk, Generation, LLMResultfrom parameterized import parameterized+from posthoganalytics.ai.langchain.callbacks import CallbackHandlerfrom posthog.schema import PersonsOnEventsMode+from posthog.llm.billing import BILLABLE_AI_PRODUCTSfrom posthog.llm.gateway_client import AIGatewayConfigfrom posthog.models import Organization, Team, Userfrom posthog.settings import BASE_DIRfrom posthog.temporal.ai.chat_agent import CHAT_AGENT_ACTIVITY_HEARTBEAT_TIMEOUTfrom ee.hogai.llm import (+ AI_GATEWAY_BILLABLE_PRODUCTS,AI_GATEWAY_FALLBACK_COUNTER,AI_GATEWAY_SERVED_KEY,AI_GATEWAY_TIMEOUT,+ AI_GATEWAY_TIMEOUT_COOLDOWN_SECONDS,BILLING_SKIPPED_COUNTER,PROJECT_ORG_USER_CONTEXT_PROMPT,+ MaxCallbackHandler,MaxChatAnthropic,MaxChatOpenAI,is_ai_gateway_served,@@ -677,6 +682,9 @@ class TestMaxChatAnthropicAIGateway(BaseTest):super().setUp()self.requests: list[httpx.Request] = []self.gateway_outcome: httpx.Response | Exception = httpx.Response(200, json=_anthropic_message_body("gateway"))+ cooldown = patch.object(MaxChatAnthropic, "_ai_gateway_timed_out_monotonic", None)+ cooldown.start()+ self.addCleanup(cooldown.stop)def _respond(self, request: httpx.Request) -> httpx.Response:self.requests.append(request)@@ -686,12 +694,12 @@ class TestMaxChatAnthropicAIGateway(BaseTest):raise self.gateway_outcomereturn self.gateway_outcome+ def _gateway_model(self, **kwargs) -> MaxChatAnthropic:+ with patch("ee.hogai.llm.resolve_ai_gateway_config", return_value=self.gateway):+ return MaxChatAnthropic(model="claude-sonnet-4-6", user=self.user, team=self.team, **kwargs)+def _model(self, **kwargs) -> MaxChatAnthropic:- model = MaxChatAnthropic.via_ai_gateway(- self.gateway,- model="claude-sonnet-4-6",- user=self.user,- team=self.team,+ model = self._gateway_model(inject_context=False,billable=True,posthog_properties={"agent_mode": "sql", "supermode": None},@@ -711,6 +719,11 @@ class TestMaxChatAnthropicAIGateway(BaseTest):assert model.ai_gateway_fallback is not Nonereturn httpx.URL(model.ai_gateway_fallback.anthropic_api_url or "").host+ def _max_handler(self, **kwargs) -> MaxCallbackHandler:+ return MaxCallbackHandler(+ Mock(privacy_mode=False), **{"distinct_id": "distinct-1", "trace_id": "trace-1", **kwargs}+ )+def _config(self, **configurable) -> dict:return {"configurable": {@@ -760,8 +773,8 @@ class TestMaxChatAnthropicAIGateway(BaseTest):return _astream- def test_via_ai_gateway_targets_the_gateway_and_keeps_a_direct_twin(self):- model = MaxChatAnthropic.via_ai_gateway(self.gateway, model="claude-sonnet-4-6", user=self.user, team=self.team)+ def test_gateway_config_targets_the_gateway_and_keeps_a_direct_twin(self):+ model = self._gateway_model()twin = model.ai_gateway_fallbackself.assertEqual(model.anthropic_api_url, "https://ai-gateway.test")@@ -787,7 +800,7 @@ class TestMaxChatAnthropicAIGateway(BaseTest):model = self._model()config = self._config(is_agent_billable=is_agent_billable, is_impersonated=is_impersonated)with patch("ee.hogai.llm.ensure_config", return_value=config):- result = await model.agenerate([[HumanMessage(content="hello")]])+ result = await model.agenerate([[HumanMessage(content="hello")]], callbacks=[self._max_handler()])self.assertEqual([str(request.url) for request in self.requests], ["https://ai-gateway.test/v1/messages"]@@ -823,24 +836,88 @@ class TestMaxChatAnthropicAIGateway(BaseTest):self.assertEqual(self.requests[0].headers.get("x-posthog-privacy-mode"), redaction_header)self.assertTrue(is_ai_gateway_served(result))- async def test_calls_for_other_products_go_direct(self):- cases: list[tuple[str, dict, dict | None]] = [- ("mcp conversation", self._config(ai_product="mcp"), None),- ("outside a posthog_ai run", self._config(ai_product=None), None),- ("product override", self._config(), {"ai_product": "alert_investigation_agent"}),+ @parameterized.expand(+ [+ ("mcp conversation is unbilled", "mcp", None, True, "mcp", "false"),+ ("posthog_ai turn is billed", "posthog_ai", None, True, "posthog_ai", None),+ ("product outside a runner", None, "alert_investigation_agent", True, "alert_investigation_agent", None),+ (+ "no handler bills nothing",+ None,+ "alert_investigation_agent",+ False,+ "alert_investigation_agent",+ "false",+ ),]- for name, config, posthog_properties in cases:- with self.subTest(name):- self.requests.clear()- model = self._model()- if posthog_properties:- model.posthog_properties = posthog_properties- with patch("ee.hogai.llm.ensure_config", return_value=config):- result = await model.agenerate([[HumanMessage(content="hello")]])+ )+ async def test_routes_every_product_with_its_label_and_current_billing(+ self, _name, configurable_product, properties_product, with_handler, expected_product, billable_header+ ):+ model = self._model()+ if properties_product:+ model.posthog_properties = {"ai_product": properties_product}+ callbacks = [self._max_handler(properties={"ai_product": configurable_product})] if with_handler else None+ with patch("ee.hogai.llm.ensure_config", return_value=self._config(ai_product=configurable_product)):+ result = await model.agenerate([[HumanMessage(content="hello")]], callbacks=callbacks)++ self.assertEqual([request.url.host for request in self.requests], ["ai-gateway.test"])+ headers = self.requests[0].headers+ self.assertEqual(headers["x-posthog-product"], expected_product)+ self.assertEqual(headers.get("x-posthog-billable"), billable_header)+ self.assertTrue(is_ai_gateway_served(result))- self.assertEqual([request.url.host for request in self.requests], [self._direct_host(model)])- self.assertFalse(any(header.lower().startswith("x-posthog-") for header in self.requests[0].headers))- self.assertFalse(is_ai_gateway_served(result))+ async def test_attached_handler_attribution_wins_over_an_inherited_config(self):+ model = self._model()+ model.posthog_properties = {"ai_product": "workflows"}+ model.billable = False+ handler = self._max_handler(+ distinct_id="workflows-user",+ trace_id="workflows-trace",+ privacy_mode=True,+ properties={"ai_product": "workflows", "alert_id": "alert-1"},+ )+ with patch("ee.hogai.llm.ensure_config", return_value=self._config(privacy_mode=False)):+ await model.agenerate([[HumanMessage(content="hello")]], callbacks=[handler])++ headers = self.requests[0].headers+ self.assertEqual(+ [+ headers["x-posthog-product"],+ headers["x-posthog-trace-id"],+ headers["x-posthog-distinct-id"],+ headers.get("x-posthog-privacy-mode"),+ headers.get("x-posthog-billable"),+ json.loads(headers["x-posthog-properties"]).get("alert_id"),+ ],+ ["workflows", "workflows-trace", "workflows-user", "true", "false", "alert-1"],+ )++ @parameterized.expand(+ [+ ("handler that would capture the generation again", "posthog_ai", "plain"),+ ("billed product the gateway does not bill", "replay_vision", "max"),+ ("capture opted out", "posthog_ai", "disabled"),+ ]+ )+ async def test_calls_that_would_change_capture_or_billing_go_direct(self, _name, ai_product, handler_kind):+ model = self._model()+ model.posthog_properties = {"ai_product": ai_product}+ handlers = {+ "max": lambda: self._max_handler(),+ "disabled": lambda: MaxCallbackHandler(Mock(privacy_mode=False, disabled=True), distinct_id="d"),+ "plain": lambda: CallbackHandler(Mock(), distinct_id="d"),+ }+ handler = handlers[handler_kind]()+ with patch("ee.hogai.llm.ensure_config", return_value=self._config(ai_product=None)):+ result = await model.agenerate([[HumanMessage(content="hello")]], callbacks=[handler])++ self.assertEqual([request.url.host for request in self.requests], [self._direct_host(model)])+ self.assertFalse(any(header.lower().startswith("x-posthog-") for header in self.requests[0].headers))+ self.assertFalse(is_ai_gateway_served(result))++ def test_gateway_billable_products_are_billed_today(self):+ self.assertLessEqual(AI_GATEWAY_BILLABLE_PRODUCTS, BILLABLE_AI_PRODUCTS)async def test_retryable_gateway_failures_fall_back_to_the_direct_twin(self):request = httpx.Request("POST", "https://ai-gateway.test/v1/messages")@@ -853,11 +930,18 @@ class TestMaxChatAnthropicAIGateway(BaseTest):("server_error", _anthropic_error(529)),("provider_billing", _anthropic_error(400, "invalid_request_error", _WORKSPACE_LIMIT)),("provider_billing", _anthropic_error(402, "billing_error", "Payment required")),+ (+ "credit_bucket",+ _anthropic_error(+ 402, "billing_error", "out", {"X-PostHog-Denial": "credit_bucket_exhausted:ai_credits"}+ ),+ ),]for reason, outcome in cases:with self.subTest(reason=reason, outcome=outcome):self.requests.clear()self.gateway_outcome = outcome+ MaxChatAnthropic._ai_gateway_timed_out_monotonic = Nonemodel = self._model()with (patch("ee.hogai.llm.ensure_config", return_value=self._config()),@@ -875,6 +959,36 @@ class TestMaxChatAnthropicAIGateway(BaseTest):self.assertFalse(is_ai_gateway_served(result))mock_labels.assert_called_once_with(reason=reason)+ async def test_a_gateway_timeout_sends_calls_direct_until_the_cooldown_ends(self):+ self.gateway_outcome = httpx.ReadTimeout("slow", request=httpx.Request("POST", "https://ai-gateway.test"))+ model = self._model()+ direct = self._direct_host(model)+ with patch("ee.hogai.llm.ensure_config", return_value=self._config()):+ await model.agenerate([[HumanMessage(content="hello")]])+ await self._model().agenerate([[HumanMessage(content="hello")]])+ with patch.object(+ ChatAnthropic,+ "get_num_tokens_from_messages",+ lambda counting_model, *args, **kwargs: 3 if counting_model.ai_gateway_fallback else 7,+ ):+ tokens = self._model().get_num_tokens_from_messages([HumanMessage(content="hello")])+ assert MaxChatAnthropic._ai_gateway_timed_out_monotonic is not None+ MaxChatAnthropic._ai_gateway_timed_out_monotonic -= AI_GATEWAY_TIMEOUT_COOLDOWN_SECONDS+ self.gateway_outcome = httpx.Response(200, json=_anthropic_message_body("gateway"))+ result = await self._model().agenerate([[HumanMessage(content="hello")]])++ self.assertEqual(+ [(request.url.host, request.url.path) for request in self.requests],+ [+ ("ai-gateway.test", "/v1/messages"),+ (direct, "/v1/messages"),+ (direct, "/v1/messages"),+ ("ai-gateway.test", "/v1/messages"),+ ],+ )+ self.assertEqual(tokens, 7)+ self.assertTrue(is_ai_gateway_served(result))+async def test_non_retryable_gateway_failures_raise_without_fallback(self):cases: list[tuple[str, httpx.Response]] = [("bad request", _anthropic_error(400, "invalid_request_error", "prompt is too long")),@@ -882,6 +996,12 @@ class TestMaxChatAnthropicAIGateway(BaseTest):"gateway's own credit denial",_anthropic_error(402, "billing_error", _WORKSPACE_LIMIT, {"X-PostHog-Denial": "insufficient_credits"}),),+ (+ "credit bucket denial of a billed call",+ _anthropic_error(+ 402, "billing_error", "out", {"X-PostHog-Denial": "credit_bucket_exhausted:ai_credits"}+ ),+ ),("forbidden", _anthropic_error(403)),("not found", _anthropic_error(404)),]@@ -895,7 +1015,7 @@ class TestMaxChatAnthropicAIGateway(BaseTest):patch.object(AI_GATEWAY_FALLBACK_COUNTER, "labels") as mock_labels,self.assertRaises(Exception),):- await model.agenerate([[HumanMessage(content="hello")]])+ await model.agenerate([[HumanMessage(content="hello")]], callbacks=[self._max_handler()])self.assertEqual([request.url.host for request in self.requests], ["ai-gateway.test"])mock_labels.assert_not_called()@@ -1017,7 +1137,7 @@ class TestMaxChatAnthropicAIGateway(BaseTest):self.assertTrue(is_ai_gateway_served(result))def test_gateway_client_has_a_bounded_timeout(self):- model = MaxChatAnthropic.via_ai_gateway(self.gateway, model="claude-sonnet-4-6", user=self.user, team=self.team)+ model = self._gateway_model()twin = model.ai_gateway_fallbackassert twin is not None@@ -1038,22 +1158,45 @@ class TestMaxChatAnthropicAIGateway(BaseTest):stalled_gateway_seconds = sum(phase or 0.0 for phase in phases) * (model.max_retries + 1)self.assertGreaterEqual(CHAT_AGENT_ACTIVITY_HEARTBEAT_TIMEOUT - stalled_gateway_seconds, 120)- def test_token_counting_uses_the_direct_twin(self):- model = MaxChatAnthropic.via_ai_gateway(self.gateway, model="claude-sonnet-4-6", user=self.user, team=self.team)+ def test_gateway_client_keeps_the_callers_timeout(self):+ model = self._gateway_model(default_request_timeout=30.0)++ self.assertEqual(model._async_client.timeout, 30.0)++ @parameterized.expand(+ [+ ("gateway counts", None, 3),+ ("gateway rate limit falls back to the twin", 429, 7),+ ("exhausted credits fall back to the twin", 402, 7),+ ]+ )+ def test_token_counting_goes_through_the_gateway(self, _name, gateway_status, expected_tokens):+ model = self._gateway_model()def count(counting_model, messages, tools=None, **kwargs):- return 0 if counting_model.ai_gateway_fallback is not None else 7+ if counting_model.ai_gateway_fallback is None:+ return 7+ if gateway_status is None:+ return 3+ request = httpx.Request(+ "POST", "https://ai-gateway.test/v1/messages/count_tokens", headers=kwargs.get("extra_headers")+ )+ response = httpx.Response(+ gateway_status, request=request, headers={"X-PostHog-Denial": "credit_bucket_exhausted"}+ )+ raise anthropic.APIStatusError("denied", response=response, body=None)with patch.object(ChatAnthropic, "get_num_tokens_from_messages", count):- self.assertEqual(model.get_num_tokens_from_messages([HumanMessage(content="hello")]), 7)+ self.assertEqual(model.get_num_tokens_from_messages([HumanMessage(content="hello")]), expected_tokens)- def test_sync_calls_use_the_direct_twin(self):+ def test_sync_calls_route_through_the_gateway(self):model = self._model()with patch("ee.hogai.llm.ensure_config", return_value=self._config()):- result = model.generate([[HumanMessage(content="hello")]])+ result = model.generate([[HumanMessage(content="hello")]], callbacks=[self._max_handler()])- self.assertEqual([request.url.host for request in self.requests], [self._direct_host(model)])- self.assertFalse(is_ai_gateway_served(result))+ self.assertEqual([request.url.host for request in self.requests], ["ai-gateway.test"])+ self.assertEqual(self.requests[0].headers["x-posthog-product"], "posthog_ai")+ self.assertTrue(is_ai_gateway_served(result))def test_is_ai_gateway_served_reads_the_generation_marker(self):served = LLMResult(diff --git a/posthog/llm/billing.py b/posthog/llm/billing.pyindex 3b936cae0751..3ce7483bf129 100644--- a/posthog/llm/billing.py+++ b/posthog/llm/billing.py@@ -1 +1,20 @@AI_COST_MARKUP_PERCENT = 0.2++# ai_product values that roll into PostHog AI (Max) billing credits.+POSTHOG_AI_PRODUCTS = [+ "posthog_ai",+ "slack_app",+ "workflows",+ "subscriptions",+ "alert_investigation_agent",+ "alert_llm_detector",+ "product_analytics",+ "surveys",+ "replay_vision",+ "hogql_decide",+]++# ai_product values billed as PostHog Desktop credits.+POSTHOG_CODE_AI_PRODUCTS = ["posthog_code"]++BILLABLE_AI_PRODUCTS = frozenset(POSTHOG_AI_PRODUCTS + POSTHOG_CODE_AI_PRODUCTS)diff --git a/posthog/tasks/usage_report.py b/posthog/tasks/usage_report.pyindex 2ffb73e4198f..c065548f0997 100644--- a/posthog/tasks/usage_report.py+++ b/posthog/tasks/usage_report.py@@ -33,7 +33,7 @@ from posthog.cloud_utils import get_cached_instance_licensefrom posthog.constants import AI_EVENT_NAME_PREFIX, MCP_EVENT_NAME_PREFIX, FlagRequestTypefrom posthog.dataclasses import frozenfrom posthog.exceptions_capture import capture_exception-from posthog.llm.billing import AI_COST_MARKUP_PERCENT+from posthog.llm.billing import AI_COST_MARKUP_PERCENT, POSTHOG_AI_PRODUCTS, POSTHOG_CODE_AI_PRODUCTSfrom posthog.logging.timing import timed_logfrom posthog.models import OrganizationMembership, Userfrom posthog.models.ai_events.sql import TABLE_BASE_NAME as AI_EVENTS_TABLE@@ -1918,23 +1918,7 @@ CLOUD_REGION_TO_URL = {}-# ai_product values that roll into PostHog AI (Max) billing credits.-POSTHOG_AI_PRODUCTS = [- "posthog_ai",- "slack_app",- "workflows",- "subscriptions",- "alert_investigation_agent",- "alert_llm_detector",- "product_analytics",- "surveys",- "replay_vision",- "hogql_decide",-]--# ai_product values billed as PostHog Desktop credits.UNBILLED_TASK_ORIGIN_PRODUCTS = ("task_analysis", "account_audit")-POSTHOG_CODE_AI_PRODUCTS = ["posthog_code"]def get_ai_billing_instance_group_type_index(team_id: int) -> int | None:diff --git a/posthog/temporal/ai/anomaly_investigation/runner.py b/posthog/temporal/ai/anomaly_investigation/runner.pyindex f958b1b35d3a..3f9ffc99c351 100644--- a/posthog/temporal/ai/anomaly_investigation/runner.py+++ b/posthog/temporal/ai/anomaly_investigation/runner.py@@ -22,7 +22,6 @@ import posthoganalyticsfrom langchain_core.callbacks import BaseCallbackHandlerfrom langchain_core.messages import HumanMessage, SystemMessage, ToolMessagefrom langchain_core.runnables import RunnableConfig-from posthoganalytics.ai.langchain.callbacks import CallbackHandlerfrom pydantic import BaseModel, ValidationErrorfrom posthog.models import Team, User@@ -433,6 +432,8 @@ async def run_investigation(def _build_callbacks(*, team: Team, alert: AlertConfiguration | None) -> list[BaseCallbackHandler]:callbacks: list[BaseCallbackHandler] = []+ from ee.hogai.llm import MaxCallbackHandler+client = posthoganalytics.default_clientif client is None:return callbacks@@ -443,7 +444,7 @@ def _build_callbacks(*, team: Team, alert: AlertConfiguration | None) -> list[Baif alert is not None:properties["alert_id"] = str(alert.id)callbacks.append(- CallbackHandler(+ MaxCallbackHandler(client,distinct_id=str(team.id),trace_id=f"alert-investigation-{uuid.uuid4()}",diff --git a/posthog/temporal/tests/ai/test_anomaly_investigation_runner.py b/posthog/temporal/tests/ai/test_anomaly_investigation_runner.pyindex 1bfd23c4ab2e..648579a2902c 100644--- a/posthog/temporal/tests/ai/test_anomaly_investigation_runner.py+++ b/posthog/temporal/tests/ai/test_anomaly_investigation_runner.py@@ -277,7 +277,7 @@ def test_build_callbacks_tags_ai_product_for_llm_analytics(alert_id, expected_prwith (patch("posthog.temporal.ai.anomaly_investigation.runner.posthoganalytics") as mock_module,- patch("posthog.temporal.ai.anomaly_investigation.runner.CallbackHandler") as mock_handler,+ patch("ee.hogai.llm.MaxCallbackHandler") as mock_handler,):mock_module.default_client = sentinel_clientdiff --git a/products/ai_observability/backend/api/personal_spend.py b/products/ai_observability/backend/api/personal_spend.pyindex b2ed34c0f453..9f4d6c951a0e 100644--- a/products/ai_observability/backend/api/personal_spend.py+++ b/products/ai_observability/backend/api/personal_spend.py@@ -47,10 +47,10 @@ from posthog.auth import (WebhookSignatureAuthentication,)from posthog.clickhouse.query_tagging import Feature, Product, tags_context+from posthog.llm.billing import BILLABLE_AI_PRODUCTSfrom posthog.models import Team, Userfrom posthog.permissions import APIScopePermissionfrom posthog.rate_limit import PersonalSpendBurstThrottle, PersonalSpendDailyThrottle, PersonalSpendSustainedThrottle-from posthog.tasks.usage_report import POSTHOG_AI_PRODUCTS, POSTHOG_CODE_AI_PRODUCTSfrom posthog.utils import relative_date_parselogger = structlog.get_logger(__name__)@@ -59,7 +59,6 @@ MAX_PRODUCT_KEY_LENGTH = 64MAX_DATE_STRING_LENGTH = 32SUPPORTED_PRODUCTS = frozenset({"posthog_code"})-BILLABLE_AI_PRODUCTS = frozenset(POSTHOG_AI_PRODUCTS + POSTHOG_CODE_AI_PRODUCTS)DEFAULT_DATE_FROM = "-30d"MAX_WINDOW_DAYS = 90diff --git a/products/workflows/backend/services/data_suggestions/llm.py b/products/workflows/backend/services/data_suggestions/llm.pyindex e2292b53198e..5196a6ccfe02 100644--- a/products/workflows/backend/services/data_suggestions/llm.py+++ b/products/workflows/backend/services/data_suggestions/llm.py@@ -5,13 +5,12 @@ from typing import Any, Literal, TypeVarimport structlogfrom langchain_core.callbacks import BaseCallbackHandlerfrom langchain_core.runnables import RunnableConfig-from posthoganalytics.ai.langchain.callbacks import CallbackHandlerfrom pydantic import BaseModel, ValidationErrorfrom posthog.exceptions_capture import capture_exceptionfrom posthog.models import Team, User-from ee.hogai.llm import MaxChatAnthropic+from ee.hogai.llm import MaxCallbackHandler, MaxChatAnthropicfrom ee.hogai.utils.feature_flags import is_privacy_mode_enabledlogger = structlog.get_logger(__name__)@@ -45,7 +44,7 @@ def invoke_structured(callbacks: list[BaseCallbackHandler] = []if analytics_client is not None:callbacks.append(- CallbackHandler(+ MaxCallbackHandler(analytics_client,distinct_id=user.distinct_id,trace_id=f"workflows-{feature}-{uuid.uuid4()}",