:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(posthog-ai): route MaxChatOpenAI through the ai-gateway (#125611)
Jonathan Mieloo9971ec112c98
diff --git a/ee/hogai/llm.py b/ee/hogai/llm.pyindex 17e3328f90f9..9160f1638a55 100644--- a/ee/hogai/llm.py+++ b/ee/hogai/llm.py@@ -1,3 +1,4 @@+import sslimport datetimefrom collections.abc import ( # noqa: F401 — Sequence resolves inherited langchain field annotations lazily in this namespaceAsyncIterator,@@ -5,27 +6,34 @@ from collections.abc import ( # noqa: F401 — Sequence resolves inherited langMapping,Sequence,)-from functools import cached_property+from functools import cache, cached_propertyfrom typing import Any, ClassVar, castfrom django.conf import settingsimport pytzimport httpx+import openaiimport anthropicimport structlogfrom asgiref.sync import sync_to_asyncfrom langchain_anthropic import ChatAnthropicfrom langchain_core.callbacks import AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun+from langchain_core.language_models import BaseChatModelfrom langchain_core.messages import BaseMessage, SystemMessagefrom langchain_core.outputs import ChatGenerationChunk, ChatResult, LLMResultfrom langchain_core.prompts import SystemMessagePromptTemplatefrom langchain_core.runnables import ensure_configfrom langchain_openai import ChatOpenAIfrom prometheus_client import Counter-from pydantic import BaseModel, ConfigDict, Field+from pydantic import BaseModel, ConfigDict, Field, model_validator-from posthog.llm.gateway_client import AIGatewayConfig, ai_gateway_headers, anthropic_gateway_base_url+from posthog.llm.gateway_client import (+ AIGatewayConfig,+ ai_gateway_headers,+ anthropic_gateway_base_url,+ resolve_ai_gateway_config,+)from posthog.models import Team, Userfrom posthog.settings import CLOUD_DEPLOYMENT@@ -79,6 +87,11 @@ OPENAI_FLEX_MODELS = ["o3", "gpt-5.6-terra", "gpt5", "gpt5-mini", "gpt5-nano"]_BYPASS_PROXY_MOUNTS: dict[str, None] = {"http://": None, "https://": None, "all://": None}+@cache+def _ai_gateway_ssl_context() -> ssl.SSLContext:+ return httpx.create_ssl_context(trust_env=False)++def is_ai_gateway_served(output: object) -> bool:if not isinstance(output, LLMResult):return False@@ -118,18 +131,18 @@ def _marked_as_gateway_served(chunks: list[ChatGenerationChunk]) -> list[ChatGendef _ai_gateway_fallback_reason(error: Exception) -> str | None:"""None re-raises: a direct retry cannot fix other errors, and the gateway's own 402/403 must stay binding."""- if isinstance(error, anthropic.APITimeoutError):+ if isinstance(error, (anthropic.APITimeoutError, openai.APITimeoutError)):return "timeout"- if isinstance(error, anthropic.APIConnectionError):+ if isinstance(error, (anthropic.APIConnectionError, openai.APIConnectionError)):return "connection"- if isinstance(error, anthropic.APIStatusError):+ if isinstance(error, (anthropic.APIStatusError, openai.APIStatusError)):if error.status_code == 401:return "unauthorized"if error.status_code == 429:return "rate_limited"if error.status_code >= 500:return "server_error"- if _is_provider_billing_block(error):+ if isinstance(error, anthropic.APIStatusError) and _is_provider_billing_block(error):return "provider_billing"return None@@ -157,6 +170,7 @@ class MaxChatMixin(BaseModel):user: Userteam: Teammax_retries: int | None = None+ streaming: bool = Falsestream_usage: bool | None = Noneconversation_start_dt: datetime.datetime | None = None"""@@ -177,6 +191,8 @@ class MaxChatMixin(BaseModel):These will be merged with the standard properties like $ai_billable and team_id."""posthog_provider: ClassVar[str]+ ai_gateway_fallback: BaseChatModel | None = Field(default=None, exclude=True)+ """Direct twin that serves calls which must not route, and gateway failures before any output."""def model_post_init(self, __context: Any) -> None:if self.max_retries is None:@@ -273,6 +289,199 @@ class MaxChatMixin(BaseModel):return new_kwargs+ def _routes_via_ai_gateway(self) -> bool:+ """Other products keep their capture and billing on the direct path."""+ if self.ai_gateway_fallback is None:+ return False+ configurable = ensure_config().get("configurable") or {}+ return (+ configurable.get("ai_product") == POSTHOG_AI_PRODUCT+ and (self.posthog_properties or {}).get("ai_product", POSTHOG_AI_PRODUCT) == POSTHOG_AI_PRODUCT+ )++ def _with_ai_gateway_headers(self, kwargs: dict[str, Any]) -> dict[str, Any]:+ configurable = ensure_config().get("configurable") or {}+ posthog_properties = kwargs["metadata"]["posthog_properties"]+ trace_id = str(configurable["trace_id"]) if configurable.get("trace_id") else None+ conversation_id = str(configurable["thread_id"]) if configurable.get("thread_id") else None+ # The gateway strips `$` keys and stamps ai_product itself; team_id names the customer it bills.+ labels = {+ key: str(value)+ for key, value in posthog_properties.items()+ if value is not None and not key.startswith("$") and key != "ai_product"+ }+ if conversation_id:+ labels["conversation_id"] = conversation_id+ # Support-trace filters read this property, which the SDK handler sets only on its own events.+ if configurable.get("is_impersonated"):+ labels["ai_support_impersonated"] = "true"+ headers = (+ ai_gateway_headers(+ ai_product=POSTHOG_AI_PRODUCT,+ trace_id=trace_id,+ session_id=conversation_id,+ properties=labels,+ distinct_id=configurable.get("distinct_id") or self.user.distinct_id,+ )+ or {}+ )+ if not posthog_properties["$ai_billable"]:+ headers["X-PostHog-Billable"] = "false"+ # The gateway records prompts and outputs unless asked not to, so an unknown privacy mode redacts too.+ if configurable.get("privacy_mode") is not False:+ headers["X-PostHog-Privacy-Mode"] = "true"+ return {**kwargs, "extra_headers": {**(kwargs.get("extra_headers") or {}), **headers}}++ def _record_ai_gateway_fallback(self, reason: str, error: Exception) -> None:+ AI_GATEWAY_FALLBACK_COUNTER.labels(reason=reason).inc()+ model_name = getattr(self, "model", None) or getattr(self, "model_name", "unknown")+ logger.warning("posthog_ai_gateway_fallback", reason=reason, error_type=type(error).__name__, model=model_name)++ def _stream(+ self,+ messages: list[BaseMessage],+ stop: list[str] | None = None,+ run_manager: CallbackManagerForLLMRun | None = None,+ **kwargs: Any,+ ) -> Iterator[ChatGenerationChunk]:+ fallback = self.ai_gateway_fallback+ if fallback is None:+ yield from cast(BaseChatModel, super())._stream(messages, stop=stop, run_manager=run_manager, **kwargs)+ return++ direct_kwargs = _without_ai_gateway_headers(kwargs)+ if _has_ai_gateway_headers(kwargs):+ served = False+ held: list[ChatGenerationChunk] = []+ try:+ for chunk in cast(BaseChatModel, super())._stream(+ messages, stop=stop, run_manager=run_manager, **kwargs+ ):+ if served:+ yield chunk+ continue+ held.append(chunk)+ if not _carries_output(chunk):+ continue+ served = True+ yield from _marked_as_gateway_served(held)+ if not served and held:+ yield from _marked_as_gateway_served(held)+ return+ except Exception as error:+ reason = _ai_gateway_fallback_reason(error)+ if served or reason is None:+ raise+ self._record_ai_gateway_fallback(reason, error)++ yield from fallback._stream(messages, stop=stop, run_manager=run_manager, **direct_kwargs)++ def _generate(+ self,+ messages: list[BaseMessage],+ stop: list[str] | None = None,+ run_manager: CallbackManagerForLLMRun | None = None,+ **kwargs: Any,+ ) -> ChatResult:+ fallback = self.ai_gateway_fallback+ if fallback is None or self.streaming:+ return cast(BaseChatModel, super())._generate(messages, stop=stop, run_manager=run_manager, **kwargs)++ direct_kwargs = _without_ai_gateway_headers(kwargs)+ if _has_ai_gateway_headers(kwargs):+ try:+ result = cast(BaseChatModel, super())._generate(messages, stop=stop, run_manager=run_manager, **kwargs)+ except Exception as error:+ reason = _ai_gateway_fallback_reason(error)+ if reason is None:+ raise+ self._record_ai_gateway_fallback(reason, error)+ else:+ for generation in result.generations:+ generation.generation_info = {**(generation.generation_info or {}), AI_GATEWAY_SERVED_KEY: True}+ return result++ return fallback._generate(messages, stop=stop, run_manager=run_manager, **direct_kwargs)++ async def _astream(+ self,+ messages: list[BaseMessage],+ stop: list[str] | None = None,+ run_manager: AsyncCallbackManagerForLLMRun | None = None,+ **kwargs: Any,+ ) -> AsyncIterator[ChatGenerationChunk]:+ fallback = self.ai_gateway_fallback+ if fallback is None:+ async for chunk in cast(BaseChatModel, super())._astream(+ messages, stop=stop, run_manager=run_manager, **kwargs+ ):+ yield chunk+ return++ direct_kwargs = _without_ai_gateway_headers(kwargs)+ if _has_ai_gateway_headers(kwargs):+ served = False+ # Anthropic opens the stream with a metadata chunk. Holding those keeps the fallback open until real+ # output, and keeps their usage off the twin's generation.+ held: list[ChatGenerationChunk] = []+ try:+ async for chunk in cast(BaseChatModel, super())._astream(+ messages, stop=stop, run_manager=run_manager, **kwargs+ ):+ if served:+ yield chunk+ continue+ held.append(chunk)+ if not _carries_output(chunk):+ continue+ served = True+ for pending in _marked_as_gateway_served(held):+ yield pending+ if not served and held:+ # A stream that ended without output was still served by the gateway.+ for pending in _marked_as_gateway_served(held):+ yield pending+ return+ except Exception as error:+ reason = _ai_gateway_fallback_reason(error)+ # After output the user has seen the turn, so a direct retry would repeat it.+ if served or reason is None:+ raise+ self._record_ai_gateway_fallback(reason, error)++ async for chunk in fallback._astream(messages, stop=stop, run_manager=run_manager, **direct_kwargs):+ yield chunk++ async def _agenerate(+ self,+ messages: list[BaseMessage],+ stop: list[str] | None = None,+ run_manager: AsyncCallbackManagerForLLMRun | None = None,+ **kwargs: Any,+ ) -> ChatResult:+ fallback = self.ai_gateway_fallback+ # A streaming call resolves through `_astream`, which routes and falls back on its own.+ if fallback is None or self.streaming:+ return await cast(BaseChatModel, super())._agenerate(messages, stop=stop, run_manager=run_manager, **kwargs)++ direct_kwargs = _without_ai_gateway_headers(kwargs)+ if _has_ai_gateway_headers(kwargs):+ try:+ result = await cast(BaseChatModel, super())._agenerate(+ messages, stop=stop, run_manager=run_manager, **kwargs+ )+ except Exception as error:+ reason = _ai_gateway_fallback_reason(error)+ if reason is None:+ raise+ self._record_ai_gateway_fallback(reason, error)+ else:+ for generation in result.generations:+ generation.generation_info = {**(generation.generation_info or {}), AI_GATEWAY_SERVED_KEY: True}+ return result++ return await fallback._agenerate(messages, stop=stop, run_manager=run_manager, **direct_kwargs)+class MaxChatOpenAI(MaxChatMixin, ChatOpenAI):"""PostHog-tuned subclass of ChatOpenAI.@@ -285,6 +494,27 @@ class MaxChatOpenAI(MaxChatMixin, ChatOpenAI):posthog_provider: ClassVar[str] = "openai"+ ai_gateway_fallback: ChatOpenAI | None = Field(default=None, exclude=True)++ @model_validator(mode="before")+ @classmethod+ def _via_ai_gateway(cls, data: Any) -> Any:+ if not isinstance(data, dict) or "ai_gateway_fallback" in data:+ return data+ ai_gateway = resolve_ai_gateway_config()+ if ai_gateway is None:+ return data+ return {+ **data,+ "base_url": ai_gateway.url,+ "api_key": ai_gateway.api_key,+ "http_client": httpx.Client(trust_env=False, verify=_ai_gateway_ssl_context()),+ "http_async_client": httpx.AsyncClient(trust_env=False, verify=_ai_gateway_ssl_context()),+ "timeout": AI_GATEWAY_TIMEOUT,+ "max_retries": 0,+ "ai_gateway_fallback": cls(**data, ai_gateway_fallback=None),+ }+def model_post_init(self, __context: Any) -> None:super().model_post_init(__context)if settings.IN_EVAL_TESTING and not self.service_tier and self.model_name in OPENAI_FLEX_MODELS:@@ -305,6 +535,8 @@ class MaxChatOpenAI(MaxChatMixin, ChatOpenAI):else:# Initialise instructions if absent or falsyself.model_kwargs["instructions"] = system_msg_content+ if self.ai_gateway_fallback is not None:+ self.ai_gateway_fallback.model_kwargs["instructions"] = self.model_kwargs["instructions"]def generate(self,@@ -320,6 +552,8 @@ class MaxChatOpenAI(MaxChatMixin, ChatOpenAI):messages = self._enrich_messages(messages, project_org_user_variables)kwargs = self._with_posthog_properties(kwargs)+ if self._routes_via_ai_gateway():+ kwargs = self._with_ai_gateway_headers(kwargs)return super().generate(messages, *args, **kwargs)@@ -337,6 +571,8 @@ class MaxChatOpenAI(MaxChatMixin, ChatOpenAI):messages = self._enrich_messages(messages, project_org_user_variables)kwargs = self._with_posthog_properties(kwargs)+ if self._routes_via_ai_gateway():+ kwargs = self._with_ai_gateway_headers(kwargs)return await super().agenerate(messages, *args, **kwargs)@@ -356,7 +592,6 @@ class MaxChatAnthropic(MaxChatMixin, ChatAnthropic):"""ai_gateway_fallback: ChatAnthropic | None = Field(default=None, exclude=True)- """Direct twin that serves calls which must not route, and gateway failures before any output."""@classmethoddef via_ai_gateway(cls, ai_gateway: AIGatewayConfig, **kwargs: Any) -> "MaxChatAnthropic":@@ -418,173 +653,12 @@ class MaxChatAnthropic(MaxChatMixin, ChatAnthropic):kwargs["timeout"] = client_params["timeout"]return kwargs- def _routes_via_ai_gateway(self) -> bool:- """Other products keep their capture and billing on the direct path."""- if self.ai_gateway_fallback is None:- return False- configurable = ensure_config().get("configurable") or {}- return (- configurable.get("ai_product") == POSTHOG_AI_PRODUCT- and (self.posthog_properties or {}).get("ai_product", POSTHOG_AI_PRODUCT) == POSTHOG_AI_PRODUCT- )-- def _with_ai_gateway_headers(self, kwargs: dict[str, Any]) -> dict[str, Any]:- configurable = ensure_config().get("configurable") or {}- posthog_properties = kwargs["metadata"]["posthog_properties"]- trace_id = str(configurable["trace_id"]) if configurable.get("trace_id") else None- conversation_id = str(configurable["thread_id"]) if configurable.get("thread_id") else None- # The gateway strips `$` keys and stamps ai_product itself; team_id names the customer it bills.- labels = {- key: str(value)- for key, value in posthog_properties.items()- if value is not None and not key.startswith("$") and key != "ai_product"- }- if conversation_id:- labels["conversation_id"] = conversation_id- # Support-trace filters read this property, which the SDK handler sets only on its own events.- if configurable.get("is_impersonated"):- labels["ai_support_impersonated"] = "true"- headers = (- ai_gateway_headers(- ai_product=POSTHOG_AI_PRODUCT,- trace_id=trace_id,- session_id=conversation_id,- properties=labels,- distinct_id=configurable.get("distinct_id") or self.user.distinct_id,- )- or {}- )- if not posthog_properties["$ai_billable"]:- headers["X-PostHog-Billable"] = "false"- # The gateway records prompts and outputs unless asked not to, so an unknown privacy mode redacts too.- if configurable.get("privacy_mode") is not False:- headers["X-PostHog-Privacy-Mode"] = "true"- return {**kwargs, "extra_headers": {**(kwargs.get("extra_headers") or {}), **headers}}-- def _record_ai_gateway_fallback(self, reason: str, error: Exception) -> None:- AI_GATEWAY_FALLBACK_COUNTER.labels(reason=reason).inc()- logger.warning("posthog_ai_gateway_fallback", reason=reason, error_type=type(error).__name__, model=self.model)-def get_num_tokens_from_messages(self, messages: list[BaseMessage], tools: Any = None, **kwargs: Any) -> int:# Token counts have no fallback, so a gateway 429 here would fail the turn.if self.ai_gateway_fallback is not None:return self.ai_gateway_fallback.get_num_tokens_from_messages(messages, tools=tools, **kwargs)return super().get_num_tokens_from_messages(messages, tools=tools, **kwargs)- def _stream(- self,- messages: list[BaseMessage],- stop: list[str] | None = None,- run_manager: CallbackManagerForLLMRun | None = None,- *,- stream_usage: bool | None = None,- **kwargs: Any,- ) -> Iterator[ChatGenerationChunk]:- # Sync calls stay direct: only `agenerate` adds the gateway headers.- if self.ai_gateway_fallback is not None:- return self.ai_gateway_fallback._stream(- messages,- stop=stop,- run_manager=run_manager,- stream_usage=stream_usage,- **_without_ai_gateway_headers(kwargs),- )- return super()._stream(messages, stop=stop, run_manager=run_manager, stream_usage=stream_usage, **kwargs)-- def _generate(- self,- messages: list[BaseMessage],- stop: list[str] | None = None,- run_manager: CallbackManagerForLLMRun | None = None,- **kwargs: Any,- ) -> ChatResult:- if self.ai_gateway_fallback is not None:- return self.ai_gateway_fallback._generate(- messages, stop=stop, run_manager=run_manager, **_without_ai_gateway_headers(kwargs)- )- return super()._generate(messages, stop=stop, run_manager=run_manager, **kwargs)-- async def _astream(- self,- messages: list[BaseMessage],- stop: list[str] | None = None,- run_manager: AsyncCallbackManagerForLLMRun | None = None,- *,- stream_usage: bool | None = None,- **kwargs: Any,- ) -> AsyncIterator[ChatGenerationChunk]:- fallback = self.ai_gateway_fallback- if fallback is None:- async for chunk in super()._astream(- messages, stop=stop, run_manager=run_manager, stream_usage=stream_usage, **kwargs- ):- yield chunk- return-- direct_kwargs = _without_ai_gateway_headers(kwargs)- if _has_ai_gateway_headers(kwargs):- served = False- # Anthropic opens the stream with a metadata chunk. Holding those keeps the fallback open until real- # output, and keeps their usage off the twin's generation.- held: list[ChatGenerationChunk] = []- try:- async for chunk in super()._astream(- messages, stop=stop, run_manager=run_manager, stream_usage=stream_usage, **kwargs- ):- if served:- yield chunk- continue- held.append(chunk)- if not _carries_output(chunk):- continue- served = True- for pending in _marked_as_gateway_served(held):- yield pending- if not served and held:- # A stream that ended without output was still served by the gateway.- for pending in _marked_as_gateway_served(held):- yield pending- return- except Exception as error:- reason = _ai_gateway_fallback_reason(error)- # After output the user has seen the turn, so a direct retry would repeat it.- if served or reason is None:- raise- self._record_ai_gateway_fallback(reason, error)-- async for chunk in fallback._astream(- messages, stop=stop, run_manager=run_manager, stream_usage=stream_usage, **direct_kwargs- ):- yield chunk-- async def _agenerate(- self,- messages: list[BaseMessage],- stop: list[str] | None = None,- run_manager: AsyncCallbackManagerForLLMRun | None = None,- **kwargs: Any,- ) -> ChatResult:- fallback = self.ai_gateway_fallback- # A streaming call resolves through `_astream`, which routes and falls back on its own.- if fallback is None or self.streaming:- return await super()._agenerate(messages, stop=stop, run_manager=run_manager, **kwargs)-- direct_kwargs = _without_ai_gateway_headers(kwargs)- if _has_ai_gateway_headers(kwargs):- try:- result = await super()._agenerate(messages, stop=stop, run_manager=run_manager, **kwargs)- except Exception as error:- reason = _ai_gateway_fallback_reason(error)- if reason is None:- raise- self._record_ai_gateway_fallback(reason, error)- else:- for generation in result.generations:- generation.generation_info = {**(generation.generation_info or {}), AI_GATEWAY_SERVED_KEY: True}- return result-- return await fallback._agenerate(messages, stop=stop, run_manager=run_manager, **direct_kwargs)-def generate(self,messages: list[list[BaseMessage]],diff --git a/ee/hogai/test/test_llm.py b/ee/hogai/test/test_llm.pyindex a11121de8473..9841ff57c0b0 100644--- a/ee/hogai/test/test_llm.py+++ b/ee/hogai/test/test_llm.py@@ -10,7 +10,9 @@ from unittest.mock import patchfrom django.test import SimpleTestCaseimport httpx+import openaiimport anthropic+from asgiref.sync import sync_to_asyncfrom langchain_anthropic import ChatAnthropicfrom langchain_core.messages import AIMessage, AIMessageChunk, BaseMessage, HumanMessage, SystemMessagefrom langchain_core.outputs import ChatGeneration, ChatGenerationChunk, Generation, LLMResult@@ -1063,3 +1065,172 @@ class TestMaxChatAnthropicAIGateway(BaseTest):self.assertTrue(is_ai_gateway_served(served))self.assertFalse(is_ai_gateway_served(unmarked))self.assertFalse(is_ai_gateway_served(ValueError("failed")))+++def _openai_body(text: str, responses_api: bool) -> dict:+ if responses_api:+ return {+ "id": "resp_test",+ "object": "response",+ "created_at": 0,+ "status": "completed",+ "model": "gpt-4.1",+ "output": [+ {+ "id": "msg_test",+ "type": "message",+ "role": "assistant",+ "status": "completed",+ "content": [{"type": "output_text", "text": text, "annotations": []}],+ }+ ],+ "parallel_tool_calls": True,+ "tool_choice": "auto",+ "tools": [],+ "usage": {+ "input_tokens": 3,+ "output_tokens": 2,+ "total_tokens": 5,+ "input_tokens_details": {"cached_tokens": 0},+ "output_tokens_details": {"reasoning_tokens": 0},+ },+ }+ return {+ "id": "chatcmpl_test",+ "object": "chat.completion",+ "created": 0,+ "model": "gpt-4.1",+ "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}],+ "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5},+ }++[email protected]("os.environ", {"OPENAI_API_KEY": "direct-api-key"})+class TestMaxChatOpenAIAIGateway(BaseTest):+ gateway = AIGatewayConfig(url="https://ai-gateway.test/v1", api_key="phs_gateway")++ def _model(self, gateway: AIGatewayConfig | None, **kwargs) -> MaxChatOpenAI:+ with patch("ee.hogai.llm.resolve_ai_gateway_config", return_value=gateway):+ return MaxChatOpenAI(model="gpt-4.1", user=self.user, team=self.team, billable=True, **kwargs)++ def _config(self, ai_product: str) -> dict:+ return {+ "configurable": {+ "trace_id": "trace-1",+ "thread_id": "conversation-1",+ "distinct_id": "distinct-1",+ "ai_product": ai_product,+ "privacy_mode": True,+ "is_agent_billable": True,+ }+ }++ def test_gateway_config_targets_the_gateway_and_keeps_a_direct_twin(self):+ model = self._model(self.gateway)+ twin = model.ai_gateway_fallback++ self.assertEqual(str(model.root_async_client.base_url), "https://ai-gateway.test/v1/")+ self.assertEqual(model.root_async_client.api_key, "phs_gateway")+ self.assertEqual(model.root_async_client.max_retries, 0)+ self.assertEqual(model.root_async_client.timeout, AI_GATEWAY_TIMEOUT)+ assert isinstance(twin, MaxChatOpenAI)+ self.assertIsNone(twin.root_async_client.timeout)+ self.assertNotIn("ai-gateway.test", str(twin.root_async_client.base_url))+ self.assertEqual(twin.root_async_client.api_key, "direct-api-key")+ self.assertEqual(twin.max_retries, 3)+ self.assertIsNone(twin.ai_gateway_fallback)+ self.assertIsNone(self._model(None).ai_gateway_fallback)++ @parameterized.expand(+ [+ ("routed_chat", "posthog_ai", False, 200, ["ai-gateway.test"], "gateway", True, False),+ ("routed_responses", "posthog_ai", True, 200, ["ai-gateway.test"], "gateway", True, False),+ ("other_product", "mcp", False, 200, ["api.openai.com"], "direct", False, False),+ ("fallback_chat", "posthog_ai", False, 503, ["ai-gateway.test", "api.openai.com"], "direct", False, False),+ (+ "fallback_responses",+ "posthog_ai",+ True,+ 503,+ ["ai-gateway.test", "api.openai.com"],+ "direct",+ False,+ False,+ ),+ ("sync_routed_responses", "posthog_ai", True, 200, ["ai-gateway.test"], "gateway", True, True),+ ("sync_other_product", "mcp", False, 200, ["api.openai.com"], "direct", False, True),+ (+ "sync_fallback_chat",+ "posthog_ai",+ False,+ 503,+ ["ai-gateway.test", "api.openai.com"],+ "direct",+ False,+ True,+ ),+ ]+ )+ async def test_generate_routing(self, _name, ai_product, responses_api, gateway_status, hosts, text, served, sync):+ requests: list[httpx.Request] = []++ def respond(request: httpx.Request) -> httpx.Response:+ requests.append(request)+ if request.url.host == "ai-gateway.test" and gateway_status != 200:+ return httpx.Response(gateway_status, json={"error": {"message": "failed", "type": "server_error"}})+ source = "gateway" if request.url.host == "ai-gateway.test" else "direct"+ return httpx.Response(200, json=_openai_body(source, responses_api))++ model = self._model(self.gateway, use_responses_api=responses_api)+ for target in (model, model.ai_gateway_fallback):+ assert target is not None+ target.root_async_client = target.root_async_client.with_options(+ http_client=httpx.AsyncClient(transport=httpx.MockTransport(respond)), max_retries=0+ )+ target.async_client = target.root_async_client.chat.completions+ target.root_client = target.root_client.with_options(+ http_client=httpx.Client(transport=httpx.MockTransport(respond)), max_retries=0+ )+ target.client = target.root_client.chat.completions++ with patch("ee.hogai.llm.ensure_config", return_value=self._config(ai_product)):+ if sync:+ result = await sync_to_async(model.generate)([[HumanMessage(content="hello")]])+ else:+ result = await model.agenerate([[HumanMessage(content="hello")]])++ self.assertEqual([request.url.host for request in requests], hosts)+ self.assertEqual(result.generations[0][0].text, text)+ self.assertEqual(is_ai_gateway_served(result), served)+ first = requests[0]+ if first.url.host == "ai-gateway.test":+ self.assertEqual(first.headers["x-posthog-product"], "posthog_ai")+ self.assertEqual(first.headers["x-posthog-trace-id"], "trace-1")+ self.assertEqual(first.headers["x-posthog-session-id"], "conversation-1")+ self.assertEqual(first.headers["x-posthog-distinct-id"], "distinct-1")+ self.assertEqual(first.headers["x-posthog-privacy-mode"], "true")+ self.assertEqual(json.loads(first.headers["x-posthog-properties"])["team_id"], str(self.team.id))+ last = requests[-1]+ if last.url.host == "api.openai.com":+ self.assertFalse(any(name.startswith("x-posthog-") for name in last.headers))+ if responses_api:+ self.assertIn("You are currently in project", json.loads(last.content)["instructions"])++ async def test_gateway_denial_raises_without_fallback(self):+ requests: list[httpx.Request] = []++ def respond(request: httpx.Request) -> httpx.Response:+ requests.append(request)+ return httpx.Response(402, json={"error": {"message": "out of balance", "type": "insufficient_quota"}})++ model = self._model(self.gateway)+ model.root_async_client = model.root_async_client.with_options(+ http_client=httpx.AsyncClient(transport=httpx.MockTransport(respond)), max_retries=0+ )+ model.async_client = model.root_async_client.chat.completions++ with patch("ee.hogai.llm.ensure_config", return_value=self._config("posthog_ai")):+ with self.assertRaises(openai.APIStatusError):+ await model.agenerate([[HumanMessage(content="hello")]])++ self.assertEqual([request.url.host for request in requests], ["ai-gateway.test"])