Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(posthog-ai): route MaxChatOpenAI through the ai-gateway (#125611)

Jonathan Mieloo9971ec112c98
diff --git a/ee/hogai/llm.py b/ee/hogai/llm.py
index 17e3328f90f9..9160f1638a55 100644
--- a/ee/hogai/llm.py
+++ b/ee/hogai/llm.py
@@ -1,3 +1,4 @@
+import ssl
import datetime
from collections.abc import ( # noqa: F401 — Sequence resolves inherited langchain field annotations lazily in this namespace
AsyncIterator,
@@ -5,27 +6,34 @@ from collections.abc import ( # noqa: F401 — Sequence resolves inherited lang
Mapping,
Sequence,
)
-from functools import cached_property
+from functools import cache, cached_property
from typing import Any, ClassVar, cast
from django.conf import settings
import pytz
import httpx
+import openai
import anthropic
import structlog
from asgiref.sync import sync_to_async
from langchain_anthropic import ChatAnthropic
from langchain_core.callbacks import AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun
+from langchain_core.language_models import BaseChatModel
from langchain_core.messages import BaseMessage, SystemMessage
from langchain_core.outputs import ChatGenerationChunk, ChatResult, LLMResult
from langchain_core.prompts import SystemMessagePromptTemplate
from langchain_core.runnables import ensure_config
from langchain_openai import ChatOpenAI
from prometheus_client import Counter
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, model_validator
-from posthog.llm.gateway_client import AIGatewayConfig, ai_gateway_headers, anthropic_gateway_base_url
+from posthog.llm.gateway_client import (
+ AIGatewayConfig,
+ ai_gateway_headers,
+ anthropic_gateway_base_url,
+ resolve_ai_gateway_config,
+)
from posthog.models import Team, User
from posthog.settings import CLOUD_DEPLOYMENT
@@ -79,6 +87,11 @@ OPENAI_FLEX_MODELS = ["o3", "gpt-5.6-terra", "gpt5", "gpt5-mini", "gpt5-nano"]
_BYPASS_PROXY_MOUNTS: dict[str, None] = {"http://": None, "https://": None, "all://": None}
+@cache
+def _ai_gateway_ssl_context() -> ssl.SSLContext:
+ return httpx.create_ssl_context(trust_env=False)
+
+
def is_ai_gateway_served(output: object) -> bool:
if not isinstance(output, LLMResult):
return False
@@ -118,18 +131,18 @@ def _marked_as_gateway_served(chunks: list[ChatGenerationChunk]) -> list[ChatGen
def _ai_gateway_fallback_reason(error: Exception) -> str | None:
"""None re-raises: a direct retry cannot fix other errors, and the gateway's own 402/403 must stay binding."""
- if isinstance(error, anthropic.APITimeoutError):
+ if isinstance(error, (anthropic.APITimeoutError, openai.APITimeoutError)):
return "timeout"
- if isinstance(error, anthropic.APIConnectionError):
+ if isinstance(error, (anthropic.APIConnectionError, openai.APIConnectionError)):
return "connection"
- if isinstance(error, anthropic.APIStatusError):
+ if isinstance(error, (anthropic.APIStatusError, openai.APIStatusError)):
if error.status_code == 401:
return "unauthorized"
if error.status_code == 429:
return "rate_limited"
if error.status_code >= 500:
return "server_error"
- if _is_provider_billing_block(error):
+ if isinstance(error, anthropic.APIStatusError) and _is_provider_billing_block(error):
return "provider_billing"
return None
@@ -157,6 +170,7 @@ class MaxChatMixin(BaseModel):
user: User
team: Team
max_retries: int | None = None
+ streaming: bool = False
stream_usage: bool | None = None
conversation_start_dt: datetime.datetime | None = None
"""
@@ -177,6 +191,8 @@ class MaxChatMixin(BaseModel):
These will be merged with the standard properties like $ai_billable and team_id.
"""
posthog_provider: ClassVar[str]
+ ai_gateway_fallback: BaseChatModel | None = Field(default=None, exclude=True)
+ """Direct twin that serves calls which must not route, and gateway failures before any output."""
def model_post_init(self, __context: Any) -> None:
if self.max_retries is None:
@@ -273,6 +289,199 @@ class MaxChatMixin(BaseModel):
return new_kwargs
+ def _routes_via_ai_gateway(self) -> bool:
+ """Other products keep their capture and billing on the direct path."""
+ if self.ai_gateway_fallback is None:
+ return False
+ configurable = ensure_config().get("configurable") or {}
+ return (
+ configurable.get("ai_product") == POSTHOG_AI_PRODUCT
+ and (self.posthog_properties or {}).get("ai_product", POSTHOG_AI_PRODUCT) == POSTHOG_AI_PRODUCT
+ )
+
+ def _with_ai_gateway_headers(self, kwargs: dict[str, Any]) -> dict[str, Any]:
+ configurable = ensure_config().get("configurable") or {}
+ posthog_properties = kwargs["metadata"]["posthog_properties"]
+ trace_id = str(configurable["trace_id"]) if configurable.get("trace_id") else None
+ conversation_id = str(configurable["thread_id"]) if configurable.get("thread_id") else None
+ # The gateway strips `$` keys and stamps ai_product itself; team_id names the customer it bills.
+ labels = {
+ key: str(value)
+ for key, value in posthog_properties.items()
+ if value is not None and not key.startswith("$") and key != "ai_product"
+ }
+ if conversation_id:
+ labels["conversation_id"] = conversation_id
+ # Support-trace filters read this property, which the SDK handler sets only on its own events.
+ if configurable.get("is_impersonated"):
+ labels["ai_support_impersonated"] = "true"
+ headers = (
+ ai_gateway_headers(
+ ai_product=POSTHOG_AI_PRODUCT,
+ trace_id=trace_id,
+ session_id=conversation_id,
+ properties=labels,
+ distinct_id=configurable.get("distinct_id") or self.user.distinct_id,
+ )
+ or {}
+ )
+ if not posthog_properties["$ai_billable"]:
+ headers["X-PostHog-Billable"] = "false"
+ # The gateway records prompts and outputs unless asked not to, so an unknown privacy mode redacts too.
+ if configurable.get("privacy_mode") is not False:
+ headers["X-PostHog-Privacy-Mode"] = "true"
+ return {**kwargs, "extra_headers": {**(kwargs.get("extra_headers") or {}), **headers}}
+
+ def _record_ai_gateway_fallback(self, reason: str, error: Exception) -> None:
+ AI_GATEWAY_FALLBACK_COUNTER.labels(reason=reason).inc()
+ model_name = getattr(self, "model", None) or getattr(self, "model_name", "unknown")
+ logger.warning("posthog_ai_gateway_fallback", reason=reason, error_type=type(error).__name__, model=model_name)
+
+ def _stream(
+ self,
+ messages: list[BaseMessage],
+ stop: list[str] | None = None,
+ run_manager: CallbackManagerForLLMRun | None = None,
+ **kwargs: Any,
+ ) -> Iterator[ChatGenerationChunk]:
+ fallback = self.ai_gateway_fallback
+ if fallback is None:
+ yield from cast(BaseChatModel, super())._stream(messages, stop=stop, run_manager=run_manager, **kwargs)
+ return
+
+ direct_kwargs = _without_ai_gateway_headers(kwargs)
+ if _has_ai_gateway_headers(kwargs):
+ served = False
+ held: list[ChatGenerationChunk] = []
+ try:
+ for chunk in cast(BaseChatModel, super())._stream(
+ messages, stop=stop, run_manager=run_manager, **kwargs
+ ):
+ if served:
+ yield chunk
+ continue
+ held.append(chunk)
+ if not _carries_output(chunk):
+ continue
+ served = True
+ yield from _marked_as_gateway_served(held)
+ if not served and held:
+ yield from _marked_as_gateway_served(held)
+ return
+ except Exception as error:
+ reason = _ai_gateway_fallback_reason(error)
+ if served or reason is None:
+ raise
+ self._record_ai_gateway_fallback(reason, error)
+
+ yield from fallback._stream(messages, stop=stop, run_manager=run_manager, **direct_kwargs)
+
+ def _generate(
+ self,
+ messages: list[BaseMessage],
+ stop: list[str] | None = None,
+ run_manager: CallbackManagerForLLMRun | None = None,
+ **kwargs: Any,
+ ) -> ChatResult:
+ fallback = self.ai_gateway_fallback
+ if fallback is None or self.streaming:
+ return cast(BaseChatModel, super())._generate(messages, stop=stop, run_manager=run_manager, **kwargs)
+
+ direct_kwargs = _without_ai_gateway_headers(kwargs)
+ if _has_ai_gateway_headers(kwargs):
+ try:
+ result = cast(BaseChatModel, super())._generate(messages, stop=stop, run_manager=run_manager, **kwargs)
+ except Exception as error:
+ reason = _ai_gateway_fallback_reason(error)
+ if reason is None:
+ raise
+ self._record_ai_gateway_fallback(reason, error)
+ else:
+ for generation in result.generations:
+ generation.generation_info = {**(generation.generation_info or {}), AI_GATEWAY_SERVED_KEY: True}
+ return result
+
+ return fallback._generate(messages, stop=stop, run_manager=run_manager, **direct_kwargs)
+
+ async def _astream(
+ self,
+ messages: list[BaseMessage],
+ stop: list[str] | None = None,
+ run_manager: AsyncCallbackManagerForLLMRun | None = None,
+ **kwargs: Any,
+ ) -> AsyncIterator[ChatGenerationChunk]:
+ fallback = self.ai_gateway_fallback
+ if fallback is None:
+ async for chunk in cast(BaseChatModel, super())._astream(
+ messages, stop=stop, run_manager=run_manager, **kwargs
+ ):
+ yield chunk
+ return
+
+ direct_kwargs = _without_ai_gateway_headers(kwargs)
+ if _has_ai_gateway_headers(kwargs):
+ served = False
+ # Anthropic opens the stream with a metadata chunk. Holding those keeps the fallback open until real
+ # output, and keeps their usage off the twin's generation.
+ held: list[ChatGenerationChunk] = []
+ try:
+ async for chunk in cast(BaseChatModel, super())._astream(
+ messages, stop=stop, run_manager=run_manager, **kwargs
+ ):
+ if served:
+ yield chunk
+ continue
+ held.append(chunk)
+ if not _carries_output(chunk):
+ continue
+ served = True
+ for pending in _marked_as_gateway_served(held):
+ yield pending
+ if not served and held:
+ # A stream that ended without output was still served by the gateway.
+ for pending in _marked_as_gateway_served(held):
+ yield pending
+ return
+ except Exception as error:
+ reason = _ai_gateway_fallback_reason(error)
+ # After output the user has seen the turn, so a direct retry would repeat it.
+ if served or reason is None:
+ raise
+ self._record_ai_gateway_fallback(reason, error)
+
+ async for chunk in fallback._astream(messages, stop=stop, run_manager=run_manager, **direct_kwargs):
+ yield chunk
+
+ async def _agenerate(
+ self,
+ messages: list[BaseMessage],
+ stop: list[str] | None = None,
+ run_manager: AsyncCallbackManagerForLLMRun | None = None,
+ **kwargs: Any,
+ ) -> ChatResult:
+ fallback = self.ai_gateway_fallback
+ # A streaming call resolves through `_astream`, which routes and falls back on its own.
+ if fallback is None or self.streaming:
+ return await cast(BaseChatModel, super())._agenerate(messages, stop=stop, run_manager=run_manager, **kwargs)
+
+ direct_kwargs = _without_ai_gateway_headers(kwargs)
+ if _has_ai_gateway_headers(kwargs):
+ try:
+ result = await cast(BaseChatModel, super())._agenerate(
+ messages, stop=stop, run_manager=run_manager, **kwargs
+ )
+ except Exception as error:
+ reason = _ai_gateway_fallback_reason(error)
+ if reason is None:
+ raise
+ self._record_ai_gateway_fallback(reason, error)
+ else:
+ for generation in result.generations:
+ generation.generation_info = {**(generation.generation_info or {}), AI_GATEWAY_SERVED_KEY: True}
+ return result
+
+ return await fallback._agenerate(messages, stop=stop, run_manager=run_manager, **direct_kwargs)
+
class MaxChatOpenAI(MaxChatMixin, ChatOpenAI):
"""PostHog-tuned subclass of ChatOpenAI.
@@ -285,6 +494,27 @@ class MaxChatOpenAI(MaxChatMixin, ChatOpenAI):
posthog_provider: ClassVar[str] = "openai"
+ ai_gateway_fallback: ChatOpenAI | None = Field(default=None, exclude=True)
+
+ @model_validator(mode="before")
+ @classmethod
+ def _via_ai_gateway(cls, data: Any) -> Any:
+ if not isinstance(data, dict) or "ai_gateway_fallback" in data:
+ return data
+ ai_gateway = resolve_ai_gateway_config()
+ if ai_gateway is None:
+ return data
+ return {
+ **data,
+ "base_url": ai_gateway.url,
+ "api_key": ai_gateway.api_key,
+ "http_client": httpx.Client(trust_env=False, verify=_ai_gateway_ssl_context()),
+ "http_async_client": httpx.AsyncClient(trust_env=False, verify=_ai_gateway_ssl_context()),
+ "timeout": AI_GATEWAY_TIMEOUT,
+ "max_retries": 0,
+ "ai_gateway_fallback": cls(**data, ai_gateway_fallback=None),
+ }
+
def model_post_init(self, __context: Any) -> None:
super().model_post_init(__context)
if settings.IN_EVAL_TESTING and not self.service_tier and self.model_name in OPENAI_FLEX_MODELS:
@@ -305,6 +535,8 @@ class MaxChatOpenAI(MaxChatMixin, ChatOpenAI):
else:
# Initialise instructions if absent or falsy
self.model_kwargs["instructions"] = system_msg_content
+ if self.ai_gateway_fallback is not None:
+ self.ai_gateway_fallback.model_kwargs["instructions"] = self.model_kwargs["instructions"]
def generate(
self,
@@ -320,6 +552,8 @@ class MaxChatOpenAI(MaxChatMixin, ChatOpenAI):
messages = self._enrich_messages(messages, project_org_user_variables)
kwargs = self._with_posthog_properties(kwargs)
+ if self._routes_via_ai_gateway():
+ kwargs = self._with_ai_gateway_headers(kwargs)
return super().generate(messages, *args, **kwargs)
@@ -337,6 +571,8 @@ class MaxChatOpenAI(MaxChatMixin, ChatOpenAI):
messages = self._enrich_messages(messages, project_org_user_variables)
kwargs = self._with_posthog_properties(kwargs)
+ if self._routes_via_ai_gateway():
+ kwargs = self._with_ai_gateway_headers(kwargs)
return await super().agenerate(messages, *args, **kwargs)
@@ -356,7 +592,6 @@ class MaxChatAnthropic(MaxChatMixin, ChatAnthropic):
"""
ai_gateway_fallback: ChatAnthropic | None = Field(default=None, exclude=True)
- """Direct twin that serves calls which must not route, and gateway failures before any output."""
@classmethod
def via_ai_gateway(cls, ai_gateway: AIGatewayConfig, **kwargs: Any) -> "MaxChatAnthropic":
@@ -418,173 +653,12 @@ class MaxChatAnthropic(MaxChatMixin, ChatAnthropic):
kwargs["timeout"] = client_params["timeout"]
return kwargs
- def _routes_via_ai_gateway(self) -> bool:
- """Other products keep their capture and billing on the direct path."""
- if self.ai_gateway_fallback is None:
- return False
- configurable = ensure_config().get("configurable") or {}
- return (
- configurable.get("ai_product") == POSTHOG_AI_PRODUCT
- and (self.posthog_properties or {}).get("ai_product", POSTHOG_AI_PRODUCT) == POSTHOG_AI_PRODUCT
- )
-
- def _with_ai_gateway_headers(self, kwargs: dict[str, Any]) -> dict[str, Any]:
- configurable = ensure_config().get("configurable") or {}
- posthog_properties = kwargs["metadata"]["posthog_properties"]
- trace_id = str(configurable["trace_id"]) if configurable.get("trace_id") else None
- conversation_id = str(configurable["thread_id"]) if configurable.get("thread_id") else None
- # The gateway strips `$` keys and stamps ai_product itself; team_id names the customer it bills.
- labels = {
- key: str(value)
- for key, value in posthog_properties.items()
- if value is not None and not key.startswith("$") and key != "ai_product"
- }
- if conversation_id:
- labels["conversation_id"] = conversation_id
- # Support-trace filters read this property, which the SDK handler sets only on its own events.
- if configurable.get("is_impersonated"):
- labels["ai_support_impersonated"] = "true"
- headers = (
- ai_gateway_headers(
- ai_product=POSTHOG_AI_PRODUCT,
- trace_id=trace_id,
- session_id=conversation_id,
- properties=labels,
- distinct_id=configurable.get("distinct_id") or self.user.distinct_id,
- )
- or {}
- )
- if not posthog_properties["$ai_billable"]:
- headers["X-PostHog-Billable"] = "false"
- # The gateway records prompts and outputs unless asked not to, so an unknown privacy mode redacts too.
- if configurable.get("privacy_mode") is not False:
- headers["X-PostHog-Privacy-Mode"] = "true"
- return {**kwargs, "extra_headers": {**(kwargs.get("extra_headers") or {}), **headers}}
-
- def _record_ai_gateway_fallback(self, reason: str, error: Exception) -> None:
- AI_GATEWAY_FALLBACK_COUNTER.labels(reason=reason).inc()
- logger.warning("posthog_ai_gateway_fallback", reason=reason, error_type=type(error).__name__, model=self.model)
-
def get_num_tokens_from_messages(self, messages: list[BaseMessage], tools: Any = None, **kwargs: Any) -> int:
# Token counts have no fallback, so a gateway 429 here would fail the turn.
if self.ai_gateway_fallback is not None:
return self.ai_gateway_fallback.get_num_tokens_from_messages(messages, tools=tools, **kwargs)
return super().get_num_tokens_from_messages(messages, tools=tools, **kwargs)
- def _stream(
- self,
- messages: list[BaseMessage],
- stop: list[str] | None = None,
- run_manager: CallbackManagerForLLMRun | None = None,
- *,
- stream_usage: bool | None = None,
- **kwargs: Any,
- ) -> Iterator[ChatGenerationChunk]:
- # Sync calls stay direct: only `agenerate` adds the gateway headers.
- if self.ai_gateway_fallback is not None:
- return self.ai_gateway_fallback._stream(
- messages,
- stop=stop,
- run_manager=run_manager,
- stream_usage=stream_usage,
- **_without_ai_gateway_headers(kwargs),
- )
- return super()._stream(messages, stop=stop, run_manager=run_manager, stream_usage=stream_usage, **kwargs)
-
- def _generate(
- self,
- messages: list[BaseMessage],
- stop: list[str] | None = None,
- run_manager: CallbackManagerForLLMRun | None = None,
- **kwargs: Any,
- ) -> ChatResult:
- if self.ai_gateway_fallback is not None:
- return self.ai_gateway_fallback._generate(
- messages, stop=stop, run_manager=run_manager, **_without_ai_gateway_headers(kwargs)
- )
- return super()._generate(messages, stop=stop, run_manager=run_manager, **kwargs)
-
- async def _astream(
- self,
- messages: list[BaseMessage],
- stop: list[str] | None = None,
- run_manager: AsyncCallbackManagerForLLMRun | None = None,
- *,
- stream_usage: bool | None = None,
- **kwargs: Any,
- ) -> AsyncIterator[ChatGenerationChunk]:
- fallback = self.ai_gateway_fallback
- if fallback is None:
- async for chunk in super()._astream(
- messages, stop=stop, run_manager=run_manager, stream_usage=stream_usage, **kwargs
- ):
- yield chunk
- return
-
- direct_kwargs = _without_ai_gateway_headers(kwargs)
- if _has_ai_gateway_headers(kwargs):
- served = False
- # Anthropic opens the stream with a metadata chunk. Holding those keeps the fallback open until real
- # output, and keeps their usage off the twin's generation.
- held: list[ChatGenerationChunk] = []
- try:
- async for chunk in super()._astream(
- messages, stop=stop, run_manager=run_manager, stream_usage=stream_usage, **kwargs
- ):
- if served:
- yield chunk
- continue
- held.append(chunk)
- if not _carries_output(chunk):
- continue
- served = True
- for pending in _marked_as_gateway_served(held):
- yield pending
- if not served and held:
- # A stream that ended without output was still served by the gateway.
- for pending in _marked_as_gateway_served(held):
- yield pending
- return
- except Exception as error:
- reason = _ai_gateway_fallback_reason(error)
- # After output the user has seen the turn, so a direct retry would repeat it.
- if served or reason is None:
- raise
- self._record_ai_gateway_fallback(reason, error)
-
- async for chunk in fallback._astream(
- messages, stop=stop, run_manager=run_manager, stream_usage=stream_usage, **direct_kwargs
- ):
- yield chunk
-
- async def _agenerate(
- self,
- messages: list[BaseMessage],
- stop: list[str] | None = None,
- run_manager: AsyncCallbackManagerForLLMRun | None = None,
- **kwargs: Any,
- ) -> ChatResult:
- fallback = self.ai_gateway_fallback
- # A streaming call resolves through `_astream`, which routes and falls back on its own.
- if fallback is None or self.streaming:
- return await super()._agenerate(messages, stop=stop, run_manager=run_manager, **kwargs)
-
- direct_kwargs = _without_ai_gateway_headers(kwargs)
- if _has_ai_gateway_headers(kwargs):
- try:
- result = await super()._agenerate(messages, stop=stop, run_manager=run_manager, **kwargs)
- except Exception as error:
- reason = _ai_gateway_fallback_reason(error)
- if reason is None:
- raise
- self._record_ai_gateway_fallback(reason, error)
- else:
- for generation in result.generations:
- generation.generation_info = {**(generation.generation_info or {}), AI_GATEWAY_SERVED_KEY: True}
- return result
-
- return await fallback._agenerate(messages, stop=stop, run_manager=run_manager, **direct_kwargs)
-
def generate(
self,
messages: list[list[BaseMessage]],
diff --git a/ee/hogai/test/test_llm.py b/ee/hogai/test/test_llm.py
index a11121de8473..9841ff57c0b0 100644
--- a/ee/hogai/test/test_llm.py
+++ b/ee/hogai/test/test_llm.py
@@ -10,7 +10,9 @@ from unittest.mock import patch
from django.test import SimpleTestCase
import httpx
+import openai
import anthropic
+from asgiref.sync import sync_to_async
from langchain_anthropic import ChatAnthropic
from langchain_core.messages import AIMessage, AIMessageChunk, BaseMessage, HumanMessage, SystemMessage
from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, Generation, LLMResult
@@ -1063,3 +1065,172 @@ class TestMaxChatAnthropicAIGateway(BaseTest):
self.assertTrue(is_ai_gateway_served(served))
self.assertFalse(is_ai_gateway_served(unmarked))
self.assertFalse(is_ai_gateway_served(ValueError("failed")))
+
+
+def _openai_body(text: str, responses_api: bool) -> dict:
+ if responses_api:
+ return {
+ "id": "resp_test",
+ "object": "response",
+ "created_at": 0,
+ "status": "completed",
+ "model": "gpt-4.1",
+ "output": [
+ {
+ "id": "msg_test",
+ "type": "message",
+ "role": "assistant",
+ "status": "completed",
+ "content": [{"type": "output_text", "text": text, "annotations": []}],
+ }
+ ],
+ "parallel_tool_calls": True,
+ "tool_choice": "auto",
+ "tools": [],
+ "usage": {
+ "input_tokens": 3,
+ "output_tokens": 2,
+ "total_tokens": 5,
+ "input_tokens_details": {"cached_tokens": 0},
+ "output_tokens_details": {"reasoning_tokens": 0},
+ },
+ }
+ return {
+ "id": "chatcmpl_test",
+ "object": "chat.completion",
+ "created": 0,
+ "model": "gpt-4.1",
+ "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5},
+ }
+
+
[email protected]("os.environ", {"OPENAI_API_KEY": "direct-api-key"})
+class TestMaxChatOpenAIAIGateway(BaseTest):
+ gateway = AIGatewayConfig(url="https://ai-gateway.test/v1", api_key="phs_gateway")
+
+ def _model(self, gateway: AIGatewayConfig | None, **kwargs) -> MaxChatOpenAI:
+ with patch("ee.hogai.llm.resolve_ai_gateway_config", return_value=gateway):
+ return MaxChatOpenAI(model="gpt-4.1", user=self.user, team=self.team, billable=True, **kwargs)
+
+ def _config(self, ai_product: str) -> dict:
+ return {
+ "configurable": {
+ "trace_id": "trace-1",
+ "thread_id": "conversation-1",
+ "distinct_id": "distinct-1",
+ "ai_product": ai_product,
+ "privacy_mode": True,
+ "is_agent_billable": True,
+ }
+ }
+
+ def test_gateway_config_targets_the_gateway_and_keeps_a_direct_twin(self):
+ model = self._model(self.gateway)
+ twin = model.ai_gateway_fallback
+
+ self.assertEqual(str(model.root_async_client.base_url), "https://ai-gateway.test/v1/")
+ self.assertEqual(model.root_async_client.api_key, "phs_gateway")
+ self.assertEqual(model.root_async_client.max_retries, 0)
+ self.assertEqual(model.root_async_client.timeout, AI_GATEWAY_TIMEOUT)
+ assert isinstance(twin, MaxChatOpenAI)
+ self.assertIsNone(twin.root_async_client.timeout)
+ self.assertNotIn("ai-gateway.test", str(twin.root_async_client.base_url))
+ self.assertEqual(twin.root_async_client.api_key, "direct-api-key")
+ self.assertEqual(twin.max_retries, 3)
+ self.assertIsNone(twin.ai_gateway_fallback)
+ self.assertIsNone(self._model(None).ai_gateway_fallback)
+
+ @parameterized.expand(
+ [
+ ("routed_chat", "posthog_ai", False, 200, ["ai-gateway.test"], "gateway", True, False),
+ ("routed_responses", "posthog_ai", True, 200, ["ai-gateway.test"], "gateway", True, False),
+ ("other_product", "mcp", False, 200, ["api.openai.com"], "direct", False, False),
+ ("fallback_chat", "posthog_ai", False, 503, ["ai-gateway.test", "api.openai.com"], "direct", False, False),
+ (
+ "fallback_responses",
+ "posthog_ai",
+ True,
+ 503,
+ ["ai-gateway.test", "api.openai.com"],
+ "direct",
+ False,
+ False,
+ ),
+ ("sync_routed_responses", "posthog_ai", True, 200, ["ai-gateway.test"], "gateway", True, True),
+ ("sync_other_product", "mcp", False, 200, ["api.openai.com"], "direct", False, True),
+ (
+ "sync_fallback_chat",
+ "posthog_ai",
+ False,
+ 503,
+ ["ai-gateway.test", "api.openai.com"],
+ "direct",
+ False,
+ True,
+ ),
+ ]
+ )
+ async def test_generate_routing(self, _name, ai_product, responses_api, gateway_status, hosts, text, served, sync):
+ requests: list[httpx.Request] = []
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+ if request.url.host == "ai-gateway.test" and gateway_status != 200:
+ return httpx.Response(gateway_status, json={"error": {"message": "failed", "type": "server_error"}})
+ source = "gateway" if request.url.host == "ai-gateway.test" else "direct"
+ return httpx.Response(200, json=_openai_body(source, responses_api))
+
+ model = self._model(self.gateway, use_responses_api=responses_api)
+ for target in (model, model.ai_gateway_fallback):
+ assert target is not None
+ target.root_async_client = target.root_async_client.with_options(
+ http_client=httpx.AsyncClient(transport=httpx.MockTransport(respond)), max_retries=0
+ )
+ target.async_client = target.root_async_client.chat.completions
+ target.root_client = target.root_client.with_options(
+ http_client=httpx.Client(transport=httpx.MockTransport(respond)), max_retries=0
+ )
+ target.client = target.root_client.chat.completions
+
+ with patch("ee.hogai.llm.ensure_config", return_value=self._config(ai_product)):
+ if sync:
+ result = await sync_to_async(model.generate)([[HumanMessage(content="hello")]])
+ else:
+ result = await model.agenerate([[HumanMessage(content="hello")]])
+
+ self.assertEqual([request.url.host for request in requests], hosts)
+ self.assertEqual(result.generations[0][0].text, text)
+ self.assertEqual(is_ai_gateway_served(result), served)
+ first = requests[0]
+ if first.url.host == "ai-gateway.test":
+ self.assertEqual(first.headers["x-posthog-product"], "posthog_ai")
+ self.assertEqual(first.headers["x-posthog-trace-id"], "trace-1")
+ self.assertEqual(first.headers["x-posthog-session-id"], "conversation-1")
+ self.assertEqual(first.headers["x-posthog-distinct-id"], "distinct-1")
+ self.assertEqual(first.headers["x-posthog-privacy-mode"], "true")
+ self.assertEqual(json.loads(first.headers["x-posthog-properties"])["team_id"], str(self.team.id))
+ last = requests[-1]
+ if last.url.host == "api.openai.com":
+ self.assertFalse(any(name.startswith("x-posthog-") for name in last.headers))
+ if responses_api:
+ self.assertIn("You are currently in project", json.loads(last.content)["instructions"])
+
+ async def test_gateway_denial_raises_without_fallback(self):
+ requests: list[httpx.Request] = []
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ requests.append(request)
+ return httpx.Response(402, json={"error": {"message": "out of balance", "type": "insufficient_quota"}})
+
+ model = self._model(self.gateway)
+ model.root_async_client = model.root_async_client.with_options(
+ http_client=httpx.AsyncClient(transport=httpx.MockTransport(respond)), max_retries=0
+ )
+ model.async_client = model.root_async_client.chat.completions
+
+ with patch("ee.hogai.llm.ensure_config", return_value=self._config("posthog_ai")):
+ with self.assertRaises(openai.APIStatusError):
+ await model.agenerate([[HumanMessage(content="hello")]])
+
+ self.assertEqual([request.url.host for request in requests], ["ai-gateway.test"])