Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

37.6 KB
Raw Blame
import ssl
import time
import datetime
from collections.abc import (  # noqa: F401 — Sequence resolves inherited langchain field annotations lazily in this namespace
    AsyncIterator,
    Iterator,
    Mapping,
    Sequence,
)
from functools import cache, cached_property
from typing import Any, ClassVar, NamedTuple, cast
from uuid import UUID

from django.conf import settings

import pytz
import httpx
import openai
import anthropic
import structlog
from asgiref.sync import sync_to_async
from langchain_anthropic import ChatAnthropic
from langchain_core.callbacks import (
    AsyncCallbackManagerForLLMRun,
    BaseCallbackManager,
    CallbackManagerForLLMRun,
    Callbacks,
)
from langchain_core.language_models import BaseChatModel
from langchain_core.messages import BaseMessage, SystemMessage
from langchain_core.outputs import ChatGenerationChunk, ChatResult, LLMResult
from langchain_core.prompts import SystemMessagePromptTemplate
from langchain_core.runnables import ensure_config
from langchain_openai import ChatOpenAI
from posthoganalytics.ai.langchain.callbacks import CallbackHandler, GenerationMetadata
from prometheus_client import Counter
from pydantic import BaseModel, ConfigDict, Field, model_validator

from posthog.llm.billing import BILLABLE_AI_PRODUCTS
from posthog.llm.gateway_client import ai_gateway_headers, anthropic_gateway_base_url, resolve_ai_gateway_config
from posthog.models import Team, User
from posthog.settings import CLOUD_DEPLOYMENT

logger = structlog.get_logger(__name__)

POSTHOG_AI_PRODUCT = "posthog_ai"

BILLING_SKIPPED_COUNTER = Counter(
    "posthog_ai_billing_skipped_total",
    "Number of AI generations where billing was skipped due to workflow-level override (e.g., impersonation)",
    ["model"],
)

AI_GATEWAY_FALLBACK_COUNTER = Counter(
    "posthog_ai_gateway_fallback_total",
    "PostHog AI model calls that fell back from the Go ai-gateway to the direct provider",
    ["reason"],
)

AI_GATEWAY_TIMEOUT = httpx.Timeout(150.0, connect=10.0, write=15.0, pool=5.0)
"""Above the gateway's 120s stream idle close. Nothing heartbeats while the gateway stalls, so every phase is bounded
to leave the twin time inside the 300s activity heartbeat."""

AI_GATEWAY_TIMEOUT_COOLDOWN_SECONDS = 300.0

AI_GATEWAY_SERVED_KEY = "ai_gateway_served"

AI_GATEWAY_BILLABLE_PRODUCTS = frozenset(
    {
        "posthog_ai",
        "slack_app",
        "workflows",
        "hogql_decide",
        "posthog_code",
        "alert_investigation_agent",
        "alert_llm_detector",
        "surveys",
    }
)

PROJECT_ORG_USER_CONTEXT_PROMPT = """
You are currently in project {{{project_name}}}, which is part of the {{{organization_name}}} organization.
The user's name appears to be {{{user_full_name}}} ({{{user_email}}}). Feel free to use their first name when greeting. DO NOT use this name if it appears possibly fake.
All PostHog app URLs must use root-relative paths starting with `/`, without a domain (no us.posthog.com, eu.posthog.com, app.posthog.com), and omit the `/project/:id/` prefix. Never include `/-/` in URLs. Never use relative paths like `../` or `./` — always start with `/`.
Use Markdown with descriptive anchor text, for example "[Cohorts view](/cohorts)".

Key URL patterns:
- Dashboard: `/dashboard/<id>`, e.g. `/dashboard/12345`
- Insights: `/insights/<short_id>`, e.g. `/insights/abc123`
- Settings: `/settings/<section-id>` where section IDs use hyphens, e.g. `/settings/organization-members`, `/settings/environment-replay`, `/settings/user-api-keys`. Section IDs are a fixed set, so never guess one because it sounds plausible; if you are not certain an ID is real, link `/settings` and name the section in the anchor text. Members and invites are organization-level (`/settings/organization-members`); there is no project members page, and per-project access is `/settings/environment-access-control`.
- Data management: `/data-management/events`, `/data-management/properties`
- Billing: `/organization/billing`
Current time in the project's timezone, {{{project_timezone}}}: {{{project_datetime}}}.
{{#person_on_events_enabled}}
Person-on-events mode is enabled. When querying `person.properties.*` on the events table, values reflect what was set at the time the event was ingested, not the person's current value. The same person can have different property values across different events. Do not suggest workarounds for "query-time" person properties.
{{/person_on_events_enabled}}
{{^person_on_events_enabled}}
Person properties are query-time in this project. `person.properties.*` on the events table always returns the person's current (latest) value, regardless of when the event occurred.
{{/person_on_events_enabled}}
""".strip()

# https://platform.openai.com/docs/guides/flex-processing
OPENAI_FLEX_MODELS = ["o3", "gpt-5.6-terra", "gpt5", "gpt5-mini", "gpt5-nano"]

# Map "http://", "https://", and "all://" to None in Client's mounts to bypass proxies for MaxChatAnthropic.
_BYPASS_PROXY_MOUNTS: dict[str, None] = {"http://": None, "https://": None, "all://": None}


@cache
def _ai_gateway_ssl_context() -> ssl.SSLContext:
    return httpx.create_ssl_context(trust_env=False)


def is_ai_gateway_served(output: object) -> bool:
    if not isinstance(output, LLMResult):
        return False
    return any(
        (generation.generation_info or {}).get(AI_GATEWAY_SERVED_KEY)
        for generations in output.generations
        for generation in generations
    )


# Anthropic's wording for a billing or usage-limit block; the gateway classifies on the same list.
_PROVIDER_BILLING_SIGNATURES = ("credit balance", "usage limit", "regain access", "plans & billing")


def _is_provider_billing_block(error: anthropic.APIStatusError) -> bool:
    """Anthropic blocking the gateway's workspace, passed through; the gateway's own 402s carry X-PostHog-Denial."""
    if error.status_code not in (400, 402) or error.response.headers.get("X-PostHog-Denial"):
        return False
    detail = error.body.get("error") if isinstance(error.body, dict) else None
    if not isinstance(detail, dict):
        return False
    message = str(detail.get("message") or "").lower()
    return detail.get("type") == "billing_error" or any(sig in message for sig in _PROVIDER_BILLING_SIGNATURES)


class AIGatewayRoute(NamedTuple):
    ai_product: str
    billable: bool
    trace_id: str | None
    distinct_id: str | None
    privacy_mode: bool | None
    properties: Mapping[str, Any]


class MaxCallbackHandler(CallbackHandler):
    """
    Skips $ai_generation for gateway-served calls, which the gateway captures. $ai_trace and $ai_span still emit
    because the AI credits query decides free turns from them.
    """

    def _capture_generation(
        self,
        trace_id: Any,
        run_id: UUID,
        run: GenerationMetadata,
        output: LLMResult | BaseException,
        parent_run_id: UUID | None = None,
        include_parent_id: bool = True,
    ):
        if is_ai_gateway_served(output):
            return
        super()._capture_generation(trace_id, run_id, run, output, parent_run_id, include_parent_id)

    @property
    def ai_product(self) -> str | None:
        return (self._properties or {}).get("ai_product")

    @property
    def trace_id(self) -> str | None:
        return str(self._trace_id) if self._trace_id is not None else None

    @property
    def distinct_id(self) -> str | None:
        return str(self._distinct_id) if self._distinct_id is not None else None

    @property
    def privacy_mode(self) -> bool:
        return self._privacy_mode or bool(getattr(self._ph_client, "privacy_mode", False))

    @property
    def capture_disabled(self) -> bool:
        return getattr(self._ph_client, "disabled", False) is True

    @property
    def properties(self) -> Mapping[str, Any]:
        return self._properties or {}


def _posthog_callback_handlers(callbacks: Callbacks) -> list[CallbackHandler]:
    handlers = callbacks.handlers if isinstance(callbacks, BaseCallbackManager) else callbacks or []
    return [handler for handler in handlers if isinstance(handler, CallbackHandler)]


def _is_unbilled_credit_bucket_denial(error: anthropic.APIStatusError) -> bool:
    if error.status_code != 402:
        return False
    if error.request.headers.get("X-PostHog-Billable") != "false":
        return False
    return error.response.headers.get("X-PostHog-Denial", "").startswith("credit_bucket_exhausted")


def _carries_output(chunk: ChatGenerationChunk) -> bool:
    """Thinking counts: the user sees it stream. An opening metadata chunk does not."""
    return bool(chunk.message.content) or bool(getattr(chunk.message, "tool_call_chunks", None))


def _marked_as_gateway_served(chunks: list[ChatGenerationChunk]) -> list[ChatGenerationChunk]:
    """One marked chunk marks the merged generation, which the SDK callback then skips."""
    first = chunks[0]
    first.generation_info = {**(first.generation_info or {}), AI_GATEWAY_SERVED_KEY: True}
    return chunks


def _ai_gateway_fallback_reason(error: Exception) -> str | None:
    """None re-raises: a direct retry cannot fix other errors, and the gateway's own 402/403 must stay binding."""
    if isinstance(error, (anthropic.APITimeoutError, openai.APITimeoutError)):
        return "timeout"
    if isinstance(error, (anthropic.APIConnectionError, openai.APIConnectionError)):
        return "connection"
    if isinstance(error, (anthropic.APIStatusError, openai.APIStatusError)):
        if error.status_code == 401:
            return "unauthorized"
        if error.status_code == 429:
            return "rate_limited"
        if error.status_code >= 500:
            return "server_error"
        if isinstance(error, anthropic.APIStatusError) and _is_provider_billing_block(error):
            return "provider_billing"
        if isinstance(error, anthropic.APIStatusError) and _is_unbilled_credit_bucket_denial(error):
            return "credit_bucket"
    return None


def _has_ai_gateway_headers(kwargs: Mapping[str, Any]) -> bool:
    return "X-PostHog-Product" in (kwargs.get("extra_headers") or {})


def _without_ai_gateway_headers(kwargs: Mapping[str, Any]) -> dict[str, Any]:
    new_kwargs = {key: value for key, value in kwargs.items() if key != "extra_headers"}
    extra_headers = {
        name: value
        for name, value in (kwargs.get("extra_headers") or {}).items()
        if not name.lower().startswith("x-posthog-")
    }
    if extra_headers:
        new_kwargs["extra_headers"] = extra_headers
    return new_kwargs


class MaxChatMixin(BaseModel):
    # We don't want to validate Django models here.
    model_config = ConfigDict(arbitrary_types_allowed=True)

    user: User
    team: Team
    max_retries: int | None = None
    streaming: bool = False
    stream_usage: bool | None = None
    conversation_start_dt: datetime.datetime | None = None
    """
    The datetime of the start of the conversation. If not provided, the current time will be used.
    """
    billable: bool = False
    """
    Whether the generation will be marked as billable in the usage report for calculating AI billing credits.
    """
    inject_context: bool = True
    """
    Whether to inject project/org/user context into the system prompt.
    Set to False to disable automatic context injection.
    """
    posthog_properties: dict[str, Any] | None = None
    """
    Additional PostHog properties to be added to the $ai_generation event.
    These will be merged with the standard properties like $ai_billable and team_id.
    """
    posthog_provider: ClassVar[str]
    ai_gateway_fallback: BaseChatModel | None = Field(default=None, exclude=True)
    """Direct twin that serves calls which must not route, and gateway failures before any output."""

    def model_post_init(self, __context: Any) -> None:
        if self.max_retries is None:
            self.max_retries = 3
        if self.stream_usage is None:
            self.stream_usage = True

    def _get_project_org_user_variables(self) -> dict[str, Any]:
        """Note: this function may perform Postgres queries on `self._team`, `self._team.organization`, and `self._user`."""
        project_timezone = self.team.timezone
        adjusted_dt = self.conversation_start_dt or datetime.datetime.now()
        project_datetime = adjusted_dt.astimezone(tz=pytz.timezone(project_timezone))

        region = CLOUD_DEPLOYMENT or "US"
        if region in ["US", "EU"]:
            region = region.lower()
        else:
            region = "us"

        return {
            "project_name": self.team.name,
            "project_timezone": project_timezone,
            "project_datetime": project_datetime.strftime("%Y-%m-%d %H:%M:%S"),
            "organization_name": self.team.organization.name,
            "user_full_name": self.user.get_full_name(),
            "user_email": self.user.email,
            "deployment_region": region,
            "person_on_events_enabled": self.team.person_on_events_querying_enabled,
        }

    @sync_to_async
    def _aget_project_org_user_variables(self) -> dict[str, Any]:
        return self._get_project_org_user_variables()

    def _get_project_org_system_message(self, project_org_user_variables: dict[str, Any]) -> BaseMessage:
        return SystemMessagePromptTemplate.from_template(
            PROJECT_ORG_USER_CONTEXT_PROMPT, template_format="mustache"
        ).format(**project_org_user_variables)

    def _enrich_messages(self, messages: list[list[BaseMessage]], project_org_user_variables: dict[str, Any]):
        messages = messages.copy()
        for i in range(len(messages)):
            message_sublist = messages[i]
            # In every sublist (which becomes a separate generation) insert our shared prompt at the very end
            # of the system messages block
            for msg_index, msg in enumerate(message_sublist):
                if isinstance(msg, SystemMessage):
                    continue  # Keep going
                else:
                    # Here's our end of the system messages block
                    copied_list = message_sublist.copy()
                    copied_list.insert(msg_index, self._get_project_org_system_message(project_org_user_variables))
                    messages[i] = copied_list
                    break
        return messages

    def _get_effective_billable(self) -> bool:
        """
        Determine the effective billable status for this generation.
        Combines model-level billable setting with workflow-level override from config.
        When is_agent_billable is False (e.g., impersonated sessions), billing is skipped
        regardless of the model's billable setting.
        """
        config = ensure_config()
        is_agent_billable = (config.get("configurable") or {}).get("is_agent_billable", True)

        effective_billable = self.billable and is_agent_billable

        if self.billable and not is_agent_billable:
            # This is really annoying given the interface differences between model providers
            # Once we are behind a proxy, this can be simplified.
            model_name = getattr(self, "model", None) or getattr(self, "model_name", "unknown")
            BILLING_SKIPPED_COUNTER.labels(model=model_name).inc()
            logger.warning("Billing skipped for generation due to workflow-level override")

        return effective_billable

    def _with_posthog_properties(
        self,
        kwargs: Mapping[str, Any] | None = None,
    ) -> dict[str, Any]:
        """Return a shallow copy of kwargs with PostHog properties, billable flag, and team_id injected into metadata."""
        new_kwargs = dict(kwargs or {})
        metadata = dict(new_kwargs.get("metadata") or {})

        posthog_props = dict(self.posthog_properties or {})
        posthog_props["$ai_billable"] = self._get_effective_billable()
        posthog_props["team_id"] = self.team.id
        posthog_props.setdefault("ai_product", POSTHOG_AI_PRODUCT)

        metadata.setdefault("ls_provider", self.posthog_provider)
        metadata["posthog_properties"] = posthog_props
        new_kwargs["metadata"] = metadata

        return new_kwargs

    def _ai_gateway_route(self, kwargs: Mapping[str, Any]) -> AIGatewayRoute | None:
        """Other products keep their capture and billing on the direct path."""
        if self.ai_gateway_fallback is None:
            return None
        configurable = ensure_config().get("configurable") or {}
        if configurable.get("ai_product") != POSTHOG_AI_PRODUCT:
            return None
        if (self.posthog_properties or {}).get("ai_product", POSTHOG_AI_PRODUCT) != POSTHOG_AI_PRODUCT:
            return None
        return self._ai_gateway_route_attributed(
            ai_product=POSTHOG_AI_PRODUCT,
            billable=kwargs["metadata"]["posthog_properties"]["$ai_billable"],
            handler=None,
        )

    def _ai_gateway_route_attributed(
        self, *, ai_product: str, billable: bool, handler: MaxCallbackHandler | None
    ) -> AIGatewayRoute:
        if handler is not None:
            return AIGatewayRoute(
                ai_product=ai_product,
                billable=billable,
                trace_id=handler.trace_id,
                distinct_id=handler.distinct_id or self.user.distinct_id,
                privacy_mode=handler.privacy_mode,
                properties=handler.properties,
            )
        configurable = ensure_config().get("configurable") or {}
        return AIGatewayRoute(
            ai_product=ai_product,
            billable=billable,
            trace_id=str(configurable["trace_id"]) if configurable.get("trace_id") else None,
            distinct_id=configurable.get("distinct_id") or self.user.distinct_id,
            privacy_mode=configurable.get("privacy_mode"),
            properties={},
        )

    def _with_ai_gateway_headers(self, kwargs: dict[str, Any], route: AIGatewayRoute) -> dict[str, Any]:
        configurable = ensure_config().get("configurable") or {}
        posthog_properties = kwargs["metadata"]["posthog_properties"]
        conversation_id = str(configurable["thread_id"]) if configurable.get("thread_id") else None
        # The gateway strips `$` keys and stamps ai_product itself; team_id names the customer it bills.
        labels = {
            key: str(value)
            for key, value in {**route.properties, **posthog_properties}.items()
            if value is not None and not key.startswith("$") and key != "ai_product"
        }
        if conversation_id:
            labels["conversation_id"] = conversation_id
        # Support-trace filters read this property, which the SDK handler sets only on its own events.
        if configurable.get("is_impersonated"):
            labels["ai_support_impersonated"] = "true"
        headers = (
            ai_gateway_headers(
                ai_product=route.ai_product,
                trace_id=route.trace_id,
                session_id=conversation_id,
                properties=labels,
                distinct_id=route.distinct_id,
            )
            or {}
        )
        if not route.billable:
            headers["X-PostHog-Billable"] = "false"
        # The gateway records prompts and outputs unless asked not to, so an unknown privacy mode redacts too.
        if route.privacy_mode is not False:
            headers["X-PostHog-Privacy-Mode"] = "true"
        return {**kwargs, "extra_headers": {**(kwargs.get("extra_headers") or {}), **headers}}

    def _record_ai_gateway_fallback(self, reason: str, error: Exception) -> None:
        AI_GATEWAY_FALLBACK_COUNTER.labels(reason=reason).inc()
        model_name = getattr(self, "model", None) or getattr(self, "model_name", "unknown")
        logger.warning("posthog_ai_gateway_fallback", reason=reason, error_type=type(error).__name__, model=model_name)

    def _stream(
        self,
        messages: list[BaseMessage],
        stop: list[str] | None = None,
        run_manager: CallbackManagerForLLMRun | None = None,
        **kwargs: Any,
    ) -> Iterator[ChatGenerationChunk]:
        fallback = self.ai_gateway_fallback
        if fallback is None:
            yield from cast(BaseChatModel, super())._stream(messages, stop=stop, run_manager=run_manager, **kwargs)
            return

        direct_kwargs = _without_ai_gateway_headers(kwargs)
        if _has_ai_gateway_headers(kwargs):
            served = False
            held: list[ChatGenerationChunk] = []
            try:
                for chunk in cast(BaseChatModel, super())._stream(
                    messages, stop=stop, run_manager=run_manager, **kwargs
                ):
                    if served:
                        yield chunk
                        continue
                    held.append(chunk)
                    if not _carries_output(chunk):
                        continue
                    served = True
                    yield from _marked_as_gateway_served(held)
                if not served and held:
                    yield from _marked_as_gateway_served(held)
                return
            except Exception as error:
                reason = _ai_gateway_fallback_reason(error)
                if served or reason is None:
                    raise
                self._record_ai_gateway_fallback(reason, error)

        yield from fallback._stream(messages, stop=stop, run_manager=run_manager, **direct_kwargs)

    def _generate(
        self,
        messages: list[BaseMessage],
        stop: list[str] | None = None,
        run_manager: CallbackManagerForLLMRun | None = None,
        **kwargs: Any,
    ) -> ChatResult:
        fallback = self.ai_gateway_fallback
        if fallback is None or self.streaming:
            return cast(BaseChatModel, super())._generate(messages, stop=stop, run_manager=run_manager, **kwargs)

        direct_kwargs = _without_ai_gateway_headers(kwargs)
        if _has_ai_gateway_headers(kwargs):
            try:
                result = cast(BaseChatModel, super())._generate(messages, stop=stop, run_manager=run_manager, **kwargs)
            except Exception as error:
                reason = _ai_gateway_fallback_reason(error)
                if reason is None:
                    raise
                self._record_ai_gateway_fallback(reason, error)
            else:
                for generation in result.generations:
                    generation.generation_info = {**(generation.generation_info or {}), AI_GATEWAY_SERVED_KEY: True}
                return result

        return fallback._generate(messages, stop=stop, run_manager=run_manager, **direct_kwargs)

    async def _astream(
        self,
        messages: list[BaseMessage],
        stop: list[str] | None = None,
        run_manager: AsyncCallbackManagerForLLMRun | None = None,
        **kwargs: Any,
    ) -> AsyncIterator[ChatGenerationChunk]:
        fallback = self.ai_gateway_fallback
        if fallback is None:
            async for chunk in cast(BaseChatModel, super())._astream(
                messages, stop=stop, run_manager=run_manager, **kwargs
            ):
                yield chunk
            return

        direct_kwargs = _without_ai_gateway_headers(kwargs)
        if _has_ai_gateway_headers(kwargs):
            served = False
            # Anthropic opens the stream with a metadata chunk. Holding those keeps the fallback open until real
            # output, and keeps their usage off the twin's generation.
            held: list[ChatGenerationChunk] = []
            try:
                async for chunk in cast(BaseChatModel, super())._astream(
                    messages, stop=stop, run_manager=run_manager, **kwargs
                ):
                    if served:
                        yield chunk
                        continue
                    held.append(chunk)
                    if not _carries_output(chunk):
                        continue
                    served = True
                    for pending in _marked_as_gateway_served(held):
                        yield pending
                if not served and held:
                    # A stream that ended without output was still served by the gateway.
                    for pending in _marked_as_gateway_served(held):
                        yield pending
                return
            except Exception as error:
                reason = _ai_gateway_fallback_reason(error)
                # After output the user has seen the turn, so a direct retry would repeat it.
                if served or reason is None:
                    raise
                self._record_ai_gateway_fallback(reason, error)

        async for chunk in fallback._astream(messages, stop=stop, run_manager=run_manager, **direct_kwargs):
            yield chunk

    async def _agenerate(
        self,
        messages: list[BaseMessage],
        stop: list[str] | None = None,
        run_manager: AsyncCallbackManagerForLLMRun | None = None,
        **kwargs: Any,
    ) -> ChatResult:
        fallback = self.ai_gateway_fallback
        # A streaming call resolves through `_astream`, which routes and falls back on its own.
        if fallback is None or self.streaming:
            return await cast(BaseChatModel, super())._agenerate(messages, stop=stop, run_manager=run_manager, **kwargs)

        direct_kwargs = _without_ai_gateway_headers(kwargs)
        if _has_ai_gateway_headers(kwargs):
            try:
                result = await cast(BaseChatModel, super())._agenerate(
                    messages, stop=stop, run_manager=run_manager, **kwargs
                )
            except Exception as error:
                reason = _ai_gateway_fallback_reason(error)
                if reason is None:
                    raise
                self._record_ai_gateway_fallback(reason, error)
            else:
                for generation in result.generations:
                    generation.generation_info = {**(generation.generation_info or {}), AI_GATEWAY_SERVED_KEY: True}
                return result

        return await fallback._agenerate(messages, stop=stop, run_manager=run_manager, **direct_kwargs)


class MaxChatOpenAI(MaxChatMixin, ChatOpenAI):
    """PostHog-tuned subclass of ChatOpenAI.

    This subclass automatically injects project, organization, and user context as the final part of the system prompt.
    It also makes sure we retry automatically in case of an OpenAI API error.
    If billable is set to True, the generation will be marked as billable in the usage report for calculating AI billing credits.
    If inject_context is set to False, no context will be included in the system prompt.
    """

    posthog_provider: ClassVar[str] = "openai"

    ai_gateway_fallback: ChatOpenAI | None = Field(default=None, exclude=True)

    @model_validator(mode="before")
    @classmethod
    def _via_ai_gateway(cls, data: Any) -> Any:
        if not isinstance(data, dict) or "ai_gateway_fallback" in data:
            return data
        ai_gateway = resolve_ai_gateway_config()
        if ai_gateway is None:
            return data
        return {
            **data,
            "base_url": ai_gateway.url,
            "api_key": ai_gateway.api_key,
            "http_client": httpx.Client(trust_env=False, verify=_ai_gateway_ssl_context()),
            "http_async_client": httpx.AsyncClient(trust_env=False, verify=_ai_gateway_ssl_context()),
            "timeout": AI_GATEWAY_TIMEOUT,
            "max_retries": 0,
            "ai_gateway_fallback": cls(**data, ai_gateway_fallback=None),
        }

    def model_post_init(self, __context: Any) -> None:
        super().model_post_init(__context)
        if settings.IN_EVAL_TESTING and not self.service_tier and self.model_name in OPENAI_FLEX_MODELS:
            self.service_tier = "flex"  # 50% cheaper than default tier, but slower

    def _enrich_responses_api_model_kwargs(self, project_org_user_variables: dict[str, Any]) -> None:
        """Mutate the provided model_kwargs dict in-place, ensuring the project/org/user context is present.

        If the caller has already supplied ``instructions`` we append our context; otherwise we set ``instructions``
        from scratch. This function is intentionally side-effectful and returns ``None``.
        """

        system_msg_content = str(self._get_project_org_system_message(project_org_user_variables).content)

        if self.model_kwargs.get("instructions"):
            # Append to existing instructions
            self.model_kwargs["instructions"] = f"{system_msg_content}\n\n{self.model_kwargs['instructions']}"
        else:
            # Initialise instructions if absent or falsy
            self.model_kwargs["instructions"] = system_msg_content
        if self.ai_gateway_fallback is not None:
            self.ai_gateway_fallback.model_kwargs["instructions"] = self.model_kwargs["instructions"]

    def generate(
        self,
        messages: list[list[BaseMessage]],
        *args,
        **kwargs,
    ) -> LLMResult:
        if self.inject_context:
            project_org_user_variables = self._get_project_org_user_variables()
            if self.use_responses_api:
                self._enrich_responses_api_model_kwargs(project_org_user_variables)
            else:
                messages = self._enrich_messages(messages, project_org_user_variables)

        kwargs = self._with_posthog_properties(kwargs)
        if route := self._ai_gateway_route(kwargs):
            kwargs = self._with_ai_gateway_headers(kwargs, route)

        return super().generate(messages, *args, **kwargs)

    async def agenerate(
        self,
        messages: list[list[BaseMessage]],
        *args,
        **kwargs,
    ) -> LLMResult:
        if self.inject_context:
            project_org_user_variables = await self._aget_project_org_user_variables()
            if self.use_responses_api:
                self._enrich_responses_api_model_kwargs(project_org_user_variables)
            else:
                messages = self._enrich_messages(messages, project_org_user_variables)

        kwargs = self._with_posthog_properties(kwargs)
        if route := self._ai_gateway_route(kwargs):
            kwargs = self._with_ai_gateway_headers(kwargs, route)

        return await super().agenerate(messages, *args, **kwargs)


class MaxChatAnthropic(MaxChatMixin, ChatAnthropic):
    """PostHog-tuned subclass of ChatAnthropic.

    This subclass automatically injects project, organization, and user context as the final part of the system prompt.
    It also makes sure we retry automatically in case of errors.
    """

    posthog_provider: ClassVar[str] = "anthropic"

    bypass_proxy: bool = False
    """
    If True, bypasses egress proxies (HTTP_PROXY/etc)—use for private LLM gateway; if False, default behavior.
    """

    ai_gateway_fallback: ChatAnthropic | None = Field(default=None, exclude=True)

    _ai_gateway_timed_out_monotonic: ClassVar[float | None] = None

    @model_validator(mode="before")
    @classmethod
    def _via_ai_gateway(cls, data: Any) -> Any:
        if not isinstance(data, dict) or "ai_gateway_fallback" in data:
            return data
        if data.keys() & {"anthropic_api_url", "base_url", "anthropic_api_key", "api_key"}:
            return data
        ai_gateway = resolve_ai_gateway_config()
        if ai_gateway is None:
            return data
        return {
            **data,
            "anthropic_api_url": anthropic_gateway_base_url(ai_gateway.url),
            "anthropic_api_key": ai_gateway.api_key,
            "bypass_proxy": True,
            # The twin is the retry; an SDK retry doubles the wait before it.
            "max_retries": 0,
            "ai_gateway_fallback": cls(**data, ai_gateway_fallback=None),
        }

    @cached_property
    def _client_params(self) -> dict[str, Any]:
        params = cast(dict[str, Any], ChatAnthropic._client_params.func(self))  # type: ignore[attr-defined]
        if self.ai_gateway_fallback is not None and self.default_request_timeout is None:
            # langchain-anthropic defaults to no timeout, so a stalled gateway would never fall back.
            params["timeout"] = AI_GATEWAY_TIMEOUT
        return params

    @cached_property
    def _client(self) -> anthropic.Client:
        if not self.bypass_proxy:
            # Defer to upstream so the lru_cache'd httpx client and default proxy behavior are preserved.
            return cast(anthropic.Client, ChatAnthropic._client.func(self))  # type: ignore[attr-defined]
        return anthropic.Client(
            **self._client_params,
            http_client=anthropic.DefaultHttpxClient(**self._bypass_http_client_kwargs()),
        )

    @cached_property
    def _async_client(self) -> anthropic.AsyncClient:
        if not self.bypass_proxy:
            return cast(anthropic.AsyncClient, ChatAnthropic._async_client.func(self))  # type: ignore[attr-defined]
        return anthropic.AsyncClient(
            **self._client_params,
            http_client=anthropic.DefaultAsyncHttpxClient(**self._bypass_http_client_kwargs()),
        )

    def _bypass_http_client_kwargs(self) -> dict[str, Any]:
        """Builds kwargs for ``anthropic.DefaultHttpxClient`` / ``DefaultAsyncHttpxClient`` to bypass the Smokescreen egress proxy without altering other SDK defaults.

        Instead of using ``trust_env=False``, which is ineffective due to SDK internals, we set ``mounts={"http://": None, "https://": None, "all://": None}`` to override environment proxy settings. This approach preserves all other Anthropic SDK connection defaults, such as timeouts, pool limits, and transport settings.

        This depends on the SDK merging the ``mounts`` kwarg on top of its proxy settings and retaining its defaults—guarded by tests in ``test_llm.py``.
        """

        client_params = self._client_params
        kwargs: dict[str, Any] = {
            "base_url": client_params["base_url"],
            "mounts": dict(_BYPASS_PROXY_MOUNTS),
        }
        if "timeout" in client_params:
            # Forward the caller's timeout (langchain-anthropic always sets this key, even when
            # the value is None) so the bypass path matches the non-bypass path's timeout exactly.
            kwargs["timeout"] = client_params["timeout"]
        return kwargs

    def get_num_tokens_from_messages(self, messages: list[BaseMessage], tools: Any = None, **kwargs: Any) -> int:
        fallback = self.ai_gateway_fallback
        if fallback is None:
            return super().get_num_tokens_from_messages(messages, tools=tools, **kwargs)
        if self._ai_gateway_cooling_down():
            AI_GATEWAY_FALLBACK_COUNTER.labels(reason="timeout_cooldown").inc()
            return fallback.get_num_tokens_from_messages(messages, tools=tools, **kwargs)
        try:
            return super().get_num_tokens_from_messages(
                messages,
                tools=tools,
                **{**kwargs, "extra_headers": {**(kwargs.get("extra_headers") or {}), "X-PostHog-Billable": "false"}},
            )
        except Exception as error:
            reason = _ai_gateway_fallback_reason(error)
            if reason is None:
                raise
            self._record_ai_gateway_fallback(reason, error)
        return fallback.get_num_tokens_from_messages(messages, tools=tools, **kwargs)

    def _record_ai_gateway_fallback(self, reason: str, error: Exception) -> None:
        super()._record_ai_gateway_fallback(reason, error)
        if reason == "timeout":
            MaxChatAnthropic._ai_gateway_timed_out_monotonic = time.monotonic()

    @staticmethod
    def _ai_gateway_cooling_down() -> bool:
        timed_out_monotonic = MaxChatAnthropic._ai_gateway_timed_out_monotonic
        if timed_out_monotonic is None:
            return False
        return time.monotonic() - timed_out_monotonic < AI_GATEWAY_TIMEOUT_COOLDOWN_SECONDS

    def _ai_gateway_route(self, kwargs: Mapping[str, Any]) -> AIGatewayRoute | None:
        if self.ai_gateway_fallback is None:
            return None
        handlers = _posthog_callback_handlers(kwargs.get("callbacks")) + _posthog_callback_handlers(self.callbacks)
        if not all(isinstance(handler, MaxCallbackHandler) for handler in handlers):
            return None
        posthog_properties = kwargs["metadata"]["posthog_properties"]
        handler = cast(MaxCallbackHandler, handlers[0]) if handlers else None
        if handler is not None and handler.capture_disabled:
            return None
        billable = False
        if handler is not None:
            ai_product = handler.ai_product or posthog_properties["ai_product"]
            if posthog_properties["$ai_billable"]:
                billable = ai_product in BILLABLE_AI_PRODUCTS
        else:
            configurable = ensure_config().get("configurable") or {}
            ai_product = configurable.get("ai_product") or posthog_properties["ai_product"]
        if billable:
            if ai_product not in AI_GATEWAY_BILLABLE_PRODUCTS:
                return None
        if self._ai_gateway_cooling_down():
            AI_GATEWAY_FALLBACK_COUNTER.labels(reason="timeout_cooldown").inc()
            return None
        return self._ai_gateway_route_attributed(ai_product=ai_product, billable=billable, handler=handler)

    def generate(
        self,
        messages: list[list[BaseMessage]],
        *args,
        **kwargs,
    ) -> LLMResult:
        if self.inject_context:
            project_org_user_variables = self._get_project_org_user_variables()
            messages = self._enrich_messages(messages, project_org_user_variables)

        kwargs = self._with_posthog_properties(kwargs)
        if route := self._ai_gateway_route(kwargs):
            kwargs = self._with_ai_gateway_headers(kwargs, route)

        return super().generate(messages, *args, **kwargs)

    async def agenerate(
        self,
        messages: list[list[BaseMessage]],
        *args,
        **kwargs,
    ) -> LLMResult:
        if self.inject_context:
            project_org_user_variables = await self._aget_project_org_user_variables()
            messages = self._enrich_messages(messages, project_org_user_variables)

        kwargs = self._with_posthog_properties(kwargs)
        if route := self._ai_gateway_route(kwargs):
            kwargs = self._with_ai_gateway_headers(kwargs, route)

        return await super().agenerate(messages, *args, **kwargs)