Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

6.2 KB
Raw Blame
import asyncio

import openai

from ifixai.core.types import ChatMessage, ProviderConfig
from ifixai.providers.base import (
    ChatProvider,
    ProviderAuthError,
    ProviderConnectionError,
    ProviderEmptyContentError,
    ProviderRateLimitError,
    ProviderResponseError,
    ProviderTimeoutError,
    create_chat_completion_json_fallback,
    openai_http_client,
    raise_for_http_status,
    raise_if_choice_errored,
    raise_if_truncated,
    sdk_error_text,
)
from ifixai.providers.client_pool import close_cached_clients

DEFAULT_MODEL = "openai/gpt-4o-mini"
DEFAULT_BASE_URL = "https://router.requesty.ai/v1"
# Hard ceiling on max_tokens for Requesty calls. Per-call ``config.max_tokens``
# is clamped to this value; unset config falls through to the ceiling. Prevents
# verbose generations from blowing wall-time and credits on long fixtures.
MAX_TOKENS_CEILING: int = 8192

ClientCacheKey = tuple[str, str | None, float, int]


class RequestyProvider(ChatProvider):
    def __init__(self) -> None:
        self._clients: dict[ClientCacheKey, openai.AsyncOpenAI] = {}
        self._client_lock = asyncio.Lock()

    async def get_client(self, config: ProviderConfig) -> openai.AsyncOpenAI:
        """Return a long-lived AsyncOpenAI client keyed on connection params.

        Caching by (base_url, api_key, timeout, max_retries) lets the
        underlying httpx pool reuse TCP/TLS across LLM calls instead of
        paying a handshake per request.
        """
        base_url = config.endpoint or DEFAULT_BASE_URL
        key: ClientCacheKey = (
            base_url,
            config.api_key,
            float(config.timeout),
            config.max_retries,
        )
        cached = self._clients.get(key)
        if cached is not None:
            return cached
        async with self._client_lock:
            cached = self._clients.get(key)
            if cached is not None:
                return cached
            client = openai.AsyncOpenAI(
                api_key=config.api_key,
                base_url=base_url,
                timeout=float(config.timeout),
                max_retries=config.max_retries,
                http_client=openai_http_client(openai),
            )
            self._clients[key] = client
            return client

    async def aclose(self) -> None:
        await close_cached_clients(self._clients)

    async def send_message(
        self,
        messages: list[ChatMessage],
        config: ProviderConfig,
    ) -> str:
        base_url = config.endpoint or DEFAULT_BASE_URL
        model = config.model or DEFAULT_MODEL
        formatted = [{"role": m.role, "content": m.content} for m in messages]

        client = await self.get_client(config)
        try:
            effective_max_tokens = (
                min(config.max_tokens, MAX_TOKENS_CEILING)
                if config.max_tokens is not None
                else MAX_TOKENS_CEILING
            )
            create_kwargs: dict = {
                "model": model,
                "messages": formatted,  # type: ignore[arg-type]
                "max_tokens": effective_max_tokens,
                "temperature": config.temperature,
            }
            if config.seed is not None:
                create_kwargs["seed"] = config.seed
            if config.json_output:
                # Constrain judge calls to valid JSON so cheap models reliably emit
                # a parseable verdict instead of breaking the contract. Falls back to
                # free text (json-repair handles parsing) if the model does not
                # support response_format.
                create_kwargs["response_format"] = {"type": "json_object"}
            response = await create_chat_completion_json_fallback(
                client, **create_kwargs
            )
            choices = response.choices
            if not choices:
                raise ProviderResponseError(
                    provider="requesty",
                    endpoint=base_url,
                    details=f"No choices in response (id={response.id})",
                )
            choice = choices[0]
            finish_reason = choice.finish_reason or "unknown"
            if choice.message is None:
                raise ProviderResponseError(
                    provider="requesty",
                    endpoint=base_url,
                    details=f"Missing message in choice (finish_reason={finish_reason})",
                )
            content = choice.message.content
            raise_if_choice_errored("requesty", base_url, choice, content or "")
            if config.reject_truncated:
                raise_if_truncated("requesty", base_url, finish_reason, content or "")
            if not content:
                raise ProviderEmptyContentError(
                    provider="requesty",
                    endpoint=base_url,
                    details=f"Empty content in response (finish_reason={finish_reason})",
                )
        except openai.AuthenticationError as exc:
            raise ProviderAuthError(
                provider="requesty", endpoint=base_url, details=sdk_error_text(exc)
            ) from exc
        except openai.RateLimitError as exc:
            raise ProviderRateLimitError(
                provider="requesty", endpoint=base_url, details=sdk_error_text(exc)
            ) from exc
        except openai.APITimeoutError as exc:
            raise ProviderTimeoutError(
                provider="requesty", endpoint=base_url, details=sdk_error_text(exc)
            ) from exc
        except openai.APIConnectionError as exc:
            raise ProviderConnectionError(
                provider="requesty", endpoint=base_url, details=sdk_error_text(exc)
            ) from exc
        except openai.APIStatusError as exc:
            # The gateway's retryable band (408 timeout, 500 internal, 502 model
            # down, 503 no routable provider, 504) is split off here as a
            # transient overload; everything else stays a response error.
            raise_for_http_status("requesty", base_url, exc)
        except openai.APIError as exc:
            raise ProviderResponseError(
                provider="requesty", endpoint=base_url, details=sdk_error_text(exc)
            ) from exc
        else:
            return content