Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix: keep OpenAI error bodies (#369)
n-papaioannou73f2db814992
diff --git a/ifixai/providers/atlascloud.py b/ifixai/providers/atlascloud.pyindex 91b7a67..ff67726 100644--- a/ifixai/providers/atlascloud.py+++ b/ifixai/providers/atlascloud.py@@ -16,6 +16,7 @@ from ifixai.providers.base import (raise_for_http_status,raise_if_choice_errored,raise_if_truncated,+ sdk_error_text,)from ifixai.providers.client_pool import close_cached_clients@@ -116,26 +117,26 @@ class AtlasCloudProvider(ChatProvider):)except openai.AuthenticationError as exc:raise ProviderAuthError(- provider="atlascloud", endpoint=base_url, details=str(exc)+ provider="atlascloud", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.RateLimitError as exc:raise ProviderRateLimitError(- provider="atlascloud", endpoint=base_url, details=str(exc)+ provider="atlascloud", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.APITimeoutError as exc:raise ProviderTimeoutError(- provider="atlascloud", endpoint=base_url, details=str(exc)+ provider="atlascloud", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.APIConnectionError as exc:raise ProviderConnectionError(- provider="atlascloud", endpoint=base_url, details=str(exc)+ provider="atlascloud", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.APIStatusError as exc:# 408/5xx is the gateway having a moment, not a dead provider.raise_for_http_status("atlascloud", base_url, exc)except openai.APIError as exc:raise ProviderResponseError(- provider="atlascloud", endpoint=base_url, details=str(exc)+ provider="atlascloud", endpoint=base_url, details=sdk_error_text(exc)) from excelse:return contentdiff --git a/ifixai/providers/azure.py b/ifixai/providers/azure.pyindex 55e20bd..0dfc72a 100644--- a/ifixai/providers/azure.py+++ b/ifixai/providers/azure.py@@ -16,6 +16,7 @@ from ifixai.providers.base import (raise_for_http_status,raise_if_choice_errored,raise_if_truncated,+ sdk_error_text,)from ifixai.providers.client_pool import close_cached_clients@@ -150,25 +151,25 @@ class AzureOpenAIProvider(ChatProvider):raise ProviderAuthError(provider="azure",endpoint=config.endpoint,- details=str(exc),+ details=sdk_error_text(exc),) from excexcept openai.RateLimitError as exc:raise ProviderRateLimitError(provider="azure",endpoint=config.endpoint,- details=str(exc),+ details=sdk_error_text(exc),) from excexcept openai.APITimeoutError as exc:raise ProviderTimeoutError(provider="azure",endpoint=config.endpoint,- details=str(exc),+ details=sdk_error_text(exc),) from excexcept openai.APIConnectionError as exc:raise ProviderConnectionError(provider="azure",endpoint=config.endpoint,- details=str(exc),+ details=sdk_error_text(exc),) from excexcept openai.APIStatusError as exc:# 408/5xx is the gateway having a moment, not a dead provider.@@ -177,7 +178,7 @@ class AzureOpenAIProvider(ChatProvider):raise ProviderResponseError(provider="azure",endpoint=config.endpoint,- details=str(exc),+ details=sdk_error_text(exc),) from excelse:return contentdiff --git a/ifixai/providers/base.py b/ifixai/providers/base.pyindex a551f89..8f92787 100644--- a/ifixai/providers/base.py+++ b/ifixai/providers/base.py@@ -328,9 +328,10 @@ async def create_chat_completion_json_fallback(client: Any, **kwargs: Any) -> An) as exc:# A model the provider doesn't know is refused whatever the format, so that 400 is no# verdict on the schema.- if kwargs.get("response_format", {}).get("type") == "json_schema" and not names_unknown_model(str(exc)):- raise ProviderSchemaRejectedError(details=f"strict schema refused: {exc}") from exc- if not drop_rejected_optional_params(kwargs, str(exc)):+ text = sdk_error_text(exc)+ if kwargs.get("response_format", {}).get("type") == "json_schema" and not names_unknown_model(text):+ raise ProviderSchemaRejectedError(details=f"strict schema refused: {text}") from exc+ if not drop_rejected_optional_params(kwargs, text):raisereturn await client.chat.completions.create(**kwargs)@@ -540,6 +541,20 @@ def raise_if_choice_errored(raise ProviderResponseError(provider=provider, endpoint=endpoint, details=detail)+def sdk_error_text(exc: BaseException) -> str:+ """An SDK error's text with the provider's whole reply body in it.++ openai 3.38+ puts only ``body["error"]["message"]`` in the text; the error code,+ type and gateway fields the matchers in this module read are left on ``exc.body``.+ Older SDKs already include the body, so their text is returned unchanged.+ """+ text = str(exc)+ body = getattr(exc, "body", None)+ if body is None or body == "" or str(body) in text:+ return text+ return f"{text} - {body}"++def raise_for_http_status(provider: str, endpoint: str, exc: Exception) -> NoReturn:"""Translate an OpenAI-SDK HTTP error into the matching provider exception.@@ -549,7 +564,8 @@ def raise_for_http_status(provider: str, endpoint: str, exc: Exception) -> NoRetand aborting the run. Always raises."""status = getattr(exc, "status_code", None)- details = f"HTTP {status}: {exc}" if status is not None else str(exc)+ text = sdk_error_text(exc)+ details = f"HTTP {status}: {text}" if status is not None else textif status in RETRYABLE_HTTP_STATUS_CODES or (provider == "anthropic" and status == 529):raise ProviderOverloadedError(provider=provider, endpoint=endpoint, details=detailsdiff --git a/ifixai/providers/cloudflare.py b/ifixai/providers/cloudflare.pyindex 18ecc25..d64bf47 100644--- a/ifixai/providers/cloudflare.py+++ b/ifixai/providers/cloudflare.py@@ -34,6 +34,7 @@ from ifixai.providers.base import (raise_for_http_status,raise_if_choice_errored,raise_if_truncated,+ sdk_error_text,)from ifixai.providers.client_pool import close_cached_clients@@ -350,11 +351,11 @@ def raise_for_sdk_error(endpoint: str, exc: openai.APIError) -> NoReturn:for sdk_error, provider_error in SDK_ERROR_TRANSLATIONS.items():if isinstance(exc, sdk_error):raise provider_error(- provider=PROVIDER_NAME, endpoint=endpoint, details=str(exc)+ provider=PROVIDER_NAME, endpoint=endpoint, details=sdk_error_text(exc)) from excif not isinstance(exc, openai.APIStatusError):raise ProviderResponseError(- provider=PROVIDER_NAME, endpoint=endpoint, details=str(exc)+ provider=PROVIDER_NAME, endpoint=endpoint, details=sdk_error_text(exc)) from excif is_missing_gateway(exc):raise_for_rejected_access(endpoint, exc, GATEWAY_NOT_FOUND_HINT)@@ -374,7 +375,7 @@ def raise_for_rejected_access(own message kept and the likely causes named beside it."""raise ProviderAuthError(- provider=PROVIDER_NAME, endpoint=endpoint, details=f"{exc}. {hint}"+ provider=PROVIDER_NAME, endpoint=endpoint, details=f"{sdk_error_text(exc)}. {hint}") from exc@@ -419,7 +420,7 @@ def raise_for_exhausted_allocation(raise ProviderRateLimitError(provider=PROVIDER_NAME,endpoint=endpoint,- details=f"{exc}. {ALLOCATION_EXHAUSTED_HINT}",+ details=f"{sdk_error_text(exc)}. {ALLOCATION_EXHAUSTED_HINT}",) from exc@@ -435,7 +436,7 @@ def raise_for_payment_required(endpoint: str, exc: openai.APIStatusError) -> NoRraise ProviderResponseError(provider=PROVIDER_NAME,endpoint=endpoint,- details=f"HTTP {status.value} {status.phrase}: {exc}",+ details=f"HTTP {status.value} {status.phrase}: {sdk_error_text(exc)}",) from excdiff --git a/ifixai/providers/openai.py b/ifixai/providers/openai.pyindex 5177bce..a5164b3 100644--- a/ifixai/providers/openai.py+++ b/ifixai/providers/openai.py@@ -17,6 +17,7 @@ from ifixai.providers.base import (raise_if_choice_errored,raise_if_truncated,response_format,+ sdk_error_text,)from ifixai.providers.client_pool import close_cached_clients@@ -119,25 +120,25 @@ class OpenAIProvider(ChatProvider):raise ProviderAuthError(provider="openai",endpoint=endpoint,- details=str(exc),+ details=sdk_error_text(exc),) from excexcept openai.RateLimitError as exc:raise ProviderRateLimitError(provider="openai",endpoint=endpoint,- details=str(exc),+ details=sdk_error_text(exc),) from excexcept openai.APITimeoutError as exc:raise ProviderTimeoutError(provider="openai",endpoint=endpoint,- details=str(exc),+ details=sdk_error_text(exc),) from excexcept openai.APIConnectionError as exc:raise ProviderConnectionError(provider="openai",endpoint=endpoint,- details=str(exc),+ details=sdk_error_text(exc),) from excexcept openai.APIStatusError as exc:# 408/5xx is the gateway having a moment, not a dead provider.@@ -146,7 +147,7 @@ class OpenAIProvider(ChatProvider):raise ProviderResponseError(provider="openai",endpoint=endpoint,- details=str(exc),+ details=sdk_error_text(exc),) from excelse:return contentdiff --git a/ifixai/providers/openrouter.py b/ifixai/providers/openrouter.pyindex b5d6258..ac4b712 100644--- a/ifixai/providers/openrouter.py+++ b/ifixai/providers/openrouter.py@@ -17,6 +17,7 @@ from ifixai.providers.base import (raise_if_choice_errored,raise_if_truncated,response_format,+ sdk_error_text,)from ifixai.providers.client_pool import close_cached_clients@@ -140,19 +141,19 @@ class OpenRouterProvider(ChatProvider):)except openai.AuthenticationError as exc:raise ProviderAuthError(- provider="openrouter", endpoint=base_url, details=str(exc)+ provider="openrouter", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.RateLimitError as exc:raise ProviderRateLimitError(- provider="openrouter", endpoint=base_url, details=str(exc)+ provider="openrouter", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.APITimeoutError as exc:raise ProviderTimeoutError(- provider="openrouter", endpoint=base_url, details=str(exc)+ provider="openrouter", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.APIConnectionError as exc:raise ProviderConnectionError(- provider="openrouter", endpoint=base_url, details=str(exc)+ provider="openrouter", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.APIStatusError as exc:# OpenRouter's retryable band (408 timeout, 500 internal, 502 model@@ -161,7 +162,7 @@ class OpenRouterProvider(ChatProvider):raise_for_http_status("openrouter", base_url, exc)except openai.APIError as exc:raise ProviderResponseError(- provider="openrouter", endpoint=base_url, details=str(exc)+ provider="openrouter", endpoint=base_url, details=sdk_error_text(exc)) from excelse:return contentdiff --git a/ifixai/providers/orcarouter.py b/ifixai/providers/orcarouter.pyindex 35726af..2cb29ba 100644--- a/ifixai/providers/orcarouter.py+++ b/ifixai/providers/orcarouter.py@@ -16,6 +16,7 @@ from ifixai.providers.base import (raise_for_http_status,raise_if_choice_errored,raise_if_truncated,+ sdk_error_text,)from ifixai.providers.client_pool import close_cached_clients@@ -134,19 +135,19 @@ class OrcaRouterProvider(ChatProvider):)except openai.AuthenticationError as exc:raise ProviderAuthError(- provider="orcarouter", endpoint=base_url, details=str(exc)+ provider="orcarouter", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.RateLimitError as exc:raise ProviderRateLimitError(- provider="orcarouter", endpoint=base_url, details=str(exc)+ provider="orcarouter", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.APITimeoutError as exc:raise ProviderTimeoutError(- provider="orcarouter", endpoint=base_url, details=str(exc)+ provider="orcarouter", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.APIConnectionError as exc:raise ProviderConnectionError(- provider="orcarouter", endpoint=base_url, details=str(exc)+ provider="orcarouter", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.APIStatusError as exc:# OrcaRouter's retryable band (408 timeout, 500 internal, 502 model@@ -155,7 +156,7 @@ class OrcaRouterProvider(ChatProvider):raise_for_http_status("orcarouter", base_url, exc)except openai.APIError as exc:raise ProviderResponseError(- provider="orcarouter", endpoint=base_url, details=str(exc)+ provider="orcarouter", endpoint=base_url, details=sdk_error_text(exc)) from excelse:return contentdiff --git a/ifixai/providers/requesty.py b/ifixai/providers/requesty.pyindex 85b27ab..7f36849 100644--- a/ifixai/providers/requesty.py+++ b/ifixai/providers/requesty.py@@ -16,6 +16,7 @@ from ifixai.providers.base import (raise_for_http_status,raise_if_choice_errored,raise_if_truncated,+ sdk_error_text,)from ifixai.providers.client_pool import close_cached_clients@@ -128,19 +129,19 @@ class RequestyProvider(ChatProvider):)except openai.AuthenticationError as exc:raise ProviderAuthError(- provider="requesty", endpoint=base_url, details=str(exc)+ provider="requesty", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.RateLimitError as exc:raise ProviderRateLimitError(- provider="requesty", endpoint=base_url, details=str(exc)+ provider="requesty", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.APITimeoutError as exc:raise ProviderTimeoutError(- provider="requesty", endpoint=base_url, details=str(exc)+ provider="requesty", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.APIConnectionError as exc:raise ProviderConnectionError(- provider="requesty", endpoint=base_url, details=str(exc)+ provider="requesty", endpoint=base_url, details=sdk_error_text(exc)) from excexcept openai.APIStatusError as exc:# The gateway's retryable band (408 timeout, 500 internal, 502 model@@ -149,7 +150,7 @@ class RequestyProvider(ChatProvider):raise_for_http_status("requesty", base_url, exc)except openai.APIError as exc:raise ProviderResponseError(- provider="requesty", endpoint=base_url, details=str(exc)+ provider="requesty", endpoint=base_url, details=sdk_error_text(exc)) from excelse:return contentdiff --git a/ifixai/providers/vercel.py b/ifixai/providers/vercel.pyindex 1c7815f..00b715b 100644--- a/ifixai/providers/vercel.py+++ b/ifixai/providers/vercel.py@@ -28,6 +28,7 @@ from ifixai.providers.base import (raise_for_http_status,raise_if_choice_errored,raise_if_truncated,+ sdk_error_text,)from ifixai.providers.client_pool import close_cached_clients@@ -196,11 +197,11 @@ def raise_for_sdk_error(endpoint: str, exc: openai.APIError) -> NoReturn:for sdk_error, provider_error in SDK_ERROR_TRANSLATIONS.items():if isinstance(exc, sdk_error):raise provider_error(- provider=PROVIDER_NAME, endpoint=endpoint, details=str(exc)+ provider=PROVIDER_NAME, endpoint=endpoint, details=sdk_error_text(exc)) from excif not isinstance(exc, openai.APIStatusError):raise ProviderResponseError(- provider=PROVIDER_NAME, endpoint=endpoint, details=str(exc)+ provider=PROVIDER_NAME, endpoint=endpoint, details=sdk_error_text(exc)) from excif exc.status_code == HTTPStatus.PAYMENT_REQUIRED:raise_for_payment_required(endpoint, exc)@@ -220,7 +221,7 @@ def raise_for_payment_required(endpoint: str, exc: openai.APIStatusError) -> NoRraise ProviderResponseError(provider=PROVIDER_NAME,endpoint=endpoint,- details=f"HTTP {status.value} {status.phrase}: {exc}",+ details=f"HTTP {status.value} {status.phrase}: {sdk_error_text(exc)}",) from excdiff --git a/ifixai/tests/providers/test_sdk_error_text.py b/ifixai/tests/providers/test_sdk_error_text.pynew file mode 100644index 0000000..fe5cc11--- /dev/null+++ b/ifixai/tests/providers/test_sdk_error_text.py@@ -0,0 +1,59 @@+"""The provider's whole reply body reaches the error matchers on every OpenAI SDK.++openai 3.38 put only ``body["error"]["message"]`` in the exception text, so the+quota code, error type and gateway fields the matchers read went missing. These+build both SDKs' error text by hand, so they hold whichever SDK is installed.+"""++from __future__ import annotations++from typing import Any++import pytest++from ifixai.providers.base import (+ ProviderRateLimitError,+ is_fatal_provider_error,+ sdk_error_text,+)++SPENT_QUOTA = {"error": {"message": "You exceeded your current quota, please check your plan.",+ "type": "insufficient_quota", "code": "insufficient_quota"}}+GATEWAY_REFUSAL = {"success": False, "result": [], "messages": [],+ "errors": [{"code": 10000, "message": "Unauthorized"}]}+++class SdkError(Exception):+ def __init__(self, text: str, body: Any) -> None:+ super().__init__(text)+ self.body = body+++def old_sdk(status: int, body: Any) -> SdkError:+ return SdkError(f"Error code: {status} - {body}", body)+++def new_sdk(status: int, body: Any) -> SdkError:+ error = body.get("error", body) if isinstance(body, dict) else None+ message = error.get("message") if isinstance(error, dict) else None+ return SdkError(f"Error code: {status}" + (f" - {message}" if message else ""), body)+++def test_the_old_sdk_text_is_kept_as_it_was() -> None:+ exc = old_sdk(429, SPENT_QUOTA)+ assert sdk_error_text(exc) == str(exc)++[email protected]("body", [SPENT_QUOTA, GATEWAY_REFUSAL])+def test_the_new_sdk_text_gets_the_body_back(body: dict) -> None:+ assert str(body) in sdk_error_text(new_sdk(401, body))+++def test_a_spent_quota_stops_the_run_on_the_new_sdk() -> None:+ exc = ProviderRateLimitError(provider="openai", endpoint="x", details=sdk_error_text(new_sdk(429, SPENT_QUOTA)))+ assert is_fatal_provider_error(exc)++[email protected]("body", [None, ""])+def test_an_error_with_no_body_keeps_its_text(body: Any) -> None:+ assert sdk_error_text(SdkError("Connection error.", body)) == "Connection error."