Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix(providers): classify embedded completion errors before empty content (#205)
Rudy Mizrahi Celekli2cf1f0897dc5
diff --git a/ifixai/providers/atlascloud.py b/ifixai/providers/atlascloud.pyindex eaa88ab..281b361 100644--- a/ifixai/providers/atlascloud.py+++ b/ifixai/providers/atlascloud.py@@ -104,6 +104,7 @@ class AtlasCloudProvider(ChatProvider):details=f"Missing message in choice (finish_reason={finish_reason})",)content = choice.message.content+ raise_if_choice_errored("atlascloud", base_url, choice, content or "")if config.reject_truncated:raise_if_truncated("atlascloud", base_url, finish_reason, content or "")if not content:@@ -112,7 +113,6 @@ class AtlasCloudProvider(ChatProvider):endpoint=base_url,details=f"Empty content in response (finish_reason={finish_reason})",)- raise_if_choice_errored("atlascloud", base_url, choice, content)except openai.AuthenticationError as exc:raise ProviderAuthError(provider="atlascloud", endpoint=base_url, details=str(exc)diff --git a/ifixai/providers/azure.py b/ifixai/providers/azure.pyindex 421ffa4..01b937e 100644--- a/ifixai/providers/azure.py+++ b/ifixai/providers/azure.py@@ -134,6 +134,7 @@ class AzureOpenAIProvider(ChatProvider):details=f"Missing message in choice (finish_reason={finish_reason})",)content = choice.message.content+ raise_if_choice_errored("azure", config.endpoint or "", choice, content or "")if config.reject_truncated:raise_if_truncated("azure", config.endpoint or "", finish_reason, content or ""@@ -144,7 +145,6 @@ class AzureOpenAIProvider(ChatProvider):endpoint=config.endpoint,details=f"Empty content in response (finish_reason={finish_reason})",)- raise_if_choice_errored("azure", config.endpoint or "", choice, content)except openai.AuthenticationError as exc:raise ProviderAuthError(provider="azure",diff --git a/ifixai/providers/huggingface.py b/ifixai/providers/huggingface.pyindex f9d739f..d0489c8 100644--- a/ifixai/providers/huggingface.py+++ b/ifixai/providers/huggingface.py@@ -183,6 +183,7 @@ def _call_chat_completion(details=f"Missing message in choice (finish_reason={finish_reason})",)content = choice.message.content+ raise_if_choice_errored("huggingface", "", choice, content or "")if reject_truncated:raise_if_truncated("huggingface", "", finish_reason, content or "")if not content:@@ -191,7 +192,6 @@ def _call_chat_completion(endpoint="",details=f"Empty content in response (finish_reason={finish_reason})",)- raise_if_choice_errored("huggingface", "", choice, content)return contentdiff --git a/ifixai/providers/openai.py b/ifixai/providers/openai.pyindex 4936c38..7c04259 100644--- a/ifixai/providers/openai.py+++ b/ifixai/providers/openai.py@@ -104,6 +104,7 @@ class OpenAIProvider(ChatProvider):details=f"Missing message in choice (finish_reason={finish_reason})",)content = choice.message.content+ raise_if_choice_errored("openai", endpoint, choice, content or "")if config.reject_truncated:raise_if_truncated("openai", endpoint, finish_reason, content or "")if not content:@@ -112,7 +113,6 @@ class OpenAIProvider(ChatProvider):endpoint=endpoint,details=f"Empty content in response (finish_reason={finish_reason})",)- raise_if_choice_errored("openai", endpoint, choice, content)except openai.AuthenticationError as exc:raise ProviderAuthError(provider="openai",diff --git a/ifixai/providers/openrouter.py b/ifixai/providers/openrouter.pyindex b9f1c53..12563b9 100644--- a/ifixai/providers/openrouter.py+++ b/ifixai/providers/openrouter.py@@ -127,6 +127,7 @@ class OpenRouterProvider(ChatProvider):details=f"Missing message in choice (finish_reason={finish_reason})",)content = choice.message.content+ raise_if_choice_errored("openrouter", base_url, choice, content or "")if config.reject_truncated:raise_if_truncated("openrouter", base_url, finish_reason, content or "")if not content:@@ -135,7 +136,6 @@ class OpenRouterProvider(ChatProvider):endpoint=base_url,details=f"Empty content in response (finish_reason={finish_reason})",)- raise_if_choice_errored("openrouter", base_url, choice, content)except openai.AuthenticationError as exc:raise ProviderAuthError(provider="openrouter", endpoint=base_url, details=str(exc)diff --git a/ifixai/providers/orcarouter.py b/ifixai/providers/orcarouter.pyindex 2196142..f5f1d3c 100644--- a/ifixai/providers/orcarouter.py+++ b/ifixai/providers/orcarouter.py@@ -122,6 +122,7 @@ class OrcaRouterProvider(ChatProvider):details=f"Missing message in choice (finish_reason={finish_reason})",)content = choice.message.content+ raise_if_choice_errored("orcarouter", base_url, choice, content or "")if config.reject_truncated:raise_if_truncated("orcarouter", base_url, finish_reason, content or "")if not content:@@ -130,7 +131,6 @@ class OrcaRouterProvider(ChatProvider):endpoint=base_url,details=f"Empty content in response (finish_reason={finish_reason})",)- raise_if_choice_errored("orcarouter", base_url, choice, content)except openai.AuthenticationError as exc:raise ProviderAuthError(provider="orcarouter", endpoint=base_url, details=str(exc)diff --git a/ifixai/providers/requesty.py b/ifixai/providers/requesty.pyindex 1055e0b..a721394 100644--- a/ifixai/providers/requesty.py+++ b/ifixai/providers/requesty.py@@ -116,6 +116,7 @@ class RequestyProvider(ChatProvider):details=f"Missing message in choice (finish_reason={finish_reason})",)content = choice.message.content+ raise_if_choice_errored("requesty", base_url, choice, content or "")if config.reject_truncated:raise_if_truncated("requesty", base_url, finish_reason, content or "")if not content:@@ -124,7 +125,6 @@ class RequestyProvider(ChatProvider):endpoint=base_url,details=f"Empty content in response (finish_reason={finish_reason})",)- raise_if_choice_errored("requesty", base_url, choice, content)except openai.AuthenticationError as exc:raise ProviderAuthError(provider="requesty", endpoint=base_url, details=str(exc)diff --git a/ifixai/tests/test_sdk_timeout_native.py b/ifixai/tests/test_sdk_timeout_native.pyindex 113349c..0892a43 100644--- a/ifixai/tests/test_sdk_timeout_native.py+++ b/ifixai/tests/test_sdk_timeout_native.py@@ -108,3 +108,60 @@ def test_connection_close_remains_a_connection_error(adapter):asyncio.run(exercise())assert received.is_set()++[email protected]("spec", [*ADAPTERS[:-1],+ ("huggingface", "HuggingFaceProvider", "huggingface_hub"),+], ids=lambda spec: spec[1])[email protected]("text", ["", "partial reply"])[email protected]("code", [429, 503])+def test_native_sdk_embedded_completion_errors_keep_their_identity(monkeypatch, spec, text, code):+ import json++ from ifixai.providers.base import (+ ProviderOverloadedError,+ ProviderRateLimitError,+ is_fatal_provider_error,+ )++ monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")+ adapter, _sdk = sdk_adapter(spec)+ seen = []+ class Handler(BaseHTTPRequestHandler):+ def do_POST(self):+ self.rfile.read(int(self.headers.get("Content-Length", "0")))+ seen.append(self.path)+ data = {"id": "owned-response", "object": "chat.completion", "created": 1, "model": "gpt-4o",+ "choices": [{"index": 0, "finish_reason": "error", "error": {"code": code, "message": "rate limit exceeded" if code == 429 else "owned gateway unavailable"},+ "message": {"role": "assistant", "content": text}}]}+ body = json.dumps(data).encode()+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)+ def log_message(self, *_args):+ pass++ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ thread = threading.Thread(target=server.serve_forever, daemon=True)+ thread.start()+ async def exercise():+ provider = adapter()+ config = ProviderConfig(provider=spec[0], api_key="owned-key", model="gpt-4o",+ endpoint=f"http://127.0.0.1:{server.server_port}/v1", max_retries=0)+ try:+ expected = ProviderRateLimitError if code == 429 else ProviderOverloadedError+ with pytest.raises(expected) as caught:+ await provider.send_message([ChatMessage(content="hello")], config)+ assert not is_fatal_provider_error(caught.value)+ assert caught.value.provider == spec[0]+ finally:+ await provider.aclose()+ try:+ asyncio.run(exercise())+ assert len(seen) == 1+ finally:+ server.shutdown()+ server.server_close()+ thread.join(2)