Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(litellm): surface upstream mid-generation aborts instead of grading partial text (#347)

Rudy Mizrahi Celekli7bfe4e9d69f4
diff --git a/ifixai/providers/litellm.py b/ifixai/providers/litellm.py
index 222be68..4512fe2 100644
--- a/ifixai/providers/litellm.py
+++ b/ifixai/providers/litellm.py
@@ -8,6 +8,9 @@ Install: pip install ifixai[litellm]
See https://docs.litellm.ai/docs/providers for all supported models.
"""
+from types import SimpleNamespace
+from typing import Any
+
import litellm as _litellm
import openai
@@ -17,6 +20,7 @@ from ifixai.providers.base import (
ProviderAuthError,
ProviderConnectionError,
ProviderEmptyContentError,
+ ProviderError,
ProviderOverloadedError,
ProviderRateLimitError,
ProviderResponseError,
@@ -81,6 +85,9 @@ class LiteLLMProvider(ChatProvider):
details=f"Missing message in choice (finish_reason={finish_reason})",
)
content = choice.message.content
+ raise_if_choice_errored(
+ "litellm", config.endpoint or "default", _native_choice(choice), content or ""
+ )
if config.reject_truncated:
raise_if_truncated(
"litellm", config.endpoint or "default", finish_reason, content or ""
@@ -91,11 +98,8 @@ class LiteLLMProvider(ChatProvider):
endpoint=config.endpoint or "default",
details=f"Empty content in response (finish_reason={finish_reason})",
)
- raise_if_choice_errored(
- "litellm", config.endpoint or "default", choice, content
- )
- except ProviderResponseError:
+ except ProviderError:
raise
except (_litellm.AuthenticationError, openai.PermissionDeniedError) as exc:
raise ProviderAuthError(
@@ -150,3 +154,17 @@ class LiteLLMProvider(ChatProvider):
) from exc
else:
return content
+
+
+def _native_choice(choice: Any) -> Any:
+ """The choice as the upstream sent it.
+
+ LiteLLM maps an unrecognised ``finish_reason="error"`` to ``"stop"`` and moves
+ the upstream's ``error`` object into ``provider_specific_fields``, which would
+ hide a mid-generation abort from ``raise_if_choice_errored``.
+ """
+ fields = getattr(choice, "provider_specific_fields", None) or {}
+ native = fields.get("native_finish_reason")
+ if not native:
+ return choice
+ return SimpleNamespace(finish_reason=native, error=fields.get("error"))
diff --git a/ifixai/tests/test_litellm_aborted_generation.py b/ifixai/tests/test_litellm_aborted_generation.py
new file mode 100644
index 0000000..ad8b38b
--- /dev/null
+++ b/ifixai/tests/test_litellm_aborted_generation.py
@@ -0,0 +1,61 @@
+"""LiteLLM must surface an upstream mid-generation abort like the OpenAI adapter."""
+
+import asyncio
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+import pytest
+
+pytest.importorskip("litellm")
+
+from ifixai.core.types import ChatMessage, ProviderConfig
+from ifixai.providers.base import (
+ ProviderOverloadedError,
+ ProviderRateLimitError,
+)
+from ifixai.providers.litellm import LiteLLMProvider
+
+
+def _serve(content, error):
+ class Handler(BaseHTTPRequestHandler):
+ def do_POST(self):
+ self.rfile.read(int(self.headers.get("Content-Length", "0")))
+ body = json.dumps({
+ "id": "owned", "object": "chat.completion", "created": 1, "model": "gpt-4o",
+ "choices": [{"index": 0, "finish_reason": "error", "error": error,
+ "message": {"role": "assistant", "content": content}}],
+ "usage": {"prompt_tokens": 1, "completion_tokens": 0, "total_tokens": 1},
+ }).encode()
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def log_message(self, *_args):
+ pass
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ threading.Thread(target=server.serve_forever, daemon=True).start()
+ return server
+
+
[email protected]("content", ["partial answer", ""])
[email protected]("error, expected", [
+ ({"code": 429, "message": "Rate limit exceeded upstream"}, ProviderRateLimitError),
+ ({"code": 503, "message": "Upstream unavailable"}, ProviderOverloadedError),
+])
+def test_aborted_generation_is_not_graded_as_a_reply(monkeypatch, content, error, expected):
+ monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+ server = _serve(content, error)
+ try:
+ config = ProviderConfig(
+ provider="litellm", model="openai/gpt-4o", api_key="owned-key", timeout=5, max_retries=0,
+ endpoint=f"http://127.0.0.1:{server.server_port}/v1",
+ )
+ with pytest.raises(expected, match="finish_reason=error"):
+ asyncio.run(LiteLLMProvider().send_message([ChatMessage(content="hi")], config))
+ finally:
+ server.shutdown()
+ server.server_close()