Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(litellm): honor the configured native retry budget (#204)

Rudy Mizrahi Celekli3270160a862d
diff --git a/ifixai/providers/litellm.py b/ifixai/providers/litellm.py
index 4e833ae..222be68 100644
--- a/ifixai/providers/litellm.py
+++ b/ifixai/providers/litellm.py
@@ -45,6 +45,7 @@ class LiteLLMProvider(ChatProvider):
"model": model,
"messages": formatted_messages,
"drop_params": True,
+ "max_retries": config.max_retries,
"timeout": float(config.timeout),
"temperature": config.temperature,
}
diff --git a/ifixai/tests/test_litellm_transport_native.py b/ifixai/tests/test_litellm_transport_native.py
index f4c58ab..f200d68 100644
--- a/ifixai/tests/test_litellm_transport_native.py
+++ b/ifixai/tests/test_litellm_transport_native.py
@@ -125,3 +125,53 @@ def test_sdk_without_optional_permission_export_keeps_transient_errors(monkeypat
send(url, monkeypatch)
assert received.is_set()
assert not is_fatal_provider_error(caught.value)
+
+
[email protected]("retries", [0, 1])
+def test_litellm_native_request_obeys_retry_budget(monkeypatch, retries):
+ monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+ sdk = pytest.importorskip("litellm")
+ from ifixai.providers.litellm import LiteLLMProvider
+
+ seen = []
+ class Handler(BaseHTTPRequestHandler):
+ def do_POST(self):
+ self.rfile.read(int(self.headers.get("Content-Length", "0")))
+ seen.append(self.path)
+ if len(seen) == 1:
+ status = 500
+ data = {"error": {"message": "owned transient server failure", "type": "server_error"}}
+ else:
+ status = 200
+ data = {"id": "owned-response", "object": "chat.completion", "created": 1, "model": "gpt-4o",
+ "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "recovered reply"}}]}
+ body = json.dumps(data).encode()
+ self.send_response(status)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+ def log_message(self, *_args):
+ pass
+
+ # A process-wide SDK default must not override the requested budget.
+ monkeypatch.setattr(sdk, "num_retries", 0)
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ async def exercise():
+ provider = LiteLLMProvider()
+ config = ProviderConfig(provider="litellm", model="openai/gpt-4o", api_key="owned-key",
+ endpoint=f"http://127.0.0.1:{server.server_port}/v1", max_retries=retries)
+ if retries:
+ assert await provider.send_message([ChatMessage(content="hello")], config) == "recovered reply"
+ else:
+ with pytest.raises(ProviderOverloadedError):
+ await provider.send_message([ChatMessage(content="hello")], config)
+ try:
+ asyncio.run(exercise())
+ assert len(seen) == retries + 1
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join(2)