Agent Zero AI framework
feat: Add retry logic for transient LiteLLM errors
linuztx143678d9d201
diff --git a/models.py b/models.pyindex c3830d981..879ee3086 100644--- a/models.py+++ b/models.py@@ -16,6 +16,7 @@ from typing import (from litellm import completion, acompletion, embeddingimport litellm+import openaifrom python.helpers import dotenvfrom python.helpers import settings@@ -220,6 +221,31 @@ def get_rate_limiter(return limiter+def _is_transient_litellm_error(exc: Exception) -> bool:+ """Uses status_code when available, else falls back to exception types"""+ # Prefer explicit status codes if present+ status_code = getattr(exc, "status_code", None)+ if isinstance(status_code, int):+ if status_code in (408, 429, 500, 502, 503, 504):+ return True+ # Treat other 5xx as retriable+ if status_code >= 500:+ return True+ return False++ # Fallback to exception classes mapped by LiteLLM/OpenAI+ transient_types = (+ getattr(openai, "APITimeoutError", Exception),+ getattr(openai, "APIConnectionError", Exception),+ getattr(openai, "RateLimitError", Exception),+ getattr(openai, "APIError", Exception),+ getattr(openai, "InternalServerError", Exception),+ # Some providers map overloads to ServiceUnavailable-like errors+ getattr(openai, "APIStatusError", Exception),+ )+ return isinstance(exc, transient_types)++async def apply_rate_limiter(model_config: ModelConfig | None,input_text: str,@@ -454,50 +480,69 @@ class LiteLLMChatWrapper(SimpleChatModel):self.a0_model_conf, str(msgs_conv), rate_limiter_callback)- # call model- _completion = await acompletion(- model=self.model_name,- messages=msgs_conv,- stream=True,- **{**self.kwargs, **kwargs},- )+ # Prepare call kwargs and retry config (strip A0-only params before calling LiteLLM)+ call_kwargs: dict[str, Any] = {**self.kwargs, **kwargs}+ max_retries: int = int(call_kwargs.pop("a0_retry_attempts", 2))+ retry_delay_s: float = float(call_kwargs.pop("a0_retry_delay_seconds", 1.5))# resultsresult = ChatGenerationResult()- # iterate over chunks- async for chunk in _completion: # type: ignore- # parse chunk- parsed = _parse_chunk(chunk)- output = result.add_chunk(parsed)-- # collect reasoning delta and call callbacks- if output["reasoning_delta"]:- if reasoning_callback:- await reasoning_callback(output["reasoning_delta"], result.reasoning)- if tokens_callback:- await tokens_callback(- output["reasoning_delta"],- approximate_tokens(output["reasoning_delta"]),- )- # Add output tokens to rate limiter if configured- if limiter:- limiter.add(output=approximate_tokens(output["reasoning_delta"]))- # collect response delta and call callbacks- if output["response_delta"]:- if response_callback:- await response_callback(output["response_delta"], result.response)- if tokens_callback:- await tokens_callback(- output["response_delta"],- approximate_tokens(output["response_delta"]),- )- # Add output tokens to rate limiter if configured- if limiter:- limiter.add(output=approximate_tokens(output["response_delta"]))+ attempt = 0+ while True:+ got_any_chunk = False+ try:+ # call model+ _completion = await acompletion(+ model=self.model_name,+ messages=msgs_conv,+ stream=True,+ **call_kwargs,+ )- # return complete results- return result.response, result.reasoning+ # iterate over chunks+ async for chunk in _completion: # type: ignore+ got_any_chunk = True+ # parse chunk+ parsed = _parse_chunk(chunk)+ output = result.add_chunk(parsed)++ # collect reasoning delta and call callbacks+ if output["reasoning_delta"]:+ if reasoning_callback:+ await reasoning_callback(output["reasoning_delta"], result.reasoning)+ if tokens_callback:+ await tokens_callback(+ output["reasoning_delta"],+ approximate_tokens(output["reasoning_delta"]),+ )+ # Add output tokens to rate limiter if configured+ if limiter:+ limiter.add(output=approximate_tokens(output["reasoning_delta"]))+ # collect response delta and call callbacks+ if output["response_delta"]:+ if response_callback:+ await response_callback(output["response_delta"], result.response)+ if tokens_callback:+ await tokens_callback(+ output["response_delta"],+ approximate_tokens(output["response_delta"]),+ )+ # Add output tokens to rate limiter if configured+ if limiter:+ limiter.add(output=approximate_tokens(output["response_delta"]))++ # Successful completion of stream+ return result.response, result.reasoning++ except Exception as e:+ import asyncio++ # Retry only if no chunks received and error is transient+ if got_any_chunk or not _is_transient_litellm_error(e) or attempt >= max_retries:+ raise+ attempt += 1+ await asyncio.sleep(retry_delay_s)class BrowserCompatibleChatWrapper(LiteLLMChatWrapper):