Skip to content

Build and run agents you can see, understand and trust.

fix(model): refine the current retry logic within the chat model modules (#1730)

qbc926108a14d58
diff --git a/src/agentscope/model/_anthropic/_model.py b/src/agentscope/model/_anthropic/_model.py
index 208b378a1..dc36659a4 100644
--- a/src/agentscope/model/_anthropic/_model.py
+++ b/src/agentscope/model/_anthropic/_model.py
@@ -61,6 +61,7 @@ class AnthropicChatModel(ChatModelBase):
parameters: "AnthropicChatModel.Parameters | None" = None,
stream: bool = True,
max_retries: int = 3,
+ retry_delay: float = 1.0,
context_size: int = 200000,
formatter: FormatterBase | None = None,
client_kwargs: dict[str, Any] | None = None,
@@ -80,6 +81,8 @@ class AnthropicChatModel(ChatModelBase):
Whether to enable streaming output.
max_retries (`int`, defaults to `3`):
The maximum number of retries for the Anthropic API.
+ retry_delay (`float`, defaults to `1.0`):
+ Seconds to sleep between retry attempts.
context_size (`int`, defaults to `200000`):
The model context size used for context compression.
formatter (`FormatterBase | None`, defaults to `None`):
@@ -97,11 +100,23 @@ class AnthropicChatModel(ChatModelBase):
parameters=parameters or self.Parameters(),
stream=stream,
max_retries=max_retries,
+ retry_delay=retry_delay,
context_size=context_size,
)
self.formatter = formatter or AnthropicChatFormatter()
self.client_kwargs = client_kwargs or {}
+ @classmethod
+ def _get_retryable_exceptions(cls) -> tuple[Type[Exception], ...]:
+ import anthropic
+
+ return (
+ anthropic.APIConnectionError,
+ anthropic.APITimeoutError,
+ anthropic.RateLimitError,
+ anthropic.InternalServerError,
+ )
+
async def _call_api(
self,
model_name: str,
diff --git a/src/agentscope/model/_base.py b/src/agentscope/model/_base.py
index 6e87e46a0..f41caf678 100644
--- a/src/agentscope/model/_base.py
+++ b/src/agentscope/model/_base.py
@@ -49,7 +49,10 @@ class ChatModelBase:
"""The enable stream output for the LLM output."""
max_retries: int
- """The maximum retries for the Anthropic API."""
+ """The maximum number of retries for the underlying API."""
+
+ retry_delay: float
+ """Seconds to sleep between retry attempts."""
context_size: int
"""The model context size that will be used in the context compression."""
@@ -61,6 +64,7 @@ class ChatModelBase:
parameters: BaseModel,
stream: bool = True,
max_retries: int = 3,
+ retry_delay: float = 1.0,
context_size: int = 32768,
) -> None:
"""Initialize the chat model base.
@@ -75,7 +79,11 @@ class ChatModelBase:
stream (`bool`, defaults to `True`):
Whether to enable streaming output for the LLM.
max_retries (`int`, defaults to `3`):
- The maximum number of retries for API calls.
+ The maximum number of retries for API calls. Only exceptions
+ listed in ``_get_retryable_exceptions()`` count against this
+ budget; other exceptions are raised immediately.
+ retry_delay (`float`, defaults to `1.0`):
+ Seconds to sleep between retry attempts.
context_size (`int`, defaults to `32768`):
The model context size used for context compression.
"""
@@ -84,8 +92,20 @@ class ChatModelBase:
self.parameters = parameters
self.stream = stream
self.max_retries = max_retries
+ self.retry_delay = retry_delay
self.context_size = context_size
+ @classmethod
+ def _get_retryable_exceptions(cls) -> tuple[Type[Exception], ...]:
+ """Return the exception types that should trigger a retry.
+
+ Defaults to an empty tuple (no retries). Subclasses can override to
+ declare provider-specific retryable exceptions. SDK exception types
+ should be imported lazily inside the override so the SDK stays an
+ optional dependency.
+ """
+ return ()
+
@classmethod
def list_models(
cls,
@@ -141,11 +161,11 @@ class ChatModelBase:
tool_choice: ToolChoice | None = None,
**kwargs: Any,
) -> ChatResponse | AsyncGenerator[ChatResponse, None]:
- """Call the model with retry and fallback logic.
+ """Call the model with retry logic.
- Formats messages using formatter if available, then attempts
- to call the model up to max_retries + 1 times. If all attempts
- fail and a fallback model is configured, retries with that model.
+ Attempts to call the model up to ``max_retries + 1`` times. Only
+ exceptions listed in ``_get_retryable_exceptions()`` count against
+ this budget; other exceptions are raised immediately.
Args:
messages (`list[Msg]`):
@@ -158,8 +178,8 @@ class ChatModelBase:
Additional keyword arguments passed to the underlying API.
"""
+ retryable = tuple(self._get_retryable_exceptions())
last_error: Exception | None = None
-
for attempt in range(self.max_retries + 1):
try:
return await self._call_api(
@@ -170,26 +190,27 @@ class ChatModelBase:
**kwargs,
)
except Exception as e:
+ if not isinstance(e, retryable):
+ raise
last_error = e
if attempt < self.max_retries:
logger.warning(
- "Attempt %d failed for model %s: %s. Retrying...",
+ "Attempt %d failed for model %s: %s. "
+ "Retrying in %.1fs...",
attempt + 1,
self.model,
str(e),
+ self.retry_delay,
)
- # Sleep before retry
- await asyncio.sleep(0.1)
+ await asyncio.sleep(self.retry_delay)
else:
logger.warning(
"All %d attempt(s) failed for model %s.",
self.max_retries + 1,
self.model,
)
-
if last_error is not None:
raise last_error
-
raise RuntimeError(
f"Failed to call model {self.model} after "
f"{self.max_retries + 1} retries.",
@@ -357,8 +378,8 @@ class ChatModelBase:
) -> StructuredResponse:
"""Generate required structured output by the given model.
- Note this function also shares the fallback model and max retries
- settings with the `__call__` method.
+ Shares the same retry settings (``max_retries``, ``retry_delay``, and
+ ``_get_retryable_exceptions()``) as the ``__call__`` method.
Args:
messages (`list[Msg]`):
@@ -377,8 +398,8 @@ class ChatModelBase:
"`generate_structured_output` method.",
)
+ retryable = tuple(self._get_retryable_exceptions())
last_error: Exception | None = None
-
for attempt in range(self.max_retries + 1):
try:
return await self._call_api_with_structured_output(
@@ -388,24 +409,27 @@ class ChatModelBase:
**kwargs,
)
except Exception as e:
+ if not isinstance(e, retryable):
+ raise
last_error = e
if attempt < self.max_retries:
logger.warning(
- "Attempt %d failed for model %s: %s. Retrying...",
+ "Attempt %d failed for model %s: %s. "
+ "Retrying in %.1fs...",
attempt + 1,
self.model,
str(e),
+ self.retry_delay,
)
+ await asyncio.sleep(self.retry_delay)
else:
logger.warning(
"All %d attempt(s) failed for model %s.",
self.max_retries + 1,
self.model,
)
-
if last_error is not None:
raise last_error
-
raise RuntimeError(
f"Failed to generate structured output after "
f"{self.max_retries + 1} retries.",
diff --git a/src/agentscope/model/_dashscope/_model.py b/src/agentscope/model/_dashscope/_model.py
index 2ed3316f4..e97c5a390 100644
--- a/src/agentscope/model/_dashscope/_model.py
+++ b/src/agentscope/model/_dashscope/_model.py
@@ -102,6 +102,7 @@ class DashScopeChatModel(ChatModelBase):
parameters: "DashScopeChatModel.Parameters | None" = None,
stream: bool = True,
max_retries: int = 3,
+ retry_delay: float = 1.0,
context_size: int = 131072,
formatter: FormatterBase | None = None,
client_kwargs: dict[str, Any] | None = None,
@@ -121,6 +122,8 @@ class DashScopeChatModel(ChatModelBase):
Whether to enable streaming output.
max_retries (`int`, defaults to `3`):
The maximum number of retries for the DashScope API.
+ retry_delay (`float`, defaults to `1.0`):
+ Seconds to sleep between retry attempts.
context_size (`int`, defaults to `131072`):
The model context size used for context compression.
formatter (`FormatterBase | None`, defaults to `None`):
@@ -137,11 +140,23 @@ class DashScopeChatModel(ChatModelBase):
parameters=parameters or self.Parameters(),
stream=stream,
max_retries=max_retries,
+ retry_delay=retry_delay,
context_size=context_size,
)
self.formatter = formatter or DashScopeChatFormatter()
self.client_kwargs = client_kwargs or {}
+ @classmethod
+ def _get_retryable_exceptions(cls) -> tuple[Type[Exception], ...]:
+ import openai
+
+ return (
+ openai.APIConnectionError,
+ openai.APITimeoutError,
+ openai.RateLimitError,
+ openai.InternalServerError,
+ )
+
async def _call_api(
self,
model_name: str,
diff --git a/src/agentscope/model/_deepseek/_model.py b/src/agentscope/model/_deepseek/_model.py
index f869f164f..56d5fbb34 100644
--- a/src/agentscope/model/_deepseek/_model.py
+++ b/src/agentscope/model/_deepseek/_model.py
@@ -82,6 +82,7 @@ class DeepSeekChatModel(ChatModelBase):
parameters: "DeepSeekChatModel.Parameters | None" = None,
stream: bool = True,
max_retries: int = 3,
+ retry_delay: float = 1.0,
context_size: int = 65536,
formatter: FormatterBase | None = None,
client_kwargs: dict[str, Any] | None = None,
@@ -101,6 +102,8 @@ class DeepSeekChatModel(ChatModelBase):
Whether to enable streaming output.
max_retries (`int`, defaults to `3`):
The maximum number of retries for the DeepSeek API.
+ retry_delay (`float`, defaults to `1.0`):
+ Seconds to sleep between retry attempts.
context_size (`int`, defaults to `65536`):
The model context size used for context compression.
formatter (`FormatterBase | None`, defaults to `None`):
@@ -117,11 +120,23 @@ class DeepSeekChatModel(ChatModelBase):
parameters=parameters or self.Parameters(),
stream=stream,
max_retries=max_retries,
+ retry_delay=retry_delay,
context_size=context_size,
)
self.formatter = formatter or DeepSeekChatFormatter()
self.client_kwargs = client_kwargs or {}
+ @classmethod
+ def _get_retryable_exceptions(cls) -> tuple[Type[Exception], ...]:
+ import openai
+
+ return (
+ openai.APIConnectionError,
+ openai.APITimeoutError,
+ openai.RateLimitError,
+ openai.InternalServerError,
+ )
+
async def _call_api(
self,
model_name: str,
diff --git a/src/agentscope/model/_gemini/_model.py b/src/agentscope/model/_gemini/_model.py
index 1790ce008..946e13841 100644
--- a/src/agentscope/model/_gemini/_model.py
+++ b/src/agentscope/model/_gemini/_model.py
@@ -5,7 +5,7 @@ import copy
import json
import uuid
from datetime import datetime
-from typing import Literal, Any, AsyncGenerator, TYPE_CHECKING, List
+from typing import Literal, Any, AsyncGenerator, TYPE_CHECKING, List, Type
from pydantic import BaseModel, Field
@@ -128,6 +128,7 @@ class GeminiChatModel(ChatModelBase):
parameters: "GeminiChatModel.Parameters | None" = None,
stream: bool = True,
max_retries: int = 3,
+ retry_delay: float = 1.0,
context_size: int = 1048576,
formatter: FormatterBase | None = None,
client_kwargs: dict[str, Any] | None = None,
@@ -147,6 +148,8 @@ class GeminiChatModel(ChatModelBase):
Whether to enable streaming output.
max_retries (`int`, defaults to `3`):
The maximum number of retries for the Gemini API.
+ retry_delay (`float`, defaults to `1.0`):
+ Seconds to sleep between retry attempts.
context_size (`int`, defaults to `1048576`):
The model context size used for context compression.
formatter (`FormatterBase | None`, defaults to `None`):
@@ -164,11 +167,23 @@ class GeminiChatModel(ChatModelBase):
parameters=parameters or self.Parameters(),
stream=stream,
max_retries=max_retries,
+ retry_delay=retry_delay,
context_size=context_size,
)
self.formatter = formatter or GeminiChatFormatter()
self.client_kwargs = client_kwargs or {}
+ @classmethod
+ def _get_retryable_exceptions(cls) -> tuple[Type[Exception], ...]:
+ from google.genai import errors
+
+ # APIError is the common parent of ClientError (4xx) and ServerError
+ # (5xx). The google-genai SDK does not expose a dedicated rate-limit
+ # subclass, and 429 surfaces as ClientError — so we accept the wider
+ # set to make sure 429s are retried, at the cost of also retrying
+ # rare 4xx like auth/bad-request a few times.
+ return (errors.APIError,)
+
async def _call_api(
self,
model_name: str,
diff --git a/src/agentscope/model/_moonshot/_model.py b/src/agentscope/model/_moonshot/_model.py
index c38d7fe66..e4dc3d2c7 100644
--- a/src/agentscope/model/_moonshot/_model.py
+++ b/src/agentscope/model/_moonshot/_model.py
@@ -71,6 +71,7 @@ class MoonshotChatModel(ChatModelBase):
parameters: "MoonshotChatModel.Parameters | None" = None,
stream: bool = True,
max_retries: int = 3,
+ retry_delay: float = 1.0,
context_size: int = 131072,
formatter: FormatterBase | None = None,
client_kwargs: dict[str, Any] | None = None,
@@ -90,6 +91,8 @@ class MoonshotChatModel(ChatModelBase):
Whether to enable streaming output.
max_retries (`int`, defaults to `3`):
The maximum number of retries for the API.
+ retry_delay (`float`, defaults to `1.0`):
+ Seconds to sleep between retry attempts.
context_size (`int`, defaults to `131072`):
The model context size used for context compression.
formatter (`FormatterBase | None`, defaults to `None`):
@@ -106,11 +109,23 @@ class MoonshotChatModel(ChatModelBase):
parameters=parameters or self.Parameters(),
stream=stream,
max_retries=max_retries,
+ retry_delay=retry_delay,
context_size=context_size,
)
self.formatter = formatter or MoonshotChatFormatter()
self.client_kwargs = client_kwargs or {}
+ @classmethod
+ def _get_retryable_exceptions(cls) -> tuple[Type[Exception], ...]:
+ import openai
+
+ return (
+ openai.APIConnectionError,
+ openai.APITimeoutError,
+ openai.RateLimitError,
+ openai.InternalServerError,
+ )
+
async def _call_api(
self,
model_name: str,
diff --git a/src/agentscope/model/_ollama/_model.py b/src/agentscope/model/_ollama/_model.py
index 93eec45e7..81b42e4da 100644
--- a/src/agentscope/model/_ollama/_model.py
+++ b/src/agentscope/model/_ollama/_model.py
@@ -3,7 +3,7 @@
import json
import uuid
from datetime import datetime
-from typing import Literal, Any, AsyncGenerator, TYPE_CHECKING, List
+from typing import Literal, Any, AsyncGenerator, TYPE_CHECKING, List, Type
from pydantic import BaseModel, Field
@@ -60,6 +60,7 @@ class OllamaChatModel(ChatModelBase):
parameters: "OllamaChatModel.Parameters | None" = None,
stream: bool = True,
max_retries: int = 3,
+ retry_delay: float = 1.0,
context_size: int = 32768,
formatter: FormatterBase | None = None,
client_kwargs: dict[str, Any] | None = None,
@@ -80,6 +81,8 @@ class OllamaChatModel(ChatModelBase):
Whether to enable streaming output.
max_retries (`int`, defaults to `3`):
The maximum number of retries for the Ollama API.
+ retry_delay (`float`, defaults to `1.0`):
+ Seconds to sleep between retry attempts.
context_size (`int`, defaults to `32768`):
The model context size used for context compression.
formatter (`FormatterBase | None`, defaults to `None`):
@@ -99,12 +102,26 @@ class OllamaChatModel(ChatModelBase):
parameters=parameters or self.Parameters(),
stream=stream,
max_retries=max_retries,
+ retry_delay=retry_delay,
context_size=context_size,
)
self.formatter = formatter or OllamaChatFormatter()
self.client_kwargs = client_kwargs or {}
+ @classmethod
+ def _get_retryable_exceptions(cls) -> tuple[Type[Exception], ...]:
+ import httpx
+
+ # Local service: retry transient transport-layer failures only.
+ # ollama.ResponseError wraps server-side errors regardless of cause
+ # (incl. 4xx like "model not found"), so we don't retry on it.
+ return (
+ httpx.ConnectError,
+ httpx.ReadTimeout,
+ httpx.RemoteProtocolError,
+ )
+
async def _call_api(
self,
model_name: str,
diff --git a/src/agentscope/model/_openai_chat/_model.py b/src/agentscope/model/_openai_chat/_model.py
index 9741b0465..d1e534ead 100644
--- a/src/agentscope/model/_openai_chat/_model.py
+++ b/src/agentscope/model/_openai_chat/_model.py
@@ -2,7 +2,7 @@
"""The OpenAI Chat Completions model implementation."""
from collections import OrderedDict
from datetime import datetime
-from typing import Literal, Any, AsyncGenerator, TYPE_CHECKING, List
+from typing import Literal, Any, AsyncGenerator, TYPE_CHECKING, List, Type
from pydantic import BaseModel, Field
@@ -97,6 +97,7 @@ class OpenAIChatModel(ChatModelBase):
parameters: "OpenAIChatModel.Parameters | None" = None,
stream: bool = True,
max_retries: int = 3,
+ retry_delay: float = 1.0,
context_size: int = 128000,
formatter: FormatterBase | None = None,
client_kwargs: dict[str, Any] | None = None,
@@ -116,6 +117,8 @@ class OpenAIChatModel(ChatModelBase):
Whether to enable streaming output.
max_retries (`int`, defaults to `3`):
The maximum number of retries for the OpenAI API.
+ retry_delay (`float`, defaults to `1.0`):
+ Seconds to sleep between retry attempts.
context_size (`int`, defaults to `128000`):
The model context size used for context compression.
formatter (`FormatterBase | None`, defaults to `None`):
@@ -132,11 +135,23 @@ class OpenAIChatModel(ChatModelBase):
parameters=parameters or self.Parameters(),
stream=stream,
max_retries=max_retries,
+ retry_delay=retry_delay,
context_size=context_size,
)
self.formatter = formatter or OpenAIChatFormatter()
self.client_kwargs = client_kwargs or {}
+ @classmethod
+ def _get_retryable_exceptions(cls) -> tuple[Type[Exception], ...]:
+ import openai
+
+ return (
+ openai.APIConnectionError,
+ openai.APITimeoutError,
+ openai.RateLimitError,
+ openai.InternalServerError,
+ )
+
async def _call_api(
self,
model_name: str,
diff --git a/src/agentscope/model/_openai_response/_model.py b/src/agentscope/model/_openai_response/_model.py
index e4aed9809..5b6a78efc 100644
--- a/src/agentscope/model/_openai_response/_model.py
+++ b/src/agentscope/model/_openai_response/_model.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""The OpenAI Responses API chat model implementation."""
from datetime import datetime
-from typing import Literal, Any, AsyncGenerator, List, TYPE_CHECKING
+from typing import Literal, Any, AsyncGenerator, List, TYPE_CHECKING, Type
from pydantic import BaseModel, Field
@@ -86,6 +86,7 @@ class OpenAIResponseModel(ChatModelBase):
parameters: "OpenAIResponseModel.Parameters | None" = None,
stream: bool = True,
max_retries: int = 3,
+ retry_delay: float = 1.0,
context_size: int = 200000,
formatter: FormatterBase | None = None,
client_kwargs: dict[str, Any] | None = None,
@@ -105,6 +106,8 @@ class OpenAIResponseModel(ChatModelBase):
Whether to enable streaming output.
max_retries (`int`, defaults to `3`):
The maximum number of retries for the OpenAI Responses API.
+ retry_delay (`float`, defaults to `1.0`):
+ Seconds to sleep between retry attempts.
context_size (`int`, defaults to `200000`):
The model context size used for context compression.
formatter (`FormatterBase | None`, defaults to `None`):
@@ -121,11 +124,23 @@ class OpenAIResponseModel(ChatModelBase):
parameters=parameters or self.Parameters(),
stream=stream,
max_retries=max_retries,
+ retry_delay=retry_delay,
context_size=context_size,
)
self.formatter = formatter or OpenAIResponseFormatter()
self.client_kwargs = client_kwargs or {}
+ @classmethod
+ def _get_retryable_exceptions(cls) -> tuple[Type[Exception], ...]:
+ import openai
+
+ return (
+ openai.APIConnectionError,
+ openai.APITimeoutError,
+ openai.RateLimitError,
+ openai.InternalServerError,
+ )
+
async def _call_api(
self,
model_name: str,
diff --git a/src/agentscope/model/_xai/_model.py b/src/agentscope/model/_xai/_model.py
index 9e360ae40..2e59c9be7 100644
--- a/src/agentscope/model/_xai/_model.py
+++ b/src/agentscope/model/_xai/_model.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""The xAI chat model implementation using the official xai_sdk."""
from datetime import datetime
-from typing import Any, AsyncGenerator, List, Literal, TYPE_CHECKING
+from typing import Any, AsyncGenerator, List, Literal, TYPE_CHECKING, Type
from pydantic import BaseModel, Field
@@ -96,6 +96,7 @@ class XAIChatModel(ChatModelBase):
parameters: "XAIChatModel.Parameters | None" = None,
stream: bool = True,
max_retries: int = 3,
+ retry_delay: float = 1.0,
context_size: int = 131072,
formatter: XAIChatFormatter | None = None,
client_kwargs: dict[str, Any] | None = None,
@@ -115,6 +116,8 @@ class XAIChatModel(ChatModelBase):
Whether to enable streaming output.
max_retries (`int`, defaults to `3`):
The maximum number of retries for the xAI API.
+ retry_delay (`float`, defaults to `1.0`):
+ Seconds to sleep between retry attempts.
context_size (`int`, defaults to `131072`):
The model context size used for context compression.
formatter (`XAIChatFormatter | None`, defaults to `None`):
@@ -134,11 +137,25 @@ class XAIChatModel(ChatModelBase):
parameters=parameters or self.Parameters(),
stream=stream,
max_retries=max_retries,
+ retry_delay=retry_delay,
context_size=context_size,
)
self.formatter = formatter or XAIChatFormatter()
self.client_kwargs = client_kwargs or {}
+ @classmethod
+ def _get_retryable_exceptions(cls) -> tuple[Type[Exception], ...]:
+ import grpc
+
+ # xai_sdk uses grpc.aio under the hood; transport/API failures surface
+ # as grpc.aio.AioRpcError (subclass of grpc.RpcError). We retry the
+ # whole class because the retry mechanism here only filters by type,
+ # not by status code — so 4xx-equivalents (UNAUTHENTICATED, INVALID
+ # _ARGUMENT) will also be retried a few times before failing, which
+ # we accept. Note xai_sdk additionally retries UNAVAILABLE 5 times at
+ # the gRPC layer with exponential backoff before raising to us.
+ return (grpc.RpcError,)
+
async def _call_api(
self,
model_name: str,