Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

fix(llm): retry throttled provider calls (#7677)

Vidit Ostwal4ed2abc7bbf5
diff --git a/lib/crewai/src/crewai/llm.py b/lib/crewai/src/crewai/llm.py
index 678055e89..c352417b4 100644
--- a/lib/crewai/src/crewai/llm.py
+++ b/lib/crewai/src/crewai/llm.py
@@ -1149,7 +1149,7 @@ class LLM(BaseLLM):
raise
except Exception as e:
error_msg = str(e)
- if LLMContextLengthExceededError._is_context_limit_error(error_msg):
+ if LLMContextLengthExceededError._is_context_length_exceeded_error(e):
raise LLMContextLengthExceededError(error_msg) from e
logging.error(f"Error in streaming response: {e!s}")
@@ -1346,7 +1346,7 @@ class LLM(BaseLLM):
raise
except Exception as e:
error_msg = str(e)
- if LLMContextLengthExceededError._is_context_limit_error(error_msg):
+ if LLMContextLengthExceededError._is_context_length_exceeded_error(e):
raise LLMContextLengthExceededError(error_msg) from e
raise
@@ -1501,7 +1501,7 @@ class LLM(BaseLLM):
raise
except Exception as e:
error_msg = str(e)
- if LLMContextLengthExceededError._is_context_limit_error(error_msg):
+ if LLMContextLengthExceededError._is_context_length_exceeded_error(e):
raise LLMContextLengthExceededError(error_msg) from e
raise
@@ -1773,7 +1773,7 @@ class LLM(BaseLLM):
raise
except Exception as e:
error_msg = str(e)
- if LLMContextLengthExceededError._is_context_limit_error(error_msg):
+ if LLMContextLengthExceededError._is_context_length_exceeded_error(e):
raise LLMContextLengthExceededError(error_msg) from e
if chunk_count == 0:
diff --git a/lib/crewai/src/crewai/llms/base_llm.py b/lib/crewai/src/crewai/llms/base_llm.py
index 9bd964a31..cdc66962d 100644
--- a/lib/crewai/src/crewai/llms/base_llm.py
+++ b/lib/crewai/src/crewai/llms/base_llm.py
@@ -11,6 +11,7 @@ from collections.abc import Generator
from contextlib import contextmanager
import contextvars
from datetime import datetime
+from functools import wraps
import json
import logging
import re
@@ -42,6 +43,7 @@ from crewai.events.types.tool_usage_events import (
ToolUsageFinishedEvent,
ToolUsageStartedEvent,
)
+from crewai.llms.retry import arun_with_rate_limit_retry, run_with_rate_limit_retry
from crewai.types.streaming import StreamSession
from crewai.types.usage_metrics import UsageMetrics
from crewai.utilities.pydantic_schema_utils import serialize_model_class
@@ -177,6 +179,47 @@ class BaseLLM(BaseModel, ABC):
model_config = ConfigDict(arbitrary_types_allowed=True, populate_by_name=True)
+ def __init_subclass__(cls, **kwargs: Any) -> None:
+ """Wrap concrete client call methods with the shared retry policy."""
+ super().__init_subclass__(**kwargs)
+ cls._wrap_call_method("call")
+ cls._wrap_call_method("acall")
+
+ @classmethod
+ def _wrap_call_method(cls, method_name: str) -> None:
+ """Install one retry wrapper around a subclass-defined public call method."""
+ method = cls.__dict__.get(method_name)
+ if method is None or getattr(method, "_crewai_rate_limit_wrapped", False):
+ return
+
+ if method_name == "call":
+
+ @wraps(method)
+ def wrapped_call(instance: BaseLLM, *args: Any, **kwargs: Any) -> Any:
+ return run_with_rate_limit_retry(
+ lambda: method(instance, *args, **kwargs)
+ )
+
+ wrapped_call._crewai_rate_limit_wrapped = True # type: ignore[attr-defined]
+ setattr(cls, method_name, wrapped_call)
+ return
+
+ if method_name == "acall":
+
+ @wraps(method)
+ async def wrapped_acall(
+ instance: BaseLLM, *args: Any, **kwargs: Any
+ ) -> Any:
+ return await arun_with_rate_limit_retry(
+ lambda: method(instance, *args, **kwargs)
+ )
+
+ wrapped_acall._crewai_rate_limit_wrapped = True # type: ignore[attr-defined]
+ setattr(cls, method_name, wrapped_acall)
+ return
+
+ raise ValueError(f"Unsupported LLM call method: {method_name}")
+
llm_type: str = "base"
model: str
temperature: float | None = None
diff --git a/lib/crewai/src/crewai/llms/retry.py b/lib/crewai/src/crewai/llms/retry.py
new file mode 100644
index 000000000..66820c61b
--- /dev/null
+++ b/lib/crewai/src/crewai/llms/retry.py
@@ -0,0 +1,225 @@
+"""Shared primitives for retrying transient LLM rate limits."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable, Iterator
+import contextvars
+import random
+import time
+from typing import Any, Final, TypeVar, cast
+
+
+_RETRYABLE_ERROR_CODES: Final[frozenset[str]] = frozenset(
+ {
+ "429",
+ "ratelimiterror",
+ "ratelimitexceeded",
+ "resourceexhausted",
+ "throttlingexception",
+ "toomanyrequests",
+ }
+)
+_NON_RETRYABLE_ERROR_CODES: Final[frozenset[str]] = frozenset(
+ {
+ "servicequotaexceededexception",
+ }
+)
+_RETRYABLE_MESSAGE_MARKERS: Final[tuple[str, ...]] = (
+ "rate limit",
+ "rate-limit",
+ "too many requests",
+ "throttled",
+ "resource exhausted",
+)
+_LLM_RATE_LIMIT_MAX_ATTEMPTS: Final = 3
+_LLM_RATE_LIMIT_INITIAL_DELAY_SECONDS: Final = 1.0
+_LLM_RATE_LIMIT_MAX_DELAY_SECONDS: Final = 8.0
+_LLM_RATE_LIMIT_JITTER_RATIO: Final = 0.2
+_T = TypeVar("_T")
+_active_llm_rate_limit_retry: contextvars.ContextVar[bool] = contextvars.ContextVar(
+ "_active_llm_rate_limit_retry", default=False
+)
+
+
+class _ThrottlingErrorClassifier:
+ """Classify provider exceptions without exposing provider SDK details."""
+
+ @classmethod
+ def is_throttling_error(cls, error: BaseException) -> bool:
+ """Return whether an error chain represents a transient provider throttle."""
+ error_chain = tuple(cls.iter_error_chain(error))
+ error_codes = tuple(cls.error_code(candidate) for candidate in error_chain)
+ if any(code in _NON_RETRYABLE_ERROR_CODES for code in error_codes):
+ return False
+
+ for candidate, error_code in zip(error_chain, error_codes, strict=True):
+ if (
+ cls.status_code(candidate) == 429
+ or error_code in _RETRYABLE_ERROR_CODES
+ ):
+ return True
+
+ return any(
+ marker in str(candidate).lower()
+ for candidate in error_chain
+ for marker in _RETRYABLE_MESSAGE_MARKERS
+ )
+
+ @staticmethod
+ def iter_error_chain(error: BaseException) -> Iterator[BaseException]:
+ """Yield an exception and its explicit or implicit causes once each."""
+ seen: set[int] = set()
+ current: BaseException | None = error
+ while current is not None and id(current) not in seen:
+ seen.add(id(current))
+ yield current
+ current = current.__cause__ or current.__context__
+
+ @staticmethod
+ def error_code(error: BaseException) -> str:
+ """Extract and normalize provider error codes across common SDK shapes."""
+ code: Any = getattr(error, "code", None)
+ response = getattr(error, "response", None)
+ if isinstance(response, dict):
+ response_error = response.get("Error") or response.get("error") or {}
+ if isinstance(response_error, dict):
+ code = response_error.get("Code") or response_error.get("code") or code
+ return str(code or error.__class__.__name__).replace("_", "").lower()
+
+ @staticmethod
+ def status_code(error: BaseException) -> int | None:
+ """Extract an HTTP status code when an SDK exposes one."""
+ status_code = getattr(error, "status_code", None)
+ response = getattr(error, "response", None)
+ if status_code is None:
+ status_code = getattr(response, "status_code", None)
+ return status_code if isinstance(status_code, int) else None
+
+
+def get_retry_delay_seconds(
+ retry_number: int,
+ *,
+ retry_after_seconds: float | None = None,
+ random_value: Callable[[], float] = random.random,
+) -> float:
+ """Calculate a jittered backoff delay for a one-based retry number.
+
+ A provider-provided retry delay takes precedence over locally calculated
+ backoff. ``random_value`` is injectable to make callers' tests deterministic.
+ """
+ if retry_number < 1:
+ raise ValueError("retry_number must be at least 1")
+ if retry_after_seconds is not None:
+ return max(0.0, retry_after_seconds)
+
+ delay = min(
+ _LLM_RATE_LIMIT_INITIAL_DELAY_SECONDS * (2 ** (retry_number - 1)),
+ _LLM_RATE_LIMIT_MAX_DELAY_SECONDS,
+ )
+ jitter = (float(random_value()) * 2 - 1) * _LLM_RATE_LIMIT_JITTER_RATIO
+ return float(delay * (1 + jitter))
+
+
+def run_with_rate_limit_retry(
+ operation: Callable[[], _T],
+ *,
+ sleep: Callable[[float], None] = time.sleep,
+) -> _T:
+ """Run an operation with retries for transient rate-limit errors only."""
+
+ def attempt() -> tuple[bool, _T | None, Exception | None]:
+ """Run one operation while retaining its retryable error."""
+ try:
+ return True, operation(), None
+ except Exception as error:
+ return False, None, error
+
+ if _active_llm_rate_limit_retry.get():
+ return operation()
+
+ token = _active_llm_rate_limit_retry.set(True)
+ try:
+ for attempt_number in range(1, _LLM_RATE_LIMIT_MAX_ATTEMPTS + 1):
+ succeeded, result, error = attempt()
+ if succeeded:
+ return cast(_T, result)
+ if error is None:
+ raise RuntimeError("failed retry attempt did not provide an error")
+ if (
+ attempt_number == _LLM_RATE_LIMIT_MAX_ATTEMPTS
+ or not _ThrottlingErrorClassifier.is_throttling_error(error)
+ ):
+ raise error
+ sleep(
+ get_retry_delay_seconds(
+ retry_number=attempt_number,
+ retry_after_seconds=_retry_after_seconds(error),
+ )
+ )
+ finally:
+ _active_llm_rate_limit_retry.reset(token)
+
+ raise RuntimeError("rate-limit retry loop completed without a result")
+
+
+async def arun_with_rate_limit_retry(
+ operation: Callable[[], Awaitable[_T]],
+ *,
+ sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+) -> _T:
+ """Asynchronously run an operation with retries for transient rate limits."""
+
+ async def attempt() -> tuple[bool, _T | None, Exception | None]:
+ """Run one operation while retaining its retryable error."""
+ try:
+ return True, await operation(), None
+ except Exception as error:
+ return False, None, error
+
+ if _active_llm_rate_limit_retry.get():
+ return await operation()
+
+ token = _active_llm_rate_limit_retry.set(True)
+ try:
+ for attempt_number in range(1, _LLM_RATE_LIMIT_MAX_ATTEMPTS + 1):
+ succeeded, result, error = await attempt()
+ if succeeded:
+ return cast(_T, result)
+ if error is None:
+ raise RuntimeError("failed retry attempt did not provide an error")
+ if (
+ attempt_number == _LLM_RATE_LIMIT_MAX_ATTEMPTS
+ or not _ThrottlingErrorClassifier.is_throttling_error(error)
+ ):
+ raise error
+ await sleep(
+ get_retry_delay_seconds(
+ retry_number=attempt_number,
+ retry_after_seconds=_retry_after_seconds(error),
+ )
+ )
+ finally:
+ _active_llm_rate_limit_retry.reset(token)
+
+ raise RuntimeError("rate-limit retry loop completed without a result")
+
+
+def _retry_after_seconds(error: BaseException) -> float | None:
+ """Read a numeric ``Retry-After`` hint from common SDK response shapes."""
+ for candidate in _ThrottlingErrorClassifier.iter_error_chain(error):
+ headers = getattr(candidate, "headers", None)
+ response = getattr(candidate, "response", None)
+ if headers is None and isinstance(response, dict):
+ metadata = response.get("ResponseMetadata") or {}
+ headers = metadata.get("HTTPHeaders") or response.get("headers")
+ if not isinstance(headers, dict):
+ continue
+ retry_after = headers.get("retry-after") or headers.get("Retry-After")
+ if retry_after is None:
+ continue
+ try:
+ return max(0.0, float(retry_after))
+ except (TypeError, ValueError):
+ continue
+ return None
diff --git a/lib/crewai/src/crewai/utilities/agent_utils.py b/lib/crewai/src/crewai/utilities/agent_utils.py
index 7e6eb3f2d..1678b0e56 100644
--- a/lib/crewai/src/crewai/utilities/agent_utils.py
+++ b/lib/crewai/src/crewai/utilities/agent_utils.py
@@ -795,9 +795,9 @@ def is_context_length_exceeded(exception: Exception) -> bool:
Returns:
bool: True if the exception is due to context length exceeding
"""
- return LLMContextLengthExceededError(str(exception))._is_context_limit_error(
+ return LLMContextLengthExceededError(
str(exception)
- )
+ )._is_context_length_exceeded_error(exception)
def handle_context_length(
diff --git a/lib/crewai/src/crewai/utilities/exceptions/context_window_exceeding_exception.py b/lib/crewai/src/crewai/utilities/exceptions/context_window_exceeding_exception.py
index 9e44ce6f4..aeb17c51a 100644
--- a/lib/crewai/src/crewai/utilities/exceptions/context_window_exceeding_exception.py
+++ b/lib/crewai/src/crewai/utilities/exceptions/context_window_exceeding_exception.py
@@ -1,5 +1,7 @@
from typing import Final
+from crewai.llms.retry import _ThrottlingErrorClassifier
+
CONTEXT_LIMIT_ERRORS: Final[list[str]] = [
"expected a string with maximum length",
@@ -30,15 +32,21 @@ class LLMContextLengthExceededError(Exception):
super().__init__(self._get_error_message(error_message))
@staticmethod
- def _is_context_limit_error(error_message: str) -> bool:
- """Check if the error message indicates a context length limit error.
+ def _is_context_length_exceeded_error(error: str | BaseException) -> bool:
+ """Check whether an error represents a context limit, not a throttle.
Args:
- error_message: The error message to check.
+ error: The provider exception or error message to check.
Returns:
True if the error message indicates a context length limit error, False otherwise.
"""
+ if isinstance(
+ error, BaseException
+ ) and _ThrottlingErrorClassifier.is_throttling_error(error):
+ return False
+
+ error_message = str(error)
return any(
phrase.lower() in error_message.lower() for phrase in CONTEXT_LIMIT_ERRORS
)
diff --git a/lib/crewai/tests/llms/bedrock/test_bedrock.py b/lib/crewai/tests/llms/bedrock/test_bedrock.py
index 8659393ee..99eccdd1e 100644
--- a/lib/crewai/tests/llms/bedrock/test_bedrock.py
+++ b/lib/crewai/tests/llms/bedrock/test_bedrock.py
@@ -1,7 +1,7 @@
import logging
import os
import threading
-from unittest.mock import patch, MagicMock
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from crewai.llm import LLM
@@ -770,12 +770,13 @@ def test_bedrock_handles_cohere_conversation_requirements():
assert "continue" in formatted_messages[-1]["content"][0]["text"].lower()
-def test_bedrock_client_error_handling():
+def test_bedrock_client_error_handling(monkeypatch):
"""
Test that Bedrock properly handles various AWS client errors
"""
from botocore.exceptions import ClientError
+ monkeypatch.setattr('crewai.llms.retry.time.sleep', lambda _: None)
llm = LLM(model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0")
with patch.object(llm._client, 'converse') as mock_converse:
@@ -804,6 +805,88 @@ def test_bedrock_client_error_handling():
llm.call("Hello")
assert "throttled" in str(exc_info.value).lower()
+ with patch.object(llm._client, 'converse') as mock_converse:
+ error_response = {
+ 'Error': {
+ 'Code': 'ServiceQuotaExceededException',
+ 'Message': 'Quota increase required',
+ }
+ }
+ mock_converse.side_effect = ClientError(error_response, 'converse')
+
+ with pytest.raises(RuntimeError) as exc_info:
+ llm.call("Hello")
+
+ assert "quota" in str(exc_info.value).lower()
+ assert mock_converse.call_count == 1
+
+
+def test_bedrock_throttling_retries_without_context_recovery(monkeypatch):
+ """Bedrock token throttles retry instead of being treated as context overflow."""
+ from botocore.exceptions import ClientError
+
+ monkeypatch.setattr('crewai.llms.retry.time.sleep', lambda _: None)
+ llm = LLM(model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0")
+ throttle_response = {
+ 'Error': {
+ 'Code': 'ThrottlingException',
+ 'Message': 'Too many tokens, please wait before trying again',
+ }
+ }
+ successful_response = {
+ 'output': {
+ 'message': {'role': 'assistant', 'content': [{'text': 'Recovered'}]}
+ },
+ 'usage': {'inputTokens': 1, 'outputTokens': 1, 'totalTokens': 2},
+ }
+
+ with patch.object(llm._client, 'converse') as mock_converse:
+ mock_converse.side_effect = [
+ ClientError(throttle_response, 'converse'),
+ successful_response,
+ ]
+
+ assert llm.call("Hello") == "Recovered"
+
+ assert mock_converse.call_count == 2
+
+
+async def test_bedrock_async_throttling_retries_without_context_recovery(monkeypatch):
+ """Async Bedrock token throttles use the same client-level retry behavior."""
+ from botocore.exceptions import ClientError
+
+ async def no_sleep(_: float) -> None:
+ return None
+
+ monkeypatch.setattr('crewai.llms.retry.asyncio.sleep', no_sleep)
+ monkeypatch.setattr(bedrock_completion, 'AIOBOTOCORE_AVAILABLE', True)
+ llm = LLM(model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0")
+ throttle_response = {
+ 'Error': {
+ 'Code': 'ThrottlingException',
+ 'Message': 'Too many tokens, please wait before trying again',
+ }
+ }
+ successful_response = {
+ 'output': {
+ 'message': {'role': 'assistant', 'content': [{'text': 'Recovered'}]}
+ },
+ 'usage': {'inputTokens': 1, 'outputTokens': 1, 'totalTokens': 2},
+ }
+ async_client = MagicMock()
+ async_client.converse = AsyncMock(
+ side_effect=[
+ ClientError(throttle_response, 'converse'),
+ successful_response,
+ ]
+ )
+
+ with patch.object(llm, '_ensure_async_client', return_value=async_client):
+ assert await llm.acall("Hello") == "Recovered"
+
+ assert async_client.converse.await_count == 2
+
def test_bedrock_stop_sequences_sync():
"""Test that stop and stop_sequences attributes stay synchronized."""
diff --git a/lib/crewai/tests/llms/test_retry.py b/lib/crewai/tests/llms/test_retry.py
new file mode 100644
index 000000000..9d76133c8
--- /dev/null
+++ b/lib/crewai/tests/llms/test_retry.py
@@ -0,0 +1,158 @@
+"""Tests for provider-neutral LLM retry behavior."""
+
+from typing import Any
+
+import pytest
+
+from crewai.llms.base_llm import BaseLLM
+from crewai.llms.retry import (
+ _ThrottlingErrorClassifier,
+ arun_with_rate_limit_retry,
+ get_retry_delay_seconds,
+ run_with_rate_limit_retry,
+)
+
+
+class _StructuredProviderError(Exception):
+ def __init__(self, code: str) -> None:
+ self.response = {"Error": {"Code": code}}
+ super().__init__(code)
+
+
+class _RetryingLLM(BaseLLM):
+ model: str = "test-model"
+ outcomes: list[Any]
+
+ def call(self, *args: Any, **kwargs: Any) -> str:
+ outcome = self.outcomes.pop(0)
+ if isinstance(outcome, BaseException):
+ raise outcome
+ return outcome
+
+ async def acall(self, *args: Any, **kwargs: Any) -> str:
+ outcome = self.outcomes.pop(0)
+ if isinstance(outcome, BaseException):
+ raise outcome
+ return outcome
+
+
+ "error",
+ [
+ type("Http429Error", (Exception,), {"status_code": 429})("busy"),
+ type("RateLimitError", (Exception,), {})("busy"),
+ _StructuredProviderError("TooManyRequests"),
+ RuntimeError("provider request was throttled"),
+ ],
+)
+def test_throttling_error_classifier_recognizes_transient_provider_throttles(
+ error: Exception,
+) -> None:
+ assert _ThrottlingErrorClassifier.is_throttling_error(error)
+
+
+def test_throttling_error_classifier_follows_exception_causes() -> None:
+ try:
+ raise _StructuredProviderError("TooManyRequests")
+ except _StructuredProviderError as cause:
+ wrapped = RuntimeError("provider request failed")
+ wrapped.__cause__ = cause
+
+ assert _ThrottlingErrorClassifier.is_throttling_error(wrapped)
+
+
+ "error",
+ [
+ _StructuredProviderError("InvalidRequest"),
+ RuntimeError("input exceeds the context window"),
+ ValueError("request validation failed"),
+ ],
+)
+def test_throttling_error_classifier_rejects_non_transient_errors(
+ error: Exception,
+) -> None:
+ assert not _ThrottlingErrorClassifier.is_throttling_error(error)
+
+
+def test_get_retry_delay_seconds_uses_exponential_backoff() -> None:
+ assert get_retry_delay_seconds(retry_number=1, random_value=lambda: 0.5) == 1
+ assert get_retry_delay_seconds(retry_number=2, random_value=lambda: 0.5) == 2
+ assert get_retry_delay_seconds(retry_number=3, random_value=lambda: 0.5) == 4
+ assert get_retry_delay_seconds(retry_number=4, random_value=lambda: 0.5) == 8
+
+
+def test_get_retry_delay_seconds_applies_jitter_and_honors_retry_after() -> None:
+ assert get_retry_delay_seconds(retry_number=1, random_value=lambda: 1) == 1.2
+ assert get_retry_delay_seconds(retry_number=1, retry_after_seconds=5) == 5
+
+
+def test_run_with_rate_limit_retry_retries_with_backoff() -> None:
+ outcomes: list[str | Exception] = [
+ RuntimeError("rate limit exceeded"),
+ "complete",
+ ]
+ delays: list[float] = []
+
+ result = run_with_rate_limit_retry(
+ lambda: _pop_outcome(outcomes), sleep=delays.append
+ )
+
+ assert result == "complete"
+ assert delays == [pytest.approx(1, abs=0.2)]
+
+
+async def test_arun_with_rate_limit_retry_retries_with_backoff() -> None:
+ outcomes: list[str | Exception] = [
+ RuntimeError("rate limit exceeded"),
+ "complete",
+ ]
+ delays: list[float] = []
+
+ async def record_delay(delay: float) -> None:
+ delays.append(delay)
+
+ async def operation() -> str:
+ return _pop_outcome(outcomes)
+
+ result = await arun_with_rate_limit_retry(operation, sleep=record_delay)
+
+ assert result == "complete"
+ assert delays == [pytest.approx(1, abs=0.2)]
+
+
+def test_base_llm_call_is_automatically_wrapped(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr("crewai.llms.retry.time.sleep", lambda _: None)
+ llm = _RetryingLLM(
+ model="test-model",
+ outcomes=[RuntimeError("rate limit exceeded"), "complete"]
+ )
+
+ assert llm.call("hello") == "complete"
+ assert llm.outcomes == []
+
+
+async def test_base_llm_acall_is_automatically_wrapped(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def no_sleep(_: float) -> None:
+ return None
+
+ monkeypatch.setattr("crewai.llms.retry.asyncio.sleep", no_sleep)
+ llm = _RetryingLLM(
+ model="test-model",
+ outcomes=[RuntimeError("rate limit exceeded"), "complete"]
+ )
+
+ assert await llm.acall("hello") == "complete"
+ assert llm.outcomes == []
+
+
+def _pop_outcome(outcomes: list[str | Exception]) -> str:
+ """Raise a scripted error or return a scripted successful result."""
+ outcome = outcomes.pop(0)
+ if isinstance(outcome, Exception):
+ raise outcome
+ return outcome
diff --git a/lib/crewai/tests/test_llm.py b/lib/crewai/tests/test_llm.py
index f90665cd3..870b3f9e5 100644
--- a/lib/crewai/tests/test_llm.py
+++ b/lib/crewai/tests/test_llm.py
@@ -13,6 +13,10 @@ from crewai.events.event_types import (
)
from crewai.llm import CONTEXT_WINDOW_USAGE_RATIO, DEFAULT_CONTEXT_WINDOW_SIZE, LLM
from crewai.llms.providers.anthropic.completion import AnthropicCompletion
+from crewai.utilities.agent_utils import is_context_length_exceeded
+from crewai.utilities.exceptions.context_window_exceeding_exception import (
+ LLMContextLengthExceededError,
+)
from crewai.utilities.token_counter_callback import TokenCalcHandler
from pydantic import BaseModel
import pytest
@@ -430,6 +434,7 @@ def test_context_window_exceeded_error_handling():
assert "context length exceeded" in str(excinfo.value).lower()
assert "8192 tokens" in str(excinfo.value)
+
llm = LLM(model="gpt-4", stream=True, is_litellm=True)
with patch("litellm.completion") as mock_completion:
mock_completion.side_effect = ContextWindowExceededError(
@@ -445,6 +450,24 @@ def test_context_window_exceeded_error_handling():
assert "8192 tokens" in str(excinfo.value)
+def test_rate_limits_are_not_treated_as_context_window_errors(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ error = RuntimeError("API throttled: too many tokens requested this minute")
+
+ assert not is_context_length_exceeded(error)
+ assert not LLMContextLengthExceededError._is_context_length_exceeded_error(error)
+
+ monkeypatch.setattr("crewai.llms.retry.time.sleep", lambda _: None)
+ llm = LLM(model="gpt-4o-mini", is_litellm=True)
+ with patch("litellm.completion", side_effect=error) as completion:
+ with pytest.raises(RuntimeError) as exc_info:
+ llm.call("Hello")
+
+ assert exc_info.value is error
+ assert completion.call_count == 3
+
+
@pytest.fixture
def anthropic_llm():
"""Fixture providing an Anthropic LLM instance."""