DSPy: The framework for programming—not prompting—language models
Python
LMs: retry with exponential backoff for a limited set of error codes (#1753)
Corey Zumarcadd6190dc4e
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex 567178432..6a55e4cfd 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -1,5 +1,4 @@import functools-from .base_lm import BaseLMimport loggingimport osimport uuid@@ -7,13 +6,15 @@ from concurrent.futures import ThreadPoolExecutorfrom datetime import datetimefrom typing import Any, Dict, List, Literal, Optional-import litellmimport ujson+from litellm import Router+from litellm.router import RetryPolicyfrom dspy.clients.finetune import FinetuneJob, TrainingMethodfrom dspy.clients.lm_finetune_utils import execute_finetune_job, get_provider_finetune_job_classfrom dspy.utils.callback import BaseCallback, with_callbacks+from .base_lm import BaseLMlogger = logging.getLogger(__name__)@@ -32,7 +33,7 @@ class LM(BaseLM):cache: bool = True,launch_kwargs: Optional[Dict[str, Any]] = None,callbacks: Optional[List[BaseCallback]] = None,- num_retries: int = 3,+ num_retries: int = 8,**kwargs,):"""@@ -174,13 +175,55 @@ def cached_litellm_completion(request, num_retries: int):def litellm_completion(request, num_retries: int, cache={"no-cache": True, "no-store": True}):kwargs = ujson.loads(request)- return litellm.completion(- num_retries=num_retries,+ router = _get_litellm_router(model=kwargs["model"], num_retries=num_retries)+ return router.completion(cache=cache,**kwargs,)[email protected]_cache(maxsize=None)+def _get_litellm_router(model: str, num_retries: int) -> Router:+ """+ Get a LiteLLM router for the given model with the specified number of retries+ for transient errors.++ Args:+ model: The name of the LiteLLM model to query (e.g. 'openai/gpt-4').+ num_retries: The number of times to retry a request if it fails transiently due to+ network error, rate limiting, etc. Requests are retried with exponential+ backoff.+ Returns:+ A LiteLLM router instance that can be used to query the given model.+ """+ retry_policy = RetryPolicy(+ TimeoutErrorRetries=num_retries,+ RateLimitErrorRetries=num_retries,+ InternalServerErrorRetries=num_retries,+ # We don't retry on errors that are unlikely to be transient+ # (e.g. bad request, invalid auth credentials)+ BadRequestErrorRetries=0,+ AuthenticationErrorRetries=0,+ ContentPolicyViolationErrorRetries=0,+ )++ return Router(+ # LiteLLM routers must specify a `model_list`, which maps model names passed+ # to `completions()` into actual LiteLLM model names. For our purposes, the+ # model name is the same as the LiteLLM model name, so we add a single+ # entry to the `model_list` that maps the model name to itself+ model_list=[+ {+ "model_name": model,+ "litellm_params": {+ "model": model,+ },+ }+ ],+ retry_policy=retry_policy,+ )++@functools.lru_cache(maxsize=None)def cached_litellm_text_completion(request, num_retries: int):return litellm_text_completion(@@ -197,6 +240,7 @@ def litellm_text_completion(request, num_retries: int, cache={"no-cache": True,# TODO: Not all the models are in the format of "provider/model"model = kwargs.pop("model").split("/", 1)provider, model = model[0] if len(model) > 1 else "openai", model[-1]+ text_completion_model_name = f"text-completion-openai/{model}"# Use the API key and base from the kwargs, or from the environment.api_key = kwargs.pop("api_key", None) or os.getenv(f"{provider}_API_KEY")@@ -205,12 +249,12 @@ def litellm_text_completion(request, num_retries: int, cache={"no-cache": True,# Build the prompt from the messages.prompt = "\n\n".join([x["content"] for x in kwargs.pop("messages")] + ["BEGIN RESPONSE:"])- return litellm.text_completion(+ router = _get_litellm_router(model=text_completion_model_name, num_retries=num_retries)+ return router.text_completion(cache=cache,- model=f"text-completion-openai/{model}",+ model=text_completion_model_name,api_key=api_key,api_base=api_base,prompt=prompt,- num_retries=num_retries,**kwargs,)diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.pyindex ef5d85a9c..61e5828ae 100644--- a/tests/clients/test_lm.py+++ b/tests/clients/test_lm.py@@ -1,25 +1,106 @@from unittest import mock-from dspy.clients.lm import LM+from litellm.router import RetryPolicy++from dspy.clients.lm import LM, _get_litellm_routerdef test_lm_chat_respects_max_retries():- lm = LM(model="openai/gpt4o", model_type="chat", max_retries=17)+ model_name = "openai/gpt4o"+ num_retries = 17+ temperature = 0.5+ max_tokens = 100+ prompt = "Hello, world!"++ lm = LM(+ model=model_name, model_type="chat", num_retries=num_retries, temperature=temperature, max_tokens=max_tokens+ )++ MockRouter = mock.MagicMock()+ mock_completion = mock.MagicMock()+ MockRouter.completion = mock_completion- with mock.patch("dspy.clients.lm.litellm.completion") as litellm_completion_api:- lm(messages=[{"content": "Hello, world!", "role": "user"}])+ with mock.patch("dspy.clients.lm.Router", return_value=MockRouter) as MockRouterConstructor:+ lm(prompt=prompt)- assert litellm_completion_api.call_count == 1- assert litellm_completion_api.call_args[1]["max_retries"] == 17- # assert litellm_completion_api.call_args[1]["retry_strategy"] == "exponential_backoff_retry"+ MockRouterConstructor.assert_called_once_with(+ model_list=[+ {+ "model_name": model_name,+ "litellm_params": {+ "model": model_name,+ },+ }+ ],+ retry_policy=RetryPolicy(+ TimeoutErrorRetries=num_retries,+ RateLimitErrorRetries=num_retries,+ InternalServerErrorRetries=num_retries,+ BadRequestErrorRetries=0,+ AuthenticationErrorRetries=0,+ ContentPolicyViolationErrorRetries=0,+ ),+ )+ mock_completion.assert_called_once_with(+ model=model_name,+ messages=[{"role": "user", "content": prompt}],+ temperature=temperature,+ max_tokens=max_tokens,+ cache=mock.ANY,+ )def test_lm_completions_respects_max_retries():- lm = LM(model="openai/gpt-3.5-turbo", model_type="completions", max_retries=17)+ model_name = "openai/gpt-3.5-turbo"+ expected_model = "text-completion-" + model_name+ num_retries = 17+ temperature = 0.5+ max_tokens = 100+ prompt = "Hello, world!"+ api_base = "http://test.com"+ api_key = "apikey"++ lm = LM(+ model=model_name,+ model_type="text",+ num_retries=num_retries,+ temperature=temperature,+ max_tokens=max_tokens,+ api_base=api_base,+ api_key=api_key,+ )++ MockRouter = mock.MagicMock()+ mock_text_completion = mock.MagicMock()+ MockRouter.text_completion = mock_text_completion- with mock.patch("dspy.clients.lm.litellm.text_completion") as litellm_completion_api:- lm(prompt="Hello, world!")+ with mock.patch("dspy.clients.lm.Router", return_value=MockRouter) as MockRouterConstructor:+ lm(prompt=prompt)- assert litellm_completion_api.call_count == 1- assert litellm_completion_api.call_args[1]["max_retries"] == 17- # assert litellm_completion_api.call_args[1]["retry_strategy"] == "exponential_backoff_retry"+ MockRouterConstructor.assert_called_once_with(+ model_list=[+ {+ "model_name": expected_model,+ "litellm_params": {+ "model": expected_model,+ },+ }+ ],+ retry_policy=RetryPolicy(+ TimeoutErrorRetries=num_retries,+ RateLimitErrorRetries=num_retries,+ InternalServerErrorRetries=num_retries,+ BadRequestErrorRetries=0,+ AuthenticationErrorRetries=0,+ ContentPolicyViolationErrorRetries=0,+ ),+ )+ mock_text_completion.assert_called_once_with(+ model=expected_model,+ prompt=prompt + "\n\nBEGIN RESPONSE:",+ temperature=temperature,+ max_tokens=max_tokens,+ api_key=api_key,+ api_base=api_base,+ cache=mock.ANY,+ )