DSPy: The framework for programming—not prompting—language models
Python
added configurable backoff max time
Arnav Singhvif4d4b7368d02
diff --git a/dsp/modules/anthropic.py b/dsp/modules/anthropic.pyindex c343e2a2a..013988281 100644--- a/dsp/modules/anthropic.py+++ b/dsp/modules/anthropic.py@@ -3,6 +3,7 @@ import osfrom typing import Any, Optionalimport backoff+import dspyfrom dsp.modules.lm import LM@@ -90,7 +91,7 @@ class Claude(LM):@backoff.on_exception(backoff.expo,(anthropic_rate_limit),- max_time=1000,+ max_time=dspy.settings.backoff_time,max_tries=8,on_backoff=backoff_hdlr,giveup=giveup_hdlr,diff --git a/dsp/modules/aws_providers.py b/dsp/modules/aws_providers.pyindex 5a6138ead..e350c8898 100644--- a/dsp/modules/aws_providers.py+++ b/dsp/modules/aws_providers.py@@ -4,6 +4,7 @@ from abc import ABC, abstractmethodfrom typing import Any, Optionalimport backoff+import dspytry:import boto3@@ -80,7 +81,7 @@ class AWSProvider(ABC):@backoff.on_exception(backoff.expo,ERRORS,- max_time=1000,+ max_time=dspy.settings.backoff_time,max_tries=8,on_backoff=backoff_hdlr,giveup=giveup_hdlr,@@ -155,7 +156,7 @@ class Sagemaker(AWSProvider):@backoff.on_exception(backoff.expo,ERRORS,- max_time=1000,+ max_time=dspy.settings.backoff_time,max_tries=8,on_backoff=backoff_hdlr,giveup=giveup_hdlr,diff --git a/dsp/modules/azure_openai.py b/dsp/modules/azure_openai.pyindex bc4fa61ce..45ca89bfd 100644--- a/dsp/modules/azure_openai.py+++ b/dsp/modules/azure_openai.py@@ -8,6 +8,7 @@ import openaifrom dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_onfrom dsp.modules.lm import LM+import dspytry:OPENAI_LEGACY = int(openai.version.__version__[0]) == 0@@ -166,7 +167,7 @@ class AzureOpenAI(LM):@backoff.on_exception(backoff.expo,ERRORS,- max_time=1000,+ max_time=dspy.settings.backoff_time,on_backoff=backoff_hdlr,)def request(self, prompt: str, **kwargs):diff --git a/dsp/modules/cloudflare.py b/dsp/modules/cloudflare.pyindex 81c20f838..abb3e5179 100644--- a/dsp/modules/cloudflare.py+++ b/dsp/modules/cloudflare.py@@ -7,6 +7,7 @@ import requestsfrom pydantic import BaseModel, ValidationErrorfrom dsp.modules.lm import LM+import dspydef backoff_hdlr(details) -> None:@@ -69,7 +70,7 @@ class CloudflareAI(LM):@backoff.on_exception(backoff.expo,requests.exceptions.RequestException,- max_time=1000,+ max_time=dspy.settings.backoff_time,on_backoff=backoff_hdlr,on_giveup=giveup_hdlr,)diff --git a/dsp/modules/cohere.py b/dsp/modules/cohere.pyindex 401fe9546..67882b027 100644--- a/dsp/modules/cohere.py+++ b/dsp/modules/cohere.py@@ -3,6 +3,7 @@ from typing import Any, Optionalimport backofffrom dsp.modules.lm import LM+import dspytry:import cohere@@ -101,7 +102,7 @@ class Cohere(LM):@backoff.on_exception(backoff.expo,(cohere_api_error),- max_time=1000,+ max_time=dspy.settings.backoff_time,on_backoff=backoff_hdlr,giveup=giveup_hdlr,)diff --git a/dsp/modules/google.py b/dsp/modules/google.pyindex 5d97534ea..07e9922e7 100644--- a/dsp/modules/google.py+++ b/dsp/modules/google.py@@ -5,6 +5,7 @@ from typing import Any, Optionalimport backofffrom dsp.modules.lm import LM+import dspytry:import google.generativeai as genai@@ -133,7 +134,7 @@ class Google(LM):@backoff.on_exception(backoff.expo,(google_api_error),- max_time=1000,+ max_time=dspy.settings.backoff_time,max_tries=8,on_backoff=backoff_hdlr,giveup=giveup_hdlr,diff --git a/dsp/modules/googlevertexai.py b/dsp/modules/googlevertexai.pyindex 118480040..5db14ebe3 100644--- a/dsp/modules/googlevertexai.py+++ b/dsp/modules/googlevertexai.py@@ -6,6 +6,7 @@ import backofffrom pydantic_core import PydanticCustomErrorfrom dsp.modules.lm import LM+import dspytry:import vertexai # type: ignore[import-untyped]@@ -184,7 +185,7 @@ class GoogleVertexAI(LM):@backoff.on_exception(backoff.expo,(Exception),- max_time=1000,+ max_time=dspy.settings.backoff_time,on_backoff=backoff_hdlr,giveup=giveup_hdlr,)diff --git a/dsp/modules/gpt3.py b/dsp/modules/gpt3.pyindex 54c0af012..c779e4359 100644--- a/dsp/modules/gpt3.py+++ b/dsp/modules/gpt3.py@@ -8,6 +8,7 @@ import openaifrom dsp.modules.cache_utils import CacheMemory, NotebookCacheMemory, cache_turn_onfrom dsp.modules.lm import LM+import dspytry:OPENAI_LEGACY = int(openai.version.__version__[0]) == 0@@ -133,11 +134,11 @@ class GPT3(LM):@backoff.on_exception(backoff.expo,ERRORS,- max_time=1000,+ max_time=dspy.settings.backoff_time,on_backoff=backoff_hdlr,)def request(self, prompt: str, **kwargs):- """Handles retreival of GPT-3 completions whilst handling rate limiting and caching."""+ """Handles retrieval of GPT-3 completions whilst handling rate limiting and caching."""if "model_type" in kwargs:del kwargs["model_type"]diff --git a/dsp/modules/groq_client.py b/dsp/modules/groq_client.pyindex 6650897ae..c62e3a3cd 100644--- a/dsp/modules/groq_client.py+++ b/dsp/modules/groq_client.py@@ -13,6 +13,7 @@ except ImportError:from dsp.modules.lm import LM+import dspydef backoff_hdlr(details):@@ -91,7 +92,7 @@ class GroqLM(LM):@backoff.on_exception(backoff.expo,groq_api_error,- max_time=1000,+ max_time=dspy.settings.backoff_time,on_backoff=backoff_hdlr,)def request(self, prompt: str, **kwargs):diff --git a/dsp/modules/hf_client.py b/dsp/modules/hf_client.pyindex 368f96c2b..f11a613e9 100644--- a/dsp/modules/hf_client.py+++ b/dsp/modules/hf_client.py@@ -11,6 +11,7 @@ import requestsfrom dsp.modules.cache_utils import CacheMemory, NotebookCacheMemoryfrom dsp.modules.hf import HFModel, openai_to_hf+import dspyERRORS = (Exception)@@ -340,7 +341,7 @@ class Together(HFModel):@backoff.on_exception(backoff.expo,ERRORS,- max_time=1000,+ max_time=dspy.settings.backoff_time,on_backoff=backoff_hdlr,)def _generate(self, prompt, use_chat_api=False, **kwargs):diff --git a/dsp/modules/mistral.py b/dsp/modules/mistral.pyindex abe5d9a23..ecb79070e 100644--- a/dsp/modules/mistral.py+++ b/dsp/modules/mistral.py@@ -3,6 +3,7 @@ from typing import Any, Optionalimport backofffrom dsp.modules.lm import LM+import dspytry:import mistralai@@ -99,7 +100,7 @@ class Mistral(LM):@backoff.on_exception(backoff.expo,(mistralai_api_error),- max_time=1000,+ max_time=dspy.settings.backoff_time,on_backoff=backoff_hdlr,giveup=giveup_hdlr,)diff --git a/dsp/modules/premai.py b/dsp/modules/premai.pyindex 80345824a..166fc72d5 100644--- a/dsp/modules/premai.py+++ b/dsp/modules/premai.py@@ -5,6 +5,7 @@ from typing import Any, Optionalimport backofffrom dsp.modules.lm import LM+import dspytry:import premai@@ -179,7 +180,7 @@ class PremAI(LM):@backoff.on_exception(backoff.expo,(premai_api_error),- max_time=1000,+ max_time=dspy.settings.backoff_time,on_backoff=backoff_hdlr,giveup=giveup_hdlr,)diff --git a/dsp/modules/snowflake.py b/dsp/modules/snowflake.pyindex 09bffd787..ad308abc0 100644--- a/dsp/modules/snowflake.py+++ b/dsp/modules/snowflake.py@@ -6,6 +6,7 @@ import backofffrom pydantic_core import PydanticCustomErrorfrom dsp.modules.lm import LM+import dspytry:from snowflake.snowpark import Session@@ -146,7 +147,7 @@ class Snowflake(LM):@backoff.on_exception(backoff.expo,(Exception),- max_time=1000,+ max_time=dspy.settings.backoff_time,on_backoff=backoff_hdlr,giveup=giveup_hdlr,)diff --git a/dsp/utils/settings.py b/dsp/utils/settings.pyindex 37cab1264..e7fe58cbd 100644--- a/dsp/utils/settings.py+++ b/dsp/utils/settings.py@@ -42,6 +42,7 @@ class Settings:suggest_failures=0,langchain_history=[],experimental=False,+ backoff_time = 10)cls._instance.__append(config)diff --git a/dspy/retrieve/chromadb_rm.py b/dspy/retrieve/chromadb_rm.pyindex aa8589d75..97a9bba8d 100644--- a/dspy/retrieve/chromadb_rm.py+++ b/dspy/retrieve/chromadb_rm.py@@ -124,7 +124,7 @@ class ChromadbRM(dspy.Retrieve):@backoff.on_exception(backoff.expo,ERRORS,- max_time=15,+ max_time=dspy.settings.backoff_time,)def _get_embeddings(self, queries: List[str]) -> List[List[float]]:"""Return query vector after creating embedding using OpenAIdiff --git a/dspy/retrieve/mongodb_atlas_rm.py b/dspy/retrieve/mongodb_atlas_rm.pyindex 3cafb05f6..8b2c76c54 100644--- a/dspy/retrieve/mongodb_atlas_rm.py+++ b/dspy/retrieve/mongodb_atlas_rm.py@@ -61,7 +61,7 @@ class Embedder:RateLimitError,UnprocessableEntityError,),- max_time=15,+ max_time=dspy.settings.backoff_time,)def __call__(self, queries) -> Any:embedding = self.client.embeddings.create(input=queries, model=self.model)diff --git a/dspy/retrieve/neo4j_rm.py b/dspy/retrieve/neo4j_rm.pyindex bb5ad2523..d8c071277 100644--- a/dspy/retrieve/neo4j_rm.py+++ b/dspy/retrieve/neo4j_rm.py@@ -42,7 +42,7 @@ class Embedder:RateLimitError,UnprocessableEntityError,),- max_time=15,+ max_time=dspy.settings.backoff_time,)def __call__(self, queries) -> Any:embedding = self.client.embeddings.create(input=queries, model=self.model)diff --git a/dspy/retrieve/pinecone_rm.py b/dspy/retrieve/pinecone_rm.pyindex 0328bc6ba..5d527f8dd 100644--- a/dspy/retrieve/pinecone_rm.py+++ b/dspy/retrieve/pinecone_rm.py@@ -178,7 +178,7 @@ class PineconeRM(dspy.Retrieve):@backoff.on_exception(backoff.expo,ERRORS,- max_time=15,+ max_time=dspy.settings.backoff_time,)def _get_embeddings(self,