DSPy: The framework for programming—not prompting—language models
Python
feat: separate the in memory cache from litellm cache (#7459)
Hung Phan6307ce41dd22
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex e4e5bbcee..44d0ae065 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -39,6 +39,7 @@ class LM(BaseLM):temperature: float = 0.0,max_tokens: int = 1000,cache: bool = True,+ cache_in_memory: bool = True,callbacks: Optional[List[BaseCallback]] = None,num_retries: int = 8,provider=None,@@ -57,6 +58,7 @@ class LM(BaseLM):max_tokens: The maximum number of tokens to generate per response.cache: Whether to cache the model responses for reuse to improve performanceand reduce costs.+ cache_in_memory: To enable additional caching with LRU in memory.callbacks: A list of callback functions to run before and after each request.num_retries: The number of times to retry a request if it fails transiently due tonetwork error, rate limiting, etc. Requests are retried with exponential@@ -69,6 +71,7 @@ class LM(BaseLM):self.model = modelself.model_type = model_typeself.cache = cache+ self.cache_in_memory = cache_in_memoryself.provider = provider or self.infer_provider()self.callbacks = callbacks or []self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)@@ -89,19 +92,29 @@ class LM(BaseLM):def __call__(self, prompt=None, messages=None, **kwargs):# Build the request.cache = kwargs.pop("cache", self.cache)+ # disable cache will also disable in memory cache+ cache_in_memory = cache and kwargs.pop("cache_in_memory", self.cache_in_memory)messages = messages or [{"role": "user", "content": prompt}]kwargs = {**self.kwargs, **kwargs}# Make the request and handle LRU & disk caching.- if self.model_type == "chat":- completion = cached_litellm_completion if cache else litellm_completion+ if cache_in_memory:+ completion = cached_litellm_completion if self.model_type == "chat" else cached_litellm_text_completion++ response = completion(+ request=dict(model=self.model, messages=messages, **kwargs),+ num_retries=self.num_retries,+ )else:- completion = cached_litellm_text_completion if cache else litellm_text_completion+ completion = litellm_completion if self.model_type == "chat" else litellm_text_completion++ response = completion(+ request=dict(model=self.model, messages=messages, **kwargs),+ num_retries=self.num_retries,+ # only leverage LiteLLM cache in this case+ cache={"no-cache": not cache, "no-store": not cache},+ )- response = completion(- request=dict(model=self.model, messages=messages, **kwargs),- num_retries=self.num_retries,- )if kwargs.get("logprobs"):outputs = [{diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.pyindex 77a942727..720b23afe 100644--- a/tests/clients/test_lm.py+++ b/tests/clients/test_lm.py@@ -29,6 +29,30 @@ def test_chat_lms_can_be_queried(litellm_test_server):assert azure_openai_lm("azure openai query") == expected_response+ ("cache", "cache_in_memory"),+ [+ (True, True),+ (True, False),+ (False, True),+ (False, False),+ ],+)+def test_chat_lms_cache(litellm_test_server, cache, cache_in_memory):+ api_base, _ = litellm_test_server+ expected_response = ["Hi!"]++ openai_lm = dspy.LM(+ model="openai/dspy-test-model",+ api_base=api_base,+ api_key="fakekey",+ model_type="chat",+ cache=cache,+ cache_in_memory=cache_in_memory,+ )+ assert openai_lm("openai query") == expected_response++def test_text_lms_can_be_queried(litellm_test_server):api_base, _ = litellm_test_serverexpected_response = ["Hi!"]