Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feat: separate the in memory cache from litellm cache (#7459)

Hung Phan6307ce41dd22
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index e4e5bbcee..44d0ae065 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -39,6 +39,7 @@ class LM(BaseLM):
temperature: float = 0.0,
max_tokens: int = 1000,
cache: bool = True,
+ cache_in_memory: bool = True,
callbacks: Optional[List[BaseCallback]] = None,
num_retries: int = 8,
provider=None,
@@ -57,6 +58,7 @@ class LM(BaseLM):
max_tokens: The maximum number of tokens to generate per response.
cache: Whether to cache the model responses for reuse to improve performance
and reduce costs.
+ cache_in_memory: To enable additional caching with LRU in memory.
callbacks: A list of callback functions to run before and after each request.
num_retries: The number of times to retry a request if it fails transiently due to
network error, rate limiting, etc. Requests are retried with exponential
@@ -69,6 +71,7 @@ class LM(BaseLM):
self.model = model
self.model_type = model_type
self.cache = cache
+ self.cache_in_memory = cache_in_memory
self.provider = provider or self.infer_provider()
self.callbacks = callbacks or []
self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)
@@ -89,19 +92,29 @@ class LM(BaseLM):
def __call__(self, prompt=None, messages=None, **kwargs):
# Build the request.
cache = kwargs.pop("cache", self.cache)
+ # disable cache will also disable in memory cache
+ cache_in_memory = cache and kwargs.pop("cache_in_memory", self.cache_in_memory)
messages = messages or [{"role": "user", "content": prompt}]
kwargs = {**self.kwargs, **kwargs}
# Make the request and handle LRU & disk caching.
- if self.model_type == "chat":
- completion = cached_litellm_completion if cache else litellm_completion
+ if cache_in_memory:
+ completion = cached_litellm_completion if self.model_type == "chat" else cached_litellm_text_completion
+
+ response = completion(
+ request=dict(model=self.model, messages=messages, **kwargs),
+ num_retries=self.num_retries,
+ )
else:
- completion = cached_litellm_text_completion if cache else litellm_text_completion
+ completion = litellm_completion if self.model_type == "chat" else litellm_text_completion
+
+ response = completion(
+ request=dict(model=self.model, messages=messages, **kwargs),
+ num_retries=self.num_retries,
+ # only leverage LiteLLM cache in this case
+ cache={"no-cache": not cache, "no-store": not cache},
+ )
- response = completion(
- request=dict(model=self.model, messages=messages, **kwargs),
- num_retries=self.num_retries,
- )
if kwargs.get("logprobs"):
outputs = [
{
diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.py
index 77a942727..720b23afe 100644
--- a/tests/clients/test_lm.py
+++ b/tests/clients/test_lm.py
@@ -29,6 +29,30 @@ def test_chat_lms_can_be_queried(litellm_test_server):
assert azure_openai_lm("azure openai query") == expected_response
+ ("cache", "cache_in_memory"),
+ [
+ (True, True),
+ (True, False),
+ (False, True),
+ (False, False),
+ ],
+)
+def test_chat_lms_cache(litellm_test_server, cache, cache_in_memory):
+ api_base, _ = litellm_test_server
+ expected_response = ["Hi!"]
+
+ openai_lm = dspy.LM(
+ model="openai/dspy-test-model",
+ api_base=api_base,
+ api_key="fakekey",
+ model_type="chat",
+ cache=cache,
+ cache_in_memory=cache_in_memory,
+ )
+ assert openai_lm("openai query") == expected_response
+
+
def test_text_lms_can_be_queried(litellm_test_server):
api_base, _ = litellm_test_server
expected_response = ["Hi!"]