Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Remove LiteLLM caching from LM (#8742)

Omar Khattab61619c16104f
diff --git a/dspy/clients/__init__.py b/dspy/clients/__init__.py
index 15917d34f..9479001bd 100644
--- a/dspy/clients/__init__.py
+++ b/dspy/clients/__init__.py
@@ -3,7 +3,6 @@ import os
from pathlib import Path
import litellm
-from litellm.caching.caching import Cache as LitellmCache
from dspy.clients.base_lm import BaseLM, inspect_history
from dspy.clients.cache import Cache
@@ -15,23 +14,12 @@ logger = logging.getLogger(__name__)
DISK_CACHE_DIR = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")
DISK_CACHE_LIMIT = int(os.environ.get("DSPY_CACHE_LIMIT", 3e10)) # 30 GB default
-
-
-def _litellm_track_cache_hit_callback(kwargs, completion_response, start_time, end_time):
- # Access the cache_hit information
- completion_response.cache_hit = kwargs.get("cache_hit", False)
-
-
-litellm.success_callback = [_litellm_track_cache_hit_callback]
-
-
def configure_cache(
enable_disk_cache: bool | None = True,
enable_memory_cache: bool | None = True,
disk_cache_dir: str | None = DISK_CACHE_DIR,
disk_size_limit_bytes: int | None = DISK_CACHE_LIMIT,
memory_max_entries: int | None = 1000000,
- enable_litellm_cache: bool = False,
):
"""Configure the cache for DSPy.
@@ -41,27 +29,7 @@ def configure_cache(
disk_cache_dir: The directory to store the on-disk cache.
disk_size_limit_bytes: The size limit of the on-disk cache.
memory_max_entries: The maximum number of entries in the in-memory cache.
- enable_litellm_cache: Whether to enable LiteLLM cache.
"""
- if enable_disk_cache and enable_litellm_cache:
- raise ValueError(
- "Cannot enable both LiteLLM and DSPy on-disk cache, please set at most one of `enable_disk_cache` or "
- "`enable_litellm_cache` to True."
- )
-
- if enable_litellm_cache:
- try:
- litellm.cache = LitellmCache(disk_cache_dir=DISK_CACHE_DIR, type="disk")
-
- if litellm.cache.cache.disk_cache.size_limit != DISK_CACHE_LIMIT:
- litellm.cache.cache.disk_cache.reset("size_limit", DISK_CACHE_LIMIT)
- except Exception as e:
- # It's possible that users don't have the write permissions to the cache directory.
- # In that case, we'll just disable the cache.
- logger.warning("Failed to initialize LiteLLM cache: %s", e)
- litellm.cache = None
- else:
- litellm.cache = None
import dspy
@@ -75,7 +43,7 @@ def configure_cache(
litellm.telemetry = False
-litellm.cache = None # By default we disable litellm cache and use DSPy on-disk cache.
+litellm.cache = None # By default we disable LiteLLM cache and use DSPy on-disk cache.
DSPY_CACHE = Cache(
enable_disk_cache=True,
diff --git a/dspy/clients/cache.py b/dspy/clients/cache.py
index 94dc8d5c3..187043538 100644
--- a/dspy/clients/cache.py
+++ b/dspy/clients/cache.py
@@ -118,6 +118,7 @@ class Cache:
if hasattr(response, "usage"):
# Clear the usage data when cache is hit, because no LM call is made
response.usage = {}
+ response.cache_hit = True
return response
def put(
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index 7cb4b8291..084da23a9 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -33,7 +33,6 @@ class LM(BaseLM):
temperature: float = 0.0,
max_tokens: int = 4000,
cache: bool = True,
- cache_in_memory: bool = True,
callbacks: list[BaseCallback] | None = None,
num_retries: int = 3,
provider: Provider | None = None,
@@ -53,7 +52,6 @@ class LM(BaseLM):
max_tokens: The maximum number of tokens to generate per response.
cache: Whether to cache the model responses for reuse to improve performance
and reduce costs.
- cache_in_memory (deprecated): To enable additional caching with LRU in memory.
callbacks: A list of callback functions to run before and after each request.
num_retries: The number of times to retry a request if it fails transiently due to
network error, rate limiting, etc. Requests are retried with exponential
@@ -66,7 +64,6 @@ class LM(BaseLM):
self.model = model
self.model_type = model_type
self.cache = cache
- self.cache_in_memory = cache_in_memory
self.provider = provider or self.infer_provider()
self.callbacks = callbacks or []
self.history = []
@@ -91,33 +88,21 @@ class LM(BaseLM):
else:
self.kwargs = dict(temperature=temperature, max_tokens=max_tokens, **kwargs)
- def _get_cached_completion_fn(self, completion_fn, cache, enable_memory_cache):
+ def _get_cached_completion_fn(self, completion_fn, cache):
ignored_args_for_cache_key = ["api_key", "api_base", "base_url"]
- if cache and enable_memory_cache:
+ if cache:
completion_fn = request_cache(
cache_arg_name="request",
ignored_args_for_cache_key=ignored_args_for_cache_key,
)(completion_fn)
- elif cache:
- completion_fn = request_cache(
- cache_arg_name="request",
- ignored_args_for_cache_key=ignored_args_for_cache_key,
- enable_memory_cache=False,
- )(completion_fn)
- else:
- completion_fn = completion_fn
- if not cache or litellm.cache is None:
- litellm_cache_args = {"no-cache": True, "no-store": True}
- else:
- litellm_cache_args = {"no-cache": False, "no-store": False}
+ litellm_cache_args = {"no-cache": True, "no-store": True}
return completion_fn, litellm_cache_args
def forward(self, prompt=None, messages=None, **kwargs):
# Build the request.
cache = kwargs.pop("cache", self.cache)
- enable_memory_cache = kwargs.pop("cache_in_memory", self.cache_in_memory)
messages = messages or [{"role": "user", "content": prompt}]
kwargs = {**self.kwargs, **kwargs}
@@ -128,7 +113,7 @@ class LM(BaseLM):
completion = litellm_text_completion
elif self.model_type == "responses":
completion = litellm_responses_completion
- completion, litellm_cache_args = self._get_cached_completion_fn(completion, cache, enable_memory_cache)
+ completion, litellm_cache_args = self._get_cached_completion_fn(completion, cache)
results = completion(
request=dict(model=self.model, messages=messages, **kwargs),
@@ -145,7 +130,6 @@ class LM(BaseLM):
async def aforward(self, prompt=None, messages=None, **kwargs):
# Build the request.
cache = kwargs.pop("cache", self.cache)
- enable_memory_cache = kwargs.pop("cache_in_memory", self.cache_in_memory)
messages = messages or [{"role": "user", "content": prompt}]
kwargs = {**self.kwargs, **kwargs}
@@ -156,7 +140,7 @@ class LM(BaseLM):
completion = alitellm_text_completion
elif self.model_type == "responses":
completion = alitellm_responses_completion
- completion, litellm_cache_args = self._get_cached_completion_fn(completion, cache, enable_memory_cache)
+ completion, litellm_cache_args = self._get_cached_completion_fn(completion, cache)
results = await completion(
request=dict(model=self.model, messages=messages, **kwargs),
@@ -246,7 +230,6 @@ class LM(BaseLM):
"model",
"model_type",
"cache",
- "cache_in_memory",
"num_retries",
"finetuning_model",
"launch_kwargs",
diff --git a/tests/clients/test_litellm_cache.py b/tests/clients/test_litellm_cache.py
deleted file mode 100644
index 9677be40b..000000000
--- a/tests/clients/test_litellm_cache.py
+++ /dev/null
@@ -1,107 +0,0 @@
-import importlib
-import shutil
-import tempfile
-from unittest.mock import patch
-
-import pytest
-
-import dspy
-from tests.test_utils.server import read_litellm_test_server_request_logs
-
-
-def temporary_blank_cache_dir(monkeypatch):
- with tempfile.TemporaryDirectory() as cache_dir_path:
- monkeypatch.setenv("DSPY_CACHEDIR", cache_dir_path)
- importlib.reload(dspy.clients)
- dspy.configure_cache(enable_memory_cache=True, enable_disk_cache=False, enable_litellm_cache=True)
- yield cache_dir_path
- dspy.configure_cache(enable_memory_cache=True, enable_disk_cache=True, enable_litellm_cache=False)
-
-
-def test_lm_calls_are_cached_across_lm_instances(litellm_test_server, temporary_blank_cache_dir):
- api_base, server_log_file_path = litellm_test_server
-
- # Call 2 LM instances with the same model & text and verify that only one API request is sent
- # to the LiteLLM server
- lm1 = dspy.LM(
- model="openai/dspy-test-model",
- api_base=api_base,
- api_key="fakekey",
- )
- lm1("Example query")
- lm2 = dspy.LM(
- model="openai/dspy-test-model",
- api_base=api_base,
- api_key="fakekey",
- )
- lm2("Example query")
- request_logs = read_litellm_test_server_request_logs(server_log_file_path)
- assert len(request_logs) == 1
-
- # Call one of the LMs with new text and verify that a new API request is sent to the
- # LiteLLM server
- lm1("New query")
- request_logs = read_litellm_test_server_request_logs(server_log_file_path)
- assert len(request_logs) == 2
-
- # Create a new LM instance with a different model and query it twice with the original text.
- # Verify that one new API request is sent to the LiteLLM server
- lm3 = dspy.LM(
- model="openai/dspy-test-model-2",
- api_base=api_base,
- api_key="fakekey",
- )
- lm3("Example query")
- lm3("Example query")
- request_logs = read_litellm_test_server_request_logs(server_log_file_path)
- assert len(request_logs) == 3
-
-
-def test_lm_calls_are_cached_in_memory_when_expected(litellm_test_server, temporary_blank_cache_dir):
- api_base, server_log_file_path = litellm_test_server
-
- lm1 = dspy.LM(
- model="openai/dspy-test-model",
- api_base=api_base,
- api_key="fakekey",
- )
- lm1("Example query")
- # Remove the disk cache, after which the LM must rely on in-memory caching
- shutil.rmtree(temporary_blank_cache_dir)
- lm1("Example query2")
- lm1("Example query2")
- lm1("Example query2")
- lm1("Example query2")
-
- request_logs = read_litellm_test_server_request_logs(server_log_file_path)
- assert len(request_logs) == 2
-
-
-def test_lm_calls_skip_in_memory_cache_if_key_not_computable():
- with patch("litellm.completion") as mock_litellm_completion:
-
- class NonJsonSerializable:
- pass
-
- lm = dspy.LM(
- model="fakemodel/fakemodel",
- non_json_serializable=NonJsonSerializable(),
- )
- lm("Example query")
- lm("Example query")
-
- assert mock_litellm_completion.call_count == 2
-
-
-def test_lms_called_expected_number_of_times_for_cache_key_generation_failures():
- with pytest.raises(RuntimeError), patch("litellm.completion") as mock_completion:
- mock_completion.side_effect = RuntimeError("Mocked exception")
- lm = dspy.LM(
- model="openai/dspy-test-model",
- api_base="fakebase",
- api_key="fakekey",
- )
- lm("Do not retry")
-
- assert mock_completion.call_count == 1
diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.py
index 2634b1ebf..491c3c594 100644
--- a/tests/clients/test_lm.py
+++ b/tests/clients/test_lm.py
@@ -62,40 +62,6 @@ def test_chat_lms_can_be_queried(litellm_test_server):
assert azure_openai_lm("azure openai query") == expected_response
- ("cache", "cache_in_memory"),
- [
- (True, True),
- (True, False),
- (False, True),
- (False, False),
- ],
-)
-def test_litellm_cache(litellm_test_server, cache, cache_in_memory):
- api_base, _ = litellm_test_server
- expected_response = ["Hi!"]
-
- original_cache = dspy.cache
- dspy.clients.configure_cache(
- enable_disk_cache=False,
- enable_memory_cache=cache_in_memory,
- enable_litellm_cache=cache,
- )
-
- openai_lm = dspy.LM(
- model="openai/dspy-test-model",
- api_base=api_base,
- api_key="fakekey",
- model_type="chat",
- cache=cache,
- cache_in_memory=cache_in_memory,
- )
- assert openai_lm("openai query") == expected_response
-
- # Reset the cache configuration
- dspy.cache = original_cache
-
-
def test_dspy_cache(litellm_test_server, tmp_path):
api_base, _ = litellm_test_server
@@ -103,7 +69,6 @@ def test_dspy_cache(litellm_test_server, tmp_path):
dspy.clients.configure_cache(
enable_disk_cache=True,
enable_memory_cache=True,
- enable_litellm_cache=False,
disk_cache_dir=tmp_path / ".disk_cache",
)
cache = dspy.cache
@@ -288,7 +253,6 @@ def test_dump_state():
"max_tokens": 100,
"num_retries": 10,
"cache": True,
- "cache_in_memory": True,
"finetuning_model": None,
"launch_kwargs": {"temperature": 1},
"train_kwargs": {"temperature": 5},
@@ -377,7 +341,6 @@ async def test_async_lm_call_with_cache(tmp_path):
dspy.clients.configure_cache(
enable_disk_cache=True,
enable_memory_cache=True,
- enable_litellm_cache=False,
disk_cache_dir=tmp_path / ".disk_cache",
)
cache = dspy.cache
@@ -400,11 +363,10 @@ async def test_async_lm_call_with_cache(tmp_path):
# Second call should hit the cache, so no new call to LiteLLM is made.
assert mock_alitellm_completion.call_count == 1
- # Test that explicitly disabling memory cache works
- await lm.acall("New query", cache_in_memory=False)
+ # A new query should result in a new LiteLLM call and a new cache entry.
+ await lm.acall("New query")
- # There should be a new call to LiteLLM on new query, but the memory cache shouldn't be written to.
- assert len(cache.memory_cache) == 1
+ assert len(cache.memory_cache) == 2
assert mock_alitellm_completion.call_count == 2
dspy.cache = original_cache
@@ -470,7 +432,6 @@ def test_responses_api(litellm_test_server):
api_key="fakekey",
model_type="responses",
cache=False,
- cache_in_memory=False,
)
assert lm("openai query") == [expected_text]
@@ -501,7 +462,6 @@ def test_responses_api_tool_calls(litellm_test_server):
api_key="fakekey",
model_type="responses",
cache=False,
- cache_in_memory=False,
)
assert lm("openai query") == expected_response
diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py
index d49d0231e..09ed2fcc1 100644
--- a/tests/predict/test_predict.py
+++ b/tests/predict/test_predict.py
@@ -56,7 +56,6 @@ def test_lm_after_dump_and_load_state():
"max_tokens": 100,
"num_retries": 10,
"cache": True,
- "cache_in_memory": True,
"finetuning_model": None,
"launch_kwargs": {},
"train_kwargs": {},