Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Dspy fanout cache (#8062)

Chen Qianff46299ead56
diff --git a/dspy/__init__.py b/dspy/__init__.py
index a8f800702..775018d61 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -16,6 +16,7 @@ from dspy.streaming.streamify import streamify
from dspy.utils.usage_tracker import track_usage
from dspy.dsp.utils.settings import settings
+from dspy.clients import DSPY_CACHE
configure_dspy_loggers(__name__)
@@ -28,3 +29,5 @@ context = settings.context
BootstrapRS = BootstrapFewShotWithRandomSearch
from .__metadata__ import __name__, __version__, __description__, __url__, __author__, __author_email__
+
+cache = DSPY_CACHE
diff --git a/dspy/clients/__init__.py b/dspy/clients/__init__.py
index 48fa730a9..3eeba1e2c 100644
--- a/dspy/clients/__init__.py
+++ b/dspy/clients/__init__.py
@@ -5,8 +5,10 @@ from dspy.clients.embedding import Embedder
import litellm
import os
from pathlib import Path
-from litellm.caching import Cache
+from dspy.clients.cache import Cache
import logging
+from typing import Optional
+from litellm.caching.caching import Cache as LitellmCache
logger = logging.getLogger(__name__)
@@ -21,20 +23,66 @@ def _litellm_track_cache_hit_callback(kwargs, completion_response, start_time, e
litellm.success_callback = [_litellm_track_cache_hit_callback]
-try:
- # TODO: There's probably value in getting litellm to support FanoutCache and to separate the limit for
- # the LM cache from the embeddings cache. Then we can lower the default 30GB limit.
- litellm.cache = Cache(disk_cache_dir=DISK_CACHE_DIR, type="disk")
- if litellm.cache.cache.disk_cache.size_limit != DISK_CACHE_LIMIT:
- litellm.cache.cache.disk_cache.reset("size_limit", DISK_CACHE_LIMIT)
-except Exception as e:
- # It's possible that users don't have the write permissions to the cache directory.
- # In that case, we'll just disable the cache.
- logger.warning("Failed to initialize LiteLLM cache: %s", e)
- litellm.cache = None
+def configure_cache(
+ enable_disk_cache: Optional[bool] = True,
+ enable_memory_cache: Optional[bool] = True,
+ disk_cache_dir: Optional[str] = DISK_CACHE_DIR,
+ disk_size_limit_bytes: Optional[int] = DISK_CACHE_LIMIT,
+ memory_max_entries: Optional[int] = 1000000,
+ enable_litellm_cache: bool = False,
+):
+ """Configure the cache for DSPy.
+
+ Args:
+ enable_disk_cache: Whether to enable on-disk cache.
+ enable_memory_cache: Whether to enable in-memory cache.
+ disk_cache_dir: The directory to store the on-disk cache.
+ disk_size_limit_bytes: The size limit of the on-disk cache.
+ memory_max_entries: The maximum number of entries in the in-memory cache.
+ enable_litellm_cache: Whether to enable LiteLLM cache.
+ """
+ if enable_disk_cache and enable_litellm_cache:
+ raise ValueError(
+ "Cannot enable both LiteLLM and DSPy on-disk cache, please set at most one of `enable_disk_cache` or "
+ "`enable_litellm_cache` to True."
+ )
+
+ if enable_litellm_cache:
+ try:
+ litellm.cache = LitellmCache(disk_cache_dir=DISK_CACHE_DIR, type="disk")
+
+ if litellm.cache.cache.disk_cache.size_limit != DISK_CACHE_LIMIT:
+ litellm.cache.cache.disk_cache.reset("size_limit", DISK_CACHE_LIMIT)
+ except Exception as e:
+ # It's possible that users don't have the write permissions to the cache directory.
+ # In that case, we'll just disable the cache.
+ logger.warning("Failed to initialize LiteLLM cache: %s", e)
+ litellm.cache = None
+ else:
+ litellm.cache = None
+
+ import dspy
+
+ dspy.cache = Cache(
+ enable_disk_cache,
+ enable_memory_cache,
+ disk_cache_dir,
+ disk_size_limit_bytes,
+ memory_max_entries,
+ )
+
litellm.telemetry = False
+litellm.cache = None # By default we disable litellm cache and use DSPy on-disk cache.
+
+DSPY_CACHE = Cache(
+ enable_disk_cache=True,
+ enable_memory_cache=True,
+ disk_cache_dir=DISK_CACHE_DIR,
+ disk_size_limit_bytes=DISK_CACHE_LIMIT,
+ memory_max_entries=1000000,
+)
# Turn off by default to avoid LiteLLM logging during every LM call.
litellm.suppress_debug_info = True
@@ -61,4 +109,5 @@ __all__ = [
"Embedder",
"enable_litellm_logging",
"disable_litellm_logging",
+ "configure_cache",
]
diff --git a/dspy/clients/cache.py b/dspy/clients/cache.py
new file mode 100644
index 000000000..c73d986ff
--- /dev/null
+++ b/dspy/clients/cache.py
@@ -0,0 +1,215 @@
+import copy
+import logging
+import threading
+from functools import wraps
+from hashlib import sha256
+from typing import Any, Dict, Optional
+
+import cloudpickle
+import pydantic
+import ujson
+from cachetools import LRUCache
+from diskcache import FanoutCache
+
+logger = logging.getLogger(__name__)
+
+
+class Cache:
+ """DSPy Cache
+
+ `Cache` provides 2 levels of caching (in the given order):
+ 1. In-memory cache - implemented with cachetools.LRUCache
+ 2. On-disk cache - implemented with diskcache.FanoutCache
+ """
+
+ def __init__(
+ self,
+ enable_disk_cache: bool,
+ enable_memory_cache: bool,
+ disk_cache_dir: str,
+ disk_size_limit_bytes: Optional[int] = 1024 * 1024 * 10,
+ memory_max_entries: Optional[int] = 1000000,
+ ignored_args_for_cache_key: Optional[list[str]] = None,
+ ):
+ """
+ Args:
+ enable_disk_cache: Whether to enable on-disk cache.
+ enable_memory_cache: Whether to enable in-memory cache.
+ disk_cache_dir: The directory where the disk cache is stored.
+ disk_size_limit_bytes: The maximum size of the disk cache (in bytes).
+ memory_max_entries: The maximum size of the in-memory cache (in number of items).
+ ignored_args_for_cache_key: A list of arguments to ignore when computing the cache key from the request.
+ """
+
+ self.enable_disk_cache = enable_disk_cache
+ self.enable_memory_cache = enable_memory_cache
+ if self.enable_memory_cache:
+ self.memory_cache = LRUCache(maxsize=memory_max_entries)
+ else:
+ self.memory_cache = {}
+ if self.enable_disk_cache:
+ self.disk_cache = FanoutCache(
+ shards=16,
+ timeout=10,
+ directory=disk_cache_dir,
+ size_limit=disk_size_limit_bytes,
+ )
+ else:
+ self.disk_cache = {}
+
+ self.ignored_args_for_cache_key = ignored_args_for_cache_key or []
+
+ self._lock = threading.RLock()
+
+ def __contains__(self, key: str) -> bool:
+ """Check if a key is in the cache."""
+ return key in self.memory_cache or key in self.disk_cache
+
+ def cache_key(self, request: Dict[str, Any]) -> str:
+ """
+ Obtain a unique cache key for the given request dictionary by hashing its JSON
+ representation. For request fields having types that are known to be JSON-incompatible,
+ convert them to a JSON-serializable format before hashing.
+ """
+
+ def transform_value(value):
+ if isinstance(value, type) and issubclass(value, pydantic.BaseModel):
+ return value.model_json_schema()
+ elif isinstance(value, pydantic.BaseModel):
+ return value.model_dump()
+ elif callable(value):
+ # Try to get the source code of the callable if available
+ import inspect
+
+ try:
+ # For regular functions, we can get the source code
+ return f"<callable_source:{inspect.getsource(value)}>"
+ except (TypeError, OSError, IOError):
+ # For lambda functions or other callables where source isn't available,
+ # use a string representation
+ return f"<callable:{value.__name__ if hasattr(value, '__name__') else 'lambda'}>"
+ elif isinstance(value, dict):
+ return {k: transform_value(v) for k, v in value.items()}
+ else:
+ return value
+
+ params = {k: transform_value(v) for k, v in request.items() if k not in self.ignored_args_for_cache_key}
+ return sha256(ujson.dumps(params, sort_keys=True).encode()).hexdigest()
+
+ def get(self, request: Dict[str, Any]) -> Any:
+ try:
+ key = self.cache_key(request)
+ except Exception:
+ logger.debug(f"Failed to generate cache key for request: {request}")
+ return None
+
+ if self.enable_memory_cache and key in self.memory_cache:
+ with self._lock:
+ response = self.memory_cache[key]
+ elif self.enable_disk_cache and key in self.disk_cache:
+ # Found on disk but not in memory cache, add to memory cache
+ response = self.disk_cache[key]
+ if self.enable_memory_cache:
+ with self._lock:
+ self.memory_cache[key] = response
+ else:
+ return None
+
+ response = copy.deepcopy(response)
+ if hasattr(response, "usage"):
+ # Clear the usage data when cache is hit, because no LM call is made
+ response.usage = {}
+ return response
+
+ def put(self, request: Dict[str, Any], value: Any) -> None:
+ try:
+ key = self.cache_key(request)
+ except Exception:
+ logger.debug(f"Failed to generate cache key for request: {request}")
+ return
+
+ if self.enable_memory_cache:
+ with self._lock:
+ self.memory_cache[key] = value
+
+ if self.enable_disk_cache:
+ try:
+ self.disk_cache[key] = value
+ except Exception as e:
+ # Disk cache writing can fail for different reasons, e.g. disk full or the `value` is not picklable.
+ logger.debug(f"Failed to put value in disk cache: {value}, {e}")
+
+ def reset_memory_cache(self) -> None:
+ if not self.enable_memory_cache:
+ return
+
+ with self._lock:
+ self.memory_cache.clear()
+
+ def save_memory_cache(self, filepath: str) -> None:
+ if not self.enable_memory_cache:
+ return
+
+ with self._lock:
+ with open(filepath, "wb") as f:
+ cloudpickle.dump(self.memory_cache, f)
+
+ def load_memory_cache(self, filepath: str) -> None:
+ if not self.enable_memory_cache:
+ return
+
+ with self._lock:
+ with open(filepath, "rb") as f:
+ self.memory_cache = cloudpickle.load(f)
+
+
+def request_cache(cache_arg_name: Optional[str] = None, ignored_args_for_cache_key: Optional[list[str]] = None):
+ """Decorator for applying caching to a function based on the request argument.
+
+ Args:
+ cache_arg_name: The name of the argument that contains the request. If not provided, the entire kwargs is used
+ as the request.
+ ignored_args_for_cache_key: A list of arguments to ignore when computing the cache key from the request.
+ """
+
+ def decorator(fn):
+ @wraps(fn)
+ def wrapper(*args, **kwargs):
+ import dspy
+
+ cache = dspy.cache
+ original_ignored_args_for_cache_key = cache.ignored_args_for_cache_key
+ cache.ignored_args_for_cache_key = ignored_args_for_cache_key or []
+
+ # Use fully qualified function name for uniqueness
+ fn_identifier = f"{fn.__module__}.{fn.__qualname__}"
+
+ # Create a modified request that includes the function identifier so that it's incorporated into the cache
+ # key. Deep copy is required because litellm sometimes modifies the kwargs in place.
+ if cache_arg_name:
+ # When `cache_arg_name` is provided, use the value of the argument with this name as the request for
+ # caching.
+ modified_request = copy.deepcopy(kwargs[cache_arg_name])
+ else:
+ # When `cache_arg_name` is not provided, use the entire kwargs as the request for caching.
+ modified_request = copy.deepcopy(kwargs)
+ for i, arg in enumerate(args):
+ modified_request[f"positional_arg_{i}"] = arg
+ modified_request["_fn_identifier"] = fn_identifier
+
+ # Retrieve from cache if available
+ cached_result = cache.get(modified_request)
+
+ if cached_result is not None:
+ return cached_result
+
+ # Otherwise, compute and store the result
+ result = fn(*args, **kwargs)
+ cache.put(modified_request, result)
+
+ cache.ignored_args_for_cache_key = original_ignored_args_for_cache_key
+ return result
+
+ return wrapper
+
+ return decorator
diff --git a/dspy/clients/embedding.py b/dspy/clients/embedding.py
index 0c08f188d..8d1a41eed 100644
--- a/dspy/clients/embedding.py
+++ b/dspy/clients/embedding.py
@@ -1,6 +1,8 @@
import litellm
import numpy as np
+from dspy.clients.cache import request_cache
+
class Embedder:
"""DSPy embedding class.
@@ -124,37 +126,52 @@ class Embedder:
"""
input_batches, caching, kwargs, is_single_input = self._preprocess(inputs, batch_size, caching, **kwargs)
+ compute_embeddings = _cached_compute_embeddings if caching else _compute_embeddings
+
embeddings_list = []
for batch in input_batches:
- if isinstance(self.model, str):
- embedding_response = litellm.embedding(model=self.model, input=batch, caching=caching, **kwargs)
- batch_embeddings = [data["embedding"] for data in embedding_response.data]
- elif callable(self.model):
- batch_embeddings = self.model(batch, **kwargs)
- else:
- raise ValueError(
- f"`model` in `dspy.Embedder` must be a string or a callable, but got {type(self.model)}."
- )
-
- embeddings_list.extend(batch_embeddings)
+ embeddings_list.extend(compute_embeddings(self.model, batch, caching=caching, **kwargs))
return self._postprocess(embeddings_list, is_single_input)
async def acall(self, inputs, batch_size=None, caching=None, **kwargs):
input_batches, caching, kwargs, is_single_input = self._preprocess(inputs, batch_size, caching, **kwargs)
embeddings_list = []
+ acompute_embeddings = _cached_acompute_embeddings if caching else _acompute_embeddings
for batch in input_batches:
- if isinstance(self.model, str):
- embedding_response = await litellm.aembedding(model=self.model, input=batch, caching=caching, **kwargs)
- batch_embeddings = [data["embedding"] for data in embedding_response.data]
- elif callable(self.model):
- batch_embeddings = self.model(batch, **kwargs)
- else:
- raise ValueError(
- f"`model` in `dspy.Embedder` must be a string or a callable, but got {type(self.model)}."
- )
-
- embeddings_list.extend(batch_embeddings)
+ embeddings_list.extend(acompute_embeddings(self.model, batch, caching=caching, **kwargs))
return self._postprocess(embeddings_list, is_single_input)
+
+
+def _compute_embeddings(model, batch_inputs, caching=False, **kwargs):
+ if isinstance(model, str):
+ caching = caching and litellm.cache is not None
+ embedding_response = litellm.embedding(model=model, input=batch_inputs, caching=caching, **kwargs)
+ return [data["embedding"] for data in embedding_response.data]
+ elif callable(model):
+ return model(batch_inputs, **kwargs)
+ else:
+ raise ValueError(f"`model` in `dspy.Embedder` must be a string or a callable, but got {type(model)}.")
+
+
+@request_cache(ignored_args_for_cache_key=["api_key", "api_base", "base_url"])
+def _cached_compute_embeddings(model, batch_inputs, caching=True, **kwargs):
+ return _compute_embeddings(model, batch_inputs, caching=caching, **kwargs)
+
+
+async def _acompute_embeddings(model, batch_inputs, caching=False, **kwargs):
+ if isinstance(model, str):
+ caching = caching and litellm.cache is not None
+ embedding_response = await litellm.aembedding(model=model, input=batch_inputs, caching=caching, **kwargs)
+ return [data["embedding"] for data in embedding_response.data]
+ elif callable(model):
+ return model(batch_inputs, **kwargs)
+ else:
+ raise ValueError(f"`model` in `dspy.Embedder` must be a string or a callable, but got {type(model)}.")
+
+
+@request_cache(ignored_args_for_cache_key=["api_key", "api_base", "base_url"])
+async def _cached_acompute_embeddings(model, batch_inputs, caching=True, **kwargs):
+ return await _acompute_embeddings(model, batch_inputs, caching=caching, **kwargs)
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index add790db1..d092808b1 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -1,20 +1,16 @@
-import functools
import logging
import os
import re
import threading
-from hashlib import sha256
from typing import Any, Dict, List, Literal, Optional, cast
import litellm
-import pydantic
-import ujson
from anyio.streams.memory import MemoryObjectSendStream
from asyncer import syncify
-from cachetools import LRUCache, cached
from litellm import RetryPolicy
import dspy
+from dspy.clients.cache import request_cache
from dspy.clients.openai import OpenAIProvider
from dspy.clients.provider import Provider, TrainingJob
from dspy.clients.utils_finetune import TrainDataFormat
@@ -209,87 +205,18 @@ class LM(BaseLM):
return {key: getattr(self, key) for key in state_keys} | self.kwargs
-def request_cache(maxsize: Optional[int] = None):
- """
- A threadsafe decorator to create an in-memory LRU cache for LM inference functions that accept
- a dictionary-like LM request. An in-memory cache for LM calls is critical for ensuring
- good performance when optimizing and evaluating DSPy LMs (disk caching alone is too slow).
-
- Args:
- maxsize: The maximum size of the cache. If unspecified, no max size is enforced (cache is unbounded).
-
- Returns:
- A decorator that wraps the target function with caching.
- """
-
- def cache_key(request: Dict[str, Any]) -> str:
- """
- Obtain a unique cache key for the given request dictionary by hashing its JSON
- representation. For request fields having types that are known to be JSON-incompatible,
- convert them to a JSON-serializable format before hashing.
-
- Note: Values that cannot be converted to JSON should *not* be ignored / discarded, since
- that would potentially lead to cache collisions. For example, consider request A
- containing only JSON-convertible values and request B containing the same JSON-convertible
- values in addition to one unconvertible value. Discarding the unconvertible value would
- lead to a cache collision between requests A and B, even though they are semantically
- different.
- """
-
- def transform_value(value):
- if isinstance(value, type) and issubclass(value, pydantic.BaseModel):
- return value.model_json_schema()
- elif isinstance(value, pydantic.BaseModel):
- return value.model_dump()
- elif callable(value) and hasattr(value, "__code__") and hasattr(value.__code__, "co_code"):
- return value.__code__.co_code.decode("utf-8")
- else:
- # Note: We don't attempt to compute a hash of the value, since the default
- # implementation of hash() is id(), which may collide if the same memory address
- # is reused for different objects at different times
- return value
-
- params = {k: transform_value(v) for k, v in request.items()}
- return sha256(ujson.dumps(params, sort_keys=True).encode()).hexdigest()
-
- def decorator(func):
- @cached(
- # NB: cachetools doesn't support maxsize=None; it recommends using float("inf") instead
- cache=LRUCache(maxsize=maxsize or float("inf")),
- key=lambda key, request, *args, **kwargs: key,
- # Use a lock to ensure thread safety for the cache when DSPy LMs are queried
- # concurrently, e.g. during optimization and evaluation
- lock=threading.RLock(),
- )
- def func_cached(key: str, request: Dict[str, Any], *args, **kwargs):
- return func(request, *args, **kwargs)
-
- @functools.wraps(func)
- def wrapper(request: dict, *args, **kwargs):
- try:
- key = cache_key(request)
- except Exception:
- # If the cache key cannot be computed (e.g. because it contains a value that cannot
- # be converted to JSON), bypass the cache and call the target function directly
- return func(request, *args, **kwargs)
- cache_hit = key in func_cached.cache
- output = func_cached(key, request, *args, **kwargs)
- if cache_hit and hasattr(output, "usage"):
- # Clear the usage data when cache is hit, because no LM call is made
- output.usage = {}
-
- return func_cached(key, request, *args, **kwargs)
-
- return wrapper
-
- return decorator
+@request_cache(cache_arg_name="request", ignored_args_for_cache_key=["api_key", "api_base", "base_url"])
+def cached_litellm_completion(request: Dict[str, Any], num_retries: int):
+ import litellm
+ if litellm.cache:
+ litellm_cache_args = {"no-cache": False, "no-store": False}
+ else:
+ litellm_cache_args = {"no-cache": True, "no-store": True}
-@request_cache(maxsize=None)
-def cached_litellm_completion(request: Dict[str, Any], num_retries: int):
return litellm_completion(
request,
- cache={"no-cache": False, "no-store": False},
+ cache=litellm_cache_args,
num_retries=num_retries,
)
@@ -339,12 +266,19 @@ def litellm_completion(request: Dict[str, Any], num_retries: int, cache={"no-cac
return stream_completion()
-@request_cache(maxsize=None)
+@request_cache(cache_arg_name="request", ignored_args_for_cache_key=["api_key", "api_base", "base_url"])
def cached_litellm_text_completion(request: Dict[str, Any], num_retries: int):
+ import litellm
+
+ if litellm.cache:
+ litellm_cache_args = {"no-cache": False, "no-store": False}
+ else:
+ litellm_cache_args = {"no-cache": True, "no-store": True}
+
return litellm_text_completion(
request,
num_retries=num_retries,
- cache={"no-cache": False, "no-store": False},
+ cache=litellm_cache_args,
)
diff --git a/dspy/dsp/cache_utils.py b/dspy/dsp/cache_utils.py
deleted file mode 100644
index c2d8c8dbd..000000000
--- a/dspy/dsp/cache_utils.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import logging
-import os
-from functools import wraps
-from pathlib import Path
-
-from joblib import Memory
-
-from dspy.dsp.utils import dotdict
-
-logger = logging.getLogger(__name__)
-
-cache_turn_on = os.environ.get('DSP_CACHEBOOL', 'True').lower() != 'false'
-
-
-def noop_decorator(arg=None, *noop_args, **noop_kwargs):
- def decorator(func):
- @wraps(func)
- def wrapper(*args, **kwargs):
- return func(*args, **kwargs)
-
- return wrapper
-
- if callable(arg):
- return decorator(arg)
- else:
- return decorator
-
-NotebookCacheMemory = dotdict()
-NotebookCacheMemory.cache = noop_decorator
-
-CacheMemory = dotdict()
-CacheMemory.cache = noop_decorator
-
-if cache_turn_on:
- try:
- cachedir = os.environ.get('DSP_CACHEDIR') or os.path.join(Path.home(), '.dspy_cache/cachedir_joblib')
- CacheMemory = Memory(location=cachedir, verbose=0)
-
- cachedir2 = os.environ.get('DSP_NOTEBOOK_CACHEDIR')
- if cachedir2:
- NotebookCacheMemory = Memory(location=cachedir2, verbose=0)
- except Exception as e:
- logger.warning("Failed to initialize cache: %s", e)
- cache_turn_on = False
diff --git a/dspy/dsp/colbertv2.py b/dspy/dsp/colbertv2.py
index 5ffaeb9f2..61b34ee5e 100644
--- a/dspy/dsp/colbertv2.py
+++ b/dspy/dsp/colbertv2.py
@@ -1,9 +1,8 @@
-import functools
from typing import Any, List, Optional, Union
import requests
-from dspy.dsp.cache_utils import CacheMemory, NotebookCacheMemory
+from dspy.clients.cache import request_cache
from dspy.dsp.utils import dotdict
# TODO: Ideally, this takes the name of the index and looks up its port.
@@ -35,7 +34,7 @@ class ColBERTv2:
return [dotdict(psg) for psg in topk]
+@request_cache()
def colbertv2_get_request_v2(url: str, query: str, k: int):
assert (
k <= 100
@@ -49,8 +48,7 @@ def colbertv2_get_request_v2(url: str, query: str, k: int):
return topk[:k]
+@request_cache()
def colbertv2_get_request_v2_wrapped(*args, **kwargs):
return colbertv2_get_request_v2(*args, **kwargs)
@@ -58,7 +56,7 @@ def colbertv2_get_request_v2_wrapped(*args, **kwargs):
colbertv2_get_request = colbertv2_get_request_v2_wrapped
+@request_cache()
def colbertv2_post_request_v2(url: str, query: str, k: int):
headers = {"Content-Type": "application/json; charset=utf-8"}
payload = {"query": query, "k": k}
@@ -67,8 +65,7 @@ def colbertv2_post_request_v2(url: str, query: str, k: int):
return res.json()["topk"][:k]
+@request_cache()
def colbertv2_post_request_v2_wrapped(*args, **kwargs):
return colbertv2_post_request_v2(*args, **kwargs)
diff --git a/tests/caching/test_caching.py b/tests/caching/test_litellm_cache.py
similarity index 92%
rename from tests/caching/test_caching.py
rename to tests/caching/test_litellm_cache.py
index e68c9a507..3d25a9ec4 100644
--- a/tests/caching/test_caching.py
+++ b/tests/caching/test_litellm_cache.py
@@ -15,7 +15,9 @@ def temporary_blank_cache_dir(monkeypatch):
with tempfile.TemporaryDirectory() as cache_dir_path:
monkeypatch.setenv("DSPY_CACHEDIR", cache_dir_path)
importlib.reload(dspy.clients)
+ dspy.configure_cache(enable_memory_cache=True, enable_disk_cache=False, enable_litellm_cache=True)
yield cache_dir_path
+ dspy.configure_cache(enable_memory_cache=True, enable_disk_cache=True, enable_litellm_cache=False)
@pytest.fixture()
@@ -31,7 +33,9 @@ def temporary_populated_cache_dir(monkeypatch):
shutil.copytree(populated_cache_path, cache_dir_path, dirs_exist_ok=True)
monkeypatch.setenv("DSPY_CACHEDIR", cache_dir_path)
importlib.reload(dspy.clients)
+ dspy.configure_cache(enable_memory_cache=True, enable_disk_cache=False, enable_litellm_cache=True)
yield cache_dir_path
+ dspy.configure_cache(enable_memory_cache=True, enable_disk_cache=True, enable_litellm_cache=False)
def test_lm_calls_are_cached_across_lm_instances(litellm_test_server, temporary_blank_cache_dir):
diff --git a/tests/clients/test_cache.py b/tests/clients/test_cache.py
new file mode 100644
index 000000000..2ced975c9
--- /dev/null
+++ b/tests/clients/test_cache.py
@@ -0,0 +1,227 @@
+import os
+import shutil
+from unittest.mock import patch
+
+import pydantic
+import pytest
+from cachetools import LRUCache
+from diskcache import FanoutCache
+
+from dspy.clients.cache import Cache
+from dataclasses import dataclass
+
+
+@dataclass
+class DummyResponse:
+ message: str
+ usage: dict
+
+
+def cache_config(tmp_path):
+ """Default cache configuration."""
+ return {
+ "enable_disk_cache": True,
+ "enable_memory_cache": True,
+ "disk_cache_dir": str(tmp_path),
+ "disk_size_limit_bytes": 1024 * 1024, # 1MB
+ "memory_max_entries": 100,
+ }
+
+
+def cache(cache_config):
+ """Create a cache instance with the default configuration."""
+ return Cache(**cache_config)
+
+
+def test_initialization(tmp_path):
+ """Test different cache initialization configurations."""
+ # Test memory-only cache
+ memory_cache = Cache(
+ enable_disk_cache=False,
+ enable_memory_cache=True,
+ disk_cache_dir="",
+ disk_size_limit_bytes=0,
+ memory_max_entries=50,
+ )
+ assert isinstance(memory_cache.memory_cache, LRUCache)
+ assert memory_cache.memory_cache.maxsize == 50
+ assert memory_cache.disk_cache == {}
+
+ # Test disk-only cache
+ disk_cache = Cache(
+ enable_disk_cache=True,
+ enable_memory_cache=False,
+ disk_cache_dir=str(tmp_path),
+ disk_size_limit_bytes=1024,
+ memory_max_entries=0,
+ )
+ assert isinstance(disk_cache.disk_cache, FanoutCache)
+ assert disk_cache.memory_cache == {}
+
+ # Test disabled cache
+ disabled_cache = Cache(
+ enable_disk_cache=False,
+ enable_memory_cache=False,
+ disk_cache_dir="",
+ disk_size_limit_bytes=0,
+ memory_max_entries=0,
+ )
+ assert disabled_cache.memory_cache == {}
+ assert disabled_cache.disk_cache == {}
+
+
+def test_cache_key_generation(cache):
+ """Test cache key generation with different types of inputs."""
+ # Test with simple dictionary
+ request = {"prompt": "Hello", "model": "openai/gpt-4o-mini", "temperature": 0.7}
+ key = cache.cache_key(request)
+ assert isinstance(key, str)
+ assert len(key) == 64 # SHA-256 hash is 64 characters
+
+ # Test with pydantic model
+ class TestModel(pydantic.BaseModel):
+ name: str
+ value: int
+
+ model = TestModel(name="test", value=42)
+ request_with_model = {"data": model}
+ key_with_model = cache.cache_key(request_with_model)
+ assert isinstance(key_with_model, str)
+
+ # Test with pydantic model class
+ request_with_model_class = {"model_class": TestModel}
+ key_with_model_class = cache.cache_key(request_with_model_class)
+ assert isinstance(key_with_model_class, str)
+
+
+def test_put_and_get(cache):
+ """Test putting and getting from cache."""
+ # Test putting and getting from memory cache
+ request = {"prompt": "Hello", "model": "openai/gpt-4o-mini", "temperature": 0.7}
+
+ value = DummyResponse(message="This is a test response", usage={"prompt_tokens": 10, "completion_tokens": 20})
+
+ cache.put(request, value)
+ result = cache.get(request)
+
+ assert result.message == value.message
+ assert result.usage == {}
+
+ # Test with disk cache
+ # First, clear memory cache to ensure we're using disk cache
+ cache.reset_memory_cache()
+
+ # Get from disk cache
+ result_from_disk = cache.get(request)
+ assert result_from_disk.message == value.message
+ assert result_from_disk.usage == {}
+
+ # Verify it was also added back to memory cache
+ assert cache.cache_key(request) in cache.memory_cache
+
+
+def test_cache_miss(cache):
+ """Test getting a non-existent key."""
+ request = {"prompt": "Non-existent", "model": "gpt-4"}
+ result = cache.get(request)
+ assert result is None
+
+
+def test_cache_key_error_handling(cache):
+ """Test error handling for unserializable objects."""
+
+ # Test with a request that can't be serialized to JSON
+ class UnserializableObject:
+ pass
+
+ request = {"data": UnserializableObject()}
+
+ # Should not raise an exception
+ result = cache.get(request)
+ assert result is None
+
+ # Should not raise an exception
+ cache.put(request, "value")
+
+
+def test_reset_memory_cache(cache):
+ """Test resetting memory cache."""
+ # Add some items to the memory cache
+ requests = [{"prompt": f"Hello {i}", "model": "openai/gpt-4o-mini"} for i in range(5)]
+ for i, req in enumerate(requests):
+ cache.put(req, f"Response {i}")
+
+ # Verify items are in memory cache
+ for req in requests:
+ key = cache.cache_key(req)
+ assert key in cache.memory_cache
+
+ # Reset memory cache
+ cache.reset_memory_cache()
+
+ # Verify memory cache is empty
+ assert len(cache.memory_cache) == 0
+
+ # But disk cache still has the items
+ for req in requests:
+ result = cache.get(req)
+ assert result is not None
+
+
+def test_save_and_load_memory_cache(cache, tmp_path):
+ """Test saving and loading memory cache."""
+ # Add some items to the memory cache
+ requests = [{"prompt": f"Hello {i}", "model": "openai/gpt-4o-mini"} for i in range(5)]
+ for i, req in enumerate(requests):
+ cache.put(req, f"Response {i}")
+
+ # Save memory cache to a temporary file
+ temp_cache_file = tmp_path / "memory_cache.pkl"
+ cache.save_memory_cache(str(temp_cache_file))
+
+ # Create a new cache instance with disk cache disabled
+ new_cache = Cache(
+ enable_memory_cache=True,
+ enable_disk_cache=False,
+ disk_cache_dir=tmp_path / "disk_cache",
+ disk_size_limit_bytes=0,
+ memory_max_entries=100,
+ )
+
+ # Load the memory cache
+ new_cache.load_memory_cache(str(temp_cache_file))
+
+ # Verify items are in the new memory cache
+ for req in requests:
+ result = new_cache.get(req)
+ assert result is not None
+ assert result == f"Response {requests.index(req)}"
+
+
+def test_lm_cache_decorator(cache):
+ """Test the lm_cache decorator."""
+ from dspy.clients.cache import request_cache
+
+ # Mock the dspy.cache attribute
+ with patch("dspy.cache", cache):
+ # Define a test function
+ @request_cache()
+ def test_function(prompt, model):
+ return f"Response for {prompt} with {model}"
+
+ # First call should compute the result
+ result1 = test_function(prompt="Hello", model="openai/gpt-4o-mini")
+ assert result1 == "Response for Hello with openai/gpt-4o-mini"
+
+ # Second call with same arguments should use cache
+ with patch.object(cache, "get") as mock_get:
+ mock_get.return_value = "Cached response"
+ result2 = test_function(prompt="Hello", model="openai/gpt-4o-mini")
+ assert result2 == "Cached response"
+ mock_get.assert_called_once()
+
+ # Call with different arguments should compute again
+ result3 = test_function(prompt="Different", model="openai/gpt-4o-mini")
+ assert result3 == "Response for Different with openai/gpt-4o-mini"
diff --git a/tests/clients/test_embedding.py b/tests/clients/test_embedding.py
index 38425cef1..625c7fc77 100644
--- a/tests/clients/test_embedding.py
+++ b/tests/clients/test_embedding.py
@@ -3,6 +3,7 @@ from unittest.mock import Mock, patch
import numpy as np
from dspy.clients.embedding import Embedder
+import dspy
# Mock response format similar to litellm's embedding response.
@@ -14,7 +15,15 @@ class MockEmbeddingResponse:
self.object = "list"
-def test_litellm_embedding():
+def cache(tmp_path):
+ original_cache = dspy.cache
+ dspy.configure_cache(disk_cache_dir=tmp_path / ".dspy_cache")
+ yield
+ dspy.cache = original_cache
+
+
+def test_litellm_embedding(cache):
model = "text-embedding-ada-002"
inputs = ["hello", "world"]
mock_embeddings = [
@@ -27,17 +36,29 @@ def test_litellm_embedding():
mock_litellm.return_value = MockEmbeddingResponse(mock_embeddings)
# Create embedding instance and call it.
- embedding = Embedder(model)
+ embedding = Embedder(model, caching=True)
result = embedding(inputs)
# Verify litellm was called with correct parameters.
- mock_litellm.assert_called_once_with(model=model, input=inputs, caching=True)
+ # Because we disable the litellm cache, it should be called with caching=False.
+ mock_litellm.assert_called_once_with(model=model, input=inputs, caching=False)
assert len(result) == len(inputs)
np.testing.assert_allclose(result, mock_embeddings)
+ # Second call should be cached.
+ result = embedding(inputs)
+ assert mock_litellm.call_count == 1
+ np.testing.assert_allclose(result, mock_embeddings)
-def test_callable_embedding():
+ # Disable cache should issue new calls.
+ embedding = Embedder(model, caching=False)
+ result = embedding(inputs)
+ assert mock_litellm.call_count == 2
+ np.testing.assert_allclose(result, mock_embeddings)
+
+
+def test_callable_embedding(cache):
inputs = ["hello", "world", "test"]
expected_embeddings = [
@@ -46,14 +67,27 @@ def test_callable_embedding():
[0.7, 0.8, 0.9], # embedding for "test"
]
- def mock_embedding_fn(texts):
- # Simple callable that returns random embeddings.
- return expected_embeddings
+ class EmbeddingFn:
+ def __init__(self):
+ self.call_count = 0
+
+ def __call__(self, texts):
+ # Simple callable that returns random embeddings.
+ self.call_count += 1
+ return expected_embeddings
+
+ embedding_fn = EmbeddingFn()
# Create embedding instance with callable
- embedding = Embedder(mock_embedding_fn)
+ embedding = Embedder(embedding_fn)
result = embedding(inputs)
+ assert embedding_fn.call_count == 1
+ np.testing.assert_allclose(result, expected_embeddings)
+
+ result = embedding(inputs)
+ # The second call should be cached.
+ assert embedding_fn.call_count == 1
np.testing.assert_allclose(result, expected_embeddings)
diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.py
index 637b087ea..c593ffb53 100644
--- a/tests/clients/test_lm.py
+++ b/tests/clients/test_lm.py
@@ -7,6 +7,7 @@ import pytest
from openai import RateLimitError
import dspy
+from dspy.utils.usage_tracker import track_usage
from tests.test_utils.server import litellm_test_server, read_litellm_test_server_request_logs
@@ -40,10 +41,17 @@ def test_chat_lms_can_be_queried(litellm_test_server):
(False, False),
],
)
-def test_chat_lms_cache(litellm_test_server, cache, cache_in_memory):
+def test_litellm_cache(litellm_test_server, cache, cache_in_memory):
api_base, _ = litellm_test_server
expected_response = ["Hi!"]
+ original_cache = dspy.cache
+ dspy.clients.configure_cache(
+ enable_disk_cache=False,
+ enable_memory_cache=cache_in_memory,
+ enable_litellm_cache=cache,
+ )
+
openai_lm = dspy.LM(
model="openai/dspy-test-model",
api_base=api_base,
@@ -54,6 +62,43 @@ def test_chat_lms_cache(litellm_test_server, cache, cache_in_memory):
)
assert openai_lm("openai query") == expected_response
+ # Reset the cache configuration
+ dspy.cache = original_cache
+
+
+def test_dspy_cache(litellm_test_server, tmp_path):
+ api_base, _ = litellm_test_server
+
+ original_cache = dspy.cache
+ dspy.clients.configure_cache(
+ enable_disk_cache=True,
+ enable_memory_cache=True,
+ enable_litellm_cache=False,
+ disk_cache_dir=tmp_path / ".disk_cache",
+ )
+ cache = dspy.cache
+
+ lm = dspy.LM(
+ model="openai/dspy-test-model",
+ api_base=api_base,
+ api_key="fakekey",
+ model_type="text",
+ )
+ with track_usage() as usage_tracker:
+ lm("Query")
+
+ assert len(cache.memory_cache) == 1
+ cache_key = next(iter(cache.memory_cache.keys()))
+ assert cache_key in cache.disk_cache
+ assert len(usage_tracker.usage_data) == 1
+
+ with track_usage() as usage_tracker:
+ lm("Query")
+
+ assert len(usage_tracker.usage_data) == 0
+
+ dspy.cache = original_cache
+
def test_text_lms_can_be_queried(litellm_test_server):
api_base, _ = litellm_test_server
@@ -86,7 +131,9 @@ def test_lm_calls_support_callables(litellm_test_server):
api_base=api_base,
api_key="fakekey",
azure_ad_token_provider=azure_ad_token_provider,
+ cache=False,
)
+
lm_with_callable("Query")
spy_completion.assert_called_once()
diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.py
index fc89b504e..c09f48063 100644
--- a/tests/streaming/test_streaming.py
+++ b/tests/streaming/test_streaming.py
@@ -14,7 +14,7 @@ async def test_streamify_yields_expected_response_chunks(litellm_test_server):
api_base=api_base,
api_key="fakekey",
)
- with dspy.context(lm=lm):
+ with dspy.context(lm=lm, adapter=dspy.JSONAdapter()):
class TestSignature(dspy.Signature):
input_text: str = dspy.InputField()
diff --git a/tests/test_utils/server/litellm_server.py b/tests/test_utils/server/litellm_server.py
index 419ea4578..58801e9e7 100644
--- a/tests/test_utils/server/litellm_server.py
+++ b/tests/test_utils/server/litellm_server.py
@@ -47,6 +47,7 @@ def _get_mock_llm_response(request_kwargs):
return litellm.completion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello world"}],
+ usage={"prompt_tokens": 10, "completion_tokens": 10, "total_tokens": 20},
mock_response="Hi!",
)