DSPy: The framework for programming—not prompting—language models
Python
Dspy fanout cache (#8062)
Chen Qianff46299ead56
diff --git a/dspy/__init__.py b/dspy/__init__.pyindex a8f800702..775018d61 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -16,6 +16,7 @@ from dspy.streaming.streamify import streamifyfrom dspy.utils.usage_tracker import track_usagefrom dspy.dsp.utils.settings import settings+from dspy.clients import DSPY_CACHEconfigure_dspy_loggers(__name__)@@ -28,3 +29,5 @@ context = settings.contextBootstrapRS = BootstrapFewShotWithRandomSearchfrom .__metadata__ import __name__, __version__, __description__, __url__, __author__, __author_email__++cache = DSPY_CACHEdiff --git a/dspy/clients/__init__.py b/dspy/clients/__init__.pyindex 48fa730a9..3eeba1e2c 100644--- a/dspy/clients/__init__.py+++ b/dspy/clients/__init__.py@@ -5,8 +5,10 @@ from dspy.clients.embedding import Embedderimport litellmimport osfrom pathlib import Path-from litellm.caching import Cache+from dspy.clients.cache import Cacheimport logging+from typing import Optional+from litellm.caching.caching import Cache as LitellmCachelogger = logging.getLogger(__name__)@@ -21,20 +23,66 @@ def _litellm_track_cache_hit_callback(kwargs, completion_response, start_time, elitellm.success_callback = [_litellm_track_cache_hit_callback]-try:- # TODO: There's probably value in getting litellm to support FanoutCache and to separate the limit for- # the LM cache from the embeddings cache. Then we can lower the default 30GB limit.- litellm.cache = Cache(disk_cache_dir=DISK_CACHE_DIR, type="disk")- if litellm.cache.cache.disk_cache.size_limit != DISK_CACHE_LIMIT:- litellm.cache.cache.disk_cache.reset("size_limit", DISK_CACHE_LIMIT)-except Exception as e:- # It's possible that users don't have the write permissions to the cache directory.- # In that case, we'll just disable the cache.- logger.warning("Failed to initialize LiteLLM cache: %s", e)- litellm.cache = None+def configure_cache(+ enable_disk_cache: Optional[bool] = True,+ enable_memory_cache: Optional[bool] = True,+ disk_cache_dir: Optional[str] = DISK_CACHE_DIR,+ disk_size_limit_bytes: Optional[int] = DISK_CACHE_LIMIT,+ memory_max_entries: Optional[int] = 1000000,+ enable_litellm_cache: bool = False,+):+ """Configure the cache for DSPy.++ Args:+ enable_disk_cache: Whether to enable on-disk cache.+ enable_memory_cache: Whether to enable in-memory cache.+ disk_cache_dir: The directory to store the on-disk cache.+ disk_size_limit_bytes: The size limit of the on-disk cache.+ memory_max_entries: The maximum number of entries in the in-memory cache.+ enable_litellm_cache: Whether to enable LiteLLM cache.+ """+ if enable_disk_cache and enable_litellm_cache:+ raise ValueError(+ "Cannot enable both LiteLLM and DSPy on-disk cache, please set at most one of `enable_disk_cache` or "+ "`enable_litellm_cache` to True."+ )++ if enable_litellm_cache:+ try:+ litellm.cache = LitellmCache(disk_cache_dir=DISK_CACHE_DIR, type="disk")++ if litellm.cache.cache.disk_cache.size_limit != DISK_CACHE_LIMIT:+ litellm.cache.cache.disk_cache.reset("size_limit", DISK_CACHE_LIMIT)+ except Exception as e:+ # It's possible that users don't have the write permissions to the cache directory.+ # In that case, we'll just disable the cache.+ logger.warning("Failed to initialize LiteLLM cache: %s", e)+ litellm.cache = None+ else:+ litellm.cache = None++ import dspy++ dspy.cache = Cache(+ enable_disk_cache,+ enable_memory_cache,+ disk_cache_dir,+ disk_size_limit_bytes,+ memory_max_entries,+ )+litellm.telemetry = False+litellm.cache = None # By default we disable litellm cache and use DSPy on-disk cache.++DSPY_CACHE = Cache(+ enable_disk_cache=True,+ enable_memory_cache=True,+ disk_cache_dir=DISK_CACHE_DIR,+ disk_size_limit_bytes=DISK_CACHE_LIMIT,+ memory_max_entries=1000000,+)# Turn off by default to avoid LiteLLM logging during every LM call.litellm.suppress_debug_info = True@@ -61,4 +109,5 @@ __all__ = ["Embedder","enable_litellm_logging","disable_litellm_logging",+ "configure_cache",]diff --git a/dspy/clients/cache.py b/dspy/clients/cache.pynew file mode 100644index 000000000..c73d986ff--- /dev/null+++ b/dspy/clients/cache.py@@ -0,0 +1,215 @@+import copy+import logging+import threading+from functools import wraps+from hashlib import sha256+from typing import Any, Dict, Optional++import cloudpickle+import pydantic+import ujson+from cachetools import LRUCache+from diskcache import FanoutCache++logger = logging.getLogger(__name__)+++class Cache:+ """DSPy Cache++ `Cache` provides 2 levels of caching (in the given order):+ 1. In-memory cache - implemented with cachetools.LRUCache+ 2. On-disk cache - implemented with diskcache.FanoutCache+ """++ def __init__(+ self,+ enable_disk_cache: bool,+ enable_memory_cache: bool,+ disk_cache_dir: str,+ disk_size_limit_bytes: Optional[int] = 1024 * 1024 * 10,+ memory_max_entries: Optional[int] = 1000000,+ ignored_args_for_cache_key: Optional[list[str]] = None,+ ):+ """+ Args:+ enable_disk_cache: Whether to enable on-disk cache.+ enable_memory_cache: Whether to enable in-memory cache.+ disk_cache_dir: The directory where the disk cache is stored.+ disk_size_limit_bytes: The maximum size of the disk cache (in bytes).+ memory_max_entries: The maximum size of the in-memory cache (in number of items).+ ignored_args_for_cache_key: A list of arguments to ignore when computing the cache key from the request.+ """++ self.enable_disk_cache = enable_disk_cache+ self.enable_memory_cache = enable_memory_cache+ if self.enable_memory_cache:+ self.memory_cache = LRUCache(maxsize=memory_max_entries)+ else:+ self.memory_cache = {}+ if self.enable_disk_cache:+ self.disk_cache = FanoutCache(+ shards=16,+ timeout=10,+ directory=disk_cache_dir,+ size_limit=disk_size_limit_bytes,+ )+ else:+ self.disk_cache = {}++ self.ignored_args_for_cache_key = ignored_args_for_cache_key or []++ self._lock = threading.RLock()++ def __contains__(self, key: str) -> bool:+ """Check if a key is in the cache."""+ return key in self.memory_cache or key in self.disk_cache++ def cache_key(self, request: Dict[str, Any]) -> str:+ """+ Obtain a unique cache key for the given request dictionary by hashing its JSON+ representation. For request fields having types that are known to be JSON-incompatible,+ convert them to a JSON-serializable format before hashing.+ """++ def transform_value(value):+ if isinstance(value, type) and issubclass(value, pydantic.BaseModel):+ return value.model_json_schema()+ elif isinstance(value, pydantic.BaseModel):+ return value.model_dump()+ elif callable(value):+ # Try to get the source code of the callable if available+ import inspect++ try:+ # For regular functions, we can get the source code+ return f"<callable_source:{inspect.getsource(value)}>"+ except (TypeError, OSError, IOError):+ # For lambda functions or other callables where source isn't available,+ # use a string representation+ return f"<callable:{value.__name__ if hasattr(value, '__name__') else 'lambda'}>"+ elif isinstance(value, dict):+ return {k: transform_value(v) for k, v in value.items()}+ else:+ return value++ params = {k: transform_value(v) for k, v in request.items() if k not in self.ignored_args_for_cache_key}+ return sha256(ujson.dumps(params, sort_keys=True).encode()).hexdigest()++ def get(self, request: Dict[str, Any]) -> Any:+ try:+ key = self.cache_key(request)+ except Exception:+ logger.debug(f"Failed to generate cache key for request: {request}")+ return None++ if self.enable_memory_cache and key in self.memory_cache:+ with self._lock:+ response = self.memory_cache[key]+ elif self.enable_disk_cache and key in self.disk_cache:+ # Found on disk but not in memory cache, add to memory cache+ response = self.disk_cache[key]+ if self.enable_memory_cache:+ with self._lock:+ self.memory_cache[key] = response+ else:+ return None++ response = copy.deepcopy(response)+ if hasattr(response, "usage"):+ # Clear the usage data when cache is hit, because no LM call is made+ response.usage = {}+ return response++ def put(self, request: Dict[str, Any], value: Any) -> None:+ try:+ key = self.cache_key(request)+ except Exception:+ logger.debug(f"Failed to generate cache key for request: {request}")+ return++ if self.enable_memory_cache:+ with self._lock:+ self.memory_cache[key] = value++ if self.enable_disk_cache:+ try:+ self.disk_cache[key] = value+ except Exception as e:+ # Disk cache writing can fail for different reasons, e.g. disk full or the `value` is not picklable.+ logger.debug(f"Failed to put value in disk cache: {value}, {e}")++ def reset_memory_cache(self) -> None:+ if not self.enable_memory_cache:+ return++ with self._lock:+ self.memory_cache.clear()++ def save_memory_cache(self, filepath: str) -> None:+ if not self.enable_memory_cache:+ return++ with self._lock:+ with open(filepath, "wb") as f:+ cloudpickle.dump(self.memory_cache, f)++ def load_memory_cache(self, filepath: str) -> None:+ if not self.enable_memory_cache:+ return++ with self._lock:+ with open(filepath, "rb") as f:+ self.memory_cache = cloudpickle.load(f)+++def request_cache(cache_arg_name: Optional[str] = None, ignored_args_for_cache_key: Optional[list[str]] = None):+ """Decorator for applying caching to a function based on the request argument.++ Args:+ cache_arg_name: The name of the argument that contains the request. If not provided, the entire kwargs is used+ as the request.+ ignored_args_for_cache_key: A list of arguments to ignore when computing the cache key from the request.+ """++ def decorator(fn):+ @wraps(fn)+ def wrapper(*args, **kwargs):+ import dspy++ cache = dspy.cache+ original_ignored_args_for_cache_key = cache.ignored_args_for_cache_key+ cache.ignored_args_for_cache_key = ignored_args_for_cache_key or []++ # Use fully qualified function name for uniqueness+ fn_identifier = f"{fn.__module__}.{fn.__qualname__}"++ # Create a modified request that includes the function identifier so that it's incorporated into the cache+ # key. Deep copy is required because litellm sometimes modifies the kwargs in place.+ if cache_arg_name:+ # When `cache_arg_name` is provided, use the value of the argument with this name as the request for+ # caching.+ modified_request = copy.deepcopy(kwargs[cache_arg_name])+ else:+ # When `cache_arg_name` is not provided, use the entire kwargs as the request for caching.+ modified_request = copy.deepcopy(kwargs)+ for i, arg in enumerate(args):+ modified_request[f"positional_arg_{i}"] = arg+ modified_request["_fn_identifier"] = fn_identifier++ # Retrieve from cache if available+ cached_result = cache.get(modified_request)++ if cached_result is not None:+ return cached_result++ # Otherwise, compute and store the result+ result = fn(*args, **kwargs)+ cache.put(modified_request, result)++ cache.ignored_args_for_cache_key = original_ignored_args_for_cache_key+ return result++ return wrapper++ return decoratordiff --git a/dspy/clients/embedding.py b/dspy/clients/embedding.pyindex 0c08f188d..8d1a41eed 100644--- a/dspy/clients/embedding.py+++ b/dspy/clients/embedding.py@@ -1,6 +1,8 @@import litellmimport numpy as np+from dspy.clients.cache import request_cache+class Embedder:"""DSPy embedding class.@@ -124,37 +126,52 @@ class Embedder:"""input_batches, caching, kwargs, is_single_input = self._preprocess(inputs, batch_size, caching, **kwargs)+ compute_embeddings = _cached_compute_embeddings if caching else _compute_embeddings+embeddings_list = []for batch in input_batches:- if isinstance(self.model, str):- embedding_response = litellm.embedding(model=self.model, input=batch, caching=caching, **kwargs)- batch_embeddings = [data["embedding"] for data in embedding_response.data]- elif callable(self.model):- batch_embeddings = self.model(batch, **kwargs)- else:- raise ValueError(- f"`model` in `dspy.Embedder` must be a string or a callable, but got {type(self.model)}."- )-- embeddings_list.extend(batch_embeddings)+ embeddings_list.extend(compute_embeddings(self.model, batch, caching=caching, **kwargs))return self._postprocess(embeddings_list, is_single_input)async def acall(self, inputs, batch_size=None, caching=None, **kwargs):input_batches, caching, kwargs, is_single_input = self._preprocess(inputs, batch_size, caching, **kwargs)embeddings_list = []+ acompute_embeddings = _cached_acompute_embeddings if caching else _acompute_embeddingsfor batch in input_batches:- if isinstance(self.model, str):- embedding_response = await litellm.aembedding(model=self.model, input=batch, caching=caching, **kwargs)- batch_embeddings = [data["embedding"] for data in embedding_response.data]- elif callable(self.model):- batch_embeddings = self.model(batch, **kwargs)- else:- raise ValueError(- f"`model` in `dspy.Embedder` must be a string or a callable, but got {type(self.model)}."- )-- embeddings_list.extend(batch_embeddings)+ embeddings_list.extend(acompute_embeddings(self.model, batch, caching=caching, **kwargs))return self._postprocess(embeddings_list, is_single_input)+++def _compute_embeddings(model, batch_inputs, caching=False, **kwargs):+ if isinstance(model, str):+ caching = caching and litellm.cache is not None+ embedding_response = litellm.embedding(model=model, input=batch_inputs, caching=caching, **kwargs)+ return [data["embedding"] for data in embedding_response.data]+ elif callable(model):+ return model(batch_inputs, **kwargs)+ else:+ raise ValueError(f"`model` in `dspy.Embedder` must be a string or a callable, but got {type(model)}.")+++@request_cache(ignored_args_for_cache_key=["api_key", "api_base", "base_url"])+def _cached_compute_embeddings(model, batch_inputs, caching=True, **kwargs):+ return _compute_embeddings(model, batch_inputs, caching=caching, **kwargs)+++async def _acompute_embeddings(model, batch_inputs, caching=False, **kwargs):+ if isinstance(model, str):+ caching = caching and litellm.cache is not None+ embedding_response = await litellm.aembedding(model=model, input=batch_inputs, caching=caching, **kwargs)+ return [data["embedding"] for data in embedding_response.data]+ elif callable(model):+ return model(batch_inputs, **kwargs)+ else:+ raise ValueError(f"`model` in `dspy.Embedder` must be a string or a callable, but got {type(model)}.")+++@request_cache(ignored_args_for_cache_key=["api_key", "api_base", "base_url"])+async def _cached_acompute_embeddings(model, batch_inputs, caching=True, **kwargs):+ return await _acompute_embeddings(model, batch_inputs, caching=caching, **kwargs)diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex add790db1..d092808b1 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -1,20 +1,16 @@-import functoolsimport loggingimport osimport reimport threading-from hashlib import sha256from typing import Any, Dict, List, Literal, Optional, castimport litellm-import pydantic-import ujsonfrom anyio.streams.memory import MemoryObjectSendStreamfrom asyncer import syncify-from cachetools import LRUCache, cachedfrom litellm import RetryPolicyimport dspy+from dspy.clients.cache import request_cachefrom dspy.clients.openai import OpenAIProviderfrom dspy.clients.provider import Provider, TrainingJobfrom dspy.clients.utils_finetune import TrainDataFormat@@ -209,87 +205,18 @@ class LM(BaseLM):return {key: getattr(self, key) for key in state_keys} | self.kwargs-def request_cache(maxsize: Optional[int] = None):- """- A threadsafe decorator to create an in-memory LRU cache for LM inference functions that accept- a dictionary-like LM request. An in-memory cache for LM calls is critical for ensuring- good performance when optimizing and evaluating DSPy LMs (disk caching alone is too slow).-- Args:- maxsize: The maximum size of the cache. If unspecified, no max size is enforced (cache is unbounded).-- Returns:- A decorator that wraps the target function with caching.- """-- def cache_key(request: Dict[str, Any]) -> str:- """- Obtain a unique cache key for the given request dictionary by hashing its JSON- representation. For request fields having types that are known to be JSON-incompatible,- convert them to a JSON-serializable format before hashing.-- Note: Values that cannot be converted to JSON should *not* be ignored / discarded, since- that would potentially lead to cache collisions. For example, consider request A- containing only JSON-convertible values and request B containing the same JSON-convertible- values in addition to one unconvertible value. Discarding the unconvertible value would- lead to a cache collision between requests A and B, even though they are semantically- different.- """-- def transform_value(value):- if isinstance(value, type) and issubclass(value, pydantic.BaseModel):- return value.model_json_schema()- elif isinstance(value, pydantic.BaseModel):- return value.model_dump()- elif callable(value) and hasattr(value, "__code__") and hasattr(value.__code__, "co_code"):- return value.__code__.co_code.decode("utf-8")- else:- # Note: We don't attempt to compute a hash of the value, since the default- # implementation of hash() is id(), which may collide if the same memory address- # is reused for different objects at different times- return value-- params = {k: transform_value(v) for k, v in request.items()}- return sha256(ujson.dumps(params, sort_keys=True).encode()).hexdigest()-- def decorator(func):- @cached(- # NB: cachetools doesn't support maxsize=None; it recommends using float("inf") instead- cache=LRUCache(maxsize=maxsize or float("inf")),- key=lambda key, request, *args, **kwargs: key,- # Use a lock to ensure thread safety for the cache when DSPy LMs are queried- # concurrently, e.g. during optimization and evaluation- lock=threading.RLock(),- )- def func_cached(key: str, request: Dict[str, Any], *args, **kwargs):- return func(request, *args, **kwargs)-- @functools.wraps(func)- def wrapper(request: dict, *args, **kwargs):- try:- key = cache_key(request)- except Exception:- # If the cache key cannot be computed (e.g. because it contains a value that cannot- # be converted to JSON), bypass the cache and call the target function directly- return func(request, *args, **kwargs)- cache_hit = key in func_cached.cache- output = func_cached(key, request, *args, **kwargs)- if cache_hit and hasattr(output, "usage"):- # Clear the usage data when cache is hit, because no LM call is made- output.usage = {}-- return func_cached(key, request, *args, **kwargs)-- return wrapper-- return decorator+@request_cache(cache_arg_name="request", ignored_args_for_cache_key=["api_key", "api_base", "base_url"])+def cached_litellm_completion(request: Dict[str, Any], num_retries: int):+ import litellm+ if litellm.cache:+ litellm_cache_args = {"no-cache": False, "no-store": False}+ else:+ litellm_cache_args = {"no-cache": True, "no-store": True}-@request_cache(maxsize=None)-def cached_litellm_completion(request: Dict[str, Any], num_retries: int):return litellm_completion(request,- cache={"no-cache": False, "no-store": False},+ cache=litellm_cache_args,num_retries=num_retries,)@@ -339,12 +266,19 @@ def litellm_completion(request: Dict[str, Any], num_retries: int, cache={"no-cacreturn stream_completion()-@request_cache(maxsize=None)+@request_cache(cache_arg_name="request", ignored_args_for_cache_key=["api_key", "api_base", "base_url"])def cached_litellm_text_completion(request: Dict[str, Any], num_retries: int):+ import litellm++ if litellm.cache:+ litellm_cache_args = {"no-cache": False, "no-store": False}+ else:+ litellm_cache_args = {"no-cache": True, "no-store": True}+return litellm_text_completion(request,num_retries=num_retries,- cache={"no-cache": False, "no-store": False},+ cache=litellm_cache_args,)diff --git a/dspy/dsp/cache_utils.py b/dspy/dsp/cache_utils.pydeleted file mode 100644index c2d8c8dbd..000000000--- a/dspy/dsp/cache_utils.py+++ /dev/null@@ -1,44 +0,0 @@-import logging-import os-from functools import wraps-from pathlib import Path--from joblib import Memory--from dspy.dsp.utils import dotdict--logger = logging.getLogger(__name__)--cache_turn_on = os.environ.get('DSP_CACHEBOOL', 'True').lower() != 'false'---def noop_decorator(arg=None, *noop_args, **noop_kwargs):- def decorator(func):- @wraps(func)- def wrapper(*args, **kwargs):- return func(*args, **kwargs)-- return wrapper-- if callable(arg):- return decorator(arg)- else:- return decorator--NotebookCacheMemory = dotdict()-NotebookCacheMemory.cache = noop_decorator--CacheMemory = dotdict()-CacheMemory.cache = noop_decorator--if cache_turn_on:- try:- cachedir = os.environ.get('DSP_CACHEDIR') or os.path.join(Path.home(), '.dspy_cache/cachedir_joblib')- CacheMemory = Memory(location=cachedir, verbose=0)-- cachedir2 = os.environ.get('DSP_NOTEBOOK_CACHEDIR')- if cachedir2:- NotebookCacheMemory = Memory(location=cachedir2, verbose=0)- except Exception as e:- logger.warning("Failed to initialize cache: %s", e)- cache_turn_on = Falsediff --git a/dspy/dsp/colbertv2.py b/dspy/dsp/colbertv2.pyindex 5ffaeb9f2..61b34ee5e 100644--- a/dspy/dsp/colbertv2.py+++ b/dspy/dsp/colbertv2.py@@ -1,9 +1,8 @@-import functoolsfrom typing import Any, List, Optional, Unionimport requests-from dspy.dsp.cache_utils import CacheMemory, NotebookCacheMemory+from dspy.clients.cache import request_cachefrom dspy.dsp.utils import dotdict# TODO: Ideally, this takes the name of the index and looks up its port.@@ -35,7 +34,7 @@ class ColBERTv2:return [dotdict(psg) for psg in topk]+@request_cache()def colbertv2_get_request_v2(url: str, query: str, k: int):assert (k <= 100@@ -49,8 +48,7 @@ def colbertv2_get_request_v2(url: str, query: str, k: int):return topk[:k]+@request_cache()def colbertv2_get_request_v2_wrapped(*args, **kwargs):return colbertv2_get_request_v2(*args, **kwargs)@@ -58,7 +56,7 @@ def colbertv2_get_request_v2_wrapped(*args, **kwargs):colbertv2_get_request = colbertv2_get_request_v2_wrapped+@request_cache()def colbertv2_post_request_v2(url: str, query: str, k: int):headers = {"Content-Type": "application/json; charset=utf-8"}payload = {"query": query, "k": k}@@ -67,8 +65,7 @@ def colbertv2_post_request_v2(url: str, query: str, k: int):return res.json()["topk"][:k]+@request_cache()def colbertv2_post_request_v2_wrapped(*args, **kwargs):return colbertv2_post_request_v2(*args, **kwargs)diff --git a/tests/caching/test_caching.py b/tests/caching/test_litellm_cache.pysimilarity index 92%rename from tests/caching/test_caching.pyrename to tests/caching/test_litellm_cache.pyindex e68c9a507..3d25a9ec4 100644--- a/tests/caching/test_caching.py+++ b/tests/caching/test_litellm_cache.py@@ -15,7 +15,9 @@ def temporary_blank_cache_dir(monkeypatch):with tempfile.TemporaryDirectory() as cache_dir_path:monkeypatch.setenv("DSPY_CACHEDIR", cache_dir_path)importlib.reload(dspy.clients)+ dspy.configure_cache(enable_memory_cache=True, enable_disk_cache=False, enable_litellm_cache=True)yield cache_dir_path+ dspy.configure_cache(enable_memory_cache=True, enable_disk_cache=True, enable_litellm_cache=False)@pytest.fixture()@@ -31,7 +33,9 @@ def temporary_populated_cache_dir(monkeypatch):shutil.copytree(populated_cache_path, cache_dir_path, dirs_exist_ok=True)monkeypatch.setenv("DSPY_CACHEDIR", cache_dir_path)importlib.reload(dspy.clients)+ dspy.configure_cache(enable_memory_cache=True, enable_disk_cache=False, enable_litellm_cache=True)yield cache_dir_path+ dspy.configure_cache(enable_memory_cache=True, enable_disk_cache=True, enable_litellm_cache=False)def test_lm_calls_are_cached_across_lm_instances(litellm_test_server, temporary_blank_cache_dir):diff --git a/tests/clients/test_cache.py b/tests/clients/test_cache.pynew file mode 100644index 000000000..2ced975c9--- /dev/null+++ b/tests/clients/test_cache.py@@ -0,0 +1,227 @@+import os+import shutil+from unittest.mock import patch++import pydantic+import pytest+from cachetools import LRUCache+from diskcache import FanoutCache++from dspy.clients.cache import Cache+from dataclasses import dataclass+++@dataclass+class DummyResponse:+ message: str+ usage: dict+++def cache_config(tmp_path):+ """Default cache configuration."""+ return {+ "enable_disk_cache": True,+ "enable_memory_cache": True,+ "disk_cache_dir": str(tmp_path),+ "disk_size_limit_bytes": 1024 * 1024, # 1MB+ "memory_max_entries": 100,+ }+++def cache(cache_config):+ """Create a cache instance with the default configuration."""+ return Cache(**cache_config)+++def test_initialization(tmp_path):+ """Test different cache initialization configurations."""+ # Test memory-only cache+ memory_cache = Cache(+ enable_disk_cache=False,+ enable_memory_cache=True,+ disk_cache_dir="",+ disk_size_limit_bytes=0,+ memory_max_entries=50,+ )+ assert isinstance(memory_cache.memory_cache, LRUCache)+ assert memory_cache.memory_cache.maxsize == 50+ assert memory_cache.disk_cache == {}++ # Test disk-only cache+ disk_cache = Cache(+ enable_disk_cache=True,+ enable_memory_cache=False,+ disk_cache_dir=str(tmp_path),+ disk_size_limit_bytes=1024,+ memory_max_entries=0,+ )+ assert isinstance(disk_cache.disk_cache, FanoutCache)+ assert disk_cache.memory_cache == {}++ # Test disabled cache+ disabled_cache = Cache(+ enable_disk_cache=False,+ enable_memory_cache=False,+ disk_cache_dir="",+ disk_size_limit_bytes=0,+ memory_max_entries=0,+ )+ assert disabled_cache.memory_cache == {}+ assert disabled_cache.disk_cache == {}+++def test_cache_key_generation(cache):+ """Test cache key generation with different types of inputs."""+ # Test with simple dictionary+ request = {"prompt": "Hello", "model": "openai/gpt-4o-mini", "temperature": 0.7}+ key = cache.cache_key(request)+ assert isinstance(key, str)+ assert len(key) == 64 # SHA-256 hash is 64 characters++ # Test with pydantic model+ class TestModel(pydantic.BaseModel):+ name: str+ value: int++ model = TestModel(name="test", value=42)+ request_with_model = {"data": model}+ key_with_model = cache.cache_key(request_with_model)+ assert isinstance(key_with_model, str)++ # Test with pydantic model class+ request_with_model_class = {"model_class": TestModel}+ key_with_model_class = cache.cache_key(request_with_model_class)+ assert isinstance(key_with_model_class, str)+++def test_put_and_get(cache):+ """Test putting and getting from cache."""+ # Test putting and getting from memory cache+ request = {"prompt": "Hello", "model": "openai/gpt-4o-mini", "temperature": 0.7}++ value = DummyResponse(message="This is a test response", usage={"prompt_tokens": 10, "completion_tokens": 20})++ cache.put(request, value)+ result = cache.get(request)++ assert result.message == value.message+ assert result.usage == {}++ # Test with disk cache+ # First, clear memory cache to ensure we're using disk cache+ cache.reset_memory_cache()++ # Get from disk cache+ result_from_disk = cache.get(request)+ assert result_from_disk.message == value.message+ assert result_from_disk.usage == {}++ # Verify it was also added back to memory cache+ assert cache.cache_key(request) in cache.memory_cache+++def test_cache_miss(cache):+ """Test getting a non-existent key."""+ request = {"prompt": "Non-existent", "model": "gpt-4"}+ result = cache.get(request)+ assert result is None+++def test_cache_key_error_handling(cache):+ """Test error handling for unserializable objects."""++ # Test with a request that can't be serialized to JSON+ class UnserializableObject:+ pass++ request = {"data": UnserializableObject()}++ # Should not raise an exception+ result = cache.get(request)+ assert result is None++ # Should not raise an exception+ cache.put(request, "value")+++def test_reset_memory_cache(cache):+ """Test resetting memory cache."""+ # Add some items to the memory cache+ requests = [{"prompt": f"Hello {i}", "model": "openai/gpt-4o-mini"} for i in range(5)]+ for i, req in enumerate(requests):+ cache.put(req, f"Response {i}")++ # Verify items are in memory cache+ for req in requests:+ key = cache.cache_key(req)+ assert key in cache.memory_cache++ # Reset memory cache+ cache.reset_memory_cache()++ # Verify memory cache is empty+ assert len(cache.memory_cache) == 0++ # But disk cache still has the items+ for req in requests:+ result = cache.get(req)+ assert result is not None+++def test_save_and_load_memory_cache(cache, tmp_path):+ """Test saving and loading memory cache."""+ # Add some items to the memory cache+ requests = [{"prompt": f"Hello {i}", "model": "openai/gpt-4o-mini"} for i in range(5)]+ for i, req in enumerate(requests):+ cache.put(req, f"Response {i}")++ # Save memory cache to a temporary file+ temp_cache_file = tmp_path / "memory_cache.pkl"+ cache.save_memory_cache(str(temp_cache_file))++ # Create a new cache instance with disk cache disabled+ new_cache = Cache(+ enable_memory_cache=True,+ enable_disk_cache=False,+ disk_cache_dir=tmp_path / "disk_cache",+ disk_size_limit_bytes=0,+ memory_max_entries=100,+ )++ # Load the memory cache+ new_cache.load_memory_cache(str(temp_cache_file))++ # Verify items are in the new memory cache+ for req in requests:+ result = new_cache.get(req)+ assert result is not None+ assert result == f"Response {requests.index(req)}"+++def test_lm_cache_decorator(cache):+ """Test the lm_cache decorator."""+ from dspy.clients.cache import request_cache++ # Mock the dspy.cache attribute+ with patch("dspy.cache", cache):+ # Define a test function+ @request_cache()+ def test_function(prompt, model):+ return f"Response for {prompt} with {model}"++ # First call should compute the result+ result1 = test_function(prompt="Hello", model="openai/gpt-4o-mini")+ assert result1 == "Response for Hello with openai/gpt-4o-mini"++ # Second call with same arguments should use cache+ with patch.object(cache, "get") as mock_get:+ mock_get.return_value = "Cached response"+ result2 = test_function(prompt="Hello", model="openai/gpt-4o-mini")+ assert result2 == "Cached response"+ mock_get.assert_called_once()++ # Call with different arguments should compute again+ result3 = test_function(prompt="Different", model="openai/gpt-4o-mini")+ assert result3 == "Response for Different with openai/gpt-4o-mini"diff --git a/tests/clients/test_embedding.py b/tests/clients/test_embedding.pyindex 38425cef1..625c7fc77 100644--- a/tests/clients/test_embedding.py+++ b/tests/clients/test_embedding.py@@ -3,6 +3,7 @@ from unittest.mock import Mock, patchimport numpy as npfrom dspy.clients.embedding import Embedder+import dspy# Mock response format similar to litellm's embedding response.@@ -14,7 +15,15 @@ class MockEmbeddingResponse:self.object = "list"-def test_litellm_embedding():+def cache(tmp_path):+ original_cache = dspy.cache+ dspy.configure_cache(disk_cache_dir=tmp_path / ".dspy_cache")+ yield+ dspy.cache = original_cache+++def test_litellm_embedding(cache):model = "text-embedding-ada-002"inputs = ["hello", "world"]mock_embeddings = [@@ -27,17 +36,29 @@ def test_litellm_embedding():mock_litellm.return_value = MockEmbeddingResponse(mock_embeddings)# Create embedding instance and call it.- embedding = Embedder(model)+ embedding = Embedder(model, caching=True)result = embedding(inputs)# Verify litellm was called with correct parameters.- mock_litellm.assert_called_once_with(model=model, input=inputs, caching=True)+ # Because we disable the litellm cache, it should be called with caching=False.+ mock_litellm.assert_called_once_with(model=model, input=inputs, caching=False)assert len(result) == len(inputs)np.testing.assert_allclose(result, mock_embeddings)+ # Second call should be cached.+ result = embedding(inputs)+ assert mock_litellm.call_count == 1+ np.testing.assert_allclose(result, mock_embeddings)-def test_callable_embedding():+ # Disable cache should issue new calls.+ embedding = Embedder(model, caching=False)+ result = embedding(inputs)+ assert mock_litellm.call_count == 2+ np.testing.assert_allclose(result, mock_embeddings)+++def test_callable_embedding(cache):inputs = ["hello", "world", "test"]expected_embeddings = [@@ -46,14 +67,27 @@ def test_callable_embedding():[0.7, 0.8, 0.9], # embedding for "test"]- def mock_embedding_fn(texts):- # Simple callable that returns random embeddings.- return expected_embeddings+ class EmbeddingFn:+ def __init__(self):+ self.call_count = 0++ def __call__(self, texts):+ # Simple callable that returns random embeddings.+ self.call_count += 1+ return expected_embeddings++ embedding_fn = EmbeddingFn()# Create embedding instance with callable- embedding = Embedder(mock_embedding_fn)+ embedding = Embedder(embedding_fn)result = embedding(inputs)+ assert embedding_fn.call_count == 1+ np.testing.assert_allclose(result, expected_embeddings)++ result = embedding(inputs)+ # The second call should be cached.+ assert embedding_fn.call_count == 1np.testing.assert_allclose(result, expected_embeddings)diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.pyindex 637b087ea..c593ffb53 100644--- a/tests/clients/test_lm.py+++ b/tests/clients/test_lm.py@@ -7,6 +7,7 @@ import pytestfrom openai import RateLimitErrorimport dspy+from dspy.utils.usage_tracker import track_usagefrom tests.test_utils.server import litellm_test_server, read_litellm_test_server_request_logs@@ -40,10 +41,17 @@ def test_chat_lms_can_be_queried(litellm_test_server):(False, False),],)-def test_chat_lms_cache(litellm_test_server, cache, cache_in_memory):+def test_litellm_cache(litellm_test_server, cache, cache_in_memory):api_base, _ = litellm_test_serverexpected_response = ["Hi!"]+ original_cache = dspy.cache+ dspy.clients.configure_cache(+ enable_disk_cache=False,+ enable_memory_cache=cache_in_memory,+ enable_litellm_cache=cache,+ )+openai_lm = dspy.LM(model="openai/dspy-test-model",api_base=api_base,@@ -54,6 +62,43 @@ def test_chat_lms_cache(litellm_test_server, cache, cache_in_memory):)assert openai_lm("openai query") == expected_response+ # Reset the cache configuration+ dspy.cache = original_cache+++def test_dspy_cache(litellm_test_server, tmp_path):+ api_base, _ = litellm_test_server++ original_cache = dspy.cache+ dspy.clients.configure_cache(+ enable_disk_cache=True,+ enable_memory_cache=True,+ enable_litellm_cache=False,+ disk_cache_dir=tmp_path / ".disk_cache",+ )+ cache = dspy.cache++ lm = dspy.LM(+ model="openai/dspy-test-model",+ api_base=api_base,+ api_key="fakekey",+ model_type="text",+ )+ with track_usage() as usage_tracker:+ lm("Query")++ assert len(cache.memory_cache) == 1+ cache_key = next(iter(cache.memory_cache.keys()))+ assert cache_key in cache.disk_cache+ assert len(usage_tracker.usage_data) == 1++ with track_usage() as usage_tracker:+ lm("Query")++ assert len(usage_tracker.usage_data) == 0++ dspy.cache = original_cache+def test_text_lms_can_be_queried(litellm_test_server):api_base, _ = litellm_test_server@@ -86,7 +131,9 @@ def test_lm_calls_support_callables(litellm_test_server):api_base=api_base,api_key="fakekey",azure_ad_token_provider=azure_ad_token_provider,+ cache=False,)+lm_with_callable("Query")spy_completion.assert_called_once()diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.pyindex fc89b504e..c09f48063 100644--- a/tests/streaming/test_streaming.py+++ b/tests/streaming/test_streaming.py@@ -14,7 +14,7 @@ async def test_streamify_yields_expected_response_chunks(litellm_test_server):api_base=api_base,api_key="fakekey",)- with dspy.context(lm=lm):+ with dspy.context(lm=lm, adapter=dspy.JSONAdapter()):class TestSignature(dspy.Signature):input_text: str = dspy.InputField()diff --git a/tests/test_utils/server/litellm_server.py b/tests/test_utils/server/litellm_server.pyindex 419ea4578..58801e9e7 100644--- a/tests/test_utils/server/litellm_server.py+++ b/tests/test_utils/server/litellm_server.py@@ -47,6 +47,7 @@ def _get_mock_llm_response(request_kwargs):return litellm.completion(model="gpt-3.5-turbo",messages=[{"role": "user", "content": "Hello world"}],+ usage={"prompt_tokens": 10, "completion_tokens": 10, "total_tokens": 20},mock_response="Hi!",)