DSPy: The framework for programming—not prompting—language models
Python
feat(clients): allow disable caching to support runtimes without a file system (#10382)
Adriaan Moors515aa8ee46e8
diff --git a/dspy/__init__.py b/dspy/__init__.pyindex 609b2ef03..e9dab23ed 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -1,3 +1,5 @@+import threading+from dspy import lm15 as lm15from dspy.predict import *from dspy.primitives import *@@ -41,7 +43,6 @@ from dspy.utils.usage_tracker import track_usagefrom dspy.dsp.utils.settings import settingsfrom dspy.dsp.colbertv2 import ColBERTv2-from dspy.clients import DSPY_CACHEfrom dspy.__metadata__ import __name__, __version__, __description__, __url__, __author__, __author_email__configure_dspy_loggers(__name__)@@ -53,4 +54,16 @@ context = settings.contextBootstrapRS = BootstrapFewShotWithRandomSearch-cache = DSPY_CACHE+_cache_lock = threading.Lock()+++def __getattr__(name):+ """Defer building the cache until it's read, so that configure_cache can fully disable caching if desired."""+ if name == "cache":+ from dspy.clients import _get_dspy_cache++ with _cache_lock:+ if "cache" not in globals():+ globals()["cache"] = _get_dspy_cache()+ return globals()["cache"]+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")diff --git a/dspy/clients/__init__.py b/dspy/clients/__init__.pyindex c371f3e06..0be5e9a73 100644--- a/dspy/clients/__init__.py+++ b/dspy/clients/__init__.py@@ -1,6 +1,5 @@import loggingimport os-from pathlib import Pathfrom typing import Anyfrom dspy.clients._litellm import get_litellm@@ -9,10 +8,11 @@ from dspy.clients.cache import Cachefrom dspy.clients.embedding import Embedderfrom dspy.clients.lm import LMfrom dspy.clients.provider import Provider, TrainingJob+from dspy.utils.caching import default_cache_dirlogger = logging.getLogger(__name__)-DISK_CACHE_DIR = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")+DISK_CACHE_DIR = default_cache_dir()DISK_CACHE_LIMIT = int(os.environ.get("DSPY_CACHE_LIMIT", 3e10)) # 30 GB default@@ -51,13 +51,15 @@ def configure_cache(import dspy- # Update the reference to point to the new cache- dspy.cache = DSPY_CACHE+ # Hold the lock the lazy build holds, or a concurrent first read of `dspy.cache` overwrites this cache+ # with the default one it was already building.+ with dspy._cache_lock:+ dspy.cache = DSPY_CACHEdef _get_dspy_cache():- disk_cache_dir = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")+ disk_cache_dir = default_cache_dir()disk_cache_limit = int(os.environ.get("DSPY_CACHE_LIMIT", 3e10))try:@@ -81,9 +83,6 @@ def _get_dspy_cache():return _dspy_cache-DSPY_CACHE = _get_dspy_cache()--def configure_litellm_logging(level: str = "ERROR"):"""Configure LiteLLM logging to the specified level."""# Litellm uses a global logger called `verbose_logger` to control all loggings.diff --git a/dspy/clients/cache.py b/dspy/clients/cache.pyindex 1db6bcc3e..5057cd92f 100644--- a/dspy/clients/cache.py+++ b/dspy/clients/cache.py@@ -10,13 +10,6 @@ from typing import Anyimport cloudpickleimport orjsonimport pydantic-from cachetools import LRUCache-from diskcache import FanoutCache--from dspy.clients.disk_serialization import (- DeserializationError,- restricted_disk,-)logger = logging.getLogger(__name__)@@ -70,7 +63,10 @@ class Cache:self.enable_disk_cache = enable_disk_cacheself.enable_memory_cache = enable_memory_cacheself.disk_cache_dir = os.fspath(disk_cache_dir) if disk_cache_dir is not None else None+ # Each tier imports its backend in its own branch, so declining a tier also drops its dependency.if self.enable_memory_cache:+ from cachetools import LRUCache+if memory_max_entries is None:raise ValueError("`memory_max_entries` cannot be None. Use `math.inf` if you need an unbounded cache.")elif memory_max_entries <= 0:@@ -79,6 +75,10 @@ class Cache:else:self.memory_cache = {}if self.enable_disk_cache:+ from diskcache import FanoutCache++ from dspy.clients.disk_serialization import restricted_disk+fanout_kwargs = dict(directory=self.disk_cache_dir,shards=16,@@ -129,6 +129,8 @@ class Cache:return self._prepare_cached_response(response)if self.enable_disk_cache:+ from dspy.clients.disk_serialization import DeserializationError+try:response = self.disk_cache.get(key)except DeserializationError:diff --git a/dspy/utils/caching.py b/dspy/utils/caching.pyindex de11b19ba..9bc76ac8b 100644--- a/dspy/utils/caching.py+++ b/dspy/utils/caching.py@@ -1,8 +1,19 @@import osfrom pathlib import Path-_DEFAULT_CACHE_DIR = os.path.join(Path.home(), ".dspy_cache")-DSPY_CACHEDIR = os.environ.get("DSPY_CACHEDIR") or _DEFAULT_CACHE_DIR++def default_cache_dir() -> str:+ """Resolve the cache directory without raising, so that `import dspy` survives having no home directory."""+ explicit = os.environ.get("DSPY_CACHEDIR")+ if explicit:+ return explicit+ try:+ return os.path.join(Path.home(), ".dspy_cache")+ except RuntimeError:+ return ".dspy_cache"+++DSPY_CACHEDIR = default_cache_dir()def create_subdir_in_cachedir(subdir: str) -> str:diff --git a/tests/clients/test_cache.py b/tests/clients/test_cache.pyindex 105dbbece..23d88d446 100644--- a/tests/clients/test_cache.py+++ b/tests/clients/test_cache.py@@ -1,4 +1,6 @@import os+import subprocess+import sysfrom dataclasses import dataclassfrom unittest.mock import patch@@ -554,3 +556,12 @@ def test_prepare_cached_response_marks_strict_models_as_cache_hit(cache):# The original response is not mutated.assert getattr(response, "cache_hit", False) is Falseassert response.usage == {"total_tokens": 3}+++def test_import_dspy_does_not_build_the_default_cache():+ """Run in a subprocess: this process imported dspy long ago."""+ probe = "import dspy; print('cache' in vars(dspy))"+ result = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True, check=False)++ assert result.returncode == 0, result.stderr+ assert result.stdout.strip() == "False", "`import dspy` built the default cache"diff --git a/tests/utils/test_caching.py b/tests/utils/test_caching.pynew file mode 100644index 000000000..84b89470a--- /dev/null+++ b/tests/utils/test_caching.py@@ -0,0 +1,26 @@+from pathlib import Path+from unittest.mock import patch++from dspy.utils.caching import default_cache_dir+++def test_default_cache_dir_prefers_explicit_env(monkeypatch):+ monkeypatch.setenv("DSPY_CACHEDIR", "/custom/cache")+ assert default_cache_dir() == "/custom/cache"+++def test_default_cache_dir_ignores_empty_env(monkeypatch):+ monkeypatch.setenv("DSPY_CACHEDIR", "")+ assert default_cache_dir() == str(Path.home() / ".dspy_cache")+++def test_default_cache_dir_uses_home(monkeypatch):+ monkeypatch.delenv("DSPY_CACHEDIR", raising=False)+ assert default_cache_dir() == str(Path.home() / ".dspy_cache")+++def test_default_cache_dir_falls_back_when_home_is_unresolvable(monkeypatch):+ """`Path.home()` raises where no home can be determined, e.g. a WebAssembly guest."""+ monkeypatch.delenv("DSPY_CACHEDIR", raising=False)+ with patch.object(Path, "home", side_effect=RuntimeError("Could not determine home directory.")):+ assert default_cache_dir() == ".dspy_cache"