Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feat(clients): allow disable caching to support runtimes without a file system (#10382)

Adriaan Moors515aa8ee46e8
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 609b2ef03..e9dab23ed 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -1,3 +1,5 @@
+import threading
+
from dspy import lm15 as lm15
from dspy.predict import *
from dspy.primitives import *
@@ -41,7 +43,6 @@ from dspy.utils.usage_tracker import track_usage
from dspy.dsp.utils.settings import settings
from dspy.dsp.colbertv2 import ColBERTv2
-from dspy.clients import DSPY_CACHE
from dspy.__metadata__ import __name__, __version__, __description__, __url__, __author__, __author_email__
configure_dspy_loggers(__name__)
@@ -53,4 +54,16 @@ context = settings.context
BootstrapRS = BootstrapFewShotWithRandomSearch
-cache = DSPY_CACHE
+_cache_lock = threading.Lock()
+
+
+def __getattr__(name):
+ """Defer building the cache until it's read, so that configure_cache can fully disable caching if desired."""
+ if name == "cache":
+ from dspy.clients import _get_dspy_cache
+
+ with _cache_lock:
+ if "cache" not in globals():
+ globals()["cache"] = _get_dspy_cache()
+ return globals()["cache"]
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/dspy/clients/__init__.py b/dspy/clients/__init__.py
index c371f3e06..0be5e9a73 100644
--- a/dspy/clients/__init__.py
+++ b/dspy/clients/__init__.py
@@ -1,6 +1,5 @@
import logging
import os
-from pathlib import Path
from typing import Any
from dspy.clients._litellm import get_litellm
@@ -9,10 +8,11 @@ from dspy.clients.cache import Cache
from dspy.clients.embedding import Embedder
from dspy.clients.lm import LM
from dspy.clients.provider import Provider, TrainingJob
+from dspy.utils.caching import default_cache_dir
logger = logging.getLogger(__name__)
-DISK_CACHE_DIR = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")
+DISK_CACHE_DIR = default_cache_dir()
DISK_CACHE_LIMIT = int(os.environ.get("DSPY_CACHE_LIMIT", 3e10)) # 30 GB default
@@ -51,13 +51,15 @@ def configure_cache(
import dspy
- # Update the reference to point to the new cache
- dspy.cache = DSPY_CACHE
+ # Hold the lock the lazy build holds, or a concurrent first read of `dspy.cache` overwrites this cache
+ # with the default one it was already building.
+ with dspy._cache_lock:
+ dspy.cache = DSPY_CACHE
def _get_dspy_cache():
- disk_cache_dir = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")
+ disk_cache_dir = default_cache_dir()
disk_cache_limit = int(os.environ.get("DSPY_CACHE_LIMIT", 3e10))
try:
@@ -81,9 +83,6 @@ def _get_dspy_cache():
return _dspy_cache
-DSPY_CACHE = _get_dspy_cache()
-
-
def configure_litellm_logging(level: str = "ERROR"):
"""Configure LiteLLM logging to the specified level."""
# Litellm uses a global logger called `verbose_logger` to control all loggings.
diff --git a/dspy/clients/cache.py b/dspy/clients/cache.py
index 1db6bcc3e..5057cd92f 100644
--- a/dspy/clients/cache.py
+++ b/dspy/clients/cache.py
@@ -10,13 +10,6 @@ from typing import Any
import cloudpickle
import orjson
import pydantic
-from cachetools import LRUCache
-from diskcache import FanoutCache
-
-from dspy.clients.disk_serialization import (
- DeserializationError,
- restricted_disk,
-)
logger = logging.getLogger(__name__)
@@ -70,7 +63,10 @@ class Cache:
self.enable_disk_cache = enable_disk_cache
self.enable_memory_cache = enable_memory_cache
self.disk_cache_dir = os.fspath(disk_cache_dir) if disk_cache_dir is not None else None
+ # Each tier imports its backend in its own branch, so declining a tier also drops its dependency.
if self.enable_memory_cache:
+ from cachetools import LRUCache
+
if memory_max_entries is None:
raise ValueError("`memory_max_entries` cannot be None. Use `math.inf` if you need an unbounded cache.")
elif memory_max_entries <= 0:
@@ -79,6 +75,10 @@ class Cache:
else:
self.memory_cache = {}
if self.enable_disk_cache:
+ from diskcache import FanoutCache
+
+ from dspy.clients.disk_serialization import restricted_disk
+
fanout_kwargs = dict(
directory=self.disk_cache_dir,
shards=16,
@@ -129,6 +129,8 @@ class Cache:
return self._prepare_cached_response(response)
if self.enable_disk_cache:
+ from dspy.clients.disk_serialization import DeserializationError
+
try:
response = self.disk_cache.get(key)
except DeserializationError:
diff --git a/dspy/utils/caching.py b/dspy/utils/caching.py
index de11b19ba..9bc76ac8b 100644
--- a/dspy/utils/caching.py
+++ b/dspy/utils/caching.py
@@ -1,8 +1,19 @@
import os
from pathlib import Path
-_DEFAULT_CACHE_DIR = os.path.join(Path.home(), ".dspy_cache")
-DSPY_CACHEDIR = os.environ.get("DSPY_CACHEDIR") or _DEFAULT_CACHE_DIR
+
+def default_cache_dir() -> str:
+ """Resolve the cache directory without raising, so that `import dspy` survives having no home directory."""
+ explicit = os.environ.get("DSPY_CACHEDIR")
+ if explicit:
+ return explicit
+ try:
+ return os.path.join(Path.home(), ".dspy_cache")
+ except RuntimeError:
+ return ".dspy_cache"
+
+
+DSPY_CACHEDIR = default_cache_dir()
def create_subdir_in_cachedir(subdir: str) -> str:
diff --git a/tests/clients/test_cache.py b/tests/clients/test_cache.py
index 105dbbece..23d88d446 100644
--- a/tests/clients/test_cache.py
+++ b/tests/clients/test_cache.py
@@ -1,4 +1,6 @@
import os
+import subprocess
+import sys
from dataclasses import dataclass
from unittest.mock import patch
@@ -554,3 +556,12 @@ def test_prepare_cached_response_marks_strict_models_as_cache_hit(cache):
# The original response is not mutated.
assert getattr(response, "cache_hit", False) is False
assert response.usage == {"total_tokens": 3}
+
+
+def test_import_dspy_does_not_build_the_default_cache():
+ """Run in a subprocess: this process imported dspy long ago."""
+ probe = "import dspy; print('cache' in vars(dspy))"
+ result = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True, check=False)
+
+ assert result.returncode == 0, result.stderr
+ assert result.stdout.strip() == "False", "`import dspy` built the default cache"
diff --git a/tests/utils/test_caching.py b/tests/utils/test_caching.py
new file mode 100644
index 000000000..84b89470a
--- /dev/null
+++ b/tests/utils/test_caching.py
@@ -0,0 +1,26 @@
+from pathlib import Path
+from unittest.mock import patch
+
+from dspy.utils.caching import default_cache_dir
+
+
+def test_default_cache_dir_prefers_explicit_env(monkeypatch):
+ monkeypatch.setenv("DSPY_CACHEDIR", "/custom/cache")
+ assert default_cache_dir() == "/custom/cache"
+
+
+def test_default_cache_dir_ignores_empty_env(monkeypatch):
+ monkeypatch.setenv("DSPY_CACHEDIR", "")
+ assert default_cache_dir() == str(Path.home() / ".dspy_cache")
+
+
+def test_default_cache_dir_uses_home(monkeypatch):
+ monkeypatch.delenv("DSPY_CACHEDIR", raising=False)
+ assert default_cache_dir() == str(Path.home() / ".dspy_cache")
+
+
+def test_default_cache_dir_falls_back_when_home_is_unresolvable(monkeypatch):
+ """`Path.home()` raises where no home can be determined, e.g. a WebAssembly guest."""
+ monkeypatch.delenv("DSPY_CACHEDIR", raising=False)
+ with patch.object(Path, "home", side_effect=RuntimeError("Could not determine home directory.")):
+ assert default_cache_dir() == ".dspy_cache"