DSPy: The framework for programming—not prompting—language models
Python
Fix: enforce positive `memory_max_entries` for in-memory cache (#9128)
Masahiro Hiramori252878dde5a1
diff --git a/dspy/clients/__init__.py b/dspy/clients/__init__.pyindex 43a5c9403..7972bba8a 100644--- a/dspy/clients/__init__.py+++ b/dspy/clients/__init__.py@@ -14,12 +14,14 @@ logger = logging.getLogger(__name__)DISK_CACHE_DIR = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")DISK_CACHE_LIMIT = int(os.environ.get("DSPY_CACHE_LIMIT", 3e10)) # 30 GB default++def configure_cache(enable_disk_cache: bool | None = True,enable_memory_cache: bool | None = True,disk_cache_dir: str | None = DISK_CACHE_DIR,disk_size_limit_bytes: int | None = DISK_CACHE_LIMIT,- memory_max_entries: int | None = 1000000,+ memory_max_entries: int = 1000000,):"""Configure the cache for DSPy.@@ -28,7 +30,8 @@ def configure_cache(enable_memory_cache: Whether to enable in-memory cache.disk_cache_dir: The directory to store the on-disk cache.disk_size_limit_bytes: The size limit of the on-disk cache.- memory_max_entries: The maximum number of entries in the in-memory cache.+ memory_max_entries: The maximum number of entries in the in-memory cache. To allow the cache to grow without+ bounds, set this parameter to `math.inf` or a similar value."""DSPY_CACHE = Cache(@@ -40,6 +43,7 @@ def configure_cache()import dspy+# Update the reference to point to the new cachedspy.cache = DSPY_CACHE@@ -47,6 +51,7 @@ def configure_cache(litellm.telemetry = Falselitellm.cache = None # By default we disable LiteLLM cache and use DSPy on-disk cache.+def _get_dspy_cache():disk_cache_dir = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")disk_cache_limit = int(os.environ.get("DSPY_CACHE_LIMIT", 3e10))@@ -71,6 +76,7 @@ def _get_dspy_cache():)return _dspy_cache+DSPY_CACHE = _get_dspy_cache()if "LITELLM_LOCAL_MODEL_COST_MAP" not in os.environ:diff --git a/dspy/clients/cache.py b/dspy/clients/cache.pyindex 4773306cc..a46f27723 100644--- a/dspy/clients/cache.py+++ b/dspy/clients/cache.py@@ -29,7 +29,7 @@ class Cache:enable_memory_cache: bool,disk_cache_dir: str,disk_size_limit_bytes: int | None = 1024 * 1024 * 10,- memory_max_entries: int | None = 1000000,+ memory_max_entries: int = 1000000,):"""Args:@@ -43,6 +43,10 @@ class Cache:self.enable_disk_cache = enable_disk_cacheself.enable_memory_cache = enable_memory_cacheif self.enable_memory_cache:+ if memory_max_entries is None:+ raise ValueError("`memory_max_entries` cannot be None. Use `math.inf` if you need an unbounded cache.")+ elif memory_max_entries <= 0:+ raise ValueError(f"`memory_max_entries` must be a positive number, but received {memory_max_entries}")self.memory_cache = LRUCache(maxsize=memory_max_entries)else:self.memory_cache = {}diff --git a/tests/clients/test_cache.py b/tests/clients/test_cache.pyindex 20e0af323..02fb27f01 100644--- a/tests/clients/test_cache.py+++ b/tests/clients/test_cache.py@@ -71,6 +71,27 @@ def test_initialization(tmp_path):assert disabled_cache.disk_cache == {}+def test_invalid_cache_initialization():+ with pytest.raises(ValueError, match=r"`memory_max_entries` must be a positive number, but received -1"):+ Cache(+ enable_disk_cache=False,+ enable_memory_cache=True,+ disk_cache_dir="",+ disk_size_limit_bytes=0,+ memory_max_entries=-1,+ )+ with pytest.raises(+ ValueError, match=r"`memory_max_entries` cannot be None. Use `math.inf` if you need an unbounded cache."+ ):+ Cache(+ enable_disk_cache=False,+ enable_memory_cache=True,+ disk_cache_dir="",+ disk_size_limit_bytes=0,+ memory_max_entries=None,+ )++def test_cache_key_generation(cache):"""Test cache key generation with different types of inputs."""# Test with simple dictionary