Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Fix: enforce positive `memory_max_entries` for in-memory cache (#9128)

Masahiro Hiramori252878dde5a1
diff --git a/dspy/clients/__init__.py b/dspy/clients/__init__.py
index 43a5c9403..7972bba8a 100644
--- a/dspy/clients/__init__.py
+++ b/dspy/clients/__init__.py
@@ -14,12 +14,14 @@ logger = logging.getLogger(__name__)
DISK_CACHE_DIR = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")
DISK_CACHE_LIMIT = int(os.environ.get("DSPY_CACHE_LIMIT", 3e10)) # 30 GB default
+
+
def configure_cache(
enable_disk_cache: bool | None = True,
enable_memory_cache: bool | None = True,
disk_cache_dir: str | None = DISK_CACHE_DIR,
disk_size_limit_bytes: int | None = DISK_CACHE_LIMIT,
- memory_max_entries: int | None = 1000000,
+ memory_max_entries: int = 1000000,
):
"""Configure the cache for DSPy.
@@ -28,7 +30,8 @@ def configure_cache(
enable_memory_cache: Whether to enable in-memory cache.
disk_cache_dir: The directory to store the on-disk cache.
disk_size_limit_bytes: The size limit of the on-disk cache.
- memory_max_entries: The maximum number of entries in the in-memory cache.
+ memory_max_entries: The maximum number of entries in the in-memory cache. To allow the cache to grow without
+ bounds, set this parameter to `math.inf` or a similar value.
"""
DSPY_CACHE = Cache(
@@ -40,6 +43,7 @@ def configure_cache(
)
import dspy
+
# Update the reference to point to the new cache
dspy.cache = DSPY_CACHE
@@ -47,6 +51,7 @@ def configure_cache(
litellm.telemetry = False
litellm.cache = None # By default we disable LiteLLM cache and use DSPy on-disk cache.
+
def _get_dspy_cache():
disk_cache_dir = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")
disk_cache_limit = int(os.environ.get("DSPY_CACHE_LIMIT", 3e10))
@@ -71,6 +76,7 @@ def _get_dspy_cache():
)
return _dspy_cache
+
DSPY_CACHE = _get_dspy_cache()
if "LITELLM_LOCAL_MODEL_COST_MAP" not in os.environ:
diff --git a/dspy/clients/cache.py b/dspy/clients/cache.py
index 4773306cc..a46f27723 100644
--- a/dspy/clients/cache.py
+++ b/dspy/clients/cache.py
@@ -29,7 +29,7 @@ class Cache:
enable_memory_cache: bool,
disk_cache_dir: str,
disk_size_limit_bytes: int | None = 1024 * 1024 * 10,
- memory_max_entries: int | None = 1000000,
+ memory_max_entries: int = 1000000,
):
"""
Args:
@@ -43,6 +43,10 @@ class Cache:
self.enable_disk_cache = enable_disk_cache
self.enable_memory_cache = enable_memory_cache
if self.enable_memory_cache:
+ if memory_max_entries is None:
+ raise ValueError("`memory_max_entries` cannot be None. Use `math.inf` if you need an unbounded cache.")
+ elif memory_max_entries <= 0:
+ raise ValueError(f"`memory_max_entries` must be a positive number, but received {memory_max_entries}")
self.memory_cache = LRUCache(maxsize=memory_max_entries)
else:
self.memory_cache = {}
diff --git a/tests/clients/test_cache.py b/tests/clients/test_cache.py
index 20e0af323..02fb27f01 100644
--- a/tests/clients/test_cache.py
+++ b/tests/clients/test_cache.py
@@ -71,6 +71,27 @@ def test_initialization(tmp_path):
assert disabled_cache.disk_cache == {}
+def test_invalid_cache_initialization():
+ with pytest.raises(ValueError, match=r"`memory_max_entries` must be a positive number, but received -1"):
+ Cache(
+ enable_disk_cache=False,
+ enable_memory_cache=True,
+ disk_cache_dir="",
+ disk_size_limit_bytes=0,
+ memory_max_entries=-1,
+ )
+ with pytest.raises(
+ ValueError, match=r"`memory_max_entries` cannot be None. Use `math.inf` if you need an unbounded cache."
+ ):
+ Cache(
+ enable_disk_cache=False,
+ enable_memory_cache=True,
+ disk_cache_dir="",
+ disk_size_limit_bytes=0,
+ memory_max_entries=None,
+ )
+
+
def test_cache_key_generation(cache):
"""Test cache key generation with different types of inputs."""
# Test with simple dictionary