DSPy: The framework for programming—not prompting—language models
Python
Fix cache bugs: os.fspath(None) crash, double disk lookups, lazy logging (#9628)
Isaac Miller8d12bdafc8f1
diff --git a/dspy/clients/cache.py b/dspy/clients/cache.pyindex a46f27723..6e52e74fe 100644--- a/dspy/clients/cache.py+++ b/dspy/clients/cache.py@@ -1,6 +1,7 @@import copyimport inspectimport logging+import osimport threadingfrom functools import wrapsfrom hashlib import sha256@@ -15,6 +16,22 @@ from diskcache import FanoutCachelogger = logging.getLogger(__name__)+def _transform_value(value):+ if isinstance(value, type) and issubclass(value, pydantic.BaseModel):+ return value.model_json_schema()+ elif isinstance(value, pydantic.BaseModel):+ return value.model_dump(mode="json")+ elif callable(value):+ try:+ return f"<callable_source:{inspect.getsource(value)}>"+ except (TypeError, OSError):+ return f"<callable:{value.__name__ if hasattr(value, '__name__') else 'lambda'}>"+ elif isinstance(value, dict):+ return {k: _transform_value(v) for k, v in value.items()}+ else:+ return value++class Cache:"""DSPy Cache@@ -27,7 +44,7 @@ class Cache:self,enable_disk_cache: bool,enable_memory_cache: bool,- disk_cache_dir: str,+ disk_cache_dir: str | None,disk_size_limit_bytes: int | None = 1024 * 1024 * 10,memory_max_entries: int = 1000000,):@@ -42,6 +59,7 @@ class Cache:self.enable_disk_cache = enable_disk_cacheself.enable_memory_cache = enable_memory_cache+ self.disk_cache_dir = os.fspath(disk_cache_dir) if disk_cache_dir is not None else Noneif self.enable_memory_cache:if memory_max_entries is None:raise ValueError("`memory_max_entries` cannot be None. Use `math.inf` if you need an unbounded cache.")@@ -52,9 +70,9 @@ class Cache:self.memory_cache = {}if self.enable_disk_cache:self.disk_cache = FanoutCache(+ directory=self.disk_cache_dir,shards=16,timeout=10,- directory=disk_cache_dir,size_limit=disk_size_limit_bytes,)else:@@ -74,54 +92,38 @@ class Cache:"""ignored_args_for_cache_key = ignored_args_for_cache_key or []-- def transform_value(value):- if isinstance(value, type) and issubclass(value, pydantic.BaseModel):- return value.model_json_schema()- elif isinstance(value, pydantic.BaseModel):- return value.model_dump(mode="json")- elif callable(value):- # Try to get the source code of the callable if available- import inspect-- try:- # For regular functions, we can get the source code- return f"<callable_source:{inspect.getsource(value)}>"- except (TypeError, OSError):- # For lambda functions or other callables where source isn't available,- # use a string representation- return f"<callable:{value.__name__ if hasattr(value, '__name__') else 'lambda'}>"- elif isinstance(value, dict):- return {k: transform_value(v) for k, v in value.items()}- else:- return value-- params = {k: transform_value(v) for k, v in request.items() if k not in ignored_args_for_cache_key}+ params = {k: _transform_value(v) for k, v in request.items() if k not in ignored_args_for_cache_key}return sha256(orjson.dumps(params, option=orjson.OPT_SORT_KEYS)).hexdigest()def get(self, request: dict[str, Any], ignored_args_for_cache_key: list[str] | None = None) -> Any:-if not self.enable_memory_cache and not self.enable_disk_cache:return Nonetry:key = self.cache_key(request, ignored_args_for_cache_key)except Exception:- logger.debug(f"Failed to generate cache key for request: {request}")+ logger.debug("Failed to generate cache key for request: %s", request)return None- if self.enable_memory_cache and key in self.memory_cache:+ if self.enable_memory_cache:with self._lock:- response = self.memory_cache[key]- elif self.enable_disk_cache and key in self.disk_cache:- # Found on disk but not in memory cache, add to memory cache- response = self.disk_cache[key]+ response = self.memory_cache.get(key)+ if response is not None:+ return self._prepare_cached_response(response)++ if self.enable_disk_cache:+ response = self.disk_cache.get(key)+ if response is None:+ return None+if self.enable_memory_cache:with self._lock:self.memory_cache[key] = response- else:- return None+ return self._prepare_cached_response(response)++ return None+ def _prepare_cached_response(self, response):response = copy.deepcopy(response)if hasattr(response, "usage"):# Clear the usage data when cache is hit, because no LM call is made@@ -145,7 +147,7 @@ class Cache:try:key = self.cache_key(request, ignored_args_for_cache_key)except Exception:- logger.debug(f"Failed to generate cache key for request: {request}")+ logger.debug("Failed to generate cache key for request: %s", request)returnif enable_memory_cache:@@ -154,10 +156,9 @@ class Cache:if self.enable_disk_cache:try:- self.disk_cache[key] = value+ self.disk_cache.set(key, value)except Exception as e:- # Disk cache writing can fail for different reasons, e.g. disk full or the `value` is not picklable.- logger.debug(f"Failed to put value in disk cache: {value}, {e}")+ logger.debug("Failed to put value in disk cache: %s, %s", value, e)def reset_memory_cache(self) -> None:if not self.enable_memory_cache:@@ -176,8 +177,10 @@ class Cache:def load_memory_cache(self, filepath: str, allow_pickle: bool = False) -> None:if not allow_pickle:- raise ValueError("Loading untrusted .pkl files can run arbitrary code, which may be dangerous. \- Set `allow_pickle=True` to load if you are running in a trusted environment and the file is from a trusted source.")+ raise ValueError(+ "Loading untrusted .pkl files can run arbitrary code, which may be dangerous. "+ "Set `allow_pickle=True` to load if you are running in a trusted environment and the file is from a trusted source."+ )if not self.enable_memory_cache:returndiff --git a/tests/clients/test_cache.py b/tests/clients/test_cache.pyindex 02fb27f01..1d7c0db83 100644--- a/tests/clients/test_cache.py+++ b/tests/clients/test_cache.py@@ -144,26 +144,17 @@ def test_put_and_get(cache):def test_cache_miss(cache):"""Test getting a non-existent key."""- request = {"prompt": "Non-existent", "model": "gpt-4"}- result = cache.get(request)- assert result is None-+ assert cache.get({"prompt": "Non-existent", "model": "gpt-4"}) is None-def test_cache_key_error_handling(cache):- """Test error handling for unserializable objects."""- # Test with a request that can't be serialized to JSON+def test_unserializable_key(cache):+ """Unserializable request key returns None without raising."""class UnserializableObject:passrequest = {"data": UnserializableObject()}-- # Should not raise an exception- result = cache.get(request)- assert result is None-- # Should not raise an exception- cache.put(request, "value")+ assert cache.get(request) is None+ cache.put(request, "value") # should not raisedef test_reset_memory_cache(cache):@@ -369,3 +360,13 @@ def test_cache_fallback_on_restricted_environment():os.environ.pop("DSPY_CACHEDIR", None)else:os.environ["DSPY_CACHEDIR"] = old_env+++def test_cache_init_with_disk_disabled_and_none_dir():+ cache = Cache(+ enable_disk_cache=False,+ enable_memory_cache=True,+ disk_cache_dir=None,+ )+ assert cache.disk_cache_dir is None+ assert cache.enable_disk_cache is False