DSPy: The framework for programming—not prompting—language models
Python
Don't crash dspy import when cache initialization fails (#7983)
Chen Qiand4b5e8e1dab5
diff --git a/dspy/clients/__init__.py b/dspy/clients/__init__.pyindex 869c544d9..19217fb7d 100644--- a/dspy/clients/__init__.py+++ b/dspy/clients/__init__.py@@ -6,16 +6,25 @@ import litellmimport osfrom pathlib import Pathfrom litellm.caching import Cache+import logging++logger = logging.getLogger(__name__)DISK_CACHE_DIR = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")DISK_CACHE_LIMIT = int(os.environ.get("DSPY_CACHE_LIMIT", 3e10)) # 30 GB default-# TODO: There's probably value in getting litellm to support FanoutCache and to separate the limit for-# the LM cache from the embeddings cache. Then we can lower the default 30GB limit.-litellm.cache = Cache(disk_cache_dir=DISK_CACHE_DIR, type="disk")--if litellm.cache.cache.disk_cache.size_limit != DISK_CACHE_LIMIT:- litellm.cache.cache.disk_cache.reset("size_limit", DISK_CACHE_LIMIT)+try:+ # TODO: There's probably value in getting litellm to support FanoutCache and to separate the limit for+ # the LM cache from the embeddings cache. Then we can lower the default 30GB limit.+ litellm.cache = Cache(disk_cache_dir=DISK_CACHE_DIR, type="disk")++ if litellm.cache.cache.disk_cache.size_limit != DISK_CACHE_LIMIT:+ litellm.cache.cache.disk_cache.reset("size_limit", DISK_CACHE_LIMIT)+except Exception as e:+ # It's possible that users don't have the write permissions to the cache directory.+ # In that case, we'll just disable the cache.+ logger.warning("Failed to initialize LiteLLM cache: %s", e)+ litellm.cache = Nonelitellm.telemetry = Falsediff --git a/dspy/dsp/cache_utils.py b/dspy/dsp/cache_utils.pyindex 15f329750..c2d8c8dbd 100644--- a/dspy/dsp/cache_utils.py+++ b/dspy/dsp/cache_utils.py@@ -1,3 +1,4 @@+import loggingimport osfrom functools import wrapsfrom pathlib import Path@@ -6,6 +7,8 @@ from joblib import Memoryfrom dspy.dsp.utils import dotdict+logger = logging.getLogger(__name__)+cache_turn_on = os.environ.get('DSP_CACHEBOOL', 'True').lower() != 'false'@@ -29,9 +32,13 @@ CacheMemory = dotdict()CacheMemory.cache = noop_decoratorif cache_turn_on:- cachedir = os.environ.get('DSP_CACHEDIR') or os.path.join(Path.home(), '.dspy_cache/cachedir_joblib')- CacheMemory = Memory(location=cachedir, verbose=0)-- cachedir2 = os.environ.get('DSP_NOTEBOOK_CACHEDIR')- if cachedir2:- NotebookCacheMemory = Memory(location=cachedir2, verbose=0)+ try:+ cachedir = os.environ.get('DSP_CACHEDIR') or os.path.join(Path.home(), '.dspy_cache/cachedir_joblib')+ CacheMemory = Memory(location=cachedir, verbose=0)++ cachedir2 = os.environ.get('DSP_NOTEBOOK_CACHEDIR')+ if cachedir2:+ NotebookCacheMemory = Memory(location=cachedir2, verbose=0)+ except Exception as e:+ logger.warning("Failed to initialize cache: %s", e)+ cache_turn_on = Falsediff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.pyindex 3085e255b..dcd7b9421 100644--- a/tests/predict/test_predict.py+++ b/tests/predict/test_predict.py@@ -5,6 +5,7 @@ from datetime import datetimeimport pydanticimport pytestimport ujson+import osimport dspyfrom dspy import Predict, Signature@@ -538,3 +539,33 @@ def test_field_constraints(adapter_type):assert "greater than or equal to: 0.0" in system_messageassert "less than or equal to: 1.0" in system_messageassert "a multiple of the given number: 2" in system_message++[email protected](os.environ.get("OPENAI_API_KEY") is None, reason="Skipping if OPENAI_API_KEY is not set")+def test_litellm_cache_initialization_failure():+ """Test that DSPy handles litellm cache initialization failure gracefully."""+ # Mock Cache to raise a permission error+ mock_cache = MagicMock()+ mock_cache.side_effect = PermissionError("Permission denied")++ import litellm++ # Normal import should have set litellm.cache+ assert litellm.cache is not None++ with patch("litellm.caching.Cache", mock_cache):+ import importlib+ import dspy.clients++ importlib.reload(dspy.clients)++ # On cache initialization failure, litellm.cache should be set to None+ assert litellm.cache is None++ # Create a simple predictor to verify it works without cache+ predictor = Predict("question -> answer")+ dspy.settings.configure(lm=dspy.LM(model="openai/gpt-4o-mini"))++ # No exception should be raised when litellm.cache is None even if we try to use the cache.+ assert dspy.settings.lm.cache == True+ predictor(question="test")