Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Don't crash dspy import when cache initialization fails (#7983)

Chen Qiand4b5e8e1dab5
diff --git a/dspy/clients/__init__.py b/dspy/clients/__init__.py
index 869c544d9..19217fb7d 100644
--- a/dspy/clients/__init__.py
+++ b/dspy/clients/__init__.py
@@ -6,16 +6,25 @@ import litellm
import os
from pathlib import Path
from litellm.caching import Cache
+import logging
+
+logger = logging.getLogger(__name__)
DISK_CACHE_DIR = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")
DISK_CACHE_LIMIT = int(os.environ.get("DSPY_CACHE_LIMIT", 3e10)) # 30 GB default
-# TODO: There's probably value in getting litellm to support FanoutCache and to separate the limit for
-# the LM cache from the embeddings cache. Then we can lower the default 30GB limit.
-litellm.cache = Cache(disk_cache_dir=DISK_CACHE_DIR, type="disk")
-
-if litellm.cache.cache.disk_cache.size_limit != DISK_CACHE_LIMIT:
- litellm.cache.cache.disk_cache.reset("size_limit", DISK_CACHE_LIMIT)
+try:
+ # TODO: There's probably value in getting litellm to support FanoutCache and to separate the limit for
+ # the LM cache from the embeddings cache. Then we can lower the default 30GB limit.
+ litellm.cache = Cache(disk_cache_dir=DISK_CACHE_DIR, type="disk")
+
+ if litellm.cache.cache.disk_cache.size_limit != DISK_CACHE_LIMIT:
+ litellm.cache.cache.disk_cache.reset("size_limit", DISK_CACHE_LIMIT)
+except Exception as e:
+ # It's possible that users don't have the write permissions to the cache directory.
+ # In that case, we'll just disable the cache.
+ logger.warning("Failed to initialize LiteLLM cache: %s", e)
+ litellm.cache = None
litellm.telemetry = False
diff --git a/dspy/dsp/cache_utils.py b/dspy/dsp/cache_utils.py
index 15f329750..c2d8c8dbd 100644
--- a/dspy/dsp/cache_utils.py
+++ b/dspy/dsp/cache_utils.py
@@ -1,3 +1,4 @@
+import logging
import os
from functools import wraps
from pathlib import Path
@@ -6,6 +7,8 @@ from joblib import Memory
from dspy.dsp.utils import dotdict
+logger = logging.getLogger(__name__)
+
cache_turn_on = os.environ.get('DSP_CACHEBOOL', 'True').lower() != 'false'
@@ -29,9 +32,13 @@ CacheMemory = dotdict()
CacheMemory.cache = noop_decorator
if cache_turn_on:
- cachedir = os.environ.get('DSP_CACHEDIR') or os.path.join(Path.home(), '.dspy_cache/cachedir_joblib')
- CacheMemory = Memory(location=cachedir, verbose=0)
-
- cachedir2 = os.environ.get('DSP_NOTEBOOK_CACHEDIR')
- if cachedir2:
- NotebookCacheMemory = Memory(location=cachedir2, verbose=0)
+ try:
+ cachedir = os.environ.get('DSP_CACHEDIR') or os.path.join(Path.home(), '.dspy_cache/cachedir_joblib')
+ CacheMemory = Memory(location=cachedir, verbose=0)
+
+ cachedir2 = os.environ.get('DSP_NOTEBOOK_CACHEDIR')
+ if cachedir2:
+ NotebookCacheMemory = Memory(location=cachedir2, verbose=0)
+ except Exception as e:
+ logger.warning("Failed to initialize cache: %s", e)
+ cache_turn_on = False
diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py
index 3085e255b..dcd7b9421 100644
--- a/tests/predict/test_predict.py
+++ b/tests/predict/test_predict.py
@@ -5,6 +5,7 @@ from datetime import datetime
import pydantic
import pytest
import ujson
+import os
import dspy
from dspy import Predict, Signature
@@ -538,3 +539,33 @@ def test_field_constraints(adapter_type):
assert "greater than or equal to: 0.0" in system_message
assert "less than or equal to: 1.0" in system_message
assert "a multiple of the given number: 2" in system_message
+
+
[email protected](os.environ.get("OPENAI_API_KEY") is None, reason="Skipping if OPENAI_API_KEY is not set")
+def test_litellm_cache_initialization_failure():
+ """Test that DSPy handles litellm cache initialization failure gracefully."""
+ # Mock Cache to raise a permission error
+ mock_cache = MagicMock()
+ mock_cache.side_effect = PermissionError("Permission denied")
+
+ import litellm
+
+ # Normal import should have set litellm.cache
+ assert litellm.cache is not None
+
+ with patch("litellm.caching.Cache", mock_cache):
+ import importlib
+ import dspy.clients
+
+ importlib.reload(dspy.clients)
+
+ # On cache initialization failure, litellm.cache should be set to None
+ assert litellm.cache is None
+
+ # Create a simple predictor to verify it works without cache
+ predictor = Predict("question -> answer")
+ dspy.settings.configure(lm=dspy.LM(model="openai/gpt-4o-mini"))
+
+ # No exception should be raised when litellm.cache is None even if we try to use the cache.
+ assert dspy.settings.lm.cache == True
+ predictor(question="test")