DSPy: The framework for programming—not prompting—language models
Python
Integrate cachetools for in-memory LM caching, including unhashable types & pydantic (#1896)
Corey Zumara8d11075daf7
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.pyindex 4dffc17e1..737de772a 100644--- a/dspy/clients/lm.py+++ b/dspy/clients/lm.py@@ -4,10 +4,13 @@ import osimport threadingimport uuidfrom datetime import datetime+from hashlib import sha256from typing import Any, Dict, List, Literal, Optionalimport litellm+import pydanticimport ujson+from cachetools import LRUCache, cachedfrom dspy.adapters.base import Adapterfrom dspy.clients.openai import OpenAIProvider@@ -92,7 +95,7 @@ class LM(BaseLM):completion = cached_litellm_text_completion if cache else litellm_text_completionresponse = completion(- request=ujson.dumps(dict(model=self.model, messages=messages, **kwargs)),+ request=dict(model=self.model, messages=messages, **kwargs),num_retries=self.num_retries,)outputs = [c.message.content if hasattr(c, "message") else c["text"] for c in response["choices"]]@@ -153,7 +156,11 @@ class LM(BaseLM):thread = threading.Thread(target=thread_function_wrapper)model_to_finetune = self.finetuning_model or self.modeljob = self.provider.TrainingJob(- thread=thread, model=model_to_finetune, train_data=train_data, train_kwargs=train_kwargs, data_format=data_format+ thread=thread,+ model=model_to_finetune,+ train_data=train_data,+ train_kwargs=train_kwargs,+ data_format=data_format,)thread.start()@@ -212,8 +219,45 @@ class LM(BaseLM):return new_instance[email protected]_cache(maxsize=None)-def cached_litellm_completion(request, num_retries: int):+def request_cache(maxsize: Optional[int] = None):+ """+ A threadsafe decorator to create an in-memory LRU cache for LM inference functions that accept+ a dictionary-like LM request. An in-memory cache for LM calls is critical for ensuring+ good performance when optimizing and evaluating DSPy LMs (disk caching alone is too slow).++ Args:+ maxsize: The maximum size of the cache. If unspecified, no max size is enforced (cache is unbounded).++ Returns:+ A decorator that wraps the target function with caching.+ """++ def cache_key(request: Dict[str, Any]) -> str:+ # Transform Pydantic models into JSON-convertible format and exclude unhashable objects+ params = {k: (v.dict() if isinstance(v, pydantic.BaseModel) else v) for k, v in request.items()}+ params = {k: v for k, v in params.items() if not callable(v)}+ return sha256(ujson.dumps(params, sort_keys=True).encode()).hexdigest()++ def decorator(func):+ @cached(+ # NB: cachetools doesn't support maxsize=None; it recommends using float("inf") instead+ cache=LRUCache(maxsize=maxsize or float("inf")),+ key=lambda request, *args, **kwargs: cache_key(request),+ # Use a lock to ensure thread safety for the cache when DSPy LMs are queried+ # concurrently, e.g. during optimization and evaluation+ lock=threading.Lock(),+ )+ @functools.wraps(func)+ def wrapper(request: dict, *args, **kwargs):+ return func(request, *args, **kwargs)++ return wrapper++ return decorator+++@request_cache(maxsize=None)+def cached_litellm_completion(request: Dict[str, Any], num_retries: int):return litellm_completion(request,cache={"no-cache": False, "no-store": False},@@ -221,17 +265,16 @@ def cached_litellm_completion(request, num_retries: int):)-def litellm_completion(request, num_retries: int, cache={"no-cache": True, "no-store": True}):- kwargs = ujson.loads(request)+def litellm_completion(request: Dict[str, Any], num_retries: int, cache={"no-cache": True, "no-store": True}):return litellm.completion(num_retries=num_retries,cache=cache,- **kwargs,+ **request,)[email protected]_cache(maxsize=None)-def cached_litellm_text_completion(request, num_retries: int):+@request_cache(maxsize=None)+def cached_litellm_text_completion(request: Dict[str, Any], num_retries: int):return litellm_text_completion(request,num_retries=num_retries,@@ -239,20 +282,18 @@ def cached_litellm_text_completion(request, num_retries: int):)-def litellm_text_completion(request, num_retries: int, cache={"no-cache": True, "no-store": True}):- kwargs = ujson.loads(request)-+def litellm_text_completion(request: Dict[str, Any], num_retries: int, cache={"no-cache": True, "no-store": True}):# Extract the provider and model from the model string.# TODO: Not all the models are in the format of "provider/model"- model = kwargs.pop("model").split("/", 1)+ model = request.pop("model").split("/", 1)provider, model = model[0] if len(model) > 1 else "openai", model[-1]- # Use the API key and base from the kwargs, or from the environment.- api_key = kwargs.pop("api_key", None) or os.getenv(f"{provider}_API_KEY")- api_base = kwargs.pop("api_base", None) or os.getenv(f"{provider}_API_BASE")+ # Use the API key and base from the request, or from the environment.+ api_key = request.pop("api_key", None) or os.getenv(f"{provider}_API_KEY")+ api_base = request.pop("api_base", None) or os.getenv(f"{provider}_API_BASE")# Build the prompt from the messages.- prompt = "\n\n".join([x["content"] for x in kwargs.pop("messages")] + ["BEGIN RESPONSE:"])+ prompt = "\n\n".join([x["content"] for x in request.pop("messages")] + ["BEGIN RESPONSE:"])return litellm.text_completion(cache=cache,@@ -261,5 +302,5 @@ def litellm_text_completion(request, num_retries: int, cache={"no-cache": True,api_base=api_base,prompt=prompt,num_retries=num_retries,- **kwargs,+ **request,)diff --git a/poetry.lock b/poetry.lockindex af67dfb63..2e13c532b 100644--- a/poetry.lock+++ b/poetry.lock@@ -579,7 +579,7 @@ virtualenv = ["virtualenv (>=20.0.35)"]name = "cachetools"version = "5.5.0"description = "Extensible memoizing collections and decorators"-optional = true+optional = falsepython-versions = ">=3.7"files = [{file = "cachetools-5.5.0-py3-none-any.whl", hash = "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292"},@@ -8523,4 +8523,4 @@ weaviate = ["weaviate-client"][metadata]lock-version = "2.0"python-versions = ">=3.9,<3.13"-content-hash = "d7c6fe24c44afc0e694de8270035ac4cd9831f9bf9d6253f1cd1dbfe03ec2835"+content-hash = "f54a12cf2b3bc3d2811e8875561b1262c8b7a546d0fb333f075a1127bc2ffcb0"diff --git a/pyproject.toml b/pyproject.tomlindex 18cc52f72..c0fe14f19 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -44,6 +44,7 @@ dependencies = ["tenacity>=8.2.3","anyio","asyncer==0.0.8",+ "cachetools",][project.optional-dependencies]@@ -138,6 +139,7 @@ falkordb = "^1.0.9"json-repair = "^0.30.0"tenacity = ">=8.2.3"asyncer = "0.0.8"+cachetools = "^5.5.0"[tool.poetry.group.dev.dependencies]pytest = "^8.3.3"diff --git a/requirements.txt b/requirements.txtindex 31874b1c9..218ca9ed7 100644--- a/requirements.txt+++ b/requirements.txt@@ -1,4 +1,7 @@+anyio+asyncer==0.0.8backoff+cachetoolsdatasetsdiskcachehttpx@@ -15,5 +18,3 @@ requeststenacity>=8.2.3tqdmujson-anyio-asyncer==0.0.8diff --git a/tests/caching/test_caching.py b/tests/caching/test_caching.pyindex d5c393374..468a81352 100644--- a/tests/caching/test_caching.py+++ b/tests/caching/test_caching.py@@ -88,3 +88,23 @@ def test_lm_calls_are_cached_across_interpreter_sessions(litellm_test_server, terequest_logs = read_litellm_test_server_request_logs(server_log_file_path)assert len(request_logs) == 0+++def test_lm_calls_are_cached_in_memory_when_expected(litellm_test_server, temporary_blank_cache_dir):+ api_base, server_log_file_path = litellm_test_server++ lm1 = dspy.LM(+ model="openai/dspy-test-model",+ api_base=api_base,+ api_key="fakekey",+ )+ lm1("Example query")+ # Remove the disk cache, after which the LM must rely on in-memory caching+ shutil.rmtree(temporary_blank_cache_dir)+ lm1("Example query2")+ lm1("Example query2")+ lm1("Example query2")+ lm1("Example query2")++ request_logs = read_litellm_test_server_request_logs(server_log_file_path)+ assert len(request_logs) == 2diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.pyindex 18c134601..519c1e24e 100644--- a/tests/clients/test_lm.py+++ b/tests/clients/test_lm.py@@ -1,24 +1,77 @@from unittest import mock+import pydanticimport pytestimport dspyfrom tests.test_utils.server import litellm_test_server-def test_lms_can_be_queried(litellm_test_server):+def test_chat_lms_can_be_queried(litellm_test_server):api_base, _ = litellm_test_server+ expected_response = ["Hi!"]openai_lm = dspy.LM(model="openai/dspy-test-model",api_base=api_base,api_key="fakekey",+ model_type="chat",)- openai_lm("openai query")+ assert openai_lm("openai query") == expected_responseazure_openai_lm = dspy.LM(model="azure/dspy-test-model",api_base=api_base,api_key="fakekey",+ model_type="chat",)- azure_openai_lm("azure openai query")+ assert azure_openai_lm("azure openai query") == expected_response+++def test_text_lms_can_be_queried(litellm_test_server):+ api_base, _ = litellm_test_server+ expected_response = ["Hi!"]++ openai_lm = dspy.LM(+ model="openai/dspy-test-model",+ api_base=api_base,+ api_key="fakekey",+ model_type="text",+ )+ assert openai_lm("openai query") == expected_response++ azure_openai_lm = dspy.LM(+ model="azure/dspy-test-model",+ api_base=api_base,+ api_key="fakekey",+ model_type="text",+ )+ assert azure_openai_lm("azure openai query") == expected_response+++def test_lm_calls_support_unhashable_types(litellm_test_server):+ api_base, server_log_file_path = litellm_test_server++ lm_with_unhashable_callable = dspy.LM(+ model="openai/dspy-test-model",+ api_base=api_base,+ api_key="fakekey",+ # Define a callable kwarg for the LM to use during inference+ azure_ad_token_provider=lambda *args, **kwargs: None,+ )+ lm_with_unhashable_callable("Query")+++def test_lm_calls_support_pydantic_models(litellm_test_server):+ api_base, server_log_file_path = litellm_test_server++ class ResponseFormat(pydantic.BaseModel):+ response: str++ lm = dspy.LM(+ model="openai/dspy-test-model",+ api_base=api_base,+ api_key="fakekey",+ response_format=ResponseFormat,+ )+ lm("Query")