DSPy: The framework for programming—not prompting—language models
Python
"""Tests for restricted pickle deserialization."""
import io
import pickle
import subprocess
import sys
from dataclasses import dataclass
import diskcache
import numpy as np
import pydantic
import pytest
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
from litellm.types.utils import EmbeddingResponse, ModelResponse, TextCompletionResponse
from openai.types.responses import ResponseFunctionToolCall, ResponseOutputMessage, ResponseOutputText
from dspy.clients.disk_serialization import (
DeserializationError,
_restricted_load,
restricted_disk,
)
from dspy.clients.lm15_boundary import response_value
from dspy.lm15 import Message, Request
class AllowedPydanticModel(pydantic.BaseModel):
name: str
value: int
@dataclass
class AllowedDataclass:
name: str
value: int
class _UnlistedModel(pydantic.BaseModel):
value: int
class _Outer:
@dataclass
class Inner:
value: int
def _allowed_for(*types):
return frozenset((cls.__module__, cls.__qualname__) for cls in types)
def _roundtrip(value, allowed=None):
"""Pickle then restricted-unpickle."""
return _restricted_load(io.BytesIO(pickle.dumps(value)), allowed or set())
# -- roundtrip tests (through diskcache) --
def _make_cache(directory, allowed):
return diskcache.FanoutCache(
directory=directory,
shards=4,
disk=restricted_disk(allowed),
timeout=10,
)
def _run_subprocess_cache_case(case, directory):
subprocess.run(
[sys.executable, "-m", "tests.clients.test_disk_serialization", case, str(directory)],
check=True,
)
def _write_tool_call_cache(directory):
cache = _make_cache(directory, set())
cache["k"] = ModelResponse(
id="chatcmpl-tool",
model="dummy",
choices=[{
"message": {
"content": None,
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city":"SF"}'},
}],
},
"index": 0,
"finish_reason": "tool_calls",
}],
usage={"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
)
cache.close()
def _assert_cached_tool_call_normalizes(directory):
response = _make_cache(directory, set())["k"]
tool_call = response.choices[0].message.tool_calls[0]
assert type(type(tool_call).__pydantic_serializer__).__name__ == "MockValSer"
assert type(type(tool_call.function).__pydantic_serializer__).__name__ == "MockValSer"
lm_response = response_value(response, "chat", Request(model="dummy", messages=(Message.user("-"),)))
part = lm_response.tool_calls[0]
assert part.name == "get_weather"
assert part.input == {"city": "SF"}
assert lm_response.provider_data["choices"][0]["message"]["tool_calls"][0]["function"]["name"] == "get_weather"
assert lm_response.usage.total_tokens == 3
def _write_responses_function_call_cache(directory):
cache = _make_cache(directory, set())
cache["k"] = ResponsesAPIResponse(
id="resp_1",
created_at=0.0,
error=None,
incomplete_details=None,
instructions=None,
model="dummy",
object="response",
output=[ResponseFunctionToolCall(
id="fc_1",
call_id="call_1",
type="function_call",
name="get_weather",
arguments='{"city":"SF"}',
status="completed",
)],
metadata={},
parallel_tool_calls=False,
temperature=1.0,
tool_choice="auto",
tools=[],
top_p=1.0,
max_output_tokens=None,
previous_response_id=None,
reasoning=None,
status="completed",
text=None,
truncation="disabled",
usage=ResponseAPIUsage(input_tokens=1, output_tokens=2, total_tokens=3),
user=None,
)
cache.close()
def _assert_cached_responses_function_call_normalizes(directory):
response = _make_cache(directory, set())["k"]
function_call = response.output[0]
assert type(type(function_call).__pydantic_serializer__).__name__ == "MockValSer"
lm_response = response_value(response, "responses", Request(model="dummy", messages=(Message.user("-"),)))
part = lm_response.tool_calls[0]
assert part.name == "get_weather"
assert part.input == {"city": "SF"}
assert lm_response.provider_data["output"][0]["call_id"] == "call_1"
assert lm_response.usage.total_tokens == 3
_SUBPROCESS_CASES = {
"write-tool-call-cache": _write_tool_call_cache,
"assert-tool-call-cache-normalizes": _assert_cached_tool_call_normalizes,
"write-responses-function-call-cache": _write_responses_function_call_cache,
"assert-responses-function-call-cache-normalizes": _assert_cached_responses_function_call_normalizes,
}
@pytest.mark.parametrize("value", [
{"a": 1, "b": [2, 3]},
[1, "two", 3.0],
"hello",
42,
(1, 2, 3),
], ids=["dict", "list", "str", "int", "tuple"])
def test_plain_values_roundtrip(tmp_path, value):
cache = _make_cache(str(tmp_path), set())
cache["k"] = value
assert cache["k"] == value
def test_pydantic_roundtrip(tmp_path):
allowed = _allowed_for(AllowedPydanticModel)
cache = _make_cache(str(tmp_path), allowed)
cache["k"] = AllowedPydanticModel(name="test", value=42)
assert cache["k"] == AllowedPydanticModel(name="test", value=42)
def test_dataclass_roundtrip(tmp_path):
allowed = _allowed_for(AllowedDataclass)
cache = _make_cache(str(tmp_path), allowed)
cache["k"] = AllowedDataclass(name="test", value=42)
assert cache["k"] == AllowedDataclass(name="test", value=42)
def test_ndarray_roundtrip(tmp_path):
cache = _make_cache(str(tmp_path), set())
value = np.arange(6, dtype=np.float32).reshape(2, 3)
cache["k"] = value
result = cache["k"]
assert isinstance(result, np.ndarray)
assert result.dtype == value.dtype
np.testing.assert_array_equal(result, value)
def test_litellm_response_roundtrip(tmp_path):
cache = _make_cache(str(tmp_path), set())
response = ModelResponse(
id="chatcmpl-test",
choices=[{"message": {"content": "Hello"}, "index": 0, "finish_reason": "stop"}],
)
cache["k"] = response
result = cache["k"]
assert isinstance(result, ModelResponse)
assert result.choices[0].message.content == "Hello"
def test_text_completion_response_roundtrip(tmp_path):
cache = _make_cache(str(tmp_path), set())
response = TextCompletionResponse(
id="cmpl-test", choices=[{"text": "Hello", "index": 0, "finish_reason": "stop"}], model="m",
)
cache["k"] = response
result = cache["k"]
assert isinstance(result, TextCompletionResponse)
assert result.choices[0]["text"] == "Hello"
def test_embedding_response_roundtrip(tmp_path):
cache = _make_cache(str(tmp_path), set())
response = EmbeddingResponse(
data=[{"embedding": [0.1, 0.2], "index": 0, "object": "embedding"}],
model="m", usage={"prompt_tokens": 1, "total_tokens": 1},
)
cache["k"] = response
result = cache["k"]
assert isinstance(result, EmbeddingResponse)
assert result.data[0]["embedding"] == pytest.approx([0.1, 0.2])
def test_responses_api_roundtrip(tmp_path):
"""ResponsesAPIResponse with nested openai.types.responses.* classes."""
cache = _make_cache(str(tmp_path), set())
response = ResponsesAPIResponse(
id="resp_1",
created_at=0.0,
error=None,
incomplete_details=None,
instructions=None,
model="test",
object="response",
output=[
ResponseOutputMessage(
id="msg_1",
type="message",
status="completed",
role="assistant",
content=[ResponseOutputText(type="output_text", text="hello", annotations=[])],
),
],
metadata={},
parallel_tool_calls=False,
temperature=1.0,
tool_choice="auto",
tools=[],
top_p=1.0,
max_output_tokens=None,
previous_response_id=None,
reasoning=None,
status="completed",
text=None,
truncation="disabled",
usage=ResponseAPIUsage(input_tokens=1, output_tokens=1, total_tokens=2),
user=None,
)
cache["k"] = response
result = cache["k"]
assert isinstance(result, ResponsesAPIResponse)
assert result.output[0].content[0].text == "hello"
def test_tool_call_response_roundtrip(tmp_path):
"""ModelResponse with tool calls uses nested litellm types."""
cache = _make_cache(str(tmp_path), set())
response = ModelResponse(
id="chatcmpl-tool",
choices=[{
"message": {
"content": None,
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city":"SF"}'},
}],
},
"index": 0,
"finish_reason": "tool_calls",
}],
)
cache["k"] = response
result = cache["k"]
assert isinstance(result, ModelResponse)
assert result.choices[0].message.tool_calls[0].function.name == "get_weather"
def test_cached_tool_call_normalizes_after_fresh_process_roundtrip(tmp_path):
_run_subprocess_cache_case("write-tool-call-cache", tmp_path)
_run_subprocess_cache_case("assert-tool-call-cache-normalizes", tmp_path)
def test_cached_responses_function_call_normalizes_after_fresh_process_roundtrip(tmp_path):
_run_subprocess_cache_case("write-responses-function-call-cache", tmp_path)
_run_subprocess_cache_case("assert-responses-function-call-cache-normalizes", tmp_path)
# -- allowlist enforcement --
def test_unlisted_type_blocked_on_read(tmp_path):
cache = _make_cache(str(tmp_path), set())
cache["k"] = _UnlistedModel(value=1)
with pytest.raises(DeserializationError, match="not in the safe_types allowlist"):
cache["k"]
def test_allowlists_are_isolated(tmp_path):
allowed = _allowed_for(AllowedPydanticModel)
cache = _make_cache(str(tmp_path / "a"), allowed)
cache["k"] = AllowedPydanticModel(name="test", value=1)
assert cache["k"] == AllowedPydanticModel(name="test", value=1)
empty_cache = _make_cache(str(tmp_path / "b"), set())
empty_cache["k"] = AllowedPydanticModel(name="test", value=1)
with pytest.raises(DeserializationError):
empty_cache["k"]
def test_numpy_ctypeslib_blocked():
"""Only specific numpy reconstruction functions are allowed, not arbitrary submodules."""
payload = (
b"\x80\x05" # PROTO 5
b"\x8c\x0fnumpy.ctypeslib" # SHORT_BINUNICODE (15 bytes)
b"\x8c\x0cload_library" # SHORT_BINUNICODE (12 bytes)
b"\x93" # STACK_GLOBAL
b"\x8c\x04evil" # SHORT_BINUNICODE "evil"
b"\x8c\x04/tmp" # SHORT_BINUNICODE "/tmp"
b"\x86" # TUPLE2
b"R" # REDUCE
b"." # STOP
)
with pytest.raises(DeserializationError):
_restricted_load(io.BytesIO(payload), set())
# -- _restricted_load unit tests --
def test_restricted_load_rejects_unknown_type():
data = pickle.dumps(_UnlistedModel(value=1))
with pytest.raises(DeserializationError, match="not in the safe_types allowlist"):
_restricted_load(io.BytesIO(data), set())
def test_corrupt_pickle_raises_deserialization_error():
"""Real pickle corruption must raise DeserializationError, not UnpicklingError."""
data = pickle.dumps({"key": "value"})
corrupted = data[:5] + b"nope" + data[9:]
with pytest.raises(DeserializationError, match="Corrupt cache entry"):
_restricted_load(io.BytesIO(corrupted), set())
def test_nested_class_safe_types(tmp_path):
"""safe_types must work for nested classes (pickle uses __qualname__)."""
allowed = _allowed_for(_Outer.Inner)
cache = _make_cache(str(tmp_path), allowed)
cache["k"] = _Outer.Inner(value=42)
assert cache["k"] == _Outer.Inner(value=42)
def test_restricted_load_allows_builtin_types():
response = ModelResponse(
id="t", choices=[{"message": {"content": "hi"}, "index": 0, "finish_reason": "stop"}],
)
result = _roundtrip(response)
assert isinstance(result, ModelResponse)
def test_all_cached_lm_types_roundtrip():
"""Every LM return type that DSPy caches must roundtrip through the restricted unpickler."""
test_values = [
ModelResponse(
id="t", choices=[{"message": {"content": "hi"}, "index": 0, "finish_reason": "stop"}],
),
TextCompletionResponse(
id="t", choices=[{"text": "hi", "index": 0, "finish_reason": "stop"}], model="m",
),
EmbeddingResponse(
data=[{"embedding": [0.1], "index": 0, "object": "embedding"}],
model="m", usage={"prompt_tokens": 1, "total_tokens": 1},
),
]
for value in test_values:
result = _roundtrip(value)
assert type(result) is type(value), f"Failed roundtrip for {type(value).__name__}"
if __name__ == "__main__":
_SUBPROCESS_CASES[sys.argv[1]](sys.argv[2])