DSPy: The framework for programming—not prompting—language models
Python
"""SandboxLM: the host LM as sandboxed code may call it."""
import pytest
from dspy.primitives.facade import SANDBOX_LM_KWARGS, SandboxLM
from dspy.utils.dummies import DummyLM
class _CapableLM(DummyLM):
supports_function_calling = True
supports_response_schema = True
supported_params = frozenset({"temperature", "response_format", "tools"})
def _lm():
return DummyLM([{"answer": "a"}, {"answer": "b"}])
def test_generation_parameters_pass_and_the_wrapped_lm_does_the_work():
lm = _lm()
outputs = SandboxLM(lm)(messages=[{"role": "user", "content": "hi"}], temperature=0.3, max_tokens=5)
assert outputs == ["[[ ## answer ## ]]\na"]
assert len(lm.history) == 1
@pytest.mark.parametrize(
"options",
[{"api_base": "http://evil.example"}, {"api_key": "sk-x"}, {"extra_headers": {"X": "y"}}, {"model": "gpt-5-pro"}],
)
def test_routing_credential_and_transport_options_are_rejected_before_any_request(options):
lm = _lm()
sandbox = SandboxLM(lm)
with pytest.raises(TypeError, match="may not set LM option"):
sandbox("prompt", temperature=0.1, **options)
with pytest.raises(TypeError, match="may not set LM option"):
sandbox.copy(**options)
assert lm.history == []
assert not options.keys() & SANDBOX_LM_KWARGS
@pytest.mark.asyncio
async def test_acall_applies_the_same_allowlist():
lm = _lm()
with pytest.raises(TypeError, match=r"may not set LM option\(s\) \['api_key'\]"):
await SandboxLM(lm).acall("prompt", api_key="sk-x")
assert lm.history == []
assert await SandboxLM(lm).acall("prompt", temperature=0.2) == ["[[ ## answer ## ]]\na"]
def test_reserve_meters_admitted_calls_only():
reserved = []
sandbox = SandboxLM(_lm(), reserve=reserved.append)
sandbox("one")
with pytest.raises(TypeError):
sandbox("two", api_key="sk-x")
sandbox.copy(temperature=0.9)("three")
sandbox.forward("four")
with pytest.raises(TypeError):
sandbox.forward("five", api_base="http://evil.example")
assert reserved == [1, 1, 1]
def test_reserve_charges_one_slot_per_requested_completion():
# `n` asks for n completions in one request; it must not multiply max_llm_calls.
class _StubLM:
model = "stub"
def __init__(self):
self.kwargs = {"n": 4, "api_key": "sk-host"}
def __call__(self, *args, **kwargs):
return ["x"] * kwargs.get("n", self.kwargs["n"])
reserved = []
sandbox = SandboxLM(_StubLM(), reserve=reserved.append)
assert len(sandbox("per-call", n=64)) == 64
sandbox("mirrored default")
sandbox("explicit one", n=1)
assert reserved == [64, 4, 1]
def test_proxy_mirrors_identity_kwargs_and_capabilities_of_the_wrapped_lm():
lm = _CapableLM([{"answer": "a"}])
lm.kwargs.update(temperature=0.7, api_key="sk-secret", api_base="http://internal.example")
sandbox = SandboxLM(lm)
assert (sandbox.model, sandbox.model_type) == (lm.model, lm.model_type)
assert sandbox.kwargs["temperature"] == 0.7
assert not {"api_key", "api_base"} & sandbox.kwargs.keys() # credentials stay on the wrapped LM
assert sandbox.supports_function_calling and sandbox.supports_response_schema and not sandbox.supports_reasoning
assert sandbox.supported_params == {"temperature", "response_format", "tools"}
assert not SandboxLM(_lm()).supports_function_calling
def test_wraps_a_callable_only_lm():
# RLM accepts any callable as sub_lm; the facade is installed by default, so the proxy must too.
class PlainLM:
def __call__(self, prompt):
return ["plain:" + prompt]
sandbox = SandboxLM(PlainLM())
assert sandbox("hi") == ["plain:hi"]
assert (sandbox.model, sandbox.kwargs, sandbox.supports_function_calling, sandbox.supported_params) == (
None, {}, False, set()
)