DSPy: The framework for programming—not prompting—language models
Python
"""Metadata loading, native capability decisions, and provider token accounting."""
import json
from unittest.mock import patch
import pytest
import dspy
import dspy.clients.model_metadata as metadata
from dspy.clients.costs import estimate_cost
from dspy.lm15 import Config, Message, Request, Response, Usage
@pytest.fixture(autouse=True)
def local_metadata(monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(metadata, "_data", None)
monkeypatch.setattr(metadata, "_source", {})
def test_snapshot_capabilities_do_not_import_litellm():
with patch("dspy.clients.lm._get_litellm", side_effect=AssertionError("LiteLLM accessed")):
lm = dspy.LM("openai/gpt-4o", engine="lm15")
assert lm.supports_function_calling
assert lm.supports_response_schema
assert "response_format" in lm.supported_params
assert not lm.supports_reasoning
claude = dspy.LM("anthropic/claude-sonnet-4-5", engine="lm15")
assert claude.supports_reasoning
assert claude.supports_response_schema
assert metadata.source_info()["is_env_forced"]
def test_unknown_and_custom_capabilities():
assert not dspy.LM("openai/unlisted").supports_function_calling
class Engine:
supports_function_calling = True
supported_params = frozenset({"tools"})
def complete(self, request):
raise AssertionError("No generation during capability lookup")
lm = dspy.LM("custom", engine=Engine())
assert lm.supports_function_calling
assert lm.supported_params == {"tools"}
def test_lm15_restriction_beats_catalog(monkeypatch):
monkeypatch.setattr(metadata, "_data", {"zai/glm-test": {
"litellm_provider": "zai", "supports_response_schema": True,
"supports_function_calling": True,
}})
lm = dspy.LM("zai:glm-test", engine="lm15")
assert not lm.supports_response_schema
assert "response_format" in lm.supported_params # JSON mode, not schema enforcement
def test_missing_flags_inherit_but_false_and_prices_do_not(monkeypatch):
monkeypatch.setattr(metadata, "_data", {
"openai/test": {"supports_reasoning": False},
"test": {"litellm_provider": "openai", "supports_reasoning": True,
"supports_function_calling": True, "input_cost_per_token": 9},
})
info = metadata.model_info("openai-chat", "test")
assert info["supports_reasoning"] is False
assert info["supports_function_calling"] is True
assert "input_cost_per_token" not in info
info["supports_reasoning"] = True
assert metadata.model_info("openai-chat", "test")["supports_reasoning"] is False
def test_remote_first_once_and_fallback(monkeypatch):
monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP")
with patch.object(metadata.urllib.request, "urlopen", side_effect=OSError("secret")) as fetch:
metadata.model_info("openai-chat", "gpt-4o")
metadata.model_info("openai-chat", "gpt-4o")
assert fetch.call_count == 1
assert metadata.source_info()["source"] == "local"
assert metadata.source_info()["fallback_reason"] == "OSError"
def test_valid_remote_map_is_adopted(monkeypatch):
import io
monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP")
data = metadata._snapshot()
data["gpt-4o"]["supports_reasoning"] = True
with patch.object(metadata.urllib.request, "urlopen", return_value=io.BytesIO(json.dumps(data).encode())):
assert metadata.model_info("openai-chat", "gpt-4o")["supports_reasoning"]
assert metadata.source_info()["source"] == "remote"
def answer(model, **usage):
return Response(id=None, model=model, message=Message.assistant("ok"),
finish_reason="stop", usage=Usage(**usage))
def test_openai_cached_input_and_reasoning_are_not_double_counted():
response = answer("gpt-4o", input_tokens=100, output_tokens=30, cache_read_tokens=40, reasoning_tokens=10)
cost, details = estimate_cost(response, provider="openai-chat", requested_model=response.model)
assert cost == pytest.approx(60 * 2.5e-6 + 40 * 1.25e-6 + 30 * 1e-5)
assert details["kind"] == "estimate"
def test_anthropic_disjoint_cache_counters():
response = answer("claude-sonnet-4-5", input_tokens=100, output_tokens=30,
cache_read_tokens=40, cache_write_tokens=20)
cost, _ = estimate_cost(response, provider="anthropic", requested_model=response.model)
assert cost == pytest.approx(100 * 3e-6 + 40 * 3e-7 + 20 * 3.75e-6 + 30 * 1.5e-5)
def test_gemini_reasoning_is_additional_output():
response = answer("gemini-2.5-flash", input_tokens=100, output_tokens=30, reasoning_tokens=10)
cost, _ = estimate_cost(response, provider="gemini", requested_model=response.model)
assert cost == pytest.approx(100 * 3e-7 + 40 * 2.5e-6)
def test_long_context_and_priority_rates(monkeypatch):
monkeypatch.setattr(metadata, "_data", {"openai/test": {
"input_cost_per_token_above_200k_tokens_priority": 2e-6,
"output_cost_per_token_above_200k_tokens_priority": 8e-6,
}})
response = answer("test", input_tokens=200001, output_tokens=30, reasoning_tokens=10)
request = Request(model="test", messages=(Message.user("hi"),), config=Config(service_tier="priority"))
cost, _ = estimate_cost(response, provider="openai-chat", requested_model="test", request=request)
assert cost == pytest.approx(200001 * 2e-6 + 30 * 8e-6)
def test_unknown_price_and_malformed_metadata_do_not_fail_generation(monkeypatch):
response = answer("unknown", input_tokens=1, output_tokens=1)
assert estimate_cost(response, provider="openai-chat", requested_model="unknown")[0] is None
monkeypatch.setattr("dspy.clients.costs.model_info", lambda *args: {"broken": object()})
assert estimate_cost(response, provider="openai-chat", requested_model="unknown")[0] is None
@pytest.mark.parametrize("router_name", ["model-router", "azure/model-router"])
@pytest.mark.parametrize("fallback", [False, True])
def test_azure_router_fee_is_not_an_inference_estimate(monkeypatch, router_name, fallback):
data = metadata._snapshot()
assert data["azure/model-router"]["input_cost_per_token"] > 0
data["openai/model-router"] = {"litellm_provider": "openai", "input_cost_per_token": 2e-6,
"output_cost_per_token": 7e-6}
monkeypatch.setattr(metadata, "_data", data)
response = answer("unlisted-selected-model" if fallback else router_name, input_tokens=1000, output_tokens=500)
for provider, namespaces in (("azure-chat", None), ("declared-azure", ("azure",))):
cost, details = estimate_cost(response, provider=provider, requested_model=router_name, namespaces=namespaces)
assert cost is None
assert details["kind"] == "unknown"
# The refusal belongs to the Azure catalog entry, not an arbitrary model name.
cost, _ = estimate_cost(answer("model-router", input_tokens=1000, output_tokens=500),
provider="openai-chat", requested_model="model-router")
assert cost == pytest.approx(0.0055)
def test_cost_provenance_survives_cache_record():
from dspy.clients.call_result import CallResult, combine
response = answer("gpt-4o", input_tokens=1, output_tokens=1)
result = CallResult.native(response)
result.cost, result.cost_details = estimate_cost(response, provider="openai-chat", requested_model=response.model)
cached = CallResult.load(result.dump())
assert cached.cost == result.cost
assert cached.cost_details == result.cost_details
assert cached.usage == {}
assert combine([result, result], model_type="chat").cost == result.cost * 2
def test_family_rule_provider_limits_only_known_entries(monkeypatch):
monkeypatch.setattr(metadata, "_data", {
"openai/gpt-test": {"litellm_provider": "openai", "supports_vision": False},
"proxy/gpt-test": {"litellm_provider": "proxy"},
"fallback_generalizations": {"rules": [{
"pattern": "^gpt-",
"fill_missing_for_providers": ["openai"],
"model_info": {"supports_reasoning": True, "supports_vision": True,
"input_cost_per_token": 99},
}]},
})
assert dspy.LM("openai/gpt-test", engine="lm15").supports_reasoning
assert metadata.model_info("openai-chat", "gpt-test")["supports_vision"] is False
assert "supports_reasoning" not in metadata.model_info("proxy", "gpt-test")
unknown = metadata.model_info("proxy", "gpt-unlisted")
assert unknown["supports_reasoning"] is True
assert "input_cost_per_token" not in unknown
@pytest.mark.parametrize("provider,namespace", [("fireworks", "fireworks_ai"), ("together", "together_ai")])
def test_native_inference_hosts_use_their_metadata_namespace(monkeypatch, provider, namespace):
monkeypatch.setattr(metadata, "_data", {
f"{namespace}/vendor/model": {
"litellm_provider": namespace, "supports_function_calling": True,
"input_cost_per_token": 2e-6, "output_cost_per_token": 7e-6,
},
"vendor/model": {"litellm_provider": "openai", "input_cost_per_token": 99},
})
lm = dspy.LM(f"{namespace}/vendor/model", engine="lm15", api_key="test")
assert lm.supports_function_calling
cost, details = estimate_cost(answer("vendor/model", input_tokens=3, output_tokens=5),
provider=provider, requested_model="vendor/model")
assert cost == pytest.approx(41e-6)
assert details["provider"] == provider