DSPy: The framework for programming—not prompting—language models
Python
import asyncio
from unittest import mock
import pytest
from pydantic import BaseModel
import dspy
from dspy.utils.usage_tracker import UsageTracker, track_usage
def test_add_usage_entry():
"""Test adding usage entries to the tracker."""
tracker = UsageTracker()
# Test with a single usage entry
usage_entry = {
"prompt_tokens": 1117,
"completion_tokens": 46,
"total_tokens": 1163,
"prompt_tokens_details": {"cached_tokens": 0, "audio_tokens": 0},
"completion_tokens_details": {
"reasoning_tokens": 0,
"audio_tokens": 0,
"accepted_prediction_tokens": 0,
"rejected_prediction_tokens": 0,
},
}
tracker.add_usage("gpt-4o-mini", usage_entry)
assert len(tracker.usage_data["gpt-4o-mini"]) == 1
assert tracker.usage_data["gpt-4o-mini"][0] == usage_entry
def test_get_total_tokens():
"""Test calculating total tokens from usage entries."""
tracker = UsageTracker()
# Add multiple usage entries for the same model
usage_entries = [
{
"prompt_tokens": 1117,
"completion_tokens": 46,
"total_tokens": 1163,
"prompt_tokens_details": {"cached_tokens": 200, "audio_tokens": 50},
"completion_tokens_details": {
"reasoning_tokens": 20,
"audio_tokens": 10,
"accepted_prediction_tokens": 16,
"rejected_prediction_tokens": 0,
},
},
{
"prompt_tokens": 800,
"completion_tokens": 100,
"total_tokens": 900,
"prompt_tokens_details": {"cached_tokens": 300, "audio_tokens": 0},
"completion_tokens_details": {
"reasoning_tokens": 50,
"audio_tokens": 0,
"accepted_prediction_tokens": 40,
"rejected_prediction_tokens": 10,
},
},
{
"prompt_tokens": 500,
"completion_tokens": 80,
"total_tokens": 580,
"prompt_tokens_details": {"cached_tokens": 100, "audio_tokens": 25},
"completion_tokens_details": {
"reasoning_tokens": 30,
"audio_tokens": 15,
"accepted_prediction_tokens": 25,
"rejected_prediction_tokens": 10,
},
},
]
for entry in usage_entries:
tracker.add_usage("gpt-4o-mini", entry)
total_usage = tracker.get_total_tokens()
assert "gpt-4o-mini" in total_usage
assert total_usage["gpt-4o-mini"]["prompt_tokens"] == 2417 # 1117 + 800 + 500
assert total_usage["gpt-4o-mini"]["completion_tokens"] == 226 # 46 + 100 + 80
assert total_usage["gpt-4o-mini"]["total_tokens"] == 2643 # 1163 + 900 + 580
assert total_usage["gpt-4o-mini"]["prompt_tokens_details"]["cached_tokens"] == 600 # 200 + 300 + 100
assert total_usage["gpt-4o-mini"]["prompt_tokens_details"]["audio_tokens"] == 75 # 50 + 0 + 25
assert total_usage["gpt-4o-mini"]["completion_tokens_details"]["reasoning_tokens"] == 100 # 20 + 50 + 30
assert total_usage["gpt-4o-mini"]["completion_tokens_details"]["audio_tokens"] == 25 # 10 + 0 + 15
assert total_usage["gpt-4o-mini"]["completion_tokens_details"]["accepted_prediction_tokens"] == 81 # 16 + 40 + 25
assert total_usage["gpt-4o-mini"]["completion_tokens_details"]["rejected_prediction_tokens"] == 20 # 0 + 10 + 10
def test_track_usage_with_multiple_models():
"""Test tracking usage across multiple models."""
tracker = UsageTracker()
# Add usage entries for different models
usage_entries = [
{
"model": "gpt-4o-mini",
"usage": {
"prompt_tokens": 1117,
"completion_tokens": 46,
"total_tokens": 1163,
"prompt_tokens_details": {"cached_tokens": 0, "audio_tokens": 0},
"completion_tokens_details": {
"reasoning_tokens": 0,
"audio_tokens": 0,
"accepted_prediction_tokens": 0,
"rejected_prediction_tokens": 0,
},
},
},
{
"model": "gpt-3.5-turbo",
"usage": {
"prompt_tokens": 800,
"completion_tokens": 100,
"total_tokens": 900,
"prompt_tokens_details": {"cached_tokens": 0, "audio_tokens": 0},
"completion_tokens_details": {
"reasoning_tokens": 0,
"audio_tokens": 0,
"accepted_prediction_tokens": 0,
"rejected_prediction_tokens": 0,
},
},
},
]
for entry in usage_entries:
tracker.add_usage(entry["model"], entry["usage"])
total_usage = tracker.get_total_tokens()
assert "gpt-4o-mini" in total_usage
assert "gpt-3.5-turbo" in total_usage
assert total_usage["gpt-4o-mini"]["total_tokens"] == 1163
assert total_usage["gpt-3.5-turbo"]["total_tokens"] == 900
def test_track_usage_context_manager(lm_for_test):
lm = dspy.LM(lm_for_test, cache=False, temperature=0.0)
dspy.configure(lm=lm)
predict = dspy.ChainOfThought("question -> answer")
with track_usage() as tracker:
predict(question="What is the capital of France?")
predict(question="What is the capital of Italy?")
assert len(tracker.usage_data) > 0
assert len(tracker.usage_data[lm_for_test]) == 2
total_usage = tracker.get_total_tokens()
assert lm_for_test in total_usage
assert len(total_usage.keys()) == 1
assert isinstance(total_usage[lm_for_test], dict)
def test_merge_usage_entries_with_new_keys():
"""Ensure merging usage entries preserves unseen keys."""
tracker = UsageTracker()
tracker.add_usage("model-x", {"prompt_tokens": 5})
tracker.add_usage("model-x", {"completion_tokens": 2})
total_usage = tracker.get_total_tokens()
assert total_usage["model-x"]["prompt_tokens"] == 5
assert total_usage["model-x"]["completion_tokens"] == 2
def test_merge_usage_entries_with_none_values():
"""Test tracking usage across multiple models."""
tracker = UsageTracker()
# Add usage entries for different models
usage_entries = [
{
"model": "gpt-4o-mini",
"usage": {
"prompt_tokens": 1117,
"completion_tokens": 46,
"total_tokens": 1163,
"prompt_tokens_details": None,
"completion_tokens_details": {},
},
},
{
"model": "gpt-4o-mini",
"usage": {
"prompt_tokens": 800,
"completion_tokens": 100,
"total_tokens": 900,
"prompt_tokens_details": None,
"completion_tokens_details": None,
},
},
{
"model": "gpt-4o-mini",
"usage": {
"prompt_tokens": 100,
"completion_tokens": 100,
"total_tokens": 200,
"prompt_tokens_details": {"cached_tokens": 50, "audio_tokens": 50},
"completion_tokens_details": None,
},
},
{
"model": "gpt-4o-mini",
"usage": {
"prompt_tokens": 800,
"completion_tokens": 100,
"total_tokens": 900,
"prompt_tokens_details": None,
"completion_tokens_details": {
"reasoning_tokens": 1,
"audio_tokens": 1,
"accepted_prediction_tokens": 1,
"rejected_prediction_tokens": 1,
},
},
},
]
for entry in usage_entries:
tracker.add_usage(entry["model"], entry["usage"])
total_usage = tracker.get_total_tokens()
assert total_usage["gpt-4o-mini"]["prompt_tokens"] == 2817
assert total_usage["gpt-4o-mini"]["completion_tokens"] == 346
assert total_usage["gpt-4o-mini"]["total_tokens"] == 3163
assert total_usage["gpt-4o-mini"]["prompt_tokens_details"]["cached_tokens"] == 50
assert total_usage["gpt-4o-mini"]["prompt_tokens_details"]["audio_tokens"] == 50
assert total_usage["gpt-4o-mini"]["completion_tokens_details"]["reasoning_tokens"] == 1
assert total_usage["gpt-4o-mini"]["completion_tokens_details"]["audio_tokens"] == 1
assert total_usage["gpt-4o-mini"]["completion_tokens_details"]["accepted_prediction_tokens"] == 1
assert total_usage["gpt-4o-mini"]["completion_tokens_details"]["rejected_prediction_tokens"] == 1
def test_merge_usage_entries_with_pydantic_models():
"""Test merging usage entries with Pydantic model objects, like `PromptTokensDetailsWrapper` from litellm."""
tracker = UsageTracker()
# Here we define a simplified version of the Pydantic models from litellm to avoid the dependency change on litellm.
class CacheCreationTokenDetails(BaseModel):
ephemeral_5m_input_tokens: int
ephemeral_1h_input_tokens: int
class PromptTokensDetailsWrapper(BaseModel):
audio_tokens: int | None
cached_tokens: int
text_tokens: int | None
image_tokens: int | None
cache_creation_tokens: int
cache_creation_token_details: CacheCreationTokenDetails
# Add usage entries for different models
usage_entries = [
{
"model": "gpt-4o-mini",
"usage": {
"prompt_tokens": 1117,
"completion_tokens": 46,
"total_tokens": 1163,
"prompt_tokens_details": PromptTokensDetailsWrapper(
audio_tokens=None,
cached_tokens=3,
text_tokens=None,
image_tokens=None,
cache_creation_tokens=0,
cache_creation_token_details=CacheCreationTokenDetails(
ephemeral_5m_input_tokens=5, ephemeral_1h_input_tokens=0
),
),
"completion_tokens_details": {},
},
},
{
"model": "gpt-4o-mini",
"usage": {
"prompt_tokens": 800,
"completion_tokens": 100,
"total_tokens": 900,
"prompt_tokens_details": PromptTokensDetailsWrapper(
audio_tokens=None,
cached_tokens=3,
text_tokens=None,
image_tokens=None,
cache_creation_tokens=0,
cache_creation_token_details=CacheCreationTokenDetails(
ephemeral_5m_input_tokens=5, ephemeral_1h_input_tokens=0
),
),
"completion_tokens_details": None,
},
},
{
"model": "gpt-4o-mini",
"usage": {
"prompt_tokens": 800,
"completion_tokens": 100,
"total_tokens": 900,
"prompt_tokens_details": PromptTokensDetailsWrapper(
audio_tokens=None,
cached_tokens=3,
text_tokens=None,
image_tokens=None,
cache_creation_tokens=0,
cache_creation_token_details=CacheCreationTokenDetails(
ephemeral_5m_input_tokens=5, ephemeral_1h_input_tokens=0
),
),
"completion_tokens_details": {
"reasoning_tokens": 1,
"audio_tokens": 1,
"accepted_prediction_tokens": 1,
"rejected_prediction_tokens": 1,
},
},
},
]
for entry in usage_entries:
tracker.add_usage(entry["model"], entry["usage"])
total_usage = tracker.get_total_tokens()
assert total_usage["gpt-4o-mini"]["prompt_tokens"] == 2717
assert total_usage["gpt-4o-mini"]["completion_tokens"] == 246
assert total_usage["gpt-4o-mini"]["total_tokens"] == 2963
assert total_usage["gpt-4o-mini"]["prompt_tokens_details"]["cached_tokens"] == 9
assert (
total_usage["gpt-4o-mini"]["prompt_tokens_details"]["cache_creation_token_details"]["ephemeral_5m_input_tokens"]
== 15
)
assert total_usage["gpt-4o-mini"]["completion_tokens_details"]["reasoning_tokens"] == 1
assert total_usage["gpt-4o-mini"]["completion_tokens_details"]["audio_tokens"] == 1
assert total_usage["gpt-4o-mini"]["completion_tokens_details"]["accepted_prediction_tokens"] == 1
assert total_usage["gpt-4o-mini"]["completion_tokens_details"]["rejected_prediction_tokens"] == 1
def test_parallel_executor_with_usage_tracker():
"""Test that usage tracking works correctly with ParallelExecutor and mocked LM calls."""
parent_tracker = UsageTracker()
# Mock LM with different responses
mock_lm = mock.MagicMock(spec=dspy.LM)
mock_lm.return_value = ['{"answer": "Mocked answer"}']
mock_lm.kwargs = {}
mock_lm.model = "openai/gpt-4o-mini"
dspy.configure(lm=mock_lm, adapter=dspy.JSONAdapter())
def task1():
# Simulate LM usage tracking for task 1
dspy.settings.usage_tracker.add_usage(
"openai/gpt-4o-mini",
{
"prompt_tokens": 50,
"completion_tokens": 10,
"total_tokens": 60,
},
)
return dspy.settings.usage_tracker.get_total_tokens()
def task2():
# Simulate LM usage tracking for task 2 with different values
dspy.settings.usage_tracker.add_usage(
"openai/gpt-4o-mini",
{
"prompt_tokens": 80,
"completion_tokens": 15,
"total_tokens": 95,
},
)
return dspy.settings.usage_tracker.get_total_tokens()
# Execute tasks in parallel
with dspy.context(track_usage=True, usage_tracker=parent_tracker):
executor = dspy.Parallel()
results = executor([(task1, {}), (task2, {})])
# Verify that the two workers had different usage
usage1 = results[0]
usage2 = results[1]
# Task 1 should have 50 prompt tokens, task 2 should have 80
assert usage1["openai/gpt-4o-mini"]["prompt_tokens"] == 50
assert usage1["openai/gpt-4o-mini"]["completion_tokens"] == 10
assert usage2["openai/gpt-4o-mini"]["prompt_tokens"] == 80
assert usage2["openai/gpt-4o-mini"]["completion_tokens"] == 15
# Parent tracker should remain unchanged (workers have independent copies)
assert len(parent_tracker.usage_data) == 0
def test_non_numeric_usage_values_are_not_accumulated():
"""Non-count fields a provider reports in `usage` are kept, not concatenated or tallied."""
tracker = UsageTracker()
tracker.add_usage(
"openai/gpt-4o-mini",
{
"prompt_tokens": 10,
"completion_tokens": 2,
"service_tier": "standard",
"inference_geo": "us",
"iterations": None,
"is_byok": False,
},
)
tracker.add_usage(
"openai/gpt-4o-mini",
{
"prompt_tokens": 10,
"completion_tokens": 2,
"service_tier": "flex",
"inference_geo": "us",
"iterations": None,
"is_byok": False,
},
)
assert tracker.get_total_tokens()["openai/gpt-4o-mini"] == {
"prompt_tokens": 20,
"completion_tokens": 4,
"service_tier": "standard", # Keeps first
"inference_geo": "us", # Keeps value
"iterations": None, # Keeps value
"is_byok": False, # Keeps bool as-is
}
def test_non_numeric_usage_value_present_in_only_some_entries():
"""A field only some calls report must not raise when merged with the entries missing it."""
tracker = UsageTracker()
tracker.add_usage("openai/gpt-4o-mini", {"prompt_tokens": 10, "service_tier": "standard"})
tracker.add_usage("openai/gpt-4o-mini", {"prompt_tokens": 10})
assert tracker.get_total_tokens()["openai/gpt-4o-mini"] == {
"prompt_tokens": 20,
"service_tier": "standard",
}
def test_nested_usage_trackers_roll_up():
"""Test that nested usage trackers still reflect usage on the parent."""
with track_usage() as orchestrator_cost:
orchestrator_cost.add_usage(
"openai/gpt-4o-mini",
{
"prompt_tokens": 100,
"completion_tokens": 10,
"total_tokens": 110,
},
)
with track_usage() as subagent_cost:
subagent_cost.add_usage(
"openai/gpt-4o-mini",
{
"prompt_tokens": 1000,
"completion_tokens": 100,
"total_tokens": 1100,
},
)
orchestrator_cost.add_usage(
"openai/gpt-4o-mini",
{
"prompt_tokens": 10,
"completion_tokens": 1,
"total_tokens": 11,
},
)
total_prompt_tokens = 1110
total_completion_tokens = 111
total_tokens = total_prompt_tokens + total_completion_tokens
assert orchestrator_cost.get_total_tokens()["openai/gpt-4o-mini"] == {
"prompt_tokens": total_prompt_tokens,
"completion_tokens": total_completion_tokens,
"total_tokens": total_tokens,
}
def test_nested_usage_trackers_roll_up_despite_errors():
"""Test that nested usage trackers still reflect usage on the parent even when child raises."""
def erroring_subagent():
with track_usage() as subagent_cost:
subagent_cost.add_usage(
"openai/gpt-4o-mini",
{
"prompt_tokens": 1000,
"completion_tokens": 100,
"total_tokens": 1100,
},
)
raise RuntimeError("!")
with track_usage() as orchestrator_cost:
orchestrator_cost.add_usage(
"openai/gpt-4o-mini",
{
"prompt_tokens": 100,
"completion_tokens": 10,
"total_tokens": 110,
},
)
try:
erroring_subagent()
except RuntimeError:
pass
total_prompt_tokens = 1100
total_completion_tokens = 110
total_tokens = total_prompt_tokens + total_completion_tokens
assert orchestrator_cost.get_total_tokens()["openai/gpt-4o-mini"] == {
"prompt_tokens": total_prompt_tokens,
"completion_tokens": total_completion_tokens,
"total_tokens": total_tokens,
}
@pytest.mark.asyncio
async def test_nested_usage_trackers_support_async_work():
"""Test that nested usage trackers work well with asyncio.gather."""
async def llm_call():
dspy.settings.usage_tracker.add_usage(
"openai/gpt-4o-mini",
{
"prompt_tokens": 10,
"completion_tokens": 1,
"total_tokens": 11,
},
)
async def subagent():
with track_usage():
await llm_call()
with track_usage() as orchestrator_cost:
tasks = [
asyncio.create_task(subagent())
for _ in range(5)
]
await asyncio.gather(*tasks)
total_prompt_tokens = 10 * 5
total_completion_tokens = 1 * 5
total_tokens = total_prompt_tokens + total_completion_tokens
assert orchestrator_cost.get_total_tokens()["openai/gpt-4o-mini"] == {
"prompt_tokens": total_prompt_tokens,
"completion_tokens": total_completion_tokens,
"total_tokens": total_tokens,
}