Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Refresh LiteLLM metadata to 1.106.0 and reject router-fee-only estimates (#10670)

Isaac Miller1a4f8372eb47
diff --git a/dspy/clients/costs.py b/dspy/clients/costs.py
index 56b0825be..70a4f2afa 100644
--- a/dspy/clients/costs.py
+++ b/dspy/clients/costs.py
@@ -46,6 +46,9 @@ def _estimate_cost(response, *, provider, requested_model, request=None, namespa
if not info:
info = model_info(provider, requested_model, namespaces=namespaces)
priced_model = requested_model
+ # This catalog entry prices only routing, not the selected model's inference.
+ if info.get("litellm_provider") == "azure" and priced_model.removeprefix("azure/") == "model-router":
+ return unknown("Azure model-router fees exclude the selected model's inference cost")
u = response.usage
if u.input_tokens is None or u.output_tokens is None:
return unknown("input or output usage was not reported")
diff --git a/dspy/clients/model_metadata/provenance.json b/dspy/clients/model_metadata/provenance.json
index 40e4518e0..da35587d7 100644
--- a/dspy/clients/model_metadata/provenance.json
+++ b/dspy/clients/model_metadata/provenance.json
@@ -1,6 +1,6 @@
{
- "source": "https://files.pythonhosted.org/packages/d7/71/72b0e4c54a8fb0e10a35adb611f2f03febc27ac4fdf9b372c4f4a7d53442/litellm-1.105.1-cp310-abi3-macosx_10_12_x86_64.whl",
- "version": "1.105.1",
- "wheel_sha256": "633886584ffb6421485349376f2c1666d4854fecc3ab8b1cdb24537817d4e1cc",
- "snapshot_sha256": "329113e5820834dc2a206500db9ec7861a17c74b86fe79601dec05478c06327e"
+ "source": "https://files.pythonhosted.org/packages/fd/96/b68c1db86a1caae616d48bbb00299bfb716b58fca02645178e0c9a0ca2ce/litellm-1.106.0-cp310-abi3-macosx_10_12_x86_64.whl",
+ "version": "1.106.0",
+ "wheel_sha256": "79bdfc1acb725e8ce75acfdb4913156a350faba835a39091338d320548e78593",
+ "snapshot_sha256": "fadace20b834db49d02765a505709ee5dc7a37e722f53cc9c393fdb02ace2dea"
}
diff --git a/dspy/clients/model_metadata/snapshot.json.gz b/dspy/clients/model_metadata/snapshot.json.gz
index 28c70db66..a316467cb 100644
Binary files a/dspy/clients/model_metadata/snapshot.json.gz and b/dspy/clients/model_metadata/snapshot.json.gz differ
diff --git a/tests/clients/test_native_metadata.py b/tests/clients/test_native_metadata.py
index 01135cbbf..58b541085 100644
--- a/tests/clients/test_native_metadata.py
+++ b/tests/clients/test_native_metadata.py
@@ -134,6 +134,25 @@ def test_unknown_price_and_malformed_metadata_do_not_fail_generation(monkeypatch
assert estimate_cost(response, provider="openai-chat", requested_model="unknown")[0] is None
[email protected]("router_name", ["model-router", "azure/model-router"])
[email protected]("fallback", [False, True])
+def test_azure_router_fee_is_not_an_inference_estimate(monkeypatch, router_name, fallback):
+ data = metadata._snapshot()
+ assert data["azure/model-router"]["input_cost_per_token"] > 0
+ data["openai/model-router"] = {"litellm_provider": "openai", "input_cost_per_token": 2e-6,
+ "output_cost_per_token": 7e-6}
+ monkeypatch.setattr(metadata, "_data", data)
+ response = answer("unlisted-selected-model" if fallback else router_name, input_tokens=1000, output_tokens=500)
+ for provider, namespaces in (("azure-chat", None), ("declared-azure", ("azure",))):
+ cost, details = estimate_cost(response, provider=provider, requested_model=router_name, namespaces=namespaces)
+ assert cost is None
+ assert details["kind"] == "unknown"
+ # The refusal belongs to the Azure catalog entry, not an arbitrary model name.
+ cost, _ = estimate_cost(answer("model-router", input_tokens=1000, output_tokens=500),
+ provider="openai-chat", requested_model="model-router")
+ assert cost == pytest.approx(0.0055)
+
+
def test_cost_provenance_survives_cache_record():
from dspy.clients.call_result import CallResult, combine