DSPy: The framework for programming—not prompting—language models
Python
Refresh LiteLLM metadata to 1.106.0 and reject router-fee-only estimates (#10670)
Isaac Miller1a4f8372eb47
diff --git a/dspy/clients/costs.py b/dspy/clients/costs.pyindex 56b0825be..70a4f2afa 100644--- a/dspy/clients/costs.py+++ b/dspy/clients/costs.py@@ -46,6 +46,9 @@ def _estimate_cost(response, *, provider, requested_model, request=None, namespaif not info:info = model_info(provider, requested_model, namespaces=namespaces)priced_model = requested_model+ # This catalog entry prices only routing, not the selected model's inference.+ if info.get("litellm_provider") == "azure" and priced_model.removeprefix("azure/") == "model-router":+ return unknown("Azure model-router fees exclude the selected model's inference cost")u = response.usageif u.input_tokens is None or u.output_tokens is None:return unknown("input or output usage was not reported")diff --git a/dspy/clients/model_metadata/provenance.json b/dspy/clients/model_metadata/provenance.jsonindex 40e4518e0..da35587d7 100644--- a/dspy/clients/model_metadata/provenance.json+++ b/dspy/clients/model_metadata/provenance.json@@ -1,6 +1,6 @@{- "source": "https://files.pythonhosted.org/packages/d7/71/72b0e4c54a8fb0e10a35adb611f2f03febc27ac4fdf9b372c4f4a7d53442/litellm-1.105.1-cp310-abi3-macosx_10_12_x86_64.whl",- "version": "1.105.1",- "wheel_sha256": "633886584ffb6421485349376f2c1666d4854fecc3ab8b1cdb24537817d4e1cc",- "snapshot_sha256": "329113e5820834dc2a206500db9ec7861a17c74b86fe79601dec05478c06327e"+ "source": "https://files.pythonhosted.org/packages/fd/96/b68c1db86a1caae616d48bbb00299bfb716b58fca02645178e0c9a0ca2ce/litellm-1.106.0-cp310-abi3-macosx_10_12_x86_64.whl",+ "version": "1.106.0",+ "wheel_sha256": "79bdfc1acb725e8ce75acfdb4913156a350faba835a39091338d320548e78593",+ "snapshot_sha256": "fadace20b834db49d02765a505709ee5dc7a37e722f53cc9c393fdb02ace2dea"}diff --git a/dspy/clients/model_metadata/snapshot.json.gz b/dspy/clients/model_metadata/snapshot.json.gzindex 28c70db66..a316467cb 100644Binary files a/dspy/clients/model_metadata/snapshot.json.gz and b/dspy/clients/model_metadata/snapshot.json.gz differdiff --git a/tests/clients/test_native_metadata.py b/tests/clients/test_native_metadata.pyindex 01135cbbf..58b541085 100644--- a/tests/clients/test_native_metadata.py+++ b/tests/clients/test_native_metadata.py@@ -134,6 +134,25 @@ def test_unknown_price_and_malformed_metadata_do_not_fail_generation(monkeypatchassert estimate_cost(response, provider="openai-chat", requested_model="unknown")[0] is None[email protected]("router_name", ["model-router", "azure/model-router"])[email protected]("fallback", [False, True])+def test_azure_router_fee_is_not_an_inference_estimate(monkeypatch, router_name, fallback):+ data = metadata._snapshot()+ assert data["azure/model-router"]["input_cost_per_token"] > 0+ data["openai/model-router"] = {"litellm_provider": "openai", "input_cost_per_token": 2e-6,+ "output_cost_per_token": 7e-6}+ monkeypatch.setattr(metadata, "_data", data)+ response = answer("unlisted-selected-model" if fallback else router_name, input_tokens=1000, output_tokens=500)+ for provider, namespaces in (("azure-chat", None), ("declared-azure", ("azure",))):+ cost, details = estimate_cost(response, provider=provider, requested_model=router_name, namespaces=namespaces)+ assert cost is None+ assert details["kind"] == "unknown"+ # The refusal belongs to the Azure catalog entry, not an arbitrary model name.+ cost, _ = estimate_cost(answer("model-router", input_tokens=1000, output_tokens=500),+ provider="openai-chat", requested_model="model-router")+ assert cost == pytest.approx(0.0055)++def test_cost_provenance_survives_cache_record():from dspy.clients.call_result import CallResult, combine