Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

fix(logging): spilled over PTU requests without base_model log the served model's rates (#47655)

Kerry Lu6b1f4ed46f5f
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index 93b45be4922..004dd7b65a5 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -2209,20 +2209,23 @@ class Logging(LiteLLMLoggingBaseClass):
usage=usage,
)
- def custom_pricing_for(self, result: object) -> bool:
- litellm_params: Final = getattr(self, "litellm_params", None)
+ def spilled_over_ptu_for(self, result: object) -> bool:
result_hidden_params: Final = getattr(result, HIDDEN_PARAMS_ATTR, None) or MappingProxyType({})
additional_headers: Final = (
result_hidden_params.get("additional_headers")
if isinstance(result_hidden_params, dict)
else getattr(result_hidden_params, "additional_headers", None)
)
- spilled_over: Final = is_spilled_over_ptu_request(
- model_info=_deployment_model_info(litellm_params),
+ return is_spilled_over_ptu_request(
+ model_info=_deployment_model_info(getattr(self, "litellm_params", None)),
response_headers=self.model_call_details.get("response_headers"),
additional_headers=additional_headers,
)
- return False if spilled_over else use_custom_pricing_for_model(litellm_params=litellm_params)
+
+ def custom_pricing_for(self, result: object) -> bool:
+ if self.spilled_over_ptu_for(result):
+ return False
+ return use_custom_pricing_for_model(litellm_params=getattr(self, "litellm_params", None))
def _prompt_for_cost_calculation(self) -> str:
"""
@@ -7245,13 +7248,16 @@ def get_standard_logging_object_payload(
saved_cache_cost: Final = (uncached_price.cost if uncached_price is not None else None) or 0.0
## Get model cost information ##
- base_model = get_base_model_from_metadata(model_call_details=kwargs)
- # The router overrides completion_response.model to the model-group alias before
- # this payload is built, so cost-map lookup via that alias always misses.
- # Fall back to the actual deployment model set by the router in metadata.
- if base_model is None:
- base_model = metadata.get("deployment")
- custom_pricing: Final = logging_obj.custom_pricing_for(init_response_obj)
+ configured_base_model: Final = get_base_model_from_metadata(model_call_details=kwargs)
+ spilled_over: Final = logging_obj.spilled_over_ptu_for(init_response_obj)
+ # The router overrides completion_response.model to the model-group alias before this payload is
+ # built, so cost-map lookup via that alias misses and the deployment model stands in for base_model.
+ # A PTU request Azure spilled onto pay-as-you-go capacity is billed at the served model's rates,
+ # which the deployment's zeroed PTU entry would misreport, so that fallback is skipped there.
+ base_model: Final = (
+ metadata.get("deployment") if configured_base_model is None and not spilled_over else configured_base_model
+ )
+ custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj)
raw_response_cost: Final = kwargs.get("response_cost")
llm_response_cost: Final[float] = raw_response_cost or 0.0
guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py
index ff82aacf393..ace57a64aae 100644
--- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py
+++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py
@@ -4,6 +4,7 @@ from functools import reduce
from typing import Final
import httpx
+from pydantic import TypeAdapter, ValidationError
from litellm.constants import LITELLM_DETAILED_TIMING
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers
@@ -101,6 +102,32 @@ def response_timing_metrics(
return {"_response_ms": total_response_time_ms, "litellm_overhead_time_ms": overhead_ms}
+_HIDDEN_PARAMS_DICT: Final = TypeAdapter(dict[str, object])
+
+
+def _hidden_params_mapping(hidden_params: object) -> Mapping[str, object]:
+ try:
+ return _HIDDEN_PARAMS_DICT.validate_python(hidden_params)
+ except ValidationError:
+ return {}
+
+
+def _existing_provider_response_model(result: object) -> str | None:
+ """The pricing model a response already carries, like the decisions path sets, wins over the response model."""
+ hidden_params: Final[object] = getattr(result, "_hidden_params", None)
+ carried: Final[object] = _hidden_params_mapping(hidden_params).get("provider_response_model") or getattr(
+ hidden_params, "provider_response_model", None
+ )
+ return carried if isinstance(carried, str) and carried else None
+
+
+def _provider_response_model(result: object) -> str | None:
+ """The model the provider answered with, kept because the proxy later rewrites ``result.model`` to the alias the
+ client asked for and cost-map lookups for the standard logging payload would otherwise miss."""
+ model: Final = getattr(result, "model", None) if isinstance(result, ModelResponse) else None
+ return model if isinstance(model, str) and model else None
+
+
class ResponseMetadata:
"""
Handles setting and managing `_hidden_params`, `response_time_ms`, and `litellm_overhead_time_ms` for LiteLLM responses
@@ -147,6 +174,8 @@ class ResponseMetadata:
preserve_litellm_internal_headers=True,
),
"litellm_model_name": model,
+ "provider_response_model": _existing_provider_response_model(self.result)
+ or _provider_response_model(self.result),
}
self._update_hidden_params(new_params)
diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py
index 00b0e2f6972..6d9d8c7997a 100644
--- a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py
+++ b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py
@@ -17,6 +17,7 @@ import pytest
import litellm
import litellm.litellm_core_utils.llm_response_utils.response_metadata as response_metadata_mod
import litellm.proxy.common_request_processing as common_request_processing_mod
+from litellm.cost_calculator import select_model_name_for_cost_calc
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
ResponseMetadata,
@@ -649,3 +650,38 @@ def test_update_response_metadata_prices_per_second_deployment_from_its_stamped_
assert result._response_ms == pytest.approx(2000)
assert result._hidden_params["response_cost"] == pytest.approx(0.02 * 2)
+
+
+def test_update_response_metadata_keeps_the_provider_model_after_the_proxy_rewrites_response_model():
+ start_time: Final = datetime.datetime(2026, 9, 21, 12, 0, 0)
+ logging_obj: Final = Logging(
+ model="azure/ptu-deployment",
+ messages=[{"role": "user", "content": "Hello"}],
+ stream=False,
+ call_type="completion",
+ start_time=start_time,
+ litellm_call_id="provider-model-response-metadata",
+ function_id="f",
+ )
+ logging_obj.update_environment_variables(
+ model="azure/ptu-deployment", litellm_params={}, optional_params={}, custom_llm_provider="azure"
+ )
+ result: Final = ModelResponse(
+ model="gpt-5.4-nano-2026-03-17",
+ usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18),
+ )
+
+ update_response_metadata(
+ result=result,
+ logging_obj=logging_obj,
+ model="azure/ptu-deployment",
+ kwargs={},
+ start_time=start_time,
+ end_time=start_time + datetime.timedelta(seconds=2),
+ )
+ result.model = "model-group-alias"
+
+ assert (
+ select_model_name_for_cost_calc(model=None, completion_response=result, custom_llm_provider="azure")
+ == "azure/gpt-5.4-nano-2026-03-17"
+ )
diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py
index b3414ebe9d7..79645ffe8f6 100644
--- a/tests/unit/litellm_core_utils/test_litellm_logging.py
+++ b/tests/unit/litellm_core_utils/test_litellm_logging.py
@@ -10168,6 +10168,7 @@ class TestAzurePTUSpilloverCost:
ROUTER_MODEL_ID: Final = "ptu-spill-router-model-id"
SERVED_MODEL: Final = "azure/spill-served-model-ptu"
+ PTU_DEPLOYMENT: Final = "azure/spill-ptu-deployment"
PTU_MODEL_INFO: Final = {
"id": ROUTER_MODEL_ID,
"team_id": "team-1",
@@ -10203,7 +10204,9 @@ class TestAzurePTUSpilloverCost:
litellm.model_cost.pop(TestAzurePTUSpilloverCost.ROUTER_MODEL_ID, None)
litellm.model_cost.pop(TestAzurePTUSpilloverCost.SERVED_MODEL, None)
- def _logging_obj(self, model_info: dict, *, flag: str, litellm_rate: float, monkeypatch) -> LitellmLogging:
+ def _logging_obj(
+ self, model_info: dict, *, flag: str, litellm_rate: float, monkeypatch, deployment: str | None = None
+ ) -> LitellmLogging:
monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", flag)
obj = LitellmLogging(
model=self.SERVED_MODEL,
@@ -10220,7 +10223,7 @@ class TestAzurePTUSpilloverCost:
optional_params={},
litellm_params={
"api_base": "",
- "metadata": {"model_info": model_info},
+ "metadata": {"model_info": model_info, "deployment": deployment},
"input_cost_per_token": litellm_rate,
"output_cost_per_token": litellm_rate,
},
@@ -10324,6 +10327,50 @@ class TestAzurePTUSpilloverCost:
finally:
self._unregister_models()
+ def test_spillover_without_base_model_reports_the_pay_as_you_go_rates_billing_used(self, monkeypatch) -> None:
+ self._register_models()
+ litellm.model_cost[self.PTU_DEPLOYMENT] = {"litellm_provider": "azure", "mode": "chat"}
+ try:
+ obj = self._logging_obj(
+ dict(self.PTU_MODEL_INFO),
+ flag="True",
+ litellm_rate=0.0,
+ monkeypatch=monkeypatch,
+ deployment=self.PTU_DEPLOYMENT,
+ )
+ response = self._response()
+ response._hidden_params["additional_headers"] = {"llm_provider-x-ms-is-spilled-over": "true"}
+ response._hidden_params["provider_response_model"] = response.model
+ response_cost: Final = obj.response_cost_calculator(result=response)
+ response.model = "spill-model-group-alias"
+
+ payload: Final = get_standard_logging_object_payload(
+ kwargs={
+ "model": self.SERVED_MODEL,
+ "custom_llm_provider": "azure",
+ "litellm_params": obj.litellm_params,
+ "response_cost": response_cost,
+ },
+ init_response_obj=response,
+ start_time=datetime.datetime.now(),
+ end_time=datetime.datetime.now(),
+ logging_obj=obj,
+ status="success",
+ )
+
+ assert response_cost == pytest.approx(self.EXPECTED_SPILL_COST)
+ assert payload is not None
+ model_map_information: Final = payload["model_map_information"]
+ assert model_map_information["model_map_key"] != self.PTU_DEPLOYMENT
+ model_map_value: Final = model_map_information["model_map_value"]
+ assert model_map_value is not None
+ assert response_cost == pytest.approx(
+ 100 * model_map_value["input_cost_per_token"] + 50 * model_map_value["output_cost_per_token"]
+ )
+ finally:
+ litellm.model_cost.pop(self.PTU_DEPLOYMENT, None)
+ self._unregister_models()
+
def test_spillover_header_does_not_touch_non_ptu_custom_pricing(self, monkeypatch) -> None:
self._register_models()
custom_model_id: Final = "non-ptu-custom-router-model-id"