The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
fix(logging): spilled over PTU requests without base_model log the served model's rates (#47655)
Kerry Lu6b1f4ed46f5f
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.pyindex 93b45be4922..004dd7b65a5 100644--- a/litellm/litellm_core_utils/litellm_logging.py+++ b/litellm/litellm_core_utils/litellm_logging.py@@ -2209,20 +2209,23 @@ class Logging(LiteLLMLoggingBaseClass):usage=usage,)- def custom_pricing_for(self, result: object) -> bool:- litellm_params: Final = getattr(self, "litellm_params", None)+ def spilled_over_ptu_for(self, result: object) -> bool:result_hidden_params: Final = getattr(result, HIDDEN_PARAMS_ATTR, None) or MappingProxyType({})additional_headers: Final = (result_hidden_params.get("additional_headers")if isinstance(result_hidden_params, dict)else getattr(result_hidden_params, "additional_headers", None))- spilled_over: Final = is_spilled_over_ptu_request(- model_info=_deployment_model_info(litellm_params),+ return is_spilled_over_ptu_request(+ model_info=_deployment_model_info(getattr(self, "litellm_params", None)),response_headers=self.model_call_details.get("response_headers"),additional_headers=additional_headers,)- return False if spilled_over else use_custom_pricing_for_model(litellm_params=litellm_params)++ def custom_pricing_for(self, result: object) -> bool:+ if self.spilled_over_ptu_for(result):+ return False+ return use_custom_pricing_for_model(litellm_params=getattr(self, "litellm_params", None))def _prompt_for_cost_calculation(self) -> str:"""@@ -7245,13 +7248,16 @@ def get_standard_logging_object_payload(saved_cache_cost: Final = (uncached_price.cost if uncached_price is not None else None) or 0.0## Get model cost information ##- base_model = get_base_model_from_metadata(model_call_details=kwargs)- # The router overrides completion_response.model to the model-group alias before- # this payload is built, so cost-map lookup via that alias always misses.- # Fall back to the actual deployment model set by the router in metadata.- if base_model is None:- base_model = metadata.get("deployment")- custom_pricing: Final = logging_obj.custom_pricing_for(init_response_obj)+ configured_base_model: Final = get_base_model_from_metadata(model_call_details=kwargs)+ spilled_over: Final = logging_obj.spilled_over_ptu_for(init_response_obj)+ # The router overrides completion_response.model to the model-group alias before this payload is+ # built, so cost-map lookup via that alias misses and the deployment model stands in for base_model.+ # A PTU request Azure spilled onto pay-as-you-go capacity is billed at the served model's rates,+ # which the deployment's zeroed PTU entry would misreport, so that fallback is skipped there.+ base_model: Final = (+ metadata.get("deployment") if configured_base_model is None and not spilled_over else configured_base_model+ )+ custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj)raw_response_cost: Final = kwargs.get("response_cost")llm_response_cost: Final[float] = raw_response_cost or 0.0guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.pyindex ff82aacf393..ace57a64aae 100644--- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py+++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py@@ -4,6 +4,7 @@ from functools import reducefrom typing import Finalimport httpx+from pydantic import TypeAdapter, ValidationErrorfrom litellm.constants import LITELLM_DETAILED_TIMINGfrom litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers@@ -101,6 +102,32 @@ def response_timing_metrics(return {"_response_ms": total_response_time_ms, "litellm_overhead_time_ms": overhead_ms}+_HIDDEN_PARAMS_DICT: Final = TypeAdapter(dict[str, object])+++def _hidden_params_mapping(hidden_params: object) -> Mapping[str, object]:+ try:+ return _HIDDEN_PARAMS_DICT.validate_python(hidden_params)+ except ValidationError:+ return {}+++def _existing_provider_response_model(result: object) -> str | None:+ """The pricing model a response already carries, like the decisions path sets, wins over the response model."""+ hidden_params: Final[object] = getattr(result, "_hidden_params", None)+ carried: Final[object] = _hidden_params_mapping(hidden_params).get("provider_response_model") or getattr(+ hidden_params, "provider_response_model", None+ )+ return carried if isinstance(carried, str) and carried else None+++def _provider_response_model(result: object) -> str | None:+ """The model the provider answered with, kept because the proxy later rewrites ``result.model`` to the alias the+ client asked for and cost-map lookups for the standard logging payload would otherwise miss."""+ model: Final = getattr(result, "model", None) if isinstance(result, ModelResponse) else None+ return model if isinstance(model, str) and model else None++class ResponseMetadata:"""Handles setting and managing `_hidden_params`, `response_time_ms`, and `litellm_overhead_time_ms` for LiteLLM responses@@ -147,6 +174,8 @@ class ResponseMetadata:preserve_litellm_internal_headers=True,),"litellm_model_name": model,+ "provider_response_model": _existing_provider_response_model(self.result)+ or _provider_response_model(self.result),}self._update_hidden_params(new_params)diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.pyindex 00b0e2f6972..6d9d8c7997a 100644--- a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py+++ b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py@@ -17,6 +17,7 @@ import pytestimport litellmimport litellm.litellm_core_utils.llm_response_utils.response_metadata as response_metadata_modimport litellm.proxy.common_request_processing as common_request_processing_mod+from litellm.cost_calculator import select_model_name_for_cost_calcfrom litellm.litellm_core_utils.litellm_logging import Loggingfrom litellm.litellm_core_utils.llm_response_utils.response_metadata import (ResponseMetadata,@@ -649,3 +650,38 @@ def test_update_response_metadata_prices_per_second_deployment_from_its_stamped_assert result._response_ms == pytest.approx(2000)assert result._hidden_params["response_cost"] == pytest.approx(0.02 * 2)+++def test_update_response_metadata_keeps_the_provider_model_after_the_proxy_rewrites_response_model():+ start_time: Final = datetime.datetime(2026, 9, 21, 12, 0, 0)+ logging_obj: Final = Logging(+ model="azure/ptu-deployment",+ messages=[{"role": "user", "content": "Hello"}],+ stream=False,+ call_type="completion",+ start_time=start_time,+ litellm_call_id="provider-model-response-metadata",+ function_id="f",+ )+ logging_obj.update_environment_variables(+ model="azure/ptu-deployment", litellm_params={}, optional_params={}, custom_llm_provider="azure"+ )+ result: Final = ModelResponse(+ model="gpt-5.4-nano-2026-03-17",+ usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18),+ )++ update_response_metadata(+ result=result,+ logging_obj=logging_obj,+ model="azure/ptu-deployment",+ kwargs={},+ start_time=start_time,+ end_time=start_time + datetime.timedelta(seconds=2),+ )+ result.model = "model-group-alias"++ assert (+ select_model_name_for_cost_calc(model=None, completion_response=result, custom_llm_provider="azure")+ == "azure/gpt-5.4-nano-2026-03-17"+ )diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.pyindex b3414ebe9d7..79645ffe8f6 100644--- a/tests/unit/litellm_core_utils/test_litellm_logging.py+++ b/tests/unit/litellm_core_utils/test_litellm_logging.py@@ -10168,6 +10168,7 @@ class TestAzurePTUSpilloverCost:ROUTER_MODEL_ID: Final = "ptu-spill-router-model-id"SERVED_MODEL: Final = "azure/spill-served-model-ptu"+ PTU_DEPLOYMENT: Final = "azure/spill-ptu-deployment"PTU_MODEL_INFO: Final = {"id": ROUTER_MODEL_ID,"team_id": "team-1",@@ -10203,7 +10204,9 @@ class TestAzurePTUSpilloverCost:litellm.model_cost.pop(TestAzurePTUSpilloverCost.ROUTER_MODEL_ID, None)litellm.model_cost.pop(TestAzurePTUSpilloverCost.SERVED_MODEL, None)- def _logging_obj(self, model_info: dict, *, flag: str, litellm_rate: float, monkeypatch) -> LitellmLogging:+ def _logging_obj(+ self, model_info: dict, *, flag: str, litellm_rate: float, monkeypatch, deployment: str | None = None+ ) -> LitellmLogging:monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", flag)obj = LitellmLogging(model=self.SERVED_MODEL,@@ -10220,7 +10223,7 @@ class TestAzurePTUSpilloverCost:optional_params={},litellm_params={"api_base": "",- "metadata": {"model_info": model_info},+ "metadata": {"model_info": model_info, "deployment": deployment},"input_cost_per_token": litellm_rate,"output_cost_per_token": litellm_rate,},@@ -10324,6 +10327,50 @@ class TestAzurePTUSpilloverCost:finally:self._unregister_models()+ def test_spillover_without_base_model_reports_the_pay_as_you_go_rates_billing_used(self, monkeypatch) -> None:+ self._register_models()+ litellm.model_cost[self.PTU_DEPLOYMENT] = {"litellm_provider": "azure", "mode": "chat"}+ try:+ obj = self._logging_obj(+ dict(self.PTU_MODEL_INFO),+ flag="True",+ litellm_rate=0.0,+ monkeypatch=monkeypatch,+ deployment=self.PTU_DEPLOYMENT,+ )+ response = self._response()+ response._hidden_params["additional_headers"] = {"llm_provider-x-ms-is-spilled-over": "true"}+ response._hidden_params["provider_response_model"] = response.model+ response_cost: Final = obj.response_cost_calculator(result=response)+ response.model = "spill-model-group-alias"++ payload: Final = get_standard_logging_object_payload(+ kwargs={+ "model": self.SERVED_MODEL,+ "custom_llm_provider": "azure",+ "litellm_params": obj.litellm_params,+ "response_cost": response_cost,+ },+ init_response_obj=response,+ start_time=datetime.datetime.now(),+ end_time=datetime.datetime.now(),+ logging_obj=obj,+ status="success",+ )++ assert response_cost == pytest.approx(self.EXPECTED_SPILL_COST)+ assert payload is not None+ model_map_information: Final = payload["model_map_information"]+ assert model_map_information["model_map_key"] != self.PTU_DEPLOYMENT+ model_map_value: Final = model_map_information["model_map_value"]+ assert model_map_value is not None+ assert response_cost == pytest.approx(+ 100 * model_map_value["input_cost_per_token"] + 50 * model_map_value["output_cost_per_token"]+ )+ finally:+ litellm.model_cost.pop(self.PTU_DEPLOYMENT, None)+ self._unregister_models()+def test_spillover_header_does_not_touch_non_ptu_custom_pricing(self, monkeypatch) -> None:self._register_models()custom_model_id: Final = "non-ptu-custom-router-model-id"