The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
fix(proxy): attribute retry attempt spend rows to their deployment and keep them out of daily request counts (#47658)
Kerry Lu5dce13d2c249
diff --git a/litellm/constants.py b/litellm/constants.pyindex 5b99a0f73dd..1c1dfb9aa1c 100644--- a/litellm/constants.py+++ b/litellm/constants.py@@ -1654,6 +1654,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"+BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.pyindex 97a2fc31fb0..54acf59ef80 100644--- a/litellm/proxy/_types.py+++ b/litellm/proxy/_types.py@@ -3731,6 +3731,7 @@ class SpendLogsMetadata(TypedDict):routing_decision: StandardLoggingRoutingDecision | Nonekubernetes_pod_routing: ReadOnly[StandardLoggingKubernetesPodRouting | None]internal_call_origin: InternalCallOrigin | None+ billed_retry_attempt: ReadOnly[NotRequired[bool | None]]litellm_roi_estimator: ReadOnly[NotRequired[bool | None]]guardrail_information: list[StandardLoggingGuardrailInformation] | Noneeval_information: Any | Nonediff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.pyindex bb6ffe675c0..4e98823ba77 100644--- a/litellm/proxy/db/db_spend_update_writer.py+++ b/litellm/proxy/db/db_spend_update_writer.py@@ -29,6 +29,7 @@ from litellm._internal_context import service_target, with_service_targetfrom litellm._logging import verbose_proxy_loggerfrom litellm.caching import RedisCachefrom litellm.constants import (+ BILLED_RETRY_ATTEMPT_METADATA_KEY,DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME,DB_SPEND_UPDATE_JOB_NAME,INTERNAL_CALL_ORIGIN_METADATA_KEY,@@ -312,6 +313,11 @@ def _spend_tables_left_to_send(return cast(DBSpendUpdateTransactions, remaining) if any(remaining.values()) else None+def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool:+ """An earlier billed retry attempt's own spend row, not a request the caller sent."""+ return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True++def _timed_request_duration_ms(payload: dict | SpendLogsPayload,request_status: Literal["success", "failure"],@@ -2805,11 +2811,12 @@ class DBSpendUpdateWriter:if not date:returnmetadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])+ internal_origin: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaultsrequest_error_accumulator.record(payload=payload,request_status=prisma_client.get_request_status(payload), # pyright: ignore[reportUnknownMemberType] # legacy payload union; this caller supplies a typed spend payloaddate=date.split("T")[0],- is_internal_call=bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)), # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults+ is_internal_call=internal_origin or _is_billed_retry_attempt(metadata),)async def _common_add_spend_log_transaction_to_daily_transaction(@@ -2874,6 +2881,7 @@ class DBSpendUpdateWriter:endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None)is_internal_call: Final = bool(_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY))+ caller_request: Final = not is_internal_call and not _is_billed_retry_attempt(_metadata)cache_read_input_tokens: Final = extract_cache_read_tokens(usage_obj)compression_saved_tokens: Final = extract_compression_saved_tokens(_metadata)savings_spend: Final = compute_savings_spend(@@ -2908,9 +2916,9 @@ class DBSpendUpdateWriter:# requests the caller made: counting them inflates request-volume# readers, and an auto-router savings figure computed on a shadow# duplicate credits savings for traffic no user sent.- api_requests=0 if is_internal_call else 1,- successful_requests=1 if not is_internal_call and request_status == "success" else 0,- failed_requests=1 if not is_internal_call and request_status != "success" else 0,+ api_requests=1 if caller_request else 0,+ successful_requests=1 if caller_request and request_status == "success" else 0,+ failed_requests=1 if caller_request and request_status != "success" else 0,cache_read_input_tokens=cache_read_input_tokens,cache_creation_input_tokens=extract_cache_creation_tokens(usage_obj),compression_saved_tokens=compression_saved_tokens,diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.pyindex 1f8b352fed9..994310c6619 100644--- a/litellm/proxy/hooks/proxy_track_cost_callback.py+++ b/litellm/proxy/hooks/proxy_track_cost_callback.py@@ -5,14 +5,14 @@ from contextlib import contextmanagerfrom contextvars import ContextVarfrom datetime import datetime, timezonefrom types import MappingProxyType-from typing import TYPE_CHECKING, Final, Protocol, cast+from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, castfrom pydantic import TypeAdapter, ValidationErrorimport litellmfrom litellm._logging import verbose_proxy_loggerfrom litellm.batches.batch_utils import batch_cost_is_final-from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED+from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED, BILLED_RETRY_ATTEMPT_METADATA_KEYfrom litellm.integrations.custom_logger import CustomLoggerfrom litellm.litellm_core_utils.core_helpers import (budget_reservation_from_metadata,@@ -54,11 +54,13 @@ from litellm.proxy.spend_tracking.spend_log_error_logger import ()from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 # legacy module exports_sanitize_error_information_for_spend_logs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export+ deployment_params_provider,get_request_model_access_groups,sanitize_error_information_for_spend_logs,should_store_prompts_and_responses_in_spend_logs,)from litellm.proxy.utils import BilledFailedAttempt, ProxyUpdateSpend+from litellm.types.router import Deploymentfrom litellm.types.utils import (CallTypes,LiteLLMBatch,@@ -187,6 +189,14 @@ def _proxy_spend_writer() -> DBSpendUpdateWriter:return proxy_logging_obj.db_spend_update_writer+DeploymentLookup: TypeAlias = Callable[[str], Deployment | None]+++def _proxy_deployment(model_id: str) -> Deployment | None:+ router: Final = get_llm_router()+ return None if router is None else router.get_deployment(model_id=model_id)++def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledFailedAttempt, ...]:"""The earlier billed attempts ``_failure_fields_to_lift`` stashed on the request, if any."""attempts: Final = request_data.get("billed_failed_attempts")@@ -197,7 +207,10 @@ def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledFdef _failure_spend_rows(- request_data: Mapping[str, object], final_cost: float, late_guardrails_snapshot: object+ request_data: Mapping[str, object],+ final_cost: float,+ late_guardrails_snapshot: object,+ deployment_lookup: DeploymentLookup,) -> tuple[tuple[float, Mapping[str, object]], ...]:"""One ``(response_cost, kwargs)`` per spend row a failure writes: the final attempt under therequest id, then every earlier attempt the provider also billed under its own row."""@@ -209,25 +222,59 @@ def _failure_spend_rows(return ((final_cost, final_row),*(- (attempt.response_cost, _attempt_spend_row_kwargs(request_data, attempt, position))+ (+ attempt.response_cost,+ _attempt_spend_row_kwargs(+ request_data, attempt, position, _attempt_deployment(attempt, deployment_lookup)+ ),+ )for position, attempt in enumerate(_billed_failed_attempts(request_data))),)+def _attempt_deployment(attempt: BilledFailedAttempt, deployment_lookup: DeploymentLookup) -> Deployment | None:+ return None if attempt.deployment_id is None else deployment_lookup(attempt.deployment_id)+++def _deployment_attribution(deployment: Deployment) -> Mapping[str, object]:+ """The spend-log attribution fields of the deployment that served an attempt: the model string it+ was called with, its provider, its configured ``api_base`` and its model group."""+ return MappingProxyType(+ {+ "model": deployment.litellm_params.model,+ "custom_llm_provider": deployment_params_provider(deployment.litellm_params),+ "api_base": deployment.litellm_params.api_base,+ "model_group": deployment.model_name,+ }+ )+++_NO_ATTRIBUTION: Final[Mapping[str, object]] = MappingProxyType({})++def _attempt_spend_row_kwargs(- request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int+ request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int, deployment: Deployment | None) -> Mapping[str, object]:"""The final attempt's row kwargs re-keyed for one earlier billed attempt: its own``request_id`` (``<id>-attempt-<position>``, since ``request_id`` is the spend-log primary keyand the retry count restarts after a fallback hop), its estimated usage and cost, its retry- count under ``attempted_retries`` and the deployment that billed it under ``model_info.id``."""+ count under ``attempted_retries``, the deployment that billed it under ``model_info.id`` and,+ when the router still knows that deployment, its model, provider, ``api_base`` and model group.+ The row is stamped ``billed_retry_attempt`` so the daily spend tables keep its spend, tokens+ and model usage without counting it as a request the caller sent."""suffix: Final = f"-attempt-{position}"+ attribution: Final = _NO_ATTRIBUTION if deployment is None else _deployment_attribution(deployment)raw_params: Final = cast("Mapping[str, object]", request_data.get("litellm_params") or {}) # cast-ok: request dict- litellm_params: Final = dict(raw_params)+ litellm_params: Final = {+ **raw_params,+ **({} if deployment is None else {"api_base": attribution["api_base"]}),+ }metadata: Final = {**cast("Mapping[str, object]", litellm_params.get("metadata") or {}), # cast-ok: request dict"attempted_retries": attempt.attempted_retries,+ BILLED_RETRY_ATTEMPT_METADATA_KEY: True,+ **({} if deployment is None else {"model_group": attribution["model_group"]}),}metadata.pop("previous_models", None)if attempt.deployment_id is not None:@@ -238,6 +285,7 @@ def _attempt_spend_row_kwargs(litellm_params["metadata"] = metadatarow: Final = {**request_data,+ **({} if deployment is None else {k: attribution[k] for k in ("model", "custom_llm_provider")}),"litellm_params": litellm_params,"combined_usage_object": attempt.usage,"response_cost": attempt.response_cost,@@ -250,7 +298,11 @@ def _attempt_spend_row_kwargs(if isinstance(standard_logging_object, dict):payload: Final = cast("Mapping[str, object]", standard_logging_object) # cast-ok: StandardLoggingPayload dictif isinstance(payload.get("id"), str):- row["standard_logging_object"] = {**payload, "id": f"{payload['id']}{suffix}"}+ row["standard_logging_object"] = {+ **payload,+ "id": f"{payload['id']}{suffix}",+ **({} if deployment is None else {**attribution, "model_id": attempt.deployment_id}),+ }return row@@ -262,10 +314,12 @@ class _ProxyDBLogger(CustomLogger):turn_off_message_logging: bool = False,message_logging: bool = True,spend_writer: Callable[[], DBSpendUpdateWriter] = _proxy_spend_writer,+ deployment_lookup: DeploymentLookup = _proxy_deployment,) -> None:super().__init__(turn_off_message_logging=turn_off_message_logging, message_logging=message_logging)self.spend_event_producer = spend_event_producerself._spend_writer: Final = spend_writer+ self._deployment_lookup: Final = deployment_lookupasync def async_log_success_event(self, kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime@@ -439,6 +493,7 @@ class _ProxyDBLogger(CustomLogger):cast("Mapping[str, object]", request_data), # cast-ok: request dict, copied per row belowrecovered_response_cost,failure_snapshot_with_late_guardrails,+ self._deployment_lookup,),)diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.pyindex 3ac270fcf2c..09f53cecb72 100644--- a/litellm/proxy/litellm_pre_call_utils.py+++ b/litellm/proxy/litellm_pre_call_utils.py@@ -29,6 +29,7 @@ from litellm._service_logger import ServiceLoggingfrom litellm._uuid import uuidfrom litellm.constants import (BAGGAGE_MAPPED_TAGS_METADATA_KEY,+ BILLED_RETRY_ATTEMPT_METADATA_KEY,CALLER_SESSION_ID_METADATA_KEY,CLIENT_OUTPUT_CEILING_METADATA_KEY,CONSUMED_REQUEST_TAGS_METADATA_KEY,@@ -426,6 +427,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = (CONSUMED_REQUEST_TAGS_METADATA_KEY,ROUTING_REQUEST_TAGS_METADATA_KEY,INTERNAL_CALL_ORIGIN_METADATA_KEY,+ BILLED_RETRY_ATTEMPT_METADATA_KEY,MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,"standard_logging_object","litellm_roi_estimator",diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.pyindex 091fb5d1305..62aee788ed0 100644--- a/litellm/proxy/spend_tracking/spend_tracking_utils.py+++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py@@ -15,6 +15,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationErrorimport litellmfrom litellm._logging import verbose_proxy_loggerfrom litellm.constants import (+ BILLED_RETRY_ATTEMPT_METADATA_KEY,CALLER_SESSION_ID_METADATA_KEY,CLI_SESSION_KEY_PREFIX,EMPTY_MAPPING,@@ -187,6 +188,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset("used_client_oauth_token",KUBERNETES_POD_ROUTING_KEY,"litellm_roi_estimator",+ BILLED_RETRY_ATTEMPT_METADATA_KEY,"user_api_key_user_email",))@@ -249,6 +251,7 @@ def _get_spend_logs_metadata(guardrail_information=None,internal_call_origin=None,litellm_roi_estimator=False,+ billed_retry_attempt=False,eval_information=None,cold_storage_object_key=cold_storage_object_key,litellm_overhead_time_ms=None,@@ -270,6 +273,8 @@ def _get_spend_logs_metadata()verbose_proxy_logger.debug("getting payload for SpendLogs, available keys in metadata: %s", [*metadata])+ retry_key: Final = BILLED_RETRY_ATTEMPT_METADATA_KEY+ billed_retry_attempt: Final = retry_key in metadata and metadata[retry_key] is True# Filter the metadata dictionary to include only the specified keysclean_metadata: Final = SpendLogsMetadata(**MappingProxyType(@@ -283,6 +288,7 @@ def _get_spend_logs_metadata(used_client_oauth_token=used_client_oauth_token,kubernetes_pod_routing=kubernetes_pod_routing,litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True,+ billed_retry_attempt=billed_retry_attempt,user_api_key_user_email=_stored_user_email(metadata),)_raw_key: Final = clean_metadata.get("user_api_key")@@ -453,7 +459,12 @@ def _sl_attribution_fallback(def _deployment_provider(deployment: DeploymentTypedDict) -> str | None:- litellm_params: Final = LiteLLM_Params.model_validate(deployment["litellm_params"])+ litellm_params: Final[LiteLLM_Params] = LiteLLM_Params.model_validate(deployment["litellm_params"])+ return deployment_params_provider(litellm_params)+++def deployment_params_provider(litellm_params: LiteLLM_Params) -> str | None:+ """The provider a deployment's ``litellm_params`` authenticate against, None when its model cannot be resolved."""if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default(litellm_params=litellm_params):return LlmProviders.LITELLM_PROXY.valuedeclared: Final = declared_authenticating_provider(litellm_params.model, litellm_params.custom_llm_provider)diff --git a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.jsonindex cfdf707c977..6e5461c3b60 100644--- a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json+++ b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json@@ -11,7 +11,7 @@"user": "","team_id": "","organization_id": "",- "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}",+ "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}","cache_key": "Cache OFF","spend": 0.00022500000000000002,"total_tokens": 30,diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.pyindex ba1b5536a07..bd997f10322 100644--- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py+++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py@@ -16,6 +16,7 @@ from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuthfrom litellm.proxy.collector import SpendEventConsumerfrom litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceededfrom litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter+from litellm.proxy.db.model_usage_rollup import build_model_usage_transactionfrom litellm.proxy.db.spend_log_tool_index import response_tool_call_namesfrom litellm.proxy.hooks.proxy_track_cost_callback import (_get_budget_reservation_from_metadata,@@ -29,6 +30,7 @@ from litellm.proxy.utils import ProxyUpdateSpendfrom litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_eventfrom litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddressfrom litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload+from litellm.types.router import Deployment, LiteLLM_Paramsfrom litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, PromptTokensDetailsWrapper, Usage@@ -154,6 +156,127 @@ async def test_each_earlier_billed_attempt_gets_its_own_spend_row():assert row["team_id"] == "t1"+def _deployment(model_name: str, model_id: str, model: str, api_base: str | None) -> Deployment:+ return Deployment(+ model_name=model_name,+ litellm_params=LiteLLM_Params(model=model, api_base=api_base),+ model_info={"id": model_id},+ )+++_TIMED_OUT_AT: Final = datetime(2026, 10, 20, 21, 24, 48, tzinfo=timezone.utc)+++async def _spend_log_payloads_of_a_retried_timeout_across_a_fallback() -> list[SpendLogsPayload]:+ """The spend-log payloads a timed-out request writes when attempt 0 and 1 ran on the anthropic+ deployment of group ``essay`` and the request then fell back to the openai deployment of group+ ``essay-fallback``: the final row first, then one row per earlier billed attempt."""+ from litellm.proxy.utils import BilledFailedAttempt++ deployments = {+ "dep-anthropic": _deployment("essay", "dep-anthropic", "anthropic/claude-sonnet-4-6", None),+ "dep-openai": _deployment("essay-fallback", "dep-openai", "openai/gpt-4.1", "https://api.openai.com/v1"),+ }+ usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24)+ request_data = {+ "model": "anthropic/claude-sonnet-4-6",+ "custom_llm_provider": "anthropic",+ "litellm_call_id": "call-1",+ "litellm_params": {"api_base": "https://api.anthropic.com/v1/messages"},+ "standard_logging_object": {+ "id": "call-1",+ "model": "anthropic/claude-sonnet-4-6",+ "custom_llm_provider": "anthropic",+ "api_base": "https://api.anthropic.com/v1/messages",+ "model_id": "dep-anthropic",+ "model_group": "essay",+ "metadata": {},+ "hidden_params": {},+ "model_map_information": {},+ },+ "metadata": {"attempted_retries": 1, "model_group": "essay", "model_info": {"id": "dep-anthropic"}},+ "combined_usage_object": usage,+ "response_cost": 7.2e-05,+ "billed_failed_attempts": (+ BilledFailedAttempt(attempted_retries=1, deployment_id="dep-anthropic", usage=usage, response_cost=7.2e-05),+ BilledFailedAttempt(attempted_retries=0, deployment_id="dep-openai", usage=usage, response_cost=4.8e-05),+ ),+ }+ exception = litellm.Timeout("Request timed out.", model="gpt-4.1", llm_provider="openai")+ with patch(+ "litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database",+ new_callable=AsyncMock,+ ) as mock_update_database:+ await ProxyDBLogger(deployment_lookup=deployments.get).async_post_call_failure_hook(+ request_data=request_data,+ original_exception=exception,+ user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key", user_id="u1"),+ )+ return [+ get_logging_payload(+ kwargs=call.kwargs["kwargs"],+ response_obj=exception,+ start_time=_TIMED_OUT_AT,+ end_time=_TIMED_OUT_AT + timedelta(seconds=11),+ )+ for call in mock_update_database.call_args_list+ ]+++async def test_attempt_rows_are_attributed_to_the_deployment_that_billed_them():+ """Regression since #47416: an earlier attempt that ran on another deployment was written with the+ final row's model, provider and api_base. Each attempt row names the deployment it ran on."""+ final_row, attempt_0, attempt_1 = await _spend_log_payloads_of_a_retried_timeout_across_a_fallback()++ assert (final_row["model"], final_row["custom_llm_provider"], final_row["model_id"]) == (+ "anthropic/claude-sonnet-4-6",+ "anthropic",+ "dep-anthropic",+ )+ assert final_row["api_base"] == "https://api.anthropic.com/v1/messages"+ assert (attempt_0["model"], attempt_0["custom_llm_provider"], attempt_0["model_id"], attempt_0["model_group"]) == (+ "anthropic/claude-sonnet-4-6",+ "anthropic",+ "dep-anthropic",+ "essay",+ )+ assert (attempt_1["model"], attempt_1["custom_llm_provider"], attempt_1["model_id"], attempt_1["model_group"]) == (+ "openai/gpt-4.1",+ "openai",+ "dep-openai",+ "essay-fallback",+ )+ assert attempt_1["api_base"] == "https://api.openai.com/v1"+ assert attempt_1["request_id"] == "call-1-attempt-1"+ assert attempt_1["spend"] == 4.8e-05+++async def test_attempt_rows_add_spend_but_not_requests_to_daily_spend():+ """Regression since #47416: one caller request with N billed earlier attempts counted N+1 requests in+ the daily spend tables. The attempt rows keep their spend, tokens and model usage and count zero requests."""+ payloads = await _spend_log_payloads_of_a_retried_timeout_across_a_fallback()+ prisma = MagicMock()+ prisma.get_request_status = MagicMock(return_value="failure")++ daily = [+ await DBSpendUpdateWriter()._common_add_spend_log_transaction_to_daily_transaction(+ payload=payload, prisma_client=prisma, type="user"+ )+ for payload in payloads+ ]++ assert all(row is not None for row in daily)+ assert [row["api_requests"] for row in daily if row is not None] == [1, 0, 0]+ assert [row["failed_requests"] for row in daily if row is not None] == [1, 0, 0]+ assert [row["spend"] for row in daily if row is not None] == [7.2e-05, 7.2e-05, 4.8e-05]+ assert [row["prompt_tokens"] for row in daily if row is not None] == [24, 24, 24]+ model_usage = [build_model_usage_transaction(payload) for payload in payloads]+ assert all(usage is not None for usage in model_usage), "attempt spend must still reach daily model usage"+ assert [usage.spend for usage in model_usage if usage is not None] == [7.2e-05, 7.2e-05, 4.8e-05]++@pytest.mark.asyncioasync def test_async_post_call_failure_hook_carries_guardrail_info_from_litellm_metadata():"""diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.pyindex ed8ae2db8b1..23bbe521670 100644--- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py+++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py@@ -3725,7 +3725,7 @@ class TestSpendLogsPayload:"model": "gpt-4o","user": "","team_id": "",- "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',+ "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',"cache_key": "Cache OFF","spend": 0.00022500000000000002,"total_tokens": 30,