Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

fix(proxy): attribute retry attempt spend rows to their deployment and keep them out of daily request counts (#47658)

Kerry Lu5dce13d2c249
diff --git a/litellm/constants.py b/litellm/constants.py
index 5b99a0f73dd..1c1dfb9aa1c 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -1654,6 +1654,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
+BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 97a2fc31fb0..54acf59ef80 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -3731,6 +3731,7 @@ class SpendLogsMetadata(TypedDict):
routing_decision: StandardLoggingRoutingDecision | None
kubernetes_pod_routing: ReadOnly[StandardLoggingKubernetesPodRouting | None]
internal_call_origin: InternalCallOrigin | None
+ billed_retry_attempt: ReadOnly[NotRequired[bool | None]]
litellm_roi_estimator: ReadOnly[NotRequired[bool | None]]
guardrail_information: list[StandardLoggingGuardrailInformation] | None
eval_information: Any | None
diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py
index bb6ffe675c0..4e98823ba77 100644
--- a/litellm/proxy/db/db_spend_update_writer.py
+++ b/litellm/proxy/db/db_spend_update_writer.py
@@ -29,6 +29,7 @@ from litellm._internal_context import service_target, with_service_target
from litellm._logging import verbose_proxy_logger
from litellm.caching import RedisCache
from litellm.constants import (
+ BILLED_RETRY_ATTEMPT_METADATA_KEY,
DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME,
DB_SPEND_UPDATE_JOB_NAME,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
@@ -312,6 +313,11 @@ def _spend_tables_left_to_send(
return cast(DBSpendUpdateTransactions, remaining) if any(remaining.values()) else None
+def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool:
+ """An earlier billed retry attempt's own spend row, not a request the caller sent."""
+ return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True
+
+
def _timed_request_duration_ms(
payload: dict | SpendLogsPayload,
request_status: Literal["success", "failure"],
@@ -2805,11 +2811,12 @@ class DBSpendUpdateWriter:
if not date:
return
metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
+ internal_origin: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults
request_error_accumulator.record(
payload=payload,
request_status=prisma_client.get_request_status(payload), # pyright: ignore[reportUnknownMemberType] # legacy payload union; this caller supplies a typed spend payload
date=date.split("T")[0],
- is_internal_call=bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)), # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults
+ is_internal_call=internal_origin or _is_billed_retry_attempt(metadata),
)
async def _common_add_spend_log_transaction_to_daily_transaction(
@@ -2874,6 +2881,7 @@ class DBSpendUpdateWriter:
endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None)
is_internal_call: Final = bool(_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY))
+ caller_request: Final = not is_internal_call and not _is_billed_retry_attempt(_metadata)
cache_read_input_tokens: Final = extract_cache_read_tokens(usage_obj)
compression_saved_tokens: Final = extract_compression_saved_tokens(_metadata)
savings_spend: Final = compute_savings_spend(
@@ -2908,9 +2916,9 @@ class DBSpendUpdateWriter:
# requests the caller made: counting them inflates request-volume
# readers, and an auto-router savings figure computed on a shadow
# duplicate credits savings for traffic no user sent.
- api_requests=0 if is_internal_call else 1,
- successful_requests=1 if not is_internal_call and request_status == "success" else 0,
- failed_requests=1 if not is_internal_call and request_status != "success" else 0,
+ api_requests=1 if caller_request else 0,
+ successful_requests=1 if caller_request and request_status == "success" else 0,
+ failed_requests=1 if caller_request and request_status != "success" else 0,
cache_read_input_tokens=cache_read_input_tokens,
cache_creation_input_tokens=extract_cache_creation_tokens(usage_obj),
compression_saved_tokens=compression_saved_tokens,
diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py
index 1f8b352fed9..994310c6619 100644
--- a/litellm/proxy/hooks/proxy_track_cost_callback.py
+++ b/litellm/proxy/hooks/proxy_track_cost_callback.py
@@ -5,14 +5,14 @@ from contextlib import contextmanager
from contextvars import ContextVar
from datetime import datetime, timezone
from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, Protocol, cast
+from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, cast
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.batches.batch_utils import batch_cost_is_final
-from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED
+from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED, BILLED_RETRY_ATTEMPT_METADATA_KEY
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import (
budget_reservation_from_metadata,
@@ -54,11 +54,13 @@ from litellm.proxy.spend_tracking.spend_log_error_logger import (
)
from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 # legacy module exports
_sanitize_error_information_for_spend_logs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
+ deployment_params_provider,
get_request_model_access_groups,
sanitize_error_information_for_spend_logs,
should_store_prompts_and_responses_in_spend_logs,
)
from litellm.proxy.utils import BilledFailedAttempt, ProxyUpdateSpend
+from litellm.types.router import Deployment
from litellm.types.utils import (
CallTypes,
LiteLLMBatch,
@@ -187,6 +189,14 @@ def _proxy_spend_writer() -> DBSpendUpdateWriter:
return proxy_logging_obj.db_spend_update_writer
+DeploymentLookup: TypeAlias = Callable[[str], Deployment | None]
+
+
+def _proxy_deployment(model_id: str) -> Deployment | None:
+ router: Final = get_llm_router()
+ return None if router is None else router.get_deployment(model_id=model_id)
+
+
def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledFailedAttempt, ...]:
"""The earlier billed attempts ``_failure_fields_to_lift`` stashed on the request, if any."""
attempts: Final = request_data.get("billed_failed_attempts")
@@ -197,7 +207,10 @@ def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledF
def _failure_spend_rows(
- request_data: Mapping[str, object], final_cost: float, late_guardrails_snapshot: object
+ request_data: Mapping[str, object],
+ final_cost: float,
+ late_guardrails_snapshot: object,
+ deployment_lookup: DeploymentLookup,
) -> tuple[tuple[float, Mapping[str, object]], ...]:
"""One ``(response_cost, kwargs)`` per spend row a failure writes: the final attempt under the
request id, then every earlier attempt the provider also billed under its own row."""
@@ -209,25 +222,59 @@ def _failure_spend_rows(
return (
(final_cost, final_row),
*(
- (attempt.response_cost, _attempt_spend_row_kwargs(request_data, attempt, position))
+ (
+ attempt.response_cost,
+ _attempt_spend_row_kwargs(
+ request_data, attempt, position, _attempt_deployment(attempt, deployment_lookup)
+ ),
+ )
for position, attempt in enumerate(_billed_failed_attempts(request_data))
),
)
+def _attempt_deployment(attempt: BilledFailedAttempt, deployment_lookup: DeploymentLookup) -> Deployment | None:
+ return None if attempt.deployment_id is None else deployment_lookup(attempt.deployment_id)
+
+
+def _deployment_attribution(deployment: Deployment) -> Mapping[str, object]:
+ """The spend-log attribution fields of the deployment that served an attempt: the model string it
+ was called with, its provider, its configured ``api_base`` and its model group."""
+ return MappingProxyType(
+ {
+ "model": deployment.litellm_params.model,
+ "custom_llm_provider": deployment_params_provider(deployment.litellm_params),
+ "api_base": deployment.litellm_params.api_base,
+ "model_group": deployment.model_name,
+ }
+ )
+
+
+_NO_ATTRIBUTION: Final[Mapping[str, object]] = MappingProxyType({})
+
+
def _attempt_spend_row_kwargs(
- request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int
+ request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int, deployment: Deployment | None
) -> Mapping[str, object]:
"""The final attempt's row kwargs re-keyed for one earlier billed attempt: its own
``request_id`` (``<id>-attempt-<position>``, since ``request_id`` is the spend-log primary key
and the retry count restarts after a fallback hop), its estimated usage and cost, its retry
- count under ``attempted_retries`` and the deployment that billed it under ``model_info.id``."""
+ count under ``attempted_retries``, the deployment that billed it under ``model_info.id`` and,
+ when the router still knows that deployment, its model, provider, ``api_base`` and model group.
+ The row is stamped ``billed_retry_attempt`` so the daily spend tables keep its spend, tokens
+ and model usage without counting it as a request the caller sent."""
suffix: Final = f"-attempt-{position}"
+ attribution: Final = _NO_ATTRIBUTION if deployment is None else _deployment_attribution(deployment)
raw_params: Final = cast("Mapping[str, object]", request_data.get("litellm_params") or {}) # cast-ok: request dict
- litellm_params: Final = dict(raw_params)
+ litellm_params: Final = {
+ **raw_params,
+ **({} if deployment is None else {"api_base": attribution["api_base"]}),
+ }
metadata: Final = {
**cast("Mapping[str, object]", litellm_params.get("metadata") or {}), # cast-ok: request dict
"attempted_retries": attempt.attempted_retries,
+ BILLED_RETRY_ATTEMPT_METADATA_KEY: True,
+ **({} if deployment is None else {"model_group": attribution["model_group"]}),
}
metadata.pop("previous_models", None)
if attempt.deployment_id is not None:
@@ -238,6 +285,7 @@ def _attempt_spend_row_kwargs(
litellm_params["metadata"] = metadata
row: Final = {
**request_data,
+ **({} if deployment is None else {k: attribution[k] for k in ("model", "custom_llm_provider")}),
"litellm_params": litellm_params,
"combined_usage_object": attempt.usage,
"response_cost": attempt.response_cost,
@@ -250,7 +298,11 @@ def _attempt_spend_row_kwargs(
if isinstance(standard_logging_object, dict):
payload: Final = cast("Mapping[str, object]", standard_logging_object) # cast-ok: StandardLoggingPayload dict
if isinstance(payload.get("id"), str):
- row["standard_logging_object"] = {**payload, "id": f"{payload['id']}{suffix}"}
+ row["standard_logging_object"] = {
+ **payload,
+ "id": f"{payload['id']}{suffix}",
+ **({} if deployment is None else {**attribution, "model_id": attempt.deployment_id}),
+ }
return row
@@ -262,10 +314,12 @@ class _ProxyDBLogger(CustomLogger):
turn_off_message_logging: bool = False,
message_logging: bool = True,
spend_writer: Callable[[], DBSpendUpdateWriter] = _proxy_spend_writer,
+ deployment_lookup: DeploymentLookup = _proxy_deployment,
) -> None:
super().__init__(turn_off_message_logging=turn_off_message_logging, message_logging=message_logging)
self.spend_event_producer = spend_event_producer
self._spend_writer: Final = spend_writer
+ self._deployment_lookup: Final = deployment_lookup
async def async_log_success_event(
self, kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime
@@ -439,6 +493,7 @@ class _ProxyDBLogger(CustomLogger):
cast("Mapping[str, object]", request_data), # cast-ok: request dict, copied per row below
recovered_response_cost,
failure_snapshot_with_late_guardrails,
+ self._deployment_lookup,
),
)
diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py
index 3ac270fcf2c..09f53cecb72 100644
--- a/litellm/proxy/litellm_pre_call_utils.py
+++ b/litellm/proxy/litellm_pre_call_utils.py
@@ -29,6 +29,7 @@ from litellm._service_logger import ServiceLogging
from litellm._uuid import uuid
from litellm.constants import (
BAGGAGE_MAPPED_TAGS_METADATA_KEY,
+ BILLED_RETRY_ATTEMPT_METADATA_KEY,
CALLER_SESSION_ID_METADATA_KEY,
CLIENT_OUTPUT_CEILING_METADATA_KEY,
CONSUMED_REQUEST_TAGS_METADATA_KEY,
@@ -426,6 +427,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = (
CONSUMED_REQUEST_TAGS_METADATA_KEY,
ROUTING_REQUEST_TAGS_METADATA_KEY,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
+ BILLED_RETRY_ATTEMPT_METADATA_KEY,
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
"standard_logging_object",
"litellm_roi_estimator",
diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py
index 091fb5d1305..62aee788ed0 100644
--- a/litellm/proxy/spend_tracking/spend_tracking_utils.py
+++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py
@@ -15,6 +15,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import (
+ BILLED_RETRY_ATTEMPT_METADATA_KEY,
CALLER_SESSION_ID_METADATA_KEY,
CLI_SESSION_KEY_PREFIX,
EMPTY_MAPPING,
@@ -187,6 +188,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset(
"used_client_oauth_token",
KUBERNETES_POD_ROUTING_KEY,
"litellm_roi_estimator",
+ BILLED_RETRY_ATTEMPT_METADATA_KEY,
"user_api_key_user_email",
)
)
@@ -249,6 +251,7 @@ def _get_spend_logs_metadata(
guardrail_information=None,
internal_call_origin=None,
litellm_roi_estimator=False,
+ billed_retry_attempt=False,
eval_information=None,
cold_storage_object_key=cold_storage_object_key,
litellm_overhead_time_ms=None,
@@ -270,6 +273,8 @@ def _get_spend_logs_metadata(
)
verbose_proxy_logger.debug("getting payload for SpendLogs, available keys in metadata: %s", [*metadata])
+ retry_key: Final = BILLED_RETRY_ATTEMPT_METADATA_KEY
+ billed_retry_attempt: Final = retry_key in metadata and metadata[retry_key] is True
# Filter the metadata dictionary to include only the specified keys
clean_metadata: Final = SpendLogsMetadata(
**MappingProxyType(
@@ -283,6 +288,7 @@ def _get_spend_logs_metadata(
used_client_oauth_token=used_client_oauth_token,
kubernetes_pod_routing=kubernetes_pod_routing,
litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True,
+ billed_retry_attempt=billed_retry_attempt,
user_api_key_user_email=_stored_user_email(metadata),
)
_raw_key: Final = clean_metadata.get("user_api_key")
@@ -453,7 +459,12 @@ def _sl_attribution_fallback(
def _deployment_provider(deployment: DeploymentTypedDict) -> str | None:
- litellm_params: Final = LiteLLM_Params.model_validate(deployment["litellm_params"])
+ litellm_params: Final[LiteLLM_Params] = LiteLLM_Params.model_validate(deployment["litellm_params"])
+ return deployment_params_provider(litellm_params)
+
+
+def deployment_params_provider(litellm_params: LiteLLM_Params) -> str | None:
+ """The provider a deployment's ``litellm_params`` authenticate against, None when its model cannot be resolved."""
if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default(litellm_params=litellm_params):
return LlmProviders.LITELLM_PROXY.value
declared: Final = declared_authenticating_provider(litellm_params.model, litellm_params.custom_llm_provider)
diff --git a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json
index cfdf707c977..6e5461c3b60 100644
--- a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json
+++ b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json
@@ -11,7 +11,7 @@
"user": "",
"team_id": "",
"organization_id": "",
- "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}",
+ "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}",
"cache_key": "Cache OFF",
"spend": 0.00022500000000000002,
"total_tokens": 30,
diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py
index ba1b5536a07..bd997f10322 100644
--- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py
+++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py
@@ -16,6 +16,7 @@ from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth
from litellm.proxy.collector import SpendEventConsumer
from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
+from litellm.proxy.db.model_usage_rollup import build_model_usage_transaction
from litellm.proxy.db.spend_log_tool_index import response_tool_call_names
from litellm.proxy.hooks.proxy_track_cost_callback import (
_get_budget_reservation_from_metadata,
@@ -29,6 +30,7 @@ from litellm.proxy.utils import ProxyUpdateSpend
from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event
from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
+from litellm.types.router import Deployment, LiteLLM_Params
from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, PromptTokensDetailsWrapper, Usage
@@ -154,6 +156,127 @@ async def test_each_earlier_billed_attempt_gets_its_own_spend_row():
assert row["team_id"] == "t1"
+def _deployment(model_name: str, model_id: str, model: str, api_base: str | None) -> Deployment:
+ return Deployment(
+ model_name=model_name,
+ litellm_params=LiteLLM_Params(model=model, api_base=api_base),
+ model_info={"id": model_id},
+ )
+
+
+_TIMED_OUT_AT: Final = datetime(2026, 10, 20, 21, 24, 48, tzinfo=timezone.utc)
+
+
+async def _spend_log_payloads_of_a_retried_timeout_across_a_fallback() -> list[SpendLogsPayload]:
+ """The spend-log payloads a timed-out request writes when attempt 0 and 1 ran on the anthropic
+ deployment of group ``essay`` and the request then fell back to the openai deployment of group
+ ``essay-fallback``: the final row first, then one row per earlier billed attempt."""
+ from litellm.proxy.utils import BilledFailedAttempt
+
+ deployments = {
+ "dep-anthropic": _deployment("essay", "dep-anthropic", "anthropic/claude-sonnet-4-6", None),
+ "dep-openai": _deployment("essay-fallback", "dep-openai", "openai/gpt-4.1", "https://api.openai.com/v1"),
+ }
+ usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24)
+ request_data = {
+ "model": "anthropic/claude-sonnet-4-6",
+ "custom_llm_provider": "anthropic",
+ "litellm_call_id": "call-1",
+ "litellm_params": {"api_base": "https://api.anthropic.com/v1/messages"},
+ "standard_logging_object": {
+ "id": "call-1",
+ "model": "anthropic/claude-sonnet-4-6",
+ "custom_llm_provider": "anthropic",
+ "api_base": "https://api.anthropic.com/v1/messages",
+ "model_id": "dep-anthropic",
+ "model_group": "essay",
+ "metadata": {},
+ "hidden_params": {},
+ "model_map_information": {},
+ },
+ "metadata": {"attempted_retries": 1, "model_group": "essay", "model_info": {"id": "dep-anthropic"}},
+ "combined_usage_object": usage,
+ "response_cost": 7.2e-05,
+ "billed_failed_attempts": (
+ BilledFailedAttempt(attempted_retries=1, deployment_id="dep-anthropic", usage=usage, response_cost=7.2e-05),
+ BilledFailedAttempt(attempted_retries=0, deployment_id="dep-openai", usage=usage, response_cost=4.8e-05),
+ ),
+ }
+ exception = litellm.Timeout("Request timed out.", model="gpt-4.1", llm_provider="openai")
+ with patch(
+ "litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database",
+ new_callable=AsyncMock,
+ ) as mock_update_database:
+ await ProxyDBLogger(deployment_lookup=deployments.get).async_post_call_failure_hook(
+ request_data=request_data,
+ original_exception=exception,
+ user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key", user_id="u1"),
+ )
+ return [
+ get_logging_payload(
+ kwargs=call.kwargs["kwargs"],
+ response_obj=exception,
+ start_time=_TIMED_OUT_AT,
+ end_time=_TIMED_OUT_AT + timedelta(seconds=11),
+ )
+ for call in mock_update_database.call_args_list
+ ]
+
+
+async def test_attempt_rows_are_attributed_to_the_deployment_that_billed_them():
+ """Regression since #47416: an earlier attempt that ran on another deployment was written with the
+ final row's model, provider and api_base. Each attempt row names the deployment it ran on."""
+ final_row, attempt_0, attempt_1 = await _spend_log_payloads_of_a_retried_timeout_across_a_fallback()
+
+ assert (final_row["model"], final_row["custom_llm_provider"], final_row["model_id"]) == (
+ "anthropic/claude-sonnet-4-6",
+ "anthropic",
+ "dep-anthropic",
+ )
+ assert final_row["api_base"] == "https://api.anthropic.com/v1/messages"
+ assert (attempt_0["model"], attempt_0["custom_llm_provider"], attempt_0["model_id"], attempt_0["model_group"]) == (
+ "anthropic/claude-sonnet-4-6",
+ "anthropic",
+ "dep-anthropic",
+ "essay",
+ )
+ assert (attempt_1["model"], attempt_1["custom_llm_provider"], attempt_1["model_id"], attempt_1["model_group"]) == (
+ "openai/gpt-4.1",
+ "openai",
+ "dep-openai",
+ "essay-fallback",
+ )
+ assert attempt_1["api_base"] == "https://api.openai.com/v1"
+ assert attempt_1["request_id"] == "call-1-attempt-1"
+ assert attempt_1["spend"] == 4.8e-05
+
+
+async def test_attempt_rows_add_spend_but_not_requests_to_daily_spend():
+ """Regression since #47416: one caller request with N billed earlier attempts counted N+1 requests in
+ the daily spend tables. The attempt rows keep their spend, tokens and model usage and count zero requests."""
+ payloads = await _spend_log_payloads_of_a_retried_timeout_across_a_fallback()
+ prisma = MagicMock()
+ prisma.get_request_status = MagicMock(return_value="failure")
+
+ daily = [
+ await DBSpendUpdateWriter()._common_add_spend_log_transaction_to_daily_transaction(
+ payload=payload, prisma_client=prisma, type="user"
+ )
+ for payload in payloads
+ ]
+
+ assert all(row is not None for row in daily)
+ assert [row["api_requests"] for row in daily if row is not None] == [1, 0, 0]
+ assert [row["failed_requests"] for row in daily if row is not None] == [1, 0, 0]
+ assert [row["spend"] for row in daily if row is not None] == [7.2e-05, 7.2e-05, 4.8e-05]
+ assert [row["prompt_tokens"] for row in daily if row is not None] == [24, 24, 24]
+ model_usage = [build_model_usage_transaction(payload) for payload in payloads]
+ assert all(usage is not None for usage in model_usage), "attempt spend must still reach daily model usage"
+ assert [usage.spend for usage in model_usage if usage is not None] == [7.2e-05, 7.2e-05, 4.8e-05]
+
+
@pytest.mark.asyncio
async def test_async_post_call_failure_hook_carries_guardrail_info_from_litellm_metadata():
"""
diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py
index ed8ae2db8b1..23bbe521670 100644
--- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py
+++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py
@@ -3725,7 +3725,7 @@ class TestSpendLogsPayload:
"model": "gpt-4o",
"user": "",
"team_id": "",
- "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
+ "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"cache_key": "Cache OFF",
"spend": 0.00022500000000000002,
"total_tokens": 30,