The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import asyncio
import datetime
import json
from collections.abc import Callable, Mapping
from datetime import timezone
from types import MappingProxyType
from typing import Any, Final, cast
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from pydantic import TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
import litellm.constants as litellm_constants
import litellm.proxy.spend_tracking.spend_tracking_utils as spend_tracking_utils
from litellm.constants import (
KUBERNETES_POD_ROUTING_KEY,
LITELLM_TRUNCATED_PAYLOAD_FIELD,
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
LITTELM_CLI_SERVICE_ACCOUNT_NAME,
LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME,
MAX_SPEND_LOG_MODEL_NAME_LENGTH,
REDACTED_BY_LITELM_STRING,
SESSION_ID_GENERATED_METADATA_KEY,
SESSION_ID_OMITTED_METADATA_KEY,
UNKNOWN_MODEL_SPEND_LOG_MODEL,
)
from litellm.constants import (
CALLER_SESSION_ID_METADATA_KEY,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
)
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy._types import SpendLogsMetadata, SpendLogsMetadataFields, SpendLogsPayload, UserAPIKeyAuth
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
from litellm.proxy.spend_tracking.spend_tracking_utils import (
_extract_usage_for_ocr_call,
_get_messages_for_spend_logs_payload,
_get_proxy_server_request_for_spend_logs_payload,
_get_request_duration_ms,
_get_response_for_spend_logs_payload,
_get_session_id_for_spend_log,
_get_spend_logs_metadata,
_get_vector_store_request_for_spend_logs_payload,
is_master_key,
_redact_logged_api_key,
_redact_prompt_leaks_in_error_string,
sanitize_error_information_for_spend_logs,
_sanitize_guardrail_information_for_spend_logs,
_sanitize_request_body_for_spend_logs_payload,
_scrub_raw_model_from_error_information,
configured_spend_logs_metadata_fields,
get_logging_payload,
get_spend_logs_id,
_stored_request_without_requester_ip_address,
requester_ip_address_logging_disabled,
should_store_prompts_and_responses_in_spend_logs,
spend_log_row_with_retained_metadata,
)
from litellm.proxy.utils import hash_token
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
BackgroundResponseSettlement,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
StandardLoggingHiddenParams,
StandardLoggingMetadata,
StandardLoggingModelInformation,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
def _get_additional_usage_values_for_usage(usage: litellm.Usage) -> dict:
payload = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
response_obj=litellm.ModelResponse(
id="chatcmpl-test",
choices=[],
usage=usage,
),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
return metadata["additional_usage_values"]
@pytest.mark.parametrize(
("metadata", "litellm_metadata", "expected"),
[
(
{
"kubernetes_pod_routing": {
"service_host": "vllm-headless.ns.svc.cluster.local",
"pod_ip": "10.0.0.1",
"pod_count": 3,
"selection": "round_robin",
}
},
{},
{
"service_host": "vllm-headless.ns.svc.cluster.local",
"pod_ip": "10.0.0.1",
"pod_count": 3,
"selection": "round_robin",
},
),
(
{},
{
"kubernetes_pod_routing": {
"service_host": "vllm-headless.ns.svc.cluster.local",
"pod_ip": "10.0.0.2",
"pod_count": 3,
"selection": "session_affinity",
}
},
{
"service_host": "vllm-headless.ns.svc.cluster.local",
"pod_ip": "10.0.0.2",
"pod_count": 3,
"selection": "session_affinity",
},
),
(
{
"kubernetes_pod_routing": {
"service_host": "vllm-headless.ns.svc.cluster.local",
"pod_ip": "10.0.0.1",
"pod_count": 3,
"selection": "round_robin",
}
},
{
"kubernetes_pod_routing": {
"service_host": "vllm-headless.ns.svc.cluster.local",
"pod_ip": "10.0.0.3",
"pod_count": 3,
"selection": "session_affinity_retry",
}
},
{
"service_host": "vllm-headless.ns.svc.cluster.local",
"pod_ip": "10.0.0.3",
"pod_count": 3,
"selection": "session_affinity_retry",
},
),
],
)
def test_get_logging_payload_includes_kubernetes_pod_routing(
metadata: dict[str, object],
litellm_metadata: dict[str, object],
expected: dict[str, object],
) -> None:
now: Final = datetime.datetime(2026, 1, 1, tzinfo=timezone.utc)
payload: Final = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_params": {"metadata": metadata, "litellm_metadata": litellm_metadata},
},
response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]),
start_time=now,
end_time=now,
)
assert json.loads(payload["metadata"])[KUBERNETES_POD_ROUTING_KEY] == expected
assert _get_spend_logs_metadata(None)["kubernetes_pod_routing"] is None
@pytest.mark.parametrize("store_prompts,redact", [(True, False), (False, False), (True, True)])
def test_classifier_audit_spend_storage_obeys_privacy_and_truncation(monkeypatch, store_prompts, redact):
from litellm.proxy import proxy_server
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": store_prompts})
audit: Final = {
"classifier_input": {"system": "rubric" * 1000, "messages": [{"role": "user", "content": "ask"}]},
"originating_request_masked": {"input": "source-only", "api_key": "REDACTED"},
}
stored: Final = json.loads(
_get_proxy_server_request_for_spend_logs_payload(
metadata={},
litellm_params={"proxy_server_request": {"body": {"model": "classifier"}}},
kwargs={
"standard_logging_object": audit,
"standard_callback_dynamic_params": {"turn_off_message_logging": redact},
},
)
)
if not store_prompts or redact:
assert "classifier_input" not in stored
assert "originating_request_masked" not in stored
else:
assert stored["classifier_input"]["messages"] == audit["classifier_input"]["messages"]
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in json.dumps(stored["classifier_input"])
assert stored["originating_request_masked"]["input"] == "source-only"
assert stored["model"] == "classifier"
assert audit["classifier_input"]["system"] == "rubric" * 1000
def test_get_logging_payload_maps_openai_cached_tokens_to_cache_read_input_tokens():
additional_usage_values = _get_additional_usage_values_for_usage(
litellm.Usage(
prompt_tokens=10,
completion_tokens=2,
total_tokens=12,
prompt_tokens_details={"cached_tokens": 123},
)
)
assert additional_usage_values["cache_read_input_tokens"] == 123
assert additional_usage_values["prompt_tokens_details"]["cached_tokens"] == 123
class _HashingCache(litellm.Cache):
def __init__(self) -> None:
pass
def get_cache_key(self, **kwargs) -> str:
raise AssertionError("a preset cache key must be reused instead of hashing the request")
def _cache_key_in_spend_log(monkeypatch: pytest.MonkeyPatch, cache: litellm.Cache | None, preset: str | None) -> str:
monkeypatch.setattr(litellm, "cache", cache)
payload: Final = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"messages": [{"role": "user", "content": "x" * 10_000}],
"litellm_params": {"metadata": {"user_api_key": "test-key"}, "preset_cache_key": preset},
},
response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
return payload["cache_key"]
def test_get_logging_payload_reuses_the_preset_cache_key_instead_of_hashing_the_body(monkeypatch):
assert _cache_key_in_spend_log(monkeypatch, _HashingCache(), "preset-key") == "preset-key"
def test_get_logging_payload_records_cache_off_without_hashing(monkeypatch):
assert _cache_key_in_spend_log(monkeypatch, None, None) == "Cache OFF"
def test_get_logging_payload_still_hashes_when_caching_is_on_and_no_preset_key_exists(monkeypatch):
class _RecordingCache(litellm.Cache):
def __init__(self) -> None:
pass
def get_cache_key(self, **kwargs) -> str:
return "hashed-from-" + kwargs["model"]
assert _cache_key_in_spend_log(monkeypatch, _RecordingCache(), None) == "hashed-from-gpt-4o-mini"
_TRACE_ONLY_STANDARD_LOGGING: Final = cast(
StandardLoggingPayload,
{
"trace_id": "trace-abc",
"session_id": "trace-abc",
"metadata": {},
"model_map_information": None,
"request_tags": [],
},
)
def _trace_only_session_id(omit_when_missing: bool) -> str | None:
"""get_litellm_params copies metadata.trace_id into litellm_session_id, so every field echoes the trace id."""
return _get_session_id_for_spend_log(
kwargs={"litellm_trace_id": "trace-abc", "litellm_session_id": "trace-abc"},
metadata={"trace_id": "trace-abc"},
standard_logging_payload=_TRACE_ONLY_STANDARD_LOGGING,
omit_when_missing=omit_when_missing,
)
def test_omit_leaves_session_id_none_when_only_a_trace_id_exists():
assert _trace_only_session_id(omit_when_missing=True) is None
def test_omit_leaves_session_id_none_without_any_ids():
assert (
_get_session_id_for_spend_log(kwargs={}, metadata=None, standard_logging_payload=None, omit_when_missing=True)
is None
)
def test_omit_records_metadata_session_id():
session_id: Final = _get_session_id_for_spend_log(
kwargs={"litellm_session_id": "chain-1"},
metadata={"trace_id": "chain-1", "session_id": "chain-1"},
standard_logging_payload=_TRACE_ONLY_STANDARD_LOGGING,
omit_when_missing=True,
)
assert session_id == "chain-1"
def test_legacy_policy_keeps_trace_id_fallback():
assert _trace_only_session_id(omit_when_missing=False) == "trace-abc"
generated: Final = _get_session_id_for_spend_log(
kwargs={}, metadata=None, standard_logging_payload=None, omit_when_missing=False
)
assert len(str(generated)) == 36
def test_batch_lifecycle_rows_derive_the_same_session_from_the_batch_id():
"""The create call's request id IS the batch id and the poller's cost row appends
_batch_cost to it, so deriving the session from the request id lands both rows in one
trace on the logs UI even though the poller builds a fresh logging context per cycle."""
from litellm.proxy.spend_tracking.spend_tracking_utils import _get_batch_trace_session_id
create_session: Final = _get_batch_trace_session_id(call_type="acreate_batch", request_id="batch-uid-1")
cost_session: Final = _get_batch_trace_session_id(call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost")
assert create_session == cost_session == "batch-uid-1"
def test_non_batch_call_types_derive_no_batch_session():
from litellm.proxy.spend_tracking.spend_tracking_utils import _get_batch_trace_session_id
assert _get_batch_trace_session_id(call_type="acompletion", request_id="chatcmpl-1") is None
def test_batch_session_outranks_the_per_request_trace_id():
"""Each batch lifecycle call carries its own auto-generated trace id, so letting the
trace id win would scatter the rows across sessions again."""
session_id: Final = _get_session_id_for_spend_log(
kwargs={"litellm_trace_id": "trace-abc"},
metadata={"trace_id": "trace-abc"},
standard_logging_payload=_TRACE_ONLY_STANDARD_LOGGING,
omit_when_missing=False,
batch_trace_session_id="batch-uid-1",
)
assert session_id == "batch-uid-1"
def _chat_session_id(
monkeypatch: pytest.MonkeyPatch,
litellm_params: Mapping[str, object],
correlation_in_logs: bool = True,
batch_trace_session_id: str | None = None,
metadata: Mapping[str, object] = MappingProxyType({}),
) -> str | None:
monkeypatch.setattr(litellm, "request_correlation_in_logs", correlation_in_logs)
return _get_session_id_for_spend_log(
kwargs={"litellm_trace_id": "trace-abc", "litellm_params": litellm_params},
metadata=metadata,
standard_logging_payload=_TRACE_ONLY_STANDARD_LOGGING,
omit_when_missing=False,
batch_trace_session_id=batch_trace_session_id,
)
_CALLER_SESSION_PARAMS: Final = MappingProxyType({"litellm_session_id": "sess-1", "metadata": {"trace_id": "otel-1"}})
def test_caller_session_wins_over_the_per_request_trace_id(monkeypatch: pytest.MonkeyPatch):
assert _chat_session_id(monkeypatch, _CALLER_SESSION_PARAMS) == "sess-1"
def test_batch_session_outranks_a_caller_session(monkeypatch: pytest.MonkeyPatch):
assert _chat_session_id(monkeypatch, _CALLER_SESSION_PARAMS, batch_trace_session_id="batch-uid-1") == "batch-uid-1"
def test_caller_session_keeps_the_legacy_trace_id_with_correlation_logging_off(monkeypatch: pytest.MonkeyPatch):
assert _chat_session_id(monkeypatch, _CALLER_SESSION_PARAMS, correlation_in_logs=False) == "trace-abc"
@pytest.mark.parametrize(("correlation_in_logs", "expected_session_id"), [(True, "convo-1"), (False, "trace-abc")])
def test_session_displaced_by_a_trace_header_wins_only_with_correlation_logging(
monkeypatch: pytest.MonkeyPatch, correlation_in_logs: bool, expected_session_id: str
):
header_chain: Final = {
"litellm_session_id": "trace-abc",
"metadata": {"trace_id": "trace-abc", "session_id": "trace-abc"},
}
session_id: Final = _chat_session_id(
monkeypatch,
header_chain,
correlation_in_logs=correlation_in_logs,
metadata={CALLER_SESSION_ID_METADATA_KEY: "convo-1"},
)
assert session_id == expected_session_id
@pytest.mark.parametrize(
("litellm_params", "metadata"),
[
pytest.param({"litellm_session_id": "convo\x00x"}, {}, id="body-session"),
pytest.param(
{"litellm_session_id": "trace-abc"},
{CALLER_SESSION_ID_METADATA_KEY: "convo\x00x"},
id="beside-trace-header",
),
],
)
def test_caller_session_postgres_cannot_store_keeps_the_trace_id(
monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object], metadata: Mapping[str, object]
):
assert _chat_session_id(monkeypatch, litellm_params, metadata=metadata) == "trace-abc"
@pytest.mark.parametrize(
"litellm_params",
[
pytest.param({"metadata": {}}, id="no-session"),
pytest.param(
{
"litellm_session_id": "minted-1",
"metadata": {"session_id": "minted-1", SESSION_ID_GENERATED_METADATA_KEY: True},
},
id="minted-by-generate-policy",
),
pytest.param({"litellm_session_id": "job-7", "metadata": {"trace_id": "job-7"}}, id="copied-from-trace-id"),
],
)
def test_session_that_is_not_a_caller_conversation_keeps_the_trace_id(
monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object]
):
assert _chat_session_id(monkeypatch, litellm_params) == "trace-abc"
def test_omit_policy_still_suppresses_batch_sessions():
session_id: Final = _get_session_id_for_spend_log(
kwargs={},
metadata=None,
standard_logging_payload=None,
omit_when_missing=True,
batch_trace_session_id="batch-uid-1",
)
assert session_id is None
def test_get_logging_payload_groups_batch_create_and_cost_rows_in_one_session():
def _payload(call_type: str) -> SpendLogsPayload:
return get_logging_payload(
kwargs={
"call_type": call_type,
"model": "gpt-4o-mini",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
response_obj=litellm.ModelResponse(id="batch-uid-1", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
create_payload: Final = _payload("acreate_batch")
cost_payload: Final = _payload("aretrieve_batch")
assert cost_payload["request_id"] == "batch-uid-1_batch_cost"
assert create_payload["session_id"] == cost_payload["session_id"] == "batch-uid-1"
@pytest.mark.parametrize(
("request_metadata", "expected"),
[
({"trace_id": "trace-abc"}, "trace-abc"),
({"trace_id": "trace-abc", SESSION_ID_OMITTED_METADATA_KEY: True}, None),
({"trace_id": "trace-abc", "session_id": "chain-1", SESSION_ID_OMITTED_METADATA_KEY: True}, "chain-1"),
],
)
def test_get_logging_payload_reads_omit_decision_stamped_on_request(
request_metadata: dict[str, object], expected: str | None
):
"""The pre-call stamp, not the live general_settings, decides the policy, so a config reload between
pre-call and spend logging cannot fabricate a session for a request accepted under `omit`."""
with patch( # test-quality-ok: proves log time ignores proxy config; general_settings is yaml, not an HTTP boundary
"litellm.proxy.proxy_server.general_settings", {"missing_session_id": "generate"}
):
payload: SpendLogsPayload = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_trace_id": "trace-abc",
"litellm_params": {"litellm_session_id": "trace-abc", "metadata": request_metadata},
"standard_logging_object": _TRACE_ONLY_STANDARD_LOGGING,
},
response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["session_id"] == expected
@pytest.mark.parametrize("policy", ["omit", "generate", None])
def test_get_logging_payload_applies_omit_to_requests_that_carry_no_stamp(policy: str | None):
"""Router-model passthrough calls `allm_passthrough_route` directly and never reaches the pre-call helper that
stamps the omit decision, so an unstamped request falls back to the configured policy. Without that fallback
`missing_session_id: omit` would fabricate a uuid session id on every passthrough spend log while its Langfuse
trace has none, which is the divergence the policy exists to remove."""
with patch( # test-quality-ok: general_settings is proxy config, loaded from yaml, not an HTTP boundary
"litellm.proxy.proxy_server.general_settings", {} if policy is None else {"missing_session_id": policy}
):
payload: SpendLogsPayload = get_logging_payload(
kwargs={
"model": "claude-opus-4",
"litellm_trace_id": "trace-abc",
"litellm_params": {"litellm_session_id": "trace-abc", "metadata": {"trace_id": "trace-abc"}},
"standard_logging_object": _TRACE_ONLY_STANDARD_LOGGING,
},
response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["session_id"] == (None if policy == "omit" else "trace-abc")
def test_get_logging_payload_preserves_anthropic_cache_read_input_tokens():
additional_usage_values = _get_additional_usage_values_for_usage(
litellm.Usage(
prompt_tokens=10,
completion_tokens=2,
total_tokens=12,
prompt_tokens_details={"cached_tokens": 123},
cache_read_input_tokens=456,
)
)
assert additional_usage_values["cache_read_input_tokens"] == 456
@pytest.mark.parametrize(
"prompt_tokens_details",
[None, {"cached_tokens": 0}],
)
def test_get_logging_payload_does_not_map_missing_or_zero_cached_tokens(prompt_tokens_details):
additional_usage_values = _get_additional_usage_values_for_usage(
litellm.Usage(
prompt_tokens=10,
completion_tokens=2,
total_tokens=12,
prompt_tokens_details=prompt_tokens_details,
)
)
assert "cache_read_input_tokens" not in additional_usage_values
def test_get_logging_payload_maps_openai_cache_write_tokens_to_cache_creation_input_tokens():
additional_usage_values = _get_additional_usage_values_for_usage(
litellm.Usage(
prompt_tokens=1000,
completion_tokens=2,
total_tokens=1002,
prompt_tokens_details={"cached_tokens": 0, "cache_write_tokens": 800},
)
)
assert additional_usage_values["cache_creation_input_tokens"] == 800
assert additional_usage_values["prompt_tokens_details"]["cache_write_tokens"] == 800
def test_get_logging_payload_maps_nested_cache_creation_input_tokens():
"""
Regression (LIT-5757): DashScope nests cache_creation_input_tokens inside
prompt_tokens_details; SpendLogs must record it as cache_creation_input_tokens.
"""
additional_usage_values: Final = _get_additional_usage_values_for_usage(
litellm.Usage(
prompt_tokens=2059,
completion_tokens=31,
total_tokens=2090,
prompt_tokens_details={
"cached_tokens": 0,
"text_tokens": 2059,
"cache_type": "ephemeral",
"cache_creation_input_tokens": 2048,
"cache_creation": {"ephemeral_5m_input_tokens": 2048},
},
)
)
assert additional_usage_values["cache_creation_input_tokens"] == 2048
assert additional_usage_values["prompt_tokens_details"]["cache_write_tokens"] == 2048
assert additional_usage_values["prompt_tokens_details"]["cache_type"] == "ephemeral"
def test_get_logging_payload_preserves_anthropic_cache_creation_input_tokens():
additional_usage_values = _get_additional_usage_values_for_usage(
litellm.Usage(
prompt_tokens=1000,
completion_tokens=2,
total_tokens=1002,
cache_creation_input_tokens=300,
)
)
assert additional_usage_values["cache_creation_input_tokens"] == 300
@pytest.mark.parametrize(
"prompt_tokens_details",
[None, {"cached_tokens": 100}, {"cached_tokens": 100, "cache_write_tokens": 0}],
)
def test_get_logging_payload_does_not_map_missing_or_zero_cache_write_tokens(prompt_tokens_details):
additional_usage_values = _get_additional_usage_values_for_usage(
litellm.Usage(
prompt_tokens=10,
completion_tokens=2,
total_tokens=12,
prompt_tokens_details=prompt_tokens_details,
)
)
assert "cache_creation_input_tokens" not in additional_usage_values
def _make_standard_logging_payload_with_usage_object(usage_object: dict) -> StandardLoggingPayload:
return StandardLoggingPayload(
id="test-id-responses",
call_type="responses",
stream=False,
response_cost=0.02,
status="success",
total_tokens=1010,
prompt_tokens=1000,
completion_tokens=10,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=None,
model_map_information=StandardLoggingModelInformation(model_map_key="gpt-5.6", model_map_value=None),
model="gpt-5.6",
model_id="model-123",
model_group="openai",
custom_llm_provider="openai",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_alias=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_user_id=None,
user_api_key_user_email=None,
user_api_key_team_alias=None,
spend_logs_metadata=None,
requester_ip_address=None,
requester_metadata=None,
user_api_key_end_user_id=None,
usage_object=usage_object,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address=None,
messages=[],
response={},
error_str=None,
model_parameters={},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.02",
litellm_overhead_time_ms=None,
additional_headers=None,
batch_models=None,
litellm_model_name=None,
usage_object=None,
),
)
def test_get_logging_payload_maps_responses_api_cache_write_tokens_from_usage_object():
"""Responses API (/v1/responses) usage is not chat-Usage-shaped, so
additional_usage_values can't derive cache tokens from response_obj.usage.
The Admin UI Logs "Cache Creation Tokens" row reads
additional_usage_values.cache_creation_input_tokens, so it must be filled
from the normalized standard_logging usage_object (LIT-4633)."""
standard_logging_payload = _make_standard_logging_payload_with_usage_object(
usage_object={
"prompt_tokens": 1000,
"completion_tokens": 10,
"total_tokens": 1010,
"prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 800, "cache_creation_tokens": 800},
}
)
payload = get_logging_payload(
kwargs={
"model": "gpt-5.6",
"call_type": "responses",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
"standard_logging_object": standard_logging_payload,
},
response_obj={
"id": "resp-test",
"usage": {
"input_tokens": 1000,
"output_tokens": 10,
"total_tokens": 1010,
"input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 800},
},
},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
additional_usage_values = json.loads(payload["metadata"])["additional_usage_values"]
assert additional_usage_values["cache_creation_input_tokens"] == 800
def test_sanitize_request_body_for_spend_logs_payload_basic():
request_body = {
"messages": [{"role": "user", "content": "Hello, how are you?"}],
}
assert _sanitize_request_body_for_spend_logs_payload(request_body) == request_body
def test_large_request_no_truncation_threshold():
"""
Test that MAX_STRING_LENGTH_PROMPT_IN_DB constant is used for request body sanitization
and that the new truncation logic keeps beginning (35%) and end (65%) of the string
"""
from litellm.constants import (
MAX_STRING_LENGTH_PROMPT_IN_DB,
LITELLM_TRUNCATED_PAYLOAD_FIELD,
)
# Create a large string that exceeds the threshold
# Use a pattern that allows us to verify beginning and end are preserved
start_pattern = "START" * 250 # 1250 chars
middle_pattern = "MIDDLE" * 200 # 1200 chars
end_pattern = "END" * 250 # 750 chars
large_content = start_pattern + middle_pattern + end_pattern
request_body = {
"messages": [{"role": "user", "content": large_content}],
"model": "gpt-5.5",
}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
# Verify the content was truncated
truncated_content = sanitized["messages"][0]["content"]
# Calculate expected character counts (35% start, 65% end)
expected_start_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * 0.35)
expected_end_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * 0.65)
# Should keep first 35% of MAX_STRING_LENGTH_PROMPT_IN_DB chars
assert truncated_content.startswith(large_content[:expected_start_chars])
# Should keep last 65% of MAX_STRING_LENGTH_PROMPT_IN_DB chars
assert truncated_content.endswith(large_content[-expected_end_chars:])
# Should have truncation marker
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in truncated_content
assert "skipped" in truncated_content
@pytest.mark.parametrize(
("max_string_length_prompt_in_db", "stored_tail"),
[(0, "claude-sonnet-5-5"), (1, "claude-sonnet-5-5"), (-1, "laude-sonnet-5-5"), (-2, "aude-sonnet-5-5")],
)
def test_a_limit_under_two_keeps_mains_tail_after_the_marker(max_string_length_prompt_in_db: int, stored_tail: str):
sanitized: Final = _sanitize_request_body_for_spend_logs_payload(
{"model": "claude-sonnet-5-5"}, max_string_length_prompt_in_db=max_string_length_prompt_in_db
)
assert sanitized["model"].startswith("...")
assert sanitized["model"].endswith(f") ...{stored_tail}")
def test_small_request_no_truncation():
"""
Test that small strings are not truncated by MAX_STRING_LENGTH_PROMPT_IN_DB
"""
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
# Create a small string that's under the threshold
small_content = "x" * (MAX_STRING_LENGTH_PROMPT_IN_DB - 100)
request_body = {
"messages": [{"role": "user", "content": small_content}],
"model": "gpt-5.5",
}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
# Verify the content was NOT truncated
assert sanitized["messages"][0]["content"] == small_content
assert len(sanitized["messages"][0]["content"]) == MAX_STRING_LENGTH_PROMPT_IN_DB - 100
def test_configurable_string_length_env_var(monkeypatch):
"""
Test that MAX_STRING_LENGTH_PROMPT_IN_DB can be configured via environment variable
"""
# Set environment variable to a custom value
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "1000")
# Import after setting env var to ensure it picks up the new value
import importlib
import litellm.constants
import litellm.proxy.spend_tracking.spend_tracking_utils
importlib.reload(litellm.constants)
importlib.reload(litellm.proxy.spend_tracking.spend_tracking_utils)
from litellm.constants import (
MAX_STRING_LENGTH_PROMPT_IN_DB,
LITELLM_TRUNCATED_PAYLOAD_FIELD,
)
from litellm.proxy.spend_tracking.spend_tracking_utils import (
_sanitize_request_body_for_spend_logs_payload,
)
# Verify the constant was set to the env var value
assert MAX_STRING_LENGTH_PROMPT_IN_DB == 1000
# Test truncation with the custom value
large_content = "A" * 500 + "B" * 800 + "C" * 500 # 1800 chars total
request_body = {
"messages": [{"role": "user", "content": large_content}],
"model": "gpt-5.5",
}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
# Verify truncation occurred with 35% beginning and 65% end preserved
truncated_content = sanitized["messages"][0]["content"]
expected_start = int(1000 * 0.35) # 350 chars from beginning
expected_end = int(1000 * 0.65) # 650 chars from end
assert truncated_content.startswith(large_content[:expected_start])
assert truncated_content.endswith(large_content[-expected_end:])
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in truncated_content
assert "skipped" in truncated_content
assert "800" in truncated_content # Should mention skipped 800 chars
def test_truncation_preserves_beginning_and_end():
"""
Test that truncation preserves the beginning (35%) and end (65%) of content for better debugging
"""
from litellm.constants import (
MAX_STRING_LENGTH_PROMPT_IN_DB,
LITELLM_TRUNCATED_PAYLOAD_FIELD,
)
# Create content with distinct beginning, middle, and end
beginning = "BEGIN_" * 200 # 1200 chars
middle = "MIDDLE_" * 300 # 2100 chars
end = "_END" * 300 # 1200 chars
large_content = beginning + middle + end
request_body = {
"messages": [{"role": "user", "content": large_content}],
"model": "gpt-5.5",
}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
truncated_content = sanitized["messages"][0]["content"]
# Calculate expected splits (35% beginning, 65% end)
expected_start_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * 0.35)
expected_end_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * 0.65)
# Check that beginning is preserved
expected_beginning = large_content[:expected_start_chars]
assert truncated_content.startswith(expected_beginning)
# Check that end is preserved
expected_end = large_content[-expected_end_chars:]
assert truncated_content.endswith(expected_end)
# Check truncation marker is present
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in truncated_content
assert "skipped" in truncated_content
# Calculate expected skipped chars
total_chars = len(large_content)
kept_chars = expected_start_chars + expected_end_chars
expected_skipped = total_chars - kept_chars
assert str(expected_skipped) in truncated_content
def test_sanitize_request_body_for_spend_logs_payload_long_string():
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
# Create a string longer than MAX_STRING_LENGTH_PROMPT_IN_DB (2048)
long_string = "a" * 3000 # Create a string longer than MAX_STRING_LENGTH_PROMPT_IN_DB
request_body = {"text": long_string, "normal_text": "short text"}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
# Calculate expected lengths: 35% start + 65% end + truncation message
start_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * 0.35)
end_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * 0.65)
total_keep = start_chars + end_chars
if total_keep > MAX_STRING_LENGTH_PROMPT_IN_DB:
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars
skipped_chars = len(long_string) - (start_chars + end_chars)
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars
assert len(sanitized["text"]) == expected_length
assert sanitized["text"].startswith("a" * start_chars)
assert sanitized["text"].endswith("a" * end_chars)
assert expected_truncation_message in sanitized["text"]
assert sanitized["normal_text"] == "short text"
def test_sanitize_request_body_for_spend_logs_payload_nested_dict():
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
# Create a string longer than MAX_STRING_LENGTH_PROMPT_IN_DB
long_string = "a" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
request_body = {"outer": {"inner": {"text": long_string, "normal": "short"}}}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
# Calculate expected lengths based on actual MAX_STRING_LENGTH_PROMPT_IN_DB
start_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * 0.35)
end_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * 0.65)
total_keep = start_chars + end_chars
if total_keep > MAX_STRING_LENGTH_PROMPT_IN_DB:
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars
skipped_chars = len(long_string) - total_keep
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars
assert len(sanitized["outer"]["inner"]["text"]) == expected_length
assert sanitized["outer"]["inner"]["normal"] == "short"
def test_sanitize_request_body_for_spend_logs_payload_nested_list():
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
# Create a string longer than MAX_STRING_LENGTH_PROMPT_IN_DB
long_string = "a" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
request_body = {"items": [{"text": long_string}, {"text": "short"}, [{"text": long_string}]]}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
# Calculate expected lengths based on actual MAX_STRING_LENGTH_PROMPT_IN_DB
start_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * 0.35)
end_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * 0.65)
total_keep = start_chars + end_chars
if total_keep > MAX_STRING_LENGTH_PROMPT_IN_DB:
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars
skipped_chars = len(long_string) - total_keep
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars
assert len(sanitized["items"][0]["text"]) == expected_length
assert sanitized["items"][1]["text"] == "short"
assert len(sanitized["items"][2][0]["text"]) == expected_length
def test_sanitize_request_body_for_spend_logs_payload_non_string_values():
request_body = {"number": 42, "boolean": True, "none": None, "float": 3.14}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
assert sanitized == request_body
def test_sanitize_request_body_for_spend_logs_payload_empty():
request_body: dict[str, Any] = {}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
assert sanitized == request_body
def test_sanitize_request_body_for_spend_logs_payload_mixed_types():
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
# Create a string longer than MAX_STRING_LENGTH_PROMPT_IN_DB
long_string = "a" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
request_body = {
"text": long_string,
"number": 42,
"nested": {"list": ["short", long_string], "dict": {"value": long_string}},
}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
# Calculate expected lengths based on actual MAX_STRING_LENGTH_PROMPT_IN_DB
start_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * 0.35)
end_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * 0.65)
total_keep = start_chars + end_chars
if total_keep > MAX_STRING_LENGTH_PROMPT_IN_DB:
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars
skipped_chars = len(long_string) - total_keep
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars
assert len(sanitized["text"]) == expected_length
assert sanitized["number"] == 42
assert sanitized["nested"]["list"][0] == "short"
assert len(sanitized["nested"]["list"][1]) == expected_length
assert len(sanitized["nested"]["dict"]["value"]) == expected_length
def test_sanitize_request_body_for_spend_logs_payload_uses_runtime_env_override(
monkeypatch: pytest.MonkeyPatch,
):
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
override_max = max(MAX_STRING_LENGTH_PROMPT_IN_DB + 1000, 6000)
test_string = "a" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
# Simulate config-loaded env var being set after module import.
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", str(override_max))
sanitized = _sanitize_request_body_for_spend_logs_payload({"text": test_string})
assert sanitized["text"] == test_string
def test_sanitize_request_body_for_spend_logs_payload_circular_reference():
# Create a circular reference
a: dict[str, Any] = {}
b: dict[str, Any] = {"a": a}
a["b"] = b
# Test that it handles circular reference without infinite recursion
sanitized = _sanitize_request_body_for_spend_logs_payload(a)
assert sanitized == {"b": {"a": {}}} # Should return empty dict for circular reference
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_get_vector_store_request_for_spend_logs_payload_store_prompts_true(
mock_should_store,
):
# When should_store_prompts_and_responses_in_spend_logs returns True
mock_should_store.return_value = True
# Sample vector store request metadata
vector_store_request = [
{"vector_store_search_response": {"data": [{"content": [{"text": "sensitive information", "type": "text"}]}]}}
]
# When store_prompts is True, the original data should be returned unchanged
result = _get_vector_store_request_for_spend_logs_payload(vector_store_request)
assert result == vector_store_request
assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"] == "sensitive information"
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_get_vector_store_request_for_spend_logs_payload_store_prompts_false(
mock_should_store,
):
# When should_store_prompts_and_responses_in_spend_logs returns False
mock_should_store.return_value = False
# Sample vector store request metadata
vector_store_request = [
{"vector_store_search_response": {"data": [{"content": [{"text": "sensitive information", "type": "text"}]}]}}
]
# When store_prompts is False, text should be redacted
result = _get_vector_store_request_for_spend_logs_payload(vector_store_request)
assert result is not None
assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"] == REDACTED_BY_LITELM_STRING
# Ensure other fields are unchanged
assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["type"] == "text"
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_get_vector_store_request_for_spend_logs_payload_null_input(mock_should_store):
# When input is None
mock_should_store.return_value = False
result = _get_vector_store_request_for_spend_logs_payload(None)
assert result is None
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_get_messages_for_spend_logs_realtime_returns_messages(mock_should_store):
"""
Test that _get_messages_for_spend_logs_payload returns messages
for realtime calls when store_prompts_in_spend_logs is True.
"""
mock_should_store.return_value = True
realtime_messages = [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "What is the weather today?"},
]
payload = cast(
StandardLoggingPayload,
{
"call_type": "_arealtime",
"messages": realtime_messages,
},
)
result = _get_messages_for_spend_logs_payload(payload)
parsed = json.loads(result)
assert len(parsed) == 2
assert parsed[0]["role"] == "system"
assert parsed[0]["content"] == "You are a helpful assistant."
assert parsed[1]["role"] == "user"
assert parsed[1]["content"] == "What is the weather today?"
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_get_messages_for_spend_logs_strips_null_bytes(mock_should_store):
"""Regression for PostgreSQL 22P05: NUL bytes must be stripped from messages."""
mock_should_store.return_value = True
payload = cast(
StandardLoggingPayload,
{
"call_type": "_arealtime",
"messages": [{"role": "user", "content": "hello\x00world"}],
},
)
result = _get_messages_for_spend_logs_payload(payload)
assert "\\u0000" not in result
parsed = json.loads(result)
assert parsed[0]["content"] == "helloworld"
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_get_messages_for_spend_logs_realtime_empty_when_disabled(mock_should_store):
"""
Test that _get_messages_for_spend_logs_payload returns '{}' for realtime calls
when store_prompts_in_spend_logs is False.
"""
mock_should_store.return_value = False
payload = cast(
StandardLoggingPayload,
{
"call_type": "_arealtime",
"messages": [{"role": "user", "content": "Hello"}],
},
)
result = _get_messages_for_spend_logs_payload(payload)
assert result == "{}"
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_get_messages_for_spend_logs_non_realtime_returns_empty(mock_should_store):
"""
Test that _get_messages_for_spend_logs_payload returns '{}' for non-realtime
calls even when store_prompts_in_spend_logs is True.
"""
mock_should_store.return_value = True
payload = cast(
StandardLoggingPayload,
{
"call_type": "acompletion",
"messages": [{"role": "user", "content": "Hello"}],
},
)
result = _get_messages_for_spend_logs_payload(payload)
assert result == "{}"
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_get_response_for_spend_logs_payload_truncates_large_base64(mock_should_store):
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
mock_should_store.return_value = True
large_text = "A" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
payload = cast(
StandardLoggingPayload,
{
"response": {
"data": [
{
"b64_json": large_text,
"other_field": "value",
}
]
}
},
)
response_json = _get_response_for_spend_logs_payload(payload)
parsed = json.loads(response_json)
truncated_value = parsed["data"][0]["b64_json"]
assert len(truncated_value) < len(large_text)
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in truncated_value
assert parsed["data"][0]["other_field"] == "value"
def test_get_response_for_spend_logs_payload_keeps_a_long_compaction_summary_whole(monkeypatch: pytest.MonkeyPatch):
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
monkeypatch.setenv("STORE_PROMPTS_IN_SPEND_LOGS", "true")
filler = "detail " * (MAX_STRING_LENGTH_PROMPT_IN_DB // 7)
summary = f"{filler}The cat is named Biscuit. {filler}"
answer = f"{filler}OK{filler}"
payload = cast(
StandardLoggingPayload,
{
"response": {
"choices": [
{
"message": {
"role": "assistant",
"content": answer,
"provider_specific_fields": {
"compaction_blocks": [{"type": "compaction", "content": summary}]
},
}
}
]
}
},
)
message = json.loads(_get_response_for_spend_logs_payload(payload))["choices"][0]["message"]
assert message["provider_specific_fields"]["compaction_blocks"] == [{"type": "compaction", "content": summary}]
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in message["content"]
def test_stored_request_keeps_only_the_summary_of_a_long_claude_compaction_input_item(monkeypatch: pytest.MonkeyPatch):
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
monkeypatch.setenv("STORE_PROMPTS_IN_SPEND_LOGS", "true")
filler = "detail " * (MAX_STRING_LENGTH_PROMPT_IN_DB // 7)
summary = f"{filler}The cat is named Biscuit. {filler}"
claude_item = json.dumps({"type": "compaction", "content": summary, "padding": filler * 10})
opaque_item = "gAAAA" + "x" * (MAX_STRING_LENGTH_PROMPT_IN_DB * 2)
request_body = {
"input": [
{"type": "compaction", "id": "cmp_1", "encrypted_content": claude_item},
{"type": "compaction", "id": "cmp_2", "encrypted_content": opaque_item},
{"role": "assistant", "content": [{"type": "compaction", "content": f"{filler}Biscuit{filler}"}]},
{"role": "user", "content": f"{filler}What is my cat called?{filler}"},
],
"metadata": {"note": {"type": "compaction", "encrypted_content": claude_item}},
}
stored = json.loads(
_get_proxy_server_request_for_spend_logs_payload(
metadata={}, litellm_params={"proxy_server_request": {"body": request_body}}
)
)
claude_stored, opaque_stored, messages_block_stored, message_stored = stored["input"]
assert {**claude_stored, "encrypted_content": json.loads(claude_stored["encrypted_content"])} == {
"type": "compaction",
"id": "cmp_1",
"encrypted_content": {"type": "compaction", "content": summary},
}
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in opaque_stored["encrypted_content"]
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in messages_block_stored["content"][0]["content"]
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in message_stored["content"]
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored["metadata"]["note"]["encrypted_content"]
def _compaction_summary_past_the_summary_cap() -> str:
from litellm.constants import MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB
filler: Final = "detail " * (MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB // 7)
return f"The cat is named Biscuit. {filler}The dog is named Rex."
def _assert_bounded_like_a_db_string(stored: str, summary: str) -> None:
from litellm.constants import MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored
assert len(stored) < MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB + 500 < len(summary) + 500
assert stored.startswith("The cat is named Biscuit.")
assert stored.endswith("The dog is named Rex.")
@pytest.mark.parametrize("operator_cap_above_the_summary", [False, True])
def test_spend_row_bounds_a_compaction_summary_at_the_larger_of_the_summary_cap_and_the_operator_cap(
monkeypatch: pytest.MonkeyPatch, operator_cap_above_the_summary: bool
):
monkeypatch.setenv("STORE_PROMPTS_IN_SPEND_LOGS", "true")
summary: Final = _compaction_summary_past_the_summary_cap()
if operator_cap_above_the_summary:
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", str(len(summary) * 2))
payload: Final = cast(
StandardLoggingPayload,
{
"response": {
"choices": [
{
"message": {
"role": "assistant",
"content": "OK",
"provider_specific_fields": {
"compaction_blocks": [{"type": "compaction", "content": summary}]
},
}
}
]
}
},
)
request_body: Final = {
"input": [
{
"type": "compaction",
"id": "cmp_1",
"encrypted_content": json.dumps({"type": "compaction", "content": summary}),
}
]
}
response_message: Final = json.loads(_get_response_for_spend_logs_payload(payload))["choices"][0]["message"]
stored_item: Final = json.loads(
_get_proxy_server_request_for_spend_logs_payload(
metadata={}, litellm_params={"proxy_server_request": {"body": request_body}}
)
)["input"][0]
stored_response_block: Final = response_message["provider_specific_fields"]["compaction_blocks"][0]
stored_request_block: Final = json.loads(stored_item["encrypted_content"])
assert stored_request_block["type"] == stored_response_block["type"] == "compaction"
if operator_cap_above_the_summary:
assert stored_response_block["content"] == stored_request_block["content"] == summary
return
_assert_bounded_like_a_db_string(stored_response_block["content"], summary)
_assert_bounded_like_a_db_string(stored_request_block["content"], summary)
@pytest.mark.parametrize("summary_past_the_summary_cap", [False, True])
def test_responses_shaped_spend_row_keeps_a_compaction_output_item_summary_bounded_like_a_chat_block(
monkeypatch: pytest.MonkeyPatch, summary_past_the_summary_cap: bool
):
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
from litellm.responses.litellm_completion_transformation.compaction_items import compaction_summary
from litellm.responses.utils import ResponsesAPIRequestUtils
monkeypatch.setenv("STORE_PROMPTS_IN_SPEND_LOGS", "true")
filler: Final = "detail " * (MAX_STRING_LENGTH_PROMPT_IN_DB // 7)
summary: Final = (
_compaction_summary_past_the_summary_cap()
if summary_past_the_summary_cap
else f"{filler}The cat is named Biscuit. {filler}"
)
payload: Final = cast(
StandardLoggingPayload,
{
"response": {
"object": "response",
"output": [
{
"type": "compaction",
"id": "cmp_1",
"encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id(
json.dumps({"type": "compaction", "content": summary}), "deployment-1"
),
},
{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "OK"}]},
],
}
},
)
stored_item: Final = json.loads(_get_response_for_spend_logs_payload(payload))["output"][0]
model_id, _ = ResponsesAPIRequestUtils.unwrap_encrypted_content_with_model_id(stored_item["encrypted_content"])
stored_summary: Final = compaction_summary(stored_item)
assert model_id == "deployment-1"
assert stored_summary is not None
if summary_past_the_summary_cap:
_assert_bounded_like_a_db_string(stored_summary, summary)
return
assert stored_summary == summary
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_get_response_for_spend_logs_payload_strips_null_bytes(mock_should_store):
"""Regression for PostgreSQL 22P05: NUL bytes must be stripped from response."""
mock_should_store.return_value = True
payload = cast(
StandardLoggingPayload,
{"response": {"content": "answer\x00here"}},
)
response_json = _get_response_for_spend_logs_payload(payload)
assert "\\u0000" not in response_json
assert json.loads(response_json)["content"] == "answerhere"
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_get_response_for_spend_logs_payload_truncates_large_embedding(
mock_should_store,
):
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
mock_should_store.return_value = True
embedding_values = [round(i * 0.0001, 6) for i in range(MAX_STRING_LENGTH_PROMPT_IN_DB + 500)]
large_embedding = json.dumps(embedding_values)
payload = cast(
StandardLoggingPayload,
{
"response": {
"data": [
{
"embedding": large_embedding,
"other_field": "value",
}
]
}
},
)
response_json = _get_response_for_spend_logs_payload(payload)
parsed = json.loads(response_json)
truncated_value = parsed["data"][0]["embedding"]
assert isinstance(truncated_value, str)
assert len(truncated_value) < len(large_embedding)
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in truncated_value
assert parsed["data"][0]["other_field"] == "value"
def test_truncation_includes_db_safeguard_note():
"""
Test that truncated content includes the DB safeguard note explaining
that full data is available in OTEL/other logging integrations.
"""
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
large_error = "Error: " + "x" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 1000)
request_body = {"error_trace": large_error}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
truncated = sanitized["error_trace"]
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in truncated
assert LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE in truncated
assert "DB storage safeguard" in truncated
assert (
"logging callbacks" in truncated.lower()
or "logging integrations" in truncated.lower()
or "logging callbacks" in truncated
)
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_response_truncation_logs_info_message(mock_should_store):
"""
Test that when response is truncated before DB storage, an info log is emitted
noting that full data is available in OTEL/other integrations.
"""
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
mock_should_store.return_value = True
large_text = "B" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
payload = cast(
StandardLoggingPayload,
{"response": {"data": [{"content": large_text}]}},
)
with patch("litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger") as mock_logger:
_get_response_for_spend_logs_payload(payload)
mock_logger.info.assert_called_once()
log_msg = mock_logger.info.call_args[0][0]
assert "response was truncated" in log_msg
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_request_body_truncation_logs_info_message(mock_should_store):
"""
Test that when request body is truncated before DB storage, an info log is emitted.
"""
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
mock_should_store.return_value = True
large_prompt = "C" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
litellm_params = {"proxy_server_request": {"body": {"messages": [{"role": "user", "content": large_prompt}]}}}
with patch("litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger") as mock_logger:
_get_proxy_server_request_for_spend_logs_payload(metadata={}, litellm_params=litellm_params, kwargs={})
mock_logger.info.assert_called_once()
log_msg = mock_logger.info.call_args[0][0]
assert "request body was truncated" in log_msg
def test_safe_dumps_handles_circular_references():
"""Test that safe_dumps can handle circular references without raising exceptions"""
# Create a circular reference
obj1 = {"name": "obj1"}
obj2 = {"name": "obj2", "ref": obj1}
obj1["ref"] = obj2 # This creates a circular reference
# This should not raise an exception
result = safe_dumps(obj1)
# Should be a valid JSON string
assert isinstance(result, str)
# Should contain placeholder for circular reference
assert "CircularReference Detected" in result
# Should be parseable as JSON
parsed = json.loads(result)
assert parsed["name"] == "obj1"
assert parsed["ref"]["name"] == "obj2"
def test_safe_dumps_normal_objects():
"""Test that safe_dumps works correctly with normal objects"""
normal_obj = {
"string": "test",
"number": 42,
"boolean": True,
"null": None,
"list": [1, 2, 3],
"nested": {"key": "value"},
}
result = safe_dumps(normal_obj)
# Should be a valid JSON string that can be parsed
assert isinstance(result, str)
parsed = json.loads(result)
assert parsed == normal_obj
def test_safe_dumps_complex_metadata_like_object():
"""Test with a complex metadata-like object similar to what caused the issue"""
# Simulate a complex metadata object
metadata = {
"user_api_key": "test-key",
"model": "gpt-4",
"usage": {"total_tokens": 100},
"mcp_tool_call_metadata": {
"name": "test_tool",
"arguments": {"param": "value"},
},
}
# Add a potential circular reference
usage_detail = {"parent_metadata": metadata}
metadata["usage"]["detail"] = usage_detail
# This should not raise an exception
result = safe_dumps(metadata)
# Should be a valid JSON string
assert isinstance(result, str)
# Should be parseable as JSON
parsed = json.loads(result)
assert parsed["user_api_key"] == "test-key"
assert parsed["model"] == "gpt-4"
_RAW_MODEL_WITH_PROMPT: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes"
_BEDROCK_INFERENCE_PROFILE_ARN: Final = (
"arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-sonnet-4-5"
)
_OVERLONG_MODEL: Final = "m" * (MAX_SPEND_LOG_MODEL_NAME_LENGTH + 1)
@pytest.mark.parametrize(
("requested_model", "failure", "expected_model"),
[
(
_RAW_MODEL_WITH_PROMPT,
ProxyModelNotFoundError(route="acompletion", model_name=_RAW_MODEL_WITH_PROMPT),
UNKNOWN_MODEL_SPEND_LOG_MODEL,
),
(
_RAW_MODEL_WITH_PROMPT,
ValueError("Upstream passthrough request failed with status 404"),
UNKNOWN_MODEL_SPEND_LOG_MODEL,
),
(_OVERLONG_MODEL, ValueError("provider timed out"), UNKNOWN_MODEL_SPEND_LOG_MODEL),
(
"gpt-5.2",
ProxyModelNotFoundError(route="acompletion", model_name="gpt-5.2"),
UNKNOWN_MODEL_SPEND_LOG_MODEL,
),
("gpt-5.2", ValueError("provider timed out"), "gpt-5.2"),
(_BEDROCK_INFERENCE_PROFILE_ARN, ValueError("provider timed out"), _BEDROCK_INFERENCE_PROFILE_ARN),
(
"MCP: deepwiki-ask_question",
ValueError("Content blocked: keyword 'confidential' detected"),
"MCP: deepwiki-ask_question",
),
],
)
def test_get_logging_payload_replaces_rejected_or_prompt_shaped_models_with_the_placeholder(
requested_model: str, failure: Exception, expected_model: str
):
kwargs: Final = {
"model": requested_model,
"messages": [{"role": "user", "content": "hi"}],
"call_type": "acompletion",
"litellm_params": {"metadata": {"user_api_key": "sk-test", "status": "failure"}},
}
payload: Final = get_logging_payload(
kwargs=kwargs,
response_obj=failure,
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["model"] == expected_model
@pytest.mark.parametrize("requested_model", [{"bad": "value"}, ["gpt-5.2"], 1])
def test_get_logging_payload_replaces_a_non_string_model_with_the_placeholder(
requested_model: dict[str, str] | list[str] | int,
):
kwargs: Final = {
"model": requested_model,
"messages": [{"role": "user", "content": "hi"}],
"call_type": "acompletion",
"litellm_params": {"metadata": {"user_api_key": "sk-test", "status": "failure"}},
}
payload: Final = get_logging_payload(
kwargs=kwargs,
response_obj=ValueError("model must be a string"),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["model"] == UNKNOWN_MODEL_SPEND_LOG_MODEL
@pytest.mark.parametrize(
("metadata", "response_obj"),
[
({"user_api_key": "sk-test"}, litellm.ModelResponse(id="chatcmpl-test", choices=[])),
(
{
"user_api_key": "sk-test",
"model_group": "team alias",
"model_info": {"id": "team-alias-deployment"},
"status": "failure",
},
ValueError("provider timed out"),
),
],
)
def test_get_logging_payload_keeps_a_whitespace_model_name_on_success_or_a_routed_failure(
metadata: dict[str, object], response_obj: litellm.ModelResponse | Exception
):
kwargs: Final = {
"model": _RAW_MODEL_WITH_PROMPT,
"messages": [{"role": "user", "content": "hi"}],
"call_type": "acompletion",
"litellm_params": {"metadata": metadata},
}
payload: Final = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["model"] == _RAW_MODEL_WITH_PROMPT
@pytest.mark.parametrize("redact_messages", [False, True])
@pytest.mark.parametrize(
("metadata", "expected_stored_model"),
[
({"user_api_key": "sk-test", "status": "failure"}, UNKNOWN_MODEL_SPEND_LOG_MODEL),
(
{"user_api_key": "sk-test", "status": "failure", "model_info": {"id": "routed-deployment"}},
_RAW_MODEL_WITH_PROMPT,
),
],
)
def test_get_logging_payload_placeholders_the_stored_request_body_model_only_when_the_row_is_placeholdered(
monkeypatch: pytest.MonkeyPatch,
metadata: dict[str, object],
expected_stored_model: str,
redact_messages: bool,
):
from litellm.proxy import proxy_server
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True})
kwargs: Final = {
"model": _RAW_MODEL_WITH_PROMPT,
"call_type": "amoderation",
"standard_callback_dynamic_params": {"turn_off_message_logging": redact_messages},
"litellm_params": {
"metadata": metadata,
"proxy_server_request": {
"url": "http://localhost:4000/v1/moderations",
"body": {"input": "hi", "model": _RAW_MODEL_WITH_PROMPT},
},
},
}
payload: Final = get_logging_payload(
kwargs=kwargs,
response_obj=ValueError("Invalid value for 'model'"),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
stored_request_body: Final = json.loads(payload["proxy_server_request"])
assert stored_request_body["model"] == expected_stored_model
@pytest.mark.parametrize(
("deployment_info", "expected_stored_model_group", "expected_stored_error_message"),
[
({}, "", f"Invalid value for 'model' = {UNKNOWN_MODEL_SPEND_LOG_MODEL}"),
(
{"model_info": {"id": "routed-deployment"}},
_RAW_MODEL_WITH_PROMPT,
f"Invalid value for 'model' = {_RAW_MODEL_WITH_PROMPT}",
),
],
)
def test_get_logging_payload_placeholders_the_metadata_copied_into_the_stored_request_body(
monkeypatch: pytest.MonkeyPatch,
deployment_info: dict[str, object],
expected_stored_model_group: str,
expected_stored_error_message: str,
):
from litellm.proxy import proxy_server
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True})
metadata: Final = {
"user_api_key": "sk-test",
"status": "failure",
"model_group": _RAW_MODEL_WITH_PROMPT,
"error_information": {
"error_code": "400",
"error_class": "BadRequestError",
"llm_provider": "openai",
"error_message": f"Invalid value for 'model' = {_RAW_MODEL_WITH_PROMPT}",
"traceback": "",
},
**deployment_info,
}
kwargs: Final = {
"model": _RAW_MODEL_WITH_PROMPT,
"call_type": "amoderation",
"litellm_params": {
"metadata": metadata,
"proxy_server_request": {
"url": "http://localhost:4000/v1/moderations",
"body": {"input": "hi", "model": _RAW_MODEL_WITH_PROMPT, "metadata": metadata},
},
},
}
payload: Final = get_logging_payload(
kwargs=kwargs,
response_obj=ValueError("Invalid value for 'model'"),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
stored_request_body: Final = json.loads(payload["proxy_server_request"])
assert stored_request_body["metadata"]["model_group"] == expected_stored_model_group
assert stored_request_body["metadata"]["error_information"]["error_message"] == expected_stored_error_message
assert stored_request_body["metadata"]["user_api_key"] == REDACTED_BY_LITELM_STRING
assert ("medical records" in payload["proxy_server_request"]) == bool(deployment_info)
_WHITESPACE_MODEL_GROUP: Final = "Broken GPT Mini"
_WHITESPACE_MODEL_GROUP_ALIAS: Final = "Broken GPT Alias"
_COOLDOWN_ERROR_MESSAGE: Final = (
f"No deployments available for selected model. Passed model={_WHITESPACE_MODEL_GROUP}. Try again in 300 seconds"
)
def _router_serving_the_whitespace_model_group() -> litellm.Router:
return litellm.Router(
model_list=[
{
"model_name": _WHITESPACE_MODEL_GROUP,
"litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "sk-test"},
}
],
model_group_alias={_WHITESPACE_MODEL_GROUP_ALIAS: _WHITESPACE_MODEL_GROUP},
)
def _router_serving_only_a_wildcard() -> litellm.Router:
return litellm.Router(
model_list=[{"model_name": "*", "litellm_params": {"model": "openai/*", "api_key": "sk-test"}}]
)
@pytest.mark.parametrize(
("requested_model", "llm_router", "expected_model", "expected_model_group", "expected_error_message"),
[
(
_WHITESPACE_MODEL_GROUP,
_router_serving_the_whitespace_model_group(),
_WHITESPACE_MODEL_GROUP,
_WHITESPACE_MODEL_GROUP,
_COOLDOWN_ERROR_MESSAGE,
),
(
_WHITESPACE_MODEL_GROUP_ALIAS,
_router_serving_the_whitespace_model_group(),
_WHITESPACE_MODEL_GROUP_ALIAS,
_WHITESPACE_MODEL_GROUP_ALIAS,
_COOLDOWN_ERROR_MESSAGE,
),
(
_WHITESPACE_MODEL_GROUP,
_router_serving_only_a_wildcard(),
UNKNOWN_MODEL_SPEND_LOG_MODEL,
"",
_COOLDOWN_ERROR_MESSAGE.replace(_WHITESPACE_MODEL_GROUP, UNKNOWN_MODEL_SPEND_LOG_MODEL),
),
(
_WHITESPACE_MODEL_GROUP,
None,
UNKNOWN_MODEL_SPEND_LOG_MODEL,
"",
_COOLDOWN_ERROR_MESSAGE.replace(_WHITESPACE_MODEL_GROUP, UNKNOWN_MODEL_SPEND_LOG_MODEL),
),
],
)
def test_get_logging_payload_keeps_a_configured_whitespace_model_group_that_failed_before_a_deployment_was_picked(
requested_model: str,
llm_router: litellm.Router | None,
expected_model: str,
expected_model_group: str,
expected_error_message: str,
):
kwargs: Final = {
"model": requested_model,
"messages": [{"role": "user", "content": "hi"}],
"call_type": "acompletion",
"litellm_params": {
"metadata": {
"user_api_key": "sk-test",
"model_group": requested_model,
"status": "failure",
"error_information": {"error_message": _COOLDOWN_ERROR_MESSAGE, "error_class": "RateLimitError"},
}
},
}
payload: Final = get_logging_payload(
kwargs=kwargs,
response_obj=litellm.RateLimitError(message=_COOLDOWN_ERROR_MESSAGE, model=requested_model, llm_provider=""),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
llm_router=llm_router,
)
persisted_error: Final = json.loads(payload["metadata"])["error_information"]
assert (payload["model"], payload["model_group"], persisted_error["error_message"]) == (
expected_model,
expected_model_group,
expected_error_message,
)
def _openai_invalid_model_error_message(model: str) -> str:
body: Final = {
"error": {
"message": f"Invalid value for 'model' = {model}. Please check the OpenAI documentation and try again.",
"type": "invalid_request_error",
"param": "model",
"code": None,
}
}
return f"Error code: 400 - {body}"
def test_get_logging_payload_persists_no_raw_model_for_a_prompt_shaped_moderation_rejected_by_the_provider():
provider_rejection: Final = litellm.BadRequestError(
message=_openai_invalid_model_error_message(_RAW_MODEL_WITH_PROMPT),
model=_RAW_MODEL_WITH_PROMPT,
llm_provider="openai",
)
error_information: Final = sanitize_error_information_for_spend_logs(
StandardLoggingPayloadSetup.get_error_information(
original_exception=provider_rejection,
traceback_str=(
f"Traceback (most recent call last):\n ...\nlitellm.exceptions.BadRequestError: {provider_rejection}"
),
),
original_exception=provider_rejection,
)
kwargs: Final = {
"model": _RAW_MODEL_WITH_PROMPT,
"input": "hi",
"call_type": "",
"litellm_params": {
"metadata": {
"user_api_key": "sk-test",
"model_group": _RAW_MODEL_WITH_PROMPT,
"status": "failure",
"error_information": error_information,
}
},
}
payload: Final = get_logging_payload(
kwargs=kwargs,
response_obj=provider_rejection,
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
persisted_error: Final = json.loads(payload["metadata"])["error_information"]
scrubbed_message: Final = (
f"litellm.BadRequestError: {_openai_invalid_model_error_message(UNKNOWN_MODEL_SPEND_LOG_MODEL)}"
)
assert (payload["model"], payload["model_group"]) == (UNKNOWN_MODEL_SPEND_LOG_MODEL, "")
assert persisted_error["error_message"] == scrubbed_message
assert persisted_error["traceback"].endswith(scrubbed_message)
assert "medical records" not in payload["metadata"]
_TRUNCATION_MARKER_TEXT: Final = (
f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped 10 chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
)
@pytest.mark.parametrize(
("error_text", "expected"),
[
(f"Invalid model {_RAW_MODEL_WITH_PROMPT}", f"Invalid model {UNKNOWN_MODEL_SPEND_LOG_MODEL}"),
(
f"OpenAIException - {{'message': {_RAW_MODEL_WITH_PROMPT!r}}}",
f"OpenAIException - {{'message': '{UNKNOWN_MODEL_SPEND_LOG_MODEL}'}}",
),
(
(
f"Invalid model {_RAW_MODEL_WITH_PROMPT[:20]}{_TRUNCATION_MARKER_TEXT}"
f"{_RAW_MODEL_WITH_PROMPT[30:]} rejected"
),
(
f"Invalid model {UNKNOWN_MODEL_SPEND_LOG_MODEL}{_TRUNCATION_MARKER_TEXT}"
f"{UNKNOWN_MODEL_SPEND_LOG_MODEL} rejected"
),
),
],
)
def test_scrub_raw_model_from_error_information_covers_literal_escaped_and_truncation_split_spellings(
error_text: str, expected: str
):
scrubbed: Final = _scrub_raw_model_from_error_information(
cast(
StandardLoggingPayloadErrorInformation,
{"error_message": error_text, "traceback": error_text, "error_class": "BadRequestError"},
),
_RAW_MODEL_WITH_PROMPT,
)
assert scrubbed == {"error_message": expected, "traceback": expected, "error_class": "BadRequestError"}
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_api_key_preserved_when_standard_logging_payload_is_none():
"""
Critical - Product incident was caused by this bug.
Test that api_key is NOT set to empty string when standard_logging_payload is None.
This is a regression test for a bug where:
- On failed requests (bad request errors), standard_logging_payload is None
- The else block was incorrectly setting api_key = ""
- This caused empty api_key in DailyUserSpend table despite SpendLogs having the correct key
Expected behavior:
- api_key from metadata should be extracted and hashed
- Even when standard_logging_payload is None, the api_key should be preserved
- The returned payload should have the hashed api_key, not empty string
"""
# Setup: Simulate a failed request scenario
test_api_key = "sk-WLi4iRn4JmbVlTaYw12IOA"
# Create kwargs similar to what's passed during a bad request error
kwargs = {
"model": "openai/gpt-4.1",
"messages": [{"role": "user", "content": "Hello"}],
"call_type": "acompletion",
"litellm_params": {
"metadata": {
"user_api_key": test_api_key, # This is the key that should be preserved
"user_api_key_user_id": "test_user",
"user_api_key_team_id": "test_team",
}
},
# Note: No 'standard_logging_object' in kwargs - simulating failure case
}
# Create a mock error response (bad request)
response_obj = Exception("BadRequestError: Invalid parameter 'usersss'")
# Create timestamps
start_time = datetime.datetime.now(timezone.utc)
end_time = datetime.datetime.now(timezone.utc)
# Call get_logging_payload
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
)
# CRITICAL ASSERTION: api_key should NOT be empty string
assert payload["api_key"] != "", (
"BUG: api_key is empty! When standard_logging_payload is None, "
"the api_key from metadata should be preserved and hashed."
)
# The api_key should be hashed (not the raw key)
assert payload["api_key"] != test_api_key, "api_key should be hashed, not the raw key"
# The api_key should be a valid hash (64 character hex string for SHA256)
assert len(payload["api_key"]) == 64, f"Expected 64 character hash, got {len(payload['api_key'])} characters"
# Verify other fields are set correctly
assert payload["model"] == "openai/gpt-4.1"
assert payload["user"] == "test_user"
@pytest.mark.asyncio
@patch("litellm.proxy.proxy_server.master_key", "sk-master-key")
@patch("litellm.proxy.proxy_server.general_settings", {})
async def test_api_key_preserved_through_failure_hook_to_database():
"""
CRITICAL E2E TEST: Validates the COMPLETE code path from failure hook to database.
This is THE comprehensive test that protects against the production incident.
It tests the EXACT flow that caused the bug:
1. async_post_call_failure_hook is called with api_key in UserAPIKeyAuth
2. Failure hook calls update_database with the token parameter
3. update_database calls get_logging_payload to create payload
4. BUG WAS HERE: get_logging_payload set api_key = "" when standard_logging_payload was None
5. Empty api_key was written to DailyUserSpend table
This test validates the ENTIRE flow to ensure the bug cannot regress.
If this test fails in CI/CD, the build MUST fail.
"""
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger
from litellm.proxy.utils import hash_token
# Setup
test_api_key = "sk-test-critical-e2e-key"
hashed_key = hash_token(test_api_key)
# Track what payload gets created
captured_payloads = []
async def mock_update_database(
token,
response_cost,
user_id,
end_user_id,
team_id,
kwargs,
completion_response,
start_time,
end_time,
org_id,
project_id=None,
):
"""Mock update_database and capture the payload it creates"""
from litellm.proxy.spend_tracking.spend_tracking_utils import (
get_logging_payload,
)
# Call get_logging_payload EXACTLY as update_database does
payload = get_logging_payload(
kwargs=kwargs,
response_obj=completion_response,
start_time=start_time,
end_time=end_time,
)
captured_payloads.append(
{
"token": token,
"payload": payload,
}
)
# Mock dependencies
mock_db_writer = MagicMock()
mock_db_writer.update_database = AsyncMock(side_effect=mock_update_database)
mock_proxy_logging_obj = MagicMock()
mock_proxy_logging_obj.db_spend_update_writer = mock_db_writer
# Create UserAPIKeyAuth (what the failure hook receives)
user_api_key_dict = UserAPIKeyAuth(
api_key=hashed_key,
user_id="test_user",
team_id="test_team",
max_budget=None,
spend=0.0,
key_alias="test-key",
budget_reset_at=None,
user_email=None,
org_id="test_org",
team_alias=None,
end_user_id=None,
request_route="/chat/completions",
metadata={},
)
# Request data with bad parameter (triggers failure)
request_data = {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "test"}],
"invalid_param": "causes_400_error", # BAD PARAMETER
"litellm_params": {
"metadata": {
"user_api_key": hashed_key,
"user_api_key_user_id": "test_user",
"user_api_key_team_id": "test_team",
}
},
}
exception = Exception("BadRequestError: Invalid parameter 'invalid_param'")
# Execute the ACTUAL failure hook code path
logger = ProxyDBLogger()
with patch("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj):
await logger.async_post_call_failure_hook(
request_data=request_data,
original_exception=exception,
user_api_key_dict=user_api_key_dict,
traceback_str=None,
)
await asyncio.sleep(0.1) # Wait for async operations
# =========================================================================
# CRITICAL ASSERTIONS - If ANY fail, the production bug has regressed!
# =========================================================================
assert len(captured_payloads) == 1, "update_database should be called once"
data = captured_payloads[0]
payload = data["payload"]
payload_api_key = payload.get("api_key")
# THE CRITICAL ASSERTION - This would fail with the original bug!
assert payload_api_key != "", (
"🚨 CRITICAL BUG: payload['api_key'] is empty! "
"This is the EXACT production incident bug. "
"get_logging_payload() is setting api_key = '' when "
"standard_logging_payload is None (failure case)."
)
assert payload_api_key is not None, "🚨 CRITICAL: payload['api_key'] is None!"
assert payload_api_key == hashed_key, f"🚨 CRITICAL: Expected api_key={hashed_key}, got {payload_api_key}"
# Verify token parameter matches
assert data["token"] == hashed_key, f"Token parameter should be {hashed_key}"
# Verify other fields
assert payload.get("model") == "gpt-3.5-turbo"
assert payload.get("user") == "test_user"
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_includes_agent_id_from_kwargs():
"""
Test that get_logging_payload extracts agent_id from kwargs and includes it in the payload.
"""
test_agent_id = "agent-uuid-12345"
kwargs = {
"model": "a2a_agent/test-agent",
"custom_llm_provider": "a2a_agent",
"agent_id": test_agent_id,
"litellm_params": {
"metadata": {
"user_api_key": "sk-test-key",
}
},
}
response_obj = {
"id": "test-response-123",
"jsonrpc": "2.0",
"result": {"status": "completed"},
}
start_time = datetime.datetime.now(timezone.utc)
end_time = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
)
assert payload["agent_id"] == test_agent_id, f"Expected agent_id '{test_agent_id}', got '{payload.get('agent_id')}'"
def test_get_logging_payload_populates_litellm_call_id_alongside_provider_request_id():
"""
LIT-6302: request_id stays the provider response id, so clients holding the
x-litellm-call-id header value could never find their row. The payload now
also carries litellm_call_id as its own column for lookups by either id.
"""
call_id = "b980eea9-5cd9-4099-93cd-8291e46c76fd"
payload = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_call_id": call_id,
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
response_obj=litellm.ModelResponse(
id="chatcmpl-provider-id",
choices=[],
usage=litellm.Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2),
),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["request_id"] == "chatcmpl-provider-id"
assert payload["litellm_call_id"] == call_id
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_includes_overhead_in_spend_logs_metadata():
"""
Test that get_logging_payload extracts litellm_overhead_time_ms from hidden_params
and stores it in spend_logs_metadata within the metadata JSON.
"""
test_overhead_ms = 123.45
# Create StandardLoggingPayload with hidden_params containing overhead
standard_logging_payload = StandardLoggingPayload(
id="test-id-123",
call_type="completion",
stream=False,
response_cost=0.001,
status="success",
total_tokens=100,
prompt_tokens=50,
completion_tokens=50,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=None,
model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None),
model="gpt-3.5-turbo",
model_id="model-123",
model_group="openai",
custom_llm_provider="openai",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_alias=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_user_id=None,
user_api_key_user_email=None,
user_api_key_team_alias=None,
spend_logs_metadata=None,
requester_ip_address=None,
requester_metadata=None,
user_api_key_end_user_id=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address=None,
messages=[],
response={},
error_str=None,
model_parameters={},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.001",
litellm_overhead_time_ms=test_overhead_ms,
additional_headers=None,
batch_models=None,
litellm_model_name=None,
usage_object=None,
),
)
kwargs = {
"model": "gpt-3.5-turbo",
"litellm_params": {
"metadata": {
"user_api_key": "sk-test-key",
}
},
"standard_logging_object": standard_logging_payload,
}
response_obj = {
"id": "test-response-123",
"choices": [{"message": {"content": "Hello!"}}],
"usage": {
"total_tokens": 100,
"prompt_tokens": 50,
"completion_tokens": 50,
},
}
start_time = datetime.datetime.now(timezone.utc)
end_time = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
)
# Parse the metadata JSON string
metadata_json = payload.get("metadata")
assert metadata_json is not None, "metadata should not be None"
metadata = json.loads(metadata_json)
# Verify overhead is stored directly in metadata
assert metadata.get("litellm_overhead_time_ms") == test_overhead_ms, (
f"Expected overhead '{test_overhead_ms}', got '{metadata.get('litellm_overhead_time_ms')}'"
)
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_strips_null_bytes_from_request_tags():
"""Regression for PostgreSQL 22P05: NUL bytes must be stripped from request_tags."""
kwargs = {
"model": "gpt-3.5-turbo",
"litellm_params": {
"metadata": {
"user_api_key": "sk-test-key",
"tags": ["clean-tag", "bad\x00tag"],
}
},
}
start_time = datetime.datetime.now(timezone.utc)
end_time = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(
kwargs=kwargs,
response_obj={},
start_time=start_time,
end_time=end_time,
)
request_tags = payload.get("request_tags")
assert request_tags is not None
assert "\\u0000" not in request_tags
assert json.loads(request_tags) == ["clean-tag", "badtag"]
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_handles_missing_overhead_gracefully():
"""
Test that get_logging_payload handles missing overhead gracefully
(backward compatibility - when overhead is not present, it should not break).
"""
# Create StandardLoggingPayload WITHOUT overhead in hidden_params
standard_logging_payload = StandardLoggingPayload(
id="test-id-456",
call_type="completion",
stream=False,
response_cost=0.001,
status="success",
total_tokens=100,
prompt_tokens=50,
completion_tokens=50,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=None,
model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None),
model="gpt-3.5-turbo",
model_id="model-123",
model_group="openai",
custom_llm_provider="openai",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_alias=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_user_id=None,
user_api_key_user_email=None,
user_api_key_team_alias=None,
spend_logs_metadata=None,
requester_ip_address=None,
requester_metadata=None,
user_api_key_end_user_id=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address=None,
messages=[],
response={},
error_str=None,
model_parameters={},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.001",
litellm_overhead_time_ms=None, # No overhead
additional_headers=None,
batch_models=None,
litellm_model_name=None,
usage_object=None,
),
)
kwargs = {
"model": "gpt-3.5-turbo",
"litellm_params": {
"metadata": {
"user_api_key": "sk-test-key",
}
},
"standard_logging_object": standard_logging_payload,
}
response_obj = {
"id": "test-response-456",
"choices": [{"message": {"content": "Hello!"}}],
"usage": {
"total_tokens": 100,
"prompt_tokens": 50,
"completion_tokens": 50,
},
}
start_time = datetime.datetime.now(timezone.utc)
end_time = datetime.datetime.now(timezone.utc)
# Should not raise an exception
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
)
# Parse the metadata JSON string
metadata_json = payload.get("metadata")
assert metadata_json is not None, "metadata should not be None"
metadata = json.loads(metadata_json)
# When overhead is None, litellm_overhead_time_ms should be None or not present
assert metadata.get("litellm_overhead_time_ms") is None, (
"litellm_overhead_time_ms should be None when overhead is not provided"
)
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_spend_logs_redacts_request_and_response_when_turn_off_message_logging_enabled(
mock_should_store,
):
"""
Test that both request body and response are redacted when turn_off_message_logging is enabled.
"""
mock_should_store.return_value = True
# Test request redaction
litellm_params = {
"proxy_server_request": {
"body": {
"messages": [{"role": "user", "content": "secret message"}],
"model": "gpt-4",
}
}
}
metadata = {}
kwargs = {
"litellm_params": litellm_params,
"standard_callback_dynamic_params": {
"turn_off_message_logging": True,
},
}
request_result = _get_proxy_server_request_for_spend_logs_payload(
metadata=metadata, litellm_params=litellm_params, kwargs=kwargs
)
parsed_request = json.loads(request_result)
assert parsed_request["messages"] == [{"role": "user", "content": "redacted-by-litellm"}]
assert parsed_request["model"] == "gpt-4"
# Test response redaction - use dict response to verify redaction
response_dict = {
"id": "test-id",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "secret response"},
}
],
"model": "gpt-4",
}
payload = cast(
StandardLoggingPayload,
{"response": response_dict},
)
response_result = _get_response_for_spend_logs_payload(payload=payload, kwargs=kwargs)
# When redaction is enabled and response is a dict (not ModelResponse),
# perform_redaction redacts content in-place within the choices structure
parsed_response = json.loads(response_result)
assert parsed_response["choices"][0]["message"]["content"] == "redacted-by-litellm"
assert parsed_response["choices"][0]["message"]["role"] == "assistant"
@patch("litellm.secret_managers.main.get_secret_bool")
def test_should_store_prompts_and_responses_in_spend_logs_case_insensitive_string(
mock_get_secret_bool,
):
"""
Test that should_store_prompts_and_responses_in_spend_logs handles
case-insensitive string values for store_prompts_in_spend_logs in general_settings.
"""
# Test case-insensitive string "true" variations
for true_value in ["true", "TRUE", "True", "TrUe"]:
with patch(
"litellm.proxy.proxy_server.general_settings",
{"store_prompts_in_spend_logs": true_value},
):
mock_get_secret_bool.return_value = False # Ensure env var is False
result = should_store_prompts_and_responses_in_spend_logs()
assert result is True, f"Expected True for '{true_value}', got {result}"
# Test boolean True
with patch(
"litellm.proxy.proxy_server.general_settings",
{"store_prompts_in_spend_logs": True},
):
mock_get_secret_bool.return_value = False
result = should_store_prompts_and_responses_in_spend_logs()
assert result is True, f"Expected True for boolean True, got {result}"
# Test that non-true values fall back to environment variable
for false_value in [False, None, "false", "FALSE", "False", "anything"]:
with patch(
"litellm.proxy.proxy_server.general_settings",
{"store_prompts_in_spend_logs": false_value},
):
# When env var is True, should return True
mock_get_secret_bool.return_value = True
result = should_store_prompts_and_responses_in_spend_logs()
assert result is True, f"Expected True (from env var) for '{false_value}', got {result}"
# When env var is False, should return False
mock_get_secret_bool.return_value = False
result = should_store_prompts_and_responses_in_spend_logs()
assert result is False, f"Expected False (from env var) for '{false_value}', got {result}"
# Test when general_settings doesn't have the key at all
with patch("litellm.proxy.proxy_server.general_settings", {}):
mock_get_secret_bool.return_value = True
result = should_store_prompts_and_responses_in_spend_logs()
assert result is True, "Expected True (from env var) when key missing, got False"
mock_get_secret_bool.return_value = False
result = should_store_prompts_and_responses_in_spend_logs()
assert result is False, "Expected False (from env var) when key missing, got True"
def test_get_spend_logs_metadata_guardrail_info_fallback_from_metadata():
"""
When standard_logging_payload is None (e.g. guardrail blocks before LLM call),
guardrail_information should fall back to reading from metadata's
standard_logging_guardrail_information field.
"""
guardrail_info = [
{
"guardrail_name": "content_filter",
"guardrail_provider": "litellm",
"guardrail_mode": "pre_call",
"guardrail_status": "guardrail_intervened",
"guardrail_response": "Content blocked",
}
]
metadata = {
"user_api_key": "test-key",
"standard_logging_guardrail_information": guardrail_info,
}
result = _get_spend_logs_metadata(
metadata=metadata,
guardrail_information=None,
)
# When guardrail_information param is None, should NOT fall back
# (the caller is responsible for passing it)
assert result["guardrail_information"] is None
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_guardrail_information_redacts_all_prompt_carrying_fields_when_flag_false(
mock_should_store,
):
"""
match_details and classification are declared as structured metadata but
in-tree writers (litellm_content_filter, block_code_execution) inline
raw prompt content into them, so they leak the same way
guardrail_request/guardrail_response do. Redaction must cover all four.
"""
mock_should_store.return_value = False
guardrail_info = [
{
"guardrail_name": "demo-echo-guard",
"guardrail_status": "success",
"guardrail_request": {"messages": [{"role": "user", "content": "hi"}]},
"guardrail_response": {"evaluated_input": "hi"},
"match_details": [{"type": "pattern", "snippet": "hi", "action_taken": "log"}],
"classification": {"intent": "x", "evidence": [{"match": "hi"}]},
"guardrail_action": "NONE",
}
]
result = _sanitize_guardrail_information_for_spend_logs(guardrail_info)
assert result is not None
entry = result[0]
assert entry["guardrail_request"] == REDACTED_BY_LITELM_STRING
assert entry["guardrail_response"] == REDACTED_BY_LITELM_STRING
assert entry["match_details"] == REDACTED_BY_LITELM_STRING
assert entry["classification"] == REDACTED_BY_LITELM_STRING
assert entry["guardrail_name"] == "demo-echo-guard"
assert entry["guardrail_status"] == "success"
assert entry["guardrail_action"] == "NONE"
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_guardrail_information_redacts_prompt_fields_when_flag_false(
mock_should_store,
):
"""
LIT-4314 Issue A regression: with store_prompts_in_spend_logs=False,
guardrail_request and guardrail_response must be redacted before they
land in LiteLLM_SpendLogs.metadata, while every other field on the
entry is preserved bit-for-bit.
"""
mock_should_store.return_value = False
guardrail_info = [
{
"guardrail_name": "demo-echo-guard",
"guardrail_provider": "custom",
"guardrail_mode": "pre_call",
"guardrail_status": "success",
"guardrail_request": {
"messages": [{"role": "user", "content": "Say hi in 3 words"}],
},
"guardrail_response": {
"evaluated_input": "Say hi in 3 words",
"verdict": "allow",
},
"start_time": 1_700_000_000.0,
"end_time": 1_700_000_000.5,
"duration": 0.5,
"guardrail_id": "gd-42",
"masked_entity_count": {"EMAIL": 1},
"violation_categories": ["prompt_injection"],
"risk_score": 3.5,
"guardrail_action": "NONE",
}
]
result = _sanitize_guardrail_information_for_spend_logs(guardrail_info)
assert result is not None
assert len(result) == 1
entry = result[0]
assert entry["guardrail_request"] == REDACTED_BY_LITELM_STRING
assert entry["guardrail_response"] == REDACTED_BY_LITELM_STRING
assert entry["guardrail_name"] == "demo-echo-guard"
assert entry["guardrail_provider"] == "custom"
assert entry["guardrail_mode"] == "pre_call"
assert entry["guardrail_status"] == "success"
assert entry["start_time"] == 1_700_000_000.0
assert entry["end_time"] == 1_700_000_000.5
assert entry["duration"] == 0.5
assert entry["guardrail_id"] == "gd-42"
assert entry["masked_entity_count"] == {"EMAIL": 1}
assert entry["violation_categories"] == ["prompt_injection"]
assert entry["risk_score"] == 3.5
assert entry["guardrail_action"] == "NONE"
assert guardrail_info[0]["guardrail_request"] == {
"messages": [{"role": "user", "content": "Say hi in 3 words"}],
}
assert guardrail_info[0]["guardrail_response"] == {
"evaluated_input": "Say hi in 3 words",
"verdict": "allow",
}
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_guardrail_information_preserves_guardrail_usage_when_flag_false(
mock_should_store,
):
"""
LIT-5650 regression: provider-reported billable usage counters live in
guardrail_usage, a sibling of guardrail_response, precisely so the
default spend-log redaction cannot drop them. The response blob (which
also embeds a usage copy) must still be redacted wholesale.
"""
mock_should_store.return_value = False
guardrail_info = [
{
"guardrail_name": "bedrock-guard",
"guardrail_status": "guardrail_intervened",
"guardrail_response": {
"action": "GUARDRAIL_INTERVENED",
"outputs": [{"text": "Sorry, the model cannot answer this question."}],
"usage": {"topicPolicyUnits": 1, "contentPolicyUnits": 1},
},
"guardrail_usage": {"topicPolicyUnits": 1, "contentPolicyUnits": 1, "wordPolicyUnits": 0},
}
]
result = _sanitize_guardrail_information_for_spend_logs(guardrail_info)
assert result is not None
entry = result[0]
assert entry["guardrail_response"] == REDACTED_BY_LITELM_STRING
assert entry["guardrail_usage"] == {"topicPolicyUnits": 1, "contentPolicyUnits": 1, "wordPolicyUnits": 0}
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_guardrail_information_passthrough_when_flag_true(
mock_should_store,
):
"""
When store_prompts_in_spend_logs=True the sanitizer must be a no-op so
operators who explicitly opted in still see full guardrail payloads.
"""
mock_should_store.return_value = True
guardrail_info = [
{
"guardrail_name": "content_filter",
"guardrail_status": "success",
"guardrail_request": {"messages": [{"role": "user", "content": "hi"}]},
"guardrail_response": {"verdict": "allow"},
}
]
result = _sanitize_guardrail_information_for_spend_logs(guardrail_info)
assert result == guardrail_info
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_guardrail_information_none_passthrough(mock_should_store):
mock_should_store.return_value = False
assert _sanitize_guardrail_information_for_spend_logs(None) is None
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_guardrail_information_normalizes_bare_dict_input(mock_should_store):
"""
Regression: xecguard (xecguard.py:246) assigns a bare dict to
standard_logging_object["guardrail_information"] even though the typed
contract is Optional[List[...]]. Without defensive normalization here,
the for-loop would iterate the dict's string keys and _redact...
would TypeError on {**"guardrail_name"}, taking down the entire
spend-log write via update_database's broad except.
"""
mock_should_store.return_value = False
bare_dict_entry = {
"guardrail_name": "xecguard",
"guardrail_status": "success",
"guardrail_response": {"decision": "SAFE", "raw_prompt": "hi"},
"start_time": 1.0,
"end_time": 2.0,
"duration": 1.0,
}
result = _sanitize_guardrail_information_for_spend_logs(bare_dict_entry)
assert result is not None
assert isinstance(result, list)
assert len(result) == 1
entry = result[0]
assert entry["guardrail_response"] == REDACTED_BY_LITELM_STRING
assert entry["guardrail_name"] == "xecguard"
assert entry["guardrail_status"] == "success"
assert entry["start_time"] == 1.0
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_guardrail_information_drops_non_dict_items_in_list(mock_should_store):
"""
A stray non-dict item in the list (e.g. from a buggy caller that
accidentally appends a string) should be silently skipped instead of
crashing the spend-log write.
"""
mock_should_store.return_value = False
mixed_input = [
{"guardrail_name": "x", "guardrail_response": {"leak": "hi"}},
"not-a-dict",
None,
]
result = _sanitize_guardrail_information_for_spend_logs(mixed_input)
assert result == [{"guardrail_name": "x", "guardrail_response": REDACTED_BY_LITELM_STRING}]
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_guardrail_information_preserves_absent_prompt_fields(mock_should_store):
"""
Entries that never carried guardrail_request or guardrail_response must
not gain those keys after sanitization; consumers keying on presence
(`"guardrail_request" in entry`) would otherwise flip from absent to
the sentinel string.
"""
mock_should_store.return_value = False
guardrail_info = [
{
"guardrail_name": "demo-echo-guard",
"guardrail_status": "success",
"guardrail_response": {"verdict": "allow", "evaluated_input": "hi"},
}
]
result = _sanitize_guardrail_information_for_spend_logs(guardrail_info)
assert result is not None
entry = result[0]
assert "guardrail_request" not in entry
assert entry["guardrail_response"] == REDACTED_BY_LITELM_STRING
assert entry["guardrail_name"] == "demo-echo-guard"
assert entry["guardrail_status"] == "success"
@patch("litellm.proxy.proxy_server.master_key", "sk-master")
@patch(
"litellm.proxy.proxy_server.general_settings",
{"store_prompts_in_spend_logs": False},
)
def test_get_logging_payload_redacts_guardrail_prompt_fields_when_flag_false():
"""
End-to-end wire-in check: get_logging_payload -> _get_spend_logs_metadata
-> sanitizer. Without the wire-in at line 139, the raw guardrail_response
lands in payload["metadata"] verbatim.
"""
guardrail_info = [
{
"guardrail_name": "demo-echo-guard",
"guardrail_provider": "custom",
"guardrail_status": "success",
"guardrail_request": {"messages": [{"role": "user", "content": "secret"}]},
"guardrail_response": {"evaluated_input": "secret"},
}
]
kwargs = {
"model": "gpt-4o-mini",
"litellm_call_id": "test-call-id",
"litellm_params": {
"metadata": {
"user_api_key": "test-key",
"standard_logging_guardrail_information": guardrail_info,
},
"proxy_server_request": {},
},
}
payload = get_logging_payload(
kwargs=kwargs,
response_obj={},
start_time=datetime.datetime.now(tz=timezone.utc),
end_time=datetime.datetime.now(tz=timezone.utc),
)
metadata_result = json.loads(payload["metadata"])
stored = metadata_result["guardrail_information"][0]
assert stored["guardrail_request"] == REDACTED_BY_LITELM_STRING
assert stored["guardrail_response"] == REDACTED_BY_LITELM_STRING
assert stored["guardrail_name"] == "demo-echo-guard"
assert stored["guardrail_status"] == "success"
def test_get_logging_payload_guardrail_info_when_no_standard_logging_payload():
"""
When a guardrail blocks a request before the LLM call, the standard_logging_object
is not set on request_data. In this case, get_logging_payload should still include
guardrail_information from the metadata.
This is the bug fix for: guardrail failures not showing GuardrailViewer in the UI.
"""
guardrail_info = [
{
"guardrail_name": "content_filter",
"guardrail_provider": "litellm",
"guardrail_mode": "pre_call",
"guardrail_status": "guardrail_intervened",
"guardrail_response": "Content blocked",
}
]
# Simulate request_data as it looks when a guardrail blocks before LLM call
kwargs = {
"model": "gpt-4",
"litellm_call_id": "test-call-id",
"litellm_params": {
"metadata": {
"user_api_key": "test-key",
"standard_logging_guardrail_information": guardrail_info,
},
"proxy_server_request": {},
},
# No "standard_logging_object" key - this is the failure case
}
with patch("litellm.proxy.proxy_server.master_key", "sk-master"):
with patch(
"litellm.proxy.proxy_server.general_settings",
{"store_prompts_in_spend_logs": True},
):
payload = get_logging_payload(
kwargs=kwargs,
response_obj={},
start_time=datetime.datetime.now(tz=timezone.utc),
end_time=datetime.datetime.now(tz=timezone.utc),
)
metadata_result = json.loads(payload["metadata"])
assert metadata_result["guardrail_information"] == guardrail_info
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_includes_retry_info_in_spend_logs_metadata():
"""
Test that retry info (attempted_retries, max_retries) from metadata
is included in the spend logs metadata JSON.
"""
kwargs = {
"model": "gpt-3.5-turbo",
"litellm_params": {
"metadata": {
"user_api_key": "sk-test-key",
"attempted_retries": 2,
"max_retries": 3,
}
},
"standard_logging_object": StandardLoggingPayload(
id="test-retry-123",
call_type="completion",
stream=False,
response_cost=0.001,
status="success",
total_tokens=100,
prompt_tokens=50,
completion_tokens=50,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=None,
model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None),
model="gpt-3.5-turbo",
model_id="model-123",
model_group="openai",
custom_llm_provider="openai",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_alias=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_user_id=None,
user_api_key_user_email=None,
user_api_key_team_alias=None,
spend_logs_metadata=None,
requester_ip_address=None,
requester_metadata=None,
user_api_key_end_user_id=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address=None,
messages=[],
response={},
error_str=None,
model_parameters={},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.001",
litellm_overhead_time_ms=None,
additional_headers=None,
batch_models=None,
litellm_model_name=None,
usage_object=None,
),
),
}
response_obj = {
"id": "test-response-retry",
"choices": [{"message": {"content": "Hello!"}}],
"usage": {
"total_tokens": 100,
"prompt_tokens": 50,
"completion_tokens": 50,
},
}
start_time = datetime.datetime.now(timezone.utc)
end_time = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
)
metadata = json.loads(payload["metadata"])
assert metadata.get("attempted_retries") == 2, (
f"Expected attempted_retries=2, got {metadata.get('attempted_retries')}"
)
assert metadata.get("max_retries") == 3, f"Expected max_retries=3, got {metadata.get('max_retries')}"
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_handles_missing_retry_info_gracefully():
"""
Test that retry fields are None when not present in metadata (backward compatibility).
"""
kwargs = {
"model": "gpt-3.5-turbo",
"litellm_params": {
"metadata": {
"user_api_key": "sk-test-key",
}
},
"standard_logging_object": StandardLoggingPayload(
id="test-no-retry-456",
call_type="completion",
stream=False,
response_cost=0.001,
status="success",
total_tokens=100,
prompt_tokens=50,
completion_tokens=50,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=None,
model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None),
model="gpt-3.5-turbo",
model_id="model-123",
model_group="openai",
custom_llm_provider="openai",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_alias=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_user_id=None,
user_api_key_user_email=None,
user_api_key_team_alias=None,
spend_logs_metadata=None,
requester_ip_address=None,
requester_metadata=None,
user_api_key_end_user_id=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address=None,
messages=[],
response={},
error_str=None,
model_parameters={},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.001",
litellm_overhead_time_ms=None,
additional_headers=None,
batch_models=None,
litellm_model_name=None,
usage_object=None,
),
),
}
response_obj = {
"id": "test-response-no-retry",
"choices": [{"message": {"content": "Hello!"}}],
"usage": {
"total_tokens": 100,
"prompt_tokens": 50,
"completion_tokens": 50,
},
}
start_time = datetime.datetime.now(timezone.utc)
end_time = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
)
metadata = json.loads(payload["metadata"])
assert metadata.get("attempted_retries") is None, "attempted_retries should be None when not provided"
assert metadata.get("max_retries") is None, "max_retries should be None when not provided"
def test_get_request_duration_ms_normal():
"""Test that request duration is correctly computed in milliseconds."""
start = datetime.datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
end = datetime.datetime(2025, 1, 1, 0, 0, 2, 500000, tzinfo=timezone.utc) # 2.5s later
result = _get_request_duration_ms(start, end)
assert result == 2500
def test_get_request_duration_ms_sub_millisecond():
"""Test that sub-millisecond durations are truncated to int."""
start = datetime.datetime(2025, 1, 1, 0, 0, 0, 0, tzinfo=timezone.utc)
end = datetime.datetime(2025, 1, 1, 0, 0, 0, 500, tzinfo=timezone.utc) # 0.5ms
result = _get_request_duration_ms(start, end)
assert result == 0
def test_get_request_duration_ms_zero():
"""Test that identical start and end times produce 0."""
t = datetime.datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
result = _get_request_duration_ms(t, t)
assert result == 0
def test_get_logging_payload_includes_request_duration_ms():
"""Test that get_logging_payload populates request_duration_ms."""
start_time = datetime.datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
end_time = datetime.datetime(2025, 1, 1, 0, 0, 3, tzinfo=timezone.utc) # 3s later
kwargs = {
"model": "gpt-4",
"litellm_params": {"api_base": "https://api.openai.com"},
"standard_logging_object": None,
}
response_obj = {"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}
with (
patch("litellm.proxy.proxy_server.master_key", None),
patch("litellm.proxy.proxy_server.general_settings", {}),
):
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
)
assert payload["request_duration_ms"] == 3000
class TestIsMasterKey:
"""Tests for _is_master_key handling None inputs without raising TypeError."""
def test_none_api_key_returns_false(self):
"""Regression: _is_master_key(None, 'sk-master') should return False, not raise TypeError."""
assert is_master_key(api_key=None, _master_key="sk-master-key") is False
def test_none_master_key_returns_false(self):
assert is_master_key(api_key="sk-some-key", _master_key=None) is False
def test_both_none_returns_false(self):
assert is_master_key(api_key=None, _master_key=None) is False
def test_matching_key_returns_true(self):
assert is_master_key(api_key="sk-master", _master_key="sk-master") is True
def test_non_matching_key_returns_false(self):
assert is_master_key(api_key="sk-other", _master_key="sk-master") is False
def test_master_key_hash_is_rejected(self):
"""
``_is_master_key`` must not accept ``hash_token(master_key)`` as
equivalent to the raw master key — only the raw value matches.
"""
from litellm.proxy.utils import hash_token
master = "sk-master-key-123"
hashed = hash_token(master)
assert is_master_key(api_key=hashed, _master_key=master) is False
def test_sanitize_request_body_strips_secret_fields():
"""
secret_fields contains raw HTTP headers (including Authorization Bearer
tokens). _sanitize_request_body_for_spend_logs_payload must strip it so
that sensitive credentials are never persisted in the spend-logs DB.
"""
request_body = {
"model": "gpt-4",
"messages": [{"role": "user", "content": "hi"}],
"secret_fields": {
"raw_headers": {
"authorization": "Bearer sk-GjX--WwRQmiX2cvASbKf5Q",
"content-type": "application/json",
"host": "litellm.example.com",
}
},
}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
assert "secret_fields" not in sanitized, "secret_fields must be stripped from the sanitized request body"
assert sanitized["model"] == "gpt-4"
assert sanitized["messages"] == [{"role": "user", "content": "hi"}]
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_proxy_server_request_payload_strips_nested_aws_credentials(mock_should_store: MagicMock) -> None:
mock_should_store.return_value = True
credentials: Final = {
"aws_access_key_id": "AKIA-canary",
"aws_secret_access_key": "secret-canary",
"aws_session_token": "token-canary",
"aws_web_identity_token": "wit-canary",
}
tool_parameters: Final = {"type": "object", "properties": {"aws_secret_access_key": {"type": "string"}}}
litellm_params: Final = {
"proxy_server_request": {
"body": {
"model": "bedrock-claude",
"messages": [{"role": "user", "content": "hello"}],
"fallbacks": [{"model": "bedrock-b", "aws_region_name": "us-west-2", **credentials}],
"extra_body": {"aws_role_name": "arn:aws:iam::123456789012:role/r", **credentials},
"tools": [{"type": "function", "function": {"name": "f", "parameters": tool_parameters}}],
**credentials,
}
}
}
parsed: Final = json.loads(
_get_proxy_server_request_for_spend_logs_payload(metadata={}, litellm_params=litellm_params, kwargs={})
)
assert "canary" not in json.dumps(parsed)
masked: Final = dict.fromkeys(credentials, REDACTED_BY_LITELM_STRING)
assert parsed["fallbacks"] == [{"model": "bedrock-b", "aws_region_name": "us-west-2", **masked}]
assert parsed["extra_body"] == {"aws_role_name": "arn:aws:iam::123456789012:role/r", **masked}
assert {name: parsed[name] for name in credentials} == masked
assert parsed["tools"][0]["function"]["parameters"] == tool_parameters
assert parsed["messages"] == [{"role": "user", "content": "hello"}]
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_proxy_server_request_payload_redacts_provider_credentials(mock_should_store: MagicMock) -> None:
mock_should_store.return_value = True
credentials: Final = {
"azure_password": "canary-azure-password",
"client_secret": "canary-client-secret",
"azure_ad_token": "canary-azure-ad-token",
"vertex_credentials": "canary-vertex-credentials",
"s3_secret_access_key": "canary-s3-secret",
"token": "canary-watsonx-token",
"apikey": "canary-watsonx-apikey",
"zen_api_key": "canary-zen-api-key",
"gemini_api_key": "canary-gemini-api-key",
"gigachat_access_token": "canary-gigachat-token",
"oci_key": "canary-oci-key",
}
metadata: Final = {"user_api_key": "custom-auth-raw-key", "requester_ip_address": "10.0.0.1"}
tool_parameters: Final = {"type": "object", "properties": {"client_secret": {"type": "string"}}}
litellm_params: Final = {
"proxy_server_request": {
"body": {
"model": "azure-gpt",
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 10,
"prompt_cache_key": "user-123-cache",
"vertex_credentials": {"private_key": "canary-private-key", "client_email": "[email protected]"},
"extra_headers": {"Authorization": "Bearer canary-extra-header"},
"tools": [
{"type": "function", "function": {"name": "f", "parameters": tool_parameters}},
{
"type": "mcp",
"server_url": "https://mcp.example.com",
"headers": {"Authorization": "canary-mcp"},
},
],
"fallbacks": [{"model": "azure-b", **credentials}],
"metadata": metadata,
**credentials,
}
}
}
parsed: Final = json.loads(
_get_proxy_server_request_for_spend_logs_payload(metadata={}, litellm_params=litellm_params, kwargs={})
)
assert "canary" not in json.dumps(parsed)
assert {name: parsed[name] for name in credentials} == dict.fromkeys(credentials, REDACTED_BY_LITELM_STRING)
assert parsed["vertex_credentials"] == REDACTED_BY_LITELM_STRING
assert parsed["extra_headers"] == {"Authorization": REDACTED_BY_LITELM_STRING}
assert parsed["tools"][0]["function"]["parameters"] == tool_parameters
assert parsed["tools"][1]["server_url"] == "https://mcp.example.com"
assert parsed["metadata"] == {"user_api_key": REDACTED_BY_LITELM_STRING, "requester_ip_address": "10.0.0.1"}
assert parsed["max_tokens"] == 10
assert parsed["prompt_cache_key"] == REDACTED_BY_LITELM_STRING
assert parsed["messages"] == [{"role": "user", "content": "hello"}]
def test_sanitize_response_redacts_credential_named_fields() -> None:
response: Final = {"access_token": "canary-oauth-token", "usage": {"prompt_tokens": 1}}
assert _sanitize_request_body_for_spend_logs_payload({"response": response}) == {
"response": {"access_token": REDACTED_BY_LITELM_STRING, "usage": {"prompt_tokens": 1}}
}
def test_sanitize_response_keeps_logprob_tokens() -> None:
response: Final = {
"system_fingerprint": "fp_x",
"choices": [
{
"logprobs": {
"content": [
{
"token": "sort",
"logprob": -0.1,
"bytes": [115],
"top_logprobs": [{"token": "sort", "logprob": -0.1}],
}
]
}
}
],
}
assert _sanitize_request_body_for_spend_logs_payload({"response": response}) == {
"response": {
"system_fingerprint": REDACTED_BY_LITELM_STRING,
"choices": [
{
"logprobs": {
"content": [
{
"token": "sort",
"logprob": -0.1,
"bytes": [115],
"top_logprobs": [{"token": "sort", "logprob": -0.1}],
}
]
}
}
],
}
}
def test_sanitize_request_body_keeps_key_named_tool_payload_fields() -> None:
request_body: Final = {
"model": "anthropic/claude",
"aws_secret_access_key": "AKIAEXAMPLESECRET",
"prompt_cache_key": "tenant-42-cache",
"metadata": {"user_api_key_alias": "tenant-user"},
"secret_fields": {"raw_headers": {"authorization": "Bearer secret"}},
"messages": [
{
"role": "assistant",
"content": [
{"type": "tool_use", "input": {"key": "order-123", "sort_key": "created_at"}},
],
},
{
"role": "assistant",
"tool_calls": [
{
"type": "function",
"function": {
"name": "get_order",
"arguments": {"key": "order-123", "sort_key": "created_at"},
},
}
],
},
{"role": "tool", "content": {"token_type": "bearer", "partition_key": "tenant_42"}},
{
"role": "user",
"content": [
{
"type": "tool_result",
"content": [{"token_type": "bearer", "partition_key": "tenant_42"}],
}
],
},
],
"input": [
{"type": "function_call", "arguments": {"key": "tenant-42", "access_level": "admin"}},
{
"type": "function_call_output",
"output": {"token_type": "bearer", "partition_key": "tenant_42"},
},
],
}
assert _sanitize_request_body_for_spend_logs_payload(request_body) == {
"model": "anthropic/claude",
"aws_secret_access_key": REDACTED_BY_LITELM_STRING,
"prompt_cache_key": REDACTED_BY_LITELM_STRING,
"metadata": {"user_api_key_alias": REDACTED_BY_LITELM_STRING},
"messages": [
{
"role": "assistant",
"content": [
{"type": "tool_use", "input": {"key": "order-123", "sort_key": "created_at"}},
],
},
{
"role": "assistant",
"tool_calls": [
{
"type": "function",
"function": {
"name": "get_order",
"arguments": {"key": "order-123", "sort_key": "created_at"},
},
}
],
},
{"role": "tool", "content": {"token_type": "bearer", "partition_key": "tenant_42"}},
{
"role": "user",
"content": [
{
"type": "tool_result",
"content": [{"token_type": "bearer", "partition_key": "tenant_42"}],
}
],
},
],
"input": [
{"type": "function_call", "arguments": {"key": "tenant-42", "access_level": "admin"}},
{
"type": "function_call_output",
"output": {"token_type": "bearer", "partition_key": "tenant_42"},
},
],
}
def test_sanitize_request_body_masks_credentials_beside_tool_blocks() -> None:
request_body: Final = {
"messages": [
{
"role": "assistant",
"content": [
{"type": "tool_use", "api_key": "sk-live", "input": {"key": "order-123"}},
{"type": {"nested": 1}, "input": {"api_key": "x"}},
],
}
]
}
assert _sanitize_request_body_for_spend_logs_payload(request_body) == {
"messages": [
{
"role": "assistant",
"content": [
{
"type": "tool_use",
"api_key": REDACTED_BY_LITELM_STRING,
"input": {"key": "order-123"},
},
{"type": {"nested": 1}, "input": {"api_key": REDACTED_BY_LITELM_STRING}},
],
}
]
}
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_proxy_server_request_payload_excludes_secret_fields(mock_should_store):
"""
End-to-end test: when the proxy_server_request body contains
secret_fields (as it did before the body-snapshot fix), the spend-log
serialization must still strip them via the sanitizer fallback.
"""
mock_should_store.return_value = True
litellm_params = {
"proxy_server_request": {
"body": {
"model": "gpt-4",
"messages": [{"role": "user", "content": "hello"}],
"secret_fields": {
"raw_headers": {
"authorization": "Bearer sk-super-secret-token",
"host": "litellm.example.com",
}
},
}
}
}
result = _get_proxy_server_request_for_spend_logs_payload(metadata={}, litellm_params=litellm_params, kwargs={})
parsed = json.loads(result)
assert "secret_fields" not in parsed, "secret_fields must never appear in the spend-log proxy_server_request column"
assert parsed["model"] == "gpt-4"
assert parsed["messages"] == [{"role": "user", "content": "hello"}]
# ---------------------------------------------------------------------------
# LIT-2992: error_information sanitization for spend logs
# ---------------------------------------------------------------------------
def test_redact_prompt_leaks_strips_input_value_python_repr():
# OpenAI-style pydantic validation error: each entry carries its own
# 'input': [...] field echoing the full conversation.
error_text = (
"OpenAIException - {'error': {'message': \"1 validation error:\\n "
"{'type': 'string_type', 'loc': ('body', 'input', 'str'), "
"'msg': 'Input should be a valid string', "
"'input': [{'role': 'user', 'content': 'super-secret-prompt'}]}"
'"}}'
)
redacted = _redact_prompt_leaks_in_error_string(error_text)
assert "super-secret-prompt" not in redacted
assert REDACTED_BY_LITELM_STRING in redacted
# Surrounding context (error class, msg, loc) is preserved.
assert "string_type" in redacted
assert "Input should be a valid string" in redacted
def test_redact_prompt_leaks_strips_input_value_json():
error_text = '{"error":{"message":"validation failed","input":[{"role":"user","content":"top-secret-content"}]}}'
redacted = _redact_prompt_leaks_in_error_string(error_text)
assert "top-secret-content" not in redacted
assert REDACTED_BY_LITELM_STRING in redacted
def test_redact_prompt_leaks_strips_messages_value():
error_text = '{"error":{"messages":[{"role":"user","content":"leak"}]}}'
redacted = _redact_prompt_leaks_in_error_string(error_text)
assert "leak" not in redacted
assert REDACTED_BY_LITELM_STRING in redacted
def test_redact_prompt_leaks_preserves_prose_mentions():
# The word "input" / "messages" in prose (not as a key) must not be
# redacted — only quoted-key matches.
error_text = "Rate limit exceeded. Reduce input size and retry."
assert _redact_prompt_leaks_in_error_string(error_text) == error_text
def test_redact_prompt_leaks_empty_string():
assert _redact_prompt_leaks_in_error_string("") == ""
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_error_information_redacts_when_not_storing_prompts(
mock_should_store,
):
mock_should_store.return_value = False
error_info = {
"error_code": "429",
"error_class": "RateLimitError",
"llm_provider": "openai",
"traceback": "Traceback (most recent call last):\n File ...",
"error_message": (
'OpenAIException - {"error":{"message":"validation failed",'
'"input":[{"role":"user","content":"leaked-prompt-content"}]}}'
),
}
sanitized = sanitize_error_information_for_spend_logs(error_info)
assert sanitized is not None
assert "leaked-prompt-content" not in sanitized["error_message"]
assert REDACTED_BY_LITELM_STRING in sanitized["error_message"]
# Non-leaking fields untouched.
assert sanitized["error_code"] == "429"
assert sanitized["error_class"] == "RateLimitError"
assert sanitized["llm_provider"] == "openai"
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_error_information_skips_redaction_when_storing_prompts(
mock_should_store,
):
mock_should_store.return_value = True
error_info = {
"error_code": "429",
"error_class": "RateLimitError",
"llm_provider": "openai",
"traceback": "",
"error_message": ('OpenAIException - {"error":{"input":[{"role":"user","content":"kept"}]}}'),
}
sanitized = sanitize_error_information_for_spend_logs(error_info)
assert sanitized is not None
# User opted in via store_prompts_in_spend_logs — no key-level redaction.
assert "kept" in sanitized["error_message"]
assert REDACTED_BY_LITELM_STRING not in sanitized["error_message"]
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_error_information_caps_size_regardless_of_prompt_flag(
mock_should_store,
):
# The DB-storage cap must apply even when prompt storage is enabled, so a
# provider error that echoes a multi-MB body can't blow up a single row.
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
mock_should_store.return_value = True
huge_error = "x" * (MAX_STRING_LENGTH_PROMPT_IN_DB * 10)
error_info = {
"error_code": "500",
"error_class": "InternalServerError",
"llm_provider": "openai",
"traceback": "x" * (MAX_STRING_LENGTH_PROMPT_IN_DB * 10),
"error_message": huge_error,
}
sanitized = sanitize_error_information_for_spend_logs(error_info)
assert sanitized is not None
assert len(sanitized["error_message"]) < len(huge_error)
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in sanitized["error_message"]
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in sanitized["traceback"]
def test_sanitize_error_information_none_passthrough():
assert sanitize_error_information_for_spend_logs(None) is None
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_error_information_reproduces_lit_2992(mock_should_store):
# Mirrors the reproduced row body from LIT-2992 — a RateLimitError whose
# message embeds 178 pydantic validation errors, each carrying a full
# 'input': [...] echo of the conversation.
mock_should_store.return_value = False
huge_conversation_blob = "user-conversation-history-" * 5000
validation_entries = []
for _ in range(50):
validation_entries.append(
"{'type': 'string_type', 'loc': ('body', 'input', 'str'), "
"'msg': 'Input should be a valid string', "
f"'input': [{{'role': 'user', 'content': '{huge_conversation_blob}'}}]}}"
)
error_message = (
"litellm.RateLimitError: RateLimitError: OpenAIException - "
'{"error":{"message":"' + "\\n ".join(validation_entries) + '"}}'
)
error_info = {
"error_code": "429",
"error_class": "RateLimitError",
"llm_provider": "openai",
"traceback": "",
"error_message": error_message,
}
sanitized = sanitize_error_information_for_spend_logs(error_info)
assert sanitized is not None
assert huge_conversation_blob not in sanitized["error_message"]
# The structural fields that aid debugging remain.
assert "RateLimitError" in sanitized["error_message"]
assert "string_type" in sanitized["error_message"]
def test_redact_prompt_leaks_handles_nested_multimodal_content():
# Multi-modal payload: 'content' is itself a list. The depth-1 regex
# would stop at the inner '['; the parser-based scanner must walk
# through balanced nested brackets.
error_text = '{"error":{"messages":[{"role":"user","content":[{"type":"text","text":"top-secret-multimodal"}]}]}}'
redacted = _redact_prompt_leaks_in_error_string(error_text)
assert "top-secret-multimodal" not in redacted
assert REDACTED_BY_LITELM_STRING in redacted
def test_redact_prompt_leaks_handles_bracket_in_prompt_text():
# Prompt text contains a literal '[' — the depth-1 regex would close
# the outer ']' prematurely. The parser must respect string quoting.
error_text = '{"error":{"input":[{"role":"user","content":"secret[123 still secret"}]}}'
redacted = _redact_prompt_leaks_in_error_string(error_text)
assert "secret[123" not in redacted
assert "still secret" not in redacted
assert REDACTED_BY_LITELM_STRING in redacted
def test_redact_prompt_leaks_handles_escaped_quote_in_prompt_text():
# Prompt with an escaped quote inside a JSON string must not break
# value scanning.
error_text = '{"error":{"input":[{"role":"user","content":"she said \\"hi[\\""}]}}'
redacted = _redact_prompt_leaks_in_error_string(error_text)
assert "she said" not in redacted
assert REDACTED_BY_LITELM_STRING in redacted
def test_redact_prompt_leaks_handles_nested_input_python_repr():
# Python dict-repr with nested list inside 'input' — single quotes.
error_text = (
"validation error: {'input': [{'role': 'user', 'content': [{'type': 'text', 'text': 'leaked-nested-text'}]}]}"
)
redacted = _redact_prompt_leaks_in_error_string(error_text)
assert "leaked-nested-text" not in redacted
assert REDACTED_BY_LITELM_STRING in redacted
def test_redact_prompt_leaks_handles_unterminated_value():
# If a value never closes (malformed error string), redact through to
# the end rather than leaving the prompt content reachable.
error_text = '{"input":[{"role":"user","content":"never-closes-leaked'
redacted = _redact_prompt_leaks_in_error_string(error_text)
assert "never-closes-leaked" not in redacted
assert REDACTED_BY_LITELM_STRING in redacted
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_error_information_redacts_traceback_when_not_storing_prompts(
mock_should_store,
):
# If a Python exception bubbles up with the request body embedded in
# its repr (e.g. ValueError(f"bad request: {body}")), the traceback
# column would carry the prompt unredacted. Verify the redaction
# covers the traceback field too.
mock_should_store.return_value = False
error_info = {
"error_code": "500",
"error_class": "ValueError",
"llm_provider": "",
"traceback": (
'Traceback (most recent call last):\n File "x.py", line 1, in <module>\n'
' raise ValueError({"input":[{"role":"user","content":"tb-leaked-prompt"}]})\n'
"ValueError: invalid request"
),
"error_message": "invalid request",
}
sanitized = sanitize_error_information_for_spend_logs(error_info)
assert sanitized is not None
assert "tb-leaked-prompt" not in sanitized["traceback"]
assert REDACTED_BY_LITELM_STRING in sanitized["traceback"]
# Surrounding traceback frames remain so the error stays debuggable.
assert "Traceback (most recent call last):" in sanitized["traceback"]
assert "ValueError: invalid request" in sanitized["traceback"]
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_error_information_skips_traceback_redaction_when_storing_prompts(
mock_should_store,
):
mock_should_store.return_value = True
error_info = {
"error_code": "500",
"error_class": "ValueError",
"llm_provider": "",
"traceback": ('raise ValueError({"input":[{"role":"user","content":"tb-kept"}]})'),
"error_message": "invalid request",
}
sanitized = sanitize_error_information_for_spend_logs(error_info)
assert sanitized is not None
assert "tb-kept" in sanitized["traceback"]
assert REDACTED_BY_LITELM_STRING not in sanitized["traceback"]
def test_redact_prompt_leaks_strips_prompt_key_completions_payload():
# /v1/completions echoes the user input under the top-level 'prompt' key
# rather than 'messages'. Without 'prompt' coverage the body would survive
# the redactor when store_prompts_in_spend_logs is False.
error_text = '{"error":{"message":"validation failed","prompt":"super-secret-completion-text"}}'
redacted = _redact_prompt_leaks_in_error_string(error_text)
assert "super-secret-completion-text" not in redacted
assert REDACTED_BY_LITELM_STRING in redacted
def test_redact_prompt_leaks_strips_prompt_key_python_repr():
error_text = "{'model': 'gpt-3.5-turbo-instruct', 'prompt': 'leaked-completion-prompt-body'}"
redacted = _redact_prompt_leaks_in_error_string(error_text)
assert "leaked-completion-prompt-body" not in redacted
assert REDACTED_BY_LITELM_STRING in redacted
def test_redact_prompt_leaks_preserves_prompt_substring_keys():
# 'prompt_tokens' / 'prompt_token_count' / etc. are not the leak key —
# the matcher requires a closing quote before ':' so substrings shouldn't
# trigger redaction.
error_text = '{"usage":{"prompt_tokens":42,"completion_tokens":7}}'
assert _redact_prompt_leaks_in_error_string(error_text) == error_text
def test_redact_prompt_leaks_strips_pydantic_input_value_list():
# Pydantic v2 validation error format — the offending value is rendered
# as a Python repr after `input_value=`. The full repr can carry the
# entire request body and must be redacted under the same gate as the
# quoted-key form.
error_text = (
"1 validation error for ChatCompletionRequest\n"
"messages\n"
" Input should be a valid list "
"[type=list_type, input_value=['secret-pydantic-prompt'], input_type=str]"
)
redacted = _redact_prompt_leaks_in_error_string(error_text)
assert "secret-pydantic-prompt" not in redacted
assert REDACTED_BY_LITELM_STRING in redacted
# Surrounding pydantic context is preserved so the error stays debuggable.
assert "list_type" in redacted
assert "input_type=str" in redacted
def test_redact_prompt_leaks_strips_pydantic_input_value_dict():
error_text = "[type=dict_type, input_value={'role': 'user', 'content': 'leaked-dict-content'}, input_type=dict]"
redacted = _redact_prompt_leaks_in_error_string(error_text)
assert "leaked-dict-content" not in redacted
assert REDACTED_BY_LITELM_STRING in redacted
assert "input_type=dict" in redacted
def test_redact_prompt_leaks_strips_pydantic_input_value_quoted_string():
error_text = "[type=string_type, input_value='leaked-string-value', input_type=str]"
redacted = _redact_prompt_leaks_in_error_string(error_text)
assert "leaked-string-value" not in redacted
assert REDACTED_BY_LITELM_STRING in redacted
def test_redact_prompt_leaks_pydantic_input_value_scalar_left_intact():
# Bare numeric / bool / None scalars in input_value are not prompt
# carriers; leaving them untouched keeps the validation error readable.
error_text = "[type=int_type, input_value=42, input_type=int]"
assert _redact_prompt_leaks_in_error_string(error_text) == error_text
def test_redact_prompt_leaks_input_value_substring_does_not_match():
# Word-boundary anchoring on `input_value` so similarly named keys (e.g.
# `my_input_value=` from another stack frame) are not mis-redacted.
error_text = "frame: my_input_value=42 elsewhere"
assert _redact_prompt_leaks_in_error_string(error_text) == error_text
def test_redact_prompt_leaks_combined_quoted_key_and_pydantic_assignment():
# A real OpenAI/pydantic error often carries BOTH forms in the same
# string — quoted JSON 'input' echoed once, then pydantic 'input_value='
# echoed per validation entry. Both must be redacted in one pass.
error_text = (
'{"error":{"input":[{"role":"user","content":"leak-via-json"}]}} '
"[type=list_type, input_value=['leak-via-pydantic'], input_type=str]"
)
redacted = _redact_prompt_leaks_in_error_string(error_text)
assert "leak-via-json" not in redacted
assert "leak-via-pydantic" not in redacted
assert redacted.count(REDACTED_BY_LITELM_STRING) >= 2
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_error_information_redacts_pydantic_assignment_form(
mock_should_store,
):
# End-to-end: a pydantic-style error that lands in error_message must be
# redacted under the spend-log path, not just the regex-level helper.
mock_should_store.return_value = False
error_info = {
"error_code": "422",
"error_class": "ValidationError",
"llm_provider": "openai",
"traceback": "",
"error_message": (
"1 validation error for ChatCompletionRequest\n"
"messages\n"
" Field required "
"[type=missing, input_value={'prompt': 'leaked-via-pydantic-msg'}, "
"input_type=dict]"
),
}
sanitized = sanitize_error_information_for_spend_logs(error_info)
assert sanitized is not None
assert "leaked-via-pydantic-msg" not in sanitized["error_message"]
assert REDACTED_BY_LITELM_STRING in sanitized["error_message"]
# ── _redact_logged_api_key unit tests ──────────────────────────────────────
@pytest.mark.parametrize(
("original_exception", "expected_error_message"),
[
(
ProxyModelNotFoundError(route="/chat/completions", model_name=_RAW_MODEL_WITH_PROMPT),
"/chat/completions: Invalid model name passed in. Call `/v1/models` to view available models for your key.",
),
(ValueError("provider timed out"), "provider timed out"),
],
)
def test_sanitize_error_information_persists_no_raw_model_for_an_unknown_model_rejection(
original_exception: Exception, expected_error_message: str
):
error_information: Final = StandardLoggingPayloadSetup.get_error_information(original_exception=original_exception)
sanitized: Final = sanitize_error_information_for_spend_logs(
error_information, original_exception=original_exception
)
assert sanitized is not None
assert sanitized["error_message"] == expected_error_message
assert "medical records" not in json.dumps(sanitized)
assert sanitized["error_class"] == type(original_exception).__name__
def test_redact_logged_api_key_none_returns_none():
assert _redact_logged_api_key(None) is None
def test_redact_logged_api_key_empty_string_returns_none():
assert _redact_logged_api_key("") is None
def test_redact_logged_api_key_sk_key_is_hashed():
raw = "sk-9876secret"
result = _redact_logged_api_key(raw)
assert result == hash_token(raw)
assert result is not None
assert not result.startswith("sk-")
assert len(result) == 64
def test_redact_logged_api_key_bearer_sk_equals_sk_hash():
raw = "sk-9876secret"
result_plain = _redact_logged_api_key(raw)
result_bearer = _redact_logged_api_key(f"Bearer {raw}")
assert result_bearer == result_plain
def test_redact_logged_api_key_bearer_case_insensitive():
raw = "sk-9876secret"
result_lower = _redact_logged_api_key(f"bearer {raw}")
result_upper = _redact_logged_api_key(f"BEARER {raw}")
expected = hash_token(raw)
assert result_lower == expected
assert result_upper == expected
def test_redact_logged_api_key_non_sk_raw_key_is_hashed():
raw = "anthropic-raw-key-xyz"
result = _redact_logged_api_key(raw)
assert result is not None
assert result != raw
assert len(result) == 64
assert result == hash_token(raw)
def test_redact_logged_api_key_already_valid_sha256_passes_through_with_flag():
already_hashed = hash_token("sk-some-key")
assert len(already_hashed) == 64
result = _redact_logged_api_key(already_hashed, already_redacted=True)
assert result == already_hashed
assert hash_token(already_hashed) != result # no double-hash
def test_redact_logged_api_key_sha256_without_flag_is_hashed():
already_hashed = hash_token("sk-some-key")
assert len(already_hashed) == 64
result = _redact_logged_api_key(already_hashed)
assert result is not None
assert result != already_hashed
assert len(result) == 64
assert result == hash_token(already_hashed)
def test_redact_logged_api_key_long_opaque_token_is_hashed():
raw = "x1" * 450
assert len(raw) == 900
result = _redact_logged_api_key(raw)
assert result is not None
assert result != raw
assert raw not in result
assert len(result) == 64
assert result == hash_token(raw)
def test_redact_logged_api_key_hashed_jwt_passes_through():
jwt_hash = "hashed-jwt-" + "a" * 64
result = _redact_logged_api_key(jwt_hash, already_redacted=True)
assert result == jwt_hash
def test_redact_logged_api_key_hashed_jwt_shape_without_provenance_is_hashed():
lookalike = "hashed-jwt-" + "a" * 64
result = _redact_logged_api_key(lookalike)
assert result == hash_token(lookalike)
assert result != lookalike
def test_redact_logged_api_key_hashed_jwt_trailing_newline_is_hashed():
trailing = "hashed-jwt-" + "a" * 64 + "\n"
result = _redact_logged_api_key(trailing, already_redacted=True)
assert result == hash_token(trailing)
assert result != trailing
def test_redact_logged_api_key_hashed_jwt_short_suffix_is_hashed():
short_jwt = "hashed-jwt-tooshort"
result = _redact_logged_api_key(short_jwt)
assert result is not None
assert result != short_jwt
assert len(result) == 64
assert result == hash_token(short_jwt)
def test_redact_logged_api_key_master_key_alias_passes_through():
from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
result = _redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS, already_redacted=True)
assert result == LITELLM_PROXY_MASTER_KEY_ALIAS
def test_redact_logged_api_key_master_key_alias_without_provenance_is_hashed():
from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
result = _redact_logged_api_key(LITELLM_PROXY_MASTER_KEY_ALIAS)
assert result == hash_token(LITELLM_PROXY_MASTER_KEY_ALIAS)
assert result != LITELLM_PROXY_MASTER_KEY_ALIAS
def test_get_spend_logs_metadata_keeps_master_key_alias_readable():
from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
meta = _get_spend_logs_metadata(
{
"user_api_key": LITELLM_PROXY_MASTER_KEY_ALIAS,
"user_api_key_hash": LITELLM_PROXY_MASTER_KEY_ALIAS,
}
)
assert meta["user_api_key"] == LITELLM_PROXY_MASTER_KEY_ALIAS
def test_get_spend_logs_metadata_keeps_user_agent():
"""`add_litellm_data_to_request` stamps the caller's User-Agent next to its IP, but
the spend log metadata dropped it, so an abusive client could not be identified
from the Logs page."""
meta = _get_spend_logs_metadata({"requester_ip_address": "203.0.113.9", "user_agent": "abusive-client/9.9"})
assert meta["requester_ip_address"] == "203.0.113.9"
assert meta["user_agent"] == "abusive-client/9.9"
assert _get_spend_logs_metadata(None)["user_agent"] is None
@pytest.mark.parametrize(
"metadata,expected",
(
(None, False),
({}, False),
({"tags": ["litellm-roi-estimator"]}, False),
({"litellm_roi_estimator": None}, False),
({"litellm_roi_estimator": "true"}, False),
({"litellm_roi_estimator": False}, False),
({"litellm_roi_estimator": True}, True),
),
)
def test_new_spend_logs_always_have_an_explicit_roi_estimator_marker(
metadata: dict[str, object] | None, expected: bool
) -> None:
assert _get_spend_logs_metadata(metadata)["litellm_roi_estimator"] is expected
@pytest.mark.parametrize(
"client_sent_oauth_token, custom_llm_provider, expected",
[
(True, "anthropic", True),
(True, "bedrock", False),
(True, "vertex_ai", False),
(False, "anthropic", False),
(None, "anthropic", None),
],
)
def test_get_logging_payload_records_used_client_oauth_token_for_the_selected_provider(
client_sent_oauth_token: bool | None, custom_llm_provider: str, expected: bool | None
):
"""The client's OAuth bearer is only forwarded to an Anthropic deployment, so a request that
the router sent to Bedrock or Vertex paid with the configured key and must not read true."""
request_metadata = (
{"user_agent": "claude-cli/2.1.0"}
if client_sent_oauth_token is None
else {"user_agent": "claude-cli/2.1.0", "used_client_oauth_token": client_sent_oauth_token}
)
payload = get_logging_payload(
kwargs={
"model": "claude-sonnet-5",
"custom_llm_provider": custom_llm_provider,
"litellm_params": {"metadata": request_metadata},
},
response_obj={},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert json.loads(payload["metadata"])["used_client_oauth_token"] is expected
assert _get_spend_logs_metadata(None)["used_client_oauth_token"] is None
@pytest.mark.parametrize(
"litellm_params, expected",
[
(
{"metadata": {"used_client_oauth_token": True}, "litellm_metadata": {"user_api_key_hash": "guardrail"}},
True,
),
(
{"metadata": {"used_client_oauth_token": True}, "litellm_metadata": {"used_client_oauth_token": False}},
False,
),
],
)
def test_get_logging_payload_reads_used_client_oauth_token_from_the_bucket_the_proxy_stamped(
litellm_params: dict, expected: bool
):
payload = get_logging_payload(
kwargs={"model": "claude-sonnet-5", "custom_llm_provider": "anthropic", "litellm_params": litellm_params},
response_obj={},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert json.loads(payload["metadata"])["used_client_oauth_token"] is expected
def test_redact_logged_api_key_bearer_only_returns_none():
# "bearer " with nothing after stripping is equivalent to no key
assert _redact_logged_api_key("bearer ") is None
assert _redact_logged_api_key("Bearer ") is None
assert _redact_logged_api_key("BEARER ") is None
# ── _get_spend_logs_metadata key-hash invariant tests ─────────────────────
def test_get_spend_logs_metadata_sk_key_hashed():
raw = "sk-9876secret"
meta = _get_spend_logs_metadata({"user_api_key": raw})
assert meta["user_api_key"] == hash_token(raw)
assert meta["user_api_key"] is not None
result = meta["user_api_key"]
assert result is not None
assert not result.startswith("sk-")
assert len(result) == 64
def test_get_spend_logs_metadata_bearer_sk_key_hashed_same_as_plain():
raw = "sk-9876secret"
meta_plain = _get_spend_logs_metadata({"user_api_key": raw})
meta_bearer = _get_spend_logs_metadata({"user_api_key": f"Bearer {raw}"})
assert meta_bearer["user_api_key"] == meta_plain["user_api_key"]
def test_get_spend_logs_metadata_non_sk_raw_key_hashed():
raw = "anthropic-raw-key-xyz"
meta = _get_spend_logs_metadata({"user_api_key": raw})
result = meta["user_api_key"]
assert result is not None
assert result != raw
assert len(result) == 64
def test_get_spend_logs_metadata_already_hashed_unchanged_with_provenance():
already_hashed = hash_token("sk-some-key")
meta = _get_spend_logs_metadata({"user_api_key": already_hashed, "user_api_key_hash": already_hashed})
assert meta["user_api_key"] == already_hashed
assert hash_token(already_hashed) != meta["user_api_key"] # no double-hash
def test_get_spend_logs_metadata_already_hashed_no_provenance_is_rehashed():
already_hashed = hash_token("sk-some-key")
meta = _get_spend_logs_metadata({"user_api_key": already_hashed})
assert meta["user_api_key"] != already_hashed
assert meta["user_api_key"] == hash_token(already_hashed)
def test_get_logging_payload_batch_attribution_keeps_verification_token_hash():
"""
Batch cost rebuilds metadata with the managed object's already-hashed api_key.
That hash must land in SpendLogs.api_key unchanged so Usage/CloudZero can join
LiteLLM_VerificationToken for api_key_alias and user_email. Regression: without
user_api_key_hash provenance, v1.99+ re-hashed the token and broke the join.
"""
token_hash = hash_token("sk-batch-creator-key")
kwargs = {
"model": "gpt-4o",
"call_type": "aretrieve_batch",
"litellm_params": {
"metadata": {
"user_api_key": token_hash,
"user_api_key_hash": token_hash,
"user_api_key_alias": "batch-creator",
"user_api_key_user_id": "alice",
"user_api_key_team_id": "team-1",
}
},
}
payload = get_logging_payload(
kwargs=kwargs,
response_obj={"id": "batch_123", "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["api_key"] == token_hash
assert payload["api_key"] != hash_token(token_hash)
parsed_meta = json.loads(payload["metadata"])
assert parsed_meta["user_api_key"] == token_hash
assert parsed_meta["user_api_key_alias"] == "batch-creator"
def test_get_spend_logs_metadata_provenance_bypass_requires_hash_match():
already_hashed = hash_token("sk-some-key")
different_hash = hash_token("sk-other-key")
meta = _get_spend_logs_metadata({"user_api_key": already_hashed, "user_api_key_hash": different_hash})
assert meta["user_api_key"] == hash_token(already_hashed)
def test_get_spend_logs_metadata_hashed_jwt_unchanged():
jwt_hash = "hashed-jwt-" + "b" * 64
meta = _get_spend_logs_metadata({"user_api_key": jwt_hash, "user_api_key_hash": jwt_hash})
assert meta["user_api_key"] == jwt_hash
def test_get_spend_logs_metadata_hashed_jwt_shape_without_provenance_is_hashed():
lookalike = "hashed-jwt-" + "b" * 64
meta = _get_spend_logs_metadata({"user_api_key": lookalike})
assert meta["user_api_key"] == hash_token(lookalike)
assert meta["user_api_key"] != lookalike
def test_get_spend_logs_metadata_none_key_is_none():
meta = _get_spend_logs_metadata({"user_api_key": None})
assert meta["user_api_key"] is None
# ── get_logging_payload key-hash invariant tests ───────────────────────────
def test_get_logging_payload_uses_recovered_combined_usage_on_failure():
"""A request that fails mid-stream has no usable response_obj usage, but the
streaming handler recovers the usage from the chunks already delivered and
the failure hook surfaces it as ``combined_usage_object``. The spend-log
payload must record those token counts instead of zero.
"""
from litellm.types.utils import Usage
kwargs = {
"model": "anthropic/claude-haiku-4-5",
"call_type": "acompletion",
"litellm_params": {"metadata": {"user_api_key": "sk-test"}},
"combined_usage_object": Usage(prompt_tokens=30, completion_tokens=1, total_tokens=31),
}
response_obj = Exception("MidStreamFallbackError: read timeout")
now = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now)
assert payload["prompt_tokens"] == 30
assert payload["completion_tokens"] == 1
assert payload["total_tokens"] == 31
def test_get_logging_payload_failure_without_recovered_usage_is_zero():
"""A failure with no recovered usage keeps zero token counts, so the
combined-usage override never invents tokens for ordinary failures.
"""
kwargs = {
"model": "anthropic/claude-haiku-4-5",
"call_type": "acompletion",
"litellm_params": {"metadata": {"user_api_key": "sk-test"}},
}
response_obj = Exception("BadRequestError")
now = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now)
assert payload["total_tokens"] == 0
def test_get_logging_payload_sets_litellm_call_id_for_correlation():
"""LIT-3868: a successful spend log must carry the x-litellm-call-id (the
trace id) in its metadata, distinct from request_id, which stays the
provider response id. Without this there is no way to correlate a DB row
with its trace for a successful call.
"""
provider_response_id = "chatcmpl-e6e6f3e9-c392-404e-9a71-5361c79d8470"
trace_call_id = "c6a77556-19ce-4406-b287-53f5fb4b2b55"
kwargs = {
"model": "openai/gpt-4o-mini",
"call_type": "acompletion",
"litellm_call_id": trace_call_id,
"litellm_params": {"metadata": {"user_api_key": "sk-test"}},
}
response_obj = {
"id": provider_response_id,
"usage": {"prompt_tokens": 5, "completion_tokens": 7, "total_tokens": 12},
}
now = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now)
metadata = json.loads(payload["metadata"])
assert payload["request_id"] == provider_response_id
assert metadata["litellm_call_id"] == trace_call_id
assert metadata["litellm_call_id"] != payload["request_id"]
def test_get_logging_payload_litellm_call_id_falls_back_to_litellm_params():
"""litellm_call_id may only be present in litellm_params; it must still land
in the spend log metadata so correlation works on that path too.
"""
trace_call_id = "fallback-7a1c-42d9-9f0e-2b6c5d4e3f21"
kwargs = {
"model": "openai/gpt-4o-mini",
"call_type": "acompletion",
"litellm_params": {
"litellm_call_id": trace_call_id,
"metadata": {"user_api_key": "sk-test"},
},
}
response_obj = {
"id": "chatcmpl-abc123",
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
now = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now)
assert json.loads(payload["metadata"])["litellm_call_id"] == trace_call_id
def test_get_logging_payload_litellm_call_id_when_response_has_no_id():
"""When the provider returns no id, request_id falls back to the call id, so
request_id and the metadata call id hold the same value and correlation
still resolves.
"""
trace_call_id = "noid-5b2e-4c7a-9d10-3f8a1c2b4e6d"
kwargs = {
"model": "openai/gpt-4o-mini",
"call_type": "acompletion",
"litellm_call_id": trace_call_id,
"litellm_params": {"metadata": {"user_api_key": "sk-test"}},
}
response_obj = {"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}
now = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now)
assert json.loads(payload["metadata"])["litellm_call_id"] == trace_call_id
assert payload["request_id"] == trace_call_id
def test_get_logging_payload_cache_hit_keeps_raw_litellm_call_id():
"""On a cache hit request_id is suffixed to stay unique, but the metadata
litellm_call_id stays the raw trace id so the row still points at its trace.
"""
trace_call_id = "cache-9a1c-42d9-9f0e-2b6c5d4e3f21"
kwargs = {
"model": "openai/gpt-4o-mini",
"call_type": "acompletion",
"litellm_call_id": trace_call_id,
"cache_hit": True,
"litellm_params": {"metadata": {"user_api_key": "sk-test"}},
}
response_obj = {
"id": "chatcmpl-cache-src",
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
now = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now)
assert json.loads(payload["metadata"])["litellm_call_id"] == trace_call_id
assert "_cache_hit" in payload["request_id"]
assert json.loads(payload["metadata"])["litellm_call_id"] != payload["request_id"]
@pytest.mark.parametrize("cached", [True, False])
def test_get_logging_payload_gives_a_response_served_from_the_cache_a_unique_request_id(cached: bool):
"""A parent call that returns an internal sub-call's cached response reuses the cached response's id,
and SpendLogs skips a duplicate request_id, so the row would be dropped unless its id is made unique."""
from litellm.litellm_core_utils.hidden_params import set_hidden_params
response_obj: Final = litellm.ModelResponse(
id="chatcmpl-cached-src",
model="gpt-5.6",
usage=litellm.Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2),
)
set_hidden_params(response_obj, {"cache_hit": True, "response_cost": 0.0} if cached else {})
now: Final = datetime.datetime.now(timezone.utc)
payload: Final = get_logging_payload(
kwargs={
"model": "openai/gpt-5.6",
"call_type": "aresponses",
"litellm_call_id": "parent-call-id",
"litellm_params": {"metadata": {"user_api_key": "sk-test"}},
},
response_obj=response_obj,
start_time=now,
end_time=now,
)
assert payload["request_id"].startswith("chatcmpl-cached-src_cache_hit") is cached
assert (payload["request_id"] == "chatcmpl-cached-src") is not cached
class TestSpendLogKeyRedaction:
"""Regression: plaintext API keys with Bearer prefix were stored in
SpendLogs for failed requests (LIT-4121)"""
def test_bearer_prefixed_sk_key_is_hashed(self):
raw = "Bearer sk-WLi4iRn4JmbVlTaYw12IOA"
result = _redact_logged_api_key(raw)
assert result is not None
assert not result.startswith("Bearer")
assert not result.startswith("sk-")
assert len(result) == 64
def test_bare_sk_key_is_hashed(self):
raw = "sk-WLi4iRn4JmbVlTaYw12IOA"
result = _redact_logged_api_key(raw)
assert result is not None
assert not result.startswith("sk-")
assert len(result) == 64
def test_bearer_lowercase_is_handled(self):
raw = "bearer sk-WLi4iRn4JmbVlTaYw12IOA"
result = _redact_logged_api_key(raw)
assert result is not None
assert not result.startswith("bearer")
assert not result.startswith("sk-")
assert len(result) == 64
def test_already_hashed_key_unchanged(self):
hashed = "bcfe8173f5447f10be0e7fb37aaa8b97829d5c9e0498232152f9d123456789ab"
assert _redact_logged_api_key(hashed, already_redacted=True) == hashed
def test_bearer_prefixed_non_sk_key_is_hashed(self):
raw = "Bearer some-other-token-format"
result = _redact_logged_api_key(raw)
assert result == hash_token("some-other-token-format")
assert result is not None
assert not result.startswith("Bearer")
def test_bearer_and_bare_produce_same_hash(self):
bare = "sk-WLi4iRn4JmbVlTaYw12IOA"
bearer = "Bearer sk-WLi4iRn4JmbVlTaYw12IOA"
assert _redact_logged_api_key(bare) == _redact_logged_api_key(bearer)
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_non_sk_raw_key_both_fields_hashed():
raw = "anthropic-raw-key-xyz"
kwargs = {
"model": "openai/gpt-4.1",
"messages": [{"role": "user", "content": "Hello"}],
"call_type": "acompletion",
"litellm_params": {
"metadata": {
"user_api_key": raw,
"user_api_key_user_id": "test_user",
"user_api_key_team_id": "test_team",
}
},
}
payload = get_logging_payload(
kwargs=kwargs,
response_obj=Exception("error"),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["api_key"] != raw
assert len(payload["api_key"]) == 64
parsed_meta = json.loads(payload["metadata"])
assert parsed_meta["user_api_key"] != raw
assert parsed_meta["user_api_key"] is not None
assert len(parsed_meta["user_api_key"]) == 64
def test_get_logging_payload_keeps_master_key_alias_readable():
from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
kwargs = {
"model": "openai/gpt-4.1",
"messages": [{"role": "user", "content": "Hello"}],
"call_type": "acompletion",
"litellm_params": {
"metadata": {
"user_api_key": LITELLM_PROXY_MASTER_KEY_ALIAS,
"user_api_key_hash": LITELLM_PROXY_MASTER_KEY_ALIAS,
"user_api_key_user_id": "test_user",
}
},
}
payload = get_logging_payload(
kwargs=kwargs,
response_obj=Exception("error"),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["api_key"] == LITELLM_PROXY_MASTER_KEY_ALIAS
parsed_meta = json.loads(payload["metadata"])
assert parsed_meta["user_api_key"] == LITELLM_PROXY_MASTER_KEY_ALIAS
@pytest.mark.parametrize(
"service_account",
[LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, LITTELM_CLI_SERVICE_ACCOUNT_NAME],
)
def test_get_logging_payload_keeps_internal_service_account_key_readable(service_account: str):
data = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
data={"metadata": {}},
user_api_key_dict=UserAPIKeyAuth(
api_key=service_account,
team_id=service_account,
key_alias=service_account,
team_alias=service_account,
),
_metadata_variable_name="metadata",
)
kwargs = {
"model": "openai/gpt-4.1",
"messages": [{"role": "user", "content": "Hello"}],
"call_type": "acompletion",
"litellm_params": {"metadata": data["metadata"]},
}
payload = get_logging_payload(
kwargs=kwargs,
response_obj=Exception("error"),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["api_key"] == service_account
parsed_meta = json.loads(payload["metadata"])
assert parsed_meta["user_api_key"] == service_account
assert parsed_meta["user_api_key_alias"] == service_account
def test_redact_logged_api_key_service_account_name_without_provenance_is_hashed():
result = _redact_logged_api_key(LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME)
assert result == hash_token(LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME)
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_hashes_bearer_prefixed_api_key():
"""Regression for LIT-4121: failed-request spend logs stored plaintext
'Bearer sk-...' in both the api_key column and metadata.user_api_key"""
raw_key = "Bearer sk-WLi4iRn4JmbVlTaYw12IOA"
kwargs = {
"model": "openai/gpt-4.1",
"call_type": "acompletion",
"litellm_params": {
"metadata": {
"user_api_key": raw_key,
"user_api_key_user_id": "test_user",
"user_api_key_team_id": "test_team",
"status": "failure",
}
},
}
payload = get_logging_payload(
kwargs=kwargs,
response_obj=Exception("model error"),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert not payload["api_key"].startswith("Bearer"), (
f"api_key column contains plaintext Bearer key: {payload['api_key']}"
)
assert not payload["api_key"].startswith("sk-"), f"api_key column contains unhashed key: {payload['api_key']}"
metadata_dict = json.loads(payload["metadata"])
assert not metadata_dict["user_api_key"].startswith("Bearer"), (
f"metadata user_api_key contains plaintext Bearer key: {metadata_dict['user_api_key']}"
)
assert not metadata_dict["user_api_key"].startswith("sk-"), (
f"metadata user_api_key contains unhashed key: {metadata_dict['user_api_key']}"
)
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_sanitize_guardrail_information_preserves_headroom_compression_token_stats(
mock_should_store,
):
"""
Headroom's compression stats live in guardrail_response, which is redacted
by default. Purely numeric token stats carry no prompt content and must
survive so daily spend aggregation can count compression_saved_tokens;
everything else in guardrail_response stays redacted.
"""
mock_should_store.return_value = False
guardrail_info = [
{
"guardrail_name": "headroom-compressor",
"guardrail_provider": "headroom",
"guardrail_status": "success",
"guardrail_response": {
"tokens_before": 1000,
"tokens_after": 400,
"tokens_saved": 600,
"compression_ratio": 0.4,
"transforms_applied": ["dedupe_messages"],
},
},
{
"guardrail_name": "echo-guard",
"guardrail_status": "success",
"guardrail_response": {
"evaluated_input": "secret prompt",
"tokens_saved": "prompt text hiding in a stat key",
},
},
]
result = _sanitize_guardrail_information_for_spend_logs(guardrail_info)
assert result is not None
assert result[0]["guardrail_response"] == {
"tokens_before": 1000,
"tokens_after": 400,
"tokens_saved": 600,
"compression_ratio": 0.4,
}
assert result[1]["guardrail_response"] == REDACTED_BY_LITELM_STRING
@pytest.mark.asyncio
async def test_compression_savings_survive_to_spend_log_payload_metadata(monkeypatch):
"""
End-to-end seam test for the native compression write path on
/v1/messages: the pre-call deployment hook records savings into the call
kwargs' litellm_metadata, the logging object captures it via
update_from_kwargs (exactly as llm_http_handler does for
anthropic_messages), and get_logging_payload lands it in
SpendLogsPayload.metadata JSON under ``compression_savings``.
"""
from litellm.integrations.compression_interception.handler import (
CompressionInterceptionLogger,
)
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.types.utils import CallTypes
monkeypatch.setattr(
"litellm.integrations.compression_interception.handler.compress",
lambda **kwargs: {
"messages": [{"role": "user", "content": "compressed"}],
"original_tokens": 12000,
"compressed_tokens": 5000,
"cache": {"auth.py": "content"},
"tools": [],
},
)
logger = CompressionInterceptionLogger()
call_kwargs = {
"model": "claude-sonnet-5",
"messages": [{"role": "user", "content": "big context"}],
"max_tokens": 512,
"litellm_call_id": "test-compression-call-id",
"litellm_metadata": {
"user_api_key": "bc46df66218d24bc910f7c95ef9d861c706d22a191f9e7378f8a51f54146474f",
"user_api_key_user_id": "u1",
"user_api_key_team_id": "t1",
},
}
hooked = await logger.async_pre_call_deployment_hook(kwargs=call_kwargs, call_type=CallTypes.anthropic_messages)
assert hooked is not None
start_time = datetime.datetime.now(timezone.utc)
logging_obj = Logging(
model="claude-sonnet-5",
messages=hooked["messages"],
stream=False,
call_type="anthropic_messages",
start_time=start_time,
litellm_call_id="test-compression-call-id",
function_id="test",
)
logging_obj.update_from_kwargs(
kwargs=hooked,
model="claude-sonnet-5",
optional_params={"max_tokens": 512},
litellm_params={
"preset_cache_key": None,
"stream_response": {},
"model_info": hooked.get("model_info"),
},
custom_llm_provider="anthropic",
)
end_time = datetime.datetime.now(timezone.utc)
logging_obj.model_call_details["completion_start_time"] = end_time
payload = get_logging_payload(
kwargs=logging_obj.model_call_details,
response_obj={
"id": "msg_test",
"usage": {"prompt_tokens": 5000, "completion_tokens": 10, "total_tokens": 5010},
},
start_time=start_time,
end_time=end_time,
)
payload_metadata = json.loads(payload["metadata"])
assert payload_metadata["compression_savings"] == {
"tokens_before": 12000,
"tokens_after": 5000,
"tokens_saved": 7000,
"source": "compression_interception",
}
def test_no_routing_decision_key_defaults_to_none_in_spend_log_metadata():
payload = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
response_obj=litellm.ModelResponse(id="chatcmpl-no-routing-decision", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["routing_decision"] is None
@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"])
def test_internal_call_origin_survives_into_spend_log_metadata(bucket):
"""The origin is only useful if it reaches the row the Logs UI reads.
_get_spend_logs_metadata projects onto SpendLogsMetadata.__annotations__, so an
undeclared key is dropped silently. Both buckets are covered because the resolver
returns litellm_metadata when present and metadata otherwise, and the classifier
sub-call populates whichever the parent route used.
"""
payload = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_params": {
bucket: {
"user_api_key": "test-key",
"internal_call_origin": "autorouter_classifier",
}
},
},
response_obj=litellm.ModelResponse(id="chatcmpl-classifier", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["internal_call_origin"] == "autorouter_classifier"
def test_user_traffic_carries_no_internal_call_origin():
"""The negative class the badge depends on: an ordinary request must be
distinguishable from a classifier call, not merely unlabelled by accident."""
payload = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
response_obj=litellm.ModelResponse(id="chatcmpl-user-traffic", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["internal_call_origin"] is None
def _spend_log_for_call_type(
call_type: str,
internal_call_origin: str | None = None,
background: bool | None = None,
managed_read: bool = False,
) -> dict:
from litellm.types.llms.openai import ResponsesAPIResponse
return cast(
dict,
get_logging_payload(
kwargs={
"model": "gpt-4o",
"call_type": call_type,
"response_cost": 0.0,
"litellm_params": {
"metadata": {
"user_api_key": "test-key",
"internal_call_origin": internal_call_origin,
**({"_managed_background_response": True} if managed_read else {}),
}
},
},
response_obj=ResponsesAPIResponse(
id="resp_lit5602",
created_at=1234567890,
model="gpt-4o",
output=[],
usage={"input_tokens": 4000, "output_tokens": 2000, "total_tokens": 6000},
background=background,
),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
),
)
def test_spend_log_for_response_retrieval_does_not_replay_the_created_responses_tokens():
"""A retrieved response carries the usage of the call that created it, so counting it again
bills the same tokens twice. Regression test for LIT-5602."""
payload = _spend_log_for_call_type("aget_responses")
assert payload["prompt_tokens"] == 0
assert payload["completion_tokens"] == 0
assert payload["total_tokens"] == 0
assert payload["spend"] == 0.0
def test_spend_log_for_background_response_cost_poll_counts_tokens():
"""The poller's read is where a background job's usage first shows up, so dropping it there
leaves the job unbilled forever."""
payload = _spend_log_for_call_type("aget_responses", internal_call_origin="background_response_cost_poll")
assert payload["total_tokens"] == 6000
def test_spend_log_for_background_response_retrieval_counts_tokens():
"""A background create answers queued carrying no usage, so its retrieval is the first and only
place the job's tokens are ever visible. Zeroing that read bills the whole job nothing on any
proxy that is not running the enterprise cost poller."""
payload = _spend_log_for_call_type("aget_responses", background=True)
assert payload["total_tokens"] == 6000
def test_spend_log_for_managed_background_response_read_counts_nothing():
"""A proxy that tracks a background response bills it once through the cost poll, so a client
read the managed files hook marked must not count the job's tokens a second time."""
payload = _spend_log_for_call_type("aget_responses", background=True, managed_read=True)
assert payload["total_tokens"] == 0
assert payload["spend"] == 0.0
def test_spend_log_for_foreground_response_retrieval_still_counts_nothing():
"""Guards the test above against a blanket exemption: an explicit background=false read was
already billed by its create and must stay at zero."""
payload = _spend_log_for_call_type("aget_responses", background=False)
assert payload["total_tokens"] == 0
def test_spend_log_for_response_creation_still_counts_tokens():
"""Guards the test above: the same response object must still be counted on the create path."""
payload = _spend_log_for_call_type("aresponses")
assert payload["total_tokens"] == 6000
REDACTED_RESPONSE_PLACEHOLDER: Final = {"text": "redacted-by-litellm"}
CONSTANT_ID_FROM_HASHED_PLACEHOLDER: Final = "00fcbef15a3b0097e14b0ca016ed30a0"
@pytest.mark.parametrize("call_type", ["aretrieve_batch", "acreate_file"])
def test_get_spend_logs_id_stays_unique_when_the_response_is_a_redaction_placeholder(call_type):
"""request_id is the LiteLLM_SpendLogs primary key and the flush inserts with
skip_duplicates, so two calls must never derive the same id from identical response
content. Message redaction replaces every body it cannot redact with one fixed
placeholder, which is what a batch and a file body both become, so hashing the
response collapsed all of them onto a single id and silently dropped every row
after the first."""
suffix = "_batch_cost" if call_type == "aretrieve_batch" else ""
first = get_spend_logs_id(call_type, dict(REDACTED_RESPONSE_PLACEHOLDER), {"litellm_call_id": "call-id-1"})
second = get_spend_logs_id(call_type, dict(REDACTED_RESPONSE_PLACEHOLDER), {"litellm_call_id": "call-id-2"})
assert first == f"call-id-1{suffix}"
assert second == f"call-id-2{suffix}"
assert first != second
assert first != CONSTANT_ID_FROM_HASHED_PLACEHOLDER
assert second != CONSTANT_ID_FROM_HASHED_PLACEHOLDER
@pytest.mark.parametrize("call_type", ["aretrieve_batch", "acreate_file"])
def test_get_spend_logs_id_prefers_the_response_id_for_batch_and_file_calls(call_type):
"""A batch or file response that survives redaction carries its own id, so the row
keys off that rather than the per-call id."""
expected = "batch_abc123_batch_cost" if call_type == "aretrieve_batch" else "batch_abc123"
assert get_spend_logs_id(call_type, {"id": "batch_abc123"}, {"litellm_call_id": "call-id-1"}) == expected
def test_get_logging_payload_gives_redacted_batch_and_file_rows_distinct_request_ids():
"""End to end at the payload level: a batch retrieve and a file create whose bodies
were both flattened to the same redaction placeholder must still produce two
insertable rows, each carrying its own spend."""
payloads = [
get_logging_payload(
kwargs={
"call_type": call_type,
"model": model,
"litellm_call_id": call_id,
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
response_obj=dict(REDACTED_RESPONSE_PLACEHOLDER),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
for call_type, model, call_id in (
("aretrieve_batch", "global.anthropic.claude-haiku-4-5-20251001-v1:0", "call-id-batch"),
("acreate_file", "vertex_ai/gemini-2.5-flash", "call-id-file"),
)
]
request_ids = [payload["request_id"] for payload in payloads]
assert request_ids == ["call-id-batch_batch_cost", "call-id-file"]
assert len(set(request_ids)) == len(request_ids)
assert CONSTANT_ID_FROM_HASHED_PLACEHOLDER not in request_ids
@pytest.mark.parametrize("call_type", ["aretrieve_batch", "acreate_file"])
def test_get_spend_logs_id_keys_off_batch_identity_when_the_body_was_redacted(call_type):
"""Retrieving one batch twice must produce one row, not two. Redaction strips the id
off the response body, so the identity has to come from the standard logging payload,
which is built from the unredacted response and keeps it. Falling through to the
per-call id here would write a second row carrying the same batch's full cost and
overstate spend by a multiple of how often the caller polled."""
standard_logging_object = {"id": "batch_abc123"}
first = get_spend_logs_id(
call_type,
dict(REDACTED_RESPONSE_PLACEHOLDER),
{"litellm_call_id": "call-id-1", "standard_logging_object": standard_logging_object},
)
second = get_spend_logs_id(
call_type,
dict(REDACTED_RESPONSE_PLACEHOLDER),
{"litellm_call_id": "call-id-2", "standard_logging_object": standard_logging_object},
)
expected = "batch_abc123_batch_cost" if call_type == "aretrieve_batch" else "batch_abc123"
assert first == second == expected
assert first != CONSTANT_ID_FROM_HASHED_PLACEHOLDER
def test_get_spend_logs_id_separates_distinct_batches_whose_bodies_were_both_redacted():
"""The flip side of idempotency: two different batches must not share a row just
because redaction flattened both bodies to the same placeholder."""
ids = [
get_spend_logs_id(
"aretrieve_batch",
dict(REDACTED_RESPONSE_PLACEHOLDER),
{"litellm_call_id": f"call-id-{index}", "standard_logging_object": {"id": batch_id}},
)
for index, batch_id in enumerate(("batch_first", "batch_second"))
]
assert ids == ["batch_first_batch_cost", "batch_second_batch_cost"]
def test_a_background_response_billing_event_writes_a_row_apart_from_the_read_that_fetched_it():
"""The one billing event of a background response logs the body a read just fetched, so both
carry the same response id. Spend logs keep one row per request id, so the billing event must
key its row apart from the read's, or the charge reaches the key and no row ever shows it."""
response: Final = {"id": "resp_shared", "object": "response", "status": "completed", "background": True}
read_metadata: Final = {"user_api_key": "test-key", MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: True}
billing_metadata: Final = {
"user_api_key": "test-key",
INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
}
request_ids: Final = tuple(
get_logging_payload(
kwargs={
"call_type": "aget_responses",
"model": "gpt-5.6",
"litellm_call_id": call_id,
"litellm_params": {"metadata": metadata},
},
response_obj=dict(response),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)["request_id"]
for call_id, metadata in (("call-read", read_metadata), ("call-bill", billing_metadata))
)
assert request_ids == ("resp_shared", "resp_shared_background_cost")
@pytest.mark.parametrize("forged_marker", [True, BackgroundResponseSettlement.SETTLES.value])
def test_a_request_body_cannot_move_its_spend_row_onto_a_background_settlement_id(forged_marker):
"""A failed call logs its raw request body as kwargs, so a body field named like the settlement
marker must not give the failure row the id a background response's billing row owns."""
request_body: Final = {
"model": "gpt-5.4-mini",
"input": "hi",
SETTLES_BACKGROUND_RESPONSE_KWARG: forged_marker,
"litellm_call_id": "call-failed",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
}
failed_at: Final = datetime.datetime(2026, 1, 1, tzinfo=timezone.utc)
row: Final = get_logging_payload(
kwargs=request_body,
response_obj=Exception("upstream rejected the request"),
start_time=failed_at,
end_time=failed_at,
)
assert row["request_id"] == "call-failed"
def test_get_spend_logs_id_prefers_the_response_id_over_the_standard_logging_id():
"""An unredacted response keeps deciding its own row key, so cache-hit ids and every
other call type behave exactly as they did before."""
assert (
get_spend_logs_id(
"acompletion",
{"id": "chatcmpl-from-response"},
{"litellm_call_id": "call-id-1", "standard_logging_object": {"id": "id-from-standard-payload"}},
)
== "chatcmpl-from-response"
)
@pytest.mark.asyncio
async def test_spend_log_request_id_is_the_message_id_a_bridged_streaming_caller_was_streamed():
"""A streaming /v1/messages call against a non-Anthropic model is served a msg_ id the
adapter mints itself, and it is the only request id that call ever shows the caller, so
GET /spend/logs?request_id=msg_... has to land on the row."""
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.anthropic.pass_through.responses_adapters.streaming_iterator import (
AnthropicResponsesStreamWrapper,
)
from litellm.types.llms.openai import (
ResponseAPIUsage,
ResponseCompletedEvent,
ResponsesAPIResponse,
)
logging_obj = Logging(
model="gpt-5.6",
messages=[{"role": "user", "content": "hi"}],
stream=True,
call_type="anthropic_messages",
start_time=datetime.datetime.now(timezone.utc),
litellm_call_id="6825cafe-0000-4000-8000-000000000001",
function_id="1234",
)
logging_obj.optional_params = {}
completed_response = ResponsesAPIResponse(
id="resp_01Lit6825Bridged",
object="response",
created_at=1767225600,
model="gpt-5.6",
status="completed",
output=[
{
"id": "msg_bridged_output",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "epsilon", "annotations": []}],
}
],
usage=ResponseAPIUsage(input_tokens=12, output_tokens=5, total_tokens=17),
)
async def _responses_stream():
yield {"type": "response.created"}
yield {"type": "response.output_text.delta", "item_id": "msg_bridged_output", "delta": "epsilon"}
yield ResponseCompletedEvent(type="response.completed", response=completed_response)
wrapper = AnthropicResponsesStreamWrapper(
responses_stream=_responses_stream(),
model="gpt-5.6",
litellm_logging_obj=logging_obj,
)
sse_frames = [frame.decode() async for frame in wrapper.async_anthropic_sse_wrapper()]
message_start_frames = [f for f in sse_frames if f.startswith("event: message_start\n")]
assert len(message_start_frames) == 1
streamed_message_id = json.loads(message_start_frames[0].split("data: ", 1)[1])["message"]["id"]
assert streamed_message_id.startswith("msg_")
_, _, logged_response = logging_obj._success_handler_helper_fn(
result=ResponseCompletedEvent(type="response.completed", response=completed_response),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert logged_response.id == streamed_message_id
payload = get_logging_payload(
kwargs={
"call_type": "anthropic_messages",
"model": "gpt-5.6",
"litellm_call_id": "6825cafe-0000-4000-8000-000000000001",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
response_obj=logged_response,
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["request_id"] == streamed_message_id
@pytest.mark.asyncio
async def test_spend_log_request_id_is_untouched_when_no_message_id_was_streamed():
"""Only the bridged streaming adapter mints a msg_ id of its own, so every other
/v1/messages call must keep the id its own response carried."""
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.types.llms.openai import (
ResponseAPIUsage,
ResponseCompletedEvent,
ResponsesAPIResponse,
)
logging_obj = Logging(
model="gpt-5.6",
messages=[{"role": "user", "content": "hi"}],
stream=False,
call_type="anthropic_messages",
start_time=datetime.datetime.now(timezone.utc),
litellm_call_id="6825cafe-0000-4000-8000-000000000002",
function_id="1234",
)
logging_obj.optional_params = {}
completed_response = ResponsesAPIResponse(
id="resp_01Lit6825Unbridged",
object="response",
created_at=1767225600,
model="gpt-5.6",
status="completed",
output=[
{
"id": "msg_unbridged_output",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "epsilon", "annotations": []}],
}
],
usage=ResponseAPIUsage(input_tokens=12, output_tokens=5, total_tokens=17),
)
_, _, logged_response = logging_obj._success_handler_helper_fn(
result=ResponseCompletedEvent(type="response.completed", response=completed_response),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert logged_response.id
assert not logged_response.id.startswith("msg_")
def test_batch_cost_row_does_not_collide_with_the_batch_creation_row():
"""Creating a batch writes a row keyed by the batch's own id, so keying the cost row
the same way makes the insert a duplicate of it. request_id is the primary key and the
flush skips duplicates, so the cost row is dropped with no error and the batch is
billed nothing. Observed against a live proxy: the poller computed and flushed the
cost, and the only row carrying that id was the acreate_batch row written when the
batch was submitted."""
batch_id = "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDphYmM7bGxtX2JhdGNoX2lkOnh5eg"
creation_row_id = get_spend_logs_id("acreate_batch", {"id": batch_id}, {"litellm_call_id": "call-create"})
cost_row_id = get_spend_logs_id(
"aretrieve_batch",
dict(REDACTED_RESPONSE_PLACEHOLDER),
{"litellm_call_id": "call-poller", "standard_logging_object": {"id": batch_id}},
)
assert creation_row_id == batch_id
assert cost_row_id != creation_row_id
assert cost_row_id == f"{batch_id}_batch_cost"
def test_batch_cost_row_id_is_stable_across_repeated_accounting():
"""The cost row stays keyed to the batch, so accounting the same batch twice collapses
to one row instead of billing it twice."""
standard_logging_object = {"id": "batch_same"}
ids = [
get_spend_logs_id(
"aretrieve_batch",
dict(REDACTED_RESPONSE_PLACEHOLDER),
{"litellm_call_id": f"call-{index}", "standard_logging_object": standard_logging_object},
)
for index in range(2)
]
assert ids[0] == ids[1] == "batch_same_batch_cost"
def _make_failed_request_standard_logging_payload() -> StandardLoggingPayload:
base: Final = _make_standard_logging_payload_with_usage_object(usage_object={})
return cast(
StandardLoggingPayload,
{
**base,
"status": "failure",
"call_type": "aresponses",
"model_id": "mid-123",
"model_group": "group-x",
"api_base": "https://api.openai.com/v1/responses",
"custom_llm_provider": "openai",
},
)
def test_get_logging_payload_failed_request_falls_back_to_standard_logging_payload():
"""Failed-request kwargs from the proxy failure hook carry no deployment info
(LIT-5795), so the attribution columns must come from the failure-time
standard_logging_object."""
payload = get_logging_payload(
kwargs={
"model": "group-x",
"litellm_params": {"metadata": {"user_api_key": "test-key", "status": "failure"}},
"standard_logging_object": _make_failed_request_standard_logging_payload(),
},
response_obj={},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["model_id"] == "mid-123"
assert payload["model_group"] == "group-x"
assert payload["api_base"] == "https://api.openai.com/v1/responses"
assert payload["custom_llm_provider"] == "openai"
def test_get_logging_payload_request_kwargs_win_over_standard_logging_payload():
payload = get_logging_payload(
kwargs={
"model": "group-y",
"custom_llm_provider": "anthropic",
"litellm_params": {
"api_base": "https://kwargs.example.com",
"metadata": {
"user_api_key": "test-key",
"model_group": "kwargs-group",
"model_info": {"id": "kwargs-mid"},
},
},
"standard_logging_object": _make_failed_request_standard_logging_payload(),
},
response_obj={},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["model_id"] == "kwargs-mid"
assert payload["model_group"] == "kwargs-group"
assert payload["api_base"] == "https://kwargs.example.com"
assert payload["custom_llm_provider"] == "anthropic"
def test_get_logging_payload_failed_request_without_standard_logging_payload_leaves_fields_empty():
payload = get_logging_payload(
kwargs={
"model": "group-x",
"litellm_params": {"metadata": {"user_api_key": "test-key", "status": "failure"}},
},
response_obj={},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["model_id"] == ""
assert payload["model_group"] == ""
assert payload["api_base"] == ""
assert payload["custom_llm_provider"] == ""
def _router_rejected_failure_payload(model_group: str, llm_router: litellm.Router | None) -> SpendLogsPayload:
return get_logging_payload(
kwargs={
"model": model_group,
"litellm_params": {
"metadata": {"user_api_key": "test-key", "model_group": model_group, "status": "failure"}
},
},
response_obj={},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
llm_router=llm_router,
)
_ProviderResolution = tuple[str, str, str | None, str | None]
def _router_init_provider_stub(
model: str,
custom_llm_provider: str | None = None,
api_base: str | None = None,
api_key: str | None = None,
litellm_params: GenericLiteLLMParams | None = None,
) -> _ProviderResolution:
prefix, _, suffix = model.partition("/")
return (suffix or model, custom_llm_provider or (prefix if suffix else "openai"), api_base, api_key)
def _oauth_tripwire(resolution_attempts: list[str]) -> Callable[..., _ProviderResolution]:
def _trip(
model: str,
custom_llm_provider: str | None = None,
api_base: str | None = None,
api_key: str | None = None,
litellm_params: GenericLiteLLMParams | None = None,
) -> _ProviderResolution:
resolution_attempts.append(model)
raise AssertionError("get_llm_provider would run the OAuth device flow")
return _trip
def _openai_and_anthropic_router() -> litellm.Router:
return litellm.Router(
model_list=[
{"model_name": "openai-group", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-a"}},
{"model_name": "openai-group", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-b"}},
{"model_name": "mixed-group", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-a"}},
{
"model_name": "mixed-group",
"litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-c"},
},
]
)
@pytest.mark.parametrize(
"model_group,expected_provider",
[("openai-group", "openai"), ("mixed-group", ""), ("not-in-router", "")],
)
def test_get_logging_payload_router_rejected_request_takes_provider_from_model_group(
model_group: str, expected_provider: str
):
payload = _router_rejected_failure_payload(model_group, _openai_and_anthropic_router())
assert payload["model_group"] == model_group
assert payload["custom_llm_provider"] == expected_provider
def test_get_logging_payload_router_rejected_request_without_router_leaves_provider_empty():
assert _router_rejected_failure_payload("openai-group", None)["custom_llm_provider"] == ""
@pytest.mark.parametrize(
"litellm_params,expected_provider",
[
({"model": "github_copilot/gpt-4o"}, "github_copilot"),
({"model": "gpt-5", "custom_llm_provider": "chatgpt"}, "chatgpt"),
],
)
def test_get_logging_payload_inferred_provider_never_resolves_declared_authenticating_providers(
monkeypatch: pytest.MonkeyPatch, litellm_params: dict[str, str], expected_provider: str
):
resolution_attempts: list[str] = []
monkeypatch.setattr(litellm, "get_llm_provider", _router_init_provider_stub)
llm_router = litellm.Router(model_list=[{"model_name": "oauth-group", "litellm_params": litellm_params}])
monkeypatch.setattr(litellm, "get_llm_provider", _oauth_tripwire(resolution_attempts))
payload = _router_rejected_failure_payload("oauth-group", llm_router)
assert payload["custom_llm_provider"] == expected_provider
assert resolution_attempts == []
@pytest.mark.parametrize(
"litellm_params",
[
{"model": "github_copilot/gpt-4o"},
{"model": "gpt-5", "custom_llm_provider": "chatgpt"},
{"model": "openai/gpt-4o-mini", "api_key": "sk-a"},
],
)
def test_get_logging_payload_inferred_provider_honours_global_litellm_proxy_override(
monkeypatch: pytest.MonkeyPatch, litellm_params: dict[str, str]
):
monkeypatch.setattr(litellm, "get_llm_provider", _router_init_provider_stub)
llm_router = litellm.Router(model_list=[{"model_name": "proxied-group", "litellm_params": litellm_params}])
monkeypatch.setattr(litellm, "get_llm_provider", _oauth_tripwire([]))
monkeypatch.setattr(litellm, "use_litellm_proxy", True)
payload = _router_rejected_failure_payload("proxied-group", llm_router)
assert payload["custom_llm_provider"] == "litellm_proxy"
def test_get_logging_payload_router_rejected_request_for_unresolvable_deployment_leaves_provider_empty(
monkeypatch: pytest.MonkeyPatch,
):
with monkeypatch.context() as router_init:
router_init.setattr(litellm, "get_llm_provider", _router_init_provider_stub)
llm_router = litellm.Router(
model_list=[{"model_name": "opaque-group", "litellm_params": {"model": "my-unprefixed-model"}}]
)
payload = _router_rejected_failure_payload("opaque-group", llm_router)
assert payload["model_group"] == "opaque-group"
assert payload["custom_llm_provider"] == ""
def test_get_logging_payload_inferred_provider_does_not_rewrite_spend_log_model():
llm_router = litellm.Router(
model_list=[
{
"model_name": "bedrock-group",
"litellm_params": {
"model": "bedrock/anthropic.claude-3-5-haiku-20241022-v1:0",
"aws_region_name": "us-east-1",
},
},
{
"model_name": "bedrock-group",
"litellm_params": {
"model": "bedrock/anthropic.claude-3-5-haiku-20241022-v1:0",
"aws_region_name": "us-west-2",
},
},
]
)
payload = _router_rejected_failure_payload("bedrock-group", llm_router)
assert payload["custom_llm_provider"] == "bedrock"
assert payload["model"] == "bedrock-group"
def test_get_logging_payload_logged_provider_wins_over_model_group_provider():
payload = get_logging_payload(
kwargs={
"model": "openai-group",
"litellm_params": {"metadata": {"user_api_key": "test-key", "model_group": "openai-group"}},
"standard_logging_object": {
**_make_failed_request_standard_logging_payload(),
"model_group": "openai-group",
"custom_llm_provider": "azure",
},
},
response_obj={},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
llm_router=_openai_and_anthropic_router(),
)
assert payload["custom_llm_provider"] == "azure"
class _ModelRouterSpendLogKwargs(TypedDict):
model: ReadOnly[str]
litellm_params: ReadOnly[dict[str, dict[str, str]]]
standard_logging_object: ReadOnly[StandardLoggingPayload]
def _model_router_spend_log_kwargs(slp_model: str | None) -> _ModelRouterSpendLogKwargs:
standard_logging_payload: Final = cast(
StandardLoggingPayload,
{
"model": slp_model,
"metadata": {},
"model_map_information": StandardLoggingModelInformation(
model_map_key="azure_ai/model_router", model_map_value=None
),
},
)
return {
"model": "azure_ai/model_router/model-router",
"litellm_params": {"metadata": {"user_api_key": "sk-test-key"}},
"standard_logging_object": standard_logging_payload,
}
def test_get_logging_payload_uses_standard_logging_payload_model():
payload = get_logging_payload(
kwargs=_model_router_spend_log_kwargs(slp_model="azure_ai/gpt-5-mini"),
response_obj={},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["model"] == "azure_ai/gpt-5-mini"
def test_get_logging_payload_falls_back_to_kwargs_model_when_slp_model_missing():
payload = get_logging_payload(
kwargs=_model_router_spend_log_kwargs(slp_model=None),
response_obj={},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["model"] == "azure_ai/model_router/model-router"
@patch("litellm.proxy.proxy_server.master_key", None)
@patch("litellm.proxy.proxy_server.general_settings", {})
def test_get_logging_payload_empty_key_slp_none_is_empty_string_not_none_literal():
kwargs = {
"model": "openai/gpt-4.1",
"messages": [{"role": "user", "content": "Hello"}],
"call_type": "acompletion",
"litellm_params": {
"metadata": {
"user_api_key_user_id": "test_user",
}
},
}
payload = get_logging_payload(
kwargs=kwargs,
response_obj=Exception("error"),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["api_key"] == "", (
f"Expected empty string but got {payload['api_key']!r}; "
"dropping _redact_logged_api_key's 'or \"\"' guard would yield 'None' here"
)
def test_get_spend_logs_metadata_sibling_fields_preserved():
raw = "anthropic-raw-key-xyz"
meta = _get_spend_logs_metadata(
{
"user_api_key": raw,
"user_api_key_alias": "my-alias",
"user_api_key_team_id": "team-123",
}
)
assert meta["user_api_key"] == hash_token(raw)
assert meta["user_api_key_alias"] == "my-alias"
assert meta["user_api_key_team_id"] == "team-123"
@pytest.mark.parametrize(
("user_email", "stored_email"),
[
("[email protected]", "[email protected]"),
("", None),
(" ", None),
(None, None),
],
)
def test_get_logging_payload_stores_user_email_in_spend_log_metadata(user_email, stored_email):
payload = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_params": {
"metadata": {
"user_api_key_user_email": user_email,
"user_api_key_user_id": "okta|abc123",
}
},
},
response_obj=litellm.ModelResponse(id="chatcmpl-user-email", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["user_api_key_user_email"] == stored_email
assert metadata["user_api_key_user_id"] == "okta|abc123"
def test_redact_logged_api_key_partial_sha256_is_hashed():
partial_hex = "a" * 63
result = _redact_logged_api_key(partial_hex)
assert result is not None
assert result != partial_hex
assert len(result) == 64
assert result == hash_token(partial_hex)
def test_redact_logged_api_key_bearer_already_hashed_passes_through_with_flag():
already_hashed = hash_token("sk-some-key")
assert len(already_hashed) == 64
result = _redact_logged_api_key(f"Bearer {already_hashed}", already_redacted=True)
assert result == already_hashed
assert hash_token(already_hashed) != result
def test_redact_logged_api_key_bearer_sha256_without_flag_is_hashed():
already_hashed = hash_token("sk-some-key")
assert len(already_hashed) == 64
result = _redact_logged_api_key(f"Bearer {already_hashed}")
assert result is not None
assert result != already_hashed
assert result == hash_token(already_hashed)
def test_autorouter_savings_flow_from_logging_payload_into_spend_log_metadata():
"""The figure the logging path computed is what the spend writer reads back, so it
is threaded from the StandardLoggingPayload like cost_breakdown, never re-derived."""
payload = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
"standard_logging_object": {"autorouter_savings": 0.42, "metadata": {}, "model_map_information": None},
},
response_obj=litellm.ModelResponse(id="chatcmpl-ar-savings", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["autorouter_savings"] == 0.42
@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"])
def test_caller_forged_autorouter_savings_is_discarded(bucket):
"""The raw request bucket is client-writable and _get_spend_logs_metadata projects
every SpendLogsMetadata key from it, so the logging payload's value must overwrite
unconditionally or a caller could report savings the router never produced."""
payload = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_params": {bucket: {"user_api_key": "test-key", "autorouter_savings": 999.0}},
},
response_obj=litellm.ModelResponse(id="chatcmpl-forged-savings", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["autorouter_savings"] is None
def test_get_logging_payload_includes_fallback_info_in_spend_logs_metadata():
"""
Test that fallback info (attempted_fallbacks, original_model_group) from metadata
is included in the spend logs metadata JSON.
"""
kwargs = {
"model": "gpt-3.5-turbo",
"litellm_params": {
"metadata": {
"user_api_key": "sk-test-key",
"attempted_fallbacks": 2,
"original_model_group": "azure-gpt-fallback",
}
},
"standard_logging_object": StandardLoggingPayload(
id="test-fallback-123",
call_type="completion",
stream=False,
response_cost=0.001,
status="success",
total_tokens=100,
prompt_tokens=50,
completion_tokens=50,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=None,
model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None),
model="gpt-3.5-turbo",
model_id="model-123",
model_group="openai",
custom_llm_provider="openai",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_alias=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_user_id=None,
user_api_key_user_email=None,
user_api_key_team_alias=None,
spend_logs_metadata=None,
requester_ip_address=None,
requester_metadata=None,
user_api_key_end_user_id=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address=None,
messages=[],
response={},
error_str=None,
model_parameters={},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.001",
litellm_overhead_time_ms=None,
additional_headers=None,
batch_models=None,
litellm_model_name=None,
usage_object=None,
),
),
}
response_obj = {
"id": "test-response-retry",
"choices": [{"message": {"content": "Hello!"}}],
"usage": {
"total_tokens": 100,
"prompt_tokens": 50,
"completion_tokens": 50,
},
}
start_time = datetime.datetime.now(timezone.utc)
end_time = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
)
metadata = json.loads(payload["metadata"])
assert metadata.get("attempted_fallbacks") == 2, (
f"Expected attempted_fallbacks=2, got {metadata.get('attempted_fallbacks')}"
)
assert metadata.get("original_model_group") == "azure-gpt-fallback", (
f"Expected original_model_group=azure-gpt-fallback, got {metadata.get('original_model_group')}"
)
def test_get_logging_payload_handles_missing_fallback_info_gracefully():
"""
Test that fallback fields are None when not present in metadata (backward compatibility).
"""
kwargs = {
"model": "gpt-3.5-turbo",
"litellm_params": {
"metadata": {
"user_api_key": "sk-test-key",
}
},
"standard_logging_object": StandardLoggingPayload(
id="test-no-fallback-456",
call_type="completion",
stream=False,
response_cost=0.001,
status="success",
total_tokens=100,
prompt_tokens=50,
completion_tokens=50,
startTime=1234567890.0,
endTime=1234567891.0,
completionStartTime=None,
model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None),
model="gpt-3.5-turbo",
model_id="model-123",
model_group="openai",
custom_llm_provider="openai",
api_base="https://api.openai.com",
metadata=StandardLoggingMetadata(
user_api_key_hash="test_hash",
user_api_key_alias=None,
user_api_key_team_id=None,
user_api_key_org_id=None,
user_api_key_user_id=None,
user_api_key_user_email=None,
user_api_key_team_alias=None,
spend_logs_metadata=None,
requester_ip_address=None,
requester_metadata=None,
user_api_key_end_user_id=None,
),
cache_hit=False,
cache_key=None,
saved_cache_cost=0.0,
request_tags=[],
end_user=None,
requester_ip_address=None,
messages=[],
response={},
error_str=None,
model_parameters={},
hidden_params=StandardLoggingHiddenParams(
model_id="model-123",
cache_key=None,
api_base="https://api.openai.com",
response_cost="0.001",
litellm_overhead_time_ms=None,
additional_headers=None,
batch_models=None,
litellm_model_name=None,
usage_object=None,
),
),
}
response_obj = {
"id": "test-response-no-fallback",
"choices": [{"message": {"content": "Hello!"}}],
"usage": {
"total_tokens": 100,
"prompt_tokens": 50,
"completion_tokens": 50,
},
}
start_time = datetime.datetime.now(timezone.utc)
end_time = datetime.datetime.now(timezone.utc)
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=start_time,
end_time=end_time,
)
metadata = json.loads(payload["metadata"])
assert metadata.get("attempted_fallbacks") is None, "attempted_fallbacks should be None when not provided"
assert metadata.get("original_model_group") is None, "original_model_group should be None when not provided"
@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"])
def test_injected_cache_breakpoints_survive_into_spend_log_metadata(bucket):
"""The injection marker only gates savings if it reaches the spend-log row.
_get_spend_logs_metadata projects onto SpendLogsMetadata.__annotations__, so an
undeclared key is dropped silently. Both buckets are covered because chat routes
stamp metadata while /v1/messages routes stamp litellm_metadata, and
record_gateway_injection writes into whichever the request carries.
"""
payload = get_logging_payload(
kwargs={
"model": "claude-sonnet-5",
"litellm_params": {
bucket: {
"user_api_key": "test-key",
"litellm_gateway_injected_cache": "dep-of-this-row",
}
},
},
response_obj=litellm.ModelResponse(id="chatcmpl-injected", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["litellm_gateway_injected_cache"] == "dep-of-this-row"
def test_passthrough_caching_carries_no_injection_marker():
"""The negative class the gate depends on: a request whose cache_control the client
supplied must read as unmarked, not merely unlabelled by accident."""
payload = get_logging_payload(
kwargs={
"model": "claude-sonnet-5",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
response_obj=litellm.ModelResponse(id="chatcmpl-passthrough", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["litellm_gateway_injected_cache"] is None
def _routed_call_kwargs(model_info: Mapping[str, object]) -> dict[str, object]:
return {
"model": "claude-haiku-4-5",
"custom_llm_provider": "azure_ai",
"litellm_call_id": "router-corr-123",
"litellm_params": {
"metadata": {
"user_api_key": "test-key",
"model_group": "internal-router/gpt-5.4",
"deployment": "azure_ai/claude-haiku-4-5",
"model_info": model_info,
}
},
}
def test_router_metadata_stamped_for_internal_router_model_deployment():
"""A deployment flagged model_info.internal_router_model gets a router_metadata
block correlating the requested model group with the selected deployment."""
payload = get_logging_payload(
kwargs=_routed_call_kwargs({"id": "mi-1", "internal_router_model": True}),
response_obj=litellm.ModelResponse(id="chatcmpl-router-meta", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["router_metadata"] == {
"requested_model": "internal-router/gpt-5.4",
"selected_model": "azure_ai/claude-haiku-4-5",
"selected_provider": "azure_ai",
"router_correlation_id": "router-corr-123",
}
def test_router_metadata_absent_without_internal_router_model_flag():
payload = get_logging_payload(
kwargs=_routed_call_kwargs({"id": "mi-1"}),
response_obj=litellm.ModelResponse(id="chatcmpl-unflagged", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["router_metadata"] is None
@pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"])
def test_caller_forged_router_metadata_is_discarded(bucket):
"""The raw request bucket is client-writable and _get_spend_logs_metadata projects
every SpendLogsMetadata key from it, so the server-derived value must overwrite
unconditionally or a caller could plant router provenance the router never produced."""
payload = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_params": {
bucket: {
"user_api_key": "test-key",
"router_metadata": {"requested_model": "forged", "router_correlation_id": "forged-id"},
}
},
},
response_obj=litellm.ModelResponse(id="chatcmpl-forged-router-meta", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["router_metadata"] is None
ANTHROPIC_MESSAGES_RESPONSE: Final = {
"id": "msg_01Lit6806NonStreaming",
"type": "message",
"role": "assistant",
"model": "claude-haiku-4-5",
"content": [{"type": "text", "text": "epsilon"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 14, "output_tokens": 4},
}
ANTHROPIC_MESSAGES_SSE_CHUNKS: Final = (
'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_01Lit6806Streaming",'
'"type":"message","role":"assistant","model":"claude-haiku-4-5","content":[],'
'"usage":{"input_tokens":14,"output_tokens":1}}}\n\n',
'event: content_block_start\ndata: {"type":"content_block_start","index":0,'
'"content_block":{"type":"text","text":""}}\n\n',
'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,'
'"delta":{"type":"text_delta","text":"epsilon"}}\n\n',
'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n',
'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},'
'"usage":{"output_tokens":4}}\n\n',
'event: message_stop\ndata: {"type":"message_stop"}\n\n',
)
def _anthropic_messages_logging_obj(*, stream: bool) -> Any:
from litellm.litellm_core_utils.litellm_logging import Logging
logging_obj = Logging(
model="claude-haiku-4-5",
messages=[{"role": "user", "content": "hi"}],
stream=stream,
call_type="anthropic_messages",
start_time=datetime.datetime.now(timezone.utc),
litellm_call_id="6806cafe-0000-4000-8000-000000000001",
function_id="1234",
)
logging_obj.optional_params = {}
logging_obj.model_call_details["custom_llm_provider"] = "anthropic"
return logging_obj
def _spend_log_request_id(response_obj: Any, kwargs: dict) -> str:
payload = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
return payload["request_id"]
def test_spend_log_request_id_is_the_message_id_a_non_streaming_messages_caller_received():
"""
POST /v1/messages hands the caller `id: msg_...`, the only request id they ever see, so
GET /spend/logs?request_id=msg_... has to find the row.
"""
logging_obj = _anthropic_messages_logging_obj(stream=False)
logged_response = logging_obj._handle_anthropic_messages_response_logging(result=ANTHROPIC_MESSAGES_RESPONSE)
assert logged_response.id == "msg_01Lit6806NonStreaming"
assert (
_spend_log_request_id(
response_obj=logged_response,
kwargs={
"call_type": "anthropic_messages",
"model": "claude-haiku-4-5",
"litellm_call_id": "6806cafe-0000-4000-8000-000000000001",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
)
== "msg_01Lit6806NonStreaming"
)
def test_spend_log_request_id_is_the_message_id_a_streaming_messages_caller_received():
"""
The streaming leg of /v1/messages logs through the Anthropic passthrough handler, which used
to stamp litellm_call_id over the msg_ id carried by the message_start event.
"""
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
AnthropicPassthroughLoggingHandler,
)
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
logging_obj = _anthropic_messages_logging_obj(stream=True)
logging_obj.model_call_details["stream"] = True
logged = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
request_body={"model": "claude-haiku-4-5"},
endpoint_type=EndpointType.ANTHROPIC,
start_time=datetime.datetime.now(timezone.utc),
all_chunks=list(ANTHROPIC_MESSAGES_SSE_CHUNKS),
end_time=datetime.datetime.now(timezone.utc),
)
assert logged["result"].id == "msg_01Lit6806Streaming"
assert (
_spend_log_request_id(
response_obj=logged["result"],
kwargs={
**logged["kwargs"],
"call_type": "anthropic_messages",
"litellm_call_id": "6806cafe-0000-4000-8000-000000000001",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
)
== "msg_01Lit6806Streaming"
)
def test_spend_log_request_id_still_falls_back_to_litellm_call_id_without_a_provider_id():
"""
Anthropic-compatible upstreams that omit `id` must keep landing on litellm_call_id rather
than on a fresh chatcmpl- uuid nobody can look up.
"""
logging_obj = _anthropic_messages_logging_obj(stream=True)
logging_obj.model_call_details["stream"] = True
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
AnthropicPassthroughLoggingHandler,
)
AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload(
litellm_model_response=litellm.ModelResponse(id="chatcmpl-generated"),
model="claude-haiku-4-5",
kwargs={},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
logging_obj=logging_obj,
)
assert logging_obj.model_call_details["complete_streaming_response"].id == ("6806cafe-0000-4000-8000-000000000001")
def test_spend_log_request_id_for_chat_completions_is_untouched():
"""
/v1/chat/completions callers look their rows up by the chatcmpl- id in the response body.
"""
assert (
_spend_log_request_id(
response_obj=litellm.ModelResponse(id="chatcmpl-EJvWIw3DAhuKYuwp3jJI4Pnhp2vjv", choices=[]),
kwargs={
"call_type": "acompletion",
"model": "gpt-5.6",
"litellm_call_id": "6806cafe-0000-4000-8000-000000000002",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
)
== "chatcmpl-EJvWIw3DAhuKYuwp3jJI4Pnhp2vjv"
)
def test_spend_log_request_id_is_the_response_id_a_bridged_messages_caller_received():
"""
/v1/messages against a non-Anthropic model answers with the Responses id the caller then
looks their row up by, so the row must not fall back to a fresh chatcmpl- uuid.
"""
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
logging_obj = _anthropic_messages_logging_obj(stream=False)
bridged_response = ResponsesAPIResponse(
id="resp_01Lit6806Bridged",
object="response",
created_at=1767225600,
model="gpt-5.6",
status="completed",
output=[
{
"id": "msg_bridged_output",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "delta", "annotations": []}],
}
],
usage=ResponseAPIUsage(input_tokens=13, output_tokens=5, total_tokens=18),
)
logged_response = logging_obj._handle_anthropic_messages_response_logging(result=bridged_response)
assert logged_response.id == "resp_01Lit6806Bridged"
assert (
_spend_log_request_id(
response_obj=logged_response,
kwargs={
"call_type": "anthropic_messages",
"model": "gpt-5.6",
"litellm_call_id": "6806cafe-0000-4000-8000-000000000003",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
},
)
== "resp_01Lit6806Bridged"
)
def test_failed_agent_request_keeps_registered_display_name():
agent_model: Final = "a2a_agent/Research Agent"
payload: Final = get_logging_payload(
kwargs={
"model": agent_model,
"call_type": "asend_message",
"litellm_params": {
"metadata": {"model_group": agent_model, "model_info": {"id": "registered-agent"}, "status": "failure"}
},
},
response_obj=ValueError("Agent action denied"),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["model"] == agent_model
assert payload["status"] == "failure"
assert payload["model_id"] == "registered-agent"
_CLI_SESSION_ALIAS: Final = "cli-session-alice"
_CLI_SESSION_TOKEN: Final = "cli-session-Qm7xJ2kP9sLw4vT1nR8yAa"
def _cli_session_request_metadata(logged_key: str) -> dict[str, str]:
return {
"user_api_key": logged_key,
"user_api_key_hash": logged_key,
"user_api_key_alias": _CLI_SESSION_ALIAS,
"user_api_key_user_id": "alice",
}
@pytest.mark.parametrize("call_type", ["acompletion", "anthropic_messages", "aresponses"])
def test_get_logging_payload_attributes_a_cli_session_to_its_alias(call_type: str):
payload = get_logging_payload(
kwargs={
"call_type": call_type,
"model": "gpt-5.4-nano",
"response_cost": 0.00001,
"litellm_params": {"metadata": _cli_session_request_metadata(_CLI_SESSION_ALIAS)},
},
response_obj=litellm.ModelResponse(id=f"{call_type}-1", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["api_key"] == _CLI_SESSION_ALIAS
assert json.loads(payload["metadata"])["user_api_key"] == _CLI_SESSION_ALIAS
assert json.loads(payload["metadata"])["user_api_key_alias"] == _CLI_SESSION_ALIAS
@pytest.mark.parametrize("call_type", ["acompletion", "anthropic_messages", "aresponses"])
def test_get_logging_payload_never_lands_a_raw_cli_session_token(call_type: str):
payload = get_logging_payload(
kwargs={
"call_type": call_type,
"model": "gpt-5.4-nano",
"litellm_params": {"metadata": _cli_session_request_metadata(_CLI_SESSION_TOKEN)},
},
response_obj=litellm.ModelResponse(id=f"{call_type}-2", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["api_key"] == hash_token(_CLI_SESSION_TOKEN)
assert json.loads(payload["metadata"])["user_api_key"] == hash_token(_CLI_SESSION_TOKEN)
def test_redact_logged_api_key_cli_session_alias_needs_alias_provenance():
assert _redact_logged_api_key(_CLI_SESSION_ALIAS, already_redacted=True, key_alias=_CLI_SESSION_ALIAS) == (
_CLI_SESSION_ALIAS
)
assert _redact_logged_api_key(_CLI_SESSION_ALIAS, already_redacted=True) == hash_token(_CLI_SESSION_ALIAS)
assert _redact_logged_api_key(_CLI_SESSION_ALIAS, key_alias=_CLI_SESSION_ALIAS) == hash_token(_CLI_SESSION_ALIAS)
assert _redact_logged_api_key("alice", already_redacted=True, key_alias="alice") == hash_token("alice")
def test_azure_spillover_stamped_from_response_headers():
"""Raw provider response headers on the logging kwargs mark the request as spilled."""
kwargs: Final = {
**_routed_call_kwargs({"id": "mi-1"}),
"response_headers": {
"x-ms-is-spilled-over": "true",
"x-ms-spillover-from-deployment": "my-ptu",
},
}
payload = get_logging_payload(
kwargs=kwargs,
response_obj=litellm.ModelResponse(id="chatcmpl-spill-raw", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["azure_spillover"] == {"from_deployment": "my-ptu"}
def test_azure_spillover_stamped_from_standard_logging_additional_headers():
"""Streaming requests carry the processed llm_provider- headers on the standard payload."""
kwargs: Final = {
**_routed_call_kwargs({"id": "mi-1"}),
"standard_logging_object": {
"hidden_params": {
"additional_headers": {
"llm_provider-x-ms-is-spilled-over": "true",
"llm_provider-x-ms-spillover-from-deployment": "my-ptu",
}
},
"metadata": {},
"model_map_information": None,
},
}
payload = get_logging_payload(
kwargs=kwargs,
response_obj=litellm.ModelResponse(id="chatcmpl-spill-sl", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["azure_spillover"] == {"from_deployment": "my-ptu"}
def test_azure_spillover_absent_without_spillover_headers():
payload = get_logging_payload(
kwargs=_routed_call_kwargs({"id": "mi-1"}),
response_obj=litellm.ModelResponse(id="chatcmpl-no-spill", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
metadata = json.loads(payload["metadata"])
assert metadata["azure_spillover"] is None
def test_baseline_estimate_metadata_comes_from_the_logging_stamp() -> None:
supplied: Final = MappingProxyType({"version": 1, "status": "estimated", "reason": "caller_supplied"})
recorded: Final = MappingProxyType({"version": 1, "status": "unknown", "reason": "history_unavailable"})
result: Final = _get_spend_logs_metadata(
{"autorouter_savings": 999.0, "autorouter_savings_estimate": supplied},
autorouter_savings=None,
autorouter_savings_estimate=recorded,
)
assert result["autorouter_savings"] is None
assert result["autorouter_savings_estimate"] == recorded
absent: Final = _get_spend_logs_metadata({"autorouter_savings_estimate": supplied})
assert absent["autorouter_savings_estimate"] is None
@pytest.mark.parametrize("billing_agent", [None, "authenticated-agent"])
def test_untrusted_agent_label_cannot_replace_verified_billing_identity(billing_agent: str | None) -> None:
kwargs = {
"model": "gpt-4",
"litellm_params": {
"metadata": {
"user_api_key": "test-key",
"agent_id": "header-selected-agent",
"billing_agent_id": billing_agent,
}
},
}
payload = get_logging_payload(
kwargs=kwargs,
response_obj={"id": "request"},
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["agent_id"] == "header-selected-agent"
assert payload["billing_agent_id"] == billing_agent
def _spend_log_row(metadata: Mapping[str, object]) -> Mapping[str, object]:
return MappingProxyType({"request_id": "req-1", "spend": 0.5, "metadata": json.dumps(dict(metadata))})
_STORED_METADATA: Final = MappingProxyType(
{
"status": "success",
"cold_storage_object_key": "logs/req-1.json",
"model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"max_tokens": 10}},
"usage_object": {"prompt_tokens": 3},
"user_api_key_alias": "alias",
}
)
def test_spend_log_row_keeps_every_metadata_field_when_unconfigured() -> None:
row: Final = _spend_log_row(_STORED_METADATA)
assert spend_log_row_with_retained_metadata(row, None) is row
def test_spend_log_row_drops_excluded_metadata_fields_only() -> None:
row: Final = _spend_log_row(_STORED_METADATA)
stored: Final = spend_log_row_with_retained_metadata(
row, SpendLogsMetadataFields(exclude=("model_map_information", "user_api_key_alias"))
)
assert json.loads(cast(str, stored["metadata"])) == {
"status": "success",
"cold_storage_object_key": "logs/req-1.json",
"usage_object": {"prompt_tokens": 3},
}
assert {name: value for name, value in stored.items() if name != "metadata"} == {
"request_id": "req-1",
"spend": 0.5,
}
def test_spend_log_row_include_keeps_listed_and_always_kept_fields() -> None:
stored: Final = spend_log_row_with_retained_metadata(
_spend_log_row(_STORED_METADATA), SpendLogsMetadataFields(include=("usage_object",))
)
assert json.loads(cast(str, stored["metadata"])) == {
"status": "success",
"cold_storage_object_key": "logs/req-1.json",
"usage_object": {"prompt_tokens": 3},
}
@pytest.mark.parametrize(
"configured",
[
{"include": ["usage_object"], "exclude": ["model_map_information"]},
{},
{"exclude": ["model_map_informaton"]},
{"include": ["usage_object", "not_a_field"]},
{"exclude": ["status"]},
{"exclude": ["cold_storage_object_key"]},
{"exclude": ["model_map_information"], "drop": ["usage_object"]},
],
)
def test_spend_logs_metadata_fields_rejects_ambiguous_or_lossy_config(configured: dict[str, list[str]]) -> None:
from pydantic import ValidationError
from litellm.proxy._types import ConfigGeneralSettings
with pytest.raises(ValidationError):
ConfigGeneralSettings.model_validate({"spend_logs_metadata_fields": configured})
def test_configured_spend_logs_metadata_fields_ignores_invalid_runtime_value() -> None:
with patch(
"litellm.proxy.proxy_server.general_settings",
{"spend_logs_metadata_fields": {"include": ["usage_object"], "exclude": ["status"]}},
):
assert configured_spend_logs_metadata_fields() is None
with patch(
"litellm.proxy.proxy_server.general_settings",
{"spend_logs_metadata_fields": {"exclude": ["model_map_information"]}},
):
assert configured_spend_logs_metadata_fields() == SpendLogsMetadataFields(exclude=("model_map_information",))
class _OcrUsageInfoDict(TypedDict, total=False):
pages_processed: ReadOnly[int]
doc_size_bytes: ReadOnly[int]
class _OcrResponseDict(TypedDict):
id: ReadOnly[str]
object: ReadOnly[str]
model: ReadOnly[str]
usage_info: ReadOnly[NotRequired[_OcrUsageInfoDict]]
class _TokenUsageDict(TypedDict):
prompt_tokens: ReadOnly[int]
completion_tokens: ReadOnly[int]
total_tokens: ReadOnly[int]
class _CompletionResponseDict(TypedDict):
id: ReadOnly[str]
object: ReadOnly[str]
model: ReadOnly[str]
usage: ReadOnly[_TokenUsageDict]
class _LoggingMetadata(TypedDict, total=False):
user_api_key_user_id: ReadOnly[str]
user_api_key_team_id: ReadOnly[str]
class _LoggingLitellmParams(TypedDict, total=False):
metadata: ReadOnly[_LoggingMetadata]
class _LoggingKwargs(TypedDict):
model: ReadOnly[str]
call_type: ReadOnly[str]
litellm_params: ReadOnly[_LoggingLitellmParams]
response_cost: ReadOnly[float]
class _AdditionalUsageValues(TypedDict, total=False):
pages_processed: ReadOnly[int | None]
doc_size_bytes: ReadOnly[int | None]
class _SpendLogMetadata(TypedDict):
additional_usage_values: ReadOnly[_AdditionalUsageValues]
_SPEND_LOG_METADATA: Final = TypeAdapter(_SpendLogMetadata)
_OCR_LOGGED_AT: Final = datetime.datetime(2026, 1, 1, tzinfo=timezone.utc)
_OCR_RESPONSE_COST: Final = 0.05
def _ocr_logging_kwargs(call_type: str = "ocr", metadata: _LoggingMetadata | None = None) -> _LoggingKwargs:
return _LoggingKwargs(
model="test-ocr-model",
call_type=call_type,
litellm_params=_LoggingLitellmParams() if metadata is None else _LoggingLitellmParams(metadata=metadata),
response_cost=_OCR_RESPONSE_COST,
)
def _ocr_payload(
kwargs: _LoggingKwargs, response_obj: _OcrResponseDict | _CompletionResponseDict | OCRResponse
) -> SpendLogsPayload:
return get_logging_payload(
kwargs=dict(kwargs),
response_obj=response_obj,
start_time=_OCR_LOGGED_AT,
end_time=_OCR_LOGGED_AT,
)
def _additional_usage_values(payload: SpendLogsPayload) -> _AdditionalUsageValues:
return _SPEND_LOG_METADATA.validate_json(payload["metadata"])["additional_usage_values"]
class TestExtractUsageForOCRCall:
def test_extract_usage_from_dict(self) -> None:
response_obj_dict: Final = {"usage_info": _OcrUsageInfoDict(pages_processed=5)}
usage: Final = _extract_usage_for_ocr_call(response_obj_dict, response_obj_dict)
assert usage == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0, "pages_processed": 5}
def test_extract_usage_from_pydantic_model(self) -> None:
response_obj: Final = OCRResponse(
pages=[],
model="test-ocr-model",
usage_info=OCRUsageInfo(pages_processed=10, doc_size_bytes=1024),
)
usage: Final = _extract_usage_for_ocr_call(response_obj, response_obj.model_dump())
assert usage["prompt_tokens"] == 0
assert usage["completion_tokens"] == 0
assert usage["total_tokens"] == 0
assert usage["pages_processed"] == 10
assert usage["doc_size_bytes"] == 1024
def test_extract_usage_with_object_attributes(self) -> None:
class _SimpleUsageInfo:
def __init__(self, pages_processed: int) -> None:
self.pages_processed = pages_processed
class _SimpleOCRResponse:
def __init__(self) -> None:
self.usage_info = _SimpleUsageInfo(pages_processed=3)
usage: Final = _extract_usage_for_ocr_call(_SimpleOCRResponse(), {})
assert usage == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0, "pages_processed": 3}
def test_extract_usage_missing_usage_info(self) -> None:
assert _extract_usage_for_ocr_call({}, {}) == {}
def test_extract_usage_empty_usage_info(self) -> None:
usage: Final = _extract_usage_for_ocr_call({"usage_info": {}}, {"usage_info": {}})
assert usage == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0, "pages_processed": 0}
class TestGetLoggingPayloadOCR:
def test_ocr_call_with_dict_response(self) -> None:
payload: Final = _ocr_payload(
_ocr_logging_kwargs(),
{
"id": "ocr-test-123",
"object": "ocr",
"model": "test-ocr-model",
"usage_info": {"pages_processed": 7, "doc_size_bytes": 2048},
},
)
assert payload["call_type"] == "ocr"
assert payload["request_id"] == "ocr-test-123"
assert payload["prompt_tokens"] == 0
assert payload["completion_tokens"] == 0
assert payload["total_tokens"] == 0
assert payload["spend"] == _OCR_RESPONSE_COST
assert _additional_usage_values(payload)["pages_processed"] == 7
assert _additional_usage_values(payload)["doc_size_bytes"] == 2048
def test_aocr_call_with_pydantic_response(self) -> None:
payload: Final = _ocr_payload(
_ocr_logging_kwargs(call_type="aocr"),
OCRResponse(pages=[], model="test-ocr-model", usage_info=OCRUsageInfo(pages_processed=12)),
)
assert payload["call_type"] == "aocr"
assert payload["prompt_tokens"] == 0
assert payload["completion_tokens"] == 0
assert payload["total_tokens"] == 0
assert payload["spend"] == _OCR_RESPONSE_COST
assert _additional_usage_values(payload)["pages_processed"] == 12
def test_ocr_call_missing_usage_info(self) -> None:
payload: Final = _ocr_payload(
_ocr_logging_kwargs(),
{"id": "ocr-test-789", "object": "ocr", "model": "test-ocr-model"},
)
assert payload["call_type"] == "ocr"
assert payload["prompt_tokens"] == 0
assert payload["completion_tokens"] == 0
assert payload["total_tokens"] == 0
assert payload["spend"] == _OCR_RESPONSE_COST
assert "pages_processed" not in _additional_usage_values(payload)
def test_ocr_call_with_zero_pages(self) -> None:
payload: Final = _ocr_payload(
_ocr_logging_kwargs(),
{
"id": "ocr-test-000",
"object": "ocr",
"model": "test-ocr-model",
"usage_info": {"pages_processed": 0},
},
)
assert payload["call_type"] == "ocr"
assert payload["prompt_tokens"] == 0
assert payload["completion_tokens"] == 0
assert payload["total_tokens"] == 0
assert payload["spend"] == _OCR_RESPONSE_COST
assert _additional_usage_values(payload)["pages_processed"] == 0
def test_non_ocr_call_uses_token_based_usage(self) -> None:
payload: Final = _ocr_payload(
_LoggingKwargs(
model="gpt-5.5", call_type="completion", litellm_params=_LoggingLitellmParams(), response_cost=0.02
),
{
"id": "completion-test-123",
"object": "chat.completion",
"model": "gpt-5.5",
"usage": {"prompt_tokens": 50, "completion_tokens": 100, "total_tokens": 150},
},
)
assert payload["call_type"] == "completion"
assert payload["prompt_tokens"] == 50
assert payload["completion_tokens"] == 100
assert payload["total_tokens"] == 150
assert payload["spend"] == 0.02
assert "pages_processed" not in _additional_usage_values(payload)
def test_ocr_with_metadata(self) -> None:
payload: Final = _ocr_payload(
_ocr_logging_kwargs(
metadata=_LoggingMetadata(user_api_key_user_id="test-user", user_api_key_team_id="test-team")
),
{
"id": "ocr-metadata-test",
"object": "ocr",
"model": "test-ocr-model",
"usage_info": {"pages_processed": 5, "doc_size_bytes": 1024},
},
)
assert payload["call_type"] == "ocr"
assert payload["user"] == "test-user"
assert payload["team_id"] == "test-team"
assert payload["prompt_tokens"] == 0
assert payload["completion_tokens"] == 0
assert payload["total_tokens"] == 0
assert payload["spend"] == _OCR_RESPONSE_COST
assert _additional_usage_values(payload)["pages_processed"] == 5
assert _additional_usage_values(payload)["doc_size_bytes"] == 1024
@pytest.mark.parametrize(
"model_id",
["chatcmpl-9XZmkzS1uPhRCoVdGQvBqqIbSgECt", "", None],
)
def test_spend_logs_payload(model_id: str | None):
kwargs: Final = {
"model": "chatgpt-v-3",
"messages": [
{"role": "system", "content": "you are a helpful assistant.\n"},
{"role": "user", "content": "bom dia"},
],
"custom_llm_provider": "azure",
"optional_params": {
"stream": False,
"max_tokens": 10,
"user": "116544810872468347480",
"extra_body": {},
},
"litellm_params": {
"acompletion": True,
"api_key": "sk-test-mock-key-707",
"force_timeout": 600,
"logger_fn": None,
"verbose": False,
"custom_llm_provider": "azure",
"api_base": "https://openai-gpt-4-test-v-1.openai.azure.com//openai/",
"litellm_call_id": "b9929bf6-7b80-4c8c-b486-034e6ac0c8b7",
"model_alias_map": {},
"completion_call_id": None,
"metadata": {
"tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"],
"user_api_key": "sk-test-mock-api-key-123",
"user_api_key_alias": "custom-key-alias",
"user_api_end_user_max_budget": None,
"litellm_api_version": "0.0.0",
"global_max_parallel_requests": None,
"user_api_key_user_id": "116544810872468347480",
"user_api_key_org_id": "custom-org-id",
"user_api_key_team_id": "custom-team-id",
"user_api_key_team_alias": "custom-team-alias",
"user_api_key_metadata": {},
"requester_ip_address": "127.0.0.1",
"spend_logs_metadata": {"hello": "world"},
"headers": {
"content-type": "application/json",
"user-agent": "PostmanRuntime/7.32.3",
"accept": "*/*",
"postman-token": "92300061-eeaa-423b-a420-0b44896ecdc4",
"host": "localhost:4000",
"accept-encoding": "gzip, deflate, br",
"connection": "keep-alive",
"content-length": "163",
},
"endpoint": "http://localhost:4000/chat/completions",
"model_group": "gpt-5-mini",
"deployment": "azure/gpt-4.1-mini",
"model_info": {
"id": "4bad40a1eb6bebd1682800f16f44b9f06c52a6703444c99c7f9f32e9de3693b4",
"db_model": False,
},
"api_base": "https://openai-gpt-4-test-v-1.openai.azure.com/",
"caching_groups": None,
"error_information": None,
"status": "success",
"proxy_server_request": "{}",
"raw_request": "\n\nPOST Request Sent from LiteLLM:\ncurl -X POST \\\nhttps://openai-gpt-4-test-v-1.openai.azure.com//openai/ \\\n-H 'Authorization: *****' \\\n-d '{'model': 'chatgpt-v-3', 'messages': [{'role': 'system', 'content': 'you are a helpful assistant.\\n'}, {'role': 'user', 'content': 'bom dia'}], 'stream': False, 'max_tokens': 10, 'user': '116544810872468347480', 'extra_body': {}}'\n",
},
"model_info": {
"id": "4bad40a1eb6bebd1682800f16f44b9f06c52a6703444c99c7f9f32e9de3693b4",
"db_model": False,
},
"proxy_server_request": {
"url": "http://localhost:4000/chat/completions",
"method": "POST",
"headers": {
"content-type": "application/json",
"user-agent": "PostmanRuntime/7.32.3",
"accept": "*/*",
"postman-token": "92300061-eeaa-423b-a420-0b44896ecdc4",
"host": "localhost:4000",
"accept-encoding": "gzip, deflate, br",
"connection": "keep-alive",
"content-length": "163",
},
"body": {
"messages": [
{
"role": "system",
"content": "you are a helpful assistant.\n",
},
{"role": "user", "content": "bom dia"},
],
"model": "gpt-5-mini",
"max_tokens": 10,
},
},
"preset_cache_key": None,
"no-log": False,
"stream_response": {},
"input_cost_per_token": None,
"input_cost_per_second": None,
"output_cost_per_token": None,
"output_cost_per_second": None,
},
"start_time": datetime.datetime(2024, 6, 7, 12, 43, 30, 307665),
"stream": False,
"user": "116544810872468347480",
"call_type": "acompletion",
"litellm_call_id": "b9929bf6-7b80-4c8c-b486-034e6ac0c8b7",
"completion_start_time": datetime.datetime(2024, 6, 7, 12, 43, 30, 954146),
"max_tokens": 10,
"extra_body": {},
"input": [
{"role": "system", "content": "you are a helpful assistant.\n"},
{"role": "user", "content": "bom dia"},
],
"api_key": "1234",
"original_response": "",
"additional_args": {
"headers": {"Authorization": "Bearer 1234"},
"api_base": "openai-gpt-4-test-v-1.openai.azure.com",
"acompletion": True,
"complete_input_dict": {
"model": "chatgpt-v-3",
"messages": [
{"role": "system", "content": "you are a helpful assistant.\n"},
{"role": "user", "content": "bom dia"},
],
"stream": False,
"max_tokens": 10,
"user": "116544810872468347480",
"extra_body": {},
},
},
"log_event_type": "post_api_call",
"end_time": datetime.datetime(2024, 6, 7, 12, 43, 30, 954146),
"cache_hit": None,
"response_cost": 2.4999999999999998e-05,
"standard_logging_object": {
"request_tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"],
"metadata": {
"user_api_key_end_user_id": "test-user",
},
"model_map_information": {
"tpm": 1000,
"rpm": 1000,
},
},
}
response_obj: Final = litellm.ModelResponse(
id=model_id,
choices=[
litellm.Choices(
finish_reason="length",
index=0,
message=litellm.Message(content="Bom dia! Como posso ajudar você", role="assistant"),
)
],
created=1717789410,
model="gpt-35-turbo",
object="chat.completion",
system_fingerprint=None,
usage=litellm.Usage(completion_tokens=10, prompt_tokens=20, total_tokens=30),
)
payload: Final[SpendLogsPayload] = get_logging_payload(
kwargs=kwargs,
response_obj=response_obj,
start_time=datetime.datetime(2024, 6, 7, 12, 43, 30, 308604),
end_time=datetime.datetime(2024, 6, 7, 12, 43, 30, 954146),
)
assert len(payload["request_id"]) > 0
expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys()
assert "metadata" in payload
assert isinstance(payload["metadata"], str)
metadata: Final[Mapping[str, object]] = json.loads(payload["metadata"])
assert set(metadata.keys()) == set(expected_metadata_keys)
assert payload["request_tags"] == '["model-anthropic-claude-v2.1", "app-ishaan-prod"]'
assert metadata["user_api_key_org_id"] == "custom-org-id"
assert metadata["user_api_key_team_id"] == "custom-team-id"
assert metadata["user_api_key_team_alias"] == "custom-team-alias"
assert metadata["user_api_key_alias"] == "custom-key-alias"
assert payload["custom_llm_provider"] == "azure"
def test_spend_logs_payload_whisper():
kwargs: Final = {
"model": "whisper-1",
"messages": [{"role": "user", "content": "audio_file"}],
"optional_params": {},
"litellm_params": {
"api_base": "",
"metadata": {
"user_api_key": "sk-test-mock-api-key-123",
"user_api_key_alias": None,
"user_api_key_end_user_id": "test-user",
"user_api_end_user_max_budget": None,
"litellm_api_version": "1.40.19",
"global_max_parallel_requests": None,
"user_api_key_user_id": "default_user_id",
"user_api_key_org_id": None,
"user_api_key_team_id": None,
"user_api_key_team_alias": None,
"user_api_key_team_max_budget": None,
"user_api_key_team_spend": None,
"user_api_key_spend": 0.0,
"user_api_key_max_budget": None,
"user_api_key_metadata": {},
"headers": {
"host": "localhost:4000",
"user-agent": "curl/7.88.1",
"accept": "*/*",
"content-length": "775501",
"content-type": "multipart/form-data; boundary=------------------------21d518e191326d20",
},
"endpoint": "http://localhost:4000/v1/audio/transcriptions",
"litellm_parent_otel_span": None,
"model_group": "whisper-1",
"deployment": "whisper-1",
"model_info": {
"id": "d7761582311451c34d83d65bc8520ce5c1537ea9ef2bec13383cf77596d49eeb",
"db_model": False,
},
"caching_groups": None,
},
},
"start_time": datetime.datetime(2024, 6, 26, 14, 20, 11, 313291),
"stream": False,
"user": "",
"call_type": "atranscription",
"litellm_call_id": "05921cf7-33f9-421c-aad9-33310c1e2702",
"completion_start_time": datetime.datetime(2024, 6, 26, 14, 20, 13, 653149),
"stream_options": None,
"input": "tmp-requestc8640aee-7d85-49c3-b3ef-bdc9255d8e37.wav",
"original_response": '{"text": "Four score and seven years ago, our fathers brought forth on this continent a new nation, conceived in liberty and dedicated to the proposition that all men are created equal. Now we are engaged in a great civil war, testing whether that nation, or any nation so conceived and so dedicated, can long endure."}',
"additional_args": {
"complete_input_dict": {
"model": "whisper-1",
"file": "<_io.BufferedReader name='tmp-requestc8640aee-7d85-49c3-b3ef-bdc9255d8e37.wav'>",
"language": None,
"prompt": None,
"response_format": None,
"temperature": None,
}
},
"log_event_type": "post_api_call",
"end_time": datetime.datetime(2024, 6, 26, 14, 20, 13, 653149),
"cache_hit": None,
"response_cost": 0.00023398580000000003,
}
response: Final = litellm.utils.TranscriptionResponse(
text="Four score and seven years ago, our fathers brought forth on this continent a new nation, conceived in liberty and dedicated to the proposition that all men are created equal. Now we are engaged in a great civil war, testing whether that nation, or any nation so conceived and so dedicated, can long endure."
)
payload: Final[SpendLogsPayload] = get_logging_payload(
kwargs=kwargs,
response_obj=response,
start_time=datetime.datetime(2026, 1, 15, 12, 0, 0),
end_time=datetime.datetime(2026, 1, 15, 12, 0, 0),
)
assert payload["call_type"] == "atranscription"
assert payload["spend"] == 0.00023398580000000003
def test_spend_logs_payload_with_prompts_enabled(monkeypatch: pytest.MonkeyPatch):
from litellm.proxy.proxy_server import general_settings
monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", True)
kwargs: Final = {
"model": "gpt-5-mini",
"messages": [{"role": "user", "content": "Hello!"}],
"litellm_params": {
"proxy_server_request": {
"body": {
"model": "gpt-5.5",
"messages": [{"role": "user", "content": "Hello!"}],
}
}
},
"standard_logging_object": {
"messages": [{"role": "user", "content": "Hello!"}],
"response": {"role": "assistant", "content": "Hi there!"},
"metadata": {
"user_api_key_end_user_id": "test-user",
},
"request_tags": ["model-anthropic-claude-v2.1", "app-ishaan-prod"],
"model_map_information": {
"tpm": 1000,
"rpm": 1000,
},
},
}
response_obj: Final = litellm.ModelResponse(
id="chatcmpl-123",
choices=[
litellm.Choices(
finish_reason="stop",
index=0,
message=litellm.Message(content="Hi there!", role="assistant"),
)
],
model="gpt-5-mini",
usage=litellm.Usage(completion_tokens=2, prompt_tokens=1, total_tokens=3),
)
logged_at: Final = datetime.datetime(2026, 1, 15, 12, 0, 0)
payload: Final[SpendLogsPayload] = get_logging_payload(
kwargs=kwargs, response_obj=response_obj, start_time=logged_at, end_time=logged_at
)
assert payload["response"] == json.dumps({"role": "assistant", "content": "Hi there!"})
proxy_server_request: Final = json.loads(payload["proxy_server_request"] or "{}")
assert proxy_server_request["model"] == "gpt-5.5"
assert proxy_server_request["messages"] == [{"role": "user", "content": "Hello!"}]
monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", False)
payload_disabled: Final[SpendLogsPayload] = get_logging_payload(
kwargs=kwargs, response_obj=response_obj, start_time=logged_at, end_time=logged_at
)
assert payload_disabled["messages"] == "{}"
assert payload_disabled["response"] == "{}"
@pytest.mark.parametrize(
("configured", "disabled"),
[
(True, True),
("true", True),
("TRUE", True),
("yes", True),
("on", True),
("1", True),
(1, True),
(False, False),
(None, False),
("false", False),
("no", False),
(0, False),
("", False),
("maybe", False),
([1], False),
],
)
def test_requester_ip_address_logging_disabled_reads_the_setting_as_the_settings_api_validates_it(
configured: object, disabled: bool
) -> None:
with patch( # test-quality-ok: general_settings is proxy config, loaded from yaml, not an HTTP boundary
"litellm.proxy.proxy_server.general_settings", {"disable_requester_ip_address_logging": configured}
):
assert requester_ip_address_logging_disabled() is disabled
def _request_ip_metadata() -> dict[str, object]:
return {
"requester_ip_address": "203.0.113.9",
"user_agent": "requester-ip-test",
"headers": {"X-Forwarded-For": "203.0.113.9, 10.0.0.1", "x-real-ip": "203.0.113.9", "accept": "*/*"},
"requester_metadata": {"headers": {"x-forwarded-for": "203.0.113.9"}, "team": "blue"},
"error_information": {
"error_code": "403",
"error_message": "Access forbidden: IP address 203.0.113.9 not allowed.",
"traceback": "HTTPException: client 2001:db8::9 via 198.51.100.7 denied at 12:30:45 (litellm_settings::modify_params, qwen2.5.1.5b)",
},
}
def _request_ip_kwargs(request_metadata: dict[str, object]) -> dict[str, object]:
return {
"model": "gpt-5-mini",
"messages": [{"role": "user", "content": "Hello!"}],
"litellm_params": {
"metadata": request_metadata,
"proxy_server_request": {
"body": {
"model": "gpt-5-mini",
"messages": [{"role": "user", "content": "Hello!"}],
"metadata": request_metadata,
}
},
},
"standard_logging_object": {
"messages": [{"role": "user", "content": "Hello!"}],
"response": {"role": "assistant", "content": "Hi there!"},
"metadata": {"requester_ip_address": "203.0.113.9"},
"model_map_information": {"tpm": 1000, "rpm": 1000},
},
}
@pytest.mark.parametrize(
("disabled", "recorded_ip", "error_message", "traceback"),
[
(
False,
"203.0.113.9",
"Access forbidden: IP address 203.0.113.9 not allowed.",
"HTTPException: client 2001:db8::9 via 198.51.100.7 denied at 12:30:45 (litellm_settings::modify_params, qwen2.5.1.5b)",
),
(
True,
None,
"Access forbidden: IP address REDACTED_BY_LITELM not allowed.",
"HTTPException: client REDACTED_BY_LITELM via REDACTED_BY_LITELM denied at 12:30:45 (litellm_settings::modify_params, qwen2.5.1.5b)",
),
],
ids=["setting-off", "setting-on"],
)
def test_spend_logs_payload_keeps_no_requester_ip_address_when_ip_logging_is_disabled(
monkeypatch: pytest.MonkeyPatch, disabled: bool, recorded_ip: str | None, error_message: str, traceback: str
) -> None:
from litellm.proxy.proxy_server import general_settings
monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", True)
monkeypatch.setitem(general_settings, "disable_requester_ip_address_logging", disabled)
logged_at: Final = datetime.datetime(2026, 1, 15, 12, 0, 0)
request_metadata: Final = _request_ip_metadata()
payload: Final[SpendLogsPayload] = get_logging_payload(
kwargs=_request_ip_kwargs(request_metadata),
response_obj=litellm.ModelResponse(id="chatcmpl-ip", model="gpt-5-mini"),
start_time=logged_at,
end_time=logged_at,
)
stored_metadata: Final = json.loads(payload["metadata"])
stored_request: Final = json.loads(payload["proxy_server_request"] or "{}")
stored_errors: Final = (stored_metadata["error_information"], stored_request["metadata"]["error_information"])
assert request_metadata == _request_ip_metadata()
assert stored_metadata["user_agent"] == "requester-ip-test"
assert stored_request["metadata"]["headers"]["accept"] == "*/*"
assert stored_request["metadata"]["requester_metadata"]["team"] == "blue"
assert payload["requester_ip_address"] == recorded_ip
assert stored_metadata["requester_ip_address"] == recorded_ip
assert stored_request["metadata"]["requester_ip_address"] == recorded_ip
assert [error["error_message"] for error in stored_errors] == [error_message, error_message]
assert [error["traceback"] for error in stored_errors] == [traceback, traceback]
assert [error["error_code"] for error in stored_errors] == ["403", "403"]
assert ("203.0.113.9" in json.dumps(payload, default=str)) is not disabled
def test_stored_request_without_requester_ip_address_drops_or_masks_only_ip_fields_from_request_metadata() -> None:
body: Final = {
"model": "gpt-5-mini",
"messages": [{"role": "user", "content": "203.0.113.9"}],
"tools": [{"type": "function", "function": {"parameters": {"headers": {"x-real-ip": "keep"}}}}],
"error_information": {"error_message": "203.0.113.9"},
"litellm_metadata": {
"requester_ip_address": "203.0.113.9",
"headers": {
"Forwarded": "for=203.0.113.9",
"CF-Connecting-IP": "203.0.113.9",
"X-Envoy-External-Address": "203.0.113.9",
"CF-Connecting-IPv6": "2001:db8::9",
"CF-Pseudo-IPv4": "203.0.113.9",
"X-AppEngine-User-IP": "203.0.113.9",
"X-Vercel-Forwarded-For": "203.0.113.9",
"X-ARR-ClientIP": "203.0.113.9",
"DO-Connecting-IP": "203.0.113.9",
"Incap-Client-IP": "2001:db8::9",
"x-edge-client": "client=203.0.113.9, peer=[2001:db8::9]:443",
"host": "10.0.0.5:4000",
"user-agent": "ua",
},
"forwarded": "not a header",
"spend_logs_metadata": [{"headers": {"true-client-ip": "203.0.113.9", "x-client": "host:2001:db8::9"}}],
"error_information": {"error_message": "denied 203.0.113.9"},
},
"originating_request_masked": {
"messages": [{"role": "user", "content": "203.0.113.9"}],
"metadata": {
"requester_ip_address": "203.0.113.9",
"headers": {"x-forwarded-for": "203.0.113.9", "host": "10.0.0.5:4000"},
},
},
"classifier_input": {
"input": [{"role": "user", "content": "203.0.113.9"}],
"metadata": {"requester_ip_address": "203.0.113.9", "user_agent": "curl"},
},
}
assert _stored_request_without_requester_ip_address(body) == {
"model": "gpt-5-mini",
"messages": [{"role": "user", "content": "203.0.113.9"}],
"tools": [{"type": "function", "function": {"parameters": {"headers": {"x-real-ip": "keep"}}}}],
"error_information": {"error_message": "203.0.113.9"},
"litellm_metadata": {
"requester_ip_address": None,
"headers": {
"DO-Connecting-IP": "REDACTED_BY_LITELM",
"Incap-Client-IP": "REDACTED_BY_LITELM",
"x-edge-client": "client=REDACTED_BY_LITELM, peer=[REDACTED_BY_LITELM]:443",
"host": "REDACTED_BY_LITELM:4000",
"user-agent": "ua",
},
"forwarded": "not a header",
"spend_logs_metadata": [{"headers": {"x-client": "host:REDACTED_BY_LITELM"}}],
"error_information": {"error_message": "denied REDACTED_BY_LITELM"},
},
"originating_request_masked": {
"messages": [{"role": "user", "content": "203.0.113.9"}],
"metadata": {"requester_ip_address": None, "headers": {"host": "REDACTED_BY_LITELM:4000"}},
},
"classifier_input": {
"input": [{"role": "user", "content": "203.0.113.9"}],
"metadata": {"requester_ip_address": None, "user_agent": "curl"},
},
}
@pytest.mark.parametrize("error_field", ["error_message", "traceback"])
def test_stored_request_masks_ip_addresses_in_long_error_text_and_at_its_truncation_cut(
monkeypatch: pytest.MonkeyPatch, error_field: str
) -> None:
from litellm.proxy.proxy_server import general_settings
monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", True)
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "1000")
text: Final = "a" * 345 + " 203.0.113.9 " + "b" * 2000 + " 2001:db8::9 " + "c" * 644
metadata: Final = {"error_information": {"error_code": "500", error_field: text}}
stored: Final = json.loads(
_get_proxy_server_request_for_spend_logs_payload(
metadata=metadata,
litellm_params={"proxy_server_request": {"body": {"model": "gpt-5-mini", "metadata": metadata}}},
drop_requester_ip_address=True,
)
)
stored_text: Final = stored["metadata"]["error_information"][error_field]
kept_head_and_tail: Final = stored_text[:350] + stored_text[-650:]
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored_text
assert kept_head_and_tail.startswith("a" * 345 + " REDA")
assert not any(char.isdigit() for char in kept_head_and_tail)
assert metadata["error_information"][error_field] == text
_TRACE_ID: Final = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
_PLANTED_MARKER: Final = (
f"x ... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped 5 chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ... "
)
@pytest.mark.parametrize("requester_metadata", [False, True], ids=["headers", "requester-metadata-headers"])
@pytest.mark.parametrize(
"value, stored_head, stored_tail",
[
(
"a" * 345 + " 203.0.113.9 " + "b" * 2000 + " 2001:db8:0:0:1:0:0:9 " + "c" * 642,
"a" * 345 + " REDACTED_BY_LITELM",
"REDACTED_BY_LITELM " + "c" * 642,
),
(
"a" * 343 + " ip:203.0.113.9 " + "b" * 2000 + " 192.0.2.1. " + "c" * 647,
"a" * 343 + " ip:REDACTED_BY_LITELM",
"REDACTED_BY_LITELM. " + "c" * 647,
),
(
"x" * 296 + f" trace={_TRACE_ID}:203.0.113.9 " + "b" * 2000 + f" 203.0.113.9:{_TRACE_ID} " + "c" * 606,
"x" * 296 + f" trace={_TRACE_ID}:REDACTED_BY_LITELM",
f"REDACTED_BY_LITELM:{_TRACE_ID} " + "c" * 606,
),
(
_PLANTED_MARKER
+ "a" * (345 - len(_PLANTED_MARKER))
+ " 203.0.113.9 "
+ "b" * 2000
+ " 2001:db8::dead:beef "
+ "c" * 640,
_PLANTED_MARKER + "a" * (345 - len(_PLANTED_MARKER)) + " REDACTED_BY_LITELM",
"REDACTED_BY_LITELM " + "c" * 640,
),
("a" * 346 + " cafe:beef " + "b" * 2000 + " deadbeef " + "c" * 646, "a" * 346 + " caf", "eef " + "c" * 646),
("0123456789abcdef" * 200, ("0123456789abcdef" * 22)[:350], ("0123456789abcdef" * 41)[-650:]),
(
"a" * 281 + " " + "ab1." * 17 + "b" * 2000 + " " + "1ba." * 17 + " " + "c" * 581,
"a" * 281 + " " + "ab1." * 6 + "REDACTED_BY_LITELM",
"REDACTED_BY_LITELMba." + "1ba." * 5 + " " + "c" * 581,
),
],
ids=[
"ipv4-and-ipv6-split",
"ipv4-after-a-label-and-before-a-dot",
"ipv4-beside-long-hex-tokens",
"after-a-planted-marker-and-ipv6-groups-without-digits",
"hex-words-without-digits-or-colons",
"hex-run-longer-than-an-ip",
"short-groups-longer-than-an-ip",
],
)
def test_stored_request_masks_the_piece_of_an_ip_address_a_header_truncation_cut_leaves(
monkeypatch: pytest.MonkeyPatch, requester_metadata: bool, value: str, stored_head: str, stored_tail: str
) -> None:
from litellm.proxy import proxy_server
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True})
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "1000")
headers: Final = {"x-edge-trace": value}
metadata: Final = {"requester_metadata": {"headers": headers}} if requester_metadata else {"headers": headers}
stored: Final = json.loads(
_get_proxy_server_request_for_spend_logs_payload(
metadata=metadata,
litellm_params={"proxy_server_request": {"body": {"model": "gpt-5-mini", "metadata": metadata}}},
drop_requester_ip_address=True,
)
)
stored_metadata: Final = stored["metadata"]["requester_metadata"] if requester_metadata else stored["metadata"]
stored_value: Final = stored_metadata["headers"]["x-edge-trace"]
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored_value
assert stored_value.startswith(stored_head + "... (")
assert stored_value.endswith(") ..." + stored_tail)
assert headers["x-edge-trace"] == value
def test_stored_request_with_ip_logging_disabled_matches_setting_off_for_bodies_without_ip_fields(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy.proxy_server import general_settings
monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", True)
loop: Final[dict[str, object]] = {}
loop["self"] = (loop,)
body: Final[dict[object, object]] = {"model": "gpt-5-mini", 7: "non-string key", "metadata": {"tags": (loop,)}}
stored: Final = [
_get_proxy_server_request_for_spend_logs_payload(
metadata={},
litellm_params={"proxy_server_request": {"body": body}},
drop_requester_ip_address=drop_requester_ip_address,
)
for drop_requester_ip_address in (False, True)
]
assert json.loads(stored[0])["model"] == "gpt-5-mini"
assert stored[1] == stored[0]
@pytest.mark.parametrize("disabled", [False, True], ids=["setting-off", "setting-on"])
def test_failure_error_information_masks_ip_addresses_in_long_text_and_at_its_truncation_cut(
monkeypatch: pytest.MonkeyPatch, disabled: bool
) -> None:
from litellm.proxy.proxy_server import general_settings
monkeypatch.setitem(general_settings, "disable_requester_ip_address_logging", disabled)
monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "1000")
text: Final = "a" * 345 + " 203.0.113.9 " + "b" * 2000 + " 2001:db8::9 " + "c" * 644
sanitized: Final = sanitize_error_information_for_spend_logs(
{"error_code": "500", "error_message": text, "traceback": text}
)
assert sanitized is not None
for stored_text in (sanitized["error_message"], sanitized["traceback"]):
kept_head_and_tail = stored_text[:350] + stored_text[-650:]
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored_text
assert any(char.isdigit() for char in kept_head_and_tail) is not disabled
assert sanitized["error_code"] == "500"
@pytest.mark.parametrize(
"address", ["203.0.113.9", "2001:db8::9", "1:2:3:4:5:6:7:8", "1:2:3:4:5:6:7::", "::1:2:3:4:5:6:7", "fe80::1"]
)
def test_failure_error_information_masks_every_ip_address_form_when_ip_logging_is_disabled(
monkeypatch: pytest.MonkeyPatch, address: str
) -> None:
from litellm.proxy.proxy_server import general_settings
monkeypatch.setitem(general_settings, "disable_requester_ip_address_logging", True)
sanitized: Final = sanitize_error_information_for_spend_logs(
{
"error_code": "403",
"error_message": f"IP address {address} not allowed.",
"traceback": f"peer [{address}]:443",
}
)
assert sanitized == {
"error_code": "403",
"error_message": "IP address REDACTED_BY_LITELM not allowed.",
"traceback": "peer [REDACTED_BY_LITELM]:443",
}
@pytest.mark.parametrize(
("text", "stored"),
[
("connect to host:2001:db8::2 failed", "connect to host:REDACTED_BY_LITELM failed"),
("peer_2001:db8::2_tail", "peer_REDACTED_BY_LITELM_tail"),
("upstream 1:2:3:4:5:6:7:8:443 refused", "upstream REDACTED_BY_LITELM refused"),
("upstream fe80::1:2:3:4:5:6:443 refused", "upstream REDACTED_BY_LITELM refused"),
("label cafe:1:2:3:4:5:6:7::", "label REDACTED_BY_LITELM"),
("label cafe:1:2:3:4:5:6:7:8:443", "label REDACTED_BY_LITELM"),
("socket_192.0.2.65 closed", "socket_REDACTED_BY_LITELM closed"),
(
"at 2026-10-10T18:20:09 Error::Foo aa:bb:cc:dd:ee:ff 1203.0.113.9 qwen2.5.1.5",
"at 2026-10-10T18:20:09 Error::Foo aa:bb:cc:dd:ee:ff 1203.0.113.9 qwen2.5.1.5",
),
("IP 192.0.2.1.443 > 10.0.0.2.80: Flags [S]", "IP REDACTED_BY_LITELM.443 > REDACTED_BY_LITELM.80: Flags [S]"),
],
)
def test_failure_error_information_masks_addresses_after_labels_and_before_ports(
monkeypatch: pytest.MonkeyPatch, text: str, stored: str
) -> None:
from litellm.proxy.proxy_server import general_settings
monkeypatch.setitem(general_settings, "disable_requester_ip_address_logging", True)
assert sanitize_error_information_for_spend_logs({"error_code": "500", "error_message": text}) == {
"error_code": "500",
"error_message": stored,
}
@pytest.mark.parametrize(("tracking_disabled", "expected_end_user"), [(True, ""), (None, "session-abc123")])
def test_get_logging_payload_stores_end_user_only_while_end_user_cost_tracking_is_on(
monkeypatch: pytest.MonkeyPatch, tracking_disabled: bool | None, expected_end_user: str
):
monkeypatch.setattr(litellm, "disable_end_user_cost_tracking", tracking_disabled)
payload: Final = get_logging_payload(
kwargs={
"model": "gpt-4o-mini",
"litellm_params": {"metadata": {"user_api_key": "test-key"}},
"standard_logging_object": cast(
StandardLoggingPayload,
{**_TRACE_ONLY_STANDARD_LOGGING, "metadata": {"user_api_key_end_user_id": "session-abc123"}},
),
},
response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[], usage=litellm.Usage()),
start_time=datetime.datetime.now(timezone.utc),
end_time=datetime.datetime.now(timezone.utc),
)
assert payload["end_user"] == expected_end_user