The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
fix(proxy): strip caller base_model from spend pricing and reject lone surrogates in multipart metadata (#47639)
Kerry Lu2a268951c4ec
diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.pyindex 521a9a15bd0..245a56678eb 100644--- a/litellm/proxy/common_utils/http_parsing_utils.py+++ b/litellm/proxy/common_utils/http_parsing_utils.py@@ -191,6 +191,23 @@ def is_otlp_trace_request(request: HTTPConnection) -> bool:return request.scope.get("method") == "POST" and get_route_path(request.scope) in {"/v1/traces", "/v1/logs"}+def _parse_form_metadata(raw_metadata: str) -> object:+ """The JSON body path rejects a lone surrogate escape (``"\\ud800"``) with a+ 400 because no provider or database can encode it. ``json.loads`` accepts+ the escape, so the multipart ``metadata`` field gets the same check."""+ parsed: Final[object] = json.loads(raw_metadata)+ try:+ json.dumps(parsed, ensure_ascii=False).encode("utf-8")+ except UnicodeEncodeError as e:+ raise ProxyException(+ message=f"Invalid form payload: metadata {e}",+ type="invalid_request_error",+ param="metadata",+ code=status.HTTP_400_BAD_REQUEST,+ ) from e+ return parsed++async def read_request_body(request: Request | None) -> dict:"""Safely read the request body and parse it as JSON.@@ -238,9 +255,13 @@ async def read_request_body(request: Request | None) -> dict:param="request_body",code=status.HTTP_400_BAD_REQUEST,)- parsed_body = dict(form_data)- if "metadata" in parsed_body and isinstance(parsed_body["metadata"], str):- parsed_body["metadata"] = json.loads(parsed_body["metadata"])+ form_body: Final = dict(form_data)+ raw_metadata: Final = form_body.get("metadata")+ parsed_body = (+ {**form_body, "metadata": _parse_form_metadata(raw_metadata)}+ if isinstance(raw_metadata, str)+ else form_body+ )else:# Read the request bodybody: Final = await request.body()diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.pyindex efc8575feb3..d69778136ee 100644--- a/litellm/proxy/litellm_pre_call_utils.py+++ b/litellm/proxy/litellm_pre_call_utils.py@@ -452,8 +452,9 @@ _ALLOW_CLIENT_MESSAGE_REDACTION_OPT_OUT_METADATA_KEY: Final = "allow_client_mess# ``litellm.model_cost`` map. Both effects belong to deployment configuration,# not to user-supplied request bodies, so the proxy strips them before they# reach the call path. Built from the Pydantic model so newly-added pricing-# fields are covered automatically.-_CLIENT_PRICING_CONTROL_FIELDS: Final = frozenset(CustomPricingLiteLLMParams.model_fields.keys())+# fields are covered automatically. ``base_model`` selects which cost-map entry+# prices the row, so a caller naming a cheaper model lowers their own spend.+_CLIENT_PRICING_CONTROL_FIELDS: Final = frozenset(CustomPricingLiteLLMParams.model_fields.keys()) | {"base_model"}# ``model_info`` carries the same pricing fields when read by# ``use_custom_pricing_for_model``; strip from metadata for the same reason.# ``standard_logging_guardrail_information`` is proxy-written telemetry summeddiff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.pyindex 2b2c55a8452..20a0d4d5053 100644--- a/litellm/proxy/utils.py+++ b/litellm/proxy/utils.py@@ -8565,7 +8565,7 @@ async def _create_spend_logs_with_poison_isolation(await repo.table.create_many(data=rows, skip_duplicates=True)return failure_budgetexcept Exception as e:- if not PrismaDBExceptionHandler.is_prisma_data_error(e):+ if not (PrismaDBExceptionHandler.is_prisma_data_error(e) or isinstance(e, UnicodeEncodeError)):raiseif PrismaDBExceptionHandler.is_database_service_unavailable_error(e):raisediff --git a/tests/unit/proxy/common_utils/test_http_parsing_utils.py b/tests/unit/proxy/common_utils/test_http_parsing_utils.pyindex 88ae5e0d84e..cd5820fa2fd 100644--- a/tests/unit/proxy/common_utils/test_http_parsing_utils.py+++ b/tests/unit/proxy/common_utils/test_http_parsing_utils.py@@ -1556,3 +1556,39 @@ def test_is_otlp_trace_request_matches_only_http_posts_to_the_otlp_routes()assert http_parsing_utils.is_otlp_trace_request(connection) is expected+++async def test_form_data_metadata_with_lone_surrogate_escape_is_rejected_like_a_json_body():+ """``json.loads`` accepts ``"\\ud800"`` and yields a lone surrogate that no+ provider or database can utf-8 encode. The JSON body path already answers+ 400, so the multipart ``metadata`` field must too, instead of letting the+ value through to the spend log writers."""+ mock_request = MagicMock()+ mock_request.form = AsyncMock(+ return_value=FormData({"model": "whisper-1", "metadata": '{"tags": ["poison-\\ud800-tag"]}'})+ )+ mock_request.headers = {"content-type": "multipart/form-data"}+ mock_request.scope = {}+ mock_request.state._cached_headers = None++ with pytest.raises(ProxyException) as exc_info:+ await read_request_body(mock_request)++ assert exc_info.value.code == "400"+ assert "metadata" in exc_info.value.message+++async def test_form_data_metadata_keeps_non_ascii_and_valid_surrogate_pairs():+ mock_request = MagicMock()+ mock_request.form = AsyncMock(+ return_value=FormData({"model": "whisper-1", "metadata": '{"tags": ["caf\\u00e9", "\\ud83d\\ude00"]}'})+ )+ mock_request.headers = {"content-type": "multipart/form-data"}+ mock_request.scope = {}+ mock_request.state._cached_headers = None++ result = await read_request_body(mock_request)++ assert result["metadata"]["tags"] == ["café", "😀"]diff --git a/tests/unit/proxy/test_pricing_field_strip.py b/tests/unit/proxy/test_pricing_field_strip.pyindex 395fdc3a16e..3e9bee9217d 100644--- a/tests/unit/proxy/test_pricing_field_strip.py+++ b/tests/unit/proxy/test_pricing_field_strip.py@@ -57,7 +57,7 @@ class TestStripClientPricingOverrides:# The strip set is built from the model so additions are picked up# automatically — this test guards against the model and the strip# set drifting apart if someone replaces the auto-derivation later.- assert _CLIENT_PRICING_CONTROL_FIELDS == frozenset(CustomPricingLiteLLMParams.model_fields.keys())+ assert _CLIENT_PRICING_CONTROL_FIELDS >= frozenset(CustomPricingLiteLLMParams.model_fields.keys())# Sanity: the obvious top-level pricing fields are in the set.for field in ("input_cost_per_token",@@ -82,6 +82,22 @@ class TestStripClientPricingOverrides:"messages": [{"role": "user", "content": "hi"}],}+ def test_root_base_model_dropped(self):+ # ``base_model`` picks the cost-map entry a spend row is priced with+ # (success rows via the standard logging payload, billed failure rows+ # via the failure lift), so a caller naming a cheaper model lowers+ # their own recorded spend.+ data = {+ "model": "gpt-4o",+ "messages": [{"role": "user", "content": "hi"}],+ "base_model": "gpt-4o-mini",+ }+ _strip_client_pricing_overrides(data)+ assert data == {+ "model": "gpt-4o",+ "messages": [{"role": "user", "content": "hi"}],+ }+def test_metadata_model_info_dropped(self):data = {"model": "gpt-4",@@ -280,6 +296,7 @@ async def test_add_litellm_data_to_request_skips_strip_with_key_opt_in():"model": "gpt-4","messages": [{"role": "user", "content": "hi"}],"input_cost_per_token": 0.0001,+ "base_model": "gpt-4o-mini","metadata": {"model_info": {"output_cost_per_token": 0.0002}},}@@ -295,6 +312,7 @@ async def test_add_litellm_data_to_request_skips_strip_with_key_opt_in():assert updated["input_cost_per_token"] == 0.0001assert updated["metadata"]["model_info"] == {"output_cost_per_token": 0.0002}+ assert updated["base_model"] == "gpt-4o-mini"@pytest.mark.asynciodiff --git a/tests/unit/proxy/test_update_spend.py b/tests/unit/proxy/test_update_spend.pyindex 62adb7303e2..571341e85e7 100644--- a/tests/unit/proxy/test_update_spend.py+++ b/tests/unit/proxy/test_update_spend.py@@ -323,3 +323,31 @@ async def test_update_spend_logs_multiple_batches_with_failure():# Verify all logs were cleared from transactionsassert len(prisma_client.spend_log_transactions) == 0+++async def test_update_spend_logs_drops_row_prisma_cannot_encode_and_keeps_the_rest():+ """A row the Prisma engine cannot utf-8 encode (a lone surrogate in a string+ column) is permanent, so it must be dropped like any other poison row instead+ of being requeued at the head of the queue where it would block every tenant."""+ prisma_client = MockPrismaClient()+ proxy_logging_obj = create_mock_proxy_logging()+ prisma_client.spend_log_transactions = [+ {"request_id": "good-1", "request_tags": '["a"]'},+ {"request_id": "poison", "request_tags": "\ud800"},+ {"request_id": "good-2", "request_tags": '["b"]'},+ ]+ written = []++ async def create_many(data, skip_duplicates):+ if any("\ud800" in row["request_tags"] for row in data):+ raise UnicodeEncodeError("utf-8", "\ud800", 0, 1, "surrogates not allowed")+ written.extend(row["request_id"] for row in data)++ prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=create_many)++ await update_spend(prisma_client, None, proxy_logging_obj)++ assert sorted(written) == ["good-1", "good-2"]+ assert prisma_client.spend_log_transactions == []+ proxy_logging_obj.failure_handler.assert_not_called()