Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

fix(proxy): strip caller base_model from spend pricing and reject lone surrogates in multipart metadata (#47639)

Kerry Lu2a268951c4ec
diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py
index 521a9a15bd0..245a56678eb 100644
--- a/litellm/proxy/common_utils/http_parsing_utils.py
+++ b/litellm/proxy/common_utils/http_parsing_utils.py
@@ -191,6 +191,23 @@ def is_otlp_trace_request(request: HTTPConnection) -> bool:
return request.scope.get("method") == "POST" and get_route_path(request.scope) in {"/v1/traces", "/v1/logs"}
+def _parse_form_metadata(raw_metadata: str) -> object:
+ """The JSON body path rejects a lone surrogate escape (``"\\ud800"``) with a
+ 400 because no provider or database can encode it. ``json.loads`` accepts
+ the escape, so the multipart ``metadata`` field gets the same check."""
+ parsed: Final[object] = json.loads(raw_metadata)
+ try:
+ json.dumps(parsed, ensure_ascii=False).encode("utf-8")
+ except UnicodeEncodeError as e:
+ raise ProxyException(
+ message=f"Invalid form payload: metadata {e}",
+ type="invalid_request_error",
+ param="metadata",
+ code=status.HTTP_400_BAD_REQUEST,
+ ) from e
+ return parsed
+
+
async def read_request_body(request: Request | None) -> dict:
"""
Safely read the request body and parse it as JSON.
@@ -238,9 +255,13 @@ async def read_request_body(request: Request | None) -> dict:
param="request_body",
code=status.HTTP_400_BAD_REQUEST,
)
- parsed_body = dict(form_data)
- if "metadata" in parsed_body and isinstance(parsed_body["metadata"], str):
- parsed_body["metadata"] = json.loads(parsed_body["metadata"])
+ form_body: Final = dict(form_data)
+ raw_metadata: Final = form_body.get("metadata")
+ parsed_body = (
+ {**form_body, "metadata": _parse_form_metadata(raw_metadata)}
+ if isinstance(raw_metadata, str)
+ else form_body
+ )
else:
# Read the request body
body: Final = await request.body()
diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py
index efc8575feb3..d69778136ee 100644
--- a/litellm/proxy/litellm_pre_call_utils.py
+++ b/litellm/proxy/litellm_pre_call_utils.py
@@ -452,8 +452,9 @@ _ALLOW_CLIENT_MESSAGE_REDACTION_OPT_OUT_METADATA_KEY: Final = "allow_client_mess
# ``litellm.model_cost`` map. Both effects belong to deployment configuration,
# not to user-supplied request bodies, so the proxy strips them before they
# reach the call path. Built from the Pydantic model so newly-added pricing
-# fields are covered automatically.
-_CLIENT_PRICING_CONTROL_FIELDS: Final = frozenset(CustomPricingLiteLLMParams.model_fields.keys())
+# fields are covered automatically. ``base_model`` selects which cost-map entry
+# prices the row, so a caller naming a cheaper model lowers their own spend.
+_CLIENT_PRICING_CONTROL_FIELDS: Final = frozenset(CustomPricingLiteLLMParams.model_fields.keys()) | {"base_model"}
# ``model_info`` carries the same pricing fields when read by
# ``use_custom_pricing_for_model``; strip from metadata for the same reason.
# ``standard_logging_guardrail_information`` is proxy-written telemetry summed
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 2b2c55a8452..20a0d4d5053 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -8565,7 +8565,7 @@ async def _create_spend_logs_with_poison_isolation(
await repo.table.create_many(data=rows, skip_duplicates=True)
return failure_budget
except Exception as e:
- if not PrismaDBExceptionHandler.is_prisma_data_error(e):
+ if not (PrismaDBExceptionHandler.is_prisma_data_error(e) or isinstance(e, UnicodeEncodeError)):
raise
if PrismaDBExceptionHandler.is_database_service_unavailable_error(e):
raise
diff --git a/tests/unit/proxy/common_utils/test_http_parsing_utils.py b/tests/unit/proxy/common_utils/test_http_parsing_utils.py
index 88ae5e0d84e..cd5820fa2fd 100644
--- a/tests/unit/proxy/common_utils/test_http_parsing_utils.py
+++ b/tests/unit/proxy/common_utils/test_http_parsing_utils.py
@@ -1556,3 +1556,39 @@ def test_is_otlp_trace_request_matches_only_http_posts_to_the_otlp_routes(
)
assert http_parsing_utils.is_otlp_trace_request(connection) is expected
+
+
+async def test_form_data_metadata_with_lone_surrogate_escape_is_rejected_like_a_json_body():
+ """``json.loads`` accepts ``"\\ud800"`` and yields a lone surrogate that no
+ provider or database can utf-8 encode. The JSON body path already answers
+ 400, so the multipart ``metadata`` field must too, instead of letting the
+ value through to the spend log writers."""
+ mock_request = MagicMock()
+ mock_request.form = AsyncMock(
+ return_value=FormData({"model": "whisper-1", "metadata": '{"tags": ["poison-\\ud800-tag"]}'})
+ )
+ mock_request.headers = {"content-type": "multipart/form-data"}
+ mock_request.scope = {}
+ mock_request.state._cached_headers = None
+
+ with pytest.raises(ProxyException) as exc_info:
+ await read_request_body(mock_request)
+
+ assert exc_info.value.code == "400"
+ assert "metadata" in exc_info.value.message
+
+
+async def test_form_data_metadata_keeps_non_ascii_and_valid_surrogate_pairs():
+ mock_request = MagicMock()
+ mock_request.form = AsyncMock(
+ return_value=FormData({"model": "whisper-1", "metadata": '{"tags": ["caf\\u00e9", "\\ud83d\\ude00"]}'})
+ )
+ mock_request.headers = {"content-type": "multipart/form-data"}
+ mock_request.scope = {}
+ mock_request.state._cached_headers = None
+
+ result = await read_request_body(mock_request)
+
+ assert result["metadata"]["tags"] == ["café", "😀"]
diff --git a/tests/unit/proxy/test_pricing_field_strip.py b/tests/unit/proxy/test_pricing_field_strip.py
index 395fdc3a16e..3e9bee9217d 100644
--- a/tests/unit/proxy/test_pricing_field_strip.py
+++ b/tests/unit/proxy/test_pricing_field_strip.py
@@ -57,7 +57,7 @@ class TestStripClientPricingOverrides:
# The strip set is built from the model so additions are picked up
# automatically — this test guards against the model and the strip
# set drifting apart if someone replaces the auto-derivation later.
- assert _CLIENT_PRICING_CONTROL_FIELDS == frozenset(CustomPricingLiteLLMParams.model_fields.keys())
+ assert _CLIENT_PRICING_CONTROL_FIELDS >= frozenset(CustomPricingLiteLLMParams.model_fields.keys())
# Sanity: the obvious top-level pricing fields are in the set.
for field in (
"input_cost_per_token",
@@ -82,6 +82,22 @@ class TestStripClientPricingOverrides:
"messages": [{"role": "user", "content": "hi"}],
}
+ def test_root_base_model_dropped(self):
+ # ``base_model`` picks the cost-map entry a spend row is priced with
+ # (success rows via the standard logging payload, billed failure rows
+ # via the failure lift), so a caller naming a cheaper model lowers
+ # their own recorded spend.
+ data = {
+ "model": "gpt-4o",
+ "messages": [{"role": "user", "content": "hi"}],
+ "base_model": "gpt-4o-mini",
+ }
+ _strip_client_pricing_overrides(data)
+ assert data == {
+ "model": "gpt-4o",
+ "messages": [{"role": "user", "content": "hi"}],
+ }
+
def test_metadata_model_info_dropped(self):
data = {
"model": "gpt-4",
@@ -280,6 +296,7 @@ async def test_add_litellm_data_to_request_skips_strip_with_key_opt_in():
"model": "gpt-4",
"messages": [{"role": "user", "content": "hi"}],
"input_cost_per_token": 0.0001,
+ "base_model": "gpt-4o-mini",
"metadata": {"model_info": {"output_cost_per_token": 0.0002}},
}
@@ -295,6 +312,7 @@ async def test_add_litellm_data_to_request_skips_strip_with_key_opt_in():
assert updated["input_cost_per_token"] == 0.0001
assert updated["metadata"]["model_info"] == {"output_cost_per_token": 0.0002}
+ assert updated["base_model"] == "gpt-4o-mini"
@pytest.mark.asyncio
diff --git a/tests/unit/proxy/test_update_spend.py b/tests/unit/proxy/test_update_spend.py
index 62adb7303e2..571341e85e7 100644
--- a/tests/unit/proxy/test_update_spend.py
+++ b/tests/unit/proxy/test_update_spend.py
@@ -323,3 +323,31 @@ async def test_update_spend_logs_multiple_batches_with_failure():
# Verify all logs were cleared from transactions
assert len(prisma_client.spend_log_transactions) == 0
+
+
+async def test_update_spend_logs_drops_row_prisma_cannot_encode_and_keeps_the_rest():
+ """A row the Prisma engine cannot utf-8 encode (a lone surrogate in a string
+ column) is permanent, so it must be dropped like any other poison row instead
+ of being requeued at the head of the queue where it would block every tenant."""
+ prisma_client = MockPrismaClient()
+ proxy_logging_obj = create_mock_proxy_logging()
+ prisma_client.spend_log_transactions = [
+ {"request_id": "good-1", "request_tags": '["a"]'},
+ {"request_id": "poison", "request_tags": "\ud800"},
+ {"request_id": "good-2", "request_tags": '["b"]'},
+ ]
+ written = []
+
+ async def create_many(data, skip_duplicates):
+ if any("\ud800" in row["request_tags"] for row in data):
+ raise UnicodeEncodeError("utf-8", "\ud800", 0, 1, "surrogates not allowed")
+ written.extend(row["request_id"] for row in data)
+
+ prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=create_many)
+
+ await update_spend(prisma_client, None, proxy_logging_obj)
+
+ assert sorted(written) == ["good-1", "good-2"]
+ assert prisma_client.spend_log_transactions == []
+ proxy_logging_obj.failure_handler.assert_not_called()