Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

fix(proxy): price a 499 spend row only for the proxy's own disconnect, and never after the call already completed (#47652)

Kerry Luffb7133d326e
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index 37df750b555..069dcffd21a 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -796,6 +796,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.defer_async_logging: bool = False
self.enqueue_deferred_logging: Callable[[], None] | None = None
self._async_success_scheduled: bool = False
+ self._async_success_enqueued: bool = False
self._sync_success_scheduled: bool = False
self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
@@ -2480,6 +2481,16 @@ class Logging(LiteLLMLoggingBaseClass):
except Exception:
return True
+ @property
+ def async_success_log_enqueued(self) -> bool:
+ """True once the success log task was actually created, not merely claimed, so a later failure raised
+ around it (a disconnect cancelling a still-pending guardrail task) must not bill the prompt a second
+ time. A claimed log parked in the deferred slot never runs once a 499 interrupts, so it stays False."""
+ return self._async_success_enqueued
+
+ def mark_async_success_log_enqueued(self) -> None:
+ self._async_success_enqueued = True
+
def claim_async_success_log(self) -> bool:
"""One async success log per request. The innermost @client wrapper always exits first,
since the outer one is awaiting it, so it claims the log here and the outer wrapper gets
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 147f9292a21..aca222c31ef 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -1386,8 +1386,9 @@ def _timed_out_reading_from_provider(exception: Exception) -> bool:
def _client_disconnected(exception: Exception) -> bool:
- """The 499 the proxy raises when the caller hung up while the provider was still working on the request."""
- return getattr(exception, "status_code", None) == LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED
+ """The 499 the proxy itself raises when the caller hung up while the provider was still working on the
+ request. A provider's own 499 arrives as a litellm exception and is a plain upstream failure."""
+ return isinstance(exception, HTTPException) and exception.status_code == LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED
def _provider_billed_the_failure(exception: Exception) -> bool:
@@ -1409,17 +1410,19 @@ def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed:
_litellm_params: Final = getattr(_logging_obj, "litellm_params", None)
_base_model: Final = get_base_model_from_metadata(model_call_details=_model_call_details)
_deployment_id: Final = _logging_obj.get_router_model_id() if isinstance(_logging_obj, Logging) else None
+ _success_already_logged: Final = isinstance(_logging_obj, Logging) and _logging_obj.async_success_log_enqueued
+ _billed: Final = provider_billed and not _success_already_logged
_usage_to_lift: Final = _failure_usage_to_lift(
model_call_details=_model_call_details,
request_body=request_data,
dispatched=_first_handoff is not None,
- provider_billed=provider_billed,
+ provider_billed=_billed,
deployment_id=_deployment_id,
base_model=_base_model,
)
_prompt_estimate: Final = (
_estimated_prompt_usage(_model_call_details, request_data)
- if provider_billed and _first_handoff is not None and not _model_call_details.get("failed_attempts_usage")
+ if _billed and _first_handoff is not None and not _model_call_details.get("failed_attempts_usage")
else None
)
_earlier_attempts: Final = (
diff --git a/litellm/utils.py b/litellm/utils.py
index 6b16fc398bd..987139b7530 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -1500,6 +1500,7 @@ def _schedule_async_success_logging(
return
def _enqueue_async_logging() -> None:
+ logging_obj.mark_async_success_log_enqueued()
asyncio.create_task(
_client_async_logging_helper(
logging_obj=logging_obj,
diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py
index 48ad3490ebb..960bbc9cba3 100644
--- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py
+++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py
@@ -828,6 +828,125 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens:
assert expected_input_cost > 0
assert request_data["response_cost"] == pytest.approx(expected_input_cost)
+ @pytest.mark.asyncio
+ async def test_upstream_499_error_is_a_plain_failure_not_a_priced_disconnect(self):
+ """A provider answering HTTP 499 surfaces as litellm.APIError with status_code 499. Only the
+ proxy's own disconnect HTTPException means the caller hung up mid-generation, so the upstream
+ error keeps the estimated usage at zero cost like any other post-dispatch provider error."""
+ from datetime import datetime
+
+ from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED
+ from litellm.types.utils import Usage
+
+ request_data = {
+ "litellm_logging_obj": self._logging_obj(
+ {
+ "first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 0),
+ "model": "gpt-3.5-turbo",
+ "messages": [{"role": "user", "content": "count these input tokens please"}],
+ "call_type": "acompletion",
+ }
+ ),
+ "metadata": {},
+ }
+ await self._run(
+ request_data,
+ original_exception=litellm.APIError(
+ status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED,
+ message="client closed request",
+ llm_provider="openai",
+ model="gpt-3.5-turbo",
+ ),
+ )
+
+ estimated = request_data["combined_usage_object"]
+ assert isinstance(estimated, Usage)
+ assert estimated.prompt_tokens > 0
+ assert request_data["response_cost"] == 0.0
+
+ @pytest.mark.asyncio
+ async def test_client_disconnect_after_the_call_already_completed_is_not_priced_again(self):
+ """The LLM task finished and claimed its success log (which carries the real usage and cost)
+ while a during_call guardrail task was still pending, then the caller hung up. The 499 raised
+ for that disconnect must not price the prompt a second time."""
+ from datetime import datetime
+
+ from fastapi import HTTPException
+
+ from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED
+ from litellm.litellm_core_utils.litellm_logging import Logging
+
+ logging_obj = Logging(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "count these input tokens please"}],
+ stream=False,
+ call_type="acompletion",
+ start_time=datetime(2026, 1, 1, 12, 0, 0),
+ litellm_call_id="completed-before-disconnect",
+ function_id="fn",
+ )
+ logging_obj.model_call_details.update(
+ {
+ "first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 1),
+ "model": "gpt-3.5-turbo",
+ "messages": [{"role": "user", "content": "count these input tokens please"}],
+ "call_type": "acompletion",
+ }
+ )
+ assert logging_obj.claim_async_success_log()
+ logging_obj.mark_async_success_log_enqueued()
+ request_data = {"litellm_logging_obj": logging_obj, "metadata": {}}
+ await self._run(
+ request_data,
+ original_exception=HTTPException(
+ status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request"
+ ),
+ )
+
+ assert request_data.get("response_cost", 0.0) == 0.0
+
+ @pytest.mark.asyncio
+ async def test_client_disconnect_with_a_claimed_but_deferred_success_log_is_still_priced(self):
+ """With a post_call guardrail registered the success log is claimed but parked in the deferred slot,
+ and a 499 raised before the flush means it never runs. Nothing else bills that request, so the
+ failure row keeps the estimated input cost."""
+ from datetime import datetime
+
+ from fastapi import HTTPException
+
+ from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED
+ from litellm.litellm_core_utils.litellm_logging import Logging
+
+ logging_obj = Logging(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "count these input tokens please"}],
+ stream=False,
+ call_type="acompletion",
+ start_time=datetime(2026, 1, 1, 12, 0, 0),
+ litellm_call_id="deferred-success-never-flushed",
+ function_id="fn",
+ )
+ logging_obj.model_call_details.update(
+ {
+ "first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 1),
+ "model": "gpt-3.5-turbo",
+ "messages": [{"role": "user", "content": "count these input tokens please"}],
+ "call_type": "acompletion",
+ }
+ )
+ logging_obj.defer_async_logging = True
+ assert logging_obj.claim_async_success_log()
+ logging_obj.enqueue_deferred_logging = lambda: None
+ request_data = {"litellm_logging_obj": logging_obj, "metadata": {}}
+ await self._run(
+ request_data,
+ original_exception=HTTPException(
+ status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request"
+ ),
+ )
+
+ assert request_data["response_cost"] > 0.0
+
@pytest.mark.asyncio
async def test_client_disconnect_before_provider_handoff_is_not_priced(self):
"""A caller that hung up while the request was still in pre-call hooks never reached the