The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
fix(proxy): price a 499 spend row only for the proxy's own disconnect, and never after the call already completed (#47652)
Kerry Luffb7133d326e
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.pyindex 37df750b555..069dcffd21a 100644--- a/litellm/litellm_core_utils/litellm_logging.py+++ b/litellm/litellm_core_utils/litellm_logging.py@@ -796,6 +796,7 @@ class Logging(LiteLLMLoggingBaseClass):self.defer_async_logging: bool = Falseself.enqueue_deferred_logging: Callable[[], None] | None = Noneself._async_success_scheduled: bool = False+ self._async_success_enqueued: bool = Falseself._sync_success_scheduled: bool = Falseself.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = Noneself.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None@@ -2480,6 +2481,16 @@ class Logging(LiteLLMLoggingBaseClass):except Exception:return True+ @property+ def async_success_log_enqueued(self) -> bool:+ """True once the success log task was actually created, not merely claimed, so a later failure raised+ around it (a disconnect cancelling a still-pending guardrail task) must not bill the prompt a second+ time. A claimed log parked in the deferred slot never runs once a 499 interrupts, so it stays False."""+ return self._async_success_enqueued++ def mark_async_success_log_enqueued(self) -> None:+ self._async_success_enqueued = True+def claim_async_success_log(self) -> bool:"""One async success log per request. The innermost @client wrapper always exits first,since the outer one is awaiting it, so it claims the log here and the outer wrapper getsdiff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.pyindex 147f9292a21..aca222c31ef 100644--- a/litellm/proxy/utils.py+++ b/litellm/proxy/utils.py@@ -1386,8 +1386,9 @@ def _timed_out_reading_from_provider(exception: Exception) -> bool:def _client_disconnected(exception: Exception) -> bool:- """The 499 the proxy raises when the caller hung up while the provider was still working on the request."""- return getattr(exception, "status_code", None) == LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED+ """The 499 the proxy itself raises when the caller hung up while the provider was still working on the+ request. A provider's own 499 arrives as a litellm exception and is a plain upstream failure."""+ return isinstance(exception, HTTPException) and exception.status_code == LITELLM_HTTP_STATUS_CLIENT_DISCONNECTEDdef _provider_billed_the_failure(exception: Exception) -> bool:@@ -1409,17 +1410,19 @@ def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed:_litellm_params: Final = getattr(_logging_obj, "litellm_params", None)_base_model: Final = get_base_model_from_metadata(model_call_details=_model_call_details)_deployment_id: Final = _logging_obj.get_router_model_id() if isinstance(_logging_obj, Logging) else None+ _success_already_logged: Final = isinstance(_logging_obj, Logging) and _logging_obj.async_success_log_enqueued+ _billed: Final = provider_billed and not _success_already_logged_usage_to_lift: Final = _failure_usage_to_lift(model_call_details=_model_call_details,request_body=request_data,dispatched=_first_handoff is not None,- provider_billed=provider_billed,+ provider_billed=_billed,deployment_id=_deployment_id,base_model=_base_model,)_prompt_estimate: Final = (_estimated_prompt_usage(_model_call_details, request_data)- if provider_billed and _first_handoff is not None and not _model_call_details.get("failed_attempts_usage")+ if _billed and _first_handoff is not None and not _model_call_details.get("failed_attempts_usage")else None)_earlier_attempts: Final = (diff --git a/litellm/utils.py b/litellm/utils.pyindex 6b16fc398bd..987139b7530 100644--- a/litellm/utils.py+++ b/litellm/utils.py@@ -1500,6 +1500,7 @@ def _schedule_async_success_logging(returndef _enqueue_async_logging() -> None:+ logging_obj.mark_async_success_log_enqueued()asyncio.create_task(_client_async_logging_helper(logging_obj=logging_obj,diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.pyindex 48ad3490ebb..960bbc9cba3 100644--- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py+++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py@@ -828,6 +828,125 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens:assert expected_input_cost > 0assert request_data["response_cost"] == pytest.approx(expected_input_cost)+ @pytest.mark.asyncio+ async def test_upstream_499_error_is_a_plain_failure_not_a_priced_disconnect(self):+ """A provider answering HTTP 499 surfaces as litellm.APIError with status_code 499. Only the+ proxy's own disconnect HTTPException means the caller hung up mid-generation, so the upstream+ error keeps the estimated usage at zero cost like any other post-dispatch provider error."""+ from datetime import datetime++ from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED+ from litellm.types.utils import Usage++ request_data = {+ "litellm_logging_obj": self._logging_obj(+ {+ "first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 0),+ "model": "gpt-3.5-turbo",+ "messages": [{"role": "user", "content": "count these input tokens please"}],+ "call_type": "acompletion",+ }+ ),+ "metadata": {},+ }+ await self._run(+ request_data,+ original_exception=litellm.APIError(+ status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED,+ message="client closed request",+ llm_provider="openai",+ model="gpt-3.5-turbo",+ ),+ )++ estimated = request_data["combined_usage_object"]+ assert isinstance(estimated, Usage)+ assert estimated.prompt_tokens > 0+ assert request_data["response_cost"] == 0.0++ @pytest.mark.asyncio+ async def test_client_disconnect_after_the_call_already_completed_is_not_priced_again(self):+ """The LLM task finished and claimed its success log (which carries the real usage and cost)+ while a during_call guardrail task was still pending, then the caller hung up. The 499 raised+ for that disconnect must not price the prompt a second time."""+ from datetime import datetime++ from fastapi import HTTPException++ from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED+ from litellm.litellm_core_utils.litellm_logging import Logging++ logging_obj = Logging(+ model="gpt-3.5-turbo",+ messages=[{"role": "user", "content": "count these input tokens please"}],+ stream=False,+ call_type="acompletion",+ start_time=datetime(2026, 1, 1, 12, 0, 0),+ litellm_call_id="completed-before-disconnect",+ function_id="fn",+ )+ logging_obj.model_call_details.update(+ {+ "first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 1),+ "model": "gpt-3.5-turbo",+ "messages": [{"role": "user", "content": "count these input tokens please"}],+ "call_type": "acompletion",+ }+ )+ assert logging_obj.claim_async_success_log()+ logging_obj.mark_async_success_log_enqueued()+ request_data = {"litellm_logging_obj": logging_obj, "metadata": {}}+ await self._run(+ request_data,+ original_exception=HTTPException(+ status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request"+ ),+ )++ assert request_data.get("response_cost", 0.0) == 0.0++ @pytest.mark.asyncio+ async def test_client_disconnect_with_a_claimed_but_deferred_success_log_is_still_priced(self):+ """With a post_call guardrail registered the success log is claimed but parked in the deferred slot,+ and a 499 raised before the flush means it never runs. Nothing else bills that request, so the+ failure row keeps the estimated input cost."""+ from datetime import datetime++ from fastapi import HTTPException++ from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED+ from litellm.litellm_core_utils.litellm_logging import Logging++ logging_obj = Logging(+ model="gpt-3.5-turbo",+ messages=[{"role": "user", "content": "count these input tokens please"}],+ stream=False,+ call_type="acompletion",+ start_time=datetime(2026, 1, 1, 12, 0, 0),+ litellm_call_id="deferred-success-never-flushed",+ function_id="fn",+ )+ logging_obj.model_call_details.update(+ {+ "first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 1),+ "model": "gpt-3.5-turbo",+ "messages": [{"role": "user", "content": "count these input tokens please"}],+ "call_type": "acompletion",+ }+ )+ logging_obj.defer_async_logging = True+ assert logging_obj.claim_async_success_log()+ logging_obj.enqueue_deferred_logging = lambda: None+ request_data = {"litellm_logging_obj": logging_obj, "metadata": {}}+ await self._run(+ request_data,+ original_exception=HTTPException(+ status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request"+ ),+ )++ assert request_data["response_cost"] > 0.0+@pytest.mark.asyncioasync def test_client_disconnect_before_provider_handoff_is_not_priced(self):"""A caller that hung up while the request was still in pre-call hooks never reached the