The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
fix(router): normalize native Responses latency per output token (#47727)
joshua-berri230f653523fa
diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.pyindex 83bb22435e6..28ed062f1ec 100644--- a/litellm/router_strategy/lowest_latency.py+++ b/litellm/router_strategy/lowest_latency.py@@ -14,7 +14,9 @@ from litellm._internal_context import with_service_targetfrom litellm.caching.caching import DualCachefrom litellm.integrations.custom_logger import CustomLoggerfrom litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs, safe_divide_seconds+from litellm.responses.utils import ResponseAPILoggingUtilsfrom litellm.router_utils.batch_utils import is_batch_retrieve_call_type+from litellm.types.llms.openai import ResponsesAPIResponsefrom litellm.types.utils import LiteLLMPydanticObjectBaseif TYPE_CHECKING:@@ -86,7 +88,7 @@ class LowestLatencyLoggingHandler(CustomLogger):{{model_group}_map: {id: {- "latency": [..]+ "latency_normalized": [..]f"{date:hour:minute}" : {"tpm": 34, "rpm": 3}}}@@ -116,11 +118,16 @@ class LowestLatencyLoggingHandler(CustomLogger):final_value: float = response_mstotal_tokens = 0- if isinstance(response_obj, ModelResponse):- _usage: Final = getattr(response_obj, "usage", None)+ if isinstance(response_obj, (ModelResponse, ResponsesAPIResponse)):+ _usage: Final = (+ ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response_obj.usage)+ if isinstance(response_obj, ResponsesAPIResponse)+ else getattr(response_obj, "usage", None)+ )if _usage is not None:completion_tokens: Final = _usage.completion_tokens- total_tokens = _usage.total_tokens+ if isinstance(response_obj, ModelResponse):+ total_tokens = _usage.total_tokens# response_ms is already normalized to float seconds aboveresponse_seconds: Final = response_ms@@ -143,10 +150,12 @@ class LowestLatencyLoggingHandler(CustomLogger):request_count_dict[id] = {}## Latency- if len(request_count_dict[id].get("latency", [])) < self.routing_args.max_latency_list_size:- request_count_dict[id].setdefault("latency", []).append(final_value)+ if len(request_count_dict[id].get("latency_normalized", [])) < self.routing_args.max_latency_list_size:+ request_count_dict[id].setdefault("latency_normalized", []).append(final_value)else:- request_count_dict[id]["latency"] = request_count_dict[id]["latency"][1:] + [final_value]+ request_count_dict[id]["latency_normalized"] = request_count_dict[id]["latency_normalized"][1:] + [+ final_value+ ]## Time to first tokenif time_to_first_token is not None:@@ -212,7 +221,7 @@ class LowestLatencyLoggingHandler(CustomLogger):{{model_group}_map: {id: {- "latency": [..]+ "latency_normalized": [..]f"{date:hour:minute}" : {"tpm": 34, "rpm": 3}}}@@ -225,10 +234,15 @@ class LowestLatencyLoggingHandler(CustomLogger):request_count_dict[id] = {}## Latency - give 1000s penalty for failing- if len(request_count_dict[id].get("latency", [])) < self.routing_args.max_latency_list_size:- request_count_dict[id].setdefault("latency", []).append(1000.0)+ if (+ len(request_count_dict[id].get("latency_normalized", []))+ < self.routing_args.max_latency_list_size+ ):+ request_count_dict[id].setdefault("latency_normalized", []).append(1000.0)else:- request_count_dict[id]["latency"] = request_count_dict[id]["latency"][1:] + [1000.0]+ request_count_dict[id]["latency_normalized"] = request_count_dict[id]["latency_normalized"][+ 1:+ ] + [1000.0]await self.router_cache.async_set_cache(key=latency_key,@@ -270,7 +284,7 @@ class LowestLatencyLoggingHandler(CustomLogger):{{model_group}_map: {id: {- "latency": [..]+ "latency_normalized": [..]"time_to_first_token_seconds": [..]f"{date:hour:minute}" : {"tpm": 34, "rpm": 3}}@@ -300,11 +314,16 @@ class LowestLatencyLoggingHandler(CustomLogger):final_value: float = response_mstotal_tokens = 0- if isinstance(response_obj, ModelResponse):- _usage: Final = getattr(response_obj, "usage", None)+ if isinstance(response_obj, (ModelResponse, ResponsesAPIResponse)):+ _usage: Final = (+ ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response_obj.usage)+ if isinstance(response_obj, ResponsesAPIResponse)+ else getattr(response_obj, "usage", None)+ )if _usage is not None:completion_tokens: Final = _usage.completion_tokens- total_tokens = _usage.total_tokens+ if isinstance(response_obj, ModelResponse):+ total_tokens = _usage.total_tokens# response_ms is already normalized to float seconds aboveresponse_seconds: Final = response_ms@@ -331,10 +350,12 @@ class LowestLatencyLoggingHandler(CustomLogger):request_count_dict[id] = {}## Latency- if len(request_count_dict[id].get("latency", [])) < self.routing_args.max_latency_list_size:- request_count_dict[id].setdefault("latency", []).append(final_value)+ if len(request_count_dict[id].get("latency_normalized", [])) < self.routing_args.max_latency_list_size:+ request_count_dict[id].setdefault("latency_normalized", []).append(final_value)else:- request_count_dict[id]["latency"] = request_count_dict[id]["latency"][1:] + [final_value]+ request_count_dict[id]["latency_normalized"] = request_count_dict[id]["latency_normalized"][1:] + [+ final_value+ ]## Time to first tokenif time_to_first_token is not None:@@ -403,7 +424,7 @@ class LowestLatencyLoggingHandler(CustomLogger):## if healthy deployment not yet usedif d["model_info"]["id"] not in all_deployments:all_deployments[d["model_info"]["id"]] = {- "latency": [0],+ "latency_normalized": [0],precise_minute: {"tpm": 0, "rpm": 0},}@@ -443,7 +464,7 @@ class LowestLatencyLoggingHandler(CustomLogger):or _deployment.get("model_info", {}).get("rpm", None)or float("inf"))- item_latency = item_map.get("latency", [])+ item_latency = item_map.get("latency_normalized", [])item_ttft_latency = item_map.get("time_to_first_token_seconds", [])item_rpm = item_map.get(precise_minute, {}).get("rpm", 0)item_tpm = item_map.get(precise_minute, {}).get("tpm", 0)diff --git a/tests/unit/router_strategy/test_lowest_latency.py b/tests/unit/router_strategy/test_lowest_latency.pyindex 0a3a92691ba..9af5def0d56 100644--- a/tests/unit/router_strategy/test_lowest_latency.py+++ b/tests/unit/router_strategy/test_lowest_latency.py@@ -17,8 +17,10 @@ from pydantic import ValidationErrorimport litellmfrom litellm.caching.caching import DualCachefrom litellm.caching.in_memory_cache import InMemoryCache+from litellm.responses.utils import ResponseAPILoggingUtilsfrom litellm.router import Routerfrom litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler, RoutingArgs+from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponseDEPLOYMENT_ID = "9876"KWARGS = {@@ -43,7 +45,7 @@ def _embedding_response() -> litellm.EmbeddingResponse:def _recorded_latencies(cache: DualCache):cached = cache.get_cache(key="gemini-embedding-001_map") or {}- return cached.get(DEPLOYMENT_ID, {}).get("latency", [])+ return cached.get(DEPLOYMENT_ID, {}).get("latency_normalized", [])def test_sync_embedding_latency_is_json_serializable():@@ -69,7 +71,7 @@ def test_sync_embedding_latency_is_json_serializable():)assert latencies[-1] == pytest.approx(2.0)# the exact failure mode from production: redis cache sync json.dumps- json.dumps({"latency": latencies})+ json.dumps({"latency_normalized": latencies})@pytest.mark.asyncio@@ -94,7 +96,7 @@ async def test_async_embedding_latency_is_json_serializable():f"raw timedelta leaked into latency list: {latencies}")assert latencies[-1] == pytest.approx(3.0)- json.dumps({"latency": latencies})+ json.dumps({"latency_normalized": latencies})def _chat_response(completion_tokens: int) -> litellm.ModelResponse:@@ -131,7 +133,7 @@ async def test_async_chat_latency_normalized_per_token():latencies = _recorded_latencies(cache)assert latencies and latencies[-1] == pytest.approx(0.5) # 2s / 4 tokens- json.dumps({"latency": latencies})+ json.dumps({"latency_normalized": latencies})@pytest.mark.asyncio@@ -151,7 +153,7 @@ async def test_async_chat_zero_completion_tokens_falls_back_to_seconds():latencies = _recorded_latencies(cache)assert latencies and latencies[-1] == pytest.approx(3.0)assert not isinstance(latencies[-1], timedelta)- json.dumps({"latency": latencies})+ json.dumps({"latency_normalized": latencies})def test_sync_chat_zero_completion_tokens_falls_back_to_seconds():@@ -168,7 +170,7 @@ def test_sync_chat_zero_completion_tokens_falls_back_to_seconds():latencies = _recorded_latencies(cache)assert latencies and latencies[-1] == pytest.approx(2.0)assert not isinstance(latencies[-1], timedelta)- json.dumps({"latency": latencies})+ json.dumps({"latency_normalized": latencies})MODEL_GROUP = "gpt-4o-mini"@@ -362,8 +364,8 @@ async def test_ttft_percentile_does_not_change_non_streaming_routing():cache.set_cache(key=f"{MODEL_GROUP}_map",value={- FAST_TTFT_ID: {"latency": [1.0], "time_to_first_token_seconds": [0.1]},- SLOW_TTFT_ID: {"latency": [0.2], "time_to_first_token_seconds": [1.5]},+ FAST_TTFT_ID: {"latency_normalized": [1.0], "time_to_first_token_seconds": [0.1]},+ SLOW_TTFT_ID: {"latency_normalized": [0.2], "time_to_first_token_seconds": [1.5]},},)@@ -380,7 +382,7 @@ async def test_ttft_percentile_does_not_change_non_streaming_routing():@pytest.mark.asyncio@pytest.mark.parametrize("cached_entry",- [{"latency": []}, {"2026-09-05-15-39": {"tpm": 28, "rpm": 1}}],+ [{"latency_normalized": []}, {"2026-09-05-15-39": {"tpm": 28, "rpm": 1}}],ids=["empty_latency_list", "minute_bucket_only_as_cost_based_routing_writes"],)async def test_async_get_available_deployments_treats_missing_samples_as_zero_latency(cached_entry):@@ -388,7 +390,7 @@ async def test_async_get_available_deployments_treats_missing_samples_as_zero_lahandler = LowestLatencyLoggingHandler(router_cache=cache)cache.set_cache(key="gemini-embedding-001_map",- value={DEPLOYMENT_ID: cached_entry, "slower": {"latency": [0.5]}},+ value={DEPLOYMENT_ID: cached_entry, "slower": {"latency_normalized": [0.5]}},)healthy_deployments = [{"model_info": {"id": DEPLOYMENT_ID}, "litellm_params": {}},@@ -605,7 +607,7 @@ def test_latency_updated():end_time=end_time,)latency_key = f"{model_group}_map"- assert end_time - start_time == test_cache.get_cache(key=latency_key)[deployment_id]["latency"][0]+ assert end_time - start_time == test_cache.get_cache(key=latency_key)[deployment_id]["latency_normalized"][0]@pytest.mark.usefixtures("frozen_latency_clock")@@ -1002,7 +1004,7 @@ def test_latency_list_trimming_discards_oldest_entry():latency_key = f"{model_group}_map"cached_data = test_cache.get_cache(key=latency_key)- latency_list = cached_data[deployment_id]["latency"]+ latency_list = cached_data[deployment_id]["latency_normalized"]assert len(latency_list) == max_size, f"Expected {max_size} entries, got {len(latency_list)}"@@ -1062,7 +1064,7 @@ async def test_latency_list_trimming_discards_oldest_entry_async():latency_key = f"{model_group}_map"cached_data = await test_cache.async_get_cache(key=latency_key)- latency_list = cached_data[deployment_id]["latency"]+ latency_list = cached_data[deployment_id]["latency_normalized"]assert len(latency_list) == max_size@@ -1129,7 +1131,7 @@ async def test_timeout_penalty_discards_oldest_entry():latency_key = f"{model_group}_map"cached_data = await test_cache.async_get_cache(key=latency_key)- latency_list = cached_data[deployment_id]["latency"]+ latency_list = cached_data[deployment_id]["latency_normalized"]assert len(latency_list) == max_size@@ -1181,7 +1183,7 @@ def test_list_order_preserved_after_multiple_trims():latency_key = f"{model_group}_map"cached_data = test_cache.get_cache(key=latency_key)- latency_list = cached_data[deployment_id]["latency"]+ latency_list = cached_data[deployment_id]["latency_normalized"]assert len(latency_list) == max_size@@ -1423,7 +1425,7 @@ def test_latency_cache_honors_custom_ttl() -> None:latency_key: Final = "ttl-group_map"cached: Final = cache.get_cache(key=latency_key)assert isinstance(cached, dict)- assert cached["deployment"]["latency"] == [1.0]+ assert cached["deployment"]["latency_normalized"] == [1.0]assert cache.in_memory_cache.ttl_dict[latency_key] == 103.0clock.return_value = 104.0@@ -1461,3 +1463,191 @@ async def test_router_model_group_usage_increases_by_logged_tokens() -> None:updated_usage: Final = await router.get_model_group_usage(model_group=model_group)assert updated_usage[0] == initial_usage[0] + 15++[email protected]("async_callback", [False, True])[email protected]("normalized_usage", [False, True])[email protected]("native_seconds", [2, 3])+async def test_responses_and_chat_latency_use_comparable_token_units(+ async_callback: bool, normalized_usage: bool, native_seconds: int+) -> None:+ cache: Final = DualCache()+ handler: Final = LowestLatencyLoggingHandler(router_cache=cache)+ usage: Final = ResponseAPIUsage(input_tokens=10, output_tokens=100, total_tokens=110)+ native: Final = ResponsesAPIResponse(id="resp-latency", created_at=0, output=[], usage=usage)+ logged_native: Final = (+ native.model_copy(update={"usage": ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage)})+ if normalized_usage+ else native+ )+ original_usage: Final = logged_native.usage+ assert original_usage is not None+ original_fields: Final = original_usage.model_dump()+ start: Final = datetime(2026, 1, 1)+ for deployment_id, response, seconds in (+ ("native", logged_native, native_seconds),+ ("bridge", _chat_response(100), 3),+ ):+ kwargs: Final = {+ "call_type": "aresponses",+ "litellm_params": {"metadata": {"model_group": "mixed"}, "model_info": {"id": deployment_id}},+ }+ if async_callback:+ await handler.async_log_success_event(kwargs, response, start, start + timedelta(seconds=seconds))+ else:+ handler.log_success_event(kwargs, response, start, start + timedelta(seconds=seconds))+ recorded: Final = cache.get_cache(key="mixed_map")+ assert recorded["native"]["latency_normalized"] == pytest.approx([native_seconds / 100])+ assert recorded["bridge"]["latency_normalized"] == pytest.approx([3 / 100])+ assert logged_native.usage is original_usage+ assert logged_native.usage.model_dump() == original_fields+ assert native.usage is usage+ deployments: Final = [+ {"model_name": "mixed", "model_info": {"id": name}, "litellm_params": {}} for name in ("native", "bridge")+ ]+ picked: Final = handler.get_available_deployments(+ model_group="mixed", healthy_deployments=deployments, request_kwargs={"stream": False, "metadata": {}}+ )+ assert picked["model_info"]["id"] in ({"native"} if native_seconds < 3 else {"native", "bridge"})+ native_counts: Final = tuple(value for key, value in recorded["native"].items() if key != "latency_normalized")+ assert native_counts == ({"tpm": 0, "rpm": 1},)++[email protected]("async_callback", [False, True])+ "response",+ [+ litellm.ModelResponse(),+ ResponsesAPIResponse(id="resp-missing", created_at=0, output=[], usage=None),+ ResponsesAPIResponse(+ id="resp-zero", created_at=0, output=[],+ usage=ResponseAPIUsage(input_tokens=10, output_tokens=0, total_tokens=10),+ ),+ ],+)+async def test_responses_without_output_tokens_keep_elapsed_seconds(+ async_callback: bool, response: litellm.ModelResponse | ResponsesAPIResponse+) -> None:+ cache: Final = DualCache()+ handler: Final = LowestLatencyLoggingHandler(router_cache=cache)+ original_usage: Final = getattr(response, "usage", None)+ if async_callback:+ await handler.async_log_success_event(KWARGS, response, 1.0, 3.0)+ else:+ handler.log_success_event(KWARGS, response, 1.0, 3.0)+ assert _recorded_latencies(cache) == [2.0]+ assert getattr(response, "usage", None) is original_usage++[email protected]("frozen_latency_clock")[email protected]("async_mode", [False, True])[email protected]("preserve_new_samples", [False, True])+async def test_legacy_latency_does_not_starve_deployments_after_upgrade(+ async_mode: bool, preserve_new_samples: bool+) -> None:+ cache: Final = DualCache()+ handler: Final = LowestLatencyLoggingHandler(router_cache=cache)+ minute: Final = FROZEN_NOW.strftime("%Y-%m-%d-%H-%M")+ legacy: Final = {+ "native": {"latency": [2.0], "time_to_first_token_seconds": [0.1], minute: {"tpm": 0, "rpm": 1}},+ "bridge": {"latency": [0.03], "time_to_first_token_seconds": [0.2], minute: {"tpm": 110, "rpm": 1}},+ }+ cache.set_cache(key="mixed_map", value=copy.deepcopy(legacy))+ deployments: Final = [+ {"model_name": "mixed", "model_info": {"id": name}, "litellm_params": {}} for name in ("native", "bridge")+ ]+ native: Final = ResponsesAPIResponse(+ id="resp-upgrade", created_at=0, output=[],+ usage=ResponseAPIUsage(input_tokens=10, output_tokens=100, total_tokens=110),+ )+ for iteration in range(2):+ bridge_kwargs: Final = {+ "litellm_params": {"metadata": {"model_group": "mixed"}, "model_info": {"id": "bridge"}},+ }+ if async_mode:+ await handler.async_log_success_event(bridge_kwargs, _chat_response(100), 0.0, 3.0)+ else:+ handler.log_success_event(bridge_kwargs, _chat_response(100), 0.0, 3.0)+ selected: Final = (+ await handler.async_get_available_deployments(model_group="mixed", healthy_deployments=deployments)+ if async_mode+ else handler.get_available_deployments(model_group="mixed", healthy_deployments=deployments)+ )+ assert selected["model_info"]["id"] == "native"+ native_kwargs: Final = {+ "litellm_params": {"metadata": {"model_group": "mixed"}, "model_info": {"id": "native"}},+ }+ if async_mode:+ await handler.async_log_success_event(native_kwargs, native, 0.0, 2.0)+ else:+ handler.log_success_event(native_kwargs, native, 0.0, 2.0)+ recorded: Final = copy.deepcopy(cache.get_cache(key="mixed_map"))+ assert recorded["native"]["latency_normalized"] == pytest.approx(+ [0.02] * (iteration + 1 if preserve_new_samples else 1)+ )+ assert recorded["native"]["latency"] == [2.0]+ assert recorded["native"]["time_to_first_token_seconds"] == [0.1]+ assert recorded["bridge"]["time_to_first_token_seconds"] == [0.2]+ assert recorded["native"][minute] == {"tpm": 0, "rpm": iteration + 2}+ assert recorded["bridge"][minute] == {"tpm": 110 + (iteration + 1) * 110, "rpm": iteration + 2}+ streaming: Final = handler.get_available_deployments(+ model_group="mixed", healthy_deployments=deployments, request_kwargs={"stream": True}+ )+ assert streaming["model_info"]["id"] == "native"+ cache.set_cache(+ key="mixed_map",+ value={+ name: {**(recorded[name] if preserve_new_samples else legacy[name]), minute: recorded[name][minute]}+ for name in ("native", "bridge")+ },+ )++[email protected]("async_mode", [False, True])+async def test_normalized_latency_window_evicts_samples_without_changing_legacy_data(async_mode: bool) -> None:+ cache: Final = DualCache()+ cache.set_cache(key="gemini-embedding-001_map", value={DEPLOYMENT_ID: {"latency": [2.0]}})+ handler: Final = LowestLatencyLoggingHandler(router_cache=cache, routing_args={"max_latency_list_size": 2})+ for elapsed, expected in ((2.0, [0.02]), (3.0, [0.02, 0.03]), (4.0, [0.03, 0.04])):+ if async_mode:+ await handler.async_log_success_event(KWARGS, _chat_response(100), 0.0, elapsed)+ else:+ handler.log_success_event(KWARGS, _chat_response(100), 0.0, elapsed)+ recorded: Final = cache.get_cache(key="gemini-embedding-001_map")[DEPLOYMENT_ID]+ assert recorded["latency_normalized"] == pytest.approx(expected)+ assert recorded["latency"] == [2.0]++[email protected]("existing_samples", [(), (0.02, 0.03)])+async def test_timeout_penalty_updates_normalized_latency_and_avoids_failed_deployment(+ existing_samples: tuple[float, ...],+) -> None:+ cache: Final = DualCache()+ handler: Final = LowestLatencyLoggingHandler(router_cache=cache, routing_args={"max_latency_list_size": 2})+ cache.set_cache(+ key="mixed_map",+ value={+ "native": {"latency": [0.001], "latency_normalized": list(existing_samples)},+ "bridge": {"latency": [2.0], "latency_normalized": [0.04]},+ },+ )+ await handler.async_log_failure_event(+ kwargs={+ "exception": litellm.Timeout(message="request timed out", model="test", llm_provider="openai"),+ "litellm_params": {"metadata": {"model_group": "mixed"}, "model_info": {"id": "native"}},+ },+ response_obj=None, start_time=0.0, end_time=3.0,+ )+ recorded: Final = cache.get_cache(key="mixed_map")["native"]+ assert recorded["latency_normalized"] == [*existing_samples[-1:], 1000.0]+ assert recorded["latency"] == [0.001]+ selected: Final = await handler.async_get_available_deployments(+ model_group="mixed",+ healthy_deployments=[{"model_info": {"id": name}, "litellm_params": {}} for name in ("native", "bridge")],+ )+ assert selected["model_info"]["id"] == "bridge"diff --git a/tests/unit/router_strategy/test_router_routing_groups.py b/tests/unit/router_strategy/test_router_routing_groups.pyindex 7e4b470f1c1..7235333f59f 100644--- a/tests/unit/router_strategy/test_router_routing_groups.py+++ b/tests/unit/router_strategy/test_router_routing_groups.py@@ -1249,7 +1249,7 @@ async def test_latency_based_override_learns_from_the_overriding_requests():)def samples(deployment_id: str) -> list[float]:- recorded = (router.cache.get_cache("grp_map") or {}).get(deployment_id, {}).get("latency", [])+ recorded = (router.cache.get_cache("grp_map") or {}).get(deployment_id, {}).get("latency_normalized", [])return [latency for latency in recorded if latency > 0]async def overriding_call() -> str:@@ -1316,7 +1316,7 @@ async def test_override_matching_a_routing_group_strategy_records_each_request_odeployment_id = response._hidden_params["model_id"]await _async_wait_until(lambda: (router.cache.get_cache("grp_map") or {}).get(deployment_id) is not None)- assert len(router.cache.get_cache("grp_map")[deployment_id]["latency"]) == 1+ assert len(router.cache.get_cache("grp_map")[deployment_id]["latency_normalized"]) == 1assert request["litellm_logging_obj"].dynamic_success_callbacks is None