The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
test(perf): add chat_completions_2rps, responses_2rps and messages_2rps direct vs bridge cases (#47752)
Kerry Lu8d9b9926d165
diff --git a/tests/e2e/gateway/perf_ci_config.yml b/tests/e2e/gateway/perf_ci_config.ymlindex 7ab6637e985..767db313847 100644--- a/tests/e2e/gateway/perf_ci_config.yml+++ b/tests/e2e/gateway/perf_ci_config.yml@@ -46,6 +46,11 @@ model_list:model: openai/gpt-6-lunaapi_base: os.environ/E2E_PERF_MOCK_OPENAI_URLapi_key: perf-mock-not-checked+ - model_name: gpt-6-luna-responses+ litellm_params:+ model: openai/responses/gpt-6-luna+ api_base: os.environ/E2E_PERF_MOCK_OPENAI_URL+ api_key: perf-mock-not-checked- model_name: claude-sonnet-5litellm_params:model: anthropic/claude-sonnet-5@@ -58,6 +63,13 @@ model_list:aws_region_name: us-east-1aws_access_key_id: perf-mock-not-checkedaws_secret_access_key: perf-mock-not-checked+ - model_name: claude-sonnet-5-bedrock-converse+ litellm_params:+ model: bedrock/converse/anthropic.claude-sonnet-5+ aws_bedrock_runtime_endpoint: os.environ/E2E_PERF_MOCK_BEDROCK_URL+ aws_region_name: us-east-1+ aws_access_key_id: perf-mock-not-checked+ aws_secret_access_key: perf-mock-not-checkedguardrails:- guardrail_name: perf-prelitellm_params:diff --git a/tests/e2e/perf/cases/chat_completions_2rps.py b/tests/e2e/perf/cases/chat_completions_2rps.pynew file mode 100644index 00000000000..9792d99c14f--- /dev/null+++ b/tests/e2e/perf/cases/chat_completions_2rps.py@@ -0,0 +1,15 @@+"""`/v1/chat/completions` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's chat completions+route, `gpt-6-luna-responses` (`openai/responses/gpt-6-luna`) is bridged to its Responses route."""++from __future__ import annotations++from typing import Final++from .direct_vs_bridge import direct_vs_bridge++CASE: Final = direct_vs_bridge(+ "chat_completions_2rps",+ "chat_completions",+ direct="gpt-6-luna",+ bridge="gpt-6-luna-responses",+)diff --git a/tests/e2e/perf/cases/direct_vs_bridge.py b/tests/e2e/perf/cases/direct_vs_bridge.pynew file mode 100644index 00000000000..13f609c8299--- /dev/null+++ b/tests/e2e/perf/cases/direct_vs_bridge.py@@ -0,0 +1,31 @@+"""The mixed case's traffic on one public API, half of it on a deployment the gateway serves on+that provider's own API (direct) and half on a deployment it has to translate to another provider+API (bridge). Same 2 rps, request count, shapes, stream share, callers, guardrails and limits as+`mixed_size_requests_2rps`, so a gap between the two halves and the mixed baseline is the+translation's cost."""++from __future__ import annotations++from dataclasses import replace++from setup.performance_test_case import Endpoint, PerformanceTestCase, VirtualKey++from .mixed_size_requests_2rps import CASE as MIXED+++def _scoped(key: VirtualKey, direct: str) -> VirtualKey:+ return replace(key, models=(direct,)) if key.models else key+++def direct_vs_bridge(name: str, endpoint: Endpoint, *, direct: str, bridge: str) -> PerformanceTestCase:+ return replace(+ MIXED,+ name=name,+ teams=(+ replace(MIXED.team("team-a"), models=(direct, bridge)),+ replace(MIXED.team("team-b"), models=(bridge,)),+ ),+ virtual_keys=tuple(_scoped(key, direct) for key in MIXED.virtual_keys),+ model_groups=(direct, bridge),+ traffic=replace(MIXED.traffic, endpoints={endpoint: 1.0}),+ )diff --git a/tests/e2e/perf/cases/messages_2rps.py b/tests/e2e/perf/cases/messages_2rps.pynew file mode 100644index 00000000000..df5d221d430--- /dev/null+++ b/tests/e2e/perf/cases/messages_2rps.py@@ -0,0 +1,16 @@+"""`/v1/messages` at 2 rps: `claude-sonnet-5` is served by the Anthropic mock's Messages route,+`claude-sonnet-5-bedrock-converse` (`bedrock/converse/anthropic.claude-sonnet-5`) is bridged to the+Bedrock mock's Converse route."""++from __future__ import annotations++from typing import Final++from .direct_vs_bridge import direct_vs_bridge++CASE: Final = direct_vs_bridge(+ "messages_2rps",+ "messages",+ direct="claude-sonnet-5",+ bridge="claude-sonnet-5-bedrock-converse",+)diff --git a/tests/e2e/perf/cases/responses_2rps.py b/tests/e2e/perf/cases/responses_2rps.pynew file mode 100644index 00000000000..dd55d536eb6--- /dev/null+++ b/tests/e2e/perf/cases/responses_2rps.py@@ -0,0 +1,15 @@+"""`/v1/responses` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's Responses route,+`claude-sonnet-5` is bridged to the Anthropic mock's Messages route."""++from __future__ import annotations++from typing import Final++from .direct_vs_bridge import direct_vs_bridge++CASE: Final = direct_vs_bridge(+ "responses_2rps",+ "responses",+ direct="gpt-6-luna",+ bridge="claude-sonnet-5",+)diff --git a/tests/e2e/perf/setup/observe.py b/tests/e2e/perf/setup/observe.pyindex ccdae6398a9..1532a78606e 100644--- a/tests/e2e/perf/setup/observe.py+++ b/tests/e2e/perf/setup/observe.py@@ -565,6 +565,7 @@ def _diagnostics(observation: Observation) -> list[str]:return [" diagnostics (not limits):",f" bodies off reply {observation.bodies_off_reply()}",+ f" mock routes {_mock_routes(observation)}",f" guardrail calls per request {len(observation.guardrail_calls) / count:.3f}",f" guardrail missed {observation.guardrail_missed() if observation.guardrail_calls else 'n/a'}",f" retried share {len(observation.retried_markers) / count:.4f}",@@ -575,6 +576,12 @@ def _diagnostics(observation: Observation) -> list[str]:]+def _mock_routes(observation: Observation) -> str:+ """Which provider route each request landed on, so a direct-vs-bridge case shows its split."""+ counts: Final = Counter(f"{seen.provider}/{seen.route}" for seen in observation.mock_requests)+ return ", ".join(f"{route} {count}" for route, count in sorted(counts.items())) or "n/a"++def _datastore_lines(observation: Observation) -> list[str]:"""What the per-request Postgres and Redis counts are made of, so a red run names the statement orcommand that grew. `plumbing` (BEGIN, COMMIT, session settings) and `unclassified` count in neither limit."""