Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

test(perf): add chat_completions_2rps, responses_2rps and messages_2rps direct vs bridge cases (#47752)

Kerry Lu8d9b9926d165
diff --git a/tests/e2e/gateway/perf_ci_config.yml b/tests/e2e/gateway/perf_ci_config.yml
index 7ab6637e985..767db313847 100644
--- a/tests/e2e/gateway/perf_ci_config.yml
+++ b/tests/e2e/gateway/perf_ci_config.yml
@@ -46,6 +46,11 @@ model_list:
model: openai/gpt-6-luna
api_base: os.environ/E2E_PERF_MOCK_OPENAI_URL
api_key: perf-mock-not-checked
+ - model_name: gpt-6-luna-responses
+ litellm_params:
+ model: openai/responses/gpt-6-luna
+ api_base: os.environ/E2E_PERF_MOCK_OPENAI_URL
+ api_key: perf-mock-not-checked
- model_name: claude-sonnet-5
litellm_params:
model: anthropic/claude-sonnet-5
@@ -58,6 +63,13 @@ model_list:
aws_region_name: us-east-1
aws_access_key_id: perf-mock-not-checked
aws_secret_access_key: perf-mock-not-checked
+ - model_name: claude-sonnet-5-bedrock-converse
+ litellm_params:
+ model: bedrock/converse/anthropic.claude-sonnet-5
+ aws_bedrock_runtime_endpoint: os.environ/E2E_PERF_MOCK_BEDROCK_URL
+ aws_region_name: us-east-1
+ aws_access_key_id: perf-mock-not-checked
+ aws_secret_access_key: perf-mock-not-checked
guardrails:
- guardrail_name: perf-pre
litellm_params:
diff --git a/tests/e2e/perf/cases/chat_completions_2rps.py b/tests/e2e/perf/cases/chat_completions_2rps.py
new file mode 100644
index 00000000000..9792d99c14f
--- /dev/null
+++ b/tests/e2e/perf/cases/chat_completions_2rps.py
@@ -0,0 +1,15 @@
+"""`/v1/chat/completions` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's chat completions
+route, `gpt-6-luna-responses` (`openai/responses/gpt-6-luna`) is bridged to its Responses route."""
+
+from __future__ import annotations
+
+from typing import Final
+
+from .direct_vs_bridge import direct_vs_bridge
+
+CASE: Final = direct_vs_bridge(
+ "chat_completions_2rps",
+ "chat_completions",
+ direct="gpt-6-luna",
+ bridge="gpt-6-luna-responses",
+)
diff --git a/tests/e2e/perf/cases/direct_vs_bridge.py b/tests/e2e/perf/cases/direct_vs_bridge.py
new file mode 100644
index 00000000000..13f609c8299
--- /dev/null
+++ b/tests/e2e/perf/cases/direct_vs_bridge.py
@@ -0,0 +1,31 @@
+"""The mixed case's traffic on one public API, half of it on a deployment the gateway serves on
+that provider's own API (direct) and half on a deployment it has to translate to another provider
+API (bridge). Same 2 rps, request count, shapes, stream share, callers, guardrails and limits as
+`mixed_size_requests_2rps`, so a gap between the two halves and the mixed baseline is the
+translation's cost."""
+
+from __future__ import annotations
+
+from dataclasses import replace
+
+from setup.performance_test_case import Endpoint, PerformanceTestCase, VirtualKey
+
+from .mixed_size_requests_2rps import CASE as MIXED
+
+
+def _scoped(key: VirtualKey, direct: str) -> VirtualKey:
+ return replace(key, models=(direct,)) if key.models else key
+
+
+def direct_vs_bridge(name: str, endpoint: Endpoint, *, direct: str, bridge: str) -> PerformanceTestCase:
+ return replace(
+ MIXED,
+ name=name,
+ teams=(
+ replace(MIXED.team("team-a"), models=(direct, bridge)),
+ replace(MIXED.team("team-b"), models=(bridge,)),
+ ),
+ virtual_keys=tuple(_scoped(key, direct) for key in MIXED.virtual_keys),
+ model_groups=(direct, bridge),
+ traffic=replace(MIXED.traffic, endpoints={endpoint: 1.0}),
+ )
diff --git a/tests/e2e/perf/cases/messages_2rps.py b/tests/e2e/perf/cases/messages_2rps.py
new file mode 100644
index 00000000000..df5d221d430
--- /dev/null
+++ b/tests/e2e/perf/cases/messages_2rps.py
@@ -0,0 +1,16 @@
+"""`/v1/messages` at 2 rps: `claude-sonnet-5` is served by the Anthropic mock's Messages route,
+`claude-sonnet-5-bedrock-converse` (`bedrock/converse/anthropic.claude-sonnet-5`) is bridged to the
+Bedrock mock's Converse route."""
+
+from __future__ import annotations
+
+from typing import Final
+
+from .direct_vs_bridge import direct_vs_bridge
+
+CASE: Final = direct_vs_bridge(
+ "messages_2rps",
+ "messages",
+ direct="claude-sonnet-5",
+ bridge="claude-sonnet-5-bedrock-converse",
+)
diff --git a/tests/e2e/perf/cases/responses_2rps.py b/tests/e2e/perf/cases/responses_2rps.py
new file mode 100644
index 00000000000..dd55d536eb6
--- /dev/null
+++ b/tests/e2e/perf/cases/responses_2rps.py
@@ -0,0 +1,15 @@
+"""`/v1/responses` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's Responses route,
+`claude-sonnet-5` is bridged to the Anthropic mock's Messages route."""
+
+from __future__ import annotations
+
+from typing import Final
+
+from .direct_vs_bridge import direct_vs_bridge
+
+CASE: Final = direct_vs_bridge(
+ "responses_2rps",
+ "responses",
+ direct="gpt-6-luna",
+ bridge="claude-sonnet-5",
+)
diff --git a/tests/e2e/perf/setup/observe.py b/tests/e2e/perf/setup/observe.py
index ccdae6398a9..1532a78606e 100644
--- a/tests/e2e/perf/setup/observe.py
+++ b/tests/e2e/perf/setup/observe.py
@@ -565,6 +565,7 @@ def _diagnostics(observation: Observation) -> list[str]:
return [
" diagnostics (not limits):",
f" bodies off reply {observation.bodies_off_reply()}",
+ f" mock routes {_mock_routes(observation)}",
f" guardrail calls per request {len(observation.guardrail_calls) / count:.3f}",
f" guardrail missed {observation.guardrail_missed() if observation.guardrail_calls else 'n/a'}",
f" retried share {len(observation.retried_markers) / count:.4f}",
@@ -575,6 +576,12 @@ def _diagnostics(observation: Observation) -> list[str]:
]
+def _mock_routes(observation: Observation) -> str:
+ """Which provider route each request landed on, so a direct-vs-bridge case shows its split."""
+ counts: Final = Counter(f"{seen.provider}/{seen.route}" for seen in observation.mock_requests)
+ return ", ".join(f"{route} {count}" for route, count in sorted(counts.items())) or "n/a"
+
+
def _datastore_lines(observation: Observation) -> list[str]:
"""What the per-request Postgres and Redis counts are made of, so a red run names the statement or
command that grew. `plumbing` (BEGIN, COMMIT, session settings) and `unclassified` count in neither limit."""