The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
test(perf): record the data request status per UI journey sample and bound each journey with a wall clock (#47622)
Kerry Lu767b75aa9db6
diff --git a/tests/e2e/perf/UI_QUERIES.md b/tests/e2e/perf/UI_QUERIES.mdindex 27780537042..9bf82c8c07e 100644--- a/tests/e2e/perf/UI_QUERIES.md+++ b/tests/e2e/perf/UI_QUERIES.md@@ -53,3 +53,5 @@ uv run pytest tests/e2e/perf/test_ui_journeys.py --reruns 0 --junitxml="$E2E_ARTEach JUnit testcase `test_ui_journey_latency[<name>]` carries `perf_ui_journeys`, `perf_ui_failures`, `perf_ui_seconds_p50`, `perf_ui_seconds_p90`, `perf_ui_seconds_max`, `perf_ui_first_seconds`, `perf_ui_repeated_seconds_p50`, `perf_ui_repeated_seconds_max` and `perf_ui_database_failures`. `ui-journey-<name>.json` holds every sample with its on-screen row count and `ui-isolation-<name>.jsonl` the SQL isolation evidenceThe Buildkite `ui_queries` lane restores the pinned snapshot, waits for migrations and indexes, runs these journeys with no retries and uploads the JUnit and JSON artifacts. Validate selectors on a small restore first, then take numbers from the completed 1TB snapshot++Each sample in `ui-journey-<name>.json` also records the `status` and `request` (method and path) of the data request the page issued, the first non-2xx one when there is one, so a failed sample names what the gateway answered. A failed sample with a 4xx or 5xx status is counted as `perf_ui_error_http_<status>` instead of `perf_ui_error_ui`. Each journey is also bounded by a wall clock of `E2E_UI_JOURNEY_TIMEOUT_SECONDS x (repeats + 2)`, so one hung browser call ends the journey with a `wall_clock` sample instead of eating the lane.diff --git a/tests/e2e/perf/setup/ui_journeys.py b/tests/e2e/perf/setup/ui_journeys.pyindex e9d4360d1a4..2b7bda77733 100644--- a/tests/e2e/perf/setup/ui_journeys.py+++ b/tests/e2e/perf/setup/ui_journeys.py@@ -10,6 +10,7 @@ from dataclasses import dataclassfrom datetime import datetimefrom itertools import chainfrom types import MappingProxyType+from urllib.parse import urlsplitfrom typing import Finalfrom playwright.sync_api import Error as PlaywrightError@@ -30,6 +31,8 @@ class Sample(BaseModel):seconds: floatrows: int = 0error: str = ""+ status: int = 0+ request: str = ""class Timings(BaseModel):@@ -80,6 +83,10 @@ def summarize(samples: tuple[Sample, ...]) -> Timings:def error_code(sample: Sample) -> str:if not sample.error:return ""+ if sample.status >= 400:+ return f"http_{sample.status}"+ if sample.error.startswith("wall_clock"):+ return "wall_clock"return "timeout" if sample.error == "timeout" else "ui"@@ -185,6 +192,19 @@ def _is_data_fetch(response: Response) -> bool:return response.request.resource_type in ("fetch", "xhr") and "/_next/" not in response.url+def _worst(responses: list[Response]) -> Response | None:+ return next((response for response in responses if response.status >= 400), responses[-1] if responses else None)+++def _with_response(sample: Sample, responses: list[Response]) -> Sample:+ worst: Final = _worst(responses)+ if worst is None:+ return sample+ return sample.model_copy(+ update={"status": worst.status, "request": f"{worst.request.method} {urlsplit(worst.url).path}"[:120]}+ )++def measure(page: Page, journey: Journey, timeout: float) -> Sample:"""Wall time from the user's action to the data being on screen. Every sample starts from thesame pre-action state so repeats measure the same journey, not a warmed client cache."""@@ -192,16 +212,27 @@ def measure(page: Page, journey: Journey, timeout: float) -> Sample:journey.start(page, timeout)except (PlaywrightError, AssertionError) as error:return Sample(seconds=0.0, error=f"start {type(error).__name__}: {error}"[:300])+ responses: Final[list[Response]] = [] # mutable-ok: filled by the playwright response listener++ def record(response: Response) -> None:+ if _is_data_fetch(response):+ responses.append(response)++ page.on("response", record)started: Final = time.monotonic()try:with page.expect_response(_is_data_fetch, timeout=timeout * 1000):journey.act(page)rows: Final = journey.settled(page, timeout)except PlaywrightTimeoutError:- return Sample(seconds=time.monotonic() - started, error="timeout")+ return _with_response(Sample(seconds=time.monotonic() - started, error="timeout"), responses)except (PlaywrightError, AssertionError) as error:- return Sample(seconds=time.monotonic() - started, error=f"{type(error).__name__}: {error}"[:300])- return Sample(seconds=time.monotonic() - started, rows=rows)+ return _with_response(+ Sample(seconds=time.monotonic() - started, error=f"{type(error).__name__}: {error}"[:300]), responses+ )+ finally:+ page.remove_listener("response", record)+ return _with_response(Sample(seconds=time.monotonic() - started, rows=rows), responses)def _visible(page: Page, test_id: str) -> Locator:diff --git a/tests/e2e/perf/test_ui_journeys.py b/tests/e2e/perf/test_ui_journeys.pyindex 888a44d2e27..db25608a00d 100644--- a/tests/e2e/perf/test_ui_journeys.py+++ b/tests/e2e/perf/test_ui_journeys.py@@ -2,11 +2,16 @@ from __future__ import annotationsimport jsonimport os+import signal+import time+from collections.abc import Generatorfrom datetime import UTC, datetimefrom pathlib import Path+from types import FrameTypefrom typing import Finalimport pytest+from playwright.sync_api import Pagefrom e2e_config import MASTER_KEY, UI_BASE_URL, UI_PASSWORD, UI_USERNAMEfrom e2e_http import send_text_external, without_retriesfrom setup.ui_journeys import (@@ -56,7 +61,30 @@ def _metrics(transport: HttpTransport) -> float:def _samples(dashboard: Dashboard, journey: Journey, timeout: float, repeats: int) -> tuple[Sample, ...]:with dashboard_page(dashboard, timeout) as page:- return tuple(measure(page, journey, timeout) for _ in range(repeats + 1))+ return tuple(_within_budget(page, journey, timeout, repeats))+++def _within_budget(page: Page, journey: Journey, timeout: float, repeats: int) -> Generator[Sample]:+ """One hanging browser call must not eat the lane: SIGALRM ends the journey at timeout x (repeats + 2),+ the budget every sample needs (start plus act plus settle) with one spare sample of slack."""+ budget: Final = timeout * (repeats + 2)++ def expire(signum: int, frame: FrameType | None) -> None:+ raise TimeoutError(f"{journey.name} exceeded its {budget:.0f}s wall clock")++ previous: Final = signal.signal(signal.SIGALRM, expire)+ started: Final = time.monotonic()+ try:+ _ = signal.setitimer(signal.ITIMER_REAL, budget)+ for _ in range(repeats + 1):+ yield measure(page, journey, timeout)+ except Exception as error:+ if time.monotonic() - started < budget:+ raise+ yield Sample(seconds=time.monotonic() - started, error=f"wall_clock {type(error).__name__}: {error}"[:300])+ finally:+ _ = signal.setitimer(signal.ITIMER_REAL, 0)+ _ = signal.signal(signal.SIGALRM, previous)def test_ui_journey_latency(journey_name: str, ui_fixture: UiFixture, request: pytest.FixtureRequest) -> None:diff --git a/tests/e2e_harness/perf/setup/test_ui_journeys.py b/tests/e2e_harness/perf/setup/test_ui_journeys.pyindex 7d5b0d531ec..dd2c1fc0301 100644--- a/tests/e2e_harness/perf/setup/test_ui_journeys.py+++ b/tests/e2e_harness/perf/setup/test_ui_journeys.py@@ -6,6 +6,7 @@ import pytestfrom setup.ui_journeys import (UI_JOURNEY_LIMITS,Sample,+ error_code,assert_within_ceiling,database_failures,journeys,@@ -142,3 +143,13 @@ def test_slow_journey_fails_its_ceiling() -> None:assert_within_ceiling("logs_open_24h", (*fast[:8], Sample(seconds=25, rows=25), Sample(seconds=25, rows=25)))with pytest.raises(AssertionError, match=r"logs_open_24h p50 11\.0s over 10s"):assert_within_ceiling("logs_open_24h", tuple(Sample(seconds=11, rows=25) for _ in range(3)))+++def test_failed_data_request_status_becomes_the_error_code() -> None:+ gateway_error: Final = Sample(seconds=0.4, error="AssertionError: Locator expected to be enabled", status=500)+ slow: Final = Sample(seconds=60.0, error="timeout", status=200, request="GET /spend/logs/ui")+ hung: Final = Sample(seconds=700.0, error="wall_clock TimeoutError: logs_next_page exceeded its 660s wall clock")+ assert (error_code(gateway_error), error_code(slow), error_code(hung)) == ("http_500", "timeout", "wall_clock")+ metrics: Final = dict(ui_junit_properties((gateway_error, slow, hung)))+ assert metrics["perf_ui_error_http_500"] == 1 and metrics["perf_ui_error_wall_clock"] == 1+ assert "perf_ui_error_ui" not in metrics