Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

test(perf): record the data request status per UI journey sample and bound each journey with a wall clock (#47622)

Kerry Lu767b75aa9db6
diff --git a/tests/e2e/perf/UI_QUERIES.md b/tests/e2e/perf/UI_QUERIES.md
index 27780537042..9bf82c8c07e 100644
--- a/tests/e2e/perf/UI_QUERIES.md
+++ b/tests/e2e/perf/UI_QUERIES.md
@@ -53,3 +53,5 @@ uv run pytest tests/e2e/perf/test_ui_journeys.py --reruns 0 --junitxml="$E2E_ART
Each JUnit testcase `test_ui_journey_latency[<name>]` carries `perf_ui_journeys`, `perf_ui_failures`, `perf_ui_seconds_p50`, `perf_ui_seconds_p90`, `perf_ui_seconds_max`, `perf_ui_first_seconds`, `perf_ui_repeated_seconds_p50`, `perf_ui_repeated_seconds_max` and `perf_ui_database_failures`. `ui-journey-<name>.json` holds every sample with its on-screen row count and `ui-isolation-<name>.jsonl` the SQL isolation evidence
The Buildkite `ui_queries` lane restores the pinned snapshot, waits for migrations and indexes, runs these journeys with no retries and uploads the JUnit and JSON artifacts. Validate selectors on a small restore first, then take numbers from the completed 1TB snapshot
+
+Each sample in `ui-journey-<name>.json` also records the `status` and `request` (method and path) of the data request the page issued, the first non-2xx one when there is one, so a failed sample names what the gateway answered. A failed sample with a 4xx or 5xx status is counted as `perf_ui_error_http_<status>` instead of `perf_ui_error_ui`. Each journey is also bounded by a wall clock of `E2E_UI_JOURNEY_TIMEOUT_SECONDS x (repeats + 2)`, so one hung browser call ends the journey with a `wall_clock` sample instead of eating the lane.
diff --git a/tests/e2e/perf/setup/ui_journeys.py b/tests/e2e/perf/setup/ui_journeys.py
index e9d4360d1a4..2b7bda77733 100644
--- a/tests/e2e/perf/setup/ui_journeys.py
+++ b/tests/e2e/perf/setup/ui_journeys.py
@@ -10,6 +10,7 @@ from dataclasses import dataclass
from datetime import datetime
from itertools import chain
from types import MappingProxyType
+from urllib.parse import urlsplit
from typing import Final
from playwright.sync_api import Error as PlaywrightError
@@ -30,6 +31,8 @@ class Sample(BaseModel):
seconds: float
rows: int = 0
error: str = ""
+ status: int = 0
+ request: str = ""
class Timings(BaseModel):
@@ -80,6 +83,10 @@ def summarize(samples: tuple[Sample, ...]) -> Timings:
def error_code(sample: Sample) -> str:
if not sample.error:
return ""
+ if sample.status >= 400:
+ return f"http_{sample.status}"
+ if sample.error.startswith("wall_clock"):
+ return "wall_clock"
return "timeout" if sample.error == "timeout" else "ui"
@@ -185,6 +192,19 @@ def _is_data_fetch(response: Response) -> bool:
return response.request.resource_type in ("fetch", "xhr") and "/_next/" not in response.url
+def _worst(responses: list[Response]) -> Response | None:
+ return next((response for response in responses if response.status >= 400), responses[-1] if responses else None)
+
+
+def _with_response(sample: Sample, responses: list[Response]) -> Sample:
+ worst: Final = _worst(responses)
+ if worst is None:
+ return sample
+ return sample.model_copy(
+ update={"status": worst.status, "request": f"{worst.request.method} {urlsplit(worst.url).path}"[:120]}
+ )
+
+
def measure(page: Page, journey: Journey, timeout: float) -> Sample:
"""Wall time from the user's action to the data being on screen. Every sample starts from the
same pre-action state so repeats measure the same journey, not a warmed client cache."""
@@ -192,16 +212,27 @@ def measure(page: Page, journey: Journey, timeout: float) -> Sample:
journey.start(page, timeout)
except (PlaywrightError, AssertionError) as error:
return Sample(seconds=0.0, error=f"start {type(error).__name__}: {error}"[:300])
+ responses: Final[list[Response]] = [] # mutable-ok: filled by the playwright response listener
+
+ def record(response: Response) -> None:
+ if _is_data_fetch(response):
+ responses.append(response)
+
+ page.on("response", record)
started: Final = time.monotonic()
try:
with page.expect_response(_is_data_fetch, timeout=timeout * 1000):
journey.act(page)
rows: Final = journey.settled(page, timeout)
except PlaywrightTimeoutError:
- return Sample(seconds=time.monotonic() - started, error="timeout")
+ return _with_response(Sample(seconds=time.monotonic() - started, error="timeout"), responses)
except (PlaywrightError, AssertionError) as error:
- return Sample(seconds=time.monotonic() - started, error=f"{type(error).__name__}: {error}"[:300])
- return Sample(seconds=time.monotonic() - started, rows=rows)
+ return _with_response(
+ Sample(seconds=time.monotonic() - started, error=f"{type(error).__name__}: {error}"[:300]), responses
+ )
+ finally:
+ page.remove_listener("response", record)
+ return _with_response(Sample(seconds=time.monotonic() - started, rows=rows), responses)
def _visible(page: Page, test_id: str) -> Locator:
diff --git a/tests/e2e/perf/test_ui_journeys.py b/tests/e2e/perf/test_ui_journeys.py
index 888a44d2e27..db25608a00d 100644
--- a/tests/e2e/perf/test_ui_journeys.py
+++ b/tests/e2e/perf/test_ui_journeys.py
@@ -2,11 +2,16 @@ from __future__ import annotations
import json
import os
+import signal
+import time
+from collections.abc import Generator
from datetime import UTC, datetime
from pathlib import Path
+from types import FrameType
from typing import Final
import pytest
+from playwright.sync_api import Page
from e2e_config import MASTER_KEY, UI_BASE_URL, UI_PASSWORD, UI_USERNAME
from e2e_http import send_text_external, without_retries
from setup.ui_journeys import (
@@ -56,7 +61,30 @@ def _metrics(transport: HttpTransport) -> float:
def _samples(dashboard: Dashboard, journey: Journey, timeout: float, repeats: int) -> tuple[Sample, ...]:
with dashboard_page(dashboard, timeout) as page:
- return tuple(measure(page, journey, timeout) for _ in range(repeats + 1))
+ return tuple(_within_budget(page, journey, timeout, repeats))
+
+
+def _within_budget(page: Page, journey: Journey, timeout: float, repeats: int) -> Generator[Sample]:
+ """One hanging browser call must not eat the lane: SIGALRM ends the journey at timeout x (repeats + 2),
+ the budget every sample needs (start plus act plus settle) with one spare sample of slack."""
+ budget: Final = timeout * (repeats + 2)
+
+ def expire(signum: int, frame: FrameType | None) -> None:
+ raise TimeoutError(f"{journey.name} exceeded its {budget:.0f}s wall clock")
+
+ previous: Final = signal.signal(signal.SIGALRM, expire)
+ started: Final = time.monotonic()
+ try:
+ _ = signal.setitimer(signal.ITIMER_REAL, budget)
+ for _ in range(repeats + 1):
+ yield measure(page, journey, timeout)
+ except Exception as error:
+ if time.monotonic() - started < budget:
+ raise
+ yield Sample(seconds=time.monotonic() - started, error=f"wall_clock {type(error).__name__}: {error}"[:300])
+ finally:
+ _ = signal.setitimer(signal.ITIMER_REAL, 0)
+ _ = signal.signal(signal.SIGALRM, previous)
def test_ui_journey_latency(journey_name: str, ui_fixture: UiFixture, request: pytest.FixtureRequest) -> None:
diff --git a/tests/e2e_harness/perf/setup/test_ui_journeys.py b/tests/e2e_harness/perf/setup/test_ui_journeys.py
index 7d5b0d531ec..dd2c1fc0301 100644
--- a/tests/e2e_harness/perf/setup/test_ui_journeys.py
+++ b/tests/e2e_harness/perf/setup/test_ui_journeys.py
@@ -6,6 +6,7 @@ import pytest
from setup.ui_journeys import (
UI_JOURNEY_LIMITS,
Sample,
+ error_code,
assert_within_ceiling,
database_failures,
journeys,
@@ -142,3 +143,13 @@ def test_slow_journey_fails_its_ceiling() -> None:
assert_within_ceiling("logs_open_24h", (*fast[:8], Sample(seconds=25, rows=25), Sample(seconds=25, rows=25)))
with pytest.raises(AssertionError, match=r"logs_open_24h p50 11\.0s over 10s"):
assert_within_ceiling("logs_open_24h", tuple(Sample(seconds=11, rows=25) for _ in range(3)))
+
+
+def test_failed_data_request_status_becomes_the_error_code() -> None:
+ gateway_error: Final = Sample(seconds=0.4, error="AssertionError: Locator expected to be enabled", status=500)
+ slow: Final = Sample(seconds=60.0, error="timeout", status=200, request="GET /spend/logs/ui")
+ hung: Final = Sample(seconds=700.0, error="wall_clock TimeoutError: logs_next_page exceeded its 660s wall clock")
+ assert (error_code(gateway_error), error_code(slow), error_code(hung)) == ("http_500", "timeout", "wall_clock")
+ metrics: Final = dict(ui_junit_properties((gateway_error, slow, hung)))
+ assert metrics["perf_ui_error_http_500"] == 1 and metrics["perf_ui_error_wall_clock"] == 1
+ assert "perf_ui_error_ui" not in metrics