The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
test(perf): record the row count and size of each table the Admin UI journeys query (#47683)
Kerry Lu17db64ef78aa
diff --git a/tests/e2e/perf/setup/ui_query_fixture.py b/tests/e2e/perf/setup/ui_query_fixture.pyindex cfffa0822e8..ed941a096bd 100644--- a/tests/e2e/perf/setup/ui_query_fixture.py+++ b/tests/e2e/perf/setup/ui_query_fixture.py@@ -3,7 +3,9 @@ from __future__ import annotationsimport hashlibimport osimport time+from collections.abc import Iterablefrom datetime import datetime, timedelta+from itertools import chainfrom typing import Finalimport psycopg@@ -18,6 +20,33 @@ class UiFixture(BaseModel):anchor: AwareDatetimeapi_key: str+ """`perf_ui_db_*` JUnit properties naming how much data the journeys ran against."""+ volume: tuple[tuple[str, int], ...] = ()+++class TableVolume(BaseModel):+ model_config = ConfigDict(frozen=True)++ table: str+ rows: int+ bytes: int+++QUERIED_TABLES: Final = (+ "LiteLLM_SpendLogs",+ "LiteLLM_DailyUserSpend",+ "LiteLLM_DailyTeamSpend",+ "LiteLLM_DailyTagSpend",+ "LiteLLM_VerificationToken",+ "LiteLLM_TeamTable",+ "LiteLLM_UserTable",+ "LiteLLM_ProxyModelTable",+)+_VOLUME_SQL: Final = """+ SELECT c.relname, GREATEST(c.reltuples, 0)::bigint, pg_total_relation_size(c.oid)+ FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace+ WHERE n.nspname = 'public' AND c.relname = ANY(%(tables)s)+"""SEEDED_TEAMS: Final = 100@@ -59,6 +88,30 @@ _SEED_SQL: Final = ()+def volume_properties(volumes: Iterable[TableVolume]) -> tuple[tuple[str, int], ...]:+ """One `perf_ui_db_rows_<table>` and one `perf_ui_db_bytes_<table>` property per table the journeys read,+ so the dashboard can say what the latencies were measured against."""+ return tuple(+ chain.from_iterable(+ ((f"perf_ui_db_rows_{v.table}", v.rows), (f"perf_ui_db_bytes_{v.table}", v.bytes)) for v in volumes+ )+ )+++def table_volumes(connection: psycopg.Connection, spend_log_rows: int) -> tuple[TableVolume, ...]:+ """Planner row estimates and on-disk size (indexes included) of the queried tables. The restore's+ `LiteLLM_SpendLogs` count comes from the seed checkpoint, exact where the estimate can be stale."""+ adapter: Final = TypeAdapter(tuple[str, int, int])+ found: Final = tuple(+ adapter.validate_python(row)+ for row in connection.execute(_VOLUME_SQL, {"tables": list(QUERIED_TABLES)}).fetchall()+ )+ return tuple(+ TableVolume(table=table, rows=spend_log_rows if table == "LiteLLM_SpendLogs" else rows, bytes=size)+ for table, rows, size in sorted(found)+ )++def prepare_fixture() -> UiFixture:validate_target(os.environ["DATABASE_URL"])_finish_upgrade()@@ -71,6 +124,7 @@ def prepare_fixture() -> UiFixture:anchor=TypeAdapter(datetime).validate_python(row[0]),api_key=hashlib.md5(b"perf-key:0", usedforsecurity=False).hexdigest()+ hashlib.md5(b"perf-key-tail:0", usedforsecurity=False).hexdigest(),+ volume=volume_properties(table_volumes(connection, TypeAdapter(int).validate_python(row[1]))),)return fixturediff --git a/tests/e2e/perf/test_ui_journeys.py b/tests/e2e/perf/test_ui_journeys.pyindex db25608a00d..63eefedec68 100644--- a/tests/e2e/perf/test_ui_journeys.py+++ b/tests/e2e/perf/test_ui_journeys.py@@ -11,14 +11,14 @@ from types import FrameTypefrom typing import Finalimport pytest-from playwright.sync_api import Pagefrom e2e_config import MASTER_KEY, UI_BASE_URL, UI_PASSWORD, UI_USERNAMEfrom e2e_http import send_text_external, without_retries+from playwright.sync_api import Pagefrom setup.ui_journeys import (- assert_within_ceiling,Dashboard,Journey,Sample,+ assert_within_ceiling,dashboard_page,database_failures,error_code,@@ -111,6 +111,7 @@ def test_ui_journey_latency(journey_name: str, ui_fixture: UiFixture, request: pitem: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untypedassert isinstance(item, pytest.Item)item.user_properties.extend(ui_junit_properties(samples))+ item.user_properties.extend(ui_fixture.volume)after: Final = settled_counter(lambda: _metrics(transport), evidence)item.user_properties.extend((diff --git a/tests/e2e_harness/perf/setup/test_ui_journeys.py b/tests/e2e_harness/perf/setup/test_ui_journeys.pyindex dd2c1fc0301..d008368052c 100644--- a/tests/e2e_harness/perf/setup/test_ui_journeys.py+++ b/tests/e2e_harness/perf/setup/test_ui_journeys.py@@ -6,9 +6,9 @@ import pytestfrom setup.ui_journeys import (UI_JOURNEY_LIMITS,Sample,- error_code,assert_within_ceiling,database_failures,+ error_code,journeys,summarize,ui_junit_properties,@@ -153,3 +153,21 @@ def test_failed_data_request_status_becomes_the_error_code() -> None:metrics: Final = dict(ui_junit_properties((gateway_error, slow, hung)))assert metrics["perf_ui_error_http_500"] == 1 and metrics["perf_ui_error_wall_clock"] == 1assert "perf_ui_error_ui" not in metrics+++def test_volume_properties_name_rows_and_bytes_per_queried_table() -> None:+ from setup.ui_query_fixture import TableVolume, volume_properties++ properties: Final = volume_properties(+ (+ TableVolume(table="LiteLLM_SpendLogs", rows=850_000_000, bytes=1_000_000_000_000),+ TableVolume(table="LiteLLM_VerificationToken", rows=5_000, bytes=4_194_304),+ )+ )++ assert properties == (+ ("perf_ui_db_rows_LiteLLM_SpendLogs", 850_000_000),+ ("perf_ui_db_bytes_LiteLLM_SpendLogs", 1_000_000_000_000),+ ("perf_ui_db_rows_LiteLLM_VerificationToken", 5_000),+ ("perf_ui_db_bytes_LiteLLM_VerificationToken", 4_194_304),+ )