Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

test(perf): emit per-location seconds, offending locations and the ten longest holds as event_loop_stalls breakdowns (#47757)

Kerry Lu647ebd63e0d0
diff --git a/tests/e2e/perf/setup/event_loop.py b/tests/e2e/perf/setup/event_loop.py
index 1c582e24fd8..58144424544 100644
--- a/tests/e2e/perf/setup/event_loop.py
+++ b/tests/e2e/perf/setup/event_loop.py
@@ -18,7 +18,7 @@ import json
import os
import re
from collections.abc import Iterable
-from dataclasses import dataclass, replace
+from dataclasses import asdict, dataclass, replace
from datetime import datetime
from itertools import groupby
from pathlib import Path
@@ -80,11 +80,24 @@ class StallRow:
total_seconds: float
max_seconds: float
allowed_by: KnownStall | None
+ """Every hold in the row, longest first."""
+ holds: tuple[float, ...] = ()
@property
def offending(self) -> bool:
return self.allowed_by is None or self.count > self.allowed_by.max_count
+ @property
+ def over_allowance(self) -> int:
+ return self.count if self.allowed_by is None else max(0, self.count - self.allowed_by.max_count)
+
+ @property
+ def note(self) -> str:
+ allowed: Final = (
+ f"{self.count} of {self.allowed_by.max_count} allowed" if self.allowed_by else "not on the allowlist"
+ )
+ return f"{allowed}, max {self.max_seconds:.2f} s"
+
def _short(path: str) -> str:
if "site-packages/" in path:
@@ -161,6 +174,7 @@ def summarize(spec: EventLoopStalls, stalls: Iterable[Stall]) -> tuple[StallRow,
total_seconds=sum(stall.seconds for stall in group),
max_seconds=max(stall.seconds for stall in group),
allowed_by=_allowed_by(spec, coroutine, where),
+ holds=tuple(sorted((stall.seconds for stall in group), reverse=True)),
)
for (coroutine, where), group in (
(key, tuple(items)) for key, items in groupby(over, key=lambda s: (s.coroutine, s.where))
@@ -182,28 +196,65 @@ def offenders(rows: Iterable[StallRow]) -> tuple[str, ...]:
)
-def _over_allowance(row: StallRow) -> int:
- return row.count if row.allowed_by is None else max(0, row.count - row.allowed_by.max_count)
+TOP_HOLDS: Final = 10
+
+
+@dataclass(frozen=True, slots=True)
+class Slice:
+ """One `_breakdown` row in the shape the other metrics use (label, value, share) plus the allowlist verdict the
+ dashboard shows under the label and colours by. `share` is None where the rows are not parts of a total."""
+
+ label: str
+ value: float
+ share: float | None
+ note: str | None
+ over: bool
+
+
+def _label(row: StallRow) -> str:
+ return f"{row.coroutine} {row.where}".strip()
+
+
+def _slice(row: StallRow, value: float, total: float) -> Slice:
+ return Slice(label=_label(row), value=value, share=value / total, note=row.note, over=row.offending)
+
+
+def _breakdown(name: str, slices: Iterable[Slice]) -> tuple[tuple[str, str], ...]:
+ rows: Final = tuple(asdict(row) for row in slices)
+ return ((f"perf_event_loop_{name}_breakdown", json.dumps(rows)),) if rows else ()
+
+
+def _longest_holds(table: tuple[StallRow, ...]) -> tuple[Slice, ...]:
+ holds: Final = sorted(((seconds, row) for row in table for seconds in row.holds), key=lambda hold: -hold[0])
+ return tuple(
+ Slice(label=_label(row), value=seconds, share=None, note=None, over=row.offending)
+ for seconds, row in holds[:TOP_HOLDS]
+ )
def junit_properties(spec: EventLoopStalls, rows: Iterable[StallRow]) -> tuple[tuple[str, float | str], ...]:
"""The stall table as `perf_event_loop_*` properties for the dashboard: every hold over the threshold with the
allowlist's total as its limit, the holds past their allowance (limit 0, the case's one assertion), the longest
- and the summed hold, and a `_breakdown` of holds per location in the shape the other metrics use."""
+ and the summed hold, each with a `_breakdown` the dashboard opens on click: holds and seconds per location, the
+ locations past their cap by how far over they are, and the ten longest single holds with where they landed."""
table: Final = tuple(rows)
holds: Final = sum(row.count for row in table)
- breakdown: Final = tuple(
- {"label": f"{row.coroutine} {row.where}".strip(), "value": row.count, "share": row.count / holds}
- for row in table
- )
+ offending: Final = sum(row.over_allowance for row in table)
+ total_seconds: Final = sum(row.total_seconds for row in table)
return (
("perf_event_loop_holds", holds),
("perf_event_loop_holds_limit", sum(known.max_count for known in spec.allowlist)),
- ("perf_event_loop_offending_holds", sum(map(_over_allowance, table))),
+ ("perf_event_loop_offending_holds", offending),
("perf_event_loop_offending_holds_limit", 0),
("perf_event_loop_hold_seconds_max", max((row.max_seconds for row in table), default=0.0)),
- ("perf_event_loop_hold_seconds_total", sum(row.total_seconds for row in table)),
- *((("perf_event_loop_holds_breakdown", json.dumps(breakdown)),) if breakdown else ()),
+ ("perf_event_loop_hold_seconds_total", total_seconds),
+ *_breakdown("holds", (_slice(row, row.count, holds) for row in table)),
+ *_breakdown(
+ "offending_holds",
+ (_slice(row, row.over_allowance, offending) for row in table if row.over_allowance),
+ ),
+ *_breakdown("hold_seconds_total", (_slice(row, row.total_seconds, total_seconds) for row in table)),
+ *_breakdown("hold_seconds_max", _longest_holds(table)),
)
diff --git a/tests/e2e_harness/perf/setup/test_event_loop.py b/tests/e2e_harness/perf/setup/test_event_loop.py
index e03e985709b..75989b38275 100644
--- a/tests/e2e_harness/perf/setup/test_event_loop.py
+++ b/tests/e2e_harness/perf/setup/test_event_loop.py
@@ -37,8 +37,14 @@ class _Slice(BaseModel):
model_config = ConfigDict(frozen=True)
label: str
- value: int
- share: float
+ value: float
+ share: float | None
+ note: str | None
+ over: bool
+
+
+def _breakdown(properties: dict[str, float | str], name: str) -> list[_Slice]:
+ return TypeAdapter(list[_Slice]).validate_json(str(properties[f"perf_event_loop_{name}_breakdown"]))
def test_parse_keeps_the_coroutine_its_location_and_the_hold() -> None:
@@ -116,13 +122,56 @@ def test_junit_properties_count_every_hold_and_only_the_holds_past_their_allowan
assert properties["perf_event_loop_offending_holds_limit"] == 0
assert properties["perf_event_loop_hold_seconds_max"] == 0.731
assert math.isclose(float(properties["perf_event_loop_hold_seconds_total"]), 0.302 + 2 * 0.412 + 3 * 0.731)
- breakdown: Final = TypeAdapter(list[_Slice]).validate_json(str(properties["perf_event_loop_holds_breakdown"]))
- assert [(slice.label, slice.value) for slice in breakdown] == [
- ("RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416", 3),
- ("TaskHandle._run_coro() anyio/_core/_tasks.py:275", 2),
- ("LifespanOn.main() uvicorn/lifespan/on.py:86", 1),
+ breakdown: Final = _breakdown(properties, "holds")
+ assert [(slice.label, slice.value, slice.note, slice.over) for slice in breakdown] == [
+ (
+ "RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416",
+ 3,
+ "3 of 2 allowed, max 0.73 s",
+ True,
+ ),
+ ("TaskHandle._run_coro() anyio/_core/_tasks.py:275", 2, "not on the allowlist, max 0.41 s", True),
+ ("LifespanOn.main() uvicorn/lifespan/on.py:86", 1, "1 of 3 allowed, max 0.30 s", False),
]
- assert math.isclose(sum(slice.share for slice in breakdown), 1.0)
+ assert math.isclose(sum(slice.share or 0.0 for slice in breakdown), 1.0)
+
+
+def test_the_offending_breakdown_lists_only_the_locations_past_their_cap_by_how_far_over_they_are() -> None:
+ rows: Final = summarize(_SPEC, parse_stalls((_STARTUP, _STREAM, _STREAM, _HANDLER, _HANDLER, _HANDLER)))
+ offending: Final = _breakdown(dict(junit_properties(_SPEC, rows)), "offending_holds")
+ assert [(slice.label, slice.value, slice.share, slice.over) for slice in offending] == [
+ ("RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416", 1, 1 / 3, True),
+ ("TaskHandle._run_coro() anyio/_core/_tasks.py:275", 2, 2 / 3, True),
+ ]
+ clean: Final = dict(junit_properties(_SPEC, summarize(_SPEC, parse_stalls((_HANDLER, _HANDLER)))))
+ assert clean["perf_event_loop_offending_holds"] == 0
+ assert "perf_event_loop_offending_holds_breakdown" not in clean
+
+
+def test_the_seconds_breakdowns_carry_the_time_per_location_and_the_longest_single_holds() -> None:
+ long_handler: Final = _HANDLER.replace("0.731", "0.950")
+ rows: Final = summarize(_SPEC, parse_stalls((_STARTUP, _STREAM, long_handler, _HANDLER)))
+ properties: Final = dict(junit_properties(_SPEC, rows))
+ per_location: Final = _breakdown(properties, "hold_seconds_total")
+ assert [(slice.label.split()[0], slice.value) for slice in per_location] == [
+ ("RequestResponseCycle.run_asgi()", 0.950 + 0.731),
+ ("TaskHandle._run_coro()", 0.412),
+ ("LifespanOn.main()", 0.302),
+ ]
+ assert math.isclose(sum(slice.share or 0.0 for slice in per_location), 1.0)
+ longest: Final = _breakdown(properties, "hold_seconds_max")
+ assert [(slice.label.split()[0], slice.value, slice.share, slice.over) for slice in longest] == [
+ ("RequestResponseCycle.run_asgi()", 0.950, None, False),
+ ("RequestResponseCycle.run_asgi()", 0.731, None, False),
+ ("TaskHandle._run_coro()", 0.412, None, True),
+ ("LifespanOn.main()", 0.302, None, False),
+ ]
+
+
+def test_the_longest_holds_breakdown_stops_at_ten() -> None:
+ rows: Final = summarize(_SPEC, parse_stalls(tuple(_STARTUP for _ in range(12))))
+ longest: Final = _breakdown(dict(junit_properties(_SPEC, rows)), "hold_seconds_max")
+ assert len(longest) == 10
def test_junit_properties_without_holds_report_zeros_and_no_breakdown() -> None:
@@ -130,4 +179,4 @@ def test_junit_properties_without_holds_report_zeros_and_no_breakdown() -> None:
assert properties["perf_event_loop_holds"] == 0
assert properties["perf_event_loop_offending_holds"] == 0
assert properties["perf_event_loop_hold_seconds_max"] == 0
- assert "perf_event_loop_holds_breakdown" not in properties
+ assert not [name for name in properties if name.endswith("_breakdown")]