The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
test(perf): emit per-location seconds, offending locations and the ten longest holds as event_loop_stalls breakdowns (#47757)
Kerry Lu647ebd63e0d0
diff --git a/tests/e2e/perf/setup/event_loop.py b/tests/e2e/perf/setup/event_loop.pyindex 1c582e24fd8..58144424544 100644--- a/tests/e2e/perf/setup/event_loop.py+++ b/tests/e2e/perf/setup/event_loop.py@@ -18,7 +18,7 @@ import jsonimport osimport refrom collections.abc import Iterable-from dataclasses import dataclass, replace+from dataclasses import asdict, dataclass, replacefrom datetime import datetimefrom itertools import groupbyfrom pathlib import Path@@ -80,11 +80,24 @@ class StallRow:total_seconds: floatmax_seconds: floatallowed_by: KnownStall | None+ """Every hold in the row, longest first."""+ holds: tuple[float, ...] = ()@propertydef offending(self) -> bool:return self.allowed_by is None or self.count > self.allowed_by.max_count+ @property+ def over_allowance(self) -> int:+ return self.count if self.allowed_by is None else max(0, self.count - self.allowed_by.max_count)++ @property+ def note(self) -> str:+ allowed: Final = (+ f"{self.count} of {self.allowed_by.max_count} allowed" if self.allowed_by else "not on the allowlist"+ )+ return f"{allowed}, max {self.max_seconds:.2f} s"+def _short(path: str) -> str:if "site-packages/" in path:@@ -161,6 +174,7 @@ def summarize(spec: EventLoopStalls, stalls: Iterable[Stall]) -> tuple[StallRow,total_seconds=sum(stall.seconds for stall in group),max_seconds=max(stall.seconds for stall in group),allowed_by=_allowed_by(spec, coroutine, where),+ holds=tuple(sorted((stall.seconds for stall in group), reverse=True)),)for (coroutine, where), group in ((key, tuple(items)) for key, items in groupby(over, key=lambda s: (s.coroutine, s.where))@@ -182,28 +196,65 @@ def offenders(rows: Iterable[StallRow]) -> tuple[str, ...]:)-def _over_allowance(row: StallRow) -> int:- return row.count if row.allowed_by is None else max(0, row.count - row.allowed_by.max_count)+TOP_HOLDS: Final = 10+++@dataclass(frozen=True, slots=True)+class Slice:+ """One `_breakdown` row in the shape the other metrics use (label, value, share) plus the allowlist verdict the+ dashboard shows under the label and colours by. `share` is None where the rows are not parts of a total."""++ label: str+ value: float+ share: float | None+ note: str | None+ over: bool+++def _label(row: StallRow) -> str:+ return f"{row.coroutine} {row.where}".strip()+++def _slice(row: StallRow, value: float, total: float) -> Slice:+ return Slice(label=_label(row), value=value, share=value / total, note=row.note, over=row.offending)+++def _breakdown(name: str, slices: Iterable[Slice]) -> tuple[tuple[str, str], ...]:+ rows: Final = tuple(asdict(row) for row in slices)+ return ((f"perf_event_loop_{name}_breakdown", json.dumps(rows)),) if rows else ()+++def _longest_holds(table: tuple[StallRow, ...]) -> tuple[Slice, ...]:+ holds: Final = sorted(((seconds, row) for row in table for seconds in row.holds), key=lambda hold: -hold[0])+ return tuple(+ Slice(label=_label(row), value=seconds, share=None, note=None, over=row.offending)+ for seconds, row in holds[:TOP_HOLDS]+ )def junit_properties(spec: EventLoopStalls, rows: Iterable[StallRow]) -> tuple[tuple[str, float | str], ...]:"""The stall table as `perf_event_loop_*` properties for the dashboard: every hold over the threshold with theallowlist's total as its limit, the holds past their allowance (limit 0, the case's one assertion), the longest- and the summed hold, and a `_breakdown` of holds per location in the shape the other metrics use."""+ and the summed hold, each with a `_breakdown` the dashboard opens on click: holds and seconds per location, the+ locations past their cap by how far over they are, and the ten longest single holds with where they landed."""table: Final = tuple(rows)holds: Final = sum(row.count for row in table)- breakdown: Final = tuple(- {"label": f"{row.coroutine} {row.where}".strip(), "value": row.count, "share": row.count / holds}- for row in table- )+ offending: Final = sum(row.over_allowance for row in table)+ total_seconds: Final = sum(row.total_seconds for row in table)return (("perf_event_loop_holds", holds),("perf_event_loop_holds_limit", sum(known.max_count for known in spec.allowlist)),- ("perf_event_loop_offending_holds", sum(map(_over_allowance, table))),+ ("perf_event_loop_offending_holds", offending),("perf_event_loop_offending_holds_limit", 0),("perf_event_loop_hold_seconds_max", max((row.max_seconds for row in table), default=0.0)),- ("perf_event_loop_hold_seconds_total", sum(row.total_seconds for row in table)),- *((("perf_event_loop_holds_breakdown", json.dumps(breakdown)),) if breakdown else ()),+ ("perf_event_loop_hold_seconds_total", total_seconds),+ *_breakdown("holds", (_slice(row, row.count, holds) for row in table)),+ *_breakdown(+ "offending_holds",+ (_slice(row, row.over_allowance, offending) for row in table if row.over_allowance),+ ),+ *_breakdown("hold_seconds_total", (_slice(row, row.total_seconds, total_seconds) for row in table)),+ *_breakdown("hold_seconds_max", _longest_holds(table)),)diff --git a/tests/e2e_harness/perf/setup/test_event_loop.py b/tests/e2e_harness/perf/setup/test_event_loop.pyindex e03e985709b..75989b38275 100644--- a/tests/e2e_harness/perf/setup/test_event_loop.py+++ b/tests/e2e_harness/perf/setup/test_event_loop.py@@ -37,8 +37,14 @@ class _Slice(BaseModel):model_config = ConfigDict(frozen=True)label: str- value: int- share: float+ value: float+ share: float | None+ note: str | None+ over: bool+++def _breakdown(properties: dict[str, float | str], name: str) -> list[_Slice]:+ return TypeAdapter(list[_Slice]).validate_json(str(properties[f"perf_event_loop_{name}_breakdown"]))def test_parse_keeps_the_coroutine_its_location_and_the_hold() -> None:@@ -116,13 +122,56 @@ def test_junit_properties_count_every_hold_and_only_the_holds_past_their_allowanassert properties["perf_event_loop_offending_holds_limit"] == 0assert properties["perf_event_loop_hold_seconds_max"] == 0.731assert math.isclose(float(properties["perf_event_loop_hold_seconds_total"]), 0.302 + 2 * 0.412 + 3 * 0.731)- breakdown: Final = TypeAdapter(list[_Slice]).validate_json(str(properties["perf_event_loop_holds_breakdown"]))- assert [(slice.label, slice.value) for slice in breakdown] == [- ("RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416", 3),- ("TaskHandle._run_coro() anyio/_core/_tasks.py:275", 2),- ("LifespanOn.main() uvicorn/lifespan/on.py:86", 1),+ breakdown: Final = _breakdown(properties, "holds")+ assert [(slice.label, slice.value, slice.note, slice.over) for slice in breakdown] == [+ (+ "RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416",+ 3,+ "3 of 2 allowed, max 0.73 s",+ True,+ ),+ ("TaskHandle._run_coro() anyio/_core/_tasks.py:275", 2, "not on the allowlist, max 0.41 s", True),+ ("LifespanOn.main() uvicorn/lifespan/on.py:86", 1, "1 of 3 allowed, max 0.30 s", False),]- assert math.isclose(sum(slice.share for slice in breakdown), 1.0)+ assert math.isclose(sum(slice.share or 0.0 for slice in breakdown), 1.0)+++def test_the_offending_breakdown_lists_only_the_locations_past_their_cap_by_how_far_over_they_are() -> None:+ rows: Final = summarize(_SPEC, parse_stalls((_STARTUP, _STREAM, _STREAM, _HANDLER, _HANDLER, _HANDLER)))+ offending: Final = _breakdown(dict(junit_properties(_SPEC, rows)), "offending_holds")+ assert [(slice.label, slice.value, slice.share, slice.over) for slice in offending] == [+ ("RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416", 1, 1 / 3, True),+ ("TaskHandle._run_coro() anyio/_core/_tasks.py:275", 2, 2 / 3, True),+ ]+ clean: Final = dict(junit_properties(_SPEC, summarize(_SPEC, parse_stalls((_HANDLER, _HANDLER)))))+ assert clean["perf_event_loop_offending_holds"] == 0+ assert "perf_event_loop_offending_holds_breakdown" not in clean+++def test_the_seconds_breakdowns_carry_the_time_per_location_and_the_longest_single_holds() -> None:+ long_handler: Final = _HANDLER.replace("0.731", "0.950")+ rows: Final = summarize(_SPEC, parse_stalls((_STARTUP, _STREAM, long_handler, _HANDLER)))+ properties: Final = dict(junit_properties(_SPEC, rows))+ per_location: Final = _breakdown(properties, "hold_seconds_total")+ assert [(slice.label.split()[0], slice.value) for slice in per_location] == [+ ("RequestResponseCycle.run_asgi()", 0.950 + 0.731),+ ("TaskHandle._run_coro()", 0.412),+ ("LifespanOn.main()", 0.302),+ ]+ assert math.isclose(sum(slice.share or 0.0 for slice in per_location), 1.0)+ longest: Final = _breakdown(properties, "hold_seconds_max")+ assert [(slice.label.split()[0], slice.value, slice.share, slice.over) for slice in longest] == [+ ("RequestResponseCycle.run_asgi()", 0.950, None, False),+ ("RequestResponseCycle.run_asgi()", 0.731, None, False),+ ("TaskHandle._run_coro()", 0.412, None, True),+ ("LifespanOn.main()", 0.302, None, False),+ ]+++def test_the_longest_holds_breakdown_stops_at_ten() -> None:+ rows: Final = summarize(_SPEC, parse_stalls(tuple(_STARTUP for _ in range(12))))+ longest: Final = _breakdown(dict(junit_properties(_SPEC, rows)), "hold_seconds_max")+ assert len(longest) == 10def test_junit_properties_without_holds_report_zeros_and_no_breakdown() -> None:@@ -130,4 +179,4 @@ def test_junit_properties_without_holds_report_zeros_and_no_breakdown() -> None:assert properties["perf_event_loop_holds"] == 0assert properties["perf_event_loop_offending_holds"] == 0assert properties["perf_event_loop_hold_seconds_max"] == 0- assert "perf_event_loop_holds_breakdown" not in properties+ assert not [name for name in properties if name.endswith("_breakdown")]