Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

3.2 KB
Raw Blame
"""Comparing one load phase against another, for tests that degrade a dependency mid-run.

Two shapes of ceiling, because the metrics divide into two kinds. RSS and CPU are
machine-shaped: RSS scales with worker count and CPU with core count, so an absolute number
calibrated on one runner means nothing on the next, and what travels is the ratio against a
healthy phase measured on the same machine in the same run. Latency and log volume are not:
a ratio there is actively misleading, because a dependency that fails fast once its breaker
opens can make the degraded phase look cheaper than the healthy one while still being far
slower or noisier than a user should ever see. Those get a flat ceiling, which is the promise
the test is actually making.
"""

from __future__ import annotations

from dataclasses import dataclass
from typing import Final, TypeAlias


def _rendered(value: float, unit: str, decimals: int) -> str:
    return f"{value:.{decimals}f}{unit}"


@dataclass(frozen=True, slots=True)
class RatioBudget:
    """One metric's healthy value, its degraded value, and how much growth is allowed."""

    name: str
    baseline: float
    degraded: float
    ratio_ceiling: float
    unit: str
    decimals: int = 1

    @property
    def ratio(self) -> float | None:
        """How many times the baseline the degraded value is, or None if there is no baseline."""
        return self.degraded / self.baseline if self.baseline > 0 else None

    def violation(self) -> str | None:
        """Why this metric fails its budget, or None if it passes."""
        ratio: Final = self.ratio
        if ratio is None:
            return (
                f"{self.name} measured {_rendered(self.baseline, self.unit, self.decimals)} in the healthy phase, "
                f"so there is nothing to compare the degraded phase against; the measurement did not happen"
            )
        if ratio > self.ratio_ceiling:
            return (
                f"{self.name} went from {_rendered(self.baseline, self.unit, self.decimals)} healthy to "
                f"{_rendered(self.degraded, self.unit, self.decimals)} degraded, {ratio:.1f}x the baseline and past "
                f"the {self.ratio_ceiling:.1f}x allowed"
            )
        return None


@dataclass(frozen=True, slots=True)
class AbsoluteBudget:
    """One metric's degraded value against a flat ceiling, for metrics a ratio cannot bound."""

    name: str
    measured: float
    ceiling: float
    unit: str
    decimals: int = 1

    def violation(self) -> str | None:
        """Why this metric fails its budget, or None if it passes."""
        if self.measured > self.ceiling:
            return (
                f"{self.name} measured {_rendered(self.measured, self.unit, self.decimals)} in the degraded phase, "
                f"past the {_rendered(self.ceiling, self.unit, self.decimals)} allowed"
            )
        return None


Budget: TypeAlias = RatioBudget | AbsoluteBudget


def violations(budgets: tuple[Budget, ...]) -> tuple[str, ...]:
    """Every budget the run blew, so one failure reports all of them instead of the first."""
    return tuple(violation for budget in budgets if (violation := budget.violation()) is not None)