The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
from datetime import datetime, timedelta, timezone
from time import monotonic
import pytest
from prometheus_client import REGISTRY
import litellm
from litellm.integrations.prometheus import PrometheusLogger
from litellm.integrations.prometheus_helpers import bounded_prometheus_series_tracker
from litellm.integrations.prometheus_helpers.bounded_prometheus_series_tracker import (
BoundedPrometheusSeriesTracker,
)
from litellm.types.integrations.prometheus import UserAPIKeyLabelValues
@pytest.fixture(autouse=True)
def cleanup_prometheus_registry():
collectors = list(REGISTRY._collector_to_names.keys())
for collector in collectors:
try:
REGISTRY.unregister(collector)
except Exception:
pass
old_enable_end_user = litellm.enable_end_user_cost_tracking_prometheus_only
old_metrics_config = litellm.prometheus_metrics_config
old_max_series = litellm.prometheus_end_user_metrics_max_series_per_metric
old_ttl_seconds = litellm.prometheus_end_user_metrics_ttl_seconds
old_cleanup_interval_seconds = (
litellm.prometheus_end_user_metrics_cleanup_interval_seconds
)
yield
litellm.enable_end_user_cost_tracking_prometheus_only = old_enable_end_user
litellm.prometheus_metrics_config = old_metrics_config
litellm.prometheus_end_user_metrics_max_series_per_metric = old_max_series
litellm.prometheus_end_user_metrics_ttl_seconds = old_ttl_seconds
litellm.prometheus_end_user_metrics_cleanup_interval_seconds = (
old_cleanup_interval_seconds
)
collectors = list(REGISTRY._collector_to_names.keys())
for collector in collectors:
try:
REGISTRY.unregister(collector)
except Exception:
pass
def test_prometheus_end_user_series_are_capped_per_metric():
litellm.enable_end_user_cost_tracking_prometheus_only = True
litellm.prometheus_metrics_config = [
{
"group": "end-user-spend",
"metrics": ["litellm_spend_metric"],
"include_labels": ["end_user"],
}
]
litellm.prometheus_end_user_metrics_max_series_per_metric = 3
litellm.prometheus_end_user_metrics_ttl_seconds = None
logger = PrometheusLogger()
for index in range(6):
PrometheusLogger._inc_labeled_counter(
logger,
logger.litellm_spend_metric,
"litellm_spend_metric",
UserAPIKeyLabelValues(end_user=f"end-user-{index}"),
amount=0.01,
)
assert len(logger.litellm_spend_metric._metrics) == 3
assert set(logger.litellm_spend_metric._metrics) == {
("end-user-3",),
("end-user-4",),
("end-user-5",),
}
def test_bounded_prometheus_series_tracker_is_label_agnostic():
class FakeMetric:
def __init__(self):
self.removed_label_values = []
def remove(self, *label_values):
self.removed_label_values.append(label_values)
metric = FakeMetric()
tracker = BoundedPrometheusSeriesTracker()
for index in range(4):
tracker.track_series(
metric=metric,
metric_name="generic_metric",
label_values=(f"route-{index}", "200"),
max_series=2,
ttl_seconds=None,
cleanup_interval_seconds=60.0,
)
assert metric.removed_label_values == [
("route-0", "200"),
("route-1", "200"),
]
def test_bounded_prometheus_series_tracker_treats_zero_max_as_unlimited():
# A misconfigured ``max_series=0`` must not silently evict every emission.
class FakeMetric:
def __init__(self):
self.removed_label_values = []
def remove(self, *label_values):
self.removed_label_values.append(label_values)
metric = FakeMetric()
tracker = BoundedPrometheusSeriesTracker()
for index in range(3):
tracker.track_series(
metric=metric,
metric_name="generic_metric",
label_values=(f"end-user-{index}",),
max_series=0,
ttl_seconds=None,
cleanup_interval_seconds=60.0,
)
assert metric.removed_label_values == []
def test_prometheus_end_user_series_expire_by_ttl(monkeypatch):
litellm.enable_end_user_cost_tracking_prometheus_only = True
litellm.prometheus_metrics_config = [
{
"group": "end-user-spend",
"metrics": ["litellm_spend_metric"],
"include_labels": ["end_user"],
}
]
litellm.prometheus_end_user_metrics_max_series_per_metric = None
litellm.prometheus_end_user_metrics_ttl_seconds = 10.0
litellm.prometheus_end_user_metrics_cleanup_interval_seconds = 0.0
logger = PrometheusLogger()
current_time = [monotonic()]
monkeypatch.setattr(
bounded_prometheus_series_tracker.time,
"monotonic",
lambda: current_time[0],
)
PrometheusLogger._inc_labeled_counter(
logger,
logger.litellm_spend_metric,
"litellm_spend_metric",
UserAPIKeyLabelValues(end_user="stale-end-user"),
amount=0.01,
)
current_time[0] += 11.0
PrometheusLogger._inc_labeled_counter(
logger,
logger.litellm_spend_metric,
"litellm_spend_metric",
UserAPIKeyLabelValues(end_user="fresh-end-user"),
amount=0.01,
)
assert set(logger.litellm_spend_metric._metrics) == {("fresh-end-user",)}
def test_prometheus_end_user_not_tracked_by_default():
litellm.enable_end_user_cost_tracking_prometheus_only = None
labels = PrometheusLogger().get_labels_for_metric("litellm_spend_metric")
assert "end_user" in labels
label_values = UserAPIKeyLabelValues(end_user="not-exported")
from litellm.integrations.prometheus import prometheus_label_factory
prometheus_labels = prometheus_label_factory(labels, label_values)
assert prometheus_labels["end_user"] is None
def test_prometheus_customer_budget_series_are_capped_per_metric(monkeypatch):
monkeypatch.setattr(litellm, "enable_end_user_cost_tracking_prometheus_only", True)
monkeypatch.setattr(litellm, "prometheus_end_user_metrics_max_series_per_metric", 2)
monkeypatch.setattr(litellm, "prometheus_end_user_metrics_ttl_seconds", None)
logger = PrometheusLogger()
for index in range(5):
logger._set_customer_budget_metrics(
end_user_id=f"customer-{index}",
spend=1.0,
max_budget=10.0,
budget_reset_at=None,
)
assert set(logger.litellm_remaining_customer_budget_metric._metrics) == {("customer-3",), ("customer-4",)}
assert set(logger.litellm_customer_max_budget_metric._metrics) == {("customer-3",), ("customer-4",)}
def test_prometheus_customer_budget_series_expire_by_ttl(monkeypatch):
monkeypatch.setattr(litellm, "enable_end_user_cost_tracking_prometheus_only", True)
monkeypatch.setattr(litellm, "prometheus_end_user_metrics_max_series_per_metric", None)
monkeypatch.setattr(litellm, "prometheus_end_user_metrics_ttl_seconds", 10.0)
monkeypatch.setattr(litellm, "prometheus_end_user_metrics_cleanup_interval_seconds", 0.0)
logger = PrometheusLogger()
current_time = [monotonic()]
monkeypatch.setattr(bounded_prometheus_series_tracker.time, "monotonic", lambda: current_time[0])
logger._set_customer_budget_metrics(
end_user_id="customer-with-removed-budget",
spend=1.0,
max_budget=10.0,
budget_reset_at=datetime.now(timezone.utc) + timedelta(hours=1),
)
current_time[0] += 11.0
logger._set_customer_budget_metrics(
end_user_id="customer-still-budgeted",
spend=1.0,
max_budget=10.0,
budget_reset_at=datetime.now(timezone.utc) + timedelta(hours=1),
)
assert set(logger.litellm_remaining_customer_budget_metric._metrics) == {("customer-still-budgeted",)}
assert set(logger.litellm_customer_max_budget_metric._metrics) == {("customer-still-budgeted",)}
assert set(logger.litellm_customer_budget_remaining_hours_metric._metrics) == {("customer-still-budgeted",)}