Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

perf(metrics): cap inbound request-path cardinality to bound memory (#3340)

Abhay Singh88a1f4e52ba2
diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.py
index 2bab01ce0..e22b8308b 100644
--- a/headroom/proxy/prometheus_metrics.py
+++ b/headroom/proxy/prometheus_metrics.py
@@ -30,6 +30,15 @@ logger = logging.getLogger("headroom.proxy")
# client-supplied model cardinality stays bounded (see record_request).
_OTHER_MODEL = "other"
+# Same idea for inbound request paths (see record_inbound_request). ``path`` is
+# client-controlled and effectively unbounded — ID-bearing passthrough routes
+# such as ``/v1/files/{id}`` or ``/v1/responses/{id}`` mint a distinct key per
+# request. Past this many distinct paths, further paths collapse into
+# ``_OTHER_PATH`` so the counter (and its exported Prometheus series) can't grow
+# without bound. The cap sits well above a proxy's real route count.
+MAX_DISTINCT_PATHS = 256
+_OTHER_PATH = "other"
+
# Closed label set for headroom_requests_rate_limited_total{source}. Two 429s
# mean opposite things to an operator: "headroom" is OUR limiter refusing the
# request (raise the cap), "upstream" is the provider refusing it (back off or
@@ -152,6 +161,7 @@ class PrometheusMetrics:
self.inbound_requests_active = 0
self.inbound_requests_by_method: dict[str, int] = defaultdict(int)
self.inbound_requests_by_path: dict[str, int] = defaultdict(int)
+ self._path_cardinality_warned = False
self.inbound_responses_by_status: dict[str, int] = defaultdict(int)
self.tokens_input_total = 0
@@ -413,6 +423,7 @@ class PrometheusMetrics:
self.inbound_requests_active = 0
self.inbound_requests_by_method.clear()
self.inbound_requests_by_path.clear()
+ self._path_cardinality_warned = False
self.inbound_responses_by_status.clear()
self.tokens_input_total = 0
@@ -773,7 +784,23 @@ class PrometheusMetrics:
self.inbound_requests_total += 1
self.inbound_requests_active += 1
self.inbound_requests_by_method[method.upper()] += 1
- self.inbound_requests_by_path[path] += 1
+ # Cap client-controlled path cardinality, mirroring the model cap in
+ # record_request. A membership test (never a defaultdict index) keeps an
+ # over-cap path from materializing a new key and defeating the bound.
+ if path in self.inbound_requests_by_path or (
+ len(self.inbound_requests_by_path) < MAX_DISTINCT_PATHS
+ ):
+ bounded_path = path
+ else:
+ bounded_path = _OTHER_PATH
+ if not self._path_cardinality_warned:
+ self._path_cardinality_warned = True
+ logger.warning(
+ "metrics.record: inbound path cardinality cap (%d) reached; "
+ 'bucketing further paths into "other"',
+ MAX_DISTINCT_PATHS,
+ )
+ self.inbound_requests_by_path[bounded_path] += 1
def record_inbound_response(self, *, status_code: int | str) -> None:
self.inbound_requests_completed += 1
diff --git a/tests/test_observability_metrics.py b/tests/test_observability_metrics.py
index ffff3438d..6aea727e8 100644
--- a/tests/test_observability_metrics.py
+++ b/tests/test_observability_metrics.py
@@ -18,7 +18,7 @@ from headroom.observability import (
set_otel_metrics,
unregister_otel_metric_attribute_provider,
)
-from headroom.proxy.prometheus_metrics import PrometheusMetrics
+from headroom.proxy.prometheus_metrics import MAX_DISTINCT_PATHS, PrometheusMetrics
from headroom.telemetry.context import MAX_DISTINCT_MODELS
from headroom.transforms.pipeline import TransformPipeline
@@ -552,6 +552,61 @@ async def test_prometheus_metrics_reset_rearms_cardinality_warning() -> None:
assert len(metrics._cache_requests_by_model) == 0
+def test_prometheus_metrics_caps_inbound_path_cardinality() -> None:
+ """A client hitting unbounded distinct paths (ID-bearing passthrough routes)
+ cannot grow inbound_requests_by_path past MAX_DISTINCT_PATHS + the "other"
+ sentinel, while the total request count stays exact."""
+ metrics = PrometheusMetrics(stateless=True)
+
+ # Fill exactly to the cap with distinct paths: no bucketing yet.
+ for i in range(MAX_DISTINCT_PATHS):
+ metrics.record_inbound_request(method="GET", path=f"/v1/files/file_{i}")
+ assert len(metrics.inbound_requests_by_path) == MAX_DISTINCT_PATHS
+ assert "other" not in metrics.inbound_requests_by_path
+
+ # New distinct paths past the cap bucket into "other", never their own key.
+ for i in range(5):
+ metrics.record_inbound_request(method="GET", path=f"/v1/responses/resp_{i}")
+ assert "/v1/responses/resp_0" not in metrics.inbound_requests_by_path
+ assert metrics.inbound_requests_by_path["other"] == 5
+ assert len(metrics.inbound_requests_by_path) == MAX_DISTINCT_PATHS + 1
+
+ # An already-tracked path keeps incrementing after the cap is reached.
+ metrics.record_inbound_request(method="GET", path="/v1/files/file_0")
+ assert metrics.inbound_requests_by_path["/v1/files/file_0"] == 2
+
+ # Accounting is preserved: every inbound request is counted somewhere.
+ total_calls = MAX_DISTINCT_PATHS + 5 + 1
+ assert metrics.inbound_requests_total == total_calls
+ assert sum(metrics.inbound_requests_by_path.values()) == total_calls
+
+
+def test_prometheus_metrics_path_cardinality_warns_once(
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ """Bucketing inbound paths into "other" logs exactly one warning."""
+ metrics = PrometheusMetrics(stateless=True)
+ with caplog.at_level(logging.WARNING, logger="headroom.proxy"):
+ for i in range(MAX_DISTINCT_PATHS + 10):
+ metrics.record_inbound_request(method="GET", path=f"/v1/files/file_{i}")
+ cap_warnings = [r for r in caplog.records if "path cardinality cap" in r.getMessage()]
+ assert len(cap_warnings) == 1
+
+
+async def test_prometheus_metrics_reset_rearms_path_cardinality_warning() -> None:
+ """reset_runtime clears the path dict and re-arms the one-shot cap warning."""
+ metrics = PrometheusMetrics(stateless=True)
+ for i in range(MAX_DISTINCT_PATHS + 5):
+ metrics.record_inbound_request(method="GET", path=f"/v1/files/file_{i}")
+ assert metrics._path_cardinality_warned is True
+
+ await metrics.reset_runtime()
+
+ assert metrics._path_cardinality_warned is False
+ assert len(metrics.inbound_requests_by_path) == 0
+
+
@pytest.mark.asyncio
async def test_prometheus_metrics_export_bounds_model_series() -> None:
"""export() emits at most MAX_DISTINCT_MODELS model series plus the 'other' bucket."""