Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
perf(metrics): cap inbound request-path cardinality to bound memory (#3340)
Abhay Singh88a1f4e52ba2
diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.pyindex 2bab01ce0..e22b8308b 100644--- a/headroom/proxy/prometheus_metrics.py+++ b/headroom/proxy/prometheus_metrics.py@@ -30,6 +30,15 @@ logger = logging.getLogger("headroom.proxy")# client-supplied model cardinality stays bounded (see record_request)._OTHER_MODEL = "other"+# Same idea for inbound request paths (see record_inbound_request). ``path`` is+# client-controlled and effectively unbounded — ID-bearing passthrough routes+# such as ``/v1/files/{id}`` or ``/v1/responses/{id}`` mint a distinct key per+# request. Past this many distinct paths, further paths collapse into+# ``_OTHER_PATH`` so the counter (and its exported Prometheus series) can't grow+# without bound. The cap sits well above a proxy's real route count.+MAX_DISTINCT_PATHS = 256+_OTHER_PATH = "other"+# Closed label set for headroom_requests_rate_limited_total{source}. Two 429s# mean opposite things to an operator: "headroom" is OUR limiter refusing the# request (raise the cap), "upstream" is the provider refusing it (back off or@@ -152,6 +161,7 @@ class PrometheusMetrics:self.inbound_requests_active = 0self.inbound_requests_by_method: dict[str, int] = defaultdict(int)self.inbound_requests_by_path: dict[str, int] = defaultdict(int)+ self._path_cardinality_warned = Falseself.inbound_responses_by_status: dict[str, int] = defaultdict(int)self.tokens_input_total = 0@@ -413,6 +423,7 @@ class PrometheusMetrics:self.inbound_requests_active = 0self.inbound_requests_by_method.clear()self.inbound_requests_by_path.clear()+ self._path_cardinality_warned = Falseself.inbound_responses_by_status.clear()self.tokens_input_total = 0@@ -773,7 +784,23 @@ class PrometheusMetrics:self.inbound_requests_total += 1self.inbound_requests_active += 1self.inbound_requests_by_method[method.upper()] += 1- self.inbound_requests_by_path[path] += 1+ # Cap client-controlled path cardinality, mirroring the model cap in+ # record_request. A membership test (never a defaultdict index) keeps an+ # over-cap path from materializing a new key and defeating the bound.+ if path in self.inbound_requests_by_path or (+ len(self.inbound_requests_by_path) < MAX_DISTINCT_PATHS+ ):+ bounded_path = path+ else:+ bounded_path = _OTHER_PATH+ if not self._path_cardinality_warned:+ self._path_cardinality_warned = True+ logger.warning(+ "metrics.record: inbound path cardinality cap (%d) reached; "+ 'bucketing further paths into "other"',+ MAX_DISTINCT_PATHS,+ )+ self.inbound_requests_by_path[bounded_path] += 1def record_inbound_response(self, *, status_code: int | str) -> None:self.inbound_requests_completed += 1diff --git a/tests/test_observability_metrics.py b/tests/test_observability_metrics.pyindex ffff3438d..6aea727e8 100644--- a/tests/test_observability_metrics.py+++ b/tests/test_observability_metrics.py@@ -18,7 +18,7 @@ from headroom.observability import (set_otel_metrics,unregister_otel_metric_attribute_provider,)-from headroom.proxy.prometheus_metrics import PrometheusMetrics+from headroom.proxy.prometheus_metrics import MAX_DISTINCT_PATHS, PrometheusMetricsfrom headroom.telemetry.context import MAX_DISTINCT_MODELSfrom headroom.transforms.pipeline import TransformPipeline@@ -552,6 +552,61 @@ async def test_prometheus_metrics_reset_rearms_cardinality_warning() -> None:assert len(metrics._cache_requests_by_model) == 0+def test_prometheus_metrics_caps_inbound_path_cardinality() -> None:+ """A client hitting unbounded distinct paths (ID-bearing passthrough routes)+ cannot grow inbound_requests_by_path past MAX_DISTINCT_PATHS + the "other"+ sentinel, while the total request count stays exact."""+ metrics = PrometheusMetrics(stateless=True)++ # Fill exactly to the cap with distinct paths: no bucketing yet.+ for i in range(MAX_DISTINCT_PATHS):+ metrics.record_inbound_request(method="GET", path=f"/v1/files/file_{i}")+ assert len(metrics.inbound_requests_by_path) == MAX_DISTINCT_PATHS+ assert "other" not in metrics.inbound_requests_by_path++ # New distinct paths past the cap bucket into "other", never their own key.+ for i in range(5):+ metrics.record_inbound_request(method="GET", path=f"/v1/responses/resp_{i}")+ assert "/v1/responses/resp_0" not in metrics.inbound_requests_by_path+ assert metrics.inbound_requests_by_path["other"] == 5+ assert len(metrics.inbound_requests_by_path) == MAX_DISTINCT_PATHS + 1++ # An already-tracked path keeps incrementing after the cap is reached.+ metrics.record_inbound_request(method="GET", path="/v1/files/file_0")+ assert metrics.inbound_requests_by_path["/v1/files/file_0"] == 2++ # Accounting is preserved: every inbound request is counted somewhere.+ total_calls = MAX_DISTINCT_PATHS + 5 + 1+ assert metrics.inbound_requests_total == total_calls+ assert sum(metrics.inbound_requests_by_path.values()) == total_calls+++def test_prometheus_metrics_path_cardinality_warns_once(+ caplog: pytest.LogCaptureFixture,+) -> None:+ """Bucketing inbound paths into "other" logs exactly one warning."""+ metrics = PrometheusMetrics(stateless=True)+ with caplog.at_level(logging.WARNING, logger="headroom.proxy"):+ for i in range(MAX_DISTINCT_PATHS + 10):+ metrics.record_inbound_request(method="GET", path=f"/v1/files/file_{i}")+ cap_warnings = [r for r in caplog.records if "path cardinality cap" in r.getMessage()]+ assert len(cap_warnings) == 1+++async def test_prometheus_metrics_reset_rearms_path_cardinality_warning() -> None:+ """reset_runtime clears the path dict and re-arms the one-shot cap warning."""+ metrics = PrometheusMetrics(stateless=True)+ for i in range(MAX_DISTINCT_PATHS + 5):+ metrics.record_inbound_request(method="GET", path=f"/v1/files/file_{i}")+ assert metrics._path_cardinality_warned is True++ await metrics.reset_runtime()++ assert metrics._path_cardinality_warned is False+ assert len(metrics.inbound_requests_by_path) == 0++@pytest.mark.asyncioasync def test_prometheus_metrics_export_bounds_model_series() -> None:"""export() emits at most MAX_DISTINCT_MODELS model series plus the 'other' bucket."""