Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(router): add max_output_tokens condition to ModelRoute (#3898)
Sudhindra Desai01185e32d393
diff --git a/docs/content/docs/configuration.mdx b/docs/content/docs/configuration.mdxindex b77aedd0e..e59cf1f6e 100644--- a/docs/content/docs/configuration.mdx+++ b/docs/content/docs/configuration.mdx@@ -480,6 +480,7 @@ Enable it with `HEADROOM_MODEL_ROUTER_ENABLED=1` and declare ordered rules in `H| `to_model` | string (required) | Model to route to when the rule matches. || `max_input_tokens` | integer | Match only when the estimated input size is at or below this. || `min_input_tokens` | integer | Match only when the estimated input size is at or above this. |+| `max_output_tokens` | integer | Match only when the request's own `max_tokens` is at or below this. A request without `max_tokens` does not match. || `require_no_tools` | boolean | Match only when the request declares no tools (a proxy for low-risk work). || `from_models` | list of strings | Restrict the rule to these source models. Omit for any source model. || `name` | string | Label surfaced in the decision log. |@@ -494,6 +495,7 @@ export HEADROOM_MODEL_ROUTES='[Notes:+- Combine `max_input_tokens` with `max_output_tokens` to downshift only turns that are cheap on both sides: a tiny prompt that asks for a long answer (a large `max_tokens`) stays on the original model.- Input size is a fast, tokenizer-free estimate over the messages, tools, and top-level `system` prompt, meant for tier selection rather than exact accounting.- A malformed rule fails open: it is skipped (never silently widened), and the rest of the rules still apply.- Routing is skipped for byte-faithful passthrough requests (`x-headroom-bypass: true` or `x-headroom-mode: passthrough`), so those are never model-rewritten.diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 00a3848fe..e49da2982 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -61,7 +61,7 @@ from headroom.proxy.identity import resolve_memory_identityfrom headroom.proxy.image_isolation import run_image_compression_isolatedfrom headroom.proxy.memory_decision import MemoryDecisionfrom headroom.proxy.memory_query import MemoryQuery-from headroom.proxy.model_router import estimate_input_tokens+from headroom.proxy.model_router import estimate_input_tokens, request_max_tokensfrom headroom.proxy.nonstream_sse_policy import should_recover_sse_replyfrom headroom.proxy.outcome import RequestOutcomefrom headroom.proxy.output_shaper import shaper_enabled_for, steering_allowed_for@@ -910,6 +910,7 @@ class AnthropicHandlerMixin:model=model,input_tokens=estimate_input_tokens(messages, body.get("tools"), body.get("system")),has_tools=bool(body.get("tools")),+ max_tokens=request_max_tokens(body),)logger.info("model routing decision: %s", decision.reason)if not decision.changed:diff --git a/headroom/proxy/model_router.py b/headroom/proxy/model_router.pyindex a46a002ac..3181aa69b 100644--- a/headroom/proxy/model_router.py+++ b/headroom/proxy/model_router.py@@ -61,7 +61,25 @@ class ModelRoute:name: str = """""Human-readable label surfaced in decision logs."""- def matches(self, *, model: str, input_tokens: int, has_tools: bool) -> bool:+ # Appended after the original fields so existing positional+ # ``ModelRoute(...)`` calls keep their meaning.+ max_output_tokens: int | None = None+ """Match only when the request's own ``max_tokens`` is <= this.++ The caller's declared output budget is the best "this turn is trivial"+ signal at routing time: a tiny prompt asking for 4000 output tokens may+ want a long answer from the strong model. A request that declares no+ ``max_tokens`` does NOT match (an unbounded response is not a trivial+ turn), so the rule can never widen to unbounded requests."""++ def matches(+ self,+ *,+ model: str,+ input_tokens: int,+ has_tools: bool,+ max_tokens: int | None = None,+ ) -> bool:"""True when every set condition is satisfied for this request."""if self.from_models and model not in self.from_models:return False@@ -73,6 +91,10 @@ class ModelRoute:return Falseif self.min_input_tokens is not None and input_tokens < self.min_input_tokens:return False+ if self.max_output_tokens is not None and (+ max_tokens is None or max_tokens > self.max_output_tokens+ ):+ return False# A rule whose ``to_model`` equals the current model still MATCHES (strict# first-match-wins): it is a no-op (``changed`` is False) that short-circuits# later rules, which lets an operator write an explicit exemption rule.@@ -132,7 +154,14 @@ class ModelRouter:def enabled(self) -> bool:return self._config.enabled and bool(self._config.routes)- def select(self, *, model: str, input_tokens: int, has_tools: bool) -> ModelDecision:+ def select(+ self,+ *,+ model: str,+ input_tokens: int,+ has_tools: bool,+ max_tokens: int | None = None,+ ) -> ModelDecision:"""Return the routing decision for a request.Never raises: on a disabled router or no matching rule, returns a@@ -144,11 +173,14 @@ class ModelRouter:return ModelDecision(model, model, matched=False, reason="no source model")for route in self._config.routes:- if route.matches(model=model, input_tokens=input_tokens, has_tools=has_tools):+ if route.matches(+ model=model, input_tokens=input_tokens, has_tools=has_tools, max_tokens=max_tokens+ ):reason = (f"matched rule {route.name or route.to_model!r}: "f"{model} -> {route.to_model} "- f"(input_tokens={input_tokens}, has_tools={has_tools})"+ f"(input_tokens={input_tokens}, has_tools={has_tools}, "+ f"max_tokens={max_tokens})")return ModelDecision(original_model=model,@@ -187,6 +219,18 @@ def estimate_input_tokens(messages: object, tools: object = None, system: objectreturn 0+def request_max_tokens(body: object) -> int | None:+ """Return the request's ``max_tokens``, or ``None`` if absent or not an integer.++ Booleans and non-integers are treated as absent so a malformed client value+ can never satisfy a ``max_output_tokens`` bound. It never raises.+ """+ value = body.get("max_tokens") if isinstance(body, dict) else None+ if isinstance(value, int) and not isinstance(value, bool):+ return value+ return None++def _truthy(value: str | None) -> bool:return (value or "").strip().lower() in {"1", "true", "yes", "on", "enable", "enabled"}@@ -219,6 +263,7 @@ _ALLOWED_ROUTE_KEYS = frozenset("to_model","max_input_tokens","min_input_tokens",+ "max_output_tokens","require_no_tools","require_tools","from_models",@@ -252,7 +297,8 @@ def _route_from_entry(entry: object, index: int) -> ModelRoute | None:max_tokens = _strict_opt_int(entry, "max_input_tokens", index)min_tokens = _strict_opt_int(entry, "min_input_tokens", index)- if max_tokens is _INVALID or min_tokens is _INVALID:+ max_output = _strict_opt_int(entry, "max_output_tokens", index)+ if max_tokens is _INVALID or min_tokens is _INVALID or max_output is _INVALID:return Nonerequire_no_tools = entry.get("require_no_tools", False)@@ -280,6 +326,7 @@ def _route_from_entry(entry: object, index: int) -> ModelRoute | None:to_model=to_model,max_input_tokens=max_tokens, # type: ignore[arg-type]min_input_tokens=min_tokens, # type: ignore[arg-type]+ max_output_tokens=max_output, # type: ignore[arg-type]require_no_tools=require_no_tools,require_tools=require_tools,from_models=tuple(from_models_raw),diff --git a/tests/test_proxy/test_model_router.py b/tests/test_proxy/test_model_router.pyindex 97b9ac20f..b2f318ee0 100644--- a/tests/test_proxy/test_model_router.py+++ b/tests/test_proxy/test_model_router.py@@ -8,6 +8,7 @@ from headroom.proxy.model_router import (ModelRouter,ModelRouterConfig,estimate_input_tokens,+ request_max_tokens,)# ---------------------------------------------------------------------------@@ -52,6 +53,45 @@ def test_route_require_tools_and_require_no_tools_never_matches() -> None:assert not route.matches(model="m", input_tokens=1, has_tools=False)+def test_route_max_output_tokens_bounds_declared_budget() -> None:+ # Issue #2765: a tiny prompt asking for a long answer must not downshift.+ route = ModelRoute(to_model="cheap", max_input_tokens=4096, max_output_tokens=512)+ assert route.matches(model="strong", input_tokens=10, has_tools=False, max_tokens=512)+ assert route.matches(model="strong", input_tokens=10, has_tools=False, max_tokens=1)+ assert not route.matches(model="strong", input_tokens=10, has_tools=False, max_tokens=4000)+++def test_route_max_output_tokens_absent_budget_does_not_match() -> None:+ # An unbounded response is not a trivial turn: absent max_tokens never+ # satisfies the bound, so the rule cannot widen to unbounded requests.+ route = ModelRoute(to_model="cheap", max_output_tokens=512)+ assert not route.matches(model="strong", input_tokens=10, has_tools=False)+ assert not route.matches(model="strong", input_tokens=10, has_tools=False, max_tokens=None)+++def test_route_without_max_output_tokens_ignores_budget() -> None:+ route = ModelRoute(to_model="cheap", max_input_tokens=4096)+ assert route.matches(model="strong", input_tokens=10, has_tools=False, max_tokens=100_000)+ assert route.matches(model="strong", input_tokens=10, has_tools=False)+++def test_route_positional_constructor_keeps_original_field_order() -> None:+ # max_output_tokens is appended last, so positional ModelRoute(...) calls+ # written before #2765 keep their meaning: here the 4th positional arg is+ # still require_no_tools, not the new output bound.+ route = ModelRoute("cheap", 4000, None, True)+ assert route.require_no_tools is True+ assert route.max_output_tokens is None+ assert not route.matches(model="strong", input_tokens=10, has_tools=True, max_tokens=1)+ assert route.matches(model="strong", input_tokens=10, has_tools=False, max_tokens=1)++ route = ModelRoute("cheap", None, None, False, True, ("strong",), "agentic")+ assert route.require_tools is True+ assert route.from_models == ("strong",)+ assert route.name == "agentic"+ assert route.max_output_tokens is None++def test_route_matches_even_for_same_model() -> None:# A same-model rule still MATCHES (strict first-match-wins); it is a no-op# that short-circuits later rules, enabling explicit exemption rules.@@ -282,3 +322,42 @@ def test_estimate_input_tokens_counts_system_string() -> None:def test_estimate_input_tokens_counts_system_blocks() -> None:blocks = [{"type": "text", "text": "x" * 4000}]assert estimate_input_tokens([{"content": "hi"}], system=blocks) > 100+++def test_select_passes_max_tokens_to_rules() -> None:+ router = ModelRouter(+ ModelRouterConfig(+ enabled=True,+ routes=(ModelRoute(to_model="cheap", max_output_tokens=512, name="trivial"),),+ )+ )+ small = router.select(model="strong", input_tokens=10, has_tools=False, max_tokens=256)+ assert small.changed and small.routed_model == "cheap"+ assert "max_tokens=256" in small.reason+ large = router.select(model="strong", input_tokens=10, has_tools=False, max_tokens=4000)+ assert not large.matched and large.routed_model == "strong"+++def test_from_env_parses_max_output_tokens() -> None:+ cfg = ModelRouterConfig.from_env(+ "1", '[{"to_model": "cheap", "max_input_tokens": 4096, "max_output_tokens": 512}]'+ )+ assert cfg.enabled+ assert cfg.routes[0].max_output_tokens == 512+++def test_from_env_malformed_max_output_tokens_skips_route() -> None:+ for bad in ('"lots"', "true", "-1"):+ cfg = ModelRouterConfig.from_env(+ "1", f'[{{"to_model": "cheap", "max_output_tokens": {bad}}}]'+ )+ assert not cfg.routes, bad+++def test_request_max_tokens() -> None:+ assert request_max_tokens({"max_tokens": 4000}) == 4000+ assert request_max_tokens({}) is None+ assert request_max_tokens({"max_tokens": True}) is None+ assert request_max_tokens({"max_tokens": "4000"}) is None+ assert request_max_tokens({"max_tokens": 1.5}) is None+ assert request_max_tokens(None) is Nonediff --git a/tests/test_proxy/test_model_router_wiring.py b/tests/test_proxy/test_model_router_wiring.pyindex ffd56822a..1ef24fa74 100644--- a/tests/test_proxy/test_model_router_wiring.py+++ b/tests/test_proxy/test_model_router_wiring.py@@ -159,6 +159,29 @@ def test_maybe_route_model_routes_when_enabled() -> None:tracker.mark_mutated.assert_called_once_with("model_router")+def test_maybe_route_model_honors_max_output_tokens() -> None:+ # Issue #2765: the request's own max_tokens reaches the router.+ host = _RouterHost()+ host.model_router = ModelRouter(+ ModelRouterConfig(+ enabled=True,+ routes=(ModelRoute(to_model="claude-haiku-4-5", max_output_tokens=512),),+ )+ )+ long_answer = {"model": "claude-opus-4-8", "max_tokens": 4000}+ out = host._maybe_route_model(+ "claude-opus-4-8", [{"content": "OK?"}], long_answer, MagicMock(), False+ )+ assert out == "claude-opus-4-8"+ assert long_answer["model"] == "claude-opus-4-8"++ trivial = {"model": "claude-opus-4-8", "max_tokens": 256}+ out = host._maybe_route_model(+ "claude-opus-4-8", [{"content": "OK?"}], trivial, MagicMock(), False+ )+ assert out == "claude-haiku-4-5"++@pytest.mark.parametrize(("routes", "expected_reason"),[