Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
refactor(output): isolate verbosity steering (#1940)
JD Davis0ce09fb63fff
diff --git a/headroom/proxy/output_shaper.py b/headroom/proxy/output_shaper.pyindex e286d4204..f469e1715 100644--- a/headroom/proxy/output_shaper.py+++ b/headroom/proxy/output_shaper.py@@ -39,9 +39,33 @@ from enum import Enumfrom typing import Anyfrom headroom.proxy import runtime_env+from headroom.proxy.output_steering import (+ apply_openai_responses_verbosity_steering,+ apply_verbosity_steering,+ replace_or_append_steering_block,+ steering_text,+)logger = logging.getLogger(__name__)+__all__ = [+ "LEGACY_THINKING_FLOOR",+ "OutputShaperSettings",+ "ShapeResult",+ "TurnKind",+ "apply_openai_responses_verbosity_steering",+ "apply_verbosity_steering",+ "classify_openai_responses_input",+ "classify_turn",+ "resolve_verbosity_level",+ "route_effort",+ "route_openai_reasoning_effort",+ "route_openai_text_verbosity",+ "shape_openai_responses_request",+ "shape_request",+ "steering_text",+]+# Documented Anthropic API minimum for thinking.budget_tokens on models# that still accept the legacy enabled/budget_tokens form.LEGACY_THINKING_FLOOR = 1024@@ -60,38 +84,7 @@ _OPENAI_RESPONSES_OUTPUT_ITEM_TYPES = frozenset(})-# Sentinel prefix marks the steering block so application is idempotent and-# the block is recognizable in logs/diffs.-_STEERING_SENTINEL = "<headroom_output_shaping>"-_STEERING_SUFFIX = "</headroom_output_shaping>"--# Levels are cumulative: each includes everything above it. Text must stay-# byte-stable across releases for prefix-cache friendliness — treat edits to-# these strings as cache-busting changes.-_VERBOSITY_LEVELS = {- 1: (- "Skip preamble and postamble. Do not announce what you are about to "- "do or recap what you just did; start with the substance."- ),- 2: (- "Skip preamble and postamble; start with the substance. Never restate "- "code, file contents, diffs, or tool output that already appear in "- "this conversation — reference them by path and line instead. After a "- "tool call succeeds, continue without narrating the result."- ),- 3: (- "Skip preamble and postamble. Never restate code, file contents, "- "diffs, or tool output already in this conversation — reference by "- "path and line. Give conclusions only; omit rationale unless the user "- "asks why. Prefer the smallest edit over rewriting whole files. Keep "- "prose to the minimum needed to be unambiguous."- ),- 4: (- "Minimum tokens. Fragments fine. No preamble, no postamble, no "- "restating context, no rationale. Answer, smallest-possible edits, "- "nothing else."- ),-}+_replace_or_append_steering_block = replace_or_append_steering_blockclass TurnKind(Enum):@@ -253,66 +246,6 @@ def classify_turn(messages: list[dict[str, Any]]) -> TurnKind:return TurnKind.UNKNOWN-def steering_text(level: int) -> str | None:- """The full steering block for a verbosity level, or None for level 0."""- text = _VERBOSITY_LEVELS.get(level)- if text is None:- return None- return f"{_STEERING_SENTINEL}\n{text}\n{_STEERING_SUFFIX}"---def _replace_or_append_steering_block(existing: str, block: str) -> tuple[str, bool]:- """Replace an existing steering block in text, or append one at the tail."""- start = existing.find(_STEERING_SENTINEL)- if start >= 0:- end = existing.find(_STEERING_SUFFIX, start)- end = len(existing) if end < 0 else end + len(_STEERING_SUFFIX)- prefix = existing[:start].rstrip()- suffix = existing[end:].lstrip("\n")- parts = [part for part in (prefix, block, suffix) if part]- updated = "\n\n".join(parts)- return updated, updated != existing-- updated = f"{existing.rstrip()}\n\n{block}" if existing.strip() else block- return updated, updated != existing---def apply_verbosity_steering(body: dict[str, Any], level: int) -> bool:- """Append the steering block to the tail of the system prompt.-- Appending AFTER the last system block keeps any ``cache_control``- breakpoint on an earlier block intact — the cached prefix is unchanged- and only the (small, byte-stable) steering block is reprocessed.-- A string system prompt is converted to block form so the original text- keeps its exact bytes as the first block.- """- text = steering_text(level)- if text is None:- return False-- system = body.get("system")- if system is None:- body["system"] = [{"type": "text", "text": text}]- return True- if isinstance(system, str):- body["system"] = [- {"type": "text", "text": system},- {"type": "text", "text": text},- ]- return True- if isinstance(system, list):- for block in system:- if isinstance(block, dict) and block.get("text", "").startswith(_STEERING_SENTINEL):- if block["text"] == text:- return False # already applied at this level- block["text"] = text # level changed mid-session- return True- system.append({"type": "text", "text": text})- return True- return False--def route_effort(body: dict[str, Any],kind: TurnKind,@@ -416,28 +349,6 @@ def classify_openai_responses_input(input_data: Any) -> TurnKind:return TurnKind.UNKNOWN-def apply_openai_responses_verbosity_steering(- body: dict[str, Any],- level: int,-) -> bool:- """Append or replace steering in OpenAI Responses ``instructions``."""- text = steering_text(level)- if text is None:- return False-- instructions = body.get("instructions")- if instructions is None:- body["instructions"] = text- return True- if not isinstance(instructions, str):- return False-- updated, changed = _replace_or_append_steering_block(instructions, text)- if changed:- body["instructions"] = updated- return changed--def route_openai_reasoning_effort(body: dict[str, Any],kind: TurnKind,diff --git a/headroom/proxy/output_steering.py b/headroom/proxy/output_steering.pynew file mode 100644index 000000000..c9ef1a263--- /dev/null+++ b/headroom/proxy/output_steering.py@@ -0,0 +1,117 @@+"""Byte-stable output verbosity steering helpers."""++from __future__ import annotations++from typing import Any++# Sentinel prefix marks the steering block so application is idempotent and+# the block is recognizable in logs/diffs.+_STEERING_SENTINEL = "<headroom_output_shaping>"+_STEERING_SUFFIX = "</headroom_output_shaping>"++# Levels are cumulative: each includes everything above it. Text must stay+# byte-stable across releases for prefix-cache friendliness; treat edits to+# these strings as cache-busting changes.+_VERBOSITY_LEVELS = {+ 1: (+ "Skip preamble and postamble. Do not announce what you are about to "+ "do or recap what you just did; start with the substance."+ ),+ 2: (+ "Skip preamble and postamble; start with the substance. Never restate "+ "code, file contents, diffs, or tool output that already appear in "+ "this conversation — reference them by path and line instead. After a "+ "tool call succeeds, continue without narrating the result."+ ),+ 3: (+ "Skip preamble and postamble. Never restate code, file contents, "+ "diffs, or tool output already in this conversation — reference by "+ "path and line. Give conclusions only; omit rationale unless the user "+ "asks why. Prefer the smallest edit over rewriting whole files. Keep "+ "prose to the minimum needed to be unambiguous."+ ),+ 4: (+ "Minimum tokens. Fragments fine. No preamble, no postamble, no "+ "restating context, no rationale. Answer, smallest-possible edits, "+ "nothing else."+ ),+}+++def steering_text(level: int) -> str | None:+ """The full steering block for a verbosity level, or None for level 0."""+ text = _VERBOSITY_LEVELS.get(level)+ if text is None:+ return None+ return f"{_STEERING_SENTINEL}\n{text}\n{_STEERING_SUFFIX}"+++def replace_or_append_steering_block(existing: str, block: str) -> tuple[str, bool]:+ """Replace an existing steering block in text, or append one at the tail."""+ start = existing.find(_STEERING_SENTINEL)+ if start >= 0:+ end = existing.find(_STEERING_SUFFIX, start)+ end = len(existing) if end < 0 else end + len(_STEERING_SUFFIX)+ prefix = existing[:start].rstrip()+ suffix = existing[end:].lstrip("\n")+ parts = [part for part in (prefix, block, suffix) if part]+ updated = "\n\n".join(parts)+ return updated, updated != existing++ updated = f"{existing.rstrip()}\n\n{block}" if existing.strip() else block+ return updated, updated != existing+++def apply_verbosity_steering(body: dict[str, Any], level: int) -> bool:+ """Append the steering block to the tail of the Anthropic system prompt.++ Appending after the last system block keeps any ``cache_control``+ breakpoint on an earlier block intact: the cached prefix is unchanged and+ only the small, byte-stable steering block is reprocessed.+ """+ text = steering_text(level)+ if text is None:+ return False++ system = body.get("system")+ if system is None:+ body["system"] = [{"type": "text", "text": text}]+ return True+ if isinstance(system, str):+ body["system"] = [+ {"type": "text", "text": system},+ {"type": "text", "text": text},+ ]+ return True+ if isinstance(system, list):+ for block in system:+ if isinstance(block, dict) and block.get("text", "").startswith(_STEERING_SENTINEL):+ if block["text"] == text:+ return False+ block["text"] = text+ return True+ system.append({"type": "text", "text": text})+ return True+ return False+++def apply_openai_responses_verbosity_steering(+ body: dict[str, Any],+ level: int,+) -> bool:+ """Append or replace steering in OpenAI Responses ``instructions``."""+ text = steering_text(level)+ if text is None:+ return False++ instructions = body.get("instructions")+ if instructions is None:+ body["instructions"] = text+ return True+ if not isinstance(instructions, str):+ return False++ updated, changed = replace_or_append_steering_block(instructions, text)+ if changed:+ body["instructions"] = updated+ return changeddiff --git a/tests/test_output_steering.py b/tests/test_output_steering.pynew file mode 100644index 000000000..144c93b74--- /dev/null+++ b/tests/test_output_steering.py@@ -0,0 +1,44 @@+"""Tests for output verbosity steering helpers."""++from __future__ import annotations++from headroom.proxy.output_steering import (+ apply_openai_responses_verbosity_steering,+ apply_verbosity_steering,+ replace_or_append_steering_block,+ steering_text,+)+++def test_replace_or_append_steering_block_replaces_existing_block() -> None:+ old = steering_text(1)+ new = steering_text(3)+ assert old is not None+ assert new is not None+ updated, changed = replace_or_append_steering_block(f"System.\n\n{old}\n\nTail.", new)++ assert changed is True+ assert old not in updated+ assert updated == f"System.\n\n{new}\n\nTail."+++def test_anthropic_steering_preserves_cached_prefix_block() -> None:+ cached = {+ "type": "text",+ "text": "Big system prompt.",+ "cache_control": {"type": "ephemeral"},+ }+ body = {"system": [cached.copy()]}++ assert apply_verbosity_steering(body, 2) is True+ assert body["system"][0] == cached+ assert body["system"][1] == {"type": "text", "text": steering_text(2)}+++def test_openai_responses_steering_is_idempotent() -> None:+ body = {"instructions": "System."}++ assert apply_openai_responses_verbosity_steering(body, 2) is True+ snapshot = body.copy()+ assert apply_openai_responses_verbosity_steering(body, 2) is False+ assert body == snapshot