Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

refactor(output): isolate verbosity steering (#1940)

JD Davis0ce09fb63fff
diff --git a/headroom/proxy/output_shaper.py b/headroom/proxy/output_shaper.py
index e286d4204..f469e1715 100644
--- a/headroom/proxy/output_shaper.py
+++ b/headroom/proxy/output_shaper.py
@@ -39,9 +39,33 @@ from enum import Enum
from typing import Any
from headroom.proxy import runtime_env
+from headroom.proxy.output_steering import (
+ apply_openai_responses_verbosity_steering,
+ apply_verbosity_steering,
+ replace_or_append_steering_block,
+ steering_text,
+)
logger = logging.getLogger(__name__)
+__all__ = [
+ "LEGACY_THINKING_FLOOR",
+ "OutputShaperSettings",
+ "ShapeResult",
+ "TurnKind",
+ "apply_openai_responses_verbosity_steering",
+ "apply_verbosity_steering",
+ "classify_openai_responses_input",
+ "classify_turn",
+ "resolve_verbosity_level",
+ "route_effort",
+ "route_openai_reasoning_effort",
+ "route_openai_text_verbosity",
+ "shape_openai_responses_request",
+ "shape_request",
+ "steering_text",
+]
+
# Documented Anthropic API minimum for thinking.budget_tokens on models
# that still accept the legacy enabled/budget_tokens form.
LEGACY_THINKING_FLOOR = 1024
@@ -60,38 +84,7 @@ _OPENAI_RESPONSES_OUTPUT_ITEM_TYPES = frozenset(
}
)
-# Sentinel prefix marks the steering block so application is idempotent and
-# the block is recognizable in logs/diffs.
-_STEERING_SENTINEL = "<headroom_output_shaping>"
-_STEERING_SUFFIX = "</headroom_output_shaping>"
-
-# Levels are cumulative: each includes everything above it. Text must stay
-# byte-stable across releases for prefix-cache friendliness — treat edits to
-# these strings as cache-busting changes.
-_VERBOSITY_LEVELS = {
- 1: (
- "Skip preamble and postamble. Do not announce what you are about to "
- "do or recap what you just did; start with the substance."
- ),
- 2: (
- "Skip preamble and postamble; start with the substance. Never restate "
- "code, file contents, diffs, or tool output that already appear in "
- "this conversation — reference them by path and line instead. After a "
- "tool call succeeds, continue without narrating the result."
- ),
- 3: (
- "Skip preamble and postamble. Never restate code, file contents, "
- "diffs, or tool output already in this conversation — reference by "
- "path and line. Give conclusions only; omit rationale unless the user "
- "asks why. Prefer the smallest edit over rewriting whole files. Keep "
- "prose to the minimum needed to be unambiguous."
- ),
- 4: (
- "Minimum tokens. Fragments fine. No preamble, no postamble, no "
- "restating context, no rationale. Answer, smallest-possible edits, "
- "nothing else."
- ),
-}
+_replace_or_append_steering_block = replace_or_append_steering_block
class TurnKind(Enum):
@@ -253,66 +246,6 @@ def classify_turn(messages: list[dict[str, Any]]) -> TurnKind:
return TurnKind.UNKNOWN
-def steering_text(level: int) -> str | None:
- """The full steering block for a verbosity level, or None for level 0."""
- text = _VERBOSITY_LEVELS.get(level)
- if text is None:
- return None
- return f"{_STEERING_SENTINEL}\n{text}\n{_STEERING_SUFFIX}"
-
-
-def _replace_or_append_steering_block(existing: str, block: str) -> tuple[str, bool]:
- """Replace an existing steering block in text, or append one at the tail."""
- start = existing.find(_STEERING_SENTINEL)
- if start >= 0:
- end = existing.find(_STEERING_SUFFIX, start)
- end = len(existing) if end < 0 else end + len(_STEERING_SUFFIX)
- prefix = existing[:start].rstrip()
- suffix = existing[end:].lstrip("\n")
- parts = [part for part in (prefix, block, suffix) if part]
- updated = "\n\n".join(parts)
- return updated, updated != existing
-
- updated = f"{existing.rstrip()}\n\n{block}" if existing.strip() else block
- return updated, updated != existing
-
-
-def apply_verbosity_steering(body: dict[str, Any], level: int) -> bool:
- """Append the steering block to the tail of the system prompt.
-
- Appending AFTER the last system block keeps any ``cache_control``
- breakpoint on an earlier block intact — the cached prefix is unchanged
- and only the (small, byte-stable) steering block is reprocessed.
-
- A string system prompt is converted to block form so the original text
- keeps its exact bytes as the first block.
- """
- text = steering_text(level)
- if text is None:
- return False
-
- system = body.get("system")
- if system is None:
- body["system"] = [{"type": "text", "text": text}]
- return True
- if isinstance(system, str):
- body["system"] = [
- {"type": "text", "text": system},
- {"type": "text", "text": text},
- ]
- return True
- if isinstance(system, list):
- for block in system:
- if isinstance(block, dict) and block.get("text", "").startswith(_STEERING_SENTINEL):
- if block["text"] == text:
- return False # already applied at this level
- block["text"] = text # level changed mid-session
- return True
- system.append({"type": "text", "text": text})
- return True
- return False
-
-
def route_effort(
body: dict[str, Any],
kind: TurnKind,
@@ -416,28 +349,6 @@ def classify_openai_responses_input(input_data: Any) -> TurnKind:
return TurnKind.UNKNOWN
-def apply_openai_responses_verbosity_steering(
- body: dict[str, Any],
- level: int,
-) -> bool:
- """Append or replace steering in OpenAI Responses ``instructions``."""
- text = steering_text(level)
- if text is None:
- return False
-
- instructions = body.get("instructions")
- if instructions is None:
- body["instructions"] = text
- return True
- if not isinstance(instructions, str):
- return False
-
- updated, changed = _replace_or_append_steering_block(instructions, text)
- if changed:
- body["instructions"] = updated
- return changed
-
-
def route_openai_reasoning_effort(
body: dict[str, Any],
kind: TurnKind,
diff --git a/headroom/proxy/output_steering.py b/headroom/proxy/output_steering.py
new file mode 100644
index 000000000..c9ef1a263
--- /dev/null
+++ b/headroom/proxy/output_steering.py
@@ -0,0 +1,117 @@
+"""Byte-stable output verbosity steering helpers."""
+
+from __future__ import annotations
+
+from typing import Any
+
+# Sentinel prefix marks the steering block so application is idempotent and
+# the block is recognizable in logs/diffs.
+_STEERING_SENTINEL = "<headroom_output_shaping>"
+_STEERING_SUFFIX = "</headroom_output_shaping>"
+
+# Levels are cumulative: each includes everything above it. Text must stay
+# byte-stable across releases for prefix-cache friendliness; treat edits to
+# these strings as cache-busting changes.
+_VERBOSITY_LEVELS = {
+ 1: (
+ "Skip preamble and postamble. Do not announce what you are about to "
+ "do or recap what you just did; start with the substance."
+ ),
+ 2: (
+ "Skip preamble and postamble; start with the substance. Never restate "
+ "code, file contents, diffs, or tool output that already appear in "
+ "this conversation — reference them by path and line instead. After a "
+ "tool call succeeds, continue without narrating the result."
+ ),
+ 3: (
+ "Skip preamble and postamble. Never restate code, file contents, "
+ "diffs, or tool output already in this conversation — reference by "
+ "path and line. Give conclusions only; omit rationale unless the user "
+ "asks why. Prefer the smallest edit over rewriting whole files. Keep "
+ "prose to the minimum needed to be unambiguous."
+ ),
+ 4: (
+ "Minimum tokens. Fragments fine. No preamble, no postamble, no "
+ "restating context, no rationale. Answer, smallest-possible edits, "
+ "nothing else."
+ ),
+}
+
+
+def steering_text(level: int) -> str | None:
+ """The full steering block for a verbosity level, or None for level 0."""
+ text = _VERBOSITY_LEVELS.get(level)
+ if text is None:
+ return None
+ return f"{_STEERING_SENTINEL}\n{text}\n{_STEERING_SUFFIX}"
+
+
+def replace_or_append_steering_block(existing: str, block: str) -> tuple[str, bool]:
+ """Replace an existing steering block in text, or append one at the tail."""
+ start = existing.find(_STEERING_SENTINEL)
+ if start >= 0:
+ end = existing.find(_STEERING_SUFFIX, start)
+ end = len(existing) if end < 0 else end + len(_STEERING_SUFFIX)
+ prefix = existing[:start].rstrip()
+ suffix = existing[end:].lstrip("\n")
+ parts = [part for part in (prefix, block, suffix) if part]
+ updated = "\n\n".join(parts)
+ return updated, updated != existing
+
+ updated = f"{existing.rstrip()}\n\n{block}" if existing.strip() else block
+ return updated, updated != existing
+
+
+def apply_verbosity_steering(body: dict[str, Any], level: int) -> bool:
+ """Append the steering block to the tail of the Anthropic system prompt.
+
+ Appending after the last system block keeps any ``cache_control``
+ breakpoint on an earlier block intact: the cached prefix is unchanged and
+ only the small, byte-stable steering block is reprocessed.
+ """
+ text = steering_text(level)
+ if text is None:
+ return False
+
+ system = body.get("system")
+ if system is None:
+ body["system"] = [{"type": "text", "text": text}]
+ return True
+ if isinstance(system, str):
+ body["system"] = [
+ {"type": "text", "text": system},
+ {"type": "text", "text": text},
+ ]
+ return True
+ if isinstance(system, list):
+ for block in system:
+ if isinstance(block, dict) and block.get("text", "").startswith(_STEERING_SENTINEL):
+ if block["text"] == text:
+ return False
+ block["text"] = text
+ return True
+ system.append({"type": "text", "text": text})
+ return True
+ return False
+
+
+def apply_openai_responses_verbosity_steering(
+ body: dict[str, Any],
+ level: int,
+) -> bool:
+ """Append or replace steering in OpenAI Responses ``instructions``."""
+ text = steering_text(level)
+ if text is None:
+ return False
+
+ instructions = body.get("instructions")
+ if instructions is None:
+ body["instructions"] = text
+ return True
+ if not isinstance(instructions, str):
+ return False
+
+ updated, changed = replace_or_append_steering_block(instructions, text)
+ if changed:
+ body["instructions"] = updated
+ return changed
diff --git a/tests/test_output_steering.py b/tests/test_output_steering.py
new file mode 100644
index 000000000..144c93b74
--- /dev/null
+++ b/tests/test_output_steering.py
@@ -0,0 +1,44 @@
+"""Tests for output verbosity steering helpers."""
+
+from __future__ import annotations
+
+from headroom.proxy.output_steering import (
+ apply_openai_responses_verbosity_steering,
+ apply_verbosity_steering,
+ replace_or_append_steering_block,
+ steering_text,
+)
+
+
+def test_replace_or_append_steering_block_replaces_existing_block() -> None:
+ old = steering_text(1)
+ new = steering_text(3)
+ assert old is not None
+ assert new is not None
+ updated, changed = replace_or_append_steering_block(f"System.\n\n{old}\n\nTail.", new)
+
+ assert changed is True
+ assert old not in updated
+ assert updated == f"System.\n\n{new}\n\nTail."
+
+
+def test_anthropic_steering_preserves_cached_prefix_block() -> None:
+ cached = {
+ "type": "text",
+ "text": "Big system prompt.",
+ "cache_control": {"type": "ephemeral"},
+ }
+ body = {"system": [cached.copy()]}
+
+ assert apply_verbosity_steering(body, 2) is True
+ assert body["system"][0] == cached
+ assert body["system"][1] == {"type": "text", "text": steering_text(2)}
+
+
+def test_openai_responses_steering_is_idempotent() -> None:
+ body = {"instructions": "System."}
+
+ assert apply_openai_responses_verbosity_steering(body, 2) is True
+ snapshot = body.copy()
+ assert apply_openai_responses_verbosity_steering(body, 2) is False
+ assert body == snapshot