Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy/openai): apply output shaping on /v1/chat/completions (#2328)
Abhay Singh1b8c11ebfb8a
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex cfacab82f..701a67261 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -3361,6 +3361,63 @@ class OpenAIHandlerMixin:# `max_completion_tokens`._normalize_openai_max_tokens(body)+ # Output shaping (opt-in via HEADROOM_OUTPUT_SHAPER): verbosity steering+ # on the chat system message. Runs after every other body mutation so the+ # turn classifier sees the final messages, and respects the same bypass+ # as compression. OpenAI-compatible clients that route through+ # /v1/chat/completions (GitHub Copilot CLI, opencode, older SDKs) never+ # reached the shaper before, so they saw zero output savings (#2302).+ # Mutating `body` in place is sufficient here — the outbound request+ # serializes `body` fresh, so no body-mutation tracker is needed.+ if not _bypass:+ from headroom.proxy import runtime_env+ from headroom.proxy.output_savings import (+ assign_arm,+ conversation_key_from_body,+ stratum_key,+ stratum_label,+ )+ from headroom.proxy.output_shaper import (+ OutputShaperSettings,+ classify_turn,+ resolve_verbosity_level,+ shape_openai_chat_request,+ )++ _shaper_settings = OutputShaperSettings.from_env()+ if _shaper_settings.enabled:+ # Conversation-stable holdout: a whole conversation is treatment+ # or control, which keeps the A/B comparison clean and the+ # provider prefix cache stable (the steering block never flips+ # mid-conversation).+ _holdout = 0.0+ try:+ _holdout = float(runtime_env.getenv("HEADROOM_OUTPUT_HOLDOUT", "0") or "0")+ except ValueError:+ _holdout = 0.0+ _arm = assign_arm(conversation_key_from_body(body), _holdout)+ _turn_kind = classify_turn(body.get("messages", [])).value+ _stratum = stratum_key(+ turn_kind=_turn_kind,+ input_tokens=original_tokens,+ model=model,+ has_tools=bool(body.get("tools")),+ )+ # Carry (arm, stratum) on the transforms channel so the outcome+ # funnel feeds the output-savings ledger from the chat path too.+ transforms_applied.append(stratum_label(_arm, _stratum))+ if _arm == "treatment":+ _level, _src = resolve_verbosity_level(_shaper_settings)+ _shape_result = shape_openai_chat_request(+ body, _shaper_settings, level_override=_level+ )+ if _shape_result.changed:+ transforms_applied.extend(_shape_result.labels or [])+ logger.info(+ f"[{request_id}] OutputShaper(chat, L{_level}/{_src}): "+ f"{_shape_result.labels}"+ )+# Route through LiteLLM/any-llm backend if configuredif self.anthropic_backend is not None:try:diff --git a/headroom/proxy/output_shaper.py b/headroom/proxy/output_shaper.pyindex 1be86e1cc..daa84f256 100644--- a/headroom/proxy/output_shaper.py+++ b/headroom/proxy/output_shaper.py@@ -58,6 +58,7 @@ from headroom.proxy.output_effort_policy import (lower_text_verbosity_value,)from headroom.proxy.output_steering import (+ apply_openai_chat_verbosity_steering,apply_openai_responses_verbosity_steering,apply_verbosity_steering,replace_or_append_steering_block,@@ -76,6 +77,7 @@ __all__ = ["OutputShaperSettings","ShapeResult","TurnKind",+ "apply_openai_chat_verbosity_steering","apply_openai_responses_verbosity_steering","apply_verbosity_steering","classify_openai_responses_input",@@ -84,6 +86,7 @@ __all__ = ["route_effort","route_openai_reasoning_effort","route_openai_text_verbosity",+ "shape_openai_chat_request","shape_openai_responses_request","shape_request","steering_text",@@ -352,6 +355,36 @@ def shape_request(return result+def shape_openai_chat_request(+ body: dict[str, Any],+ settings: OutputShaperSettings | None = None,+ level_override: int | None = None,+) -> ShapeResult:+ """Apply output-shaping levers to an OpenAI chat/completions body in place.++ The chat counterpart of :func:`shape_request`. Chat carries the system+ prompt as a ``role: "system"`` message, so verbosity steering uses the+ chat-specific injector. Effort routing is intentionally not applied here:+ the ``route_effort`` levers write Anthropic-shaped config and there is no+ portable chat/completions equivalent, so only the verbosity steering lever+ (the one that reduces output tokens) runs on this path.+ """+ if settings is None:+ settings = OutputShaperSettings.from_env()+ result = ShapeResult()+ if not settings.enabled:+ return result++ assert result.labels is not None # __post_init__ guarantees this++ level = settings.verbosity_level if level_override is None else level_override+ if level > 0 and apply_openai_chat_verbosity_steering(body, level):+ result.changed = True+ result.labels.append(f"output_shaper:verbosity:L{level}")++ return result++# ---------------------------------------------------------------------------# OpenAI Responses format (Codex, /v1/responses HTTP + WebSocket)# ---------------------------------------------------------------------------diff --git a/headroom/proxy/output_steering.py b/headroom/proxy/output_steering.pyindex 6eb31b64e..2d86a86bc 100644--- a/headroom/proxy/output_steering.py+++ b/headroom/proxy/output_steering.py@@ -46,6 +46,68 @@ def apply_verbosity_steering(body: dict[str, Any], level: int) -> bool:return False+def apply_openai_chat_verbosity_steering(+ body: dict[str, Any],+ level: int,+) -> bool:+ """Append or replace the steering block in an OpenAI chat/completions body.++ OpenAI ``/v1/chat/completions`` carries the system prompt as a+ ``role: "system"`` (or ``"developer"``) message inside ``messages`` rather+ than a top-level field, so it needs its own injector (the Anthropic+ ``system`` and Responses ``instructions`` variants do not reach it — the+ root cause of GitHub Copilot CLI seeing zero output savings, #2302).++ The block is appended to the tail of the last system/developer message so a+ treatment conversation's steering stays byte-stable across turns (and+ re-applies idempotently via the sentinel). When the request carries no+ system message at all, one is inserted at the front. Returns True only when+ the body actually changed.+ """+ text = steering_text(level)+ if text is None:+ return False++ messages = body.get("messages")+ if not isinstance(messages, list):+ return False++ target: dict[str, Any] | None = None+ for message in messages:+ if isinstance(message, dict) and message.get("role") in ("system", "developer"):+ target = message+ if target is None:+ # No system prompt to append to — insert one carrying just the block.+ messages.insert(0, {"role": "system", "content": text})+ return True++ content = target.get("content")+ if content is None:+ target["content"] = text+ return True+ if isinstance(content, str):+ updated, changed = replace_or_append_steering_block(content, text)+ if changed:+ target["content"] = updated+ return changed+ if isinstance(content, list):+ # OpenAI also accepts a content-part list ([{"type": "text", ...}]).+ for part in content:+ if (+ isinstance(part, dict)+ and part.get("type") == "text"+ and isinstance(part.get("text"), str)+ and part["text"].startswith(_STEERING_SENTINEL)+ ):+ if part["text"] == text:+ return False+ part["text"] = text+ return True+ content.append({"type": "text", "text": text})+ return True+ return False++def apply_openai_responses_verbosity_steering(body: dict[str, Any],level: int,diff --git a/tests/test_output_shaper.py b/tests/test_output_shaper.pyindex 516478942..00e274e3d 100644--- a/tests/test_output_shaper.py+++ b/tests/test_output_shaper.py@@ -20,6 +20,7 @@ from headroom.proxy.output_shaper import (route_effort,route_openai_reasoning_effort,route_openai_text_verbosity,+ shape_openai_chat_request,shape_openai_responses_request,shape_request,steering_text,@@ -402,3 +403,40 @@ class TestOpenAIResponsesTextVerbosity:assert steering_text(2) in body["instructions"]assert body["reasoning"]["effort"] == "low"assert body["text"]["verbosity"] == "low"+++class TestShapeOpenAIChatRequest:+ def test_disabled_is_noop(self):+ body = {"messages": [{"role": "system", "content": "Sys."}]}+ snapshot = copy.deepcopy(body)+ result = shape_openai_chat_request(body, OutputShaperSettings(enabled=False))+ assert result.changed is False+ assert body == snapshot++ def test_enabled_applies_verbosity_steering(self):+ body = {+ "messages": [+ {"role": "system", "content": "Sys."},+ {"role": "user", "content": "hi"},+ ]+ }+ result = shape_openai_chat_request(body, ENABLED)+ assert result.changed is True+ assert result.labels == ["output_shaper:verbosity:L2"]+ assert steering_text(2) in body["messages"][0]["content"]+ # User turn is untouched.+ assert body["messages"][1] == {"role": "user", "content": "hi"}++ def test_level_override_supersedes_settings(self):+ body = {"messages": [{"role": "system", "content": "Sys."}]}+ result = shape_openai_chat_request(body, ENABLED, level_override=4)+ assert result.labels == ["output_shaper:verbosity:L4"]+ assert steering_text(4) in body["messages"][0]["content"]++ def test_second_pass_is_stable(self):+ body = {"messages": [{"role": "system", "content": "Sys."}]}+ shape_openai_chat_request(body, ENABLED)+ snapshot = copy.deepcopy(body)+ second = shape_openai_chat_request(body, ENABLED)+ assert second.changed is False+ assert body == snapshotdiff --git a/tests/test_output_steering.py b/tests/test_output_steering.pyindex 144c93b74..8b94f66c8 100644--- a/tests/test_output_steering.py+++ b/tests/test_output_steering.py@@ -42,3 +42,66 @@ def test_openai_responses_steering_is_idempotent() -> None:snapshot = body.copy()assert apply_openai_responses_verbosity_steering(body, 2) is Falseassert body == snapshot+++def test_openai_chat_steering_appends_to_system_message() -> None:+ from headroom.proxy.output_steering import apply_openai_chat_verbosity_steering++ body = {+ "messages": [+ {"role": "system", "content": "You are helpful."},+ {"role": "user", "content": "hi"},+ ]+ }+ assert apply_openai_chat_verbosity_steering(body, 2) is True+ sys_content = body["messages"][0]["content"]+ assert "You are helpful." in sys_content+ assert steering_text(2) in sys_content+ # Other messages and ordering are untouched.+ assert body["messages"][1] == {"role": "user", "content": "hi"}+ assert [m["role"] for m in body["messages"]] == ["system", "user"]+++def test_openai_chat_steering_is_idempotent_and_swaps_level() -> None:+ from headroom.proxy.output_steering import apply_openai_chat_verbosity_steering++ body = {"messages": [{"role": "system", "content": "S."}]}+ assert apply_openai_chat_verbosity_steering(body, 2) is True+ first = body["messages"][0]["content"]+ # Same level again: no change.+ assert apply_openai_chat_verbosity_steering(body, 2) is False+ assert body["messages"][0]["content"] == first+ # Different level: replace, still exactly one block.+ assert apply_openai_chat_verbosity_steering(body, 4) is True+ swapped = body["messages"][0]["content"]+ assert steering_text(4) in swapped+ assert swapped.count("<headroom_output_shaping>") == 1+++def test_openai_chat_steering_inserts_system_when_absent() -> None:+ from headroom.proxy.output_steering import apply_openai_chat_verbosity_steering++ body = {"messages": [{"role": "user", "content": "hi"}]}+ assert apply_openai_chat_verbosity_steering(body, 3) is True+ assert body["messages"][0]["role"] == "system"+ assert body["messages"][0]["content"] == steering_text(3)+ assert body["messages"][1] == {"role": "user", "content": "hi"}+++def test_openai_chat_steering_handles_list_content() -> None:+ from headroom.proxy.output_steering import apply_openai_chat_verbosity_steering++ body = {"messages": [{"role": "system", "content": [{"type": "text", "text": "base"}]}]}+ assert apply_openai_chat_verbosity_steering(body, 1) is True+ parts = body["messages"][0]["content"]+ assert parts[0] == {"type": "text", "text": "base"}+ assert parts[1]["type"] == "text"+ assert parts[1]["text"] == steering_text(1)+++def test_openai_chat_steering_level_zero_is_noop() -> None:+ from headroom.proxy.output_steering import apply_openai_chat_verbosity_steering++ body = {"messages": [{"role": "system", "content": "S."}]}+ assert apply_openai_chat_verbosity_steering(body, 0) is False+ assert body["messages"][0]["content"] == "S."