Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy/openai): apply output shaping on /v1/chat/completions (#2328)

Abhay Singh1b8c11ebfb8a
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index cfacab82f..701a67261 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -3361,6 +3361,63 @@ class OpenAIHandlerMixin:
# `max_completion_tokens`.
_normalize_openai_max_tokens(body)
+ # Output shaping (opt-in via HEADROOM_OUTPUT_SHAPER): verbosity steering
+ # on the chat system message. Runs after every other body mutation so the
+ # turn classifier sees the final messages, and respects the same bypass
+ # as compression. OpenAI-compatible clients that route through
+ # /v1/chat/completions (GitHub Copilot CLI, opencode, older SDKs) never
+ # reached the shaper before, so they saw zero output savings (#2302).
+ # Mutating `body` in place is sufficient here — the outbound request
+ # serializes `body` fresh, so no body-mutation tracker is needed.
+ if not _bypass:
+ from headroom.proxy import runtime_env
+ from headroom.proxy.output_savings import (
+ assign_arm,
+ conversation_key_from_body,
+ stratum_key,
+ stratum_label,
+ )
+ from headroom.proxy.output_shaper import (
+ OutputShaperSettings,
+ classify_turn,
+ resolve_verbosity_level,
+ shape_openai_chat_request,
+ )
+
+ _shaper_settings = OutputShaperSettings.from_env()
+ if _shaper_settings.enabled:
+ # Conversation-stable holdout: a whole conversation is treatment
+ # or control, which keeps the A/B comparison clean and the
+ # provider prefix cache stable (the steering block never flips
+ # mid-conversation).
+ _holdout = 0.0
+ try:
+ _holdout = float(runtime_env.getenv("HEADROOM_OUTPUT_HOLDOUT", "0") or "0")
+ except ValueError:
+ _holdout = 0.0
+ _arm = assign_arm(conversation_key_from_body(body), _holdout)
+ _turn_kind = classify_turn(body.get("messages", [])).value
+ _stratum = stratum_key(
+ turn_kind=_turn_kind,
+ input_tokens=original_tokens,
+ model=model,
+ has_tools=bool(body.get("tools")),
+ )
+ # Carry (arm, stratum) on the transforms channel so the outcome
+ # funnel feeds the output-savings ledger from the chat path too.
+ transforms_applied.append(stratum_label(_arm, _stratum))
+ if _arm == "treatment":
+ _level, _src = resolve_verbosity_level(_shaper_settings)
+ _shape_result = shape_openai_chat_request(
+ body, _shaper_settings, level_override=_level
+ )
+ if _shape_result.changed:
+ transforms_applied.extend(_shape_result.labels or [])
+ logger.info(
+ f"[{request_id}] OutputShaper(chat, L{_level}/{_src}): "
+ f"{_shape_result.labels}"
+ )
+
# Route through LiteLLM/any-llm backend if configured
if self.anthropic_backend is not None:
try:
diff --git a/headroom/proxy/output_shaper.py b/headroom/proxy/output_shaper.py
index 1be86e1cc..daa84f256 100644
--- a/headroom/proxy/output_shaper.py
+++ b/headroom/proxy/output_shaper.py
@@ -58,6 +58,7 @@ from headroom.proxy.output_effort_policy import (
lower_text_verbosity_value,
)
from headroom.proxy.output_steering import (
+ apply_openai_chat_verbosity_steering,
apply_openai_responses_verbosity_steering,
apply_verbosity_steering,
replace_or_append_steering_block,
@@ -76,6 +77,7 @@ __all__ = [
"OutputShaperSettings",
"ShapeResult",
"TurnKind",
+ "apply_openai_chat_verbosity_steering",
"apply_openai_responses_verbosity_steering",
"apply_verbosity_steering",
"classify_openai_responses_input",
@@ -84,6 +86,7 @@ __all__ = [
"route_effort",
"route_openai_reasoning_effort",
"route_openai_text_verbosity",
+ "shape_openai_chat_request",
"shape_openai_responses_request",
"shape_request",
"steering_text",
@@ -352,6 +355,36 @@ def shape_request(
return result
+def shape_openai_chat_request(
+ body: dict[str, Any],
+ settings: OutputShaperSettings | None = None,
+ level_override: int | None = None,
+) -> ShapeResult:
+ """Apply output-shaping levers to an OpenAI chat/completions body in place.
+
+ The chat counterpart of :func:`shape_request`. Chat carries the system
+ prompt as a ``role: "system"`` message, so verbosity steering uses the
+ chat-specific injector. Effort routing is intentionally not applied here:
+ the ``route_effort`` levers write Anthropic-shaped config and there is no
+ portable chat/completions equivalent, so only the verbosity steering lever
+ (the one that reduces output tokens) runs on this path.
+ """
+ if settings is None:
+ settings = OutputShaperSettings.from_env()
+ result = ShapeResult()
+ if not settings.enabled:
+ return result
+
+ assert result.labels is not None # __post_init__ guarantees this
+
+ level = settings.verbosity_level if level_override is None else level_override
+ if level > 0 and apply_openai_chat_verbosity_steering(body, level):
+ result.changed = True
+ result.labels.append(f"output_shaper:verbosity:L{level}")
+
+ return result
+
+
# ---------------------------------------------------------------------------
# OpenAI Responses format (Codex, /v1/responses HTTP + WebSocket)
# ---------------------------------------------------------------------------
diff --git a/headroom/proxy/output_steering.py b/headroom/proxy/output_steering.py
index 6eb31b64e..2d86a86bc 100644
--- a/headroom/proxy/output_steering.py
+++ b/headroom/proxy/output_steering.py
@@ -46,6 +46,68 @@ def apply_verbosity_steering(body: dict[str, Any], level: int) -> bool:
return False
+def apply_openai_chat_verbosity_steering(
+ body: dict[str, Any],
+ level: int,
+) -> bool:
+ """Append or replace the steering block in an OpenAI chat/completions body.
+
+ OpenAI ``/v1/chat/completions`` carries the system prompt as a
+ ``role: "system"`` (or ``"developer"``) message inside ``messages`` rather
+ than a top-level field, so it needs its own injector (the Anthropic
+ ``system`` and Responses ``instructions`` variants do not reach it — the
+ root cause of GitHub Copilot CLI seeing zero output savings, #2302).
+
+ The block is appended to the tail of the last system/developer message so a
+ treatment conversation's steering stays byte-stable across turns (and
+ re-applies idempotently via the sentinel). When the request carries no
+ system message at all, one is inserted at the front. Returns True only when
+ the body actually changed.
+ """
+ text = steering_text(level)
+ if text is None:
+ return False
+
+ messages = body.get("messages")
+ if not isinstance(messages, list):
+ return False
+
+ target: dict[str, Any] | None = None
+ for message in messages:
+ if isinstance(message, dict) and message.get("role") in ("system", "developer"):
+ target = message
+ if target is None:
+ # No system prompt to append to — insert one carrying just the block.
+ messages.insert(0, {"role": "system", "content": text})
+ return True
+
+ content = target.get("content")
+ if content is None:
+ target["content"] = text
+ return True
+ if isinstance(content, str):
+ updated, changed = replace_or_append_steering_block(content, text)
+ if changed:
+ target["content"] = updated
+ return changed
+ if isinstance(content, list):
+ # OpenAI also accepts a content-part list ([{"type": "text", ...}]).
+ for part in content:
+ if (
+ isinstance(part, dict)
+ and part.get("type") == "text"
+ and isinstance(part.get("text"), str)
+ and part["text"].startswith(_STEERING_SENTINEL)
+ ):
+ if part["text"] == text:
+ return False
+ part["text"] = text
+ return True
+ content.append({"type": "text", "text": text})
+ return True
+ return False
+
+
def apply_openai_responses_verbosity_steering(
body: dict[str, Any],
level: int,
diff --git a/tests/test_output_shaper.py b/tests/test_output_shaper.py
index 516478942..00e274e3d 100644
--- a/tests/test_output_shaper.py
+++ b/tests/test_output_shaper.py
@@ -20,6 +20,7 @@ from headroom.proxy.output_shaper import (
route_effort,
route_openai_reasoning_effort,
route_openai_text_verbosity,
+ shape_openai_chat_request,
shape_openai_responses_request,
shape_request,
steering_text,
@@ -402,3 +403,40 @@ class TestOpenAIResponsesTextVerbosity:
assert steering_text(2) in body["instructions"]
assert body["reasoning"]["effort"] == "low"
assert body["text"]["verbosity"] == "low"
+
+
+class TestShapeOpenAIChatRequest:
+ def test_disabled_is_noop(self):
+ body = {"messages": [{"role": "system", "content": "Sys."}]}
+ snapshot = copy.deepcopy(body)
+ result = shape_openai_chat_request(body, OutputShaperSettings(enabled=False))
+ assert result.changed is False
+ assert body == snapshot
+
+ def test_enabled_applies_verbosity_steering(self):
+ body = {
+ "messages": [
+ {"role": "system", "content": "Sys."},
+ {"role": "user", "content": "hi"},
+ ]
+ }
+ result = shape_openai_chat_request(body, ENABLED)
+ assert result.changed is True
+ assert result.labels == ["output_shaper:verbosity:L2"]
+ assert steering_text(2) in body["messages"][0]["content"]
+ # User turn is untouched.
+ assert body["messages"][1] == {"role": "user", "content": "hi"}
+
+ def test_level_override_supersedes_settings(self):
+ body = {"messages": [{"role": "system", "content": "Sys."}]}
+ result = shape_openai_chat_request(body, ENABLED, level_override=4)
+ assert result.labels == ["output_shaper:verbosity:L4"]
+ assert steering_text(4) in body["messages"][0]["content"]
+
+ def test_second_pass_is_stable(self):
+ body = {"messages": [{"role": "system", "content": "Sys."}]}
+ shape_openai_chat_request(body, ENABLED)
+ snapshot = copy.deepcopy(body)
+ second = shape_openai_chat_request(body, ENABLED)
+ assert second.changed is False
+ assert body == snapshot
diff --git a/tests/test_output_steering.py b/tests/test_output_steering.py
index 144c93b74..8b94f66c8 100644
--- a/tests/test_output_steering.py
+++ b/tests/test_output_steering.py
@@ -42,3 +42,66 @@ def test_openai_responses_steering_is_idempotent() -> None:
snapshot = body.copy()
assert apply_openai_responses_verbosity_steering(body, 2) is False
assert body == snapshot
+
+
+def test_openai_chat_steering_appends_to_system_message() -> None:
+ from headroom.proxy.output_steering import apply_openai_chat_verbosity_steering
+
+ body = {
+ "messages": [
+ {"role": "system", "content": "You are helpful."},
+ {"role": "user", "content": "hi"},
+ ]
+ }
+ assert apply_openai_chat_verbosity_steering(body, 2) is True
+ sys_content = body["messages"][0]["content"]
+ assert "You are helpful." in sys_content
+ assert steering_text(2) in sys_content
+ # Other messages and ordering are untouched.
+ assert body["messages"][1] == {"role": "user", "content": "hi"}
+ assert [m["role"] for m in body["messages"]] == ["system", "user"]
+
+
+def test_openai_chat_steering_is_idempotent_and_swaps_level() -> None:
+ from headroom.proxy.output_steering import apply_openai_chat_verbosity_steering
+
+ body = {"messages": [{"role": "system", "content": "S."}]}
+ assert apply_openai_chat_verbosity_steering(body, 2) is True
+ first = body["messages"][0]["content"]
+ # Same level again: no change.
+ assert apply_openai_chat_verbosity_steering(body, 2) is False
+ assert body["messages"][0]["content"] == first
+ # Different level: replace, still exactly one block.
+ assert apply_openai_chat_verbosity_steering(body, 4) is True
+ swapped = body["messages"][0]["content"]
+ assert steering_text(4) in swapped
+ assert swapped.count("<headroom_output_shaping>") == 1
+
+
+def test_openai_chat_steering_inserts_system_when_absent() -> None:
+ from headroom.proxy.output_steering import apply_openai_chat_verbosity_steering
+
+ body = {"messages": [{"role": "user", "content": "hi"}]}
+ assert apply_openai_chat_verbosity_steering(body, 3) is True
+ assert body["messages"][0]["role"] == "system"
+ assert body["messages"][0]["content"] == steering_text(3)
+ assert body["messages"][1] == {"role": "user", "content": "hi"}
+
+
+def test_openai_chat_steering_handles_list_content() -> None:
+ from headroom.proxy.output_steering import apply_openai_chat_verbosity_steering
+
+ body = {"messages": [{"role": "system", "content": [{"type": "text", "text": "base"}]}]}
+ assert apply_openai_chat_verbosity_steering(body, 1) is True
+ parts = body["messages"][0]["content"]
+ assert parts[0] == {"type": "text", "text": "base"}
+ assert parts[1]["type"] == "text"
+ assert parts[1]["text"] == steering_text(1)
+
+
+def test_openai_chat_steering_level_zero_is_noop() -> None:
+ from headroom.proxy.output_steering import apply_openai_chat_verbosity_steering
+
+ body = {"messages": [{"role": "system", "content": "S."}]}
+ assert apply_openai_chat_verbosity_steering(body, 0) is False
+ assert body["messages"][0]["content"] == "S."