Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(output-savings): seed the holdout key on the whole first user message (#3209)

gglucass117ff72ec9aa
diff --git a/headroom/proxy/output_savings_policy.py b/headroom/proxy/output_savings_policy.py
index eb30661be..da56c9441 100644
--- a/headroom/proxy/output_savings_policy.py
+++ b/headroom/proxy/output_savings_policy.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import hashlib
+from collections.abc import Iterable
from typing import Any, cast
# Coarse input-token buckets. Coarse on purpose: too many strata make
@@ -68,6 +69,33 @@ def stratum_key(
)
+def _absorb(digest: Any, text: str) -> None:
+ """Fold one seed field into ``digest``.
+
+ Incremental so no length cap is needed: the seed is consumed a field at a
+ time instead of being concatenated into one string first. The NUL is the
+ field separator, and feeding it separately is byte-identical to hashing
+ ``"\x00" + text``.
+ """
+ digest.update(b"\x00")
+ digest.update(text.encode("utf-8", "ignore"))
+
+
+def _absorb_text_blocks(digest: Any, blocks: Iterable[str]) -> None:
+ """Fold every text block of one message into ``digest`` as one field.
+
+ Byte-identical to absorbing ``"\x00".join(blocks)``, including the empty
+ case: a message with no text blocks still contributes its separator.
+ """
+ absorbed = False
+ for text in blocks:
+ digest.update(b"\x00")
+ digest.update(text.encode("utf-8", "ignore"))
+ absorbed = True
+ if not absorbed:
+ digest.update(b"\x00")
+
+
def _unwrap_response_create_body(body: dict[str, Any]) -> dict[str, Any]:
response = body.get("response")
if body.get("type") == "response.create" and isinstance(response, dict):
@@ -113,52 +141,87 @@ def _stable_response_identifier(body: dict[str, Any]) -> str:
def conversation_key_from_body(body: dict[str, Any]) -> str:
- """Derive a conversation-stable key for holdout assignment."""
+ """Derive a conversation-stable key for holdout assignment.
+
+ Every text block of the first user message feeds the seed, in full: the
+ digest is built incrementally, field by field, so there is no length cap to
+ collapse behind.
+
+ Seeding on the first 512 characters of the *first* block collapsed whole
+ populations of conversations onto one key, because agent clients open a
+ conversation with injected context -- CLAUDE.md, IDE selection, memory
+ digests -- that is byte-identical for every conversation in a project and
+ far longer than 512 characters. The user's own words, the part that makes
+ one conversation different from the next, sat past the cut.
+
+ One key means one arm, permanently, since the assignment is deterministic:
+ on a real 78k-request ledger that left fable at 22,222 treatment requests
+ against 1 control and sonnet at 15,401 against 0, while opus accumulated
+ 3,532 control samples -- a holdout nominally set to 3% that had in fact
+ frozen each client into a single arm. Reading the whole message restores
+ the intended unit of randomization without weakening stability: user turns
+ are never compressed, so the first message is immutable for the life of
+ the conversation.
+ """
body = _unwrap_response_create_body(body)
- model = str(body.get("model", ""))
- seed = model
+ digest = hashlib.sha256()
+ digest.update(str(body.get("model", "")).encode("utf-8", "ignore"))
for msg in body.get("messages", []):
if isinstance(msg, dict) and msg.get("role") == "user":
content = msg.get("content")
if isinstance(content, str):
- seed += "\x00" + content[:512]
+ _absorb(digest, content)
elif isinstance(content, list):
- for block in content:
- if isinstance(block, dict) and block.get("type") == "text":
- seed += "\x00" + str(block.get("text", ""))[:512]
- break
+ _absorb_text_blocks(
+ digest,
+ (
+ str(block.get("text", ""))
+ for block in content
+ if isinstance(block, dict) and block.get("type") == "text"
+ ),
+ )
break
if "input" in body:
stable_response_key = _stable_response_identifier(body)
if stable_response_key:
- seed += "\x00" + stable_response_key
+ _absorb(digest, stable_response_key)
elif not body.get("messages"):
- seed += "\x00responses"
- return hashlib.sha256(seed.encode("utf-8", "ignore")).hexdigest()
+ _absorb(digest, "responses")
+ return digest.hexdigest()
def conversation_key_from_responses_body(body: dict[str, Any]) -> str:
- """Conversation-stable key for an OpenAI Responses payload."""
+ """Conversation-stable key for an OpenAI Responses payload.
+
+ Same seeding rule as :func:`conversation_key_from_body`, and for the same
+ reason: this path carried the identical 512-character cut, on the first
+ text part alone, so a Codex-style client whose opener begins with injected
+ context keyed every conversation the same way and never left one arm.
+ """
body = _unwrap_response_create_body(body)
- model = str(body.get("model", ""))
- seed = model
+ digest = hashlib.sha256()
+ digest.update(str(body.get("model", "")).encode("utf-8", "ignore"))
input_data = body.get("input")
if isinstance(input_data, str):
- seed += "\x00" + input_data[:512]
+ _absorb(digest, input_data)
elif isinstance(input_data, list):
for item in input_data:
if not isinstance(item, dict) or item.get("role") != "user":
continue
content = item.get("content")
if isinstance(content, str):
- seed += "\x00" + content[:512]
+ _absorb(digest, content)
elif isinstance(content, list):
- for part in content:
- if isinstance(part, dict) and isinstance(part.get("text"), str):
- seed += "\x00" + part["text"][:512]
- break
+ _absorb_text_blocks(
+ digest,
+ (
+ part["text"]
+ for part in content
+ if isinstance(part, dict) and isinstance(part.get("text"), str)
+ ),
+ )
break
- return hashlib.sha256(seed.encode("utf-8", "ignore")).hexdigest()
+ return digest.hexdigest()
def assign_arm(conversation_key: str, holdout_fraction: float) -> str:
diff --git a/tests/test_output_savings_policy.py b/tests/test_output_savings_policy.py
index bb71543ba..8ab6048d6 100644
--- a/tests/test_output_savings_policy.py
+++ b/tests/test_output_savings_policy.py
@@ -5,6 +5,7 @@ from __future__ import annotations
from headroom.proxy.output_savings_policy import (
assign_arm,
conversation_key_from_body,
+ conversation_key_from_responses_body,
input_bucket,
model_family,
parse_stratum_label,
@@ -52,9 +53,183 @@ def test_conversation_key_uses_response_create_payload() -> None:
assert conversation_key_from_body(http_body) == conversation_key_from_body(ws_body)
+def test_conversation_key_survives_a_long_identical_client_prologue() -> None:
+ """Two conversations differing only past a long injected prologue.
+
+ Agent clients open every conversation with the same context blocks
+ (project instructions, IDE state, memory), commonly kilobytes long. Keying
+ on a 512-character prefix gave all of them one key, so a whole client sat
+ in one arm permanently and never produced holdout data.
+ """
+ prologue = "<system-reminder>project instructions. </system-reminder>" * 40
+ assert len(prologue) > 512
+
+ def body(question: str) -> dict[str, object]:
+ return {
+ "model": "claude-opus-4-5",
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": prologue},
+ {"type": "text", "text": question},
+ ],
+ }
+ ],
+ }
+
+ assert conversation_key_from_body(body("add a cache")) != conversation_key_from_body(
+ body("delete the cache")
+ )
+ # Same words, same conversation: still one key, so the arm stays stable.
+ assert conversation_key_from_body(body("add a cache")) == conversation_key_from_body(
+ body("add a cache")
+ )
+
+
+def test_conversation_key_separates_conversations_behind_a_64kb_prologue() -> None:
+ """No cutoff can hide the user's words, however long the prologue.
+
+ A character cap is a cutoff wherever it sits: put a prologue past it and
+ every conversation behind that prologue keys the same way again, which is
+ the collapse this key exists to avoid. The seed is hashed incrementally, so
+ there is no cap -- this prologue is deliberately larger than the 64KB guard
+ an earlier revision of this fix used.
+ """
+ prologue = "x" * 96_000
+
+ def body(question: str) -> dict[str, object]:
+ return {
+ "model": "claude-opus-4-5",
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": prologue},
+ {"type": "text", "text": question},
+ ],
+ }
+ ],
+ }
+
+ assert conversation_key_from_body(body("add a cache")) != conversation_key_from_body(
+ body("delete the cache")
+ )
+ # And the same opener is still one key, so the arm holds for the conversation.
+ assert conversation_key_from_body(body("add a cache")) == conversation_key_from_body(
+ body("add a cache")
+ )
+
+
+def test_responses_conversation_key_separates_past_a_long_prologue() -> None:
+ """The Responses path carried the same cut and needs the same seeding."""
+ prologue = "x" * 96_000
+
+ def body(question: str) -> dict[str, object]:
+ return {
+ "model": "gpt-5",
+ "input": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "input_text", "text": prologue},
+ {"type": "input_text", "text": question},
+ ],
+ }
+ ],
+ }
+
+ assert conversation_key_from_responses_body(
+ body("add a cache")
+ ) != conversation_key_from_responses_body(body("delete the cache"))
+ assert conversation_key_from_responses_body(
+ body("add a cache")
+ ) == conversation_key_from_responses_body(body("add a cache"))
+
+
+def test_conversation_key_ignores_non_text_blocks() -> None:
+ """An image block carries no conversation identity and must not seed."""
+ text_only = {
+ "model": "claude-opus-4-5",
+ "messages": [{"role": "user", "content": [{"type": "text", "text": "ship it"}]}],
+ }
+ with_image = {
+ "model": "claude-opus-4-5",
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "image", "source": {"data": "iVBORw0KGgo="}},
+ {"type": "text", "text": "ship it"},
+ ],
+ }
+ ],
+ }
+
+ assert conversation_key_from_body(text_only) == conversation_key_from_body(with_image)
+
+
+def test_responses_conversation_key_reads_past_non_user_items() -> None:
+ """Seeding starts at the first user item, whatever precedes it."""
+
+ def body(question: str) -> dict[str, object]:
+ return {
+ "model": "gpt-5",
+ "input": [
+ {"role": "system", "content": "you are a helpful assistant"},
+ {"role": "user", "content": question},
+ ],
+ }
+
+ assert conversation_key_from_responses_body(
+ body("add a cache")
+ ) != conversation_key_from_responses_body(body("delete the cache"))
+
+
+def test_conversation_key_handles_a_message_with_no_text() -> None:
+ """An image-only opener still keys, and keys apart from an empty one."""
+ image_only = {
+ "model": "claude-opus-4-5",
+ "messages": [{"role": "user", "content": [{"type": "image", "source": {"data": "iVBOR"}}]}],
+ }
+ empty = {"model": "claude-opus-4-5", "messages": [{"role": "user", "content": []}]}
+ no_message = {"model": "claude-opus-4-5", "messages": []}
+
+ # No text either way, so these two agree; a message-less body does not.
+ assert conversation_key_from_body(image_only) == conversation_key_from_body(empty)
+ assert conversation_key_from_body(image_only) != conversation_key_from_body(no_message)
+
+
def test_stratum_label_round_trips_arm_and_key() -> None:
key = "opus|code|m|tools"
assert parse_stratum_label(stratum_label("treatment", key)) == ("treatment", key)
assert parse_stratum_label(stratum_label("control", key)) == ("control", key)
assert parse_stratum_label("unrelated") is None
+
+
+def test_responses_conversation_key_seeds_on_a_plain_string_input() -> None:
+ """/v1/responses accepts `input` as a bare string, not only an item list.
+
+ That shape skips the item loop entirely, so it needs its own seeding path
+ or every string-input conversation hashes to the same key and lands in one
+ arm together.
+ """
+
+ def body(question: str) -> dict:
+ return {"model": "gpt-5.2-codex", "input": question}
+
+ assert conversation_key_from_responses_body(
+ body("add a cache")
+ ) != conversation_key_from_responses_body(body("delete the cache"))
+ assert conversation_key_from_responses_body(
+ body("add a cache")
+ ) == conversation_key_from_responses_body(body("add a cache"))
+ # And it agrees with the equivalent item-list form. The seed is the opening
+ # user text, not the wire shape carrying it, so a client that switches
+ # between the two mid-session stays in the arm it was assigned.
+ assert conversation_key_from_responses_body(body("add a cache")) == (
+ conversation_key_from_responses_body(
+ {"model": "gpt-5.2-codex", "input": [{"role": "user", "content": "add a cache"}]}
+ )
+ )