Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(output-savings): seed the holdout key on the whole first user message (#3209)
gglucass117ff72ec9aa
diff --git a/headroom/proxy/output_savings_policy.py b/headroom/proxy/output_savings_policy.pyindex eb30661be..da56c9441 100644--- a/headroom/proxy/output_savings_policy.py+++ b/headroom/proxy/output_savings_policy.py@@ -3,6 +3,7 @@from __future__ import annotationsimport hashlib+from collections.abc import Iterablefrom typing import Any, cast# Coarse input-token buckets. Coarse on purpose: too many strata make@@ -68,6 +69,33 @@ def stratum_key()+def _absorb(digest: Any, text: str) -> None:+ """Fold one seed field into ``digest``.++ Incremental so no length cap is needed: the seed is consumed a field at a+ time instead of being concatenated into one string first. The NUL is the+ field separator, and feeding it separately is byte-identical to hashing+ ``"\x00" + text``.+ """+ digest.update(b"\x00")+ digest.update(text.encode("utf-8", "ignore"))+++def _absorb_text_blocks(digest: Any, blocks: Iterable[str]) -> None:+ """Fold every text block of one message into ``digest`` as one field.++ Byte-identical to absorbing ``"\x00".join(blocks)``, including the empty+ case: a message with no text blocks still contributes its separator.+ """+ absorbed = False+ for text in blocks:+ digest.update(b"\x00")+ digest.update(text.encode("utf-8", "ignore"))+ absorbed = True+ if not absorbed:+ digest.update(b"\x00")++def _unwrap_response_create_body(body: dict[str, Any]) -> dict[str, Any]:response = body.get("response")if body.get("type") == "response.create" and isinstance(response, dict):@@ -113,52 +141,87 @@ def _stable_response_identifier(body: dict[str, Any]) -> str:def conversation_key_from_body(body: dict[str, Any]) -> str:- """Derive a conversation-stable key for holdout assignment."""+ """Derive a conversation-stable key for holdout assignment.++ Every text block of the first user message feeds the seed, in full: the+ digest is built incrementally, field by field, so there is no length cap to+ collapse behind.++ Seeding on the first 512 characters of the *first* block collapsed whole+ populations of conversations onto one key, because agent clients open a+ conversation with injected context -- CLAUDE.md, IDE selection, memory+ digests -- that is byte-identical for every conversation in a project and+ far longer than 512 characters. The user's own words, the part that makes+ one conversation different from the next, sat past the cut.++ One key means one arm, permanently, since the assignment is deterministic:+ on a real 78k-request ledger that left fable at 22,222 treatment requests+ against 1 control and sonnet at 15,401 against 0, while opus accumulated+ 3,532 control samples -- a holdout nominally set to 3% that had in fact+ frozen each client into a single arm. Reading the whole message restores+ the intended unit of randomization without weakening stability: user turns+ are never compressed, so the first message is immutable for the life of+ the conversation.+ """body = _unwrap_response_create_body(body)- model = str(body.get("model", ""))- seed = model+ digest = hashlib.sha256()+ digest.update(str(body.get("model", "")).encode("utf-8", "ignore"))for msg in body.get("messages", []):if isinstance(msg, dict) and msg.get("role") == "user":content = msg.get("content")if isinstance(content, str):- seed += "\x00" + content[:512]+ _absorb(digest, content)elif isinstance(content, list):- for block in content:- if isinstance(block, dict) and block.get("type") == "text":- seed += "\x00" + str(block.get("text", ""))[:512]- break+ _absorb_text_blocks(+ digest,+ (+ str(block.get("text", ""))+ for block in content+ if isinstance(block, dict) and block.get("type") == "text"+ ),+ )breakif "input" in body:stable_response_key = _stable_response_identifier(body)if stable_response_key:- seed += "\x00" + stable_response_key+ _absorb(digest, stable_response_key)elif not body.get("messages"):- seed += "\x00responses"- return hashlib.sha256(seed.encode("utf-8", "ignore")).hexdigest()+ _absorb(digest, "responses")+ return digest.hexdigest()def conversation_key_from_responses_body(body: dict[str, Any]) -> str:- """Conversation-stable key for an OpenAI Responses payload."""+ """Conversation-stable key for an OpenAI Responses payload.++ Same seeding rule as :func:`conversation_key_from_body`, and for the same+ reason: this path carried the identical 512-character cut, on the first+ text part alone, so a Codex-style client whose opener begins with injected+ context keyed every conversation the same way and never left one arm.+ """body = _unwrap_response_create_body(body)- model = str(body.get("model", ""))- seed = model+ digest = hashlib.sha256()+ digest.update(str(body.get("model", "")).encode("utf-8", "ignore"))input_data = body.get("input")if isinstance(input_data, str):- seed += "\x00" + input_data[:512]+ _absorb(digest, input_data)elif isinstance(input_data, list):for item in input_data:if not isinstance(item, dict) or item.get("role") != "user":continuecontent = item.get("content")if isinstance(content, str):- seed += "\x00" + content[:512]+ _absorb(digest, content)elif isinstance(content, list):- for part in content:- if isinstance(part, dict) and isinstance(part.get("text"), str):- seed += "\x00" + part["text"][:512]- break+ _absorb_text_blocks(+ digest,+ (+ part["text"]+ for part in content+ if isinstance(part, dict) and isinstance(part.get("text"), str)+ ),+ )break- return hashlib.sha256(seed.encode("utf-8", "ignore")).hexdigest()+ return digest.hexdigest()def assign_arm(conversation_key: str, holdout_fraction: float) -> str:diff --git a/tests/test_output_savings_policy.py b/tests/test_output_savings_policy.pyindex bb71543ba..8ab6048d6 100644--- a/tests/test_output_savings_policy.py+++ b/tests/test_output_savings_policy.py@@ -5,6 +5,7 @@ from __future__ import annotationsfrom headroom.proxy.output_savings_policy import (assign_arm,conversation_key_from_body,+ conversation_key_from_responses_body,input_bucket,model_family,parse_stratum_label,@@ -52,9 +53,183 @@ def test_conversation_key_uses_response_create_payload() -> None:assert conversation_key_from_body(http_body) == conversation_key_from_body(ws_body)+def test_conversation_key_survives_a_long_identical_client_prologue() -> None:+ """Two conversations differing only past a long injected prologue.++ Agent clients open every conversation with the same context blocks+ (project instructions, IDE state, memory), commonly kilobytes long. Keying+ on a 512-character prefix gave all of them one key, so a whole client sat+ in one arm permanently and never produced holdout data.+ """+ prologue = "<system-reminder>project instructions. </system-reminder>" * 40+ assert len(prologue) > 512++ def body(question: str) -> dict[str, object]:+ return {+ "model": "claude-opus-4-5",+ "messages": [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": prologue},+ {"type": "text", "text": question},+ ],+ }+ ],+ }++ assert conversation_key_from_body(body("add a cache")) != conversation_key_from_body(+ body("delete the cache")+ )+ # Same words, same conversation: still one key, so the arm stays stable.+ assert conversation_key_from_body(body("add a cache")) == conversation_key_from_body(+ body("add a cache")+ )+++def test_conversation_key_separates_conversations_behind_a_64kb_prologue() -> None:+ """No cutoff can hide the user's words, however long the prologue.++ A character cap is a cutoff wherever it sits: put a prologue past it and+ every conversation behind that prologue keys the same way again, which is+ the collapse this key exists to avoid. The seed is hashed incrementally, so+ there is no cap -- this prologue is deliberately larger than the 64KB guard+ an earlier revision of this fix used.+ """+ prologue = "x" * 96_000++ def body(question: str) -> dict[str, object]:+ return {+ "model": "claude-opus-4-5",+ "messages": [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": prologue},+ {"type": "text", "text": question},+ ],+ }+ ],+ }++ assert conversation_key_from_body(body("add a cache")) != conversation_key_from_body(+ body("delete the cache")+ )+ # And the same opener is still one key, so the arm holds for the conversation.+ assert conversation_key_from_body(body("add a cache")) == conversation_key_from_body(+ body("add a cache")+ )+++def test_responses_conversation_key_separates_past_a_long_prologue() -> None:+ """The Responses path carried the same cut and needs the same seeding."""+ prologue = "x" * 96_000++ def body(question: str) -> dict[str, object]:+ return {+ "model": "gpt-5",+ "input": [+ {+ "role": "user",+ "content": [+ {"type": "input_text", "text": prologue},+ {"type": "input_text", "text": question},+ ],+ }+ ],+ }++ assert conversation_key_from_responses_body(+ body("add a cache")+ ) != conversation_key_from_responses_body(body("delete the cache"))+ assert conversation_key_from_responses_body(+ body("add a cache")+ ) == conversation_key_from_responses_body(body("add a cache"))+++def test_conversation_key_ignores_non_text_blocks() -> None:+ """An image block carries no conversation identity and must not seed."""+ text_only = {+ "model": "claude-opus-4-5",+ "messages": [{"role": "user", "content": [{"type": "text", "text": "ship it"}]}],+ }+ with_image = {+ "model": "claude-opus-4-5",+ "messages": [+ {+ "role": "user",+ "content": [+ {"type": "image", "source": {"data": "iVBORw0KGgo="}},+ {"type": "text", "text": "ship it"},+ ],+ }+ ],+ }++ assert conversation_key_from_body(text_only) == conversation_key_from_body(with_image)+++def test_responses_conversation_key_reads_past_non_user_items() -> None:+ """Seeding starts at the first user item, whatever precedes it."""++ def body(question: str) -> dict[str, object]:+ return {+ "model": "gpt-5",+ "input": [+ {"role": "system", "content": "you are a helpful assistant"},+ {"role": "user", "content": question},+ ],+ }++ assert conversation_key_from_responses_body(+ body("add a cache")+ ) != conversation_key_from_responses_body(body("delete the cache"))+++def test_conversation_key_handles_a_message_with_no_text() -> None:+ """An image-only opener still keys, and keys apart from an empty one."""+ image_only = {+ "model": "claude-opus-4-5",+ "messages": [{"role": "user", "content": [{"type": "image", "source": {"data": "iVBOR"}}]}],+ }+ empty = {"model": "claude-opus-4-5", "messages": [{"role": "user", "content": []}]}+ no_message = {"model": "claude-opus-4-5", "messages": []}++ # No text either way, so these two agree; a message-less body does not.+ assert conversation_key_from_body(image_only) == conversation_key_from_body(empty)+ assert conversation_key_from_body(image_only) != conversation_key_from_body(no_message)++def test_stratum_label_round_trips_arm_and_key() -> None:key = "opus|code|m|tools"assert parse_stratum_label(stratum_label("treatment", key)) == ("treatment", key)assert parse_stratum_label(stratum_label("control", key)) == ("control", key)assert parse_stratum_label("unrelated") is None+++def test_responses_conversation_key_seeds_on_a_plain_string_input() -> None:+ """/v1/responses accepts `input` as a bare string, not only an item list.++ That shape skips the item loop entirely, so it needs its own seeding path+ or every string-input conversation hashes to the same key and lands in one+ arm together.+ """++ def body(question: str) -> dict:+ return {"model": "gpt-5.2-codex", "input": question}++ assert conversation_key_from_responses_body(+ body("add a cache")+ ) != conversation_key_from_responses_body(body("delete the cache"))+ assert conversation_key_from_responses_body(+ body("add a cache")+ ) == conversation_key_from_responses_body(body("add a cache"))+ # And it agrees with the equivalent item-list form. The seed is the opening+ # user text, not the wire shape carrying it, so a client that switches+ # between the two mid-session stays in the arm it was assigned.+ assert conversation_key_from_responses_body(body("add a cache")) == (+ conversation_key_from_responses_body(+ {"model": "gpt-5.2-codex", "input": [{"role": "user", "content": "add a cache"}]}+ )+ )