Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
perf(proxy): lighter request path: alias the message snapshot, keep the token cache on re-counts, decompress off the event loop (#3909)
Billc873d13fe07d
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex d49d7d151..587e91547 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -1139,12 +1139,25 @@ class AnthropicHandlerMixin:# aborting multi-turn sessions. Canonicalizing here (in place, so body,# original, forwarded, and the recorded/replayed prefix are all identical)# keeps it cache-safe: overlay_cached_prefix replays the same stripped bytes.+ # (Also applied in read_request_json_with_bytes for every parsed request;+ # the re-run here is idempotent and covers paths that bypass that helper.)_strip_streaming_only_content_fields(messages)pipeline_provider = provider_namepipeline_path = request.url.path if upstream_base_url else "/v1/messages"pipeline_stream = bool(body.get("stream", False) or force_stream)+ # O1 (2026-09-27 perf audit): the snapshot of the original+ # conversation aliases the live list unless hooks or pipeline+ # extensions are configured - those can mutate `messages` in place+ # (pre_compress receives the live list), so then the snapshot is an+ # independently owned copy (snapshot_original_messages). The+ # deep_copy stage is always measured so the timing summary keeps+ # its key.+ from headroom.proxy.helpers import snapshot_original_messages+with stage_timer.measure("deep_copy"):- original_client_messages = copy.deepcopy(messages)+ original_client_messages = snapshot_original_messages(+ messages, hooks=self.config.hooks, extensions=self.pipeline_extensions+ )input_event = self.pipeline_extensions.emit(PipelineStage.INPUT_RECEIVED,operation="proxy.request",@@ -1158,7 +1171,9 @@ class AnthropicHandlerMixin:if input_event.messages is not None:messages = input_event.messageswith stage_timer.measure("deep_copy"):- original_client_messages = copy.deepcopy(messages)+ original_client_messages = snapshot_original_messages(+ messages, hooks=self.config.hooks, extensions=self.pipeline_extensions+ )if input_event.tools is not None:body["tools"] = input_event.tools@@ -5413,6 +5428,9 @@ class AnthropicHandlerMixin:compressed_requests = []pipeline_timing: dict[str, float] = {}+ # O1 (2026-09-27 perf audit): the per-request snapshot aliases unless+ # hooks or extensions are configured (see the main handler).+ from headroom.proxy.helpers import snapshot_original_messagesfrom headroom.transforms.cold_prefix import anthropic_cache_ttl_seconds# Apply compression to each request in the batch@@ -5422,7 +5440,9 @@ class AnthropicHandlerMixin:canonical_params = dict(params)original_tools = canonical_params.get("tools")messages = params.get("messages", [])- original_messages = copy.deepcopy(messages)+ original_messages = snapshot_original_messages(+ messages, hooks=self.config.hooks, extensions=self.pipeline_extensions+ )model = params.get("model", "unknown")cache_ttl_seconds = anthropic_cache_ttl_seconds(model, original_messages, params.get("system")diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 4dbee1e0a..70df986c3 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -3583,7 +3583,16 @@ class OpenAIHandlerMixin:)model = body.get("model", "unknown")messages = body.get("messages", [])- original_client_messages = copy.deepcopy(messages)+ # O1 (2026-09-27 perf audit): the snapshot of the original+ # conversation aliases the live list unless hooks or pipeline+ # extensions are configured; those can mutate `messages` in place+ # (pre_compress receives the live list), so then it is an+ # independently owned copy (see snapshot_original_messages).+ from headroom.proxy.helpers import snapshot_original_messages++ original_client_messages = snapshot_original_messages(+ messages, hooks=self.config.hooks, extensions=self.pipeline_extensions+ )custom_upstream_base_url = _resolve_openai_upstream_base(request.headers)upstream_base_url = self._resolve_openai_upstream(request)handler_path_suffix = _resolve_openai_chat_handler_path(@@ -3607,7 +3616,9 @@ class OpenAIHandlerMixin:)if input_event.messages is not None:messages = input_event.messages- original_client_messages = copy.deepcopy(messages)+ original_client_messages = snapshot_original_messages(+ messages, hooks=self.config.hooks, extensions=self.pipeline_extensions+ )if input_event.tools is not None:body["tools"] = input_event.toolsdiff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.pyindex f987a1a15..472d5d18e 100644--- a/headroom/proxy/helpers.py+++ b/headroom/proxy/helpers.py@@ -9,6 +9,7 @@ Extracted from server.py for maintainability.from __future__ import annotationsimport asyncio+import copyimport hashlibimport jsonimport logging@@ -3031,6 +3032,65 @@ def _brotli_bounded(raw: bytes) -> bytes:return bytes(out)+def _decompress_bounded(raw: bytes, encoding: str) -> bytes:+ """Decode a ``Content-Encoding:``-compressed body off the event loop.++ One dispatch point for the four bounded decompressors below, run inside+ a worker thread by :func:`_read_request_body_bytes`. Message behavior is+ identical to the previous inline branches: ``ValueError`` (including the+ ``RequestBodyTooLarge`` subclass and the "Failed to decompress ..."+ diagnostics, which also cover a missing optional codec — a zstd/br+ ``ImportError`` is reworded, never let through raw) propagates+ verbatim; any other error type is reworded into a decompression+ ``ValueError`` with the codec named.+ """+ if encoding in ("zstd", "zstandard"):+ try:+ return _zstd_bounded(raw)+ except ValueError:+ # Covers RequestBodyTooLarge and the explicit stream diagnostics.+ raise+ except ImportError:+ raise ValueError(+ "Request body is zstd-compressed but the 'zstandard' package is not installed. "+ "Install it with: pip install zstandard"+ ) from None+ except Exception as exc:+ raise ValueError(f"Failed to decompress zstd request body: {exc}") from exc+ if encoding == "gzip":+ import zlib++ try:+ return _inflate_bounded(raw, wbits=16 + zlib.MAX_WBITS, label="gzip", multi_member=True)+ except ValueError:+ # Covers RequestBodyTooLarge and the explicit stream diagnostics,+ # both already carrying the message we want.+ raise+ except Exception as exc:+ raise ValueError(f"Failed to decompress gzip request body: {exc}") from exc+ if encoding == "deflate":+ import zlib++ try:+ return _inflate_bounded(raw, wbits=zlib.MAX_WBITS, label="deflate")+ except ValueError:+ raise+ except Exception as exc:+ raise ValueError(f"Failed to decompress deflate request body: {exc}") from exc+ if encoding == "br":+ try:+ return _brotli_bounded(raw)+ except ValueError:+ raise+ except ImportError:+ raise ValueError(+ "Request body is brotli-compressed but the 'brotli' package is not installed."+ ) from None+ except Exception as exc:+ raise ValueError(f"Failed to decompress brotli request body: {exc}") from exc+ raise ValueError(f"Unsupported Content-Encoding: {encoding}")++async def _read_request_body_bytes(request: Request) -> bytes:"""Read and (if needed) decompress the request body, returning raw UTF-8 bytes.@@ -3078,49 +3138,41 @@ async def _read_request_body_bytes(request: Request) -> bytes:# MAX_DECOMPRESSED_BODY_SIZE. RequestBodyTooLarge is re-raised ahead of the# generic handlers so the size refusal is not reworded into a vague# "failed to decompress" (#3284).- if encoding in ("zstd", "zstandard"):- try:- raw = _zstd_bounded(raw)- except RequestBodyTooLarge:- raise- except ImportError:- raise ValueError(- "Request body is zstd-compressed but the 'zstandard' package is not installed. "- "Install it with: pip install zstandard"- ) from None- except Exception as exc:- raise ValueError(f"Failed to decompress zstd request body: {exc}") from exc- elif encoding == "gzip":- import zlib-- try:- raw = _inflate_bounded(raw, wbits=16 + zlib.MAX_WBITS, label="gzip", multi_member=True)- except ValueError:- # Covers RequestBodyTooLarge and the explicit stream diagnostics,- # both already carrying the message we want.- raise- except Exception as exc:- raise ValueError(f"Failed to decompress gzip request body: {exc}") from exc- elif encoding == "deflate":- import zlib-- try:- raw = _inflate_bounded(raw, wbits=zlib.MAX_WBITS, label="deflate")- except ValueError:- raise- except Exception as exc:- raise ValueError(f"Failed to decompress deflate request body: {exc}") from exc- elif encoding == "br":+ #+ # The decode itself is offloaded to a worker thread: it is the CPU-heavy+ # part of this function (a gzip/zstd/br body can be megabytes of+ # decompressed JSON) and running it inline on the event loop stalls every+ # other in-flight request for its duration (#1701 pattern, see+ # _run_compression_in_executor). The bounded decompressors keep every+ # intermediate allocation capped, so the thread cannot OOM the process;+ # exceptions propagate verbatim across asyncio.to_thread.+ if encoding in (+ "zstd",+ "zstandard",+ "gzip",+ "deflate",+ "br",+ ):try:- raw = _brotli_bounded(raw)+ raw = await asyncio.to_thread(+ _decompress_bounded,+ raw,+ encoding,+ )except ValueError:+ # RequestBodyTooLarge is a ValueError subclass; the bounded+ # decompressors raise ValueError with their own message on a+ # truncated/stalled stream. Re-raise unchanged — the wrapping+ # below would reword a size refusal into a vague failure.raiseexcept ImportError:+ # _decompress_bounded rewords every codec ImportError into a+ # ValueError, so this clause is defensive (the to_thread hop+ # cannot turn one into the other) — kept so a codec-import+ # failure can never surface as a raw 500.raise ValueError(- "Request body is brotli-compressed but the 'brotli' package is not installed."+ "Failed to decompress request body: optional codec unavailable") from None- except Exception as exc:- raise ValueError(f"Failed to decompress brotli request body: {exc}") from excelif encoding and encoding != "identity":raise ValueError(f"Unsupported Content-Encoding: {encoding}")@@ -3211,10 +3263,18 @@ async def _read_request_json(request: Request) -> dict[str, Any]:if strip_output_only_request_blocks(result.get("messages")):logger.warning("removed output-only content block(s) (%s) from request messages "- "before forwarding (not valid on the request path)",+ "(not valid on the request path)",",".join(sorted(OUTPUT_ONLY_REQUEST_BLOCK_TYPES)),)+ # Canonicalize streaming-only ``index`` keys here too, so every parsed+ # request (both body readers) is schema-valid before any handler+ # snapshots or forwards it. Idempotent and a no-op for well-formed+ # requests.+ from headroom.utils import strip_streaming_only_content_fields_in_place++ strip_streaming_only_content_fields_in_place(result.get("messages"))+return result@@ -3251,6 +3311,18 @@ async def read_request_json_with_bytes(request: Request) -> tuple[dict[str, Any]",".join(sorted(OUTPUT_ONLY_REQUEST_BLOCK_TYPES)),)+ # Canonicalize streaming-only ``index`` keys on every parsed request, in+ # place, before any handler deep-copies ``messages``: the snapshot the+ # handler takes (the ``original_client_messages`` deepcopy) must be+ # schema-valid for session-id hashing and prefix replay, and re-running+ # the strip later in a single handler only would leave the snapshot (and+ # the replayed prefix) carrying keys the upstream rejects with a 400.+ # Idempotent and a no-op for well-formed requests, so existing behavior+ # is unchanged.+ from headroom.utils import strip_streaming_only_content_fields_in_place++ strip_streaming_only_content_fields_in_place(result.get("messages"))+return result, raw@@ -4603,3 +4675,20 @@ def apply_keep_last_turns(if dropped == 0:return messages, 0return result, dropped+++def snapshot_original_messages(+ messages: list[Any], *, hooks: Any = None, extensions: Any = None+) -> list[Any]:+ """The request's original messages, as later stages must see them.++ Aliasing the live list is safe only while nothing between ingress and the+ pipeline's own deepcopy can mutate it in place. A configured+ ``config.hooks`` (``pre_compress`` and friends receive the live list) or an+ enabled pipeline extension can, so in that case the snapshot is an+ independently owned deep copy; with neither configured the alias is kept+ and costs nothing.+ """+ if hooks is not None or bool(getattr(extensions, "enabled", False)):+ return copy.deepcopy(messages)+ return messagesdiff --git a/headroom/tokenizers/base.py b/headroom/tokenizers/base.pyindex 8c0e16743..cfcca7ba0 100644--- a/headroom/tokenizers/base.py+++ b/headroom/tokenizers/base.py@@ -41,9 +41,14 @@ class TokenCountCache:characters are capped.No lock: ``dict`` get/set/clear are atomic under the GIL, and the pipeline- runs on a thread pool. An LRU would need ``move_to_end``, which is not- atomic — hence clear-on-full rather than eviction. A cleared cache costs one- re-encode, never a wrong answer.+ runs on a thread pool. A true LRU would need ``move_to_end`` on every hit,+ which is not atomic — hence no per-key eviction. A full cache still clears,+ but only on the one put that admits a new key past the cap: repeated+ re-counts of *known* texts (an agent loop re-counting a stable prefix on+ every request) are pure dict hits and never trip the eviction, so the+ classic storm — one fresh tool_result per request evicting the whole+ cache and forcing a full-prefix re-encode — cannot happen. A cleared+ cache costs one re-encode, never a wrong answer."""__slots__ = ("_chars", "_counts", "_max_chars", "_max_entries", "_min_chars")@@ -66,7 +71,17 @@ class TokenCountCache:return self._counts.get(text)def put(self, text: str, count: int) -> None:- """Store *count* for *text* if it is worth caching."""+ """Store *count* for *text* if it is worth caching.++ A re-count of an entry already in the cache is a hit, not an+ admission: updating the stored value cannot grow the cache, so it+ must not run the eviction check (a dict hit would clear the whole+ cache and force a full re-encode of every cached message on the+ next miss — the storm this exists to prevent).+ """+ if text in self._counts:+ self._counts[text] = count+ returnif len(text) < self._min_chars:returnif len(self._counts) >= self._max_entries or self._chars >= self._max_chars:diff --git a/headroom/utils.py b/headroom/utils.pyindex b6489f0ea..05c31edf5 100644--- a/headroom/utils.py+++ b/headroom/utils.py@@ -278,3 +278,32 @@ def deep_copy_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:serialisation overhead on large conversation histories)."""return copy.deepcopy(messages)+++def strip_streaming_only_content_fields_in_place(messages: Any) -> None:+ """Remove streaming-only ``index`` keys from request content blocks, in place.++ Shared in-place canonicalizer used right after the shared body readers+ parse a request (the Anthropic handler has a private copy of this walk).+ ``index`` is a field some providers emit on streaming RESPONSE content-block+ deltas; it is not part of the request-message schema, so forwarding it+ upstream triggers a 400 that aborts multi-turn sessions once a client+ echoes a reconstructed assistant turn back. Stripping it before any+ deepcopy / prefix recording keeps every derived copy identical. Nested+ tool_result content lists are walked too.+ """+ if not isinstance(messages, list):+ return+ for message in messages:+ if isinstance(message, dict):+ _strip_index_from_content_blocks_in_place(message.get("content"))+++def _strip_index_from_content_blocks_in_place(content: Any) -> None:+ if not isinstance(content, list):+ return+ for block in content:+ if isinstance(block, dict):+ block.pop("index", None)+ # tool_result blocks nest their own content list of blocks.+ _strip_index_from_content_blocks_in_place(block.get("content"))diff --git a/tests/test_original_snapshot_in_place_hook.py b/tests/test_original_snapshot_in_place_hook.pynew file mode 100644index 000000000..3b75a87e2--- /dev/null+++ b/tests/test_original_snapshot_in_place_hook.py@@ -0,0 +1,177 @@+"""The recorded original messages stay pre-hook when a hook mutates in place.++O1 aliases ``original_client_messages`` to the live ``messages`` list to skip a+deepcopy per request. That is only safe while nothing mutates the list before+the pipeline's own copy; ``config.hooks.pre_compress`` receives the live list,+so a hook that edits it in place (and returns the same list) used to rewrite the+recorded original too. With hooks or extensions configured the snapshot is an+independently owned copy; with neither, the alias stays (no env switch).++Observed at the handlers' own consumers: ``compute_session_id`` derives the+session from the ORIGINAL client messages after the hook has run, and the+upstream body carries the forwarded (hooked) messages.+"""++from __future__ import annotations++import copy+import json++import httpx+import pytest++pytest.importorskip("fastapi")++from fastapi.testclient import TestClient # noqa: E402++from headroom.hooks import CompressionHooks # noqa: E402+from headroom.proxy.helpers import snapshot_original_messages # noqa: E402+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402++ORIGINAL = "the original user text"+HOOKED = "HOOKED IN PLACE"+++class _InPlaceHooks(CompressionHooks):+ """Edits the live list in place and returns the same list."""++ def pre_compress(self, messages, ctx):+ for m in messages:+ if m.get("role") == "user":+ m["content"] = HOOKED+ return messages+++def _texts(messages) -> list[str]:+ out = []+ for m in messages:+ c = m.get("content")+ if isinstance(c, str):+ out.append(c)+ elif isinstance(c, list):+ out.extend(b.get("text", "") for b in c if isinstance(b, dict))+ return out+++def _config(**over) -> ProxyConfig:+ base = {+ "optimize": False,+ "rate_limit_enabled": False,+ "cost_tracking_enabled": False,+ "log_requests": False,+ "ccr_inject_tool": False,+ "ccr_handle_responses": False,+ "ccr_context_tracking": False,+ "image_optimize": False,+ "hooks": _InPlaceHooks(),+ }+ base.update(over)+ return ProxyConfig(**base)+++def _record_session_messages(proxy) -> list:+ seen: list = []+ store = proxy.session_tracker_store+ real = store.compute_session_id++ def spy(request, model, messages, *a, **k):+ seen.append(copy.deepcopy(messages))+ return real(request, model, messages, *a, **k)++ store.compute_session_id = spy+ return seen+++def test_openai_default_cache_path_keeps_the_original_pre_hook() -> None:+ forwarded: list = []+ with TestClient(create_app(_config(cache_enabled=True))) as client:+ proxy = client.app.state.proxy+ seen = _record_session_messages(proxy)++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ forwarded.append(copy.deepcopy(body))+ return httpx.Response(+ 200,+ json={+ "id": "chatcmpl_1",+ "object": "chat.completion",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},+ },+ )++ proxy._retry_request = _fake_retry+ r = client.post(+ "/v1/chat/completions",+ headers={"authorization": "Bearer sk-test"},+ json={"model": "gpt-4o", "messages": [{"role": "user", "content": ORIGINAL}]},+ )+ assert r.status_code == 200, r.text++ assert seen, "compute_session_id was not reached"+ assert ORIGINAL in _texts(seen[-1]) and HOOKED not in _texts(seen[-1])+ assert forwarded and HOOKED in _texts(forwarded[-1]["messages"])+++def test_anthropic_token_mode_keeps_the_original_pre_hook() -> None:+ forwarded: list = []+ with TestClient(create_app(_config(mode="token", cache_enabled=False))) as client:+ proxy = client.app.state.proxy+ seen = _record_session_messages(proxy)++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ forwarded.append(copy.deepcopy(body))+ return httpx.Response(+ 200,+ content=json.dumps(+ {+ "id": "msg_1",+ "type": "message",+ "role": "assistant",+ "model": "claude-sonnet-4-5",+ "content": [{"type": "text", "text": "ok"}],+ "stop_reason": "end_turn",+ "usage": {"input_tokens": 5, "output_tokens": 1},+ }+ ).encode(),+ headers={"content-type": "application/json"},+ )++ proxy._retry_request = _fake_retry+ r = client.post(+ "/v1/messages",+ headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},+ json={+ "model": "claude-sonnet-4-5",+ "max_tokens": 16,+ "messages": [{"role": "user", "content": ORIGINAL}],+ },+ )+ assert r.status_code == 200, r.text++ assert seen, "compute_session_id was not reached"+ assert ORIGINAL in _texts(seen[-1]) and HOOKED not in _texts(seen[-1])+ assert forwarded and HOOKED in _texts(forwarded[-1]["messages"])+++class _Ext:+ enabled = True+++class _NoExt:+ enabled = False+++def test_the_snapshot_aliases_only_when_nothing_can_mutate() -> None:+ msgs = [{"role": "user", "content": ORIGINAL}]+ assert snapshot_original_messages(msgs) is msgs+ assert snapshot_original_messages(msgs, extensions=_NoExt()) is msgs+ for kw in ({"hooks": _InPlaceHooks()}, {"extensions": _Ext()}):+ snap = snapshot_original_messages(msgs, **kw)+ assert snap == msgs and snap is not msgs and snap[0] is not msgs[0]diff --git a/tests/test_output_only_request_blocks.py b/tests/test_output_only_request_blocks.pyindex 1df9924e8..684a479d1 100644--- a/tests/test_output_only_request_blocks.py+++ b/tests/test_output_only_request_blocks.py@@ -103,3 +103,123 @@ def test_reader_leaves_clean_requests_byte_identical():)_, out_raw = asyncio.run(read_request_json_with_bytes(_FakeRequest(raw)))assert out_raw == raw+++# ---------------------------------------------------------------------------+# Streaming-only ``index`` keys (2026-09-27 perf audit): both shared body+# readers must canonicalize them in place so the snapshot, the forwarded body,+# and the recorded/replayed prefix are all schema-valid for EVERY provider,+# not just the Anthropic handler. ``index`` is a streaming-RESPONSE field;+# echoing it into request content 400s upstreams that reject extra inputs.+# Idempotent and a no-op for well-formed requests (byte-identical passthrough+# is preserved — the readers only re-encode raw when output-only blocks are+# removed).+# ---------------------------------------------------------------------------+++def test_with_bytes_reader_strips_streaming_index_keys():+ body = {+ "model": "claude-fable-5",+ "messages": [+ {+ "role": "assistant",+ "content": [+ {"type": "text", "text": "hi", "index": 0},+ {"type": "tool_use", "id": "t1", "name": "f", "input": {}, "index": 1},+ ],+ },+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": "t1",+ "content": [{"type": "text", "text": "ok", "index": 0}],+ "index": 0,+ }+ ],+ },+ ],+ }+ raw = json.dumps(body).encode("utf-8")+ result, out_raw = asyncio.run(read_request_json_with_bytes(_FakeRequest(raw)))+ for msg in result["messages"]:+ for block in msg["content"]:+ assert "index" not in block+ if block.get("type") == "tool_result":+ for inner in block.get("content", []):+ assert "index" not in inner+++def test_with_bytes_reader_index_strip_preserves_clean_bytes():+ """No fallback blocks + index stripped: raw bytes stay the original wire bytes."""+ body = {+ "model": "x",+ "messages": [+ {"role": "assistant", "content": [{"type": "text", "text": "hi", "index": 0}]},+ ],+ }+ raw = json.dumps(body).encode("utf-8")+ result, out_raw = asyncio.run(read_request_json_with_bytes(_FakeRequest(raw)))+ assert result["messages"][0]["content"][0] == {"type": "text", "text": "hi"}+ # Dict is stripped; raw is NOT re-encoded (index strip alone must not+ # force a canonical re-serialization — byte-faithful passthrough holds).+ assert out_raw == raw+++def test_bytes_less_reader_strips_streaming_index_keys():+ """The bytes-less reader (Gemini + OpenAI chat paths) canonicalizes too."""+ from headroom.proxy.helpers import _read_request_json++ body = {+ "model": "x",+ "messages": [+ {"role": "assistant", "content": [{"type": "text", "text": "hi", "index": 3}]},+ ],+ }+ raw = json.dumps(body).encode("utf-8")+ result = asyncio.run(_read_request_json(_FakeRequest(raw)))+ assert result["messages"][0]["content"][0] == {"type": "text", "text": "hi"}+++def test_strip_streaming_index_is_idempotent():+ """The in-place canonicalizer must be safe to re-run (the readers apply it,+ and the Anthropic handler re-runs its private copy downstream)."""+ from headroom.utils import strip_streaming_only_content_fields_in_place++ messages = [+ {+ "role": "assistant",+ "content": [+ {"type": "text", "text": "hi", "index": 0},+ {"type": "tool_use", "id": "t1", "name": "f", "input": {}, "index": 1},+ ],+ },+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": "t1",+ "content": [{"type": "text", "text": "ok", "index": 0}],+ "index": 0,+ }+ ],+ },+ {"role": "user", "content": "plain string content"},+ ]+ first = strip_streaming_only_content_fields_in_place(messages)+ second = strip_streaming_only_content_fields_in_place(messages)+ # returns None (in-place); idempotency is asserted by the walk below+ assert first is None and second is None+ for msg in messages:+ content = msg.get("content")+ if isinstance(content, list):+ for block in content:+ assert "index" not in block+ if isinstance(block.get("content"), list):+ for inner in block["content"]:+ assert "index" not in inner+ # non-list input is a safe no-op+ strip_streaming_only_content_fields_in_place(None)+ strip_streaming_only_content_fields_in_place({"not": "a list"})diff --git a/tests/test_token_count_cache.py b/tests/test_token_count_cache.pyindex c90427870..cfb6a1df3 100644--- a/tests/test_token_count_cache.py+++ b/tests/test_token_count_cache.py@@ -106,3 +106,54 @@ def test_concurrent_counting_is_consistent() -> None:with ThreadPoolExecutor(max_workers=8) as pool:got = list(pool.map(counter.count_text, bodies))assert got == [expected[b] for b in bodies]+++# ---------------------------------------------------------------------------+# O2 (2026-09-27 perf audit): a re-count of an entry already in the cache is a+# hit, not an admission — it must update the stored value without running the+# eviction check. Before this change the admission path ran for every put(),+# so re-counting one cached message while the cache sat at capacity cleared+# the WHOLE cache and forced a full re-encode of every cached message on the+# next miss (the storm the cache exists to prevent).+# ---------------------------------------------------------------------------+++def test_recput_of_existing_key_updates_value_without_evicting() -> None:+ """Re-counting a cached key at capacity must not clear the cache."""+ cache = TokenCountCache(min_chars=1, max_entries=2, max_chars=1000)+ cache.put("alpha", 10)+ cache.put("beta", 20) # cache now at max_entries+ cache.put("alpha", 99) # hit: update in place, skip admission+ assert cache.get("alpha") == 99+ assert cache.get("beta") == 20 # not evicted+ assert len(cache._counts) == 2+++def test_recput_of_existing_key_at_char_budget_does_not_evict() -> None:+ """The char-budget branch of admission must be skipped on a hit too."""+ cache = TokenCountCache(min_chars=1, max_entries=100, max_chars=40)+ cache.put("a" * 20, 10)+ cache.put("b" * 20, 20) # _chars == 40 == max_chars+ cache.put("a" * 20, 99) # hit: no admission, no clear+ assert cache.get("a" * 20) == 99+ assert cache.get("b" * 20) == 20+ assert len(cache._counts) == 2+++def test_new_key_at_capacity_still_admits_via_clear() -> None:+ """The hit-path early return must not change admission for NEW keys."""+ cache = TokenCountCache(min_chars=1, max_entries=2, max_chars=1000)+ cache.put("alpha", 10)+ cache.put("beta", 20)+ cache.put("gamma", 30) # new key at capacity: documented clear + admit+ assert cache._counts == {"gamma": 30}+++def test_recput_of_below_min_key_never_admits_and_never_evicts() -> None:+ """Keys below min_chars are rejected by admission; re-puts must not evict."""+ cache = TokenCountCache(min_chars=10, max_entries=2, max_chars=1000)+ cache.put("aaaaaaaaaa-1", 10)+ cache.put("x", 1) # below min_chars: rejected+ assert cache._counts == {"aaaaaaaaaa-1": 10}+ cache.put("x", 5) # re-put: still no admission, no eviction+ assert cache._counts == {"aaaaaaaaaa-1": 10}