Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

perf(proxy): lighter request path: alias the message snapshot, keep the token cache on re-counts, decompress off the event loop (#3909)

Billc873d13fe07d
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index d49d7d151..587e91547 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -1139,12 +1139,25 @@ class AnthropicHandlerMixin:
# aborting multi-turn sessions. Canonicalizing here (in place, so body,
# original, forwarded, and the recorded/replayed prefix are all identical)
# keeps it cache-safe: overlay_cached_prefix replays the same stripped bytes.
+ # (Also applied in read_request_json_with_bytes for every parsed request;
+ # the re-run here is idempotent and covers paths that bypass that helper.)
_strip_streaming_only_content_fields(messages)
pipeline_provider = provider_name
pipeline_path = request.url.path if upstream_base_url else "/v1/messages"
pipeline_stream = bool(body.get("stream", False) or force_stream)
+ # O1 (2026-09-27 perf audit): the snapshot of the original
+ # conversation aliases the live list unless hooks or pipeline
+ # extensions are configured - those can mutate `messages` in place
+ # (pre_compress receives the live list), so then the snapshot is an
+ # independently owned copy (snapshot_original_messages). The
+ # deep_copy stage is always measured so the timing summary keeps
+ # its key.
+ from headroom.proxy.helpers import snapshot_original_messages
+
with stage_timer.measure("deep_copy"):
- original_client_messages = copy.deepcopy(messages)
+ original_client_messages = snapshot_original_messages(
+ messages, hooks=self.config.hooks, extensions=self.pipeline_extensions
+ )
input_event = self.pipeline_extensions.emit(
PipelineStage.INPUT_RECEIVED,
operation="proxy.request",
@@ -1158,7 +1171,9 @@ class AnthropicHandlerMixin:
if input_event.messages is not None:
messages = input_event.messages
with stage_timer.measure("deep_copy"):
- original_client_messages = copy.deepcopy(messages)
+ original_client_messages = snapshot_original_messages(
+ messages, hooks=self.config.hooks, extensions=self.pipeline_extensions
+ )
if input_event.tools is not None:
body["tools"] = input_event.tools
@@ -5413,6 +5428,9 @@ class AnthropicHandlerMixin:
compressed_requests = []
pipeline_timing: dict[str, float] = {}
+ # O1 (2026-09-27 perf audit): the per-request snapshot aliases unless
+ # hooks or extensions are configured (see the main handler).
+ from headroom.proxy.helpers import snapshot_original_messages
from headroom.transforms.cold_prefix import anthropic_cache_ttl_seconds
# Apply compression to each request in the batch
@@ -5422,7 +5440,9 @@ class AnthropicHandlerMixin:
canonical_params = dict(params)
original_tools = canonical_params.get("tools")
messages = params.get("messages", [])
- original_messages = copy.deepcopy(messages)
+ original_messages = snapshot_original_messages(
+ messages, hooks=self.config.hooks, extensions=self.pipeline_extensions
+ )
model = params.get("model", "unknown")
cache_ttl_seconds = anthropic_cache_ttl_seconds(
model, original_messages, params.get("system")
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 4dbee1e0a..70df986c3 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -3583,7 +3583,16 @@ class OpenAIHandlerMixin:
)
model = body.get("model", "unknown")
messages = body.get("messages", [])
- original_client_messages = copy.deepcopy(messages)
+ # O1 (2026-09-27 perf audit): the snapshot of the original
+ # conversation aliases the live list unless hooks or pipeline
+ # extensions are configured; those can mutate `messages` in place
+ # (pre_compress receives the live list), so then it is an
+ # independently owned copy (see snapshot_original_messages).
+ from headroom.proxy.helpers import snapshot_original_messages
+
+ original_client_messages = snapshot_original_messages(
+ messages, hooks=self.config.hooks, extensions=self.pipeline_extensions
+ )
custom_upstream_base_url = _resolve_openai_upstream_base(request.headers)
upstream_base_url = self._resolve_openai_upstream(request)
handler_path_suffix = _resolve_openai_chat_handler_path(
@@ -3607,7 +3616,9 @@ class OpenAIHandlerMixin:
)
if input_event.messages is not None:
messages = input_event.messages
- original_client_messages = copy.deepcopy(messages)
+ original_client_messages = snapshot_original_messages(
+ messages, hooks=self.config.hooks, extensions=self.pipeline_extensions
+ )
if input_event.tools is not None:
body["tools"] = input_event.tools
diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.py
index f987a1a15..472d5d18e 100644
--- a/headroom/proxy/helpers.py
+++ b/headroom/proxy/helpers.py
@@ -9,6 +9,7 @@ Extracted from server.py for maintainability.
from __future__ import annotations
import asyncio
+import copy
import hashlib
import json
import logging
@@ -3031,6 +3032,65 @@ def _brotli_bounded(raw: bytes) -> bytes:
return bytes(out)
+def _decompress_bounded(raw: bytes, encoding: str) -> bytes:
+ """Decode a ``Content-Encoding:``-compressed body off the event loop.
+
+ One dispatch point for the four bounded decompressors below, run inside
+ a worker thread by :func:`_read_request_body_bytes`. Message behavior is
+ identical to the previous inline branches: ``ValueError`` (including the
+ ``RequestBodyTooLarge`` subclass and the "Failed to decompress ..."
+ diagnostics, which also cover a missing optional codec — a zstd/br
+ ``ImportError`` is reworded, never let through raw) propagates
+ verbatim; any other error type is reworded into a decompression
+ ``ValueError`` with the codec named.
+ """
+ if encoding in ("zstd", "zstandard"):
+ try:
+ return _zstd_bounded(raw)
+ except ValueError:
+ # Covers RequestBodyTooLarge and the explicit stream diagnostics.
+ raise
+ except ImportError:
+ raise ValueError(
+ "Request body is zstd-compressed but the 'zstandard' package is not installed. "
+ "Install it with: pip install zstandard"
+ ) from None
+ except Exception as exc:
+ raise ValueError(f"Failed to decompress zstd request body: {exc}") from exc
+ if encoding == "gzip":
+ import zlib
+
+ try:
+ return _inflate_bounded(raw, wbits=16 + zlib.MAX_WBITS, label="gzip", multi_member=True)
+ except ValueError:
+ # Covers RequestBodyTooLarge and the explicit stream diagnostics,
+ # both already carrying the message we want.
+ raise
+ except Exception as exc:
+ raise ValueError(f"Failed to decompress gzip request body: {exc}") from exc
+ if encoding == "deflate":
+ import zlib
+
+ try:
+ return _inflate_bounded(raw, wbits=zlib.MAX_WBITS, label="deflate")
+ except ValueError:
+ raise
+ except Exception as exc:
+ raise ValueError(f"Failed to decompress deflate request body: {exc}") from exc
+ if encoding == "br":
+ try:
+ return _brotli_bounded(raw)
+ except ValueError:
+ raise
+ except ImportError:
+ raise ValueError(
+ "Request body is brotli-compressed but the 'brotli' package is not installed."
+ ) from None
+ except Exception as exc:
+ raise ValueError(f"Failed to decompress brotli request body: {exc}") from exc
+ raise ValueError(f"Unsupported Content-Encoding: {encoding}")
+
+
async def _read_request_body_bytes(request: Request) -> bytes:
"""Read and (if needed) decompress the request body, returning raw UTF-8 bytes.
@@ -3078,49 +3138,41 @@ async def _read_request_body_bytes(request: Request) -> bytes:
# MAX_DECOMPRESSED_BODY_SIZE. RequestBodyTooLarge is re-raised ahead of the
# generic handlers so the size refusal is not reworded into a vague
# "failed to decompress" (#3284).
- if encoding in ("zstd", "zstandard"):
- try:
- raw = _zstd_bounded(raw)
- except RequestBodyTooLarge:
- raise
- except ImportError:
- raise ValueError(
- "Request body is zstd-compressed but the 'zstandard' package is not installed. "
- "Install it with: pip install zstandard"
- ) from None
- except Exception as exc:
- raise ValueError(f"Failed to decompress zstd request body: {exc}") from exc
- elif encoding == "gzip":
- import zlib
-
- try:
- raw = _inflate_bounded(raw, wbits=16 + zlib.MAX_WBITS, label="gzip", multi_member=True)
- except ValueError:
- # Covers RequestBodyTooLarge and the explicit stream diagnostics,
- # both already carrying the message we want.
- raise
- except Exception as exc:
- raise ValueError(f"Failed to decompress gzip request body: {exc}") from exc
- elif encoding == "deflate":
- import zlib
-
- try:
- raw = _inflate_bounded(raw, wbits=zlib.MAX_WBITS, label="deflate")
- except ValueError:
- raise
- except Exception as exc:
- raise ValueError(f"Failed to decompress deflate request body: {exc}") from exc
- elif encoding == "br":
+ #
+ # The decode itself is offloaded to a worker thread: it is the CPU-heavy
+ # part of this function (a gzip/zstd/br body can be megabytes of
+ # decompressed JSON) and running it inline on the event loop stalls every
+ # other in-flight request for its duration (#1701 pattern, see
+ # _run_compression_in_executor). The bounded decompressors keep every
+ # intermediate allocation capped, so the thread cannot OOM the process;
+ # exceptions propagate verbatim across asyncio.to_thread.
+ if encoding in (
+ "zstd",
+ "zstandard",
+ "gzip",
+ "deflate",
+ "br",
+ ):
try:
- raw = _brotli_bounded(raw)
+ raw = await asyncio.to_thread(
+ _decompress_bounded,
+ raw,
+ encoding,
+ )
except ValueError:
+ # RequestBodyTooLarge is a ValueError subclass; the bounded
+ # decompressors raise ValueError with their own message on a
+ # truncated/stalled stream. Re-raise unchanged — the wrapping
+ # below would reword a size refusal into a vague failure.
raise
except ImportError:
+ # _decompress_bounded rewords every codec ImportError into a
+ # ValueError, so this clause is defensive (the to_thread hop
+ # cannot turn one into the other) — kept so a codec-import
+ # failure can never surface as a raw 500.
raise ValueError(
- "Request body is brotli-compressed but the 'brotli' package is not installed."
+ "Failed to decompress request body: optional codec unavailable"
) from None
- except Exception as exc:
- raise ValueError(f"Failed to decompress brotli request body: {exc}") from exc
elif encoding and encoding != "identity":
raise ValueError(f"Unsupported Content-Encoding: {encoding}")
@@ -3211,10 +3263,18 @@ async def _read_request_json(request: Request) -> dict[str, Any]:
if strip_output_only_request_blocks(result.get("messages")):
logger.warning(
"removed output-only content block(s) (%s) from request messages "
- "before forwarding (not valid on the request path)",
+ "(not valid on the request path)",
",".join(sorted(OUTPUT_ONLY_REQUEST_BLOCK_TYPES)),
)
+ # Canonicalize streaming-only ``index`` keys here too, so every parsed
+ # request (both body readers) is schema-valid before any handler
+ # snapshots or forwards it. Idempotent and a no-op for well-formed
+ # requests.
+ from headroom.utils import strip_streaming_only_content_fields_in_place
+
+ strip_streaming_only_content_fields_in_place(result.get("messages"))
+
return result
@@ -3251,6 +3311,18 @@ async def read_request_json_with_bytes(request: Request) -> tuple[dict[str, Any]
",".join(sorted(OUTPUT_ONLY_REQUEST_BLOCK_TYPES)),
)
+ # Canonicalize streaming-only ``index`` keys on every parsed request, in
+ # place, before any handler deep-copies ``messages``: the snapshot the
+ # handler takes (the ``original_client_messages`` deepcopy) must be
+ # schema-valid for session-id hashing and prefix replay, and re-running
+ # the strip later in a single handler only would leave the snapshot (and
+ # the replayed prefix) carrying keys the upstream rejects with a 400.
+ # Idempotent and a no-op for well-formed requests, so existing behavior
+ # is unchanged.
+ from headroom.utils import strip_streaming_only_content_fields_in_place
+
+ strip_streaming_only_content_fields_in_place(result.get("messages"))
+
return result, raw
@@ -4603,3 +4675,20 @@ def apply_keep_last_turns(
if dropped == 0:
return messages, 0
return result, dropped
+
+
+def snapshot_original_messages(
+ messages: list[Any], *, hooks: Any = None, extensions: Any = None
+) -> list[Any]:
+ """The request's original messages, as later stages must see them.
+
+ Aliasing the live list is safe only while nothing between ingress and the
+ pipeline's own deepcopy can mutate it in place. A configured
+ ``config.hooks`` (``pre_compress`` and friends receive the live list) or an
+ enabled pipeline extension can, so in that case the snapshot is an
+ independently owned deep copy; with neither configured the alias is kept
+ and costs nothing.
+ """
+ if hooks is not None or bool(getattr(extensions, "enabled", False)):
+ return copy.deepcopy(messages)
+ return messages
diff --git a/headroom/tokenizers/base.py b/headroom/tokenizers/base.py
index 8c0e16743..cfcca7ba0 100644
--- a/headroom/tokenizers/base.py
+++ b/headroom/tokenizers/base.py
@@ -41,9 +41,14 @@ class TokenCountCache:
characters are capped.
No lock: ``dict`` get/set/clear are atomic under the GIL, and the pipeline
- runs on a thread pool. An LRU would need ``move_to_end``, which is not
- atomic — hence clear-on-full rather than eviction. A cleared cache costs one
- re-encode, never a wrong answer.
+ runs on a thread pool. A true LRU would need ``move_to_end`` on every hit,
+ which is not atomic — hence no per-key eviction. A full cache still clears,
+ but only on the one put that admits a new key past the cap: repeated
+ re-counts of *known* texts (an agent loop re-counting a stable prefix on
+ every request) are pure dict hits and never trip the eviction, so the
+ classic storm — one fresh tool_result per request evicting the whole
+ cache and forcing a full-prefix re-encode — cannot happen. A cleared
+ cache costs one re-encode, never a wrong answer.
"""
__slots__ = ("_chars", "_counts", "_max_chars", "_max_entries", "_min_chars")
@@ -66,7 +71,17 @@ class TokenCountCache:
return self._counts.get(text)
def put(self, text: str, count: int) -> None:
- """Store *count* for *text* if it is worth caching."""
+ """Store *count* for *text* if it is worth caching.
+
+ A re-count of an entry already in the cache is a hit, not an
+ admission: updating the stored value cannot grow the cache, so it
+ must not run the eviction check (a dict hit would clear the whole
+ cache and force a full re-encode of every cached message on the
+ next miss — the storm this exists to prevent).
+ """
+ if text in self._counts:
+ self._counts[text] = count
+ return
if len(text) < self._min_chars:
return
if len(self._counts) >= self._max_entries or self._chars >= self._max_chars:
diff --git a/headroom/utils.py b/headroom/utils.py
index b6489f0ea..05c31edf5 100644
--- a/headroom/utils.py
+++ b/headroom/utils.py
@@ -278,3 +278,32 @@ def deep_copy_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
serialisation overhead on large conversation histories).
"""
return copy.deepcopy(messages)
+
+
+def strip_streaming_only_content_fields_in_place(messages: Any) -> None:
+ """Remove streaming-only ``index`` keys from request content blocks, in place.
+
+ Shared in-place canonicalizer used right after the shared body readers
+ parse a request (the Anthropic handler has a private copy of this walk).
+ ``index`` is a field some providers emit on streaming RESPONSE content-block
+ deltas; it is not part of the request-message schema, so forwarding it
+ upstream triggers a 400 that aborts multi-turn sessions once a client
+ echoes a reconstructed assistant turn back. Stripping it before any
+ deepcopy / prefix recording keeps every derived copy identical. Nested
+ tool_result content lists are walked too.
+ """
+ if not isinstance(messages, list):
+ return
+ for message in messages:
+ if isinstance(message, dict):
+ _strip_index_from_content_blocks_in_place(message.get("content"))
+
+
+def _strip_index_from_content_blocks_in_place(content: Any) -> None:
+ if not isinstance(content, list):
+ return
+ for block in content:
+ if isinstance(block, dict):
+ block.pop("index", None)
+ # tool_result blocks nest their own content list of blocks.
+ _strip_index_from_content_blocks_in_place(block.get("content"))
diff --git a/tests/test_original_snapshot_in_place_hook.py b/tests/test_original_snapshot_in_place_hook.py
new file mode 100644
index 000000000..3b75a87e2
--- /dev/null
+++ b/tests/test_original_snapshot_in_place_hook.py
@@ -0,0 +1,177 @@
+"""The recorded original messages stay pre-hook when a hook mutates in place.
+
+O1 aliases ``original_client_messages`` to the live ``messages`` list to skip a
+deepcopy per request. That is only safe while nothing mutates the list before
+the pipeline's own copy; ``config.hooks.pre_compress`` receives the live list,
+so a hook that edits it in place (and returns the same list) used to rewrite the
+recorded original too. With hooks or extensions configured the snapshot is an
+independently owned copy; with neither, the alias stays (no env switch).
+
+Observed at the handlers' own consumers: ``compute_session_id`` derives the
+session from the ORIGINAL client messages after the hook has run, and the
+upstream body carries the forwarded (hooked) messages.
+"""
+
+from __future__ import annotations
+
+import copy
+import json
+
+import httpx
+import pytest
+
+pytest.importorskip("fastapi")
+
+from fastapi.testclient import TestClient # noqa: E402
+
+from headroom.hooks import CompressionHooks # noqa: E402
+from headroom.proxy.helpers import snapshot_original_messages # noqa: E402
+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
+
+ORIGINAL = "the original user text"
+HOOKED = "HOOKED IN PLACE"
+
+
+class _InPlaceHooks(CompressionHooks):
+ """Edits the live list in place and returns the same list."""
+
+ def pre_compress(self, messages, ctx):
+ for m in messages:
+ if m.get("role") == "user":
+ m["content"] = HOOKED
+ return messages
+
+
+def _texts(messages) -> list[str]:
+ out = []
+ for m in messages:
+ c = m.get("content")
+ if isinstance(c, str):
+ out.append(c)
+ elif isinstance(c, list):
+ out.extend(b.get("text", "") for b in c if isinstance(b, dict))
+ return out
+
+
+def _config(**over) -> ProxyConfig:
+ base = {
+ "optimize": False,
+ "rate_limit_enabled": False,
+ "cost_tracking_enabled": False,
+ "log_requests": False,
+ "ccr_inject_tool": False,
+ "ccr_handle_responses": False,
+ "ccr_context_tracking": False,
+ "image_optimize": False,
+ "hooks": _InPlaceHooks(),
+ }
+ base.update(over)
+ return ProxyConfig(**base)
+
+
+def _record_session_messages(proxy) -> list:
+ seen: list = []
+ store = proxy.session_tracker_store
+ real = store.compute_session_id
+
+ def spy(request, model, messages, *a, **k):
+ seen.append(copy.deepcopy(messages))
+ return real(request, model, messages, *a, **k)
+
+ store.compute_session_id = spy
+ return seen
+
+
+def test_openai_default_cache_path_keeps_the_original_pre_hook() -> None:
+ forwarded: list = []
+ with TestClient(create_app(_config(cache_enabled=True))) as client:
+ proxy = client.app.state.proxy
+ seen = _record_session_messages(proxy)
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ forwarded.append(copy.deepcopy(body))
+ return httpx.Response(
+ 200,
+ json={
+ "id": "chatcmpl_1",
+ "object": "chat.completion",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
+ },
+ )
+
+ proxy._retry_request = _fake_retry
+ r = client.post(
+ "/v1/chat/completions",
+ headers={"authorization": "Bearer sk-test"},
+ json={"model": "gpt-4o", "messages": [{"role": "user", "content": ORIGINAL}]},
+ )
+ assert r.status_code == 200, r.text
+
+ assert seen, "compute_session_id was not reached"
+ assert ORIGINAL in _texts(seen[-1]) and HOOKED not in _texts(seen[-1])
+ assert forwarded and HOOKED in _texts(forwarded[-1]["messages"])
+
+
+def test_anthropic_token_mode_keeps_the_original_pre_hook() -> None:
+ forwarded: list = []
+ with TestClient(create_app(_config(mode="token", cache_enabled=False))) as client:
+ proxy = client.app.state.proxy
+ seen = _record_session_messages(proxy)
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ forwarded.append(copy.deepcopy(body))
+ return httpx.Response(
+ 200,
+ content=json.dumps(
+ {
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-sonnet-4-5",
+ "content": [{"type": "text", "text": "ok"}],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 5, "output_tokens": 1},
+ }
+ ).encode(),
+ headers={"content-type": "application/json"},
+ )
+
+ proxy._retry_request = _fake_retry
+ r = client.post(
+ "/v1/messages",
+ headers={"x-api-key": "sk-ant-test", "anthropic-version": "2023-06-01"},
+ json={
+ "model": "claude-sonnet-4-5",
+ "max_tokens": 16,
+ "messages": [{"role": "user", "content": ORIGINAL}],
+ },
+ )
+ assert r.status_code == 200, r.text
+
+ assert seen, "compute_session_id was not reached"
+ assert ORIGINAL in _texts(seen[-1]) and HOOKED not in _texts(seen[-1])
+ assert forwarded and HOOKED in _texts(forwarded[-1]["messages"])
+
+
+class _Ext:
+ enabled = True
+
+
+class _NoExt:
+ enabled = False
+
+
+def test_the_snapshot_aliases_only_when_nothing_can_mutate() -> None:
+ msgs = [{"role": "user", "content": ORIGINAL}]
+ assert snapshot_original_messages(msgs) is msgs
+ assert snapshot_original_messages(msgs, extensions=_NoExt()) is msgs
+ for kw in ({"hooks": _InPlaceHooks()}, {"extensions": _Ext()}):
+ snap = snapshot_original_messages(msgs, **kw)
+ assert snap == msgs and snap is not msgs and snap[0] is not msgs[0]
diff --git a/tests/test_output_only_request_blocks.py b/tests/test_output_only_request_blocks.py
index 1df9924e8..684a479d1 100644
--- a/tests/test_output_only_request_blocks.py
+++ b/tests/test_output_only_request_blocks.py
@@ -103,3 +103,123 @@ def test_reader_leaves_clean_requests_byte_identical():
)
_, out_raw = asyncio.run(read_request_json_with_bytes(_FakeRequest(raw)))
assert out_raw == raw
+
+
+# ---------------------------------------------------------------------------
+# Streaming-only ``index`` keys (2026-09-27 perf audit): both shared body
+# readers must canonicalize them in place so the snapshot, the forwarded body,
+# and the recorded/replayed prefix are all schema-valid for EVERY provider,
+# not just the Anthropic handler. ``index`` is a streaming-RESPONSE field;
+# echoing it into request content 400s upstreams that reject extra inputs.
+# Idempotent and a no-op for well-formed requests (byte-identical passthrough
+# is preserved — the readers only re-encode raw when output-only blocks are
+# removed).
+# ---------------------------------------------------------------------------
+
+
+def test_with_bytes_reader_strips_streaming_index_keys():
+ body = {
+ "model": "claude-fable-5",
+ "messages": [
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "text", "text": "hi", "index": 0},
+ {"type": "tool_use", "id": "t1", "name": "f", "input": {}, "index": 1},
+ ],
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "t1",
+ "content": [{"type": "text", "text": "ok", "index": 0}],
+ "index": 0,
+ }
+ ],
+ },
+ ],
+ }
+ raw = json.dumps(body).encode("utf-8")
+ result, out_raw = asyncio.run(read_request_json_with_bytes(_FakeRequest(raw)))
+ for msg in result["messages"]:
+ for block in msg["content"]:
+ assert "index" not in block
+ if block.get("type") == "tool_result":
+ for inner in block.get("content", []):
+ assert "index" not in inner
+
+
+def test_with_bytes_reader_index_strip_preserves_clean_bytes():
+ """No fallback blocks + index stripped: raw bytes stay the original wire bytes."""
+ body = {
+ "model": "x",
+ "messages": [
+ {"role": "assistant", "content": [{"type": "text", "text": "hi", "index": 0}]},
+ ],
+ }
+ raw = json.dumps(body).encode("utf-8")
+ result, out_raw = asyncio.run(read_request_json_with_bytes(_FakeRequest(raw)))
+ assert result["messages"][0]["content"][0] == {"type": "text", "text": "hi"}
+ # Dict is stripped; raw is NOT re-encoded (index strip alone must not
+ # force a canonical re-serialization — byte-faithful passthrough holds).
+ assert out_raw == raw
+
+
+def test_bytes_less_reader_strips_streaming_index_keys():
+ """The bytes-less reader (Gemini + OpenAI chat paths) canonicalizes too."""
+ from headroom.proxy.helpers import _read_request_json
+
+ body = {
+ "model": "x",
+ "messages": [
+ {"role": "assistant", "content": [{"type": "text", "text": "hi", "index": 3}]},
+ ],
+ }
+ raw = json.dumps(body).encode("utf-8")
+ result = asyncio.run(_read_request_json(_FakeRequest(raw)))
+ assert result["messages"][0]["content"][0] == {"type": "text", "text": "hi"}
+
+
+def test_strip_streaming_index_is_idempotent():
+ """The in-place canonicalizer must be safe to re-run (the readers apply it,
+ and the Anthropic handler re-runs its private copy downstream)."""
+ from headroom.utils import strip_streaming_only_content_fields_in_place
+
+ messages = [
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "text", "text": "hi", "index": 0},
+ {"type": "tool_use", "id": "t1", "name": "f", "input": {}, "index": 1},
+ ],
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "t1",
+ "content": [{"type": "text", "text": "ok", "index": 0}],
+ "index": 0,
+ }
+ ],
+ },
+ {"role": "user", "content": "plain string content"},
+ ]
+ first = strip_streaming_only_content_fields_in_place(messages)
+ second = strip_streaming_only_content_fields_in_place(messages)
+ # returns None (in-place); idempotency is asserted by the walk below
+ assert first is None and second is None
+ for msg in messages:
+ content = msg.get("content")
+ if isinstance(content, list):
+ for block in content:
+ assert "index" not in block
+ if isinstance(block.get("content"), list):
+ for inner in block["content"]:
+ assert "index" not in inner
+ # non-list input is a safe no-op
+ strip_streaming_only_content_fields_in_place(None)
+ strip_streaming_only_content_fields_in_place({"not": "a list"})
diff --git a/tests/test_token_count_cache.py b/tests/test_token_count_cache.py
index c90427870..cfb6a1df3 100644
--- a/tests/test_token_count_cache.py
+++ b/tests/test_token_count_cache.py
@@ -106,3 +106,54 @@ def test_concurrent_counting_is_consistent() -> None:
with ThreadPoolExecutor(max_workers=8) as pool:
got = list(pool.map(counter.count_text, bodies))
assert got == [expected[b] for b in bodies]
+
+
+# ---------------------------------------------------------------------------
+# O2 (2026-09-27 perf audit): a re-count of an entry already in the cache is a
+# hit, not an admission — it must update the stored value without running the
+# eviction check. Before this change the admission path ran for every put(),
+# so re-counting one cached message while the cache sat at capacity cleared
+# the WHOLE cache and forced a full re-encode of every cached message on the
+# next miss (the storm the cache exists to prevent).
+# ---------------------------------------------------------------------------
+
+
+def test_recput_of_existing_key_updates_value_without_evicting() -> None:
+ """Re-counting a cached key at capacity must not clear the cache."""
+ cache = TokenCountCache(min_chars=1, max_entries=2, max_chars=1000)
+ cache.put("alpha", 10)
+ cache.put("beta", 20) # cache now at max_entries
+ cache.put("alpha", 99) # hit: update in place, skip admission
+ assert cache.get("alpha") == 99
+ assert cache.get("beta") == 20 # not evicted
+ assert len(cache._counts) == 2
+
+
+def test_recput_of_existing_key_at_char_budget_does_not_evict() -> None:
+ """The char-budget branch of admission must be skipped on a hit too."""
+ cache = TokenCountCache(min_chars=1, max_entries=100, max_chars=40)
+ cache.put("a" * 20, 10)
+ cache.put("b" * 20, 20) # _chars == 40 == max_chars
+ cache.put("a" * 20, 99) # hit: no admission, no clear
+ assert cache.get("a" * 20) == 99
+ assert cache.get("b" * 20) == 20
+ assert len(cache._counts) == 2
+
+
+def test_new_key_at_capacity_still_admits_via_clear() -> None:
+ """The hit-path early return must not change admission for NEW keys."""
+ cache = TokenCountCache(min_chars=1, max_entries=2, max_chars=1000)
+ cache.put("alpha", 10)
+ cache.put("beta", 20)
+ cache.put("gamma", 30) # new key at capacity: documented clear + admit
+ assert cache._counts == {"gamma": 30}
+
+
+def test_recput_of_below_min_key_never_admits_and_never_evicts() -> None:
+ """Keys below min_chars are rejected by admission; re-puts must not evict."""
+ cache = TokenCountCache(min_chars=10, max_entries=2, max_chars=1000)
+ cache.put("aaaaaaaaaa-1", 10)
+ cache.put("x", 1) # below min_chars: rejected
+ assert cache._counts == {"aaaaaaaaaa-1": 10}
+ cache.put("x", 5) # re-put: still no admission, no eviction
+ assert cache._counts == {"aaaaaaaaaa-1": 10}