Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): break the compact-every-turn loop near the real context window (#2942)

gglucass76cceeb48728
diff --git a/headroom/proxy/context_guard.py b/headroom/proxy/context_guard.py
new file mode 100644
index 000000000..b1bd1802e
--- /dev/null
+++ b/headroom/proxy/context_guard.py
@@ -0,0 +1,516 @@
+"""Context-limit guard: keep client auto-compaction working under compression.
+
+Compression makes the usage reported back to the client reflect the
+*forwarded* (compressed) request, not the client's raw transcript. Clients
+whose auto-compaction keys off reported usage (Claude Code) therefore defer
+compaction — the selling point — but with no floor: the raw transcript can
+grow until even the compressed request exceeds the model's real context
+window. From that point every turn fails with a 400 ``prompt is too long``,
+the client force-compacts on each error, and the session degrades into a
+compact-every-other-prompt loop (field report 2026-08-12).
+
+Three cooperating pieces, all passthrough until the danger zone:
+
+1. ``effective_context_limit`` — the window the forwarded request is really
+ subject to: the model's registry limit, raised to 1M when the client sent
+ a ``context-1m`` beta token, capped by any limit *learned* from an actual
+ ``prompt is too long`` error (piece 2). Also used for compression
+ pressure, so 1M sessions stop being treated as 5x over budget.
+2. ``note_prompt_too_long`` — parses ``prompt is too long: N tokens > M
+ maximum`` from upstream 400 bodies and records M per (model, 1m-beta)
+ key. This is how a client that *believes* 1M but whose account is capped
+ at 200k stops looping after exactly one error.
+3. ``StreamUsageGuard`` — an SSE transformer for the streamed bytes only:
+ when the ``message_start`` usage total reaches ``TRIGGER_FRACTION`` of
+ the effective limit, it inflates the reported ``input_tokens`` — in both
+ ``message_start`` and the final cumulative-usage ``message_delta``, which
+ clients merge over the former — so the client's gauge reads
+ ``REPORT_FRACTION`` of the window the client believes in, which trips
+ the client's own graceful compaction before the 400 wall. Savings
+ metrics and telemetry parse the original upstream bytes, never the
+ nudged ones.
+
+The nudged number is deliberately unmarked on the wire. It exists to be read
+by the client's own compaction gauge, which is a third-party parser we do not
+control, so anything added beside it is either ignored or a wire-shape change
+against a client we cannot test. Nothing inside headroom needs the marker:
+savings, telemetry and billing all read the original upstream bytes, never the
+client-bound ones, and each rewrite emits ``event=context_guard_nudge`` with
+the forwarded and reported totals for reconciliation.
+
+Kill switch: ``HEADROOM_CONTEXT_GUARD=0``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import re
+import time
+from typing import Any
+
+from headroom.proxy.sse_byte_buffer_policy import find_sse_event_terminator
+
+logger = logging.getLogger("headroom.proxy")
+
+# Forwarded-usage fraction of the *effective* (real) limit at which the nudge
+# arms. 0.90 leaves one normal turn of headroom before the hard wall.
+TRIGGER_FRACTION = 0.90
+# Reported-usage fraction of the *believed* limit the nudge inflates to.
+# Claude Code auto-compacts around 92%; 0.95 clears that with margin.
+REPORT_FRACTION = 0.95
+
+_CONTEXT_1M_PREFIX = "context-1m"
+
+_PROMPT_TOO_LONG_RE = re.compile(
+ r"prompt is too long:\s*(\d+)\s*tokens?\s*>\s*(\d+)\s*maximum", re.IGNORECASE
+)
+
+# A learned maximum is believed only inside this band. Below the floor it is
+# not a context window: gateways return the same "prompt is too long: N > M"
+# shape for per-request quota and per-key throttles, and one such number would
+# clamp every later request on that model for the life of the process. Above
+# the ceiling it cannot cap anything anyway.
+_MIN_LEARNED_LIMIT = 8_000
+_MAX_LEARNED_LIMIT = 10_000_000
+
+# A learned limit is evidence about one account at one moment, not a fact about
+# the model: entitlements get provisioned, gateways get repointed. It expires so
+# a stale cap cannot hold a budget down forever -- the cost of expiry is at most
+# one more prompt-too-long error, which is what taught it the first time.
+_LEARNED_LIMIT_TTL_SECONDS = 3600.0
+
+# (model, has_1m_beta, credential scope) -> (maximum, learned-at monotonic time).
+#
+# The 1m-beta dimension matters: an account with real 1M access must not inherit
+# a 200k limit learned from a request that never sent the beta. The credential
+# scope matters for the same reason one step out -- this dict is process-global,
+# and on a shared proxy one tenant's capped key would otherwise clamp the budget
+# of every other tenant on that model.
+_learned_limits: dict[tuple[str, bool, str], tuple[int, float]] = {}
+
+# Give up scanning for message_start beyond this many buffered bytes.
+_MAX_GUARD_BUFFER = 256 * 1024
+
+
+def context_guard_enabled() -> bool:
+ """True unless explicitly disabled via HEADROOM_CONTEXT_GUARD=0."""
+ from headroom.proxy import runtime_env
+
+ return (runtime_env.getenv("HEADROOM_CONTEXT_GUARD", "1") or "1").strip().lower() not in (
+ "0",
+ "false",
+ "no",
+ "off",
+ )
+
+
+def has_context_1m_beta(beta_header: str | None) -> bool:
+ """True when the anthropic-beta header carries a context-1m token."""
+ if not beta_header:
+ return False
+ return any(
+ token.strip().lower().startswith(_CONTEXT_1M_PREFIX) for token in beta_header.split(",")
+ )
+
+
+def believed_context_limit(model_limit: int, beta_header: str | None) -> int:
+ """The context window the *client* thinks it has."""
+ if has_context_1m_beta(beta_header):
+ return max(model_limit, 1_000_000)
+ return model_limit
+
+
+def credential_scope(credential: str | None) -> str:
+ """A stable, non-reversible tag for the upstream credential.
+
+ Only ever a truncated digest: the credential itself is never stored, never
+ logged, and never compared outside this function.
+ """
+ if not credential:
+ return ""
+ return hashlib.sha256(credential.strip().encode("utf-8", "ignore")).hexdigest()[:16]
+
+
+def credential_scope_from_headers(headers: Any) -> str:
+ """Credential scope for an outbound header mapping, best effort.
+
+ Never raises: an unusable header mapping just means the unscoped bucket,
+ which is the behaviour of a single-tenant proxy anyway.
+ """
+ try:
+ for name in ("x-api-key", "authorization"):
+ value = headers.get(name) or headers.get(name.title())
+ if isinstance(value, str) and value:
+ return credential_scope(value)
+ except Exception:
+ return ""
+ return ""
+
+
+def _learned_key(model: str, beta_header: str | None, scope: str) -> tuple[str, bool, str]:
+ return (model, has_context_1m_beta(beta_header), scope)
+
+
+def _learned_limit(key: tuple[str, bool, str]) -> int | None:
+ """The unexpired learned maximum for ``key``, dropping it once stale."""
+ entry = _learned_limits.get(key)
+ if entry is None:
+ return None
+ maximum, learned_at = entry
+ if time.monotonic() - learned_at >= _LEARNED_LIMIT_TTL_SECONDS:
+ _learned_limits.pop(key, None)
+ return None
+ return maximum
+
+
+def effective_context_limit(
+ model: str,
+ model_limit: int,
+ beta_header: str | None,
+ *,
+ scope: str = "",
+) -> int:
+ """The window the forwarded request is actually subject to.
+
+ Optimistically the believed limit; capped by a limit learned from a real
+ ``prompt is too long`` error for the same (model, beta, credential) shape,
+ and only while that lesson is still fresh.
+ """
+ believed = believed_context_limit(model_limit, beta_header)
+ learned = _learned_limit(_learned_key(model, beta_header, scope))
+ if learned is not None:
+ return min(believed, learned)
+ return believed
+
+
+def note_prompt_too_long(
+ model: str,
+ beta_header: str | None,
+ error_text: Any,
+ *,
+ scope: str = "",
+) -> int | None:
+ """Learn the real context limit from an upstream prompt-too-long error.
+
+ Returns the learned maximum, or None when there is nothing to learn.
+
+ This runs on the error path, where the client is still waiting for the real
+ upstream body, so it accepts whatever the body turned out to be: upstreams
+ and gateways send ``"message": null``, nested objects, and lists in the
+ slot this reads, and a diagnostic helper must not turn one of those into an
+ exception that replaces the error the caller was about to forward.
+ """
+ if isinstance(error_text, bytes | bytearray):
+ error_text = bytes(error_text).decode("utf-8", errors="replace")
+ if not isinstance(error_text, str):
+ return None
+ match = _PROMPT_TOO_LONG_RE.search(error_text)
+ if not match:
+ return None
+ maximum = int(match.group(2))
+ if not _MIN_LEARNED_LIMIT <= maximum <= _MAX_LEARNED_LIMIT:
+ # Not a context window. Learning it would clamp the budget on a number
+ # that never described one.
+ logger.debug(
+ "event=context_guard_limit_out_of_band model=%s limit=%d (ignored)",
+ model,
+ maximum,
+ )
+ return None
+ key = _learned_key(model, beta_header, scope)
+ previous = _learned_limit(key)
+ _learned_limits[key] = (maximum, time.monotonic())
+ if previous != maximum:
+ logger.warning(
+ "event=context_guard_learned_limit model=%s context_1m=%s limit=%d "
+ "attempted=%s (client compaction gauge will be nudged near this limit)",
+ model,
+ key[1],
+ maximum,
+ match.group(1),
+ )
+ return maximum
+
+
+def reset_learned_limits() -> None:
+ """Test hook: clear limits learned from prompt-too-long errors."""
+ _learned_limits.clear()
+
+
+def _data_line(payload: dict[str, Any], original: bytes) -> bytes:
+ """A rewritten ``data:`` line keeping the original's CRLF ending."""
+ eol = b"\r" if original.endswith(b"\r") else b""
+ return b"data: " + json.dumps(payload, separators=(",", ":")).encode() + eol
+
+
+def _sse_event_type(event: bytes) -> bytes | None:
+ """The event type of one SSE frame, or ``None`` when it carries none.
+
+ Read from the ``event:`` line, not matched as a substring of the frame: a
+ substring match reads the *payload* too, so an assistant that writes the
+ words ``message_delta`` into its own reply ended the scan and lost the
+ final nudge.
+
+ ``None`` means the frame is not part of the message protocol at all --
+ an SSE comment, a ``retry:`` field, the keepalives an intermediary shim
+ injects. Those pass through without arming or disarming anything; treating
+ one as an unknown event disabled the guard for the whole stream.
+ """
+ saw_data = False
+ for line in event.split(b"\n"):
+ if line.startswith(b"event:"):
+ return line[len(b"event:") :].strip()
+ if line.startswith(b"data:"):
+ saw_data = True
+ if not saw_data:
+ return None
+ # A data frame with no event: line defaults to the type inside the payload.
+ for line in event.split(b"\n"):
+ if not line.startswith(b"data:"):
+ continue
+ try:
+ payload = json.loads(line[len(b"data:") :].strip())
+ except Exception:
+ return b""
+ value = payload.get("type") if isinstance(payload, dict) else None
+ return value.encode() if isinstance(value, str) else b""
+ return b""
+
+
+def nudged_input_tokens(
+ usage: dict[str, Any],
+ *,
+ believed_limit: int,
+ effective_limit: int,
+) -> int | None:
+ """The inflated ``input_tokens`` for this usage, or ``None`` to leave it be.
+
+ One implementation for every client-bound surface -- ``message_start``, the
+ final ``message_delta``, and the non-streaming JSON body -- so a request
+ cannot get the raised compression budget on one path and the un-nudged
+ gauge on another.
+ """
+ if believed_limit <= 0 or effective_limit <= 0:
+ return None
+ input_tokens = int(usage.get("input_tokens") or 0)
+ cache_read = int(usage.get("cache_read_input_tokens") or 0)
+ cache_creation = int(usage.get("cache_creation_input_tokens") or 0)
+ total = input_tokens + cache_read + cache_creation
+ if total < TRIGGER_FRACTION * effective_limit:
+ return None
+ target_total = int(believed_limit * REPORT_FRACTION)
+ if total >= target_total:
+ # Already reads as full to the client -- never deflate.
+ return None
+ return target_total - cache_read - cache_creation
+
+
+def nudge_response_usage(
+ resp_json: Any,
+ *,
+ believed_limit: int,
+ effective_limit: int,
+ request_id: str = "",
+) -> bool:
+ """Nudge the usage of a non-streaming Anthropic message body, in place.
+
+ The buffered path raises the compression budget exactly like the streaming
+ one, so it has to move the client's gauge exactly like the streaming one
+ too; otherwise a non-streaming client gets the larger forwarded request and
+ none of the warning that it is near the wall.
+
+ Returns True when the body was modified.
+ """
+ if not isinstance(resp_json, dict):
+ return False
+ usage = resp_json.get("usage")
+ if not isinstance(usage, dict):
+ return False
+ new_input = nudged_input_tokens(
+ usage, believed_limit=believed_limit, effective_limit=effective_limit
+ )
+ if new_input is None:
+ return False
+ cache_read = int(usage.get("cache_read_input_tokens") or 0)
+ cache_creation = int(usage.get("cache_creation_input_tokens") or 0)
+ forwarded_total = int(usage.get("input_tokens") or 0) + cache_read + cache_creation
+ usage["input_tokens"] = new_input
+ logger.warning(
+ "[%s] event=context_guard_nudge path=non_streaming forwarded_total=%d "
+ "effective_limit=%d reported_total=%d believed_limit=%d (nudging client to "
+ "compact before the prompt-too-long wall)",
+ request_id,
+ forwarded_total,
+ effective_limit,
+ new_input + cache_read + cache_creation,
+ believed_limit,
+ )
+ return True
+
+
+class StreamUsageGuard:
+ """Rewrites near-the-wall usage in the client-bound SSE bytes.
+
+ Feed every chunk headed to the client through :meth:`feed`; call
+ :meth:`flush` when the stream ends. The first ``message_start`` decides
+ whether the nudge is active. When it is not (the common case), the guard
+ goes inert immediately and chunks pass through untouched — steady-state
+ overhead is one ``if``. When it is, the guard rewrites the
+ ``message_start`` usage AND stays armed for the final ``message_delta``:
+ since mid-2025 that event carries cumulative input/cache usage and
+ clients (verified live with Claude Code 2026-08-12) let it override the
+ ``message_start`` values, so nudging only the first event loses the
+ merge. Events are released as soon as their ``\\n\\n`` terminator
+ arrives, so no latency is added beyond SSE's own event framing.
+ """
+
+ def __init__(
+ self,
+ *,
+ believed_limit: int,
+ effective_limit: int,
+ request_id: str = "",
+ ) -> None:
+ self._believed_limit = believed_limit
+ self._effective_limit = effective_limit
+ self._request_id = request_id
+ self._buf = bytearray()
+ self._seen_message_start = False
+ self._target_total: int | None = None
+ self._start_cache_read = 0
+ self._start_cache_creation = 0
+ # Inert immediately when limits are unusable.
+ self._done = believed_limit <= 0 or effective_limit <= 0
+
+ def feed(self, chunk: bytes) -> bytes:
+ if self._done:
+ return chunk
+ self._buf.extend(chunk)
+ out = bytearray()
+ while not self._done:
+ # LF or CRLF framing: a CRLF stream never contains ``\n\n`` and
+ # was held until the buffer cap, so it stopped streaming.
+ terminator = find_sse_event_terminator(self._buf)
+ if terminator is None:
+ break
+ end = terminator[0] + terminator[1]
+ event = bytes(self._buf[:end])
+ del self._buf[:end]
+ out += self._process_event(event)
+ # Capped on what is still unframed, so one large chunk carrying
+ # message_start is read before the cap can disarm the guard.
+ if self._done or len(self._buf) > _MAX_GUARD_BUFFER:
+ out += self._finish()
+ return bytes(out)
+
+ def flush(self) -> bytes:
+ """Return any held-back bytes; the guard goes inert."""
+ return self._finish()
+
+ def _finish(self) -> bytes:
+ self._done = True
+ remaining = bytes(self._buf)
+ self._buf = bytearray()
+ return remaining
+
+ def _process_event(self, event: bytes) -> bytes:
+ event_type = _sse_event_type(event)
+ # Pings, comments, retry fields, keepalives: not part of the message
+ # protocol, so they neither arm nor disarm the guard.
+ if event_type is None or event_type == b"ping":
+ return event
+ if not self._seen_message_start:
+ # Anything data-bearing that isn't message_start (error events,
+ # protocol changes) ends the scan.
+ self._seen_message_start = True
+ if event_type != b"message_start":
+ self._done = True
+ return event
+ try:
+ rewritten = self._rewrite_message_start(event)
+ except Exception:
+ logger.debug("context_guard: message_start rewrite skipped", exc_info=True)
+ self._done = True
+ return event
+ if self._target_total is None:
+ # Below trigger (or already reads full): nothing to nudge.
+ self._done = True
+ return rewritten
+ # Armed: pass content deltas through, rewrite the final usage delta.
+ if event_type != b"message_delta":
+ return event
+ self._done = True
+ try:
+ return self._rewrite_message_delta(event)
+ except Exception:
+ logger.debug("context_guard: message_delta rewrite skipped", exc_info=True)
+ return event
+
+ def _rewrite_message_start(self, event: bytes) -> bytes:
+ lines = event.split(b"\n")
+ for i, line in enumerate(lines):
+ if not line.startswith(b"data:"):
+ continue
+ payload = json.loads(line[len(b"data:") :].strip())
+ if payload.get("type") != "message_start":
+ return event
+ usage = payload.get("message", {}).get("usage")
+ if not isinstance(usage, dict):
+ return event
+ cache_read = int(usage.get("cache_read_input_tokens") or 0)
+ cache_creation = int(usage.get("cache_creation_input_tokens") or 0)
+ total = int(usage.get("input_tokens") or 0) + cache_read + cache_creation
+ new_input = nudged_input_tokens(
+ usage,
+ believed_limit=self._believed_limit,
+ effective_limit=self._effective_limit,
+ )
+ if new_input is None:
+ return event
+ target_total = new_input + cache_read + cache_creation
+ self._target_total = target_total
+ self._start_cache_read = cache_read
+ self._start_cache_creation = cache_creation
+ usage["input_tokens"] = new_input
+ logger.warning(
+ "[%s] event=context_guard_nudge forwarded_total=%d effective_limit=%d "
+ "reported_total=%d believed_limit=%d (nudging client to compact "
+ "before the prompt-too-long wall)",
+ self._request_id,
+ total,
+ self._effective_limit,
+ target_total,
+ self._believed_limit,
+ )
+ lines[i] = _data_line(payload, line)
+ return b"\n".join(lines)
+ return event
+
+ def _rewrite_message_delta(self, event: bytes) -> bytes:
+ assert self._target_total is not None
+ lines = event.split(b"\n")
+ for i, line in enumerate(lines):
+ if not line.startswith(b"data:"):
+ continue
+ payload = json.loads(line[len(b"data:") :].strip())
+ if payload.get("type") != "message_delta":
+ return event
+ usage = payload.get("usage")
+ # A delta without cumulative input usage can't override the
+ # message_start values we already rewrote — leave it alone.
+ if not isinstance(usage, dict) or "input_tokens" not in usage:
+ return event
+ cache_read = int(usage.get("cache_read_input_tokens", self._start_cache_read) or 0)
+ cache_creation = int(
+ usage.get("cache_creation_input_tokens", self._start_cache_creation) or 0
+ )
+ new_input = self._target_total - cache_read - cache_creation
+ if int(usage.get("input_tokens") or 0) >= new_input:
+ # Never deflate.
+ return event
+ usage["input_tokens"] = new_input
+ lines[i] = _data_line(payload, line)
+ return b"\n".join(lines)
+ return event
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 6f97dda20..00a3848fe 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -1857,6 +1857,24 @@ class AnthropicHandlerMixin:
from headroom.proxy.helpers import COMPRESSION_TIMEOUT_SECONDS
context_limit = self.anthropic_provider.get_context_limit(model)
+ # Context pressure must be computed against the window the
+ # request is really subject to: raised to 1M when the
+ # client sent a context-1m beta (a 1M session is NOT 5x
+ # over budget), capped by a limit learned from an actual
+ # prompt-too-long error. See headroom/proxy/context_guard.py.
+ from headroom.proxy.context_guard import (
+ context_guard_enabled,
+ credential_scope_from_headers,
+ effective_context_limit,
+ )
+
+ if context_guard_enabled():
+ context_limit = effective_context_limit(
+ model,
+ context_limit,
+ headers.get("anthropic-beta"),
+ scope=credential_scope_from_headers(headers),
+ )
result = None
biases = (
self.config.hooks.compute_biases(messages, _hook_ctx)
@@ -4122,6 +4140,7 @@ class AnthropicHandlerMixin:
outcome_provider=provider_name,
session_key=session_key,
server_memory_tool_names=server_memory_tool_names,
+ client_beta=client_anthropic_beta or "",
)
else:
# Whatever set it — the client's own ``stream: false`` or
@@ -4251,6 +4270,30 @@ class AnthropicHandlerMixin:
)
err_type = "parse_error"
+ if response.status_code == 400:
+ # Diagnostic only, and outside the parse above:
+ # `err_msg` is whatever upstream put in that slot
+ # (null, an object, a list have all been seen), and
+ # the client is still waiting for the real error
+ # body below.
+ try:
+ from headroom.proxy.context_guard import (
+ credential_scope_from_headers,
+ note_prompt_too_long,
+ )
+
+ note_prompt_too_long(
+ model,
+ headers.get("anthropic-beta"),
+ err_msg,
+ scope=credential_scope_from_headers(headers),
+ )
+ except Exception:
+ logger.debug(
+ f"[{request_id}] context_guard: limit learning skipped",
+ exc_info=True,
+ )
+
logger.warning(
f"[{request_id}] UPSTREAM_ERROR "
f"status={response.status_code} "
@@ -5022,7 +5065,14 @@ class AnthropicHandlerMixin:
)
if not buffered_stream_ccr:
return Response(
- content=response.content,
+ content=self._context_guard_nudge_body(
+ response.content,
+ status_code=response.status_code,
+ model=model,
+ headers=headers,
+ request_id=request_id,
+ client_beta=client_anthropic_beta or "",
+ ),
status_code=response.status_code,
headers=response_headers,
)
@@ -5144,6 +5194,15 @@ class AnthropicHandlerMixin:
status_code=502,
)
+ # This stream is resynthesized from the parsed message,
+ # so the streaming guard never sees it: nudge here.
+ self._context_guard_nudge_message(
+ resp_json,
+ model=model,
+ headers=headers,
+ request_id=request_id,
+ client_beta=client_anthropic_beta or "",
+ )
try:
sse_events = self._response_to_sse(resp_json, "anthropic")
except ValueError as sse_err:
@@ -5175,7 +5234,14 @@ class AnthropicHandlerMixin:
)
return Response(
- content=response.content,
+ content=self._context_guard_nudge_body(
+ response.content,
+ status_code=response.status_code,
+ model=model,
+ headers=headers,
+ request_id=request_id,
+ client_beta=client_anthropic_beta or "",
+ ),
status_code=response.status_code,
headers=response_headers,
)
@@ -5306,6 +5372,96 @@ class AnthropicHandlerMixin:
# permanently. The emit function is idempotent.
await _finalize_pre_upstream()
+ def _context_guard_nudge_body(
+ self,
+ content: bytes,
+ *,
+ status_code: int,
+ model: str,
+ headers: dict[str, str],
+ request_id: str,
+ client_beta: str | None = None,
+ ) -> bytes:
+ """Apply the context guard to a buffered (non-streaming) message body.
+
+ The buffered path raises the compression budget through
+ ``effective_context_limit`` exactly like the streaming path, so it has
+ to move the client's gauge exactly like the streaming path too. Without
+ this, a non-streaming client got the larger forwarded request and none
+ of the warning, and walked into the prompt-too-long wall the guard
+ exists to keep it away from.
+
+ Returns ``content`` unchanged unless the usage genuinely needed
+ nudging, so byte-faithful forwarding still holds for every response
+ that is not near the wall.
+ """
+ if status_code != 200 or not content:
+ return content
+ try:
+ payload = json.loads(content)
+ except Exception:
+ return content
+ if not self._context_guard_nudge_message(
+ payload,
+ model=model,
+ headers=headers,
+ request_id=request_id,
+ client_beta=client_beta,
+ ):
+ return content
+ return json.dumps(payload).encode()
+
+ def _context_guard_nudge_message(
+ self,
+ payload: Any,
+ *,
+ model: str,
+ headers: dict[str, str],
+ request_id: str,
+ client_beta: str | None = None,
+ ) -> bool:
+ """Nudge a parsed message's usage in place; True when it changed.
+
+ Shared by the buffered body above and the buffered-CCR stream, which
+ resynthesizes SSE from the parsed message and so never passes through
+ the streaming guard. ``client_beta`` is the beta the client itself
+ sent (its gauge follows it; session-sticky merging can add context-1m
+ to the outbound header); None falls back to the outbound header.
+ """
+ try:
+ from headroom.proxy.context_guard import (
+ believed_context_limit,
+ context_guard_enabled,
+ credential_scope_from_headers,
+ effective_context_limit,
+ nudge_response_usage,
+ )
+
+ if not context_guard_enabled():
+ return False
+ if not isinstance(payload, dict) or payload.get("type") != "message":
+ return False
+ model_limit = self.anthropic_provider.get_context_limit(model)
+ beta_header = headers.get("anthropic-beta")
+ return bool(
+ nudge_response_usage(
+ payload,
+ believed_limit=believed_context_limit(
+ model_limit, beta_header if client_beta is None else client_beta
+ ),
+ effective_limit=effective_context_limit(
+ model,
+ model_limit,
+ beta_header,
+ scope=credential_scope_from_headers(headers),
+ ),
+ request_id=request_id,
+ )
+ )
+ except Exception:
+ logger.debug(f"[{request_id}] context_guard: buffered nudge skipped", exc_info=True)
+ return False
+
def _anthropic_batch_capability_error(self) -> Response | None:
"""Return the stable client error for a Copilot batch target."""
if not is_copilot_upstream_url(self.ANTHROPIC_API_URL):
@@ -5478,6 +5634,23 @@ class AnthropicHandlerMixin:
original_tokens = 0 # Initialize before try to prevent UnboundLocalError
try:
context_limit = self.anthropic_provider.get_context_limit(model)
+ # Same window the streaming path computes: a batch item for a
+ # 1M-beta model was otherwise compressed against the registry
+ # limit, so identical requests got two different budgets
+ # depending on which endpoint they arrived through.
+ from headroom.proxy.context_guard import (
+ context_guard_enabled,
+ credential_scope_from_headers,
+ effective_context_limit,
+ )
+
+ if context_guard_enabled():
+ context_limit = effective_context_limit(
+ model,
+ context_limit,
+ headers.get("anthropic-beta"),
+ scope=credential_scope_from_headers(headers),
+ )
frozen_message_count = (
self._strict_previous_turn_frozen_count(original_messages, 0)
if is_cache_mode(self.config.mode)
diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py
index 2bc0d7731..4a8808af3 100644
--- a/headroom/proxy/handlers/streaming.py
+++ b/headroom/proxy/handlers/streaming.py
@@ -1029,6 +1029,7 @@ class StreamingMixin:
conversation_key: str | None = None,
conversation_tokens_saved: int | None = None,
server_memory_tool_names: frozenset[str] | None = None,
+ client_beta: str | None = None,
) -> Response | StreamingResponse:
"""Stream response with metrics tracking and memory tool handling.
@@ -1075,6 +1076,7 @@ class StreamingMixin:
conversation_key=conversation_key,
conversation_tokens_saved=conversation_tokens_saved,
server_memory_tool_names=server_memory_tool_names,
+ client_beta=client_beta,
)
except (Exception, asyncio.CancelledError):
self._cleanup_mid_turn_stream(session_key)
@@ -1108,6 +1110,7 @@ class StreamingMixin:
conversation_key: str | None = None,
conversation_tokens_saved: int | None = None,
server_memory_tool_names: frozenset[str] | None = None,
+ client_beta: str | None = None,
) -> Response | StreamingResponse:
"""Actual streaming implementation, guarded by _stream_response's cleanup wrapper."""
from fastapi.responses import Response, StreamingResponse
@@ -1128,7 +1131,63 @@ class StreamingMixin:
# of being swallowed. (#1608)
if supports_mid_turn_coalescing(client):
self._active_streams.add(session_key)
+
+ # Before the guard: its learned-limit lookup is keyed on the credential
+ # actually sent, the same one the 400 learning below keys on.
headers = await apply_copilot_api_auth(headers, url=url)
+
+ # Context-limit guard (see headroom/proxy/context_guard.py): rewrites
+ # only the client-bound message_start bytes when the forwarded request
+ # is near the model's real window, so clients whose auto-compaction
+ # keys off reported usage compact gracefully instead of looping on
+ # prompt-too-long 400s. Metrics below parse the original upstream
+ # bytes and are unaffected.
+ context_guard = None
+ if provider == "anthropic":
+ from headroom.proxy.context_guard import (
+ StreamUsageGuard,
+ believed_context_limit,
+ context_guard_enabled,
+ credential_scope_from_headers,
+ effective_context_limit,
+ )
+
+ try:
+ if context_guard_enabled():
+ _guard_model_limit = self.anthropic_provider.get_context_limit(model)
+ _guard_beta = headers.get("anthropic-beta")
+ _guard_scope = credential_scope_from_headers(headers)
+ context_guard = StreamUsageGuard(
+ # The client's gauge follows the beta the client
+ # sent; session-sticky merging can add context-1m to
+ # the outbound header after the client dropped it.
+ # None: the caller did not say, use the outbound one.
+ believed_limit=believed_context_limit(
+ _guard_model_limit,
+ _guard_beta if client_beta is None else client_beta,
+ ),
+ effective_limit=effective_context_limit(
+ model, _guard_model_limit, _guard_beta, scope=_guard_scope
+ ),
+ request_id=request_id,
+ )
+ except Exception:
+ logger.debug("context_guard setup skipped", exc_info=True)
+
+ def _guard_client_bytes(data: bytes) -> bytes:
+ """Client-bound bytes through the context guard, when there is one."""
+ return context_guard.feed(data) if context_guard is not None else data
+
+ def _drain_context_guard() -> bytes:
+ """Held-back guard bytes, once. Safe to call on any exit path."""
+ if context_guard is None:
+ return b""
+ try:
+ return context_guard.flush()
+ except Exception:
+ logger.debug("context_guard flush skipped", exc_info=True)
+ return b""
+
start_time = time.time()
# Byte-faithful forwarding (PR-A3, fixes P0-2). Resolve outbound
@@ -1403,6 +1462,24 @@ class StreamingMixin:
finally:
await upstream_response.aclose()
+ if provider == "anthropic" and upstream_response.status_code == 400:
+ # Diagnostic only: whatever it makes of the body, the client
+ # still gets the upstream error below.
+ try:
+ from headroom.proxy.context_guard import (
+ credential_scope_from_headers,
+ note_prompt_too_long,
+ )
+
+ note_prompt_too_long(
+ model,
+ headers.get("anthropic-beta"),
+ error_content,
+ scope=credential_scope_from_headers(headers),
+ )
+ except Exception:
+ logger.debug("context_guard: limit learning skipped", exc_info=True)
+
if _codex_wire_debug:
_error_text: str | None = None
_error_body: Any = None
@@ -1542,11 +1619,16 @@ class StreamingMixin:
# Always stream immediately — buffering breaks
# real-time clients (LangGraph, LangChain, etc.)
- if memory_filter is None:
- yield chunk
- else:
- for frame in memory_filter.feed(chunk):
- yield frame
+ # The context guard runs last, on exactly what the
+ # client receives (after the memory filter), and may
+ # hold back bytes only until the first complete event.
+ client_frames = (
+ [chunk] if memory_filter is None else memory_filter.feed(chunk)
+ )
+ for frame in client_frames:
+ guarded_frame = _guard_client_bytes(frame)
+ if guarded_frame:
+ yield guarded_frame
if _codex_wire_debug:
capture_codex_wire_debug(
@@ -1625,7 +1707,12 @@ class StreamingMixin:
)
if memory_filter is not None:
for frame in memory_filter.closing_frames():
- yield frame
+ guarded_frame = _guard_client_bytes(frame)
+ if guarded_frame:
+ yield guarded_frame
+ guarded_tail = _drain_context_guard()
+ if guarded_tail:
+ yield guarded_tail
return
# Parse SSE to get response JSON
@@ -1644,7 +1731,9 @@ class StreamingMixin:
stream_state=stream_state,
request_id=request_id,
):
- yield frame
+ guarded_frame = _guard_client_bytes(frame)
+ if guarded_frame:
+ yield guarded_frame
memory_filter = None
parsed_response = (
stream_state.pop("memory_client_message", None) or parsed_response
@@ -1687,7 +1776,20 @@ class StreamingMixin:
stream_state=stream_state,
request_id=request_id,
):
- yield frame
+ guarded_frame = _guard_client_bytes(frame)
+ if guarded_frame:
+ yield guarded_frame
+
+ # A stream that ended before its first complete event may
+ # leave held-back bytes in the context guard; release them,
+ # after the memory filter's held tail so the final
+ # message_delta is still nudged. Every exit does this,
+ # including the error paths below: bytes the guard is holding
+ # are bytes the upstream already sent, and dropping them
+ # silently truncates the client's stream.
+ guarded_tail = _drain_context_guard()
+ if guarded_tail:
+ yield guarded_tail
# CCR Feedback: Record headroom_retrieve tool calls for TOIN learning.
# In streaming mode, the client handles actual retrieval, but we
@@ -1725,6 +1827,9 @@ class StreamingMixin:
except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:
logger.error(f"[{request_id}] Connection error to upstream API: {e}")
+ guarded_tail = _drain_context_guard()
+ if guarded_tail:
+ yield guarded_tail
error_event = public_errors.anthropic_error_body(
public_errors.classify_or_internal(e),
request_id=request_id,
@@ -1733,6 +1838,9 @@ class StreamingMixin:
yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
except httpx.HTTPStatusError as e:
logger.error(f"[{request_id}] HTTP error from upstream API: {e}")
+ guarded_tail = _drain_context_guard()
+ if guarded_tail:
+ yield guarded_tail
# Forward the upstream error response
yield e.response.content
except MemoryToolStreamOverflowError as e:
@@ -1745,6 +1853,9 @@ class StreamingMixin:
yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()
except Exception as e:
logger.error(f"[{request_id}] Unexpected streaming error: {e}")
+ guarded_tail = _drain_context_guard()
+ if guarded_tail:
+ yield guarded_tail
error_event = public_errors.anthropic_error_body(
public_errors.classify_or_internal(e),
request_id=request_id,
diff --git a/tests/test_anthropic_buffered_ccr_usage.py b/tests/test_anthropic_buffered_ccr_usage.py
index 5002147c1..46569cce4 100644
--- a/tests/test_anthropic_buffered_ccr_usage.py
+++ b/tests/test_anthropic_buffered_ccr_usage.py
@@ -179,3 +179,124 @@ def test_buffered_ccr_records_usage_with_compression_on(monkeypatch, ccr_marker:
f"buffered-CCR turn dropped usage under compression: "
f"cr={o.cache_read_tokens} cw={o.cache_write_tokens} out={o.output_tokens}"
)
+
+
+def test_buffered_ccr_stream_gets_the_context_guard_nudge(monkeypatch, ccr_marker: str) -> None:
+ """The buffered-CCR stream is resynthesized from the parsed message, so the
+ streaming guard never sees it. Near the wall its gauge must move anyway, or
+ the client gets the raised compression budget without the warning."""
+ import json
+
+ from headroom.proxy.context_guard import REPORT_FRACTION, reset_learned_limits
+
+ reset_learned_limits()
+ app, _ = _app_and_outcomes(monkeypatch)
+ near_wall = {**_RESPONSE, "usage": {**_USAGE, "cache_read_input_tokens": 900_000}}
+ respx.post("https://api.anthropic.com/v1/messages").mock(
+ return_value=httpx.Response(200, json=near_wall)
+ )
+
+ r = _post(app, stream=True, tools=[_RETRIEVE_TOOL], marker=ccr_marker)
+
+ assert r.status_code == 200
+ start = next(
+ json.loads(line[len("data:") :])
+ for line in r.text.splitlines()
+ if line.startswith("data:") and '"message_start"' in line
+ )
+ usage = start["message"]["usage"]
+ reported = (
+ usage["input_tokens"]
+ + usage["cache_read_input_tokens"]
+ + usage["cache_creation_input_tokens"]
+ )
+ # claude-opus-5 is a 1M model; 909k forwarded is past the trigger.
+ assert reported == int(1_000_000 * REPORT_FRACTION)
+
+
+def _reported_input(usage: dict) -> int:
+ return (
+ usage["input_tokens"]
+ + usage["cache_read_input_tokens"]
+ + usage["cache_creation_input_tokens"]
+ )
+
+
+def test_non_stream_json_gets_the_context_guard_nudge(monkeypatch) -> None:
+ """The plain non-stream JSON body is nudged near the wall, like the stream."""
+ from headroom.proxy.context_guard import REPORT_FRACTION, reset_learned_limits
+
+ reset_learned_limits()
+ app, _ = _app_and_outcomes(monkeypatch)
+ near_wall = {**_RESPONSE, "usage": {**_USAGE, "cache_read_input_tokens": 900_000}}
+ respx.post("https://api.anthropic.com/v1/messages").mock(
+ return_value=httpx.Response(200, json=near_wall)
+ )
+
+ r = _post(app, stream=False, tools=None)
+
+ assert r.status_code == 200
+ usage = r.json()["usage"]
+ # claude-opus-5 is a 1M model; 909k forwarded is past the trigger.
+ assert _reported_input(usage) == int(1_000_000 * REPORT_FRACTION)
+ assert usage["output_tokens"] == _USAGE["output_tokens"]
+
+
+def test_non_stream_json_is_untouched_with_the_guard_off(monkeypatch) -> None:
+ from headroom.proxy.context_guard import reset_learned_limits
+
+ reset_learned_limits()
+ monkeypatch.setenv("HEADROOM_CONTEXT_GUARD", "0")
+ app, _ = _app_and_outcomes(monkeypatch)
+ near_wall = {**_RESPONSE, "usage": {**_USAGE, "cache_read_input_tokens": 900_000}}
+ respx.post("https://api.anthropic.com/v1/messages").mock(
+ return_value=httpx.Response(200, json=near_wall)
+ )
+
+ r = _post(app, stream=False, tools=None)
+
+ assert r.status_code == 200
+ assert r.json()["usage"] == near_wall["usage"]
+
+
+def test_buffered_nudge_follows_the_client_beta_not_the_sticky_one(monkeypatch) -> None:
+ """Session-sticky merging can keep context-1m outbound after the client
+ dropped it. The learned cap keys on the forwarded beta; the client's gauge
+ is its own 200k window, so the nudge targets 95% of 200k, not of 1M."""
+ from headroom.proxy.context_guard import (
+ REPORT_FRACTION,
+ credential_scope,
+ note_prompt_too_long,
+ reset_learned_limits,
+ )
+
+ reset_learned_limits()
+ beta_1m = "context-1m-2025-08-07"
+ note_prompt_too_long(
+ "claude-sonnet-4-5",
+ beta_1m,
+ "prompt is too long: 213021 tokens > 200000 maximum",
+ scope=credential_scope("sk-sticky"),
+ )
+ app, _ = _app_and_outcomes(monkeypatch)
+ proxy = app.state.proxy
+ monkeypatch.setattr(proxy.anthropic_provider, "get_context_limit", lambda _model: 200_000)
+ payload = {
+ **_RESPONSE,
+ "model": "claude-sonnet-4-5",
+ "usage": {**_USAGE, "cache_read_input_tokens": 185_000 - 300 - 8197},
+ }
+
+ changed = proxy._context_guard_nudge_message(
+ payload,
+ model="claude-sonnet-4-5",
+ headers={"x-api-key": "sk-sticky", "anthropic-beta": beta_1m},
+ request_id="test-sticky-buffered",
+ client_beta="",
+ )
+
+ assert changed
+ assert _reported_input(payload["usage"]) == int(200_000 * REPORT_FRACTION)
diff --git a/tests/test_proxy/test_context_guard.py b/tests/test_proxy/test_context_guard.py
new file mode 100644
index 000000000..d5e20d105
--- /dev/null
+++ b/tests/test_proxy/test_context_guard.py
@@ -0,0 +1,694 @@
+"""Tests for the context-limit guard (headroom/proxy/context_guard.py)."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from headroom.proxy.context_guard import (
+ REPORT_FRACTION,
+ StreamUsageGuard,
+ believed_context_limit,
+ context_guard_enabled,
+ credential_scope,
+ effective_context_limit,
+ has_context_1m_beta,
+ note_prompt_too_long,
+ nudge_response_usage,
+ reset_learned_limits,
+)
+
+
[email protected](autouse=True)
+def _clean_learned_limits():
+ reset_learned_limits()
+ yield
+ reset_learned_limits()
+
+
+def _message_start_event(
+ input_tokens: int,
+ cache_read: int = 0,
+ cache_creation: int = 0,
+) -> bytes:
+ payload = {
+ "type": "message_start",
+ "message": {
+ "id": "msg_test",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-sonnet-4-5",
+ "content": [],
+ "usage": {
+ "input_tokens": input_tokens,
+ "cache_read_input_tokens": cache_read,
+ "cache_creation_input_tokens": cache_creation,
+ "output_tokens": 1,
+ },
+ },
+ }
+ return b"event: message_start\ndata: " + json.dumps(payload).encode() + b"\n\n"
+
+
+def _parse_usage(event_bytes: bytes) -> dict:
+ for line in event_bytes.split(b"\n"):
+ if line.startswith(b"data:"):
+ return json.loads(line[5:].strip())["message"]["usage"]
+ raise AssertionError("no data line in event")
+
+
+def _message_delta_event(
+ input_tokens: int,
+ cache_read: int = 0,
+ cache_creation: int = 0,
+) -> bytes:
+ payload = {
+ "type": "message_delta",
+ "delta": {"stop_reason": "end_turn", "stop_sequence": None},
+ "usage": {
+ "input_tokens": input_tokens,
+ "cache_read_input_tokens": cache_read,
+ "cache_creation_input_tokens": cache_creation,
+ "output_tokens": 42,
+ },
+ }
+ return b"event: message_delta\ndata: " + json.dumps(payload).encode() + b"\n\n"
+
+
+class TestBetaAndLimits:
+ def test_has_context_1m_beta_detects_token(self):
+ assert has_context_1m_beta("context-1m-2025-08-07")
+ assert has_context_1m_beta("oauth-2025-04-20, context-1m-2025-08-07")
+ assert not has_context_1m_beta("oauth-2025-04-20")
+ assert not has_context_1m_beta(None)
+ assert not has_context_1m_beta("")
+
+ def test_believed_limit_raised_by_1m_beta(self):
+ assert believed_context_limit(200_000, "context-1m-2025-08-07") == 1_000_000
+ assert believed_context_limit(200_000, "oauth-2025-04-20") == 200_000
+
+ def test_believed_limit_never_lowered(self):
+ assert believed_context_limit(2_000_000, "context-1m-2025-08-07") == 2_000_000
+
+ def test_effective_limit_optimistic_until_learned(self):
+ beta = "context-1m-2025-08-07"
+ assert effective_context_limit("claude-sonnet-4-5", 200_000, beta) == 1_000_000
+
+ def test_learned_limit_caps_effective(self):
+ beta = "context-1m-2025-08-07"
+ learned = note_prompt_too_long(
+ "claude-sonnet-4-5",
+ beta,
+ '{"error": {"message": "prompt is too long: 213021 tokens > 200000 maximum"}}',
+ )
+ assert learned == 200_000
+ assert effective_context_limit("claude-sonnet-4-5", 200_000, beta) == 200_000
+
+ def test_learned_limit_keyed_by_beta_presence(self):
+ # A limit learned WITHOUT the 1m beta must not clamp sessions that
+ # send it (their account may have real 1M access).
+ note_prompt_too_long(
+ "claude-sonnet-4-5",
+ None,
+ "prompt is too long: 201000 tokens > 200000 maximum",
+ )
+ assert (
+ effective_context_limit("claude-sonnet-4-5", 200_000, "context-1m-2025-08-07")
+ == 1_000_000
+ )
+
+ def test_note_prompt_too_long_ignores_other_errors(self):
+ assert note_prompt_too_long("m", None, "rate limit exceeded") is None
+ assert note_prompt_too_long("m", None, b"") is None
+
+ def test_note_prompt_too_long_accepts_bytes(self):
+ assert (
+ note_prompt_too_long("m", None, b"prompt is too long: 250000 tokens > 200000 maximum")
+ == 200_000
+ )
+
+ def test_enabled_by_default_and_kill_switch(self, monkeypatch):
+ monkeypatch.delenv("HEADROOM_CONTEXT_GUARD", raising=False)
+ assert context_guard_enabled()
+ monkeypatch.setenv("HEADROOM_CONTEXT_GUARD", "0")
+ assert not context_guard_enabled()
+ monkeypatch.setenv("HEADROOM_CONTEXT_GUARD", "false")
+ assert not context_guard_enabled()
+
+
+class TestLearnedLimitHygiene:
+ """Regressions for what a learned limit is allowed to do."""
+
+ def test_non_string_error_bodies_do_not_raise(self):
+ # Upstreams and gateways put null, objects and lists in the message
+ # slot. This runs while the client is still waiting for the real error
+ # body, so it must return empty-handed rather than raise over it.
+ for body in (None, {"message": {"nested": True}}, ["prompt is too long"], 42):
+ assert note_prompt_too_long("claude-sonnet-4-5", None, body) is None
+ assert effective_context_limit("claude-sonnet-4-5", 200_000, None) == 200_000
+
+ def test_out_of_band_maximum_is_not_learned(self):
+ # Same wire shape, different subject: gateways report per-request quota
+ # and per-key throttles as "prompt is too long: N > M". A limit that is
+ # not a context window must not clamp the budget for the process.
+ assert (
+ note_prompt_too_long(
+ "claude-sonnet-4-5", None, "prompt is too long: 900 tokens > 1 maximum"
+ )
+ is None
+ )
+ assert (
+ note_prompt_too_long(
+ "claude-sonnet-4-5",
+ None,
+ "prompt is too long: 99000000 tokens > 50000000 maximum",
+ )
+ is None
+ )
+ assert effective_context_limit("claude-sonnet-4-5", 200_000, None) == 200_000
+
+ def test_learned_limit_is_scoped_to_one_credential(self):
+ # One tenant's capped key on a shared proxy must not clamp another's.
+ note_prompt_too_long(
+ "claude-sonnet-4-5",
+ None,
+ "prompt is too long: 213021 tokens > 200000 maximum",
+ scope=credential_scope("sk-tenant-a"),
+ )
+ assert (
+ effective_context_limit(
+ "claude-sonnet-4-5", 400_000, None, scope=credential_scope("sk-tenant-a")
+ )
+ == 200_000
+ )
+ assert (
+ effective_context_limit(
+ "claude-sonnet-4-5", 400_000, None, scope=credential_scope("sk-tenant-b")
+ )
+ == 400_000
+ )
+
+ def test_credential_scope_never_echoes_the_credential(self):
+ scope = credential_scope("sk-ant-secret-value")
+ assert "secret" not in scope
+ assert scope and scope != credential_scope("sk-ant-other-value")
+ assert credential_scope(None) == credential_scope("") == ""
+
+ def test_learned_limit_expires(self, monkeypatch):
+ # Entitlements get provisioned and gateways get repointed; a lesson
+ # from an hour ago must not hold the budget down forever.
+ from headroom.proxy import context_guard
+
+ clock = {"now": 1_000.0}
+ monkeypatch.setattr(context_guard.time, "monotonic", lambda: clock["now"])
+ note_prompt_too_long(
+ "claude-sonnet-4-5", None, "prompt is too long: 213021 tokens > 200000 maximum"
+ )
+ assert effective_context_limit("claude-sonnet-4-5", 400_000, None) == 200_000
+ clock["now"] += context_guard._LEARNED_LIMIT_TTL_SECONDS + 1
+ assert effective_context_limit("claude-sonnet-4-5", 400_000, None) == 400_000
+
+
+class TestNonStreamingNudge:
+ """The buffered path raises the same budget, so it owes the same warning."""
+
+ def test_near_limit_body_is_nudged(self):
+ payload = {
+ "type": "message",
+ "usage": {
+ "input_tokens": 185_000,
+ "cache_read_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
+ "output_tokens": 7,
+ },
+ }
+ assert nudge_response_usage(payload, believed_limit=200_000, effective_limit=200_000)
+ assert payload["usage"]["input_tokens"] == int(200_000 * REPORT_FRACTION)
+
+ def test_far_from_limit_body_is_untouched(self):
+ payload = {"type": "message", "usage": {"input_tokens": 5_000, "output_tokens": 7}}
+ assert not nudge_response_usage(payload, believed_limit=200_000, effective_limit=200_000)
+ assert payload["usage"]["input_tokens"] == 5_000
+
+ def test_bodies_without_usage_are_untouched(self):
+ for payload in ({"type": "message"}, {"type": "error"}, "not a body", None):
+ assert not nudge_response_usage(
+ payload, believed_limit=200_000, effective_limit=200_000
+ )
+
+
+class TestStreamUsageGuard:
+ def test_below_trigger_passes_through_byte_identical(self):
+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)
+ event = _message_start_event(50_000, cache_read=100_000)
+ assert guard.feed(event) == event
+ # Inert afterwards: later chunks untouched even if they look like events.
+ tail = b"event: content_block_delta\ndata: {}\n\n"
+ assert guard.feed(tail) == tail
+
+ def test_above_trigger_inflates_to_report_fraction_of_believed(self):
+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)
+ # 185k total forwarded = 92.5% of the real 200k window.
+ event = _message_start_event(5_000, cache_read=170_000, cache_creation=10_000)
+ out = guard.feed(event)
+ usage = _parse_usage(out)
+ target_total = int(1_000_000 * REPORT_FRACTION)
+ assert (
+ usage["input_tokens"]
+ + usage["cache_read_input_tokens"]
+ + usage["cache_creation_input_tokens"]
+ == target_total
+ )
+ # Cache components are never touched — only input_tokens absorbs the nudge.
+ assert usage["cache_read_input_tokens"] == 170_000
+ assert usage["cache_creation_input_tokens"] == 10_000
+ assert usage["output_tokens"] == 1
+
+ def test_same_limits_nudges_gauge_over_compact_threshold(self):
+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)
+ event = _message_start_event(185_000)
+ usage = _parse_usage(guard.feed(event))
+ assert usage["input_tokens"] == int(200_000 * REPORT_FRACTION)
+
+ def test_never_deflates(self):
+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)
+ event = _message_start_event(199_000) # above the 95% report target
+ assert guard.feed(event) == event
+
+ def test_split_chunks_across_event_boundary(self):
+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)
+ event = _message_start_event(190_000)
+ first, second = event[:40], event[40:]
+ assert guard.feed(first) == b"" # held back: no complete event yet
+ out = guard.feed(second)
+ assert _parse_usage(out)["input_tokens"] == int(1_000_000 * REPORT_FRACTION)
+
+ def test_crlf_framed_events_are_released_and_rewritten(self):
+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)
+ event = _message_start_event(190_000).replace(b"\n", b"\r\n")
+ out = guard.feed(event)
+ # Released at its own terminator, not held until the buffer cap.
+ assert out.endswith(b"\r\n\r\n")
+ assert b"\n" not in out.replace(b"\r\n", b"")
+ assert _parse_usage(out)["input_tokens"] == int(1_000_000 * REPORT_FRACTION)
+
+ def test_a_chunk_over_the_buffer_cap_still_gets_its_message_start_read(self):
+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)
+ filler = b"event: content_block_delta\ndata: " + b"x" * (300 * 1024) + b"\n\n"
+ out = guard.feed(_message_start_event(190_000) + filler)
+ assert _parse_usage(out)["input_tokens"] == int(1_000_000 * REPORT_FRACTION)
+ assert out.endswith(filler)
+
+ def test_ping_events_pass_through_before_message_start(self):
+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)
+ ping = b'event: ping\ndata: {"type": "ping"}\n\n'
+ assert guard.feed(ping) == ping
+ out = guard.feed(_message_start_event(190_000))
+ assert _parse_usage(out)["input_tokens"] == int(1_000_000 * REPORT_FRACTION)
+
+ def test_non_message_start_first_event_disarms(self):
+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)
+ error_event = b'event: error\ndata: {"type": "error"}\n\n'
+ assert guard.feed(error_event) == error_event
+ # Even a later message_start is untouched (protocol says it is first).
+ event = _message_start_event(190_000)
+ assert guard.feed(event) == event
+
+ def test_malformed_json_passes_through(self):
+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)
+ event = b"event: message_start\ndata: {not json\n\n"
+ assert guard.feed(event) == event
+
+ def test_buffer_cap_flushes_verbatim(self):
+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)
+ blob = b"x" * (300 * 1024) # no event boundary anywhere
+ out = guard.feed(blob)
+ assert out == blob
+ assert guard.feed(b"more") == b"more" # inert afterwards
+
+ def test_flush_returns_held_bytes(self):
+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)
+ partial = b"event: message_start\ndata: {"
+ assert guard.feed(partial) == b""
+ assert guard.flush() == partial
+
+ def test_unusable_limits_make_guard_inert(self):
+ guard = StreamUsageGuard(believed_limit=0, effective_limit=200_000)
+ event = _message_start_event(190_000)
+ assert guard.feed(event) == event
+
+ def test_rewritten_event_is_valid_sse(self):
+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)
+ out = guard.feed(_message_start_event(185_000))
+ assert out.startswith(b"event: message_start\n")
+ assert out.endswith(b"\n\n")
+ payload = _parse_usage(out) # parses => data line is intact JSON
+ assert payload["input_tokens"] > 185_000
+
+ def test_armed_guard_rewrites_final_message_delta(self):
+ # Clients merge the final cumulative-usage message_delta over
+ # message_start (verified live with Claude Code 2026-08-12), so an
+ # armed guard must rewrite both or the nudge loses the merge.
+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)
+ guard.feed(_message_start_event(180_000, cache_creation=5_000))
+ content = b'event: content_block_delta\ndata: {"type":"content_block_delta"}\n\n'
+ assert guard.feed(content) == content # armed but passthrough
+ delta = _message_delta_event(180_000, cache_creation=5_000)
+ out = guard.feed(delta)
+ usage = json.loads(out.split(b"data: ")[1])["usage"]
+ assert usage["input_tokens"] + usage["cache_creation_input_tokens"] == int(
+ 200_000 * REPORT_FRACTION
+ )
+ assert usage["output_tokens"] == 42
+ # Inert afterwards.
+ stop = b'event: message_stop\ndata: {"type":"message_stop"}\n\n'
+ assert guard.feed(stop) == stop
+
+ def test_output_only_message_delta_untouched(self):
+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)
+ guard.feed(_message_start_event(185_000))
+ delta = (
+ b'event: message_delta\ndata: {"type":"message_delta","usage":{"output_tokens":9}}\n\n'
+ )
+ assert guard.feed(delta) == delta
+
+ def test_disarmed_guard_leaves_message_delta_alone(self):
+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)
+ guard.feed(_message_start_event(50_000)) # below trigger -> inert
+ delta = _message_delta_event(50_000)
+ assert guard.feed(delta) == delta
+
+ def test_message_delta_never_deflates(self):
+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)
+ guard.feed(_message_start_event(185_000))
+ delta = _message_delta_event(199_000) # already above the 190k target
+ assert guard.feed(delta) == delta
+
+
+class TestSseFraming:
+ """The scan reads the SSE envelope, never the payload text."""
+
+ def _armed_guard(self) -> StreamUsageGuard:
+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)
+ guard.feed(_message_start_event(190_000))
+ return guard
+
+ def test_assistant_text_naming_an_event_does_not_disarm(self):
+ # The model writing about message_delta -- explaining SSE, quoting a
+ # log line -- used to end the scan, so the final cumulative usage went
+ # out un-nudged and the client's gauge snapped back.
+ guard = self._armed_guard()
+ chatter = (
+ b"event: content_block_delta\ndata: "
+ + json.dumps(
+ {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {
+ "type": "text_delta",
+ "text": "the final message_delta carries cumulative usage",
+ },
+ }
+ ).encode()
+ + b"\n\n"
+ )
+ assert guard.feed(chatter) == chatter
+ nudged = guard.feed(_message_delta_event(190_000))
+ assert json.loads(nudged.split(b"data: ", 1)[1])["usage"]["input_tokens"] == int(
+ 200_000 * REPORT_FRACTION
+ )
+
+ def test_keepalive_frames_before_message_start_do_not_disarm(self):
+ # An intermediary shim (nginx, a corporate egress proxy) injects SSE
+ # comments and retry fields. They are not events, so they must not be
+ # read as "a protocol we don't recognize" and switch the guard off.
+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)
+ for keepalive in (b": keepalive\n\n", b"retry: 3000\n\n", b":\n\n"):
+ assert guard.feed(keepalive) == keepalive
+ nudged = guard.feed(_message_start_event(190_000))
+ assert _parse_usage(nudged)["input_tokens"] == int(200_000 * REPORT_FRACTION)
+
+
+class TestStreamResponseIntegration:
+ """The guard wired through _stream_response with a mocked upstream."""
+
+ def _create_mock_proxy(self):
+ from unittest.mock import AsyncMock, MagicMock
+
+ import httpx
+
+ from headroom.proxy.server import HeadroomProxy
+
+ proxy = object.__new__(HeadroomProxy)
+ proxy.http_client = MagicMock(spec=httpx.AsyncClient)
+ proxy.metrics = MagicMock()
+ proxy.metrics.record_request = AsyncMock(return_value=None)
+ proxy.metrics.record_failed = AsyncMock(return_value=None)
+ proxy.cost_tracker = MagicMock()
+ proxy.cost_tracker.estimate_cost.return_value = 0.001
+ proxy.cost_tracker.record_request.return_value = None
+ proxy.stats = {
+ "requests_total": 0,
+ "requests_optimized": 0,
+ "tokens": {"original": 0, "optimized": 0, "saved": 0},
+ "cost": {"total_usd": 0, "savings_usd": 0},
+ "errors": 0,
+ "active_requests": 0,
+ "requests_per_model": {},
+ }
+ proxy.memory_manager = None
+ proxy._config = MagicMock()
+ proxy._config.memory_enabled = False
+ proxy._config.ccr_inject_tool = False
+ proxy._config.retry_max_attempts = 3
+ proxy._config.retry_base_delay_ms = 0
+ proxy._config.retry_max_delay_ms = 0
+ proxy.config = proxy._config
+ proxy._parse_sse_usage_from_buffer = MagicMock(return_value=None)
+ proxy.memory_handler = None
+ proxy.anthropic_provider = MagicMock()
+ proxy.anthropic_provider.get_context_limit.return_value = 200_000
+ return proxy
+
+ def _mock_upstream(self, sse_bytes: bytes, status_code: int = 200):
+ from unittest.mock import AsyncMock
+
+ import httpx
+
+ mock_response = AsyncMock()
+ mock_response.headers = httpx.Headers({"content-type": "text/event-stream"})
+ mock_response.status_code = status_code
+
+ async def aiter_bytes():
+ yield sse_bytes
+
+ mock_response.aiter_bytes = aiter_bytes
+ mock_response.aclose = AsyncMock()
+ mock_response.aread = AsyncMock(return_value=sse_bytes)
+ return mock_response
+
+ async def _run(self, proxy, mock_response, headers, **kwargs):
+ from unittest.mock import AsyncMock, MagicMock
+
+ proxy.http_client.build_request = MagicMock(return_value=MagicMock())
+ proxy.http_client.send = AsyncMock(return_value=mock_response)
+ return await proxy._stream_response(
+ url="https://api.anthropic.com/v1/messages",
+ headers=headers,
+ body={
+ "model": "claude-sonnet-4-5",
+ "max_tokens": 100,
+ "stream": True,
+ "messages": [{"role": "user", "content": "hi"}],
+ },
+ provider="anthropic",
+ model="claude-sonnet-4-5",
+ request_id="test-guard",
+ original_tokens=10,
+ optimized_tokens=10,
+ tokens_saved=0,
+ transforms_applied=[],
+ tags={},
+ optimization_latency=0.0,
+ **kwargs,
+ )
+
+ @pytest.mark.asyncio
+ async def test_near_limit_message_start_is_nudged_on_the_wire(self):
+ proxy = self._create_mock_proxy()
+ # 185k = above the 90% trigger, below the 95% report target — the
+ # assertions below can only pass if the rewrites actually happened.
+ upstream = self._mock_upstream(
+ _message_start_event(185_000)
+ + b'event: content_block_delta\ndata: {"type":"content_block_delta"}\n\n'
+ + _message_delta_event(185_000)
+ + b'event: message_stop\ndata: {"type":"message_stop"}\n\n'
+ )
+ result = await self._run(proxy, upstream, {"x-api-key": "sk-test"})
+ client_bytes = b"".join([chunk async for chunk in result.body_iterator])
+ events = client_bytes.split(b"\n\n")
+ usage = _parse_usage(events[0] + b"\n\n")
+ assert usage["input_tokens"] == int(200_000 * REPORT_FRACTION)
+ # The final cumulative-usage message_delta is nudged too — clients
+ # merge it over message_start, so it must agree.
+ delta_usage = json.loads(events[2].split(b"data: ")[1])["usage"]
+ assert delta_usage["input_tokens"] == int(200_000 * REPORT_FRACTION)
+ assert delta_usage["output_tokens"] == 42
+ # Everything else is untouched.
+ assert (
+ b'event: content_block_delta\ndata: {"type":"content_block_delta"}\n\n' in client_bytes
+ )
+ assert b'event: message_stop\ndata: {"type":"message_stop"}\n\n' in client_bytes
+
+ @pytest.mark.asyncio
+ async def test_far_from_limit_stream_is_byte_identical(self):
+ proxy = self._create_mock_proxy()
+ sse = (
+ _message_start_event(50_000) + b'event: message_stop\ndata: {"type":"message_stop"}\n\n'
+ )
+ result = await self._run(proxy, self._mock_upstream(sse), {"x-api-key": "sk-test"})
+ client_bytes = b"".join([chunk async for chunk in result.body_iterator])
+ assert client_bytes == sse
+
+ @pytest.mark.asyncio
+ async def test_kill_switch_disables_nudge(self, monkeypatch):
+ monkeypatch.setenv("HEADROOM_CONTEXT_GUARD", "0")
+ proxy = self._create_mock_proxy()
+ sse = _message_start_event(190_000)
+ result = await self._run(proxy, self._mock_upstream(sse), {"x-api-key": "sk-test"})
+ client_bytes = b"".join([chunk async for chunk in result.body_iterator])
+ assert client_bytes == sse
+
+ @pytest.mark.asyncio
+ async def test_streaming_400_learns_real_limit(self):
+ proxy = self._create_mock_proxy()
+ error_body = json.dumps(
+ {
+ "type": "error",
+ "error": {
+ "type": "invalid_request_error",
+ "message": "prompt is too long: 213021 tokens > 200000 maximum",
+ },
+ }
+ ).encode()
+ upstream = self._mock_upstream(error_body, status_code=400)
+ upstream.headers = __import__("httpx").Headers({"content-type": "application/json"})
+ result = await self._run(
+ proxy, upstream, {"x-api-key": "sk-test", "anthropic-beta": "context-1m-2025-08-07"}
+ )
+ assert result.status_code == 400
+ # Learned against the credential that hit the wall, not process-wide.
+ scope = credential_scope("sk-test")
+ assert (
+ effective_context_limit(
+ "claude-sonnet-4-5", 200_000, "context-1m-2025-08-07", scope=scope
+ )
+ == 200_000
+ )
+ # A different key on the same proxy keeps its own (optimistic) budget.
+ assert (
+ effective_context_limit(
+ "claude-sonnet-4-5",
+ 200_000,
+ "context-1m-2025-08-07",
+ scope=credential_scope("sk-other"),
+ )
+ == 1_000_000
+ )
+
+ @pytest.mark.asyncio
+ async def test_sticky_1m_beta_does_not_inflate_a_200k_client_gauge(self):
+ """The client dropped context-1m; session-sticky merging kept it outbound.
+
+ The learned cap still keys on the forwarded beta, but the client's
+ gauge is its own 200k window, so the nudge must target 95% of 200k.
+ """
+ beta_1m = "context-1m-2025-08-07"
+ note_prompt_too_long(
+ "claude-sonnet-4-5",
+ beta_1m,
+ "prompt is too long: 213021 tokens > 200000 maximum",
+ scope=credential_scope("sk-sticky"),
+ )
+ proxy = self._create_mock_proxy()
+ sse = _message_start_event(185_000) + _message_delta_event(185_000)
+ result = await self._run(
+ proxy,
+ self._mock_upstream(sse),
+ {"x-api-key": "sk-sticky", "anthropic-beta": beta_1m},
+ client_beta="",
+ )
+ client_bytes = b"".join([chunk async for chunk in result.body_iterator])
+ events = client_bytes.split(b"\n\n")
+ assert _parse_usage(events[0] + b"\n\n")["input_tokens"] == int(200_000 * REPORT_FRACTION)
+ delta_usage = json.loads(events[1].split(b"data: ")[1])["usage"]
+ assert delta_usage["input_tokens"] == int(200_000 * REPORT_FRACTION)
+
+ @pytest.mark.asyncio
+ async def test_guard_nudges_what_the_memory_filter_releases(self):
+ """The guard runs after the memory filter, on the client-bound frames.
+
+ The filter withholds the proxy's memory call and holds message_delta
+ until the turn's outcome is known; that held delta is still the one
+ the client merges over message_start, so it must come out nudged.
+ """
+ from unittest.mock import AsyncMock, MagicMock
+
+ def frame(payload: dict) -> bytes:
+ return f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n".encode()
+
+ def tool_block(index: int, tool_id: str, name: str) -> bytes:
+ return (
+ frame(
+ {
+ "type": "content_block_start",
+ "index": index,
+ "content_block": {"type": "tool_use", "id": tool_id, "name": name},
+ }
+ )
+ + frame(
+ {
+ "type": "content_block_delta",
+ "index": index,
+ "delta": {"type": "input_json_delta", "partial_json": "{}"},
+ }
+ )
+ + frame({"type": "content_block_stop", "index": index})
+ )
+
+ delta = json.loads(_message_delta_event(185_000).split(b"data: ")[1])
+ delta["delta"]["stop_reason"] = "tool_use"
+ sse = (
+ _message_start_event(185_000)
+ + tool_block(0, "toolu_mem", "memory_save")
+ + tool_block(1, "toolu_bash", "Bash")
+ + frame(delta)
+ + frame({"type": "message_stop"})
+ )
+ proxy = self._create_mock_proxy()
+ proxy.memory_handler = MagicMock()
+ proxy.memory_handler.handle_memory_tool_calls = AsyncMock(
+ return_value=[{"type": "tool_result", "tool_use_id": "toolu_mem", "content": "ok"}]
+ )
+ result = await self._run(
+ proxy,
+ self._mock_upstream(sse),
+ {"x-api-key": "sk-test"},
+ memory_user_id="user-1",
+ server_memory_tool_names=frozenset({"memory_save"}),
+ )
+ client_bytes = b"".join([chunk async for chunk in result.body_iterator])
+ events = [
+ json.loads(line[len(b"data: ") :])
+ for line in client_bytes.split(b"\n")
+ if line.startswith(b"data: ")
+ ]
+
+ assert b"memory_save" not in client_bytes
+ assert events[0]["message"]["usage"]["input_tokens"] == int(200_000 * REPORT_FRACTION)
+ final = [e for e in events if e["type"] == "message_delta"]
+ assert len(final) == 1
+ assert final[0]["delta"]["stop_reason"] == "tool_use"
+ assert final[0]["usage"]["input_tokens"] == int(200_000 * REPORT_FRACTION)
+ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()