Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): break the compact-every-turn loop near the real context window (#2942)
gglucass76cceeb48728
diff --git a/headroom/proxy/context_guard.py b/headroom/proxy/context_guard.pynew file mode 100644index 000000000..b1bd1802e--- /dev/null+++ b/headroom/proxy/context_guard.py@@ -0,0 +1,516 @@+"""Context-limit guard: keep client auto-compaction working under compression.++Compression makes the usage reported back to the client reflect the+*forwarded* (compressed) request, not the client's raw transcript. Clients+whose auto-compaction keys off reported usage (Claude Code) therefore defer+compaction — the selling point — but with no floor: the raw transcript can+grow until even the compressed request exceeds the model's real context+window. From that point every turn fails with a 400 ``prompt is too long``,+the client force-compacts on each error, and the session degrades into a+compact-every-other-prompt loop (field report 2026-08-12).++Three cooperating pieces, all passthrough until the danger zone:++1. ``effective_context_limit`` — the window the forwarded request is really+ subject to: the model's registry limit, raised to 1M when the client sent+ a ``context-1m`` beta token, capped by any limit *learned* from an actual+ ``prompt is too long`` error (piece 2). Also used for compression+ pressure, so 1M sessions stop being treated as 5x over budget.+2. ``note_prompt_too_long`` — parses ``prompt is too long: N tokens > M+ maximum`` from upstream 400 bodies and records M per (model, 1m-beta)+ key. This is how a client that *believes* 1M but whose account is capped+ at 200k stops looping after exactly one error.+3. ``StreamUsageGuard`` — an SSE transformer for the streamed bytes only:+ when the ``message_start`` usage total reaches ``TRIGGER_FRACTION`` of+ the effective limit, it inflates the reported ``input_tokens`` — in both+ ``message_start`` and the final cumulative-usage ``message_delta``, which+ clients merge over the former — so the client's gauge reads+ ``REPORT_FRACTION`` of the window the client believes in, which trips+ the client's own graceful compaction before the 400 wall. Savings+ metrics and telemetry parse the original upstream bytes, never the+ nudged ones.++The nudged number is deliberately unmarked on the wire. It exists to be read+by the client's own compaction gauge, which is a third-party parser we do not+control, so anything added beside it is either ignored or a wire-shape change+against a client we cannot test. Nothing inside headroom needs the marker:+savings, telemetry and billing all read the original upstream bytes, never the+client-bound ones, and each rewrite emits ``event=context_guard_nudge`` with+the forwarded and reported totals for reconciliation.++Kill switch: ``HEADROOM_CONTEXT_GUARD=0``.+"""++from __future__ import annotations++import hashlib+import json+import logging+import re+import time+from typing import Any++from headroom.proxy.sse_byte_buffer_policy import find_sse_event_terminator++logger = logging.getLogger("headroom.proxy")++# Forwarded-usage fraction of the *effective* (real) limit at which the nudge+# arms. 0.90 leaves one normal turn of headroom before the hard wall.+TRIGGER_FRACTION = 0.90+# Reported-usage fraction of the *believed* limit the nudge inflates to.+# Claude Code auto-compacts around 92%; 0.95 clears that with margin.+REPORT_FRACTION = 0.95++_CONTEXT_1M_PREFIX = "context-1m"++_PROMPT_TOO_LONG_RE = re.compile(+ r"prompt is too long:\s*(\d+)\s*tokens?\s*>\s*(\d+)\s*maximum", re.IGNORECASE+)++# A learned maximum is believed only inside this band. Below the floor it is+# not a context window: gateways return the same "prompt is too long: N > M"+# shape for per-request quota and per-key throttles, and one such number would+# clamp every later request on that model for the life of the process. Above+# the ceiling it cannot cap anything anyway.+_MIN_LEARNED_LIMIT = 8_000+_MAX_LEARNED_LIMIT = 10_000_000++# A learned limit is evidence about one account at one moment, not a fact about+# the model: entitlements get provisioned, gateways get repointed. It expires so+# a stale cap cannot hold a budget down forever -- the cost of expiry is at most+# one more prompt-too-long error, which is what taught it the first time.+_LEARNED_LIMIT_TTL_SECONDS = 3600.0++# (model, has_1m_beta, credential scope) -> (maximum, learned-at monotonic time).+#+# The 1m-beta dimension matters: an account with real 1M access must not inherit+# a 200k limit learned from a request that never sent the beta. The credential+# scope matters for the same reason one step out -- this dict is process-global,+# and on a shared proxy one tenant's capped key would otherwise clamp the budget+# of every other tenant on that model.+_learned_limits: dict[tuple[str, bool, str], tuple[int, float]] = {}++# Give up scanning for message_start beyond this many buffered bytes.+_MAX_GUARD_BUFFER = 256 * 1024+++def context_guard_enabled() -> bool:+ """True unless explicitly disabled via HEADROOM_CONTEXT_GUARD=0."""+ from headroom.proxy import runtime_env++ return (runtime_env.getenv("HEADROOM_CONTEXT_GUARD", "1") or "1").strip().lower() not in (+ "0",+ "false",+ "no",+ "off",+ )+++def has_context_1m_beta(beta_header: str | None) -> bool:+ """True when the anthropic-beta header carries a context-1m token."""+ if not beta_header:+ return False+ return any(+ token.strip().lower().startswith(_CONTEXT_1M_PREFIX) for token in beta_header.split(",")+ )+++def believed_context_limit(model_limit: int, beta_header: str | None) -> int:+ """The context window the *client* thinks it has."""+ if has_context_1m_beta(beta_header):+ return max(model_limit, 1_000_000)+ return model_limit+++def credential_scope(credential: str | None) -> str:+ """A stable, non-reversible tag for the upstream credential.++ Only ever a truncated digest: the credential itself is never stored, never+ logged, and never compared outside this function.+ """+ if not credential:+ return ""+ return hashlib.sha256(credential.strip().encode("utf-8", "ignore")).hexdigest()[:16]+++def credential_scope_from_headers(headers: Any) -> str:+ """Credential scope for an outbound header mapping, best effort.++ Never raises: an unusable header mapping just means the unscoped bucket,+ which is the behaviour of a single-tenant proxy anyway.+ """+ try:+ for name in ("x-api-key", "authorization"):+ value = headers.get(name) or headers.get(name.title())+ if isinstance(value, str) and value:+ return credential_scope(value)+ except Exception:+ return ""+ return ""+++def _learned_key(model: str, beta_header: str | None, scope: str) -> tuple[str, bool, str]:+ return (model, has_context_1m_beta(beta_header), scope)+++def _learned_limit(key: tuple[str, bool, str]) -> int | None:+ """The unexpired learned maximum for ``key``, dropping it once stale."""+ entry = _learned_limits.get(key)+ if entry is None:+ return None+ maximum, learned_at = entry+ if time.monotonic() - learned_at >= _LEARNED_LIMIT_TTL_SECONDS:+ _learned_limits.pop(key, None)+ return None+ return maximum+++def effective_context_limit(+ model: str,+ model_limit: int,+ beta_header: str | None,+ *,+ scope: str = "",+) -> int:+ """The window the forwarded request is actually subject to.++ Optimistically the believed limit; capped by a limit learned from a real+ ``prompt is too long`` error for the same (model, beta, credential) shape,+ and only while that lesson is still fresh.+ """+ believed = believed_context_limit(model_limit, beta_header)+ learned = _learned_limit(_learned_key(model, beta_header, scope))+ if learned is not None:+ return min(believed, learned)+ return believed+++def note_prompt_too_long(+ model: str,+ beta_header: str | None,+ error_text: Any,+ *,+ scope: str = "",+) -> int | None:+ """Learn the real context limit from an upstream prompt-too-long error.++ Returns the learned maximum, or None when there is nothing to learn.++ This runs on the error path, where the client is still waiting for the real+ upstream body, so it accepts whatever the body turned out to be: upstreams+ and gateways send ``"message": null``, nested objects, and lists in the+ slot this reads, and a diagnostic helper must not turn one of those into an+ exception that replaces the error the caller was about to forward.+ """+ if isinstance(error_text, bytes | bytearray):+ error_text = bytes(error_text).decode("utf-8", errors="replace")+ if not isinstance(error_text, str):+ return None+ match = _PROMPT_TOO_LONG_RE.search(error_text)+ if not match:+ return None+ maximum = int(match.group(2))+ if not _MIN_LEARNED_LIMIT <= maximum <= _MAX_LEARNED_LIMIT:+ # Not a context window. Learning it would clamp the budget on a number+ # that never described one.+ logger.debug(+ "event=context_guard_limit_out_of_band model=%s limit=%d (ignored)",+ model,+ maximum,+ )+ return None+ key = _learned_key(model, beta_header, scope)+ previous = _learned_limit(key)+ _learned_limits[key] = (maximum, time.monotonic())+ if previous != maximum:+ logger.warning(+ "event=context_guard_learned_limit model=%s context_1m=%s limit=%d "+ "attempted=%s (client compaction gauge will be nudged near this limit)",+ model,+ key[1],+ maximum,+ match.group(1),+ )+ return maximum+++def reset_learned_limits() -> None:+ """Test hook: clear limits learned from prompt-too-long errors."""+ _learned_limits.clear()+++def _data_line(payload: dict[str, Any], original: bytes) -> bytes:+ """A rewritten ``data:`` line keeping the original's CRLF ending."""+ eol = b"\r" if original.endswith(b"\r") else b""+ return b"data: " + json.dumps(payload, separators=(",", ":")).encode() + eol+++def _sse_event_type(event: bytes) -> bytes | None:+ """The event type of one SSE frame, or ``None`` when it carries none.++ Read from the ``event:`` line, not matched as a substring of the frame: a+ substring match reads the *payload* too, so an assistant that writes the+ words ``message_delta`` into its own reply ended the scan and lost the+ final nudge.++ ``None`` means the frame is not part of the message protocol at all --+ an SSE comment, a ``retry:`` field, the keepalives an intermediary shim+ injects. Those pass through without arming or disarming anything; treating+ one as an unknown event disabled the guard for the whole stream.+ """+ saw_data = False+ for line in event.split(b"\n"):+ if line.startswith(b"event:"):+ return line[len(b"event:") :].strip()+ if line.startswith(b"data:"):+ saw_data = True+ if not saw_data:+ return None+ # A data frame with no event: line defaults to the type inside the payload.+ for line in event.split(b"\n"):+ if not line.startswith(b"data:"):+ continue+ try:+ payload = json.loads(line[len(b"data:") :].strip())+ except Exception:+ return b""+ value = payload.get("type") if isinstance(payload, dict) else None+ return value.encode() if isinstance(value, str) else b""+ return b""+++def nudged_input_tokens(+ usage: dict[str, Any],+ *,+ believed_limit: int,+ effective_limit: int,+) -> int | None:+ """The inflated ``input_tokens`` for this usage, or ``None`` to leave it be.++ One implementation for every client-bound surface -- ``message_start``, the+ final ``message_delta``, and the non-streaming JSON body -- so a request+ cannot get the raised compression budget on one path and the un-nudged+ gauge on another.+ """+ if believed_limit <= 0 or effective_limit <= 0:+ return None+ input_tokens = int(usage.get("input_tokens") or 0)+ cache_read = int(usage.get("cache_read_input_tokens") or 0)+ cache_creation = int(usage.get("cache_creation_input_tokens") or 0)+ total = input_tokens + cache_read + cache_creation+ if total < TRIGGER_FRACTION * effective_limit:+ return None+ target_total = int(believed_limit * REPORT_FRACTION)+ if total >= target_total:+ # Already reads as full to the client -- never deflate.+ return None+ return target_total - cache_read - cache_creation+++def nudge_response_usage(+ resp_json: Any,+ *,+ believed_limit: int,+ effective_limit: int,+ request_id: str = "",+) -> bool:+ """Nudge the usage of a non-streaming Anthropic message body, in place.++ The buffered path raises the compression budget exactly like the streaming+ one, so it has to move the client's gauge exactly like the streaming one+ too; otherwise a non-streaming client gets the larger forwarded request and+ none of the warning that it is near the wall.++ Returns True when the body was modified.+ """+ if not isinstance(resp_json, dict):+ return False+ usage = resp_json.get("usage")+ if not isinstance(usage, dict):+ return False+ new_input = nudged_input_tokens(+ usage, believed_limit=believed_limit, effective_limit=effective_limit+ )+ if new_input is None:+ return False+ cache_read = int(usage.get("cache_read_input_tokens") or 0)+ cache_creation = int(usage.get("cache_creation_input_tokens") or 0)+ forwarded_total = int(usage.get("input_tokens") or 0) + cache_read + cache_creation+ usage["input_tokens"] = new_input+ logger.warning(+ "[%s] event=context_guard_nudge path=non_streaming forwarded_total=%d "+ "effective_limit=%d reported_total=%d believed_limit=%d (nudging client to "+ "compact before the prompt-too-long wall)",+ request_id,+ forwarded_total,+ effective_limit,+ new_input + cache_read + cache_creation,+ believed_limit,+ )+ return True+++class StreamUsageGuard:+ """Rewrites near-the-wall usage in the client-bound SSE bytes.++ Feed every chunk headed to the client through :meth:`feed`; call+ :meth:`flush` when the stream ends. The first ``message_start`` decides+ whether the nudge is active. When it is not (the common case), the guard+ goes inert immediately and chunks pass through untouched — steady-state+ overhead is one ``if``. When it is, the guard rewrites the+ ``message_start`` usage AND stays armed for the final ``message_delta``:+ since mid-2025 that event carries cumulative input/cache usage and+ clients (verified live with Claude Code 2026-08-12) let it override the+ ``message_start`` values, so nudging only the first event loses the+ merge. Events are released as soon as their ``\\n\\n`` terminator+ arrives, so no latency is added beyond SSE's own event framing.+ """++ def __init__(+ self,+ *,+ believed_limit: int,+ effective_limit: int,+ request_id: str = "",+ ) -> None:+ self._believed_limit = believed_limit+ self._effective_limit = effective_limit+ self._request_id = request_id+ self._buf = bytearray()+ self._seen_message_start = False+ self._target_total: int | None = None+ self._start_cache_read = 0+ self._start_cache_creation = 0+ # Inert immediately when limits are unusable.+ self._done = believed_limit <= 0 or effective_limit <= 0++ def feed(self, chunk: bytes) -> bytes:+ if self._done:+ return chunk+ self._buf.extend(chunk)+ out = bytearray()+ while not self._done:+ # LF or CRLF framing: a CRLF stream never contains ``\n\n`` and+ # was held until the buffer cap, so it stopped streaming.+ terminator = find_sse_event_terminator(self._buf)+ if terminator is None:+ break+ end = terminator[0] + terminator[1]+ event = bytes(self._buf[:end])+ del self._buf[:end]+ out += self._process_event(event)+ # Capped on what is still unframed, so one large chunk carrying+ # message_start is read before the cap can disarm the guard.+ if self._done or len(self._buf) > _MAX_GUARD_BUFFER:+ out += self._finish()+ return bytes(out)++ def flush(self) -> bytes:+ """Return any held-back bytes; the guard goes inert."""+ return self._finish()++ def _finish(self) -> bytes:+ self._done = True+ remaining = bytes(self._buf)+ self._buf = bytearray()+ return remaining++ def _process_event(self, event: bytes) -> bytes:+ event_type = _sse_event_type(event)+ # Pings, comments, retry fields, keepalives: not part of the message+ # protocol, so they neither arm nor disarm the guard.+ if event_type is None or event_type == b"ping":+ return event+ if not self._seen_message_start:+ # Anything data-bearing that isn't message_start (error events,+ # protocol changes) ends the scan.+ self._seen_message_start = True+ if event_type != b"message_start":+ self._done = True+ return event+ try:+ rewritten = self._rewrite_message_start(event)+ except Exception:+ logger.debug("context_guard: message_start rewrite skipped", exc_info=True)+ self._done = True+ return event+ if self._target_total is None:+ # Below trigger (or already reads full): nothing to nudge.+ self._done = True+ return rewritten+ # Armed: pass content deltas through, rewrite the final usage delta.+ if event_type != b"message_delta":+ return event+ self._done = True+ try:+ return self._rewrite_message_delta(event)+ except Exception:+ logger.debug("context_guard: message_delta rewrite skipped", exc_info=True)+ return event++ def _rewrite_message_start(self, event: bytes) -> bytes:+ lines = event.split(b"\n")+ for i, line in enumerate(lines):+ if not line.startswith(b"data:"):+ continue+ payload = json.loads(line[len(b"data:") :].strip())+ if payload.get("type") != "message_start":+ return event+ usage = payload.get("message", {}).get("usage")+ if not isinstance(usage, dict):+ return event+ cache_read = int(usage.get("cache_read_input_tokens") or 0)+ cache_creation = int(usage.get("cache_creation_input_tokens") or 0)+ total = int(usage.get("input_tokens") or 0) + cache_read + cache_creation+ new_input = nudged_input_tokens(+ usage,+ believed_limit=self._believed_limit,+ effective_limit=self._effective_limit,+ )+ if new_input is None:+ return event+ target_total = new_input + cache_read + cache_creation+ self._target_total = target_total+ self._start_cache_read = cache_read+ self._start_cache_creation = cache_creation+ usage["input_tokens"] = new_input+ logger.warning(+ "[%s] event=context_guard_nudge forwarded_total=%d effective_limit=%d "+ "reported_total=%d believed_limit=%d (nudging client to compact "+ "before the prompt-too-long wall)",+ self._request_id,+ total,+ self._effective_limit,+ target_total,+ self._believed_limit,+ )+ lines[i] = _data_line(payload, line)+ return b"\n".join(lines)+ return event++ def _rewrite_message_delta(self, event: bytes) -> bytes:+ assert self._target_total is not None+ lines = event.split(b"\n")+ for i, line in enumerate(lines):+ if not line.startswith(b"data:"):+ continue+ payload = json.loads(line[len(b"data:") :].strip())+ if payload.get("type") != "message_delta":+ return event+ usage = payload.get("usage")+ # A delta without cumulative input usage can't override the+ # message_start values we already rewrote — leave it alone.+ if not isinstance(usage, dict) or "input_tokens" not in usage:+ return event+ cache_read = int(usage.get("cache_read_input_tokens", self._start_cache_read) or 0)+ cache_creation = int(+ usage.get("cache_creation_input_tokens", self._start_cache_creation) or 0+ )+ new_input = self._target_total - cache_read - cache_creation+ if int(usage.get("input_tokens") or 0) >= new_input:+ # Never deflate.+ return event+ usage["input_tokens"] = new_input+ lines[i] = _data_line(payload, line)+ return b"\n".join(lines)+ return eventdiff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 6f97dda20..00a3848fe 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -1857,6 +1857,24 @@ class AnthropicHandlerMixin:from headroom.proxy.helpers import COMPRESSION_TIMEOUT_SECONDScontext_limit = self.anthropic_provider.get_context_limit(model)+ # Context pressure must be computed against the window the+ # request is really subject to: raised to 1M when the+ # client sent a context-1m beta (a 1M session is NOT 5x+ # over budget), capped by a limit learned from an actual+ # prompt-too-long error. See headroom/proxy/context_guard.py.+ from headroom.proxy.context_guard import (+ context_guard_enabled,+ credential_scope_from_headers,+ effective_context_limit,+ )++ if context_guard_enabled():+ context_limit = effective_context_limit(+ model,+ context_limit,+ headers.get("anthropic-beta"),+ scope=credential_scope_from_headers(headers),+ )result = Nonebiases = (self.config.hooks.compute_biases(messages, _hook_ctx)@@ -4122,6 +4140,7 @@ class AnthropicHandlerMixin:outcome_provider=provider_name,session_key=session_key,server_memory_tool_names=server_memory_tool_names,+ client_beta=client_anthropic_beta or "",)else:# Whatever set it — the client's own ``stream: false`` or@@ -4251,6 +4270,30 @@ class AnthropicHandlerMixin:)err_type = "parse_error"+ if response.status_code == 400:+ # Diagnostic only, and outside the parse above:+ # `err_msg` is whatever upstream put in that slot+ # (null, an object, a list have all been seen), and+ # the client is still waiting for the real error+ # body below.+ try:+ from headroom.proxy.context_guard import (+ credential_scope_from_headers,+ note_prompt_too_long,+ )++ note_prompt_too_long(+ model,+ headers.get("anthropic-beta"),+ err_msg,+ scope=credential_scope_from_headers(headers),+ )+ except Exception:+ logger.debug(+ f"[{request_id}] context_guard: limit learning skipped",+ exc_info=True,+ )+logger.warning(f"[{request_id}] UPSTREAM_ERROR "f"status={response.status_code} "@@ -5022,7 +5065,14 @@ class AnthropicHandlerMixin:)if not buffered_stream_ccr:return Response(- content=response.content,+ content=self._context_guard_nudge_body(+ response.content,+ status_code=response.status_code,+ model=model,+ headers=headers,+ request_id=request_id,+ client_beta=client_anthropic_beta or "",+ ),status_code=response.status_code,headers=response_headers,)@@ -5144,6 +5194,15 @@ class AnthropicHandlerMixin:status_code=502,)+ # This stream is resynthesized from the parsed message,+ # so the streaming guard never sees it: nudge here.+ self._context_guard_nudge_message(+ resp_json,+ model=model,+ headers=headers,+ request_id=request_id,+ client_beta=client_anthropic_beta or "",+ )try:sse_events = self._response_to_sse(resp_json, "anthropic")except ValueError as sse_err:@@ -5175,7 +5234,14 @@ class AnthropicHandlerMixin:)return Response(- content=response.content,+ content=self._context_guard_nudge_body(+ response.content,+ status_code=response.status_code,+ model=model,+ headers=headers,+ request_id=request_id,+ client_beta=client_anthropic_beta or "",+ ),status_code=response.status_code,headers=response_headers,)@@ -5306,6 +5372,96 @@ class AnthropicHandlerMixin:# permanently. The emit function is idempotent.await _finalize_pre_upstream()+ def _context_guard_nudge_body(+ self,+ content: bytes,+ *,+ status_code: int,+ model: str,+ headers: dict[str, str],+ request_id: str,+ client_beta: str | None = None,+ ) -> bytes:+ """Apply the context guard to a buffered (non-streaming) message body.++ The buffered path raises the compression budget through+ ``effective_context_limit`` exactly like the streaming path, so it has+ to move the client's gauge exactly like the streaming path too. Without+ this, a non-streaming client got the larger forwarded request and none+ of the warning, and walked into the prompt-too-long wall the guard+ exists to keep it away from.++ Returns ``content`` unchanged unless the usage genuinely needed+ nudging, so byte-faithful forwarding still holds for every response+ that is not near the wall.+ """+ if status_code != 200 or not content:+ return content+ try:+ payload = json.loads(content)+ except Exception:+ return content+ if not self._context_guard_nudge_message(+ payload,+ model=model,+ headers=headers,+ request_id=request_id,+ client_beta=client_beta,+ ):+ return content+ return json.dumps(payload).encode()++ def _context_guard_nudge_message(+ self,+ payload: Any,+ *,+ model: str,+ headers: dict[str, str],+ request_id: str,+ client_beta: str | None = None,+ ) -> bool:+ """Nudge a parsed message's usage in place; True when it changed.++ Shared by the buffered body above and the buffered-CCR stream, which+ resynthesizes SSE from the parsed message and so never passes through+ the streaming guard. ``client_beta`` is the beta the client itself+ sent (its gauge follows it; session-sticky merging can add context-1m+ to the outbound header); None falls back to the outbound header.+ """+ try:+ from headroom.proxy.context_guard import (+ believed_context_limit,+ context_guard_enabled,+ credential_scope_from_headers,+ effective_context_limit,+ nudge_response_usage,+ )++ if not context_guard_enabled():+ return False+ if not isinstance(payload, dict) or payload.get("type") != "message":+ return False+ model_limit = self.anthropic_provider.get_context_limit(model)+ beta_header = headers.get("anthropic-beta")+ return bool(+ nudge_response_usage(+ payload,+ believed_limit=believed_context_limit(+ model_limit, beta_header if client_beta is None else client_beta+ ),+ effective_limit=effective_context_limit(+ model,+ model_limit,+ beta_header,+ scope=credential_scope_from_headers(headers),+ ),+ request_id=request_id,+ )+ )+ except Exception:+ logger.debug(f"[{request_id}] context_guard: buffered nudge skipped", exc_info=True)+ return False+def _anthropic_batch_capability_error(self) -> Response | None:"""Return the stable client error for a Copilot batch target."""if not is_copilot_upstream_url(self.ANTHROPIC_API_URL):@@ -5478,6 +5634,23 @@ class AnthropicHandlerMixin:original_tokens = 0 # Initialize before try to prevent UnboundLocalErrortry:context_limit = self.anthropic_provider.get_context_limit(model)+ # Same window the streaming path computes: a batch item for a+ # 1M-beta model was otherwise compressed against the registry+ # limit, so identical requests got two different budgets+ # depending on which endpoint they arrived through.+ from headroom.proxy.context_guard import (+ context_guard_enabled,+ credential_scope_from_headers,+ effective_context_limit,+ )++ if context_guard_enabled():+ context_limit = effective_context_limit(+ model,+ context_limit,+ headers.get("anthropic-beta"),+ scope=credential_scope_from_headers(headers),+ )frozen_message_count = (self._strict_previous_turn_frozen_count(original_messages, 0)if is_cache_mode(self.config.mode)diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex 2bc0d7731..4a8808af3 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -1029,6 +1029,7 @@ class StreamingMixin:conversation_key: str | None = None,conversation_tokens_saved: int | None = None,server_memory_tool_names: frozenset[str] | None = None,+ client_beta: str | None = None,) -> Response | StreamingResponse:"""Stream response with metrics tracking and memory tool handling.@@ -1075,6 +1076,7 @@ class StreamingMixin:conversation_key=conversation_key,conversation_tokens_saved=conversation_tokens_saved,server_memory_tool_names=server_memory_tool_names,+ client_beta=client_beta,)except (Exception, asyncio.CancelledError):self._cleanup_mid_turn_stream(session_key)@@ -1108,6 +1110,7 @@ class StreamingMixin:conversation_key: str | None = None,conversation_tokens_saved: int | None = None,server_memory_tool_names: frozenset[str] | None = None,+ client_beta: str | None = None,) -> Response | StreamingResponse:"""Actual streaming implementation, guarded by _stream_response's cleanup wrapper."""from fastapi.responses import Response, StreamingResponse@@ -1128,7 +1131,63 @@ class StreamingMixin:# of being swallowed. (#1608)if supports_mid_turn_coalescing(client):self._active_streams.add(session_key)++ # Before the guard: its learned-limit lookup is keyed on the credential+ # actually sent, the same one the 400 learning below keys on.headers = await apply_copilot_api_auth(headers, url=url)++ # Context-limit guard (see headroom/proxy/context_guard.py): rewrites+ # only the client-bound message_start bytes when the forwarded request+ # is near the model's real window, so clients whose auto-compaction+ # keys off reported usage compact gracefully instead of looping on+ # prompt-too-long 400s. Metrics below parse the original upstream+ # bytes and are unaffected.+ context_guard = None+ if provider == "anthropic":+ from headroom.proxy.context_guard import (+ StreamUsageGuard,+ believed_context_limit,+ context_guard_enabled,+ credential_scope_from_headers,+ effective_context_limit,+ )++ try:+ if context_guard_enabled():+ _guard_model_limit = self.anthropic_provider.get_context_limit(model)+ _guard_beta = headers.get("anthropic-beta")+ _guard_scope = credential_scope_from_headers(headers)+ context_guard = StreamUsageGuard(+ # The client's gauge follows the beta the client+ # sent; session-sticky merging can add context-1m to+ # the outbound header after the client dropped it.+ # None: the caller did not say, use the outbound one.+ believed_limit=believed_context_limit(+ _guard_model_limit,+ _guard_beta if client_beta is None else client_beta,+ ),+ effective_limit=effective_context_limit(+ model, _guard_model_limit, _guard_beta, scope=_guard_scope+ ),+ request_id=request_id,+ )+ except Exception:+ logger.debug("context_guard setup skipped", exc_info=True)++ def _guard_client_bytes(data: bytes) -> bytes:+ """Client-bound bytes through the context guard, when there is one."""+ return context_guard.feed(data) if context_guard is not None else data++ def _drain_context_guard() -> bytes:+ """Held-back guard bytes, once. Safe to call on any exit path."""+ if context_guard is None:+ return b""+ try:+ return context_guard.flush()+ except Exception:+ logger.debug("context_guard flush skipped", exc_info=True)+ return b""+start_time = time.time()# Byte-faithful forwarding (PR-A3, fixes P0-2). Resolve outbound@@ -1403,6 +1462,24 @@ class StreamingMixin:finally:await upstream_response.aclose()+ if provider == "anthropic" and upstream_response.status_code == 400:+ # Diagnostic only: whatever it makes of the body, the client+ # still gets the upstream error below.+ try:+ from headroom.proxy.context_guard import (+ credential_scope_from_headers,+ note_prompt_too_long,+ )++ note_prompt_too_long(+ model,+ headers.get("anthropic-beta"),+ error_content,+ scope=credential_scope_from_headers(headers),+ )+ except Exception:+ logger.debug("context_guard: limit learning skipped", exc_info=True)+if _codex_wire_debug:_error_text: str | None = None_error_body: Any = None@@ -1542,11 +1619,16 @@ class StreamingMixin:# Always stream immediately — buffering breaks# real-time clients (LangGraph, LangChain, etc.)- if memory_filter is None:- yield chunk- else:- for frame in memory_filter.feed(chunk):- yield frame+ # The context guard runs last, on exactly what the+ # client receives (after the memory filter), and may+ # hold back bytes only until the first complete event.+ client_frames = (+ [chunk] if memory_filter is None else memory_filter.feed(chunk)+ )+ for frame in client_frames:+ guarded_frame = _guard_client_bytes(frame)+ if guarded_frame:+ yield guarded_frameif _codex_wire_debug:capture_codex_wire_debug(@@ -1625,7 +1707,12 @@ class StreamingMixin:)if memory_filter is not None:for frame in memory_filter.closing_frames():- yield frame+ guarded_frame = _guard_client_bytes(frame)+ if guarded_frame:+ yield guarded_frame+ guarded_tail = _drain_context_guard()+ if guarded_tail:+ yield guarded_tailreturn# Parse SSE to get response JSON@@ -1644,7 +1731,9 @@ class StreamingMixin:stream_state=stream_state,request_id=request_id,):- yield frame+ guarded_frame = _guard_client_bytes(frame)+ if guarded_frame:+ yield guarded_framememory_filter = Noneparsed_response = (stream_state.pop("memory_client_message", None) or parsed_response@@ -1687,7 +1776,20 @@ class StreamingMixin:stream_state=stream_state,request_id=request_id,):- yield frame+ guarded_frame = _guard_client_bytes(frame)+ if guarded_frame:+ yield guarded_frame++ # A stream that ended before its first complete event may+ # leave held-back bytes in the context guard; release them,+ # after the memory filter's held tail so the final+ # message_delta is still nudged. Every exit does this,+ # including the error paths below: bytes the guard is holding+ # are bytes the upstream already sent, and dropping them+ # silently truncates the client's stream.+ guarded_tail = _drain_context_guard()+ if guarded_tail:+ yield guarded_tail# CCR Feedback: Record headroom_retrieve tool calls for TOIN learning.# In streaming mode, the client handles actual retrieval, but we@@ -1725,6 +1827,9 @@ class StreamingMixin:except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:logger.error(f"[{request_id}] Connection error to upstream API: {e}")+ guarded_tail = _drain_context_guard()+ if guarded_tail:+ yield guarded_tailerror_event = public_errors.anthropic_error_body(public_errors.classify_or_internal(e),request_id=request_id,@@ -1733,6 +1838,9 @@ class StreamingMixin:yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()except httpx.HTTPStatusError as e:logger.error(f"[{request_id}] HTTP error from upstream API: {e}")+ guarded_tail = _drain_context_guard()+ if guarded_tail:+ yield guarded_tail# Forward the upstream error responseyield e.response.contentexcept MemoryToolStreamOverflowError as e:@@ -1745,6 +1853,9 @@ class StreamingMixin:yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()except Exception as e:logger.error(f"[{request_id}] Unexpected streaming error: {e}")+ guarded_tail = _drain_context_guard()+ if guarded_tail:+ yield guarded_tailerror_event = public_errors.anthropic_error_body(public_errors.classify_or_internal(e),request_id=request_id,diff --git a/tests/test_anthropic_buffered_ccr_usage.py b/tests/test_anthropic_buffered_ccr_usage.pyindex 5002147c1..46569cce4 100644--- a/tests/test_anthropic_buffered_ccr_usage.py+++ b/tests/test_anthropic_buffered_ccr_usage.py@@ -179,3 +179,124 @@ def test_buffered_ccr_records_usage_with_compression_on(monkeypatch, ccr_marker:f"buffered-CCR turn dropped usage under compression: "f"cr={o.cache_read_tokens} cw={o.cache_write_tokens} out={o.output_tokens}")+++def test_buffered_ccr_stream_gets_the_context_guard_nudge(monkeypatch, ccr_marker: str) -> None:+ """The buffered-CCR stream is resynthesized from the parsed message, so the+ streaming guard never sees it. Near the wall its gauge must move anyway, or+ the client gets the raised compression budget without the warning."""+ import json++ from headroom.proxy.context_guard import REPORT_FRACTION, reset_learned_limits++ reset_learned_limits()+ app, _ = _app_and_outcomes(monkeypatch)+ near_wall = {**_RESPONSE, "usage": {**_USAGE, "cache_read_input_tokens": 900_000}}+ respx.post("https://api.anthropic.com/v1/messages").mock(+ return_value=httpx.Response(200, json=near_wall)+ )++ r = _post(app, stream=True, tools=[_RETRIEVE_TOOL], marker=ccr_marker)++ assert r.status_code == 200+ start = next(+ json.loads(line[len("data:") :])+ for line in r.text.splitlines()+ if line.startswith("data:") and '"message_start"' in line+ )+ usage = start["message"]["usage"]+ reported = (+ usage["input_tokens"]+ + usage["cache_read_input_tokens"]+ + usage["cache_creation_input_tokens"]+ )+ # claude-opus-5 is a 1M model; 909k forwarded is past the trigger.+ assert reported == int(1_000_000 * REPORT_FRACTION)+++def _reported_input(usage: dict) -> int:+ return (+ usage["input_tokens"]+ + usage["cache_read_input_tokens"]+ + usage["cache_creation_input_tokens"]+ )+++def test_non_stream_json_gets_the_context_guard_nudge(monkeypatch) -> None:+ """The plain non-stream JSON body is nudged near the wall, like the stream."""+ from headroom.proxy.context_guard import REPORT_FRACTION, reset_learned_limits++ reset_learned_limits()+ app, _ = _app_and_outcomes(monkeypatch)+ near_wall = {**_RESPONSE, "usage": {**_USAGE, "cache_read_input_tokens": 900_000}}+ respx.post("https://api.anthropic.com/v1/messages").mock(+ return_value=httpx.Response(200, json=near_wall)+ )++ r = _post(app, stream=False, tools=None)++ assert r.status_code == 200+ usage = r.json()["usage"]+ # claude-opus-5 is a 1M model; 909k forwarded is past the trigger.+ assert _reported_input(usage) == int(1_000_000 * REPORT_FRACTION)+ assert usage["output_tokens"] == _USAGE["output_tokens"]+++def test_non_stream_json_is_untouched_with_the_guard_off(monkeypatch) -> None:+ from headroom.proxy.context_guard import reset_learned_limits++ reset_learned_limits()+ monkeypatch.setenv("HEADROOM_CONTEXT_GUARD", "0")+ app, _ = _app_and_outcomes(monkeypatch)+ near_wall = {**_RESPONSE, "usage": {**_USAGE, "cache_read_input_tokens": 900_000}}+ respx.post("https://api.anthropic.com/v1/messages").mock(+ return_value=httpx.Response(200, json=near_wall)+ )++ r = _post(app, stream=False, tools=None)++ assert r.status_code == 200+ assert r.json()["usage"] == near_wall["usage"]+++def test_buffered_nudge_follows_the_client_beta_not_the_sticky_one(monkeypatch) -> None:+ """Session-sticky merging can keep context-1m outbound after the client+ dropped it. The learned cap keys on the forwarded beta; the client's gauge+ is its own 200k window, so the nudge targets 95% of 200k, not of 1M."""+ from headroom.proxy.context_guard import (+ REPORT_FRACTION,+ credential_scope,+ note_prompt_too_long,+ reset_learned_limits,+ )++ reset_learned_limits()+ beta_1m = "context-1m-2025-08-07"+ note_prompt_too_long(+ "claude-sonnet-4-5",+ beta_1m,+ "prompt is too long: 213021 tokens > 200000 maximum",+ scope=credential_scope("sk-sticky"),+ )+ app, _ = _app_and_outcomes(monkeypatch)+ proxy = app.state.proxy+ monkeypatch.setattr(proxy.anthropic_provider, "get_context_limit", lambda _model: 200_000)+ payload = {+ **_RESPONSE,+ "model": "claude-sonnet-4-5",+ "usage": {**_USAGE, "cache_read_input_tokens": 185_000 - 300 - 8197},+ }++ changed = proxy._context_guard_nudge_message(+ payload,+ model="claude-sonnet-4-5",+ headers={"x-api-key": "sk-sticky", "anthropic-beta": beta_1m},+ request_id="test-sticky-buffered",+ client_beta="",+ )++ assert changed+ assert _reported_input(payload["usage"]) == int(200_000 * REPORT_FRACTION)diff --git a/tests/test_proxy/test_context_guard.py b/tests/test_proxy/test_context_guard.pynew file mode 100644index 000000000..d5e20d105--- /dev/null+++ b/tests/test_proxy/test_context_guard.py@@ -0,0 +1,694 @@+"""Tests for the context-limit guard (headroom/proxy/context_guard.py)."""++from __future__ import annotations++import json++import pytest++from headroom.proxy.context_guard import (+ REPORT_FRACTION,+ StreamUsageGuard,+ believed_context_limit,+ context_guard_enabled,+ credential_scope,+ effective_context_limit,+ has_context_1m_beta,+ note_prompt_too_long,+ nudge_response_usage,+ reset_learned_limits,+)++[email protected](autouse=True)+def _clean_learned_limits():+ reset_learned_limits()+ yield+ reset_learned_limits()+++def _message_start_event(+ input_tokens: int,+ cache_read: int = 0,+ cache_creation: int = 0,+) -> bytes:+ payload = {+ "type": "message_start",+ "message": {+ "id": "msg_test",+ "type": "message",+ "role": "assistant",+ "model": "claude-sonnet-4-5",+ "content": [],+ "usage": {+ "input_tokens": input_tokens,+ "cache_read_input_tokens": cache_read,+ "cache_creation_input_tokens": cache_creation,+ "output_tokens": 1,+ },+ },+ }+ return b"event: message_start\ndata: " + json.dumps(payload).encode() + b"\n\n"+++def _parse_usage(event_bytes: bytes) -> dict:+ for line in event_bytes.split(b"\n"):+ if line.startswith(b"data:"):+ return json.loads(line[5:].strip())["message"]["usage"]+ raise AssertionError("no data line in event")+++def _message_delta_event(+ input_tokens: int,+ cache_read: int = 0,+ cache_creation: int = 0,+) -> bytes:+ payload = {+ "type": "message_delta",+ "delta": {"stop_reason": "end_turn", "stop_sequence": None},+ "usage": {+ "input_tokens": input_tokens,+ "cache_read_input_tokens": cache_read,+ "cache_creation_input_tokens": cache_creation,+ "output_tokens": 42,+ },+ }+ return b"event: message_delta\ndata: " + json.dumps(payload).encode() + b"\n\n"+++class TestBetaAndLimits:+ def test_has_context_1m_beta_detects_token(self):+ assert has_context_1m_beta("context-1m-2025-08-07")+ assert has_context_1m_beta("oauth-2025-04-20, context-1m-2025-08-07")+ assert not has_context_1m_beta("oauth-2025-04-20")+ assert not has_context_1m_beta(None)+ assert not has_context_1m_beta("")++ def test_believed_limit_raised_by_1m_beta(self):+ assert believed_context_limit(200_000, "context-1m-2025-08-07") == 1_000_000+ assert believed_context_limit(200_000, "oauth-2025-04-20") == 200_000++ def test_believed_limit_never_lowered(self):+ assert believed_context_limit(2_000_000, "context-1m-2025-08-07") == 2_000_000++ def test_effective_limit_optimistic_until_learned(self):+ beta = "context-1m-2025-08-07"+ assert effective_context_limit("claude-sonnet-4-5", 200_000, beta) == 1_000_000++ def test_learned_limit_caps_effective(self):+ beta = "context-1m-2025-08-07"+ learned = note_prompt_too_long(+ "claude-sonnet-4-5",+ beta,+ '{"error": {"message": "prompt is too long: 213021 tokens > 200000 maximum"}}',+ )+ assert learned == 200_000+ assert effective_context_limit("claude-sonnet-4-5", 200_000, beta) == 200_000++ def test_learned_limit_keyed_by_beta_presence(self):+ # A limit learned WITHOUT the 1m beta must not clamp sessions that+ # send it (their account may have real 1M access).+ note_prompt_too_long(+ "claude-sonnet-4-5",+ None,+ "prompt is too long: 201000 tokens > 200000 maximum",+ )+ assert (+ effective_context_limit("claude-sonnet-4-5", 200_000, "context-1m-2025-08-07")+ == 1_000_000+ )++ def test_note_prompt_too_long_ignores_other_errors(self):+ assert note_prompt_too_long("m", None, "rate limit exceeded") is None+ assert note_prompt_too_long("m", None, b"") is None++ def test_note_prompt_too_long_accepts_bytes(self):+ assert (+ note_prompt_too_long("m", None, b"prompt is too long: 250000 tokens > 200000 maximum")+ == 200_000+ )++ def test_enabled_by_default_and_kill_switch(self, monkeypatch):+ monkeypatch.delenv("HEADROOM_CONTEXT_GUARD", raising=False)+ assert context_guard_enabled()+ monkeypatch.setenv("HEADROOM_CONTEXT_GUARD", "0")+ assert not context_guard_enabled()+ monkeypatch.setenv("HEADROOM_CONTEXT_GUARD", "false")+ assert not context_guard_enabled()+++class TestLearnedLimitHygiene:+ """Regressions for what a learned limit is allowed to do."""++ def test_non_string_error_bodies_do_not_raise(self):+ # Upstreams and gateways put null, objects and lists in the message+ # slot. This runs while the client is still waiting for the real error+ # body, so it must return empty-handed rather than raise over it.+ for body in (None, {"message": {"nested": True}}, ["prompt is too long"], 42):+ assert note_prompt_too_long("claude-sonnet-4-5", None, body) is None+ assert effective_context_limit("claude-sonnet-4-5", 200_000, None) == 200_000++ def test_out_of_band_maximum_is_not_learned(self):+ # Same wire shape, different subject: gateways report per-request quota+ # and per-key throttles as "prompt is too long: N > M". A limit that is+ # not a context window must not clamp the budget for the process.+ assert (+ note_prompt_too_long(+ "claude-sonnet-4-5", None, "prompt is too long: 900 tokens > 1 maximum"+ )+ is None+ )+ assert (+ note_prompt_too_long(+ "claude-sonnet-4-5",+ None,+ "prompt is too long: 99000000 tokens > 50000000 maximum",+ )+ is None+ )+ assert effective_context_limit("claude-sonnet-4-5", 200_000, None) == 200_000++ def test_learned_limit_is_scoped_to_one_credential(self):+ # One tenant's capped key on a shared proxy must not clamp another's.+ note_prompt_too_long(+ "claude-sonnet-4-5",+ None,+ "prompt is too long: 213021 tokens > 200000 maximum",+ scope=credential_scope("sk-tenant-a"),+ )+ assert (+ effective_context_limit(+ "claude-sonnet-4-5", 400_000, None, scope=credential_scope("sk-tenant-a")+ )+ == 200_000+ )+ assert (+ effective_context_limit(+ "claude-sonnet-4-5", 400_000, None, scope=credential_scope("sk-tenant-b")+ )+ == 400_000+ )++ def test_credential_scope_never_echoes_the_credential(self):+ scope = credential_scope("sk-ant-secret-value")+ assert "secret" not in scope+ assert scope and scope != credential_scope("sk-ant-other-value")+ assert credential_scope(None) == credential_scope("") == ""++ def test_learned_limit_expires(self, monkeypatch):+ # Entitlements get provisioned and gateways get repointed; a lesson+ # from an hour ago must not hold the budget down forever.+ from headroom.proxy import context_guard++ clock = {"now": 1_000.0}+ monkeypatch.setattr(context_guard.time, "monotonic", lambda: clock["now"])+ note_prompt_too_long(+ "claude-sonnet-4-5", None, "prompt is too long: 213021 tokens > 200000 maximum"+ )+ assert effective_context_limit("claude-sonnet-4-5", 400_000, None) == 200_000+ clock["now"] += context_guard._LEARNED_LIMIT_TTL_SECONDS + 1+ assert effective_context_limit("claude-sonnet-4-5", 400_000, None) == 400_000+++class TestNonStreamingNudge:+ """The buffered path raises the same budget, so it owes the same warning."""++ def test_near_limit_body_is_nudged(self):+ payload = {+ "type": "message",+ "usage": {+ "input_tokens": 185_000,+ "cache_read_input_tokens": 0,+ "cache_creation_input_tokens": 0,+ "output_tokens": 7,+ },+ }+ assert nudge_response_usage(payload, believed_limit=200_000, effective_limit=200_000)+ assert payload["usage"]["input_tokens"] == int(200_000 * REPORT_FRACTION)++ def test_far_from_limit_body_is_untouched(self):+ payload = {"type": "message", "usage": {"input_tokens": 5_000, "output_tokens": 7}}+ assert not nudge_response_usage(payload, believed_limit=200_000, effective_limit=200_000)+ assert payload["usage"]["input_tokens"] == 5_000++ def test_bodies_without_usage_are_untouched(self):+ for payload in ({"type": "message"}, {"type": "error"}, "not a body", None):+ assert not nudge_response_usage(+ payload, believed_limit=200_000, effective_limit=200_000+ )+++class TestStreamUsageGuard:+ def test_below_trigger_passes_through_byte_identical(self):+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)+ event = _message_start_event(50_000, cache_read=100_000)+ assert guard.feed(event) == event+ # Inert afterwards: later chunks untouched even if they look like events.+ tail = b"event: content_block_delta\ndata: {}\n\n"+ assert guard.feed(tail) == tail++ def test_above_trigger_inflates_to_report_fraction_of_believed(self):+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)+ # 185k total forwarded = 92.5% of the real 200k window.+ event = _message_start_event(5_000, cache_read=170_000, cache_creation=10_000)+ out = guard.feed(event)+ usage = _parse_usage(out)+ target_total = int(1_000_000 * REPORT_FRACTION)+ assert (+ usage["input_tokens"]+ + usage["cache_read_input_tokens"]+ + usage["cache_creation_input_tokens"]+ == target_total+ )+ # Cache components are never touched — only input_tokens absorbs the nudge.+ assert usage["cache_read_input_tokens"] == 170_000+ assert usage["cache_creation_input_tokens"] == 10_000+ assert usage["output_tokens"] == 1++ def test_same_limits_nudges_gauge_over_compact_threshold(self):+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)+ event = _message_start_event(185_000)+ usage = _parse_usage(guard.feed(event))+ assert usage["input_tokens"] == int(200_000 * REPORT_FRACTION)++ def test_never_deflates(self):+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)+ event = _message_start_event(199_000) # above the 95% report target+ assert guard.feed(event) == event++ def test_split_chunks_across_event_boundary(self):+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)+ event = _message_start_event(190_000)+ first, second = event[:40], event[40:]+ assert guard.feed(first) == b"" # held back: no complete event yet+ out = guard.feed(second)+ assert _parse_usage(out)["input_tokens"] == int(1_000_000 * REPORT_FRACTION)++ def test_crlf_framed_events_are_released_and_rewritten(self):+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)+ event = _message_start_event(190_000).replace(b"\n", b"\r\n")+ out = guard.feed(event)+ # Released at its own terminator, not held until the buffer cap.+ assert out.endswith(b"\r\n\r\n")+ assert b"\n" not in out.replace(b"\r\n", b"")+ assert _parse_usage(out)["input_tokens"] == int(1_000_000 * REPORT_FRACTION)++ def test_a_chunk_over_the_buffer_cap_still_gets_its_message_start_read(self):+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)+ filler = b"event: content_block_delta\ndata: " + b"x" * (300 * 1024) + b"\n\n"+ out = guard.feed(_message_start_event(190_000) + filler)+ assert _parse_usage(out)["input_tokens"] == int(1_000_000 * REPORT_FRACTION)+ assert out.endswith(filler)++ def test_ping_events_pass_through_before_message_start(self):+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)+ ping = b'event: ping\ndata: {"type": "ping"}\n\n'+ assert guard.feed(ping) == ping+ out = guard.feed(_message_start_event(190_000))+ assert _parse_usage(out)["input_tokens"] == int(1_000_000 * REPORT_FRACTION)++ def test_non_message_start_first_event_disarms(self):+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)+ error_event = b'event: error\ndata: {"type": "error"}\n\n'+ assert guard.feed(error_event) == error_event+ # Even a later message_start is untouched (protocol says it is first).+ event = _message_start_event(190_000)+ assert guard.feed(event) == event++ def test_malformed_json_passes_through(self):+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)+ event = b"event: message_start\ndata: {not json\n\n"+ assert guard.feed(event) == event++ def test_buffer_cap_flushes_verbatim(self):+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)+ blob = b"x" * (300 * 1024) # no event boundary anywhere+ out = guard.feed(blob)+ assert out == blob+ assert guard.feed(b"more") == b"more" # inert afterwards++ def test_flush_returns_held_bytes(self):+ guard = StreamUsageGuard(believed_limit=1_000_000, effective_limit=200_000)+ partial = b"event: message_start\ndata: {"+ assert guard.feed(partial) == b""+ assert guard.flush() == partial++ def test_unusable_limits_make_guard_inert(self):+ guard = StreamUsageGuard(believed_limit=0, effective_limit=200_000)+ event = _message_start_event(190_000)+ assert guard.feed(event) == event++ def test_rewritten_event_is_valid_sse(self):+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)+ out = guard.feed(_message_start_event(185_000))+ assert out.startswith(b"event: message_start\n")+ assert out.endswith(b"\n\n")+ payload = _parse_usage(out) # parses => data line is intact JSON+ assert payload["input_tokens"] > 185_000++ def test_armed_guard_rewrites_final_message_delta(self):+ # Clients merge the final cumulative-usage message_delta over+ # message_start (verified live with Claude Code 2026-08-12), so an+ # armed guard must rewrite both or the nudge loses the merge.+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)+ guard.feed(_message_start_event(180_000, cache_creation=5_000))+ content = b'event: content_block_delta\ndata: {"type":"content_block_delta"}\n\n'+ assert guard.feed(content) == content # armed but passthrough+ delta = _message_delta_event(180_000, cache_creation=5_000)+ out = guard.feed(delta)+ usage = json.loads(out.split(b"data: ")[1])["usage"]+ assert usage["input_tokens"] + usage["cache_creation_input_tokens"] == int(+ 200_000 * REPORT_FRACTION+ )+ assert usage["output_tokens"] == 42+ # Inert afterwards.+ stop = b'event: message_stop\ndata: {"type":"message_stop"}\n\n'+ assert guard.feed(stop) == stop++ def test_output_only_message_delta_untouched(self):+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)+ guard.feed(_message_start_event(185_000))+ delta = (+ b'event: message_delta\ndata: {"type":"message_delta","usage":{"output_tokens":9}}\n\n'+ )+ assert guard.feed(delta) == delta++ def test_disarmed_guard_leaves_message_delta_alone(self):+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)+ guard.feed(_message_start_event(50_000)) # below trigger -> inert+ delta = _message_delta_event(50_000)+ assert guard.feed(delta) == delta++ def test_message_delta_never_deflates(self):+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)+ guard.feed(_message_start_event(185_000))+ delta = _message_delta_event(199_000) # already above the 190k target+ assert guard.feed(delta) == delta+++class TestSseFraming:+ """The scan reads the SSE envelope, never the payload text."""++ def _armed_guard(self) -> StreamUsageGuard:+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)+ guard.feed(_message_start_event(190_000))+ return guard++ def test_assistant_text_naming_an_event_does_not_disarm(self):+ # The model writing about message_delta -- explaining SSE, quoting a+ # log line -- used to end the scan, so the final cumulative usage went+ # out un-nudged and the client's gauge snapped back.+ guard = self._armed_guard()+ chatter = (+ b"event: content_block_delta\ndata: "+ + json.dumps(+ {+ "type": "content_block_delta",+ "index": 0,+ "delta": {+ "type": "text_delta",+ "text": "the final message_delta carries cumulative usage",+ },+ }+ ).encode()+ + b"\n\n"+ )+ assert guard.feed(chatter) == chatter+ nudged = guard.feed(_message_delta_event(190_000))+ assert json.loads(nudged.split(b"data: ", 1)[1])["usage"]["input_tokens"] == int(+ 200_000 * REPORT_FRACTION+ )++ def test_keepalive_frames_before_message_start_do_not_disarm(self):+ # An intermediary shim (nginx, a corporate egress proxy) injects SSE+ # comments and retry fields. They are not events, so they must not be+ # read as "a protocol we don't recognize" and switch the guard off.+ guard = StreamUsageGuard(believed_limit=200_000, effective_limit=200_000)+ for keepalive in (b": keepalive\n\n", b"retry: 3000\n\n", b":\n\n"):+ assert guard.feed(keepalive) == keepalive+ nudged = guard.feed(_message_start_event(190_000))+ assert _parse_usage(nudged)["input_tokens"] == int(200_000 * REPORT_FRACTION)+++class TestStreamResponseIntegration:+ """The guard wired through _stream_response with a mocked upstream."""++ def _create_mock_proxy(self):+ from unittest.mock import AsyncMock, MagicMock++ import httpx++ from headroom.proxy.server import HeadroomProxy++ proxy = object.__new__(HeadroomProxy)+ proxy.http_client = MagicMock(spec=httpx.AsyncClient)+ proxy.metrics = MagicMock()+ proxy.metrics.record_request = AsyncMock(return_value=None)+ proxy.metrics.record_failed = AsyncMock(return_value=None)+ proxy.cost_tracker = MagicMock()+ proxy.cost_tracker.estimate_cost.return_value = 0.001+ proxy.cost_tracker.record_request.return_value = None+ proxy.stats = {+ "requests_total": 0,+ "requests_optimized": 0,+ "tokens": {"original": 0, "optimized": 0, "saved": 0},+ "cost": {"total_usd": 0, "savings_usd": 0},+ "errors": 0,+ "active_requests": 0,+ "requests_per_model": {},+ }+ proxy.memory_manager = None+ proxy._config = MagicMock()+ proxy._config.memory_enabled = False+ proxy._config.ccr_inject_tool = False+ proxy._config.retry_max_attempts = 3+ proxy._config.retry_base_delay_ms = 0+ proxy._config.retry_max_delay_ms = 0+ proxy.config = proxy._config+ proxy._parse_sse_usage_from_buffer = MagicMock(return_value=None)+ proxy.memory_handler = None+ proxy.anthropic_provider = MagicMock()+ proxy.anthropic_provider.get_context_limit.return_value = 200_000+ return proxy++ def _mock_upstream(self, sse_bytes: bytes, status_code: int = 200):+ from unittest.mock import AsyncMock++ import httpx++ mock_response = AsyncMock()+ mock_response.headers = httpx.Headers({"content-type": "text/event-stream"})+ mock_response.status_code = status_code++ async def aiter_bytes():+ yield sse_bytes++ mock_response.aiter_bytes = aiter_bytes+ mock_response.aclose = AsyncMock()+ mock_response.aread = AsyncMock(return_value=sse_bytes)+ return mock_response++ async def _run(self, proxy, mock_response, headers, **kwargs):+ from unittest.mock import AsyncMock, MagicMock++ proxy.http_client.build_request = MagicMock(return_value=MagicMock())+ proxy.http_client.send = AsyncMock(return_value=mock_response)+ return await proxy._stream_response(+ url="https://api.anthropic.com/v1/messages",+ headers=headers,+ body={+ "model": "claude-sonnet-4-5",+ "max_tokens": 100,+ "stream": True,+ "messages": [{"role": "user", "content": "hi"}],+ },+ provider="anthropic",+ model="claude-sonnet-4-5",+ request_id="test-guard",+ original_tokens=10,+ optimized_tokens=10,+ tokens_saved=0,+ transforms_applied=[],+ tags={},+ optimization_latency=0.0,+ **kwargs,+ )++ @pytest.mark.asyncio+ async def test_near_limit_message_start_is_nudged_on_the_wire(self):+ proxy = self._create_mock_proxy()+ # 185k = above the 90% trigger, below the 95% report target — the+ # assertions below can only pass if the rewrites actually happened.+ upstream = self._mock_upstream(+ _message_start_event(185_000)+ + b'event: content_block_delta\ndata: {"type":"content_block_delta"}\n\n'+ + _message_delta_event(185_000)+ + b'event: message_stop\ndata: {"type":"message_stop"}\n\n'+ )+ result = await self._run(proxy, upstream, {"x-api-key": "sk-test"})+ client_bytes = b"".join([chunk async for chunk in result.body_iterator])+ events = client_bytes.split(b"\n\n")+ usage = _parse_usage(events[0] + b"\n\n")+ assert usage["input_tokens"] == int(200_000 * REPORT_FRACTION)+ # The final cumulative-usage message_delta is nudged too — clients+ # merge it over message_start, so it must agree.+ delta_usage = json.loads(events[2].split(b"data: ")[1])["usage"]+ assert delta_usage["input_tokens"] == int(200_000 * REPORT_FRACTION)+ assert delta_usage["output_tokens"] == 42+ # Everything else is untouched.+ assert (+ b'event: content_block_delta\ndata: {"type":"content_block_delta"}\n\n' in client_bytes+ )+ assert b'event: message_stop\ndata: {"type":"message_stop"}\n\n' in client_bytes++ @pytest.mark.asyncio+ async def test_far_from_limit_stream_is_byte_identical(self):+ proxy = self._create_mock_proxy()+ sse = (+ _message_start_event(50_000) + b'event: message_stop\ndata: {"type":"message_stop"}\n\n'+ )+ result = await self._run(proxy, self._mock_upstream(sse), {"x-api-key": "sk-test"})+ client_bytes = b"".join([chunk async for chunk in result.body_iterator])+ assert client_bytes == sse++ @pytest.mark.asyncio+ async def test_kill_switch_disables_nudge(self, monkeypatch):+ monkeypatch.setenv("HEADROOM_CONTEXT_GUARD", "0")+ proxy = self._create_mock_proxy()+ sse = _message_start_event(190_000)+ result = await self._run(proxy, self._mock_upstream(sse), {"x-api-key": "sk-test"})+ client_bytes = b"".join([chunk async for chunk in result.body_iterator])+ assert client_bytes == sse++ @pytest.mark.asyncio+ async def test_streaming_400_learns_real_limit(self):+ proxy = self._create_mock_proxy()+ error_body = json.dumps(+ {+ "type": "error",+ "error": {+ "type": "invalid_request_error",+ "message": "prompt is too long: 213021 tokens > 200000 maximum",+ },+ }+ ).encode()+ upstream = self._mock_upstream(error_body, status_code=400)+ upstream.headers = __import__("httpx").Headers({"content-type": "application/json"})+ result = await self._run(+ proxy, upstream, {"x-api-key": "sk-test", "anthropic-beta": "context-1m-2025-08-07"}+ )+ assert result.status_code == 400+ # Learned against the credential that hit the wall, not process-wide.+ scope = credential_scope("sk-test")+ assert (+ effective_context_limit(+ "claude-sonnet-4-5", 200_000, "context-1m-2025-08-07", scope=scope+ )+ == 200_000+ )+ # A different key on the same proxy keeps its own (optimistic) budget.+ assert (+ effective_context_limit(+ "claude-sonnet-4-5",+ 200_000,+ "context-1m-2025-08-07",+ scope=credential_scope("sk-other"),+ )+ == 1_000_000+ )++ @pytest.mark.asyncio+ async def test_sticky_1m_beta_does_not_inflate_a_200k_client_gauge(self):+ """The client dropped context-1m; session-sticky merging kept it outbound.++ The learned cap still keys on the forwarded beta, but the client's+ gauge is its own 200k window, so the nudge must target 95% of 200k.+ """+ beta_1m = "context-1m-2025-08-07"+ note_prompt_too_long(+ "claude-sonnet-4-5",+ beta_1m,+ "prompt is too long: 213021 tokens > 200000 maximum",+ scope=credential_scope("sk-sticky"),+ )+ proxy = self._create_mock_proxy()+ sse = _message_start_event(185_000) + _message_delta_event(185_000)+ result = await self._run(+ proxy,+ self._mock_upstream(sse),+ {"x-api-key": "sk-sticky", "anthropic-beta": beta_1m},+ client_beta="",+ )+ client_bytes = b"".join([chunk async for chunk in result.body_iterator])+ events = client_bytes.split(b"\n\n")+ assert _parse_usage(events[0] + b"\n\n")["input_tokens"] == int(200_000 * REPORT_FRACTION)+ delta_usage = json.loads(events[1].split(b"data: ")[1])["usage"]+ assert delta_usage["input_tokens"] == int(200_000 * REPORT_FRACTION)++ @pytest.mark.asyncio+ async def test_guard_nudges_what_the_memory_filter_releases(self):+ """The guard runs after the memory filter, on the client-bound frames.++ The filter withholds the proxy's memory call and holds message_delta+ until the turn's outcome is known; that held delta is still the one+ the client merges over message_start, so it must come out nudged.+ """+ from unittest.mock import AsyncMock, MagicMock++ def frame(payload: dict) -> bytes:+ return f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n".encode()++ def tool_block(index: int, tool_id: str, name: str) -> bytes:+ return (+ frame(+ {+ "type": "content_block_start",+ "index": index,+ "content_block": {"type": "tool_use", "id": tool_id, "name": name},+ }+ )+ + frame(+ {+ "type": "content_block_delta",+ "index": index,+ "delta": {"type": "input_json_delta", "partial_json": "{}"},+ }+ )+ + frame({"type": "content_block_stop", "index": index})+ )++ delta = json.loads(_message_delta_event(185_000).split(b"data: ")[1])+ delta["delta"]["stop_reason"] = "tool_use"+ sse = (+ _message_start_event(185_000)+ + tool_block(0, "toolu_mem", "memory_save")+ + tool_block(1, "toolu_bash", "Bash")+ + frame(delta)+ + frame({"type": "message_stop"})+ )+ proxy = self._create_mock_proxy()+ proxy.memory_handler = MagicMock()+ proxy.memory_handler.handle_memory_tool_calls = AsyncMock(+ return_value=[{"type": "tool_result", "tool_use_id": "toolu_mem", "content": "ok"}]+ )+ result = await self._run(+ proxy,+ self._mock_upstream(sse),+ {"x-api-key": "sk-test"},+ memory_user_id="user-1",+ server_memory_tool_names=frozenset({"memory_save"}),+ )+ client_bytes = b"".join([chunk async for chunk in result.body_iterator])+ events = [+ json.loads(line[len(b"data: ") :])+ for line in client_bytes.split(b"\n")+ if line.startswith(b"data: ")+ ]++ assert b"memory_save" not in client_bytes+ assert events[0]["message"]["usage"]["input_tokens"] == int(200_000 * REPORT_FRACTION)+ final = [e for e in events if e["type"] == "message_delta"]+ assert len(final) == 1+ assert final[0]["delta"]["stop_reason"] == "tool_use"+ assert final[0]["usage"]["input_tokens"] == int(200_000 * REPORT_FRACTION)+ proxy.memory_handler.handle_memory_tool_calls.assert_awaited_once()