Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): make handler failures debuggable from the logs (#4127)
Tejas Chopra7dc20b6fe027
diff --git a/headroom/log_safety.py b/headroom/log_safety.pyindex 8282195fc..abe110d39 100644--- a/headroom/log_safety.py+++ b/headroom/log_safety.py@@ -61,6 +61,7 @@ _SUPPRESS_CONTEXT = vars(BaseException)["__suppress_context__"]_TRACEBACK = vars(BaseException)["__traceback__"]_ERRNO = vars(OSError)["errno"]_QUALNAME = vars(type)["__qualname__"]+_CLASS_MODULE = vars(type)["__module__"]# Ints above this many bits have more than 4300 digits, which repr refuses._MAX_ID_INT_BITS = 14_000# The module __dict__ descriptor itself, so a ModuleType subclass cannot override it.@@ -302,6 +303,15 @@ def _int_repr(value: int) -> str:return int.__repr__(value)+def describe_class(cls: type) -> str:+ """Name a class without emitting arbitrary plugin-defined metadata."""+ name = _class_name(cls)+ module = _CLASS_MODULE.__get__(cls)+ if name is None or type(module) is not str or not _QUALIFIED_NAME.fullmatch(module):+ return "<class>"+ return f"{module}.{name}"++def _class_name(cls: type) -> str | None:"""The class's qualified name when it is plain identifiers, else None.diff --git a/headroom/proxy/handlers/_failure_logging.py b/headroom/proxy/handlers/_failure_logging.pynew file mode 100644index 000000000..228f5a753--- /dev/null+++ b/headroom/proxy/handlers/_failure_logging.py@@ -0,0 +1,78 @@+"""Failure logging for the proxy request path.++Exceptions on this path can carry request content: a compression hook sees the+messages, and a catch-all handler catches whatever a compressor or router raised.+So no line, at any level, carries an exception message or a traceback. A failure+is named by the request id, provider, model, hook, stage and+``describe_exception`` (exception types, code locations, errno). That holds for+upstream transport errors too: an httpx ``ProxyError`` quotes the proxy's reason+phrase, which a peer controls. Messages and tracebacks appear only under the+operator's content opt-in, ``HEADROOM_DEBUG_DUMP=full``.+"""++from __future__ import annotations++import logging++from headroom.log_safety import WarnOnce, describe_class, describe_exception, safe_id++logger = logging.getLogger("headroom.proxy")++# One warning per (hook class, stage, exception type), so a hook failing on+# every request warns once and a different hook or stage still warns.+_HOOK_WARNINGS = WarnOnce(256, "failing compression hooks")+++class _HookClassKey:+ """Use class identity without running plugin metaclass hashing/equality.++ Retaining the class for the bounded warning window also prevents id reuse.+ """++ def __init__(self, cls: type) -> None:+ self.cls = cls++ def __hash__(self) -> int:+ return id(self.cls)++ def __eq__(self, other: object) -> bool:+ return type(other) is _HookClassKey and self.cls is other.cls+++def _hook_name(hooks: object) -> str:+ return describe_class(type(hooks))+++def log_hook_failure(+ request_id: str,+ stage: str,+ hooks: object,+ err: BaseException,+ outcome: str = "continuing without it",+) -> None:+ """Log one compression-hook failure: a warning the first time per key, else debug.++ ``outcome`` says what the caller does next, e.g. that it skips compression.+ """+ hook = _hook_name(hooks)+ first = _HOOK_WARNINGS.first((_HookClassKey(type(hooks)), stage, type(err).__name__), logger)+ logger.log(+ logging.WARNING if first else logging.DEBUG,+ "[%s] hook %s %s failed; %s: %s%s",+ request_id,+ hook,+ stage,+ outcome,+ describe_exception(err),+ " (repeats log at debug)" if first else "",+ )+++def log_request_failure(request_id: str, what: str, err: BaseException, **context: object) -> None:+ """Log a request-path failure at error.++ ``what`` names the operation (``"Request"``, ``"OpenAI request"``); ``context``+ adds ``key=value`` fields such as provider and model.+ """+ fields = "".join(f"{key}={safe_id(value)} " for key, value in context.items())+ logger.error("[%s] %s failed: %s%s", request_id, what, fields, describe_exception(err))diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex a5232b2ae..d24c688c2 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -39,6 +39,7 @@ from headroom.ccr.context_tracker import looks_like_claude_code_compact_summaryfrom headroom.ccr.marker_resolution import resolve_markers_in_responsefrom headroom.ccr.tool_calls import has_ccr_retrieve_toolfrom headroom.copilot_auth import apply_copilot_api_auth, is_copilot_upstream_url+from headroom.log_safety import describe_exceptionfrom headroom.pipeline import PipelineStage, summarize_routing_markersfrom headroom.proxy import public_errorsfrom headroom.proxy.anthropic_wire import (@@ -61,6 +62,7 @@ from headroom.proxy.buffered_ccr_response import ()from headroom.proxy.compression_decision import CompressionDecisionfrom headroom.proxy.handlers._debug_dump import _debug_dump_mode, _redact_debug_value+from headroom.proxy.handlers._failure_logging import log_hook_failure, log_request_failurefrom headroom.proxy.helpers import (extract_tags,invalid_request_body_message,@@ -338,8 +340,15 @@ def _dump_prefix_mismatch(request_id: str, current: list, previous: list) -> Nonfh,default=str,)- except Exception: # noqa: BLE001 - debug aid only- pass+ except (OSError, TypeError, ValueError, RecursionError) as e: # debug aid; never fail the turn+ # describe_exception keeps an OSError's errno and strerror; other errors+ # could quote message content, so no message is logged.+ logger.warning(+ "[%s] prefix_mismatch_debug dump to %s failed: %s",+ request_id,+ os.environ.get("HEADROOM_DEBUG_PREFIX_MISMATCH"),+ describe_exception(e),+ )class AnthropicHandlerMixin:@@ -1683,7 +1692,7 @@ class AnthropicHandlerMixin:try:messages = self.config.hooks.pre_compress(messages, _hook_ctx)except Exception as e:- logger.debug(f"[{request_id}] pre_compress hook error: {e}")+ log_hook_failure(request_id, "pre_compress", self.config.hooks, e)else:_hook_ctx = None@@ -1981,21 +1990,31 @@ class AnthropicHandlerMixin:scope=credential_scope_from_headers(headers),)result = None- biases = (- self.config.hooks.compute_biases(messages, _hook_ctx)- if self.config.hooks and _hook_ctx is not None- else None- )+ biases = None# Hard per-message veto. Separate from ``biases`` because a# bias is a soft multiplier that several strategies clamp or# ignore, so it cannot express "leave this one alone".- from headroom.hooks import collect_protected-- protect = (- collect_protected(self.config.hooks, messages, _hook_ctx)- if self.config.hooks and _hook_ctx is not None- else None- )+ protect = None+ if self.config.hooks and _hook_ctx is not None:+ from headroom.hooks import collect_protected++ # A failure still aborts optimization for this request (the+ # outer handler forwards it uncompressed); this only names+ # the stage. Compressing without a veto would be unsafe.+ _hook_stage = "compute_biases"+ try:+ biases = self.config.hooks.compute_biases(messages, _hook_ctx)+ _hook_stage = "protect_messages"+ protect = collect_protected(self.config.hooks, messages, _hook_ctx)+ except Exception as e:+ log_hook_failure(+ request_id,+ _hook_stage,+ self.config.hooks,+ e,+ outcome="skipping compression for this turn",+ )+ raise# F2.1 c5/5: derive the per-request CompressionPolicy# from the auth_mode classified at request entry. The@@ -2377,8 +2396,12 @@ class AnthropicHandlerMixin:# enabled" with Before == After (issue #2357).tags["passthrough_reason"] = "cache_mode_prefix_mismatch"logger.info(- "[%s] Compression skipped: reason=cache_mode_prefix_mismatch",+ "[%s] Compression skipped: reason=cache_mode_prefix_mismatch "+ "model=%s current_len=%d previous_len=%d",request_id,+ model,+ len(original_client_messages),+ len(previous_original_messages),)if os.environ.get("HEADROOM_DEBUG_PREFIX_MISMATCH"):_dump_prefix_mismatch(@@ -2397,9 +2420,11 @@ class AnthropicHandlerMixin:if result and result.waste_signals:waste_signals_dict = result.waste_signals.to_dict()except Exception as e:- # Include type so TimeoutError vs other failures is distinguishable- # in bug reports — str(asyncio.TimeoutError()) is empty otherwise.- logger.warning(f"[{request_id}] Optimization failed: {type(e).__name__}: {e}")+ # Types and code locations only: a compressor or hook error can+ # quote the messages it was given.+ logger.warning(+ "[%s] Optimization failed: %s", request_id, describe_exception(e)+ )# Flag compression failure for observability_compression_failed = True# Split timeout from other errors: a timeout means the@@ -2625,7 +2650,7 @@ class AnthropicHandlerMixin:))except Exception as e:- logger.debug(f"[{request_id}] post_compress hook error: {e}")+ log_hook_failure(request_id, "post_compress", self.config.hooks, e)# CCR Tool Injection: Inject retrieval tool if compression occurred# OR if this session has previously done CCR (PR-B7 sticky-on).@@ -3143,7 +3168,12 @@ class AnthropicHandlerMixin:def _count_tool_tokens(value: object) -> int:try:return tokenizer.count_text(json.dumps(value, default=str))- except Exception:+ except Exception as e:+ logger.debug(+ "[%s] tool token count failed; tool savings not counted: %s",+ request_id,+ describe_exception(e),+ )return 0_tools_compaction_started = time.time()@@ -4614,7 +4644,8 @@ class AnthropicHandlerMixin:# Reuse main client for CCR continuations (connection pooling)logger.info(- f"CCR: Making continuation request with {len(msgs)} messages"+ f"[{request_id}] CCR: Making continuation request "+ f"with {len(msgs)} messages")assert self.http_client is not None, "HTTP client not initialized"# Byte-faithful (PR-A3, fixes P0-2). The CCR@@ -4665,6 +4696,7 @@ class AnthropicHandlerMixin:timeout=self._anthropic_buffered_request_timeout(),)+ cont_response: httpx.Response | None = Nonetry:cont_response = await self._send_buffered_as_stream(_post_continuation,@@ -4674,7 +4706,8 @@ class AnthropicHandlerMixin:request_id=request_id,)logger.info(- f"CCR: Got response status={cont_response.status_code}, "+ f"[{request_id}] CCR: Got response "+ f"status={cont_response.status_code}, "f"content-encoding={cont_response.headers.get('content-encoding')}")if not 200 <= cont_response.status_code < 300:@@ -4683,16 +4716,23 @@ class AnthropicHandlerMixin:f"{cont_response.status_code}")result: dict[str, Any] = cont_response.json()- logger.info("CCR: Parsed JSON successfully")+ logger.info(f"[{request_id}] CCR: Parsed JSON successfully")return resultexcept Exception as e:- resp_headers: str | dict[str, str] = "N/A"- try:- resp_headers = dict(cont_response.headers)- except Exception:- pass- logger.error(- f"CCR: API call failed: {e}, response headers: {resp_headers}"+ # Status and the upstream request id only:+ # the other response headers are not ours+ # to copy into a log.+ upstream_request_id = (+ cont_response.headers.get("request-id")+ if cont_response is not None+ else None+ )+ log_request_failure(+ request_id,+ "CCR continuation",+ e,+ status=getattr(cont_response, "status_code", None),+ upstream_request_id=upstream_request_id,)raise@@ -5530,8 +5570,11 @@ class AnthropicHandlerMixin:raiseexcept Exception as e:await self.metrics.record_failed(provider=provider_name)- # Log full error details internally for debugging- logger.error(f"[{request_id}] Request failed: {type(e).__name__}: {e}")+ # Error names the failure; text and traceback go to debug.+ # `model` is unbound when the failure precedes body parsing.+ log_request_failure(+ request_id, "Request", e, provider=provider_name, model=locals().get("model")+ )# An untrusted TLS-inspection root is the one failure worth# spelling out: it is environmental, never transient, and the@@ -6043,7 +6086,7 @@ class AnthropicHandlerMixin:except Exception as e:await self.metrics.record_failed(provider="anthropic")- logger.error(f"[{request_id}] Batch request failed: {type(e).__name__}: {e}")+ log_request_failure(request_id, "Anthropic batch request", e)return JSONResponse(status_code=502,content={diff --git a/headroom/proxy/handlers/batch.py b/headroom/proxy/handlers/batch.pyindex f028bdfcb..622ad44ee 100644--- a/headroom/proxy/handlers/batch.py+++ b/headroom/proxy/handlers/batch.py@@ -15,6 +15,7 @@ if TYPE_CHECKING:from fastapi.responses import Responsefrom headroom.proxy.auth_mode import classify_client+from headroom.proxy.handlers._failure_logging import log_request_failurefrom headroom.proxy.helpers import (COMPRESSION_TIMEOUT_SECONDS,_headroom_bypass_enabled,@@ -393,7 +394,7 @@ class BatchHandlerMixin:)except Exception as e:- logger.error(f"[{request_id}] Google batch request failed: {e}")+ log_request_failure(request_id, "Google batch request", e, model=model)return JSONResponse(status_code=500,content={diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.pyindex 8a78e0a6b..a9bdb9970 100644--- a/headroom/proxy/handlers/gemini.py+++ b/headroom/proxy/handlers/gemini.py@@ -22,6 +22,7 @@ from headroom.agent_savings import proxy_pipeline_kwargsfrom headroom.copilot_auth import build_copilot_upstream_urlfrom headroom.proxy.auth_mode import classify_clientfrom headroom.proxy.compression_decision import CompressionDecision+from headroom.proxy.handlers._failure_logging import log_request_failurefrom headroom.proxy.helpers import (COMPRESSION_TIMEOUT_SECONDS,extract_tags,@@ -1111,7 +1112,9 @@ class GeminiHandlerMixin:return Response(content=e.content, status_code=e.status_code, headers=response_headers)except Exception as e:await self.metrics.record_failed(provider=provider_name)- logger.error(f"[{request_id}] Gemini request failed: {type(e).__name__}: {e}")+ log_request_failure(+ request_id, "Gemini request", e, provider=provider_name, model=model+ )return JSONResponse(status_code=502,content={@@ -1581,7 +1584,9 @@ class GeminiHandlerMixin:)except Exception as e:await self.metrics.record_failed(provider=provider_name)- logger.error(f"[{request_id}] Gemini countTokens failed: {type(e).__name__}: {e}")+ log_request_failure(+ request_id, "Gemini countTokens", e, provider=provider_name, model=model+ )return JSONResponse(status_code=502,content={diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 64861b0df..145cbcea8 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -25,7 +25,9 @@ from datetime import datetimefrom typing import TYPE_CHECKING, Anyfrom urllib.parse import quote, unquote, urlparse+from headroom.log_safety import describe_exceptionfrom headroom.proxy.conversation_savings import savings_conversation_key+from headroom.proxy.handlers._failure_logging import log_hook_failure, log_request_failurefrom headroom.proxy.helpers import (COMPRESSION_TIMEOUT_SECONDS,_headroom_bypass_enabled,@@ -2702,8 +2704,12 @@ class OpenAIHandlerMixin:- tokenizer.count_text(_json_debug_dumps(working.get("tools"))),)_add_timing("compression_tool_schema_token_count", tool_token_started)- except Exception:- pass+ except Exception as e:+ logger.debug(+ "[%s] tool schema compaction token count failed; savings not counted: %s",+ request_id,+ describe_exception(e),+ )# Layer 2: Tool description truncation (opt-in via# HEADROOM_TOOL_DESC_MAX_CHARS).@@ -2731,10 +2737,17 @@ class OpenAIHandlerMixin:tokenizer.count_text(_json_debug_dumps(payload.get("tools")))- tokenizer.count_text(_json_debug_dumps(working.get("tools"))),)- except Exception:- pass- except Exception:- pass+ except Exception as e:+ logger.debug(+ "[%s] tool desc compaction token count failed; savings not counted: %s",+ request_id,+ describe_exception(e),+ )+ except Exception as e:+ # Tool schemas are client content: log the failure, never its message.+ logger.warning(+ "[%s] tool desc compaction step failed: %s", request_id, describe_exception(e)+ )# Server-side Tool Search deferral (OpenAI Responses, gpt-5.4+): mark# non-core function/MCP tools defer_loading + inject {"type": "tool_search"}@@ -2856,8 +2869,12 @@ class OpenAIHandlerMixin:)if _mt_after < _mt_before:tokens_saved += _mt_before - _mt_after- except Exception:- pass+ except Exception as e:+ logger.debug(+ "[%s] turn-hook message fold token count failed; savings not counted: %s",+ request_id,+ describe_exception(e),+ )modified = Truetransforms.append("openai:responses:turn_hook")@@ -2906,8 +2923,12 @@ class OpenAIHandlerMixin:"compression_tool_schema_attempted_token_count",attempted_token_started,)- except Exception:- pass+ except Exception as e:+ logger.debug(+ "[%s] tool schema attempted-token count failed: %s",+ request_id,+ describe_exception(e),+ )dedupe_started = time.perf_counter()deduped: list[str] = []@@ -3539,12 +3560,15 @@ class OpenAIHandlerMixin:from headroom.hooks import CompressContext, collect_protected_hook_ctx = CompressContext(model=model, provider="openai")+ _hook_stage = "pre_compress"try:messages = self.config.hooks.pre_compress(messages, _hook_ctx)+ _hook_stage = "compute_biases"_hook_biases = self.config.hooks.compute_biases(messages, _hook_ctx)+ _hook_stage = "protect_messages"_hook_protect = collect_protected(self.config.hooks, messages, _hook_ctx)except Exception as e:- logger.debug(f"[{request_id}] Hook error: {e}")+ log_hook_failure(request_id, _hook_stage, self.config.hooks, e)# x-headroom-keep-last-turns: N — trim history before optimization.# Consumed here (after bypass check, after _strip_internal_headers)@@ -3949,7 +3973,7 @@ class OpenAIHandlerMixin:))except Exception as e:- logger.debug(f"[{request_id}] post_compress hook error: {e}")+ log_hook_failure(request_id, "post_compress", self.config.hooks, e)# CCR Tool Injection: Inject retrieval tool if compression occurred# OR if this session has previously done CCR (PR-B7 sticky-on).@@ -4781,8 +4805,12 @@ class OpenAIHandlerMixin:current_forwarded_messages=optimized_messages,),)- except Exception:- pass+ except Exception as e:+ logger.debug(+ "[%s] cache observation (openai chat, backend) failed: %s",+ request_id,+ describe_exception(e),+ )openai_prefix_tracker.update_from_response(cache_read_tokens=cache_read_tokens,@@ -5281,8 +5309,12 @@ class OpenAIHandlerMixin:current_forwarded_messages=optimized_messages,),)- except Exception:- pass+ except Exception as e:+ logger.debug(+ "[%s] cache observation (openai chat, direct) failed: %s",+ request_id,+ describe_exception(e),+ )openai_prefix_tracker.update_from_response(cache_read_tokens=cache_read_tokens,@@ -5475,8 +5507,14 @@ class OpenAIHandlerMixin:)except Exception as e:await self.metrics.record_failed(provider=openai_chat_outcome_provider)- # Log full error details internally for debugging- logger.error(f"[{request_id}] OpenAI request failed: {type(e).__name__}: {e}")+ # Error names the failure; text and traceback go to debug.+ log_request_failure(+ request_id,+ "OpenAI request",+ e,+ provider=openai_chat_outcome_provider,+ model=model,+ )# Return sanitized error message to client (don't expose internal details)return JSONResponse(status_code=502,@@ -6907,7 +6945,7 @@ class OpenAIHandlerMixin:return await _buffered_ccr_operation()except Exception as e:await self.metrics.record_failed(provider="openai")- logger.error(f"[{request_id}] OpenAI responses request failed: {type(e).__name__}: {e}")+ log_request_failure(request_id, "OpenAI responses request", e, model=model)return JSONResponse(status_code=502,content={diff --git a/tests/test_buffered_ccr_upstream_stream.py b/tests/test_buffered_ccr_upstream_stream.pyindex acb3c7822..fb191fa67 100644--- a/tests/test_buffered_ccr_upstream_stream.py+++ b/tests/test_buffered_ccr_upstream_stream.py@@ -505,6 +505,114 @@ def test_the_buffered_turn_and_its_ccr_continuation_stream_upstream() -> None:}+class _FailingContinuationClient:+ """A CCR continuation that dies on the wire or comes back non-2xx."""++ def __init__(self, reply: httpx.Response | None) -> None:+ self.reply = reply++ async def post(self, url, *, content=None, headers=None, timeout=None): # noqa: ANN001+ if self.reply is None:+ raise httpx.ConnectError("peer said CONTENT_CANARY")+ return self.reply++ async def aclose(self) -> None:+ return None++[email protected]("failure", ["transport", "status"])+def test_a_failed_ccr_continuation_logs_only_safe_metadata(monkeypatch, failure) -> None:+ import logging++ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)+ marker = get_compression_store().store(+ original=json.dumps({"secret": "retrieved answer"}), compressed="{}", original_item_count=1+ )+ retrieve_turn = [+ _message_start(),+ {+ "type": "content_block_start",+ "index": 0,+ "content_block": {+ "type": "tool_use",+ "id": "t1",+ "name": "headroom_retrieve",+ "input": {},+ },+ },+ {+ "type": "content_block_delta",+ "index": 0,+ "delta": {"type": "input_json_delta", "partial_json": json.dumps({"hash": marker})},+ },+ {"type": "content_block_stop", "index": 0},+ {+ "type": "message_delta",+ "delta": {"stop_reason": "tool_use", "stop_sequence": None},+ "usage": {"output_tokens": 9},+ },+ {"type": "message_stop"},+ ]+ reply = (+ None+ if failure == "transport"+ else httpx.Response(+ 503,+ content=b"CONTENT_CANARY",+ headers={"request-id": "req_upstream_1", "x-private": "HEADER_CANARY"},+ )+ )+ records: list[logging.LogRecord] = []++ class _Collect(logging.Handler):+ def emit(self, record: logging.LogRecord) -> None:+ records.append(record)++ app = create_app(_config())+ with TestClient(app) as client:+ # Attached after startup: startup reconfigures headroom.* logging.+ logger = logging.getLogger("headroom.proxy")+ handler = _Collect(level=logging.DEBUG)+ logger.addHandler(handler)+ proxy = client.app.state.proxy++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ return _stream_reply(*retrieve_turn)++ proxy._retry_request = _fake_retry # type: ignore[assignment]+ proxy.http_client = _FailingContinuationClient(reply)+ try:+ client.post(+ "/v1/messages",+ headers={"x-api-key": "k", "anthropic-version": "2023-06-01"},+ json={+ "model": "claude-opus-5",+ "max_tokens": 64,+ "stream": True,+ "tools": [create_ccr_tool_definition("anthropic")],+ "messages": [{"role": "user", "content": f"use <<ccr:{marker}>>"}],+ },+ )+ finally:+ logger.removeHandler(handler)++ [failure_log] = [+ logging.Formatter().format(r)+ for r in records+ if "CCR continuation failed" in r.getMessage()+ ]+ if failure == "transport":+ assert "ConnectError" in failure_log+ assert "status=None" in failure_log+ else:+ assert "status=503" in failure_log+ assert "upstream_request_id='req_upstream_1'" in failure_log+ for record in records:+ rendered = logging.Formatter().format(record)+ assert "CONTENT_CANARY" not in rendered+ assert "HEADER_CANARY" not in rendered++def test_a_client_stream_false_request_is_left_alone() -> None:marker = get_compression_store().store(original=json.dumps({"earlier": "tool output"}),diff --git a/tests/test_hook_failure_logging.py b/tests/test_hook_failure_logging.pynew file mode 100644index 000000000..bc9afa7b9--- /dev/null+++ b/tests/test_hook_failure_logging.py@@ -0,0 +1,327 @@+"""Request-path failure logging: named at warning/error, no payload at any level."""++from __future__ import annotations++import contextlib+import logging+import socket+import threading++import httpx+import pytest++from headroom.hooks import CompressionHooks+from headroom.log_safety import WarnOnce+from headroom.proxy.handlers import _failure_logging+from headroom.proxy.handlers._failure_logging import log_hook_failure, log_request_failure++CONTENT = "CANARY-USER-CONTENT"+CREDENTIAL = "sk-CANARY-CREDENTIAL"+++class _Capture(logging.Handler):+ def __init__(self) -> None:+ super().__init__(level=logging.DEBUG)+ self.records: list[logging.LogRecord] = []++ def emit(self, record: logging.LogRecord) -> None:+ self.records.append(record)++ def at(self, level: int) -> list[logging.LogRecord]:+ return [r for r in self.records if r.levelno == level]++ def hook_warnings(self) -> list[str]:+ return [r.getMessage() for r in self.at(logging.WARNING) if "] hook " in r.getMessage()]++ def assert_no_canaries(self) -> None:+ """Every record, rendered the way a log file renders it (traceback included)."""+ formatter = logging.Formatter()+ assert self.records+ for record in self.records:+ text = formatter.format(record)+ assert CONTENT not in text+ assert CREDENTIAL not in text+++def capture(monkeypatch: pytest.MonkeyPatch):+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)+ monkeypatch.setattr(_failure_logging, "_HOOK_WARNINGS", WarnOnce(256, "failing hooks"))+ log = logging.getLogger("headroom.proxy")+ handler = _Capture()+ old_level = log.level+ log.addHandler(handler)+ log.setLevel(logging.DEBUG)+ try:+ yield handler+ finally:+ log.removeHandler(handler)+ log.setLevel(old_level)+++class _HooksA(CompressionHooks):+ pass+++class _HooksB(CompressionHooks):+ pass+++def _raised(exc: BaseException) -> BaseException:+ """``exc`` with a traceback, as a caught exception has."""+ try:+ raise exc+ except BaseException as caught: # noqa: BLE001 - test helper+ return caught+++def test_hook_failure_warns_once_and_no_record_carries_the_payload(capture) -> None:+ for request_id in ("req-1", "req-2"):+ err = _raised(ValueError(f"bad message {CONTENT} key={CREDENTIAL}"))+ log_hook_failure(request_id, "compute_biases", _HooksA(), err)++ (warning,) = capture.hook_warnings()+ assert warning.startswith("[req-1] hook ")+ assert "_HooksA compute_biases failed; continuing without it: ValueError" in warning+ (repeat,) = capture.at(logging.DEBUG)+ assert repeat.getMessage().startswith("[req-2] hook ")+ capture.assert_no_canaries()+++def test_hook_identity_stage_and_error_type_each_get_their_own_warning(capture) -> None:+ log_hook_failure("r1", "pre_compress", _HooksA(), ValueError("x"))+ log_hook_failure("r2", "pre_compress", _HooksB(), ValueError("x")) # other hook+ log_hook_failure("r3", "post_compress", _HooksA(), ValueError("x")) # other stage+ log_hook_failure("r4", "pre_compress", _HooksA(), KeyError("x")) # other type+ log_hook_failure("r5", "pre_compress", _HooksA(), ValueError("y")) # repeat++ assert [w.split()[0] for w in capture.hook_warnings()] == ["[r1]", "[r2]", "[r3]", "[r4]"]+++def test_a_suppressed_warning_is_still_emitted_later(capture) -> None:+ log = logging.getLogger("headroom.proxy")+ log.setLevel(logging.ERROR)+ log_hook_failure("r1", "post_compress", _HooksA(), ValueError("x"))+ log.setLevel(logging.INFO)+ log_hook_failure("r2", "post_compress", _HooksA(), ValueError("x"))++ assert [w.split()[0] for w in capture.hook_warnings()] == ["[r2]"]++[email protected]("level", [logging.INFO, logging.DEBUG])[email protected]("attribute", ["__module__", "__qualname__"])+def test_hook_metadata_cannot_disclose_content_or_forge_log_lines(+ capture, level, attribute+) -> None:+ hooks_type = type("UntrustedHook", (), {})+ setattr(hooks_type, attribute, f"{CONTENT}\nFORGED_LOG_ENTRY")+ logging.getLogger("headroom.proxy").setLevel(level)+ for request_id in ("first", "repeat"):+ log_hook_failure(request_id, "pre_compress", hooks_type(), ValueError("failure"))+ assert len(capture.hook_warnings()) == 1+ capture.assert_no_canaries()+ for record in capture.records:+ rendered = logging.Formatter().format(record)+ assert "FORGED_LOG_ENTRY" not in rendered+ assert "\n" not in rendered+++def test_invalid_hook_names_keep_separate_warning_budgets(capture) -> None:+ for name in (f"{CONTENT}\nfirst", f"{CONTENT}\nsecond"):+ hooks_type = type(name, (), {})+ log_hook_failure("r1", "pre_compress", hooks_type(), ValueError("failure"))+ log_hook_failure("r2", "pre_compress", hooks_type(), ValueError("failure"))+ assert len(capture.hook_warnings()) == 2+ capture.assert_no_canaries()+++def test_hook_warning_keys_do_not_run_metaclass_hash_or_equality(capture) -> None:+ class PluginMeta(type):+ def __hash__(cls):+ raise AssertionError("plugin hash executed")++ def __eq__(cls, other):+ raise AssertionError("plugin equality executed")++ class PluginHook(metaclass=PluginMeta):+ pass++ for request_id in ("first", "repeat"):+ log_hook_failure(request_id, "pre_compress", PluginHook(), ValueError("failure"))+ assert len(capture.hook_warnings()) == 1+ capture.assert_no_canaries()+++def test_request_failure_names_the_failure_without_payload(capture) -> None:+ log_request_failure(+ "r1", "Request", _raised(ValueError(CONTENT)), provider="anthropic", model="m"+ )++ (error,) = (r.getMessage() for r in capture.at(logging.ERROR))+ assert error.startswith("[r1] Request failed: provider='anthropic' model='m' ValueError")+ capture.assert_no_canaries()+++def _proxy_rejecting_connect(reason: str):+ """A real local HTTP proxy that answers every CONNECT with ``502 <reason>``."""+ server = socket.socket()+ server.bind(("127.0.0.1", 0))+ server.listen()+ server.settimeout(5)++ def serve() -> None:+ try:+ conn, _ = server.accept()+ except OSError: # closed before a client came+ return+ with conn:+ conn.recv(65536)+ conn.sendall(f"HTTP/1.1 502 {reason}\r\nContent-Length: 0\r\n\r\n".encode())++ thread = threading.Thread(target=serve, daemon=True)+ thread.start()+ try:+ yield f"http://127.0.0.1:{server.getsockname()[1]}"+ finally:+ server.close()+ thread.join(timeout=5)+++def _proxy_error(reason: str) -> httpx.ProxyError:+ with _proxy_rejecting_connect(reason) as proxy_url:+ with httpx.Client(proxy=proxy_url, timeout=5) as client:+ with pytest.raises(httpx.ProxyError) as caught:+ client.get("https://upstream.example/v1/messages")+ assert reason in str(caught.value) # the peer's text really is in the exception+ return caught.value+++def test_a_proxy_reason_phrase_stays_out_of_the_request_failure_line(capture) -> None:+ log_request_failure("r1", "Request", _proxy_error(CONTENT), provider="anthropic", model="m")++ (error,) = (r.getMessage() for r in capture.at(logging.ERROR))+ assert error.startswith("[r1] Request failed: provider='anthropic' model='m' ProxyError")+ capture.assert_no_canaries()+++def test_the_content_opt_in_restores_the_proxy_text(+ capture, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")+ log_request_failure("r1", "Request", _proxy_error(CONTENT), model="m")++ (error,) = capture.at(logging.ERROR)+ assert CONTENT in logging.Formatter().format(error)+++def _proxy_app(hooks: CompressionHooks, fake_retry, *, optimize: bool): # noqa: ANN001, ANN202+ pytest.importorskip("fastapi")+ from headroom.proxy.server import ProxyConfig, create_app++ app = create_app(+ ProxyConfig(optimize=optimize, cache_enabled=False, rate_limit_enabled=False, hooks=hooks)+ )+ return app+++def _chat_response() -> httpx.Response:+ return httpx.Response(+ 200,+ json={+ "id": "c",+ "object": "chat.completion",+ "model": "gpt-4o",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},+ },+ )+++def test_openai_chat_warns_for_each_failing_stage(capture) -> None:+ """Through the real handler: after a pre_compress failure, a later compute_biases+ failure of the same type still warns, under its own stage."""+ from fastapi.testclient import TestClient++ class Flaky(CompressionHooks):+ failing = "pre_compress"++ def pre_compress(self, messages, ctx): # noqa: ANN001, ANN201+ if self.failing == "pre_compress":+ raise ValueError(CONTENT)+ return messages++ def compute_biases(self, messages, ctx): # noqa: ANN001, ANN201+ raise ValueError(CONTENT)++ async def fake_retry(method, url, headers, body, *args, **kwargs): # noqa: ANN001, ANN202+ return _chat_response()++ hooks = Flaky()+ with TestClient(_proxy_app(hooks, fake_retry, optimize=False)) as client:+ # Proxy startup may reset logger levels; re-assert capture afterwards.+ logging.getLogger("headroom.proxy").setLevel(logging.DEBUG)+ client.app.state.proxy._retry_request = fake_retry+ for failing in ("pre_compress", "compute_biases"):+ hooks.failing = failing+ resp = client.post(+ "/v1/chat/completions",+ json={"model": "gpt-4o", "messages": [{"role": "user", "content": CONTENT}]},+ headers={"Authorization": f"Bearer {CREDENTIAL}"},+ )+ assert resp.status_code == 200, resp.text++ warnings = capture.hook_warnings()+ assert len(warnings) == 2+ assert "Flaky pre_compress failed" in warnings[0]+ assert "Flaky compute_biases failed" in warnings[1]+ capture.assert_no_canaries()+++def test_anthropic_bias_failure_says_compression_is_skipped(capture) -> None:+ """Through the real handler: Anthropic aborts optimization when a bias hook fails,+ so the warning must say compression is skipped, not that it continues."""+ from fastapi.testclient import TestClient++ class BrokenBiases(CompressionHooks):+ def compute_biases(self, messages, ctx): # noqa: ANN001, ANN201+ raise ValueError(CONTENT)++ async def fake_retry(method, url, headers, body, *args, **kwargs): # noqa: ANN001, ANN202+ return httpx.Response(+ 200,+ json={+ "id": "msg_1",+ "type": "message",+ "role": "assistant",+ "model": "claude-sonnet-4-5",+ "content": [{"type": "text", "text": "ok"}],+ "stop_reason": "end_turn",+ "usage": {"input_tokens": 5, "output_tokens": 1},+ },+ )++ with TestClient(_proxy_app(BrokenBiases(), fake_retry, optimize=True)) as client:+ logging.getLogger("headroom.proxy").setLevel(logging.DEBUG)+ client.app.state.proxy._retry_request = fake_retry+ resp = client.post(+ "/v1/messages",+ json={+ "model": "claude-sonnet-4-5",+ "max_tokens": 16,+ "messages": [{"role": "user", "content": CONTENT}],+ },+ headers={"x-api-key": CREDENTIAL, "anthropic-version": "2023-06-01"},+ )++ assert resp.status_code == 200, resp.text+ (warning,) = capture.hook_warnings()+ assert "BrokenBiases compute_biases failed; skipping compression for this turn" in warning+ assert "continuing without it" not in warning+ capture.assert_no_canaries()diff --git a/tests/test_proxy_anthropic_compression_diagnostics.py b/tests/test_proxy_anthropic_compression_diagnostics.pyindex dad3f75bb..3f80d2c39 100644--- a/tests/test_proxy_anthropic_compression_diagnostics.py+++ b/tests/test_proxy_anthropic_compression_diagnostics.py@@ -142,7 +142,7 @@ def test_optimization_failure_logs_exception_type() -> None:assert response.status_code == 200, response.textwarning_msgs = [- call.args[0]+ str(call.args[0]) % call.args[1:]for call in mock_warning.call_args_listif call.args and "Optimization failed" in str(call.args[0])]