Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): make handler failures debuggable from the logs (#4127)

Tejas Chopra7dc20b6fe027
diff --git a/headroom/log_safety.py b/headroom/log_safety.py
index 8282195fc..abe110d39 100644
--- a/headroom/log_safety.py
+++ b/headroom/log_safety.py
@@ -61,6 +61,7 @@ _SUPPRESS_CONTEXT = vars(BaseException)["__suppress_context__"]
_TRACEBACK = vars(BaseException)["__traceback__"]
_ERRNO = vars(OSError)["errno"]
_QUALNAME = vars(type)["__qualname__"]
+_CLASS_MODULE = vars(type)["__module__"]
# Ints above this many bits have more than 4300 digits, which repr refuses.
_MAX_ID_INT_BITS = 14_000
# The module __dict__ descriptor itself, so a ModuleType subclass cannot override it.
@@ -302,6 +303,15 @@ def _int_repr(value: int) -> str:
return int.__repr__(value)
+def describe_class(cls: type) -> str:
+ """Name a class without emitting arbitrary plugin-defined metadata."""
+ name = _class_name(cls)
+ module = _CLASS_MODULE.__get__(cls)
+ if name is None or type(module) is not str or not _QUALIFIED_NAME.fullmatch(module):
+ return "<class>"
+ return f"{module}.{name}"
+
+
def _class_name(cls: type) -> str | None:
"""The class's qualified name when it is plain identifiers, else None.
diff --git a/headroom/proxy/handlers/_failure_logging.py b/headroom/proxy/handlers/_failure_logging.py
new file mode 100644
index 000000000..228f5a753
--- /dev/null
+++ b/headroom/proxy/handlers/_failure_logging.py
@@ -0,0 +1,78 @@
+"""Failure logging for the proxy request path.
+
+Exceptions on this path can carry request content: a compression hook sees the
+messages, and a catch-all handler catches whatever a compressor or router raised.
+So no line, at any level, carries an exception message or a traceback. A failure
+is named by the request id, provider, model, hook, stage and
+``describe_exception`` (exception types, code locations, errno). That holds for
+upstream transport errors too: an httpx ``ProxyError`` quotes the proxy's reason
+phrase, which a peer controls. Messages and tracebacks appear only under the
+operator's content opt-in, ``HEADROOM_DEBUG_DUMP=full``.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from headroom.log_safety import WarnOnce, describe_class, describe_exception, safe_id
+
+logger = logging.getLogger("headroom.proxy")
+
+# One warning per (hook class, stage, exception type), so a hook failing on
+# every request warns once and a different hook or stage still warns.
+_HOOK_WARNINGS = WarnOnce(256, "failing compression hooks")
+
+
+class _HookClassKey:
+ """Use class identity without running plugin metaclass hashing/equality.
+
+ Retaining the class for the bounded warning window also prevents id reuse.
+ """
+
+ def __init__(self, cls: type) -> None:
+ self.cls = cls
+
+ def __hash__(self) -> int:
+ return id(self.cls)
+
+ def __eq__(self, other: object) -> bool:
+ return type(other) is _HookClassKey and self.cls is other.cls
+
+
+def _hook_name(hooks: object) -> str:
+ return describe_class(type(hooks))
+
+
+def log_hook_failure(
+ request_id: str,
+ stage: str,
+ hooks: object,
+ err: BaseException,
+ outcome: str = "continuing without it",
+) -> None:
+ """Log one compression-hook failure: a warning the first time per key, else debug.
+
+ ``outcome`` says what the caller does next, e.g. that it skips compression.
+ """
+ hook = _hook_name(hooks)
+ first = _HOOK_WARNINGS.first((_HookClassKey(type(hooks)), stage, type(err).__name__), logger)
+ logger.log(
+ logging.WARNING if first else logging.DEBUG,
+ "[%s] hook %s %s failed; %s: %s%s",
+ request_id,
+ hook,
+ stage,
+ outcome,
+ describe_exception(err),
+ " (repeats log at debug)" if first else "",
+ )
+
+
+def log_request_failure(request_id: str, what: str, err: BaseException, **context: object) -> None:
+ """Log a request-path failure at error.
+
+ ``what`` names the operation (``"Request"``, ``"OpenAI request"``); ``context``
+ adds ``key=value`` fields such as provider and model.
+ """
+ fields = "".join(f"{key}={safe_id(value)} " for key, value in context.items())
+ logger.error("[%s] %s failed: %s%s", request_id, what, fields, describe_exception(err))
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index a5232b2ae..d24c688c2 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -39,6 +39,7 @@ from headroom.ccr.context_tracker import looks_like_claude_code_compact_summary
from headroom.ccr.marker_resolution import resolve_markers_in_response
from headroom.ccr.tool_calls import has_ccr_retrieve_tool
from headroom.copilot_auth import apply_copilot_api_auth, is_copilot_upstream_url
+from headroom.log_safety import describe_exception
from headroom.pipeline import PipelineStage, summarize_routing_markers
from headroom.proxy import public_errors
from headroom.proxy.anthropic_wire import (
@@ -61,6 +62,7 @@ from headroom.proxy.buffered_ccr_response import (
)
from headroom.proxy.compression_decision import CompressionDecision
from headroom.proxy.handlers._debug_dump import _debug_dump_mode, _redact_debug_value
+from headroom.proxy.handlers._failure_logging import log_hook_failure, log_request_failure
from headroom.proxy.helpers import (
extract_tags,
invalid_request_body_message,
@@ -338,8 +340,15 @@ def _dump_prefix_mismatch(request_id: str, current: list, previous: list) -> Non
fh,
default=str,
)
- except Exception: # noqa: BLE001 - debug aid only
- pass
+ except (OSError, TypeError, ValueError, RecursionError) as e: # debug aid; never fail the turn
+ # describe_exception keeps an OSError's errno and strerror; other errors
+ # could quote message content, so no message is logged.
+ logger.warning(
+ "[%s] prefix_mismatch_debug dump to %s failed: %s",
+ request_id,
+ os.environ.get("HEADROOM_DEBUG_PREFIX_MISMATCH"),
+ describe_exception(e),
+ )
class AnthropicHandlerMixin:
@@ -1683,7 +1692,7 @@ class AnthropicHandlerMixin:
try:
messages = self.config.hooks.pre_compress(messages, _hook_ctx)
except Exception as e:
- logger.debug(f"[{request_id}] pre_compress hook error: {e}")
+ log_hook_failure(request_id, "pre_compress", self.config.hooks, e)
else:
_hook_ctx = None
@@ -1981,21 +1990,31 @@ class AnthropicHandlerMixin:
scope=credential_scope_from_headers(headers),
)
result = None
- biases = (
- self.config.hooks.compute_biases(messages, _hook_ctx)
- if self.config.hooks and _hook_ctx is not None
- else None
- )
+ biases = None
# Hard per-message veto. Separate from ``biases`` because a
# bias is a soft multiplier that several strategies clamp or
# ignore, so it cannot express "leave this one alone".
- from headroom.hooks import collect_protected
-
- protect = (
- collect_protected(self.config.hooks, messages, _hook_ctx)
- if self.config.hooks and _hook_ctx is not None
- else None
- )
+ protect = None
+ if self.config.hooks and _hook_ctx is not None:
+ from headroom.hooks import collect_protected
+
+ # A failure still aborts optimization for this request (the
+ # outer handler forwards it uncompressed); this only names
+ # the stage. Compressing without a veto would be unsafe.
+ _hook_stage = "compute_biases"
+ try:
+ biases = self.config.hooks.compute_biases(messages, _hook_ctx)
+ _hook_stage = "protect_messages"
+ protect = collect_protected(self.config.hooks, messages, _hook_ctx)
+ except Exception as e:
+ log_hook_failure(
+ request_id,
+ _hook_stage,
+ self.config.hooks,
+ e,
+ outcome="skipping compression for this turn",
+ )
+ raise
# F2.1 c5/5: derive the per-request CompressionPolicy
# from the auth_mode classified at request entry. The
@@ -2377,8 +2396,12 @@ class AnthropicHandlerMixin:
# enabled" with Before == After (issue #2357).
tags["passthrough_reason"] = "cache_mode_prefix_mismatch"
logger.info(
- "[%s] Compression skipped: reason=cache_mode_prefix_mismatch",
+ "[%s] Compression skipped: reason=cache_mode_prefix_mismatch "
+ "model=%s current_len=%d previous_len=%d",
request_id,
+ model,
+ len(original_client_messages),
+ len(previous_original_messages),
)
if os.environ.get("HEADROOM_DEBUG_PREFIX_MISMATCH"):
_dump_prefix_mismatch(
@@ -2397,9 +2420,11 @@ class AnthropicHandlerMixin:
if result and result.waste_signals:
waste_signals_dict = result.waste_signals.to_dict()
except Exception as e:
- # Include type so TimeoutError vs other failures is distinguishable
- # in bug reports — str(asyncio.TimeoutError()) is empty otherwise.
- logger.warning(f"[{request_id}] Optimization failed: {type(e).__name__}: {e}")
+ # Types and code locations only: a compressor or hook error can
+ # quote the messages it was given.
+ logger.warning(
+ "[%s] Optimization failed: %s", request_id, describe_exception(e)
+ )
# Flag compression failure for observability
_compression_failed = True
# Split timeout from other errors: a timeout means the
@@ -2625,7 +2650,7 @@ class AnthropicHandlerMixin:
)
)
except Exception as e:
- logger.debug(f"[{request_id}] post_compress hook error: {e}")
+ log_hook_failure(request_id, "post_compress", self.config.hooks, e)
# CCR Tool Injection: Inject retrieval tool if compression occurred
# OR if this session has previously done CCR (PR-B7 sticky-on).
@@ -3143,7 +3168,12 @@ class AnthropicHandlerMixin:
def _count_tool_tokens(value: object) -> int:
try:
return tokenizer.count_text(json.dumps(value, default=str))
- except Exception:
+ except Exception as e:
+ logger.debug(
+ "[%s] tool token count failed; tool savings not counted: %s",
+ request_id,
+ describe_exception(e),
+ )
return 0
_tools_compaction_started = time.time()
@@ -4614,7 +4644,8 @@ class AnthropicHandlerMixin:
# Reuse main client for CCR continuations (connection pooling)
logger.info(
- f"CCR: Making continuation request with {len(msgs)} messages"
+ f"[{request_id}] CCR: Making continuation request "
+ f"with {len(msgs)} messages"
)
assert self.http_client is not None, "HTTP client not initialized"
# Byte-faithful (PR-A3, fixes P0-2). The CCR
@@ -4665,6 +4696,7 @@ class AnthropicHandlerMixin:
timeout=self._anthropic_buffered_request_timeout(),
)
+ cont_response: httpx.Response | None = None
try:
cont_response = await self._send_buffered_as_stream(
_post_continuation,
@@ -4674,7 +4706,8 @@ class AnthropicHandlerMixin:
request_id=request_id,
)
logger.info(
- f"CCR: Got response status={cont_response.status_code}, "
+ f"[{request_id}] CCR: Got response "
+ f"status={cont_response.status_code}, "
f"content-encoding={cont_response.headers.get('content-encoding')}"
)
if not 200 <= cont_response.status_code < 300:
@@ -4683,16 +4716,23 @@ class AnthropicHandlerMixin:
f"{cont_response.status_code}"
)
result: dict[str, Any] = cont_response.json()
- logger.info("CCR: Parsed JSON successfully")
+ logger.info(f"[{request_id}] CCR: Parsed JSON successfully")
return result
except Exception as e:
- resp_headers: str | dict[str, str] = "N/A"
- try:
- resp_headers = dict(cont_response.headers)
- except Exception:
- pass
- logger.error(
- f"CCR: API call failed: {e}, response headers: {resp_headers}"
+ # Status and the upstream request id only:
+ # the other response headers are not ours
+ # to copy into a log.
+ upstream_request_id = (
+ cont_response.headers.get("request-id")
+ if cont_response is not None
+ else None
+ )
+ log_request_failure(
+ request_id,
+ "CCR continuation",
+ e,
+ status=getattr(cont_response, "status_code", None),
+ upstream_request_id=upstream_request_id,
)
raise
@@ -5530,8 +5570,11 @@ class AnthropicHandlerMixin:
raise
except Exception as e:
await self.metrics.record_failed(provider=provider_name)
- # Log full error details internally for debugging
- logger.error(f"[{request_id}] Request failed: {type(e).__name__}: {e}")
+ # Error names the failure; text and traceback go to debug.
+ # `model` is unbound when the failure precedes body parsing.
+ log_request_failure(
+ request_id, "Request", e, provider=provider_name, model=locals().get("model")
+ )
# An untrusted TLS-inspection root is the one failure worth
# spelling out: it is environmental, never transient, and the
@@ -6043,7 +6086,7 @@ class AnthropicHandlerMixin:
except Exception as e:
await self.metrics.record_failed(provider="anthropic")
- logger.error(f"[{request_id}] Batch request failed: {type(e).__name__}: {e}")
+ log_request_failure(request_id, "Anthropic batch request", e)
return JSONResponse(
status_code=502,
content={
diff --git a/headroom/proxy/handlers/batch.py b/headroom/proxy/handlers/batch.py
index f028bdfcb..622ad44ee 100644
--- a/headroom/proxy/handlers/batch.py
+++ b/headroom/proxy/handlers/batch.py
@@ -15,6 +15,7 @@ if TYPE_CHECKING:
from fastapi.responses import Response
from headroom.proxy.auth_mode import classify_client
+from headroom.proxy.handlers._failure_logging import log_request_failure
from headroom.proxy.helpers import (
COMPRESSION_TIMEOUT_SECONDS,
_headroom_bypass_enabled,
@@ -393,7 +394,7 @@ class BatchHandlerMixin:
)
except Exception as e:
- logger.error(f"[{request_id}] Google batch request failed: {e}")
+ log_request_failure(request_id, "Google batch request", e, model=model)
return JSONResponse(
status_code=500,
content={
diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.py
index 8a78e0a6b..a9bdb9970 100644
--- a/headroom/proxy/handlers/gemini.py
+++ b/headroom/proxy/handlers/gemini.py
@@ -22,6 +22,7 @@ from headroom.agent_savings import proxy_pipeline_kwargs
from headroom.copilot_auth import build_copilot_upstream_url
from headroom.proxy.auth_mode import classify_client
from headroom.proxy.compression_decision import CompressionDecision
+from headroom.proxy.handlers._failure_logging import log_request_failure
from headroom.proxy.helpers import (
COMPRESSION_TIMEOUT_SECONDS,
extract_tags,
@@ -1111,7 +1112,9 @@ class GeminiHandlerMixin:
return Response(content=e.content, status_code=e.status_code, headers=response_headers)
except Exception as e:
await self.metrics.record_failed(provider=provider_name)
- logger.error(f"[{request_id}] Gemini request failed: {type(e).__name__}: {e}")
+ log_request_failure(
+ request_id, "Gemini request", e, provider=provider_name, model=model
+ )
return JSONResponse(
status_code=502,
content={
@@ -1581,7 +1584,9 @@ class GeminiHandlerMixin:
)
except Exception as e:
await self.metrics.record_failed(provider=provider_name)
- logger.error(f"[{request_id}] Gemini countTokens failed: {type(e).__name__}: {e}")
+ log_request_failure(
+ request_id, "Gemini countTokens", e, provider=provider_name, model=model
+ )
return JSONResponse(
status_code=502,
content={
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 64861b0df..145cbcea8 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -25,7 +25,9 @@ from datetime import datetime
from typing import TYPE_CHECKING, Any
from urllib.parse import quote, unquote, urlparse
+from headroom.log_safety import describe_exception
from headroom.proxy.conversation_savings import savings_conversation_key
+from headroom.proxy.handlers._failure_logging import log_hook_failure, log_request_failure
from headroom.proxy.helpers import (
COMPRESSION_TIMEOUT_SECONDS,
_headroom_bypass_enabled,
@@ -2702,8 +2704,12 @@ class OpenAIHandlerMixin:
- tokenizer.count_text(_json_debug_dumps(working.get("tools"))),
)
_add_timing("compression_tool_schema_token_count", tool_token_started)
- except Exception:
- pass
+ except Exception as e:
+ logger.debug(
+ "[%s] tool schema compaction token count failed; savings not counted: %s",
+ request_id,
+ describe_exception(e),
+ )
# Layer 2: Tool description truncation (opt-in via
# HEADROOM_TOOL_DESC_MAX_CHARS).
@@ -2731,10 +2737,17 @@ class OpenAIHandlerMixin:
tokenizer.count_text(_json_debug_dumps(payload.get("tools")))
- tokenizer.count_text(_json_debug_dumps(working.get("tools"))),
)
- except Exception:
- pass
- except Exception:
- pass
+ except Exception as e:
+ logger.debug(
+ "[%s] tool desc compaction token count failed; savings not counted: %s",
+ request_id,
+ describe_exception(e),
+ )
+ except Exception as e:
+ # Tool schemas are client content: log the failure, never its message.
+ logger.warning(
+ "[%s] tool desc compaction step failed: %s", request_id, describe_exception(e)
+ )
# Server-side Tool Search deferral (OpenAI Responses, gpt-5.4+): mark
# non-core function/MCP tools defer_loading + inject {"type": "tool_search"}
@@ -2856,8 +2869,12 @@ class OpenAIHandlerMixin:
)
if _mt_after < _mt_before:
tokens_saved += _mt_before - _mt_after
- except Exception:
- pass
+ except Exception as e:
+ logger.debug(
+ "[%s] turn-hook message fold token count failed; savings not counted: %s",
+ request_id,
+ describe_exception(e),
+ )
modified = True
transforms.append("openai:responses:turn_hook")
@@ -2906,8 +2923,12 @@ class OpenAIHandlerMixin:
"compression_tool_schema_attempted_token_count",
attempted_token_started,
)
- except Exception:
- pass
+ except Exception as e:
+ logger.debug(
+ "[%s] tool schema attempted-token count failed: %s",
+ request_id,
+ describe_exception(e),
+ )
dedupe_started = time.perf_counter()
deduped: list[str] = []
@@ -3539,12 +3560,15 @@ class OpenAIHandlerMixin:
from headroom.hooks import CompressContext, collect_protected
_hook_ctx = CompressContext(model=model, provider="openai")
+ _hook_stage = "pre_compress"
try:
messages = self.config.hooks.pre_compress(messages, _hook_ctx)
+ _hook_stage = "compute_biases"
_hook_biases = self.config.hooks.compute_biases(messages, _hook_ctx)
+ _hook_stage = "protect_messages"
_hook_protect = collect_protected(self.config.hooks, messages, _hook_ctx)
except Exception as e:
- logger.debug(f"[{request_id}] Hook error: {e}")
+ log_hook_failure(request_id, _hook_stage, self.config.hooks, e)
# x-headroom-keep-last-turns: N — trim history before optimization.
# Consumed here (after bypass check, after _strip_internal_headers)
@@ -3949,7 +3973,7 @@ class OpenAIHandlerMixin:
)
)
except Exception as e:
- logger.debug(f"[{request_id}] post_compress hook error: {e}")
+ log_hook_failure(request_id, "post_compress", self.config.hooks, e)
# CCR Tool Injection: Inject retrieval tool if compression occurred
# OR if this session has previously done CCR (PR-B7 sticky-on).
@@ -4781,8 +4805,12 @@ class OpenAIHandlerMixin:
current_forwarded_messages=optimized_messages,
),
)
- except Exception:
- pass
+ except Exception as e:
+ logger.debug(
+ "[%s] cache observation (openai chat, backend) failed: %s",
+ request_id,
+ describe_exception(e),
+ )
openai_prefix_tracker.update_from_response(
cache_read_tokens=cache_read_tokens,
@@ -5281,8 +5309,12 @@ class OpenAIHandlerMixin:
current_forwarded_messages=optimized_messages,
),
)
- except Exception:
- pass
+ except Exception as e:
+ logger.debug(
+ "[%s] cache observation (openai chat, direct) failed: %s",
+ request_id,
+ describe_exception(e),
+ )
openai_prefix_tracker.update_from_response(
cache_read_tokens=cache_read_tokens,
@@ -5475,8 +5507,14 @@ class OpenAIHandlerMixin:
)
except Exception as e:
await self.metrics.record_failed(provider=openai_chat_outcome_provider)
- # Log full error details internally for debugging
- logger.error(f"[{request_id}] OpenAI request failed: {type(e).__name__}: {e}")
+ # Error names the failure; text and traceback go to debug.
+ log_request_failure(
+ request_id,
+ "OpenAI request",
+ e,
+ provider=openai_chat_outcome_provider,
+ model=model,
+ )
# Return sanitized error message to client (don't expose internal details)
return JSONResponse(
status_code=502,
@@ -6907,7 +6945,7 @@ class OpenAIHandlerMixin:
return await _buffered_ccr_operation()
except Exception as e:
await self.metrics.record_failed(provider="openai")
- logger.error(f"[{request_id}] OpenAI responses request failed: {type(e).__name__}: {e}")
+ log_request_failure(request_id, "OpenAI responses request", e, model=model)
return JSONResponse(
status_code=502,
content={
diff --git a/tests/test_buffered_ccr_upstream_stream.py b/tests/test_buffered_ccr_upstream_stream.py
index acb3c7822..fb191fa67 100644
--- a/tests/test_buffered_ccr_upstream_stream.py
+++ b/tests/test_buffered_ccr_upstream_stream.py
@@ -505,6 +505,114 @@ def test_the_buffered_turn_and_its_ccr_continuation_stream_upstream() -> None:
}
+class _FailingContinuationClient:
+ """A CCR continuation that dies on the wire or comes back non-2xx."""
+
+ def __init__(self, reply: httpx.Response | None) -> None:
+ self.reply = reply
+
+ async def post(self, url, *, content=None, headers=None, timeout=None): # noqa: ANN001
+ if self.reply is None:
+ raise httpx.ConnectError("peer said CONTENT_CANARY")
+ return self.reply
+
+ async def aclose(self) -> None:
+ return None
+
+
[email protected]("failure", ["transport", "status"])
+def test_a_failed_ccr_continuation_logs_only_safe_metadata(monkeypatch, failure) -> None:
+ import logging
+
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+ marker = get_compression_store().store(
+ original=json.dumps({"secret": "retrieved answer"}), compressed="{}", original_item_count=1
+ )
+ retrieve_turn = [
+ _message_start(),
+ {
+ "type": "content_block_start",
+ "index": 0,
+ "content_block": {
+ "type": "tool_use",
+ "id": "t1",
+ "name": "headroom_retrieve",
+ "input": {},
+ },
+ },
+ {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "input_json_delta", "partial_json": json.dumps({"hash": marker})},
+ },
+ {"type": "content_block_stop", "index": 0},
+ {
+ "type": "message_delta",
+ "delta": {"stop_reason": "tool_use", "stop_sequence": None},
+ "usage": {"output_tokens": 9},
+ },
+ {"type": "message_stop"},
+ ]
+ reply = (
+ None
+ if failure == "transport"
+ else httpx.Response(
+ 503,
+ content=b"CONTENT_CANARY",
+ headers={"request-id": "req_upstream_1", "x-private": "HEADER_CANARY"},
+ )
+ )
+ records: list[logging.LogRecord] = []
+
+ class _Collect(logging.Handler):
+ def emit(self, record: logging.LogRecord) -> None:
+ records.append(record)
+
+ app = create_app(_config())
+ with TestClient(app) as client:
+ # Attached after startup: startup reconfigures headroom.* logging.
+ logger = logging.getLogger("headroom.proxy")
+ handler = _Collect(level=logging.DEBUG)
+ logger.addHandler(handler)
+ proxy = client.app.state.proxy
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ return _stream_reply(*retrieve_turn)
+
+ proxy._retry_request = _fake_retry # type: ignore[assignment]
+ proxy.http_client = _FailingContinuationClient(reply)
+ try:
+ client.post(
+ "/v1/messages",
+ headers={"x-api-key": "k", "anthropic-version": "2023-06-01"},
+ json={
+ "model": "claude-opus-5",
+ "max_tokens": 64,
+ "stream": True,
+ "tools": [create_ccr_tool_definition("anthropic")],
+ "messages": [{"role": "user", "content": f"use <<ccr:{marker}>>"}],
+ },
+ )
+ finally:
+ logger.removeHandler(handler)
+
+ [failure_log] = [
+ logging.Formatter().format(r)
+ for r in records
+ if "CCR continuation failed" in r.getMessage()
+ ]
+ if failure == "transport":
+ assert "ConnectError" in failure_log
+ assert "status=None" in failure_log
+ else:
+ assert "status=503" in failure_log
+ assert "upstream_request_id='req_upstream_1'" in failure_log
+ for record in records:
+ rendered = logging.Formatter().format(record)
+ assert "CONTENT_CANARY" not in rendered
+ assert "HEADER_CANARY" not in rendered
+
+
def test_a_client_stream_false_request_is_left_alone() -> None:
marker = get_compression_store().store(
original=json.dumps({"earlier": "tool output"}),
diff --git a/tests/test_hook_failure_logging.py b/tests/test_hook_failure_logging.py
new file mode 100644
index 000000000..bc9afa7b9
--- /dev/null
+++ b/tests/test_hook_failure_logging.py
@@ -0,0 +1,327 @@
+"""Request-path failure logging: named at warning/error, no payload at any level."""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import socket
+import threading
+
+import httpx
+import pytest
+
+from headroom.hooks import CompressionHooks
+from headroom.log_safety import WarnOnce
+from headroom.proxy.handlers import _failure_logging
+from headroom.proxy.handlers._failure_logging import log_hook_failure, log_request_failure
+
+CONTENT = "CANARY-USER-CONTENT"
+CREDENTIAL = "sk-CANARY-CREDENTIAL"
+
+
+class _Capture(logging.Handler):
+ def __init__(self) -> None:
+ super().__init__(level=logging.DEBUG)
+ self.records: list[logging.LogRecord] = []
+
+ def emit(self, record: logging.LogRecord) -> None:
+ self.records.append(record)
+
+ def at(self, level: int) -> list[logging.LogRecord]:
+ return [r for r in self.records if r.levelno == level]
+
+ def hook_warnings(self) -> list[str]:
+ return [r.getMessage() for r in self.at(logging.WARNING) if "] hook " in r.getMessage()]
+
+ def assert_no_canaries(self) -> None:
+ """Every record, rendered the way a log file renders it (traceback included)."""
+ formatter = logging.Formatter()
+ assert self.records
+ for record in self.records:
+ text = formatter.format(record)
+ assert CONTENT not in text
+ assert CREDENTIAL not in text
+
+
+def capture(monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+ monkeypatch.setattr(_failure_logging, "_HOOK_WARNINGS", WarnOnce(256, "failing hooks"))
+ log = logging.getLogger("headroom.proxy")
+ handler = _Capture()
+ old_level = log.level
+ log.addHandler(handler)
+ log.setLevel(logging.DEBUG)
+ try:
+ yield handler
+ finally:
+ log.removeHandler(handler)
+ log.setLevel(old_level)
+
+
+class _HooksA(CompressionHooks):
+ pass
+
+
+class _HooksB(CompressionHooks):
+ pass
+
+
+def _raised(exc: BaseException) -> BaseException:
+ """``exc`` with a traceback, as a caught exception has."""
+ try:
+ raise exc
+ except BaseException as caught: # noqa: BLE001 - test helper
+ return caught
+
+
+def test_hook_failure_warns_once_and_no_record_carries_the_payload(capture) -> None:
+ for request_id in ("req-1", "req-2"):
+ err = _raised(ValueError(f"bad message {CONTENT} key={CREDENTIAL}"))
+ log_hook_failure(request_id, "compute_biases", _HooksA(), err)
+
+ (warning,) = capture.hook_warnings()
+ assert warning.startswith("[req-1] hook ")
+ assert "_HooksA compute_biases failed; continuing without it: ValueError" in warning
+ (repeat,) = capture.at(logging.DEBUG)
+ assert repeat.getMessage().startswith("[req-2] hook ")
+ capture.assert_no_canaries()
+
+
+def test_hook_identity_stage_and_error_type_each_get_their_own_warning(capture) -> None:
+ log_hook_failure("r1", "pre_compress", _HooksA(), ValueError("x"))
+ log_hook_failure("r2", "pre_compress", _HooksB(), ValueError("x")) # other hook
+ log_hook_failure("r3", "post_compress", _HooksA(), ValueError("x")) # other stage
+ log_hook_failure("r4", "pre_compress", _HooksA(), KeyError("x")) # other type
+ log_hook_failure("r5", "pre_compress", _HooksA(), ValueError("y")) # repeat
+
+ assert [w.split()[0] for w in capture.hook_warnings()] == ["[r1]", "[r2]", "[r3]", "[r4]"]
+
+
+def test_a_suppressed_warning_is_still_emitted_later(capture) -> None:
+ log = logging.getLogger("headroom.proxy")
+ log.setLevel(logging.ERROR)
+ log_hook_failure("r1", "post_compress", _HooksA(), ValueError("x"))
+ log.setLevel(logging.INFO)
+ log_hook_failure("r2", "post_compress", _HooksA(), ValueError("x"))
+
+ assert [w.split()[0] for w in capture.hook_warnings()] == ["[r2]"]
+
+
[email protected]("level", [logging.INFO, logging.DEBUG])
[email protected]("attribute", ["__module__", "__qualname__"])
+def test_hook_metadata_cannot_disclose_content_or_forge_log_lines(
+ capture, level, attribute
+) -> None:
+ hooks_type = type("UntrustedHook", (), {})
+ setattr(hooks_type, attribute, f"{CONTENT}\nFORGED_LOG_ENTRY")
+ logging.getLogger("headroom.proxy").setLevel(level)
+ for request_id in ("first", "repeat"):
+ log_hook_failure(request_id, "pre_compress", hooks_type(), ValueError("failure"))
+ assert len(capture.hook_warnings()) == 1
+ capture.assert_no_canaries()
+ for record in capture.records:
+ rendered = logging.Formatter().format(record)
+ assert "FORGED_LOG_ENTRY" not in rendered
+ assert "\n" not in rendered
+
+
+def test_invalid_hook_names_keep_separate_warning_budgets(capture) -> None:
+ for name in (f"{CONTENT}\nfirst", f"{CONTENT}\nsecond"):
+ hooks_type = type(name, (), {})
+ log_hook_failure("r1", "pre_compress", hooks_type(), ValueError("failure"))
+ log_hook_failure("r2", "pre_compress", hooks_type(), ValueError("failure"))
+ assert len(capture.hook_warnings()) == 2
+ capture.assert_no_canaries()
+
+
+def test_hook_warning_keys_do_not_run_metaclass_hash_or_equality(capture) -> None:
+ class PluginMeta(type):
+ def __hash__(cls):
+ raise AssertionError("plugin hash executed")
+
+ def __eq__(cls, other):
+ raise AssertionError("plugin equality executed")
+
+ class PluginHook(metaclass=PluginMeta):
+ pass
+
+ for request_id in ("first", "repeat"):
+ log_hook_failure(request_id, "pre_compress", PluginHook(), ValueError("failure"))
+ assert len(capture.hook_warnings()) == 1
+ capture.assert_no_canaries()
+
+
+def test_request_failure_names_the_failure_without_payload(capture) -> None:
+ log_request_failure(
+ "r1", "Request", _raised(ValueError(CONTENT)), provider="anthropic", model="m"
+ )
+
+ (error,) = (r.getMessage() for r in capture.at(logging.ERROR))
+ assert error.startswith("[r1] Request failed: provider='anthropic' model='m' ValueError")
+ capture.assert_no_canaries()
+
+
+def _proxy_rejecting_connect(reason: str):
+ """A real local HTTP proxy that answers every CONNECT with ``502 <reason>``."""
+ server = socket.socket()
+ server.bind(("127.0.0.1", 0))
+ server.listen()
+ server.settimeout(5)
+
+ def serve() -> None:
+ try:
+ conn, _ = server.accept()
+ except OSError: # closed before a client came
+ return
+ with conn:
+ conn.recv(65536)
+ conn.sendall(f"HTTP/1.1 502 {reason}\r\nContent-Length: 0\r\n\r\n".encode())
+
+ thread = threading.Thread(target=serve, daemon=True)
+ thread.start()
+ try:
+ yield f"http://127.0.0.1:{server.getsockname()[1]}"
+ finally:
+ server.close()
+ thread.join(timeout=5)
+
+
+def _proxy_error(reason: str) -> httpx.ProxyError:
+ with _proxy_rejecting_connect(reason) as proxy_url:
+ with httpx.Client(proxy=proxy_url, timeout=5) as client:
+ with pytest.raises(httpx.ProxyError) as caught:
+ client.get("https://upstream.example/v1/messages")
+ assert reason in str(caught.value) # the peer's text really is in the exception
+ return caught.value
+
+
+def test_a_proxy_reason_phrase_stays_out_of_the_request_failure_line(capture) -> None:
+ log_request_failure("r1", "Request", _proxy_error(CONTENT), provider="anthropic", model="m")
+
+ (error,) = (r.getMessage() for r in capture.at(logging.ERROR))
+ assert error.startswith("[r1] Request failed: provider='anthropic' model='m' ProxyError")
+ capture.assert_no_canaries()
+
+
+def test_the_content_opt_in_restores_the_proxy_text(
+ capture, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
+ log_request_failure("r1", "Request", _proxy_error(CONTENT), model="m")
+
+ (error,) = capture.at(logging.ERROR)
+ assert CONTENT in logging.Formatter().format(error)
+
+
+def _proxy_app(hooks: CompressionHooks, fake_retry, *, optimize: bool): # noqa: ANN001, ANN202
+ pytest.importorskip("fastapi")
+ from headroom.proxy.server import ProxyConfig, create_app
+
+ app = create_app(
+ ProxyConfig(optimize=optimize, cache_enabled=False, rate_limit_enabled=False, hooks=hooks)
+ )
+ return app
+
+
+def _chat_response() -> httpx.Response:
+ return httpx.Response(
+ 200,
+ json={
+ "id": "c",
+ "object": "chat.completion",
+ "model": "gpt-4o",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
+ },
+ )
+
+
+def test_openai_chat_warns_for_each_failing_stage(capture) -> None:
+ """Through the real handler: after a pre_compress failure, a later compute_biases
+ failure of the same type still warns, under its own stage."""
+ from fastapi.testclient import TestClient
+
+ class Flaky(CompressionHooks):
+ failing = "pre_compress"
+
+ def pre_compress(self, messages, ctx): # noqa: ANN001, ANN201
+ if self.failing == "pre_compress":
+ raise ValueError(CONTENT)
+ return messages
+
+ def compute_biases(self, messages, ctx): # noqa: ANN001, ANN201
+ raise ValueError(CONTENT)
+
+ async def fake_retry(method, url, headers, body, *args, **kwargs): # noqa: ANN001, ANN202
+ return _chat_response()
+
+ hooks = Flaky()
+ with TestClient(_proxy_app(hooks, fake_retry, optimize=False)) as client:
+ # Proxy startup may reset logger levels; re-assert capture afterwards.
+ logging.getLogger("headroom.proxy").setLevel(logging.DEBUG)
+ client.app.state.proxy._retry_request = fake_retry
+ for failing in ("pre_compress", "compute_biases"):
+ hooks.failing = failing
+ resp = client.post(
+ "/v1/chat/completions",
+ json={"model": "gpt-4o", "messages": [{"role": "user", "content": CONTENT}]},
+ headers={"Authorization": f"Bearer {CREDENTIAL}"},
+ )
+ assert resp.status_code == 200, resp.text
+
+ warnings = capture.hook_warnings()
+ assert len(warnings) == 2
+ assert "Flaky pre_compress failed" in warnings[0]
+ assert "Flaky compute_biases failed" in warnings[1]
+ capture.assert_no_canaries()
+
+
+def test_anthropic_bias_failure_says_compression_is_skipped(capture) -> None:
+ """Through the real handler: Anthropic aborts optimization when a bias hook fails,
+ so the warning must say compression is skipped, not that it continues."""
+ from fastapi.testclient import TestClient
+
+ class BrokenBiases(CompressionHooks):
+ def compute_biases(self, messages, ctx): # noqa: ANN001, ANN201
+ raise ValueError(CONTENT)
+
+ async def fake_retry(method, url, headers, body, *args, **kwargs): # noqa: ANN001, ANN202
+ return httpx.Response(
+ 200,
+ json={
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-sonnet-4-5",
+ "content": [{"type": "text", "text": "ok"}],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 5, "output_tokens": 1},
+ },
+ )
+
+ with TestClient(_proxy_app(BrokenBiases(), fake_retry, optimize=True)) as client:
+ logging.getLogger("headroom.proxy").setLevel(logging.DEBUG)
+ client.app.state.proxy._retry_request = fake_retry
+ resp = client.post(
+ "/v1/messages",
+ json={
+ "model": "claude-sonnet-4-5",
+ "max_tokens": 16,
+ "messages": [{"role": "user", "content": CONTENT}],
+ },
+ headers={"x-api-key": CREDENTIAL, "anthropic-version": "2023-06-01"},
+ )
+
+ assert resp.status_code == 200, resp.text
+ (warning,) = capture.hook_warnings()
+ assert "BrokenBiases compute_biases failed; skipping compression for this turn" in warning
+ assert "continuing without it" not in warning
+ capture.assert_no_canaries()
diff --git a/tests/test_proxy_anthropic_compression_diagnostics.py b/tests/test_proxy_anthropic_compression_diagnostics.py
index dad3f75bb..3f80d2c39 100644
--- a/tests/test_proxy_anthropic_compression_diagnostics.py
+++ b/tests/test_proxy_anthropic_compression_diagnostics.py
@@ -142,7 +142,7 @@ def test_optimization_failure_logs_exception_type() -> None:
assert response.status_code == 200, response.text
warning_msgs = [
- call.args[0]
+ str(call.args[0]) % call.args[1:]
for call in mock_warning.call_args_list
if call.args and "Optimization failed" in str(call.args[0])
]