Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): log the savings and metrics failures that currently vanish (#4084)

Tejas Chopra2800932e1827
diff --git a/headroom/log_safety.py b/headroom/log_safety.py
new file mode 100644
index 000000000..8282195fc
--- /dev/null
+++ b/headroom/log_safety.py
@@ -0,0 +1,383 @@
+"""Payload-safe building blocks for log lines.
+
+Log records are kept, rotated, attached to bug reports and shipped to log
+collectors, so no level may carry request content, tool output or
+credentials. Exception messages are the usual leak: a compressor, hook or
+storage backend can quote the payload or a connection URL in its error. These
+helpers describe a failure by what is safe to keep (exception types, code
+locations, errno, ids, counts) and leave the message out.
+
+Content is logged only when the operator opts in to it explicitly with
+``HEADROOM_DEBUG_DUMP=full``, the same switch that gates full request dumps.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import logging
+import math
+import os
+import re
+import sys
+import threading
+import time
+import traceback
+from collections import deque
+from collections.abc import Hashable
+from types import FrameType, ModuleType, TracebackType
+from urllib.parse import urlsplit
+
+CONTENT_OPT_IN_ENV = "HEADROOM_DEBUG_DUMP"
+_CONTENT_OPT_IN_VALUES = frozenset({"full", "all", "content"})
+_MAX_FRAMES = 3
+_MAX_CHAIN = 4
+_ID_MAX_CHARS = 80
+_MAX_ERRNO = 2**31 - 1
+# Schemes whose URLs have no host ("file:///path").
+_LOCAL_SCHEMES = frozenset({"file", "sqlite", "unix"})
+# Strict DNS labels. "_" is left out on purpose: it is not valid in a public
+# hostname but is common in keys (sk_live_...), so a host with it is masked.
+_HOSTNAME = re.compile(
+ r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*\.?"
+)
+_HOSTLESS_URL = re.compile(r"([A-Za-z][A-Za-z0-9+.-]*):///")
+# What urlsplit removes before parsing: leading C0 controls and spaces, and
+# every tab and newline. The hostless check must look at the same text.
+_URL_LEADING_JUNK = "".join(chr(code) for code in range(33))
+_URL_REMOVED = str.maketrans("", "", "\t\r\n")
+_HEADROOM_MODULE = re.compile(r"headroom(?:\.[A-Za-z_][A-Za-z0-9_]*)*")
+_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+# A class's __qualname__: identifiers joined by dots, with <locals> for nested ones.
+_QUALIFIED_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(?:\.(?:<locals>|[A-Za-z_][A-Za-z0-9_]*))*")
+_FIXED_CODE_NAMES = frozenset(
+ {"<module>", "<lambda>", "<genexpr>", "<listcomp>", "<dictcomp>", "<setcomp>"}
+)
+# The base classes' own attribute descriptors. Reading through them means no
+# subclass property (a __cause__ or errno that raises, a __qualname__ chosen by
+# a metaclass) can run while a failure is being described.
+_CAUSE = vars(BaseException)["__cause__"]
+_CONTEXT = vars(BaseException)["__context__"]
+_SUPPRESS_CONTEXT = vars(BaseException)["__suppress_context__"]
+_TRACEBACK = vars(BaseException)["__traceback__"]
+_ERRNO = vars(OSError)["errno"]
+_QUALNAME = vars(type)["__qualname__"]
+# Ints above this many bits have more than 4300 digits, which repr refuses.
+_MAX_ID_INT_BITS = 14_000
+# The module __dict__ descriptor itself, so a ModuleType subclass cannot override it.
+_MODULE_DICT = vars(ModuleType)["__dict__"]
+# Schemes a log may name; any other text before "://" could be a key someone
+# pasted where a URL belongs, so it is replaced.
+_KNOWN_SCHEMES = frozenset(
+ {
+ "http",
+ "https",
+ "ws",
+ "wss",
+ "grpc",
+ "grpcs",
+ "redis",
+ "rediss",
+ "postgres",
+ "postgresql",
+ "mysql",
+ "mongodb",
+ "mongodb+srv",
+ "sqlite",
+ "file",
+ "unix",
+ "s3",
+ "gs",
+ }
+)
+
+
+def content_logging_enabled() -> bool:
+ """True when the operator opted in to content in diagnostics (``HEADROOM_DEBUG_DUMP=full``)."""
+ return os.environ.get(CONTENT_OPT_IN_ENV, "").strip().lower() in _CONTENT_OPT_IN_VALUES
+
+
+def describe_exception(exc: BaseException) -> str:
+ """Describe ``exc`` for a log line without its message.
+
+ Gives each exception in the cause chain as its type plus the innermost code
+ locations, e.g. ``ValueError at headroom/x.py:12 in parse; caused by
+ KeyError at ...``. An ``OSError`` keeps its errno and the system's text for
+ it, never the exception's own strerror. With ``HEADROOM_DEBUG_DUMP=full`` it returns the full
+ traceback, messages included.
+ """
+ if content_logging_enabled():
+ return "".join(traceback.format_exception(exc)).rstrip()
+ parts: list[str] = []
+ seen: set[int] = set()
+ current: BaseException | None = exc
+ while current is not None and id(current) not in seen and len(parts) < _MAX_CHAIN:
+ seen.add(id(current))
+ parts.append(_describe_one(current))
+ current = _CAUSE.__get__(current) or (
+ None if _SUPPRESS_CONTEXT.__get__(current) else _CONTEXT.__get__(current)
+ )
+ return "; caused by ".join(parts)
+
+
+def _describe_one(exc: BaseException) -> str:
+ # A class name is chosen by whoever defined it (type("Err\nforged", ...)).
+ text = _class_name(type(exc)) or "<exception>"
+ # An int subclass errno could format itself as anything, so only a plain int.
+ errno = _ERRNO.__get__(exc) if issubclass(type(exc), OSError) else None
+ if type(errno) is int:
+ # The system's own text for the errno; exc.strerror is free text in some
+ # subclasses (ssl.SSLError) and can quote a path or a peer's message.
+ # An out-of-range errno is not formatted at all: converting a huge int
+ # to text can itself raise, and a log helper must never raise.
+ # Negative codes (socket.gaierror's EAI_*) have no os.strerror text.
+ if 0 <= errno <= _MAX_ERRNO:
+ text += f" [Errno {errno}] {_errno_text(errno)}"
+ elif -_MAX_ERRNO - 1 <= errno < 0:
+ text += f" [Errno {errno}]"
+ else:
+ text += " [Errno out of range]"
+ frames = _innermost_headroom_frames(_TRACEBACK.__get__(exc))
+ if frames:
+ text += " at " + " <- ".join(reversed(frames))
+ return text
+
+
+def _errno_text(errno: int) -> str:
+ try:
+ return os.strerror(errno)
+ except (ValueError, OverflowError):
+ return "unknown error"
+
+
+def _innermost_headroom_frames(tb: TracebackType | None) -> list[str]:
+ """Locations of the innermost Headroom frames, at most ``_MAX_FRAMES``, in memory bounded by that."""
+ frames: deque[str] = deque(maxlen=_MAX_FRAMES)
+ while tb is not None:
+ location = _headroom_frame_location(tb.tb_frame, tb.tb_lineno)
+ if location is not None:
+ frames.append(location)
+ tb = tb.tb_next
+ return list(frames)
+
+
+def _headroom_frame_location(frame: FrameType, lineno: int) -> str | None:
+ """``headroom/<module>.py:line in function`` for a frame of Headroom's own code, else None.
+
+ Only Headroom's own frames are named; library and stdlib frames are skipped,
+ since any text a frame carries (file, module or function name) can be set by
+ code compiled at runtime and quote request data. A frame counts as Headroom's
+ only when all of these hold, so none of its text is chosen at runtime:
+ its globals are the namespace of the module ``sys.modules`` holds under that
+ name; that name is a plain dotted ``headroom.*`` identifier; the code's file
+ is the module's own file (compared by stem, so ``.pyc``-only and zip installs
+ match); and the function name is a plain identifier. Headroom never execs code.
+ """
+ namespace = frame.f_globals
+ # Read through dict's own methods: globals can be a dict subclass.
+ name = dict.get(namespace, "__name__")
+ if type(name) is not str or not _HEADROOM_MODULE.fullmatch(name):
+ return None
+ module = sys.modules.get(name)
+ if not isinstance(module, ModuleType) or _MODULE_DICT.__get__(module) is not namespace:
+ return None
+ code = frame.f_code
+ module_file = dict.get(namespace, "__file__")
+ if type(module_file) is not str or _stem(module_file) != _stem(code.co_filename):
+ return None
+ function = code.co_name
+ if not (_IDENTIFIER.fullmatch(function) or function in _FIXED_CODE_NAMES):
+ return None
+ package = dict.__contains__(namespace, "__path__")
+ path = name.replace(".", "/") + ("/__init__.py" if package else ".py")
+ return f"{path}:{lineno} in {function}"
+
+
+def _stem(path: str) -> str:
+ return os.path.splitext(path.replace("\\", "/").rsplit("/", 1)[-1])[0]
+
+
+def redact_url(url: str) -> str:
+ """Return ``url`` as scheme, host and port only, safe for any log line.
+
+ Userinfo, query and fragment are always dropped, and so is the path, since
+ a path segment can carry a credential (``/bot<token>/``, webhook URLs) and
+ no rule tells such a segment from a route name. The path is kept only with
+ the explicit content opt-in, ``HEADROOM_DEBUG_DUMP=full``. The host comes
+ back lowercased, and only when it is a DNS name or IP address; a URL with
+ no host is accepted only for ``file``, ``sqlite`` and ``unix``.
+ """
+ try:
+ parts = urlsplit(url)
+ host = parts.hostname or ""
+ port = parts.port
+ except ValueError:
+ return "<unparseable url>"
+ # Without "//" urlsplit reads "sk-key:secret" as scheme "sk-key", so a string
+ # with no host is logged only for the hostless schemes ("file:///path").
+ if not parts.netloc and not _is_hostless_local_url(url, parts.scheme):
+ return "<unparseable url>"
+ if parts.netloc:
+ # A host that is not an IP or a strict DNS name may be a pasted secret;
+ # it is masked, and scheme and port still say which endpoint was meant.
+ host = _checked_host(host) or "<host>"
+ scheme = parts.scheme if parts.scheme in _KNOWN_SCHEMES else "<scheme>"
+ if ":" in host and host != "<host>":
+ host = f"[{host}]"
+ netloc = f"{host}:{port}" if port is not None else host
+ if content_logging_enabled():
+ path = parts.path
+ else:
+ path = "/<path>" if parts.path not in ("", "/") else parts.path
+ redacted = f"{scheme}://{netloc}{path}"
+ return f"{redacted}?<redacted>" if parts.query else redacted
+
+
+def _is_hostless_local_url(url: str, scheme: str) -> bool:
+ """True for ``file:///…``, ``sqlite:///…`` or ``unix:///…`` as urlsplit read it."""
+ cleaned = url.lstrip(_URL_LEADING_JUNK).translate(_URL_REMOVED)
+ match = _HOSTLESS_URL.match(cleaned)
+ return scheme in _LOCAL_SCHEMES and match is not None and match.group(1).lower() == scheme
+
+
+def _checked_host(host: str) -> str | None:
+ """``host`` as a loggable name or address, or None when it may not be one.
+
+ An IPv6 zone id (``fe80::1%eth0``) is free text, so it is dropped. A
+ non-ASCII name is given in its ASCII (IDNA) form.
+ """
+ address = host.split("%", 1)[0]
+ try:
+ ipaddress.ip_address(address)
+ except ValueError:
+ pass
+ else:
+ return address
+ try:
+ ascii_host = host.encode("idna").decode("ascii").lower()
+ except UnicodeError:
+ return None
+ return ascii_host if _HOSTNAME.fullmatch(ascii_host) else None
+
+
+def safe_id(value: object) -> str:
+ """Render a client- or model-supplied id as one bounded log token.
+
+ Uses the built-in ``repr`` of ``str``, ``bytes`` and numbers, so a newline
+ in the id cannot forge a second log line and no ``__repr__`` of the
+ caller's object runs; any other type is shown by its name. Cuts the result
+ to a fixed length so an oversized id cannot flood the log.
+ """
+ text = _builtin_repr(value)
+ if len(text) <= _ID_MAX_CHARS:
+ return text
+ return f"{text[:_ID_MAX_CHARS]}…(+{len(text) - _ID_MAX_CHARS} chars)"
+
+
+def _builtin_repr(value: object) -> str:
+ # type() rather than isinstance: isinstance consults a __class__ the object
+ # can fake, and then a built-in repr would run on the wrong type.
+ kind = type(value)
+ if value is None or kind is bool:
+ return "None" if value is None else ("True" if value else "False")
+ if issubclass(kind, str):
+ return str.__repr__(value) # type: ignore[arg-type]
+ if issubclass(kind, bytes):
+ return bytes.__repr__(value) # type: ignore[arg-type]
+ if issubclass(kind, int):
+ return _int_repr(value) # type: ignore[arg-type]
+ if issubclass(kind, float):
+ return float.__repr__(value) # type: ignore[arg-type]
+ return f"<{_class_name(kind) or 'object'}>"
+
+
+def _int_repr(value: int) -> str:
+ bits = int.bit_length(value)
+ # Digits an int of this many bits can have, against the process's own limit
+ # (sys.set_int_max_str_digits); past it, repr raises ValueError.
+ # The limit and its getter arrived in 3.10.7; earlier releases have no limit.
+ get_limit = getattr(sys, "get_int_max_str_digits", None)
+ limit = get_limit() if get_limit is not None else 0
+ if bits > _MAX_ID_INT_BITS or (limit and bits * 0.30103 + 1 >= limit):
+ return f"<int of {bits} bits>"
+ return int.__repr__(value)
+
+
+def _class_name(cls: type) -> str | None:
+ """The class's qualified name when it is plain identifiers, else None.
+
+ Whoever defines a class chooses its name (``type("sk-...\\r", ...)``), so only
+ names shaped like source identifiers are logged.
+ """
+ name = _QUALNAME.__get__(cls)
+ return name if type(name) is str and _QUALIFIED_NAME.fullmatch(name) else None
+
+
+class WarnOnce:
+ """Warn once per key per window, for at most ``limit`` keys per window.
+
+ The first warning opens a window of ``window_seconds`` (an hour by
+ default). Inside it each key warns once; past ``limit`` keys it logs one
+ overflow WARNING and stays quiet, so a failure that hits many keys
+ (paths, rows, models) cannot flood the log: at most ``limit + 1`` warnings
+ per window. When the window ends the guard starts over, so a failure that
+ keeps recurring, or a newly broken key, warns again once an hour. A key is
+ used up only when WARNING is enabled, so a warning suppressed by the log
+ level is still emitted later. ``forget`` re-arms a key once its failure has
+ cleared; a re-armed key that fails again counts against the same window's
+ budget, so forgetting can never push a window past ``limit + 1`` warnings.
+ """
+
+ def __init__(self, limit: int, what: str, *, window_seconds: float = 3600.0) -> None:
+ if limit < 1:
+ raise ValueError(f"WarnOnce limit must be at least 1, got {limit}")
+ if not 0 < window_seconds < math.inf: # also rejects nan
+ raise ValueError(
+ f"WarnOnce window_seconds must be positive and finite, got {window_seconds}"
+ )
+ self._limit = limit
+ self._what = what
+ self._window = window_seconds
+ self._keys: set[Hashable] = set()
+ self._issued = 0 # warnings issued this window; forget() does not refund them
+ self._window_start: float | None = None
+ self._overflowed = False
+ self._lock = threading.Lock()
+
+ def first(self, key: Hashable, log: logging.Logger) -> bool:
+ """Return True if the caller should log this key's WARNING now."""
+ if not log.isEnabledFor(logging.WARNING):
+ return False
+ with self._lock:
+ now = time.monotonic()
+ if self._window_start is not None and now - self._window_start >= self._window:
+ self._keys.clear()
+ self._issued = 0
+ self._overflowed = False
+ self._window_start = None
+ if key in self._keys:
+ return False
+ if not self._overflowed and self._issued < self._limit:
+ self._keys.add(key)
+ self._issued += 1
+ if self._window_start is None:
+ self._window_start = now
+ return True
+ report_overflow = not self._overflowed
+ self._overflowed = True
+ started = now if self._window_start is None else self._window_start
+ remaining = self._window - (now - started)
+ if report_overflow:
+ log.warning(
+ "More than %d warnings about %s this window; further ones are not logged "
+ "at WARNING "
+ "for the next %d minutes",
+ self._limit,
+ self._what,
+ max(1, math.ceil(remaining / 60)),
+ )
+ return False
+
+ def forget(self, key: Hashable) -> None:
+ """Re-arm ``key`` so its next failure warns again."""
+ with self._lock:
+ self._keys.discard(key)
diff --git a/headroom/proxy/handlers/_debug_dump.py b/headroom/proxy/handlers/_debug_dump.py
index b04459fb3..f58fc9fb6 100644
--- a/headroom/proxy/handlers/_debug_dump.py
+++ b/headroom/proxy/handlers/_debug_dump.py
@@ -14,7 +14,8 @@ import os
from datetime import datetime
from pathlib import Path
from typing import Any
-from urllib.parse import urlsplit, urlunsplit
+
+from headroom.log_safety import content_logging_enabled, redact_url
logger = logging.getLogger(__name__)
@@ -29,9 +30,9 @@ def _debug_dump_mode(config: Any) -> str:
"""
if getattr(config, "stateless", False):
return "off"
- raw = os.environ.get("HEADROOM_DEBUG_DUMP", "").strip().lower()
- if raw in ("full", "all", "content"):
+ if content_logging_enabled():
return "full"
+ raw = os.environ.get("HEADROOM_DEBUG_DUMP", "").strip().lower()
if raw in ("1", "true", "yes", "on", "redacted"):
return "redacted"
return "off"
@@ -54,23 +55,6 @@ def _redact_debug_value(value: Any, _max_len: int = 80) -> Any:
return value
-def _redact_url(url: str) -> str:
- """Drop the query string and fragment from ``url``.
-
- Some upstreams carry credentials in the query (Gemini's ``?key=``), and the
- query is never needed to debug a rejected request, so it is elided in every
- mode — ``full`` opts in to prompt content, not to credentials.
- """
- try:
- parts = urlsplit(url)
- except ValueError:
- return "<unparseable url>"
- if not parts.query and not parts.fragment:
- return url
- redacted = urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
- return f"{redacted}?<redacted>" if parts.query else redacted
-
-
def _decode_sent_body(body: Any) -> Any:
"""Decode the exact bytes sent upstream back to JSON for the dump."""
if not isinstance(body, (bytes, bytearray)):
@@ -119,7 +103,7 @@ def write_upstream_error_dump(
payload = json.dumps(
{
"request_id": request_id,
- "url": _redact_url(url),
+ "url": redact_url(url),
"status": status,
"provider": provider,
"model": model,
diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.py
index 1a6d66694..9b53b69b6 100644
--- a/headroom/proxy/outcome.py
+++ b/headroom/proxy/outcome.py
@@ -31,6 +31,7 @@ from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any
+from headroom.log_safety import describe_exception
from headroom.proxy.conversation_savings import get_conversation_savings
from headroom.proxy.tool_schema_savings_policy import (
headline_tokens_saved,
@@ -629,8 +630,12 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:
# does a full read-modify-write of the ledger file — run them
# together off the event loop (#18) so a slow flush can't stall it.
output_tokens_saved_est = await asyncio.to_thread(_record_and_estimate)
- except Exception: # pragma: no cover - defensive
- pass
+ except Exception as exc: # pragma: no cover - defensive
+ logger.debug(
+ "Output savings record/estimate failed for request %s; booking 0: %s",
+ outcome.request_id,
+ describe_exception(exc),
+ )
# Project attribution: explicit outcome field wins, else the value the
# HTTP middleware / WS accept captured from ``X-Headroom-Project``.
diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.py
index d07622ce0..1593b7727 100644
--- a/headroom/proxy/savings_tracker.py
+++ b/headroom/proxy/savings_tracker.py
@@ -24,6 +24,7 @@ from pathlib import Path
from typing import Any
from headroom import paths as _paths
+from headroom.log_safety import WarnOnce, describe_exception, safe_id
from headroom.proxy import project_name_policy
from headroom.proxy.persistent_metrics import PersistentMetricsState
@@ -355,14 +356,23 @@ def _resolve_litellm_model(model: str) -> str:
info = litellm.model_cost.get(resolved)
if info and info.get("input_cost_per_token") is not None:
return resolved
- except Exception:
- pass
+ except Exception as exc:
+ logger.debug(
+ "Shared litellm model resolution failed for model=%s: %s",
+ safe_id(model),
+ describe_exception(exc),
+ )
+ # litellm raises a bare Exception for a model it has not mapped, so this is
+ # the narrowest catch that works. A miss means "not this spelling, try the
+ # provider prefixes next". Runs once per model (lru_cache above).
try:
litellm.cost_per_token(model=model, prompt_tokens=1, completion_tokens=0)
return model
- except Exception:
- pass
+ except Exception as exc:
+ logger.debug(
+ "litellm cannot price model=%s as given: %s", safe_id(model), describe_exception(exc)
+ )
prefixes = {
"claude-": "anthropic/",
@@ -388,6 +398,25 @@ def _resolve_litellm_model(model: str) -> str:
return model
+# An unpriced model (a local model, a gateway alias, a model newer than the
+# bundled litellm) is an expected case, and the estimators below run on every
+# request, so each model is logged once. Debug, not warning: proxy/cost.py
+# already warns once per unpriceable model on the live path. Model names come
+# from clients, so the guard is bounded like _resolve_litellm_model's cache.
+_unpriced_models_logged = WarnOnce(_MODEL_RESOLUTION_CACHE_MAXSIZE, "unpriced models")
+
+
+def _log_unpriced_model_once(model: str, field: str) -> None:
+ if not logger.isEnabledFor(logging.DEBUG):
+ return
+ if _unpriced_models_logged.first(model, logger):
+ logger.debug(
+ "No litellm %s for model=%s; savings and cost use the fallback rate",
+ field,
+ safe_id(model),
+ )
+
+
def _estimate_compression_savings_usd(model: str, tokens_saved: int) -> float:
"""Estimate compression savings in USD from saved input tokens."""
litellm = _get_litellm_module()
@@ -405,9 +434,15 @@ def _estimate_compression_savings_usd(model: str, tokens_saved: int) -> float:
# a real 0.0 as unavailable and billed the $3/M fallback — phantom savings
# for a model that costs nothing.
if input_cost_per_token is None:
- raise RuntimeError("input cost unavailable")
+ _log_unpriced_model_once(model, "input_cost_per_token")
+ return float(tokens_saved) * float(DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN)
return float(tokens_saved) * float(input_cost_per_token)
- except Exception:
+ except Exception as exc:
+ logger.debug(
+ "Compression savings priced at the fallback rate for model=%s: %s",
+ safe_id(model),
+ describe_exception(exc),
+ )
return float(tokens_saved) * float(DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN)
@@ -432,9 +467,15 @@ def _estimate_output_savings_usd(model: str, tokens_saved: int) -> float:
# rate -> phantom output savings for a model that costs nothing. Mirrors
# the fix already applied to `_estimate_compression_savings_usd`.
if output_cost_per_token is None:
- raise RuntimeError("output cost unavailable")
+ _log_unpriced_model_once(model, "output_cost_per_token")
+ return float(tokens_saved) * float(DEFAULT_FALLBACK_OUTPUT_COST_PER_TOKEN)
return float(tokens_saved) * float(output_cost_per_token)
- except Exception:
+ except Exception as exc:
+ logger.debug(
+ "Output savings priced at the fallback rate for model=%s: %s",
+ safe_id(model),
+ describe_exception(exc),
+ )
return float(tokens_saved) * float(DEFAULT_FALLBACK_OUTPUT_COST_PER_TOKEN)
@@ -481,7 +522,12 @@ def _estimate_cache_savings_usd(model: str, cache_read_tokens: int) -> float:
if discount <= 0:
return 0.0
return float(cache_read_tokens) * discount
- except Exception:
+ except Exception as exc:
+ logger.debug(
+ "Cache savings pricing failed for model=%s; booking 0: %s",
+ safe_id(model),
+ describe_exception(exc),
+ )
return 0.0
@@ -628,7 +674,8 @@ def _estimate_input_cost_usd(
# A missing key means the model is unknown → fall back to a blended rate.
# A present 0.0 means the model is free and must cost $0, not the fallback.
if input_cost_per_token is None:
- raise RuntimeError("input cost unavailable")
+ _log_unpriced_model_once(model, "input_cost_per_token")
+ return float(chargeable_tokens) * float(DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN)
if use_breakdown:
cache_read_cost = info.get(
@@ -646,7 +693,12 @@ def _estimate_input_cost_usd(
)
return float(total_input_tokens) * float(input_cost_per_token)
- except Exception:
+ except Exception as exc:
+ logger.debug(
+ "Input cost priced at the fallback rate for model=%s: %s",
+ safe_id(model),
+ describe_exception(exc),
+ )
return float(chargeable_tokens) * float(DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN)
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 2f6323176..e07abfc8f 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -86,6 +86,7 @@ from headroom.config import (
ReadLifecycleConfig,
)
from headroom.dashboard import get_dashboard_html
+from headroom.log_safety import describe_exception, safe_id
from headroom.observability import (
LangfuseTracingConfig,
OTelMetricsConfig,
@@ -3129,12 +3130,24 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
# that construct the app without going through the `headroom` CLI entrypoint
# (which already applies them before Click parsing). setdefault keeps
# explicit env exports authoritative; fail-open so it never blocks startup.
+ # settings_store.load() already reports an unreadable or corrupt file; this
+ # catches anything else that goes wrong while applying it.
+ settings_file: object = "the saved settings file"
try:
+ from headroom import paths as _hr_paths
from headroom import settings_store
+ settings_file = _hr_paths.settings_path()
settings_store.apply_to_environ(settings_store.load())
- except Exception: # noqa: BLE001 — settings load must never break startup
- pass
+ except Exception as exc: # noqa: BLE001 — settings load must never break startup
+ # Settings values can hold credentials, so the exception is described by
+ # type and code location only, never its message.
+ logger.warning(
+ "Could not apply saved settings from %s (%s); continuing with env and "
+ "defaults. Fix or remove that file and restart the proxy.",
+ settings_file,
+ describe_exception(exc),
+ )
# Air-gap master switch. Propagate config.offline to the env so the
# env-based egress predicates (telemetry, update check, license) all honor
@@ -4148,8 +4161,14 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
action="admin_request",
status_code=response.status_code,
)
- except Exception:
- logger.debug("admin audit emission failed", exc_info=True)
+ except Exception as exc:
+ logger.warning(
+ "Admin audit record not written for %s %s (%s); this request is missing "
+ "from the headroom.audit trail. Please report this as a bug.",
+ request.method,
+ safe_id(request.url.path),
+ describe_exception(exc),
+ )
return response
# The gate above is http-only (BaseHTTPMiddleware ignores every other
@@ -4994,8 +5013,8 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
_oest = get_recorder().estimate(_olevel)
output_reduction = _output_reduction_payload(_shaper_active, _oest)
- except Exception: # pragma: no cover - defensive
- pass
+ except Exception as exc: # pragma: no cover - defensive
+ logger.debug("/stats output_reduction section failed: %s", describe_exception(exc))
# Model-routing section: populated only when an extension registered a
# routing-stats provider (headroom.proxy.routing_stats); None otherwise.
diff --git a/headroom/proxy/tool_schema_savings_policy.py b/headroom/proxy/tool_schema_savings_policy.py
index 7b2bd8f81..c75f09e94 100644
--- a/headroom/proxy/tool_schema_savings_policy.py
+++ b/headroom/proxy/tool_schema_savings_policy.py
@@ -26,8 +26,32 @@ name to :data:`TOOL_SCHEMA_SAVINGS_TAGS` and every surface picks it up.
from __future__ import annotations
import copy
+import logging
from typing import Any
+from headroom.log_safety import WarnOnce, describe_exception
+
+logger = logging.getLogger(__name__)
+
+# A failure that repeats on every request warns once per exception type; every
+# occurrence also goes to debug.
+_reconcile_warned = WarnOnce(32, "tool-schema reconcile failure types")
+
+
+def _log_reconcile_failure(exc: Exception, booked: int) -> None:
+ # The exception comes from tokenizing client tool schemas and can quote
+ # them, so it is described by type and code location only, at every level.
+ where = describe_exception(exc)
+ if _reconcile_warned.first(type(exc).__qualname__, logger):
+ logger.warning(
+ "tool-schema deferral reconcile failed (%s); credit left at %d booked tokens, "
+ "so /stats over-reports tool-search savings for this request. Please report "
+ "this as a bug.",
+ where,
+ booked,
+ )
+ logger.debug("tool-schema deferral reconcile failed; booked=%d: %s", booked, where)
+
def without_deferral_flags(tools: object) -> object:
"""The tool array as it is MEASURED for a hook's tool delta.
@@ -128,7 +152,8 @@ def reconcile_deferred_tokens(
tags["tool_search_deferred_tools"] = len(kept)
if isinstance(entry, dict):
entry["tokens"] = max(0, int(entry.get("tokens") or 0) - released)
- except Exception: # accounting must never break a request
+ except Exception as exc: # accounting must never break a request
+ _log_reconcile_failure(exc, booked)
return
diff --git a/tests/test_debug_dump_gating.py b/tests/test_debug_dump_gating.py
index 924c9d3fa..40a87d27d 100644
--- a/tests/test_debug_dump_gating.py
+++ b/tests/test_debug_dump_gating.py
@@ -189,10 +189,25 @@ def test_upstream_dump_never_writes_url_credentials(dump_dir, monkeypatch, mode)
assert path is not None
text = path.read_text()
assert "AIzaSECRET" not in text
- assert json.loads(text)["url"] == (
- "https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"
- "?<redacted>"
- )
+ if mode == "full": # the content opt-in keeps the route, still never the key
+ expected = (
+ "https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"
+ )
+ else: # a path segment can carry a credential, so it is dropped without the opt-in
+ expected = "https://generativelanguage.googleapis.com/<path>"
+ assert json.loads(text)["url"] == expected + "?<redacted>"
+
+
+ "url",
+ ["https://upstream.example/botCANARY/v1/messages", "https://upstream.example/bot%43ANARY/v1"],
+)
+def test_redacted_dump_never_writes_path_credentials(dump_dir, monkeypatch, url):
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "redacted")
+ path = _write(url=url)
+ assert path is not None
+ text = path.read_text()
+ assert "CANARY" not in text and "%43ANARY" not in text
def test_upstream_dump_keeps_query_free_url(dump_dir, monkeypatch):
diff --git a/tests/test_log_safety.py b/tests/test_log_safety.py
new file mode 100644
index 000000000..dfe95d473
--- /dev/null
+++ b/tests/test_log_safety.py
@@ -0,0 +1,555 @@
+"""Tests for the payload-safe log helpers in ``headroom.log_safety``."""
+
+from __future__ import annotations
+
+import importlib
+import importlib.machinery
+import importlib.util
+import json
+import logging
+import math
+import os
+import py_compile
+import sys
+import types
+import zipfile
+import zipimport
+from pathlib import Path
+
+import pytest
+
+from headroom import log_safety
+from headroom.log_safety import WarnOnce, describe_exception, redact_url, safe_id
+
+CANARY = "sk-live-CANARY tool output: patient record 42"
+
+
+def _raise_chain() -> BaseException:
+ try:
+ try:
+ raise KeyError(CANARY)
+ except KeyError as inner:
+ raise ValueError(f"could not parse {CANARY}") from inner
+ except ValueError as exc:
+ return exc
+
+
+def test_exception_description_names_types_and_places_but_not_messages(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+ text = describe_exception(_raise_chain())
+
+ assert "CANARY" not in text
+ # The chain is raised from this test file, which is not Headroom's code,
+ # so only the types are named.
+ assert text == "ValueError; caused by KeyError"
+
+
+def test_oserror_keeps_errno_but_not_its_filename(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+ text = describe_exception(PermissionError(13, "Permission denied", f"/tmp/{CANARY}"))
+
+ assert text == "PermissionError [Errno 13] Permission denied"
+
+
+def test_oserror_free_text_strerror_is_replaced_by_the_system_text() -> None:
+ text = describe_exception(OSError(2, f"token={CANARY}"))
+
+ assert "CANARY" not in text
+ assert text == f"FileNotFoundError [Errno 2] {os.strerror(2)}"
+
+
+def test_content_opt_in_gives_the_full_traceback(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
+
+ text = describe_exception(_raise_chain())
+
+ assert "Traceback" in text
+ assert CANARY in text
+
+
+ ("url", "expected"),
+ [
+ (
+ "https://user:[email protected]:8443/v1?key=abc#frag",
+ "https://proxy.example:8443/<path>?<redacted>",
+ ),
+ ("https://upstream.example/botSECRET/v1/messages", "https://upstream.example/<path>"),
+ ("https://upstream.example/bot%53ECRET/v1", "https://upstream.example/<path>"),
+ ("http://127.0.0.1:8787", "http://127.0.0.1:8787"),
+ ("http://[::1]:8787/", "http://[::1]:8787/"),
+ ("http://[::1", "<unparseable url>"),
+ ],
+)
+def test_redact_url_keeps_only_scheme_host_and_port(
+ url: str, expected: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+ assert redact_url(url) == expected
+
+
+def test_content_opt_in_keeps_the_path_but_never_credentials(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")
+
+ assert redact_url("https://u:[email protected]/v1/messages?key=abc") == (
+ "https://upstream.example/v1/messages?<redacted>"
+ )
+
+
+def test_safe_id_escapes_newlines_and_bounds_length() -> None:
+ assert safe_id("m1\nWARNING forged") == "'m1\\nWARNING forged'"
+ long = safe_id("x" * 10_000)
+ assert len(long) < 120
+ assert long.endswith("chars)")
+
+
+def _warnings(caplog: pytest.LogCaptureFixture) -> list[str]:
+ return [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]
+
+
+def test_warn_once_stays_bounded_across_repeated_passes(caplog: pytest.LogCaptureFixture) -> None:
+ log = logging.getLogger("headroom.test_log_safety.bounded")
+ guard = WarnOnce(limit=64, what="unwritable ledgers")
+
+ with caplog.at_level(logging.INFO, logger=log.name):
+ for _ in range(3):
+ for path in range(65):
+ if guard.first(path, log):
+ log.warning("ledger %d failed", path)
+
+ messages = _warnings(caplog)
+ assert len(messages) == 65 # 64 keys plus one overflow notice, not 65 per pass
+ assert messages[-1] == (
+ "More than 64 warnings about unwritable ledgers this window; further ones are not logged "
+ "at WARNING "
+ "for the next 60 minutes"
+ )
+
+
+def test_warn_once_keeps_the_key_while_warnings_are_disabled() -> None:
+ log = logging.getLogger("headroom.test_log_safety.level")
+ guard = WarnOnce(limit=4, what="failures")
+ log.setLevel(logging.ERROR)
+ try:
+ assert guard.first("k", log) is False
+ log.setLevel(logging.INFO)
+ assert guard.first("k", log) is True
+ assert guard.first("k", log) is False
+ finally:
+ log.setLevel(logging.NOTSET)
+
+
+def test_forget_rearms_a_key() -> None:
+ log = logging.getLogger("headroom.test_log_safety.forget")
+ guard = WarnOnce(limit=4, what="failures")
+
+ assert guard.first("path", log) is True
+ guard.forget("path")
+ assert guard.first("path", log) is True
+
+
+def test_raise_from_none_hides_the_suppressed_context(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+ try:
+ try:
+ raise KeyError(CANARY)
+ except KeyError:
+ raise ValueError("bad") from None
+ except ValueError as exc:
+ text = describe_exception(exc)
+
+ assert "KeyError" not in text
+
+
+def test_a_cause_cycle_is_described_once() -> None:
+ first = ValueError(CANARY)
+ second = KeyError(CANARY)
+ first.__cause__ = second
+ second.__cause__ = first
+
+ assert describe_exception(first) == "ValueError; caused by KeyError"
+
+
+def test_forgetting_after_overflow_does_not_reopen_the_cap(
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ log = logging.getLogger("headroom.test_log_safety.reopen")
+ guard = WarnOnce(limit=2, what="failures")
+
+ with caplog.at_level(logging.INFO, logger=log.name):
+ assert guard.first("a", log) and guard.first("b", log)
+ assert guard.first("c", log) is False # overflow notice
+ for cycle in range(5):
+ guard.forget("a")
+ assert guard.first(f"new-{cycle}", log) is False
+
+ assert len(_warnings(caplog)) == 1
+
+
+def test_warn_once_starts_over_after_the_window(
+ caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ log = logging.getLogger("headroom.test_log_safety.window")
+ guard = WarnOnce(limit=2, what="hook failures", window_seconds=600)
+ clock = [1000.0]
+ monkeypatch.setattr(log_safety.time, "monotonic", lambda: clock[0])
+
+ results = []
+ with caplog.at_level(logging.INFO, logger=log.name):
+ steps = (("a", 0), ("b", 0), ("c", 0), ("a", 300), ("d", 299), ("a", 1), ("new-hook", 0))
+ for key, advance in steps:
+ clock[0] += advance
+ results.append(guard.first(key, log))
+
+ # a and b warn and c is the overflow notice; inside the window a stays quiet
+ # and d is suppressed; once the window ends, the still-recurring a and a
+ # newly broken key both warn again.
+ assert results == [True, True, False, False, False, True, True]
+ assert len(_warnings(caplog)) == 1
+
+
+def test_warn_once_needs_a_positive_window() -> None:
+ with pytest.raises(ValueError, match="window_seconds must be positive"):
+ WarnOnce(limit=1, what="failures", window_seconds=0)
+
+
[email protected]("digits", [19, 10_001])
+def test_out_of_range_errno_never_breaks_the_description(digits: int) -> None:
+ """An errno too large to print (Python caps int-to-text at 4300 digits) is not printed."""
+ exc = OSError()
+ exc.errno = 10 ** (digits - 1) # computed, not a literal, so collection never formats it
+
+ assert describe_exception(exc) == "OSError [Errno out of range]"
+
+
+def test_negative_dns_error_codes_are_kept() -> None:
+ import socket
+
+ assert describe_exception(socket.gaierror(-2, "Name or service not known")) == (
+ "gaierror [Errno -2]"
+ )
+
+
+def test_overflow_notice_states_the_time_left_in_the_window(
+ caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ log = logging.getLogger("headroom.test_log_safety.remaining")
+ guard = WarnOnce(limit=1, what="failures", window_seconds=3600)
+ clock = [0.0]
+ monkeypatch.setattr(log_safety.time, "monotonic", lambda: clock[0])
+
+ with caplog.at_level(logging.INFO, logger=log.name):
+ guard.first("a", log)
+ clock[0] = 3000.0 # ten minutes left in the window
+ guard.first("b", log)
+
+ assert _warnings(caplog)[-1].endswith("for the next 10 minutes")
+
+
+def test_forget_cycles_cannot_exceed_the_window_budget(caplog: pytest.LogCaptureFixture) -> None:
+ log = logging.getLogger("headroom.test_log_safety.forget_budget")
+ guard = WarnOnce(limit=3, what="failures")
+
+ with caplog.at_level(logging.INFO, logger=log.name):
+ issued = 0
+ for cycle in range(50):
+ if guard.first(f"path-{cycle}", log):
+ issued += 1
+ log.warning("path-%d failed", cycle)
+ guard.forget(f"path-{cycle}")
+
+ assert issued == 3
+ assert len(_warnings(caplog)) == 4 # three warnings plus one overflow notice
+
+
+ "value",
+ ["sk-ant-abc123:x", "AKIAABCDEFGH:secret", "myuser:hunter2@db:5432", "AKIASECRET://host/x"],
+)
+def test_text_that_is_not_a_url_never_reaches_the_log(value: str) -> None:
+ text = redact_url(value)
+ for secret in ("sk-ant", "abc123", "AKIA", "akia", "hunter2", "myuser", "SECRET", "secret"):
+ assert secret not in text
+
+
[email protected]("host", ["a
b", "a\x0bb\x0cc", "a\x85b"])
+def test_hosts_with_line_breaks_are_not_logged(host: str) -> None:
+ assert redact_url(f"http://{host}/") == "http://<host>/"
+
+
+def test_class_names_cannot_forge_a_log_line(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+ forged = type("Err\nWARNING forged sk-QUAL", (Exception,), {})
+
+ text = describe_exception(forged("x"))
+
+ assert "\n" not in text and "sk-QUAL" not in text
+ assert text.startswith("<exception>")
+
+
+def test_odd_errno_and_module_objects_never_break_the_description(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+
+ class FormatsItself(int):
+ def __format__(self, spec: str) -> str:
+ return "sk-ERRNO-CANARY"
+
+ exc = OSError()
+ exc.errno = FormatsItself(2)
+ assert "CANARY" not in describe_exception(exc)
+
+ class RaisingGet(dict): # type: ignore[type-arg]
+ def get(self, *args: object) -> object:
+ raise RuntimeError("get")
+
+ namespace = RaisingGet()
+ exec(compile("def boom():\n raise ValueError('x')\n", "<x>", "exec"), namespace)
+ with pytest.raises(ValueError) as caught:
+ namespace["boom"]()
+ assert " at " not in describe_exception(caught.value)
+
+ lazy = types.ModuleType("lazy_sk_mod")
+
+ def lazy_getattr(attr: str) -> str:
+ return "sk-FILE-CANARY.py"
+
+ lazy.__getattr__ = lazy_getattr # type: ignore[method-assign]
+ monkeypatch.setitem(sys.modules, "lazy_sk_mod", lazy)
+ exec(compile("def boom():\n raise ValueError('x')\n", "<x>", "exec"), vars(lazy))
+ with pytest.raises(ValueError) as caught:
+ lazy.boom()
+ assert "CANARY" not in describe_exception(caught.value)
+
+
+def test_safe_id_never_raises() -> None:
+ class BadRepr:
+ def __repr__(self) -> str:
+ raise RuntimeError("repr")
+
+ huge = 10 ** int("5000")
+ assert safe_id(huge) == "<int of 16610 bits>"
+ assert safe_id(BadRepr()) == f"<{BadRepr.__qualname__}>"
+
+
[email protected]("window", [math.inf, math.nan, 0, -1])
+def test_warn_once_needs_a_finite_positive_window(window: float) -> None:
+ with pytest.raises(ValueError, match="window_seconds must be positive and finite"):
+ WarnOnce(limit=1, what="failures", window_seconds=window)
+
+
+def test_headroom_frames_are_named_by_module_path() -> None:
+ with pytest.raises(ValueError) as caught:
+ WarnOnce(limit=0, what="failures")
+
+ assert describe_exception(caught.value) == (
+ f"ValueError at headroom/log_safety.py:{caught.traceback[-1].lineno + 1} in __init__"
+ )
+
+
+def test_library_frames_are_skipped_but_the_headroom_caller_is_kept() -> None:
+ with pytest.raises(json.JSONDecodeError) as caught:
+ json.loads("{")
+
+ assert describe_exception(caught.value) == "JSONDecodeError"
+
+
+ ("filename", "module_name"),
+ [
+ ("sk-FILE-CANARY\nforged", None),
+ (os.__file__, "os"),
+ (log_safety.__file__, "headroom.log_safety"),
+ ],
+ ids=["forged-name", "claims-stdlib-file", "claims-headroom-module"],
+)
+def test_runtime_compiled_frames_are_not_logged(
+ monkeypatch: pytest.MonkeyPatch, filename: str, module_name: str | None
+) -> None:
+ """Runtime code may claim a real file and module name; it is still not named."""
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+ code = compile("def sk_FUNC_CANARY():\n raise ValueError('x')\n", filename, "exec")
+ namespace: dict[str, object] = {} if module_name is None else {"__name__": module_name}
+ exec(code, namespace)
+ with pytest.raises(ValueError) as caught:
+ namespace["sk_FUNC_CANARY"]() # type: ignore[operator]
+
+ assert describe_exception(caught.value) == "ValueError"
+
+
+def test_code_execd_into_a_headroom_module_is_not_named(monkeypatch: pytest.MonkeyPatch) -> None:
+ namespace = vars(log_safety)
+ code = compile("def sk_GEN_CANARY():\n raise ValueError('x')\n", "<generated>", "exec")
+ exec(code, namespace)
+ monkeypatch.delitem(namespace, "sk_GEN_CANARY")
+ function = code.co_consts[0]
+ generated = types.FunctionType(function, namespace)
+
+ with pytest.raises(ValueError) as caught:
+ generated()
+
+ assert describe_exception(caught.value) == "ValueError"
+
+
+ "module_name", ["headroom.x\nWARNING forged", "headroom.sk-live-CANARY", "headroomx"]
+)
+def test_odd_module_names_are_not_named(monkeypatch: pytest.MonkeyPatch, module_name: str) -> None:
+ module = types.ModuleType(module_name)
+ module.__file__ = "probe.py"
+ monkeypatch.setitem(sys.modules, module_name, module)
+ exec(compile("def boom():\n raise ValueError('x')\n", "probe.py", "exec"), vars(module))
+
+ with pytest.raises(ValueError) as caught:
+ module.boom()
+
+ assert describe_exception(caught.value) == "ValueError"
+
+
+def test_zip_and_sourceless_headroom_modules_keep_their_locations(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ source = "def boom():\n raise ValueError('x')\n"
+ with zipfile.ZipFile(tmp_path / "bundle.zip", "w") as bundle:
+ bundle.writestr("_zip_probe.py", source)
+ (tmp_path / "_pyc_probe.py").write_text(source)
+ py_compile.compile(str(tmp_path / "_pyc_probe.py"), cfile=str(tmp_path / "_pyc_probe.pyc"))
+ (tmp_path / "_pyc_probe.py").unlink()
+
+ zip_spec = zipimport.zipimporter(str(tmp_path / "bundle.zip")).find_spec("headroom._zip_probe")
+ pyc_loader = importlib.machinery.SourcelessFileLoader(
+ "headroom._pyc_probe", str(tmp_path / "_pyc_probe.pyc")
+ )
+ pyc_spec = importlib.util.spec_from_loader("headroom._pyc_probe", pyc_loader)
+ texts = []
+ for spec in (zip_spec, pyc_spec):
+ assert spec is not None and spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ monkeypatch.setitem(sys.modules, spec.name, module)
+ spec.loader.exec_module(module)
+ with pytest.raises(ValueError) as caught:
+ module.boom()
+ texts.append(describe_exception(caught.value))
+
+ assert texts == [
+ "ValueError at headroom/_zip_probe.py:2 in boom",
+ "ValueError at headroom/_pyc_probe.py:2 in boom",
+ ]
+
+
+def test_deep_tracebacks_keep_only_the_innermost_headroom_frames() -> None:
+ def recurse(depth: int) -> None:
+ if depth == 0:
+ WarnOnce(limit=0, what="failures")
+ recurse(depth - 1)
+
+ with pytest.raises(ValueError) as caught:
+ recurse(500)
+
+ assert describe_exception(caught.value).count(" in ") == 1
+
+
+ ("url", "expected"),
+ [
+ ("file:///var/lib/headroom/ledger.db", "file:///<path>"),
+ ("sqlite:///tmp/x.db", "sqlite:///<path>"),
+ ("unix:///run/headroom.sock", "unix:///<path>"),
+ ("https://sk_live_CANARY@/v1", "https://<host>/<path>"),
+ ("https://sk_live_CANARY/v1", "https://<host>/<path>"),
+ ("https://abcDEF+ghi=/v1", "https://<host>/<path>"),
+ ("http://model_gateway:8000/v1", "http://<host>:8000/<path>"),
+ ("https://bücher.example/v1", "https://xn--bcher-kva.example/<path>"),
+ ("http://[fe80::1%25sk-ZONE-CANARY]:8787/", "http://[fe80::1]:8787/"),
+ ("https://upstream.example/file:///etc", "https://upstream.example/<path>"),
+ ("sk-key-CANARY:file:///x", "<unparseable url>"),
+ ("https://xn--bcher-kva.example:8443/v1", "https://xn--bcher-kva.example:8443/<path>"),
+ ("http://10.0.0.7:8787", "http://10.0.0.7:8787"),
+ ],
+)
+def test_redact_url_hosts_and_hostless_schemes(
+ url: str, expected: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+ assert redact_url(url) == expected
+
+
[email protected]("error", [RuntimeError, LookupError, OSError, OverflowError, KeyError])
+def test_safe_id_never_runs_the_callers_repr(error: type[Exception]) -> None:
+ class RaisingStr(str):
+ def __repr__(self) -> str:
+ raise error("repr")
+
+ assert safe_id(RaisingStr("model-x")) == "'model-x'"
+
+
+def test_broken_exception_attributes_never_break_the_description(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+
+ class Broken(OSError):
+ @property # type: ignore[override]
+ def __cause__(self) -> BaseException:
+ raise LookupError("cause")
+
+ @property # type: ignore[override]
+ def __traceback__(self) -> None:
+ raise RuntimeError("traceback")
+
+ @property # type: ignore[override]
+ def errno(self) -> int:
+ raise OverflowError("errno")
+
+ error = Broken(2, "x")
+ error.__context__ = KeyError("k")
+
+ assert describe_exception(error) == (
+ f"{Broken.__qualname__} [Errno 2] {os.strerror(2)}; caused by KeyError"
+ )
+
+
+def test_hostless_urls_need_the_triple_slash_right_after_the_scheme() -> None:
+ assert redact_url("file:sk-CANARY-x://y") == "<unparseable url>"
+ assert redact_url("file:///var/x") == "file:///<path>"
+
+
+def test_int_ids_respect_the_process_digit_limit(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(sys, "get_int_max_str_digits", lambda: 640)
+ big = 10 ** int("700")
+
+ assert safe_id(big) == f"<int of {big.bit_length()} bits>"
+ assert safe_id(12345) == "12345"
+
+
+def test_safe_id_ignores_a_faked_class() -> None:
+ class Pretender:
+ @property # type: ignore[misc]
+ def __class__(self) -> type:
+ return str
+
+ assert safe_id(Pretender()) == f"<{Pretender.__qualname__}>"
+
+
[email protected]("name", ["Err\rforged", "sk-live-CANARY", "Err\u2028x"])
+def test_class_names_must_be_plain_identifiers(name: str) -> None:
+ cls = type(name, (Exception,), {})
+
+ assert describe_exception(cls("x")) == "<exception>"
+ assert safe_id(cls("x")) == "<object>"
+
+
[email protected]("url", [" \x01file:///var/x", "fi\tle:///var/x", "file:/\n//var/x"])
+def test_hostless_check_reads_the_url_the_way_urlsplit_does(url: str) -> None:
+ assert redact_url(url) == "file:///<path>"
+
+
+def test_int_ids_work_without_a_digit_limit_getter(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delattr(sys, "get_int_max_str_digits")
+
+ assert safe_id(12345) == "12345"
diff --git a/tests/test_saved_settings_apply_logging.py b/tests/test_saved_settings_apply_logging.py
new file mode 100644
index 000000000..5f8e3facf
--- /dev/null
+++ b/tests/test_saved_settings_apply_logging.py
@@ -0,0 +1,59 @@
+"""A failure applying saved settings is reported without the settings' values.
+
+Saved settings can hold credentials, and an error raised while applying them can
+quote a value, so the warning names the file and the failure's type and code
+location, never its message, at any level.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import pytest
+
+CREDENTIAL_CANARY = "sk-ant-SETTINGS-CANARY-4d18"
+
+
+def test_settings_apply_failure_warns_without_credential(monkeypatch, tmp_path) -> None:
+ from headroom import settings_store
+ from headroom.proxy.server import ProxyConfig, create_app
+
+ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path / "ws"))
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+
+ def _apply_fails(_values: object) -> None:
+ raise ValueError(f"cannot apply ANTHROPIC_API_KEY={CREDENTIAL_CANARY}")
+
+ monkeypatch.setattr(settings_store, "apply_to_environ", _apply_fails)
+
+ records: list[logging.LogRecord] = []
+
+ class _Capture(logging.Handler):
+ def emit(self, record: logging.LogRecord) -> None:
+ records.append(record)
+
+ proxy_logger = logging.getLogger("headroom.proxy")
+ handler = _Capture(level=logging.DEBUG)
+ old_level = proxy_logger.level
+ proxy_logger.addHandler(handler)
+ proxy_logger.setLevel(logging.DEBUG)
+ try:
+ create_app(ProxyConfig(port=0))
+ finally:
+ proxy_logger.removeHandler(handler)
+ proxy_logger.setLevel(old_level)
+
+ warnings = [r for r in records if "Could not apply saved settings" in r.getMessage()]
+ assert len(warnings) == 1
+ assert warnings[0].levelno == logging.WARNING
+ assert "ValueError" in warnings[0].getMessage()
+ assert "settings.json" in warnings[0].getMessage()
+ formatter = logging.Formatter()
+ for record in records:
+ assert CREDENTIAL_CANARY not in formatter.format(record)
+
+
[email protected](autouse=True)
+def _isolated_home(monkeypatch, tmp_path):
+ monkeypatch.setenv("HOME", str(tmp_path / "home"))
+ monkeypatch.setenv("CODEX_HOME", str(tmp_path / "codex"))
diff --git a/tests/test_savings_pricing_fallback_logging.py b/tests/test_savings_pricing_fallback_logging.py
new file mode 100644
index 000000000..c12750f8c
--- /dev/null
+++ b/tests/test_savings_pricing_fallback_logging.py
@@ -0,0 +1,105 @@
+"""Savings pricing fallbacks are debuggable without flooding or leaking into the log.
+
+An unpriced model is expected (local models, gateway aliases) and the estimators
+run on every request, so it is logged once per model. Unexpected pricing errors
+are logged by exception type and code location only: their text can carry a
+credential (a provider error quoting an API key) and must not reach any level.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import pytest
+
+import headroom.proxy.savings_tracker as st
+from headroom.log_safety import WarnOnce
+
+CREDENTIAL_CANARY = "sk-ant-CREDENTIAL-CANARY-91c2"
+
+
+class _Capture(logging.Handler):
+ def __init__(self) -> None:
+ super().__init__(level=logging.DEBUG)
+ self.records: list[logging.LogRecord] = []
+
+ def emit(self, record: logging.LogRecord) -> None:
+ self.records.append(record)
+
+
+class _CredentialEchoingPrice:
+ """A price value whose conversion fails with a message quoting a key."""
+
+ def __float__(self) -> float:
+ raise ValueError(f"bad price from provider config api_key={CREDENTIAL_CANARY}")
+
+
+def captured(monkeypatch: pytest.MonkeyPatch):
+ class _FakeLiteLLM:
+ model_cost = {"weird-priced-model": {"input_cost_per_token": _CredentialEchoingPrice()}}
+
+ @staticmethod
+ def cost_per_token(**_kwargs):
+ raise Exception(f"This model isn't mapped yet. key={CREDENTIAL_CANARY}")
+
+ monkeypatch.setattr(st, "_get_litellm_module", lambda: _FakeLiteLLM())
+ monkeypatch.setattr(st, "_unpriced_models_logged", WarnOnce(256, "test unpriced models"))
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+ st._resolve_litellm_model.cache_clear()
+ handler = _Capture()
+ old_level = st.logger.level
+ st.logger.addHandler(handler)
+ st.logger.setLevel(logging.DEBUG)
+ try:
+ yield handler.records
+ finally:
+ st.logger.removeHandler(handler)
+ st.logger.setLevel(old_level)
+ st._resolve_litellm_model.cache_clear()
+
+
+def _assert_no_canary(records: list[logging.LogRecord]) -> None:
+ formatter = logging.Formatter()
+ for record in records:
+ assert CREDENTIAL_CANARY not in formatter.format(record)
+
+
+def test_unpriced_model_logs_once(captured) -> None:
+ for _ in range(3):
+ assert st._estimate_compression_savings_usd("local-llama-7b", 1000) == pytest.approx(
+ 1000 * st.DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN
+ )
+ st._estimate_output_savings_usd("local-llama-7b", 1000)
+ st._estimate_input_cost_usd("local-llama-7b", 1000)
+
+ unpriced = [r for r in captured if "No litellm" in r.getMessage()]
+ assert len(unpriced) == 1
+ assert "model='local-llama-7b'" in unpriced[0].getMessage()
+ assert all(r.exc_info is None for r in captured)
+ # The litellm probe error quoted a key; it never reaches the log.
+ _assert_no_canary(captured)
+
+
+def test_credential_bearing_pricing_error_is_described_not_quoted(captured) -> None:
+ assert st._estimate_compression_savings_usd("weird-priced-model", 1000) == pytest.approx(
+ 1000 * st.DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN
+ )
+
+ failures = [
+ r for r in captured if "fallback rate for model='weird-priced-model'" in r.getMessage()
+ ]
+ assert len(failures) == 1
+ assert "ValueError" in failures[0].getMessage()
+ assert not any("No litellm" in r.getMessage() for r in captured)
+ _assert_no_canary(captured)
+
+
+def test_unpriced_model_not_marked_when_debug_disabled(captured) -> None:
+ st.logger.setLevel(logging.INFO)
+ st._estimate_compression_savings_usd("quiet-model", 10)
+ st.logger.setLevel(logging.DEBUG)
+ st._estimate_compression_savings_usd("quiet-model", 10)
+ assert [
+ r for r in captured if "quiet-model" in r.getMessage() and "No litellm" in r.getMessage()
+ ]
diff --git a/tests/test_tool_schema_savings_policy.py b/tests/test_tool_schema_savings_policy.py
index a44c9b124..9365e9a2f 100644
--- a/tests/test_tool_schema_savings_policy.py
+++ b/tests/test_tool_schema_savings_policy.py
@@ -140,3 +140,95 @@ def test_tool_schema_compaction_saves_real_tokens_not_just_bytes() -> None:
assert tokens_saved == tool_before - tool_after
assert original_tokens - optimized_tokens == tokens_saved
assert headline_tokens_saved(tokens_saved, {}) == tokens_saved
+
+
+SCHEMA_CANARY = "PRIVATE-SCHEMA-CANARY-7f3a"
+
+
+def _capture_policy_logs(monkeypatch):
+ import logging
+
+ from headroom.log_safety import WarnOnce
+ from headroom.proxy import tool_schema_savings_policy as policy
+
+ records: list[logging.LogRecord] = []
+
+ class _Capture(logging.Handler):
+ def emit(self, record: logging.LogRecord) -> None:
+ records.append(record)
+
+ handler = _Capture(level=logging.DEBUG)
+ policy.logger.addHandler(handler)
+ old_level = policy.logger.level
+ policy.logger.setLevel(logging.DEBUG)
+ monkeypatch.setattr(
+ policy, "_reconcile_warned", WarnOnce(32, "test reconcile types"), raising=False
+ )
+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)
+ return policy, records, handler, old_level
+
+
+def test_reconcile_failure_keeps_private_schema_out_of_every_log_record(monkeypatch) -> None:
+ import logging
+
+ policy, records, handler, old_level = _capture_policy_logs(monkeypatch)
+
+ tool = {"name": "Read", "defer_loading": True, "description": SCHEMA_CANARY}
+
+ def _tokenizer_echoes_schema(tools: object) -> int:
+ # A tokenizer error that quotes the schema it choked on.
+ raise ValueError(f"cannot tokenize {tools!r}")
+
+ tags = {"tool_search_deferred_tokens": 100}
+ booking = (100, {"Read": dict(tool)}, None)
+ try:
+ policy.reconcile_deferred_tokens(tags, booking, [dict(tool)], _tokenizer_echoes_schema)
+ policy.reconcile_deferred_tokens(tags, booking, [dict(tool)], _tokenizer_echoes_schema)
+ finally:
+ policy.logger.removeHandler(handler)
+ policy.logger.setLevel(old_level)
+
+ # The booked credit is left as it was.
+ assert tags["tool_search_deferred_tokens"] == 100
+
+ levels = [r.levelno for r in records]
+ assert levels.count(logging.WARNING) == 1, "warns once per exception type"
+ assert levels.count(logging.DEBUG) == 2, "every failure is recorded at debug"
+ formatter = logging.Formatter()
+ for record in records:
+ rendered = formatter.format(record)
+ assert SCHEMA_CANARY not in rendered
+ assert "ValueError" in rendered
+ assert "100 booked tokens" in records[0].getMessage()
+
+
+def test_reconcile_warns_again_for_a_new_exception_type(monkeypatch) -> None:
+ import logging
+
+ policy, records, handler, old_level = _capture_policy_logs(monkeypatch)
+ tool = {"name": "Read", "defer_loading": True, "description": "x"}
+ booking = (100, {"Read": dict(tool)}, None)
+
+ def _runtime(tools: object) -> int:
+ raise RuntimeError("boom")
+
+ def _value(tools: object) -> int:
+ raise ValueError("boom")
+
+ try:
+ # A healthy reconcile logs nothing.
+ policy.reconcile_deferred_tokens(
+ {"tool_search_deferred_tokens": 100}, booking, [dict(tool)], lambda t: 10
+ )
+ assert records == []
+ for counter in (_runtime, _runtime, _value):
+ policy.reconcile_deferred_tokens(
+ {"tool_search_deferred_tokens": 100}, booking, [dict(tool)], counter
+ )
+ finally:
+ policy.logger.removeHandler(handler)
+ policy.logger.setLevel(old_level)
+
+ warnings = [r.getMessage() for r in records if r.levelno == logging.WARNING]
+ assert len(warnings) == 2
+ assert "RuntimeError" in warnings[0] and "ValueError" in warnings[1]