Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): log the savings and metrics failures that currently vanish (#4084)
Tejas Chopra2800932e1827
diff --git a/headroom/log_safety.py b/headroom/log_safety.pynew file mode 100644index 000000000..8282195fc--- /dev/null+++ b/headroom/log_safety.py@@ -0,0 +1,383 @@+"""Payload-safe building blocks for log lines.++Log records are kept, rotated, attached to bug reports and shipped to log+collectors, so no level may carry request content, tool output or+credentials. Exception messages are the usual leak: a compressor, hook or+storage backend can quote the payload or a connection URL in its error. These+helpers describe a failure by what is safe to keep (exception types, code+locations, errno, ids, counts) and leave the message out.++Content is logged only when the operator opts in to it explicitly with+``HEADROOM_DEBUG_DUMP=full``, the same switch that gates full request dumps.+"""++from __future__ import annotations++import ipaddress+import logging+import math+import os+import re+import sys+import threading+import time+import traceback+from collections import deque+from collections.abc import Hashable+from types import FrameType, ModuleType, TracebackType+from urllib.parse import urlsplit++CONTENT_OPT_IN_ENV = "HEADROOM_DEBUG_DUMP"+_CONTENT_OPT_IN_VALUES = frozenset({"full", "all", "content"})+_MAX_FRAMES = 3+_MAX_CHAIN = 4+_ID_MAX_CHARS = 80+_MAX_ERRNO = 2**31 - 1+# Schemes whose URLs have no host ("file:///path").+_LOCAL_SCHEMES = frozenset({"file", "sqlite", "unix"})+# Strict DNS labels. "_" is left out on purpose: it is not valid in a public+# hostname but is common in keys (sk_live_...), so a host with it is masked.+_HOSTNAME = re.compile(+ r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*\.?"+)+_HOSTLESS_URL = re.compile(r"([A-Za-z][A-Za-z0-9+.-]*):///")+# What urlsplit removes before parsing: leading C0 controls and spaces, and+# every tab and newline. The hostless check must look at the same text.+_URL_LEADING_JUNK = "".join(chr(code) for code in range(33))+_URL_REMOVED = str.maketrans("", "", "\t\r\n")+_HEADROOM_MODULE = re.compile(r"headroom(?:\.[A-Za-z_][A-Za-z0-9_]*)*")+_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")+# A class's __qualname__: identifiers joined by dots, with <locals> for nested ones.+_QUALIFIED_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(?:\.(?:<locals>|[A-Za-z_][A-Za-z0-9_]*))*")+_FIXED_CODE_NAMES = frozenset(+ {"<module>", "<lambda>", "<genexpr>", "<listcomp>", "<dictcomp>", "<setcomp>"}+)+# The base classes' own attribute descriptors. Reading through them means no+# subclass property (a __cause__ or errno that raises, a __qualname__ chosen by+# a metaclass) can run while a failure is being described.+_CAUSE = vars(BaseException)["__cause__"]+_CONTEXT = vars(BaseException)["__context__"]+_SUPPRESS_CONTEXT = vars(BaseException)["__suppress_context__"]+_TRACEBACK = vars(BaseException)["__traceback__"]+_ERRNO = vars(OSError)["errno"]+_QUALNAME = vars(type)["__qualname__"]+# Ints above this many bits have more than 4300 digits, which repr refuses.+_MAX_ID_INT_BITS = 14_000+# The module __dict__ descriptor itself, so a ModuleType subclass cannot override it.+_MODULE_DICT = vars(ModuleType)["__dict__"]+# Schemes a log may name; any other text before "://" could be a key someone+# pasted where a URL belongs, so it is replaced.+_KNOWN_SCHEMES = frozenset(+ {+ "http",+ "https",+ "ws",+ "wss",+ "grpc",+ "grpcs",+ "redis",+ "rediss",+ "postgres",+ "postgresql",+ "mysql",+ "mongodb",+ "mongodb+srv",+ "sqlite",+ "file",+ "unix",+ "s3",+ "gs",+ }+)+++def content_logging_enabled() -> bool:+ """True when the operator opted in to content in diagnostics (``HEADROOM_DEBUG_DUMP=full``)."""+ return os.environ.get(CONTENT_OPT_IN_ENV, "").strip().lower() in _CONTENT_OPT_IN_VALUES+++def describe_exception(exc: BaseException) -> str:+ """Describe ``exc`` for a log line without its message.++ Gives each exception in the cause chain as its type plus the innermost code+ locations, e.g. ``ValueError at headroom/x.py:12 in parse; caused by+ KeyError at ...``. An ``OSError`` keeps its errno and the system's text for+ it, never the exception's own strerror. With ``HEADROOM_DEBUG_DUMP=full`` it returns the full+ traceback, messages included.+ """+ if content_logging_enabled():+ return "".join(traceback.format_exception(exc)).rstrip()+ parts: list[str] = []+ seen: set[int] = set()+ current: BaseException | None = exc+ while current is not None and id(current) not in seen and len(parts) < _MAX_CHAIN:+ seen.add(id(current))+ parts.append(_describe_one(current))+ current = _CAUSE.__get__(current) or (+ None if _SUPPRESS_CONTEXT.__get__(current) else _CONTEXT.__get__(current)+ )+ return "; caused by ".join(parts)+++def _describe_one(exc: BaseException) -> str:+ # A class name is chosen by whoever defined it (type("Err\nforged", ...)).+ text = _class_name(type(exc)) or "<exception>"+ # An int subclass errno could format itself as anything, so only a plain int.+ errno = _ERRNO.__get__(exc) if issubclass(type(exc), OSError) else None+ if type(errno) is int:+ # The system's own text for the errno; exc.strerror is free text in some+ # subclasses (ssl.SSLError) and can quote a path or a peer's message.+ # An out-of-range errno is not formatted at all: converting a huge int+ # to text can itself raise, and a log helper must never raise.+ # Negative codes (socket.gaierror's EAI_*) have no os.strerror text.+ if 0 <= errno <= _MAX_ERRNO:+ text += f" [Errno {errno}] {_errno_text(errno)}"+ elif -_MAX_ERRNO - 1 <= errno < 0:+ text += f" [Errno {errno}]"+ else:+ text += " [Errno out of range]"+ frames = _innermost_headroom_frames(_TRACEBACK.__get__(exc))+ if frames:+ text += " at " + " <- ".join(reversed(frames))+ return text+++def _errno_text(errno: int) -> str:+ try:+ return os.strerror(errno)+ except (ValueError, OverflowError):+ return "unknown error"+++def _innermost_headroom_frames(tb: TracebackType | None) -> list[str]:+ """Locations of the innermost Headroom frames, at most ``_MAX_FRAMES``, in memory bounded by that."""+ frames: deque[str] = deque(maxlen=_MAX_FRAMES)+ while tb is not None:+ location = _headroom_frame_location(tb.tb_frame, tb.tb_lineno)+ if location is not None:+ frames.append(location)+ tb = tb.tb_next+ return list(frames)+++def _headroom_frame_location(frame: FrameType, lineno: int) -> str | None:+ """``headroom/<module>.py:line in function`` for a frame of Headroom's own code, else None.++ Only Headroom's own frames are named; library and stdlib frames are skipped,+ since any text a frame carries (file, module or function name) can be set by+ code compiled at runtime and quote request data. A frame counts as Headroom's+ only when all of these hold, so none of its text is chosen at runtime:+ its globals are the namespace of the module ``sys.modules`` holds under that+ name; that name is a plain dotted ``headroom.*`` identifier; the code's file+ is the module's own file (compared by stem, so ``.pyc``-only and zip installs+ match); and the function name is a plain identifier. Headroom never execs code.+ """+ namespace = frame.f_globals+ # Read through dict's own methods: globals can be a dict subclass.+ name = dict.get(namespace, "__name__")+ if type(name) is not str or not _HEADROOM_MODULE.fullmatch(name):+ return None+ module = sys.modules.get(name)+ if not isinstance(module, ModuleType) or _MODULE_DICT.__get__(module) is not namespace:+ return None+ code = frame.f_code+ module_file = dict.get(namespace, "__file__")+ if type(module_file) is not str or _stem(module_file) != _stem(code.co_filename):+ return None+ function = code.co_name+ if not (_IDENTIFIER.fullmatch(function) or function in _FIXED_CODE_NAMES):+ return None+ package = dict.__contains__(namespace, "__path__")+ path = name.replace(".", "/") + ("/__init__.py" if package else ".py")+ return f"{path}:{lineno} in {function}"+++def _stem(path: str) -> str:+ return os.path.splitext(path.replace("\\", "/").rsplit("/", 1)[-1])[0]+++def redact_url(url: str) -> str:+ """Return ``url`` as scheme, host and port only, safe for any log line.++ Userinfo, query and fragment are always dropped, and so is the path, since+ a path segment can carry a credential (``/bot<token>/``, webhook URLs) and+ no rule tells such a segment from a route name. The path is kept only with+ the explicit content opt-in, ``HEADROOM_DEBUG_DUMP=full``. The host comes+ back lowercased, and only when it is a DNS name or IP address; a URL with+ no host is accepted only for ``file``, ``sqlite`` and ``unix``.+ """+ try:+ parts = urlsplit(url)+ host = parts.hostname or ""+ port = parts.port+ except ValueError:+ return "<unparseable url>"+ # Without "//" urlsplit reads "sk-key:secret" as scheme "sk-key", so a string+ # with no host is logged only for the hostless schemes ("file:///path").+ if not parts.netloc and not _is_hostless_local_url(url, parts.scheme):+ return "<unparseable url>"+ if parts.netloc:+ # A host that is not an IP or a strict DNS name may be a pasted secret;+ # it is masked, and scheme and port still say which endpoint was meant.+ host = _checked_host(host) or "<host>"+ scheme = parts.scheme if parts.scheme in _KNOWN_SCHEMES else "<scheme>"+ if ":" in host and host != "<host>":+ host = f"[{host}]"+ netloc = f"{host}:{port}" if port is not None else host+ if content_logging_enabled():+ path = parts.path+ else:+ path = "/<path>" if parts.path not in ("", "/") else parts.path+ redacted = f"{scheme}://{netloc}{path}"+ return f"{redacted}?<redacted>" if parts.query else redacted+++def _is_hostless_local_url(url: str, scheme: str) -> bool:+ """True for ``file:///…``, ``sqlite:///…`` or ``unix:///…`` as urlsplit read it."""+ cleaned = url.lstrip(_URL_LEADING_JUNK).translate(_URL_REMOVED)+ match = _HOSTLESS_URL.match(cleaned)+ return scheme in _LOCAL_SCHEMES and match is not None and match.group(1).lower() == scheme+++def _checked_host(host: str) -> str | None:+ """``host`` as a loggable name or address, or None when it may not be one.++ An IPv6 zone id (``fe80::1%eth0``) is free text, so it is dropped. A+ non-ASCII name is given in its ASCII (IDNA) form.+ """+ address = host.split("%", 1)[0]+ try:+ ipaddress.ip_address(address)+ except ValueError:+ pass+ else:+ return address+ try:+ ascii_host = host.encode("idna").decode("ascii").lower()+ except UnicodeError:+ return None+ return ascii_host if _HOSTNAME.fullmatch(ascii_host) else None+++def safe_id(value: object) -> str:+ """Render a client- or model-supplied id as one bounded log token.++ Uses the built-in ``repr`` of ``str``, ``bytes`` and numbers, so a newline+ in the id cannot forge a second log line and no ``__repr__`` of the+ caller's object runs; any other type is shown by its name. Cuts the result+ to a fixed length so an oversized id cannot flood the log.+ """+ text = _builtin_repr(value)+ if len(text) <= _ID_MAX_CHARS:+ return text+ return f"{text[:_ID_MAX_CHARS]}…(+{len(text) - _ID_MAX_CHARS} chars)"+++def _builtin_repr(value: object) -> str:+ # type() rather than isinstance: isinstance consults a __class__ the object+ # can fake, and then a built-in repr would run on the wrong type.+ kind = type(value)+ if value is None or kind is bool:+ return "None" if value is None else ("True" if value else "False")+ if issubclass(kind, str):+ return str.__repr__(value) # type: ignore[arg-type]+ if issubclass(kind, bytes):+ return bytes.__repr__(value) # type: ignore[arg-type]+ if issubclass(kind, int):+ return _int_repr(value) # type: ignore[arg-type]+ if issubclass(kind, float):+ return float.__repr__(value) # type: ignore[arg-type]+ return f"<{_class_name(kind) or 'object'}>"+++def _int_repr(value: int) -> str:+ bits = int.bit_length(value)+ # Digits an int of this many bits can have, against the process's own limit+ # (sys.set_int_max_str_digits); past it, repr raises ValueError.+ # The limit and its getter arrived in 3.10.7; earlier releases have no limit.+ get_limit = getattr(sys, "get_int_max_str_digits", None)+ limit = get_limit() if get_limit is not None else 0+ if bits > _MAX_ID_INT_BITS or (limit and bits * 0.30103 + 1 >= limit):+ return f"<int of {bits} bits>"+ return int.__repr__(value)+++def _class_name(cls: type) -> str | None:+ """The class's qualified name when it is plain identifiers, else None.++ Whoever defines a class chooses its name (``type("sk-...\\r", ...)``), so only+ names shaped like source identifiers are logged.+ """+ name = _QUALNAME.__get__(cls)+ return name if type(name) is str and _QUALIFIED_NAME.fullmatch(name) else None+++class WarnOnce:+ """Warn once per key per window, for at most ``limit`` keys per window.++ The first warning opens a window of ``window_seconds`` (an hour by+ default). Inside it each key warns once; past ``limit`` keys it logs one+ overflow WARNING and stays quiet, so a failure that hits many keys+ (paths, rows, models) cannot flood the log: at most ``limit + 1`` warnings+ per window. When the window ends the guard starts over, so a failure that+ keeps recurring, or a newly broken key, warns again once an hour. A key is+ used up only when WARNING is enabled, so a warning suppressed by the log+ level is still emitted later. ``forget`` re-arms a key once its failure has+ cleared; a re-armed key that fails again counts against the same window's+ budget, so forgetting can never push a window past ``limit + 1`` warnings.+ """++ def __init__(self, limit: int, what: str, *, window_seconds: float = 3600.0) -> None:+ if limit < 1:+ raise ValueError(f"WarnOnce limit must be at least 1, got {limit}")+ if not 0 < window_seconds < math.inf: # also rejects nan+ raise ValueError(+ f"WarnOnce window_seconds must be positive and finite, got {window_seconds}"+ )+ self._limit = limit+ self._what = what+ self._window = window_seconds+ self._keys: set[Hashable] = set()+ self._issued = 0 # warnings issued this window; forget() does not refund them+ self._window_start: float | None = None+ self._overflowed = False+ self._lock = threading.Lock()++ def first(self, key: Hashable, log: logging.Logger) -> bool:+ """Return True if the caller should log this key's WARNING now."""+ if not log.isEnabledFor(logging.WARNING):+ return False+ with self._lock:+ now = time.monotonic()+ if self._window_start is not None and now - self._window_start >= self._window:+ self._keys.clear()+ self._issued = 0+ self._overflowed = False+ self._window_start = None+ if key in self._keys:+ return False+ if not self._overflowed and self._issued < self._limit:+ self._keys.add(key)+ self._issued += 1+ if self._window_start is None:+ self._window_start = now+ return True+ report_overflow = not self._overflowed+ self._overflowed = True+ started = now if self._window_start is None else self._window_start+ remaining = self._window - (now - started)+ if report_overflow:+ log.warning(+ "More than %d warnings about %s this window; further ones are not logged "+ "at WARNING "+ "for the next %d minutes",+ self._limit,+ self._what,+ max(1, math.ceil(remaining / 60)),+ )+ return False++ def forget(self, key: Hashable) -> None:+ """Re-arm ``key`` so its next failure warns again."""+ with self._lock:+ self._keys.discard(key)diff --git a/headroom/proxy/handlers/_debug_dump.py b/headroom/proxy/handlers/_debug_dump.pyindex b04459fb3..f58fc9fb6 100644--- a/headroom/proxy/handlers/_debug_dump.py+++ b/headroom/proxy/handlers/_debug_dump.py@@ -14,7 +14,8 @@ import osfrom datetime import datetimefrom pathlib import Pathfrom typing import Any-from urllib.parse import urlsplit, urlunsplit++from headroom.log_safety import content_logging_enabled, redact_urllogger = logging.getLogger(__name__)@@ -29,9 +30,9 @@ def _debug_dump_mode(config: Any) -> str:"""if getattr(config, "stateless", False):return "off"- raw = os.environ.get("HEADROOM_DEBUG_DUMP", "").strip().lower()- if raw in ("full", "all", "content"):+ if content_logging_enabled():return "full"+ raw = os.environ.get("HEADROOM_DEBUG_DUMP", "").strip().lower()if raw in ("1", "true", "yes", "on", "redacted"):return "redacted"return "off"@@ -54,23 +55,6 @@ def _redact_debug_value(value: Any, _max_len: int = 80) -> Any:return value-def _redact_url(url: str) -> str:- """Drop the query string and fragment from ``url``.-- Some upstreams carry credentials in the query (Gemini's ``?key=``), and the- query is never needed to debug a rejected request, so it is elided in every- mode — ``full`` opts in to prompt content, not to credentials.- """- try:- parts = urlsplit(url)- except ValueError:- return "<unparseable url>"- if not parts.query and not parts.fragment:- return url- redacted = urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))- return f"{redacted}?<redacted>" if parts.query else redacted--def _decode_sent_body(body: Any) -> Any:"""Decode the exact bytes sent upstream back to JSON for the dump."""if not isinstance(body, (bytes, bytearray)):@@ -119,7 +103,7 @@ def write_upstream_error_dump(payload = json.dumps({"request_id": request_id,- "url": _redact_url(url),+ "url": redact_url(url),"status": status,"provider": provider,"model": model,diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.pyindex 1a6d66694..9b53b69b6 100644--- a/headroom/proxy/outcome.py+++ b/headroom/proxy/outcome.py@@ -31,6 +31,7 @@ from dataclasses import dataclass, fieldfrom datetime import datetime, timezonefrom typing import Any+from headroom.log_safety import describe_exceptionfrom headroom.proxy.conversation_savings import get_conversation_savingsfrom headroom.proxy.tool_schema_savings_policy import (headline_tokens_saved,@@ -629,8 +630,12 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:# does a full read-modify-write of the ledger file — run them# together off the event loop (#18) so a slow flush can't stall it.output_tokens_saved_est = await asyncio.to_thread(_record_and_estimate)- except Exception: # pragma: no cover - defensive- pass+ except Exception as exc: # pragma: no cover - defensive+ logger.debug(+ "Output savings record/estimate failed for request %s; booking 0: %s",+ outcome.request_id,+ describe_exception(exc),+ )# Project attribution: explicit outcome field wins, else the value the# HTTP middleware / WS accept captured from ``X-Headroom-Project``.diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.pyindex d07622ce0..1593b7727 100644--- a/headroom/proxy/savings_tracker.py+++ b/headroom/proxy/savings_tracker.py@@ -24,6 +24,7 @@ from pathlib import Pathfrom typing import Anyfrom headroom import paths as _paths+from headroom.log_safety import WarnOnce, describe_exception, safe_idfrom headroom.proxy import project_name_policyfrom headroom.proxy.persistent_metrics import PersistentMetricsState@@ -355,14 +356,23 @@ def _resolve_litellm_model(model: str) -> str:info = litellm.model_cost.get(resolved)if info and info.get("input_cost_per_token") is not None:return resolved- except Exception:- pass+ except Exception as exc:+ logger.debug(+ "Shared litellm model resolution failed for model=%s: %s",+ safe_id(model),+ describe_exception(exc),+ )+ # litellm raises a bare Exception for a model it has not mapped, so this is+ # the narrowest catch that works. A miss means "not this spelling, try the+ # provider prefixes next". Runs once per model (lru_cache above).try:litellm.cost_per_token(model=model, prompt_tokens=1, completion_tokens=0)return model- except Exception:- pass+ except Exception as exc:+ logger.debug(+ "litellm cannot price model=%s as given: %s", safe_id(model), describe_exception(exc)+ )prefixes = {"claude-": "anthropic/",@@ -388,6 +398,25 @@ def _resolve_litellm_model(model: str) -> str:return model+# An unpriced model (a local model, a gateway alias, a model newer than the+# bundled litellm) is an expected case, and the estimators below run on every+# request, so each model is logged once. Debug, not warning: proxy/cost.py+# already warns once per unpriceable model on the live path. Model names come+# from clients, so the guard is bounded like _resolve_litellm_model's cache.+_unpriced_models_logged = WarnOnce(_MODEL_RESOLUTION_CACHE_MAXSIZE, "unpriced models")+++def _log_unpriced_model_once(model: str, field: str) -> None:+ if not logger.isEnabledFor(logging.DEBUG):+ return+ if _unpriced_models_logged.first(model, logger):+ logger.debug(+ "No litellm %s for model=%s; savings and cost use the fallback rate",+ field,+ safe_id(model),+ )++def _estimate_compression_savings_usd(model: str, tokens_saved: int) -> float:"""Estimate compression savings in USD from saved input tokens."""litellm = _get_litellm_module()@@ -405,9 +434,15 @@ def _estimate_compression_savings_usd(model: str, tokens_saved: int) -> float:# a real 0.0 as unavailable and billed the $3/M fallback — phantom savings# for a model that costs nothing.if input_cost_per_token is None:- raise RuntimeError("input cost unavailable")+ _log_unpriced_model_once(model, "input_cost_per_token")+ return float(tokens_saved) * float(DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN)return float(tokens_saved) * float(input_cost_per_token)- except Exception:+ except Exception as exc:+ logger.debug(+ "Compression savings priced at the fallback rate for model=%s: %s",+ safe_id(model),+ describe_exception(exc),+ )return float(tokens_saved) * float(DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN)@@ -432,9 +467,15 @@ def _estimate_output_savings_usd(model: str, tokens_saved: int) -> float:# rate -> phantom output savings for a model that costs nothing. Mirrors# the fix already applied to `_estimate_compression_savings_usd`.if output_cost_per_token is None:- raise RuntimeError("output cost unavailable")+ _log_unpriced_model_once(model, "output_cost_per_token")+ return float(tokens_saved) * float(DEFAULT_FALLBACK_OUTPUT_COST_PER_TOKEN)return float(tokens_saved) * float(output_cost_per_token)- except Exception:+ except Exception as exc:+ logger.debug(+ "Output savings priced at the fallback rate for model=%s: %s",+ safe_id(model),+ describe_exception(exc),+ )return float(tokens_saved) * float(DEFAULT_FALLBACK_OUTPUT_COST_PER_TOKEN)@@ -481,7 +522,12 @@ def _estimate_cache_savings_usd(model: str, cache_read_tokens: int) -> float:if discount <= 0:return 0.0return float(cache_read_tokens) * discount- except Exception:+ except Exception as exc:+ logger.debug(+ "Cache savings pricing failed for model=%s; booking 0: %s",+ safe_id(model),+ describe_exception(exc),+ )return 0.0@@ -628,7 +674,8 @@ def _estimate_input_cost_usd(# A missing key means the model is unknown → fall back to a blended rate.# A present 0.0 means the model is free and must cost $0, not the fallback.if input_cost_per_token is None:- raise RuntimeError("input cost unavailable")+ _log_unpriced_model_once(model, "input_cost_per_token")+ return float(chargeable_tokens) * float(DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN)if use_breakdown:cache_read_cost = info.get(@@ -646,7 +693,12 @@ def _estimate_input_cost_usd()return float(total_input_tokens) * float(input_cost_per_token)- except Exception:+ except Exception as exc:+ logger.debug(+ "Input cost priced at the fallback rate for model=%s: %s",+ safe_id(model),+ describe_exception(exc),+ )return float(chargeable_tokens) * float(DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN)diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 2f6323176..e07abfc8f 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -86,6 +86,7 @@ from headroom.config import (ReadLifecycleConfig,)from headroom.dashboard import get_dashboard_html+from headroom.log_safety import describe_exception, safe_idfrom headroom.observability import (LangfuseTracingConfig,OTelMetricsConfig,@@ -3129,12 +3130,24 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:# that construct the app without going through the `headroom` CLI entrypoint# (which already applies them before Click parsing). setdefault keeps# explicit env exports authoritative; fail-open so it never blocks startup.+ # settings_store.load() already reports an unreadable or corrupt file; this+ # catches anything else that goes wrong while applying it.+ settings_file: object = "the saved settings file"try:+ from headroom import paths as _hr_pathsfrom headroom import settings_store+ settings_file = _hr_paths.settings_path()settings_store.apply_to_environ(settings_store.load())- except Exception: # noqa: BLE001 — settings load must never break startup- pass+ except Exception as exc: # noqa: BLE001 — settings load must never break startup+ # Settings values can hold credentials, so the exception is described by+ # type and code location only, never its message.+ logger.warning(+ "Could not apply saved settings from %s (%s); continuing with env and "+ "defaults. Fix or remove that file and restart the proxy.",+ settings_file,+ describe_exception(exc),+ )# Air-gap master switch. Propagate config.offline to the env so the# env-based egress predicates (telemetry, update check, license) all honor@@ -4148,8 +4161,14 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:action="admin_request",status_code=response.status_code,)- except Exception:- logger.debug("admin audit emission failed", exc_info=True)+ except Exception as exc:+ logger.warning(+ "Admin audit record not written for %s %s (%s); this request is missing "+ "from the headroom.audit trail. Please report this as a bug.",+ request.method,+ safe_id(request.url.path),+ describe_exception(exc),+ )return response# The gate above is http-only (BaseHTTPMiddleware ignores every other@@ -4994,8 +5013,8 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:_oest = get_recorder().estimate(_olevel)output_reduction = _output_reduction_payload(_shaper_active, _oest)- except Exception: # pragma: no cover - defensive- pass+ except Exception as exc: # pragma: no cover - defensive+ logger.debug("/stats output_reduction section failed: %s", describe_exception(exc))# Model-routing section: populated only when an extension registered a# routing-stats provider (headroom.proxy.routing_stats); None otherwise.diff --git a/headroom/proxy/tool_schema_savings_policy.py b/headroom/proxy/tool_schema_savings_policy.pyindex 7b2bd8f81..c75f09e94 100644--- a/headroom/proxy/tool_schema_savings_policy.py+++ b/headroom/proxy/tool_schema_savings_policy.py@@ -26,8 +26,32 @@ name to :data:`TOOL_SCHEMA_SAVINGS_TAGS` and every surface picks it up.from __future__ import annotationsimport copy+import loggingfrom typing import Any+from headroom.log_safety import WarnOnce, describe_exception++logger = logging.getLogger(__name__)++# A failure that repeats on every request warns once per exception type; every+# occurrence also goes to debug.+_reconcile_warned = WarnOnce(32, "tool-schema reconcile failure types")+++def _log_reconcile_failure(exc: Exception, booked: int) -> None:+ # The exception comes from tokenizing client tool schemas and can quote+ # them, so it is described by type and code location only, at every level.+ where = describe_exception(exc)+ if _reconcile_warned.first(type(exc).__qualname__, logger):+ logger.warning(+ "tool-schema deferral reconcile failed (%s); credit left at %d booked tokens, "+ "so /stats over-reports tool-search savings for this request. Please report "+ "this as a bug.",+ where,+ booked,+ )+ logger.debug("tool-schema deferral reconcile failed; booked=%d: %s", booked, where)+def without_deferral_flags(tools: object) -> object:"""The tool array as it is MEASURED for a hook's tool delta.@@ -128,7 +152,8 @@ def reconcile_deferred_tokens(tags["tool_search_deferred_tools"] = len(kept)if isinstance(entry, dict):entry["tokens"] = max(0, int(entry.get("tokens") or 0) - released)- except Exception: # accounting must never break a request+ except Exception as exc: # accounting must never break a request+ _log_reconcile_failure(exc, booked)returndiff --git a/tests/test_debug_dump_gating.py b/tests/test_debug_dump_gating.pyindex 924c9d3fa..40a87d27d 100644--- a/tests/test_debug_dump_gating.py+++ b/tests/test_debug_dump_gating.py@@ -189,10 +189,25 @@ def test_upstream_dump_never_writes_url_credentials(dump_dir, monkeypatch, mode)assert path is not Nonetext = path.read_text()assert "AIzaSECRET" not in text- assert json.loads(text)["url"] == (- "https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"- "?<redacted>"- )+ if mode == "full": # the content opt-in keeps the route, still never the key+ expected = (+ "https://generativelanguage.googleapis.com/v1beta/models/gemini:streamGenerateContent"+ )+ else: # a path segment can carry a credential, so it is dropped without the opt-in+ expected = "https://generativelanguage.googleapis.com/<path>"+ assert json.loads(text)["url"] == expected + "?<redacted>"+++ "url",+ ["https://upstream.example/botCANARY/v1/messages", "https://upstream.example/bot%43ANARY/v1"],+)+def test_redacted_dump_never_writes_path_credentials(dump_dir, monkeypatch, url):+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "redacted")+ path = _write(url=url)+ assert path is not None+ text = path.read_text()+ assert "CANARY" not in text and "%43ANARY" not in textdef test_upstream_dump_keeps_query_free_url(dump_dir, monkeypatch):diff --git a/tests/test_log_safety.py b/tests/test_log_safety.pynew file mode 100644index 000000000..dfe95d473--- /dev/null+++ b/tests/test_log_safety.py@@ -0,0 +1,555 @@+"""Tests for the payload-safe log helpers in ``headroom.log_safety``."""++from __future__ import annotations++import importlib+import importlib.machinery+import importlib.util+import json+import logging+import math+import os+import py_compile+import sys+import types+import zipfile+import zipimport+from pathlib import Path++import pytest++from headroom import log_safety+from headroom.log_safety import WarnOnce, describe_exception, redact_url, safe_id++CANARY = "sk-live-CANARY tool output: patient record 42"+++def _raise_chain() -> BaseException:+ try:+ try:+ raise KeyError(CANARY)+ except KeyError as inner:+ raise ValueError(f"could not parse {CANARY}") from inner+ except ValueError as exc:+ return exc+++def test_exception_description_names_types_and_places_but_not_messages(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)+ text = describe_exception(_raise_chain())++ assert "CANARY" not in text+ # The chain is raised from this test file, which is not Headroom's code,+ # so only the types are named.+ assert text == "ValueError; caused by KeyError"+++def test_oserror_keeps_errno_but_not_its_filename(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)+ text = describe_exception(PermissionError(13, "Permission denied", f"/tmp/{CANARY}"))++ assert text == "PermissionError [Errno 13] Permission denied"+++def test_oserror_free_text_strerror_is_replaced_by_the_system_text() -> None:+ text = describe_exception(OSError(2, f"token={CANARY}"))++ assert "CANARY" not in text+ assert text == f"FileNotFoundError [Errno 2] {os.strerror(2)}"+++def test_content_opt_in_gives_the_full_traceback(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")++ text = describe_exception(_raise_chain())++ assert "Traceback" in text+ assert CANARY in text+++ ("url", "expected"),+ [+ (+ "https://user:[email protected]:8443/v1?key=abc#frag",+ "https://proxy.example:8443/<path>?<redacted>",+ ),+ ("https://upstream.example/botSECRET/v1/messages", "https://upstream.example/<path>"),+ ("https://upstream.example/bot%53ECRET/v1", "https://upstream.example/<path>"),+ ("http://127.0.0.1:8787", "http://127.0.0.1:8787"),+ ("http://[::1]:8787/", "http://[::1]:8787/"),+ ("http://[::1", "<unparseable url>"),+ ],+)+def test_redact_url_keeps_only_scheme_host_and_port(+ url: str, expected: str, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)+ assert redact_url(url) == expected+++def test_content_opt_in_keeps_the_path_but_never_credentials(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", "full")++ assert redact_url("https://u:[email protected]/v1/messages?key=abc") == (+ "https://upstream.example/v1/messages?<redacted>"+ )+++def test_safe_id_escapes_newlines_and_bounds_length() -> None:+ assert safe_id("m1\nWARNING forged") == "'m1\\nWARNING forged'"+ long = safe_id("x" * 10_000)+ assert len(long) < 120+ assert long.endswith("chars)")+++def _warnings(caplog: pytest.LogCaptureFixture) -> list[str]:+ return [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]+++def test_warn_once_stays_bounded_across_repeated_passes(caplog: pytest.LogCaptureFixture) -> None:+ log = logging.getLogger("headroom.test_log_safety.bounded")+ guard = WarnOnce(limit=64, what="unwritable ledgers")++ with caplog.at_level(logging.INFO, logger=log.name):+ for _ in range(3):+ for path in range(65):+ if guard.first(path, log):+ log.warning("ledger %d failed", path)++ messages = _warnings(caplog)+ assert len(messages) == 65 # 64 keys plus one overflow notice, not 65 per pass+ assert messages[-1] == (+ "More than 64 warnings about unwritable ledgers this window; further ones are not logged "+ "at WARNING "+ "for the next 60 minutes"+ )+++def test_warn_once_keeps_the_key_while_warnings_are_disabled() -> None:+ log = logging.getLogger("headroom.test_log_safety.level")+ guard = WarnOnce(limit=4, what="failures")+ log.setLevel(logging.ERROR)+ try:+ assert guard.first("k", log) is False+ log.setLevel(logging.INFO)+ assert guard.first("k", log) is True+ assert guard.first("k", log) is False+ finally:+ log.setLevel(logging.NOTSET)+++def test_forget_rearms_a_key() -> None:+ log = logging.getLogger("headroom.test_log_safety.forget")+ guard = WarnOnce(limit=4, what="failures")++ assert guard.first("path", log) is True+ guard.forget("path")+ assert guard.first("path", log) is True+++def test_raise_from_none_hides_the_suppressed_context(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)+ try:+ try:+ raise KeyError(CANARY)+ except KeyError:+ raise ValueError("bad") from None+ except ValueError as exc:+ text = describe_exception(exc)++ assert "KeyError" not in text+++def test_a_cause_cycle_is_described_once() -> None:+ first = ValueError(CANARY)+ second = KeyError(CANARY)+ first.__cause__ = second+ second.__cause__ = first++ assert describe_exception(first) == "ValueError; caused by KeyError"+++def test_forgetting_after_overflow_does_not_reopen_the_cap(+ caplog: pytest.LogCaptureFixture,+) -> None:+ log = logging.getLogger("headroom.test_log_safety.reopen")+ guard = WarnOnce(limit=2, what="failures")++ with caplog.at_level(logging.INFO, logger=log.name):+ assert guard.first("a", log) and guard.first("b", log)+ assert guard.first("c", log) is False # overflow notice+ for cycle in range(5):+ guard.forget("a")+ assert guard.first(f"new-{cycle}", log) is False++ assert len(_warnings(caplog)) == 1+++def test_warn_once_starts_over_after_the_window(+ caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch+) -> None:+ log = logging.getLogger("headroom.test_log_safety.window")+ guard = WarnOnce(limit=2, what="hook failures", window_seconds=600)+ clock = [1000.0]+ monkeypatch.setattr(log_safety.time, "monotonic", lambda: clock[0])++ results = []+ with caplog.at_level(logging.INFO, logger=log.name):+ steps = (("a", 0), ("b", 0), ("c", 0), ("a", 300), ("d", 299), ("a", 1), ("new-hook", 0))+ for key, advance in steps:+ clock[0] += advance+ results.append(guard.first(key, log))++ # a and b warn and c is the overflow notice; inside the window a stays quiet+ # and d is suppressed; once the window ends, the still-recurring a and a+ # newly broken key both warn again.+ assert results == [True, True, False, False, False, True, True]+ assert len(_warnings(caplog)) == 1+++def test_warn_once_needs_a_positive_window() -> None:+ with pytest.raises(ValueError, match="window_seconds must be positive"):+ WarnOnce(limit=1, what="failures", window_seconds=0)++[email protected]("digits", [19, 10_001])+def test_out_of_range_errno_never_breaks_the_description(digits: int) -> None:+ """An errno too large to print (Python caps int-to-text at 4300 digits) is not printed."""+ exc = OSError()+ exc.errno = 10 ** (digits - 1) # computed, not a literal, so collection never formats it++ assert describe_exception(exc) == "OSError [Errno out of range]"+++def test_negative_dns_error_codes_are_kept() -> None:+ import socket++ assert describe_exception(socket.gaierror(-2, "Name or service not known")) == (+ "gaierror [Errno -2]"+ )+++def test_overflow_notice_states_the_time_left_in_the_window(+ caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch+) -> None:+ log = logging.getLogger("headroom.test_log_safety.remaining")+ guard = WarnOnce(limit=1, what="failures", window_seconds=3600)+ clock = [0.0]+ monkeypatch.setattr(log_safety.time, "monotonic", lambda: clock[0])++ with caplog.at_level(logging.INFO, logger=log.name):+ guard.first("a", log)+ clock[0] = 3000.0 # ten minutes left in the window+ guard.first("b", log)++ assert _warnings(caplog)[-1].endswith("for the next 10 minutes")+++def test_forget_cycles_cannot_exceed_the_window_budget(caplog: pytest.LogCaptureFixture) -> None:+ log = logging.getLogger("headroom.test_log_safety.forget_budget")+ guard = WarnOnce(limit=3, what="failures")++ with caplog.at_level(logging.INFO, logger=log.name):+ issued = 0+ for cycle in range(50):+ if guard.first(f"path-{cycle}", log):+ issued += 1+ log.warning("path-%d failed", cycle)+ guard.forget(f"path-{cycle}")++ assert issued == 3+ assert len(_warnings(caplog)) == 4 # three warnings plus one overflow notice+++ "value",+ ["sk-ant-abc123:x", "AKIAABCDEFGH:secret", "myuser:hunter2@db:5432", "AKIASECRET://host/x"],+)+def test_text_that_is_not_a_url_never_reaches_the_log(value: str) -> None:+ text = redact_url(value)+ for secret in ("sk-ant", "abc123", "AKIA", "akia", "hunter2", "myuser", "SECRET", "secret"):+ assert secret not in text++[email protected]("host", ["a b", "a\x0bb\x0cc", "a\x85b"])+def test_hosts_with_line_breaks_are_not_logged(host: str) -> None:+ assert redact_url(f"http://{host}/") == "http://<host>/"+++def test_class_names_cannot_forge_a_log_line(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)+ forged = type("Err\nWARNING forged sk-QUAL", (Exception,), {})++ text = describe_exception(forged("x"))++ assert "\n" not in text and "sk-QUAL" not in text+ assert text.startswith("<exception>")+++def test_odd_errno_and_module_objects_never_break_the_description(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)++ class FormatsItself(int):+ def __format__(self, spec: str) -> str:+ return "sk-ERRNO-CANARY"++ exc = OSError()+ exc.errno = FormatsItself(2)+ assert "CANARY" not in describe_exception(exc)++ class RaisingGet(dict): # type: ignore[type-arg]+ def get(self, *args: object) -> object:+ raise RuntimeError("get")++ namespace = RaisingGet()+ exec(compile("def boom():\n raise ValueError('x')\n", "<x>", "exec"), namespace)+ with pytest.raises(ValueError) as caught:+ namespace["boom"]()+ assert " at " not in describe_exception(caught.value)++ lazy = types.ModuleType("lazy_sk_mod")++ def lazy_getattr(attr: str) -> str:+ return "sk-FILE-CANARY.py"++ lazy.__getattr__ = lazy_getattr # type: ignore[method-assign]+ monkeypatch.setitem(sys.modules, "lazy_sk_mod", lazy)+ exec(compile("def boom():\n raise ValueError('x')\n", "<x>", "exec"), vars(lazy))+ with pytest.raises(ValueError) as caught:+ lazy.boom()+ assert "CANARY" not in describe_exception(caught.value)+++def test_safe_id_never_raises() -> None:+ class BadRepr:+ def __repr__(self) -> str:+ raise RuntimeError("repr")++ huge = 10 ** int("5000")+ assert safe_id(huge) == "<int of 16610 bits>"+ assert safe_id(BadRepr()) == f"<{BadRepr.__qualname__}>"++[email protected]("window", [math.inf, math.nan, 0, -1])+def test_warn_once_needs_a_finite_positive_window(window: float) -> None:+ with pytest.raises(ValueError, match="window_seconds must be positive and finite"):+ WarnOnce(limit=1, what="failures", window_seconds=window)+++def test_headroom_frames_are_named_by_module_path() -> None:+ with pytest.raises(ValueError) as caught:+ WarnOnce(limit=0, what="failures")++ assert describe_exception(caught.value) == (+ f"ValueError at headroom/log_safety.py:{caught.traceback[-1].lineno + 1} in __init__"+ )+++def test_library_frames_are_skipped_but_the_headroom_caller_is_kept() -> None:+ with pytest.raises(json.JSONDecodeError) as caught:+ json.loads("{")++ assert describe_exception(caught.value) == "JSONDecodeError"+++ ("filename", "module_name"),+ [+ ("sk-FILE-CANARY\nforged", None),+ (os.__file__, "os"),+ (log_safety.__file__, "headroom.log_safety"),+ ],+ ids=["forged-name", "claims-stdlib-file", "claims-headroom-module"],+)+def test_runtime_compiled_frames_are_not_logged(+ monkeypatch: pytest.MonkeyPatch, filename: str, module_name: str | None+) -> None:+ """Runtime code may claim a real file and module name; it is still not named."""+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)+ code = compile("def sk_FUNC_CANARY():\n raise ValueError('x')\n", filename, "exec")+ namespace: dict[str, object] = {} if module_name is None else {"__name__": module_name}+ exec(code, namespace)+ with pytest.raises(ValueError) as caught:+ namespace["sk_FUNC_CANARY"]() # type: ignore[operator]++ assert describe_exception(caught.value) == "ValueError"+++def test_code_execd_into_a_headroom_module_is_not_named(monkeypatch: pytest.MonkeyPatch) -> None:+ namespace = vars(log_safety)+ code = compile("def sk_GEN_CANARY():\n raise ValueError('x')\n", "<generated>", "exec")+ exec(code, namespace)+ monkeypatch.delitem(namespace, "sk_GEN_CANARY")+ function = code.co_consts[0]+ generated = types.FunctionType(function, namespace)++ with pytest.raises(ValueError) as caught:+ generated()++ assert describe_exception(caught.value) == "ValueError"+++ "module_name", ["headroom.x\nWARNING forged", "headroom.sk-live-CANARY", "headroomx"]+)+def test_odd_module_names_are_not_named(monkeypatch: pytest.MonkeyPatch, module_name: str) -> None:+ module = types.ModuleType(module_name)+ module.__file__ = "probe.py"+ monkeypatch.setitem(sys.modules, module_name, module)+ exec(compile("def boom():\n raise ValueError('x')\n", "probe.py", "exec"), vars(module))++ with pytest.raises(ValueError) as caught:+ module.boom()++ assert describe_exception(caught.value) == "ValueError"+++def test_zip_and_sourceless_headroom_modules_keep_their_locations(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ source = "def boom():\n raise ValueError('x')\n"+ with zipfile.ZipFile(tmp_path / "bundle.zip", "w") as bundle:+ bundle.writestr("_zip_probe.py", source)+ (tmp_path / "_pyc_probe.py").write_text(source)+ py_compile.compile(str(tmp_path / "_pyc_probe.py"), cfile=str(tmp_path / "_pyc_probe.pyc"))+ (tmp_path / "_pyc_probe.py").unlink()++ zip_spec = zipimport.zipimporter(str(tmp_path / "bundle.zip")).find_spec("headroom._zip_probe")+ pyc_loader = importlib.machinery.SourcelessFileLoader(+ "headroom._pyc_probe", str(tmp_path / "_pyc_probe.pyc")+ )+ pyc_spec = importlib.util.spec_from_loader("headroom._pyc_probe", pyc_loader)+ texts = []+ for spec in (zip_spec, pyc_spec):+ assert spec is not None and spec.loader is not None+ module = importlib.util.module_from_spec(spec)+ monkeypatch.setitem(sys.modules, spec.name, module)+ spec.loader.exec_module(module)+ with pytest.raises(ValueError) as caught:+ module.boom()+ texts.append(describe_exception(caught.value))++ assert texts == [+ "ValueError at headroom/_zip_probe.py:2 in boom",+ "ValueError at headroom/_pyc_probe.py:2 in boom",+ ]+++def test_deep_tracebacks_keep_only_the_innermost_headroom_frames() -> None:+ def recurse(depth: int) -> None:+ if depth == 0:+ WarnOnce(limit=0, what="failures")+ recurse(depth - 1)++ with pytest.raises(ValueError) as caught:+ recurse(500)++ assert describe_exception(caught.value).count(" in ") == 1+++ ("url", "expected"),+ [+ ("file:///var/lib/headroom/ledger.db", "file:///<path>"),+ ("sqlite:///tmp/x.db", "sqlite:///<path>"),+ ("unix:///run/headroom.sock", "unix:///<path>"),+ ("https://sk_live_CANARY@/v1", "https://<host>/<path>"),+ ("https://sk_live_CANARY/v1", "https://<host>/<path>"),+ ("https://abcDEF+ghi=/v1", "https://<host>/<path>"),+ ("http://model_gateway:8000/v1", "http://<host>:8000/<path>"),+ ("https://bücher.example/v1", "https://xn--bcher-kva.example/<path>"),+ ("http://[fe80::1%25sk-ZONE-CANARY]:8787/", "http://[fe80::1]:8787/"),+ ("https://upstream.example/file:///etc", "https://upstream.example/<path>"),+ ("sk-key-CANARY:file:///x", "<unparseable url>"),+ ("https://xn--bcher-kva.example:8443/v1", "https://xn--bcher-kva.example:8443/<path>"),+ ("http://10.0.0.7:8787", "http://10.0.0.7:8787"),+ ],+)+def test_redact_url_hosts_and_hostless_schemes(+ url: str, expected: str, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)+ assert redact_url(url) == expected++[email protected]("error", [RuntimeError, LookupError, OSError, OverflowError, KeyError])+def test_safe_id_never_runs_the_callers_repr(error: type[Exception]) -> None:+ class RaisingStr(str):+ def __repr__(self) -> str:+ raise error("repr")++ assert safe_id(RaisingStr("model-x")) == "'model-x'"+++def test_broken_exception_attributes_never_break_the_description(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)++ class Broken(OSError):+ @property # type: ignore[override]+ def __cause__(self) -> BaseException:+ raise LookupError("cause")++ @property # type: ignore[override]+ def __traceback__(self) -> None:+ raise RuntimeError("traceback")++ @property # type: ignore[override]+ def errno(self) -> int:+ raise OverflowError("errno")++ error = Broken(2, "x")+ error.__context__ = KeyError("k")++ assert describe_exception(error) == (+ f"{Broken.__qualname__} [Errno 2] {os.strerror(2)}; caused by KeyError"+ )+++def test_hostless_urls_need_the_triple_slash_right_after_the_scheme() -> None:+ assert redact_url("file:sk-CANARY-x://y") == "<unparseable url>"+ assert redact_url("file:///var/x") == "file:///<path>"+++def test_int_ids_respect_the_process_digit_limit(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(sys, "get_int_max_str_digits", lambda: 640)+ big = 10 ** int("700")++ assert safe_id(big) == f"<int of {big.bit_length()} bits>"+ assert safe_id(12345) == "12345"+++def test_safe_id_ignores_a_faked_class() -> None:+ class Pretender:+ @property # type: ignore[misc]+ def __class__(self) -> type:+ return str++ assert safe_id(Pretender()) == f"<{Pretender.__qualname__}>"++[email protected]("name", ["Err\rforged", "sk-live-CANARY", "Err\u2028x"])+def test_class_names_must_be_plain_identifiers(name: str) -> None:+ cls = type(name, (Exception,), {})++ assert describe_exception(cls("x")) == "<exception>"+ assert safe_id(cls("x")) == "<object>"++[email protected]("url", [" \x01file:///var/x", "fi\tle:///var/x", "file:/\n//var/x"])+def test_hostless_check_reads_the_url_the_way_urlsplit_does(url: str) -> None:+ assert redact_url(url) == "file:///<path>"+++def test_int_ids_work_without_a_digit_limit_getter(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delattr(sys, "get_int_max_str_digits")++ assert safe_id(12345) == "12345"diff --git a/tests/test_saved_settings_apply_logging.py b/tests/test_saved_settings_apply_logging.pynew file mode 100644index 000000000..5f8e3facf--- /dev/null+++ b/tests/test_saved_settings_apply_logging.py@@ -0,0 +1,59 @@+"""A failure applying saved settings is reported without the settings' values.++Saved settings can hold credentials, and an error raised while applying them can+quote a value, so the warning names the file and the failure's type and code+location, never its message, at any level.+"""++from __future__ import annotations++import logging++import pytest++CREDENTIAL_CANARY = "sk-ant-SETTINGS-CANARY-4d18"+++def test_settings_apply_failure_warns_without_credential(monkeypatch, tmp_path) -> None:+ from headroom import settings_store+ from headroom.proxy.server import ProxyConfig, create_app++ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path / "ws"))+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)++ def _apply_fails(_values: object) -> None:+ raise ValueError(f"cannot apply ANTHROPIC_API_KEY={CREDENTIAL_CANARY}")++ monkeypatch.setattr(settings_store, "apply_to_environ", _apply_fails)++ records: list[logging.LogRecord] = []++ class _Capture(logging.Handler):+ def emit(self, record: logging.LogRecord) -> None:+ records.append(record)++ proxy_logger = logging.getLogger("headroom.proxy")+ handler = _Capture(level=logging.DEBUG)+ old_level = proxy_logger.level+ proxy_logger.addHandler(handler)+ proxy_logger.setLevel(logging.DEBUG)+ try:+ create_app(ProxyConfig(port=0))+ finally:+ proxy_logger.removeHandler(handler)+ proxy_logger.setLevel(old_level)++ warnings = [r for r in records if "Could not apply saved settings" in r.getMessage()]+ assert len(warnings) == 1+ assert warnings[0].levelno == logging.WARNING+ assert "ValueError" in warnings[0].getMessage()+ assert "settings.json" in warnings[0].getMessage()+ formatter = logging.Formatter()+ for record in records:+ assert CREDENTIAL_CANARY not in formatter.format(record)++[email protected](autouse=True)+def _isolated_home(monkeypatch, tmp_path):+ monkeypatch.setenv("HOME", str(tmp_path / "home"))+ monkeypatch.setenv("CODEX_HOME", str(tmp_path / "codex"))diff --git a/tests/test_savings_pricing_fallback_logging.py b/tests/test_savings_pricing_fallback_logging.pynew file mode 100644index 000000000..c12750f8c--- /dev/null+++ b/tests/test_savings_pricing_fallback_logging.py@@ -0,0 +1,105 @@+"""Savings pricing fallbacks are debuggable without flooding or leaking into the log.++An unpriced model is expected (local models, gateway aliases) and the estimators+run on every request, so it is logged once per model. Unexpected pricing errors+are logged by exception type and code location only: their text can carry a+credential (a provider error quoting an API key) and must not reach any level.+"""++from __future__ import annotations++import logging++import pytest++import headroom.proxy.savings_tracker as st+from headroom.log_safety import WarnOnce++CREDENTIAL_CANARY = "sk-ant-CREDENTIAL-CANARY-91c2"+++class _Capture(logging.Handler):+ def __init__(self) -> None:+ super().__init__(level=logging.DEBUG)+ self.records: list[logging.LogRecord] = []++ def emit(self, record: logging.LogRecord) -> None:+ self.records.append(record)+++class _CredentialEchoingPrice:+ """A price value whose conversion fails with a message quoting a key."""++ def __float__(self) -> float:+ raise ValueError(f"bad price from provider config api_key={CREDENTIAL_CANARY}")+++def captured(monkeypatch: pytest.MonkeyPatch):+ class _FakeLiteLLM:+ model_cost = {"weird-priced-model": {"input_cost_per_token": _CredentialEchoingPrice()}}++ @staticmethod+ def cost_per_token(**_kwargs):+ raise Exception(f"This model isn't mapped yet. key={CREDENTIAL_CANARY}")++ monkeypatch.setattr(st, "_get_litellm_module", lambda: _FakeLiteLLM())+ monkeypatch.setattr(st, "_unpriced_models_logged", WarnOnce(256, "test unpriced models"))+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)+ st._resolve_litellm_model.cache_clear()+ handler = _Capture()+ old_level = st.logger.level+ st.logger.addHandler(handler)+ st.logger.setLevel(logging.DEBUG)+ try:+ yield handler.records+ finally:+ st.logger.removeHandler(handler)+ st.logger.setLevel(old_level)+ st._resolve_litellm_model.cache_clear()+++def _assert_no_canary(records: list[logging.LogRecord]) -> None:+ formatter = logging.Formatter()+ for record in records:+ assert CREDENTIAL_CANARY not in formatter.format(record)+++def test_unpriced_model_logs_once(captured) -> None:+ for _ in range(3):+ assert st._estimate_compression_savings_usd("local-llama-7b", 1000) == pytest.approx(+ 1000 * st.DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN+ )+ st._estimate_output_savings_usd("local-llama-7b", 1000)+ st._estimate_input_cost_usd("local-llama-7b", 1000)++ unpriced = [r for r in captured if "No litellm" in r.getMessage()]+ assert len(unpriced) == 1+ assert "model='local-llama-7b'" in unpriced[0].getMessage()+ assert all(r.exc_info is None for r in captured)+ # The litellm probe error quoted a key; it never reaches the log.+ _assert_no_canary(captured)+++def test_credential_bearing_pricing_error_is_described_not_quoted(captured) -> None:+ assert st._estimate_compression_savings_usd("weird-priced-model", 1000) == pytest.approx(+ 1000 * st.DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN+ )++ failures = [+ r for r in captured if "fallback rate for model='weird-priced-model'" in r.getMessage()+ ]+ assert len(failures) == 1+ assert "ValueError" in failures[0].getMessage()+ assert not any("No litellm" in r.getMessage() for r in captured)+ _assert_no_canary(captured)+++def test_unpriced_model_not_marked_when_debug_disabled(captured) -> None:+ st.logger.setLevel(logging.INFO)+ st._estimate_compression_savings_usd("quiet-model", 10)+ st.logger.setLevel(logging.DEBUG)+ st._estimate_compression_savings_usd("quiet-model", 10)+ assert [+ r for r in captured if "quiet-model" in r.getMessage() and "No litellm" in r.getMessage()+ ]diff --git a/tests/test_tool_schema_savings_policy.py b/tests/test_tool_schema_savings_policy.pyindex a44c9b124..9365e9a2f 100644--- a/tests/test_tool_schema_savings_policy.py+++ b/tests/test_tool_schema_savings_policy.py@@ -140,3 +140,95 @@ def test_tool_schema_compaction_saves_real_tokens_not_just_bytes() -> None:assert tokens_saved == tool_before - tool_afterassert original_tokens - optimized_tokens == tokens_savedassert headline_tokens_saved(tokens_saved, {}) == tokens_saved+++SCHEMA_CANARY = "PRIVATE-SCHEMA-CANARY-7f3a"+++def _capture_policy_logs(monkeypatch):+ import logging++ from headroom.log_safety import WarnOnce+ from headroom.proxy import tool_schema_savings_policy as policy++ records: list[logging.LogRecord] = []++ class _Capture(logging.Handler):+ def emit(self, record: logging.LogRecord) -> None:+ records.append(record)++ handler = _Capture(level=logging.DEBUG)+ policy.logger.addHandler(handler)+ old_level = policy.logger.level+ policy.logger.setLevel(logging.DEBUG)+ monkeypatch.setattr(+ policy, "_reconcile_warned", WarnOnce(32, "test reconcile types"), raising=False+ )+ monkeypatch.delenv("HEADROOM_DEBUG_DUMP", raising=False)+ return policy, records, handler, old_level+++def test_reconcile_failure_keeps_private_schema_out_of_every_log_record(monkeypatch) -> None:+ import logging++ policy, records, handler, old_level = _capture_policy_logs(monkeypatch)++ tool = {"name": "Read", "defer_loading": True, "description": SCHEMA_CANARY}++ def _tokenizer_echoes_schema(tools: object) -> int:+ # A tokenizer error that quotes the schema it choked on.+ raise ValueError(f"cannot tokenize {tools!r}")++ tags = {"tool_search_deferred_tokens": 100}+ booking = (100, {"Read": dict(tool)}, None)+ try:+ policy.reconcile_deferred_tokens(tags, booking, [dict(tool)], _tokenizer_echoes_schema)+ policy.reconcile_deferred_tokens(tags, booking, [dict(tool)], _tokenizer_echoes_schema)+ finally:+ policy.logger.removeHandler(handler)+ policy.logger.setLevel(old_level)++ # The booked credit is left as it was.+ assert tags["tool_search_deferred_tokens"] == 100++ levels = [r.levelno for r in records]+ assert levels.count(logging.WARNING) == 1, "warns once per exception type"+ assert levels.count(logging.DEBUG) == 2, "every failure is recorded at debug"+ formatter = logging.Formatter()+ for record in records:+ rendered = formatter.format(record)+ assert SCHEMA_CANARY not in rendered+ assert "ValueError" in rendered+ assert "100 booked tokens" in records[0].getMessage()+++def test_reconcile_warns_again_for_a_new_exception_type(monkeypatch) -> None:+ import logging++ policy, records, handler, old_level = _capture_policy_logs(monkeypatch)+ tool = {"name": "Read", "defer_loading": True, "description": "x"}+ booking = (100, {"Read": dict(tool)}, None)++ def _runtime(tools: object) -> int:+ raise RuntimeError("boom")++ def _value(tools: object) -> int:+ raise ValueError("boom")++ try:+ # A healthy reconcile logs nothing.+ policy.reconcile_deferred_tokens(+ {"tool_search_deferred_tokens": 100}, booking, [dict(tool)], lambda t: 10+ )+ assert records == []+ for counter in (_runtime, _runtime, _value):+ policy.reconcile_deferred_tokens(+ {"tool_search_deferred_tokens": 100}, booking, [dict(tool)], counter+ )+ finally:+ policy.logger.removeHandler(handler)+ policy.logger.setLevel(old_level)++ warnings = [r.getMessage() for r in records if r.levelno == logging.WARNING]+ assert len(warnings) == 2+ assert "RuntimeError" in warnings[0] and "ValueError" in warnings[1]