Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): let HEADROOM_LOG_LEVEL=debug turn on Headroom's own debug logs (#4137)

Tejas Chopra00093ceddcf4
diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.py
index e3675ca3a..7c66d85ee 100644
--- a/headroom/proxy/helpers.py
+++ b/headroom/proxy/helpers.py
@@ -1882,6 +1882,54 @@ def _warn_once_if_owner_only_unsupported(log_path: Path) -> None:
)
+def _headroom_log_level() -> int:
+ """Level for the ``headroom`` logger and the proxy.log handler.
+
+ ``HEADROOM_LOG_LEVEL=debug`` (or ``trace``) turns on Headroom's own debug
+ lines as well as uvicorn's; anything else keeps the INFO default, so the
+ default (``warning``, which only quiets uvicorn) never hides Headroom's
+ INFO events.
+ """
+ raw = os.environ.get("HEADROOM_LOG_LEVEL", "").strip().lower()
+ return logging.DEBUG if raw in ("debug", "trace") else logging.INFO
+
+
+class _HeadroomDebugStaysInProxyLog(logging.Filter):
+ """Drop ``headroom.*`` DEBUG records on every handler except proxy.log.
+
+ Headroom's debug lines can carry request-derived data (error messages that
+ quote a request; with HEADROOM_DEBUG_DUMP=full, the tool output the router
+ compresses). proxy.log is owner-only; stdout, container logs
+ and wrap's stdio capture are not, so debug stays out of them, exactly as
+ when debug was unreachable.
+ """
+
+ def filter(self, record: logging.LogRecord) -> bool:
+ if record.levelno >= logging.INFO:
+ return True
+ return record.name != "headroom" and not record.name.startswith("headroom.")
+
+
+def _keep_headroom_debug_out_of_other_handlers() -> None:
+ """Filter every handler that can see ``headroom.*`` records except proxy.log.
+
+ That is the root handlers, plus any handler already attached to ``headroom``
+ or a child logger (e.g. an extension's). Records stop at proxy.log only.
+ """
+ loggers: list[logging.Logger] = [logging.getLogger(), logging.getLogger("headroom")]
+ loggers += [
+ candidate
+ for name, candidate in logging.Logger.manager.loggerDict.items()
+ if name.startswith("headroom.") and isinstance(candidate, logging.Logger)
+ ]
+ for each in loggers:
+ for handler in each.handlers:
+ if handler.get_name() == _PROXY_LOG_HANDLER_NAME:
+ continue
+ if not any(isinstance(f, _HeadroomDebugStaysInProxyLog) for f in handler.filters):
+ handler.addFilter(_HeadroomDebugStaysInProxyLog())
+
+
def _setup_file_logging(
port: int | None = None,
*,
@@ -1928,7 +1976,8 @@ def _setup_file_logging(
# Keep root propagation enabled for container stdout/stderr while
# this handler writes the separate port/worker-specific proxy log.
headroom_logger = logging.getLogger("headroom")
- headroom_logger.setLevel(logging.INFO)
+ level = _headroom_log_level()
+ headroom_logger.setLevel(level)
# Decide BEFORE constructing the handler: constructing a
# RotatingFileHandler opens (creates) the file, so building one only to
# discard it would leave an empty stray worker log and leak
@@ -1940,7 +1989,12 @@ def _setup_file_logging(
for h in headroom_logger.handlers
if isinstance(h, RotatingFileHandler) and h.name == _PROXY_LOG_HANDLER_NAME
]
- if any(Path(h.baseFilename) == log_path for h in existing):
+ if level == logging.DEBUG:
+ _keep_headroom_debug_out_of_other_handlers()
+ reused = [h for h in existing if Path(h.baseFilename) == log_path]
+ if reused:
+ for h in reused:
+ h.setLevel(level)
return
handler = handler_cls(
log_path,
@@ -1949,7 +2003,7 @@ def _setup_file_logging(
encoding="utf-8",
)
handler.set_name(_PROXY_LOG_HANDLER_NAME)
- handler.setLevel(logging.INFO)
+ handler.setLevel(level)
handler.setFormatter(
logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
)
@@ -1957,6 +2011,15 @@ def _setup_file_logging(
headroom_logger.removeHandler(stale)
stale.close()
headroom_logger.addHandler(handler)
+ if level == logging.DEBUG:
+ logger.warning(
+ "Headroom debug logging is on (HEADROOM_LOG_LEVEL). %s now records "
+ "request-derived data such as error messages that can quote requests; the "
+ "file is owner-only and debug lines stay out of stdout. Full tool-output "
+ "dumps also need HEADROOM_DEBUG_DUMP=full. Unset HEADROOM_LOG_LEVEL when "
+ "you finish diagnosing.",
+ log_path,
+ )
except OSError as exc:
# Non-fatal: can't write logs (read-only fs, permissions, etc.). Say so,
# or the missing proxy.log is the only clue. The OSError names the path.
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 4e0de0fc0..0ff77f423 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -63,6 +63,7 @@ from ..config import (
is_tool_excluded,
unwrap_tool_call,
)
+from ..log_safety import content_logging_enabled
from ..offline import OfflineEgressBlocked
from ..parser import CCR_RETRIEVAL_MARKER_RE
from ..tokenizer import Tokenizer
@@ -953,8 +954,14 @@ def _bash_command_is_search(command: str, search_commands: frozenset[str]) -> bo
return prog in search_commands
+def _router_debug_enabled() -> bool:
+ """Router debug events carry the tool output being routed, so DEBUG alone is
+ not enough: they also need the explicit content opt-in."""
+ return logger.isEnabledFor(logging.DEBUG) and content_logging_enabled()
+
+
def _log_router_debug(event: str, **payload: Any) -> None:
- if not logger.isEnabledFor(logging.DEBUG):
+ if not _router_debug_enabled():
return
payload = {"event": event, **payload}
logger.debug("event=%s %s", event, _router_debug_dumps(payload))
@@ -2886,7 +2893,7 @@ class ContentRouter(Transform):
RouterCompressionResult with compressed content and routing metadata.
"""
context = context or ""
- debug_enabled = logger.isEnabledFor(logging.DEBUG)
+ debug_enabled = _router_debug_enabled()
request_debug = (
{
"chars": len(content),
@@ -3184,7 +3191,7 @@ class ContentRouter(Transform):
sections_source,
isolate=tuple(placeholder for placeholder, _ in protected),
)
- if logger.isEnabledFor(logging.DEBUG):
+ if _router_debug_enabled():
_log_router_debug(
"content_router_mixed_sections",
section_count=len(sections),
@@ -4406,7 +4413,7 @@ class ContentRouter(Transform):
compressed, compressed_tokens = dense
strategy_chain.append("dense_elide")
decision_reason = f"{decision_reason}_dense_elide"
- if logger.isEnabledFor(logging.DEBUG):
+ if _router_debug_enabled():
_log_router_debug(
"content_router_strategy_result",
requested_strategy=requested_strategy.value,
@@ -4466,7 +4473,7 @@ class ContentRouter(Transform):
)
return elided, elided_tokens, strategy_chain
strategy_chain.append(CompressionStrategy.PASSTHROUGH.value)
- if logger.isEnabledFor(logging.DEBUG):
+ if _router_debug_enabled():
_log_router_debug(
"content_router_strategy_result",
requested_strategy=requested_strategy.value,
diff --git a/tests/test_perf_per_port_logs.py b/tests/test_perf_per_port_logs.py
index 4447bd999..fd8c449f3 100644
--- a/tests/test_perf_per_port_logs.py
+++ b/tests/test_perf_per_port_logs.py
@@ -253,3 +253,151 @@ def test_parse_log_files_aggregates_per_port_and_legacy(workspace: Path) -> None
# Non-PERF files never ingested.
assert models.isdisjoint({"model-STDIO", "model-STDIO2", "model-ERR"})
assert rids.isdisjoint({"hr_stdio", "hr_stdio2", "hr_err"})
+
+
+def headroom_log_state(request: pytest.FixtureRequest):
+ """Restore the headroom logger level and drop the debug filter from root handlers."""
+ from headroom.proxy import helpers
+
+ headroom_logger = logging.getLogger("headroom")
+ level = headroom_logger.level
+
+ def restore() -> None:
+ headroom_logger.setLevel(level)
+ loggers = [logging.getLogger(), headroom_logger] + [
+ lg
+ for name, lg in logging.Logger.manager.loggerDict.items()
+ if name.startswith("headroom.") and isinstance(lg, logging.Logger)
+ ]
+ for handler in [h for lg in loggers for h in lg.handlers]:
+ for f in list(handler.filters):
+ if isinstance(f, helpers._HeadroomDebugStaysInProxyLog):
+ handler.removeFilter(f)
+
+ request.addfinalizer(restore)
+ return headroom_logger
+
+
+def _flush(headroom_logger: logging.Logger) -> None:
+ for handler in headroom_logger.handlers:
+ handler.flush()
+
+
+ ("env", "debug_written"),
+ [("debug", True), ("TRACE", True), ("warning", False), ("info", False), (None, False)],
+)
+def test_headroom_log_level_debug_reaches_proxy_log(
+ workspace: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ headroom_log_state: logging.Logger,
+ env: str | None,
+ debug_written: bool,
+) -> None:
+ """``HEADROOM_LOG_LEVEL=debug`` must surface Headroom's own debug lines, not only uvicorn's."""
+ if env is None:
+ monkeypatch.delenv("HEADROOM_LOG_LEVEL", raising=False)
+ else:
+ monkeypatch.setenv("HEADROOM_LOG_LEVEL", env)
+
+ _setup_file_logging(port=9911)
+ probe = logging.getLogger("headroom.proxy.log_level_probe")
+ probe.debug("debug-probe")
+ probe.info("info-probe")
+ _flush(headroom_log_state)
+
+ text = _paths.proxy_log_path(9911).read_text()
+ assert "info-probe" in text
+ assert ("debug-probe" in text) is debug_written
+ assert ("Headroom debug logging is on" in text) is debug_written
+
+
+def test_headroom_debug_stays_out_of_stdout(
+ workspace: Path, monkeypatch: pytest.MonkeyPatch, headroom_log_state: logging.Logger
+) -> None:
+ """Debug lines can carry tool output: only the owner-only proxy.log may receive them."""
+ import io
+
+ stdout = io.StringIO()
+ root_handler = logging.StreamHandler(stdout)
+ logging.getLogger().addHandler(root_handler)
+ try:
+ monkeypatch.setenv("HEADROOM_LOG_LEVEL", "debug")
+ _setup_file_logging(port=9912)
+ probe = logging.getLogger("headroom.transforms.log_level_probe")
+ probe.debug("secret-tool-output")
+ probe.info("info-probe")
+ logging.getLogger("thirdparty.log_level_probe").warning("thirdparty-warning")
+ _flush(headroom_log_state)
+ finally:
+ logging.getLogger().removeHandler(root_handler)
+
+ assert "secret-tool-output" in _paths.proxy_log_path(9912).read_text()
+ assert "secret-tool-output" not in stdout.getvalue()
+ assert "info-probe" in stdout.getvalue()
+ assert "thirdparty-warning" in stdout.getvalue()
+
+
+def test_reused_proxy_log_picks_up_debug(
+ workspace: Path, monkeypatch: pytest.MonkeyPatch, headroom_log_state: logging.Logger
+) -> None:
+ """A second same-port setup after switching to debug must not keep the old INFO handler."""
+ monkeypatch.setenv("HEADROOM_LOG_LEVEL", "info")
+ _setup_file_logging(port=9913)
+ monkeypatch.setenv("HEADROOM_LOG_LEVEL", "debug")
+ _setup_file_logging(port=9913)
+
+ logging.getLogger("headroom.proxy.log_level_probe").debug("debug-after-reuse")
+ _flush(headroom_log_state)
+
+ assert "debug-after-reuse" in _paths.proxy_log_path(9913).read_text()
+
+
+def test_headroom_debug_stays_out_of_child_logger_handlers(
+ workspace: Path, monkeypatch: pytest.MonkeyPatch, headroom_log_state: logging.Logger
+) -> None:
+ """A handler on a headroom.* logger (e.g. an extension's) must not receive debug lines either."""
+ import io
+
+ stream = io.StringIO()
+ child = logging.getLogger("headroom.extension_probe")
+ child_handler = logging.StreamHandler(stream)
+ child.addHandler(child_handler)
+ try:
+ monkeypatch.setenv("HEADROOM_LOG_LEVEL", "debug")
+ _setup_file_logging(port=9914)
+ child.debug("secret-tool-output")
+ child.info("info-probe")
+ _flush(headroom_log_state)
+ finally:
+ child.removeHandler(child_handler)
+
+ assert "secret-tool-output" in _paths.proxy_log_path(9914).read_text()
+ assert "secret-tool-output" not in stream.getvalue()
+ assert "info-probe" in stream.getvalue()
+
+
[email protected](("dump", "logged"), [("", False), ("full", True)])
+def test_router_content_dump_needs_the_content_opt_in(
+ monkeypatch: pytest.MonkeyPatch, headroom_log_state: logging.Logger, dump: str, logged: bool
+) -> None:
+ """HEADROOM_LOG_LEVEL=debug alone must not write the tool output the router compresses."""
+ import io
+
+ from headroom.transforms import content_router
+
+ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", dump)
+ stream = io.StringIO()
+ handler = logging.StreamHandler(stream)
+ router_logger = logging.getLogger(content_router.__name__)
+ previous = router_logger.level
+ router_logger.addHandler(handler)
+ router_logger.setLevel(logging.DEBUG)
+ try:
+ content_router._log_router_debug("routing_decision", content="secret-tool-output")
+ finally:
+ router_logger.removeHandler(handler)
+ router_logger.setLevel(previous)
+
+ assert ("secret-tool-output" in stream.getvalue()) is logged