Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): let HEADROOM_LOG_LEVEL=debug turn on Headroom's own debug logs (#4137)
Tejas Chopra00093ceddcf4
diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.pyindex e3675ca3a..7c66d85ee 100644--- a/headroom/proxy/helpers.py+++ b/headroom/proxy/helpers.py@@ -1882,6 +1882,54 @@ def _warn_once_if_owner_only_unsupported(log_path: Path) -> None:)+def _headroom_log_level() -> int:+ """Level for the ``headroom`` logger and the proxy.log handler.++ ``HEADROOM_LOG_LEVEL=debug`` (or ``trace``) turns on Headroom's own debug+ lines as well as uvicorn's; anything else keeps the INFO default, so the+ default (``warning``, which only quiets uvicorn) never hides Headroom's+ INFO events.+ """+ raw = os.environ.get("HEADROOM_LOG_LEVEL", "").strip().lower()+ return logging.DEBUG if raw in ("debug", "trace") else logging.INFO+++class _HeadroomDebugStaysInProxyLog(logging.Filter):+ """Drop ``headroom.*`` DEBUG records on every handler except proxy.log.++ Headroom's debug lines can carry request-derived data (error messages that+ quote a request; with HEADROOM_DEBUG_DUMP=full, the tool output the router+ compresses). proxy.log is owner-only; stdout, container logs+ and wrap's stdio capture are not, so debug stays out of them, exactly as+ when debug was unreachable.+ """++ def filter(self, record: logging.LogRecord) -> bool:+ if record.levelno >= logging.INFO:+ return True+ return record.name != "headroom" and not record.name.startswith("headroom.")+++def _keep_headroom_debug_out_of_other_handlers() -> None:+ """Filter every handler that can see ``headroom.*`` records except proxy.log.++ That is the root handlers, plus any handler already attached to ``headroom``+ or a child logger (e.g. an extension's). Records stop at proxy.log only.+ """+ loggers: list[logging.Logger] = [logging.getLogger(), logging.getLogger("headroom")]+ loggers += [+ candidate+ for name, candidate in logging.Logger.manager.loggerDict.items()+ if name.startswith("headroom.") and isinstance(candidate, logging.Logger)+ ]+ for each in loggers:+ for handler in each.handlers:+ if handler.get_name() == _PROXY_LOG_HANDLER_NAME:+ continue+ if not any(isinstance(f, _HeadroomDebugStaysInProxyLog) for f in handler.filters):+ handler.addFilter(_HeadroomDebugStaysInProxyLog())++def _setup_file_logging(port: int | None = None,*,@@ -1928,7 +1976,8 @@ def _setup_file_logging(# Keep root propagation enabled for container stdout/stderr while# this handler writes the separate port/worker-specific proxy log.headroom_logger = logging.getLogger("headroom")- headroom_logger.setLevel(logging.INFO)+ level = _headroom_log_level()+ headroom_logger.setLevel(level)# Decide BEFORE constructing the handler: constructing a# RotatingFileHandler opens (creates) the file, so building one only to# discard it would leave an empty stray worker log and leak@@ -1940,7 +1989,12 @@ def _setup_file_logging(for h in headroom_logger.handlersif isinstance(h, RotatingFileHandler) and h.name == _PROXY_LOG_HANDLER_NAME]- if any(Path(h.baseFilename) == log_path for h in existing):+ if level == logging.DEBUG:+ _keep_headroom_debug_out_of_other_handlers()+ reused = [h for h in existing if Path(h.baseFilename) == log_path]+ if reused:+ for h in reused:+ h.setLevel(level)returnhandler = handler_cls(log_path,@@ -1949,7 +2003,7 @@ def _setup_file_logging(encoding="utf-8",)handler.set_name(_PROXY_LOG_HANDLER_NAME)- handler.setLevel(logging.INFO)+ handler.setLevel(level)handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))@@ -1957,6 +2011,15 @@ def _setup_file_logging(headroom_logger.removeHandler(stale)stale.close()headroom_logger.addHandler(handler)+ if level == logging.DEBUG:+ logger.warning(+ "Headroom debug logging is on (HEADROOM_LOG_LEVEL). %s now records "+ "request-derived data such as error messages that can quote requests; the "+ "file is owner-only and debug lines stay out of stdout. Full tool-output "+ "dumps also need HEADROOM_DEBUG_DUMP=full. Unset HEADROOM_LOG_LEVEL when "+ "you finish diagnosing.",+ log_path,+ )except OSError as exc:# Non-fatal: can't write logs (read-only fs, permissions, etc.). Say so,# or the missing proxy.log is the only clue. The OSError names the path.diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 4e0de0fc0..0ff77f423 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -63,6 +63,7 @@ from ..config import (is_tool_excluded,unwrap_tool_call,)+from ..log_safety import content_logging_enabledfrom ..offline import OfflineEgressBlockedfrom ..parser import CCR_RETRIEVAL_MARKER_REfrom ..tokenizer import Tokenizer@@ -953,8 +954,14 @@ def _bash_command_is_search(command: str, search_commands: frozenset[str]) -> boreturn prog in search_commands+def _router_debug_enabled() -> bool:+ """Router debug events carry the tool output being routed, so DEBUG alone is+ not enough: they also need the explicit content opt-in."""+ return logger.isEnabledFor(logging.DEBUG) and content_logging_enabled()++def _log_router_debug(event: str, **payload: Any) -> None:- if not logger.isEnabledFor(logging.DEBUG):+ if not _router_debug_enabled():returnpayload = {"event": event, **payload}logger.debug("event=%s %s", event, _router_debug_dumps(payload))@@ -2886,7 +2893,7 @@ class ContentRouter(Transform):RouterCompressionResult with compressed content and routing metadata."""context = context or ""- debug_enabled = logger.isEnabledFor(logging.DEBUG)+ debug_enabled = _router_debug_enabled()request_debug = ({"chars": len(content),@@ -3184,7 +3191,7 @@ class ContentRouter(Transform):sections_source,isolate=tuple(placeholder for placeholder, _ in protected),)- if logger.isEnabledFor(logging.DEBUG):+ if _router_debug_enabled():_log_router_debug("content_router_mixed_sections",section_count=len(sections),@@ -4406,7 +4413,7 @@ class ContentRouter(Transform):compressed, compressed_tokens = densestrategy_chain.append("dense_elide")decision_reason = f"{decision_reason}_dense_elide"- if logger.isEnabledFor(logging.DEBUG):+ if _router_debug_enabled():_log_router_debug("content_router_strategy_result",requested_strategy=requested_strategy.value,@@ -4466,7 +4473,7 @@ class ContentRouter(Transform):)return elided, elided_tokens, strategy_chainstrategy_chain.append(CompressionStrategy.PASSTHROUGH.value)- if logger.isEnabledFor(logging.DEBUG):+ if _router_debug_enabled():_log_router_debug("content_router_strategy_result",requested_strategy=requested_strategy.value,diff --git a/tests/test_perf_per_port_logs.py b/tests/test_perf_per_port_logs.pyindex 4447bd999..fd8c449f3 100644--- a/tests/test_perf_per_port_logs.py+++ b/tests/test_perf_per_port_logs.py@@ -253,3 +253,151 @@ def test_parse_log_files_aggregates_per_port_and_legacy(workspace: Path) -> None# Non-PERF files never ingested.assert models.isdisjoint({"model-STDIO", "model-STDIO2", "model-ERR"})assert rids.isdisjoint({"hr_stdio", "hr_stdio2", "hr_err"})+++def headroom_log_state(request: pytest.FixtureRequest):+ """Restore the headroom logger level and drop the debug filter from root handlers."""+ from headroom.proxy import helpers++ headroom_logger = logging.getLogger("headroom")+ level = headroom_logger.level++ def restore() -> None:+ headroom_logger.setLevel(level)+ loggers = [logging.getLogger(), headroom_logger] + [+ lg+ for name, lg in logging.Logger.manager.loggerDict.items()+ if name.startswith("headroom.") and isinstance(lg, logging.Logger)+ ]+ for handler in [h for lg in loggers for h in lg.handlers]:+ for f in list(handler.filters):+ if isinstance(f, helpers._HeadroomDebugStaysInProxyLog):+ handler.removeFilter(f)++ request.addfinalizer(restore)+ return headroom_logger+++def _flush(headroom_logger: logging.Logger) -> None:+ for handler in headroom_logger.handlers:+ handler.flush()+++ ("env", "debug_written"),+ [("debug", True), ("TRACE", True), ("warning", False), ("info", False), (None, False)],+)+def test_headroom_log_level_debug_reaches_proxy_log(+ workspace: Path,+ monkeypatch: pytest.MonkeyPatch,+ headroom_log_state: logging.Logger,+ env: str | None,+ debug_written: bool,+) -> None:+ """``HEADROOM_LOG_LEVEL=debug`` must surface Headroom's own debug lines, not only uvicorn's."""+ if env is None:+ monkeypatch.delenv("HEADROOM_LOG_LEVEL", raising=False)+ else:+ monkeypatch.setenv("HEADROOM_LOG_LEVEL", env)++ _setup_file_logging(port=9911)+ probe = logging.getLogger("headroom.proxy.log_level_probe")+ probe.debug("debug-probe")+ probe.info("info-probe")+ _flush(headroom_log_state)++ text = _paths.proxy_log_path(9911).read_text()+ assert "info-probe" in text+ assert ("debug-probe" in text) is debug_written+ assert ("Headroom debug logging is on" in text) is debug_written+++def test_headroom_debug_stays_out_of_stdout(+ workspace: Path, monkeypatch: pytest.MonkeyPatch, headroom_log_state: logging.Logger+) -> None:+ """Debug lines can carry tool output: only the owner-only proxy.log may receive them."""+ import io++ stdout = io.StringIO()+ root_handler = logging.StreamHandler(stdout)+ logging.getLogger().addHandler(root_handler)+ try:+ monkeypatch.setenv("HEADROOM_LOG_LEVEL", "debug")+ _setup_file_logging(port=9912)+ probe = logging.getLogger("headroom.transforms.log_level_probe")+ probe.debug("secret-tool-output")+ probe.info("info-probe")+ logging.getLogger("thirdparty.log_level_probe").warning("thirdparty-warning")+ _flush(headroom_log_state)+ finally:+ logging.getLogger().removeHandler(root_handler)++ assert "secret-tool-output" in _paths.proxy_log_path(9912).read_text()+ assert "secret-tool-output" not in stdout.getvalue()+ assert "info-probe" in stdout.getvalue()+ assert "thirdparty-warning" in stdout.getvalue()+++def test_reused_proxy_log_picks_up_debug(+ workspace: Path, monkeypatch: pytest.MonkeyPatch, headroom_log_state: logging.Logger+) -> None:+ """A second same-port setup after switching to debug must not keep the old INFO handler."""+ monkeypatch.setenv("HEADROOM_LOG_LEVEL", "info")+ _setup_file_logging(port=9913)+ monkeypatch.setenv("HEADROOM_LOG_LEVEL", "debug")+ _setup_file_logging(port=9913)++ logging.getLogger("headroom.proxy.log_level_probe").debug("debug-after-reuse")+ _flush(headroom_log_state)++ assert "debug-after-reuse" in _paths.proxy_log_path(9913).read_text()+++def test_headroom_debug_stays_out_of_child_logger_handlers(+ workspace: Path, monkeypatch: pytest.MonkeyPatch, headroom_log_state: logging.Logger+) -> None:+ """A handler on a headroom.* logger (e.g. an extension's) must not receive debug lines either."""+ import io++ stream = io.StringIO()+ child = logging.getLogger("headroom.extension_probe")+ child_handler = logging.StreamHandler(stream)+ child.addHandler(child_handler)+ try:+ monkeypatch.setenv("HEADROOM_LOG_LEVEL", "debug")+ _setup_file_logging(port=9914)+ child.debug("secret-tool-output")+ child.info("info-probe")+ _flush(headroom_log_state)+ finally:+ child.removeHandler(child_handler)++ assert "secret-tool-output" in _paths.proxy_log_path(9914).read_text()+ assert "secret-tool-output" not in stream.getvalue()+ assert "info-probe" in stream.getvalue()++[email protected](("dump", "logged"), [("", False), ("full", True)])+def test_router_content_dump_needs_the_content_opt_in(+ monkeypatch: pytest.MonkeyPatch, headroom_log_state: logging.Logger, dump: str, logged: bool+) -> None:+ """HEADROOM_LOG_LEVEL=debug alone must not write the tool output the router compresses."""+ import io++ from headroom.transforms import content_router++ monkeypatch.setenv("HEADROOM_DEBUG_DUMP", dump)+ stream = io.StringIO()+ handler = logging.StreamHandler(stream)+ router_logger = logging.getLogger(content_router.__name__)+ previous = router_logger.level+ router_logger.addHandler(handler)+ router_logger.setLevel(logging.DEBUG)+ try:+ content_router._log_router_debug("routing_decision", content="secret-tool-output")+ finally:+ router_logger.removeHandler(handler)+ router_logger.setLevel(previous)++ assert ("secret-tool-output" in stream.getvalue()) is logged