Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(proxy): add loopback-only debug introspection endpoints

Adryan Eka Vandraa4010aaa1b6f
diff --git a/headroom/proxy/debug_introspection.py b/headroom/proxy/debug_introspection.py
new file mode 100644
index 000000000..2ad60a883
--- /dev/null
+++ b/headroom/proxy/debug_introspection.py
@@ -0,0 +1,180 @@
+"""Pure serializers for the loopback-only /debug/* introspection endpoints.
+
+Unit 5 of the Codex-proxy resilience plan. These helpers transform live
+runtime state (asyncio tasks, WS session registry, warmup registry) into
+JSON-serializable dicts that ``/debug/tasks``, ``/debug/ws-sessions`` and
+``/debug/warmup`` return.
+
+Design constraints
+------------------
+* **No state mutation.** Every helper is a pure read of the current
+ state. Calling any of them N times in a row must not change registry
+ contents or task counts.
+* **No blocking I/O.** The helpers only call ``asyncio.all_tasks()``,
+ ``task.get_name()`` and attribute reads on already-materialized
+ registry state.
+* **No privacy leaks.** Task serialization deliberately excludes
+ ``cr_frame.f_locals``, coro ``locals()``, request bodies and anything
+ that could accidentally carry user data. Only the task *name* and the
+ coroutine's *qualname* (the static code symbol) are exposed.
+
+Age tracking for generic tasks
+------------------------------
+:mod:`asyncio.Task` does not record a creation time natively. For tasks
+that belong to a tracked WS session, the WS registry holds a
+``started_at`` we can correlate by task name (the handler names relay
+tasks ``codex-ws-c2u-<sid>`` / ``codex-ws-u2c-<sid>``). For every other
+task we report ``age_seconds=None`` rather than faking it — the plan
+explicitly prefers this minimal approach over invasive wrapper
+instrumentation.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from headroom.proxy.warmup import WarmupRegistry
+ from headroom.proxy.ws_session_registry import WebSocketSessionRegistry
+
+__all__ = [
+ "collect_tasks",
+ "collect_ws_sessions",
+ "collect_warmup",
+]
+
+
+# Task-name prefixes emitted by the Codex WS handler for its relay tasks.
+# See ``headroom/proxy/handlers/openai.py`` ``handle_openai_responses_ws``.
+_CODEX_WS_RELAY_PREFIXES: tuple[str, ...] = (
+ "codex-ws-c2u-",
+ "codex-ws-u2c-",
+)
+
+
+def _coro_qualname(task: asyncio.Task[Any]) -> str | None:
+ """Return the coroutine qualname for ``task`` without touching locals.
+
+ The qualname is a static code symbol (``Class.method``) and never
+ carries request data. We intentionally do not touch ``cr_frame`` or
+ any mutable coroutine state beyond ``cr_code.co_qualname``.
+ """
+ try:
+ coro = task.get_coro()
+ except Exception:
+ return None
+ if coro is None:
+ return None
+ code = getattr(coro, "cr_code", None)
+ if code is None:
+ return None
+ qualname = getattr(code, "co_qualname", None)
+ if qualname is None:
+ qualname = getattr(code, "co_name", None)
+ return qualname if isinstance(qualname, str) else None
+
+
+def _stack_depth(task: asyncio.Task[Any]) -> int | None:
+ """Return a short stack-depth summary for ``task``.
+
+ Uses :meth:`asyncio.Task.get_stack` which returns frame objects only
+ up to a bounded limit; we report the *count*, never the frame
+ contents. Frame locals are never inspected.
+ """
+ try:
+ stack = task.get_stack(limit=32)
+ except Exception:
+ return None
+ return len(stack)
+
+
+def _age_for_named_task(
+ task_name: str,
+ ws_registry: WebSocketSessionRegistry | None,
+) -> float | None:
+ """Resolve an age for tasks named by the WS handler.
+
+ Codex relay task names embed the session id after a known prefix.
+ When the WS registry is present and holds that session, we can
+ derive an age from the session's ``started_at``. For everything
+ else we return ``None`` — the plan prefers a truthful ``null`` to
+ an invented age.
+ """
+ if ws_registry is None:
+ return None
+ for prefix in _CODEX_WS_RELAY_PREFIXES:
+ if task_name.startswith(prefix):
+ session_id = task_name[len(prefix):]
+ handle = ws_registry.get(session_id)
+ if handle is None:
+ return None
+ return max(0.0, time.perf_counter() - handle.started_at)
+ return None
+
+
+def collect_tasks(
+ ws_registry: WebSocketSessionRegistry | None = None,
+) -> list[dict[str, Any]]:
+ """Enumerate ``asyncio.all_tasks()`` for /debug/tasks.
+
+ Each entry carries: ``name``, ``coro_qualname``, ``age_seconds``
+ (``None`` unless the task is a tracked WS relay), ``stack_depth``,
+ and ``done``. Sorted by age descending with ``None`` ages sorted
+ after known ages. System noise (``None`` tasks, tasks with no
+ coroutine) is filtered out.
+ """
+ try:
+ tasks = asyncio.all_tasks()
+ except RuntimeError:
+ # No running loop — e.g. called outside of an event loop.
+ return []
+
+ entries: list[dict[str, Any]] = []
+ for task in tasks:
+ if task is None:
+ continue
+ try:
+ name = task.get_name()
+ except Exception:
+ name = None
+ qualname = _coro_qualname(task)
+ if qualname is None and name is None:
+ # No stable identity — skip rather than emit a blank row.
+ continue
+ age = _age_for_named_task(name or "", ws_registry)
+ entry: dict[str, Any] = {
+ "name": name,
+ "coro_qualname": qualname,
+ "age_seconds": age,
+ "stack_depth": _stack_depth(task),
+ "done": bool(task.done()),
+ }
+ entries.append(entry)
+
+ # Sort by age descending; None ages sort last (treat as -inf for desc).
+ def _sort_key(entry: dict[str, Any]) -> tuple[int, float]:
+ age = entry.get("age_seconds")
+ if age is None:
+ return (1, 0.0)
+ return (0, -float(age))
+
+ entries.sort(key=_sort_key)
+ return entries
+
+
+def collect_ws_sessions(
+ registry: WebSocketSessionRegistry | None,
+) -> list[dict[str, Any]]:
+ """Return the registry snapshot, or an empty list if no registry."""
+ if registry is None:
+ return []
+ return registry.snapshot()
+
+
+def collect_warmup(registry: WarmupRegistry | None) -> dict[str, Any]:
+ """Return the warmup registry dict, or an empty dict if none."""
+ if registry is None:
+ return {}
+ return registry.to_dict()
diff --git a/headroom/proxy/loopback_guard.py b/headroom/proxy/loopback_guard.py
new file mode 100644
index 000000000..4341fc729
--- /dev/null
+++ b/headroom/proxy/loopback_guard.py
@@ -0,0 +1,71 @@
+"""Loopback-only access guard for /debug/* endpoints.
+
+Unit 5 of the Codex-proxy resilience plan. A FastAPI dependency that
+raises :class:`fastapi.HTTPException` with status 404 — *not* 403 — for
+any request whose client address is not the loopback interface. 404 is
+deliberate: debug endpoints should be invisible to external scanners,
+not merely forbidden.
+
+The guard is a ``Depends(...)``-friendly function (rather than a
+middleware) because:
+
+* FastAPI's dependency injection makes the guard explicit on each
+ route, so ``ruff``/reviewers can see which endpoints are guarded.
+* ``TestClient`` lets us override a dependency with
+ ``app.dependency_overrides``, which is the cleanest way to simulate
+ a non-loopback client in tests.
+* The set of debug endpoints is small and co-located; a middleware
+ would be disproportionate.
+"""
+
+try:
+ from fastapi import HTTPException, Request
+except ImportError: # pragma: no cover - fastapi is a hard dep in practice
+ HTTPException = None # type: ignore[assignment,misc]
+ Request = None # type: ignore[assignment,misc]
+
+
+__all__ = [
+ "LOOPBACK_HOSTS",
+ "is_loopback_host",
+ "require_loopback",
+]
+
+
+# Accepted loopback hostnames. ``None`` covers ``TestClient`` which does
+# not populate ``request.client`` by default; we treat that as loopback
+# because it means the call did not originate from a real socket.
+LOOPBACK_HOSTS: frozenset[str] = frozenset({"127.0.0.1", "::1", "localhost"})
+
+
+def is_loopback_host(host: str | None) -> bool:
+ """Return True if ``host`` represents a loopback interface.
+
+ ``None`` is treated as loopback — this covers ``TestClient``
+ requests where FastAPI does not populate ``request.client``.
+ """
+ if host is None:
+ return True
+ return host in LOOPBACK_HOSTS
+
+
+def require_loopback(request: Request) -> None: # type: ignore[valid-type]
+ """FastAPI dependency: 404 any non-loopback caller.
+
+ Usage::
+
+ @app.get("/debug/tasks", dependencies=[Depends(require_loopback)])
+ async def debug_tasks() -> list[dict]:
+ ...
+
+ Returning 404 (not 403) keeps debug endpoints invisible to
+ external scanners — indistinguishable from "no such route".
+ """
+ if HTTPException is None: # pragma: no cover - defensive
+ raise RuntimeError("FastAPI is required for the loopback guard")
+
+ client = getattr(request, "client", None)
+ host = getattr(client, "host", None) if client is not None else None
+ if not is_loopback_host(host):
+ # No body: minimal FastAPI default, behaves like "no route".
+ raise HTTPException(status_code=404)
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 058650ba7..ef6507223 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -45,7 +45,7 @@ import httpx
try:
import uvicorn
- from fastapi import FastAPI, HTTPException, Request, Response, WebSocket
+ from fastapi import Depends, FastAPI, HTTPException, Request, Response, WebSocket
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse
@@ -1310,6 +1310,34 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
payload = _health_payload(include_config=True)
return JSONResponse(status_code=200, content=payload)
+ # Loopback-only debug introspection (Unit 5). A remote IP gets 404 —
+ # debug endpoints are invisible to external scanners.
+ from headroom.proxy.debug_introspection import (
+ collect_tasks as _collect_tasks,
+ )
+ from headroom.proxy.debug_introspection import (
+ collect_warmup as _collect_warmup,
+ )
+ from headroom.proxy.debug_introspection import (
+ collect_ws_sessions as _collect_ws_sessions,
+ )
+ from headroom.proxy.loopback_guard import require_loopback as _require_loopback
+
+ @app.get("/debug/tasks", dependencies=[Depends(_require_loopback)])
+ async def debug_tasks():
+ ws_registry = getattr(proxy, "ws_sessions", None)
+ return JSONResponse(status_code=200, content=_collect_tasks(ws_registry))
+
+ @app.get("/debug/ws-sessions", dependencies=[Depends(_require_loopback)])
+ async def debug_ws_sessions():
+ ws_registry = getattr(proxy, "ws_sessions", None)
+ return JSONResponse(status_code=200, content=_collect_ws_sessions(ws_registry))
+
+ @app.get("/debug/warmup", dependencies=[Depends(_require_loopback)])
+ async def debug_warmup():
+ warmup_registry = getattr(proxy, "warmup", None)
+ return JSONResponse(status_code=200, content=_collect_warmup(warmup_registry))
+
@app.get("/dashboard", response_class=HTMLResponse)
async def dashboard():
"""Serve the Headroom dashboard UI."""
diff --git a/tests/test_proxy_debug_endpoints.py b/tests/test_proxy_debug_endpoints.py
new file mode 100644
index 000000000..f7b1afb18
--- /dev/null
+++ b/tests/test_proxy_debug_endpoints.py
@@ -0,0 +1,342 @@
+"""Tests for the loopback-only /debug/* introspection endpoints (Unit 5)."""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+
+pytest.importorskip("fastapi")
+pytest.importorskip("httpx")
+
+from fastapi import HTTPException
+from fastapi.testclient import TestClient
+
+from headroom.proxy.debug_introspection import (
+ collect_tasks,
+ collect_warmup,
+ collect_ws_sessions,
+)
+from headroom.proxy.loopback_guard import (
+ LOOPBACK_HOSTS,
+ is_loopback_host,
+ require_loopback,
+)
+from headroom.proxy.server import ProxyConfig, create_app
+from headroom.proxy.warmup import WarmupRegistry
+from headroom.proxy.ws_session_registry import (
+ WebSocketSessionRegistry,
+ WSSessionHandle,
+)
+
+# ---------------------------------------------------------------------------
+# Shared fixtures
+# ---------------------------------------------------------------------------
+
+
+def client():
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ )
+ app = create_app(config)
+ # Pin the simulated client address to loopback so the /debug/* guard
+ # accepts the request. Without this, FastAPI's TestClient reports
+ # the host as ``testclient`` and the guard correctly 404s us.
+ with TestClient(app, client=("127.0.0.1", 12345)) as test_client:
+ yield test_client
+
+
+def app_and_client():
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ )
+ app = create_app(config)
+ with TestClient(app, client=("127.0.0.1", 12345)) as test_client:
+ yield app, test_client
+
+
+def app_and_external_client():
+ """TestClient that reports a non-loopback address (to exercise 404)."""
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ )
+ app = create_app(config)
+ with TestClient(app, client=("10.0.0.1", 54321)) as test_client:
+ yield app, test_client
+
+
+# ---------------------------------------------------------------------------
+# Loopback guard unit tests
+# ---------------------------------------------------------------------------
+
+
+def test_is_loopback_host_accepts_canonical_hosts():
+ for host in LOOPBACK_HOSTS:
+ assert is_loopback_host(host) is True
+ # None (TestClient with no client info) is treated as loopback.
+ assert is_loopback_host(None) is True
+
+
+def test_is_loopback_host_rejects_external_hosts():
+ assert is_loopback_host("10.0.0.1") is False
+ assert is_loopback_host("192.168.1.100") is False
+ assert is_loopback_host("8.8.8.8") is False
+
+
+def test_require_loopback_raises_404_for_external_client():
+ class _FakeClient:
+ host = "10.0.0.1"
+
+ class _FakeRequest:
+ client = _FakeClient()
+
+ with pytest.raises(HTTPException) as exc_info:
+ require_loopback(_FakeRequest()) # type: ignore[arg-type]
+
+ assert exc_info.value.status_code == 404
+ # Privacy: 404 explicitly, not 403 — endpoints should be invisible.
+ assert exc_info.value.status_code != 403
+
+
+def test_require_loopback_accepts_loopback_client():
+ class _FakeClient:
+ host = "127.0.0.1"
+
+ class _FakeRequest:
+ client = _FakeClient()
+
+ # Should not raise.
+ require_loopback(_FakeRequest()) # type: ignore[arg-type]
+
+
+# ---------------------------------------------------------------------------
+# Serializer unit tests
+# ---------------------------------------------------------------------------
+
+
+def test_collect_warmup_handles_none_registry():
+ assert collect_warmup(None) == {}
+
+
+def test_collect_warmup_returns_registry_shape():
+ registry = WarmupRegistry()
+ registry.kompress.mark_loaded(handle=object(), source_status="enabled")
+ registry.memory_backend.mark_error("boom")
+
+ payload = collect_warmup(registry)
+
+ assert payload["kompress"]["status"] == "loaded"
+ assert payload["memory_backend"]["status"] == "error"
+ assert payload["memory_backend"]["error"] == "boom"
+ # Raw handle must never leak into the serialized payload.
+ assert "handle" not in payload["kompress"]
+
+
+def test_collect_ws_sessions_handles_none_registry():
+ assert collect_ws_sessions(None) == []
+
+
+def test_collect_ws_sessions_returns_snapshot():
+ reg = WebSocketSessionRegistry()
+ handle = WSSessionHandle(
+ session_id="sess-debug-1",
+ request_id="req-debug-1",
+ client_addr="127.0.0.1:9999",
+ upstream_url="wss://upstream/test",
+ )
+ reg.register(handle)
+
+ payload = collect_ws_sessions(reg)
+ assert len(payload) == 1
+ assert payload[0]["session_id"] == "sess-debug-1"
+ assert payload[0]["request_id"] == "req-debug-1"
+
+
+async def test_collect_tasks_returns_current_tasks_with_metadata():
+ async def _noop_task():
+ await asyncio.sleep(0.05)
+
+ task = asyncio.create_task(_noop_task(), name="debug-test-task")
+ try:
+ entries = collect_tasks()
+ matching = [e for e in entries if e["name"] == "debug-test-task"]
+ assert matching, "expected the named task to appear in collect_tasks output"
+ entry = matching[0]
+ assert entry["coro_qualname"] is not None
+ # Privacy: no frame locals, no coroutine args.
+ assert "locals" not in entry
+ assert "cr_frame" not in entry
+ assert "args" not in entry
+ assert entry["stack_depth"] is None or isinstance(entry["stack_depth"], int)
+ finally:
+ task.cancel()
+ try:
+ await task
+ except (asyncio.CancelledError, BaseException):
+ pass
+
+
+async def test_collect_tasks_derives_age_from_ws_registry_for_codex_relays():
+ reg = WebSocketSessionRegistry()
+ sid = "relay-sess-1"
+ reg.register(
+ WSSessionHandle(
+ session_id=sid,
+ request_id="req-relay-1",
+ client_addr="127.0.0.1:1",
+ upstream_url="wss://upstream",
+ )
+ )
+
+ async def _long_relay():
+ await asyncio.sleep(0.2)
+
+ relay_task = asyncio.create_task(_long_relay(), name=f"codex-ws-c2u-{sid}")
+ try:
+ await asyncio.sleep(0.02) # let some age accrue
+ entries = collect_tasks(ws_registry=reg)
+ named = [e for e in entries if e["name"] == f"codex-ws-c2u-{sid}"]
+ assert named, "expected relay task in output"
+ entry = named[0]
+ assert entry["age_seconds"] is not None
+ assert entry["age_seconds"] >= 0.0
+ finally:
+ relay_task.cancel()
+ try:
+ await relay_task
+ except (asyncio.CancelledError, BaseException):
+ pass
+
+
+# ---------------------------------------------------------------------------
+# HTTP endpoint tests (loopback)
+# ---------------------------------------------------------------------------
+
+
+def test_debug_tasks_returns_json_array_for_loopback(client):
+ response = client.get("/debug/tasks")
+ assert response.status_code == 200
+ data = response.json()
+ assert isinstance(data, list)
+ # Each entry at least has name + coro_qualname fields.
+ for entry in data:
+ assert "name" in entry
+ assert "coro_qualname" in entry
+
+
+def test_debug_warmup_reports_registry_slots(client):
+ response = client.get("/debug/warmup")
+ assert response.status_code == 200
+ data = response.json()
+ # Registry surfaces all canonical slot names.
+ assert "kompress" in data
+ assert "magika" in data
+ assert "memory_backend" in data
+ assert "memory_embedder" in data
+ # Each slot has at least a status field.
+ assert "status" in data["memory_backend"]
+
+
+def test_debug_ws_sessions_reports_live_session(app_and_client):
+ app, client = app_and_client
+ proxy = app.state.proxy
+ assert proxy is not None, "create_app must wire app.state.proxy"
+
+ sid = "sess-debug-http"
+ proxy.ws_sessions.register(
+ WSSessionHandle(
+ session_id=sid,
+ request_id="req-debug-http",
+ client_addr="127.0.0.1:12345",
+ upstream_url="wss://upstream/test",
+ )
+ )
+ try:
+ response = client.get("/debug/ws-sessions")
+ assert response.status_code == 200
+ data = response.json()
+ matching = [entry for entry in data if entry["session_id"] == sid]
+ assert matching, "expected live session in /debug/ws-sessions output"
+ assert matching[0]["request_id"] == "req-debug-http"
+ finally:
+ proxy.ws_sessions.deregister(sid, cause="response_completed")
+
+ # After cleanup the session is gone.
+ response = client.get("/debug/ws-sessions")
+ assert response.status_code == 200
+ assert all(entry["session_id"] != sid for entry in response.json())
+
+
+def test_debug_endpoints_do_not_mutate_state(client):
+ # Call each endpoint 100 times and confirm the second read equals
+ # the first — no accidental mutation from serialization.
+ first_tasks = client.get("/debug/tasks").json()
+ first_warmup = client.get("/debug/warmup").json()
+ first_ws = client.get("/debug/ws-sessions").json()
+
+ for _ in range(100):
+ client.get("/debug/tasks")
+ client.get("/debug/warmup")
+ client.get("/debug/ws-sessions")
+
+ # Warmup and ws-sessions are deterministic (no background work touches
+ # them in this test config), so they must be identical.
+ assert client.get("/debug/warmup").json() == first_warmup
+ assert client.get("/debug/ws-sessions").json() == first_ws
+ # Tasks may vary naturally, but the call itself never raises and the
+ # shape never changes.
+ new_tasks = client.get("/debug/tasks").json()
+ assert isinstance(new_tasks, list)
+ for entry in new_tasks:
+ assert set(entry.keys()) == set(first_tasks[0].keys()) if first_tasks else True
+
+
+def test_debug_tasks_does_not_leak_coro_locals(client):
+ response = client.get("/debug/tasks")
+ assert response.status_code == 200
+ for entry in response.json():
+ # Privacy check: the serializer must not leak coroutine locals,
+ # frame state, or request bodies. Only name / qualname / age /
+ # depth / done are allowed.
+ assert set(entry.keys()) <= {
+ "name",
+ "coro_qualname",
+ "age_seconds",
+ "stack_depth",
+ "done",
+ }
+
+
+# ---------------------------------------------------------------------------
+# HTTP endpoint tests (non-loopback)
+# ---------------------------------------------------------------------------
+
+
+def test_debug_endpoints_return_404_for_non_loopback_client(app_and_external_client):
+ _, client = app_and_external_client
+ for path in ("/debug/tasks", "/debug/ws-sessions", "/debug/warmup"):
+ response = client.get(path)
+ assert response.status_code == 404, path
+ # Must be 404, not 403 — invisible to scanners.
+ assert response.status_code != 403
+
+
+def test_existing_health_routes_unchanged(client):
+ # Invariant: Unit 5 must not regress the existing health endpoints.
+ for path in ("/livez", "/readyz", "/health"):
+ response = client.get(path)
+ assert response.status_code == 200, path