Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(health): readyz verifies upstream connectivity, not just process liveness (#744)

yehsuf5dfb446da1fb
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index f7b954474..badc6a530 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -1601,6 +1601,12 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
native_tool=bool(memory_status.get("native_tool", False)),
bridge_enabled=bool(memory_status.get("bridge_enabled", False)),
),
+ "upstream": _component_health(
+ enabled=os.environ.get("HEADROOM_SKIP_UPSTREAM_CHECK", "").strip() != "1",
+ ready=bool(_upstream_check_cache["ok"]),
+ url=_upstream_check_cache["url"],
+ error=_upstream_check_cache["error"],
+ ),
}
def _runtime_payload() -> dict[str, Any]:
@@ -1703,6 +1709,70 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
}
return payload
+ # ---------------------------------------------------------------------------
+ # Upstream connectivity check — cached to avoid hammering the upstream on
+ # every /readyz poll. Set HEADROOM_SKIP_UPSTREAM_CHECK=1 to opt out (e.g.
+ # in air-gapped or test environments where the upstream isn't reachable at
+ # startup time).
+ # ---------------------------------------------------------------------------
+
+ _UPSTREAM_CHECK_TTL = 30.0 # seconds
+ _upstream_check_cache: dict[str, Any] = {
+ "expires_at": 0.0,
+ "ok": True,
+ "error": None,
+ "url": None,
+ }
+ _upstream_check_lock = asyncio.Lock()
+
+ def _upstream_target_url() -> str:
+ """Return the primary upstream base URL to probe."""
+ # Use the resolved API target from the provider runtime so we respect
+ # any overrides set by ProxyConfig.anthropic_api_url / env vars.
+ return proxy.provider_runtime.api_targets.anthropic
+
+ async def _check_upstream() -> None:
+ """Probe the upstream API endpoint and update the cached result.
+
+ Uses a HEAD request with a 5-second timeout — just enough to verify
+ TLS + TCP reachability without triggering an inference call.
+ """
+ if os.environ.get("HEADROOM_SKIP_UPSTREAM_CHECK", "").strip() == "1":
+ # Opt-out: treat upstream as always reachable.
+ _upstream_check_cache["ok"] = True
+ _upstream_check_cache["error"] = None
+ _upstream_check_cache["expires_at"] = time.monotonic() + _UPSTREAM_CHECK_TTL
+ return
+
+ now = time.monotonic()
+ # Fast-path: return if the cached result is still fresh (no lock needed
+ # for a simple float comparison — worst case we re-check twice).
+ if now < _upstream_check_cache["expires_at"]:
+ return
+
+ async with _upstream_check_lock:
+ # Re-check inside the lock to handle concurrent waiters.
+ if time.monotonic() < _upstream_check_cache["expires_at"]:
+ return
+ url = _upstream_target_url()
+ _upstream_check_cache["url"] = url
+ client = proxy.http_client
+ if client is None:
+ _upstream_check_cache["ok"] = False
+ _upstream_check_cache["error"] = "proxy client not initialised"
+ _upstream_check_cache["expires_at"] = time.monotonic() + _UPSTREAM_CHECK_TTL
+ return
+ try:
+ resp = await client.head(url, timeout=5.0)
+ # Any HTTP response (even 4xx/5xx) means TLS+TCP worked.
+ _ = resp.status_code
+ _upstream_check_cache["ok"] = True
+ _upstream_check_cache["error"] = None
+ except Exception as exc: # noqa: BLE001
+ _upstream_check_cache["ok"] = False
+ _upstream_check_cache["error"] = str(exc)
+ _upstream_check_cache["expires_at"] = time.monotonic() + _UPSTREAM_CHECK_TTL
+
# CORS
app.add_middleware(
CORSMiddleware,
@@ -1838,11 +1908,13 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
@app.get("/readyz")
async def readyz():
+ await _check_upstream()
payload = _health_payload(include_config=False)
return JSONResponse(status_code=200 if payload["ready"] else 503, content=payload)
@app.get("/health")
async def health():
+ await _check_upstream()
payload = _health_payload(include_config=True)
return JSONResponse(status_code=200, content=payload)
diff --git a/tests/test_proxy_healthchecks.py b/tests/test_proxy_healthchecks.py
index 58b7d22f6..1d7761b32 100644
--- a/tests/test_proxy_healthchecks.py
+++ b/tests/test_proxy_healthchecks.py
@@ -12,7 +12,10 @@ from headroom.proxy.server import ProxyConfig, create_app
@pytest.fixture
-def client():
+def client(monkeypatch):
+ # Skip the live upstream connectivity probe in unit tests — tests verify
+ # the check logic separately (see test_readyz_upstream_check_* below).
+ monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
config = ProxyConfig(
optimize=False,
cache_enabled=False,
@@ -80,6 +83,7 @@ def test_health_preserves_backwards_compatible_config_payload(client):
def test_health_includes_deployment_metadata_when_present(monkeypatch):
+ monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
monkeypatch.setenv("HEADROOM_DEPLOYMENT_PROFILE", "default")
monkeypatch.setenv("HEADROOM_DEPLOYMENT_PRESET", "persistent-service")
monkeypatch.setenv("HEADROOM_DEPLOYMENT_RUNTIME", "python")
@@ -116,7 +120,8 @@ def test_health_remains_200_when_proxy_is_not_ready(client):
assert response.json()["ready"] is False
-def test_readyz_reports_memory_backend_when_enabled(tmp_path):
+def test_readyz_reports_memory_backend_when_enabled(tmp_path, monkeypatch):
+ monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
config = ProxyConfig(
optimize=False,
cache_enabled=False,
@@ -141,6 +146,7 @@ def test_readyz_reports_memory_backend_when_enabled(tmp_path):
def test_readyz_initializes_qdrant_memory_backend(monkeypatch):
+ monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
from headroom.memory.backends import direct_mem0
init_calls: list[str] = []
@@ -175,7 +181,8 @@ def test_readyz_initializes_qdrant_memory_backend(monkeypatch):
assert data["checks"]["memory"]["initialized"] is True
-def test_shutdown_tolerates_stubbed_memory_handler():
+def test_shutdown_tolerates_stubbed_memory_handler(monkeypatch):
+ monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
config = ProxyConfig(
optimize=False,
cache_enabled=False,
@@ -197,3 +204,86 @@ def test_shutdown_tolerates_stubbed_memory_handler():
response = client.get("/health")
assert response.status_code == 200
+
+
+# ---------------------------------------------------------------------------
+# Upstream connectivity check tests
+# ---------------------------------------------------------------------------
+
+
+def test_readyz_upstream_check_disabled_by_env_var(monkeypatch):
+ """HEADROOM_SKIP_UPSTREAM_CHECK=1 suppresses the probe and reports ready."""
+ monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ )
+ app = create_app(config)
+
+ with TestClient(app) as test_client:
+ response = test_client.get("/readyz")
+
+ assert response.status_code == 200
+ data = response.json()
+ assert data["ready"] is True
+ # When the check is skipped the component is reported as "disabled"
+ assert data["checks"]["upstream"]["enabled"] is False
+ assert data["checks"]["upstream"]["ready"] is True
+
+
+def test_readyz_upstream_check_failure_returns_503(monkeypatch):
+ """A failed upstream probe makes /readyz return HTTP 503."""
+ from unittest.mock import AsyncMock, patch
+
+ import httpx
+
+ monkeypatch.delenv("HEADROOM_SKIP_UPSTREAM_CHECK", raising=False)
+
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ )
+ app = create_app(config)
+
+ # Patch the proxy's shared http_client.head so the probe uses the same
+ # client as real traffic (which also means TLS/CA config is consistent).
+ with TestClient(app) as test_client:
+ with patch.object(
+ test_client.app.state.proxy.http_client,
+ "head",
+ new=AsyncMock(side_effect=httpx.ConnectError("connection refused (test)")),
+ ):
+ response = test_client.get("/readyz")
+
+ assert response.status_code == 503
+ data = response.json()
+ assert data["ready"] is False
+ assert data["checks"]["upstream"]["ready"] is False
+ assert "connection refused" in data["checks"]["upstream"]["error"]
+
+
+def test_health_includes_upstream_check_result(monkeypatch):
+ """/health always returns 200 but exposes the upstream check result."""
+ monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ )
+ app = create_app(config)
+
+ with TestClient(app) as test_client:
+ response = test_client.get("/health")
+
+ assert response.status_code == 200
+ data = response.json()
+ assert "upstream" in data["checks"]
+ upstream = data["checks"]["upstream"]
+ assert "enabled" in upstream
+ assert "ready" in upstream
+ assert "status" in upstream