Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): forward operator-listed guarded upstreams through a proxy (#3804)

gglucass4b7e5d246e54
diff --git a/headroom/proxy/upstream_guard.py b/headroom/proxy/upstream_guard.py
index 50aba3f3a..1872895e8 100644
--- a/headroom/proxy/upstream_guard.py
+++ b/headroom/proxy/upstream_guard.py
@@ -205,8 +205,11 @@ def clear_validated_addresses() -> None:
_PIN_SCOPE.set(None)
-def _allowlisted_destinations() -> tuple[set[str], set[tuple[str, str, int]]] | None:
- raw = os.environ.get(ALLOWED_BASE_URLS_ENV)
+def allowlisted_destinations(
+ env: str = ALLOWED_BASE_URLS_ENV,
+) -> tuple[set[str], set[tuple[str, str, int]]] | None:
+ """Parse a comma-separated host/URL list from ``env``; ``None`` when unset."""
+ raw = os.environ.get(env)
if not raw or not raw.strip():
return None
hosts: set[str] = set()
@@ -233,6 +236,26 @@ def _allowlisted_destinations() -> tuple[set[str], set[tuple[str, str, int]]] |
return hosts, origins
+def destination_listed(
+ allow: tuple[set[str], set[tuple[str, str, int]]],
+ scheme: str,
+ host: str,
+ port: int | None,
+) -> bool:
+ """Whether a destination matches a parsed host/origin list.
+
+ A bare host admits every port for that host; a URL admits only its exact
+ origin, with the scheme's default port filled in when ``port`` is ``None``.
+ """
+ hosts, origins = allow
+ if host.lower() in hosts:
+ return True
+ scheme = scheme.lower()
+ if port is None:
+ port = 443 if scheme in {"https", "wss"} else 80
+ return (scheme, host.lower(), port) in origins
+
+
# RFC 6052 / RFC 8215: these IPv6 prefixes embed an IPv4 address in their low
# 32 bits, and `ipaddress` reports the well-known one as globally routable. On a
# NAT64 network `64:ff9b::7f00:1` reaches 127.0.0.1, so the embedded address is
@@ -307,18 +330,13 @@ def is_safe_upstream_url(url: str, *, scope: dict[str, UpstreamPin] | None = Non
if not host:
return False
- allow = _allowlisted_destinations()
+ allow = allowlisted_destinations()
if allow is not None:
- hosts, origins = allow
- if host.lower() in hosts:
- return True
try:
port = parsed.port
except ValueError:
- return False
- if port is None:
- port = 443 if parsed.scheme.lower() in {"https", "wss"} else 80
- return (parsed.scheme.lower(), host.lower(), port) in origins
+ port = -1 # matches no origin; a listed bare host still passes, as before
+ return destination_listed(allow, parsed.scheme, host, port)
try:
infos = _RESOLVER_POOL.submit(
diff --git a/headroom/proxy/upstream_pinning.py b/headroom/proxy/upstream_pinning.py
index 404258de7..d3274e402 100644
--- a/headroom/proxy/upstream_pinning.py
+++ b/headroom/proxy/upstream_pinning.py
@@ -59,6 +59,18 @@ and are unaffected, so proxied deployments keep working exactly as before unless
they also accept ``x-headroom-base-url`` -- in which case the refusal is the
honest answer, and ``HEADROOM_ALLOWED_BASE_URLS`` is the supported way to admit
specific internal endpoints through a proxy.
+
+A deployment whose ``x-headroom-base-url`` callers are local agent
+integrations (OpenCode, Grok) cannot use that allowlist: it switches the guard
+to allow-only mode, and every provider those callers target would have to be
+listed there. ``HEADROOM_PROXIED_BASE_URLS`` (same syntax: bare hosts admit
+every port, URLs their exact origin) names the destinations whose resolution
+the operator is willing to leave to the proxy. A guarded request to one of
+them still has to pass the guard, and then leaves through the proxy by name;
+every other guarded destination is refused on a proxy route exactly as above,
+before the proxy is contacted, so the proxy never gets to resolve a name the
+operator did not choose. Direct routes stay pinned whether or not a host is
+listed. Unset, nothing changes.
"""
from __future__ import annotations
@@ -69,7 +81,23 @@ from typing import Any
import httpcore
import httpx
-from headroom.proxy.upstream_guard import guarded_pin
+from headroom.proxy.upstream_guard import (
+ allowlisted_destinations,
+ destination_listed,
+ guarded_pin,
+)
+
+PROXIED_BASE_URLS_ENV = "HEADROOM_PROXIED_BASE_URLS"
+
+
+def proxied_upstream_listed(url: httpx.URL) -> bool:
+ """Whether the operator listed ``url``'s destination in HEADROOM_PROXIED_BASE_URLS.
+
+ Read per request, like the guard's own allowlist, so a test or an operator
+ changing the environment does not need a new client.
+ """
+ allow = allowlisted_destinations(PROXIED_BASE_URLS_ENV)
+ return allow is not None and destination_listed(allow, url.scheme, url.host, url.port)
class UnpinnableUpstreamError(RuntimeError):
@@ -89,7 +117,8 @@ def _refusal(host: str, reason: str) -> UnpinnableUpstreamError:
"those addresses cannot be enforced on this connection, so the name "
"would be resolved a second time and could answer differently (DNS "
"rebinding). Retry the request, or allowlist the endpoint with "
- "HEADROOM_ALLOWED_BASE_URLS — see headroom/proxy/upstream_pinning.py."
+ "HEADROOM_ALLOWED_BASE_URLS (HEADROOM_PROXIED_BASE_URLS for a proxy "
+ "route) — see headroom/proxy/upstream_pinning.py."
)
@@ -181,13 +210,18 @@ class GuardedUpstreamRefusingTransport(httpx.AsyncBaseTransport):
the task whose contextvar scope holds the guard's verdict.
"""
- def __init__(self, inner: httpx.AsyncBaseTransport, reason: str) -> None:
+ def __init__(
+ self, inner: httpx.AsyncBaseTransport, reason: str, *, via_proxy: bool = False
+ ) -> None:
self._inner = inner
self._reason = reason
+ self._via_proxy = via_proxy
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
host = request.url.host
- if guarded_pin(host) is not None:
+ if guarded_pin(host) is not None and not (
+ self._via_proxy and proxied_upstream_listed(request.url)
+ ):
raise _refusal(host, self._reason)
return await self._inner.handle_async_request(request)
@@ -247,6 +281,7 @@ def _install(transport: Any) -> tuple[Any, bool]:
transport,
"it routes through a proxy, which resolves the target hostname "
"itself and cannot be told which address the guard accepted",
+ via_proxy=True,
),
False,
)
diff --git a/tests/test_upstream_guard.py b/tests/test_upstream_guard.py
index deac1feb1..188d25833 100644
--- a/tests/test_upstream_guard.py
+++ b/tests/test_upstream_guard.py
@@ -846,6 +846,136 @@ async def test_a_guarded_upstream_through_a_proxy_is_refused(
assert "proxy" in str(refused.value)
+class _ResolvingProxy(httpx.AsyncBaseTransport):
+ """Stands in for the proxy: resolves the target by name, as a real one does.
+
+ Its lookup is the second, independent resolution the guard cannot see, so a
+ test can hand it a different answer from the one the guard judged.
+ """
+
+ def __init__(self) -> None:
+ self.dialled: list[str] = []
+
+ async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+ infos = socket.getaddrinfo(request.url.host, None)
+ self.dialled.append(str(infos[0][4][0]))
+ return httpx.Response(200)
+
+
+def _proxied_transport(client: httpx.AsyncClient, url: str) -> _ResolvingProxy:
+ from headroom.proxy.upstream_pinning import GuardedUpstreamRefusingTransport
+
+ transport = client._transport_for_url(httpx.URL(url))
+ assert isinstance(transport, GuardedUpstreamRefusingTransport)
+ proxy = _ResolvingProxy()
+ transport._inner = proxy
+ return proxy
+
+
[email protected]("listed", [None, "api.example.com"])
+async def test_proxy_rebinding_to_an_internal_address_stays_refused(
+ monkeypatch: pytest.MonkeyPatch, listed: str | None
+) -> None:
+ """Headroom sees a public answer, the proxy would see metadata: refused.
+
+ Neither with HEADROOM_PROXIED_BASE_URLS unset nor with it naming some other
+ host may a guarded destination reach the proxy, because the proxy's own
+ lookup is the one an attacker who controls the name answers differently.
+ The refusal lands before the proxy is contacted, so that lookup never runs.
+ """
+ from headroom.proxy import upstream_guard
+ from headroom.proxy.upstream_pinning import UnpinnableUpstreamError, install_upstream_pinning
+
+ resolver = _RebindingResolver("rebind.example", first="8.8.8.8", then="169.254.169.254")
+ monkeypatch.setattr(socket, "getaddrinfo", resolver)
+ if listed is None:
+ monkeypatch.delenv("HEADROOM_PROXIED_BASE_URLS", raising=False)
+ else:
+ monkeypatch.setenv("HEADROOM_PROXIED_BASE_URLS", listed)
+ upstream_guard.clear_validated_addresses()
+
+ client = install_upstream_pinning(_client_through_proxy("http://proxy.internal:3128"))
+ try:
+ proxy = _proxied_transport(client, "https://rebind.example/v1")
+ assert is_safe_upstream_url("https://rebind.example/v1") is True
+ with pytest.raises(UnpinnableUpstreamError):
+ await client._transport_for_url(
+ httpx.URL("https://rebind.example/v1")
+ ).handle_async_request(httpx.Request("POST", "https://rebind.example/v1"))
+ finally:
+ await client.aclose()
+
+ assert proxy.dialled == []
+ assert resolver.calls == 1 # the guard's lookup only
+
+
+async def test_a_listed_host_leaves_through_the_proxy(monkeypatch: pytest.MonkeyPatch) -> None:
+ """HEADROOM_PROXIED_BASE_URLS forwards the destinations the operator named.
+
+ Behind a proxy a local agent integration (Grok, OpenCode) was refused
+ outright. Listing its provider lets the guarded request out through the
+ proxy after the guard has judged it; an unlisted host on the same client,
+ and a listed URL's other ports, are still refused.
+ """
+ from headroom.proxy import upstream_guard
+ from headroom.proxy.upstream_pinning import UnpinnableUpstreamError, install_upstream_pinning
+
+ monkeypatch.setattr(
+ socket, "getaddrinfo", lambda *a, **k: [(None, None, None, None, ("8.8.8.8", 443))]
+ )
+ monkeypatch.setenv("HEADROOM_PROXIED_BASE_URLS", "api.example, https://gw.example")
+ upstream_guard.clear_validated_addresses()
+
+ client = install_upstream_pinning(_client_through_proxy("http://proxy.internal:3128"))
+ try:
+ for url, forwarded in [
+ ("https://api.example/v1", True),
+ ("http://api.example:8080/v1", True),
+ ("https://gw.example/v1", True),
+ ("https://gw.example:8443/v1", False),
+ ("https://other.example/v1", False),
+ ]:
+ proxy = _proxied_transport(client, url)
+ transport = client._transport_for_url(httpx.URL(url))
+ assert is_safe_upstream_url(url) is True
+ request = httpx.Request("POST", url)
+ if forwarded:
+ assert (await transport.handle_async_request(request)).status_code == 200, url
+ assert proxy.dialled == ["8.8.8.8"], url
+ else:
+ with pytest.raises(UnpinnableUpstreamError):
+ await transport.handle_async_request(request)
+ assert proxy.dialled == [], url
+ finally:
+ await client.aclose()
+
+
+async def test_a_listed_host_keeps_every_other_refusal(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Listing covers proxy routes only, and never admits an internal answer."""
+ from headroom.proxy import upstream_guard
+ from headroom.proxy.upstream_pinning import (
+ GuardedUpstreamRefusingTransport,
+ UnpinnableUpstreamError,
+ )
+
+ monkeypatch.setattr(
+ socket, "getaddrinfo", lambda *a, **k: [(None, None, None, None, ("8.8.8.8", 443))]
+ )
+ monkeypatch.setenv("HEADROOM_PROXIED_BASE_URLS", "api.example,internal.example")
+ upstream_guard.clear_validated_addresses()
+
+ assert is_safe_upstream_url("https://api.example/v1") is True
+ poolless = GuardedUpstreamRefusingTransport(_ResolvingProxy(), "no pool")
+ with pytest.raises(UnpinnableUpstreamError):
+ await poolless.handle_async_request(httpx.Request("POST", "https://api.example/v1"))
+
+ monkeypatch.setattr(
+ socket, "getaddrinfo", lambda *a, **k: [(None, None, None, None, ("10.0.0.5", 443))]
+ )
+ upstream_guard.clear_validated_addresses()
+ assert is_safe_upstream_url("https://internal.example/v1") is False
+
+
async def test_a_proxied_client_still_pins_its_direct_transport(
monkeypatch: pytest.MonkeyPatch,
) -> None: