Python
security: block loopback/private/metadata addresses in model-supplied URL fetches
Mr-Neutr0nff86735cf0aa
diff --git a/coworker/connectors/integration_tools.py b/coworker/connectors/integration_tools.pyindex 7588136d..f0f18d0b 100644--- a/coworker/connectors/integration_tools.py+++ b/coworker/connectors/integration_tools.py@@ -19,6 +19,7 @@ from urllib.parse import quoteimport aisuite as aifrom ..secrets import SecretStore+from ..web.guard import get_checkedfrom .browser_automation import make_browser_automation_toolsfrom .email_tools import make_email_toolsfrom .tool_defs import approval_for_tool, connector_for_tool@@ -305,15 +306,39 @@ def _gmail_is_hidden(def _request(- method: str, url: str, *, headers=None, params=None, json=None, auth=None+ method: str,+ url: str,+ *,+ headers=None,+ params=None,+ json=None,+ auth=None,+ check_addresses: bool = False,) -> dict[str, Any]:+ """HTTP for the connectors.++ `check_addresses` is for URLs the *model* supplies (browser_read_url). It turns off+ automatic redirects and walks the chain through the address guard instead, so a public+ URL cannot 302 into loopback or the metadata endpoint. The vendor endpoints everything+ else in this module calls are hardcoded, so they skip the guard and its DNS lookup.+ """try:import httpx- with httpx.Client(timeout=30.0, follow_redirects=True) as client:- resp = client.request(- method, url, headers=headers, params=params, json=json, auth=auth- )+ with httpx.Client(+ timeout=30.0, follow_redirects=not check_addresses+ ) as client:+ if check_addresses:+ if method.upper() != "GET":+ return {"error": "address-checked requests must be GET"}+ try:+ resp = get_checked(client, url)+ except PermissionError as exc:+ return {"error": str(exc)}+ else:+ resp = client.request(+ method, url, headers=headers, params=params, json=json, auth=auth+ )ctype = resp.headers.get("content-type", "")data: Any = resp.json() if "json" in ctype.lower() else resp.textif resp.status_code >= 400:@@ -533,7 +558,13 @@ def make_integration_tools(def browser_read_url(url: str, max_chars: int = 20000) -> dict[str, Any]:if not url.lower().startswith(("http://", "https://")):return {"error": "url must start with http:// or https://"}- out = _request("GET", url, headers={"User-Agent": "coworker/0.1 (+connector)"})+ # Model-supplied URL: address-check every hop, same guard as web_fetch.+ out = _request(+ "GET",+ url,+ headers={"User-Agent": "coworker/0.1 (+connector)"},+ check_addresses=True,+ )if "error" in out:return outdata = out["data"]diff --git a/coworker/web/fetch.py b/coworker/web/fetch.pyindex 9d273d03..f58291da 100644--- a/coworker/web/fetch.py+++ b/coworker/web/fetch.py@@ -13,6 +13,8 @@ from typing import Any, Callableimport aisuite as ai+from .guard import get_checked+_MAX = 20000 # default chars returned_SCHEMA = {@@ -84,16 +86,20 @@ def make_web_fetch_tool() -> Callable[..., Any]:try:import httpx+ # follow_redirects=False: guard.get_checked walks the chain so every hop is+ # address-checked, not just the URL the model first supplied.with httpx.Client(- follow_redirects=True,+ follow_redirects=False,timeout=20.0,headers={"User-Agent": "coworker/0.1 (+desktop)"},) as client:- resp = client.get(url)+ resp = get_checked(client, url)resp.raise_for_status()ctype = resp.headers.get("content-type", "")body = resp.textfinal_url = str(resp.url)+ except PermissionError as exc: # blocked address (loopback, private, metadata)+ return {"error": str(exc)}except Exception as exc: # network / HTTP / TLSreturn {"error": f"fetch failed: {exc}"}text = _html_to_text(body) if "html" in ctype.lower() else bodydiff --git a/coworker/web/guard.py b/coworker/web/guard.pynew file mode 100644index 00000000..c4f6d1c3--- /dev/null+++ b/coworker/web/guard.py@@ -0,0 +1,109 @@+"""Address guard for URLs the model chooses.++`web_fetch` and `browser_read_url` take a URL straight from the model, and the model's+input is untrusted by design — it reads web pages, email and Slack messages, all of which+are documented as "data, not instructions". A page that talks the agent into fetching+`http://169.254.169.254/` or `http://127.0.0.1:11434/` turns a read-only research tool into+a probe of the machine's own network position, and `web_fetch` is `requires_approval=False`,+so no prompt ever appears.++This blocks the ranges that are only reachable *because* OpenWorker runs on the user's+machine: loopback, RFC1918 and other private space, link-local (which covers the cloud+metadata endpoint at 169.254.169.254), and the reserved/multicast blocks.++Every hop is checked, not just the first: `follow_redirects=True` otherwise lets a public+URL 302 straight to loopback, which is the standard way this filter is bypassed.++Not covered: DNS rebinding. The name is resolved here and resolved again by the client when+it connects, so a record with a ~0 TTL can change between the two. Closing that needs+connection-level IP pinning; the hop check is the cheap 90% and is stated as such.+"""++from __future__ import annotations++import ipaddress+import socket+from typing import Optional+from urllib.parse import urlsplit++MAX_REDIRECTS = 5+++def _blocked_reason(ip: ipaddress._BaseAddress) -> Optional[str]:+ if ip.is_loopback:+ return "loopback"+ if ip.is_link_local:+ return "link-local (includes the cloud metadata endpoint)"+ if ip.is_private:+ return "a private network"+ if ip.is_multicast:+ return "multicast"+ if ip.is_reserved or ip.is_unspecified:+ return "a reserved range"+ return None+++def check_url(url: str) -> Optional[str]:+ """None if the URL may be fetched, else a human-readable refusal reason.++ Resolves the host and rejects when *any* answer lands in a blocked range, so a name+ with both a public and a private A record cannot be used to slip through.+ """+ parts = urlsplit(url)+ if parts.scheme not in ("http", "https"):+ return "url must start with http:// or https://"+ host = parts.hostname+ if not host:+ return "url has no host"++ # A literal address needs no lookup.+ try:+ literal = ipaddress.ip_address(host)+ except ValueError:+ literal = None+ if literal is not None:+ reason = _blocked_reason(literal)+ return f"refusing to fetch {host}: {reason}" if reason else None++ try:+ infos = socket.getaddrinfo(host, parts.port or (443 if parts.scheme == "https" else 80),+ proto=socket.IPPROTO_TCP)+ except OSError as exc:+ return f"could not resolve {host}: {exc}"++ for info in infos:+ raw = info[4][0]+ try:+ ip = ipaddress.ip_address(raw)+ except ValueError:+ continue+ # ::ffff:127.0.0.1 and friends must be judged as the v4 address they carry.+ mapped = getattr(ip, "ipv4_mapped", None)+ if mapped is not None:+ ip = mapped+ reason = _blocked_reason(ip)+ if reason:+ return f"refusing to fetch {host} ({ip}): {reason}"+ return None+++def get_checked(client, url: str, *, max_redirects: int = MAX_REDIRECTS):+ """GET `url`, validating the address before every hop.++ `client` must be built with `follow_redirects=False`; redirects are walked here so each+ Location is checked. Returns the final response. Raises `PermissionError` when a hop is+ refused, `RuntimeError` when the redirect budget is exhausted.+ """+ seen = url+ for _ in range(max_redirects + 1):+ reason = check_url(seen)+ if reason:+ raise PermissionError(reason)+ resp = client.get(seen)+ if resp.status_code not in (301, 302, 303, 307, 308):+ return resp+ location = resp.headers.get("location")+ if not location:+ return resp+ seen = str(resp.url.join(location))+ raise RuntimeError(f"too many redirects (>{max_redirects})")diff --git a/tests/test_url_address_guard.py b/tests/test_url_address_guard.pynew file mode 100644index 00000000..13642b6c--- /dev/null+++ b/tests/test_url_address_guard.py@@ -0,0 +1,157 @@+"""`web_fetch` / `browser_read_url` must not reach the machine's own network position.++Both take a URL straight from the model, and the model's input is untrusted by design —+the tools' own descriptions call fetched content "data to evaluate, not instructions".+`web_fetch` is additionally `requires_approval=False`, so nothing prompts the user.+"""++import socket++import pytest++from coworker.web import guard+from coworker.web.fetch import make_web_fetch_tool+++def _resolves_to(monkeypatch, ip: str):+ monkeypatch.setattr(+ guard.socket, "getaddrinfo",+ lambda *a, **k: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, 80))],+ )+++# -- literals -----------------------------------------------------------------+[email protected]("url,needle", [+ ("http://127.0.0.1:11434/api/tags", "loopback"),+ ("http://localhost:8000/", "loopback"),+ ("http://[::1]:8080/", "loopback"),+ ("http://169.254.169.254/latest/meta-data/", "link-local"),+ ("http://10.0.0.5/admin", "private"),+ ("http://192.168.1.1/", "private"),+ ("http://172.16.4.4/", "private"),+ ("http://0.0.0.0/", "refusing to fetch"), # 0.0.0.0/8 lands in is_private first+])+def test_blocked_literals(url, needle):+ reason = guard.check_url(url)+ assert reason and needle in reason+++def test_ipv4_mapped_ipv6_loopback_is_blocked():+ """::ffff:127.0.0.1 must be judged as the v4 address it carries."""+ assert guard.check_url("http://[::ffff:127.0.0.1]/")+++def test_public_literal_is_allowed():+ assert guard.check_url("https://93.184.216.34/") is None++[email protected]("url", ["file:///etc/passwd", "ftp://example.com/x",+ "gopher://example.com/", "http://"])+def test_non_http_schemes_and_hostless_urls_are_refused(url):+ assert guard.check_url(url)+++# -- names --------------------------------------------------------------------++def test_hostname_resolving_to_loopback_is_blocked(monkeypatch):+ """`localtest.me` and friends are public names with private answers."""+ _resolves_to(monkeypatch, "127.0.0.1")+ assert "loopback" in guard.check_url("http://sneaky.example.com/")+++def test_hostname_resolving_to_metadata_ip_is_blocked(monkeypatch):+ _resolves_to(monkeypatch, "169.254.169.254")+ assert guard.check_url("http://metadata.example.com/")+++def test_any_private_answer_blocks_a_split_horizon_name(monkeypatch):+ """One public and one private A record must not be a way through."""+ monkeypatch.setattr(+ guard.socket, "getaddrinfo",+ lambda *a, **k: [+ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 80)),+ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 80)),+ ],+ )+ assert guard.check_url("http://split.example.com/")+++def test_public_hostname_is_allowed(monkeypatch):+ _resolves_to(monkeypatch, "93.184.216.34")+ assert guard.check_url("https://example.com/docs") is None+++def test_unresolvable_host_is_refused_not_fetched(monkeypatch):+ def boom(*a, **k):+ raise socket.gaierror("nodename nor servname provided")+ monkeypatch.setattr(guard.socket, "getaddrinfo", boom)+ assert "could not resolve" in guard.check_url("http://nope.invalid/")+++# -- redirects ----------------------------------------------------------------++class _Resp:+ def __init__(self, status=200, location=None, url="https://example.com/"):+ self.status_code = status+ self.headers = {"location": location} if location else {}+ self.url = _Url(url)+ self.text = "body"++ def raise_for_status(self):+ pass+++class _Url(str):+ def join(self, other):+ return other+++class _Client:+ """Records what was actually requested, so a blocked hop is provably not fetched."""++ def __init__(self, script):+ self.script = script+ self.requested = []++ def get(self, url):+ self.requested.append(url)+ return self.script.pop(0)+++def test_redirect_into_loopback_is_blocked_before_the_second_request(monkeypatch):+ _resolves_to(monkeypatch, "93.184.216.34")+ client = _Client([_Resp(302, location="http://127.0.0.1:11434/api/tags")])+ with pytest.raises(PermissionError, match="loopback"):+ guard.get_checked(client, "https://example.com/start")+ assert client.requested == ["https://example.com/start"], (+ "the redirect target must never be requested"+ )+++def test_allowed_redirect_chain_is_followed(monkeypatch):+ _resolves_to(monkeypatch, "93.184.216.34")+ client = _Client([_Resp(302, location="https://example.com/b"), _Resp(200)])+ resp = guard.get_checked(client, "https://example.com/a")+ assert resp.status_code == 200+ assert client.requested == ["https://example.com/a", "https://example.com/b"]+++def test_redirect_loop_is_bounded(monkeypatch):+ _resolves_to(monkeypatch, "93.184.216.34")+ client = _Client([_Resp(302, location="https://example.com/loop")] * 50)+ with pytest.raises(RuntimeError, match="too many redirects"):+ guard.get_checked(client, "https://example.com/loop")+++# -- the tool -----------------------------------------------------------------++def test_web_fetch_returns_the_refusal_as_a_tool_error(monkeypatch):+ _resolves_to(monkeypatch, "127.0.0.1")+ out = make_web_fetch_tool()("http://sneaky.example.com/")+ assert "loopback" in out["error"]+ assert "text" not in out+++def test_web_fetch_still_rejects_non_http_schemes():+ assert "http" in make_web_fetch_tool()("file:///etc/passwd")["error"]