Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

security: SSRF gate resolves the host httpx connects to; allowlist matches its spellings (#161)

Francois du Plessis545946ec651a
diff --git a/pyproject.toml b/pyproject.toml
index 29aa4c2..3517a6d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -32,6 +32,7 @@ dependencies = [
"Crawl4AI>=0.7.3",
"pymupdf>=1.24",
"httpx>=0.27",
+ "idna",
]
[project.optional-dependencies]
diff --git a/src/hyperresearch/web/safe_http.py b/src/hyperresearch/web/safe_http.py
index 309dd78..e104e5b 100644
--- a/src/hyperresearch/web/safe_http.py
+++ b/src/hyperresearch/web/safe_http.py
@@ -32,6 +32,7 @@ from dataclasses import dataclass, field
from urllib.parse import urlparse
import httpx
+import idna
_ALLOWED_SCHEMES = frozenset({"http", "https"})
@@ -118,6 +119,30 @@ def _is_blocked_ip(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
)
+def _normalize_hostname(host: str) -> str:
+ """Canonical form for allowlist matching: lowercase, ACE, no trailing dot.
+
+ A trailing dot marks the name as absolute: the same host as the bare name
+ whenever the resolver looks that up as absolute, but a different string
+ to an exact match. IDN hosts arrive as unicode or punycode depending on
+ who typed them. Allowlist matching must not depend on which spelling the
+ URL or the config file used.
+
+ A non-ASCII name goes through idna.encode, the IDNA 2008 encoder httpx
+ applies to a URL host, so it becomes the name httpx would connect to
+ (labels split on the full stops U+3002, U+FF0E and U+FF61 as well as
+ "."), and a name httpx would refuse in a URL raises idna.IDNAError. The
+ stdlib "idna" codec is IDNA 2003 and maps some characters away: it
+ encodes "straße" as "strasse", a different, separately registrable name
+ from the "xn--strae-oqa" that httpx connects to. Matching through that
+ codec would let an allowlist entry for one admit the other.
+ """
+ host = host.lower()
+ if not host.isascii():
+ host = idna.encode(host).decode("ascii")
+ return host.rstrip(".")
+
+
def _resolve(host: str) -> list[ipaddress.IPv4Address | ipaddress.IPv6Address]:
try:
return [ipaddress.ip_address(host.strip("[]"))]
@@ -125,7 +150,11 @@ def _resolve(host: str) -> list[ipaddress.IPv4Address | ipaddress.IPv6Address]:
pass
try:
infos = socket.getaddrinfo(host, None)
- except socket.gaierror as exc:
+ except (UnicodeError, OSError) as exc:
+ # gaierror is an OSError. UnicodeError covers names the IDNA codec
+ # rejects while getaddrinfo encodes them (a 300-character label, for
+ # one). Anything but SafeHTTPError escapes the batch fetch path's
+ # per-URL handler and fails the whole batch on one hostile hostname.
raise SafeHTTPError(f"DNS lookup failed for {host!r}: {exc}") from exc
addrs: list[ipaddress.IPv4Address | ipaddress.IPv6Address] = []
for info in infos:
@@ -143,7 +172,8 @@ def _parse_allowlist(
An entry that parses as an IP network (a bare address counts as a
/32 or /128) matches by resolved address; anything else matches the
- URL hostname exactly, case-insensitively. A malformed entry raises
+ URL hostname after both are normalized (case, trailing root dot,
+ IDN/punycode spelling). A malformed entry raises
:class:`SafeHTTPError` naming it — silently ignoring one would turn
a typo into a lockout that looks like an SSRF refusal.
"""
@@ -162,6 +192,14 @@ def _parse_allowlist(
raise SafeHTTPError(
f"invalid allow_private_hosts entry {entry!r}: not a valid CIDR"
) from None
+ if "%" in entry:
+ # Only a scoped IPv6 literal (fe80::1%eth0) legitimately
+ # carries '%', and those parsed as networks above. As a
+ # hostname the entry could never match.
+ raise SafeHTTPError(
+ f"invalid allow_private_hosts entry {entry!r}: '%' is only "
+ "valid as an IPv6 scope id on an address literal"
+ )
if ":" in entry:
# host:port. URL hostnames never carry the port, so the entry
# would become a hostname that can never match: a silent dead
@@ -170,10 +208,23 @@ def _parse_allowlist(
f"invalid allow_private_hosts entry {entry!r}: entries are "
"hostnames or CIDRs; drop the port"
)
- hostnames.add(entry.lower())
+ try:
+ hostnames.add(_normalize_hostname(entry))
+ except UnicodeError as exc: # idna.IDNAError is a UnicodeError
+ # httpx refuses this host in a URL, so the entry could never
+ # match: the same silent dead entry as above.
+ raise SafeHTTPError(
+ f"invalid allow_private_hosts entry {entry!r}: not a valid "
+ f"IDNA hostname ({exc})"
+ ) from None
return frozenset(hostnames), networks
+# What urlparse strips from the start of a URL (urllib.parse's
+# _WHATWG_C0_CONTROL_OR_SPACE, a private name).
+_C0_CONTROL_OR_SPACE = "".join(chr(c) for c in range(0x20)) + " "
+
+
def check_url(url: str, allow_private_hosts: tuple[str, ...] = ()) -> None:
"""Validate the URL is safe to fetch. Raises :class:`SafeHTTPError`.
@@ -186,17 +237,43 @@ def check_url(url: str, allow_private_hosts: tuple[str, ...] = ()) -> None:
made afterwards re-resolves it (see the module docstring on DNS
rebinding).
"""
- parsed = urlparse(url)
+ try:
+ parsed = urlparse(url)
+ hostname = parsed.hostname
+ except ValueError as exc:
+ # urlparse raises ValueError for some malformed netlocs: unbalanced
+ # IPv6 brackets give "Invalid IPv6 URL". A bare ValueError escapes
+ # the batch fetch path's SafeHTTPError-only per-URL handler.
+ raise SafeHTTPError(f"refusing unparseable URL {url!r}: {exc}") from exc
if parsed.scheme.lower() not in _ALLOWED_SCHEMES:
raise SafeHTTPError(
f"refusing URL with scheme {parsed.scheme!r}; only http/https are allowed"
)
- if not parsed.hostname:
+ if not hostname:
+ raise SafeHTTPError(f"refusing URL with no hostname: {url!r}")
+ # Check the name the connection will look up, not urlparse's spelling of
+ # it. httpx sends an IDN host in its IDNA 2008 ASCII form, while
+ # getaddrinfo encodes a non-ASCII name with the stdlib "idna" codec (IDNA
+ # 2003), which maps some characters away: "straße" would be looked up as
+ # "strasse", a different name from the "xn--strae-oqa" httpx connects to.
+ #
+ # urlparse drops leading C0 controls and spaces (the WHATWG rule); httpx
+ # reads " http://host/" as a relative URL with no host. Strip the same
+ # characters so both parsers see one host, and refuse a URL httpx still
+ # finds no host in rather than look up an empty name.
+ try:
+ hostname = httpx.URL(url.lstrip(_C0_CONTROL_OR_SPACE)).raw_host.decode("ascii")
+ except (httpx.InvalidURL, UnicodeError) as exc:
+ # UnicodeError: httpx cannot encode a lone surrogate in the path,
+ # query or fragment, which is what a non-UTF-8 byte in a command-line
+ # argument becomes.
+ raise SafeHTTPError(f"refusing unparseable URL {url!r}: {exc}") from exc
+ if not hostname:
raise SafeHTTPError(f"refusing URL with no hostname: {url!r}")
allowed_hosts, allowed_nets = _parse_allowlist(allow_private_hosts)
- if parsed.hostname.lower() in allowed_hosts:
+ if _normalize_hostname(hostname) in allowed_hosts:
return
- addrs = _resolve(parsed.hostname)
+ addrs = _resolve(hostname)
if not addrs:
raise SafeHTTPError(f"refusing URL {url!r}: hostname did not resolve")
for addr in addrs:
@@ -204,7 +281,7 @@ def check_url(url: str, allow_private_hosts: tuple[str, ...] = ()) -> None:
addr.version == net.version and addr in net for net in allowed_nets
):
raise SafeHTTPError(
- f"refusing URL {url!r}: hostname {parsed.hostname} resolves to "
+ f"refusing URL {url!r}: hostname {hostname} resolves to "
f"non-public address {addr}"
)
diff --git a/tests/test_core/test_oa_recovery.py b/tests/test_core/test_oa_recovery.py
index 266850e..6dd4597 100644
--- a/tests/test_core/test_oa_recovery.py
+++ b/tests/test_core/test_oa_recovery.py
@@ -131,7 +131,9 @@ class TestCheckOaUrl:
monkeypatch.setattr(socket, "getaddrinfo", boom)
ok, reason = oa.check_oa_url("https://nowhere.example.org/p.pdf")
- assert ok is False and "DNS resolution failed" in reason
+ # check_url turns resolver errors into SafeHTTPError, so the refusal
+ # carries the gate's reason.
+ assert ok is False and "DNS lookup failed" in reason
class TestNeedsRecovery:
diff --git a/tests/test_web/test_fetch_many_gate.py b/tests/test_web/test_fetch_many_gate.py
index 95a5cd3..ec26a62 100644
--- a/tests/test_web/test_fetch_many_gate.py
+++ b/tests/test_web/test_fetch_many_gate.py
@@ -46,6 +46,37 @@ def _run(coro):
return box["result"]
+def test_fetch_many_skips_malformed_url_and_fetches_the_rest(monkeypatch, caplog):
+ """A URL check_url cannot parse must be skipped like a refused one. urlparse
+ raises ValueError for unbalanced IPv6 brackets, and httpx.URL raises
+ UnicodeEncodeError for a lone surrogate (a non-UTF-8 byte in a
+ command-line argument becomes one); if either escapes check_url as
+ anything but SafeHTTPError, the per-URL handler in _fetch_many_async
+ does not catch it and every other URL in the batch is lost with it."""
+ fetched: list[str] = []
+
+ def fake_fetch_pdf(url, settings=None):
+ fetched.append(url)
+ return WebResult(url=url, title="ok", content="pdf text")
+
+ monkeypatch.setattr(provider, "_fetch_pdf", fake_fetch_pdf)
+
+ inst = provider.Crawl4AIProvider.__new__(provider.Crawl4AIProvider)
+ inst._settings = provider.FetchSettings()
+
+ urls = [
+ "http://[::1/broken.pdf",
+ "http://8.8.8.8/pa\udcffper.pdf",
+ "http://8.8.8.8/paper.pdf",
+ ]
+ with caplog.at_level(logging.WARNING, logger="hyperresearch.web"):
+ results = _run(inst._fetch_many_async(urls))
+
+ assert [r.url for r in results] == ["http://8.8.8.8/paper.pdf"]
+ assert fetched == ["http://8.8.8.8/paper.pdf"]
+ assert "refused batch fetch" in caplog.text
+
+
def test_fetch_many_skips_refused_urls_and_fetches_the_rest(monkeypatch, caplog):
fetched: list[str] = []
diff --git a/tests/test_web/test_safe_http.py b/tests/test_web/test_safe_http.py
index fa9ba5c..488fbd9 100644
--- a/tests/test_web/test_safe_http.py
+++ b/tests/test_web/test_safe_http.py
@@ -411,6 +411,187 @@ def test_safe_get_redirect_to_allowlisted_private_host_is_followed():
assert resp.content == b"mirror content"
+# ---------------------------------------------------------------------------
+# Hostname spellings: allowlist matching follows DNS equivalence, not string
+# equality. The resolver is stubbed to answer with a private address, so a
+# spelling that misses its entry is refused rather than looked up for real.
+# ---------------------------------------------------------------------------
+
+
+ ("url", "entry"),
+ [
+ # A trailing dot marks an absolute name: same host when looked up as one.
+ ("http://mirror.internal./x", "mirror.internal"),
+ ("http://mirror.internal/x", "mirror.internal."),
+ ("http://Mirror.INTERNAL./x", "mirror.internal"),
+ # IDN hosts arrive as unicode or punycode depending on who typed them.
+ ("http://münchen.example/x", "xn--mnchen-3ya.example"),
+ ("http://xn--mnchen-3ya.example/x", "münchen.example"),
+ ],
+)
+def test_allowlist_hostname_matches_dns_equivalent_spelling(url, entry):
+ with patch("hyperresearch.web.safe_http.socket.getaddrinfo") as gai:
+ gai.return_value = [(socket.AF_INET, None, None, "", ("10.0.0.5", 0))]
+ check_url(url, allow_private_hosts=(entry,))
+
+
+ ("url", "entry"),
+ [
+ ("http://strasse.example/x", "straße.example"),
+ ("http://straße.example/x", "strasse.example"),
+ ],
+)
+def test_allowlist_hostname_does_not_match_idna2003_mapping(url, entry):
+ """IDNA 2003 (the stdlib "idna" codec) maps "straße" to "strasse", but
+ httpx sends "xn--strae-oqa": two separately registrable names. An entry
+ for one must not admit the other."""
+ with patch("hyperresearch.web.safe_http.socket.getaddrinfo") as gai:
+ gai.return_value = [(socket.AF_INET, None, None, "", ("10.0.0.5", 0))]
+ with pytest.raises(SafeHTTPError, match="non-public address"):
+ check_url(url, allow_private_hosts=(entry,))
+
+
+def test_allowlist_entry_with_ideographic_full_stop_names_the_dotted_host():
+ """IDNA and httpx separate labels on U+3002, so this entry names
+ "mirror.internal", not the single label its punycode would spell."""
+ entry = "mirror\u3002internal"
+ one_label = "xn--" + entry.encode("punycode").decode("ascii")
+ with patch("hyperresearch.web.safe_http.socket.getaddrinfo") as gai:
+ gai.return_value = [(socket.AF_INET, None, None, "", ("10.0.0.5", 0))]
+ check_url("http://mirror.internal/x", allow_private_hosts=(entry,))
+ with pytest.raises(SafeHTTPError, match="non-public address"):
+ check_url(f"http://{one_label}/x", allow_private_hosts=(entry,))
+
+
[email protected]("entry", ["fe80::1%eth0", "[fe80::1%eth0]"])
+def test_allowlist_scoped_ipv6_entry_parses_as_its_address(entry):
+ """A scoped entry is an address, not a hostname, and matches by address
+ bits whatever the scope."""
+ import ipaddress
+
+ from hyperresearch.web.safe_http import _parse_allowlist
+
+ hostnames, networks = _parse_allowlist((entry,))
+ assert hostnames == frozenset()
+ assert len(networks) == 1
+ assert ipaddress.ip_address("fe80::1") in networks[0]
+
+
+def test_allowlist_scoped_ipv6_url_allowed_by_bracketed_scoped_entry():
+ check_url("http://[fe80::1%eth0]/x", allow_private_hosts=("[fe80::1%eth0]",))
+
+
+def test_scoped_ipv6_url_refused_without_entry():
+ with pytest.raises(SafeHTTPError, match="non-public address"):
+ check_url("http://[fe80::1%eth0]/x")
+
+
+def test_allowlist_percent_in_non_address_entry_is_a_loud_error():
+ with pytest.raises(SafeHTTPError, match="scope id"):
+ check_url("http://8.8.8.8/", allow_private_hosts=("my%host",))
+
+
+ "entry",
+ [
+ "\uff45xample.com", # fullwidth "e"
+ "mu\u0308nchen.example", # NFD: "u" plus a combining diaeresis
+ "ex\u00adample.com", # soft hyphen
+ "\u200b.example", # zero-width space
+ ],
+)
+def test_allowlist_entry_httpx_would_refuse_is_a_loud_error(entry):
+ """httpx refuses each of these hosts in a URL (IDNA 2008 rejects them), so
+ the entry could never match and must not be accepted silently."""
+ with pytest.raises(SafeHTTPError, match="not a valid IDNA hostname"):
+ check_url("http://8.8.8.8/", allow_private_hosts=(entry,))
+
+
+# ---------------------------------------------------------------------------
+# The gate looks up the name the connection will. httpx sends an IDN host in
+# its IDNA 2008 form, while getaddrinfo encodes a unicode name with the stdlib
+# IDNA 2003 codec, which maps the deviation characters (ß, ς, ZWJ, ZWNJ) to a
+# different name. A zone owner could point the two at different addresses.
+# ---------------------------------------------------------------------------
+
+
+_WIRE_FORMS = [
+ ("http://straße.example/x", "xn--strae-oqa.example"), # IDNA 2003: strasse
+ # Final sigma; IDNA 2003 folds it to the medial sigma, "xn--4xa".
+ ("http://ς.example/x", "xn--3xa.example"),
+ # Devanagari ka, virama, ZERO WIDTH JOINER, ssa; IDNA 2003 drops the ZWJ
+ ("http://\u0915\u094d\u200d\u0937.example/x", "xn--11b2ezcw70k.example"),
+]
+
+
[email protected](("url", "wire_host"), _WIRE_FORMS)
+def test_check_url_resolves_the_name_httpx_connects_to(url, wire_host):
+ assert httpx.URL(url).raw_host.decode("ascii") == wire_host
+ with patch("hyperresearch.web.safe_http.socket.getaddrinfo") as gai:
+ gai.return_value = [(socket.AF_INET, None, None, "", ("8.8.8.8", 0))]
+ check_url(url)
+ assert [c.args[0] for c in gai.call_args_list] == [wire_host]
+
+
+def test_safe_get_resolves_the_name_each_hop_requests():
+ """Every hop, the entry URL included, is checked on the host its request
+ is sent to."""
+ requested: list[str] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ requested.append(request.url.raw_host.decode("ascii"))
+ if request.url.path == "/start":
+ location = "http://ς.example/paper".encode()
+ return httpx.Response(302, headers=[(b"location", location)])
+ return httpx.Response(200, content=b"ok")
+
+ with patch("hyperresearch.web.safe_http.socket.getaddrinfo") as gai:
+ gai.return_value = [(socket.AF_INET, None, None, "", ("8.8.8.8", 0))]
+ resp = safe_get(
+ "http://straße.example/start", max_bytes=1024,
+ transport=httpx.MockTransport(handler),
+ )
+ assert resp.content == b"ok"
+ assert requested == ["xn--strae-oqa.example", "xn--3xa.example"]
+ assert [c.args[0] for c in gai.call_args_list] == requested
+
+
+# ---------------------------------------------------------------------------
+# Exception containment: check_url raises SafeHTTPError and nothing else, so
+# the batch fetch path's per-URL handler can skip a bad URL
+# ---------------------------------------------------------------------------
+
+
+def test_check_url_malformed_ipv6_brackets_raise_safe_error():
+ """urlparse() itself raises ValueError("Invalid IPv6 URL") here."""
+ with pytest.raises(SafeHTTPError, match="unparseable"):
+ check_url("http://[::1/x")
+
+
+def test_check_url_overlong_label_raises_safe_error():
+ """getaddrinfo raises UnicodeError, not gaierror, when IDNA encoding
+ rejects a 300-character label. No lookup is sent."""
+ with pytest.raises(SafeHTTPError, match="DNS lookup failed"):
+ check_url("http://" + "a" * 300 + ".com/x")
+
+
+ "url",
+ [
+ "http://8.8.8.8/pa\udcffper.pdf",
+ "http://8.8.8.8/x?q=\udcff",
+ "http://8.8.8.8/x#\ud800",
+ ],
+)
+def test_check_url_unencodable_url_raises_safe_error(url):
+ """A lone surrogate (what a non-UTF-8 byte in a command-line argument
+ becomes) makes httpx.URL raise UnicodeEncodeError."""
+ with pytest.raises(SafeHTTPError, match="unparseable"):
+ check_url(url)
+
+
# ---------------------------------------------------------------------------
# allow_private_hosts entry hygiene
# ---------------------------------------------------------------------------
@@ -429,3 +610,17 @@ def test_allowlist_bracketed_ipv6_entry_admits_the_literal():
"""A bracketed IPv6 entry (URL notation) parses as the address itself
instead of becoming a hostname that never matches."""
check_url("http://[::1]/", allow_private_hosts=("[::1]",))
+
+
[email protected]("url", [" http://8.8.8.8/", "\t http://8.8.8.8/", "\x00http://8.8.8.8/"])
+def test_check_url_strips_leading_whitespace_as_urlparse_does(url):
+ """urlparse drops leading C0 controls and spaces; httpx alone reads the
+ URL as relative and finds no host. The gate checks the same host both
+ see instead of looking up an empty name."""
+ check_url(url)
+
+
[email protected]("url", [" http://127.0.0.1/", "\x0bhttp://10.0.0.1/"])
+def test_check_url_leading_whitespace_still_ip_gated(url):
+ with pytest.raises(SafeHTTPError, match="non-public address"):
+ check_url(url)