Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
security: SSRF gate resolves the host httpx connects to; allowlist matches its spellings (#161)
Francois du Plessis545946ec651a
diff --git a/pyproject.toml b/pyproject.tomlindex 29aa4c2..3517a6d 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -32,6 +32,7 @@ dependencies = ["Crawl4AI>=0.7.3","pymupdf>=1.24","httpx>=0.27",+ "idna",][project.optional-dependencies]diff --git a/src/hyperresearch/web/safe_http.py b/src/hyperresearch/web/safe_http.pyindex 309dd78..e104e5b 100644--- a/src/hyperresearch/web/safe_http.py+++ b/src/hyperresearch/web/safe_http.py@@ -32,6 +32,7 @@ from dataclasses import dataclass, fieldfrom urllib.parse import urlparseimport httpx+import idna_ALLOWED_SCHEMES = frozenset({"http", "https"})@@ -118,6 +119,30 @@ def _is_blocked_ip(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:)+def _normalize_hostname(host: str) -> str:+ """Canonical form for allowlist matching: lowercase, ACE, no trailing dot.++ A trailing dot marks the name as absolute: the same host as the bare name+ whenever the resolver looks that up as absolute, but a different string+ to an exact match. IDN hosts arrive as unicode or punycode depending on+ who typed them. Allowlist matching must not depend on which spelling the+ URL or the config file used.++ A non-ASCII name goes through idna.encode, the IDNA 2008 encoder httpx+ applies to a URL host, so it becomes the name httpx would connect to+ (labels split on the full stops U+3002, U+FF0E and U+FF61 as well as+ "."), and a name httpx would refuse in a URL raises idna.IDNAError. The+ stdlib "idna" codec is IDNA 2003 and maps some characters away: it+ encodes "straße" as "strasse", a different, separately registrable name+ from the "xn--strae-oqa" that httpx connects to. Matching through that+ codec would let an allowlist entry for one admit the other.+ """+ host = host.lower()+ if not host.isascii():+ host = idna.encode(host).decode("ascii")+ return host.rstrip(".")++def _resolve(host: str) -> list[ipaddress.IPv4Address | ipaddress.IPv6Address]:try:return [ipaddress.ip_address(host.strip("[]"))]@@ -125,7 +150,11 @@ def _resolve(host: str) -> list[ipaddress.IPv4Address | ipaddress.IPv6Address]:passtry:infos = socket.getaddrinfo(host, None)- except socket.gaierror as exc:+ except (UnicodeError, OSError) as exc:+ # gaierror is an OSError. UnicodeError covers names the IDNA codec+ # rejects while getaddrinfo encodes them (a 300-character label, for+ # one). Anything but SafeHTTPError escapes the batch fetch path's+ # per-URL handler and fails the whole batch on one hostile hostname.raise SafeHTTPError(f"DNS lookup failed for {host!r}: {exc}") from excaddrs: list[ipaddress.IPv4Address | ipaddress.IPv6Address] = []for info in infos:@@ -143,7 +172,8 @@ def _parse_allowlist(An entry that parses as an IP network (a bare address counts as a/32 or /128) matches by resolved address; anything else matches the- URL hostname exactly, case-insensitively. A malformed entry raises+ URL hostname after both are normalized (case, trailing root dot,+ IDN/punycode spelling). A malformed entry raises:class:`SafeHTTPError` naming it — silently ignoring one would turna typo into a lockout that looks like an SSRF refusal."""@@ -162,6 +192,14 @@ def _parse_allowlist(raise SafeHTTPError(f"invalid allow_private_hosts entry {entry!r}: not a valid CIDR") from None+ if "%" in entry:+ # Only a scoped IPv6 literal (fe80::1%eth0) legitimately+ # carries '%', and those parsed as networks above. As a+ # hostname the entry could never match.+ raise SafeHTTPError(+ f"invalid allow_private_hosts entry {entry!r}: '%' is only "+ "valid as an IPv6 scope id on an address literal"+ )if ":" in entry:# host:port. URL hostnames never carry the port, so the entry# would become a hostname that can never match: a silent dead@@ -170,10 +208,23 @@ def _parse_allowlist(f"invalid allow_private_hosts entry {entry!r}: entries are ""hostnames or CIDRs; drop the port")- hostnames.add(entry.lower())+ try:+ hostnames.add(_normalize_hostname(entry))+ except UnicodeError as exc: # idna.IDNAError is a UnicodeError+ # httpx refuses this host in a URL, so the entry could never+ # match: the same silent dead entry as above.+ raise SafeHTTPError(+ f"invalid allow_private_hosts entry {entry!r}: not a valid "+ f"IDNA hostname ({exc})"+ ) from Nonereturn frozenset(hostnames), networks+# What urlparse strips from the start of a URL (urllib.parse's+# _WHATWG_C0_CONTROL_OR_SPACE, a private name).+_C0_CONTROL_OR_SPACE = "".join(chr(c) for c in range(0x20)) + " "++def check_url(url: str, allow_private_hosts: tuple[str, ...] = ()) -> None:"""Validate the URL is safe to fetch. Raises :class:`SafeHTTPError`.@@ -186,17 +237,43 @@ def check_url(url: str, allow_private_hosts: tuple[str, ...] = ()) -> None:made afterwards re-resolves it (see the module docstring on DNSrebinding)."""- parsed = urlparse(url)+ try:+ parsed = urlparse(url)+ hostname = parsed.hostname+ except ValueError as exc:+ # urlparse raises ValueError for some malformed netlocs: unbalanced+ # IPv6 brackets give "Invalid IPv6 URL". A bare ValueError escapes+ # the batch fetch path's SafeHTTPError-only per-URL handler.+ raise SafeHTTPError(f"refusing unparseable URL {url!r}: {exc}") from excif parsed.scheme.lower() not in _ALLOWED_SCHEMES:raise SafeHTTPError(f"refusing URL with scheme {parsed.scheme!r}; only http/https are allowed")- if not parsed.hostname:+ if not hostname:+ raise SafeHTTPError(f"refusing URL with no hostname: {url!r}")+ # Check the name the connection will look up, not urlparse's spelling of+ # it. httpx sends an IDN host in its IDNA 2008 ASCII form, while+ # getaddrinfo encodes a non-ASCII name with the stdlib "idna" codec (IDNA+ # 2003), which maps some characters away: "straße" would be looked up as+ # "strasse", a different name from the "xn--strae-oqa" httpx connects to.+ #+ # urlparse drops leading C0 controls and spaces (the WHATWG rule); httpx+ # reads " http://host/" as a relative URL with no host. Strip the same+ # characters so both parsers see one host, and refuse a URL httpx still+ # finds no host in rather than look up an empty name.+ try:+ hostname = httpx.URL(url.lstrip(_C0_CONTROL_OR_SPACE)).raw_host.decode("ascii")+ except (httpx.InvalidURL, UnicodeError) as exc:+ # UnicodeError: httpx cannot encode a lone surrogate in the path,+ # query or fragment, which is what a non-UTF-8 byte in a command-line+ # argument becomes.+ raise SafeHTTPError(f"refusing unparseable URL {url!r}: {exc}") from exc+ if not hostname:raise SafeHTTPError(f"refusing URL with no hostname: {url!r}")allowed_hosts, allowed_nets = _parse_allowlist(allow_private_hosts)- if parsed.hostname.lower() in allowed_hosts:+ if _normalize_hostname(hostname) in allowed_hosts:return- addrs = _resolve(parsed.hostname)+ addrs = _resolve(hostname)if not addrs:raise SafeHTTPError(f"refusing URL {url!r}: hostname did not resolve")for addr in addrs:@@ -204,7 +281,7 @@ def check_url(url: str, allow_private_hosts: tuple[str, ...] = ()) -> None:addr.version == net.version and addr in net for net in allowed_nets):raise SafeHTTPError(- f"refusing URL {url!r}: hostname {parsed.hostname} resolves to "+ f"refusing URL {url!r}: hostname {hostname} resolves to "f"non-public address {addr}")diff --git a/tests/test_core/test_oa_recovery.py b/tests/test_core/test_oa_recovery.pyindex 266850e..6dd4597 100644--- a/tests/test_core/test_oa_recovery.py+++ b/tests/test_core/test_oa_recovery.py@@ -131,7 +131,9 @@ class TestCheckOaUrl:monkeypatch.setattr(socket, "getaddrinfo", boom)ok, reason = oa.check_oa_url("https://nowhere.example.org/p.pdf")- assert ok is False and "DNS resolution failed" in reason+ # check_url turns resolver errors into SafeHTTPError, so the refusal+ # carries the gate's reason.+ assert ok is False and "DNS lookup failed" in reasonclass TestNeedsRecovery:diff --git a/tests/test_web/test_fetch_many_gate.py b/tests/test_web/test_fetch_many_gate.pyindex 95a5cd3..ec26a62 100644--- a/tests/test_web/test_fetch_many_gate.py+++ b/tests/test_web/test_fetch_many_gate.py@@ -46,6 +46,37 @@ def _run(coro):return box["result"]+def test_fetch_many_skips_malformed_url_and_fetches_the_rest(monkeypatch, caplog):+ """A URL check_url cannot parse must be skipped like a refused one. urlparse+ raises ValueError for unbalanced IPv6 brackets, and httpx.URL raises+ UnicodeEncodeError for a lone surrogate (a non-UTF-8 byte in a+ command-line argument becomes one); if either escapes check_url as+ anything but SafeHTTPError, the per-URL handler in _fetch_many_async+ does not catch it and every other URL in the batch is lost with it."""+ fetched: list[str] = []++ def fake_fetch_pdf(url, settings=None):+ fetched.append(url)+ return WebResult(url=url, title="ok", content="pdf text")++ monkeypatch.setattr(provider, "_fetch_pdf", fake_fetch_pdf)++ inst = provider.Crawl4AIProvider.__new__(provider.Crawl4AIProvider)+ inst._settings = provider.FetchSettings()++ urls = [+ "http://[::1/broken.pdf",+ "http://8.8.8.8/pa\udcffper.pdf",+ "http://8.8.8.8/paper.pdf",+ ]+ with caplog.at_level(logging.WARNING, logger="hyperresearch.web"):+ results = _run(inst._fetch_many_async(urls))++ assert [r.url for r in results] == ["http://8.8.8.8/paper.pdf"]+ assert fetched == ["http://8.8.8.8/paper.pdf"]+ assert "refused batch fetch" in caplog.text++def test_fetch_many_skips_refused_urls_and_fetches_the_rest(monkeypatch, caplog):fetched: list[str] = []diff --git a/tests/test_web/test_safe_http.py b/tests/test_web/test_safe_http.pyindex fa9ba5c..488fbd9 100644--- a/tests/test_web/test_safe_http.py+++ b/tests/test_web/test_safe_http.py@@ -411,6 +411,187 @@ def test_safe_get_redirect_to_allowlisted_private_host_is_followed():assert resp.content == b"mirror content"+# ---------------------------------------------------------------------------+# Hostname spellings: allowlist matching follows DNS equivalence, not string+# equality. The resolver is stubbed to answer with a private address, so a+# spelling that misses its entry is refused rather than looked up for real.+# ---------------------------------------------------------------------------+++ ("url", "entry"),+ [+ # A trailing dot marks an absolute name: same host when looked up as one.+ ("http://mirror.internal./x", "mirror.internal"),+ ("http://mirror.internal/x", "mirror.internal."),+ ("http://Mirror.INTERNAL./x", "mirror.internal"),+ # IDN hosts arrive as unicode or punycode depending on who typed them.+ ("http://münchen.example/x", "xn--mnchen-3ya.example"),+ ("http://xn--mnchen-3ya.example/x", "münchen.example"),+ ],+)+def test_allowlist_hostname_matches_dns_equivalent_spelling(url, entry):+ with patch("hyperresearch.web.safe_http.socket.getaddrinfo") as gai:+ gai.return_value = [(socket.AF_INET, None, None, "", ("10.0.0.5", 0))]+ check_url(url, allow_private_hosts=(entry,))+++ ("url", "entry"),+ [+ ("http://strasse.example/x", "straße.example"),+ ("http://straße.example/x", "strasse.example"),+ ],+)+def test_allowlist_hostname_does_not_match_idna2003_mapping(url, entry):+ """IDNA 2003 (the stdlib "idna" codec) maps "straße" to "strasse", but+ httpx sends "xn--strae-oqa": two separately registrable names. An entry+ for one must not admit the other."""+ with patch("hyperresearch.web.safe_http.socket.getaddrinfo") as gai:+ gai.return_value = [(socket.AF_INET, None, None, "", ("10.0.0.5", 0))]+ with pytest.raises(SafeHTTPError, match="non-public address"):+ check_url(url, allow_private_hosts=(entry,))+++def test_allowlist_entry_with_ideographic_full_stop_names_the_dotted_host():+ """IDNA and httpx separate labels on U+3002, so this entry names+ "mirror.internal", not the single label its punycode would spell."""+ entry = "mirror\u3002internal"+ one_label = "xn--" + entry.encode("punycode").decode("ascii")+ with patch("hyperresearch.web.safe_http.socket.getaddrinfo") as gai:+ gai.return_value = [(socket.AF_INET, None, None, "", ("10.0.0.5", 0))]+ check_url("http://mirror.internal/x", allow_private_hosts=(entry,))+ with pytest.raises(SafeHTTPError, match="non-public address"):+ check_url(f"http://{one_label}/x", allow_private_hosts=(entry,))++[email protected]("entry", ["fe80::1%eth0", "[fe80::1%eth0]"])+def test_allowlist_scoped_ipv6_entry_parses_as_its_address(entry):+ """A scoped entry is an address, not a hostname, and matches by address+ bits whatever the scope."""+ import ipaddress++ from hyperresearch.web.safe_http import _parse_allowlist++ hostnames, networks = _parse_allowlist((entry,))+ assert hostnames == frozenset()+ assert len(networks) == 1+ assert ipaddress.ip_address("fe80::1") in networks[0]+++def test_allowlist_scoped_ipv6_url_allowed_by_bracketed_scoped_entry():+ check_url("http://[fe80::1%eth0]/x", allow_private_hosts=("[fe80::1%eth0]",))+++def test_scoped_ipv6_url_refused_without_entry():+ with pytest.raises(SafeHTTPError, match="non-public address"):+ check_url("http://[fe80::1%eth0]/x")+++def test_allowlist_percent_in_non_address_entry_is_a_loud_error():+ with pytest.raises(SafeHTTPError, match="scope id"):+ check_url("http://8.8.8.8/", allow_private_hosts=("my%host",))+++ "entry",+ [+ "\uff45xample.com", # fullwidth "e"+ "mu\u0308nchen.example", # NFD: "u" plus a combining diaeresis+ "ex\u00adample.com", # soft hyphen+ "\u200b.example", # zero-width space+ ],+)+def test_allowlist_entry_httpx_would_refuse_is_a_loud_error(entry):+ """httpx refuses each of these hosts in a URL (IDNA 2008 rejects them), so+ the entry could never match and must not be accepted silently."""+ with pytest.raises(SafeHTTPError, match="not a valid IDNA hostname"):+ check_url("http://8.8.8.8/", allow_private_hosts=(entry,))+++# ---------------------------------------------------------------------------+# The gate looks up the name the connection will. httpx sends an IDN host in+# its IDNA 2008 form, while getaddrinfo encodes a unicode name with the stdlib+# IDNA 2003 codec, which maps the deviation characters (ß, ς, ZWJ, ZWNJ) to a+# different name. A zone owner could point the two at different addresses.+# ---------------------------------------------------------------------------+++_WIRE_FORMS = [+ ("http://straße.example/x", "xn--strae-oqa.example"), # IDNA 2003: strasse+ # Final sigma; IDNA 2003 folds it to the medial sigma, "xn--4xa".+ ("http://ς.example/x", "xn--3xa.example"),+ # Devanagari ka, virama, ZERO WIDTH JOINER, ssa; IDNA 2003 drops the ZWJ+ ("http://\u0915\u094d\u200d\u0937.example/x", "xn--11b2ezcw70k.example"),+]++[email protected](("url", "wire_host"), _WIRE_FORMS)+def test_check_url_resolves_the_name_httpx_connects_to(url, wire_host):+ assert httpx.URL(url).raw_host.decode("ascii") == wire_host+ with patch("hyperresearch.web.safe_http.socket.getaddrinfo") as gai:+ gai.return_value = [(socket.AF_INET, None, None, "", ("8.8.8.8", 0))]+ check_url(url)+ assert [c.args[0] for c in gai.call_args_list] == [wire_host]+++def test_safe_get_resolves_the_name_each_hop_requests():+ """Every hop, the entry URL included, is checked on the host its request+ is sent to."""+ requested: list[str] = []++ def handler(request: httpx.Request) -> httpx.Response:+ requested.append(request.url.raw_host.decode("ascii"))+ if request.url.path == "/start":+ location = "http://ς.example/paper".encode()+ return httpx.Response(302, headers=[(b"location", location)])+ return httpx.Response(200, content=b"ok")++ with patch("hyperresearch.web.safe_http.socket.getaddrinfo") as gai:+ gai.return_value = [(socket.AF_INET, None, None, "", ("8.8.8.8", 0))]+ resp = safe_get(+ "http://straße.example/start", max_bytes=1024,+ transport=httpx.MockTransport(handler),+ )+ assert resp.content == b"ok"+ assert requested == ["xn--strae-oqa.example", "xn--3xa.example"]+ assert [c.args[0] for c in gai.call_args_list] == requested+++# ---------------------------------------------------------------------------+# Exception containment: check_url raises SafeHTTPError and nothing else, so+# the batch fetch path's per-URL handler can skip a bad URL+# ---------------------------------------------------------------------------+++def test_check_url_malformed_ipv6_brackets_raise_safe_error():+ """urlparse() itself raises ValueError("Invalid IPv6 URL") here."""+ with pytest.raises(SafeHTTPError, match="unparseable"):+ check_url("http://[::1/x")+++def test_check_url_overlong_label_raises_safe_error():+ """getaddrinfo raises UnicodeError, not gaierror, when IDNA encoding+ rejects a 300-character label. No lookup is sent."""+ with pytest.raises(SafeHTTPError, match="DNS lookup failed"):+ check_url("http://" + "a" * 300 + ".com/x")+++ "url",+ [+ "http://8.8.8.8/pa\udcffper.pdf",+ "http://8.8.8.8/x?q=\udcff",+ "http://8.8.8.8/x#\ud800",+ ],+)+def test_check_url_unencodable_url_raises_safe_error(url):+ """A lone surrogate (what a non-UTF-8 byte in a command-line argument+ becomes) makes httpx.URL raise UnicodeEncodeError."""+ with pytest.raises(SafeHTTPError, match="unparseable"):+ check_url(url)++# ---------------------------------------------------------------------------# allow_private_hosts entry hygiene# ---------------------------------------------------------------------------@@ -429,3 +610,17 @@ def test_allowlist_bracketed_ipv6_entry_admits_the_literal():"""A bracketed IPv6 entry (URL notation) parses as the address itselfinstead of becoming a hostname that never matches."""check_url("http://[::1]/", allow_private_hosts=("[::1]",))++[email protected]("url", [" http://8.8.8.8/", "\t http://8.8.8.8/", "\x00http://8.8.8.8/"])+def test_check_url_strips_leading_whitespace_as_urlparse_does(url):+ """urlparse drops leading C0 controls and spaces; httpx alone reads the+ URL as relative and finds no host. The gate checks the same host both+ see instead of looking up an empty name."""+ check_url(url)++[email protected]("url", [" http://127.0.0.1/", "\x0bhttp://10.0.0.1/"])+def test_check_url_leading_whitespace_still_ip_gated(url):+ with pytest.raises(SafeHTTPError, match="non-public address"):+ check_url(url)