Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
fix: the builtin provider raises CertVerificationError on a bad TLS certificate (#130)
Francois du Plessisd72267fa2a7d
diff --git a/src/hyperresearch/web/builtin.py b/src/hyperresearch/web/builtin.pyindex 2ecf7b7..cf5b63e 100644--- a/src/hyperresearch/web/builtin.py+++ b/src/hyperresearch/web/builtin.py@@ -113,13 +113,25 @@ class BuiltinProvider:def _get(self, url: str):"""Download URL through the SSRF gate in :mod:`hyperresearch.web.safe_http`."""- from hyperresearch.web.safe_http import safe_get+ from hyperresearch.web.pdf import _is_cert_error+ from hyperresearch.web.safe_http import CertVerificationError, safe_get- resp = safe_get(- url,- max_bytes=self._settings.max_html_bytes,- allow_private_hosts=self._settings.allow_private_hosts,- )+ try:+ resp = safe_get(+ url,+ max_bytes=self._settings.max_html_bytes,+ allow_private_hosts=self._settings.allow_private_hosts,+ )+ except Exception as exc:+ # Raise a certificate failure as its own type, the way the PDF lane+ # and the crawl4ai provider do, instead of letting the raw+ # httpx.ConnectError through looking like any other failed fetch.+ # This lane always verifies TLS; pdf_verify_tls covers PDFs only.+ if _is_cert_error(exc):+ raise CertVerificationError(+ f"certificate verification failed for {url!r}: {exc}"+ ) from exc+ raiseif resp.status_code >= 400:raise RuntimeError(f"HTTP {resp.status_code} fetching {url}")return respdiff --git a/tests/test_web/test_builtin_cert_refusal.py b/tests/test_web/test_builtin_cert_refusal.pynew file mode 100644index 0000000..d58a471--- /dev/null+++ b/tests/test_web/test_builtin_cert_refusal.py@@ -0,0 +1,229 @@+"""The builtin provider refuses a bad TLS certificate as CertVerificationError.++The PDF lane (`web.pdf.safe_get_pdf`) and the crawl4ai provider already raise+`CertVerificationError` on a certificate failure. The builtin provider's HTML+lane did not: `safe_get` let the raw `httpx.ConnectError` escape, so the same+refusal surfaced as an opaque ssl string and callers could not tell it apart+from a refused connection or a DNS failure.++Two layers here. The unit tests fake the httpx error chain at `safe_get`. The+end-to-end tests run a real TLS server on loopback with a self-signed+certificate, which also checks the error-chain walk against the nesting httpx+produces today, and count completed handshakes: a client that verifies never+completes one, so a nonzero count means some lane retried without+verification.+"""++from __future__ import annotations++import datetime+import ipaddress+import json+import socket+import ssl+import threading++import httpx+import pytest+from typer.testing import CliRunner++from hyperresearch.cli import app+from hyperresearch.core.config import FetchSettings+from hyperresearch.web.builtin import BuiltinProvider+from hyperresearch.web.safe_http import CertVerificationError++runner = CliRunner()+++def _cert_error() -> Exception:+ err = httpx.ConnectError("TLS handshake failed")+ err.__cause__ = ssl.SSLCertVerificationError("CERTIFICATE_VERIFY_FAILED")+ return err+++def test_cert_failure_raises_cert_verification_error(monkeypatch):+ calls: list[bool | str] = []++ def fake_safe_get(url, *, max_bytes, timeout=None, headers=None, verify=True,+ allow_private_hosts=()):+ calls.append(verify)+ raise _cert_error()++ monkeypatch.setattr("hyperresearch.web.safe_http.safe_get", fake_safe_get)++ with pytest.raises(CertVerificationError, match="certificate verification failed"):+ BuiltinProvider().fetch("https://broken-cert.example.edu/page")++ assert calls == [True], "exactly one verified attempt, no unverified retry"+++def test_non_cert_connect_error_propagates_untranslated(monkeypatch):+ def fake_safe_get(url, *, max_bytes, timeout=None, headers=None, verify=True,+ allow_private_hosts=()):+ raise httpx.ConnectError("connection refused")++ monkeypatch.setattr("hyperresearch.web.safe_http.safe_get", fake_safe_get)++ with pytest.raises(httpx.ConnectError, match="connection refused"):+ BuiltinProvider().fetch("https://down.example.com/page")+++# ---------------------------------------------------------------------------+# End to end: a real self-signed TLS server on loopback+# ---------------------------------------------------------------------------+++def self_signed_server(tmp_path):+ """HTTPS server on 127.0.0.1 with a self-signed cert. Yields (base_url, stats)."""+ pytest.importorskip("cryptography")+ from cryptography import x509+ from cryptography.hazmat.primitives import hashes, serialization+ from cryptography.hazmat.primitives.asymmetric import ec+ from cryptography.x509.oid import NameOID++ key = ec.generate_private_key(ec.SECP256R1())+ name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")])+ now = datetime.datetime.now(datetime.UTC)+ cert = (+ x509.CertificateBuilder()+ .subject_name(name)+ .issuer_name(name)+ .public_key(key.public_key())+ .serial_number(x509.random_serial_number())+ .not_valid_before(now - datetime.timedelta(days=1))+ .not_valid_after(now + datetime.timedelta(days=1))+ .add_extension(+ x509.SubjectAlternativeName([x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]),+ critical=False,+ )+ .sign(key, hashes.SHA256())+ )+ cert_path = tmp_path / "cert.pem"+ key_path = tmp_path / "key.pem"+ cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM))+ key_path.write_bytes(+ key.private_bytes(+ serialization.Encoding.PEM,+ serialization.PrivateFormat.PKCS8,+ serialization.NoEncryption(),+ )+ )+ ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)+ ctx.load_cert_chain(cert_path, key_path)++ stats = {"connections": 0, "handshakes_completed": 0}+ body = b"<html><title>served</title><body>" + b"page text " * 200 + b"</body></html>"+ listener = socket.socket()+ listener.bind(("127.0.0.1", 0))+ listener.listen(8)+ listener.settimeout(0.05) # so the accept loop notices `stop` promptly+ stop = threading.Event()++ def serve() -> None:+ while not stop.is_set():+ try:+ conn, _ = listener.accept()+ except TimeoutError:+ continue+ except OSError:+ return+ conn.settimeout(5)+ stats["connections"] += 1+ try:+ with ctx.wrap_socket(conn, server_side=True) as tls:+ stats["handshakes_completed"] += 1+ tls.recv(4096)+ tls.sendall(+ b"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n"+ b"Content-Length: %d\r\nConnection: close\r\n\r\n" % len(body)+ + body+ )+ except (OSError, ssl.SSLError):+ conn.close()++ thread = threading.Thread(target=serve, daemon=True)+ thread.start()+ try:+ yield f"https://127.0.0.1:{listener.getsockname()[1]}", stats+ finally:+ stop.set()+ thread.join(timeout=5)+ listener.close()+++def test_real_self_signed_cert_raises_cert_verification_error(self_signed_server):+ base, stats = self_signed_server+ prov = BuiltinProvider(FetchSettings(allow_private_hosts=("127.0.0.1",)))++ with pytest.raises(CertVerificationError, match="certificate verification failed"):+ prov.fetch(f"{base}/page")++ assert stats["connections"] == 1+ assert stats["handshakes_completed"] == 0+++def cert_vault(tmp_path, monkeypatch, self_signed_server):+ """A builtin-provider vault that may reach loopback, with the open-access+ rescue armed to hand back the refused host itself: the worst case for a+ lane that might retry a cert-refused source."""+ base, stats = self_signed_server+ root = tmp_path / "kb"+ assert runner.invoke(app, ["init", str(root), "--name", "Cert Test"]).exit_code == 0+ cfg = root / ".hyperresearch" / "config.toml"+ text = cfg.read_text(encoding="utf-8")+ assert text.count("allow_private_hosts = []") == 1+ cfg.write_text(+ text.replace("allow_private_hosts = []", 'allow_private_hosts = ["127.0.0.1"]'),+ encoding="utf-8",+ )++ from hyperresearch.core import oa, scholar++ rescues: list[str] = []++ def candidates(conn, doi, ttl_days, email=None, prefer_published=False):+ rescues.append(doi)+ yield oa.OALocation(url=f"{base}/open-copy", resolver="unpaywall", kind="page")++ # A doi.org link whose publisher fails TLS carries its DOI in the URL; a+ # loopback URL cannot, so supply the DOI directly.+ monkeypatch.setattr(scholar, "extract_doi", lambda url, raw_html=None, content=None: "10.1234/x")+ monkeypatch.setattr(oa, "iter_oa_candidates", candidates)+ monkeypatch.setattr(oa, "check_oa_url", lambda url: (True, ""))+ monkeypatch.chdir(root)+ return base, stats, rescues+++def test_cli_single_fetch_reports_the_cert_refusal(cert_vault):+ base, stats, rescues = cert_vault++ result = runner.invoke(app, ["fetch", f"{base}/paper", "--provider", "builtin", "-j"])++ payload = json.loads(result.stdout)+ assert result.exit_code == 1+ assert payload["ok"] is False+ assert "certificate verification failed" in payload["error"]+ assert rescues == ["10.1234/x"]+ # Two connections: the fetch, then the rescue's attempt at the same host.+ # Neither completed a handshake, so neither lane skipped verification.+ assert stats["connections"] == 2+ assert stats["handshakes_completed"] == 0+++def test_cli_batch_fetch_reports_the_cert_refusal(cert_vault):+ base, stats, rescues = cert_vault++ result = runner.invoke(app, ["fetch-batch", f"{base}/paper", "--provider", "builtin", "-j"])++ payload = json.loads(result.stdout)+ assert result.exit_code == 0+ failed = payload["data"]["failed_urls"]+ assert [f["url"] for f in failed] == [f"{base}/paper"]+ assert "certificate verification failed" in failed[0]["error"]+ assert rescues == ["10.1234/x"]+ # Two connections: the fetch, then the rescue's attempt at the same host.+ # Neither completed a handshake, so neither lane skipped verification.+ assert stats["connections"] == 2+ assert stats["handshakes_completed"] == 0