Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
fix: give the builtin provider a PDF lane, and say why a PDF was declined (#124)
Jordan Gibbs9ed3232d6514
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ffe35ba..9c904de 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -2,6 +2,7 @@## [Unreleased]+- **The builtin provider fetches PDFs (#82, reported by @earldodd).** The PDF lane (`_is_pdf_url` / `_fetch_pdf`) lived inside the crawl4ai provider only, so a vault still on `provider = "builtin"` (the default until `hpr install` switches it) had no PDF handling at all: a direct `.pdf` link was decoded as HTML text and rejected by the junk gate as "Binary PDF garbage in content", identically for every mirror of the same document, and an arXiv `/abs/` link saved the 900-word abstract page instead of the paper. The lane now lives in `web/pdf.py` and both providers use it; the crawl4ai module keeps the old names. The builtin provider also detects a PDF from its bytes when the URL does not look like one. When the PDF lane declines a URL and the HTML fallback turns out to be junk, `hpr fetch -j` now says why the lane declined it (`PDF lane: HTTP 403`, `no extractable text layer`, ...) instead of the generic junk verdict, and the pymupdf document handle is closed on the exception path instead of leaking one per encrypted PDF.- **`serply` web provider.** Google organic results with page fetch through the same `SERPLY_API_KEY`; opt-in via `[web] provider = "serply"`, no new dependency.## [0.11.1] - 2026-09-11diff --git a/src/hyperresearch/cli/fetch.py b/src/hyperresearch/cli/fetch.pyindex 6e946d9..90c4fde 100644--- a/src/hyperresearch/cli/fetch.py+++ b/src/hyperresearch/cli/fetch.py@@ -230,6 +230,7 @@ def fetch(from hyperresearch.core.sync import compute_sync_plan, execute_syncfrom hyperresearch.core.vault import Vault, VaultErrorfrom hyperresearch.web.base import get_provider+ from hyperresearch.web.pdf import PDF_FAILURE_KEYtry:vault = Vault.discover()@@ -391,7 +392,11 @@ def fetch(item_id = _escalate_blocked(vault, url, reason, tags, suggested_by, utility_score,detail=junk_reason)escalated = f" Queued for browser-lane escalation (#{item_id})." if item_id else ""- msg = f"Skipped junk content from {url}: {junk_reason}.{escalated}"+ # A PDF that failed its own lane lands here as "binary garbage";+ # say why the PDF lane declined it, or the message is useless (#82).+ pdf_failure = result.metadata.get(PDF_FAILURE_KEY)+ pdf_note = f" PDF lane: {pdf_failure}." if pdf_failure else ""+ msg = f"Skipped junk content from {url}: {junk_reason}.{pdf_note}{escalated}"if json_output:output(error(msg, "JUNK_ESCALATED" if item_id else "JUNK_CONTENT"), json_mode=True)else:diff --git a/src/hyperresearch/core/oa.py b/src/hyperresearch/core/oa.pyindex 051ba1e..4d1b635 100644--- a/src/hyperresearch/core/oa.py+++ b/src/hyperresearch/core/oa.py@@ -796,9 +796,9 @@ def _try_candidates(vault, prov, doi: str, settings, *, fallback_title, beat_chaattempts += 1try:if loc.kind == "pdf":- from hyperresearch.web.crawl4ai_provider import _fetch_pdf+ from hyperresearch.web.pdf import fetch_pdf- recovered = _fetch_pdf(loc.url, vault.config.fetch)+ recovered = fetch_pdf(loc.url, vault.config.fetch)elif loc.kind == "jats":recovered = _fetch_jats(loc.url, fallback_title)elif loc.kind == "coretext":diff --git a/src/hyperresearch/web/builtin.py b/src/hyperresearch/web/builtin.pyindex b90a076..2ecf7b7 100644--- a/src/hyperresearch/web/builtin.py+++ b/src/hyperresearch/web/builtin.py@@ -1,4 +1,10 @@-"""Builtin web provider — SSRF-gated fetching via safe_http, beautifulsoup4 extraction if available."""+"""Builtin web provider — SSRF-gated fetching via safe_http, beautifulsoup4 extraction if available.++PDFs go through the shared lane in :mod:`hyperresearch.web.pdf`, the same one+the crawl4ai provider uses. Before that, a vault on this provider had no PDF+handling at all: the bytes were decoded as HTML and every PDF on every host+was rejected by the junk gate with the same generic message (#82).+"""from __future__ import annotations@@ -7,6 +13,13 @@ from datetime import UTC, datetimefrom html.parser import HTMLParserfrom hyperresearch.web.base import WebResult+from hyperresearch.web.pdf import (+ PDF_FAILURE_KEY,+ extract_pdf,+ failure_reason,+ fetch_pdf,+ is_pdf_url,+)class _TextExtractor(HTMLParser):@@ -58,15 +71,39 @@ class BuiltinProvider:self._settings = settings or FetchSettings()def fetch(self, url: str) -> WebResult:- html, final_url = self._download(url)+ # PDF detection by URL shape: download directly, extract with pymupdf.+ # A miss falls through to the HTML lane (the URL may be a landing page)+ # and the reason travels with the result for the junk gate to report.+ pdf_failure: str | None = None+ if is_pdf_url(url):+ result = fetch_pdf(url, self._settings)+ if result is not None:+ return result+ pdf_failure = failure_reason(url)++ resp = self._get(url)+ # Post-download PDF detection: a PDF behind a URL that does not look+ # like one (download?id=…, a redirect) is decoded here as text and+ # would otherwise be reported as binary garbage.+ if resp.content.startswith(b"%PDF-") or "application/pdf" in resp.headers.get("content-type", "").lower():+ result, pdf_failure = extract_pdf(+ resp.url, resp.content, self._settings, resp.headers.get("content-type", "")+ )+ if result is not None:+ return result++ html = resp.texttitle, content = self._extract(html)- return WebResult(- url=final_url,+ result = WebResult(+ url=resp.url,title=title,content=content,raw_html=html,fetched_at=datetime.now(UTC),)+ if pdf_failure:+ result.metadata[PDF_FAILURE_KEY] = pdf_failure+ return resultdef search(self, query: str, max_results: int = 5) -> list[WebResult]:raise NotImplementedError(@@ -74,9 +111,8 @@ class BuiltinProvider:"Use your agent's built-in search, then pipe URLs into 'hyperresearch fetch'.")- def _download(self, url: str) -> tuple[str, str]:- """Download URL, return (html, final_url). All requests go through the- SSRF gate in :mod:`hyperresearch.web.safe_http`."""+ def _get(self, url: str):+ """Download URL through the SSRF gate in :mod:`hyperresearch.web.safe_http`."""from hyperresearch.web.safe_http import safe_getresp = safe_get(@@ -86,6 +122,11 @@ class BuiltinProvider:)if resp.status_code >= 400:raise RuntimeError(f"HTTP {resp.status_code} fetching {url}")+ return resp++ def _download(self, url: str) -> tuple[str, str]:+ """Download URL, return (html, final_url)."""+ resp = self._get(url)return resp.text, resp.urldef _extract(self, html: str) -> tuple[str, str]:diff --git a/src/hyperresearch/web/crawl4ai_provider.py b/src/hyperresearch/web/crawl4ai_provider.pyindex 1a3a13b..94bc55e 100644--- a/src/hyperresearch/web/crawl4ai_provider.py+++ b/src/hyperresearch/web/crawl4ai_provider.py@@ -23,6 +23,13 @@ from crawl4ai.content_filter_strategy import PruningContentFilterfrom hyperresearch.core.config import FetchSettings, JunkGatesfrom hyperresearch.web.base import WebResult, is_binary_garbage+from hyperresearch.web.pdf import PDF_FAILURE_KEY+from hyperresearch.web.pdf import failure_reason as _pdf_failure_reason+from hyperresearch.web.pdf import fetch_pdf as _fetch_pdf+from hyperresearch.web.pdf import is_pdf_url as _is_pdf_url+from hyperresearch.web.pdf import (+ safe_get_pdf as _safe_get_pdf, # noqa: F401 # kept for callers that import it here+)# Fix Windows encoding before crawl4ai's managed browser tries to log Unicodeif sys.platform == "win32":@@ -63,22 +70,6 @@ def _run_coro(coro):return box["result"]-def _is_pdf_url(url: str) -> bool:- """Check if URL likely points to a PDF."""- from urllib.parse import urlparse-- parsed = urlparse(url.lower())- path = parsed.path- # Direct .pdf links- if path.endswith(".pdf"):- return True- # Common academic PDF patterns- if "/pdf/" in path or "/pdfs/" in path:- return True- # arXiv PDF links- return "arxiv.org" in parsed.netloc and ("/pdf/" in path or "/abs/" in path)--def _looks_like_binary(text: str, gates: JunkGates | None = None) -> bool:"""Check if extracted 'content' is actually binary garbage from a PDF."""if not text:@@ -119,196 +110,6 @@ def _smart_wait_js(settings: FetchSettings) -> str:)-_PYMUPDF_MISSING_LOGGED = False---def _pdf_log() -> logging.Logger:- return logging.getLogger("hyperresearch.pdf")---def _import_pymupdf():- """Import pymupdf, warning loudly (once) if it is unavailable.-- Without this warning a missing/broken pymupdf is invisible: every PDF falls- through to the browser lane, arrives as binary, and is discarded as junk —- across every domain at once, with nothing explaining why.- """- global _PYMUPDF_MISSING_LOGGED- try:- import pymupdf-- return pymupdf- except ImportError as exc:- if not _PYMUPDF_MISSING_LOGGED:- _PYMUPDF_MISSING_LOGGED = True- _pdf_log().error(- "pymupdf could not be imported (%s) — PDF text extraction is disabled, "- "so every PDF will be discarded as junk content. Reinstall it with "- "`pip install --force-reinstall pymupdf`.",- exc,- )- return None---_PDF_FETCH_HEADERS = {- "User-Agent": (- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "- "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"- ),- "Accept": "application/pdf,application/octet-stream;q=0.9,*/*;q=0.8",-}---def _is_cert_error(exc: Exception) -> bool:- import ssl-- # httpx nests the ssl error two causes deep (httpx.ConnectError ->- # httpcore.ConnectError -> SSLCertVerificationError), so walk the chain- # rather than checking only the direct cause. String match as fallback.- cause: BaseException | None = exc- for _ in range(5):- if cause is None:- break- if isinstance(cause, ssl.SSLCertVerificationError):- return True- cause = cause.__cause__- return "CERTIFICATE_VERIFY_FAILED" in str(exc)---def _safe_get_pdf(url: str, settings: FetchSettings):- """SSRF-gated, size-capped PDF download.-- TLS verification follows ``pdf_verify_tls`` (default on) with NO- automatic unverified retry: a MITM can serve a bad certificate- precisely to force such a retry, which would turn verify-by-default- into something the attacker controls. Mirrors with known-broken- certificates are handled by the explicit, user-declared- ``pdf_verify_tls = false`` opt-out.- """- from hyperresearch.web.safe_http import CertVerificationError, safe_get-- try:- return safe_get(url, max_bytes=settings.max_pdf_bytes,- timeout=settings.pdf_timeout_s,- headers=_PDF_FETCH_HEADERS,- verify=settings.pdf_verify_tls,- allow_private_hosts=settings.allow_private_hosts)- except Exception as exc:- if settings.pdf_verify_tls and _is_cert_error(exc):- # Refuse, but say how to opt out for a trusted cert-broken mirror.- # Raised as its own type: the browser lane runs with TLS errors- # ignored, so treating this like any failed PDF would hand the- # URL to a lane that amounts to an automatic unverified retry.- raise CertVerificationError(- f"certificate verification failed for {url!r}: {exc}. "- "If this host is a known cert-broken mirror you trust, set "- "pdf_verify_tls = false under [fetch] in config.toml."- ) from exc- raise---def _fetch_pdf(url: str, settings: FetchSettings | None = None) -> WebResult | None:- """Download a PDF and extract text using pymupdf. Returns None if extraction fails.-- Every failure path logs its reason. A silent None here is indistinguishable- from "this URL is not a PDF", which is what made missing-PDF failures so hard- to diagnose.- """- pymupdf = _import_pymupdf()- if pymupdf is None:- return None-- settings = settings or FetchSettings()-- from hyperresearch.web.safe_http import CertVerificationError, SafeHTTPError-- try:- # Convert arXiv abs links to PDF links- if "arxiv.org/abs/" in url:- url = url.replace("/abs/", "/pdf/")- if not url.endswith(".pdf"):- url += ".pdf"-- try:- resp = _safe_get_pdf(url, settings)- except CertVerificationError:- # Propagates: callers must not fold a cert refusal into the- # generic PDF-failed None, whose fallback is the TLS-ignoring- # browser lane.- raise- except SafeHTTPError as exc:- _pdf_log().warning("refused PDF fetch for %s: %s", url, exc)- return None-- if resp.status_code != 200:- _pdf_log().warning("PDF fetch for %s returned HTTP %s", url, resp.status_code)- return None-- pdf_bytes = resp.content- content_type = resp.headers.get("content-type", "")-- # Magic bytes are authoritative. Servers mislabel PDFs as octet-stream,- # and plenty of PDF URLs carry no .pdf suffix, so trusting the header or- # the URL shape alone silently drops real PDFs.- if not pdf_bytes.startswith(b"%PDF-"):- _pdf_log().warning(- "PDF fetch for %s did not return PDF data (content-type=%r, first bytes=%r)",- url, content_type, pdf_bytes[:8],- )- return None-- if len(pdf_bytes) < settings.min_pdf_bytes:- _pdf_log().warning("PDF fetch for %s returned only %d bytes", url, len(pdf_bytes))- return None-- doc = pymupdf.open(stream=pdf_bytes, filetype="pdf")-- # Extract text from all pages- pages = []- for page in doc:- text = page.get_text("text")- if text.strip():- pages.append(text)-- page_count = doc.page_count- doc.close()-- if not pages:- _pdf_log().warning(- "PDF at %s has %d page(s) but no extractable text layer — "- "likely a scanned document requiring OCR.", url, page_count,- )- return None-- # Build markdown from extracted text- full_text = "\n\n---\n\n".join(pages)- title = ""- # Try to get title from first page (first non-empty line)- for line in pages[0].split("\n"):- line = line.strip()- if len(line) > 10:- title = line- break-- return WebResult(- url=url,- title=title or f"PDF: {url.split('/')[-1]}",- content=full_text,- fetched_at=datetime.now(UTC),- metadata={"content_type": "application/pdf", "pages": len(pages)},- raw_bytes=pdf_bytes,- raw_content_type="application/pdf",- )-- except CertVerificationError:- # Second guard: the catch-all below must not convert a cert refusal- # into the generic None either.- raise- except Exception as e:- _pdf_log().warning("PDF extraction failed for %s: %s", url, e)- return None--def _check_final_url(entry_url: str, final_url: str | None, settings: FetchSettings) -> None:"""Re-validate the URL the browser actually ended up on.@@ -413,25 +214,32 @@ class Crawl4AIProvider:check_url(url, self._settings.allow_private_hosts)# PDF detection: fetch directly with httpx, extract text with pymupdf+ pdf_failure: str | None = Noneif _is_pdf_url(url):result = _fetch_pdf(url, self._settings)if result is not None:return result# Fallback to browser if PDF fetch failed (might be a landing page, not actual PDF)+ pdf_failure = _pdf_failure_reason(url)# When visible + profile: use Playwright directly (crawl4ai managed browser ignores headless=False)if not self._headless and self._data_dir:- return _run_coro(self._fetch_visible(url))-- result = _run_coro(self._fetch_async(url))-- # Post-fetch PDF detection: if the browser got binary garbage (PDF served- # inline without proper content-type handling), re-fetch as a direct PDF download.- if result.content and _looks_like_binary(result.content, self._gates):- pdf_result = _fetch_pdf(url, self._settings)- if pdf_result is not None:- return pdf_result+ result = _run_coro(self._fetch_visible(url))+ else:+ result = _run_coro(self._fetch_async(url))+ # Post-fetch PDF detection: if the browser got binary garbage (PDF served+ # inline without proper content-type handling), re-fetch as a direct PDF download.+ if result.content and _looks_like_binary(result.content, self._gates):+ pdf_result = _fetch_pdf(url, self._settings)+ if pdf_result is not None:+ return pdf_result+ pdf_failure = _pdf_failure_reason(url) or pdf_failure++ # The junk gate downstream only sees "binary garbage"; the reason the+ # PDF lane gave up is what the operator actually needs (#82).+ if pdf_failure:+ result.metadata[PDF_FAILURE_KEY] = pdf_failurereturn resultasync def _fetch_visible(self, url: str) -> WebResult:diff --git a/src/hyperresearch/web/pdf.py b/src/hyperresearch/web/pdf.pynew file mode 100644index 0000000..7afaa18--- /dev/null+++ b/src/hyperresearch/web/pdf.py@@ -0,0 +1,298 @@+"""PDF lane shared by every web provider.++Detects PDF URLs, downloads them through the SSRF gate, and extracts text+with pymupdf. This used to live inside the crawl4ai provider only, so a+vault on the builtin provider had no PDF handling at all: a direct .pdf+link was decoded as HTML text and rejected by the junk gate as "Binary PDF+garbage in content", identically for every mirror of the same document,+and an arXiv /abs/ link saved the abstract page instead of the paper (#82).++Every failure path here produces a reason string. Providers carry that+reason on the fallback result's ``metadata["pdf_failure"]`` so the CLI can+print it next to the junk verdict instead of a bare generic message.+"""++from __future__ import annotations++import logging+from datetime import UTC, datetime++from hyperresearch.core.config import FetchSettings+from hyperresearch.web.base import WebResult++# Key under which a provider records why the PDF lane declined a URL when it+# then falls back to the HTML lane. Read by `hpr fetch` when the fallback+# result turns out to be junk.+PDF_FAILURE_KEY = "pdf_failure"++_PYMUPDF_MISSING_LOGGED = False+++def pdf_log() -> logging.Logger:+ return logging.getLogger("hyperresearch.pdf")+++def is_pdf_url(url: str) -> bool:+ """Check if URL likely points to a PDF."""+ from urllib.parse import urlparse++ parsed = urlparse(url.lower())+ path = parsed.path+ # Direct .pdf links+ if path.endswith(".pdf"):+ return True+ # Common academic PDF patterns+ if "/pdf/" in path or "/pdfs/" in path:+ return True+ # arXiv PDF links+ return "arxiv.org" in parsed.netloc and ("/pdf/" in path or "/abs/" in path)+++def import_pymupdf():+ """Import pymupdf, warning loudly (once) if it is unavailable.++ Without this warning a missing/broken pymupdf is invisible: every PDF falls+ through to the HTML lane, arrives as binary, and is discarded as junk —+ across every domain at once, with nothing explaining why.+ """+ global _PYMUPDF_MISSING_LOGGED+ try:+ import pymupdf++ return pymupdf+ except ImportError as exc:+ if not _PYMUPDF_MISSING_LOGGED:+ _PYMUPDF_MISSING_LOGGED = True+ pdf_log().error(+ "pymupdf could not be imported (%s) — PDF text extraction is disabled, "+ "so every PDF will be discarded as junk content. Reinstall it with "+ "`pip install --force-reinstall pymupdf`.",+ exc,+ )+ return None+++PDF_FETCH_HEADERS = {+ "User-Agent": (+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "+ "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"+ ),+ "Accept": "application/pdf,application/octet-stream;q=0.9,*/*;q=0.8",+}+++def _is_cert_error(exc: Exception) -> bool:+ import ssl++ # httpx nests the ssl error two causes deep (httpx.ConnectError ->+ # httpcore.ConnectError -> SSLCertVerificationError), so walk the chain+ # rather than checking only the direct cause. String match as fallback.+ cause: BaseException | None = exc+ for _ in range(5):+ if cause is None:+ break+ if isinstance(cause, ssl.SSLCertVerificationError):+ return True+ cause = cause.__cause__+ return "CERTIFICATE_VERIFY_FAILED" in str(exc)+++def safe_get_pdf(url: str, settings: FetchSettings):+ """SSRF-gated, size-capped PDF download.++ TLS verification follows ``pdf_verify_tls`` (default on) with NO+ automatic unverified retry: a MITM can serve a bad certificate+ precisely to force such a retry, which would turn verify-by-default+ into something the attacker controls. Mirrors with known-broken+ certificates are handled by the explicit, user-declared+ ``pdf_verify_tls = false`` opt-out.+ """+ from hyperresearch.web.safe_http import CertVerificationError, safe_get++ try:+ return safe_get(url, max_bytes=settings.max_pdf_bytes,+ timeout=settings.pdf_timeout_s,+ headers=PDF_FETCH_HEADERS,+ verify=settings.pdf_verify_tls,+ allow_private_hosts=settings.allow_private_hosts)+ except Exception as exc:+ if settings.pdf_verify_tls and _is_cert_error(exc):+ # Refuse, but say how to opt out for a trusted cert-broken mirror.+ # Raised as its own type: the browser lane runs with TLS errors+ # ignored, so treating this like any failed PDF would hand the+ # URL to a lane that amounts to an automatic unverified retry.+ raise CertVerificationError(+ f"certificate verification failed for {url!r}: {exc}. "+ "If this host is a known cert-broken mirror you trust, set "+ "pdf_verify_tls = false under [fetch] in config.toml."+ ) from exc+ raise+++def pdf_url_for(url: str) -> str:+ """The URL the PDF lane actually downloads: arXiv abs links become pdf links."""+ if "arxiv.org/abs/" in url:+ url = url.replace("/abs/", "/pdf/")+ if not url.endswith(".pdf"):+ url += ".pdf"+ return url+++def extract_pdf(+ url: str,+ pdf_bytes: bytes,+ settings: FetchSettings | None = None,+ content_type: str = "",+) -> tuple[WebResult | None, str | None]:+ """Extract text from downloaded bytes. Returns (result, None) or (None, reason).++ Magic bytes are authoritative. Servers mislabel PDFs as octet-stream, and+ plenty of PDF URLs carry no .pdf suffix, so trusting the header or the URL+ shape alone silently drops real PDFs.+ """+ settings = settings or FetchSettings()+ log = pdf_log()++ pymupdf = import_pymupdf()+ if pymupdf is None:+ return None, "pymupdf is not importable"++ if not pdf_bytes.startswith(b"%PDF-"):+ reason = (+ f"did not return PDF data (content-type={content_type!r}, "+ f"first bytes={pdf_bytes[:8]!r})"+ )+ log.warning("PDF fetch for %s %s", url, reason)+ return None, reason++ if len(pdf_bytes) < settings.min_pdf_bytes:+ reason = f"returned only {len(pdf_bytes)} bytes"+ log.warning("PDF fetch for %s %s", url, reason)+ return None, reason++ try:+ doc = pymupdf.open(stream=pdf_bytes, filetype="pdf")+ except Exception as e:+ reason = f"pymupdf could not open the document: {e}"+ log.warning("PDF fetch for %s: %s", url, reason)+ return None, reason++ # Extract text from all pages. The handle is closed on the way out even+ # when page iteration raises (encrypted or damaged documents), which used+ # to leak one open document per failed PDF.+ pages = []+ try:+ for page in doc:+ text = page.get_text("text")+ if text.strip():+ pages.append(text)+ page_count = doc.page_count+ except Exception as e:+ reason = f"pymupdf failed while reading pages: {e}"+ log.warning("PDF fetch for %s: %s", url, reason)+ return None, reason+ finally:+ doc.close()++ if not pages:+ reason = (+ f"has {page_count} page(s) but no extractable text layer — "+ "likely a scanned document requiring OCR"+ )+ log.warning("PDF at %s %s.", url, reason)+ return None, reason++ # Build markdown from extracted text+ full_text = "\n\n---\n\n".join(pages)+ title = ""+ # Try to get title from first page (first non-empty line)+ for line in pages[0].split("\n"):+ line = line.strip()+ if len(line) > 10:+ title = line+ break++ return WebResult(+ url=url,+ title=title or f"PDF: {url.split('/')[-1]}",+ content=full_text,+ fetched_at=datetime.now(UTC),+ metadata={"content_type": "application/pdf", "pages": len(pages)},+ raw_bytes=pdf_bytes,+ raw_content_type="application/pdf",+ ), None+++def fetch_pdf_ex(+ url: str, settings: FetchSettings | None = None,+) -> tuple[WebResult | None, str | None]:+ """Download a PDF and extract its text. Returns (result, None) or (None, reason).++ The reason is a short phrase suitable for showing next to a junk verdict+ ("HTTP 403", "no extractable text layer"). A silent None here is+ indistinguishable from "this URL is not a PDF", which is what made+ missing-PDF failures so hard to diagnose.++ A certificate refusal propagates as ``CertVerificationError``: callers+ must not fold it into the generic failure, whose fallback is the+ TLS-ignoring browser lane.+ """+ from hyperresearch.web.safe_http import CertVerificationError, SafeHTTPError++ settings = settings or FetchSettings()+ log = pdf_log()++ if import_pymupdf() is None:+ return None, "pymupdf is not importable"++ url = pdf_url_for(url)+ try:+ try:+ resp = safe_get_pdf(url, settings)+ except CertVerificationError:+ raise+ except SafeHTTPError as exc:+ log.warning("refused PDF fetch for %s: %s", url, exc)+ return None, f"refused: {exc}"++ if resp.status_code != 200:+ log.warning("PDF fetch for %s returned HTTP %s", url, resp.status_code)+ return None, f"HTTP {resp.status_code}"++ return extract_pdf(url, resp.content, settings, resp.headers.get("content-type", ""))++ except CertVerificationError:+ # Second guard: the catch-all below must not convert a cert refusal+ # into the generic failure either.+ raise+ except Exception as e:+ log.warning("PDF extraction failed for %s: %s", url, e)+ return None, f"extraction failed: {e}"+++# Why the PDF lane last declined each URL, keyed by the URL as requested.+# Providers call `fetch_pdf` (the patchable, result-only entry point) and read+# the reason back through `failure_reason` when they fall back to HTML.+_FAILURES: dict[str, str] = {}+_FAILURES_MAX = 256+++def fetch_pdf(url: str, settings: FetchSettings | None = None) -> WebResult | None:+ """Download a PDF and extract text using pymupdf. Returns None if extraction fails.++ Result-only form of :func:`fetch_pdf_ex`. The failure reason is kept and+ can be read back with :func:`failure_reason`.+ """+ result, reason = fetch_pdf_ex(url, settings)+ if reason is not None:+ if len(_FAILURES) >= _FAILURES_MAX:+ _FAILURES.clear()+ _FAILURES[url] = reason+ else:+ _FAILURES.pop(url, None)+ return result+++def failure_reason(url: str) -> str | None:+ """Why the last `fetch_pdf` for this URL returned None, if it did."""+ return _FAILURES.get(url)diff --git a/tests/test_cli/test_oa_fetch.py b/tests/test_cli/test_oa_fetch.pyindex 791fb50..aca626d 100644--- a/tests/test_cli/test_oa_fetch.py+++ b/tests/test_cli/test_oa_fetch.py@@ -86,7 +86,7 @@ def vault_dir(tmp_path: Path, monkeypatch) -> Path:)from hyperresearch.core import oa, scholar- from hyperresearch.web import crawl4ai_provider+ from hyperresearch.web import pdf as pdf_lanemonkeypatch.setattr("hyperresearch.web.base.get_provider", lambda *a, **k: _AbstractOnlyProvider())monkeypatch.setattr(@@ -94,8 +94,8 @@ def vault_dir(tmp_path: Path, monkeypatch) -> Path:)monkeypatch.setattr(oa.socket, "getaddrinfo", lambda h, p: [(2, 1, 6, "", ("93.184.216.34", 0))])monkeypatch.setattr(- crawl4ai_provider,- "_fetch_pdf",+ pdf_lane,+ "fetch_pdf",lambda url, settings: WebResult(url=url, title="Widget Paper", content=FULL_TEXT),)return rootdiff --git a/tests/test_core/test_oa_core_resolver.py b/tests/test_core/test_oa_core_resolver.pyindex dc162c8..ad68987 100644--- a/tests/test_core/test_oa_core_resolver.py+++ b/tests/test_core/test_oa_core_resolver.py@@ -132,7 +132,7 @@ def _stub_core(monkeypatch, responses: dict[str, object]):def _stub_pdf(monkeypatch, returned):- from hyperresearch.web import crawl4ai_provider+ from hyperresearch.web import pdf as pdf_lanetried: list[str] = []@@ -140,7 +140,7 @@ def _stub_pdf(monkeypatch, returned):tried.append(url)return returned(url) if callable(returned) else returned- monkeypatch.setattr(crawl4ai_provider, "_fetch_pdf", fake)+ monkeypatch.setattr(pdf_lane, "fetch_pdf", fake)return trieddiff --git a/tests/test_core/test_oa_recovery.py b/tests/test_core/test_oa_recovery.pyindex 47058ac..9328a85 100644--- a/tests/test_core/test_oa_recovery.py+++ b/tests/test_core/test_oa_recovery.py@@ -310,9 +310,9 @@ class TestRecoverFullText:return tmp_vaultdef _stub_pdf(self, monkeypatch, returned):- from hyperresearch.web import crawl4ai_provider+ from hyperresearch.web import pdf as pdf_lane- monkeypatch.setattr(crawl4ai_provider, "_fetch_pdf", lambda url, settings: returned)+ monkeypatch.setattr(pdf_lane, "fetch_pdf", lambda url, settings: returned)def test_happy_path_swaps_the_body(self, vault, monkeypatch, public_dns):_stub_http(monkeypatch, {"unpaywall": UNPAYWALL_PDF})@@ -368,7 +368,7 @@ class TestRecoverFullText:}_stub_http(monkeypatch, {"unpaywall": payload})- from hyperresearch.web import crawl4ai_provider+ from hyperresearch.web import pdf as pdf_lanetried: list[str] = []@@ -376,7 +376,7 @@ class TestRecoverFullText:tried.append(url)return None if "blocked" in url else _result(FULL_TEXT, url=url)- monkeypatch.setattr(crawl4ai_provider, "_fetch_pdf", flaky)+ monkeypatch.setattr(pdf_lane, "fetch_pdf", flaky)out, loc = oa.recover_full_text(vault, None, "https://p/x", "10.1/x", _result(ABSTRACT))assert tried == ["https://blocked.example.org/a.pdf", "https://mirror.example.org/b.pdf"]assert loc.url == "https://mirror.example.org/b.pdf"@@ -393,11 +393,11 @@ class TestRecoverFullText:}_stub_http(monkeypatch, {"unpaywall": payload})- from hyperresearch.web import crawl4ai_provider+ from hyperresearch.web import pdf as pdf_lanetried: list[str] = []monkeypatch.setattr(- crawl4ai_provider, "_fetch_pdf", lambda url, s: tried.append(url) or None+ pdf_lane, "fetch_pdf", lambda url, s: tried.append(url) or None)original = _result(ABSTRACT)out, loc = oa.recover_full_text(vault, None, "https://p/x", "10.1/x", original)@@ -426,13 +426,13 @@ class TestRecoverFullText:assert out is original and loc is Nonedef test_raising_fetcher_is_soft(self, vault, monkeypatch, public_dns):- from hyperresearch.web import crawl4ai_provider+ from hyperresearch.web import pdf as pdf_lanedef boom(url, settings):raise RuntimeError("connection reset")_stub_http(monkeypatch, {"unpaywall": UNPAYWALL_PDF})- monkeypatch.setattr(crawl4ai_provider, "_fetch_pdf", boom)+ monkeypatch.setattr(pdf_lane, "fetch_pdf", boom)original = _result(ABSTRACT)out, loc = oa.recover_full_text(vault, None, "https://p/x", "10.1/x", original)assert out is original and loc is None@@ -542,9 +542,9 @@ class TestRescueFullText:return tmp_vaultdef _stub_pdf(self, monkeypatch, returned):- from hyperresearch.web import crawl4ai_provider+ from hyperresearch.web import pdf as pdf_lane- monkeypatch.setattr(crawl4ai_provider, "_fetch_pdf", lambda url, settings: returned)+ monkeypatch.setattr(pdf_lane, "fetch_pdf", lambda url, settings: returned)def test_recovers_with_no_original_to_beat(self, vault, monkeypatch, public_dns):_stub_http(monkeypatch, {"unpaywall": UNPAYWALL_PDF})diff --git a/tests/test_web/test_builtin_pdf_lane.py b/tests/test_web/test_builtin_pdf_lane.pynew file mode 100644index 0000000..17927ec--- /dev/null+++ b/tests/test_web/test_builtin_pdf_lane.py@@ -0,0 +1,147 @@+"""The builtin provider has a PDF lane (#82).++Before this, `_fetch_pdf` lived inside the crawl4ai provider only. A vault on+`provider = "builtin"` (the default until `hpr install` switches it) decoded+every PDF as HTML text and the junk gate rejected it as "Binary PDF garbage+in content" — the same message for every mirror of the same document, which+is exactly the all-hosts-at-once shape reported in #82. An arXiv /abs/ link+saved the abstract page instead of the paper.++Offline: the SSRF-gated download (`safe_get`) is stubbed at the source, so+both the PDF lane and the HTML lane see canned responses.+"""++from __future__ import annotations++import json++import pytest+from typer.testing import CliRunner++from hyperresearch.cli import app+from hyperresearch.web import pdf as pdf_lane+from hyperresearch.web.builtin import BuiltinProvider+from hyperresearch.web.safe_http import SafeResponse++runner = CliRunner()++HTML_PAGE = (+ "<html><head><title>[2005.14165] Language Models are Few-Shot Learners</title></head>"+ "<body><h1>Abstract</h1><p>" + "We demonstrate that scaling language models improves few-shot performance. " * 12+ + "</p></body></html>"+)+++def pdf_bytes() -> bytes:+ pytest.importorskip("pymupdf")+ import pymupdf++ doc = pymupdf.open()+ page = doc.new_page()+ page.insert_text((36, 60), "Language Models are Few-Shot Learners: full paper text.")+ for i in range(20):+ page.insert_text((36, 80 + i * 14), f"Line {i}: body text of the paper for extraction. " * 2)+ data = doc.tobytes()+ doc.close()+ return data+++def responses(monkeypatch):+ """Route every safe_get through a table of URL -> SafeResponse. Records requests."""+ table: dict[str, SafeResponse] = {}+ seen: list[str] = []++ def fake_safe_get(url, *, max_bytes, timeout=None, headers=None, verify=True,+ allow_private_hosts=()):+ seen.append(url)+ if url not in table:+ raise AssertionError(f"unexpected fetch of {url}")+ return table[url]++ monkeypatch.setattr("hyperresearch.web.safe_http.safe_get", fake_safe_get)+ # Keep the SSRF hostname check offline.+ monkeypatch.setattr("hyperresearch.web.safe_http.check_url", lambda url, allow=(): None)+ table["__seen__"] = seen # type: ignore[assignment]+ return table+++def _resp(url: str, body: bytes, content_type: str, status: int = 200) -> SafeResponse:+ return SafeResponse(url=url, status_code=status, headers={"content-type": content_type}, content=body)+++def test_direct_pdf_url_is_extracted_not_decoded_as_html(responses, pdf_bytes):+ url = "https://mirror.example.org/papers/2005.14165.pdf"+ responses[url] = _resp(url, pdf_bytes, "application/pdf")++ result = BuiltinProvider().fetch(url)++ assert result.raw_content_type == "application/pdf"+ assert "full paper text" in result.content+ assert result.looks_like_junk() is None+++def test_arxiv_abs_link_fetches_the_paper_not_the_abstract_page(responses, pdf_bytes):+ abs_url = "https://arxiv.org/abs/2005.14165"+ pdf_url = "https://arxiv.org/pdf/2005.14165.pdf"+ responses[abs_url] = _resp(abs_url, HTML_PAGE.encode(), "text/html")+ responses[pdf_url] = _resp(pdf_url, pdf_bytes, "application/pdf")++ result = BuiltinProvider().fetch(abs_url)++ assert responses["__seen__"] == [pdf_url], "the abs page must not be fetched at all"+ assert "full paper text" in result.content+ assert result.metadata.get("pages") == 1+++def test_pdf_lane_failure_falls_back_to_html_and_carries_the_reason(responses):+ abs_url = "https://arxiv.org/abs/2005.14165"+ pdf_url = "https://arxiv.org/pdf/2005.14165.pdf"+ responses[pdf_url] = _resp(pdf_url, b"", "text/html", status=403)+ responses[abs_url] = _resp(abs_url, HTML_PAGE.encode(), "text/html")++ result = BuiltinProvider().fetch(abs_url)++ assert "Few-Shot Learners" in result.title+ assert result.metadata[pdf_lane.PDF_FAILURE_KEY] == "HTTP 403"+++def test_pdf_behind_a_non_pdf_url_is_detected_from_the_bytes(responses, pdf_bytes):+ url = "https://repo.example.edu/download?id=123"+ responses[url] = _resp(url, pdf_bytes, "application/octet-stream")++ result = BuiltinProvider().fetch(url)++ assert result.raw_content_type == "application/pdf"+ assert "full paper text" in result.content+++def test_cli_junk_verdict_names_the_pdf_lane_reason(tmp_path, responses, monkeypatch):+ """A PDF whose lane failed used to surface only as 'Binary PDF garbage in+ content' with no hint of why. The -j error now says what the PDF lane saw."""+ root = tmp_path / "kb"+ assert runner.invoke(app, ["init", str(root), "--name", "PDF Test"]).exit_code == 0+ monkeypatch.chdir(root)++ url = "https://mirror.example.org/papers/x.pdf"+ calls = {"n": 0}++ def two_lanes(url_, *, max_bytes, timeout=None, headers=None, verify=True, allow_private_hosts=()):+ calls["n"] += 1+ if calls["n"] == 1:+ # PDF lane: the mirror answers with an HTML interstitial.+ return _resp(url_, b"<html>Access denied</html>", "text/html")+ # HTML lane: the same mirror now streams the raw document, which the+ # provider decodes as text and the junk gate calls binary garbage.+ return _resp(url_, bytes(range(256)) * 8, "text/html")++ monkeypatch.setattr("hyperresearch.web.safe_http.safe_get", two_lanes)++ result = runner.invoke(app, ["fetch", url, "-j"])+ payload = json.loads(result.stdout)++ assert payload["ok"] is False+ assert payload["error_code"] == "JUNK_CONTENT"+ assert "PDF lane: did not return PDF data" in payload["error"]+ assert "text/html" in payload["error"]diff --git a/tests/test_web/test_pdf_diagnostics.py b/tests/test_web/test_pdf_diagnostics.pyindex 9528bf3..cd7f3fd 100644--- a/tests/test_web/test_pdf_diagnostics.py+++ b/tests/test_web/test_pdf_diagnostics.py@@ -1,13 +1,14 @@"""PDF fetch diagnostics.-Every `_fetch_pdf` failure used to return a bare `None`, indistinguishable from+Every `fetch_pdf` failure used to return a bare `None`, indistinguishable from"this URL is not a PDF". When pymupdf was missing or broken — e.g. no wheel forthe platform — every PDF on every domain silently fell through to the browserlane, arrived as binary, and was discarded as junk, with nothing logged to saywhy. These tests pin the diagnostics, not just the happy path.-Offline: the download layer (`_safe_get_pdf` / `safe_get`) is stubbed, no-network is touched.+Offline: the download layer (`safe_get_pdf` / `safe_get`) is stubbed, no+network is touched. The lane lives in `hyperresearch.web.pdf` and is shared+by every provider; the crawl4ai module re-exports it under the old names."""from __future__ import annotations@@ -17,7 +18,7 @@ import loggingimport pytest-import hyperresearch.web.crawl4ai_provider as provider+import hyperresearch.web.pdf as providerclass _Resp:@@ -32,7 +33,7 @@ def stub_httpx(monkeypatch):"""Replace the SSRF-gated download call with a canned response."""def _install(resp: _Resp):- monkeypatch.setattr(provider, "_safe_get_pdf", lambda url, settings: resp)+ monkeypatch.setattr(provider, "safe_get_pdf", lambda url, settings: resp)return _install@@ -56,7 +57,7 @@ def test_missing_pymupdf_logs_the_consequence(monkeypatch, caplog):monkeypatch.setattr(builtins, "__import__", fake_import)with caplog.at_level(logging.ERROR, logger="hyperresearch.pdf"):- assert provider._import_pymupdf() is None+ assert provider.import_pymupdf() is Noneassert "pymupdf could not be imported" in caplog.textassert "discarded as junk" in caplog.text, "the log must name the actual consequence"@@ -75,7 +76,7 @@ def test_missing_pymupdf_warning_is_not_repeated(monkeypatch, caplog):with caplog.at_level(logging.ERROR, logger="hyperresearch.pdf"):for _ in range(5):- provider._import_pymupdf()+ provider.import_pymupdf()assert caplog.text.count("pymupdf could not be imported") == 1@@ -85,7 +86,7 @@ def test_non_pdf_response_logs_content_type_and_first_bytes(stub_httpx, caplog):stub_httpx(_Resp(b"<!doctype html><html>...", content_type="text/html"))with caplog.at_level(logging.WARNING, logger="hyperresearch.pdf"):- assert provider._fetch_pdf("https://example.com/paper") is None+ assert provider.fetch_pdf("https://example.com/paper") is Noneassert "did not return PDF data" in caplog.textassert "text/html" in caplog.text@@ -96,7 +97,7 @@ def test_http_error_status_is_logged(stub_httpx, caplog):stub_httpx(_Resp(b"", status=403, content_type="text/html"))with caplog.at_level(logging.WARNING, logger="hyperresearch.pdf"):- assert provider._fetch_pdf("https://example.com/x.pdf") is None+ assert provider.fetch_pdf("https://example.com/x.pdf") is Noneassert "403" in caplog.text@@ -122,7 +123,7 @@ def test_magic_bytes_beat_a_wrong_content_type(stub_httpx):stub_httpx(_Resp(pdf_bytes, content_type="application/octet-stream"))- result = provider._fetch_pdf("https://example.com/download?id=123")+ result = provider.fetch_pdf("https://example.com/download?id=123")assert result is not None, "mislabelled PDF was rejected"assert "Extractable text layer" in result.contentassert result.looks_like_junk() is None@@ -133,7 +134,7 @@ def test_html_masquerading_as_pdf_content_type_is_rejected(stub_httpx, caplog):stub_httpx(_Resp(b"<html><body>Access denied</body></html>"))with caplog.at_level(logging.WARNING, logger="hyperresearch.pdf"):- assert provider._fetch_pdf("https://example.com/x.pdf") is None+ assert provider.fetch_pdf("https://example.com/x.pdf") is Noneassert "did not return PDF data" in caplog.text@@ -151,7 +152,7 @@ def test_scanned_pdf_without_text_layer_explains_itself(stub_httpx, caplog):stub_httpx(_Resp(pdf_bytes))with caplog.at_level(logging.WARNING, logger="hyperresearch.pdf"):- assert provider._fetch_pdf("https://example.com/scan.pdf") is None+ assert provider.fetch_pdf("https://example.com/scan.pdf") is Noneassert "no extractable text layer" in caplog.textassert "OCR" in caplog.text@@ -184,7 +185,7 @@ def test_cert_error_refuses_with_no_unverified_retry(monkeypatch):monkeypatch.setattr("hyperresearch.web.safe_http.safe_get", fake_safe_get)with pytest.raises(SafeHTTPError, match="pdf_verify_tls"):- provider._safe_get_pdf(+ provider.safe_get_pdf("https://broken-cert.example.edu/paper.pdf", provider.FetchSettings())@@ -206,7 +207,7 @@ def test_non_cert_error_propagates_untranslated(monkeypatch):monkeypatch.setattr("hyperresearch.web.safe_http.safe_get", fake_safe_get)with pytest.raises(httpx.ConnectError):- provider._safe_get_pdf("https://down.example.com/x.pdf", provider.FetchSettings())+ provider.safe_get_pdf("https://down.example.com/x.pdf", provider.FetchSettings())assert calls == [True]@@ -223,9 +224,75 @@ def test_pdf_verify_tls_false_fetches_unverified(monkeypatch):monkeypatch.setattr("hyperresearch.web.safe_http.safe_get", fake_safe_get)- resp = provider._safe_get_pdf(+ resp = provider.safe_get_pdf("https://mirror.example.org/x.pdf", provider.FetchSettings(pdf_verify_tls=False))assert calls == [False], "config opt-out must fetch unverified on the first attempt"assert resp.content == b"%PDF- direct"+++def test_failure_reason_is_kept_for_the_requested_url(stub_httpx):+ """The reason a PDF was declined is readable afterwards, keyed by the URL+ as the caller asked for it — not the rewritten arXiv pdf URL."""+ stub_httpx(_Resp(b"", status=403, content_type="text/html"))++ assert provider.fetch_pdf("https://arxiv.org/abs/2005.14165") is None+ assert provider.failure_reason("https://arxiv.org/abs/2005.14165") == "HTTP 403"+++def test_failure_reason_is_cleared_on_success(stub_httpx):+ pytest.importorskip("pymupdf")+ import pymupdf++ doc = pymupdf.open()+ page = doc.new_page()+ for i in range(20):+ page.insert_text((36, 60 + i * 14), f"Line {i}: body text for the regression test. " * 2)+ pdf_bytes = doc.tobytes()+ doc.close()++ stub_httpx(_Resp(b"", status=500))+ assert provider.fetch_pdf("https://example.com/x.pdf") is None+ assert provider.failure_reason("https://example.com/x.pdf") == "HTTP 500"++ stub_httpx(_Resp(pdf_bytes))+ assert provider.fetch_pdf("https://example.com/x.pdf") is not None+ assert provider.failure_reason("https://example.com/x.pdf") is None+++def test_document_handle_is_closed_when_page_iteration_raises(stub_httpx, monkeypatch):+ """An encrypted or damaged PDF that raises mid-iteration must not leak+ the open document."""+ closed: list[bool] = []++ class _Doc:+ page_count = 3++ def __iter__(self):+ raise RuntimeError("cannot read page: document is encrypted")++ def close(self):+ closed.append(True)++ class _PyMuPDF:+ @staticmethod+ def open(stream, filetype):+ return _Doc()++ monkeypatch.setattr(provider, "import_pymupdf", lambda: _PyMuPDF())+ stub_httpx(_Resp(b"%PDF-1.7 " + b"x" * 4096))++ assert provider.fetch_pdf("https://example.com/locked.pdf") is None+ assert closed == [True]+ assert "encrypted" in (provider.failure_reason("https://example.com/locked.pdf") or "")+++def test_crawl4ai_module_keeps_the_old_names():+ """Callers and tests patched `crawl4ai_provider._fetch_pdf` for a long time;+ the aliases stay so the move is not a silent break."""+ crawl4ai_provider = pytest.importorskip("hyperresearch.web.crawl4ai_provider")++ assert crawl4ai_provider._fetch_pdf is provider.fetch_pdf+ assert crawl4ai_provider._is_pdf_url is provider.is_pdf_url+ assert crawl4ai_provider._safe_get_pdf is provider.safe_get_pdf