Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai
Python agentsagentskillscitationsclaude-codecodexcodex-plugincodex-skilldeep-researchdeep-research-agentdeepresearchmcp-serverresearchresearch-agentresearch-tools
fix: make PDF fetch failures diagnosable instead of silent (#42)
synqingd48f184a4c1a
diff --git a/src/hyperresearch/web/crawl4ai_provider.py b/src/hyperresearch/web/crawl4ai_provider.pyindex bfe253f..7ba9027 100644--- a/src/hyperresearch/web/crawl4ai_provider.py+++ b/src/hyperresearch/web/crawl4ai_provider.py@@ -9,6 +9,7 @@ Supports authenticated crawling via crawl4ai browser profiles:from __future__ import annotationsimport asyncio+import loggingimport osimport pathlibimport sys@@ -64,11 +65,46 @@ def _looks_like_binary(text: str) -> bool:return binary_garbage_ratio(sample) > 0.05-def _fetch_pdf(url: str) -> WebResult | None:- """Download a PDF and extract text using pymupdf. Returns None if extraction fails."""+_PYMUPDF_MISSING_LOGGED = False+++def _pdf_log() -> logging.Logger:+ return logging.getLogger("hyperresearch.pdf")+++def _import_pymupdf():+ """Import pymupdf, warning loudly (once) if it is unavailable.++ Without this warning a missing/broken pymupdf is invisible: every PDF falls+ through to the browser lane, arrives as binary, and is discarded as junk —+ across every domain at once, with nothing explaining why.+ """+ global _PYMUPDF_MISSING_LOGGEDtry:import pymupdf- except ImportError:++ return pymupdf+ except ImportError as exc:+ if not _PYMUPDF_MISSING_LOGGED:+ _PYMUPDF_MISSING_LOGGED = True+ _pdf_log().error(+ "pymupdf could not be imported (%s) — PDF text extraction is disabled, "+ "so every PDF will be discarded as junk content. Reinstall it with "+ "`pip install --force-reinstall pymupdf`.",+ exc,+ )+ return None+++def _fetch_pdf(url: str) -> WebResult | None:+ """Download a PDF and extract text using pymupdf. Returns None if extraction fails.++ Every failure path logs its reason. A silent None here is indistinguishable+ from "this URL is not a PDF", which is what made missing-PDF failures so hard+ to diagnose.+ """+ pymupdf = _import_pymupdf()+ if pymupdf is None:return Noneimport httpx@@ -81,17 +117,31 @@ def _fetch_pdf(url: str) -> WebResult | None:url += ".pdf"resp = httpx.get(url, follow_redirects=True, timeout=30, verify=False, headers={- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"+ "User-Agent": (+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "+ "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"+ ),+ "Accept": "application/pdf,application/octet-stream;q=0.9,*/*;q=0.8",})if resp.status_code != 200:+ _pdf_log().warning("PDF fetch for %s returned HTTP %s", url, resp.status_code)return None+ pdf_bytes = resp.contentcontent_type = resp.headers.get("content-type", "")- if "pdf" not in content_type and not url.lower().endswith(".pdf"):- return None # Not actually a PDF- pdf_bytes = resp.content+ # Magic bytes are authoritative. Servers mislabel PDFs as octet-stream,+ # and plenty of PDF URLs carry no .pdf suffix, so trusting the header or+ # the URL shape alone silently drops real PDFs.+ if not pdf_bytes.startswith(b"%PDF-"):+ _pdf_log().warning(+ "PDF fetch for %s did not return PDF data (content-type=%r, first bytes=%r)",+ url, content_type, pdf_bytes[:8],+ )+ return None+if len(pdf_bytes) < 100:+ _pdf_log().warning("PDF fetch for %s returned only %d bytes", url, len(pdf_bytes))return Nonedoc = pymupdf.open(stream=pdf_bytes, filetype="pdf")@@ -103,9 +153,14 @@ def _fetch_pdf(url: str) -> WebResult | None:if text.strip():pages.append(text)+ page_count = doc.page_countdoc.close()if not pages:+ _pdf_log().warning(+ "PDF at %s has %d page(s) but no extractable text layer — "+ "likely a scanned document requiring OCR.", url, page_count,+ )return None# Build markdown from extracted text@@ -129,8 +184,7 @@ def _fetch_pdf(url: str) -> WebResult | None:)except Exception as e:- import logging- logging.getLogger("hyperresearch.pdf").warning(f"PDF extraction failed for {url}: {e}")+ _pdf_log().warning("PDF extraction failed for %s: %s", url, e)return Nonediff --git a/tests/test_web/test_pdf_diagnostics.py b/tests/test_web/test_pdf_diagnostics.pynew file mode 100644index 0000000..cafdc27--- /dev/null+++ b/tests/test_web/test_pdf_diagnostics.py@@ -0,0 +1,158 @@+"""PDF fetch diagnostics.++Every `_fetch_pdf` failure used to return a bare `None`, indistinguishable from+"this URL is not a PDF". When pymupdf was missing or broken — e.g. no wheel for+the platform — every PDF on every domain silently fell through to the browser+lane, arrived as binary, and was discarded as junk, with nothing logged to say+why. These tests pin the diagnostics, not just the happy path.++Offline: httpx is stubbed, no network is touched.+"""++from __future__ import annotations++import builtins+import logging++import pytest++import hyperresearch.web.crawl4ai_provider as provider+++class _Resp:+ def __init__(self, content: bytes, status: int = 200, content_type: str = "application/pdf"):+ self.content = content+ self.status_code = status+ self.headers = {"content-type": content_type}+++def stub_httpx(monkeypatch):+ """Replace httpx.get with a canned response."""++ def _install(resp: _Resp):+ import httpx++ monkeypatch.setattr(httpx, "get", lambda *a, **k: resp)++ return _install++[email protected](autouse=True)+def _reset_warning_latch():+ provider._PYMUPDF_MISSING_LOGGED = False+ yield+ provider._PYMUPDF_MISSING_LOGGED = False+++def test_missing_pymupdf_logs_the_consequence(monkeypatch, caplog):+ """A missing pymupdf must not fail silently — it disables all PDF ingestion."""+ real_import = builtins.__import__++ def fake_import(name, *args, **kwargs):+ if name == "pymupdf":+ raise ImportError("no wheel for this platform")+ return real_import(name, *args, **kwargs)++ monkeypatch.setattr(builtins, "__import__", fake_import)++ with caplog.at_level(logging.ERROR, logger="hyperresearch.pdf"):+ assert provider._import_pymupdf() is None++ assert "pymupdf could not be imported" in caplog.text+ assert "discarded as junk" in caplog.text, "the log must name the actual consequence"+++def test_missing_pymupdf_warning_is_not_repeated(monkeypatch, caplog):+ """One clear error, not one per fetched URL."""+ real_import = builtins.__import__++ def fake_import(name, *args, **kwargs):+ if name == "pymupdf":+ raise ImportError("nope")+ return real_import(name, *args, **kwargs)++ monkeypatch.setattr(builtins, "__import__", fake_import)++ with caplog.at_level(logging.ERROR, logger="hyperresearch.pdf"):+ for _ in range(5):+ provider._import_pymupdf()++ assert caplog.text.count("pymupdf could not be imported") == 1+++def test_non_pdf_response_logs_content_type_and_first_bytes(stub_httpx, caplog):+ """When the server returns HTML, say so — don't just return None."""+ stub_httpx(_Resp(b"<!doctype html><html>...", content_type="text/html"))++ with caplog.at_level(logging.WARNING, logger="hyperresearch.pdf"):+ assert provider._fetch_pdf("https://example.com/paper") is None++ assert "did not return PDF data" in caplog.text+ assert "text/html" in caplog.text+++def test_http_error_status_is_logged(stub_httpx, caplog):+ """A 403 must be visible, not silently indistinguishable from 'not a PDF'."""+ stub_httpx(_Resp(b"", status=403, content_type="text/html"))++ with caplog.at_level(logging.WARNING, logger="hyperresearch.pdf"):+ assert provider._fetch_pdf("https://example.com/x.pdf") is None++ assert "403" in caplog.text+++def test_magic_bytes_beat_a_wrong_content_type(stub_httpx):+ """A real PDF mislabelled as octet-stream must still be accepted.++ Servers routinely mislabel PDFs, and many PDF URLs carry no .pdf suffix.+ Trusting the header or the URL shape alone drops real PDFs on the floor.+ """+ pytest.importorskip("pymupdf")+ import pymupdf++ doc = pymupdf.open()+ page = doc.new_page()+ # Needs to clear the >=300-char "near-empty content" rule to reach the+ # binary/junk checks this test is actually about.+ page.insert_text((36, 60), "Extractable text layer for the regression test.")+ for i in range(20):+ page.insert_text((36, 80 + i * 14), f"Line {i}: representative body text for extraction. " * 2)+ pdf_bytes = doc.tobytes()+ doc.close()++ stub_httpx(_Resp(pdf_bytes, content_type="application/octet-stream"))++ result = provider._fetch_pdf("https://example.com/download?id=123")+ assert result is not None, "mislabelled PDF was rejected"+ assert "Extractable text layer" in result.content+ assert result.looks_like_junk() is None+++def test_html_masquerading_as_pdf_content_type_is_rejected(stub_httpx, caplog):+ """The inverse: a text/html body labelled application/pdf is not a PDF."""+ stub_httpx(_Resp(b"<html><body>Access denied</body></html>"))++ with caplog.at_level(logging.WARNING, logger="hyperresearch.pdf"):+ assert provider._fetch_pdf("https://example.com/x.pdf") is None++ assert "did not return PDF data" in caplog.text+++def test_scanned_pdf_without_text_layer_explains_itself(stub_httpx, caplog):+ """An image-only PDF needs OCR — say that rather than returning a bare None."""+ pytest.importorskip("pymupdf")+ import pymupdf++ doc = pymupdf.open()+ doc.new_page() # blank page, no text layer+ pdf_bytes = doc.tobytes()+ doc.close()++ stub_httpx(_Resp(pdf_bytes))++ with caplog.at_level(logging.WARNING, logger="hyperresearch.pdf"):+ assert provider._fetch_pdf("https://example.com/scan.pdf") is None++ assert "no extractable text layer" in caplog.text+ assert "OCR" in caplog.text