Skip to content

Convert Claude Code or Codex into the most intelligent Deep Research Agent. Collect, search, and synthesize web research into a persistent, searchable wiki that builds on itself. Hosted API + MCP: hyperresearch.ai

fix: make PDF fetch failures diagnosable instead of silent (#42)

synqingd48f184a4c1a
diff --git a/src/hyperresearch/web/crawl4ai_provider.py b/src/hyperresearch/web/crawl4ai_provider.py
index bfe253f..7ba9027 100644
--- a/src/hyperresearch/web/crawl4ai_provider.py
+++ b/src/hyperresearch/web/crawl4ai_provider.py
@@ -9,6 +9,7 @@ Supports authenticated crawling via crawl4ai browser profiles:
from __future__ import annotations
import asyncio
+import logging
import os
import pathlib
import sys
@@ -64,11 +65,46 @@ def _looks_like_binary(text: str) -> bool:
return binary_garbage_ratio(sample) > 0.05
-def _fetch_pdf(url: str) -> WebResult | None:
- """Download a PDF and extract text using pymupdf. Returns None if extraction fails."""
+_PYMUPDF_MISSING_LOGGED = False
+
+
+def _pdf_log() -> logging.Logger:
+ return logging.getLogger("hyperresearch.pdf")
+
+
+def _import_pymupdf():
+ """Import pymupdf, warning loudly (once) if it is unavailable.
+
+ Without this warning a missing/broken pymupdf is invisible: every PDF falls
+ through to the browser lane, arrives as binary, and is discarded as junk —
+ across every domain at once, with nothing explaining why.
+ """
+ global _PYMUPDF_MISSING_LOGGED
try:
import pymupdf
- except ImportError:
+
+ return pymupdf
+ except ImportError as exc:
+ if not _PYMUPDF_MISSING_LOGGED:
+ _PYMUPDF_MISSING_LOGGED = True
+ _pdf_log().error(
+ "pymupdf could not be imported (%s) — PDF text extraction is disabled, "
+ "so every PDF will be discarded as junk content. Reinstall it with "
+ "`pip install --force-reinstall pymupdf`.",
+ exc,
+ )
+ return None
+
+
+def _fetch_pdf(url: str) -> WebResult | None:
+ """Download a PDF and extract text using pymupdf. Returns None if extraction fails.
+
+ Every failure path logs its reason. A silent None here is indistinguishable
+ from "this URL is not a PDF", which is what made missing-PDF failures so hard
+ to diagnose.
+ """
+ pymupdf = _import_pymupdf()
+ if pymupdf is None:
return None
import httpx
@@ -81,17 +117,31 @@ def _fetch_pdf(url: str) -> WebResult | None:
url += ".pdf"
resp = httpx.get(url, follow_redirects=True, timeout=30, verify=False, headers={
- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+ "User-Agent": (
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+ "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+ ),
+ "Accept": "application/pdf,application/octet-stream;q=0.9,*/*;q=0.8",
})
if resp.status_code != 200:
+ _pdf_log().warning("PDF fetch for %s returned HTTP %s", url, resp.status_code)
return None
+ pdf_bytes = resp.content
content_type = resp.headers.get("content-type", "")
- if "pdf" not in content_type and not url.lower().endswith(".pdf"):
- return None # Not actually a PDF
- pdf_bytes = resp.content
+ # Magic bytes are authoritative. Servers mislabel PDFs as octet-stream,
+ # and plenty of PDF URLs carry no .pdf suffix, so trusting the header or
+ # the URL shape alone silently drops real PDFs.
+ if not pdf_bytes.startswith(b"%PDF-"):
+ _pdf_log().warning(
+ "PDF fetch for %s did not return PDF data (content-type=%r, first bytes=%r)",
+ url, content_type, pdf_bytes[:8],
+ )
+ return None
+
if len(pdf_bytes) < 100:
+ _pdf_log().warning("PDF fetch for %s returned only %d bytes", url, len(pdf_bytes))
return None
doc = pymupdf.open(stream=pdf_bytes, filetype="pdf")
@@ -103,9 +153,14 @@ def _fetch_pdf(url: str) -> WebResult | None:
if text.strip():
pages.append(text)
+ page_count = doc.page_count
doc.close()
if not pages:
+ _pdf_log().warning(
+ "PDF at %s has %d page(s) but no extractable text layer — "
+ "likely a scanned document requiring OCR.", url, page_count,
+ )
return None
# Build markdown from extracted text
@@ -129,8 +184,7 @@ def _fetch_pdf(url: str) -> WebResult | None:
)
except Exception as e:
- import logging
- logging.getLogger("hyperresearch.pdf").warning(f"PDF extraction failed for {url}: {e}")
+ _pdf_log().warning("PDF extraction failed for %s: %s", url, e)
return None
diff --git a/tests/test_web/test_pdf_diagnostics.py b/tests/test_web/test_pdf_diagnostics.py
new file mode 100644
index 0000000..cafdc27
--- /dev/null
+++ b/tests/test_web/test_pdf_diagnostics.py
@@ -0,0 +1,158 @@
+"""PDF fetch diagnostics.
+
+Every `_fetch_pdf` failure used to return a bare `None`, indistinguishable from
+"this URL is not a PDF". When pymupdf was missing or broken — e.g. no wheel for
+the platform — every PDF on every domain silently fell through to the browser
+lane, arrived as binary, and was discarded as junk, with nothing logged to say
+why. These tests pin the diagnostics, not just the happy path.
+
+Offline: httpx is stubbed, no network is touched.
+"""
+
+from __future__ import annotations
+
+import builtins
+import logging
+
+import pytest
+
+import hyperresearch.web.crawl4ai_provider as provider
+
+
+class _Resp:
+ def __init__(self, content: bytes, status: int = 200, content_type: str = "application/pdf"):
+ self.content = content
+ self.status_code = status
+ self.headers = {"content-type": content_type}
+
+
+def stub_httpx(monkeypatch):
+ """Replace httpx.get with a canned response."""
+
+ def _install(resp: _Resp):
+ import httpx
+
+ monkeypatch.setattr(httpx, "get", lambda *a, **k: resp)
+
+ return _install
+
+
[email protected](autouse=True)
+def _reset_warning_latch():
+ provider._PYMUPDF_MISSING_LOGGED = False
+ yield
+ provider._PYMUPDF_MISSING_LOGGED = False
+
+
+def test_missing_pymupdf_logs_the_consequence(monkeypatch, caplog):
+ """A missing pymupdf must not fail silently — it disables all PDF ingestion."""
+ real_import = builtins.__import__
+
+ def fake_import(name, *args, **kwargs):
+ if name == "pymupdf":
+ raise ImportError("no wheel for this platform")
+ return real_import(name, *args, **kwargs)
+
+ monkeypatch.setattr(builtins, "__import__", fake_import)
+
+ with caplog.at_level(logging.ERROR, logger="hyperresearch.pdf"):
+ assert provider._import_pymupdf() is None
+
+ assert "pymupdf could not be imported" in caplog.text
+ assert "discarded as junk" in caplog.text, "the log must name the actual consequence"
+
+
+def test_missing_pymupdf_warning_is_not_repeated(monkeypatch, caplog):
+ """One clear error, not one per fetched URL."""
+ real_import = builtins.__import__
+
+ def fake_import(name, *args, **kwargs):
+ if name == "pymupdf":
+ raise ImportError("nope")
+ return real_import(name, *args, **kwargs)
+
+ monkeypatch.setattr(builtins, "__import__", fake_import)
+
+ with caplog.at_level(logging.ERROR, logger="hyperresearch.pdf"):
+ for _ in range(5):
+ provider._import_pymupdf()
+
+ assert caplog.text.count("pymupdf could not be imported") == 1
+
+
+def test_non_pdf_response_logs_content_type_and_first_bytes(stub_httpx, caplog):
+ """When the server returns HTML, say so — don't just return None."""
+ stub_httpx(_Resp(b"<!doctype html><html>...", content_type="text/html"))
+
+ with caplog.at_level(logging.WARNING, logger="hyperresearch.pdf"):
+ assert provider._fetch_pdf("https://example.com/paper") is None
+
+ assert "did not return PDF data" in caplog.text
+ assert "text/html" in caplog.text
+
+
+def test_http_error_status_is_logged(stub_httpx, caplog):
+ """A 403 must be visible, not silently indistinguishable from 'not a PDF'."""
+ stub_httpx(_Resp(b"", status=403, content_type="text/html"))
+
+ with caplog.at_level(logging.WARNING, logger="hyperresearch.pdf"):
+ assert provider._fetch_pdf("https://example.com/x.pdf") is None
+
+ assert "403" in caplog.text
+
+
+def test_magic_bytes_beat_a_wrong_content_type(stub_httpx):
+ """A real PDF mislabelled as octet-stream must still be accepted.
+
+ Servers routinely mislabel PDFs, and many PDF URLs carry no .pdf suffix.
+ Trusting the header or the URL shape alone drops real PDFs on the floor.
+ """
+ pytest.importorskip("pymupdf")
+ import pymupdf
+
+ doc = pymupdf.open()
+ page = doc.new_page()
+ # Needs to clear the >=300-char "near-empty content" rule to reach the
+ # binary/junk checks this test is actually about.
+ page.insert_text((36, 60), "Extractable text layer for the regression test.")
+ for i in range(20):
+ page.insert_text((36, 80 + i * 14), f"Line {i}: representative body text for extraction. " * 2)
+ pdf_bytes = doc.tobytes()
+ doc.close()
+
+ stub_httpx(_Resp(pdf_bytes, content_type="application/octet-stream"))
+
+ result = provider._fetch_pdf("https://example.com/download?id=123")
+ assert result is not None, "mislabelled PDF was rejected"
+ assert "Extractable text layer" in result.content
+ assert result.looks_like_junk() is None
+
+
+def test_html_masquerading_as_pdf_content_type_is_rejected(stub_httpx, caplog):
+ """The inverse: a text/html body labelled application/pdf is not a PDF."""
+ stub_httpx(_Resp(b"<html><body>Access denied</body></html>"))
+
+ with caplog.at_level(logging.WARNING, logger="hyperresearch.pdf"):
+ assert provider._fetch_pdf("https://example.com/x.pdf") is None
+
+ assert "did not return PDF data" in caplog.text
+
+
+def test_scanned_pdf_without_text_layer_explains_itself(stub_httpx, caplog):
+ """An image-only PDF needs OCR — say that rather than returning a bare None."""
+ pytest.importorskip("pymupdf")
+ import pymupdf
+
+ doc = pymupdf.open()
+ doc.new_page() # blank page, no text layer
+ pdf_bytes = doc.tobytes()
+ doc.close()
+
+ stub_httpx(_Resp(pdf_bytes))
+
+ with caplog.at_level(logging.WARNING, logger="hyperresearch.pdf"):
+ assert provider._fetch_pdf("https://example.com/scan.pdf") is None
+
+ assert "no extractable text layer" in caplog.text
+ assert "OCR" in caplog.text