~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
harden(arxiv): route PDF downloads through the SSRF-validated session (#5807)
LearningCircuit994df671330f
diff --git a/.github/scripts/check-file-writes.sh b/.github/scripts/check-file-writes.shindex fbf63b2fda..fdffe9f1e8 100755--- a/.github/scripts/check-file-writes.sh+++ b/.github/scripts/check-file-writes.sh@@ -252,10 +252,14 @@ if [ -n "$ALL_MATCHES" ]; then# - vector_stores/legacy_cleanup.py — writes the text-free# ``.idmap.json`` migration sidecar (plain position->uuid JSON map,# no chunk text; see the module docstring / `_ID_RE`)+ # - web_search_engines/engines/search_engine_arxiv.py — writes the+ # downloaded PDF of a public arXiv paper into the caller-provided+ # download dir (public published document, no PII/secrets; the fetch+ # itself goes through the SSRF-validated SafeSession egress gate)# If you add an entry here, document WHY the file's writes are safe# (public data, not user-specific, not encrypted at rest by design).if [ "$skip_line" -eq 0 ]; then- if echo "$line" | grep -qE "web/app_factory\.py|document_loaders/bytes_loader\.py|journal_quality/downloader\.py|journal_quality/data_sources/.+\.py|vector_stores/implementations/faiss_store\.py|vector_stores/legacy_cleanup\.py"; then+ if echo "$line" | grep -qE "web/app_factory\.py|document_loaders/bytes_loader\.py|journal_quality/downloader\.py|journal_quality/data_sources/.+\.py|vector_stores/implementations/faiss_store\.py|vector_stores/legacy_cleanup\.py|web_search_engines/engines/search_engine_arxiv\.py"; thenskip_line=1fifidiff --git a/changelog.d/5807.security.md b/changelog.d/5807.security.mdnew file mode 100644index 0000000000..b95aa9171d--- /dev/null+++ b/changelog.d/5807.security.md@@ -0,0 +1,20 @@+arXiv PDF downloads now go through the same SSRF-validated session+(with SSRF pre-validation and DNS pinning) as the rest of the arXiv+integration, instead of the arxiv library's built-in download path, and+fetch from export.arxiv.org (arXiv's designated host for automated+access, matching the arxiv library's own default `download_domain` and+the host already used for the metadata API) rather than the public+arxiv.org host.+This fetch now also passes `stream=True`, so SafeSession's response-size+cap runs before the body is buffered rather than after: an oversized+response with a `Content-Length` header is now rejected immediately,+instead of being fully read into memory first and only then discarded.+When `Content-Length` is absent, the cap's guard now correctly bounds+memory for a connection-close-delimited body, but not for one sent with+`Transfer-Encoding: chunked` (the common case for a CDN of unknown+length) -- `urllib3` reads chunked bodies through a path that bypasses+the guard. That remaining gap lives in SafeSession's response-size cap+itself (`_check_response_size`/`_install_body_guard` in+`safe_requests.py`), affects every `SafeSession` caller that already+streams, and needs its own follow-up; it is not the same issue as+#6172, which tracks callers that don't set `stream=True` at all.diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py b/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.pyindex 9c73f3a862..ffe30a534a 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py@@ -1,13 +1,26 @@+import re+from pathlib import Pathfrom typing import Any, Dict, List, Optionalimport arxivfrom langchain_core.language_models import BaseLLMfrom ...constants import SNIPPET_LENGTH_SHORT+from ...security import SafeSession+from ...security.directory_creation import create_directory+from ...security.safe_requests import DEFAULT_TIMEOUTfrom ...security.secure_logging import loggerfrom ..rate_limiting import RateLimitErrorfrom ..search_engine_base import BaseSearchEngine, Exposure, Sensitivity+# Canonical arXiv identifier shapes, anchored so nothing else slips through+# before we build an egress URL from the value:+# - new style: 2301.12345 / 2301.12345v2 (4-or-5 digit sequence)+# - old style: math.GT/0309136 or cond-mat/0501234 (with optional version)+_ARXIV_ID_RE = re.compile(+ r"^(?:\d{4}\.\d{4,5}(?:v\d+)?|[a-z-]+(?:\.[A-Z]{2})?/\d{7}(?:v\d+)?)$"+)+class ArXivSearchEngine(BaseSearchEngine):"""arXiv search engine implementation with two-phase approach"""@@ -123,6 +136,122 @@ class ArXivSearchEngine(BaseSearchEngine):# Get the search resultsreturn list(client.results(search))+ @staticmethod+ def _validated_arxiv_id(paper: Any) -> Optional[str]:+ """Return a canonical, validated arXiv id for ``paper`` or ``None``.++ The id is derived from the arxiv-provided ``entry_id`` (e.g.+ ``http://arxiv.org/abs/2301.12345v1``) and matched against+ ``_ARXIV_ID_RE`` before it is ever used to build an egress URL,+ so a malformed or unexpected value cannot be interpolated into a+ request target.+ """+ entry_id = getattr(paper, "entry_id", "") or ""+ match = re.search(r"arxiv\.org/abs/(.+)$", entry_id)+ candidate = (match.group(1) if match else entry_id).strip()+ if _ARXIV_ID_RE.match(candidate):+ return candidate+ return None++ def _download_pdf_safely(self, paper: Any, dirpath: str) -> str:+ """Download a paper PDF through the SSRF-validated ``SafeSession``.++ Invariant: every arXiv PDF fetch goes through the same egress gate+ the rest of the arXiv integration uses. ``SafeSession`` applies SSRF+ pre-validation and DNS pinning to this fetch. This replaces+ ``arxiv.Result.download_pdf``, which fetches via+ ``urllib.request.urlretrieve`` and therefore bypasses all of those+ controls. Behavior is preserved: the PDF is written into ``dirpath``+ and the resulting path is returned.++ Note: this call passes ``stream=True``, so ``requests.Session.send()``+ does *not* drain the body itself (it only does that when+ ``stream`` is falsy — see ``requests.sessions.Session.send``);+ ``SafeSession``'s response-size cap (``_check_response_size``) runs+ while the connection is still open, before the call site reads+ ``response.content``. That fixes the case this note used to+ describe: with ``Content-Length`` present and over the limit, the+ cap now rejects immediately after the headers arrive, before any+ body bytes are buffered, instead of buffering the whole oversized+ body first and only then discarding it.++ It does not fully fix the ``Content-Length``-absent case. The cap+ installs a guard on ``response.raw.read()``+ (``_install_body_guard``), and that guard does run correctly — but+ only when the connection is delimited by the socket closing (no+ ``Content-Length``, no ``Transfer-Encoding``). When the origin uses+ ``Transfer-Encoding: chunked`` instead — the common case for a CDN+ serving a PDF of unknown length — ``urllib3``'s+ ``HTTPResponse.stream()`` (what ``requests`` uses under+ ``iter_content``/``.content``) reads such bodies through+ ``read_chunked()``, which pulls bytes directly off the socket via+ ``self._fp._safe_read()`` and never calls the patched+ ``read()`` — verified by reading the installed ``urllib3``+ (2.7.0) source and confirming with a live chunked-response+ server: the patched ``read()`` recorded zero calls while a full+ chunked body was consumed. So for a genuinely chunked response,+ peak memory is still unbounded here, same as before this change.++ That gap is a defect in ``_check_response_size``/+ ``_install_body_guard`` in ``safe_requests.py`` itself — every+ ``SafeSession`` caller that already uses ``stream=True`` (e.g.+ ``research_library/downloaders/generic.py``) has the same+ exposure for chunked responses. It is independent of whether this+ call streams and is not the same issue as #6172, which is about+ callers that never set ``stream=True`` at all (this one now+ does). It needs its own follow-up rather than being fixed here.+ """+ arxiv_id = self._validated_arxiv_id(paper)+ if not arxiv_id:+ raise ValueError(+ "Could not derive a valid arXiv id for PDF download"+ )++ # Build the download URL from the validated id rather than trusting+ # an arbitrary attribute value; SafeSession re-validates it anyway.+ # Use export.arxiv.org (not the public arxiv.org CDN-fronted host)+ # to match arxiv.Result.download_pdf's default `download_domain`+ # (see the installed `arxiv` package, ~v2.4) and the host+ # `arxiv.Client`/`arxiv.Search` already use for the metadata API+ # (`query_url_format = "https://export.arxiv.org/api/query?..."`).+ # export.arxiv.org is arXiv's designated host for automated/scripted+ # access; the public arxiv.org host is rate-limited and+ # bot-challenged for that traffic.+ pdf_url = f"https://export.arxiv.org/pdf/{arxiv_id}.pdf"++ directory = Path(dirpath)+ create_directory(directory, context="arXiv PDF download directory")+ target_path = directory / f"{arxiv_id.replace('/', '_')}.pdf"++ with SafeSession() as session:+ # stream=True is intentional here — DO NOT remove it. It lets+ # SafeSession's response-size cap (_check_response_size) run+ # while the connection is still open, instead of after+ # `requests` has already buffered the whole body (see the+ # docstring note above). The `with ... as response:` ensures+ # the connection is released on every path, including a+ # rejection raised by the cap itself, mirroring the same+ # stream=True + context-manager pattern already used in+ # research_library/downloaders/generic.py.+ with session.get(+ pdf_url,+ timeout=DEFAULT_TIMEOUT,+ allow_redirects=True,+ stream=True,+ ) as response:+ response.raise_for_status()+ # `.content` still returns the full body unchanged — see+ # the docstring note above for what moving the read here+ # (rather than inside `send()`) does and does not fix.+ pdf_bytes = response.content++ # Public arXiv PDF (a published paper, no PII/secrets) written into the+ # caller-provided download dir; see the allowlist note in+ # .github/scripts/check-file-writes.sh.+ target_path.write_bytes(pdf_bytes)++ return str(target_path)+def _get_previews(self, query: str) -> List[Dict[str, Any]]:"""Get preview information for arXiv papers.@@ -271,8 +400,8 @@ class ArXivSearchEngine(BaseSearchEngine):# Apply rate limiting before PDF downloadself.rate_tracker.apply_rate_limit(self.engine_type)- paper_path = paper.download_pdf(- dirpath=self.download_dir+ paper_path = self._download_pdf_safely(+ paper, self.download_dir)result["pdf_path"] = str(paper_path)@@ -456,8 +585,10 @@ class ArXivSearchEngine(BaseSearchEngine):# Apply rate limiting before PDF downloadself.rate_tracker.apply_rate_limit(self.engine_type)- # Download the paper- paper_path = paper.download_pdf(dirpath=self.download_dir)+ # Download the paper through the SSRF-validated session+ paper_path = self._download_pdf_safely(+ paper, self.download_dir+ )result["pdf_path"] = str(paper_path)except Exception as e:safe_msg = self._scrub_error(e)diff --git a/tests/web_search_engines/engines/test_arxiv_coverage.py b/tests/web_search_engines/engines/test_arxiv_coverage.pyindex f22f388620..3691aa17e9 100644--- a/tests/web_search_engines/engines/test_arxiv_coverage.py+++ b/tests/web_search_engines/engines/test_arxiv_coverage.py@@ -421,6 +421,11 @@ class TestGetFullContent:with (patch("builtins.open", mock_open()),patch.dict("sys.modules", {"pypdf": MagicMock()}),+ patch.object(+ engine_with_pdf,+ "_download_pdf_safely",+ return_value="/tmp/paper.pdf",+ ),):# We need to mock pypdf inside the methodimport sys@@ -529,11 +534,15 @@ class TestGetFullContent:def test_pdf_download_fails(self, engine_with_pdf):"""Download failure sets pdf_path to None and decrements counter."""paper = _make_mock_paper()- paper.download_pdf.side_effect = Exception("Network error")engine_with_pdf._papers = {paper.entry_id: paper}items = [{"id": paper.entry_id, "title": "T"}]- result = engine_with_pdf._get_full_content(items)+ with patch.object(+ engine_with_pdf,+ "_download_pdf_safely",+ side_effect=Exception("Network error"),+ ):+ result = engine_with_pdf._get_full_content(items)assert result[0]["pdf_path"] is Nonedef test_pdf_limit_reached(self, engine_with_pdf):@@ -669,6 +678,11 @@ class TestGetPaperDetails:with (patch.object(arxiv, "Client") as mock_client_cls,patch.object(arxiv, "Search"),+ patch.object(+ engine_with_pdf,+ "_download_pdf_safely",+ return_value="/tmp/paper.pdf",+ ) as mock_download,):mock_client = Mock()mock_client.results.return_value = [paper]@@ -676,17 +690,21 @@ class TestGetPaperDetails:result = engine_with_pdf.get_paper_details("2101.00001")assert result["pdf_path"] == "/tmp/paper.pdf"- paper.download_pdf.assert_called_once()+ mock_download.assert_called_once()def test_paper_details_pdf_download_fails(self, engine_with_pdf):"""PDF download failure in get_paper_details is handled gracefully."""import arxivpaper = _make_mock_paper()- paper.download_pdf.side_effect = Exception("download error")with (patch.object(arxiv, "Client") as mock_client_cls,patch.object(arxiv, "Search"),+ patch.object(+ engine_with_pdf,+ "_download_pdf_safely",+ side_effect=Exception("download error"),+ ),):mock_client = Mock()mock_client.results.return_value = [paper]diff --git a/tests/web_search_engines/engines/test_arxiv_pdf_egress_gating.py b/tests/web_search_engines/engines/test_arxiv_pdf_egress_gating.pynew file mode 100644index 0000000000..de67c06f4c--- /dev/null+++ b/tests/web_search_engines/engines/test_arxiv_pdf_egress_gating.py@@ -0,0 +1,227 @@+"""+Regression tests for the arXiv PDF download egress-gating hardening.++Invariant under test: every arXiv PDF fetch is routed through the project's+SSRF-validated ``SafeSession`` (SSRF pre-validation and DNS pinning) rather+than ``arxiv.Result.download_pdf``, which fetches via+``urllib.request.urlretrieve`` and bypasses all of those controls. The+fetch targets ``export.arxiv.org`` (arXiv's designated host for automated+access — the same default ``arxiv.Result.download_pdf`` uses and the host+already used for the metadata API), not the public ``arxiv.org`` host.++Note: this fetch passes ``stream=True``, so ``SafeSession``'s+response-size cap (``_check_response_size``) runs while the connection+is still open rather than after ``requests`` has already buffered the+whole body. An oversized response with a ``Content-Length`` header is+now rejected immediately, before any body bytes are read. When+``Content-Length`` is absent, the cap's guard correctly bounds memory+for a connection-close-delimited body, but not for one sent with+``Transfer-Encoding: chunked`` (the common case for a CDN of unknown+length): ``urllib3`` reads chunked bodies through ``read_chunked()``,+which bypasses the guard's patched ``read()``. That gap lives in the+cap itself (``_check_response_size``/``_install_body_guard`` in+``safe_requests.py``), not in this call site, and is unrelated to+#6172 (which tracks callers that never set ``stream=True`` at all —+this one now does).++These tests mock ``SafeSession`` wholesale, so they don't exercise+``_check_response_size`` — they only pin that the call site asks for+``stream=True`` and handles the response as a context manager.++These tests are written to be run by CI; they are not executed here.+"""++from pathlib import Path+from unittest.mock import MagicMock, Mock, patch++import pytest++ENGINE_MODULE = (+ "local_deep_research.web_search_engines.engines.search_engine_arxiv"+)+++def _make_engine(**kwargs):+ from local_deep_research.web_search_engines.engines.search_engine_arxiv import (+ ArXivSearchEngine,+ )++ with patch(+ "local_deep_research.advanced_search_system.filters."+ "journal_reputation_filter.JournalReputationFilter.create_default",+ return_value=None,+ ):+ return ArXivSearchEngine(**kwargs)+++def _mock_safe_session(pdf_bytes=b"%PDF-1.4 test"):+ """Return (SafeSession class mock, session instance mock, response mock)."""+ response = MagicMock()+ response.content = pdf_bytes+ response.raise_for_status = Mock()+ # The response is used as a context manager (`with session.get(...) as+ # response:`), same as a real requests.Response, whose __enter__+ # returns self.+ response.__enter__.return_value = response+ response.__exit__.return_value = False++ session = MagicMock()+ session.get.return_value = response++ session_cls = MagicMock()+ session_cls.return_value.__enter__.return_value = session+ session_cls.return_value.__exit__.return_value = False+ return session_cls, session, response+++class TestValidatedArxivId:+ """The id used to build the egress URL must be validated first."""++ @pytest.mark.parametrize(+ "entry_id,expected",+ [+ ("https://arxiv.org/abs/2101.12345", "2101.12345"),+ ("http://arxiv.org/abs/2301.12345v2", "2301.12345v2"),+ ("https://arxiv.org/abs/2401.01234", "2401.01234"),+ ("http://arxiv.org/abs/cond-mat/0501234", "cond-mat/0501234"),+ ("http://arxiv.org/abs/math.GT/0309136v1", "math.GT/0309136v1"),+ ],+ )+ def test_accepts_canonical_ids(self, entry_id, expected):+ from local_deep_research.web_search_engines.engines.search_engine_arxiv import (+ ArXivSearchEngine,+ )++ paper = Mock()+ paper.entry_id = entry_id+ assert ArXivSearchEngine._validated_arxiv_id(paper) == expected++ @pytest.mark.parametrize(+ "entry_id",+ [+ "https://arxiv.org/abs/../../etc/passwd",+ "http://arxiv.org/abs/2101.12345/../../../secret",+ "https://evil.example.com/abs/[email protected]",+ "http://arxiv.org/abs/not an id",+ "",+ "https://arxiv.org/abs/12345",+ ],+ )+ def test_rejects_malformed_ids(self, entry_id):+ from local_deep_research.web_search_engines.engines.search_engine_arxiv import (+ ArXivSearchEngine,+ )++ paper = Mock()+ paper.entry_id = entry_id+ assert ArXivSearchEngine._validated_arxiv_id(paper) is None+++class TestDownloadPdfSafely:+ """The download helper must use SafeSession and never download_pdf."""++ def test_routes_through_safe_session(self, tmp_path):+ engine = _make_engine()+ paper = Mock()+ paper.entry_id = "https://arxiv.org/abs/2101.12345"++ session_cls, session, _ = _mock_safe_session(b"%PDF-1.4 payload")++ with patch(f"{ENGINE_MODULE}.SafeSession", session_cls):+ path = engine._download_pdf_safely(paper, str(tmp_path))++ # SafeSession was constructed and used as the fetch transport.+ assert session_cls.called+ session.get.assert_called_once()+ called_url = session.get.call_args.args[0]+ assert called_url == "https://export.arxiv.org/pdf/2101.12345.pdf"++ # arxiv.Result.download_pdf (the ungated urlretrieve path) is never used.+ assert not paper.download_pdf.called++ # Bytes were written to the returned path inside dirpath.+ assert Path(path).parent == tmp_path+ assert Path(path).is_file()+ with open(path, "rb") as fh:+ assert fh.read() == b"%PDF-1.4 payload"++ def test_uses_streaming_get(self, tmp_path):+ """stream=True lets SafeSession's response-size cap run before the+ body is buffered instead of after — see the module docstring."""+ engine = _make_engine()+ paper = Mock()+ paper.entry_id = "https://arxiv.org/abs/2101.12345"++ session_cls, session, _ = _mock_safe_session()+ with patch(f"{ENGINE_MODULE}.SafeSession", session_cls):+ engine._download_pdf_safely(paper, str(tmp_path))++ assert session.get.call_args.kwargs["stream"] is True++ def test_url_is_built_from_validated_id(self, tmp_path):+ engine = _make_engine()+ paper = Mock()+ paper.entry_id = "http://arxiv.org/abs/2301.12345v3"++ session_cls, session, _ = _mock_safe_session()+ with patch(f"{ENGINE_MODULE}.SafeSession", session_cls):+ engine._download_pdf_safely(paper, str(tmp_path))++ assert (+ session.get.call_args.args[0]+ == "https://export.arxiv.org/pdf/2301.12345v3.pdf"+ )++ def test_raises_and_makes_no_request_for_bad_id(self, tmp_path):+ engine = _make_engine()+ paper = Mock()+ paper.entry_id = "https://evil.example.com/abs/2101.12345"++ session_cls, session, _ = _mock_safe_session()+ with patch(f"{ENGINE_MODULE}.SafeSession", session_cls):+ with pytest.raises(ValueError):+ engine._download_pdf_safely(paper, str(tmp_path))++ # No egress attempted for an id that fails validation.+ assert not session.get.called+ assert not session_cls.called+++class TestFullContentUsesGatedPath:+ """The higher-level full-content path must also avoid download_pdf."""++ def test_get_full_content_uses_safe_session(self, tmp_path):+ engine = _make_engine(+ include_full_text=True,+ download_dir=str(tmp_path),+ max_full_text=1,+ )++ paper = Mock()+ paper.entry_id = "https://arxiv.org/abs/2101.12345"+ paper.pdf_url = "https://arxiv.org/pdf/2101.12345.pdf"+ paper.authors = []+ paper.published = None+ paper.updated = None+ paper.categories = ["cs.AI"]+ paper.summary = "Summary text"+ paper.comment = None+ paper.doi = None+ paper.journal_ref = None+ engine._papers = {"https://arxiv.org/abs/2101.12345": paper}++ item = {"id": "https://arxiv.org/abs/2101.12345"}++ session_cls, session, _ = _mock_safe_session(b"%PDF-1.4 body")+ # Neutralise the PDF text extractors so the test stays focused on+ # the egress transport rather than parsing.+ with (+ patch(f"{ENGINE_MODULE}.SafeSession", session_cls),+ patch("pypdf.PdfReader", side_effect=Exception("skip")),+ ):+ results = engine._get_full_content([item])++ # The gated session performed the fetch; download_pdf was never called.+ assert session.get.called+ assert not paper.download_pdf.called+ assert results[0]["pdf_path"] is not None