Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

harden(arxiv): route PDF downloads through the SSRF-validated session (#5807)

LearningCircuit994df671330f
diff --git a/.github/scripts/check-file-writes.sh b/.github/scripts/check-file-writes.sh
index fbf63b2fda..fdffe9f1e8 100755
--- a/.github/scripts/check-file-writes.sh
+++ b/.github/scripts/check-file-writes.sh
@@ -252,10 +252,14 @@ if [ -n "$ALL_MATCHES" ]; then
# - vector_stores/legacy_cleanup.py — writes the text-free
# ``.idmap.json`` migration sidecar (plain position->uuid JSON map,
# no chunk text; see the module docstring / `_ID_RE`)
+ # - web_search_engines/engines/search_engine_arxiv.py — writes the
+ # downloaded PDF of a public arXiv paper into the caller-provided
+ # download dir (public published document, no PII/secrets; the fetch
+ # itself goes through the SSRF-validated SafeSession egress gate)
# If you add an entry here, document WHY the file's writes are safe
# (public data, not user-specific, not encrypted at rest by design).
if [ "$skip_line" -eq 0 ]; then
- if echo "$line" | grep -qE "web/app_factory\.py|document_loaders/bytes_loader\.py|journal_quality/downloader\.py|journal_quality/data_sources/.+\.py|vector_stores/implementations/faiss_store\.py|vector_stores/legacy_cleanup\.py"; then
+ if echo "$line" | grep -qE "web/app_factory\.py|document_loaders/bytes_loader\.py|journal_quality/downloader\.py|journal_quality/data_sources/.+\.py|vector_stores/implementations/faiss_store\.py|vector_stores/legacy_cleanup\.py|web_search_engines/engines/search_engine_arxiv\.py"; then
skip_line=1
fi
fi
diff --git a/changelog.d/5807.security.md b/changelog.d/5807.security.md
new file mode 100644
index 0000000000..b95aa9171d
--- /dev/null
+++ b/changelog.d/5807.security.md
@@ -0,0 +1,20 @@
+arXiv PDF downloads now go through the same SSRF-validated session
+(with SSRF pre-validation and DNS pinning) as the rest of the arXiv
+integration, instead of the arxiv library's built-in download path, and
+fetch from export.arxiv.org (arXiv's designated host for automated
+access, matching the arxiv library's own default `download_domain` and
+the host already used for the metadata API) rather than the public
+arxiv.org host.
+This fetch now also passes `stream=True`, so SafeSession's response-size
+cap runs before the body is buffered rather than after: an oversized
+response with a `Content-Length` header is now rejected immediately,
+instead of being fully read into memory first and only then discarded.
+When `Content-Length` is absent, the cap's guard now correctly bounds
+memory for a connection-close-delimited body, but not for one sent with
+`Transfer-Encoding: chunked` (the common case for a CDN of unknown
+length) -- `urllib3` reads chunked bodies through a path that bypasses
+the guard. That remaining gap lives in SafeSession's response-size cap
+itself (`_check_response_size`/`_install_body_guard` in
+`safe_requests.py`), affects every `SafeSession` caller that already
+streams, and needs its own follow-up; it is not the same issue as
+#6172, which tracks callers that don't set `stream=True` at all.
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py b/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py
index 9c73f3a862..ffe30a534a 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py
@@ -1,13 +1,26 @@
+import re
+from pathlib import Path
from typing import Any, Dict, List, Optional
import arxiv
from langchain_core.language_models import BaseLLM
from ...constants import SNIPPET_LENGTH_SHORT
+from ...security import SafeSession
+from ...security.directory_creation import create_directory
+from ...security.safe_requests import DEFAULT_TIMEOUT
from ...security.secure_logging import logger
from ..rate_limiting import RateLimitError
from ..search_engine_base import BaseSearchEngine, Exposure, Sensitivity
+# Canonical arXiv identifier shapes, anchored so nothing else slips through
+# before we build an egress URL from the value:
+# - new style: 2301.12345 / 2301.12345v2 (4-or-5 digit sequence)
+# - old style: math.GT/0309136 or cond-mat/0501234 (with optional version)
+_ARXIV_ID_RE = re.compile(
+ r"^(?:\d{4}\.\d{4,5}(?:v\d+)?|[a-z-]+(?:\.[A-Z]{2})?/\d{7}(?:v\d+)?)$"
+)
+
class ArXivSearchEngine(BaseSearchEngine):
"""arXiv search engine implementation with two-phase approach"""
@@ -123,6 +136,122 @@ class ArXivSearchEngine(BaseSearchEngine):
# Get the search results
return list(client.results(search))
+ @staticmethod
+ def _validated_arxiv_id(paper: Any) -> Optional[str]:
+ """Return a canonical, validated arXiv id for ``paper`` or ``None``.
+
+ The id is derived from the arxiv-provided ``entry_id`` (e.g.
+ ``http://arxiv.org/abs/2301.12345v1``) and matched against
+ ``_ARXIV_ID_RE`` before it is ever used to build an egress URL,
+ so a malformed or unexpected value cannot be interpolated into a
+ request target.
+ """
+ entry_id = getattr(paper, "entry_id", "") or ""
+ match = re.search(r"arxiv\.org/abs/(.+)$", entry_id)
+ candidate = (match.group(1) if match else entry_id).strip()
+ if _ARXIV_ID_RE.match(candidate):
+ return candidate
+ return None
+
+ def _download_pdf_safely(self, paper: Any, dirpath: str) -> str:
+ """Download a paper PDF through the SSRF-validated ``SafeSession``.
+
+ Invariant: every arXiv PDF fetch goes through the same egress gate
+ the rest of the arXiv integration uses. ``SafeSession`` applies SSRF
+ pre-validation and DNS pinning to this fetch. This replaces
+ ``arxiv.Result.download_pdf``, which fetches via
+ ``urllib.request.urlretrieve`` and therefore bypasses all of those
+ controls. Behavior is preserved: the PDF is written into ``dirpath``
+ and the resulting path is returned.
+
+ Note: this call passes ``stream=True``, so ``requests.Session.send()``
+ does *not* drain the body itself (it only does that when
+ ``stream`` is falsy — see ``requests.sessions.Session.send``);
+ ``SafeSession``'s response-size cap (``_check_response_size``) runs
+ while the connection is still open, before the call site reads
+ ``response.content``. That fixes the case this note used to
+ describe: with ``Content-Length`` present and over the limit, the
+ cap now rejects immediately after the headers arrive, before any
+ body bytes are buffered, instead of buffering the whole oversized
+ body first and only then discarding it.
+
+ It does not fully fix the ``Content-Length``-absent case. The cap
+ installs a guard on ``response.raw.read()``
+ (``_install_body_guard``), and that guard does run correctly — but
+ only when the connection is delimited by the socket closing (no
+ ``Content-Length``, no ``Transfer-Encoding``). When the origin uses
+ ``Transfer-Encoding: chunked`` instead — the common case for a CDN
+ serving a PDF of unknown length — ``urllib3``'s
+ ``HTTPResponse.stream()`` (what ``requests`` uses under
+ ``iter_content``/``.content``) reads such bodies through
+ ``read_chunked()``, which pulls bytes directly off the socket via
+ ``self._fp._safe_read()`` and never calls the patched
+ ``read()`` — verified by reading the installed ``urllib3``
+ (2.7.0) source and confirming with a live chunked-response
+ server: the patched ``read()`` recorded zero calls while a full
+ chunked body was consumed. So for a genuinely chunked response,
+ peak memory is still unbounded here, same as before this change.
+
+ That gap is a defect in ``_check_response_size``/
+ ``_install_body_guard`` in ``safe_requests.py`` itself — every
+ ``SafeSession`` caller that already uses ``stream=True`` (e.g.
+ ``research_library/downloaders/generic.py``) has the same
+ exposure for chunked responses. It is independent of whether this
+ call streams and is not the same issue as #6172, which is about
+ callers that never set ``stream=True`` at all (this one now
+ does). It needs its own follow-up rather than being fixed here.
+ """
+ arxiv_id = self._validated_arxiv_id(paper)
+ if not arxiv_id:
+ raise ValueError(
+ "Could not derive a valid arXiv id for PDF download"
+ )
+
+ # Build the download URL from the validated id rather than trusting
+ # an arbitrary attribute value; SafeSession re-validates it anyway.
+ # Use export.arxiv.org (not the public arxiv.org CDN-fronted host)
+ # to match arxiv.Result.download_pdf's default `download_domain`
+ # (see the installed `arxiv` package, ~v2.4) and the host
+ # `arxiv.Client`/`arxiv.Search` already use for the metadata API
+ # (`query_url_format = "https://export.arxiv.org/api/query?..."`).
+ # export.arxiv.org is arXiv's designated host for automated/scripted
+ # access; the public arxiv.org host is rate-limited and
+ # bot-challenged for that traffic.
+ pdf_url = f"https://export.arxiv.org/pdf/{arxiv_id}.pdf"
+
+ directory = Path(dirpath)
+ create_directory(directory, context="arXiv PDF download directory")
+ target_path = directory / f"{arxiv_id.replace('/', '_')}.pdf"
+
+ with SafeSession() as session:
+ # stream=True is intentional here — DO NOT remove it. It lets
+ # SafeSession's response-size cap (_check_response_size) run
+ # while the connection is still open, instead of after
+ # `requests` has already buffered the whole body (see the
+ # docstring note above). The `with ... as response:` ensures
+ # the connection is released on every path, including a
+ # rejection raised by the cap itself, mirroring the same
+ # stream=True + context-manager pattern already used in
+ # research_library/downloaders/generic.py.
+ with session.get(
+ pdf_url,
+ timeout=DEFAULT_TIMEOUT,
+ allow_redirects=True,
+ stream=True,
+ ) as response:
+ response.raise_for_status()
+ # `.content` still returns the full body unchanged — see
+ # the docstring note above for what moving the read here
+ # (rather than inside `send()`) does and does not fix.
+ pdf_bytes = response.content
+
+ # Public arXiv PDF (a published paper, no PII/secrets) written into the
+ # caller-provided download dir; see the allowlist note in
+ # .github/scripts/check-file-writes.sh.
+ target_path.write_bytes(pdf_bytes)
+
+ return str(target_path)
+
def _get_previews(self, query: str) -> List[Dict[str, Any]]:
"""
Get preview information for arXiv papers.
@@ -271,8 +400,8 @@ class ArXivSearchEngine(BaseSearchEngine):
# Apply rate limiting before PDF download
self.rate_tracker.apply_rate_limit(self.engine_type)
- paper_path = paper.download_pdf(
- dirpath=self.download_dir
+ paper_path = self._download_pdf_safely(
+ paper, self.download_dir
)
result["pdf_path"] = str(paper_path)
@@ -456,8 +585,10 @@ class ArXivSearchEngine(BaseSearchEngine):
# Apply rate limiting before PDF download
self.rate_tracker.apply_rate_limit(self.engine_type)
- # Download the paper
- paper_path = paper.download_pdf(dirpath=self.download_dir)
+ # Download the paper through the SSRF-validated session
+ paper_path = self._download_pdf_safely(
+ paper, self.download_dir
+ )
result["pdf_path"] = str(paper_path)
except Exception as e:
safe_msg = self._scrub_error(e)
diff --git a/tests/web_search_engines/engines/test_arxiv_coverage.py b/tests/web_search_engines/engines/test_arxiv_coverage.py
index f22f388620..3691aa17e9 100644
--- a/tests/web_search_engines/engines/test_arxiv_coverage.py
+++ b/tests/web_search_engines/engines/test_arxiv_coverage.py
@@ -421,6 +421,11 @@ class TestGetFullContent:
with (
patch("builtins.open", mock_open()),
patch.dict("sys.modules", {"pypdf": MagicMock()}),
+ patch.object(
+ engine_with_pdf,
+ "_download_pdf_safely",
+ return_value="/tmp/paper.pdf",
+ ),
):
# We need to mock pypdf inside the method
import sys
@@ -529,11 +534,15 @@ class TestGetFullContent:
def test_pdf_download_fails(self, engine_with_pdf):
"""Download failure sets pdf_path to None and decrements counter."""
paper = _make_mock_paper()
- paper.download_pdf.side_effect = Exception("Network error")
engine_with_pdf._papers = {paper.entry_id: paper}
items = [{"id": paper.entry_id, "title": "T"}]
- result = engine_with_pdf._get_full_content(items)
+ with patch.object(
+ engine_with_pdf,
+ "_download_pdf_safely",
+ side_effect=Exception("Network error"),
+ ):
+ result = engine_with_pdf._get_full_content(items)
assert result[0]["pdf_path"] is None
def test_pdf_limit_reached(self, engine_with_pdf):
@@ -669,6 +678,11 @@ class TestGetPaperDetails:
with (
patch.object(arxiv, "Client") as mock_client_cls,
patch.object(arxiv, "Search"),
+ patch.object(
+ engine_with_pdf,
+ "_download_pdf_safely",
+ return_value="/tmp/paper.pdf",
+ ) as mock_download,
):
mock_client = Mock()
mock_client.results.return_value = [paper]
@@ -676,17 +690,21 @@ class TestGetPaperDetails:
result = engine_with_pdf.get_paper_details("2101.00001")
assert result["pdf_path"] == "/tmp/paper.pdf"
- paper.download_pdf.assert_called_once()
+ mock_download.assert_called_once()
def test_paper_details_pdf_download_fails(self, engine_with_pdf):
"""PDF download failure in get_paper_details is handled gracefully."""
import arxiv
paper = _make_mock_paper()
- paper.download_pdf.side_effect = Exception("download error")
with (
patch.object(arxiv, "Client") as mock_client_cls,
patch.object(arxiv, "Search"),
+ patch.object(
+ engine_with_pdf,
+ "_download_pdf_safely",
+ side_effect=Exception("download error"),
+ ),
):
mock_client = Mock()
mock_client.results.return_value = [paper]
diff --git a/tests/web_search_engines/engines/test_arxiv_pdf_egress_gating.py b/tests/web_search_engines/engines/test_arxiv_pdf_egress_gating.py
new file mode 100644
index 0000000000..de67c06f4c
--- /dev/null
+++ b/tests/web_search_engines/engines/test_arxiv_pdf_egress_gating.py
@@ -0,0 +1,227 @@
+"""
+Regression tests for the arXiv PDF download egress-gating hardening.
+
+Invariant under test: every arXiv PDF fetch is routed through the project's
+SSRF-validated ``SafeSession`` (SSRF pre-validation and DNS pinning) rather
+than ``arxiv.Result.download_pdf``, which fetches via
+``urllib.request.urlretrieve`` and bypasses all of those controls. The
+fetch targets ``export.arxiv.org`` (arXiv's designated host for automated
+access — the same default ``arxiv.Result.download_pdf`` uses and the host
+already used for the metadata API), not the public ``arxiv.org`` host.
+
+Note: this fetch passes ``stream=True``, so ``SafeSession``'s
+response-size cap (``_check_response_size``) runs while the connection
+is still open rather than after ``requests`` has already buffered the
+whole body. An oversized response with a ``Content-Length`` header is
+now rejected immediately, before any body bytes are read. When
+``Content-Length`` is absent, the cap's guard correctly bounds memory
+for a connection-close-delimited body, but not for one sent with
+``Transfer-Encoding: chunked`` (the common case for a CDN of unknown
+length): ``urllib3`` reads chunked bodies through ``read_chunked()``,
+which bypasses the guard's patched ``read()``. That gap lives in the
+cap itself (``_check_response_size``/``_install_body_guard`` in
+``safe_requests.py``), not in this call site, and is unrelated to
+#6172 (which tracks callers that never set ``stream=True`` at all —
+this one now does).
+
+These tests mock ``SafeSession`` wholesale, so they don't exercise
+``_check_response_size`` — they only pin that the call site asks for
+``stream=True`` and handles the response as a context manager.
+
+These tests are written to be run by CI; they are not executed here.
+"""
+
+from pathlib import Path
+from unittest.mock import MagicMock, Mock, patch
+
+import pytest
+
+ENGINE_MODULE = (
+ "local_deep_research.web_search_engines.engines.search_engine_arxiv"
+)
+
+
+def _make_engine(**kwargs):
+ from local_deep_research.web_search_engines.engines.search_engine_arxiv import (
+ ArXivSearchEngine,
+ )
+
+ with patch(
+ "local_deep_research.advanced_search_system.filters."
+ "journal_reputation_filter.JournalReputationFilter.create_default",
+ return_value=None,
+ ):
+ return ArXivSearchEngine(**kwargs)
+
+
+def _mock_safe_session(pdf_bytes=b"%PDF-1.4 test"):
+ """Return (SafeSession class mock, session instance mock, response mock)."""
+ response = MagicMock()
+ response.content = pdf_bytes
+ response.raise_for_status = Mock()
+ # The response is used as a context manager (`with session.get(...) as
+ # response:`), same as a real requests.Response, whose __enter__
+ # returns self.
+ response.__enter__.return_value = response
+ response.__exit__.return_value = False
+
+ session = MagicMock()
+ session.get.return_value = response
+
+ session_cls = MagicMock()
+ session_cls.return_value.__enter__.return_value = session
+ session_cls.return_value.__exit__.return_value = False
+ return session_cls, session, response
+
+
+class TestValidatedArxivId:
+ """The id used to build the egress URL must be validated first."""
+
+ @pytest.mark.parametrize(
+ "entry_id,expected",
+ [
+ ("https://arxiv.org/abs/2101.12345", "2101.12345"),
+ ("http://arxiv.org/abs/2301.12345v2", "2301.12345v2"),
+ ("https://arxiv.org/abs/2401.01234", "2401.01234"),
+ ("http://arxiv.org/abs/cond-mat/0501234", "cond-mat/0501234"),
+ ("http://arxiv.org/abs/math.GT/0309136v1", "math.GT/0309136v1"),
+ ],
+ )
+ def test_accepts_canonical_ids(self, entry_id, expected):
+ from local_deep_research.web_search_engines.engines.search_engine_arxiv import (
+ ArXivSearchEngine,
+ )
+
+ paper = Mock()
+ paper.entry_id = entry_id
+ assert ArXivSearchEngine._validated_arxiv_id(paper) == expected
+
+ @pytest.mark.parametrize(
+ "entry_id",
+ [
+ "https://arxiv.org/abs/../../etc/passwd",
+ "http://arxiv.org/abs/2101.12345/../../../secret",
+ "https://evil.example.com/abs/[email protected]",
+ "http://arxiv.org/abs/not an id",
+ "",
+ "https://arxiv.org/abs/12345",
+ ],
+ )
+ def test_rejects_malformed_ids(self, entry_id):
+ from local_deep_research.web_search_engines.engines.search_engine_arxiv import (
+ ArXivSearchEngine,
+ )
+
+ paper = Mock()
+ paper.entry_id = entry_id
+ assert ArXivSearchEngine._validated_arxiv_id(paper) is None
+
+
+class TestDownloadPdfSafely:
+ """The download helper must use SafeSession and never download_pdf."""
+
+ def test_routes_through_safe_session(self, tmp_path):
+ engine = _make_engine()
+ paper = Mock()
+ paper.entry_id = "https://arxiv.org/abs/2101.12345"
+
+ session_cls, session, _ = _mock_safe_session(b"%PDF-1.4 payload")
+
+ with patch(f"{ENGINE_MODULE}.SafeSession", session_cls):
+ path = engine._download_pdf_safely(paper, str(tmp_path))
+
+ # SafeSession was constructed and used as the fetch transport.
+ assert session_cls.called
+ session.get.assert_called_once()
+ called_url = session.get.call_args.args[0]
+ assert called_url == "https://export.arxiv.org/pdf/2101.12345.pdf"
+
+ # arxiv.Result.download_pdf (the ungated urlretrieve path) is never used.
+ assert not paper.download_pdf.called
+
+ # Bytes were written to the returned path inside dirpath.
+ assert Path(path).parent == tmp_path
+ assert Path(path).is_file()
+ with open(path, "rb") as fh:
+ assert fh.read() == b"%PDF-1.4 payload"
+
+ def test_uses_streaming_get(self, tmp_path):
+ """stream=True lets SafeSession's response-size cap run before the
+ body is buffered instead of after — see the module docstring."""
+ engine = _make_engine()
+ paper = Mock()
+ paper.entry_id = "https://arxiv.org/abs/2101.12345"
+
+ session_cls, session, _ = _mock_safe_session()
+ with patch(f"{ENGINE_MODULE}.SafeSession", session_cls):
+ engine._download_pdf_safely(paper, str(tmp_path))
+
+ assert session.get.call_args.kwargs["stream"] is True
+
+ def test_url_is_built_from_validated_id(self, tmp_path):
+ engine = _make_engine()
+ paper = Mock()
+ paper.entry_id = "http://arxiv.org/abs/2301.12345v3"
+
+ session_cls, session, _ = _mock_safe_session()
+ with patch(f"{ENGINE_MODULE}.SafeSession", session_cls):
+ engine._download_pdf_safely(paper, str(tmp_path))
+
+ assert (
+ session.get.call_args.args[0]
+ == "https://export.arxiv.org/pdf/2301.12345v3.pdf"
+ )
+
+ def test_raises_and_makes_no_request_for_bad_id(self, tmp_path):
+ engine = _make_engine()
+ paper = Mock()
+ paper.entry_id = "https://evil.example.com/abs/2101.12345"
+
+ session_cls, session, _ = _mock_safe_session()
+ with patch(f"{ENGINE_MODULE}.SafeSession", session_cls):
+ with pytest.raises(ValueError):
+ engine._download_pdf_safely(paper, str(tmp_path))
+
+ # No egress attempted for an id that fails validation.
+ assert not session.get.called
+ assert not session_cls.called
+
+
+class TestFullContentUsesGatedPath:
+ """The higher-level full-content path must also avoid download_pdf."""
+
+ def test_get_full_content_uses_safe_session(self, tmp_path):
+ engine = _make_engine(
+ include_full_text=True,
+ download_dir=str(tmp_path),
+ max_full_text=1,
+ )
+
+ paper = Mock()
+ paper.entry_id = "https://arxiv.org/abs/2101.12345"
+ paper.pdf_url = "https://arxiv.org/pdf/2101.12345.pdf"
+ paper.authors = []
+ paper.published = None
+ paper.updated = None
+ paper.categories = ["cs.AI"]
+ paper.summary = "Summary text"
+ paper.comment = None
+ paper.doi = None
+ paper.journal_ref = None
+ engine._papers = {"https://arxiv.org/abs/2101.12345": paper}
+
+ item = {"id": "https://arxiv.org/abs/2101.12345"}
+
+ session_cls, session, _ = _mock_safe_session(b"%PDF-1.4 body")
+ # Neutralise the PDF text extractors so the test stays focused on
+ # the egress transport rather than parsing.
+ with (
+ patch(f"{ENGINE_MODULE}.SafeSession", session_cls),
+ patch("pypdf.PdfReader", side_effect=Exception("skip")),
+ ):
+ results = engine._get_full_content([item])
+
+ # The gated session performed the fetch; download_pdf was never called.
+ assert session.get.called
+ assert not paper.download_pdf.called
+ assert results[0]["pdf_path"] is not None