~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(downloaders): preserve semicolon suffixes when extracting DOI URLs (#8001)
HPH Proton4be446622e86
diff --git a/changelog.d/7984.bugfix.md b/changelog.d/7984.bugfix.mdnew file mode 100644index 0000000000..3c15cdf07b--- /dev/null+++ b/changelog.d/7984.bugfix.md@@ -0,0 +1 @@+DOI open-access downloads keep semicolon suffixes in legacy SICI identifiers and decode percent-encoded punctuation in doi.org URLs.diff --git a/src/local_deep_research/research_library/downloaders/open_access_doi.py b/src/local_deep_research/research_library/downloaders/open_access_doi.pyindex 45c5281105..83bf360ec6 100644--- a/src/local_deep_research/research_library/downloaders/open_access_doi.py+++ b/src/local_deep_research/research_library/downloaders/open_access_doi.py@@ -19,7 +19,7 @@ see ``library/download_management/failure_classifier.py``):"""from typing import Optional, Tuple-from urllib.parse import urlparse+from urllib.parse import unquote, urlsplitimport requestsfrom loguru import logger@@ -52,7 +52,7 @@ class OpenAccessDOIDownloader(BaseDownloader):def can_handle(self, url: str) -> bool:"""Handle canonical DOI URLs (doi.org / dx.doi.org)."""try:- hostname = (urlparse(url).hostname or "").lower()+ hostname = (urlsplit(url).hostname or "").lower()except (ValueError, AttributeError, TypeError):return Falsereturn hostname in ("doi.org", "dx.doi.org")@@ -109,15 +109,23 @@ class OpenAccessDOIDownloader(BaseDownloader):@staticmethoddef _extract_doi(url: str) -> Optional[str]:- """Extract the DOI from a doi.org / dx.doi.org URL."""+ """Extract the DOI from a doi.org / dx.doi.org URL.++ ``urlsplit`` keeps a semicolon inside the path. ``urlparse`` would+ move a legacy SICI suffix such as ``;2-8`` into ``params`` and drop+ it. The query and fragment are removed before percent-decoding, so+ ``?query`` / ``#fragment`` stay out of the identifier while ``%3C``,+ ``%3E``, and ``%3B`` in the path become ``<``, ``>``, and ``;``.+ """try:- parsed = urlparse(url)+ parsed = urlsplit(url)except (ValueError, AttributeError, TypeError):return Nonehostname = (parsed.hostname or "").lower()if hostname not in ("doi.org", "dx.doi.org"):return None- return parsed.path.lstrip("/") or None+ doi = unquote(parsed.path.lstrip("/")).strip()+ return doi or Nonedef _resolve_oa_pdf(self, doi: str) -> Tuple[Optional[str], str]:"""diff --git a/tests/research_library/downloaders/test_open_access_doi_downloader.py b/tests/research_library/downloaders/test_open_access_doi_downloader.pyindex e09889ae5c..193a032baa 100644--- a/tests/research_library/downloaders/test_open_access_doi_downloader.py+++ b/tests/research_library/downloaders/test_open_access_doi_downloader.py@@ -53,6 +53,16 @@ def _pdf_response(pdf_content):return r+# Canonical legacy SICI DOI. Percent-encoded and literal doi.org URLs+# must both normalize to this string, and a query or fragment must not.+_SICI_DOI = "10.1002/(SICI)1099-1409(199908/10)3:6/7<672::AID-JPP192>3.0.CO;2-8"+_SICI_PATHS = (+ "10.1002/(SICI)1099-1409(199908/10)3:6/7%3C672::AID-JPP192%3E3.0.CO;2-8",+ "10.1002/(SICI)1099-1409(199908/10)3:6/7<672::AID-JPP192>3.0.CO;2-8",+ "10.1002/(SICI)1099-1409(199908/10)3:6/7%3C672::AID-JPP192%3E3.0.CO%3B2-8",+)++class TestOpenAccessDOIDownloaderInit:def test_preserves_download_service_context(self):snapshot = {"research_library.auto_persist_research_sources": False}@@ -128,6 +138,21 @@ class TestOpenAccessDOIDownloaderExtractDoi:OpenAccessDOIDownloader()._extract_doi("https://doi.org/") is None)+ @pytest.mark.parametrize("host", ["doi.org", "dx.doi.org"])+ @pytest.mark.parametrize("path", _SICI_PATHS)+ @pytest.mark.parametrize("suffix", ["", "?query=1#frag"])+ def test_preserves_semicolon_suffix(self, host, path, suffix):+ """SICI suffixes survive parsing and percent-decoding.++ Literal ``<``/``>``/``;`` and ``%3C``/``%3E``/``%3B`` normalize to+ the same DOI. ``?query=1#frag`` is removed with the split, before+ decoding, so it is not part of the identifier.+ """+ doi = OpenAccessDOIDownloader()._extract_doi(+ f"https://{host}/{path}{suffix}"+ )+ assert doi == _SICI_DOI+class TestOpenAccessDOIDownloaderResolve:def test_best_location_pdf(self):@@ -371,6 +396,38 @@ class TestOpenAccessDOIDownloaderDownloadWithResult:assert result.is_success is Falseassert result.skip_reason is not None+ @pytest.mark.parametrize("host", ["doi.org", "dx.doi.org"])+ def test_sici_suffix_is_sent_in_openalex_filter(+ self, host, mock_pdf_content+ ):+ """The OpenAlex filter receives the full normalized SICI DOI."""+ d = OpenAccessDOIDownloader()+ url = (+ f"https://{host}/10.1002/(SICI)1099-1409(199908/10)3:6/7"+ "%3C672::AID-JPP192%3E3.0.CO;2-8"+ )+ with patch.object(+ d.session,+ "get",+ side_effect=[+ _api_response(+ _oa_work(pdf_url="https://example.com/paper.pdf")+ ),+ _pdf_response(mock_pdf_content),+ ],+ ) as session_get:+ result = d.download_with_result(url)++ assert result.is_success is True+ assert result.content == mock_pdf_content+ filters = [+ call.kwargs["params"]["filter"]+ for call in session_get.call_args_list+ if isinstance(call.kwargs.get("params"), dict)+ and "filter" in call.kwargs["params"]+ ]+ assert filters == [f"doi:{_SICI_DOI}"]+@pytest.fixturedef doi_logs():