Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(downloaders): preserve semicolon suffixes when extracting DOI URLs (#8001)

HPH Proton4be446622e86
diff --git a/changelog.d/7984.bugfix.md b/changelog.d/7984.bugfix.md
new file mode 100644
index 0000000000..3c15cdf07b
--- /dev/null
+++ b/changelog.d/7984.bugfix.md
@@ -0,0 +1 @@
+DOI open-access downloads keep semicolon suffixes in legacy SICI identifiers and decode percent-encoded punctuation in doi.org URLs.
diff --git a/src/local_deep_research/research_library/downloaders/open_access_doi.py b/src/local_deep_research/research_library/downloaders/open_access_doi.py
index 45c5281105..83bf360ec6 100644
--- a/src/local_deep_research/research_library/downloaders/open_access_doi.py
+++ b/src/local_deep_research/research_library/downloaders/open_access_doi.py
@@ -19,7 +19,7 @@ see ``library/download_management/failure_classifier.py``):
"""
from typing import Optional, Tuple
-from urllib.parse import urlparse
+from urllib.parse import unquote, urlsplit
import requests
from loguru import logger
@@ -52,7 +52,7 @@ class OpenAccessDOIDownloader(BaseDownloader):
def can_handle(self, url: str) -> bool:
"""Handle canonical DOI URLs (doi.org / dx.doi.org)."""
try:
- hostname = (urlparse(url).hostname or "").lower()
+ hostname = (urlsplit(url).hostname or "").lower()
except (ValueError, AttributeError, TypeError):
return False
return hostname in ("doi.org", "dx.doi.org")
@@ -109,15 +109,23 @@ class OpenAccessDOIDownloader(BaseDownloader):
@staticmethod
def _extract_doi(url: str) -> Optional[str]:
- """Extract the DOI from a doi.org / dx.doi.org URL."""
+ """Extract the DOI from a doi.org / dx.doi.org URL.
+
+ ``urlsplit`` keeps a semicolon inside the path. ``urlparse`` would
+ move a legacy SICI suffix such as ``;2-8`` into ``params`` and drop
+ it. The query and fragment are removed before percent-decoding, so
+ ``?query`` / ``#fragment`` stay out of the identifier while ``%3C``,
+ ``%3E``, and ``%3B`` in the path become ``<``, ``>``, and ``;``.
+ """
try:
- parsed = urlparse(url)
+ parsed = urlsplit(url)
except (ValueError, AttributeError, TypeError):
return None
hostname = (parsed.hostname or "").lower()
if hostname not in ("doi.org", "dx.doi.org"):
return None
- return parsed.path.lstrip("/") or None
+ doi = unquote(parsed.path.lstrip("/")).strip()
+ return doi or None
def _resolve_oa_pdf(self, doi: str) -> Tuple[Optional[str], str]:
"""
diff --git a/tests/research_library/downloaders/test_open_access_doi_downloader.py b/tests/research_library/downloaders/test_open_access_doi_downloader.py
index e09889ae5c..193a032baa 100644
--- a/tests/research_library/downloaders/test_open_access_doi_downloader.py
+++ b/tests/research_library/downloaders/test_open_access_doi_downloader.py
@@ -53,6 +53,16 @@ def _pdf_response(pdf_content):
return r
+# Canonical legacy SICI DOI. Percent-encoded and literal doi.org URLs
+# must both normalize to this string, and a query or fragment must not.
+_SICI_DOI = "10.1002/(SICI)1099-1409(199908/10)3:6/7<672::AID-JPP192>3.0.CO;2-8"
+_SICI_PATHS = (
+ "10.1002/(SICI)1099-1409(199908/10)3:6/7%3C672::AID-JPP192%3E3.0.CO;2-8",
+ "10.1002/(SICI)1099-1409(199908/10)3:6/7<672::AID-JPP192>3.0.CO;2-8",
+ "10.1002/(SICI)1099-1409(199908/10)3:6/7%3C672::AID-JPP192%3E3.0.CO%3B2-8",
+)
+
+
class TestOpenAccessDOIDownloaderInit:
def test_preserves_download_service_context(self):
snapshot = {"research_library.auto_persist_research_sources": False}
@@ -128,6 +138,21 @@ class TestOpenAccessDOIDownloaderExtractDoi:
OpenAccessDOIDownloader()._extract_doi("https://doi.org/") is None
)
+ @pytest.mark.parametrize("host", ["doi.org", "dx.doi.org"])
+ @pytest.mark.parametrize("path", _SICI_PATHS)
+ @pytest.mark.parametrize("suffix", ["", "?query=1#frag"])
+ def test_preserves_semicolon_suffix(self, host, path, suffix):
+ """SICI suffixes survive parsing and percent-decoding.
+
+ Literal ``<``/``>``/``;`` and ``%3C``/``%3E``/``%3B`` normalize to
+ the same DOI. ``?query=1#frag`` is removed with the split, before
+ decoding, so it is not part of the identifier.
+ """
+ doi = OpenAccessDOIDownloader()._extract_doi(
+ f"https://{host}/{path}{suffix}"
+ )
+ assert doi == _SICI_DOI
+
class TestOpenAccessDOIDownloaderResolve:
def test_best_location_pdf(self):
@@ -371,6 +396,38 @@ class TestOpenAccessDOIDownloaderDownloadWithResult:
assert result.is_success is False
assert result.skip_reason is not None
+ @pytest.mark.parametrize("host", ["doi.org", "dx.doi.org"])
+ def test_sici_suffix_is_sent_in_openalex_filter(
+ self, host, mock_pdf_content
+ ):
+ """The OpenAlex filter receives the full normalized SICI DOI."""
+ d = OpenAccessDOIDownloader()
+ url = (
+ f"https://{host}/10.1002/(SICI)1099-1409(199908/10)3:6/7"
+ "%3C672::AID-JPP192%3E3.0.CO;2-8"
+ )
+ with patch.object(
+ d.session,
+ "get",
+ side_effect=[
+ _api_response(
+ _oa_work(pdf_url="https://example.com/paper.pdf")
+ ),
+ _pdf_response(mock_pdf_content),
+ ],
+ ) as session_get:
+ result = d.download_with_result(url)
+
+ assert result.is_success is True
+ assert result.content == mock_pdf_content
+ filters = [
+ call.kwargs["params"]["filter"]
+ for call in session_get.call_args_list
+ if isinstance(call.kwargs.get("params"), dict)
+ and "filter" in call.kwargs["params"]
+ ]
+ assert filters == [f"doi:{_SICI_DOI}"]
+
@pytest.fixture
def doi_logs():