Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(security): pin https on journal dataset fetches via require_https (#6441)

LearningCircuitaadff6480b71
diff --git a/changelog.d/6441.security.md b/changelog.d/6441.security.md
new file mode 100644
index 0000000000..6a8d893566
--- /dev/null
+++ b/changelog.d/6441.security.md
@@ -0,0 +1 @@
+Journal-quality dataset fetches now pin HTTPS end to end. `safe_get` validated redirect hops for host and IP scope only, so a hop from `https://` to `http://` on a public host passed validation — a compromised upstream (the OpenAlex S3 bucket, DOAJ, GitHub raw) or any open redirect could move the fetch to cleartext, where an on-path attacker can tamper with the bytes that later drive journal reputation scores. A new opt-in `require_https` flag on `safe_get` refuses a non-https initial URL and any redirect hop that resolves to a non-https URL (relative and protocol-relative `Location` headers inherit https and are still followed) before the cleartext request is made, and refuses to run with `verify=False`. It is opt-in because localhost SearXNG instances are legitimately `http://`. All six journal-data fetch sites now set it — the DOAJ CSV, the JabRef and predatory-journal CSVs, and the OpenAlex sources and institutions manifests and their partitions — so every hop of each is carried over certificate-verified TLS. This protects transport only: it does not defend against a compromised origin, and the OpenAlex manifest's size/md5 check (#6438) remains a corruption guard rather than an authenticity check, since its declared values come from the same origin as the bytes they describe.
diff --git a/src/local_deep_research/journal_quality/data_sources/_openalex_common.py b/src/local_deep_research/journal_quality/data_sources/_openalex_common.py
index 1cc2d8d795..0b3ff9f444 100644
--- a/src/local_deep_research/journal_quality/data_sources/_openalex_common.py
+++ b/src/local_deep_research/journal_quality/data_sources/_openalex_common.py
@@ -213,7 +213,9 @@ def iter_partitions(
so body-stream transients (``ChunkedEncodingError``,
``ReadTimeout``) raised during ``resp.content`` are
retried inside the wrapper, not propagated to abort the
- whole multi-partition pull.
+ whole multi-partition pull. Must also accept
+ ``require_https=True``: partitions are fetched with the
+ scheme pinned so no redirect hop can move them to cleartext.
timeout: Per-partition HTTP timeout (seconds).
max_retries: Per-partition retry budget. Defaults higher than
``safe_get_with_retries``' generic 3 because partition
@@ -323,6 +325,7 @@ def iter_partitions(
consume_body=True,
max_retries=max_retries,
backoff_times=backoff_times,
+ require_https=True,
)
resp.raise_for_status()
meta = entry.get("meta") or {}
diff --git a/src/local_deep_research/journal_quality/data_sources/doaj.py b/src/local_deep_research/journal_quality/data_sources/doaj.py
index 6401ab21da..a148221687 100644
--- a/src/local_deep_research/journal_quality/data_sources/doaj.py
+++ b/src/local_deep_research/journal_quality/data_sources/doaj.py
@@ -67,7 +67,12 @@ class DOAJSource(DataSource):
# ChunkedEncodingError / ReadTimeout is a realistic failure
# mode worth retrying. Without this flag the body-read fires
# outside safe_get_with_retries' retry loop.
- resp = safe_get(_DOAJ_CSV_URL, timeout=120, consume_body=True)
+ resp = safe_get(
+ _DOAJ_CSV_URL,
+ timeout=120,
+ consume_body=True,
+ require_https=True,
+ )
resp.raise_for_status()
# DOAJ serves UTF-8 CSV. Parse in-memory — the whole file is
diff --git a/src/local_deep_research/journal_quality/data_sources/institutions.py b/src/local_deep_research/journal_quality/data_sources/institutions.py
index c60f00b8db..59a2208691 100644
--- a/src/local_deep_research/journal_quality/data_sources/institutions.py
+++ b/src/local_deep_research/journal_quality/data_sources/institutions.py
@@ -76,7 +76,10 @@ class InstitutionSource(DataSource):
# consume_body=True: small JSON but serial bottleneck — a body
# transient here aborts the whole 10-min institutions pull.
manifest_resp = safe_get(
- _OPENALEX_INSTITUTIONS_MANIFEST, timeout=30, consume_body=True
+ _OPENALEX_INSTITUTIONS_MANIFEST,
+ timeout=30,
+ consume_body=True,
+ require_https=True,
)
manifest_resp.raise_for_status()
manifest = manifest_resp.json()
diff --git a/src/local_deep_research/journal_quality/data_sources/jabref.py b/src/local_deep_research/journal_quality/data_sources/jabref.py
index 4a8e2611e6..58c28bf731 100644
--- a/src/local_deep_research/journal_quality/data_sources/jabref.py
+++ b/src/local_deep_research/journal_quality/data_sources/jabref.py
@@ -63,7 +63,9 @@ class JabRefSource(DataSource):
for filename in _JABREF_FILES:
url = f"{_JABREF_BASE}/{filename}"
try:
- resp = safe_get(url, timeout=30, consume_body=True)
+ resp = safe_get(
+ url, timeout=30, consume_body=True, require_https=True
+ )
resp.raise_for_status()
reader = csv.reader(io.StringIO(resp.text))
for row in reader:
diff --git a/src/local_deep_research/journal_quality/data_sources/openalex.py b/src/local_deep_research/journal_quality/data_sources/openalex.py
index 924481bed0..386b192776 100644
--- a/src/local_deep_research/journal_quality/data_sources/openalex.py
+++ b/src/local_deep_research/journal_quality/data_sources/openalex.py
@@ -90,7 +90,10 @@ class OpenAlexSource(DataSource):
# bottleneck for the whole download. A body-read transient
# here aborts everything.
manifest_resp = safe_get(
- _OPENALEX_SOURCES_MANIFEST, timeout=30, consume_body=True
+ _OPENALEX_SOURCES_MANIFEST,
+ timeout=30,
+ consume_body=True,
+ require_https=True,
)
manifest_resp.raise_for_status()
manifest = manifest_resp.json()
diff --git a/src/local_deep_research/journal_quality/data_sources/predatory.py b/src/local_deep_research/journal_quality/data_sources/predatory.py
index 31a4e22989..58da5b54f3 100644
--- a/src/local_deep_research/journal_quality/data_sources/predatory.py
+++ b/src/local_deep_research/journal_quality/data_sources/predatory.py
@@ -69,7 +69,9 @@ class PredatorySource(DataSource):
def _read_csv(filename: str) -> list[dict]:
url = f"{_PREDATORY_BASE}/{filename}"
- resp = safe_get(url, timeout=30, consume_body=True)
+ resp = safe_get(
+ url, timeout=30, consume_body=True, require_https=True
+ )
resp.raise_for_status()
reader = csv.DictReader(io.StringIO(resp.text))
return [
diff --git a/src/local_deep_research/security/safe_requests.py b/src/local_deep_research/security/safe_requests.py
index 9e2076a7d7..a3c97db1df 100644
--- a/src/local_deep_research/security/safe_requests.py
+++ b/src/local_deep_research/security/safe_requests.py
@@ -206,6 +206,7 @@ def safe_get(
timeout: int = DEFAULT_TIMEOUT,
allow_localhost: bool = False,
allow_private_ips: bool = False,
+ require_https: bool = False,
**kwargs,
) -> requests.Response:
"""
@@ -226,15 +227,43 @@ def safe_get(
Note: cloud metadata endpoints (AWS / Azure / OCI / DigitalOcean /
AlibabaCloud / Tencent / ECS) are ALWAYS blocked — see
``ssrf_validator.ALWAYS_BLOCKED_METADATA_IPS``.
+ require_https: Refuse the fetch (and any redirect hop) whose URL
+ is not https://. Opt-in per caller for integrity-sensitive
+ downloads (e.g. the journal-quality datasets): a hop to
+ cleartext passes SSRF validation, letting a compromised
+ upstream or open-redirect move the fetch onto a tamperable
+ path. Redirect hops are judged after resolving the Location
+ against the current URL, so relative and protocol-relative
+ Locations inherit https. Combining it with ``verify=False``
+ raises ValueError — https without certificate verification
+ is not an authenticated transport. NOT global — localhost
+ SearXNG instances are legitimately http://.
**kwargs: Additional arguments to pass to requests.get()
Returns:
Response object
Raises:
- ValueError: If URL fails SSRF validation
+ ValueError: If URL fails SSRF validation, or (with
+ ``require_https``) the URL or a redirect hop is not https, or
+ ``verify=False`` is passed
requests.RequestException: If request fails
"""
+ if require_https:
+ # Pinning the scheme is meaningless if the certificate is not
+ # checked: an on-path attacker can terminate TLS with any cert.
+ # requests disables verification for any falsy ``verify``
+ # (``False``, ``0``, ``""``); ``None`` merges to the session
+ # default (verified) and is allowed.
+ verify = kwargs.get("verify")
+ if verify is not None and not verify:
+ raise ValueError(
+ "require_https cannot be combined with verify=False"
+ )
+ if not url.lower().startswith("https://"):
+ raise ValueError(
+ f"URL must use https when require_https is set: {url}"
+ )
# Validate URL to prevent SSRF
if not ssrf_validator.validate_url(
url,
@@ -294,6 +323,22 @@ def safe_get(
response.url or current_url, redirect_url
)
+ # Scheme pin: refuse a hop that would move the fetch to
+ # cleartext. Checked after urljoin so relative and
+ # protocol-relative Locations inherit the https scheme
+ # before being judged.
+ if require_https and not redirect_url.lower().startswith(
+ "https://"
+ ):
+ logger.warning(
+ f"Redirect to {redirect_url} would downgrade to "
+ f"cleartext (from {url}, hop {redirects_followed + 1})"
+ )
+ response.close()
+ raise ValueError(
+ f"Redirect would downgrade to cleartext: {redirect_url}"
+ )
+
# Validate redirect target against SSRF rules
if not ssrf_validator.validate_url(
redirect_url,
diff --git a/tests/journal_quality/test_openalex_common.py b/tests/journal_quality/test_openalex_common.py
index 646224fb69..aa7eaa8754 100644
--- a/tests/journal_quality/test_openalex_common.py
+++ b/tests/journal_quality/test_openalex_common.py
@@ -238,6 +238,36 @@ class TestIterPartitions:
f"got kwargs={kwargs!r}"
)
+ def test_passes_require_https_to_safe_get(self, tmp_path):
+ """``iter_partitions`` must pin partition fetches to https.
+
+ ``s3_to_https`` produces an https URL, but ``safe_get`` only
+ refuses a redirect hop to cleartext when ``require_https=True``
+ is passed. Without it a compromised bucket or open redirect can
+ move the MB-sized partition bodies onto a tamperable path.
+ """
+ entries = [{"url": "s3://openalex/data/jsonl/sources/part_0.gz"}]
+ safe_get = MagicMock(
+ return_value=_response(_gz_lines([b'{"id": "S1"}']))
+ )
+
+ list(
+ iter_partitions(
+ entries,
+ tmp_path,
+ file_prefix="require_https_test",
+ label="test",
+ safe_get=safe_get,
+ )
+ )
+
+ assert safe_get.call_count == 1
+ _, kwargs = safe_get.call_args
+ assert kwargs.get("require_https") is True, (
+ f"iter_partitions must call safe_get with require_https=True; "
+ f"got kwargs={kwargs!r}"
+ )
+
def test_uses_higher_retry_budget_than_safe_get_default(self, tmp_path):
"""Partition fetches must override the generic safe_get retry budget.
diff --git a/tests/security/test_dataset_fetch_https_pinning.py b/tests/security/test_dataset_fetch_https_pinning.py
new file mode 100644
index 0000000000..13047bf4b4
--- /dev/null
+++ b/tests/security/test_dataset_fetch_https_pinning.py
@@ -0,0 +1,294 @@
+"""HTTPS scheme pinning for dataset fetches (``safe_get``).
+
+``safe_get`` validates every redirect hop against the SSRF rules —
+host and IP scope only. The scheme is never pinned: a hop from
+``https://`` to ``http://`` on a public host passes validation, so a
+compromised upstream (the journal-quality datasets come from a public
+S3 bucket, DOAJ, and GitHub raw) or any open-redirect on the path can
+move the fetch to cleartext, where an on-path attacker can tamper with
+the bytes that later drive journal reputation scores.
+
+The fix must be opt-in per caller: localhost SearXNG instances are
+legitimately ``http://`` (``ALLOWED_SCHEMES`` includes http for that
+reason), so the pin is a ``require_https`` flag rather than a global
+restriction.
+
+These tests pin the flag: with it, an initial non-https URL and any
+redirect hop that would downgrade to cleartext are refused; without
+it, current behaviour is unchanged; https-preserving hops still
+follow.
+"""
+
+from contextlib import contextmanager, nullcontext
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from local_deep_research.security import safe_requests
+
+_SR = "local_deep_research.security.safe_requests"
+
+
+def _redirect(location: str) -> MagicMock:
+ r = MagicMock()
+ r.status_code = 302
+ r.headers = {"Location": location}
+ r.url = "https://example.com/"
+ r.close = MagicMock()
+ return r
+
+
+def _ok(url: str = "https://example.com/final") -> MagicMock:
+ r = MagicMock()
+ r.status_code = 200
+ r.headers = {}
+ r.url = url
+ r.close = MagicMock()
+ return r
+
+
+@contextmanager
+def _patches(responses):
+ """Patch validator, DNS pinning, and requests.get — no network."""
+ with (
+ patch.object(
+ safe_requests.ssrf_validator, "validate_url", return_value=True
+ ) as validator,
+ patch.object(
+ safe_requests.dns_pinning,
+ "pinned_request",
+ MagicMock(return_value=nullcontext()),
+ ) as pin,
+ patch(f"{_SR}.requests.get", side_effect=responses) as gets,
+ ):
+ yield validator, pin, gets
+
+
+def test_https_to_http_redirect_downgrade_is_refused():
+ """A redirect hop to cleartext must abort the pinned fetch."""
+ with _patches([_redirect("http://example.com/data"), _ok()]) as (
+ _v,
+ _p,
+ gets,
+ ):
+ with pytest.raises(ValueError, match="downgrade"):
+ safe_requests.safe_get("https://example.com/", require_https=True)
+ assert gets.call_count == 1, "the cleartext hop must never be fetched"
+
+
+def test_non_https_initial_url_is_refused():
+ """The pin applies to the initial URL, not just redirect hops."""
+ with _patches([_ok()]) as (_v, _p, gets):
+ with pytest.raises(ValueError, match="https"):
+ safe_requests.safe_get(
+ "http://example.com/data", require_https=True
+ )
+ assert gets.call_count == 0
+
+
+def test_https_to_https_redirect_still_followed():
+ """Scheme-preserving hops keep working under the pin."""
+ with _patches(
+ [_redirect("https://example.com/data"), _ok("https://example.com/data")]
+ ) as (_v, _p, gets):
+ response = safe_requests.safe_get(
+ "https://example.com/", require_https=True
+ )
+
+ assert response.status_code == 200
+ assert gets.call_count == 2
+
+
+def test_downgrade_without_the_flag_keeps_current_behaviour():
+ """Control: without require_https the hop is followed as today."""
+ with _patches(
+ [_redirect("http://example.com/data"), _ok("http://example.com/data")]
+ ) as (_v, _p, gets):
+ response = safe_requests.safe_get("https://example.com/")
+
+ assert response.status_code == 200
+ assert gets.call_count == 2
+
+
+# --- Redirect hops are judged after urljoin -------------------------------
+
+
+def test_relative_location_inherits_https_and_is_followed():
+ """A relative ``Location`` resolves against the https URL, so it is
+ followed. Judging the raw header ("/x" does not start with https://)
+ would wrongly refuse every same-origin relative redirect."""
+ with _patches([_redirect("/x"), _ok("https://example.com/x")]) as (
+ _v,
+ _p,
+ gets,
+ ):
+ response = safe_requests.safe_get(
+ "https://example.com/", require_https=True
+ )
+
+ assert response.status_code == 200
+ assert gets.call_count == 2
+ assert gets.call_args_list[1].args[0] == "https://example.com/x"
+
+
+def test_protocol_relative_location_inherits_https_and_is_followed():
+ """``//cdn.example/x`` from an https page resolves to https."""
+ with _patches(
+ [_redirect("//cdn.example/x"), _ok("https://cdn.example/x")]
+ ) as (_v, _p, gets):
+ response = safe_requests.safe_get(
+ "https://example.com/", require_https=True
+ )
+
+ assert response.status_code == 200
+ assert gets.call_count == 2
+ assert gets.call_args_list[1].args[0] == "https://cdn.example/x"
+
+
+def test_downgrade_on_a_later_hop_is_refused():
+ """https -> https -> http: the pin applies to every hop, not just the
+ first one."""
+ with _patches(
+ [
+ _redirect("https://mirror.example/1"),
+ _redirect("http://mirror.example/2"),
+ _ok("http://mirror.example/2"),
+ ]
+ ) as (_v, _p, gets):
+ with pytest.raises(ValueError, match="downgrade"):
+ safe_requests.safe_get("https://example.com/", require_https=True)
+
+ assert gets.call_count == 2, "the cleartext hop must never be fetched"
+
+
+# --- https without certificate verification is refused ---------------------
+
+
[email protected]("verify", [False, 0, ""])
+def test_require_https_with_verify_false_is_refused(verify):
+ """Pinning the scheme while disabling certificate verification gives
+ no authenticated transport; refuse the combination outright. requests
+ treats any falsy ``verify`` as "don't verify", so all of them are
+ refused."""
+ with _patches([_ok()]) as (_v, _p, gets):
+ with pytest.raises(ValueError, match="verify=False"):
+ safe_requests.safe_get(
+ "https://example.com/", require_https=True, verify=verify
+ )
+ assert gets.call_count == 0
+
+
+def test_require_https_with_default_verify_none_is_allowed():
+ """Control: ``verify=None`` merges to the session default (verified),
+ so it is not refused."""
+ with _patches([_ok()]) as (_v, _p, gets):
+ response = safe_requests.safe_get(
+ "https://example.com/", require_https=True, verify=None
+ )
+ assert response.status_code == 200
+ assert gets.call_count == 1
+
+
+def test_retry_wrapper_forwards_require_https():
+ """The six dataset sites call ``safe_get_with_retries``; the pin only
+ holds if the wrapper hands the flag to ``safe_get`` on every attempt."""
+ fake = MagicMock(return_value=_ok())
+ with patch.object(safe_requests, "safe_get", fake):
+ safe_requests.safe_get_with_retries(
+ "https://example.com/", require_https=True
+ )
+ assert fake.call_count == 1
+ assert fake.call_args.kwargs.get("require_https") is True
+
+
+def test_require_https_with_ca_bundle_verify_is_allowed():
+ """Control: a CA-bundle path for ``verify`` still verifies, so it is
+ not refused."""
+ with _patches([_ok()]) as (_v, _p, gets):
+ response = safe_requests.safe_get(
+ "https://example.com/",
+ require_https=True,
+ verify="/etc/ssl/certs/ca-certificates.crt",
+ )
+ assert response.status_code == 200
+ assert gets.call_count == 1
+
+
+# --- Wiring: every journal-quality dataset fetch sets require_https --------
+#
+# The flag is opt-in, so the protection exists only where a call site
+# passes it. Each test drives one source's ``fetch`` against a fake
+# ``safe_get_with_retries`` (the sources import it lazily from
+# ``security.safe_requests``) and asserts the kwarg. The fake raises to
+# stop the fetch at the first request — no parsing, no network.
+
+_RETRIES = "local_deep_research.security.safe_requests.safe_get_with_retries"
+
+
+class _StopFetch(Exception):
+ pass
+
+
+def _assert_pinned(fake, expected_calls=None):
+ assert fake.call_count >= 1
+ if expected_calls is not None:
+ assert fake.call_count == expected_calls
+ for call in fake.call_args_list:
+ assert call.kwargs.get("require_https") is True, (
+ f"dataset fetch must pass require_https=True; got {call!r}"
+ )
+
+
+def test_doaj_fetch_pins_https(tmp_path):
+ from local_deep_research.journal_quality.data_sources.doaj import (
+ DOAJSource,
+ )
+
+ with patch(_RETRIES, side_effect=_StopFetch) as fake:
+ with pytest.raises(_StopFetch):
+ DOAJSource().fetch(tmp_path)
+ _assert_pinned(fake, expected_calls=1)
+
+
+def test_predatory_fetch_pins_https(tmp_path):
+ from local_deep_research.journal_quality.data_sources.predatory import (
+ PredatorySource,
+ )
+
+ with patch(_RETRIES, side_effect=_StopFetch) as fake:
+ with pytest.raises(_StopFetch):
+ PredatorySource().fetch(tmp_path)
+ _assert_pinned(fake, expected_calls=1)
+
+
+def test_jabref_fetch_pins_https_for_every_file(tmp_path):
+ from local_deep_research.journal_quality.data_sources import jabref
+
+ # JabRef tolerates per-file failures and then refuses on the floor,
+ # so every file is requested — each request must carry the pin.
+ with patch(_RETRIES, side_effect=_StopFetch) as fake:
+ with pytest.raises(RuntimeError, match="suspiciously few"):
+ jabref.JabRefSource().fetch(tmp_path)
+ _assert_pinned(fake, expected_calls=len(jabref._JABREF_FILES))
+
+
+def test_openalex_manifest_fetch_pins_https(tmp_path):
+ from local_deep_research.journal_quality.data_sources.openalex import (
+ OpenAlexSource,
+ )
+
+ with patch(_RETRIES, side_effect=_StopFetch) as fake:
+ with pytest.raises(_StopFetch):
+ OpenAlexSource().fetch(tmp_path)
+ _assert_pinned(fake, expected_calls=1)
+
+
+def test_institutions_manifest_fetch_pins_https(tmp_path):
+ from local_deep_research.journal_quality.data_sources.institutions import (
+ InstitutionSource,
+ )
+
+ with patch(_RETRIES, side_effect=_StopFetch) as fake:
+ with pytest.raises(_StopFetch):
+ InstitutionSource().fetch(tmp_path)
+ _assert_pinned(fake, expected_calls=1)