~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(security): pin https on journal dataset fetches via require_https (#6441)
LearningCircuitaadff6480b71
diff --git a/changelog.d/6441.security.md b/changelog.d/6441.security.mdnew file mode 100644index 0000000000..6a8d893566--- /dev/null+++ b/changelog.d/6441.security.md@@ -0,0 +1 @@+Journal-quality dataset fetches now pin HTTPS end to end. `safe_get` validated redirect hops for host and IP scope only, so a hop from `https://` to `http://` on a public host passed validation — a compromised upstream (the OpenAlex S3 bucket, DOAJ, GitHub raw) or any open redirect could move the fetch to cleartext, where an on-path attacker can tamper with the bytes that later drive journal reputation scores. A new opt-in `require_https` flag on `safe_get` refuses a non-https initial URL and any redirect hop that resolves to a non-https URL (relative and protocol-relative `Location` headers inherit https and are still followed) before the cleartext request is made, and refuses to run with `verify=False`. It is opt-in because localhost SearXNG instances are legitimately `http://`. All six journal-data fetch sites now set it — the DOAJ CSV, the JabRef and predatory-journal CSVs, and the OpenAlex sources and institutions manifests and their partitions — so every hop of each is carried over certificate-verified TLS. This protects transport only: it does not defend against a compromised origin, and the OpenAlex manifest's size/md5 check (#6438) remains a corruption guard rather than an authenticity check, since its declared values come from the same origin as the bytes they describe.diff --git a/src/local_deep_research/journal_quality/data_sources/_openalex_common.py b/src/local_deep_research/journal_quality/data_sources/_openalex_common.pyindex 1cc2d8d795..0b3ff9f444 100644--- a/src/local_deep_research/journal_quality/data_sources/_openalex_common.py+++ b/src/local_deep_research/journal_quality/data_sources/_openalex_common.py@@ -213,7 +213,9 @@ def iter_partitions(so body-stream transients (``ChunkedEncodingError``,``ReadTimeout``) raised during ``resp.content`` areretried inside the wrapper, not propagated to abort the- whole multi-partition pull.+ whole multi-partition pull. Must also accept+ ``require_https=True``: partitions are fetched with the+ scheme pinned so no redirect hop can move them to cleartext.timeout: Per-partition HTTP timeout (seconds).max_retries: Per-partition retry budget. Defaults higher than``safe_get_with_retries``' generic 3 because partition@@ -323,6 +325,7 @@ def iter_partitions(consume_body=True,max_retries=max_retries,backoff_times=backoff_times,+ require_https=True,)resp.raise_for_status()meta = entry.get("meta") or {}diff --git a/src/local_deep_research/journal_quality/data_sources/doaj.py b/src/local_deep_research/journal_quality/data_sources/doaj.pyindex 6401ab21da..a148221687 100644--- a/src/local_deep_research/journal_quality/data_sources/doaj.py+++ b/src/local_deep_research/journal_quality/data_sources/doaj.py@@ -67,7 +67,12 @@ class DOAJSource(DataSource):# ChunkedEncodingError / ReadTimeout is a realistic failure# mode worth retrying. Without this flag the body-read fires# outside safe_get_with_retries' retry loop.- resp = safe_get(_DOAJ_CSV_URL, timeout=120, consume_body=True)+ resp = safe_get(+ _DOAJ_CSV_URL,+ timeout=120,+ consume_body=True,+ require_https=True,+ )resp.raise_for_status()# DOAJ serves UTF-8 CSV. Parse in-memory — the whole file isdiff --git a/src/local_deep_research/journal_quality/data_sources/institutions.py b/src/local_deep_research/journal_quality/data_sources/institutions.pyindex c60f00b8db..59a2208691 100644--- a/src/local_deep_research/journal_quality/data_sources/institutions.py+++ b/src/local_deep_research/journal_quality/data_sources/institutions.py@@ -76,7 +76,10 @@ class InstitutionSource(DataSource):# consume_body=True: small JSON but serial bottleneck — a body# transient here aborts the whole 10-min institutions pull.manifest_resp = safe_get(- _OPENALEX_INSTITUTIONS_MANIFEST, timeout=30, consume_body=True+ _OPENALEX_INSTITUTIONS_MANIFEST,+ timeout=30,+ consume_body=True,+ require_https=True,)manifest_resp.raise_for_status()manifest = manifest_resp.json()diff --git a/src/local_deep_research/journal_quality/data_sources/jabref.py b/src/local_deep_research/journal_quality/data_sources/jabref.pyindex 4a8e2611e6..58c28bf731 100644--- a/src/local_deep_research/journal_quality/data_sources/jabref.py+++ b/src/local_deep_research/journal_quality/data_sources/jabref.py@@ -63,7 +63,9 @@ class JabRefSource(DataSource):for filename in _JABREF_FILES:url = f"{_JABREF_BASE}/{filename}"try:- resp = safe_get(url, timeout=30, consume_body=True)+ resp = safe_get(+ url, timeout=30, consume_body=True, require_https=True+ )resp.raise_for_status()reader = csv.reader(io.StringIO(resp.text))for row in reader:diff --git a/src/local_deep_research/journal_quality/data_sources/openalex.py b/src/local_deep_research/journal_quality/data_sources/openalex.pyindex 924481bed0..386b192776 100644--- a/src/local_deep_research/journal_quality/data_sources/openalex.py+++ b/src/local_deep_research/journal_quality/data_sources/openalex.py@@ -90,7 +90,10 @@ class OpenAlexSource(DataSource):# bottleneck for the whole download. A body-read transient# here aborts everything.manifest_resp = safe_get(- _OPENALEX_SOURCES_MANIFEST, timeout=30, consume_body=True+ _OPENALEX_SOURCES_MANIFEST,+ timeout=30,+ consume_body=True,+ require_https=True,)manifest_resp.raise_for_status()manifest = manifest_resp.json()diff --git a/src/local_deep_research/journal_quality/data_sources/predatory.py b/src/local_deep_research/journal_quality/data_sources/predatory.pyindex 31a4e22989..58da5b54f3 100644--- a/src/local_deep_research/journal_quality/data_sources/predatory.py+++ b/src/local_deep_research/journal_quality/data_sources/predatory.py@@ -69,7 +69,9 @@ class PredatorySource(DataSource):def _read_csv(filename: str) -> list[dict]:url = f"{_PREDATORY_BASE}/{filename}"- resp = safe_get(url, timeout=30, consume_body=True)+ resp = safe_get(+ url, timeout=30, consume_body=True, require_https=True+ )resp.raise_for_status()reader = csv.DictReader(io.StringIO(resp.text))return [diff --git a/src/local_deep_research/security/safe_requests.py b/src/local_deep_research/security/safe_requests.pyindex 9e2076a7d7..a3c97db1df 100644--- a/src/local_deep_research/security/safe_requests.py+++ b/src/local_deep_research/security/safe_requests.py@@ -206,6 +206,7 @@ def safe_get(timeout: int = DEFAULT_TIMEOUT,allow_localhost: bool = False,allow_private_ips: bool = False,+ require_https: bool = False,**kwargs,) -> requests.Response:"""@@ -226,15 +227,43 @@ def safe_get(Note: cloud metadata endpoints (AWS / Azure / OCI / DigitalOcean /AlibabaCloud / Tencent / ECS) are ALWAYS blocked — see``ssrf_validator.ALWAYS_BLOCKED_METADATA_IPS``.+ require_https: Refuse the fetch (and any redirect hop) whose URL+ is not https://. Opt-in per caller for integrity-sensitive+ downloads (e.g. the journal-quality datasets): a hop to+ cleartext passes SSRF validation, letting a compromised+ upstream or open-redirect move the fetch onto a tamperable+ path. Redirect hops are judged after resolving the Location+ against the current URL, so relative and protocol-relative+ Locations inherit https. Combining it with ``verify=False``+ raises ValueError — https without certificate verification+ is not an authenticated transport. NOT global — localhost+ SearXNG instances are legitimately http://.**kwargs: Additional arguments to pass to requests.get()Returns:Response objectRaises:- ValueError: If URL fails SSRF validation+ ValueError: If URL fails SSRF validation, or (with+ ``require_https``) the URL or a redirect hop is not https, or+ ``verify=False`` is passedrequests.RequestException: If request fails"""+ if require_https:+ # Pinning the scheme is meaningless if the certificate is not+ # checked: an on-path attacker can terminate TLS with any cert.+ # requests disables verification for any falsy ``verify``+ # (``False``, ``0``, ``""``); ``None`` merges to the session+ # default (verified) and is allowed.+ verify = kwargs.get("verify")+ if verify is not None and not verify:+ raise ValueError(+ "require_https cannot be combined with verify=False"+ )+ if not url.lower().startswith("https://"):+ raise ValueError(+ f"URL must use https when require_https is set: {url}"+ )# Validate URL to prevent SSRFif not ssrf_validator.validate_url(url,@@ -294,6 +323,22 @@ def safe_get(response.url or current_url, redirect_url)+ # Scheme pin: refuse a hop that would move the fetch to+ # cleartext. Checked after urljoin so relative and+ # protocol-relative Locations inherit the https scheme+ # before being judged.+ if require_https and not redirect_url.lower().startswith(+ "https://"+ ):+ logger.warning(+ f"Redirect to {redirect_url} would downgrade to "+ f"cleartext (from {url}, hop {redirects_followed + 1})"+ )+ response.close()+ raise ValueError(+ f"Redirect would downgrade to cleartext: {redirect_url}"+ )+# Validate redirect target against SSRF rulesif not ssrf_validator.validate_url(redirect_url,diff --git a/tests/journal_quality/test_openalex_common.py b/tests/journal_quality/test_openalex_common.pyindex 646224fb69..aa7eaa8754 100644--- a/tests/journal_quality/test_openalex_common.py+++ b/tests/journal_quality/test_openalex_common.py@@ -238,6 +238,36 @@ class TestIterPartitions:f"got kwargs={kwargs!r}")+ def test_passes_require_https_to_safe_get(self, tmp_path):+ """``iter_partitions`` must pin partition fetches to https.++ ``s3_to_https`` produces an https URL, but ``safe_get`` only+ refuses a redirect hop to cleartext when ``require_https=True``+ is passed. Without it a compromised bucket or open redirect can+ move the MB-sized partition bodies onto a tamperable path.+ """+ entries = [{"url": "s3://openalex/data/jsonl/sources/part_0.gz"}]+ safe_get = MagicMock(+ return_value=_response(_gz_lines([b'{"id": "S1"}']))+ )++ list(+ iter_partitions(+ entries,+ tmp_path,+ file_prefix="require_https_test",+ label="test",+ safe_get=safe_get,+ )+ )++ assert safe_get.call_count == 1+ _, kwargs = safe_get.call_args+ assert kwargs.get("require_https") is True, (+ f"iter_partitions must call safe_get with require_https=True; "+ f"got kwargs={kwargs!r}"+ )+def test_uses_higher_retry_budget_than_safe_get_default(self, tmp_path):"""Partition fetches must override the generic safe_get retry budget.diff --git a/tests/security/test_dataset_fetch_https_pinning.py b/tests/security/test_dataset_fetch_https_pinning.pynew file mode 100644index 0000000000..13047bf4b4--- /dev/null+++ b/tests/security/test_dataset_fetch_https_pinning.py@@ -0,0 +1,294 @@+"""HTTPS scheme pinning for dataset fetches (``safe_get``).++``safe_get`` validates every redirect hop against the SSRF rules —+host and IP scope only. The scheme is never pinned: a hop from+``https://`` to ``http://`` on a public host passes validation, so a+compromised upstream (the journal-quality datasets come from a public+S3 bucket, DOAJ, and GitHub raw) or any open-redirect on the path can+move the fetch to cleartext, where an on-path attacker can tamper with+the bytes that later drive journal reputation scores.++The fix must be opt-in per caller: localhost SearXNG instances are+legitimately ``http://`` (``ALLOWED_SCHEMES`` includes http for that+reason), so the pin is a ``require_https`` flag rather than a global+restriction.++These tests pin the flag: with it, an initial non-https URL and any+redirect hop that would downgrade to cleartext are refused; without+it, current behaviour is unchanged; https-preserving hops still+follow.+"""++from contextlib import contextmanager, nullcontext+from unittest.mock import MagicMock, patch++import pytest++from local_deep_research.security import safe_requests++_SR = "local_deep_research.security.safe_requests"+++def _redirect(location: str) -> MagicMock:+ r = MagicMock()+ r.status_code = 302+ r.headers = {"Location": location}+ r.url = "https://example.com/"+ r.close = MagicMock()+ return r+++def _ok(url: str = "https://example.com/final") -> MagicMock:+ r = MagicMock()+ r.status_code = 200+ r.headers = {}+ r.url = url+ r.close = MagicMock()+ return r+++@contextmanager+def _patches(responses):+ """Patch validator, DNS pinning, and requests.get — no network."""+ with (+ patch.object(+ safe_requests.ssrf_validator, "validate_url", return_value=True+ ) as validator,+ patch.object(+ safe_requests.dns_pinning,+ "pinned_request",+ MagicMock(return_value=nullcontext()),+ ) as pin,+ patch(f"{_SR}.requests.get", side_effect=responses) as gets,+ ):+ yield validator, pin, gets+++def test_https_to_http_redirect_downgrade_is_refused():+ """A redirect hop to cleartext must abort the pinned fetch."""+ with _patches([_redirect("http://example.com/data"), _ok()]) as (+ _v,+ _p,+ gets,+ ):+ with pytest.raises(ValueError, match="downgrade"):+ safe_requests.safe_get("https://example.com/", require_https=True)+ assert gets.call_count == 1, "the cleartext hop must never be fetched"+++def test_non_https_initial_url_is_refused():+ """The pin applies to the initial URL, not just redirect hops."""+ with _patches([_ok()]) as (_v, _p, gets):+ with pytest.raises(ValueError, match="https"):+ safe_requests.safe_get(+ "http://example.com/data", require_https=True+ )+ assert gets.call_count == 0+++def test_https_to_https_redirect_still_followed():+ """Scheme-preserving hops keep working under the pin."""+ with _patches(+ [_redirect("https://example.com/data"), _ok("https://example.com/data")]+ ) as (_v, _p, gets):+ response = safe_requests.safe_get(+ "https://example.com/", require_https=True+ )++ assert response.status_code == 200+ assert gets.call_count == 2+++def test_downgrade_without_the_flag_keeps_current_behaviour():+ """Control: without require_https the hop is followed as today."""+ with _patches(+ [_redirect("http://example.com/data"), _ok("http://example.com/data")]+ ) as (_v, _p, gets):+ response = safe_requests.safe_get("https://example.com/")++ assert response.status_code == 200+ assert gets.call_count == 2+++# --- Redirect hops are judged after urljoin -------------------------------+++def test_relative_location_inherits_https_and_is_followed():+ """A relative ``Location`` resolves against the https URL, so it is+ followed. Judging the raw header ("/x" does not start with https://)+ would wrongly refuse every same-origin relative redirect."""+ with _patches([_redirect("/x"), _ok("https://example.com/x")]) as (+ _v,+ _p,+ gets,+ ):+ response = safe_requests.safe_get(+ "https://example.com/", require_https=True+ )++ assert response.status_code == 200+ assert gets.call_count == 2+ assert gets.call_args_list[1].args[0] == "https://example.com/x"+++def test_protocol_relative_location_inherits_https_and_is_followed():+ """``//cdn.example/x`` from an https page resolves to https."""+ with _patches(+ [_redirect("//cdn.example/x"), _ok("https://cdn.example/x")]+ ) as (_v, _p, gets):+ response = safe_requests.safe_get(+ "https://example.com/", require_https=True+ )++ assert response.status_code == 200+ assert gets.call_count == 2+ assert gets.call_args_list[1].args[0] == "https://cdn.example/x"+++def test_downgrade_on_a_later_hop_is_refused():+ """https -> https -> http: the pin applies to every hop, not just the+ first one."""+ with _patches(+ [+ _redirect("https://mirror.example/1"),+ _redirect("http://mirror.example/2"),+ _ok("http://mirror.example/2"),+ ]+ ) as (_v, _p, gets):+ with pytest.raises(ValueError, match="downgrade"):+ safe_requests.safe_get("https://example.com/", require_https=True)++ assert gets.call_count == 2, "the cleartext hop must never be fetched"+++# --- https without certificate verification is refused ---------------------++[email protected]("verify", [False, 0, ""])+def test_require_https_with_verify_false_is_refused(verify):+ """Pinning the scheme while disabling certificate verification gives+ no authenticated transport; refuse the combination outright. requests+ treats any falsy ``verify`` as "don't verify", so all of them are+ refused."""+ with _patches([_ok()]) as (_v, _p, gets):+ with pytest.raises(ValueError, match="verify=False"):+ safe_requests.safe_get(+ "https://example.com/", require_https=True, verify=verify+ )+ assert gets.call_count == 0+++def test_require_https_with_default_verify_none_is_allowed():+ """Control: ``verify=None`` merges to the session default (verified),+ so it is not refused."""+ with _patches([_ok()]) as (_v, _p, gets):+ response = safe_requests.safe_get(+ "https://example.com/", require_https=True, verify=None+ )+ assert response.status_code == 200+ assert gets.call_count == 1+++def test_retry_wrapper_forwards_require_https():+ """The six dataset sites call ``safe_get_with_retries``; the pin only+ holds if the wrapper hands the flag to ``safe_get`` on every attempt."""+ fake = MagicMock(return_value=_ok())+ with patch.object(safe_requests, "safe_get", fake):+ safe_requests.safe_get_with_retries(+ "https://example.com/", require_https=True+ )+ assert fake.call_count == 1+ assert fake.call_args.kwargs.get("require_https") is True+++def test_require_https_with_ca_bundle_verify_is_allowed():+ """Control: a CA-bundle path for ``verify`` still verifies, so it is+ not refused."""+ with _patches([_ok()]) as (_v, _p, gets):+ response = safe_requests.safe_get(+ "https://example.com/",+ require_https=True,+ verify="/etc/ssl/certs/ca-certificates.crt",+ )+ assert response.status_code == 200+ assert gets.call_count == 1+++# --- Wiring: every journal-quality dataset fetch sets require_https --------+#+# The flag is opt-in, so the protection exists only where a call site+# passes it. Each test drives one source's ``fetch`` against a fake+# ``safe_get_with_retries`` (the sources import it lazily from+# ``security.safe_requests``) and asserts the kwarg. The fake raises to+# stop the fetch at the first request — no parsing, no network.++_RETRIES = "local_deep_research.security.safe_requests.safe_get_with_retries"+++class _StopFetch(Exception):+ pass+++def _assert_pinned(fake, expected_calls=None):+ assert fake.call_count >= 1+ if expected_calls is not None:+ assert fake.call_count == expected_calls+ for call in fake.call_args_list:+ assert call.kwargs.get("require_https") is True, (+ f"dataset fetch must pass require_https=True; got {call!r}"+ )+++def test_doaj_fetch_pins_https(tmp_path):+ from local_deep_research.journal_quality.data_sources.doaj import (+ DOAJSource,+ )++ with patch(_RETRIES, side_effect=_StopFetch) as fake:+ with pytest.raises(_StopFetch):+ DOAJSource().fetch(tmp_path)+ _assert_pinned(fake, expected_calls=1)+++def test_predatory_fetch_pins_https(tmp_path):+ from local_deep_research.journal_quality.data_sources.predatory import (+ PredatorySource,+ )++ with patch(_RETRIES, side_effect=_StopFetch) as fake:+ with pytest.raises(_StopFetch):+ PredatorySource().fetch(tmp_path)+ _assert_pinned(fake, expected_calls=1)+++def test_jabref_fetch_pins_https_for_every_file(tmp_path):+ from local_deep_research.journal_quality.data_sources import jabref++ # JabRef tolerates per-file failures and then refuses on the floor,+ # so every file is requested — each request must carry the pin.+ with patch(_RETRIES, side_effect=_StopFetch) as fake:+ with pytest.raises(RuntimeError, match="suspiciously few"):+ jabref.JabRefSource().fetch(tmp_path)+ _assert_pinned(fake, expected_calls=len(jabref._JABREF_FILES))+++def test_openalex_manifest_fetch_pins_https(tmp_path):+ from local_deep_research.journal_quality.data_sources.openalex import (+ OpenAlexSource,+ )++ with patch(_RETRIES, side_effect=_StopFetch) as fake:+ with pytest.raises(_StopFetch):+ OpenAlexSource().fetch(tmp_path)+ _assert_pinned(fake, expected_calls=1)+++def test_institutions_manifest_fetch_pins_https(tmp_path):+ from local_deep_research.journal_quality.data_sources.institutions import (+ InstitutionSource,+ )++ with patch(_RETRIES, side_effect=_StopFetch) as fake:+ with pytest.raises(_StopFetch):+ InstitutionSource().fetch(tmp_path)+ _assert_pinned(fake, expected_calls=1)