~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""HTTPS scheme pinning for dataset fetches (``safe_get``).
``safe_get`` validates every redirect hop against the SSRF rules —
host and IP scope only. The scheme is never pinned: a hop from
``https://`` to ``http://`` on a public host passes validation, so a
compromised upstream (the journal-quality datasets come from a public
S3 bucket, DOAJ, and GitHub raw) or any open-redirect on the path can
move the fetch to cleartext, where an on-path attacker can tamper with
the bytes that later drive journal reputation scores.
The fix must be opt-in per caller: localhost SearXNG instances are
legitimately ``http://`` (``ALLOWED_SCHEMES`` includes http for that
reason), so the pin is a ``require_https`` flag rather than a global
restriction.
These tests pin the flag: with it, an initial non-https URL and any
redirect hop that would downgrade to cleartext are refused; without
it, current behaviour is unchanged; https-preserving hops still
follow.
"""
from contextlib import contextmanager, nullcontext
from unittest.mock import MagicMock, patch
import pytest
from local_deep_research.security import safe_requests
_SR = "local_deep_research.security.safe_requests"
def _redirect(location: str) -> MagicMock:
r = MagicMock()
r.status_code = 302
r.headers = {"Location": location}
r.url = "https://example.com/"
r.close = MagicMock()
return r
def _ok(url: str = "https://example.com/final") -> MagicMock:
r = MagicMock()
r.status_code = 200
r.headers = {}
r.url = url
r.close = MagicMock()
return r
@contextmanager
def _patches(responses):
"""Patch validator, DNS pinning, and requests.get — no network."""
with (
patch.object(
safe_requests.ssrf_validator, "validate_url", return_value=True
) as validator,
patch.object(
safe_requests.dns_pinning,
"pinned_request",
MagicMock(return_value=nullcontext()),
) as pin,
patch(f"{_SR}.requests.get", side_effect=responses) as gets,
):
yield validator, pin, gets
def test_https_to_http_redirect_downgrade_is_refused():
"""A redirect hop to cleartext must abort the pinned fetch."""
with _patches([_redirect("http://example.com/data"), _ok()]) as (
_v,
_p,
gets,
):
with pytest.raises(ValueError, match="downgrade"):
safe_requests.safe_get("https://example.com/", require_https=True)
assert gets.call_count == 1, "the cleartext hop must never be fetched"
def test_non_https_initial_url_is_refused():
"""The pin applies to the initial URL, not just redirect hops."""
with _patches([_ok()]) as (_v, _p, gets):
with pytest.raises(ValueError, match="https"):
safe_requests.safe_get(
"http://example.com/data", require_https=True
)
assert gets.call_count == 0
def test_https_to_https_redirect_still_followed():
"""Scheme-preserving hops keep working under the pin."""
with _patches(
[_redirect("https://example.com/data"), _ok("https://example.com/data")]
) as (_v, _p, gets):
response = safe_requests.safe_get(
"https://example.com/", require_https=True
)
assert response.status_code == 200
assert gets.call_count == 2
def test_downgrade_without_the_flag_keeps_current_behaviour():
"""Control: without require_https the hop is followed as today."""
with _patches(
[_redirect("http://example.com/data"), _ok("http://example.com/data")]
) as (_v, _p, gets):
response = safe_requests.safe_get("https://example.com/")
assert response.status_code == 200
assert gets.call_count == 2
# --- Redirect hops are judged after urljoin -------------------------------
def test_relative_location_inherits_https_and_is_followed():
"""A relative ``Location`` resolves against the https URL, so it is
followed. Judging the raw header ("/x" does not start with https://)
would wrongly refuse every same-origin relative redirect."""
with _patches([_redirect("/x"), _ok("https://example.com/x")]) as (
_v,
_p,
gets,
):
response = safe_requests.safe_get(
"https://example.com/", require_https=True
)
assert response.status_code == 200
assert gets.call_count == 2
assert gets.call_args_list[1].args[0] == "https://example.com/x"
def test_protocol_relative_location_inherits_https_and_is_followed():
"""``//cdn.example/x`` from an https page resolves to https."""
with _patches(
[_redirect("//cdn.example/x"), _ok("https://cdn.example/x")]
) as (_v, _p, gets):
response = safe_requests.safe_get(
"https://example.com/", require_https=True
)
assert response.status_code == 200
assert gets.call_count == 2
assert gets.call_args_list[1].args[0] == "https://cdn.example/x"
def test_downgrade_on_a_later_hop_is_refused():
"""https -> https -> http: the pin applies to every hop, not just the
first one."""
with _patches(
[
_redirect("https://mirror.example/1"),
_redirect("http://mirror.example/2"),
_ok("http://mirror.example/2"),
]
) as (_v, _p, gets):
with pytest.raises(ValueError, match="downgrade"):
safe_requests.safe_get("https://example.com/", require_https=True)
assert gets.call_count == 2, "the cleartext hop must never be fetched"
# --- https without certificate verification is refused ---------------------
@pytest.mark.parametrize("verify", [False, 0, ""])
def test_require_https_with_verify_false_is_refused(verify):
"""Pinning the scheme while disabling certificate verification gives
no authenticated transport; refuse the combination outright. requests
treats any falsy ``verify`` as "don't verify", so all of them are
refused."""
with _patches([_ok()]) as (_v, _p, gets):
with pytest.raises(ValueError, match="verify=False"):
safe_requests.safe_get(
"https://example.com/", require_https=True, verify=verify
)
assert gets.call_count == 0
def test_require_https_with_default_verify_none_is_allowed():
"""Control: ``verify=None`` merges to the session default (verified),
so it is not refused."""
with _patches([_ok()]) as (_v, _p, gets):
response = safe_requests.safe_get(
"https://example.com/", require_https=True, verify=None
)
assert response.status_code == 200
assert gets.call_count == 1
def test_retry_wrapper_forwards_require_https():
"""The six dataset sites call ``safe_get_with_retries``; the pin only
holds if the wrapper hands the flag to ``safe_get`` on every attempt."""
fake = MagicMock(return_value=_ok())
with patch.object(safe_requests, "safe_get", fake):
safe_requests.safe_get_with_retries(
"https://example.com/", require_https=True
)
assert fake.call_count == 1
assert fake.call_args.kwargs.get("require_https") is True
def test_require_https_with_ca_bundle_verify_is_allowed():
"""Control: a CA-bundle path for ``verify`` still verifies, so it is
not refused."""
with _patches([_ok()]) as (_v, _p, gets):
response = safe_requests.safe_get(
"https://example.com/",
require_https=True,
verify="/etc/ssl/certs/ca-certificates.crt",
)
assert response.status_code == 200
assert gets.call_count == 1
# --- Wiring: every journal-quality dataset fetch sets require_https --------
#
# The flag is opt-in, so the protection exists only where a call site
# passes it. Each test drives one source's ``fetch`` against a fake
# ``safe_get_with_retries`` (the sources import it lazily from
# ``security.safe_requests``) and asserts the kwarg. The fake raises to
# stop the fetch at the first request — no parsing, no network.
_RETRIES = "local_deep_research.security.safe_requests.safe_get_with_retries"
class _StopFetch(Exception):
pass
def _assert_pinned(fake, expected_calls=None):
assert fake.call_count >= 1
if expected_calls is not None:
assert fake.call_count == expected_calls
for call in fake.call_args_list:
assert call.kwargs.get("require_https") is True, (
f"dataset fetch must pass require_https=True; got {call!r}"
)
def test_doaj_fetch_pins_https(tmp_path):
from local_deep_research.journal_quality.data_sources.doaj import (
DOAJSource,
)
with patch(_RETRIES, side_effect=_StopFetch) as fake:
with pytest.raises(_StopFetch):
DOAJSource().fetch(tmp_path)
_assert_pinned(fake, expected_calls=1)
def test_predatory_fetch_pins_https(tmp_path):
from local_deep_research.journal_quality.data_sources.predatory import (
PredatorySource,
)
with patch(_RETRIES, side_effect=_StopFetch) as fake:
with pytest.raises(_StopFetch):
PredatorySource().fetch(tmp_path)
_assert_pinned(fake, expected_calls=1)
def test_jabref_fetch_pins_https_for_every_file(tmp_path):
from local_deep_research.journal_quality.data_sources import jabref
# JabRef tolerates per-file failures and then refuses on the floor,
# so every file is requested — each request must carry the pin.
with patch(_RETRIES, side_effect=_StopFetch) as fake:
with pytest.raises(RuntimeError, match="suspiciously few"):
jabref.JabRefSource().fetch(tmp_path)
_assert_pinned(fake, expected_calls=len(jabref._JABREF_FILES))
def test_openalex_manifest_fetch_pins_https(tmp_path):
from local_deep_research.journal_quality.data_sources.openalex import (
OpenAlexSource,
)
with patch(_RETRIES, side_effect=_StopFetch) as fake:
with pytest.raises(_StopFetch):
OpenAlexSource().fetch(tmp_path)
_assert_pinned(fake, expected_calls=1)
def test_institutions_manifest_fetch_pins_https(tmp_path):
from local_deep_research.journal_quality.data_sources.institutions import (
InstitutionSource,
)
with patch(_RETRIES, side_effect=_StopFetch) as fake:
with pytest.raises(_StopFetch):
InstitutionSource().fetch(tmp_path)
_assert_pinned(fake, expected_calls=1)