Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

10.1 KB
Raw Blame
"""HTTPS scheme pinning for dataset fetches (``safe_get``).

``safe_get`` validates every redirect hop against the SSRF rules —
host and IP scope only. The scheme is never pinned: a hop from
``https://`` to ``http://`` on a public host passes validation, so a
compromised upstream (the journal-quality datasets come from a public
S3 bucket, DOAJ, and GitHub raw) or any open-redirect on the path can
move the fetch to cleartext, where an on-path attacker can tamper with
the bytes that later drive journal reputation scores.

The fix must be opt-in per caller: localhost SearXNG instances are
legitimately ``http://`` (``ALLOWED_SCHEMES`` includes http for that
reason), so the pin is a ``require_https`` flag rather than a global
restriction.

These tests pin the flag: with it, an initial non-https URL and any
redirect hop that would downgrade to cleartext are refused; without
it, current behaviour is unchanged; https-preserving hops still
follow.
"""

from contextlib import contextmanager, nullcontext
from unittest.mock import MagicMock, patch

import pytest

from local_deep_research.security import safe_requests

_SR = "local_deep_research.security.safe_requests"


def _redirect(location: str) -> MagicMock:
    r = MagicMock()
    r.status_code = 302
    r.headers = {"Location": location}
    r.url = "https://example.com/"
    r.close = MagicMock()
    return r


def _ok(url: str = "https://example.com/final") -> MagicMock:
    r = MagicMock()
    r.status_code = 200
    r.headers = {}
    r.url = url
    r.close = MagicMock()
    return r


@contextmanager
def _patches(responses):
    """Patch validator, DNS pinning, and requests.get — no network."""
    with (
        patch.object(
            safe_requests.ssrf_validator, "validate_url", return_value=True
        ) as validator,
        patch.object(
            safe_requests.dns_pinning,
            "pinned_request",
            MagicMock(return_value=nullcontext()),
        ) as pin,
        patch(f"{_SR}.requests.get", side_effect=responses) as gets,
    ):
        yield validator, pin, gets


def test_https_to_http_redirect_downgrade_is_refused():
    """A redirect hop to cleartext must abort the pinned fetch."""
    with _patches([_redirect("http://example.com/data"), _ok()]) as (
        _v,
        _p,
        gets,
    ):
        with pytest.raises(ValueError, match="downgrade"):
            safe_requests.safe_get("https://example.com/", require_https=True)
    assert gets.call_count == 1, "the cleartext hop must never be fetched"


def test_non_https_initial_url_is_refused():
    """The pin applies to the initial URL, not just redirect hops."""
    with _patches([_ok()]) as (_v, _p, gets):
        with pytest.raises(ValueError, match="https"):
            safe_requests.safe_get(
                "http://example.com/data", require_https=True
            )
    assert gets.call_count == 0


def test_https_to_https_redirect_still_followed():
    """Scheme-preserving hops keep working under the pin."""
    with _patches(
        [_redirect("https://example.com/data"), _ok("https://example.com/data")]
    ) as (_v, _p, gets):
        response = safe_requests.safe_get(
            "https://example.com/", require_https=True
        )

    assert response.status_code == 200
    assert gets.call_count == 2


def test_downgrade_without_the_flag_keeps_current_behaviour():
    """Control: without require_https the hop is followed as today."""
    with _patches(
        [_redirect("http://example.com/data"), _ok("http://example.com/data")]
    ) as (_v, _p, gets):
        response = safe_requests.safe_get("https://example.com/")

    assert response.status_code == 200
    assert gets.call_count == 2


# --- Redirect hops are judged after urljoin -------------------------------


def test_relative_location_inherits_https_and_is_followed():
    """A relative ``Location`` resolves against the https URL, so it is
    followed. Judging the raw header ("/x" does not start with https://)
    would wrongly refuse every same-origin relative redirect."""
    with _patches([_redirect("/x"), _ok("https://example.com/x")]) as (
        _v,
        _p,
        gets,
    ):
        response = safe_requests.safe_get(
            "https://example.com/", require_https=True
        )

    assert response.status_code == 200
    assert gets.call_count == 2
    assert gets.call_args_list[1].args[0] == "https://example.com/x"


def test_protocol_relative_location_inherits_https_and_is_followed():
    """``//cdn.example/x`` from an https page resolves to https."""
    with _patches(
        [_redirect("//cdn.example/x"), _ok("https://cdn.example/x")]
    ) as (_v, _p, gets):
        response = safe_requests.safe_get(
            "https://example.com/", require_https=True
        )

    assert response.status_code == 200
    assert gets.call_count == 2
    assert gets.call_args_list[1].args[0] == "https://cdn.example/x"


def test_downgrade_on_a_later_hop_is_refused():
    """https -> https -> http: the pin applies to every hop, not just the
    first one."""
    with _patches(
        [
            _redirect("https://mirror.example/1"),
            _redirect("http://mirror.example/2"),
            _ok("http://mirror.example/2"),
        ]
    ) as (_v, _p, gets):
        with pytest.raises(ValueError, match="downgrade"):
            safe_requests.safe_get("https://example.com/", require_https=True)

    assert gets.call_count == 2, "the cleartext hop must never be fetched"


# --- https without certificate verification is refused ---------------------


@pytest.mark.parametrize("verify", [False, 0, ""])
def test_require_https_with_verify_false_is_refused(verify):
    """Pinning the scheme while disabling certificate verification gives
    no authenticated transport; refuse the combination outright. requests
    treats any falsy ``verify`` as "don't verify", so all of them are
    refused."""
    with _patches([_ok()]) as (_v, _p, gets):
        with pytest.raises(ValueError, match="verify=False"):
            safe_requests.safe_get(
                "https://example.com/", require_https=True, verify=verify
            )
    assert gets.call_count == 0


def test_require_https_with_default_verify_none_is_allowed():
    """Control: ``verify=None`` merges to the session default (verified),
    so it is not refused."""
    with _patches([_ok()]) as (_v, _p, gets):
        response = safe_requests.safe_get(
            "https://example.com/", require_https=True, verify=None
        )
    assert response.status_code == 200
    assert gets.call_count == 1


def test_retry_wrapper_forwards_require_https():
    """The six dataset sites call ``safe_get_with_retries``; the pin only
    holds if the wrapper hands the flag to ``safe_get`` on every attempt."""
    fake = MagicMock(return_value=_ok())
    with patch.object(safe_requests, "safe_get", fake):
        safe_requests.safe_get_with_retries(
            "https://example.com/", require_https=True
        )
    assert fake.call_count == 1
    assert fake.call_args.kwargs.get("require_https") is True


def test_require_https_with_ca_bundle_verify_is_allowed():
    """Control: a CA-bundle path for ``verify`` still verifies, so it is
    not refused."""
    with _patches([_ok()]) as (_v, _p, gets):
        response = safe_requests.safe_get(
            "https://example.com/",
            require_https=True,
            verify="/etc/ssl/certs/ca-certificates.crt",
        )
    assert response.status_code == 200
    assert gets.call_count == 1


# --- Wiring: every journal-quality dataset fetch sets require_https --------
#
# The flag is opt-in, so the protection exists only where a call site
# passes it. Each test drives one source's ``fetch`` against a fake
# ``safe_get_with_retries`` (the sources import it lazily from
# ``security.safe_requests``) and asserts the kwarg. The fake raises to
# stop the fetch at the first request — no parsing, no network.

_RETRIES = "local_deep_research.security.safe_requests.safe_get_with_retries"


class _StopFetch(Exception):
    pass


def _assert_pinned(fake, expected_calls=None):
    assert fake.call_count >= 1
    if expected_calls is not None:
        assert fake.call_count == expected_calls
    for call in fake.call_args_list:
        assert call.kwargs.get("require_https") is True, (
            f"dataset fetch must pass require_https=True; got {call!r}"
        )


def test_doaj_fetch_pins_https(tmp_path):
    from local_deep_research.journal_quality.data_sources.doaj import (
        DOAJSource,
    )

    with patch(_RETRIES, side_effect=_StopFetch) as fake:
        with pytest.raises(_StopFetch):
            DOAJSource().fetch(tmp_path)
    _assert_pinned(fake, expected_calls=1)


def test_predatory_fetch_pins_https(tmp_path):
    from local_deep_research.journal_quality.data_sources.predatory import (
        PredatorySource,
    )

    with patch(_RETRIES, side_effect=_StopFetch) as fake:
        with pytest.raises(_StopFetch):
            PredatorySource().fetch(tmp_path)
    _assert_pinned(fake, expected_calls=1)


def test_jabref_fetch_pins_https_for_every_file(tmp_path):
    from local_deep_research.journal_quality.data_sources import jabref

    # JabRef tolerates per-file failures and then refuses on the floor,
    # so every file is requested — each request must carry the pin.
    with patch(_RETRIES, side_effect=_StopFetch) as fake:
        with pytest.raises(RuntimeError, match="suspiciously few"):
            jabref.JabRefSource().fetch(tmp_path)
    _assert_pinned(fake, expected_calls=len(jabref._JABREF_FILES))


def test_openalex_manifest_fetch_pins_https(tmp_path):
    from local_deep_research.journal_quality.data_sources.openalex import (
        OpenAlexSource,
    )

    with patch(_RETRIES, side_effect=_StopFetch) as fake:
        with pytest.raises(_StopFetch):
            OpenAlexSource().fetch(tmp_path)
    _assert_pinned(fake, expected_calls=1)


def test_institutions_manifest_fetch_pins_https(tmp_path):
    from local_deep_research.journal_quality.data_sources.institutions import (
        InstitutionSource,
    )

    with patch(_RETRIES, side_effect=_StopFetch) as fake:
        with pytest.raises(_StopFetch):
            InstitutionSource().fetch(tmp_path)
    _assert_pinned(fake, expected_calls=1)