Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

11.5 KB
Raw Blame
"""HTTPS integrity policy for SafeSession-backed library downloads.

Covers ``SafeSession``'s downgrade guard and the research-library paths that
apply it: the library download service's downloaders and the Zotero import.
Every transport is a scripted adapter, so nothing here touches the network.
"""

import io
from unittest.mock import MagicMock

import pytest
import requests

from local_deep_research.research_library.downloaders import base
from local_deep_research.research_library.downloaders.direct_pdf import (
    DirectPDFDownloader,
)
from local_deep_research.research_library.downloaders.playwright_html import (
    AutoHTMLDownloader,
    PlaywrightHTMLDownloader,
)
from local_deep_research.research_library.services import download_service
from local_deep_research.research_library.zotero import (
    client as zotero_client_module,
)
from local_deep_research.research_library.zotero.client import ZoteroClient
from local_deep_research.security.safe_requests import SafeSession


START = "https://93.184.216.34/paper.pdf"


class ScriptedAdapter(requests.adapters.BaseAdapter):
    """Answer redirect chains locally and record every attempted transport.

    Each scripted response is ``(status, location)`` or
    ``(status, location, body)``.
    """

    def __init__(self, responses):
        self.responses = list(responses)
        self.sent = []

    def send(self, request, **kwargs):
        self.sent.append(request.url)
        status, location, *body = self.responses.pop(0)
        response = requests.Response()
        response.status_code = status
        response.url = request.url
        response.request = request
        response.raw = io.BytesIO(body[0] if body else b"")
        if location:
            response.headers["Location"] = location
        return response

    def close(self):
        pass


def session_with_responses(responses, *, block_https_downgrade=True):
    session = SafeSession(block_https_downgrade=block_https_downgrade)
    adapter = ScriptedAdapter(responses)
    session.mount("https://", adapter)
    session.mount("http://", adapter)
    return session, adapter


@pytest.mark.parametrize(
    ("location", "expected"),
    [
        ("/next.pdf", "https://93.184.216.34/next.pdf"),
        ("//1.1.1.1/next.pdf", "https://1.1.1.1/next.pdf"),
    ],
)
def test_relative_redirects_remain_on_https(location, expected):
    session, adapter = session_with_responses([(302, location), (200, None)])

    with session:
        response = session.get(START)

    assert response.status_code == 200
    assert adapter.sent == [START, expected]


@pytest.mark.parametrize(
    "responses, expected_sent",
    [
        ([(302, "http://1.1.1.1/next.pdf")], [START]),
        (
            [
                (302, "https://1.1.1.1/first.pdf"),
                (302, "http://1.1.1.1/next.pdf"),
            ],
            [START, "https://1.1.1.1/first.pdf"],
        ),
    ],
)
def test_https_to_http_redirect_never_reaches_transport(
    responses, expected_sent
):
    session, adapter = session_with_responses(responses)

    with session:
        with pytest.raises(ValueError, match="downgrade to cleartext"):
            session.get(START)

    assert adapter.sent == expected_sent


def test_http_starting_url_can_follow_http_redirect():
    session, adapter = session_with_responses([(302, "/next.pdf"), (200, None)])

    with session:
        response = session.get("http://93.184.216.34/paper.pdf")

    assert response.status_code == 200
    assert adapter.sent == [
        "http://93.184.216.34/paper.pdf",
        "http://93.184.216.34/next.pdf",
    ]


def test_disabled_redirect_following_returns_response_without_fetching_http():
    session, adapter = session_with_responses(
        [(302, "http://1.1.1.1/next.pdf")]
    )

    with session:
        response = session.get(START, allow_redirects=False)

    assert response.status_code == 302
    assert adapter.sent == [START]


@pytest.mark.parametrize("disable_at_session", [False, True])
def test_https_policy_rejects_disabled_certificate_verification(
    disable_at_session,
):
    session, adapter = session_with_responses([(200, None)])
    if disable_at_session:
        session.verify = False

    with session:
        with pytest.raises(ValueError, match="verify=False"):
            session.get(
                START, **({} if disable_at_session else {"verify": False})
            )

    assert adapter.sent == []


def test_prepared_https_request_rejects_none_verification():
    session, adapter = session_with_responses([(200, None)])
    prepared = session.prepare_request(requests.Request("GET", START))

    with session:
        with pytest.raises(ValueError, match="verify=False"):
            session.send(prepared, verify=None)

    assert adapter.sent == []


@pytest.mark.parametrize("verify_options", [{}, {"verify": True}])
def test_prepared_https_request_keeps_verified_transport(verify_options):
    session, adapter = session_with_responses([(200, None)])
    prepared = session.prepare_request(requests.Request("GET", START))

    with session:
        response = session.send(prepared, **verify_options)

    assert response.status_code == 200
    assert adapter.sent == [START]


def test_default_session_retains_legacy_redirect_behavior():
    session, adapter = session_with_responses(
        [(302, "http://1.1.1.1/next.pdf"), (200, None)],
        block_https_downgrade=False,
    )

    with session:
        response = session.get(START)

    assert response.status_code == 200
    assert adapter.sent == [START, "http://1.1.1.1/next.pdf"]


def test_research_library_pdf_downloader_enables_https_policy(monkeypatch):
    tracker = MagicMock()
    tracker.apply_rate_limit.return_value = 0
    monkeypatch.setattr(base, "AdaptiveRateLimitTracker", lambda **_: tracker)
    adapter = ScriptedAdapter([(302, "http://1.1.1.1/next.pdf")])

    with DirectPDFDownloader() as downloader:
        assert downloader.session.block_https_downgrade is True
        downloader.session.mount("https://", adapter)
        downloader.session.mount("http://", adapter)
        assert downloader._download_pdf(START) is None

    assert adapter.sent == [START]
    tracker.record_outcome.assert_called_once()


@pytest.fixture
def library_download_service(monkeypatch, tmp_path):
    """The research library's DownloadService, built without a user database."""
    tracker = MagicMock()
    tracker.apply_rate_limit.return_value = 0
    monkeypatch.setattr(base, "AdaptiveRateLimitTracker", lambda **_: tracker)
    settings = MagicMock()
    settings.get_setting.side_effect = lambda key, default=None: (
        str(tmp_path) if key == "research_library.storage_path" else default
    )
    monkeypatch.setattr(
        download_service, "get_settings_manager", lambda **_: settings
    )
    monkeypatch.setattr(
        download_service, "get_library_directory", lambda: tmp_path
    )
    monkeypatch.setattr(download_service, "RetryManager", MagicMock())
    service = download_service.DownloadService("library_user", "pw")
    yield service
    service.close()


def test_every_library_download_service_downloader_refuses_the_downgrade(
    library_download_service,
):
    """Every downloader the library downloads through refuses HTTPS to HTTP.

    None of them may be a browser downloader: the Playwright and Crawl4AI
    route guards follow redirects without a scheme check.
    """
    downloaders = library_download_service.downloaders
    assert downloaders
    for downloader in downloaders:
        name = type(downloader).__name__
        assert not isinstance(
            downloader, (AutoHTMLDownloader, PlaywrightHTMLDownloader)
        ), name
        adapter = ScriptedAdapter([(302, "http://1.1.1.1/next.pdf")])
        downloader.session.mount("https://", adapter)
        downloader.session.mount("http://", adapter)

        with pytest.raises(ValueError, match="downgrade to cleartext"):
            downloader.session.get(START)

        assert adapter.sent == [START], name


ZOTERO_PDF = b"%PDF-1.7 attachment bytes"


def scripted_zotero_client(
    monkeypatch, *, base_url, api_responses, storage_responses, local=False
):
    """A ZoteroClient whose API and storage sessions answer from scripts.

    ``base_url`` replaces the API base with an IP literal, so URL validation
    needs no DNS lookup.
    """
    client = ZoteroClient(
        api_key="" if local else "KEY",
        library_type="user",
        library_id="1",
        local=local,
    )
    client._base = base_url
    api = ScriptedAdapter(api_responses)
    client.session.mount("https://", api)
    client.session.mount("http://", api)
    storage = ScriptedAdapter(storage_responses)

    def storage_session(*args, **kwargs):
        session = SafeSession(*args, **kwargs)
        session.mount("https://", storage)
        session.mount("http://", storage)
        return session

    monkeypatch.setattr(zotero_client_module, "SafeSession", storage_session)
    return client, api, storage


@pytest.mark.parametrize(
    (
        "base_url",
        "local",
        "location",
        "storage_responses",
        "imported",
        "storage_sent",
    ),
    [
        pytest.param(
            "https://93.184.216.34",
            False,
            "http://1.1.1.1/file.pdf",
            [(200, None, ZOTERO_PDF)],
            None,
            [],
            id="api-redirects-to-http",
        ),
        pytest.param(
            "https://93.184.216.34",
            False,
            "HTTP://1.1.1.1/file.pdf",
            [(200, None, ZOTERO_PDF)],
            None,
            [],
            id="api-redirects-to-uppercase-http",
        ),
        pytest.param(
            "https://93.184.216.34",
            False,
            "https://1.1.1.1/file.pdf",
            [(302, "http://1.1.1.1/plain.pdf"), (200, None, ZOTERO_PDF)],
            None,
            ["https://1.1.1.1/file.pdf"],
            id="storage-redirects-to-http",
        ),
        pytest.param(
            "https://93.184.216.34",
            False,
            "https://1.1.1.1/file.pdf",
            [(200, None, ZOTERO_PDF)],
            ZOTERO_PDF,
            ["https://1.1.1.1/file.pdf"],
            id="https-storage-is-fetched",
        ),
        pytest.param(
            "http://93.184.216.34/api",
            True,
            "http://1.1.1.1/file.pdf",
            [(200, None, ZOTERO_PDF)],
            ZOTERO_PDF,
            ["http://1.1.1.1/file.pdf"],
            id="local-http-api-has-no-https-to-leave",
        ),
    ],
)
def test_zotero_attachment_import_refuses_https_to_http(
    monkeypatch,
    base_url,
    local,
    location,
    storage_responses,
    imported,
    storage_sent,
):
    client, api, storage = scripted_zotero_client(
        monkeypatch,
        base_url=base_url,
        local=local,
        api_responses=[(302, location)],
        storage_responses=storage_responses,
    )

    with client:
        assert client.download_attachment("ATT1") == imported

    assert api.sent == [f"{base_url}/users/1/items/ATT1/file"]
    assert storage.sent == storage_sent


def test_zotero_api_redirect_from_https_to_http_is_refused(monkeypatch):
    """Text-only imports store Zotero's full text instead of the PDF."""
    client, api, _ = scripted_zotero_client(
        monkeypatch,
        base_url="https://93.184.216.34",
        api_responses=[
            (302, "http://1.1.1.1/fulltext"),
            (200, None, b'{"content": "replaced text"}'),
        ],
        storage_responses=[],
    )

    with client:
        assert client.get_attachment_fulltext("ATT1") is None

    assert api.sent == ["https://93.184.216.34/users/1/items/ATT1/fulltext"]