~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""HTTPS integrity policy for SafeSession-backed library downloads.
Covers ``SafeSession``'s downgrade guard and the research-library paths that
apply it: the library download service's downloaders and the Zotero import.
Every transport is a scripted adapter, so nothing here touches the network.
"""
import io
from unittest.mock import MagicMock
import pytest
import requests
from local_deep_research.research_library.downloaders import base
from local_deep_research.research_library.downloaders.direct_pdf import (
DirectPDFDownloader,
)
from local_deep_research.research_library.downloaders.playwright_html import (
AutoHTMLDownloader,
PlaywrightHTMLDownloader,
)
from local_deep_research.research_library.services import download_service
from local_deep_research.research_library.zotero import (
client as zotero_client_module,
)
from local_deep_research.research_library.zotero.client import ZoteroClient
from local_deep_research.security.safe_requests import SafeSession
START = "https://93.184.216.34/paper.pdf"
class ScriptedAdapter(requests.adapters.BaseAdapter):
"""Answer redirect chains locally and record every attempted transport.
Each scripted response is ``(status, location)`` or
``(status, location, body)``.
"""
def __init__(self, responses):
self.responses = list(responses)
self.sent = []
def send(self, request, **kwargs):
self.sent.append(request.url)
status, location, *body = self.responses.pop(0)
response = requests.Response()
response.status_code = status
response.url = request.url
response.request = request
response.raw = io.BytesIO(body[0] if body else b"")
if location:
response.headers["Location"] = location
return response
def close(self):
pass
def session_with_responses(responses, *, block_https_downgrade=True):
session = SafeSession(block_https_downgrade=block_https_downgrade)
adapter = ScriptedAdapter(responses)
session.mount("https://", adapter)
session.mount("http://", adapter)
return session, adapter
@pytest.mark.parametrize(
("location", "expected"),
[
("/next.pdf", "https://93.184.216.34/next.pdf"),
("//1.1.1.1/next.pdf", "https://1.1.1.1/next.pdf"),
],
)
def test_relative_redirects_remain_on_https(location, expected):
session, adapter = session_with_responses([(302, location), (200, None)])
with session:
response = session.get(START)
assert response.status_code == 200
assert adapter.sent == [START, expected]
@pytest.mark.parametrize(
"responses, expected_sent",
[
([(302, "http://1.1.1.1/next.pdf")], [START]),
(
[
(302, "https://1.1.1.1/first.pdf"),
(302, "http://1.1.1.1/next.pdf"),
],
[START, "https://1.1.1.1/first.pdf"],
),
],
)
def test_https_to_http_redirect_never_reaches_transport(
responses, expected_sent
):
session, adapter = session_with_responses(responses)
with session:
with pytest.raises(ValueError, match="downgrade to cleartext"):
session.get(START)
assert adapter.sent == expected_sent
def test_http_starting_url_can_follow_http_redirect():
session, adapter = session_with_responses([(302, "/next.pdf"), (200, None)])
with session:
response = session.get("http://93.184.216.34/paper.pdf")
assert response.status_code == 200
assert adapter.sent == [
"http://93.184.216.34/paper.pdf",
"http://93.184.216.34/next.pdf",
]
def test_disabled_redirect_following_returns_response_without_fetching_http():
session, adapter = session_with_responses(
[(302, "http://1.1.1.1/next.pdf")]
)
with session:
response = session.get(START, allow_redirects=False)
assert response.status_code == 302
assert adapter.sent == [START]
@pytest.mark.parametrize("disable_at_session", [False, True])
def test_https_policy_rejects_disabled_certificate_verification(
disable_at_session,
):
session, adapter = session_with_responses([(200, None)])
if disable_at_session:
session.verify = False
with session:
with pytest.raises(ValueError, match="verify=False"):
session.get(
START, **({} if disable_at_session else {"verify": False})
)
assert adapter.sent == []
def test_prepared_https_request_rejects_none_verification():
session, adapter = session_with_responses([(200, None)])
prepared = session.prepare_request(requests.Request("GET", START))
with session:
with pytest.raises(ValueError, match="verify=False"):
session.send(prepared, verify=None)
assert adapter.sent == []
@pytest.mark.parametrize("verify_options", [{}, {"verify": True}])
def test_prepared_https_request_keeps_verified_transport(verify_options):
session, adapter = session_with_responses([(200, None)])
prepared = session.prepare_request(requests.Request("GET", START))
with session:
response = session.send(prepared, **verify_options)
assert response.status_code == 200
assert adapter.sent == [START]
def test_default_session_retains_legacy_redirect_behavior():
session, adapter = session_with_responses(
[(302, "http://1.1.1.1/next.pdf"), (200, None)],
block_https_downgrade=False,
)
with session:
response = session.get(START)
assert response.status_code == 200
assert adapter.sent == [START, "http://1.1.1.1/next.pdf"]
def test_research_library_pdf_downloader_enables_https_policy(monkeypatch):
tracker = MagicMock()
tracker.apply_rate_limit.return_value = 0
monkeypatch.setattr(base, "AdaptiveRateLimitTracker", lambda **_: tracker)
adapter = ScriptedAdapter([(302, "http://1.1.1.1/next.pdf")])
with DirectPDFDownloader() as downloader:
assert downloader.session.block_https_downgrade is True
downloader.session.mount("https://", adapter)
downloader.session.mount("http://", adapter)
assert downloader._download_pdf(START) is None
assert adapter.sent == [START]
tracker.record_outcome.assert_called_once()
@pytest.fixture
def library_download_service(monkeypatch, tmp_path):
"""The research library's DownloadService, built without a user database."""
tracker = MagicMock()
tracker.apply_rate_limit.return_value = 0
monkeypatch.setattr(base, "AdaptiveRateLimitTracker", lambda **_: tracker)
settings = MagicMock()
settings.get_setting.side_effect = lambda key, default=None: (
str(tmp_path) if key == "research_library.storage_path" else default
)
monkeypatch.setattr(
download_service, "get_settings_manager", lambda **_: settings
)
monkeypatch.setattr(
download_service, "get_library_directory", lambda: tmp_path
)
monkeypatch.setattr(download_service, "RetryManager", MagicMock())
service = download_service.DownloadService("library_user", "pw")
yield service
service.close()
def test_every_library_download_service_downloader_refuses_the_downgrade(
library_download_service,
):
"""Every downloader the library downloads through refuses HTTPS to HTTP.
None of them may be a browser downloader: the Playwright and Crawl4AI
route guards follow redirects without a scheme check.
"""
downloaders = library_download_service.downloaders
assert downloaders
for downloader in downloaders:
name = type(downloader).__name__
assert not isinstance(
downloader, (AutoHTMLDownloader, PlaywrightHTMLDownloader)
), name
adapter = ScriptedAdapter([(302, "http://1.1.1.1/next.pdf")])
downloader.session.mount("https://", adapter)
downloader.session.mount("http://", adapter)
with pytest.raises(ValueError, match="downgrade to cleartext"):
downloader.session.get(START)
assert adapter.sent == [START], name
ZOTERO_PDF = b"%PDF-1.7 attachment bytes"
def scripted_zotero_client(
monkeypatch, *, base_url, api_responses, storage_responses, local=False
):
"""A ZoteroClient whose API and storage sessions answer from scripts.
``base_url`` replaces the API base with an IP literal, so URL validation
needs no DNS lookup.
"""
client = ZoteroClient(
api_key="" if local else "KEY",
library_type="user",
library_id="1",
local=local,
)
client._base = base_url
api = ScriptedAdapter(api_responses)
client.session.mount("https://", api)
client.session.mount("http://", api)
storage = ScriptedAdapter(storage_responses)
def storage_session(*args, **kwargs):
session = SafeSession(*args, **kwargs)
session.mount("https://", storage)
session.mount("http://", storage)
return session
monkeypatch.setattr(zotero_client_module, "SafeSession", storage_session)
return client, api, storage
@pytest.mark.parametrize(
(
"base_url",
"local",
"location",
"storage_responses",
"imported",
"storage_sent",
),
[
pytest.param(
"https://93.184.216.34",
False,
"http://1.1.1.1/file.pdf",
[(200, None, ZOTERO_PDF)],
None,
[],
id="api-redirects-to-http",
),
pytest.param(
"https://93.184.216.34",
False,
"HTTP://1.1.1.1/file.pdf",
[(200, None, ZOTERO_PDF)],
None,
[],
id="api-redirects-to-uppercase-http",
),
pytest.param(
"https://93.184.216.34",
False,
"https://1.1.1.1/file.pdf",
[(302, "http://1.1.1.1/plain.pdf"), (200, None, ZOTERO_PDF)],
None,
["https://1.1.1.1/file.pdf"],
id="storage-redirects-to-http",
),
pytest.param(
"https://93.184.216.34",
False,
"https://1.1.1.1/file.pdf",
[(200, None, ZOTERO_PDF)],
ZOTERO_PDF,
["https://1.1.1.1/file.pdf"],
id="https-storage-is-fetched",
),
pytest.param(
"http://93.184.216.34/api",
True,
"http://1.1.1.1/file.pdf",
[(200, None, ZOTERO_PDF)],
ZOTERO_PDF,
["http://1.1.1.1/file.pdf"],
id="local-http-api-has-no-https-to-leave",
),
],
)
def test_zotero_attachment_import_refuses_https_to_http(
monkeypatch,
base_url,
local,
location,
storage_responses,
imported,
storage_sent,
):
client, api, storage = scripted_zotero_client(
monkeypatch,
base_url=base_url,
local=local,
api_responses=[(302, location)],
storage_responses=storage_responses,
)
with client:
assert client.download_attachment("ATT1") == imported
assert api.sent == [f"{base_url}/users/1/items/ATT1/file"]
assert storage.sent == storage_sent
def test_zotero_api_redirect_from_https_to_http_is_refused(monkeypatch):
"""Text-only imports store Zotero's full text instead of the PDF."""
client, api, _ = scripted_zotero_client(
monkeypatch,
base_url="https://93.184.216.34",
api_responses=[
(302, "http://1.1.1.1/fulltext"),
(200, None, b'{"content": "replaced text"}'),
],
storage_responses=[],
)
with client:
assert client.get_attachment_fulltext("ATT1") is None
assert api.sent == ["https://93.184.216.34/users/1/items/ATT1/fulltext"]