~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Only complete source text may satisfy a later full-document request."""
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from local_deep_research.content_fetcher.fetcher import ContentFetcher
from local_deep_research.utilities.research_content_cache import (
ResearchContentCache,
)
from local_deep_research.web_search_engines.engines.search_engine_arxiv import (
ArXivSearchEngine,
FullTextFetchFailedError,
FullTextNotExtractedError,
)
URL = "https://arxiv.org/abs/2301.12345"
ABSTRACT = "An abstract that is longer than the complete test paper."
BODY = "Complete paper."
@pytest.fixture
def arxiv_engine(tmp_path):
with patch.object(
ArXivSearchEngine, "_create_journal_filter", return_value=None
):
engine = ArXivSearchEngine(
include_full_text=True,
download_dir=str(tmp_path),
settings_snapshot={},
)
engine._configure_programmatic_mode(True)
engine.search_snippets_only = False
engine.content_cache = ResearchContentCache()
engine.rate_tracker = MagicMock()
paper = SimpleNamespace(
entry_id=URL,
summary=ABSTRACT,
authors=[],
published=None,
updated=None,
categories=[],
comment="",
doi="",
journal_ref="",
title="Test paper",
)
def previews(query):
engine._papers = {URL: paper}
return [
{"id": URL, "link": URL, "title": paper.title, "snippet": ABSTRACT}
]
engine._get_previews = previews
with patch(
"local_deep_research.security.ssrf_validator.validate_url",
return_value=True,
):
yield engine
@pytest.mark.parametrize(
"initial",
[
FullTextFetchFailedError("temporary outage"),
FullTextNotExtractedError("empty extraction"),
None,
" ",
],
)
def test_arxiv_retries_partial_content_after_recovery(arxiv_engine, initial):
engine = arxiv_engine
engine._fetch_full_text = MagicMock(side_effect=[initial, BODY])
assert engine.run("first search")[0]["full_content"] == ABSTRACT
assert engine.content_cache.get(URL) is None
assert engine.run("later search")[0]["full_content"] == BODY
assert engine.run("repeat search")[0]["full_content"] == BODY
assert engine._fetch_full_text.call_count == 2
assert engine.content_cache.get(URL)["content"] == BODY
@pytest.mark.parametrize("reason", ["disabled", "budget"])
def test_abstract_only_search_does_not_satisfy_full_fetch(arxiv_engine, reason):
engine = arxiv_engine
if reason == "disabled":
engine.include_full_text = False
else:
engine.max_full_text = 0
engine._fetch_full_text = MagicMock(
side_effect=AssertionError("search must not fetch")
)
assert engine.run("search")[0]["full_content"] == ABSTRACT
downloader = MagicMock()
downloader.download_with_result.return_value = SimpleNamespace(
is_success=True, content=BODY.encode(), skip_reason=None
)
downloader.get_metadata.return_value = {"title": "Test paper"}
with (
patch(
"local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
return_value=True,
),
patch.object(
ContentFetcher, "_get_downloader", return_value=downloader
),
):
with ContentFetcher(content_cache=engine.content_cache) as fetcher:
assert fetcher.fetch(URL)["content"] == BODY
assert fetcher.fetch(URL)["content"] == BODY
assert downloader.download_with_result.call_count == 1
def test_complete_cache_replaces_partial_preview(arxiv_engine):
engine = arxiv_engine
engine.content_cache.put(URL, BODY)
engine._fetch_full_text = MagicMock(
side_effect=AssertionError("must reuse complete text")
)
result = engine._get_full_content_cached(
[
{
"link": URL,
"full_content": ABSTRACT,
"full_content_is_complete": False,
}
]
)
assert result[0]["full_content"] == BODY
assert result[0]["full_content_is_complete"] is True
def test_complete_preview_is_preserved(arxiv_engine):
arxiv_engine.content_cache.put(URL, "Previously fetched text")
result = arxiv_engine._get_full_content_cached(
[
{
"link": URL,
"full_content": BODY,
"full_content_is_complete": True,
}
]
)
assert result[0]["full_content"] == BODY
@pytest.mark.parametrize("complete", [None, False, "true", 1, True])
def test_unverified_search_content_cannot_fill_document_cache(
arxiv_engine, complete
):
item = {"link": URL, "full_content": ABSTRACT}
if complete is not None:
item["full_content_is_complete"] = complete
arxiv_engine._store_full_content_in_cache([item])
assert (arxiv_engine.content_cache.get(URL) is not None) is (
complete is True
)
def test_pubmed_abstract_fallback_is_retried():
from local_deep_research.web_search_engines.engines.search_engine_pubmed import (
PubMedSearchEngine,
)
with patch.object(
PubMedSearchEngine, "_create_journal_filter", return_value=None
):
engine = PubMedSearchEngine(get_full_text=True, settings_snapshot={})
engine.content_cache = ResearchContentCache()
engine._get_article_abstracts = MagicMock(return_value={"123": ABSTRACT})
engine._get_article_detailed_metadata = MagicMock(return_value={})
engine._find_pmc_ids = MagicMock(return_value={"123": "PMC123"})
engine._get_pmc_full_text = MagicMock(side_effect=[None, BODY])
url = "https://pubmed.ncbi.nlm.nih.gov/123/"
def search():
return engine._get_full_content_cached(
[{"_pmid": "123", "link": url, "title": "Paper"}]
)[0]
with patch(
"local_deep_research.security.ssrf_validator.validate_url",
return_value=True,
):
assert search()["content_type"] == "abstract"
assert engine.content_cache.get(url) is None
assert BODY in search()["full_content"]
assert BODY in search()["full_content"]
assert engine._get_pmc_full_text.call_count == 2
@pytest.mark.parametrize(
"initial", [None, {}, RuntimeError("temporary outage")]
)
def test_paperless_snippet_fallback_is_retried(initial):
from local_deep_research.web_search_engines.engines.search_engine_paperless import (
PaperlessSearchEngine,
)
engine = PaperlessSearchEngine.__new__(PaperlessSearchEngine)
engine.include_content = True
engine.content_cache = ResearchContentCache()
engine._make_request = MagicMock(side_effect=[initial, {"content": BODY}])
engine._scrub_error = MagicMock(return_value="request failed")
engine._is_cache_url_allowed = MagicMock(return_value=True)
url = "https://paperless.example/documents/123/"
def search():
return engine._get_full_content_cached(
[
{
"link": url,
"snippet": ABSTRACT,
"_raw_data": {},
"metadata": {"doc_id": "123"},
}
]
)[0]
assert search()["full_content"] == ABSTRACT
assert engine.content_cache.get(url) is None
assert search()["full_content"] == BODY
assert search()["full_content"] == BODY
assert engine._make_request.call_count == 2
def test_wayback_error_text_is_not_a_complete_cached_page():
from local_deep_research.web_search_engines.engines.search_engine_wayback import (
WaybackSearchEngine,
)
engine = WaybackSearchEngine.__new__(WaybackSearchEngine)
engine.content_cache = ResearchContentCache()
engine.rate_tracker = MagicMock()
engine.engine_type = "wayback"
engine._is_cache_url_allowed = MagicMock(return_value=True)
engine._get_wayback_content = MagicMock(
side_effect=[
("", "Error retrieving content: temporary outage"),
("<p>Complete paper.</p>", BODY),
]
)
url = "https://web.archive.org/web/20200101/https://example.com/"
assert (
"Error retrieving"
in engine._get_full_content_cached([{"link": url}])[0]["full_content"]
)
assert engine.content_cache.get(url) is None
assert (
engine._get_full_content_cached([{"link": url}])[0]["full_content"]
== BODY
)
assert (
engine._get_full_content_cached([{"link": url}])[0]["full_content"]
== BODY
)
assert engine._get_wayback_content.call_count == 2
@pytest.mark.parametrize("kind", ["issue", "user"])
def test_github_summaries_do_not_fill_document_cache(kind):
from local_deep_research.web_search_engines.engines.search_engine_github import (
GitHubSearchEngine,
)
engine = GitHubSearchEngine.__new__(GitHubSearchEngine)
engine.content_cache = ResearchContentCache()
engine._is_cache_url_allowed = MagicMock(return_value=True)
url = "https://github.com/example/repo/issues/1"
result = engine._get_full_content_cached(
[
{
"link": url,
"snippet": ABSTRACT,
"search_type": kind,
}
]
)
assert ABSTRACT in result[0]["full_content"]
assert engine.content_cache.get(url) is None
def test_sofya_snippet_then_extracted_content():
from local_deep_research.web_search_engines.engines.search_engine_sofya import (
SofyaSearchEngine,
)
engine = SofyaSearchEngine.__new__(SofyaSearchEngine)
engine.content_cache = ResearchContentCache()
engine._is_cache_url_allowed = MagicMock(return_value=True)
preview = {"link": URL, "snippet": ABSTRACT}
assert (
engine._get_full_content_cached([preview.copy()])[0]["full_content"]
== ABSTRACT
)
assert engine.content_cache.get(URL) is None
preview["_full_result"] = {"content": BODY}
assert engine._get_full_content_cached([preview])[0]["full_content"] == BODY
assert engine.content_cache.get(URL)["content"] == BODY
def test_missing_library_chunk_can_be_retried():
from local_deep_research.web_search_engines.engines.search_engine_library import (
LibraryRAGSearchEngine,
)
engine = LibraryRAGSearchEngine.__new__(LibraryRAGSearchEngine)
engine.username = "cache-test-user"
engine.content_cache = ResearchContentCache()
engine._is_cache_url_allowed = MagicMock(return_value=True)
document_id = "12345678-1234-5678-1234-567812345678"
url = f"/library/document/{document_id}#chunk-1"
session = MagicMock()
query = session.query.return_value
query.filter.return_value = query
query.order_by.return_value.first.side_effect = [
None,
None,
SimpleNamespace(chunk_text=BODY),
]
query.limit.return_value.all.return_value = []
def search():
return engine._get_full_content_cached(
[
{
"link": url,
"metadata": {
"source_id": document_id,
"chunk_index": 1,
"collection_id": "test",
},
}
]
)[0]
with patch(
"local_deep_research.database.session_context.get_user_db_session"
) as context:
context.return_value.__enter__.return_value = session
assert "not found" in search()["full_content"]
assert engine.content_cache.get(url, collection_id="test") is None
assert search()["full_content"] == BODY
assert search()["full_content"] == BODY
assert context.call_count == 2
@pytest.mark.parametrize(
"mode,unrestricted_cloud",
[
("full", True),
("summary_focus", True),
("summary_focus_query", True),
("summary_focus", False),
("summary_focus_query", False),
],
)
@pytest.mark.parametrize("limit", [1_000, 10_000, 50_000])
def test_repeated_tool_fetch_uses_same_limit_and_keeps_raw_text(
mode, unrestricted_cloud, limit
):
from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
SearchResultsCollector,
)
from local_deep_research.advanced_search_system.tools.fetch import (
build_fetch_tool,
)
from local_deep_research.config.llm_config import (
wrap_llm_without_think_tags,
)
settings = {
"search.fetch.content_max_length": {
"value": limit,
"ui_element": "number",
},
"search.fetch.summary_input_max_length": {
"value": limit,
"ui_element": "number",
},
"llm.provider": "openai",
"llm.context_window_unrestricted": True,
"llm.local_context_window_size": 8192,
"llm.max_tokens": 8192,
"llm.supports_max_tokens": True,
"rate_limiting.llm_enabled": False,
}
cache = ResearchContentCache()
raw = "Ω" * (limit + 2000)
downloader = MagicMock()
downloader.download_with_result.return_value = SimpleNamespace(
is_success=True, content=raw.encode(), skip_reason=None
)
downloader.get_metadata.return_value = {"title": "Paper"}
model = MagicMock()
model.invoke.return_value = SimpleNamespace(content="Selected text")
# Use the real wrapper's build metadata for unrestricted cloud input.
# An unwrapped model remains bounded even if the tool snapshot names a
# cloud provider, since it does not describe how that model was built.
tool_model = (
wrap_llm_without_think_tags(
model, provider="openai", settings_snapshot=settings
)
if unrestricted_cloud
else model
)
tool = build_fetch_tool(
mode,
SearchResultsCollector([]),
model=tool_model,
overall_query="Question",
settings_snapshot=settings,
max_fetches=0,
content_cache=cache,
)
args = {"url": URL}
if mode != "full":
args["focus"] = "Question"
with (
patch(
"local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
return_value=True,
),
patch(
"local_deep_research.security.egress.fetch.policy_aware_validate_url",
return_value=True,
),
patch.object(
ContentFetcher, "_get_downloader", return_value=downloader
),
):
outputs = [tool.invoke(args), tool.invoke(args)]
if mode != "full":
assert model.invoke.call_count == 2
outputs = [call.args[0] for call in model.invoke.call_args_list]
else:
model.invoke.assert_not_called()
expected_limit = limit
context_trims_page = False
if not unrestricted_cloud:
# Independent whole-prompt oracle: an 8192-token window reserves
# int(8192 * 0.8) = 6553 tokens for output, leaving 6556 characters.
# The fixed prompt occupies 1345/1381 characters in these cases.
page_limit = 5211 if mode == "summary_focus" else 5175
expected_limit = min(limit, page_limit)
if limit > page_limit:
context_trims_page = True
assert [len(text) for text in outputs] == [6556, 6556]
assert [text.count("Ω") for text in outputs] == [expected_limit] * 2
if mode != "full":
pages = [
text.split("Page content:\n", 1)[1].split("\n\nOutput rules:", 1)[0]
for text in outputs
]
expected_page = raw[:expected_limit]
if not context_trims_page:
# Both cold and warm ContentFetcher results carry this marker;
# the whole-prompt context cap can remove it with the page tail.
expected_page += "\n\n[... content truncated ...]"
assert pages == [expected_page] * 2
assert cache.get(URL)["content"] == raw
assert downloader.download_with_result.call_count == 1
@pytest.mark.parametrize("surface", ["fetcher", "tool", "pipeline", "batch"])
def test_arxiv_download_metadata_does_not_prevent_later_full_text(
monkeypatch, surface
):
from local_deep_research.research_library.downloaders.arxiv import (
ArxivDownloader,
ArxivTextResult,
ArxivTextSource,
)
from local_deep_research.research_library.downloaders.extraction import (
pipeline,
)
raw = BODY * 20
source = MagicMock(
side_effect=[
ArxivTextResult(ABSTRACT, ArxivTextSource.ARXIV_API),
ArxivTextResult(raw, ArxivTextSource.ARXIV_HTML),
]
)
# Keep the real downloader's result construction and downstream cache
# paths. Only replace its transport/source selection and resource setup.
monkeypatch.setattr(
ArxivDownloader, "__init__", lambda self, **kwargs: None
)
monkeypatch.setattr(ArxivDownloader, "download_text_with_source", source)
monkeypatch.setattr(ArxivDownloader, "get_metadata", lambda self, url: {})
monkeypatch.setattr(ArxivDownloader, "close", lambda self: None)
monkeypatch.setattr(
"local_deep_research.security.ssrf_validator.validate_url",
lambda *args, **kwargs: True,
)
monkeypatch.setattr(
"local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
lambda *args, **kwargs: True,
)
monkeypatch.setattr(
"local_deep_research.security.egress.fetch.policy_aware_validate_url",
lambda *args, **kwargs: True,
)
cache = ResearchContentCache()
if surface in {"fetcher", "tool"}:
monkeypatch.setattr(
ContentFetcher,
"_get_downloader",
lambda self, url_type: ArxivDownloader(),
)
if surface == "fetcher":
def fetch():
with ContentFetcher(content_cache=cache) as fetcher:
return fetcher.fetch(URL)["content"]
elif surface == "tool":
from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
SearchResultsCollector,
)
from local_deep_research.advanced_search_system.tools.fetch import (
build_fetch_tool,
)
tool = build_fetch_tool(
"full",
SearchResultsCollector([]),
content_cache=cache,
max_fetches=0,
)
def fetch():
return tool.invoke({"url": URL})
elif surface == "pipeline":
def fetch():
return pipeline.fetch_and_extract(URL, content_cache=cache)
else:
def fetch():
return pipeline.batch_fetch_and_extract([URL], content_cache=cache)[
URL
]
first = fetch()
if surface in {"pipeline", "batch"}:
assert first is None
else:
assert ABSTRACT in first
assert cache.get(URL) is None
assert raw in fetch()
assert raw in fetch()
assert cache.get(URL)["content"] == raw
assert source.call_count == 2