Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

18.7 KB
Raw Blame
"""Only complete source text may satisfy a later full-document request."""

from types import SimpleNamespace
from unittest.mock import MagicMock, patch

import pytest

from local_deep_research.content_fetcher.fetcher import ContentFetcher
from local_deep_research.utilities.research_content_cache import (
    ResearchContentCache,
)
from local_deep_research.web_search_engines.engines.search_engine_arxiv import (
    ArXivSearchEngine,
    FullTextFetchFailedError,
    FullTextNotExtractedError,
)


URL = "https://arxiv.org/abs/2301.12345"
ABSTRACT = "An abstract that is longer than the complete test paper."
BODY = "Complete paper."


@pytest.fixture
def arxiv_engine(tmp_path):
    with patch.object(
        ArXivSearchEngine, "_create_journal_filter", return_value=None
    ):
        engine = ArXivSearchEngine(
            include_full_text=True,
            download_dir=str(tmp_path),
            settings_snapshot={},
        )
    engine._configure_programmatic_mode(True)
    engine.search_snippets_only = False
    engine.content_cache = ResearchContentCache()
    engine.rate_tracker = MagicMock()
    paper = SimpleNamespace(
        entry_id=URL,
        summary=ABSTRACT,
        authors=[],
        published=None,
        updated=None,
        categories=[],
        comment="",
        doi="",
        journal_ref="",
        title="Test paper",
    )

    def previews(query):
        engine._papers = {URL: paper}
        return [
            {"id": URL, "link": URL, "title": paper.title, "snippet": ABSTRACT}
        ]

    engine._get_previews = previews
    with patch(
        "local_deep_research.security.ssrf_validator.validate_url",
        return_value=True,
    ):
        yield engine


@pytest.mark.parametrize(
    "initial",
    [
        FullTextFetchFailedError("temporary outage"),
        FullTextNotExtractedError("empty extraction"),
        None,
        "   ",
    ],
)
def test_arxiv_retries_partial_content_after_recovery(arxiv_engine, initial):
    engine = arxiv_engine
    engine._fetch_full_text = MagicMock(side_effect=[initial, BODY])
    assert engine.run("first search")[0]["full_content"] == ABSTRACT
    assert engine.content_cache.get(URL) is None
    assert engine.run("later search")[0]["full_content"] == BODY
    assert engine.run("repeat search")[0]["full_content"] == BODY
    assert engine._fetch_full_text.call_count == 2
    assert engine.content_cache.get(URL)["content"] == BODY


@pytest.mark.parametrize("reason", ["disabled", "budget"])
def test_abstract_only_search_does_not_satisfy_full_fetch(arxiv_engine, reason):
    engine = arxiv_engine
    if reason == "disabled":
        engine.include_full_text = False
    else:
        engine.max_full_text = 0
    engine._fetch_full_text = MagicMock(
        side_effect=AssertionError("search must not fetch")
    )
    assert engine.run("search")[0]["full_content"] == ABSTRACT
    downloader = MagicMock()
    downloader.download_with_result.return_value = SimpleNamespace(
        is_success=True, content=BODY.encode(), skip_reason=None
    )
    downloader.get_metadata.return_value = {"title": "Test paper"}
    with (
        patch(
            "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
            return_value=True,
        ),
        patch.object(
            ContentFetcher, "_get_downloader", return_value=downloader
        ),
    ):
        with ContentFetcher(content_cache=engine.content_cache) as fetcher:
            assert fetcher.fetch(URL)["content"] == BODY
            assert fetcher.fetch(URL)["content"] == BODY
    assert downloader.download_with_result.call_count == 1


def test_complete_cache_replaces_partial_preview(arxiv_engine):
    engine = arxiv_engine
    engine.content_cache.put(URL, BODY)
    engine._fetch_full_text = MagicMock(
        side_effect=AssertionError("must reuse complete text")
    )
    result = engine._get_full_content_cached(
        [
            {
                "link": URL,
                "full_content": ABSTRACT,
                "full_content_is_complete": False,
            }
        ]
    )
    assert result[0]["full_content"] == BODY
    assert result[0]["full_content_is_complete"] is True


def test_complete_preview_is_preserved(arxiv_engine):
    arxiv_engine.content_cache.put(URL, "Previously fetched text")
    result = arxiv_engine._get_full_content_cached(
        [
            {
                "link": URL,
                "full_content": BODY,
                "full_content_is_complete": True,
            }
        ]
    )
    assert result[0]["full_content"] == BODY


@pytest.mark.parametrize("complete", [None, False, "true", 1, True])
def test_unverified_search_content_cannot_fill_document_cache(
    arxiv_engine, complete
):
    item = {"link": URL, "full_content": ABSTRACT}
    if complete is not None:
        item["full_content_is_complete"] = complete
    arxiv_engine._store_full_content_in_cache([item])
    assert (arxiv_engine.content_cache.get(URL) is not None) is (
        complete is True
    )


def test_pubmed_abstract_fallback_is_retried():
    from local_deep_research.web_search_engines.engines.search_engine_pubmed import (
        PubMedSearchEngine,
    )

    with patch.object(
        PubMedSearchEngine, "_create_journal_filter", return_value=None
    ):
        engine = PubMedSearchEngine(get_full_text=True, settings_snapshot={})
    engine.content_cache = ResearchContentCache()
    engine._get_article_abstracts = MagicMock(return_value={"123": ABSTRACT})
    engine._get_article_detailed_metadata = MagicMock(return_value={})
    engine._find_pmc_ids = MagicMock(return_value={"123": "PMC123"})
    engine._get_pmc_full_text = MagicMock(side_effect=[None, BODY])
    url = "https://pubmed.ncbi.nlm.nih.gov/123/"

    def search():
        return engine._get_full_content_cached(
            [{"_pmid": "123", "link": url, "title": "Paper"}]
        )[0]

    with patch(
        "local_deep_research.security.ssrf_validator.validate_url",
        return_value=True,
    ):
        assert search()["content_type"] == "abstract"
        assert engine.content_cache.get(url) is None
        assert BODY in search()["full_content"]
        assert BODY in search()["full_content"]
    assert engine._get_pmc_full_text.call_count == 2


@pytest.mark.parametrize(
    "initial", [None, {}, RuntimeError("temporary outage")]
)
def test_paperless_snippet_fallback_is_retried(initial):
    from local_deep_research.web_search_engines.engines.search_engine_paperless import (
        PaperlessSearchEngine,
    )

    engine = PaperlessSearchEngine.__new__(PaperlessSearchEngine)
    engine.include_content = True
    engine.content_cache = ResearchContentCache()
    engine._make_request = MagicMock(side_effect=[initial, {"content": BODY}])
    engine._scrub_error = MagicMock(return_value="request failed")
    engine._is_cache_url_allowed = MagicMock(return_value=True)
    url = "https://paperless.example/documents/123/"

    def search():
        return engine._get_full_content_cached(
            [
                {
                    "link": url,
                    "snippet": ABSTRACT,
                    "_raw_data": {},
                    "metadata": {"doc_id": "123"},
                }
            ]
        )[0]

    assert search()["full_content"] == ABSTRACT
    assert engine.content_cache.get(url) is None
    assert search()["full_content"] == BODY
    assert search()["full_content"] == BODY
    assert engine._make_request.call_count == 2


def test_wayback_error_text_is_not_a_complete_cached_page():
    from local_deep_research.web_search_engines.engines.search_engine_wayback import (
        WaybackSearchEngine,
    )

    engine = WaybackSearchEngine.__new__(WaybackSearchEngine)
    engine.content_cache = ResearchContentCache()
    engine.rate_tracker = MagicMock()
    engine.engine_type = "wayback"
    engine._is_cache_url_allowed = MagicMock(return_value=True)
    engine._get_wayback_content = MagicMock(
        side_effect=[
            ("", "Error retrieving content: temporary outage"),
            ("<p>Complete paper.</p>", BODY),
        ]
    )
    url = "https://web.archive.org/web/20200101/https://example.com/"
    assert (
        "Error retrieving"
        in engine._get_full_content_cached([{"link": url}])[0]["full_content"]
    )
    assert engine.content_cache.get(url) is None
    assert (
        engine._get_full_content_cached([{"link": url}])[0]["full_content"]
        == BODY
    )
    assert (
        engine._get_full_content_cached([{"link": url}])[0]["full_content"]
        == BODY
    )
    assert engine._get_wayback_content.call_count == 2


@pytest.mark.parametrize("kind", ["issue", "user"])
def test_github_summaries_do_not_fill_document_cache(kind):
    from local_deep_research.web_search_engines.engines.search_engine_github import (
        GitHubSearchEngine,
    )

    engine = GitHubSearchEngine.__new__(GitHubSearchEngine)
    engine.content_cache = ResearchContentCache()
    engine._is_cache_url_allowed = MagicMock(return_value=True)
    url = "https://github.com/example/repo/issues/1"
    result = engine._get_full_content_cached(
        [
            {
                "link": url,
                "snippet": ABSTRACT,
                "search_type": kind,
            }
        ]
    )
    assert ABSTRACT in result[0]["full_content"]
    assert engine.content_cache.get(url) is None


def test_sofya_snippet_then_extracted_content():
    from local_deep_research.web_search_engines.engines.search_engine_sofya import (
        SofyaSearchEngine,
    )

    engine = SofyaSearchEngine.__new__(SofyaSearchEngine)
    engine.content_cache = ResearchContentCache()
    engine._is_cache_url_allowed = MagicMock(return_value=True)
    preview = {"link": URL, "snippet": ABSTRACT}
    assert (
        engine._get_full_content_cached([preview.copy()])[0]["full_content"]
        == ABSTRACT
    )
    assert engine.content_cache.get(URL) is None
    preview["_full_result"] = {"content": BODY}
    assert engine._get_full_content_cached([preview])[0]["full_content"] == BODY
    assert engine.content_cache.get(URL)["content"] == BODY


def test_missing_library_chunk_can_be_retried():
    from local_deep_research.web_search_engines.engines.search_engine_library import (
        LibraryRAGSearchEngine,
    )

    engine = LibraryRAGSearchEngine.__new__(LibraryRAGSearchEngine)
    engine.username = "cache-test-user"
    engine.content_cache = ResearchContentCache()
    engine._is_cache_url_allowed = MagicMock(return_value=True)
    document_id = "12345678-1234-5678-1234-567812345678"
    url = f"/library/document/{document_id}#chunk-1"
    session = MagicMock()
    query = session.query.return_value
    query.filter.return_value = query
    query.order_by.return_value.first.side_effect = [
        None,
        None,
        SimpleNamespace(chunk_text=BODY),
    ]
    query.limit.return_value.all.return_value = []

    def search():
        return engine._get_full_content_cached(
            [
                {
                    "link": url,
                    "metadata": {
                        "source_id": document_id,
                        "chunk_index": 1,
                        "collection_id": "test",
                    },
                }
            ]
        )[0]

    with patch(
        "local_deep_research.database.session_context.get_user_db_session"
    ) as context:
        context.return_value.__enter__.return_value = session
        assert "not found" in search()["full_content"]
        assert engine.content_cache.get(url, collection_id="test") is None
        assert search()["full_content"] == BODY
        assert search()["full_content"] == BODY
        assert context.call_count == 2


@pytest.mark.parametrize(
    "mode,unrestricted_cloud",
    [
        ("full", True),
        ("summary_focus", True),
        ("summary_focus_query", True),
        ("summary_focus", False),
        ("summary_focus_query", False),
    ],
)
@pytest.mark.parametrize("limit", [1_000, 10_000, 50_000])
def test_repeated_tool_fetch_uses_same_limit_and_keeps_raw_text(
    mode, unrestricted_cloud, limit
):
    from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
        SearchResultsCollector,
    )
    from local_deep_research.advanced_search_system.tools.fetch import (
        build_fetch_tool,
    )
    from local_deep_research.config.llm_config import (
        wrap_llm_without_think_tags,
    )

    settings = {
        "search.fetch.content_max_length": {
            "value": limit,
            "ui_element": "number",
        },
        "search.fetch.summary_input_max_length": {
            "value": limit,
            "ui_element": "number",
        },
        "llm.provider": "openai",
        "llm.context_window_unrestricted": True,
        "llm.local_context_window_size": 8192,
        "llm.max_tokens": 8192,
        "llm.supports_max_tokens": True,
        "rate_limiting.llm_enabled": False,
    }

    cache = ResearchContentCache()
    raw = "Ω" * (limit + 2000)
    downloader = MagicMock()
    downloader.download_with_result.return_value = SimpleNamespace(
        is_success=True, content=raw.encode(), skip_reason=None
    )
    downloader.get_metadata.return_value = {"title": "Paper"}
    model = MagicMock()
    model.invoke.return_value = SimpleNamespace(content="Selected text")
    # Use the real wrapper's build metadata for unrestricted cloud input.
    # An unwrapped model remains bounded even if the tool snapshot names a
    # cloud provider, since it does not describe how that model was built.
    tool_model = (
        wrap_llm_without_think_tags(
            model, provider="openai", settings_snapshot=settings
        )
        if unrestricted_cloud
        else model
    )
    tool = build_fetch_tool(
        mode,
        SearchResultsCollector([]),
        model=tool_model,
        overall_query="Question",
        settings_snapshot=settings,
        max_fetches=0,
        content_cache=cache,
    )
    args = {"url": URL}
    if mode != "full":
        args["focus"] = "Question"
    with (
        patch(
            "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
            return_value=True,
        ),
        patch(
            "local_deep_research.security.egress.fetch.policy_aware_validate_url",
            return_value=True,
        ),
        patch.object(
            ContentFetcher, "_get_downloader", return_value=downloader
        ),
    ):
        outputs = [tool.invoke(args), tool.invoke(args)]
    if mode != "full":
        assert model.invoke.call_count == 2
        outputs = [call.args[0] for call in model.invoke.call_args_list]
    else:
        model.invoke.assert_not_called()
    expected_limit = limit
    context_trims_page = False
    if not unrestricted_cloud:
        # Independent whole-prompt oracle: an 8192-token window reserves
        # int(8192 * 0.8) = 6553 tokens for output, leaving 6556 characters.
        # The fixed prompt occupies 1345/1381 characters in these cases.
        page_limit = 5211 if mode == "summary_focus" else 5175
        expected_limit = min(limit, page_limit)
        if limit > page_limit:
            context_trims_page = True
            assert [len(text) for text in outputs] == [6556, 6556]
    assert [text.count("Ω") for text in outputs] == [expected_limit] * 2
    if mode != "full":
        pages = [
            text.split("Page content:\n", 1)[1].split("\n\nOutput rules:", 1)[0]
            for text in outputs
        ]
        expected_page = raw[:expected_limit]
        if not context_trims_page:
            # Both cold and warm ContentFetcher results carry this marker;
            # the whole-prompt context cap can remove it with the page tail.
            expected_page += "\n\n[... content truncated ...]"
        assert pages == [expected_page] * 2
    assert cache.get(URL)["content"] == raw
    assert downloader.download_with_result.call_count == 1


@pytest.mark.parametrize("surface", ["fetcher", "tool", "pipeline", "batch"])
def test_arxiv_download_metadata_does_not_prevent_later_full_text(
    monkeypatch, surface
):
    from local_deep_research.research_library.downloaders.arxiv import (
        ArxivDownloader,
        ArxivTextResult,
        ArxivTextSource,
    )
    from local_deep_research.research_library.downloaders.extraction import (
        pipeline,
    )

    raw = BODY * 20
    source = MagicMock(
        side_effect=[
            ArxivTextResult(ABSTRACT, ArxivTextSource.ARXIV_API),
            ArxivTextResult(raw, ArxivTextSource.ARXIV_HTML),
        ]
    )
    # Keep the real downloader's result construction and downstream cache
    # paths. Only replace its transport/source selection and resource setup.
    monkeypatch.setattr(
        ArxivDownloader, "__init__", lambda self, **kwargs: None
    )
    monkeypatch.setattr(ArxivDownloader, "download_text_with_source", source)
    monkeypatch.setattr(ArxivDownloader, "get_metadata", lambda self, url: {})
    monkeypatch.setattr(ArxivDownloader, "close", lambda self: None)
    monkeypatch.setattr(
        "local_deep_research.security.ssrf_validator.validate_url",
        lambda *args, **kwargs: True,
    )
    monkeypatch.setattr(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        lambda *args, **kwargs: True,
    )
    monkeypatch.setattr(
        "local_deep_research.security.egress.fetch.policy_aware_validate_url",
        lambda *args, **kwargs: True,
    )
    cache = ResearchContentCache()
    if surface in {"fetcher", "tool"}:
        monkeypatch.setattr(
            ContentFetcher,
            "_get_downloader",
            lambda self, url_type: ArxivDownloader(),
        )
    if surface == "fetcher":

        def fetch():
            with ContentFetcher(content_cache=cache) as fetcher:
                return fetcher.fetch(URL)["content"]
    elif surface == "tool":
        from local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy import (
            SearchResultsCollector,
        )
        from local_deep_research.advanced_search_system.tools.fetch import (
            build_fetch_tool,
        )

        tool = build_fetch_tool(
            "full",
            SearchResultsCollector([]),
            content_cache=cache,
            max_fetches=0,
        )

        def fetch():
            return tool.invoke({"url": URL})
    elif surface == "pipeline":

        def fetch():
            return pipeline.fetch_and_extract(URL, content_cache=cache)
    else:

        def fetch():
            return pipeline.batch_fetch_and_extract([URL], content_cache=cache)[
                URL
            ]

    first = fetch()
    if surface in {"pipeline", "batch"}:
        assert first is None
    else:
        assert ABSTRACT in first
    assert cache.get(URL) is None
    assert raw in fetch()
    assert raw in fetch()
    assert cache.get(URL)["content"] == raw
    assert source.call_count == 2