Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

6.5 KB
Raw Blame
"""Formatter failures must not hide synthesis errors from the worker."""

from contextlib import ExitStack
from unittest.mock import MagicMock, patch

import pytest

from local_deep_research.advanced_search_system.findings.repository import (
    FindingsRepository,
)
from local_deep_research.advanced_search_system.strategies.source_based_strategy import (
    SourceBasedSearchStrategy,
)
from local_deep_research.database.models import ResearchHistory
from local_deep_research.error_handling.report_generator import (
    ErrorReportGenerator,
)
from local_deep_research.storage.database import DatabaseReportStorage
from tests.web.services.helpers import (
    MODULE,
    _base_run_patches,
    _egress_and_search_patches,
    _get_raw_run_research_process,
)


@pytest.mark.parametrize("formatting_fails", [False, True])
@pytest.mark.parametrize("outcome", ["answer", "partial", "all_errors"])
def test_worker_saves_safe_content_after_formatting(
    report_db, formatting_fails, outcome
):
    marker = "/srv/private/provider-internals.txt"
    answer = "A complete and useful synthesized answer."
    partial = "Useful safe partial finding."
    synthesized = (
        answer
        if outcome == "answer"
        else f"Error: Failed to synthesize final answer. LLM error: {marker}"
    )
    findings = []
    if outcome == "partial":
        findings.append(
            {"phase": "Search", "content": partial, "search_results": []}
        )
    findings.append(
        {
            "phase": "Final synthesis",
            "content": synthesized,
            "search_results": [],
        }
    )
    repository = FindingsRepository(MagicMock())
    # Exercise the real formatter's failure path with a malformed question
    # group. Mocking formatted_findings missed the old prefix-changing wrapper.
    repository.set_questions_by_iteration(
        {1: None} if formatting_fails else {1: ["test question"]}
    )
    formatted = repository.format_findings_to_text(findings, synthesized)
    system = MagicMock()
    system.analyze_topic.return_value = {
        "findings": findings,
        "iterations": 1,
        "formatted_findings": formatted,
        "current_knowledge": synthesized,
    }
    system.all_links_of_system = []
    patches = _base_run_patches()
    patches[f"{MODULE}.get_llm"] = MagicMock(return_value=MagicMock())
    patches[f"{MODULE}.AdvancedSearchSystem"] = MagicMock(return_value=system)
    patches[f"{MODULE}.ErrorReportGenerator"] = ErrorReportGenerator

    # Keep the real worker, error renderer, citation formatter and storage.
    # Read the committed row so this checks the report actually saved for users.
    with ExitStack() as stack:
        for cm in _egress_and_search_patches():
            stack.enter_context(cm)
        for name, value in patches.items():
            stack.enter_context(patch(name, value))
        stack.enter_context(
            patch(
                "local_deep_research.storage.get_report_storage",
                return_value=DatabaseReportStorage(report_db),
            )
        )
        _get_raw_run_research_process()(
            "report-errors",
            "test query",
            "quick",
            username="user1",
            search_engine="searxng",
            settings_snapshot={
                "llm.provider": "ollama",
                "llm.model": "m",
                "search.tool": "searxng",
            },
        )

    report_db.expire_all()
    report = report_db.get(ResearchHistory, "report-errors").report_content
    assert report
    assert marker not in report
    if outcome == "answer":
        assert report == answer
    elif outcome == "partial":
        assert partial in report
        assert "Fallback Mode" in report
    else:
        assert "Research Failed" in report
        assert "What happened:" in report


@pytest.mark.parametrize(
    "failure,expected_hint,expected_category",
    [
        (
            "Connection refused [Errno 111]",
            "Cannot connect to the LLM service",
            "Connection Issue",
        ),
        (
            "No auth credentials found",
            "API key is missing",
            "LLM Service Error",
        ),
        (
            "database is locked",
            "database is temporarily locked",
            "File System Error",
        ),
    ],
)
def test_all_error_results_keep_safe_category_and_hint(
    report_db, failure, expected_hint, expected_category
):
    marker = "/srv/private/provider-internals.txt"
    strategy = SourceBasedSearchStrategy(
        search=MagicMock(),
        model=MagicMock(),
        citation_handler=MagicMock(),
        use_cross_engine_filter=False,
        include_text_content=False,
        max_iterations=1,
        questions_per_iteration=1,
        settings_snapshot={"app.max_user_query_length": 300},
    )
    # Let the real strategy create its error result from a failed LLM call.
    with patch(
        "local_deep_research.advanced_search_system.questions.standard_question.invoke_llm_sync",
        side_effect=RuntimeError(f"{failure}: {marker}"),
    ):
        results = strategy.analyze_topic("test query")
    assert results["findings"][0]["phase"] == "Error"
    assert failure in results["formatted_findings"]

    system = MagicMock()
    system.analyze_topic.return_value = results
    system.all_links_of_system = []
    patches = _base_run_patches()
    patches[f"{MODULE}.get_llm"] = MagicMock(return_value=MagicMock())
    patches[f"{MODULE}.AdvancedSearchSystem"] = MagicMock(return_value=system)
    patches[f"{MODULE}.ErrorReportGenerator"] = ErrorReportGenerator
    with ExitStack() as stack:
        for cm in _egress_and_search_patches():
            stack.enter_context(cm)
        for name, value in patches.items():
            stack.enter_context(patch(name, value))
        stack.enter_context(
            patch(
                "local_deep_research.storage.get_report_storage",
                return_value=DatabaseReportStorage(report_db),
            )
        )
        _get_raw_run_research_process()(
            "report-errors",
            "test query",
            "quick",
            username="user1",
            search_engine="searxng",
            settings_snapshot={
                "llm.provider": "ollama",
                "llm.model": "m",
                "search.tool": "searxng",
            },
        )

    report_db.expire_all()
    report = report_db.get(ResearchHistory, "report-errors").report_content
    assert expected_hint in report
    assert f"**Error Type:** {expected_category}" in report
    assert marker not in report