Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

18.4 KB
Raw Blame
"""
Tests for run_research_process() core execution logic.

Covers:
- Quick vs detailed mode branching
- Settings context creation
- Progress callback invocation
- Termination handling
- Error handling
- Research status updates
"""

from contextlib import ExitStack
from unittest.mock import MagicMock, patch

import pytest

from local_deep_research.settings.manager import SnapshotSettingsContext
from local_deep_research.web.services.research_service import (
    _DETAILED_REPORT_PROGRESS_END,
    _DETAILED_SEARCH_PROGRESS_CAP,
    _IN_PROGRESS_CAP,
    _REPORT_PHASES,
    _RUN_COMPLETE_FLAG,
)
from tests.web.services.helpers import (
    MODULE,
    _base_run_patches,
    _get_raw_run_research_process,
    run_quick_mode_with_analyze_result,
    run_quick_mode_with_search_error,
)
from tests.web.services.test_research_service_progress_integration import (
    captured_progress_callback,
)


class TestSettingsContext:
    """Tests for SettingsContext created inside run_research_process.

    ``run_research_process`` builds its settings context via the real
    ``SnapshotSettingsContext`` class (settings/manager.py) -- the same
    one TestSettingsContextSetup in test_research_service_core.py already
    exercises directly. These calls use it too, instead of a local
    ``get_setting`` closure that could never fail no matter what
    ``SnapshotSettingsContext`` did.
    """

    def test_settings_context_extracts_values_from_setting_objects(self):
        """SettingsContext extracts 'value' from setting dicts."""
        snapshot = {
            "llm.provider": {"value": "openai", "type": "string"},
            "search.tool": "google",  # plain value
        }
        ctx = SnapshotSettingsContext(snapshot, username="testuser")

        assert ctx.get_setting("llm.provider") == "openai"
        assert ctx.get_setting("search.tool") == "google"

    def test_settings_context_get_setting_from_snapshot(self):
        """get_setting returns value from snapshot, default for missing."""
        ctx = SnapshotSettingsContext(
            {"llm.provider": "openai"}, username="testuser"
        )

        assert ctx.get_setting("llm.provider") == "openai"
        assert ctx.get_setting("missing.key", "fallback") == "fallback"

    def test_settings_context_empty_snapshot(self):
        """Empty snapshot → all get_setting calls return default."""
        ctx = SnapshotSettingsContext({}, username="testuser")

        assert ctx.get_setting("any.key", 42) == 42


class TestProgressCallback:
    """Tests for progress callback logic.

    These drive the REAL ``progress_callback`` closure defined inside
    ``run_research_process`` via ``captured_progress_callback``
    (test_research_service_progress_integration.py), instead of a local
    if/else copy of its adjustment rules.
    """

    def test_progress_adjusted_for_detailed_output_generation(self):
        """Detailed mode output_generation → capped at search cap."""
        with captured_progress_callback("detailed") as (cb, state, _):
            cb("x", 90, {"phase": "output_generation"})
            assert state[0] == _DETAILED_SEARCH_PROGRESS_CAP

    def test_progress_adjusted_for_detailed_report_generation(self):
        """Detailed mode report_generation → passes through (wrapper maps range)."""
        with captured_progress_callback("detailed") as (cb, state, _):
            cb("x", 50, {"phase": "report_generation"})
            assert state[0] == 50

    def test_progress_adjusted_for_quick_output_generation(self):
        """Quick mode output_generation → scaled 85-95%."""
        with captured_progress_callback("quick") as (cb, state, _):
            cb("x", 50, {"phase": "output_generation"})
            assert state[0] == 90.0

    def test_quick_output_generation_with_none_progress_passes_through(self):
        """Quick mode output_generation with None progress must not crash
        and must not move the stored progress value."""
        with captured_progress_callback("quick") as (cb, state, _):
            cb("baseline", 10, {"phase": "search"})
            baseline = state[0]
            cb("x", None, {"phase": "output_generation"})  # must not raise
            assert state[0] == baseline

    def test_quick_output_generation_starts_at_eighty_five_percent(self):
        with captured_progress_callback("quick") as (callback, state, _):
            callback("starting output", 0, {"phase": "output_generation"})
        assert state[0] == 85

    def test_search_plan_extracted_from_message(self):
        """SEARCH_PLAN: in message → engines extracted into metadata."""
        with captured_progress_callback("detailed") as (cb, _, __):
            metadata = {}
            cb("Planning SEARCH_PLAN: google, bing, wikipedia", 5, metadata)
            assert metadata["planned_engines"] == "google, bing, wikipedia"
            assert metadata["phase"] == "search_planning"

    def test_engine_selected_extracted_from_message(self):
        """ENGINE_SELECTED: in message → engine extracted into metadata."""
        with captured_progress_callback("detailed") as (cb, _, __):
            metadata = {}
            cb("Selected ENGINE_SELECTED: google", 5, metadata)
            assert metadata["selected_engine"] == "google"
            assert metadata["phase"] == "search"


class TestDetailedModeSearchCap:
    """Tests for the search-phase cap in detailed mode.

    Each of these now drives the real closure via
    ``captured_progress_callback`` instead of the local
    ``_apply_detailed_progress`` mirror the file used to keep. Global
    monotonicity (enforced by ``update_progress_and_check_active``) means
    a value must be reasserted from a higher baseline to see it get
    clamped back down, so each case that isn't monotonically increasing
    uses its own fresh callback instance.
    """

    def test_high_search_values_clamp_to_cap(self):
        """Search progress above the cap clamps to the cap."""
        with captured_progress_callback("detailed") as (cb, state, _):
            cb("x", 90, {"phase": "search"})
            assert state[0] == _DETAILED_SEARCH_PROGRESS_CAP
        with captured_progress_callback("detailed") as (cb, state, _):
            cb("x", 100, {"phase": "search"})
            assert state[0] == _DETAILED_SEARCH_PROGRESS_CAP

    def test_low_search_values_pass_through_below_cap(self):
        """Search progress below the cap passes through unchanged.

        The harness's own initial "Starting research process" emission
        sets a baseline of 5 (research_service.py's first progress_
        callback call) before this test's own calls run, and
        update_progress_and_check_active enforces monotonicity -- so
        these values must climb from (and stay above) that baseline to
        actually observe pass-through rather than rejection.
        """
        with captured_progress_callback("detailed") as (cb, state, _):
            assert state[0] == 5  # the harness's own baseline emission
            cb("x", 6, {"phase": "search"})
            assert state[0] == 6
            cb("x", 7, {"phase": "search"})
            assert state[0] == 7
            cb("x", 8, {"phase": "search"})
            assert state[0] == 8

    def test_none_progress_passes_through_for_error_phase(self):
        """None progress with phase='error' must not crash (regression #3806)."""
        with captured_progress_callback("detailed") as (cb, state, _):
            cb("baseline", 5, {"phase": "search"})
            baseline = state[0]
            cb("x", None, {"phase": "error"})  # must not raise
            assert state[0] == baseline

    def test_none_progress_passes_through_for_sub_search(self):
        """None progress from constrained-search sub-callback must not crash.

        The sub-callback in constrained_search_strategy.py emits None with
        phase in {'search_complete', 'final_results'}.
        """
        for phase in ("search_complete", "final_results"):
            with captured_progress_callback("detailed") as (cb, state, _):
                cb("baseline", 5, {"phase": "search"})
                baseline = state[0]
                cb("x", None, {"phase": phase})  # must not raise
                assert state[0] == baseline, (
                    f"None should pass through for phase={phase}"
                )

    def test_report_phase_unaffected_by_search_cap(self):
        """Report phases pass through; the wrapper already maps the range."""
        for phase in (
            "report_generation",
            "report_section_research",
            "report_formatting",
            "report_structure",
            "report_complete",
        ):
            with captured_progress_callback("detailed") as (cb, state, _):
                cb("x", 55, {"phase": phase})
                assert state[0] == 55, f"phase={phase} should pass through"

    def test_strategy_complete_phase_does_not_jump_bar_to_100_mid_report(self):
        """phase='complete' from a strategy mid-report must NOT pin bar to 100.

        Regression: every search strategy emits {'phase': 'complete'} when its
        analyze_topic finishes (e.g. standard_strategy.py:334 at value 95).
        report_generator runs analyze_topic per subsection via
        self.search_system.analyze_topic, and the SearchSystem's
        progress_callback is the outer callback. So a strategy 'complete'
        fires AFTER each subsection and would jump the bar to 100
        mid-report if treated as the final marker.

        Expected behavior: 'complete' is treated as a search-phase
        emission — capped at the search cap. The legitimate 100% emit
        uses phase='report_complete' (in _REPORT_PHASES) and passes
        through.
        """
        with captured_progress_callback("detailed") as (cb, state, _):
            cb("baseline", 10, {"phase": "report_generation"})
            assert state[0] == 10

            # First subsection's strategy finishes: emits phase='complete' at 95
            cb("Research complete", 95, {"phase": "complete"})
            assert state[0] != 100, (
                "regression: strategy 'complete' jumped bar to 100 mid-report"
            )
            assert state[0] == 10, (
                "'complete' is capped at the search cap (8), which the "
                "monotonic guard rejects since 10 > 8 -- the bar must stay "
                "at 10"
            )

    def test_only_the_flagged_run_completion_reaches_100(self):
        """report_complete ends the report range but not the run.

        Citation formatting and the database save still follow it, so it is
        held just below 100. Only run_research_process's own completion
        emit, flagged with _RUN_COMPLETE_FLAG, reaches 100 -- in detailed
        mode too, where an unflagged phase="complete" is capped at the
        search cap.
        """
        assert "report_complete" in _REPORT_PHASES
        assert "complete" not in _REPORT_PHASES
        with captured_progress_callback("detailed") as (cb, state, _):
            cb(
                "complete",
                _DETAILED_REPORT_PROGRESS_END,
                {"phase": "report_complete"},
            )
            assert state[0] == _IN_PROGRESS_CAP
            cb(
                "Research completed successfully",
                100,
                {"phase": "complete", _RUN_COMPLETE_FLAG: True},
            )
            assert state[0] == 100


class TestTerminationHandling:
    """Tests for termination checks in research process."""

    def test_termination_during_progress_raises(self):
        """Termination requested during progress → the run short-circuits
        via handle_termination instead of ever reaching analyze_topic.

        ``is_termination_requested`` is checked once before start (returns
        False here) and again inside the progress_callback closure on the
        very first emission ("Starting research process") -- returns True
        the second time via the side_effect list below, so the run should
        call ``handle_termination`` and never call ``analyze_topic``.
        """
        system = MagicMock()
        system.analyze_topic.return_value = {
            "findings": [],
            "formatted_findings": "",
            "iterations": 0,
            "current_knowledge": "",
        }
        patches = _base_run_patches()
        patches[f"{MODULE}.get_llm"] = MagicMock(return_value=MagicMock())
        patches[f"{MODULE}.AdvancedSearchSystem"] = MagicMock(
            return_value=system
        )
        patches[
            "local_deep_research.web.research_state.is_termination_requested"
        ] = MagicMock(side_effect=[False, True, True, True, True])
        snapshot = {
            "llm.provider": "ollama",
            "llm.model": "m",
            "search.tool": "searxng",
        }
        with ExitStack() as stack:
            stack.enter_context(
                patch(
                    "local_deep_research.config.search_config.factory_get_search",
                    MagicMock(return_value=MagicMock()),
                )
            )
            stack.enter_context(
                patch(
                    "local_deep_research.security.egress.policy.context_from_snapshot",
                    return_value=MagicMock(),
                )
            )
            stack.enter_context(
                patch(
                    "local_deep_research.security.egress.run_classification.audit_run_from_snapshot",
                    return_value=MagicMock(allowed=True),
                )
            )
            for target, mock_obj in patches.items():
                stack.enter_context(patch(target, mock_obj))
            _get_raw_run_research_process()(
                1,
                "test",
                "quick",
                username="user1",
                settings_snapshot=snapshot,
                search_engine="searxng",
            )
            handle_termination_mock = patches[f"{MODULE}.handle_termination"]

        assert handle_termination_mock.called, (
            "handle_termination should have been invoked once "
            "is_termination_requested flipped True mid-run"
        )
        assert not system.analyze_topic.called, (
            "the search should never have started once termination was "
            "detected in the first progress_callback emission"
        )


class TestErrorClassification:
    """Tests for search error classification in run_research_process.

    Drives the REAL classification in run_research_process (the
    ``except Exception as search_error`` / ``except Exception as e`` pair
    around ``system.analyze_topic()``) via the shared
    ``run_quick_mode_with_search_error`` harness in helpers.py, instead of
    a local ``_classify_error`` copy of the string-matching rules.
    """

    @pytest.mark.parametrize(
        "error_message, expected_substring",
        [
            (
                "Request failed with status code: 503",
                "Ollama AI service is unavailable",
            ),
            ("status code: 404 not found", "model not found"),
            ("status code: 429 rate limited", "rejected the request"),
            ("status code: 500 internal error", "rejected the request"),
            (
                "Connection refused to localhost:11434",
                "Connection error",
            ),
            ("TCP connection reset by peer", "Connection error"),
            ("Something unexpected happened", "unexpected error"),
        ],
    )
    def test_error_classification(self, error_message, expected_substring):
        message = run_quick_mode_with_search_error(error_message)
        assert expected_substring.lower() in message.lower()

    def test_503_takes_priority_over_generic_status_code(self):
        """503 is matched specifically before the generic 'status code:'
        pattern that also matches it (both are substrings of the raw
        message)."""
        message = run_quick_mode_with_search_error("status code: 503")
        assert "Ollama AI service is unavailable" in message
        assert "rejected the request" not in message

    def test_404_takes_priority_over_generic_status_code(self):
        """404 is matched specifically before the generic 'status code:'
        pattern."""
        message = run_quick_mode_with_search_error("status code: 404")
        assert "model not found" in message.lower()
        assert "rejected the request" not in message


class TestResearchModes:
    """Tests for mode-specific behavior.

    Drives the real quick-mode output logic (research_service.py
    ~1736-1900) via the shared ``run_quick_mode_with_analyze_result``
    harness, instead of re-deriving ``is_error`` / ``has_findings``
    locals from a results dict the test built itself and never passed
    anywhere.
    """

    def test_quick_mode_checks_findings(self):
        """A results dict with findings/formatted_findings reaches the
        citation formatter (i.e. produces a report), rather than falling
        through to the "nothing to summarize" branch."""
        result = run_quick_mode_with_analyze_result(
            {
                "findings": [{"content": "f1", "phase": "Final synthesis"}],
                "formatted_findings": "Summary text",
                "iterations": 1,
                "current_knowledge": "",
            }
        )
        assert "clean_markdown" in result

    def test_quick_mode_detects_error_in_findings(self):
        """An 'Error:' prefixed formatted_findings triggers the fallback
        path -- the saved content differs from (recovers past) the raw
        error text, using current_knowledge as the recovery source."""
        result = run_quick_mode_with_analyze_result(
            {
                "findings": [],
                "formatted_findings": "Error: token limit exceeded",
                "iterations": 1,
                "current_knowledge": "fallback knowledge",
            }
        )
        assert result == {"clean_markdown": "fallback knowledge"}

    def test_quick_mode_no_error_prefix(self):
        """A normal (non-'Error:') formatted_findings is NOT routed
        through the fallback machinery -- the synthesized answer passes
        straight through unchanged."""
        result = run_quick_mode_with_analyze_result(
            {
                "findings": [
                    {
                        "content": "This is a normal summary.",
                        "phase": "Final synthesis",
                    }
                ],
                "formatted_findings": "This is a normal summary.",
                "iterations": 1,
                "current_knowledge": "",
            }
        )
        assert result == {"clean_markdown": "This is a normal summary."}