~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for run_research_process() core execution logic.
Covers:
- Quick vs detailed mode branching
- Settings context creation
- Progress callback invocation
- Termination handling
- Error handling
- Research status updates
"""
from contextlib import ExitStack
from unittest.mock import MagicMock, patch
import pytest
from local_deep_research.settings.manager import SnapshotSettingsContext
from local_deep_research.web.services.research_service import (
_DETAILED_REPORT_PROGRESS_END,
_DETAILED_SEARCH_PROGRESS_CAP,
_IN_PROGRESS_CAP,
_REPORT_PHASES,
_RUN_COMPLETE_FLAG,
)
from tests.web.services.helpers import (
MODULE,
_base_run_patches,
_get_raw_run_research_process,
run_quick_mode_with_analyze_result,
run_quick_mode_with_search_error,
)
from tests.web.services.test_research_service_progress_integration import (
captured_progress_callback,
)
class TestSettingsContext:
"""Tests for SettingsContext created inside run_research_process.
``run_research_process`` builds its settings context via the real
``SnapshotSettingsContext`` class (settings/manager.py) -- the same
one TestSettingsContextSetup in test_research_service_core.py already
exercises directly. These calls use it too, instead of a local
``get_setting`` closure that could never fail no matter what
``SnapshotSettingsContext`` did.
"""
def test_settings_context_extracts_values_from_setting_objects(self):
"""SettingsContext extracts 'value' from setting dicts."""
snapshot = {
"llm.provider": {"value": "openai", "type": "string"},
"search.tool": "google", # plain value
}
ctx = SnapshotSettingsContext(snapshot, username="testuser")
assert ctx.get_setting("llm.provider") == "openai"
assert ctx.get_setting("search.tool") == "google"
def test_settings_context_get_setting_from_snapshot(self):
"""get_setting returns value from snapshot, default for missing."""
ctx = SnapshotSettingsContext(
{"llm.provider": "openai"}, username="testuser"
)
assert ctx.get_setting("llm.provider") == "openai"
assert ctx.get_setting("missing.key", "fallback") == "fallback"
def test_settings_context_empty_snapshot(self):
"""Empty snapshot → all get_setting calls return default."""
ctx = SnapshotSettingsContext({}, username="testuser")
assert ctx.get_setting("any.key", 42) == 42
class TestProgressCallback:
"""Tests for progress callback logic.
These drive the REAL ``progress_callback`` closure defined inside
``run_research_process`` via ``captured_progress_callback``
(test_research_service_progress_integration.py), instead of a local
if/else copy of its adjustment rules.
"""
def test_progress_adjusted_for_detailed_output_generation(self):
"""Detailed mode output_generation → capped at search cap."""
with captured_progress_callback("detailed") as (cb, state, _):
cb("x", 90, {"phase": "output_generation"})
assert state[0] == _DETAILED_SEARCH_PROGRESS_CAP
def test_progress_adjusted_for_detailed_report_generation(self):
"""Detailed mode report_generation → passes through (wrapper maps range)."""
with captured_progress_callback("detailed") as (cb, state, _):
cb("x", 50, {"phase": "report_generation"})
assert state[0] == 50
def test_progress_adjusted_for_quick_output_generation(self):
"""Quick mode output_generation → scaled 85-95%."""
with captured_progress_callback("quick") as (cb, state, _):
cb("x", 50, {"phase": "output_generation"})
assert state[0] == 90.0
def test_quick_output_generation_with_none_progress_passes_through(self):
"""Quick mode output_generation with None progress must not crash
and must not move the stored progress value."""
with captured_progress_callback("quick") as (cb, state, _):
cb("baseline", 10, {"phase": "search"})
baseline = state[0]
cb("x", None, {"phase": "output_generation"}) # must not raise
assert state[0] == baseline
def test_quick_output_generation_starts_at_eighty_five_percent(self):
with captured_progress_callback("quick") as (callback, state, _):
callback("starting output", 0, {"phase": "output_generation"})
assert state[0] == 85
def test_search_plan_extracted_from_message(self):
"""SEARCH_PLAN: in message → engines extracted into metadata."""
with captured_progress_callback("detailed") as (cb, _, __):
metadata = {}
cb("Planning SEARCH_PLAN: google, bing, wikipedia", 5, metadata)
assert metadata["planned_engines"] == "google, bing, wikipedia"
assert metadata["phase"] == "search_planning"
def test_engine_selected_extracted_from_message(self):
"""ENGINE_SELECTED: in message → engine extracted into metadata."""
with captured_progress_callback("detailed") as (cb, _, __):
metadata = {}
cb("Selected ENGINE_SELECTED: google", 5, metadata)
assert metadata["selected_engine"] == "google"
assert metadata["phase"] == "search"
class TestDetailedModeSearchCap:
"""Tests for the search-phase cap in detailed mode.
Each of these now drives the real closure via
``captured_progress_callback`` instead of the local
``_apply_detailed_progress`` mirror the file used to keep. Global
monotonicity (enforced by ``update_progress_and_check_active``) means
a value must be reasserted from a higher baseline to see it get
clamped back down, so each case that isn't monotonically increasing
uses its own fresh callback instance.
"""
def test_high_search_values_clamp_to_cap(self):
"""Search progress above the cap clamps to the cap."""
with captured_progress_callback("detailed") as (cb, state, _):
cb("x", 90, {"phase": "search"})
assert state[0] == _DETAILED_SEARCH_PROGRESS_CAP
with captured_progress_callback("detailed") as (cb, state, _):
cb("x", 100, {"phase": "search"})
assert state[0] == _DETAILED_SEARCH_PROGRESS_CAP
def test_low_search_values_pass_through_below_cap(self):
"""Search progress below the cap passes through unchanged.
The harness's own initial "Starting research process" emission
sets a baseline of 5 (research_service.py's first progress_
callback call) before this test's own calls run, and
update_progress_and_check_active enforces monotonicity -- so
these values must climb from (and stay above) that baseline to
actually observe pass-through rather than rejection.
"""
with captured_progress_callback("detailed") as (cb, state, _):
assert state[0] == 5 # the harness's own baseline emission
cb("x", 6, {"phase": "search"})
assert state[0] == 6
cb("x", 7, {"phase": "search"})
assert state[0] == 7
cb("x", 8, {"phase": "search"})
assert state[0] == 8
def test_none_progress_passes_through_for_error_phase(self):
"""None progress with phase='error' must not crash (regression #3806)."""
with captured_progress_callback("detailed") as (cb, state, _):
cb("baseline", 5, {"phase": "search"})
baseline = state[0]
cb("x", None, {"phase": "error"}) # must not raise
assert state[0] == baseline
def test_none_progress_passes_through_for_sub_search(self):
"""None progress from constrained-search sub-callback must not crash.
The sub-callback in constrained_search_strategy.py emits None with
phase in {'search_complete', 'final_results'}.
"""
for phase in ("search_complete", "final_results"):
with captured_progress_callback("detailed") as (cb, state, _):
cb("baseline", 5, {"phase": "search"})
baseline = state[0]
cb("x", None, {"phase": phase}) # must not raise
assert state[0] == baseline, (
f"None should pass through for phase={phase}"
)
def test_report_phase_unaffected_by_search_cap(self):
"""Report phases pass through; the wrapper already maps the range."""
for phase in (
"report_generation",
"report_section_research",
"report_formatting",
"report_structure",
"report_complete",
):
with captured_progress_callback("detailed") as (cb, state, _):
cb("x", 55, {"phase": phase})
assert state[0] == 55, f"phase={phase} should pass through"
def test_strategy_complete_phase_does_not_jump_bar_to_100_mid_report(self):
"""phase='complete' from a strategy mid-report must NOT pin bar to 100.
Regression: every search strategy emits {'phase': 'complete'} when its
analyze_topic finishes (e.g. standard_strategy.py:334 at value 95).
report_generator runs analyze_topic per subsection via
self.search_system.analyze_topic, and the SearchSystem's
progress_callback is the outer callback. So a strategy 'complete'
fires AFTER each subsection and would jump the bar to 100
mid-report if treated as the final marker.
Expected behavior: 'complete' is treated as a search-phase
emission — capped at the search cap. The legitimate 100% emit
uses phase='report_complete' (in _REPORT_PHASES) and passes
through.
"""
with captured_progress_callback("detailed") as (cb, state, _):
cb("baseline", 10, {"phase": "report_generation"})
assert state[0] == 10
# First subsection's strategy finishes: emits phase='complete' at 95
cb("Research complete", 95, {"phase": "complete"})
assert state[0] != 100, (
"regression: strategy 'complete' jumped bar to 100 mid-report"
)
assert state[0] == 10, (
"'complete' is capped at the search cap (8), which the "
"monotonic guard rejects since 10 > 8 -- the bar must stay "
"at 10"
)
def test_only_the_flagged_run_completion_reaches_100(self):
"""report_complete ends the report range but not the run.
Citation formatting and the database save still follow it, so it is
held just below 100. Only run_research_process's own completion
emit, flagged with _RUN_COMPLETE_FLAG, reaches 100 -- in detailed
mode too, where an unflagged phase="complete" is capped at the
search cap.
"""
assert "report_complete" in _REPORT_PHASES
assert "complete" not in _REPORT_PHASES
with captured_progress_callback("detailed") as (cb, state, _):
cb(
"complete",
_DETAILED_REPORT_PROGRESS_END,
{"phase": "report_complete"},
)
assert state[0] == _IN_PROGRESS_CAP
cb(
"Research completed successfully",
100,
{"phase": "complete", _RUN_COMPLETE_FLAG: True},
)
assert state[0] == 100
class TestTerminationHandling:
"""Tests for termination checks in research process."""
def test_termination_during_progress_raises(self):
"""Termination requested during progress → the run short-circuits
via handle_termination instead of ever reaching analyze_topic.
``is_termination_requested`` is checked once before start (returns
False here) and again inside the progress_callback closure on the
very first emission ("Starting research process") -- returns True
the second time via the side_effect list below, so the run should
call ``handle_termination`` and never call ``analyze_topic``.
"""
system = MagicMock()
system.analyze_topic.return_value = {
"findings": [],
"formatted_findings": "",
"iterations": 0,
"current_knowledge": "",
}
patches = _base_run_patches()
patches[f"{MODULE}.get_llm"] = MagicMock(return_value=MagicMock())
patches[f"{MODULE}.AdvancedSearchSystem"] = MagicMock(
return_value=system
)
patches[
"local_deep_research.web.research_state.is_termination_requested"
] = MagicMock(side_effect=[False, True, True, True, True])
snapshot = {
"llm.provider": "ollama",
"llm.model": "m",
"search.tool": "searxng",
}
with ExitStack() as stack:
stack.enter_context(
patch(
"local_deep_research.config.search_config.factory_get_search",
MagicMock(return_value=MagicMock()),
)
)
stack.enter_context(
patch(
"local_deep_research.security.egress.policy.context_from_snapshot",
return_value=MagicMock(),
)
)
stack.enter_context(
patch(
"local_deep_research.security.egress.run_classification.audit_run_from_snapshot",
return_value=MagicMock(allowed=True),
)
)
for target, mock_obj in patches.items():
stack.enter_context(patch(target, mock_obj))
_get_raw_run_research_process()(
1,
"test",
"quick",
username="user1",
settings_snapshot=snapshot,
search_engine="searxng",
)
handle_termination_mock = patches[f"{MODULE}.handle_termination"]
assert handle_termination_mock.called, (
"handle_termination should have been invoked once "
"is_termination_requested flipped True mid-run"
)
assert not system.analyze_topic.called, (
"the search should never have started once termination was "
"detected in the first progress_callback emission"
)
class TestErrorClassification:
"""Tests for search error classification in run_research_process.
Drives the REAL classification in run_research_process (the
``except Exception as search_error`` / ``except Exception as e`` pair
around ``system.analyze_topic()``) via the shared
``run_quick_mode_with_search_error`` harness in helpers.py, instead of
a local ``_classify_error`` copy of the string-matching rules.
"""
@pytest.mark.parametrize(
"error_message, expected_substring",
[
(
"Request failed with status code: 503",
"Ollama AI service is unavailable",
),
("status code: 404 not found", "model not found"),
("status code: 429 rate limited", "rejected the request"),
("status code: 500 internal error", "rejected the request"),
(
"Connection refused to localhost:11434",
"Connection error",
),
("TCP connection reset by peer", "Connection error"),
("Something unexpected happened", "unexpected error"),
],
)
def test_error_classification(self, error_message, expected_substring):
message = run_quick_mode_with_search_error(error_message)
assert expected_substring.lower() in message.lower()
def test_503_takes_priority_over_generic_status_code(self):
"""503 is matched specifically before the generic 'status code:'
pattern that also matches it (both are substrings of the raw
message)."""
message = run_quick_mode_with_search_error("status code: 503")
assert "Ollama AI service is unavailable" in message
assert "rejected the request" not in message
def test_404_takes_priority_over_generic_status_code(self):
"""404 is matched specifically before the generic 'status code:'
pattern."""
message = run_quick_mode_with_search_error("status code: 404")
assert "model not found" in message.lower()
assert "rejected the request" not in message
class TestResearchModes:
"""Tests for mode-specific behavior.
Drives the real quick-mode output logic (research_service.py
~1736-1900) via the shared ``run_quick_mode_with_analyze_result``
harness, instead of re-deriving ``is_error`` / ``has_findings``
locals from a results dict the test built itself and never passed
anywhere.
"""
def test_quick_mode_checks_findings(self):
"""A results dict with findings/formatted_findings reaches the
citation formatter (i.e. produces a report), rather than falling
through to the "nothing to summarize" branch."""
result = run_quick_mode_with_analyze_result(
{
"findings": [{"content": "f1", "phase": "Final synthesis"}],
"formatted_findings": "Summary text",
"iterations": 1,
"current_knowledge": "",
}
)
assert "clean_markdown" in result
def test_quick_mode_detects_error_in_findings(self):
"""An 'Error:' prefixed formatted_findings triggers the fallback
path -- the saved content differs from (recovers past) the raw
error text, using current_knowledge as the recovery source."""
result = run_quick_mode_with_analyze_result(
{
"findings": [],
"formatted_findings": "Error: token limit exceeded",
"iterations": 1,
"current_knowledge": "fallback knowledge",
}
)
assert result == {"clean_markdown": "fallback knowledge"}
def test_quick_mode_no_error_prefix(self):
"""A normal (non-'Error:') formatted_findings is NOT routed
through the fallback machinery -- the synthesized answer passes
straight through unchanged."""
result = run_quick_mode_with_analyze_result(
{
"findings": [
{
"content": "This is a normal summary.",
"phase": "Final synthesis",
}
],
"formatted_findings": "This is a normal summary.",
"iterations": 1,
"current_knowledge": "",
}
)
assert result == {"clean_markdown": "This is a normal summary."}