~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
refactor: remove dead SEARCH_SNIPPETS_ONLY checks (DRY PR 2/23) (#4459)
LearningCircuit2b96e1d53675
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py b/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.pyindex 82747e33dd..d26f16fda6 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py@@ -7,7 +7,6 @@ from loguru import loggerfrom ...advanced_search_system.filters.journal_reputation_filter import (JournalReputationFilter,)-from ...config import search_configfrom ...constants import SNIPPET_LENGTH_SHORTfrom ..rate_limiting import RateLimitErrorfrom ..search_engine_base import BaseSearchEngine@@ -204,14 +203,6 @@ class ArXivSearchEngine(BaseSearchEngine):Returns:List of result dictionaries with full content"""- # Check if we should get full content- if (- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")- and search_config.SEARCH_SNIPPETS_ONLY- ):- logger.info("Snippet-only mode, skipping full content retrieval")- return relevant_items-logger.info("Getting full content for relevant arXiv papers")results = []@@ -422,49 +413,42 @@ class ArXivSearchEngine(BaseSearchEngine):"journal_ref": paper.journal_ref,}- # Add full content if not in snippet-only mode- if (- not hasattr(search_config, "SEARCH_SNIPPETS_ONLY")- or not search_config.SEARCH_SNIPPETS_ONLY- ):- result.update(- {- "pdf_url": paper.pdf_url,- "authors": [- author.name for author in paper.authors- ], # All authors- "published": (- paper.published.strftime("%Y-%m-%d")- if paper.published- else None- ),- "updated": (- paper.updated.strftime("%Y-%m-%d")- if paper.updated- else None- ),- "categories": paper.categories,- "summary": paper.summary, # Full summary- "comment": paper.comment,- "doi": paper.doi,- "content": paper.summary, # Use summary as content- "full_content": paper.summary, # For consistency- }- )-- # Download PDF if requested- if self.include_full_text and self.download_dir:- try:- # Apply rate limiting before PDF download- self.rate_tracker.apply_rate_limit(self.engine_type)+ result.update(+ {+ "pdf_url": paper.pdf_url,+ "authors": [+ author.name for author in paper.authors+ ], # All authors+ "published": (+ paper.published.strftime("%Y-%m-%d")+ if paper.published+ else None+ ),+ "updated": (+ paper.updated.strftime("%Y-%m-%d")+ if paper.updated+ else None+ ),+ "categories": paper.categories,+ "summary": paper.summary, # Full summary+ "comment": paper.comment,+ "doi": paper.doi,+ "content": paper.summary, # Use summary as content+ "full_content": paper.summary, # For consistency+ }+ )- # Download the paper- paper_path = paper.download_pdf(- dirpath=self.download_dir- )- result["pdf_path"] = str(paper_path)- except Exception:- logger.exception("Error downloading paper")+ # Download PDF if requested+ if self.include_full_text and self.download_dir:+ try:+ # Apply rate limiting before PDF download+ self.rate_tracker.apply_rate_limit(self.engine_type)++ # Download the paper+ paper_path = paper.download_pdf(dirpath=self.download_dir)+ result["pdf_path"] = str(paper_path)+ except Exception:+ logger.exception("Error downloading paper")return resultdiff --git a/src/local_deep_research/web_search_engines/engines/search_engine_elasticsearch.py b/src/local_deep_research/web_search_engines/engines/search_engine_elasticsearch.pyindex 58927f9e2b..31db4991a1 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_elasticsearch.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_elasticsearch.py@@ -4,7 +4,6 @@ from typing import Any, Dict, List, Optionalfrom elasticsearch import Elasticsearchfrom langchain_core.language_models import BaseLLM-from ...config import search_configfrom ...constants import SNIPPET_LENGTH_SHORTfrom ...security.log_sanitizer import redact_secrets, sanitize_error_messagefrom ..search_engine_base import BaseSearchEngine@@ -302,14 +301,6 @@ class ElasticsearchSearchEngine(BaseSearchEngine):Returns:List of result dictionaries with full content"""- # Check if we should get full content- if (- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")- and search_config.SEARCH_SNIPPETS_ONLY- ):- logger.info("Snippet-only mode, skipping full content retrieval")- return relevant_items-logger.info("Getting full content for relevant Elasticsearch documents")results = []diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_github.py b/src/local_deep_research/web_search_engines/engines/search_engine_github.pyindex 8a7d54d918..31b94bfc22 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_github.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_github.py@@ -6,7 +6,7 @@ from typing import Any, Dict, List, Optionalfrom langchain_core.language_models import BaseLLMfrom loguru import logger-from ...config import llm_config, search_config+from ...config import llm_configfrom ...constants import USER_AGENTfrom ...security.log_sanitizer import redact_secrets, sanitize_error_messagefrom ...security.safe_requests import safe_get@@ -633,14 +633,6 @@ class GitHubSearchEngine(BaseSearchEngine):Returns:List of result dictionaries with full content"""- # Check if we should add full content- if (- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")- and search_config.SEARCH_SNIPPETS_ONLY- ):- logger.info("Snippet-only mode, skipping full content retrieval")- return relevant_items-logger.info(f"Getting full content for {len(relevant_items)} GitHub results")@@ -806,14 +798,7 @@ class GitHubSearchEngine(BaseSearchEngine):# Format resultspreviews = [self._format_code_preview(result) for result in results]- # Get full content if requested- if (- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")- and not search_config.SEARCH_SNIPPETS_ONLY- ):- return self._get_full_content(previews)-- return previews+ return self._get_full_content(previews)finally:# Restore original search typediff --git a/src/local_deep_research/web_search_engines/engines/search_engine_library.py b/src/local_deep_research/web_search_engines/engines/search_engine_library.pyindex fb0824979e..3a16a3d1c9 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_library.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_library.py@@ -366,16 +366,6 @@ class LibraryRAGSearchEngine(BaseSearchEngine):Get full content for relevant library documents.Retrieves complete document text instead of just snippets."""- # Check if we should get full content- from ...config import search_config-- if (- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")- and search_config.SEARCH_SNIPPETS_ONLY- ):- logger.info("Snippet-only mode, skipping full content retrieval")- return relevant_items-if not self.username:logger.error("Cannot retrieve full content without username")return relevant_itemsdiff --git a/src/local_deep_research/web_search_engines/engines/search_engine_mojeek.py b/src/local_deep_research/web_search_engines/engines/search_engine_mojeek.pyindex a9a111778b..5e75650338 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_mojeek.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_mojeek.py@@ -3,7 +3,6 @@ from typing import Any, Dict, List, Optionalfrom langchain_core.language_models import BaseLLMfrom loguru import logger-from ...config import search_configfrom ...security.log_sanitizer import redact_secretsfrom ...security.safe_requests import safe_getfrom ..rate_limiting import RateLimitError@@ -230,13 +229,6 @@ class MojeekSearchEngine(BaseSearchEngine):Returns:List of result dictionaries with full content"""- if (- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")- and search_config.SEARCH_SNIPPETS_ONLY- ):- logger.info("Snippet-only mode, skipping full content retrieval")- return relevant_items-if self.include_full_content and hasattr(self, "full_search"):logger.info("Retrieving full webpage content")try:diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.pyindex 6e948ad465..1c496eb558 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py@@ -6,7 +6,6 @@ from defusedxml import ElementTree as ETfrom langchain_core.language_models import BaseLLMfrom loguru import logger-from ...config import search_configfrom ...constants import SNIPPET_LENGTH_LONGfrom ...security.log_sanitizer import redact_secretsfrom ...security.safe_requests import safe_get@@ -1504,19 +1503,6 @@ The default assumption should be that medical and scientific queries want RECENTReturns:List of result dictionaries with full content"""- # Check if we should add full content- snippets_only_mode = (- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")- and search_config.SEARCH_SNIPPETS_ONLY- )-- if snippets_only_mode:- logger.info(- "Snippet-only mode enabled, will fetch abstracts as snippets"- )- # For PubMed, we still need to fetch abstracts as they serve as snippets- # But we'll skip full-text retrieval-logger.info(f"Getting content for {len(relevant_items)} PubMed articles")@@ -1530,7 +1516,7 @@ The default assumption should be that medical and scientific queries want RECENT# Get abstracts if requested and PMIDs exist# In snippet-only mode, always get abstracts as they serve as snippetsabstracts = {}- if (self.get_abstracts or snippets_only_mode) and pmids:+ if self.get_abstracts and pmids:abstracts = self._get_article_abstracts(pmids)# Get detailed metadata for all articles (publication types, MeSH terms, etc.)@@ -1540,7 +1526,7 @@ The default assumption should be that medical and scientific queries want RECENT# Find PMC IDs for full-text retrieval (if enabled and not in snippet-only mode)pmid_to_pmcid = {}- if self.get_full_text and pmids and not snippets_only_mode:+ if self.get_full_text and pmids:pmid_to_pmcid = self._find_pmc_ids(pmids)# Add content to results@@ -1673,13 +1659,8 @@ The default assumption should be that medical and scientific queries want RECENTf"Title: {title}\n\nAbstract: {abstract_text}")- # In snippet-only mode, use enriched content- if snippets_only_mode:- result["full_content"] = enriched_content- result["content"] = enriched_content- result["content_type"] = "abstract"# Use abstract as content if no full text- elif pmid not in pmid_to_pmcid:+ if pmid not in pmid_to_pmcid:result["full_content"] = enriched_contentresult["content"] = enriched_contentresult["content_type"] = "abstract"diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_searxng.py b/src/local_deep_research/web_search_engines/engines/search_engine_searxng.pyindex 91bd1072fe..038a375e52 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_searxng.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_searxng.py@@ -7,7 +7,6 @@ import requestsfrom langchain_core.language_models import BaseLLMfrom loguru import logger-from ...config import search_configfrom ...security.safe_requests import safe_getfrom ..search_engine_base import BaseSearchEngine@@ -488,13 +487,6 @@ class SearXNGSearchEngine(BaseSearchEngine):if not self.is_available:return relevant_items- if (- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")- and search_config.SEARCH_SNIPPETS_ONLY- ):- logger.info("Snippet-only mode, skipping full content retrieval")- return relevant_items-if not hasattr(self, "full_search"):return relevant_itemsdiff --git a/src/local_deep_research/web_search_engines/engines/search_engine_wayback.py b/src/local_deep_research/web_search_engines/engines/search_engine_wayback.pyindex e8c8fcbace..767930b48c 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_wayback.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_wayback.py@@ -4,7 +4,6 @@ from typing import Any, Dict, List, Optional, Tuplefrom langchain_core.language_models import BaseLLMfrom loguru import logger-from ...config import search_configfrom ...research_library.downloaders.extraction import extract_contentfrom ...security.safe_requests import safe_getfrom ..rate_limiting import RateLimitError@@ -329,14 +328,6 @@ class WaybackSearchEngine(BaseSearchEngine):Returns:List of result dictionaries with full content"""- # Check if we should add full content- if (- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")- and search_config.SEARCH_SNIPPETS_ONLY- ):- logger.info("Snippet-only mode, skipping full content retrieval")- return relevant_items-logger.info(f"Getting full content for {len(relevant_items)} Wayback Machine snapshots")@@ -399,14 +390,7 @@ class WaybackSearchEngine(BaseSearchEngine):}previews.append(preview)- # Get full content if not in snippets-only mode- if (- not hasattr(search_config, "SEARCH_SNIPPETS_ONLY")- or not search_config.SEARCH_SNIPPETS_ONLY- ):- return self._get_full_content(previews)-- return previews+ return self._get_full_content(previews)def search_by_date_range(self, url: str, start_date: str, end_date: str@@ -464,14 +448,7 @@ class WaybackSearchEngine(BaseSearchEngine):})- # Get full content if not in snippets-only mode- if (- not hasattr(search_config, "SEARCH_SNIPPETS_ONLY")- or not search_config.SEARCH_SNIPPETS_ONLY- ):- return self._get_full_content(snapshots)-- return snapshots+ return self._get_full_content(snapshots)except Exception:logger.exception(f"Error searching date range for {url}")@@ -509,16 +486,9 @@ class WaybackSearchEngine(BaseSearchEngine):"formatted_date": self._format_timestamp(timestamp),}- # Get full content if not in snippets-only mode- if (- not hasattr(search_config, "SEARCH_SNIPPETS_ONLY")- or not search_config.SEARCH_SNIPPETS_ONLY- ):- raw_html, full_content = self._get_wayback_content(- wayback_url- )- result["raw_html"] = raw_html- result["full_content"] = full_content+ raw_html, full_content = self._get_wayback_content(wayback_url)+ result["raw_html"] = raw_html+ result["full_content"] = full_contentreturn resultdiff --git a/src/local_deep_research/web_search_engines/engines/search_engine_wikipedia.py b/src/local_deep_research/web_search_engines/engines/search_engine_wikipedia.pyindex ce6d055232..378abe9063 100644--- a/src/local_deep_research/web_search_engines/engines/search_engine_wikipedia.py+++ b/src/local_deep_research/web_search_engines/engines/search_engine_wikipedia.py@@ -6,7 +6,6 @@ import wikipediafrom langchain_core.language_models import BaseLLMfrom loguru import logger-from ...config import search_configfrom ..search_engine_base import BaseSearchEngine@@ -208,14 +207,6 @@ class WikipediaSearchEngine(BaseSearchEngine):Returns:List of result dictionaries with full content"""- # Check if we should add full content- if (- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")- and search_config.SEARCH_SNIPPETS_ONLY- ):- logger.info("Snippet-only mode, skipping full content retrieval")- return relevant_items-logger.info(f"Getting full content for {len(relevant_items)} relevant Wikipedia pages")@@ -309,13 +300,8 @@ class WikipediaSearchEngine(BaseSearchEngine):Returns:Dictionary with page information"""- # Initialize include_content with our instance valueinclude_content = self.include_content- # Check if we should override with config setting- if hasattr(search_config, "SEARCH_SNIPPETS_ONLY"):- include_content = not search_config.SEARCH_SNIPPETS_ONLY-try:page = wikipedia.page(title, auto_suggest=False)diff --git a/tests/search_engines/test_search_engine_wayback.py b/tests/search_engines/test_search_engine_wayback.pyindex 5512c1b306..9c94cb3a38 100644--- a/tests/search_engines/test_search_engine_wayback.py+++ b/tests/search_engines/test_search_engine_wayback.py@@ -458,11 +458,10 @@ class TestWaybackSpecialMethods:Mock(return_value=mock_response),)- # Mock snippets only mode+ # Mock _get_wayback_content to avoid network callsmonkeypatch.setattr(- "local_deep_research.web_search_engines.engines.search_engine_wayback.search_config.SEARCH_SNIPPETS_ONLY",- True,- raising=False,+ "local_deep_research.web_search_engines.engines.search_engine_wayback.WaybackSearchEngine._get_wayback_content",+ Mock(return_value=("", "")),)engine.closest_only = True@@ -491,11 +490,10 @@ class TestWaybackSpecialMethods:Mock(return_value=mock_response),)- # Mock snippets only mode+ # Mock _get_wayback_content to avoid network callsmonkeypatch.setattr(- "local_deep_research.web_search_engines.engines.search_engine_wayback.search_config.SEARCH_SNIPPETS_ONLY",- True,- raising=False,+ "local_deep_research.web_search_engines.engines.search_engine_wayback.WaybackSearchEngine._get_wayback_content",+ Mock(return_value=("", "")),)result = engine.get_latest_snapshot("https://example.com")@@ -545,11 +543,10 @@ class TestWaybackDateRangeSearch:Mock(return_value=mock_response),)- # Mock snippets only mode+ # Mock _get_wayback_content to avoid network callsmonkeypatch.setattr(- "local_deep_research.web_search_engines.engines.search_engine_wayback.search_config.SEARCH_SNIPPETS_ONLY",- True,- raising=False,+ "local_deep_research.web_search_engines.engines.search_engine_wayback.WaybackSearchEngine._get_wayback_content",+ Mock(return_value=("", "")),)results = engine.search_by_date_range(@@ -603,35 +600,8 @@ class TestWaybackFullContent:return WaybackSearchEngine()- def test_get_full_content_snippets_only_mode(self, engine, monkeypatch):- """Test full content retrieval in snippets only mode."""- monkeypatch.setattr(- "local_deep_research.web_search_engines.engines.search_engine_wayback.search_config.SEARCH_SNIPPETS_ONLY",- True,- raising=False,- )-- items = [- {- "title": "Test Archive",- "link": "https://web.archive.org/web/20240115/https://example.com",- "snippet": "Test snippet",- }- ]-- results = engine._get_full_content(items)-- # Should return items as-is in snippets only mode- assert results == items-def test_get_full_content_with_retrieval(self, engine, monkeypatch):"""Test full content retrieval with actual fetching."""- # Ensure snippets only is False- monkeypatch.setattr(- "local_deep_research.web_search_engines.engines.search_engine_wayback.search_config",- Mock(spec=[]), # Empty spec means hasattr returns False- )-mock_response = Mock()mock_response.text = "<html><body><p>Full content</p></body></html>"diff --git a/tests/web_search_engines/engines/test_arxiv_coverage.py b/tests/web_search_engines/engines/test_arxiv_coverage.pyindex edf096a8a3..0a4bc7fd95 100644--- a/tests/web_search_engines/engines/test_arxiv_coverage.py+++ b/tests/web_search_engines/engines/test_arxiv_coverage.py@@ -361,16 +361,6 @@ class TestGetPreviews:class TestGetFullContent:- def test_snippets_only_mode(self, engine):- """SEARCH_SNIPPETS_ONLY returns items unchanged."""- items = [{"id": "1", "title": "T"}]- with patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"- ) as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = True- result = engine._get_full_content(items)- assert result == items-def test_no_paper_in_cache(self, engine):"""Item not in _papers cache is returned as-is."""engine._papers = {}@@ -431,11 +421,7 @@ class TestGetFullContent:with (patch("builtins.open", mock_open()),patch.dict("sys.modules", {"pypdf": MagicMock()}),- patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"- ) as mock_cfg,):- mock_cfg.SEARCH_SNIPPETS_ONLY = False# We need to mock pypdf inside the methodimport sys@@ -465,11 +451,7 @@ class TestGetFullContent:with (patch("builtins.open", mock_open()),- patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"- ) as mock_cfg,):- mock_cfg.SEARCH_SNIPPETS_ONLY = Falseimport sysmock_pypdf2 = MagicMock()@@ -498,11 +480,7 @@ class TestGetFullContent:with (patch("builtins.open", mock_open()),- patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"- ) as mock_cfg,):- mock_cfg.SEARCH_SNIPPETS_ONLY = Falseimport sys# pypdf fails with ImportError@@ -529,11 +507,7 @@ class TestGetFullContent:with (patch("builtins.open", mock_open()),- patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"- ) as mock_cfg,):- mock_cfg.SEARCH_SNIPPETS_ONLY = Falseimport sysmock_pypdf2 = MagicMock()@@ -559,12 +533,8 @@ class TestGetFullContent:engine_with_pdf._papers = {paper.entry_id: paper}items = [{"id": paper.entry_id, "title": "T"}]- with patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"- ) as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = False- result = engine_with_pdf._get_full_content(items)- assert result[0]["pdf_path"] is None+ result = engine_with_pdf._get_full_content(items)+ assert result[0]["pdf_path"] is Nonedef test_pdf_limit_reached(self, engine_with_pdf):"""Once max_full_text PDFs processed, remaining use summary."""@@ -587,11 +557,7 @@ class TestGetFullContent:with (patch("builtins.open", mock_open()),- patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"- ) as mock_cfg,):- mock_cfg.SEARCH_SNIPPETS_ONLY = Falseimport sysmock_pypdf2 = MagicMock()@@ -638,18 +604,14 @@ class TestRun:class TestGetPaperDetails:def test_paper_found_full_mode(self, engine):- """Paper found with full content (not snippet-only)."""+ """Paper found with full content."""import arxivpaper = _make_mock_paper()with (patch.object(arxiv, "Client") as mock_client_cls,patch.object(arxiv, "Search"),- patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"- ) as mock_cfg,):- mock_cfg.SEARCH_SNIPPETS_ONLY = Falsemock_client = Mock()mock_client.results.return_value = [paper]mock_client_cls.return_value = mock_client@@ -682,19 +644,15 @@ class TestGetPaperDetails:result = engine.get_paper_details("2101.00001")assert result == {}- def test_paper_snippet_only_mode(self, engine):- """Snippet-only mode returns basic info without full content."""+ def test_paper_long_summary_snippet_truncated(self, engine):+ """Long summary gets truncated snippet with ellipsis."""import arxivpaper = _make_mock_paper(summary="A" * 300)with (patch.object(arxiv, "Client") as mock_client_cls,patch.object(arxiv, "Search"),- patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"- ) as mock_cfg,):- mock_cfg.SEARCH_SNIPPETS_ONLY = Truemock_client = Mock()mock_client.results.return_value = [paper]mock_client_cls.return_value = mock_client@@ -702,7 +660,6 @@ class TestGetPaperDetails:result = engine.get_paper_details("2101.00001")assert result["title"] == "Test Paper"assert result["snippet"].endswith("...")- assert "content" not in resultdef test_paper_details_with_pdf_download(self, engine_with_pdf):"""PDF download happens in get_paper_details when configured."""@@ -712,11 +669,7 @@ class TestGetPaperDetails:with (patch.object(arxiv, "Client") as mock_client_cls,patch.object(arxiv, "Search"),- patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"- ) as mock_cfg,):- mock_cfg.SEARCH_SNIPPETS_ONLY = Falsemock_client = Mock()mock_client.results.return_value = [paper]mock_client_cls.return_value = mock_client@@ -734,11 +687,7 @@ class TestGetPaperDetails:with (patch.object(arxiv, "Client") as mock_client_cls,patch.object(arxiv, "Search"),- patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"- ) as mock_cfg,):- mock_cfg.SEARCH_SNIPPETS_ONLY = Falsemock_client = Mock()mock_client.results.return_value = [paper]mock_client_cls.return_value = mock_client@@ -755,11 +704,7 @@ class TestGetPaperDetails:with (patch.object(arxiv, "Client") as mock_client_cls,patch.object(arxiv, "Search"),- patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"- ) as mock_cfg,):- mock_cfg.SEARCH_SNIPPETS_ONLY = Falsemock_client = Mock()mock_client.results.return_value = [paper]mock_client_cls.return_value = mock_client@@ -776,11 +721,7 @@ class TestGetPaperDetails:with (patch.object(arxiv, "Client") as mock_client_cls,patch.object(arxiv, "Search"),- patch(- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"- ) as mock_cfg,):- mock_cfg.SEARCH_SNIPPETS_ONLY = Truemock_client = Mock()mock_client.results.return_value = [paper]mock_client_cls.return_value = mock_clientdiff --git a/tests/web_search_engines/engines/test_search_engine_arxiv.py b/tests/web_search_engines/engines/test_search_engine_arxiv.pyindex 822a556d77..0002a4de19 100644--- a/tests/web_search_engines/engines/test_search_engine_arxiv.py+++ b/tests/web_search_engines/engines/test_search_engine_arxiv.py@@ -463,32 +463,6 @@ class TestGetFullContent:assert len(results) == 1assert results[0]["title"] == "Test Paper"- def test_get_full_content_snippets_only_mode(self):- """Get full content in snippets only mode."""- from local_deep_research.web_search_engines.engines.search_engine_arxiv import (- ArXivSearchEngine,- )- from local_deep_research.web_search_engines.engines import (- search_engine_arxiv,- )-- with patch(- "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",- return_value=None,- ):- engine = ArXivSearchEngine()-- with patch.object(- search_engine_arxiv.search_config,- "SEARCH_SNIPPETS_ONLY",- True,- create=True,- ):- items = [{"id": "paper-id", "title": "Test Paper"}]- results = engine._get_full_content(items)-- assert results == items-class TestRun:"""Tests for run method."""diff --git a/tests/web_search_engines/engines/test_search_engine_elasticsearch.py b/tests/web_search_engines/engines/test_search_engine_elasticsearch.pyindex 539e92ba08..bfd90a4f0a 100644--- a/tests/web_search_engines/engines/test_search_engine_elasticsearch.py+++ b/tests/web_search_engines/engines/test_search_engine_elasticsearch.py@@ -602,34 +602,6 @@ class TestGetFullContent:# Should still have the original dataassert results[0]["title"] == "Test"- def test_get_full_content_snippets_only_mode(self):- """Get full content respects snippets-only mode."""- from local_deep_research.web_search_engines.engines.search_engine_elasticsearch import (- ElasticsearchSearchEngine,- )-- mock_client = MagicMock()- mock_client.info.return_value = {- "cluster_name": "test",- "version": {"number": "8.0"},- }-- with patch(- "local_deep_research.web_search_engines.engines.search_engine_elasticsearch.Elasticsearch",- return_value=mock_client,- ):- with patch(- "local_deep_research.web_search_engines.engines.search_engine_elasticsearch.search_config"- ) as mock_config:- mock_config.SEARCH_SNIPPETS_ONLY = True- engine = ElasticsearchSearchEngine()- items = [{"id": "doc1", "title": "Test", "snippet": "Snippet"}]- results = engine._get_full_content(items)-- # Should return items unchanged- assert results == items- mock_client.get.assert_not_called()-class TestSearchByQueryString:"""Tests for search_by_query_string method."""diff --git a/tests/web_search_engines/engines/test_search_engine_github_coverage.py b/tests/web_search_engines/engines/test_search_engine_github_coverage.pyindex 55c17b3bd5..2876bef7f9 100644--- a/tests/web_search_engines/engines/test_search_engine_github_coverage.py+++ b/tests/web_search_engines/engines/test_search_engine_github_coverage.py@@ -315,17 +315,6 @@ class TestGetPreviewsCoverage:class TestGetFullContentCoverage:"""Edge cases for _get_full_content."""- def test_snippets_only_mode(self):- """When SEARCH_SNIPPETS_ONLY is True, return items without fetching content."""- engine = _make_engine()- items = [- {"id": "1", "search_type": "repository", "repo_full_name": "o/r"}- ]- with patch(f"{MODULE}.search_config") as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = True- result = engine._get_full_content(items)- assert result == items-def test_issue_content_uses_snippet(self):"""Issue search type sets full_content to snippet."""engine = _make_engine()@@ -336,9 +325,7 @@ class TestGetFullContentCoverage:"snippet": "Bug description here",}]- with patch(f"{MODULE}.search_config") as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = False- result = engine._get_full_content(items)+ result = engine._get_full_content(items)assert result[0]["full_content"] == "Bug description here"assert result[0]["content_type"] == "issue"@@ -357,9 +344,7 @@ class TestGetFullContentCoverage:"public_repos": 20,}]- with patch(f"{MODULE}.search_config") as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = False- result = engine._get_full_content(items)+ result = engine._get_full_content(items)assert "Bio: Full-stack developer" in result[0]["full_content"]assert "Name: Dev Person" in result[0]["full_content"]assert "Location: NYC" in result[0]["full_content"]@@ -379,20 +364,16 @@ class TestGetFullContentCoverage:"public_repos": 0,}]- with patch(f"{MODULE}.search_config") as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = False- result = engine._get_full_content(items)+ result = engine._get_full_content(items)assert "Bio:" not in result[0]["full_content"]def test_repository_missing_full_name(self):"""Repository with empty repo_full_name skips readme fetch."""engine = _make_engine(include_readme=True)items = [{"id": "1", "search_type": "repository", "repo_full_name": ""}]- with patch(f"{MODULE}.search_config") as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = False- with patch.object(engine, "_get_readme_content") as mock_readme:- result = engine._get_full_content(items)- mock_readme.assert_not_called()+ with patch.object(engine, "_get_readme_content") as mock_readme:+ result = engine._get_full_content(items)+ mock_readme.assert_not_called()assert "full_content" not in result[0]@@ -618,7 +599,7 @@ class TestSearchCodeCoverage:assert engine.search_type == "repositories"def test_full_content_retrieval(self):- """search_code calls _get_full_content when SEARCH_SNIPPETS_ONLY is False."""+ """search_code calls _get_full_content."""engine = _make_engine()mock_results = [{@@ -634,15 +615,13 @@ class TestSearchCodeCoverage:}]with patch.object(engine, "_search_github", return_value=mock_results):- with patch(f"{MODULE}.search_config") as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = False- with patch.object(- engine,- "_get_full_content",- return_value=[{"full_content": "code"}],- ) as mock_fc:- engine.search_code("test")- mock_fc.assert_called_once()+ with patch.object(+ engine,+ "_get_full_content",+ return_value=[{"full_content": "code"}],+ ) as mock_fc:+ engine.search_code("test")+ mock_fc.assert_called_once()# ---------------------------------------------------------------------------diff --git a/tests/web_search_engines/engines/test_search_engine_github_deep_coverage.py b/tests/web_search_engines/engines/test_search_engine_github_deep_coverage.pyindex 26e61fab67..6dc6d57350 100644--- a/tests/web_search_engines/engines/test_search_engine_github_deep_coverage.py+++ b/tests/web_search_engines/engines/test_search_engine_github_deep_coverage.py@@ -403,11 +403,9 @@ class TestGetFullContentDeep:"repo_full_name": "owner/repo",}]- with patch(f"{MODULE}.search_config") as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = False- with patch.object(engine, "_get_readme_content") as mock_readme:- result = engine._get_full_content(items)- mock_readme.assert_not_called()+ with patch.object(engine, "_get_readme_content") as mock_readme:+ result = engine._get_full_content(items)+ mock_readme.assert_not_called()# No full_content key addedassert "full_content" not in result[0]@@ -421,15 +419,13 @@ class TestGetFullContentDeep:"file_url": "https://api.github.com/repos/o/r/contents/f.py",}]- with patch(f"{MODULE}.search_config") as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = False- with patch.object(- engine, "_get_file_content", return_value="file body"- ) as mock_fc:- result = engine._get_full_content(items)- mock_fc.assert_called_once_with(- "https://api.github.com/repos/o/r/contents/f.py"- )+ with patch.object(+ engine, "_get_file_content", return_value="file body"+ ) as mock_fc:+ result = engine._get_full_content(items)+ mock_fc.assert_called_once_with(+ "https://api.github.com/repos/o/r/contents/f.py"+ )assert result[0]["full_content"] == "file body"assert result[0]["content_type"] == "file"@@ -437,11 +433,9 @@ class TestGetFullContentDeep:"""Code item without file_url does not attempt fetch."""engine = _make_engine()items = [{"id": "code_abc", "search_type": "code", "file_url": ""}]- with patch(f"{MODULE}.search_config") as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = False- with patch.object(engine, "_get_file_content") as mock_fc:- engine._get_full_content(items)- mock_fc.assert_not_called()+ with patch.object(engine, "_get_file_content") as mock_fc:+ engine._get_full_content(items)+ mock_fc.assert_not_called()def test_repository_with_issues_fetches_issues(self):"""Repository item with include_issues=True fetches issues."""@@ -453,16 +447,14 @@ class TestGetFullContentDeep:"repo_full_name": "owner/repo",}]- with patch(f"{MODULE}.search_config") as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = False+ with patch.object(+ engine, "_get_readme_content", return_value="# README"+ ):with patch.object(- engine, "_get_readme_content", return_value="# README"- ):- with patch.object(- engine, "_get_recent_issues", return_value=[{"number": 1}]- ) as mock_iss:- result = engine._get_full_content(items)- mock_iss.assert_called_once_with("owner/repo")+ engine, "_get_recent_issues", return_value=[{"number": 1}]+ ) as mock_iss:+ result = engine._get_full_content(items)+ mock_iss.assert_called_once_with("owner/repo")assert result[0]["recent_issues"] == [{"number": 1}]def test_user_with_no_name_or_location_no_extra_lines(self):@@ -480,9 +472,7 @@ class TestGetFullContentDeep:"public_repos": 0,}]- with patch(f"{MODULE}.search_config") as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = False- result = engine._get_full_content(items)+ result = engine._get_full_content(items)assert "Name:" not in result[0]["full_content"]assert "Location:" not in result[0]["full_content"]assert result[0]["content_type"] == "user_profile"@@ -601,29 +591,6 @@ class TestSearchCodeDeep:call_arg = mock_search.call_args[0][0]assert call_arg == "import numpy"- def test_snippets_only_returns_previews_without_full_content(self):- """When SEARCH_SNIPPETS_ONLY is True, skip _get_full_content."""- engine = _make_engine()- mock_result = {- "sha": "x",- "name": "f.py",- "path": "f.py",- "html_url": "https://github.com/o/r/blob/main/f.py",- "url": "https://api.github.com/repos/o/r/contents/f.py",- "repository": {- "full_name": "o/r",- "html_url": "https://github.com/o/r",- },- }- with patch.object(engine, "_search_github", return_value=[mock_result]):- with patch(f"{MODULE}.search_config") as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = True- with patch.object(engine, "_get_full_content") as mock_fc:- results = engine.search_code("test")- mock_fc.assert_not_called()- assert len(results) == 1- assert results[0]["search_type"] == "code"-# ---------------------------------------------------------------------------# search_issues deep pathsdiff --git a/tests/web_search_engines/engines/test_search_engine_library_coverage.py b/tests/web_search_engines/engines/test_search_engine_library_coverage.pyindex c550b067ca..bf8358b32d 100644--- a/tests/web_search_engines/engines/test_search_engine_library_coverage.py+++ b/tests/web_search_engines/engines/test_search_engine_library_coverage.py@@ -5,7 +5,7 @@ Covers:- __init__: initialization with/without username- search: no username, no collections, exception handling- _get_previews: delegates to search-- _get_full_content: snippet-only mode, no username, no doc_id, success path+- _get_full_content: no username, no doc_id, success path- close: no-op"""@@ -233,32 +233,12 @@ class TestGetPreviews:class TestGetFullContent:- def _patch_search_config(self, snippet_only=False):- """Patch the search_config module that gets imported inside _get_full_content."""- mock_config = MagicMock()- mock_config.SEARCH_SNIPPETS_ONLY = snippet_only- return patch.dict(- "sys.modules",- {"local_deep_research.config.search_config": mock_config},- )-- def test_snippet_only_mode_returns_items(self):- """When SEARCH_SNIPPETS_ONLY is True, returns items unchanged."""+ def test_items_without_document_id_returned_unchanged(self):+ """Items lacking metadata.document_id are skipped and returned as-is."""engine = _make_engine()items = [{"title": "test", "snippet": "content"}]- from local_deep_research.config import search_config-- original = getattr(search_config, "SEARCH_SNIPPETS_ONLY", None)- search_config.SEARCH_SNIPPETS_ONLY = True- try:- result = engine._get_full_content(items)- finally:- if original is None:- if hasattr(search_config, "SEARCH_SNIPPETS_ONLY"):- delattr(search_config, "SEARCH_SNIPPETS_ONLY")- else:- search_config.SEARCH_SNIPPETS_ONLY = original+ result = engine._get_full_content(items)assert result == itemsdiff --git a/tests/web_search_engines/engines/test_search_engine_mojeek.py b/tests/web_search_engines/engines/test_search_engine_mojeek.pyindex 8a164067c8..324ffb9113 100644--- a/tests/web_search_engines/engines/test_search_engine_mojeek.py+++ b/tests/web_search_engines/engines/test_search_engine_mojeek.py@@ -548,31 +548,6 @@ class TestGetPreviews:class TestGetFullContent:"""Tests for _get_full_content method."""- def test_snippets_only_mode(self):- """Test snippet-only mode returns items unchanged."""- from local_deep_research.web_search_engines.engines import (- search_engine_mojeek,- )- from local_deep_research.web_search_engines.engines.search_engine_mojeek import (- MojeekSearchEngine,- )-- engine = MojeekSearchEngine(- api_key="test-key", include_full_content=False- )-- items = [{"link": "https://example.com", "snippet": "test"}]-- with patch.object(- search_engine_mojeek.search_config,- "SEARCH_SNIPPETS_ONLY",- True,- create=True,- ):- results = engine._get_full_content(items)-- assert results == items-def test_with_full_search_attr(self):"""Test delegation to full_search._get_full_content."""from local_deep_research.web_search_engines.engines.search_engine_mojeek import (diff --git a/tests/web_search_engines/engines/test_search_engine_pubmed_coverage.py b/tests/web_search_engines/engines/test_search_engine_pubmed_coverage.pyindex 03b951a57f..4facd23b16 100644--- a/tests/web_search_engines/engines/test_search_engine_pubmed_coverage.py+++ b/tests/web_search_engines/engines/test_search_engine_pubmed_coverage.py@@ -423,24 +423,26 @@ class TestGetFullContentPMCIntegration:class TestGetFullContentSnippetsOnlyMode:def test_snippets_only_mode_uses_abstract(self):- engine = _make_engine(get_full_text=True, get_abstracts=True)+ # Snippets-only behavior is now driven by get_full_text=False; the+ # dead module-level SEARCH_SNIPPETS_ONLY switch this test used to+ # patch was removed in this PR. With full text disabled, no PMC+ # lookup happens and the content is the abstract.+ engine = _make_engine(get_full_text=False, get_abstracts=True)items = [_make_preview_item(pmid="10003")]- with patch(f"{MODULE}.search_config") as mock_config:- mock_config.SEARCH_SNIPPETS_ONLY = True+ with patch.object(+ engine,+ "_get_article_abstracts",+ return_value={"10003": "Snippet abstract"},+ ):with patch.object(- engine,- "_get_article_abstracts",- return_value={"10003": "Snippet abstract"},+ engine, "_get_article_detailed_metadata", return_value={}):with patch.object(- engine, "_get_article_detailed_metadata", return_value={}- ):- with patch.object(- engine, "_find_pmc_ids", return_value={}- ) as mock_pmc:- results = engine._get_full_content(items)- assert results[0]["content_type"] == "abstract"- mock_pmc.assert_not_called()+ engine, "_find_pmc_ids", return_value={}+ ) as mock_pmc:+ results = engine._get_full_content(items)+ assert results[0]["content_type"] == "abstract"+ mock_pmc.assert_not_called()class TestGetFullContentPMCAvailability:diff --git a/tests/web_search_engines/engines/test_search_engine_searxng_coverage.py b/tests/web_search_engines/engines/test_search_engine_searxng_coverage.pyindex f8befd5f84..d1349ecc88 100644--- a/tests/web_search_engines/engines/test_search_engine_searxng_coverage.py+++ b/tests/web_search_engines/engines/test_search_engine_searxng_coverage.py@@ -347,25 +347,6 @@ class TestGetSearchResultsURLFromTitleHref:assert results[0]["url"] == "https://example.com/padded"-# ----------------------------------------------------------------------------# _get_full_content – snippets-only mode-# ------------------------------------------------------------------------------class TestGetFullContentSnippetsOnly:- """_get_full_content returns items unchanged in snippets-only mode."""-- def test_snippets_only_skips_full_content(self):- engine = _make_engine()- items = [{"title": "T", "link": "https://example.com", "snippet": "S"}]-- with patch(f"{MODULE}.search_config") as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = True- result = engine._get_full_content(items)-- assert result is items--# ---------------------------------------------------------------------------# _get_full_content – exception handling# ---------------------------------------------------------------------------@@ -378,15 +359,11 @@ class TestGetFullContentException:engine = _make_engine()items = [{"title": "T", "link": "https://example.com", "snippet": "S"}]- with patch(f"{MODULE}.search_config") as mock_cfg:- mock_cfg.SEARCH_SNIPPETS_ONLY = False- # Make full_search._get_full_content raise- engine.full_search = Mock()- engine.full_search._get_full_content.side_effect = RuntimeError(- "boom"- )+ # Make full_search._get_full_content raise+ engine.full_search = Mock()+ engine.full_search._get_full_content.side_effect = RuntimeError("boom")- result = engine._get_full_content(items)+ result = engine._get_full_content(items)assert result is itemsdiff --git a/tests/web_search_engines/engines/test_search_engine_wayback_coverage.py b/tests/web_search_engines/engines/test_search_engine_wayback_coverage.pyindex fcc13306d2..8f67d5e470 100644--- a/tests/web_search_engines/engines/test_search_engine_wayback_coverage.py+++ b/tests/web_search_engines/engines/test_search_engine_wayback_coverage.py@@ -206,37 +206,20 @@ class TestGetWaybackContentExtra:class TestGetFullContentExtra:- def test_snippet_only_mode_skips_fetch(self):- engine = _engine()- items = [- {- "title": "T",- "link": "https://web.archive.org/web/x/https://t.com",- }- ]- with patch(- f"{MODULE}.search_config",- SEARCH_SNIPPETS_ONLY=True,- ):- result = engine._get_full_content(items)- assert result == items-def test_item_without_link_appended_as_is(self):engine = _engine()items = [{"title": "No link item"}] # no 'link' key- with patch(f"{MODULE}.search_config", SEARCH_SNIPPETS_ONLY=False):- result = engine._get_full_content(items)+ result = engine._get_full_content(items)assert result[0]["title"] == "No link item"assert "raw_html" not in result[0]def test_exception_in_loop_appends_item_unchanged(self):engine = _engine()items = [{"title": "X", "link": "https://web.archive.org/web/x"}]- with patch(f"{MODULE}.search_config", SEARCH_SNIPPETS_ONLY=False):- with patch.object(- engine, "_get_wayback_content", side_effect=OSError("bad")- ):- result = engine._get_full_content(items)+ with patch.object(+ engine, "_get_wayback_content", side_effect=OSError("bad")+ ):+ result = engine._get_full_content(items)assert result[0] == items[0]@@ -246,24 +229,6 @@ class TestGetFullContentExtra:class TestSearchByUrlExtra:- def test_snippet_only_mode_returns_previews_without_full_content(self):- engine = _engine()- snapshots = [- {- "timestamp": "20240101000000",- "formatted_date": "2024-01-01 00:00:00",- "url": "https://web.archive.org/web/20240101/https://example.com",- "original_url": "https://example.com",- }- ]- with patch.object(- engine, "_get_wayback_snapshots", return_value=snapshots- ):- with patch(f"{MODULE}.search_config", SEARCH_SNIPPETS_ONLY=True):- result = engine.search_by_url("https://example.com")- assert len(result) == 1- assert "raw_html" not in result[0]-def test_max_snapshots_parameter_honoured(self):engine = _engine(max_snapshots_per_url=10)snapshots = [@@ -278,7 +243,9 @@ class TestSearchByUrlExtra:with patch.object(engine, "_get_wayback_snapshots", return_value=snapshots):- with patch(f"{MODULE}.search_config", SEARCH_SNIPPETS_ONLY=True):+ with patch.object(+ engine, "_get_wayback_content", return_value=("", "")+ ):result = engine.search_by_url("https://example.com", max_snapshots=2)@@ -299,7 +266,9 @@ class TestSearchByDateRange:resp = _mock_response(status=200, json_body=data)engine = _engine()with patch(f"{MODULE}.safe_get", return_value=resp):- with patch(f"{MODULE}.search_config", SEARCH_SNIPPETS_ONLY=True):+ with patch.object(+ engine, "_get_wayback_content", return_value=("", "")+ ):result = engine.search_by_date_range("https://example.com", "20230101", "20231231")@@ -336,28 +305,3 @@ class TestGetLatestSnapshotExtra:with patch(f"{MODULE}.safe_get", side_effect=OSError("fail")):result = engine.get_latest_snapshot("https://example.com")assert result is None-- def test_snippets_only_mode_skips_content_fetch(self):- """In snippet-only mode get_latest_snapshot must NOT call _get_wayback_content."""- engine = _engine()- resp = _mock_response(- status=200,- json_body={- "archived_snapshots": {- "closest": {- "url": "https://web.archive.org/web/20240101/https://example.com",- "timestamp": "20240101000000",- }- }- },- )- with patch(f"{MODULE}.safe_get", return_value=resp):- with patch(f"{MODULE}.search_config", SEARCH_SNIPPETS_ONLY=True):- with patch.object(- engine,- "_get_wayback_content",- side_effect=AssertionError("must not be called"),- ):- result = engine.get_latest_snapshot("https://example.com")- assert result is not None- assert result["timestamp"] == "20240101000000"