Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

refactor: remove dead SEARCH_SNIPPETS_ONLY checks (DRY PR 2/23) (#4459)

LearningCircuit2b96e1d53675
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py b/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py
index 82747e33dd..d26f16fda6 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_arxiv.py
@@ -7,7 +7,6 @@ from loguru import logger
from ...advanced_search_system.filters.journal_reputation_filter import (
JournalReputationFilter,
)
-from ...config import search_config
from ...constants import SNIPPET_LENGTH_SHORT
from ..rate_limiting import RateLimitError
from ..search_engine_base import BaseSearchEngine
@@ -204,14 +203,6 @@ class ArXivSearchEngine(BaseSearchEngine):
Returns:
List of result dictionaries with full content
"""
- # Check if we should get full content
- if (
- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
- and search_config.SEARCH_SNIPPETS_ONLY
- ):
- logger.info("Snippet-only mode, skipping full content retrieval")
- return relevant_items
-
logger.info("Getting full content for relevant arXiv papers")
results = []
@@ -422,49 +413,42 @@ class ArXivSearchEngine(BaseSearchEngine):
"journal_ref": paper.journal_ref,
}
- # Add full content if not in snippet-only mode
- if (
- not hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
- or not search_config.SEARCH_SNIPPETS_ONLY
- ):
- result.update(
- {
- "pdf_url": paper.pdf_url,
- "authors": [
- author.name for author in paper.authors
- ], # All authors
- "published": (
- paper.published.strftime("%Y-%m-%d")
- if paper.published
- else None
- ),
- "updated": (
- paper.updated.strftime("%Y-%m-%d")
- if paper.updated
- else None
- ),
- "categories": paper.categories,
- "summary": paper.summary, # Full summary
- "comment": paper.comment,
- "doi": paper.doi,
- "content": paper.summary, # Use summary as content
- "full_content": paper.summary, # For consistency
- }
- )
-
- # Download PDF if requested
- if self.include_full_text and self.download_dir:
- try:
- # Apply rate limiting before PDF download
- self.rate_tracker.apply_rate_limit(self.engine_type)
+ result.update(
+ {
+ "pdf_url": paper.pdf_url,
+ "authors": [
+ author.name for author in paper.authors
+ ], # All authors
+ "published": (
+ paper.published.strftime("%Y-%m-%d")
+ if paper.published
+ else None
+ ),
+ "updated": (
+ paper.updated.strftime("%Y-%m-%d")
+ if paper.updated
+ else None
+ ),
+ "categories": paper.categories,
+ "summary": paper.summary, # Full summary
+ "comment": paper.comment,
+ "doi": paper.doi,
+ "content": paper.summary, # Use summary as content
+ "full_content": paper.summary, # For consistency
+ }
+ )
- # Download the paper
- paper_path = paper.download_pdf(
- dirpath=self.download_dir
- )
- result["pdf_path"] = str(paper_path)
- except Exception:
- logger.exception("Error downloading paper")
+ # Download PDF if requested
+ if self.include_full_text and self.download_dir:
+ try:
+ # Apply rate limiting before PDF download
+ self.rate_tracker.apply_rate_limit(self.engine_type)
+
+ # Download the paper
+ paper_path = paper.download_pdf(dirpath=self.download_dir)
+ result["pdf_path"] = str(paper_path)
+ except Exception:
+ logger.exception("Error downloading paper")
return result
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_elasticsearch.py b/src/local_deep_research/web_search_engines/engines/search_engine_elasticsearch.py
index 58927f9e2b..31db4991a1 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_elasticsearch.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_elasticsearch.py
@@ -4,7 +4,6 @@ from typing import Any, Dict, List, Optional
from elasticsearch import Elasticsearch
from langchain_core.language_models import BaseLLM
-from ...config import search_config
from ...constants import SNIPPET_LENGTH_SHORT
from ...security.log_sanitizer import redact_secrets, sanitize_error_message
from ..search_engine_base import BaseSearchEngine
@@ -302,14 +301,6 @@ class ElasticsearchSearchEngine(BaseSearchEngine):
Returns:
List of result dictionaries with full content
"""
- # Check if we should get full content
- if (
- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
- and search_config.SEARCH_SNIPPETS_ONLY
- ):
- logger.info("Snippet-only mode, skipping full content retrieval")
- return relevant_items
-
logger.info("Getting full content for relevant Elasticsearch documents")
results = []
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_github.py b/src/local_deep_research/web_search_engines/engines/search_engine_github.py
index 8a7d54d918..31b94bfc22 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_github.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_github.py
@@ -6,7 +6,7 @@ from typing import Any, Dict, List, Optional
from langchain_core.language_models import BaseLLM
from loguru import logger
-from ...config import llm_config, search_config
+from ...config import llm_config
from ...constants import USER_AGENT
from ...security.log_sanitizer import redact_secrets, sanitize_error_message
from ...security.safe_requests import safe_get
@@ -633,14 +633,6 @@ class GitHubSearchEngine(BaseSearchEngine):
Returns:
List of result dictionaries with full content
"""
- # Check if we should add full content
- if (
- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
- and search_config.SEARCH_SNIPPETS_ONLY
- ):
- logger.info("Snippet-only mode, skipping full content retrieval")
- return relevant_items
-
logger.info(
f"Getting full content for {len(relevant_items)} GitHub results"
)
@@ -806,14 +798,7 @@ class GitHubSearchEngine(BaseSearchEngine):
# Format results
previews = [self._format_code_preview(result) for result in results]
- # Get full content if requested
- if (
- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
- and not search_config.SEARCH_SNIPPETS_ONLY
- ):
- return self._get_full_content(previews)
-
- return previews
+ return self._get_full_content(previews)
finally:
# Restore original search type
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_library.py b/src/local_deep_research/web_search_engines/engines/search_engine_library.py
index fb0824979e..3a16a3d1c9 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_library.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_library.py
@@ -366,16 +366,6 @@ class LibraryRAGSearchEngine(BaseSearchEngine):
Get full content for relevant library documents.
Retrieves complete document text instead of just snippets.
"""
- # Check if we should get full content
- from ...config import search_config
-
- if (
- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
- and search_config.SEARCH_SNIPPETS_ONLY
- ):
- logger.info("Snippet-only mode, skipping full content retrieval")
- return relevant_items
-
if not self.username:
logger.error("Cannot retrieve full content without username")
return relevant_items
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_mojeek.py b/src/local_deep_research/web_search_engines/engines/search_engine_mojeek.py
index a9a111778b..5e75650338 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_mojeek.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_mojeek.py
@@ -3,7 +3,6 @@ from typing import Any, Dict, List, Optional
from langchain_core.language_models import BaseLLM
from loguru import logger
-from ...config import search_config
from ...security.log_sanitizer import redact_secrets
from ...security.safe_requests import safe_get
from ..rate_limiting import RateLimitError
@@ -230,13 +229,6 @@ class MojeekSearchEngine(BaseSearchEngine):
Returns:
List of result dictionaries with full content
"""
- if (
- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
- and search_config.SEARCH_SNIPPETS_ONLY
- ):
- logger.info("Snippet-only mode, skipping full content retrieval")
- return relevant_items
-
if self.include_full_content and hasattr(self, "full_search"):
logger.info("Retrieving full webpage content")
try:
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py
index 6e948ad465..1c496eb558 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_pubmed.py
@@ -6,7 +6,6 @@ from defusedxml import ElementTree as ET
from langchain_core.language_models import BaseLLM
from loguru import logger
-from ...config import search_config
from ...constants import SNIPPET_LENGTH_LONG
from ...security.log_sanitizer import redact_secrets
from ...security.safe_requests import safe_get
@@ -1504,19 +1503,6 @@ The default assumption should be that medical and scientific queries want RECENT
Returns:
List of result dictionaries with full content
"""
- # Check if we should add full content
- snippets_only_mode = (
- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
- and search_config.SEARCH_SNIPPETS_ONLY
- )
-
- if snippets_only_mode:
- logger.info(
- "Snippet-only mode enabled, will fetch abstracts as snippets"
- )
- # For PubMed, we still need to fetch abstracts as they serve as snippets
- # But we'll skip full-text retrieval
-
logger.info(
f"Getting content for {len(relevant_items)} PubMed articles"
)
@@ -1530,7 +1516,7 @@ The default assumption should be that medical and scientific queries want RECENT
# Get abstracts if requested and PMIDs exist
# In snippet-only mode, always get abstracts as they serve as snippets
abstracts = {}
- if (self.get_abstracts or snippets_only_mode) and pmids:
+ if self.get_abstracts and pmids:
abstracts = self._get_article_abstracts(pmids)
# Get detailed metadata for all articles (publication types, MeSH terms, etc.)
@@ -1540,7 +1526,7 @@ The default assumption should be that medical and scientific queries want RECENT
# Find PMC IDs for full-text retrieval (if enabled and not in snippet-only mode)
pmid_to_pmcid = {}
- if self.get_full_text and pmids and not snippets_only_mode:
+ if self.get_full_text and pmids:
pmid_to_pmcid = self._find_pmc_ids(pmids)
# Add content to results
@@ -1673,13 +1659,8 @@ The default assumption should be that medical and scientific queries want RECENT
f"Title: {title}\n\nAbstract: {abstract_text}"
)
- # In snippet-only mode, use enriched content
- if snippets_only_mode:
- result["full_content"] = enriched_content
- result["content"] = enriched_content
- result["content_type"] = "abstract"
# Use abstract as content if no full text
- elif pmid not in pmid_to_pmcid:
+ if pmid not in pmid_to_pmcid:
result["full_content"] = enriched_content
result["content"] = enriched_content
result["content_type"] = "abstract"
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_searxng.py b/src/local_deep_research/web_search_engines/engines/search_engine_searxng.py
index 91bd1072fe..038a375e52 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_searxng.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_searxng.py
@@ -7,7 +7,6 @@ import requests
from langchain_core.language_models import BaseLLM
from loguru import logger
-from ...config import search_config
from ...security.safe_requests import safe_get
from ..search_engine_base import BaseSearchEngine
@@ -488,13 +487,6 @@ class SearXNGSearchEngine(BaseSearchEngine):
if not self.is_available:
return relevant_items
- if (
- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
- and search_config.SEARCH_SNIPPETS_ONLY
- ):
- logger.info("Snippet-only mode, skipping full content retrieval")
- return relevant_items
-
if not hasattr(self, "full_search"):
return relevant_items
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_wayback.py b/src/local_deep_research/web_search_engines/engines/search_engine_wayback.py
index e8c8fcbace..767930b48c 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_wayback.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_wayback.py
@@ -4,7 +4,6 @@ from typing import Any, Dict, List, Optional, Tuple
from langchain_core.language_models import BaseLLM
from loguru import logger
-from ...config import search_config
from ...research_library.downloaders.extraction import extract_content
from ...security.safe_requests import safe_get
from ..rate_limiting import RateLimitError
@@ -329,14 +328,6 @@ class WaybackSearchEngine(BaseSearchEngine):
Returns:
List of result dictionaries with full content
"""
- # Check if we should add full content
- if (
- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
- and search_config.SEARCH_SNIPPETS_ONLY
- ):
- logger.info("Snippet-only mode, skipping full content retrieval")
- return relevant_items
-
logger.info(
f"Getting full content for {len(relevant_items)} Wayback Machine snapshots"
)
@@ -399,14 +390,7 @@ class WaybackSearchEngine(BaseSearchEngine):
}
previews.append(preview)
- # Get full content if not in snippets-only mode
- if (
- not hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
- or not search_config.SEARCH_SNIPPETS_ONLY
- ):
- return self._get_full_content(previews)
-
- return previews
+ return self._get_full_content(previews)
def search_by_date_range(
self, url: str, start_date: str, end_date: str
@@ -464,14 +448,7 @@ class WaybackSearchEngine(BaseSearchEngine):
}
)
- # Get full content if not in snippets-only mode
- if (
- not hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
- or not search_config.SEARCH_SNIPPETS_ONLY
- ):
- return self._get_full_content(snapshots)
-
- return snapshots
+ return self._get_full_content(snapshots)
except Exception:
logger.exception(f"Error searching date range for {url}")
@@ -509,16 +486,9 @@ class WaybackSearchEngine(BaseSearchEngine):
"formatted_date": self._format_timestamp(timestamp),
}
- # Get full content if not in snippets-only mode
- if (
- not hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
- or not search_config.SEARCH_SNIPPETS_ONLY
- ):
- raw_html, full_content = self._get_wayback_content(
- wayback_url
- )
- result["raw_html"] = raw_html
- result["full_content"] = full_content
+ raw_html, full_content = self._get_wayback_content(wayback_url)
+ result["raw_html"] = raw_html
+ result["full_content"] = full_content
return result
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_wikipedia.py b/src/local_deep_research/web_search_engines/engines/search_engine_wikipedia.py
index ce6d055232..378abe9063 100644
--- a/src/local_deep_research/web_search_engines/engines/search_engine_wikipedia.py
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_wikipedia.py
@@ -6,7 +6,6 @@ import wikipedia
from langchain_core.language_models import BaseLLM
from loguru import logger
-from ...config import search_config
from ..search_engine_base import BaseSearchEngine
@@ -208,14 +207,6 @@ class WikipediaSearchEngine(BaseSearchEngine):
Returns:
List of result dictionaries with full content
"""
- # Check if we should add full content
- if (
- hasattr(search_config, "SEARCH_SNIPPETS_ONLY")
- and search_config.SEARCH_SNIPPETS_ONLY
- ):
- logger.info("Snippet-only mode, skipping full content retrieval")
- return relevant_items
-
logger.info(
f"Getting full content for {len(relevant_items)} relevant Wikipedia pages"
)
@@ -309,13 +300,8 @@ class WikipediaSearchEngine(BaseSearchEngine):
Returns:
Dictionary with page information
"""
- # Initialize include_content with our instance value
include_content = self.include_content
- # Check if we should override with config setting
- if hasattr(search_config, "SEARCH_SNIPPETS_ONLY"):
- include_content = not search_config.SEARCH_SNIPPETS_ONLY
-
try:
page = wikipedia.page(title, auto_suggest=False)
diff --git a/tests/search_engines/test_search_engine_wayback.py b/tests/search_engines/test_search_engine_wayback.py
index 5512c1b306..9c94cb3a38 100644
--- a/tests/search_engines/test_search_engine_wayback.py
+++ b/tests/search_engines/test_search_engine_wayback.py
@@ -458,11 +458,10 @@ class TestWaybackSpecialMethods:
Mock(return_value=mock_response),
)
- # Mock snippets only mode
+ # Mock _get_wayback_content to avoid network calls
monkeypatch.setattr(
- "local_deep_research.web_search_engines.engines.search_engine_wayback.search_config.SEARCH_SNIPPETS_ONLY",
- True,
- raising=False,
+ "local_deep_research.web_search_engines.engines.search_engine_wayback.WaybackSearchEngine._get_wayback_content",
+ Mock(return_value=("", "")),
)
engine.closest_only = True
@@ -491,11 +490,10 @@ class TestWaybackSpecialMethods:
Mock(return_value=mock_response),
)
- # Mock snippets only mode
+ # Mock _get_wayback_content to avoid network calls
monkeypatch.setattr(
- "local_deep_research.web_search_engines.engines.search_engine_wayback.search_config.SEARCH_SNIPPETS_ONLY",
- True,
- raising=False,
+ "local_deep_research.web_search_engines.engines.search_engine_wayback.WaybackSearchEngine._get_wayback_content",
+ Mock(return_value=("", "")),
)
result = engine.get_latest_snapshot("https://example.com")
@@ -545,11 +543,10 @@ class TestWaybackDateRangeSearch:
Mock(return_value=mock_response),
)
- # Mock snippets only mode
+ # Mock _get_wayback_content to avoid network calls
monkeypatch.setattr(
- "local_deep_research.web_search_engines.engines.search_engine_wayback.search_config.SEARCH_SNIPPETS_ONLY",
- True,
- raising=False,
+ "local_deep_research.web_search_engines.engines.search_engine_wayback.WaybackSearchEngine._get_wayback_content",
+ Mock(return_value=("", "")),
)
results = engine.search_by_date_range(
@@ -603,35 +600,8 @@ class TestWaybackFullContent:
return WaybackSearchEngine()
- def test_get_full_content_snippets_only_mode(self, engine, monkeypatch):
- """Test full content retrieval in snippets only mode."""
- monkeypatch.setattr(
- "local_deep_research.web_search_engines.engines.search_engine_wayback.search_config.SEARCH_SNIPPETS_ONLY",
- True,
- raising=False,
- )
-
- items = [
- {
- "title": "Test Archive",
- "link": "https://web.archive.org/web/20240115/https://example.com",
- "snippet": "Test snippet",
- }
- ]
-
- results = engine._get_full_content(items)
-
- # Should return items as-is in snippets only mode
- assert results == items
-
def test_get_full_content_with_retrieval(self, engine, monkeypatch):
"""Test full content retrieval with actual fetching."""
- # Ensure snippets only is False
- monkeypatch.setattr(
- "local_deep_research.web_search_engines.engines.search_engine_wayback.search_config",
- Mock(spec=[]), # Empty spec means hasattr returns False
- )
-
mock_response = Mock()
mock_response.text = "<html><body><p>Full content</p></body></html>"
diff --git a/tests/web_search_engines/engines/test_arxiv_coverage.py b/tests/web_search_engines/engines/test_arxiv_coverage.py
index edf096a8a3..0a4bc7fd95 100644
--- a/tests/web_search_engines/engines/test_arxiv_coverage.py
+++ b/tests/web_search_engines/engines/test_arxiv_coverage.py
@@ -361,16 +361,6 @@ class TestGetPreviews:
class TestGetFullContent:
- def test_snippets_only_mode(self, engine):
- """SEARCH_SNIPPETS_ONLY returns items unchanged."""
- items = [{"id": "1", "title": "T"}]
- with patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"
- ) as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = True
- result = engine._get_full_content(items)
- assert result == items
-
def test_no_paper_in_cache(self, engine):
"""Item not in _papers cache is returned as-is."""
engine._papers = {}
@@ -431,11 +421,7 @@ class TestGetFullContent:
with (
patch("builtins.open", mock_open()),
patch.dict("sys.modules", {"pypdf": MagicMock()}),
- patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"
- ) as mock_cfg,
):
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
# We need to mock pypdf inside the method
import sys
@@ -465,11 +451,7 @@ class TestGetFullContent:
with (
patch("builtins.open", mock_open()),
- patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"
- ) as mock_cfg,
):
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
import sys
mock_pypdf2 = MagicMock()
@@ -498,11 +480,7 @@ class TestGetFullContent:
with (
patch("builtins.open", mock_open()),
- patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"
- ) as mock_cfg,
):
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
import sys
# pypdf fails with ImportError
@@ -529,11 +507,7 @@ class TestGetFullContent:
with (
patch("builtins.open", mock_open()),
- patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"
- ) as mock_cfg,
):
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
import sys
mock_pypdf2 = MagicMock()
@@ -559,12 +533,8 @@ class TestGetFullContent:
engine_with_pdf._papers = {paper.entry_id: paper}
items = [{"id": paper.entry_id, "title": "T"}]
- with patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"
- ) as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
- result = engine_with_pdf._get_full_content(items)
- assert result[0]["pdf_path"] is None
+ result = engine_with_pdf._get_full_content(items)
+ assert result[0]["pdf_path"] is None
def test_pdf_limit_reached(self, engine_with_pdf):
"""Once max_full_text PDFs processed, remaining use summary."""
@@ -587,11 +557,7 @@ class TestGetFullContent:
with (
patch("builtins.open", mock_open()),
- patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"
- ) as mock_cfg,
):
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
import sys
mock_pypdf2 = MagicMock()
@@ -638,18 +604,14 @@ class TestRun:
class TestGetPaperDetails:
def test_paper_found_full_mode(self, engine):
- """Paper found with full content (not snippet-only)."""
+ """Paper found with full content."""
import arxiv
paper = _make_mock_paper()
with (
patch.object(arxiv, "Client") as mock_client_cls,
patch.object(arxiv, "Search"),
- patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"
- ) as mock_cfg,
):
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
mock_client = Mock()
mock_client.results.return_value = [paper]
mock_client_cls.return_value = mock_client
@@ -682,19 +644,15 @@ class TestGetPaperDetails:
result = engine.get_paper_details("2101.00001")
assert result == {}
- def test_paper_snippet_only_mode(self, engine):
- """Snippet-only mode returns basic info without full content."""
+ def test_paper_long_summary_snippet_truncated(self, engine):
+ """Long summary gets truncated snippet with ellipsis."""
import arxiv
paper = _make_mock_paper(summary="A" * 300)
with (
patch.object(arxiv, "Client") as mock_client_cls,
patch.object(arxiv, "Search"),
- patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"
- ) as mock_cfg,
):
- mock_cfg.SEARCH_SNIPPETS_ONLY = True
mock_client = Mock()
mock_client.results.return_value = [paper]
mock_client_cls.return_value = mock_client
@@ -702,7 +660,6 @@ class TestGetPaperDetails:
result = engine.get_paper_details("2101.00001")
assert result["title"] == "Test Paper"
assert result["snippet"].endswith("...")
- assert "content" not in result
def test_paper_details_with_pdf_download(self, engine_with_pdf):
"""PDF download happens in get_paper_details when configured."""
@@ -712,11 +669,7 @@ class TestGetPaperDetails:
with (
patch.object(arxiv, "Client") as mock_client_cls,
patch.object(arxiv, "Search"),
- patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"
- ) as mock_cfg,
):
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
mock_client = Mock()
mock_client.results.return_value = [paper]
mock_client_cls.return_value = mock_client
@@ -734,11 +687,7 @@ class TestGetPaperDetails:
with (
patch.object(arxiv, "Client") as mock_client_cls,
patch.object(arxiv, "Search"),
- patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"
- ) as mock_cfg,
):
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
mock_client = Mock()
mock_client.results.return_value = [paper]
mock_client_cls.return_value = mock_client
@@ -755,11 +704,7 @@ class TestGetPaperDetails:
with (
patch.object(arxiv, "Client") as mock_client_cls,
patch.object(arxiv, "Search"),
- patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"
- ) as mock_cfg,
):
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
mock_client = Mock()
mock_client.results.return_value = [paper]
mock_client_cls.return_value = mock_client
@@ -776,11 +721,7 @@ class TestGetPaperDetails:
with (
patch.object(arxiv, "Client") as mock_client_cls,
patch.object(arxiv, "Search"),
- patch(
- "local_deep_research.web_search_engines.engines.search_engine_arxiv.search_config"
- ) as mock_cfg,
):
- mock_cfg.SEARCH_SNIPPETS_ONLY = True
mock_client = Mock()
mock_client.results.return_value = [paper]
mock_client_cls.return_value = mock_client
diff --git a/tests/web_search_engines/engines/test_search_engine_arxiv.py b/tests/web_search_engines/engines/test_search_engine_arxiv.py
index 822a556d77..0002a4de19 100644
--- a/tests/web_search_engines/engines/test_search_engine_arxiv.py
+++ b/tests/web_search_engines/engines/test_search_engine_arxiv.py
@@ -463,32 +463,6 @@ class TestGetFullContent:
assert len(results) == 1
assert results[0]["title"] == "Test Paper"
- def test_get_full_content_snippets_only_mode(self):
- """Get full content in snippets only mode."""
- from local_deep_research.web_search_engines.engines.search_engine_arxiv import (
- ArXivSearchEngine,
- )
- from local_deep_research.web_search_engines.engines import (
- search_engine_arxiv,
- )
-
- with patch(
- "local_deep_research.advanced_search_system.filters.journal_reputation_filter.JournalReputationFilter.create_default",
- return_value=None,
- ):
- engine = ArXivSearchEngine()
-
- with patch.object(
- search_engine_arxiv.search_config,
- "SEARCH_SNIPPETS_ONLY",
- True,
- create=True,
- ):
- items = [{"id": "paper-id", "title": "Test Paper"}]
- results = engine._get_full_content(items)
-
- assert results == items
-
class TestRun:
"""Tests for run method."""
diff --git a/tests/web_search_engines/engines/test_search_engine_elasticsearch.py b/tests/web_search_engines/engines/test_search_engine_elasticsearch.py
index 539e92ba08..bfd90a4f0a 100644
--- a/tests/web_search_engines/engines/test_search_engine_elasticsearch.py
+++ b/tests/web_search_engines/engines/test_search_engine_elasticsearch.py
@@ -602,34 +602,6 @@ class TestGetFullContent:
# Should still have the original data
assert results[0]["title"] == "Test"
- def test_get_full_content_snippets_only_mode(self):
- """Get full content respects snippets-only mode."""
- from local_deep_research.web_search_engines.engines.search_engine_elasticsearch import (
- ElasticsearchSearchEngine,
- )
-
- mock_client = MagicMock()
- mock_client.info.return_value = {
- "cluster_name": "test",
- "version": {"number": "8.0"},
- }
-
- with patch(
- "local_deep_research.web_search_engines.engines.search_engine_elasticsearch.Elasticsearch",
- return_value=mock_client,
- ):
- with patch(
- "local_deep_research.web_search_engines.engines.search_engine_elasticsearch.search_config"
- ) as mock_config:
- mock_config.SEARCH_SNIPPETS_ONLY = True
- engine = ElasticsearchSearchEngine()
- items = [{"id": "doc1", "title": "Test", "snippet": "Snippet"}]
- results = engine._get_full_content(items)
-
- # Should return items unchanged
- assert results == items
- mock_client.get.assert_not_called()
-
class TestSearchByQueryString:
"""Tests for search_by_query_string method."""
diff --git a/tests/web_search_engines/engines/test_search_engine_github_coverage.py b/tests/web_search_engines/engines/test_search_engine_github_coverage.py
index 55c17b3bd5..2876bef7f9 100644
--- a/tests/web_search_engines/engines/test_search_engine_github_coverage.py
+++ b/tests/web_search_engines/engines/test_search_engine_github_coverage.py
@@ -315,17 +315,6 @@ class TestGetPreviewsCoverage:
class TestGetFullContentCoverage:
"""Edge cases for _get_full_content."""
- def test_snippets_only_mode(self):
- """When SEARCH_SNIPPETS_ONLY is True, return items without fetching content."""
- engine = _make_engine()
- items = [
- {"id": "1", "search_type": "repository", "repo_full_name": "o/r"}
- ]
- with patch(f"{MODULE}.search_config") as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = True
- result = engine._get_full_content(items)
- assert result == items
-
def test_issue_content_uses_snippet(self):
"""Issue search type sets full_content to snippet."""
engine = _make_engine()
@@ -336,9 +325,7 @@ class TestGetFullContentCoverage:
"snippet": "Bug description here",
}
]
- with patch(f"{MODULE}.search_config") as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
- result = engine._get_full_content(items)
+ result = engine._get_full_content(items)
assert result[0]["full_content"] == "Bug description here"
assert result[0]["content_type"] == "issue"
@@ -357,9 +344,7 @@ class TestGetFullContentCoverage:
"public_repos": 20,
}
]
- with patch(f"{MODULE}.search_config") as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
- result = engine._get_full_content(items)
+ result = engine._get_full_content(items)
assert "Bio: Full-stack developer" in result[0]["full_content"]
assert "Name: Dev Person" in result[0]["full_content"]
assert "Location: NYC" in result[0]["full_content"]
@@ -379,20 +364,16 @@ class TestGetFullContentCoverage:
"public_repos": 0,
}
]
- with patch(f"{MODULE}.search_config") as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
- result = engine._get_full_content(items)
+ result = engine._get_full_content(items)
assert "Bio:" not in result[0]["full_content"]
def test_repository_missing_full_name(self):
"""Repository with empty repo_full_name skips readme fetch."""
engine = _make_engine(include_readme=True)
items = [{"id": "1", "search_type": "repository", "repo_full_name": ""}]
- with patch(f"{MODULE}.search_config") as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
- with patch.object(engine, "_get_readme_content") as mock_readme:
- result = engine._get_full_content(items)
- mock_readme.assert_not_called()
+ with patch.object(engine, "_get_readme_content") as mock_readme:
+ result = engine._get_full_content(items)
+ mock_readme.assert_not_called()
assert "full_content" not in result[0]
@@ -618,7 +599,7 @@ class TestSearchCodeCoverage:
assert engine.search_type == "repositories"
def test_full_content_retrieval(self):
- """search_code calls _get_full_content when SEARCH_SNIPPETS_ONLY is False."""
+ """search_code calls _get_full_content."""
engine = _make_engine()
mock_results = [
{
@@ -634,15 +615,13 @@ class TestSearchCodeCoverage:
}
]
with patch.object(engine, "_search_github", return_value=mock_results):
- with patch(f"{MODULE}.search_config") as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
- with patch.object(
- engine,
- "_get_full_content",
- return_value=[{"full_content": "code"}],
- ) as mock_fc:
- engine.search_code("test")
- mock_fc.assert_called_once()
+ with patch.object(
+ engine,
+ "_get_full_content",
+ return_value=[{"full_content": "code"}],
+ ) as mock_fc:
+ engine.search_code("test")
+ mock_fc.assert_called_once()
# ---------------------------------------------------------------------------
diff --git a/tests/web_search_engines/engines/test_search_engine_github_deep_coverage.py b/tests/web_search_engines/engines/test_search_engine_github_deep_coverage.py
index 26e61fab67..6dc6d57350 100644
--- a/tests/web_search_engines/engines/test_search_engine_github_deep_coverage.py
+++ b/tests/web_search_engines/engines/test_search_engine_github_deep_coverage.py
@@ -403,11 +403,9 @@ class TestGetFullContentDeep:
"repo_full_name": "owner/repo",
}
]
- with patch(f"{MODULE}.search_config") as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
- with patch.object(engine, "_get_readme_content") as mock_readme:
- result = engine._get_full_content(items)
- mock_readme.assert_not_called()
+ with patch.object(engine, "_get_readme_content") as mock_readme:
+ result = engine._get_full_content(items)
+ mock_readme.assert_not_called()
# No full_content key added
assert "full_content" not in result[0]
@@ -421,15 +419,13 @@ class TestGetFullContentDeep:
"file_url": "https://api.github.com/repos/o/r/contents/f.py",
}
]
- with patch(f"{MODULE}.search_config") as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
- with patch.object(
- engine, "_get_file_content", return_value="file body"
- ) as mock_fc:
- result = engine._get_full_content(items)
- mock_fc.assert_called_once_with(
- "https://api.github.com/repos/o/r/contents/f.py"
- )
+ with patch.object(
+ engine, "_get_file_content", return_value="file body"
+ ) as mock_fc:
+ result = engine._get_full_content(items)
+ mock_fc.assert_called_once_with(
+ "https://api.github.com/repos/o/r/contents/f.py"
+ )
assert result[0]["full_content"] == "file body"
assert result[0]["content_type"] == "file"
@@ -437,11 +433,9 @@ class TestGetFullContentDeep:
"""Code item without file_url does not attempt fetch."""
engine = _make_engine()
items = [{"id": "code_abc", "search_type": "code", "file_url": ""}]
- with patch(f"{MODULE}.search_config") as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
- with patch.object(engine, "_get_file_content") as mock_fc:
- engine._get_full_content(items)
- mock_fc.assert_not_called()
+ with patch.object(engine, "_get_file_content") as mock_fc:
+ engine._get_full_content(items)
+ mock_fc.assert_not_called()
def test_repository_with_issues_fetches_issues(self):
"""Repository item with include_issues=True fetches issues."""
@@ -453,16 +447,14 @@ class TestGetFullContentDeep:
"repo_full_name": "owner/repo",
}
]
- with patch(f"{MODULE}.search_config") as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
+ with patch.object(
+ engine, "_get_readme_content", return_value="# README"
+ ):
with patch.object(
- engine, "_get_readme_content", return_value="# README"
- ):
- with patch.object(
- engine, "_get_recent_issues", return_value=[{"number": 1}]
- ) as mock_iss:
- result = engine._get_full_content(items)
- mock_iss.assert_called_once_with("owner/repo")
+ engine, "_get_recent_issues", return_value=[{"number": 1}]
+ ) as mock_iss:
+ result = engine._get_full_content(items)
+ mock_iss.assert_called_once_with("owner/repo")
assert result[0]["recent_issues"] == [{"number": 1}]
def test_user_with_no_name_or_location_no_extra_lines(self):
@@ -480,9 +472,7 @@ class TestGetFullContentDeep:
"public_repos": 0,
}
]
- with patch(f"{MODULE}.search_config") as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
- result = engine._get_full_content(items)
+ result = engine._get_full_content(items)
assert "Name:" not in result[0]["full_content"]
assert "Location:" not in result[0]["full_content"]
assert result[0]["content_type"] == "user_profile"
@@ -601,29 +591,6 @@ class TestSearchCodeDeep:
call_arg = mock_search.call_args[0][0]
assert call_arg == "import numpy"
- def test_snippets_only_returns_previews_without_full_content(self):
- """When SEARCH_SNIPPETS_ONLY is True, skip _get_full_content."""
- engine = _make_engine()
- mock_result = {
- "sha": "x",
- "name": "f.py",
- "path": "f.py",
- "html_url": "https://github.com/o/r/blob/main/f.py",
- "url": "https://api.github.com/repos/o/r/contents/f.py",
- "repository": {
- "full_name": "o/r",
- "html_url": "https://github.com/o/r",
- },
- }
- with patch.object(engine, "_search_github", return_value=[mock_result]):
- with patch(f"{MODULE}.search_config") as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = True
- with patch.object(engine, "_get_full_content") as mock_fc:
- results = engine.search_code("test")
- mock_fc.assert_not_called()
- assert len(results) == 1
- assert results[0]["search_type"] == "code"
-
# ---------------------------------------------------------------------------
# search_issues deep paths
diff --git a/tests/web_search_engines/engines/test_search_engine_library_coverage.py b/tests/web_search_engines/engines/test_search_engine_library_coverage.py
index c550b067ca..bf8358b32d 100644
--- a/tests/web_search_engines/engines/test_search_engine_library_coverage.py
+++ b/tests/web_search_engines/engines/test_search_engine_library_coverage.py
@@ -5,7 +5,7 @@ Covers:
- __init__: initialization with/without username
- search: no username, no collections, exception handling
- _get_previews: delegates to search
-- _get_full_content: snippet-only mode, no username, no doc_id, success path
+- _get_full_content: no username, no doc_id, success path
- close: no-op
"""
@@ -233,32 +233,12 @@ class TestGetPreviews:
class TestGetFullContent:
- def _patch_search_config(self, snippet_only=False):
- """Patch the search_config module that gets imported inside _get_full_content."""
- mock_config = MagicMock()
- mock_config.SEARCH_SNIPPETS_ONLY = snippet_only
- return patch.dict(
- "sys.modules",
- {"local_deep_research.config.search_config": mock_config},
- )
-
- def test_snippet_only_mode_returns_items(self):
- """When SEARCH_SNIPPETS_ONLY is True, returns items unchanged."""
+ def test_items_without_document_id_returned_unchanged(self):
+ """Items lacking metadata.document_id are skipped and returned as-is."""
engine = _make_engine()
items = [{"title": "test", "snippet": "content"}]
- from local_deep_research.config import search_config
-
- original = getattr(search_config, "SEARCH_SNIPPETS_ONLY", None)
- search_config.SEARCH_SNIPPETS_ONLY = True
- try:
- result = engine._get_full_content(items)
- finally:
- if original is None:
- if hasattr(search_config, "SEARCH_SNIPPETS_ONLY"):
- delattr(search_config, "SEARCH_SNIPPETS_ONLY")
- else:
- search_config.SEARCH_SNIPPETS_ONLY = original
+ result = engine._get_full_content(items)
assert result == items
diff --git a/tests/web_search_engines/engines/test_search_engine_mojeek.py b/tests/web_search_engines/engines/test_search_engine_mojeek.py
index 8a164067c8..324ffb9113 100644
--- a/tests/web_search_engines/engines/test_search_engine_mojeek.py
+++ b/tests/web_search_engines/engines/test_search_engine_mojeek.py
@@ -548,31 +548,6 @@ class TestGetPreviews:
class TestGetFullContent:
"""Tests for _get_full_content method."""
- def test_snippets_only_mode(self):
- """Test snippet-only mode returns items unchanged."""
- from local_deep_research.web_search_engines.engines import (
- search_engine_mojeek,
- )
- from local_deep_research.web_search_engines.engines.search_engine_mojeek import (
- MojeekSearchEngine,
- )
-
- engine = MojeekSearchEngine(
- api_key="test-key", include_full_content=False
- )
-
- items = [{"link": "https://example.com", "snippet": "test"}]
-
- with patch.object(
- search_engine_mojeek.search_config,
- "SEARCH_SNIPPETS_ONLY",
- True,
- create=True,
- ):
- results = engine._get_full_content(items)
-
- assert results == items
-
def test_with_full_search_attr(self):
"""Test delegation to full_search._get_full_content."""
from local_deep_research.web_search_engines.engines.search_engine_mojeek import (
diff --git a/tests/web_search_engines/engines/test_search_engine_pubmed_coverage.py b/tests/web_search_engines/engines/test_search_engine_pubmed_coverage.py
index 03b951a57f..4facd23b16 100644
--- a/tests/web_search_engines/engines/test_search_engine_pubmed_coverage.py
+++ b/tests/web_search_engines/engines/test_search_engine_pubmed_coverage.py
@@ -423,24 +423,26 @@ class TestGetFullContentPMCIntegration:
class TestGetFullContentSnippetsOnlyMode:
def test_snippets_only_mode_uses_abstract(self):
- engine = _make_engine(get_full_text=True, get_abstracts=True)
+ # Snippets-only behavior is now driven by get_full_text=False; the
+ # dead module-level SEARCH_SNIPPETS_ONLY switch this test used to
+ # patch was removed in this PR. With full text disabled, no PMC
+ # lookup happens and the content is the abstract.
+ engine = _make_engine(get_full_text=False, get_abstracts=True)
items = [_make_preview_item(pmid="10003")]
- with patch(f"{MODULE}.search_config") as mock_config:
- mock_config.SEARCH_SNIPPETS_ONLY = True
+ with patch.object(
+ engine,
+ "_get_article_abstracts",
+ return_value={"10003": "Snippet abstract"},
+ ):
with patch.object(
- engine,
- "_get_article_abstracts",
- return_value={"10003": "Snippet abstract"},
+ engine, "_get_article_detailed_metadata", return_value={}
):
with patch.object(
- engine, "_get_article_detailed_metadata", return_value={}
- ):
- with patch.object(
- engine, "_find_pmc_ids", return_value={}
- ) as mock_pmc:
- results = engine._get_full_content(items)
- assert results[0]["content_type"] == "abstract"
- mock_pmc.assert_not_called()
+ engine, "_find_pmc_ids", return_value={}
+ ) as mock_pmc:
+ results = engine._get_full_content(items)
+ assert results[0]["content_type"] == "abstract"
+ mock_pmc.assert_not_called()
class TestGetFullContentPMCAvailability:
diff --git a/tests/web_search_engines/engines/test_search_engine_searxng_coverage.py b/tests/web_search_engines/engines/test_search_engine_searxng_coverage.py
index f8befd5f84..d1349ecc88 100644
--- a/tests/web_search_engines/engines/test_search_engine_searxng_coverage.py
+++ b/tests/web_search_engines/engines/test_search_engine_searxng_coverage.py
@@ -347,25 +347,6 @@ class TestGetSearchResultsURLFromTitleHref:
assert results[0]["url"] == "https://example.com/padded"
-# ---------------------------------------------------------------------------
-# _get_full_content – snippets-only mode
-# ---------------------------------------------------------------------------
-
-
-class TestGetFullContentSnippetsOnly:
- """_get_full_content returns items unchanged in snippets-only mode."""
-
- def test_snippets_only_skips_full_content(self):
- engine = _make_engine()
- items = [{"title": "T", "link": "https://example.com", "snippet": "S"}]
-
- with patch(f"{MODULE}.search_config") as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = True
- result = engine._get_full_content(items)
-
- assert result is items
-
-
# ---------------------------------------------------------------------------
# _get_full_content – exception handling
# ---------------------------------------------------------------------------
@@ -378,15 +359,11 @@ class TestGetFullContentException:
engine = _make_engine()
items = [{"title": "T", "link": "https://example.com", "snippet": "S"}]
- with patch(f"{MODULE}.search_config") as mock_cfg:
- mock_cfg.SEARCH_SNIPPETS_ONLY = False
- # Make full_search._get_full_content raise
- engine.full_search = Mock()
- engine.full_search._get_full_content.side_effect = RuntimeError(
- "boom"
- )
+ # Make full_search._get_full_content raise
+ engine.full_search = Mock()
+ engine.full_search._get_full_content.side_effect = RuntimeError("boom")
- result = engine._get_full_content(items)
+ result = engine._get_full_content(items)
assert result is items
diff --git a/tests/web_search_engines/engines/test_search_engine_wayback_coverage.py b/tests/web_search_engines/engines/test_search_engine_wayback_coverage.py
index fcc13306d2..8f67d5e470 100644
--- a/tests/web_search_engines/engines/test_search_engine_wayback_coverage.py
+++ b/tests/web_search_engines/engines/test_search_engine_wayback_coverage.py
@@ -206,37 +206,20 @@ class TestGetWaybackContentExtra:
class TestGetFullContentExtra:
- def test_snippet_only_mode_skips_fetch(self):
- engine = _engine()
- items = [
- {
- "title": "T",
- "link": "https://web.archive.org/web/x/https://t.com",
- }
- ]
- with patch(
- f"{MODULE}.search_config",
- SEARCH_SNIPPETS_ONLY=True,
- ):
- result = engine._get_full_content(items)
- assert result == items
-
def test_item_without_link_appended_as_is(self):
engine = _engine()
items = [{"title": "No link item"}] # no 'link' key
- with patch(f"{MODULE}.search_config", SEARCH_SNIPPETS_ONLY=False):
- result = engine._get_full_content(items)
+ result = engine._get_full_content(items)
assert result[0]["title"] == "No link item"
assert "raw_html" not in result[0]
def test_exception_in_loop_appends_item_unchanged(self):
engine = _engine()
items = [{"title": "X", "link": "https://web.archive.org/web/x"}]
- with patch(f"{MODULE}.search_config", SEARCH_SNIPPETS_ONLY=False):
- with patch.object(
- engine, "_get_wayback_content", side_effect=OSError("bad")
- ):
- result = engine._get_full_content(items)
+ with patch.object(
+ engine, "_get_wayback_content", side_effect=OSError("bad")
+ ):
+ result = engine._get_full_content(items)
assert result[0] == items[0]
@@ -246,24 +229,6 @@ class TestGetFullContentExtra:
class TestSearchByUrlExtra:
- def test_snippet_only_mode_returns_previews_without_full_content(self):
- engine = _engine()
- snapshots = [
- {
- "timestamp": "20240101000000",
- "formatted_date": "2024-01-01 00:00:00",
- "url": "https://web.archive.org/web/20240101/https://example.com",
- "original_url": "https://example.com",
- }
- ]
- with patch.object(
- engine, "_get_wayback_snapshots", return_value=snapshots
- ):
- with patch(f"{MODULE}.search_config", SEARCH_SNIPPETS_ONLY=True):
- result = engine.search_by_url("https://example.com")
- assert len(result) == 1
- assert "raw_html" not in result[0]
-
def test_max_snapshots_parameter_honoured(self):
engine = _engine(max_snapshots_per_url=10)
snapshots = [
@@ -278,7 +243,9 @@ class TestSearchByUrlExtra:
with patch.object(
engine, "_get_wayback_snapshots", return_value=snapshots
):
- with patch(f"{MODULE}.search_config", SEARCH_SNIPPETS_ONLY=True):
+ with patch.object(
+ engine, "_get_wayback_content", return_value=("", "")
+ ):
result = engine.search_by_url(
"https://example.com", max_snapshots=2
)
@@ -299,7 +266,9 @@ class TestSearchByDateRange:
resp = _mock_response(status=200, json_body=data)
engine = _engine()
with patch(f"{MODULE}.safe_get", return_value=resp):
- with patch(f"{MODULE}.search_config", SEARCH_SNIPPETS_ONLY=True):
+ with patch.object(
+ engine, "_get_wayback_content", return_value=("", "")
+ ):
result = engine.search_by_date_range(
"https://example.com", "20230101", "20231231"
)
@@ -336,28 +305,3 @@ class TestGetLatestSnapshotExtra:
with patch(f"{MODULE}.safe_get", side_effect=OSError("fail")):
result = engine.get_latest_snapshot("https://example.com")
assert result is None
-
- def test_snippets_only_mode_skips_content_fetch(self):
- """In snippet-only mode get_latest_snapshot must NOT call _get_wayback_content."""
- engine = _engine()
- resp = _mock_response(
- status=200,
- json_body={
- "archived_snapshots": {
- "closest": {
- "url": "https://web.archive.org/web/20240101/https://example.com",
- "timestamp": "20240101000000",
- }
- }
- },
- )
- with patch(f"{MODULE}.safe_get", return_value=resp):
- with patch(f"{MODULE}.search_config", SEARCH_SNIPPETS_ONLY=True):
- with patch.object(
- engine,
- "_get_wayback_content",
- side_effect=AssertionError("must not be called"),
- ):
- result = engine.get_latest_snapshot("https://example.com")
- assert result is not None
- assert result["timestamp"] == "20240101000000"