Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

feat(search): add TinyFish search engine (#4057)

Pranav Janakiramanf9be28fbded4
diff --git a/changelog.d/4057.feature.md b/changelog.d/4057.feature.md
new file mode 100644
index 0000000000..23b085b519
--- /dev/null
+++ b/changelog.d/4057.feature.md
@@ -0,0 +1 @@
+Add TinyFish as an optional hosted search engine with structured search results and browser-rendered content extraction.
diff --git a/src/local_deep_research/defaults/settings/search_engines/tinyfish.json b/src/local_deep_research/defaults/settings/search_engines/tinyfish.json
new file mode 100644
index 0000000000..d5fb1e690c
--- /dev/null
+++ b/src/local_deep_research/defaults/settings/search_engines/tinyfish.json
@@ -0,0 +1,211 @@
+{
+ "search.engine.web.tinyfish.display_name": {
+ "category": "tinyfish",
+ "description": "Display name to use in the U.I. for this search engine.",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Display Name",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "text",
+ "value": "TinyFish",
+ "visible": false
+ },
+ "search.engine.web.tinyfish.description": {
+ "category": "tinyfish",
+ "description": "Human-readable description of the search engine.",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Description",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "text",
+ "value": "Fast hosted web search with optional browser-rendered content extraction.",
+ "visible": false
+ },
+ "search.engine.web.tinyfish.api_key": {
+ "category": "tinyfish",
+ "description": "The TinyFish API key to use. Can also be supplied with LDR_SEARCH_ENGINE_WEB_TINYFISH_API_KEY.",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Api Key",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "password",
+ "value": "",
+ "visible": true
+ },
+ "search.engine.web.tinyfish.default_params.location": {
+ "category": "tinyfish",
+ "description": "Country code for geo-targeted results.",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Location",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "text",
+ "value": "US",
+ "visible": true
+ },
+ "search.engine.web.tinyfish.default_params.language": {
+ "category": "tinyfish",
+ "description": "Language code for search results.",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Language",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "text",
+ "value": "en",
+ "visible": true
+ },
+ "search.engine.web.tinyfish.default_params.include_full_content": {
+ "category": "tinyfish",
+ "description": "Fetch browser-rendered page content for search results.",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Include Full Content",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "checkbox",
+ "value": true,
+ "visible": true
+ },
+ "search.engine.web.tinyfish.default_params.fetch_format": {
+ "category": "tinyfish",
+ "description": "Output format for fetched page content.",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Fetch Format",
+ "options": [
+ "markdown",
+ "html",
+ "json"
+ ],
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "select",
+ "value": "markdown",
+ "visible": true
+ },
+ "search.engine.web.tinyfish.requires_api_key": {
+ "category": "tinyfish",
+ "description": "Whether this search engine requires an API key.",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Requires Api Key",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "checkbox",
+ "value": true,
+ "visible": false
+ },
+ "search.engine.web.tinyfish.reliability": {
+ "category": "tinyfish",
+ "description": "Reliability score (0-1) for TinyFish search.",
+ "editable": true,
+ "max_value": 1.0,
+ "min_value": 0.0,
+ "name": "Reliability",
+ "options": null,
+ "step": 0.05,
+ "type": "SEARCH",
+ "ui_element": "range",
+ "value": 0.85,
+ "visible": true
+ },
+ "search.engine.web.tinyfish.requires_llm": {
+ "category": "tinyfish",
+ "description": "Whether this search engine requires an LLM.",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Requires Llm",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "checkbox",
+ "value": false,
+ "visible": false
+ },
+ "search.engine.web.tinyfish.supports_full_search": {
+ "category": "tinyfish",
+ "description": "Whether this search engine can return extracted page content.",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Supports Full Search",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "checkbox",
+ "value": true,
+ "visible": false
+ },
+ "search.engine.web.tinyfish.strengths": {
+ "category": "tinyfish",
+ "description": "Strengths of this search engine.",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Strengths",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "json",
+ "value": [
+ "fast structured web search",
+ "browser-rendered content extraction",
+ "clean markdown output for LLM consumption",
+ "batch fetches multiple URLs"
+ ],
+ "visible": true
+ },
+ "search.engine.web.tinyfish.weaknesses": {
+ "category": "tinyfish",
+ "description": "Weaknesses of this search engine.",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Weaknesses",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "json",
+ "value": [
+ "requires hosted API key",
+ "external network requests leave the local environment",
+ "fetch latency depends on target page rendering"
+ ],
+ "visible": true
+ },
+ "search.engine.web.tinyfish.use_in_auto_search": {
+ "category": "tinyfish",
+ "description": "Include TinyFish in auto search mode",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Include in Auto Search",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "checkbox",
+ "value": false,
+ "visible": true
+ }
+}
diff --git a/src/local_deep_research/security/module_whitelist.py b/src/local_deep_research/security/module_whitelist.py
index 22add19ea3..efd868c300 100644
--- a/src/local_deep_research/security/module_whitelist.py
+++ b/src/local_deep_research/security/module_whitelist.py
@@ -51,6 +51,7 @@ ALLOWED_MODULE_PATHS: frozenset[str] = frozenset(
".engines.search_engine_serper",
".engines.search_engine_serpapi",
".engines.search_engine_tavily",
+ ".engines.search_engine_tinyfish",
".engines.search_engine_gutenberg",
".engines.search_engine_openlibrary",
".engines.search_engine_pubchem",
@@ -100,6 +101,7 @@ ALLOWED_CLASS_NAMES: frozenset[str] = frozenset(
"SerperSearchEngine",
"StackExchangeSearchEngine",
"TavilySearchEngine",
+ "TinyFishSearchEngine",
"WaybackSearchEngine",
"WikinewsSearchEngine",
"WikipediaSearchEngine",
diff --git a/src/local_deep_research/web_search_engines/engine_registry.py b/src/local_deep_research/web_search_engines/engine_registry.py
index 4aae627a8a..775f416073 100644
--- a/src/local_deep_research/web_search_engines/engine_registry.py
+++ b/src/local_deep_research/web_search_engines/engine_registry.py
@@ -76,6 +76,10 @@ ENGINE_REGISTRY: Dict[str, EngineEntry] = {
module_path=".engines.search_engine_tavily",
class_name="TavilySearchEngine",
),
+ "tinyfish": EngineEntry(
+ module_path=".engines.search_engine_tinyfish",
+ class_name="TinyFishSearchEngine",
+ ),
"wayback": EngineEntry(
module_path=".engines.search_engine_wayback",
class_name="WaybackSearchEngine",
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_tinyfish.py b/src/local_deep_research/web_search_engines/engines/search_engine_tinyfish.py
new file mode 100644
index 0000000000..affadcfefa
--- /dev/null
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_tinyfish.py
@@ -0,0 +1,247 @@
+from typing import Any, Dict, List, Optional
+from urllib.parse import urlparse
+
+import requests
+from langchain_core.language_models import BaseLLM
+from loguru import logger
+
+from ...security import safe_get, safe_post
+from ..rate_limiting import RateLimitError
+from ..search_engine_base import BaseSearchEngine
+
+
+class TinyFishSearchEngine(BaseSearchEngine):
+ """TinyFish Search + Fetch API search engine implementation."""
+
+ is_public = True
+ is_generic = True
+
+ SEARCH_URL = "https://api.search.tinyfish.ai"
+ FETCH_URL = "https://api.fetch.tinyfish.ai"
+ SEARCH_TIMEOUT = 10
+ FETCH_TIMEOUT = 150
+ MAX_QUERY_LEN = 400
+ MAX_FETCH_URLS = 10
+
+ def __init__(
+ self,
+ max_results: int = 10,
+ location: str = "US",
+ language: str = "en",
+ api_key: Optional[str] = None,
+ llm: Optional[BaseLLM] = None,
+ include_full_content: bool = True,
+ fetch_format: str = "markdown",
+ max_filtered_results: Optional[int] = None,
+ search_snippets_only: Optional[bool] = None,
+ settings_snapshot: Optional[Dict[str, Any]] = None,
+ **kwargs,
+ ):
+ """
+ Initialize the TinyFish search engine.
+
+ Args:
+ max_results: Maximum number of search results
+ location: Country code for geo-targeted search results
+ language: Language code for search results
+ api_key: TinyFish API key
+ llm: Language model for relevance filtering
+ include_full_content: Whether to fetch extracted page content
+ fetch_format: TinyFish Fetch output format: markdown, html, or json
+ max_filtered_results: Maximum results to keep after filtering
+ search_snippets_only: Whether to return only search snippets
+ settings_snapshot: Settings snapshot for thread context
+ **kwargs: Additional parameters ignored for compatibility
+ """
+ if search_snippets_only is None:
+ search_snippets_only = not include_full_content
+
+ super().__init__(
+ llm=llm,
+ max_filtered_results=max_filtered_results,
+ max_results=max_results,
+ include_full_content=include_full_content,
+ search_snippets_only=search_snippets_only,
+ settings_snapshot=settings_snapshot,
+ )
+ self.location = self._normalize_location(location)
+ self.language = self._normalize_language(language)
+ self.fetch_format = fetch_format
+ self.api_key = self._resolve_api_key(
+ api_key,
+ "search.engine.web.tinyfish.api_key",
+ engine_name="TinyFish",
+ settings_snapshot=settings_snapshot,
+ )
+
+ @staticmethod
+ def _normalize_location(location: str) -> str:
+ return (location or "US").strip().upper()
+
+ @staticmethod
+ def _normalize_language(language: str) -> str:
+ language_value = (language or "en").strip()
+ language_map = {
+ "english": "en",
+ "spanish": "es",
+ "french": "fr",
+ "german": "de",
+ "italian": "it",
+ "portuguese": "pt",
+ "japanese": "ja",
+ "chinese": "zh",
+ "korean": "ko",
+ }
+ return language_map.get(language_value.lower(), language_value)
+
+ def _headers(self) -> Dict[str, str]:
+ return {"X-API-Key": self.api_key}
+
+ def _raise_request_exception_if_rate_limited(
+ self, error: requests.exceptions.RequestException
+ ) -> None:
+ response = getattr(error, "response", None)
+ status_code = getattr(response, "status_code", None)
+ if status_code is not None:
+ self._raise_if_rate_limit(status_code)
+
+ def _get_previews(self, query: str) -> List[Dict[str, Any]]:
+ """
+ Get preview information from TinyFish Search.
+
+ Args:
+ query: The search query
+
+ Returns:
+ List of preview dictionaries
+ """
+ logger.info("Getting search results from TinyFish")
+
+ try:
+ self._last_wait_time = self.rate_tracker.apply_rate_limit(
+ self.engine_type
+ )
+
+ response = safe_get(
+ self.SEARCH_URL,
+ params={
+ "query": query[: self.MAX_QUERY_LEN],
+ "location": self.location,
+ "language": self.language,
+ },
+ headers=self._headers(),
+ timeout=self.SEARCH_TIMEOUT,
+ )
+ self._raise_if_rate_limit(response.status_code)
+ response.raise_for_status()
+
+ data = response.json()
+ results = data.get("results", [])[: self.max_results]
+
+ previews = []
+ for idx, result in enumerate(results):
+ link = result.get("url", "")
+ display_link = result.get("site_name", "")
+ if not display_link and link:
+ try:
+ display_link = urlparse(link).netloc or ""
+ except Exception:
+ logger.debug(
+ f"Failed to parse URL for display: {link[:50]}"
+ )
+
+ preview = {
+ "id": link or str(idx),
+ "title": result.get("title", ""),
+ "link": link,
+ "snippet": result.get("snippet", ""),
+ "displayed_link": display_link,
+ "position": result.get("position", idx + 1),
+ }
+ preview["_full_result"] = result
+ previews.append(preview)
+
+ self._search_results = previews
+ return previews
+
+ except RateLimitError:
+ raise
+ except requests.exceptions.RequestException as e:
+ logger.exception("Error getting TinyFish search results")
+ self._raise_request_exception_if_rate_limited(e)
+ return []
+ except Exception:
+ logger.exception("Unexpected error getting TinyFish search results")
+ return []
+
+ def _get_full_content(
+ self, relevant_items: List[Dict[str, Any]]
+ ) -> List[Dict[str, Any]]:
+ """
+ Fetch clean page content from TinyFish Fetch for selected results.
+
+ Args:
+ relevant_items: List of relevant preview dictionaries
+
+ Returns:
+ List of result dictionaries enriched with extracted content
+ """
+ results = [
+ {k: v for k, v in item.items() if k != "_full_result"}
+ for item in relevant_items
+ ]
+ if not self.include_full_content:
+ return results
+
+ url_to_result = {
+ result.get("link") or result.get("url"): result
+ for result in results
+ if result.get("link") or result.get("url")
+ }
+ urls = list(url_to_result.keys())[: self.MAX_FETCH_URLS]
+ if not urls:
+ return results
+
+ try:
+ response = safe_post(
+ self.FETCH_URL,
+ json={"urls": urls, "format": self.fetch_format},
+ headers={**self._headers(), "Content-Type": "application/json"},
+ timeout=self.FETCH_TIMEOUT,
+ )
+ self._raise_if_rate_limit(response.status_code)
+ response.raise_for_status()
+
+ data = response.json()
+ for page in data.get("results", []):
+ url = page.get("url") or page.get("final_url")
+ result = url_to_result.get(url)
+ if result is None:
+ result = url_to_result.get(page.get("final_url"))
+ if result is None:
+ continue
+
+ content = page.get("text")
+ if content:
+ result["content"] = content
+ for field in ("final_url", "description", "language"):
+ if page.get(field):
+ result[field] = page[field]
+ if page.get("title") and not result.get("title"):
+ result["title"] = page["title"]
+
+ errors = data.get("errors", [])
+ if errors:
+ logger.info(
+ f"TinyFish Fetch returned {len(errors)} per-URL errors"
+ )
+
+ except RateLimitError:
+ raise
+ except requests.exceptions.RequestException as e:
+ logger.exception("Error fetching TinyFish page content")
+ self._raise_request_exception_if_rate_limited(e)
+ except Exception:
+ logger.exception("Unexpected error fetching TinyFish page content")
+
+ return results
diff --git a/src/local_deep_research/web_search_engines/search_engine_factory.py b/src/local_deep_research/web_search_engines/search_engine_factory.py
index cf2c85fe9f..6958717279 100644
--- a/src/local_deep_research/web_search_engines/search_engine_factory.py
+++ b/src/local_deep_research/web_search_engines/search_engine_factory.py
@@ -660,6 +660,11 @@ def get_search(
if search_tool in ["serpapi", "brave", "google_pse", "wikinews"]:
params["search_language"] = search_language
+ if search_tool == "tinyfish":
+ params["location"] = region.upper()
+ params["language"] = search_language
+ params["search_snippets_only"] = search_snippets_only
+
if search_tool == "wikinews":
params["search_snippets_only"] = search_snippets_only
params["adaptive_search"] = bool(
diff --git a/tests/web_search_engines/engines/test_search_engine_tinyfish.py b/tests/web_search_engines/engines/test_search_engine_tinyfish.py
new file mode 100644
index 0000000000..44c6f07bc9
--- /dev/null
+++ b/tests/web_search_engines/engines/test_search_engine_tinyfish.py
@@ -0,0 +1,345 @@
+from unittest.mock import Mock, patch
+
+import pytest
+
+
+class TestTinyFishSearchEngineInit:
+ """Tests for TinyFishSearchEngine initialization."""
+
+ def test_init_with_api_key(self):
+ from local_deep_research.web_search_engines.engines.search_engine_tinyfish import (
+ TinyFishSearchEngine,
+ )
+
+ engine = TinyFishSearchEngine(api_key="test-api-key")
+
+ assert engine.api_key == "test-api-key"
+ assert engine.max_results == 10
+ assert engine.location == "US"
+ assert engine.language == "en"
+ assert engine.fetch_format == "markdown"
+ assert engine.include_full_content is True
+ assert engine.search_snippets_only is False
+
+ def test_init_without_api_key_raises(self):
+ from local_deep_research.web_search_engines.engines.search_engine_tinyfish import (
+ TinyFishSearchEngine,
+ )
+
+ with pytest.raises(ValueError, match="No valid API key found for TinyFish"):
+ TinyFishSearchEngine(settings_snapshot={})
+
+ def test_init_with_api_key_from_settings(self):
+ from local_deep_research.web_search_engines.engines.search_engine_tinyfish import (
+ TinyFishSearchEngine,
+ )
+
+ engine = TinyFishSearchEngine(
+ settings_snapshot={
+ "search.engine.web.tinyfish.api_key": "settings-key"
+ }
+ )
+
+ assert engine.api_key == "settings-key"
+
+ def test_init_with_snippet_only_override(self):
+ from local_deep_research.web_search_engines.engines.search_engine_tinyfish import (
+ TinyFishSearchEngine,
+ )
+
+ engine = TinyFishSearchEngine(
+ api_key="test-key",
+ include_full_content=True,
+ search_snippets_only=True,
+ )
+
+ assert engine.include_full_content is True
+ assert engine.search_snippets_only is True
+
+ def test_init_normalizes_location_and_common_language_name(self):
+ from local_deep_research.web_search_engines.engines.search_engine_tinyfish import (
+ TinyFishSearchEngine,
+ )
+
+ engine = TinyFishSearchEngine(
+ api_key="test-key",
+ location="us",
+ language="English",
+ )
+
+ assert engine.location == "US"
+ assert engine.language == "en"
+
+
+class TestTinyFishGetPreviews:
+ """Tests for TinyFish preview search."""
+
+ def test_get_previews_returns_formatted_results(self):
+ from local_deep_research.web_search_engines.engines.search_engine_tinyfish import (
+ TinyFishSearchEngine,
+ )
+
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {
+ "results": [
+ {
+ "position": 1,
+ "site_name": "example.com",
+ "title": "Result 1",
+ "snippet": "Snippet 1",
+ "url": "https://example.com/page",
+ },
+ {
+ "position": 2,
+ "title": "Result 2",
+ "snippet": "Snippet 2",
+ "url": "https://docs.example.org/post",
+ },
+ ]
+ }
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_tinyfish.safe_get",
+ return_value=mock_response,
+ ) as mock_get:
+ engine = TinyFishSearchEngine(api_key="test-key", max_results=5)
+ previews = engine._get_previews("test query")
+
+ assert len(previews) == 2
+ assert previews[0]["id"] == "https://example.com/page"
+ assert previews[0]["title"] == "Result 1"
+ assert previews[0]["link"] == "https://example.com/page"
+ assert previews[0]["snippet"] == "Snippet 1"
+ assert previews[0]["displayed_link"] == "example.com"
+ assert previews[1]["displayed_link"] == "docs.example.org"
+
+ call_kwargs = mock_get.call_args.kwargs
+ assert call_kwargs["params"] == {
+ "query": "test query",
+ "location": "US",
+ "language": "en",
+ }
+ assert call_kwargs["headers"] == {"X-API-Key": "test-key"}
+ assert call_kwargs["timeout"] == 10
+
+ def test_get_previews_limits_query_and_results(self):
+ from local_deep_research.web_search_engines.engines.search_engine_tinyfish import (
+ TinyFishSearchEngine,
+ )
+
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {
+ "results": [
+ {
+ "title": f"Result {idx}",
+ "url": f"https://example.com/{idx}",
+ "snippet": "Snippet",
+ }
+ for idx in range(5)
+ ]
+ }
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_tinyfish.safe_get",
+ return_value=mock_response,
+ ) as mock_get:
+ engine = TinyFishSearchEngine(api_key="test-key", max_results=2)
+ previews = engine._get_previews("x" * 500)
+
+ assert len(previews) == 2
+ assert (
+ len(mock_get.call_args.kwargs["params"]["query"])
+ == TinyFishSearchEngine.MAX_QUERY_LEN
+ )
+
+ def test_get_previews_rate_limit_error(self):
+ from local_deep_research.web_search_engines.engines.search_engine_tinyfish import (
+ TinyFishSearchEngine,
+ )
+ from local_deep_research.web_search_engines.rate_limiting import (
+ RateLimitError,
+ )
+
+ mock_response = Mock()
+ mock_response.status_code = 429
+ mock_response.text = "Rate limit exceeded"
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_tinyfish.safe_get",
+ return_value=mock_response,
+ ):
+ engine = TinyFishSearchEngine(api_key="test-key")
+
+ with pytest.raises(RateLimitError):
+ engine._get_previews("test query")
+
+ def test_get_previews_request_exception(self):
+ import requests
+
+ from local_deep_research.web_search_engines.engines.search_engine_tinyfish import (
+ TinyFishSearchEngine,
+ )
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_tinyfish.safe_get",
+ side_effect=requests.exceptions.RequestException(
+ "Connection error"
+ ),
+ ):
+ engine = TinyFishSearchEngine(api_key="test-key")
+
+ assert engine._get_previews("test query") == []
+
+ def test_get_previews_request_exception_with_rate_limit_response(self):
+ import requests
+
+ from local_deep_research.web_search_engines.engines.search_engine_tinyfish import (
+ TinyFishSearchEngine,
+ )
+ from local_deep_research.web_search_engines.rate_limiting import (
+ RateLimitError,
+ )
+
+ response = Mock()
+ response.status_code = 429
+ response.text = "Rate limit exceeded"
+ error = requests.exceptions.RequestException("rate limited")
+ error.response = response
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_tinyfish.safe_get",
+ side_effect=error,
+ ):
+ engine = TinyFishSearchEngine(api_key="test-key")
+
+ with pytest.raises(RateLimitError):
+ engine._get_previews("test query")
+
+
+class TestTinyFishFullContent:
+ """Tests for TinyFish Fetch enrichment."""
+
+ def test_get_full_content_enriches_results(self):
+ from local_deep_research.web_search_engines.engines.search_engine_tinyfish import (
+ TinyFishSearchEngine,
+ )
+
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {
+ "results": [
+ {
+ "url": "https://example.com/page",
+ "final_url": "https://example.com/page",
+ "title": "Fetched title",
+ "description": "Fetched description",
+ "language": "en",
+ "text": "# Extracted content",
+ }
+ ],
+ "errors": [],
+ }
+
+ relevant_items = [
+ {
+ "title": "Search title",
+ "link": "https://example.com/page",
+ "snippet": "Search snippet",
+ }
+ ]
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_tinyfish.safe_post",
+ return_value=mock_response,
+ ) as mock_post:
+ engine = TinyFishSearchEngine(api_key="test-key")
+ results = engine._get_full_content(relevant_items)
+
+ assert results[0]["title"] == "Search title"
+ assert results[0]["content"] == "# Extracted content"
+ assert results[0]["description"] == "Fetched description"
+ assert results[0]["language"] == "en"
+
+ call_kwargs = mock_post.call_args.kwargs
+ assert call_kwargs["json"] == {
+ "urls": ["https://example.com/page"],
+ "format": "markdown",
+ }
+ assert call_kwargs["headers"] == {
+ "X-API-Key": "test-key",
+ "Content-Type": "application/json",
+ }
+ assert call_kwargs["timeout"] == 150
+
+ def test_get_full_content_skips_fetch_when_disabled(self):
+ from local_deep_research.web_search_engines.engines.search_engine_tinyfish import (
+ TinyFishSearchEngine,
+ )
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_tinyfish.safe_post"
+ ) as mock_post:
+ engine = TinyFishSearchEngine(
+ api_key="test-key", include_full_content=False
+ )
+ results = engine._get_full_content(
+ [{"title": "A", "link": "https://example.com"}]
+ )
+
+ assert results == [{"title": "A", "link": "https://example.com"}]
+ mock_post.assert_not_called()
+
+ def test_get_full_content_limits_batch_to_ten_urls(self):
+ from local_deep_research.web_search_engines.engines.search_engine_tinyfish import (
+ TinyFishSearchEngine,
+ )
+
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {"results": [], "errors": []}
+
+ relevant_items = [
+ {"title": str(idx), "link": f"https://example.com/{idx}"}
+ for idx in range(12)
+ ]
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_tinyfish.safe_post",
+ return_value=mock_response,
+ ) as mock_post:
+ engine = TinyFishSearchEngine(api_key="test-key")
+ engine._get_full_content(relevant_items)
+
+ assert (
+ len(mock_post.call_args.kwargs["json"]["urls"])
+ == TinyFishSearchEngine.MAX_FETCH_URLS
+ )
+
+ def test_get_full_content_request_exception_with_rate_limit_response(self):
+ import requests
+
+ from local_deep_research.web_search_engines.engines.search_engine_tinyfish import (
+ TinyFishSearchEngine,
+ )
+ from local_deep_research.web_search_engines.rate_limiting import (
+ RateLimitError,
+ )
+
+ response = Mock()
+ response.status_code = 429
+ response.text = "Rate limit exceeded"
+ error = requests.exceptions.RequestException("rate limited")
+ error.response = response
+
+ with patch(
+ "local_deep_research.web_search_engines.engines.search_engine_tinyfish.safe_post",
+ side_effect=error,
+ ):
+ engine = TinyFishSearchEngine(api_key="test-key")
+
+ with pytest.raises(RateLimitError):
+ engine._get_full_content(
+ [{"title": "A", "link": "https://example.com"}]
+ )
diff --git a/tests/web_search_engines/test_search_engine_factory.py b/tests/web_search_engines/test_search_engine_factory.py
index e44c61639e..f4e5ddd348 100644
--- a/tests/web_search_engines/test_search_engine_factory.py
+++ b/tests/web_search_engines/test_search_engine_factory.py
@@ -147,7 +147,6 @@ class TestCreateSearchEngineRequirements:
assert result is None
-
class TestDisplayLabelFallback:
"""Tests for display label fallback functionality."""