~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
feat(search): add Sofya search engine (#4434)
Yusuf Gürdoğan6770fced6090
diff --git a/changelog.d/+sofya-search-engine.feature.md b/changelog.d/+sofya-search-engine.feature.mdnew file mode 100644index 0000000000..2c0fd336b5--- /dev/null+++ b/changelog.d/+sofya-search-engine.feature.md@@ -0,0 +1 @@+**Sofya search engine** — Added [Sofya](https://sofya.co) as an optional hosted web-search engine. Sofya's `/v1/search` returns ranked results with SERP snippets and extracted page content (markdown) in a single call, fitting LDR's two-phase retrieval model without a separate fetch round trip. Supports `basic`/`snippets` depth, `general`/`news` topics, recency filtering, and domain include/exclude. Configure your API key under Settings → Search → Sofya (`search.engine.web.sofya.api_key`) or the `LDR_SEARCH_ENGINE_WEB_SOFYA_API_KEY` environment variable; GitHub sign-up grants 1000 free credits/month.diff --git a/src/local_deep_research/defaults/settings/search_engines/sofya.json b/src/local_deep_research/defaults/settings/search_engines/sofya.jsonnew file mode 100644index 0000000000..9f6c9b9ae8--- /dev/null+++ b/src/local_deep_research/defaults/settings/search_engines/sofya.json@@ -0,0 +1,223 @@+{+ "search.engine.web.sofya.display_name": {+ "category": "sofya",+ "description": "Display name to use in the U.I. for this search engine.",+ "editable": false,+ "max_value": null,+ "min_value": null,+ "name": "Display Name",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "text",+ "value": "Sofya",+ "visible": false+ },+ "search.engine.web.sofya.description": {+ "category": "sofya",+ "description": "Human-readable description of the search engine.",+ "editable": false,+ "max_value": null,+ "min_value": null,+ "name": "Description",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "text",+ "value": "Hosted web-search API for AI agents. Returns ranked results with SERP snippets and extracted page content (markdown) in a single call.",+ "visible": false+ },+ "search.engine.web.sofya.api_key": {+ "category": "sofya",+ "description": "The Sofya API key to use. Get yours at https://sofya.co/dashboard (GitHub sign-up grants 1000 free credits/month).",+ "editable": true,+ "max_value": null,+ "min_value": null,+ "name": "Api Key",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "password",+ "value": "",+ "visible": true+ },+ "search.engine.web.sofya.default_params.topic": {+ "category": "sofya",+ "description": "Search topic. 'general' for broad web search, 'news' for current events.",+ "editable": true,+ "max_value": null,+ "min_value": null,+ "name": "Topic",+ "options": [+ {+ "label": "General",+ "value": "general"+ },+ {+ "label": "News",+ "value": "news"+ }+ ],+ "step": null,+ "type": "SEARCH",+ "ui_element": "select",+ "value": "general",+ "visible": true+ },+ "search.engine.web.sofya.default_params.freshness": {+ "category": "sofya",+ "description": "Filter results by recency ('day', 'week', 'month', 'year', or null for all time).",+ "editable": true,+ "max_value": null,+ "min_value": null,+ "name": "Freshness",+ "options": [+ {+ "label": "All Time",+ "value": null+ },+ {+ "label": "Past Day",+ "value": "day"+ },+ {+ "label": "Past Week",+ "value": "week"+ },+ {+ "label": "Past Month",+ "value": "month"+ },+ {+ "label": "Past Year",+ "value": "year"+ }+ ],+ "step": null,+ "type": "SEARCH",+ "ui_element": "select",+ "value": null,+ "visible": true+ },+ "search.engine.web.sofya.default_params.include_full_content": {+ "category": "sofya",+ "description": "Fetch extracted page content. On uses Sofya 'basic' depth (content, 3 credits); off uses cheaper 'snippets' depth (1 credit).",+ "editable": true,+ "max_value": null,+ "min_value": null,+ "name": "Include Full Content",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "checkbox",+ "value": true,+ "visible": true+ },+ "search.engine.web.sofya.reliability": {+ "category": "sofya",+ "description": "Reliability of the Sofya search engine",+ "editable": true,+ "max_value": 1.0,+ "min_value": 0.0,+ "name": "Reliability",+ "options": null,+ "step": 0.05,+ "type": "SEARCH",+ "ui_element": "range",+ "value": 0.85,+ "visible": true+ },+ "search.engine.web.sofya.requires_api_key": {+ "category": "sofya",+ "description": "Whether this search engine requires an API key",+ "editable": false,+ "max_value": null,+ "min_value": null,+ "name": "Requires Api Key",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "checkbox",+ "value": true,+ "visible": false+ },+ "search.engine.web.sofya.requires_llm": {+ "category": "sofya",+ "description": "Whether this search engine requires an LLM for relevance filtering",+ "editable": false,+ "max_value": null,+ "min_value": null,+ "name": "Requires LLM",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "checkbox",+ "value": false,+ "visible": false+ },+ "search.engine.web.sofya.supports_full_search": {+ "category": "sofya",+ "description": "Whether this search engine supports full document search",+ "editable": false,+ "max_value": null,+ "min_value": null,+ "name": "Supports Full Search",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "checkbox",+ "value": true,+ "visible": false+ },+ "search.engine.web.sofya.strengths": {+ "category": "sofya",+ "description": "Strengths of this search engine",+ "editable": false,+ "max_value": null,+ "min_value": null,+ "name": "Strengths",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "json",+ "value": [+ "Returns SERP snippets and extracted page content (markdown) in one call",+ "Domain include/exclude filtering and recency filtering",+ "General and news topics",+ "1000 free credits/month for GitHub sign-ups",+ "Built for AI agents"+ ],+ "visible": false+ },+ "search.engine.web.sofya.weaknesses": {+ "category": "sofya",+ "description": "Weaknesses of this search engine",+ "editable": false,+ "max_value": null,+ "min_value": null,+ "name": "Weaknesses",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "json",+ "value": [+ "Requires API key with credit-based usage limits",+ "Not specialized for academic content"+ ],+ "visible": false+ },+ "search.engine.web.sofya.use_in_auto_search": {+ "category": "sofya",+ "description": "Whether to include this engine in auto/meta search selection",+ "editable": true,+ "max_value": null,+ "min_value": null,+ "name": "Use In Auto Search",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "checkbox",+ "value": false,+ "visible": true+ }+}diff --git a/src/local_deep_research/security/module_whitelist.py b/src/local_deep_research/security/module_whitelist.pyindex efd868c300..e42e8e374b 100644--- a/src/local_deep_research/security/module_whitelist.py+++ b/src/local_deep_research/security/module_whitelist.py@@ -50,6 +50,7 @@ ALLOWED_MODULE_PATHS: frozenset[str] = frozenset(".engines.search_engine_semantic_scholar",".engines.search_engine_serper",".engines.search_engine_serpapi",+ ".engines.search_engine_sofya",".engines.search_engine_tavily",".engines.search_engine_tinyfish",".engines.search_engine_gutenberg",@@ -99,6 +100,7 @@ ALLOWED_CLASS_NAMES: frozenset[str] = frozenset("SemanticScholarSearchEngine","SerpAPISearchEngine","SerperSearchEngine",+ "SofyaSearchEngine","StackExchangeSearchEngine","TavilySearchEngine","TinyFishSearchEngine",diff --git a/src/local_deep_research/web_search_engines/engine_registry.py b/src/local_deep_research/web_search_engines/engine_registry.pyindex 775f416073..645ff3ada2 100644--- a/src/local_deep_research/web_search_engines/engine_registry.py+++ b/src/local_deep_research/web_search_engines/engine_registry.py@@ -109,6 +109,10 @@ ENGINE_REGISTRY: Dict[str, EngineEntry] = {module_path=".engines.search_engine_serper",class_name="SerperSearchEngine",),+ "sofya": EngineEntry(+ module_path=".engines.search_engine_sofya",+ class_name="SofyaSearchEngine",+ ),# --- Engines from defaults/settings_*.json ---"nasa_ads": EngineEntry(module_path=".engines.search_engine_nasa_ads",diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_sofya.py b/src/local_deep_research/web_search_engines/engines/search_engine_sofya.pynew file mode 100644index 0000000000..6e9d0e6e68--- /dev/null+++ b/src/local_deep_research/web_search_engines/engines/search_engine_sofya.py@@ -0,0 +1,259 @@+"""Sofya search engine implementation for Local Deep Research.++Sofya (https://sofya.co) is a hosted web-search API for AI agents. Its+``POST /v1/search`` endpoint returns ranked results with a SERP snippet+(``description``) and, in ``basic`` depth, the extracted page content+(``content``) in the same response. A single request therefore covers both+phases of LDR's two-phase retrieval model: previews come from+``description`` and full content from ``content`` — no separate fetch round+trip is needed.++Sign-up gives 1000 free credits/month to GitHub-authenticated accounts, so+the engine is usable without a paid plan. Get an API key at+https://sofya.co/dashboard.+"""++from typing import Any, Dict, List, Optional++import requests+from langchain_core.language_models import BaseLLM++from ...security.safe_requests import safe_post+from ...security.secure_logging import logger+from ..rate_limiting import RateLimitError+from ..search_engine_base import BaseSearchEngine, Exposure, Sensitivity+++class SofyaSearchEngine(BaseSearchEngine):+ """Sofya search engine implementation with two-phase approach."""++ # Mark as a public, generic (general web) search engine.+ is_public = True+ egress_sensitivity = Sensitivity.NON_SENSITIVE+ egress_exposure = Exposure.EXPOSING+ is_generic = True++ #: Base URL for the Sofya REST API.+ BASE_URL = "https://sofya.co/v1"+ #: Sofya caps ``max_results`` at 20 per request.+ MAX_RESULTS_CAP = 20+ #: Sofya accepts at most 10 include/exclude domains per request.+ MAX_DOMAINS = 10+ #: Characters of extracted content to use as a preview snippet when the+ #: SERP ``description`` is missing.+ SNIPPET_PREVIEW_CHARS = 500+ #: HTTP request timeout in seconds.+ REQUEST_TIMEOUT = 30++ def __init__(+ self,+ max_results: int = 10,+ api_key: Optional[str] = None,+ llm: Optional[BaseLLM] = None,+ topic: str = "general",+ freshness: Optional[str] = None,+ include_domains: Optional[List[str]] = None,+ exclude_domains: Optional[List[str]] = None,+ include_full_content: bool = True,+ search_snippets_only: Optional[bool] = None,+ max_filtered_results: Optional[int] = None,+ settings_snapshot: Optional[Dict[str, Any]] = None,+ **kwargs,+ ):+ """Initialize the Sofya search engine.++ Args:+ max_results: Maximum number of search results to request.+ api_key: Sofya API key. May also be supplied via the+ ``search.engine.web.sofya.api_key`` setting or the+ ``LDR_SEARCH_ENGINE_WEB_SOFYA_API_KEY`` environment variable.+ llm: Language model for relevance filtering.+ topic: ``"general"`` or ``"news"``.+ freshness: Recency filter — ``"day"``, ``"week"``, ``"month"``,+ ``"year"``, or a ``"YYYY-MM-DD:YYYY-MM-DD"`` range. ``None``+ means no filter.+ include_domains: Restrict results to these domains (max 10).+ exclude_domains: Exclude results from these domains (max 10).+ include_full_content: Whether to fetch extracted page content.+ When ``True`` (and full content will be consumed, see+ ``search_snippets_only``) the request uses Sofya's ``basic``+ depth (content included); otherwise it uses the cheaper+ ``snippets`` depth (SERP snippets only).+ search_snippets_only: Whether to return only search snippets.+ ``None`` (default) derives it from ``include_full_content``,+ like TinyFish; an explicit value (e.g. forwarded from the+ ``search.snippets_only`` setting by the factory) wins.+ max_filtered_results: Maximum number of results to keep after+ relevance filtering.+ settings_snapshot: Settings snapshot for thread context.+ **kwargs: Additional base-class parameters (e.g.+ ``programmatic_mode``); forwarded to ``super().__init__``.+ """+ if search_snippets_only is None:+ search_snippets_only = not include_full_content+ super().__init__(+ llm=llm,+ max_filtered_results=max_filtered_results,+ max_results=max_results,+ include_full_content=include_full_content,+ search_snippets_only=search_snippets_only,+ settings_snapshot=settings_snapshot,+ **kwargs,+ )++ self.topic = topic if topic in ("general", "news") else "general"+ self.freshness = freshness or None+ self.include_domains = self._ensure_list(include_domains)[+ : self.MAX_DOMAINS+ ]+ self.exclude_domains = self._ensure_list(exclude_domains)[+ : self.MAX_DOMAINS+ ]++ # Resolve API key from params, settings snapshot, or env var.+ self.api_key = self._resolve_api_key(+ api_key,+ "search.engine.web.sofya.api_key",+ engine_name="Sofya",+ settings_snapshot=settings_snapshot,+ )++ def _build_payload(self, query: str) -> Dict[str, Any]:+ """Build the request body for the Sofya ``/search`` endpoint."""+ # "basic" depth (3 credits) returns extracted page content; "snippets"+ # (1 credit) returns SERP snippets only. Only pay for content when it+ # will actually be consumed: run() calls _get_full_content (which+ # promotes the content) only when search_snippets_only is False.+ wants_content = (+ self.include_full_content and not self.search_snippets_only+ )+ payload: Dict[str, Any] = {+ "query": query,+ "search_depth": "basic" if wants_content else "snippets",+ "max_results": min(self.MAX_RESULTS_CAP, self.max_results),+ "topic": self.topic,+ }+ if self.freshness:+ payload["freshness"] = self.freshness+ if self.include_domains:+ payload["include_domains"] = self.include_domains+ if self.exclude_domains:+ payload["exclude_domains"] = self.exclude_domains+ return payload++ def _get_previews(self, query: str) -> List[Dict[str, Any]]:+ """Get preview information from the Sofya search API.++ Args:+ query: The search query.++ Returns:+ List of preview dictionaries.+ """+ logger.info("Getting search results from Sofya")++ try:+ # Apply rate limiting before the request.+ self._last_wait_time = self.rate_tracker.apply_rate_limit(+ self.engine_type+ )++ response = safe_post(+ f"{self.BASE_URL}/search",+ json=self._build_payload(query),+ headers={+ "Authorization": f"Bearer {self.api_key}",+ "Content-Type": "application/json",+ },+ timeout=self.REQUEST_TIMEOUT,+ )++ # Detect rate limiting from the status code.+ self._raise_if_rate_limit(response.status_code)+ response.raise_for_status()++ data = response.json()+ results = data.get("results", []) or []++ previews = []+ for i, result in enumerate(results):+ url = result.get("url", "")+ # Keep the short SERP ``description`` as the preview snippet+ # (used for relevance filtering and display); fall back to a+ # slice of the extracted content when no description is+ # provided. The extracted page content itself stays in the+ # ``_full_result`` stash until _get_full_content promotes it,+ # so snippets-only mode never carries full page text.+ snippet = result.get("description") or ""+ if not snippet:+ snippet = (result.get("content") or "")[+ : self.SNIPPET_PREVIEW_CHARS+ ]++ preview = {+ "id": url or str(i),+ "title": result.get("title", ""),+ "link": url,+ "snippet": snippet,+ "displayed_link": url,+ "position": i,+ "published_date": result.get("published_date"),+ # Store the full Sofya result for _get_full_content().+ "_full_result": result,+ }+ previews.append(preview)++ self._search_results = previews+ return previews++ except RateLimitError:+ raise # Re-raise rate limit errors+ except requests.exceptions.RequestException as e:+ safe_msg = self._scrub_error(e)+ logger.warning(f"Error getting Sofya results: {safe_msg}")+ # Re-check for rate limiting using the response status code, not+ # the exception object, so _raise_if_rate_limit gets an int.+ status_code = getattr(+ getattr(e, "response", None), "status_code", None+ )+ if status_code is not None:+ self._raise_if_rate_limit(status_code)+ return []+ except Exception as e:+ safe_msg = self._scrub_error(e)+ logger.warning(+ f"Unexpected error getting Sofya results: {safe_msg}"+ )+ return []++ def _get_full_content(+ self, relevant_items: List[Dict[str, Any]]+ ) -> List[Dict[str, Any]]:+ """Return full content for the relevant results.++ Sofya returns the extracted page text inline (``basic`` depth), and+ ``_get_previews`` stashes it on each preview under ``_full_result``.+ Promote it here to ``full_content`` (the field the report's citation+ handler reads before falling back to ``snippet``) while dropping the+ internal ``_full_result`` marker and preserving the preview fields,+ notably ``link``, which the citation handler uses as the source URL.+ Falls back to the snippet when no extracted content is available+ (e.g. ``snippets`` depth), so ``full_content`` is always usable.++ Args:+ relevant_items: List of relevant preview dictionaries.++ Returns:+ List of result dictionaries with full content where available.+ """+ results = []+ for item in relevant_items:+ result = {k: v for k, v in item.items() if k != "_full_result"}+ content = (item.get("_full_result") or {}).get("content") or ""+ result["full_content"] = (+ content+ or result.get("full_content")+ or result.get("snippet", "")+ )+ results.append(result)+ return resultsdiff --git a/src/local_deep_research/web_search_engines/search_engine_factory.py b/src/local_deep_research/web_search_engines/search_engine_factory.pyindex 7519c9b606..2ed661b51c 100644--- a/src/local_deep_research/web_search_engines/search_engine_factory.py+++ b/src/local_deep_research/web_search_engines/search_engine_factory.py@@ -693,6 +693,11 @@ def get_search(params["language"] = search_languageparams["search_snippets_only"] = search_snippets_only+ if search_tool == "sofya":+ # Sofya derives its request depth from this: the paid "basic" depth+ # (inline page content) is only requested when full content is used.+ params["search_snippets_only"] = search_snippets_only+if search_tool == "wikinews":params["search_snippets_only"] = search_snippets_onlyparams["adaptive_search"] = bool(diff --git a/tests/settings/golden_master_settings.json b/tests/settings/golden_master_settings.jsonindex 749982fce8..5e32a571b1 100644--- a/tests/settings/golden_master_settings.json+++ b/tests/settings/golden_master_settings.json@@ -6647,6 +6647,227 @@],"visible": false},+ "search.engine.web.sofya.api_key": {+ "category": "sofya",+ "description": "The Sofya API key to use. Get yours at https://sofya.co/dashboard (GitHub sign-up grants 1000 free credits/month).",+ "editable": true,+ "max_value": null,+ "min_value": null,+ "name": "Api Key",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "password",+ "value": "",+ "visible": true+ },+ "search.engine.web.sofya.default_params.freshness": {+ "category": "sofya",+ "description": "Filter results by recency ('day', 'week', 'month', 'year', or null for all time).",+ "editable": true,+ "max_value": null,+ "min_value": null,+ "name": "Freshness",+ "options": [+ {+ "label": "All Time",+ "value": null+ },+ {+ "label": "Past Day",+ "value": "day"+ },+ {+ "label": "Past Week",+ "value": "week"+ },+ {+ "label": "Past Month",+ "value": "month"+ },+ {+ "label": "Past Year",+ "value": "year"+ }+ ],+ "step": null,+ "type": "SEARCH",+ "ui_element": "select",+ "value": null,+ "visible": true+ },+ "search.engine.web.sofya.default_params.include_full_content": {+ "category": "sofya",+ "description": "Fetch extracted page content. On uses Sofya 'basic' depth (content, 3 credits); off uses cheaper 'snippets' depth (1 credit).",+ "editable": true,+ "max_value": null,+ "min_value": null,+ "name": "Include Full Content",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "checkbox",+ "value": true,+ "visible": true+ },+ "search.engine.web.sofya.default_params.topic": {+ "category": "sofya",+ "description": "Search topic. 'general' for broad web search, 'news' for current events.",+ "editable": true,+ "max_value": null,+ "min_value": null,+ "name": "Topic",+ "options": [+ {+ "label": "General",+ "value": "general"+ },+ {+ "label": "News",+ "value": "news"+ }+ ],+ "step": null,+ "type": "SEARCH",+ "ui_element": "select",+ "value": "general",+ "visible": true+ },+ "search.engine.web.sofya.description": {+ "category": "sofya",+ "description": "Human-readable description of the search engine.",+ "editable": false,+ "max_value": null,+ "min_value": null,+ "name": "Description",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "text",+ "value": "Hosted web-search API for AI agents. Returns ranked results with SERP snippets and extracted page content (markdown) in a single call.",+ "visible": false+ },+ "search.engine.web.sofya.display_name": {+ "category": "sofya",+ "description": "Display name to use in the U.I. for this search engine.",+ "editable": false,+ "max_value": null,+ "min_value": null,+ "name": "Display Name",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "text",+ "value": "Sofya",+ "visible": false+ },+ "search.engine.web.sofya.reliability": {+ "category": "sofya",+ "description": "Reliability of the Sofya search engine",+ "editable": true,+ "max_value": 1.0,+ "min_value": 0.0,+ "name": "Reliability",+ "options": null,+ "step": 0.05,+ "type": "SEARCH",+ "ui_element": "range",+ "value": 0.85,+ "visible": true+ },+ "search.engine.web.sofya.requires_api_key": {+ "category": "sofya",+ "description": "Whether this search engine requires an API key",+ "editable": false,+ "max_value": null,+ "min_value": null,+ "name": "Requires Api Key",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "checkbox",+ "value": true,+ "visible": false+ },+ "search.engine.web.sofya.requires_llm": {+ "category": "sofya",+ "description": "Whether this search engine requires an LLM for relevance filtering",+ "editable": false,+ "max_value": null,+ "min_value": null,+ "name": "Requires LLM",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "checkbox",+ "value": false,+ "visible": false+ },+ "search.engine.web.sofya.strengths": {+ "category": "sofya",+ "description": "Strengths of this search engine",+ "editable": false,+ "max_value": null,+ "min_value": null,+ "name": "Strengths",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "json",+ "value": [+ "Returns SERP snippets and extracted page content (markdown) in one call",+ "Domain include/exclude filtering and recency filtering",+ "General and news topics",+ "1000 free credits/month for GitHub sign-ups",+ "Built for AI agents"+ ],+ "visible": false+ },+ "search.engine.web.sofya.supports_full_search": {+ "category": "sofya",+ "description": "Whether this search engine supports full document search",+ "editable": false,+ "max_value": null,+ "min_value": null,+ "name": "Supports Full Search",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "checkbox",+ "value": true,+ "visible": false+ },+ "search.engine.web.sofya.use_in_auto_search": {+ "category": "sofya",+ "description": "Whether to include this engine in auto/meta search selection",+ "editable": true,+ "max_value": null,+ "min_value": null,+ "name": "Use In Auto Search",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "checkbox",+ "value": false,+ "visible": true+ },+ "search.engine.web.sofya.weaknesses": {+ "category": "sofya",+ "description": "Weaknesses of this search engine",+ "editable": false,+ "max_value": null,+ "min_value": null,+ "name": "Weaknesses",+ "options": null,+ "step": null,+ "type": "SEARCH",+ "ui_element": "json",+ "value": [+ "Requires API key with credit-based usage limits",+ "Not specialized for academic content"+ ],+ "visible": false+ },"search.engine.web.stackexchange": {"category": "stackexchange","description": "Stack Exchange search engine configuration",diff --git a/tests/web_search_engines/engines/test_search_engine_sofya.py b/tests/web_search_engines/engines/test_search_engine_sofya.pynew file mode 100644index 0000000000..25d21de155--- /dev/null+++ b/tests/web_search_engines/engines/test_search_engine_sofya.py@@ -0,0 +1,525 @@+"""+Tests for the SofyaSearchEngine class.++Tests cover:+- Initialization and configuration+- API key handling+- Request payload construction+- Preview generation+- Full content retrieval+- Rate limit error handling+- Run method+- Class attributes+"""++from unittest.mock import Mock, patch++import pytest++MODULE = "local_deep_research.web_search_engines.engines.search_engine_sofya"+++def _make_engine(**kwargs):+ from local_deep_research.web_search_engines.engines.search_engine_sofya import (+ SofyaSearchEngine,+ )++ kwargs.setdefault("api_key", "test-api-key")+ return SofyaSearchEngine(**kwargs)+++class TestSofyaSearchEngineInit:+ """Tests for SofyaSearchEngine initialization."""++ def test_init_with_api_key(self):+ """Initialize with API key and defaults."""+ engine = _make_engine()++ assert engine.api_key == "test-api-key"+ assert engine.max_results == 10+ assert engine.topic == "general"+ assert engine.include_full_content is True++ def test_init_with_custom_max_results(self):+ """Initialize with custom max_results."""+ engine = _make_engine(max_results=25)++ assert engine.max_results == 25++ def test_init_invalid_topic_falls_back_to_general(self):+ """An unsupported topic falls back to 'general'."""+ engine = _make_engine(topic="sports")++ assert engine.topic == "general"++ def test_init_with_domain_filters(self):+ """Initialize with domain filters."""+ engine = _make_engine(+ include_domains=["example.com", "test.com"],+ exclude_domains=["spam.com"],+ )++ assert engine.include_domains == ["example.com", "test.com"]+ assert engine.exclude_domains == ["spam.com"]++ def test_domain_filters_capped_at_max(self):+ """Domain lists are capped at MAX_DOMAINS entries."""+ from local_deep_research.web_search_engines.engines.search_engine_sofya import (+ SofyaSearchEngine,+ )++ too_many = [f"domain{i}.com" for i in range(15)]+ engine = _make_engine(include_domains=too_many)++ assert len(engine.include_domains) == SofyaSearchEngine.MAX_DOMAINS++ def test_init_without_api_key_raises(self):+ """Initialize without API key raises ValueError."""+ from local_deep_research.web_search_engines.engines.search_engine_sofya import (+ SofyaSearchEngine,+ )++ with pytest.raises(+ ValueError, match="No valid API key found for Sofya"+ ):+ SofyaSearchEngine()++ def test_init_with_api_key_from_settings(self):+ """Initialize with API key from settings snapshot."""+ from local_deep_research.web_search_engines.engines.search_engine_sofya import (+ SofyaSearchEngine,+ )++ engine = SofyaSearchEngine(+ settings_snapshot={+ "search.engine.web.sofya.api_key": "settings-api-key"+ }+ )++ assert engine.api_key == "settings-api-key"++ def test_init_with_llm(self):+ """Initialize with LLM."""+ mock_llm = Mock()+ engine = _make_engine(llm=mock_llm)++ assert engine.llm is mock_llm++ def test_init_derives_snippets_only_from_include_full_content(self):+ """Like TinyFish: search_snippets_only defaults to the inverse of+ include_full_content, so the full-content path actually runs when+ content is requested."""+ engine = _make_engine()+ assert engine.search_snippets_only is False++ engine = _make_engine(include_full_content=False)+ assert engine.search_snippets_only is True++ def test_init_explicit_snippets_only_wins_over_derivation(self):+ """An explicit search_snippets_only (e.g. forwarded from the+ search.snippets_only setting by the factory) beats the derived value."""+ engine = _make_engine(+ include_full_content=True, search_snippets_only=True+ )++ assert engine.search_snippets_only is True+++class TestBuildPayload:+ """Tests for _build_payload."""++ def test_payload_basics(self):+ """Payload includes query, depth, max_results, and topic."""+ engine = _make_engine(max_results=5)++ payload = engine._build_payload("hello world")++ assert payload["query"] == "hello world"+ # include_full_content defaults to True (and search_snippets_only+ # derives to False), so the content will be consumed -> basic depth.+ assert payload["search_depth"] == "basic"+ assert payload["max_results"] == 5+ assert payload["topic"] == "general"+ # Optional fields are omitted when unset.+ assert "freshness" not in payload+ assert "include_domains" not in payload+ assert "exclude_domains" not in payload++ def test_payload_snippets_depth_when_no_full_content(self):+ """include_full_content=False maps to the cheaper snippets depth."""+ engine = _make_engine(include_full_content=False)++ payload = engine._build_payload("q")++ assert payload["search_depth"] == "snippets"++ def test_payload_snippets_depth_when_snippets_only(self):+ """search_snippets_only=True uses the snippets depth even when+ include_full_content=True: run() would discard the content, so the+ 3x-priced basic depth must not be paid for."""+ engine = _make_engine(+ include_full_content=True, search_snippets_only=True+ )++ payload = engine._build_payload("q")++ assert payload["search_depth"] == "snippets"++ def test_payload_caps_max_results(self):+ """max_results is capped at MAX_RESULTS_CAP in the payload."""+ engine = _make_engine(max_results=100)++ payload = engine._build_payload("q")++ assert payload["max_results"] == engine.MAX_RESULTS_CAP++ def test_payload_includes_optional_fields(self):+ """Freshness and domain filters are included when set."""+ engine = _make_engine(+ freshness="week",+ include_domains=["example.com"],+ exclude_domains=["spam.com"],+ )++ payload = engine._build_payload("q")++ assert payload["freshness"] == "week"+ assert payload["include_domains"] == ["example.com"]+ assert payload["exclude_domains"] == ["spam.com"]+++class TestGetPreviews:+ """Tests for _get_previews method."""++ def test_get_previews_returns_results(self):+ """Get previews returns formatted results."""+ mock_response = Mock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "results": [+ {+ "title": "Result 1",+ "url": "https://example1.com",+ "description": "Snippet 1",+ "content": "Full content 1",+ },+ {+ "title": "Result 2",+ "url": "https://example2.com",+ "description": "Snippet 2",+ "content": "Full content 2",+ },+ ]+ }++ with patch(f"{MODULE}.safe_post", return_value=mock_response):+ engine = _make_engine()+ previews = engine._get_previews("test query")++ assert len(previews) == 2+ assert previews[0]["title"] == "Result 1"+ assert previews[0]["snippet"] == "Snippet 1"+ assert previews[0]["link"] == "https://example1.com"+ assert previews[0]["_full_result"]["content"] == "Full content 1"+ # The extracted content must stay in the _full_result stash until+ # _get_full_content promotes it: previews are returned as-is in+ # snippets-only mode, so full_content here would leak full page+ # text into the synthesis prompt.+ assert "full_content" not in previews[0]++ def test_get_previews_snippet_falls_back_to_content(self):+ """When description is missing, snippet falls back to content slice."""+ long_content = "x" * 1000+ mock_response = Mock()+ mock_response.status_code = 200+ mock_response.json.return_value = {+ "results": [+ {+ "title": "No description",+ "url": "https://example.com",+ "content": long_content,+ }+ ]+ }++ with patch(f"{MODULE}.safe_post", return_value=mock_response):+ engine = _make_engine()+ previews = engine._get_previews("test query")++ assert len(previews[0]["snippet"]) == engine.SNIPPET_PREVIEW_CHARS++ def test_get_previews_sends_auth_header(self):+ """Get previews sends the bearer auth header."""+ mock_response = Mock()+ mock_response.status_code = 200+ mock_response.json.return_value = {"results": []}++ with patch(+ f"{MODULE}.safe_post", return_value=mock_response+ ) as mock_post:+ engine = _make_engine()+ engine._get_previews("test query")++ headers = mock_post.call_args[1]["headers"]+ assert headers["Authorization"] == "Bearer test-api-key"++ def test_get_previews_rate_limit_error(self):+ """Get previews raises RateLimitError on 429."""+ from local_deep_research.web_search_engines.rate_limiting import (+ RateLimitError,+ )++ mock_response = Mock()+ mock_response.status_code = 429+ mock_response.text = "Rate limit exceeded"++ with patch(f"{MODULE}.safe_post", return_value=mock_response):+ engine = _make_engine()++ with pytest.raises(RateLimitError):+ engine._get_previews("test query")++ def test_get_previews_empty_results(self):+ """Get previews handles empty results."""+ mock_response = Mock()+ mock_response.status_code = 200+ mock_response.json.return_value = {"results": []}++ with patch(f"{MODULE}.safe_post", return_value=mock_response):+ engine = _make_engine()+ previews = engine._get_previews("test query")++ assert previews == []++ def test_get_previews_request_exception(self):+ """Get previews handles request exceptions gracefully."""+ import requests++ with patch(+ f"{MODULE}.safe_post",+ side_effect=requests.exceptions.RequestException(+ "Connection error"+ ),+ ):+ engine = _make_engine()+ previews = engine._get_previews("test query")++ assert previews == []++ def test_get_previews_request_exception_with_rate_limit_status(self):+ """A RequestException carrying a 429 response raises RateLimitError."""+ import requests++ from local_deep_research.web_search_engines.rate_limiting import (+ RateLimitError,+ )++ err = requests.exceptions.RequestException("boom")+ err.response = Mock(status_code=429)++ with patch(f"{MODULE}.safe_post", side_effect=err):+ engine = _make_engine()++ with pytest.raises(RateLimitError):+ engine._get_previews("test query")++ def test_get_previews_unexpected_error(self):+ """Get previews handles unexpected errors."""+ mock_response = Mock()+ mock_response.status_code = 200+ mock_response.json.side_effect = Exception("Parse error")+ mock_response.raise_for_status = Mock()++ with patch(f"{MODULE}.safe_post", return_value=mock_response):+ engine = _make_engine()+ previews = engine._get_previews("test query")++ assert previews == []+++class TestGetFullContent:+ """Tests for _get_full_content method."""++ def test_get_full_content_promotes_stashed_content(self):+ """The extracted content is promoted from the _full_result stash to+ full_content, the citation link survives, and the internal+ _full_result marker is dropped."""+ engine = _make_engine(include_full_content=True)++ items = [+ {+ "title": "Result",+ "link": "https://example.com",+ "snippet": "Short snippet",+ "_full_result": {+ "url": "https://example.com",+ "content": "Full extracted markdown content",+ },+ }+ ]++ results = engine._get_full_content(items)++ assert len(results) == 1+ assert results[0]["full_content"] == "Full extracted markdown content"+ assert results[0]["link"] == "https://example.com"+ assert "_full_result" not in results[0]++ def test_get_full_content_falls_back_to_snippet(self):+ """When the stash carries no extracted content (e.g. snippets depth),+ fall back to the snippet."""+ engine = _make_engine(include_full_content=True)++ items = [+ {+ "title": "Result",+ "link": "https://example.com",+ "snippet": "Only a snippet",+ "_full_result": {"url": "https://example.com"},+ }+ ]++ results = engine._get_full_content(items)++ assert results[0]["full_content"] == "Only a snippet"++ def test_get_full_content_without_full_result(self):+ """Get full content handles items without _full_result."""+ engine = _make_engine(include_full_content=False)++ items = [{"title": "Result", "link": "https://example.com"}]++ results = engine._get_full_content(items)++ assert len(results) == 1+ assert results[0]["title"] == "Result"+++class TestRun:+ """Tests for run method."""++ def test_run_returns_results(self):+ """Run promotes the stashed content to full_content."""+ engine = _make_engine()++ with patch.object(+ engine,+ "_get_previews",+ return_value=[+ {+ "id": "https://example.com",+ "title": "Result",+ "snippet": "Snippet",+ "link": "https://example.com",+ "_full_result": {+ "url": "https://example.com",+ "content": "Full page text",+ },+ }+ ],+ ):+ results = engine.run("test query")++ assert len(results) == 1+ # full_content is the field the report's citation handler reads+ # before falling back to the snippet.+ assert results[0]["full_content"] == "Full page text"+ assert "_full_result" not in results[0]++ def test_run_surfaces_full_content_end_to_end(self):+ """End-to-end (real _get_previews): the inline extracted page content+ reaches run() output under full_content, while the snippet stays the+ short SERP description. Guards against the depth/credits being paid for+ content that is then discarded."""+ mock_response = Mock()+ mock_response.status_code = 200+ mock_response.raise_for_status = Mock()+ mock_response.json.return_value = {+ "results": [+ {+ "title": "Result",+ "url": "https://example.com",+ "description": "Short SERP snippet",+ "content": "Full extracted page markdown",+ }+ ]+ }++ with patch(f"{MODULE}.safe_post", return_value=mock_response):+ engine = _make_engine(include_full_content=True)+ results = engine.run("test query")++ assert len(results) == 1+ assert results[0]["full_content"] == "Full extracted page markdown"+ assert results[0]["snippet"] == "Short SERP snippet"+ assert results[0]["link"] == "https://example.com"++ def test_run_snippets_only_carries_no_full_content(self):+ """Regression: in snippets-only mode run() returns the previews+ directly (no _get_full_content call), so they must not carry+ full_content - otherwise full page text would leak into the+ synthesis prompt despite the setting."""+ mock_response = Mock()+ mock_response.status_code = 200+ mock_response.raise_for_status = Mock()+ mock_response.json.return_value = {+ "results": [+ {+ "title": "Result",+ "url": "https://example.com",+ "description": "Short SERP snippet",+ "content": "Full extracted page markdown",+ }+ ]+ }++ with patch(f"{MODULE}.safe_post", return_value=mock_response):+ engine = _make_engine(search_snippets_only=True)+ results = engine.run("test query")++ assert len(results) == 1+ assert "full_content" not in results[0]+ assert results[0]["snippet"] == "Short SERP snippet"++ def test_run_handles_empty_results(self):+ """Run handles empty results."""+ engine = _make_engine()++ with patch.object(engine, "_get_previews", return_value=[]):+ results = engine.run("test query")++ assert results == []+++class TestClassAttributes:+ """Tests for class attributes."""++ def test_is_public(self):+ """SofyaSearchEngine is marked as public."""+ from local_deep_research.web_search_engines.engines.search_engine_sofya import (+ SofyaSearchEngine,+ )++ assert SofyaSearchEngine.is_public is True++ def test_is_generic(self):+ """SofyaSearchEngine is marked as generic."""+ from local_deep_research.web_search_engines.engines.search_engine_sofya import (+ SofyaSearchEngine,+ )++ assert SofyaSearchEngine.is_generic is True++ def test_egress_labels_match_public_engine(self):+ """A public engine must declare NON_SENSITIVE/EXPOSING egress labels+ (enforced by test_registered_engines_declare_consistent_labels)."""+ from local_deep_research.security.egress.classification import (+ Exposure,+ Sensitivity,+ )+ from local_deep_research.web_search_engines.engines.search_engine_sofya import (+ SofyaSearchEngine,+ )++ assert SofyaSearchEngine.egress_sensitivity is Sensitivity.NON_SENSITIVE+ assert SofyaSearchEngine.egress_exposure is Exposure.EXPOSINGdiff --git a/tests/web_search_engines/test_search_engine_factory_coverage.py b/tests/web_search_engines/test_search_engine_factory_coverage.pyindex 8885125a6d..b457a21b01 100644--- a/tests/web_search_engines/test_search_engine_factory_coverage.py+++ b/tests/web_search_engines/test_search_engine_factory_coverage.py@@ -912,6 +912,28 @@ class TestGetSearchParameterRouting:assert call_kwargs["search_language"] == "French"assert call_kwargs["adaptive_search"] is False+ def test_get_search_sofya_params(self):+ """Sofya gets search_snippets_only forwarded: it derives its request+ depth from it (paid 'basic' depth only when content is consumed)."""+ from local_deep_research.web_search_engines.search_engine_factory import (+ get_search,+ )++ with patch(+ "local_deep_research.web_search_engines.search_engine_factory.create_search_engine"+ ) as mock_create:+ mock_create.return_value = Mock(run=Mock())++ get_search(+ search_tool="sofya",+ llm_instance=Mock(),+ search_snippets_only=True,+ settings_snapshot={"x": 1},+ )++ call_kwargs = mock_create.call_args[1]+ assert call_kwargs["search_snippets_only"] is True+def test_get_search_max_filtered_results(self):"""max_filtered_results is passed when provided."""from local_deep_research.web_search_engines.search_engine_factory import (