Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

feat(search): add Sofya search engine (#4434)

Yusuf Gürdoğan6770fced6090
diff --git a/changelog.d/+sofya-search-engine.feature.md b/changelog.d/+sofya-search-engine.feature.md
new file mode 100644
index 0000000000..2c0fd336b5
--- /dev/null
+++ b/changelog.d/+sofya-search-engine.feature.md
@@ -0,0 +1 @@
+**Sofya search engine** — Added [Sofya](https://sofya.co) as an optional hosted web-search engine. Sofya's `/v1/search` returns ranked results with SERP snippets and extracted page content (markdown) in a single call, fitting LDR's two-phase retrieval model without a separate fetch round trip. Supports `basic`/`snippets` depth, `general`/`news` topics, recency filtering, and domain include/exclude. Configure your API key under Settings → Search → Sofya (`search.engine.web.sofya.api_key`) or the `LDR_SEARCH_ENGINE_WEB_SOFYA_API_KEY` environment variable; GitHub sign-up grants 1000 free credits/month.
diff --git a/src/local_deep_research/defaults/settings/search_engines/sofya.json b/src/local_deep_research/defaults/settings/search_engines/sofya.json
new file mode 100644
index 0000000000..9f6c9b9ae8
--- /dev/null
+++ b/src/local_deep_research/defaults/settings/search_engines/sofya.json
@@ -0,0 +1,223 @@
+{
+ "search.engine.web.sofya.display_name": {
+ "category": "sofya",
+ "description": "Display name to use in the U.I. for this search engine.",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Display Name",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "text",
+ "value": "Sofya",
+ "visible": false
+ },
+ "search.engine.web.sofya.description": {
+ "category": "sofya",
+ "description": "Human-readable description of the search engine.",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Description",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "text",
+ "value": "Hosted web-search API for AI agents. Returns ranked results with SERP snippets and extracted page content (markdown) in a single call.",
+ "visible": false
+ },
+ "search.engine.web.sofya.api_key": {
+ "category": "sofya",
+ "description": "The Sofya API key to use. Get yours at https://sofya.co/dashboard (GitHub sign-up grants 1000 free credits/month).",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Api Key",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "password",
+ "value": "",
+ "visible": true
+ },
+ "search.engine.web.sofya.default_params.topic": {
+ "category": "sofya",
+ "description": "Search topic. 'general' for broad web search, 'news' for current events.",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Topic",
+ "options": [
+ {
+ "label": "General",
+ "value": "general"
+ },
+ {
+ "label": "News",
+ "value": "news"
+ }
+ ],
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "select",
+ "value": "general",
+ "visible": true
+ },
+ "search.engine.web.sofya.default_params.freshness": {
+ "category": "sofya",
+ "description": "Filter results by recency ('day', 'week', 'month', 'year', or null for all time).",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Freshness",
+ "options": [
+ {
+ "label": "All Time",
+ "value": null
+ },
+ {
+ "label": "Past Day",
+ "value": "day"
+ },
+ {
+ "label": "Past Week",
+ "value": "week"
+ },
+ {
+ "label": "Past Month",
+ "value": "month"
+ },
+ {
+ "label": "Past Year",
+ "value": "year"
+ }
+ ],
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "select",
+ "value": null,
+ "visible": true
+ },
+ "search.engine.web.sofya.default_params.include_full_content": {
+ "category": "sofya",
+ "description": "Fetch extracted page content. On uses Sofya 'basic' depth (content, 3 credits); off uses cheaper 'snippets' depth (1 credit).",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Include Full Content",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "checkbox",
+ "value": true,
+ "visible": true
+ },
+ "search.engine.web.sofya.reliability": {
+ "category": "sofya",
+ "description": "Reliability of the Sofya search engine",
+ "editable": true,
+ "max_value": 1.0,
+ "min_value": 0.0,
+ "name": "Reliability",
+ "options": null,
+ "step": 0.05,
+ "type": "SEARCH",
+ "ui_element": "range",
+ "value": 0.85,
+ "visible": true
+ },
+ "search.engine.web.sofya.requires_api_key": {
+ "category": "sofya",
+ "description": "Whether this search engine requires an API key",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Requires Api Key",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "checkbox",
+ "value": true,
+ "visible": false
+ },
+ "search.engine.web.sofya.requires_llm": {
+ "category": "sofya",
+ "description": "Whether this search engine requires an LLM for relevance filtering",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Requires LLM",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "checkbox",
+ "value": false,
+ "visible": false
+ },
+ "search.engine.web.sofya.supports_full_search": {
+ "category": "sofya",
+ "description": "Whether this search engine supports full document search",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Supports Full Search",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "checkbox",
+ "value": true,
+ "visible": false
+ },
+ "search.engine.web.sofya.strengths": {
+ "category": "sofya",
+ "description": "Strengths of this search engine",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Strengths",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "json",
+ "value": [
+ "Returns SERP snippets and extracted page content (markdown) in one call",
+ "Domain include/exclude filtering and recency filtering",
+ "General and news topics",
+ "1000 free credits/month for GitHub sign-ups",
+ "Built for AI agents"
+ ],
+ "visible": false
+ },
+ "search.engine.web.sofya.weaknesses": {
+ "category": "sofya",
+ "description": "Weaknesses of this search engine",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Weaknesses",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "json",
+ "value": [
+ "Requires API key with credit-based usage limits",
+ "Not specialized for academic content"
+ ],
+ "visible": false
+ },
+ "search.engine.web.sofya.use_in_auto_search": {
+ "category": "sofya",
+ "description": "Whether to include this engine in auto/meta search selection",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Use In Auto Search",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "checkbox",
+ "value": false,
+ "visible": true
+ }
+}
diff --git a/src/local_deep_research/security/module_whitelist.py b/src/local_deep_research/security/module_whitelist.py
index efd868c300..e42e8e374b 100644
--- a/src/local_deep_research/security/module_whitelist.py
+++ b/src/local_deep_research/security/module_whitelist.py
@@ -50,6 +50,7 @@ ALLOWED_MODULE_PATHS: frozenset[str] = frozenset(
".engines.search_engine_semantic_scholar",
".engines.search_engine_serper",
".engines.search_engine_serpapi",
+ ".engines.search_engine_sofya",
".engines.search_engine_tavily",
".engines.search_engine_tinyfish",
".engines.search_engine_gutenberg",
@@ -99,6 +100,7 @@ ALLOWED_CLASS_NAMES: frozenset[str] = frozenset(
"SemanticScholarSearchEngine",
"SerpAPISearchEngine",
"SerperSearchEngine",
+ "SofyaSearchEngine",
"StackExchangeSearchEngine",
"TavilySearchEngine",
"TinyFishSearchEngine",
diff --git a/src/local_deep_research/web_search_engines/engine_registry.py b/src/local_deep_research/web_search_engines/engine_registry.py
index 775f416073..645ff3ada2 100644
--- a/src/local_deep_research/web_search_engines/engine_registry.py
+++ b/src/local_deep_research/web_search_engines/engine_registry.py
@@ -109,6 +109,10 @@ ENGINE_REGISTRY: Dict[str, EngineEntry] = {
module_path=".engines.search_engine_serper",
class_name="SerperSearchEngine",
),
+ "sofya": EngineEntry(
+ module_path=".engines.search_engine_sofya",
+ class_name="SofyaSearchEngine",
+ ),
# --- Engines from defaults/settings_*.json ---
"nasa_ads": EngineEntry(
module_path=".engines.search_engine_nasa_ads",
diff --git a/src/local_deep_research/web_search_engines/engines/search_engine_sofya.py b/src/local_deep_research/web_search_engines/engines/search_engine_sofya.py
new file mode 100644
index 0000000000..6e9d0e6e68
--- /dev/null
+++ b/src/local_deep_research/web_search_engines/engines/search_engine_sofya.py
@@ -0,0 +1,259 @@
+"""Sofya search engine implementation for Local Deep Research.
+
+Sofya (https://sofya.co) is a hosted web-search API for AI agents. Its
+``POST /v1/search`` endpoint returns ranked results with a SERP snippet
+(``description``) and, in ``basic`` depth, the extracted page content
+(``content``) in the same response. A single request therefore covers both
+phases of LDR's two-phase retrieval model: previews come from
+``description`` and full content from ``content`` — no separate fetch round
+trip is needed.
+
+Sign-up gives 1000 free credits/month to GitHub-authenticated accounts, so
+the engine is usable without a paid plan. Get an API key at
+https://sofya.co/dashboard.
+"""
+
+from typing import Any, Dict, List, Optional
+
+import requests
+from langchain_core.language_models import BaseLLM
+
+from ...security.safe_requests import safe_post
+from ...security.secure_logging import logger
+from ..rate_limiting import RateLimitError
+from ..search_engine_base import BaseSearchEngine, Exposure, Sensitivity
+
+
+class SofyaSearchEngine(BaseSearchEngine):
+ """Sofya search engine implementation with two-phase approach."""
+
+ # Mark as a public, generic (general web) search engine.
+ is_public = True
+ egress_sensitivity = Sensitivity.NON_SENSITIVE
+ egress_exposure = Exposure.EXPOSING
+ is_generic = True
+
+ #: Base URL for the Sofya REST API.
+ BASE_URL = "https://sofya.co/v1"
+ #: Sofya caps ``max_results`` at 20 per request.
+ MAX_RESULTS_CAP = 20
+ #: Sofya accepts at most 10 include/exclude domains per request.
+ MAX_DOMAINS = 10
+ #: Characters of extracted content to use as a preview snippet when the
+ #: SERP ``description`` is missing.
+ SNIPPET_PREVIEW_CHARS = 500
+ #: HTTP request timeout in seconds.
+ REQUEST_TIMEOUT = 30
+
+ def __init__(
+ self,
+ max_results: int = 10,
+ api_key: Optional[str] = None,
+ llm: Optional[BaseLLM] = None,
+ topic: str = "general",
+ freshness: Optional[str] = None,
+ include_domains: Optional[List[str]] = None,
+ exclude_domains: Optional[List[str]] = None,
+ include_full_content: bool = True,
+ search_snippets_only: Optional[bool] = None,
+ max_filtered_results: Optional[int] = None,
+ settings_snapshot: Optional[Dict[str, Any]] = None,
+ **kwargs,
+ ):
+ """Initialize the Sofya search engine.
+
+ Args:
+ max_results: Maximum number of search results to request.
+ api_key: Sofya API key. May also be supplied via the
+ ``search.engine.web.sofya.api_key`` setting or the
+ ``LDR_SEARCH_ENGINE_WEB_SOFYA_API_KEY`` environment variable.
+ llm: Language model for relevance filtering.
+ topic: ``"general"`` or ``"news"``.
+ freshness: Recency filter — ``"day"``, ``"week"``, ``"month"``,
+ ``"year"``, or a ``"YYYY-MM-DD:YYYY-MM-DD"`` range. ``None``
+ means no filter.
+ include_domains: Restrict results to these domains (max 10).
+ exclude_domains: Exclude results from these domains (max 10).
+ include_full_content: Whether to fetch extracted page content.
+ When ``True`` (and full content will be consumed, see
+ ``search_snippets_only``) the request uses Sofya's ``basic``
+ depth (content included); otherwise it uses the cheaper
+ ``snippets`` depth (SERP snippets only).
+ search_snippets_only: Whether to return only search snippets.
+ ``None`` (default) derives it from ``include_full_content``,
+ like TinyFish; an explicit value (e.g. forwarded from the
+ ``search.snippets_only`` setting by the factory) wins.
+ max_filtered_results: Maximum number of results to keep after
+ relevance filtering.
+ settings_snapshot: Settings snapshot for thread context.
+ **kwargs: Additional base-class parameters (e.g.
+ ``programmatic_mode``); forwarded to ``super().__init__``.
+ """
+ if search_snippets_only is None:
+ search_snippets_only = not include_full_content
+ super().__init__(
+ llm=llm,
+ max_filtered_results=max_filtered_results,
+ max_results=max_results,
+ include_full_content=include_full_content,
+ search_snippets_only=search_snippets_only,
+ settings_snapshot=settings_snapshot,
+ **kwargs,
+ )
+
+ self.topic = topic if topic in ("general", "news") else "general"
+ self.freshness = freshness or None
+ self.include_domains = self._ensure_list(include_domains)[
+ : self.MAX_DOMAINS
+ ]
+ self.exclude_domains = self._ensure_list(exclude_domains)[
+ : self.MAX_DOMAINS
+ ]
+
+ # Resolve API key from params, settings snapshot, or env var.
+ self.api_key = self._resolve_api_key(
+ api_key,
+ "search.engine.web.sofya.api_key",
+ engine_name="Sofya",
+ settings_snapshot=settings_snapshot,
+ )
+
+ def _build_payload(self, query: str) -> Dict[str, Any]:
+ """Build the request body for the Sofya ``/search`` endpoint."""
+ # "basic" depth (3 credits) returns extracted page content; "snippets"
+ # (1 credit) returns SERP snippets only. Only pay for content when it
+ # will actually be consumed: run() calls _get_full_content (which
+ # promotes the content) only when search_snippets_only is False.
+ wants_content = (
+ self.include_full_content and not self.search_snippets_only
+ )
+ payload: Dict[str, Any] = {
+ "query": query,
+ "search_depth": "basic" if wants_content else "snippets",
+ "max_results": min(self.MAX_RESULTS_CAP, self.max_results),
+ "topic": self.topic,
+ }
+ if self.freshness:
+ payload["freshness"] = self.freshness
+ if self.include_domains:
+ payload["include_domains"] = self.include_domains
+ if self.exclude_domains:
+ payload["exclude_domains"] = self.exclude_domains
+ return payload
+
+ def _get_previews(self, query: str) -> List[Dict[str, Any]]:
+ """Get preview information from the Sofya search API.
+
+ Args:
+ query: The search query.
+
+ Returns:
+ List of preview dictionaries.
+ """
+ logger.info("Getting search results from Sofya")
+
+ try:
+ # Apply rate limiting before the request.
+ self._last_wait_time = self.rate_tracker.apply_rate_limit(
+ self.engine_type
+ )
+
+ response = safe_post(
+ f"{self.BASE_URL}/search",
+ json=self._build_payload(query),
+ headers={
+ "Authorization": f"Bearer {self.api_key}",
+ "Content-Type": "application/json",
+ },
+ timeout=self.REQUEST_TIMEOUT,
+ )
+
+ # Detect rate limiting from the status code.
+ self._raise_if_rate_limit(response.status_code)
+ response.raise_for_status()
+
+ data = response.json()
+ results = data.get("results", []) or []
+
+ previews = []
+ for i, result in enumerate(results):
+ url = result.get("url", "")
+ # Keep the short SERP ``description`` as the preview snippet
+ # (used for relevance filtering and display); fall back to a
+ # slice of the extracted content when no description is
+ # provided. The extracted page content itself stays in the
+ # ``_full_result`` stash until _get_full_content promotes it,
+ # so snippets-only mode never carries full page text.
+ snippet = result.get("description") or ""
+ if not snippet:
+ snippet = (result.get("content") or "")[
+ : self.SNIPPET_PREVIEW_CHARS
+ ]
+
+ preview = {
+ "id": url or str(i),
+ "title": result.get("title", ""),
+ "link": url,
+ "snippet": snippet,
+ "displayed_link": url,
+ "position": i,
+ "published_date": result.get("published_date"),
+ # Store the full Sofya result for _get_full_content().
+ "_full_result": result,
+ }
+ previews.append(preview)
+
+ self._search_results = previews
+ return previews
+
+ except RateLimitError:
+ raise # Re-raise rate limit errors
+ except requests.exceptions.RequestException as e:
+ safe_msg = self._scrub_error(e)
+ logger.warning(f"Error getting Sofya results: {safe_msg}")
+ # Re-check for rate limiting using the response status code, not
+ # the exception object, so _raise_if_rate_limit gets an int.
+ status_code = getattr(
+ getattr(e, "response", None), "status_code", None
+ )
+ if status_code is not None:
+ self._raise_if_rate_limit(status_code)
+ return []
+ except Exception as e:
+ safe_msg = self._scrub_error(e)
+ logger.warning(
+ f"Unexpected error getting Sofya results: {safe_msg}"
+ )
+ return []
+
+ def _get_full_content(
+ self, relevant_items: List[Dict[str, Any]]
+ ) -> List[Dict[str, Any]]:
+ """Return full content for the relevant results.
+
+ Sofya returns the extracted page text inline (``basic`` depth), and
+ ``_get_previews`` stashes it on each preview under ``_full_result``.
+ Promote it here to ``full_content`` (the field the report's citation
+ handler reads before falling back to ``snippet``) while dropping the
+ internal ``_full_result`` marker and preserving the preview fields,
+ notably ``link``, which the citation handler uses as the source URL.
+ Falls back to the snippet when no extracted content is available
+ (e.g. ``snippets`` depth), so ``full_content`` is always usable.
+
+ Args:
+ relevant_items: List of relevant preview dictionaries.
+
+ Returns:
+ List of result dictionaries with full content where available.
+ """
+ results = []
+ for item in relevant_items:
+ result = {k: v for k, v in item.items() if k != "_full_result"}
+ content = (item.get("_full_result") or {}).get("content") or ""
+ result["full_content"] = (
+ content
+ or result.get("full_content")
+ or result.get("snippet", "")
+ )
+ results.append(result)
+ return results
diff --git a/src/local_deep_research/web_search_engines/search_engine_factory.py b/src/local_deep_research/web_search_engines/search_engine_factory.py
index 7519c9b606..2ed661b51c 100644
--- a/src/local_deep_research/web_search_engines/search_engine_factory.py
+++ b/src/local_deep_research/web_search_engines/search_engine_factory.py
@@ -693,6 +693,11 @@ def get_search(
params["language"] = search_language
params["search_snippets_only"] = search_snippets_only
+ if search_tool == "sofya":
+ # Sofya derives its request depth from this: the paid "basic" depth
+ # (inline page content) is only requested when full content is used.
+ params["search_snippets_only"] = search_snippets_only
+
if search_tool == "wikinews":
params["search_snippets_only"] = search_snippets_only
params["adaptive_search"] = bool(
diff --git a/tests/settings/golden_master_settings.json b/tests/settings/golden_master_settings.json
index 749982fce8..5e32a571b1 100644
--- a/tests/settings/golden_master_settings.json
+++ b/tests/settings/golden_master_settings.json
@@ -6647,6 +6647,227 @@
],
"visible": false
},
+ "search.engine.web.sofya.api_key": {
+ "category": "sofya",
+ "description": "The Sofya API key to use. Get yours at https://sofya.co/dashboard (GitHub sign-up grants 1000 free credits/month).",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Api Key",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "password",
+ "value": "",
+ "visible": true
+ },
+ "search.engine.web.sofya.default_params.freshness": {
+ "category": "sofya",
+ "description": "Filter results by recency ('day', 'week', 'month', 'year', or null for all time).",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Freshness",
+ "options": [
+ {
+ "label": "All Time",
+ "value": null
+ },
+ {
+ "label": "Past Day",
+ "value": "day"
+ },
+ {
+ "label": "Past Week",
+ "value": "week"
+ },
+ {
+ "label": "Past Month",
+ "value": "month"
+ },
+ {
+ "label": "Past Year",
+ "value": "year"
+ }
+ ],
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "select",
+ "value": null,
+ "visible": true
+ },
+ "search.engine.web.sofya.default_params.include_full_content": {
+ "category": "sofya",
+ "description": "Fetch extracted page content. On uses Sofya 'basic' depth (content, 3 credits); off uses cheaper 'snippets' depth (1 credit).",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Include Full Content",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "checkbox",
+ "value": true,
+ "visible": true
+ },
+ "search.engine.web.sofya.default_params.topic": {
+ "category": "sofya",
+ "description": "Search topic. 'general' for broad web search, 'news' for current events.",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Topic",
+ "options": [
+ {
+ "label": "General",
+ "value": "general"
+ },
+ {
+ "label": "News",
+ "value": "news"
+ }
+ ],
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "select",
+ "value": "general",
+ "visible": true
+ },
+ "search.engine.web.sofya.description": {
+ "category": "sofya",
+ "description": "Human-readable description of the search engine.",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Description",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "text",
+ "value": "Hosted web-search API for AI agents. Returns ranked results with SERP snippets and extracted page content (markdown) in a single call.",
+ "visible": false
+ },
+ "search.engine.web.sofya.display_name": {
+ "category": "sofya",
+ "description": "Display name to use in the U.I. for this search engine.",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Display Name",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "text",
+ "value": "Sofya",
+ "visible": false
+ },
+ "search.engine.web.sofya.reliability": {
+ "category": "sofya",
+ "description": "Reliability of the Sofya search engine",
+ "editable": true,
+ "max_value": 1.0,
+ "min_value": 0.0,
+ "name": "Reliability",
+ "options": null,
+ "step": 0.05,
+ "type": "SEARCH",
+ "ui_element": "range",
+ "value": 0.85,
+ "visible": true
+ },
+ "search.engine.web.sofya.requires_api_key": {
+ "category": "sofya",
+ "description": "Whether this search engine requires an API key",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Requires Api Key",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "checkbox",
+ "value": true,
+ "visible": false
+ },
+ "search.engine.web.sofya.requires_llm": {
+ "category": "sofya",
+ "description": "Whether this search engine requires an LLM for relevance filtering",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Requires LLM",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "checkbox",
+ "value": false,
+ "visible": false
+ },
+ "search.engine.web.sofya.strengths": {
+ "category": "sofya",
+ "description": "Strengths of this search engine",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Strengths",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "json",
+ "value": [
+ "Returns SERP snippets and extracted page content (markdown) in one call",
+ "Domain include/exclude filtering and recency filtering",
+ "General and news topics",
+ "1000 free credits/month for GitHub sign-ups",
+ "Built for AI agents"
+ ],
+ "visible": false
+ },
+ "search.engine.web.sofya.supports_full_search": {
+ "category": "sofya",
+ "description": "Whether this search engine supports full document search",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Supports Full Search",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "checkbox",
+ "value": true,
+ "visible": false
+ },
+ "search.engine.web.sofya.use_in_auto_search": {
+ "category": "sofya",
+ "description": "Whether to include this engine in auto/meta search selection",
+ "editable": true,
+ "max_value": null,
+ "min_value": null,
+ "name": "Use In Auto Search",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "checkbox",
+ "value": false,
+ "visible": true
+ },
+ "search.engine.web.sofya.weaknesses": {
+ "category": "sofya",
+ "description": "Weaknesses of this search engine",
+ "editable": false,
+ "max_value": null,
+ "min_value": null,
+ "name": "Weaknesses",
+ "options": null,
+ "step": null,
+ "type": "SEARCH",
+ "ui_element": "json",
+ "value": [
+ "Requires API key with credit-based usage limits",
+ "Not specialized for academic content"
+ ],
+ "visible": false
+ },
"search.engine.web.stackexchange": {
"category": "stackexchange",
"description": "Stack Exchange search engine configuration",
diff --git a/tests/web_search_engines/engines/test_search_engine_sofya.py b/tests/web_search_engines/engines/test_search_engine_sofya.py
new file mode 100644
index 0000000000..25d21de155
--- /dev/null
+++ b/tests/web_search_engines/engines/test_search_engine_sofya.py
@@ -0,0 +1,525 @@
+"""
+Tests for the SofyaSearchEngine class.
+
+Tests cover:
+- Initialization and configuration
+- API key handling
+- Request payload construction
+- Preview generation
+- Full content retrieval
+- Rate limit error handling
+- Run method
+- Class attributes
+"""
+
+from unittest.mock import Mock, patch
+
+import pytest
+
+MODULE = "local_deep_research.web_search_engines.engines.search_engine_sofya"
+
+
+def _make_engine(**kwargs):
+ from local_deep_research.web_search_engines.engines.search_engine_sofya import (
+ SofyaSearchEngine,
+ )
+
+ kwargs.setdefault("api_key", "test-api-key")
+ return SofyaSearchEngine(**kwargs)
+
+
+class TestSofyaSearchEngineInit:
+ """Tests for SofyaSearchEngine initialization."""
+
+ def test_init_with_api_key(self):
+ """Initialize with API key and defaults."""
+ engine = _make_engine()
+
+ assert engine.api_key == "test-api-key"
+ assert engine.max_results == 10
+ assert engine.topic == "general"
+ assert engine.include_full_content is True
+
+ def test_init_with_custom_max_results(self):
+ """Initialize with custom max_results."""
+ engine = _make_engine(max_results=25)
+
+ assert engine.max_results == 25
+
+ def test_init_invalid_topic_falls_back_to_general(self):
+ """An unsupported topic falls back to 'general'."""
+ engine = _make_engine(topic="sports")
+
+ assert engine.topic == "general"
+
+ def test_init_with_domain_filters(self):
+ """Initialize with domain filters."""
+ engine = _make_engine(
+ include_domains=["example.com", "test.com"],
+ exclude_domains=["spam.com"],
+ )
+
+ assert engine.include_domains == ["example.com", "test.com"]
+ assert engine.exclude_domains == ["spam.com"]
+
+ def test_domain_filters_capped_at_max(self):
+ """Domain lists are capped at MAX_DOMAINS entries."""
+ from local_deep_research.web_search_engines.engines.search_engine_sofya import (
+ SofyaSearchEngine,
+ )
+
+ too_many = [f"domain{i}.com" for i in range(15)]
+ engine = _make_engine(include_domains=too_many)
+
+ assert len(engine.include_domains) == SofyaSearchEngine.MAX_DOMAINS
+
+ def test_init_without_api_key_raises(self):
+ """Initialize without API key raises ValueError."""
+ from local_deep_research.web_search_engines.engines.search_engine_sofya import (
+ SofyaSearchEngine,
+ )
+
+ with pytest.raises(
+ ValueError, match="No valid API key found for Sofya"
+ ):
+ SofyaSearchEngine()
+
+ def test_init_with_api_key_from_settings(self):
+ """Initialize with API key from settings snapshot."""
+ from local_deep_research.web_search_engines.engines.search_engine_sofya import (
+ SofyaSearchEngine,
+ )
+
+ engine = SofyaSearchEngine(
+ settings_snapshot={
+ "search.engine.web.sofya.api_key": "settings-api-key"
+ }
+ )
+
+ assert engine.api_key == "settings-api-key"
+
+ def test_init_with_llm(self):
+ """Initialize with LLM."""
+ mock_llm = Mock()
+ engine = _make_engine(llm=mock_llm)
+
+ assert engine.llm is mock_llm
+
+ def test_init_derives_snippets_only_from_include_full_content(self):
+ """Like TinyFish: search_snippets_only defaults to the inverse of
+ include_full_content, so the full-content path actually runs when
+ content is requested."""
+ engine = _make_engine()
+ assert engine.search_snippets_only is False
+
+ engine = _make_engine(include_full_content=False)
+ assert engine.search_snippets_only is True
+
+ def test_init_explicit_snippets_only_wins_over_derivation(self):
+ """An explicit search_snippets_only (e.g. forwarded from the
+ search.snippets_only setting by the factory) beats the derived value."""
+ engine = _make_engine(
+ include_full_content=True, search_snippets_only=True
+ )
+
+ assert engine.search_snippets_only is True
+
+
+class TestBuildPayload:
+ """Tests for _build_payload."""
+
+ def test_payload_basics(self):
+ """Payload includes query, depth, max_results, and topic."""
+ engine = _make_engine(max_results=5)
+
+ payload = engine._build_payload("hello world")
+
+ assert payload["query"] == "hello world"
+ # include_full_content defaults to True (and search_snippets_only
+ # derives to False), so the content will be consumed -> basic depth.
+ assert payload["search_depth"] == "basic"
+ assert payload["max_results"] == 5
+ assert payload["topic"] == "general"
+ # Optional fields are omitted when unset.
+ assert "freshness" not in payload
+ assert "include_domains" not in payload
+ assert "exclude_domains" not in payload
+
+ def test_payload_snippets_depth_when_no_full_content(self):
+ """include_full_content=False maps to the cheaper snippets depth."""
+ engine = _make_engine(include_full_content=False)
+
+ payload = engine._build_payload("q")
+
+ assert payload["search_depth"] == "snippets"
+
+ def test_payload_snippets_depth_when_snippets_only(self):
+ """search_snippets_only=True uses the snippets depth even when
+ include_full_content=True: run() would discard the content, so the
+ 3x-priced basic depth must not be paid for."""
+ engine = _make_engine(
+ include_full_content=True, search_snippets_only=True
+ )
+
+ payload = engine._build_payload("q")
+
+ assert payload["search_depth"] == "snippets"
+
+ def test_payload_caps_max_results(self):
+ """max_results is capped at MAX_RESULTS_CAP in the payload."""
+ engine = _make_engine(max_results=100)
+
+ payload = engine._build_payload("q")
+
+ assert payload["max_results"] == engine.MAX_RESULTS_CAP
+
+ def test_payload_includes_optional_fields(self):
+ """Freshness and domain filters are included when set."""
+ engine = _make_engine(
+ freshness="week",
+ include_domains=["example.com"],
+ exclude_domains=["spam.com"],
+ )
+
+ payload = engine._build_payload("q")
+
+ assert payload["freshness"] == "week"
+ assert payload["include_domains"] == ["example.com"]
+ assert payload["exclude_domains"] == ["spam.com"]
+
+
+class TestGetPreviews:
+ """Tests for _get_previews method."""
+
+ def test_get_previews_returns_results(self):
+ """Get previews returns formatted results."""
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {
+ "results": [
+ {
+ "title": "Result 1",
+ "url": "https://example1.com",
+ "description": "Snippet 1",
+ "content": "Full content 1",
+ },
+ {
+ "title": "Result 2",
+ "url": "https://example2.com",
+ "description": "Snippet 2",
+ "content": "Full content 2",
+ },
+ ]
+ }
+
+ with patch(f"{MODULE}.safe_post", return_value=mock_response):
+ engine = _make_engine()
+ previews = engine._get_previews("test query")
+
+ assert len(previews) == 2
+ assert previews[0]["title"] == "Result 1"
+ assert previews[0]["snippet"] == "Snippet 1"
+ assert previews[0]["link"] == "https://example1.com"
+ assert previews[0]["_full_result"]["content"] == "Full content 1"
+ # The extracted content must stay in the _full_result stash until
+ # _get_full_content promotes it: previews are returned as-is in
+ # snippets-only mode, so full_content here would leak full page
+ # text into the synthesis prompt.
+ assert "full_content" not in previews[0]
+
+ def test_get_previews_snippet_falls_back_to_content(self):
+ """When description is missing, snippet falls back to content slice."""
+ long_content = "x" * 1000
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {
+ "results": [
+ {
+ "title": "No description",
+ "url": "https://example.com",
+ "content": long_content,
+ }
+ ]
+ }
+
+ with patch(f"{MODULE}.safe_post", return_value=mock_response):
+ engine = _make_engine()
+ previews = engine._get_previews("test query")
+
+ assert len(previews[0]["snippet"]) == engine.SNIPPET_PREVIEW_CHARS
+
+ def test_get_previews_sends_auth_header(self):
+ """Get previews sends the bearer auth header."""
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {"results": []}
+
+ with patch(
+ f"{MODULE}.safe_post", return_value=mock_response
+ ) as mock_post:
+ engine = _make_engine()
+ engine._get_previews("test query")
+
+ headers = mock_post.call_args[1]["headers"]
+ assert headers["Authorization"] == "Bearer test-api-key"
+
+ def test_get_previews_rate_limit_error(self):
+ """Get previews raises RateLimitError on 429."""
+ from local_deep_research.web_search_engines.rate_limiting import (
+ RateLimitError,
+ )
+
+ mock_response = Mock()
+ mock_response.status_code = 429
+ mock_response.text = "Rate limit exceeded"
+
+ with patch(f"{MODULE}.safe_post", return_value=mock_response):
+ engine = _make_engine()
+
+ with pytest.raises(RateLimitError):
+ engine._get_previews("test query")
+
+ def test_get_previews_empty_results(self):
+ """Get previews handles empty results."""
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {"results": []}
+
+ with patch(f"{MODULE}.safe_post", return_value=mock_response):
+ engine = _make_engine()
+ previews = engine._get_previews("test query")
+
+ assert previews == []
+
+ def test_get_previews_request_exception(self):
+ """Get previews handles request exceptions gracefully."""
+ import requests
+
+ with patch(
+ f"{MODULE}.safe_post",
+ side_effect=requests.exceptions.RequestException(
+ "Connection error"
+ ),
+ ):
+ engine = _make_engine()
+ previews = engine._get_previews("test query")
+
+ assert previews == []
+
+ def test_get_previews_request_exception_with_rate_limit_status(self):
+ """A RequestException carrying a 429 response raises RateLimitError."""
+ import requests
+
+ from local_deep_research.web_search_engines.rate_limiting import (
+ RateLimitError,
+ )
+
+ err = requests.exceptions.RequestException("boom")
+ err.response = Mock(status_code=429)
+
+ with patch(f"{MODULE}.safe_post", side_effect=err):
+ engine = _make_engine()
+
+ with pytest.raises(RateLimitError):
+ engine._get_previews("test query")
+
+ def test_get_previews_unexpected_error(self):
+ """Get previews handles unexpected errors."""
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_response.json.side_effect = Exception("Parse error")
+ mock_response.raise_for_status = Mock()
+
+ with patch(f"{MODULE}.safe_post", return_value=mock_response):
+ engine = _make_engine()
+ previews = engine._get_previews("test query")
+
+ assert previews == []
+
+
+class TestGetFullContent:
+ """Tests for _get_full_content method."""
+
+ def test_get_full_content_promotes_stashed_content(self):
+ """The extracted content is promoted from the _full_result stash to
+ full_content, the citation link survives, and the internal
+ _full_result marker is dropped."""
+ engine = _make_engine(include_full_content=True)
+
+ items = [
+ {
+ "title": "Result",
+ "link": "https://example.com",
+ "snippet": "Short snippet",
+ "_full_result": {
+ "url": "https://example.com",
+ "content": "Full extracted markdown content",
+ },
+ }
+ ]
+
+ results = engine._get_full_content(items)
+
+ assert len(results) == 1
+ assert results[0]["full_content"] == "Full extracted markdown content"
+ assert results[0]["link"] == "https://example.com"
+ assert "_full_result" not in results[0]
+
+ def test_get_full_content_falls_back_to_snippet(self):
+ """When the stash carries no extracted content (e.g. snippets depth),
+ fall back to the snippet."""
+ engine = _make_engine(include_full_content=True)
+
+ items = [
+ {
+ "title": "Result",
+ "link": "https://example.com",
+ "snippet": "Only a snippet",
+ "_full_result": {"url": "https://example.com"},
+ }
+ ]
+
+ results = engine._get_full_content(items)
+
+ assert results[0]["full_content"] == "Only a snippet"
+
+ def test_get_full_content_without_full_result(self):
+ """Get full content handles items without _full_result."""
+ engine = _make_engine(include_full_content=False)
+
+ items = [{"title": "Result", "link": "https://example.com"}]
+
+ results = engine._get_full_content(items)
+
+ assert len(results) == 1
+ assert results[0]["title"] == "Result"
+
+
+class TestRun:
+ """Tests for run method."""
+
+ def test_run_returns_results(self):
+ """Run promotes the stashed content to full_content."""
+ engine = _make_engine()
+
+ with patch.object(
+ engine,
+ "_get_previews",
+ return_value=[
+ {
+ "id": "https://example.com",
+ "title": "Result",
+ "snippet": "Snippet",
+ "link": "https://example.com",
+ "_full_result": {
+ "url": "https://example.com",
+ "content": "Full page text",
+ },
+ }
+ ],
+ ):
+ results = engine.run("test query")
+
+ assert len(results) == 1
+ # full_content is the field the report's citation handler reads
+ # before falling back to the snippet.
+ assert results[0]["full_content"] == "Full page text"
+ assert "_full_result" not in results[0]
+
+ def test_run_surfaces_full_content_end_to_end(self):
+ """End-to-end (real _get_previews): the inline extracted page content
+ reaches run() output under full_content, while the snippet stays the
+ short SERP description. Guards against the depth/credits being paid for
+ content that is then discarded."""
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_response.raise_for_status = Mock()
+ mock_response.json.return_value = {
+ "results": [
+ {
+ "title": "Result",
+ "url": "https://example.com",
+ "description": "Short SERP snippet",
+ "content": "Full extracted page markdown",
+ }
+ ]
+ }
+
+ with patch(f"{MODULE}.safe_post", return_value=mock_response):
+ engine = _make_engine(include_full_content=True)
+ results = engine.run("test query")
+
+ assert len(results) == 1
+ assert results[0]["full_content"] == "Full extracted page markdown"
+ assert results[0]["snippet"] == "Short SERP snippet"
+ assert results[0]["link"] == "https://example.com"
+
+ def test_run_snippets_only_carries_no_full_content(self):
+ """Regression: in snippets-only mode run() returns the previews
+ directly (no _get_full_content call), so they must not carry
+ full_content - otherwise full page text would leak into the
+ synthesis prompt despite the setting."""
+ mock_response = Mock()
+ mock_response.status_code = 200
+ mock_response.raise_for_status = Mock()
+ mock_response.json.return_value = {
+ "results": [
+ {
+ "title": "Result",
+ "url": "https://example.com",
+ "description": "Short SERP snippet",
+ "content": "Full extracted page markdown",
+ }
+ ]
+ }
+
+ with patch(f"{MODULE}.safe_post", return_value=mock_response):
+ engine = _make_engine(search_snippets_only=True)
+ results = engine.run("test query")
+
+ assert len(results) == 1
+ assert "full_content" not in results[0]
+ assert results[0]["snippet"] == "Short SERP snippet"
+
+ def test_run_handles_empty_results(self):
+ """Run handles empty results."""
+ engine = _make_engine()
+
+ with patch.object(engine, "_get_previews", return_value=[]):
+ results = engine.run("test query")
+
+ assert results == []
+
+
+class TestClassAttributes:
+ """Tests for class attributes."""
+
+ def test_is_public(self):
+ """SofyaSearchEngine is marked as public."""
+ from local_deep_research.web_search_engines.engines.search_engine_sofya import (
+ SofyaSearchEngine,
+ )
+
+ assert SofyaSearchEngine.is_public is True
+
+ def test_is_generic(self):
+ """SofyaSearchEngine is marked as generic."""
+ from local_deep_research.web_search_engines.engines.search_engine_sofya import (
+ SofyaSearchEngine,
+ )
+
+ assert SofyaSearchEngine.is_generic is True
+
+ def test_egress_labels_match_public_engine(self):
+ """A public engine must declare NON_SENSITIVE/EXPOSING egress labels
+ (enforced by test_registered_engines_declare_consistent_labels)."""
+ from local_deep_research.security.egress.classification import (
+ Exposure,
+ Sensitivity,
+ )
+ from local_deep_research.web_search_engines.engines.search_engine_sofya import (
+ SofyaSearchEngine,
+ )
+
+ assert SofyaSearchEngine.egress_sensitivity is Sensitivity.NON_SENSITIVE
+ assert SofyaSearchEngine.egress_exposure is Exposure.EXPOSING
diff --git a/tests/web_search_engines/test_search_engine_factory_coverage.py b/tests/web_search_engines/test_search_engine_factory_coverage.py
index 8885125a6d..b457a21b01 100644
--- a/tests/web_search_engines/test_search_engine_factory_coverage.py
+++ b/tests/web_search_engines/test_search_engine_factory_coverage.py
@@ -912,6 +912,28 @@ class TestGetSearchParameterRouting:
assert call_kwargs["search_language"] == "French"
assert call_kwargs["adaptive_search"] is False
+ def test_get_search_sofya_params(self):
+ """Sofya gets search_snippets_only forwarded: it derives its request
+ depth from it (paid 'basic' depth only when content is consumed)."""
+ from local_deep_research.web_search_engines.search_engine_factory import (
+ get_search,
+ )
+
+ with patch(
+ "local_deep_research.web_search_engines.search_engine_factory.create_search_engine"
+ ) as mock_create:
+ mock_create.return_value = Mock(run=Mock())
+
+ get_search(
+ search_tool="sofya",
+ llm_instance=Mock(),
+ search_snippets_only=True,
+ settings_snapshot={"x": 1},
+ )
+
+ call_kwargs = mock_create.call_args[1]
+ assert call_kwargs["search_snippets_only"] is True
+
def test_get_search_max_filtered_results(self):
"""max_filtered_results is passed when provided."""
from local_deep_research.web_search_engines.search_engine_factory import (