Skip to content

Personal AI, On Personal Devices

Python
32.4 KB
Raw Blame
"""Web search tool — You.com or Tavily, with a DuckDuckGo fallback.

Engine selection is explicit (``engine=`` or ``OPENJARVIS_WEB_SEARCH_ENGINE``)
rather than a chain of ``try``/``except`` layers. The default, ``"auto"``,
resolves to whichever engine the environment can actually serve, preferring an
API-backed engine over the DuckDuckGo HTML scrape:

* ``TAVILY_API_KEY`` set → Tavily (unchanged for existing installs)
* ``YOUDOTCOM_API_KEY`` set → You.com, keyed
* ``SERPLY_API_KEY`` set → Serply, a Google SERP proxy
* ``FIRECRAWL_API_KEY`` set, or ``FIRECRAWL_API_URL`` pointing at a
  self-hosted instance → Firecrawl, which also returns URL lookups (including
  PDFs) as markdown
* none of them → You.com, keyless free tier (no signup, rate limited per IP)

The keyless tier is what makes a fresh install API-backed with zero config.
DuckDuckGo remains the last resort for every engine, and dropping to it is now
logged at WARNING with the reason, so a typo'd key no longer looks identical to
a working install with quieter results.
"""

from __future__ import annotations

import logging
import os
from typing import Any
from urllib.parse import urljoin

from openjarvis import __version__
from openjarvis.core.registry import ToolRegistry
from openjarvis.core.types import ToolResult
from openjarvis.security.ssrf import check_ssrf
from openjarvis.tools._stubs import BaseTool, ToolSpec

logger = logging.getLogger(__name__)


YOUCOM_KEYED_SEARCH_URL = "https://api.you.com/v1/search"
YOUCOM_KEYLESS_SEARCH_URL = "https://api.you.com/v1/agents/search"
YOUCOM_CONTENTS_URL = "https://api.you.com/v1/contents"
YOUCOM_API_KEY_ENV = "YOUDOTCOM_API_KEY"

SERPLY_SEARCH_URL = "https://api.serply.io/v1/search"
SERPLY_API_KEY_ENV = "SERPLY_API_KEY"
# Two-letter country code asking Serply for a locale-specific Google result
# set. Unset means the API answers from its own default region.
SERPLY_LOCATION_ENV = "SERPLY_PROXY_LOCATION"

FIRECRAWL_API_KEY_ENV = "FIRECRAWL_API_KEY"
# Base URL of the Firecrawl API. Point it at a self-hosted Firecrawl to run
# page fetches on an instance you operate; that instance still sends search
# queries to its own configured search backend.
FIRECRAWL_API_URL_ENV = "FIRECRAWL_API_URL"
FIRECRAWL_DEFAULT_API_URL = "https://api.firecrawl.dev"
FIRECRAWL_ORIGIN = "openjarvis"
# PDFs are billed per page, and only the first max_chars of the markdown are
# kept, so parsing stops after a few pages.
FIRECRAWL_PDF_MAX_PAGES = 3
# Server-side scrape timeout, below the client timeout so a slow page fails on
# Firecrawl's side first instead of being abandoned mid-scrape.
FIRECRAWL_SCRAPE_TIMEOUT_MS = 45000
# Search descriptions can run to several thousand characters of page text, so
# each one is cut to a snippet that a small local model can take five of.
FIRECRAWL_SNIPPET_CHARS = 500
FIRECRAWL_KEYS_URL = (
    "https://www.firecrawl.dev/app/api-keys"
    "?utm_source=openjarvis&utm_medium=integration"
)

ENGINE_ENV = "OPENJARVIS_WEB_SEARCH_ENGINE"
ENGINES = ("auto", "youcom", "tavily", "duckduckgo", "serply", "firecrawl")

# Identifies OpenJarvis to You.com. The keyless tier carries no API key, so the
# User-Agent is the only attribution signal; sent to You.com hosts only.
YOUCOM_USER_AGENT = (
    f"openjarvis/{__version__} youdotcom-integration/open-jarvis-openjarvis"
)

_MAX_FETCH_REDIRECTS = 5
_REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})

# Keyless tier exhaustion (402) and per-IP throttling (429) both mean "get a
# key", which is a different remedy from a generic HTTP failure.
_KEYLESS_LIMIT_STATUSES = (402, 429)
YOUCOM_PLATFORM_URL = (
    "https://you.com/platform"
    "?utm_source=open-jarvis-openjarvis&utm_medium=oss_integration"
    "&utm_campaign=2026-09-oss-integrations&utm_content=error-message"
)
_KEY_UPGRADE_HINT = (
    f"You.com keyless free-tier limit reached. Set {YOUCOM_API_KEY_ENV} for "
    f"higher limits — a free key is available at {YOUCOM_PLATFORM_URL}"
)


class _WebSearchEngineError(RuntimeError):
    """An engine failed in a way that should be reported, not swallowed."""


@ToolRegistry.register("web_search")
class WebSearchTool(BaseTool):
    """Search the web via You.com or Tavily, falling back to DuckDuckGo."""

    tool_id = "web_search"
    is_local = False

    def __init__(
        self,
        api_key: str | None = None,
        max_results: int = 5,
        *,
        engine: str | None = None,
        youcom_api_key: str | None = None,
        serply_api_key: str | None = None,
        serply_location: str | None = None,
        firecrawl_api_key: str | None = None,
        firecrawl_api_url: str | None = None,
    ):
        """Configure the search engine.

        ``api_key`` remains the Tavily key, positionally, for backwards
        compatibility. ``engine`` is one of :data:`ENGINES`; when omitted it
        comes from ``OPENJARVIS_WEB_SEARCH_ENGINE`` and defaults to ``"auto"``.
        An unknown engine name falls back to ``"auto"`` with a warning rather
        than raising, so a typo in the environment cannot break tool loading.
        """
        self._api_key = api_key or os.environ.get("TAVILY_API_KEY")
        self._youcom_api_key = youcom_api_key or os.environ.get(YOUCOM_API_KEY_ENV)
        self._serply_api_key = serply_api_key or os.environ.get(SERPLY_API_KEY_ENV)
        self._serply_location = serply_location or os.environ.get(SERPLY_LOCATION_ENV)
        self._firecrawl_api_key = (
            firecrawl_api_key or os.environ.get(FIRECRAWL_API_KEY_ENV) or ""
        ).strip() or None
        self._firecrawl_api_url = (
            (firecrawl_api_url or os.environ.get(FIRECRAWL_API_URL_ENV) or "")
            .strip()
            .rstrip("/")
        ) or FIRECRAWL_DEFAULT_API_URL
        # Self-hosted instances usually run without auth, so a custom URL
        # enables the engine even when no key is set.
        self._firecrawl_self_hosted = (
            self._firecrawl_api_url != FIRECRAWL_DEFAULT_API_URL
        )
        self._max_results = max_results

        requested = (engine or os.environ.get(ENGINE_ENV) or "auto").strip().lower()
        if requested not in ENGINES:
            logger.warning(
                "Unknown web search engine %r (expected one of %s); using 'auto'",
                requested,
                ", ".join(ENGINES),
            )
            requested = "auto"
        self._engine = requested

    def _resolve_engine(self) -> str:
        """Resolve ``auto`` to a concrete engine from what the env can serve.

        Keyed engines come first, in the order they were added, so every
        install that already had a key resolves exactly as it did before:
        Tavily, then keyed You.com, then Serply, then Firecrawl. The keyless
        You.com tier is last and remains the zero-config API-backed path, which
        means a Serply or Firecrawl key only ever wins over having no key at
        all.
        """
        if self._engine != "auto":
            return self._engine
        if self._api_key:
            return "tavily"
        if self._youcom_api_key:
            return "youcom"
        if self._serply_api_key:
            return "serply"
        if self._firecrawl_api_key or self._firecrawl_self_hosted:
            return "firecrawl"
        return "youcom"

    @property
    def spec(self) -> ToolSpec:
        engine = self._resolve_engine()
        return ToolSpec(
            name="web_search",
            description=(
                "Search the web for current information."
                " Returns relevant search results."
            ),
            parameters={
                "type": "object",
                "properties": {
                    "query": {"type": "string", "description": "Search query."},
                    "max_results": {
                        "type": "integer",
                        "description": "Maximum results to return.",
                    },
                },
                "required": ["query"],
            },
            category="search",
            metadata={
                # Kept for backwards compatibility: existing readers (Settings
                # UI, tools endpoint) expect a single Tavily key name here.
                "requires_api_key": "TAVILY_API_KEY",
                # web_search needs no key at all on the You.com keyless tier,
                # so the keys above are upgrades rather than prerequisites.
                "optional_api_keys": [
                    "TAVILY_API_KEY",
                    YOUCOM_API_KEY_ENV,
                    SERPLY_API_KEY_ENV,
                    FIRECRAWL_API_KEY_ENV,
                ],
                "engine": engine,
                "engines": list(ENGINES),
                "fallback": "duckduckgo",
            },
        )

    @staticmethod
    def _is_url(text: str) -> bool:
        """Check if text is a URL."""
        stripped = text.strip()
        return stripped.startswith("http://") or stripped.startswith("https://")

    @staticmethod
    def _extract_url(text: str) -> str | None:
        """Extract the first URL from text, if any."""
        import re as _re

        match = _re.search(r"https?://[^\s,;\"'<>]+", text)
        return match.group(0).rstrip(".,;)") if match else None

    @staticmethod
    def _normalize_url(url: str) -> str:
        """Convert known PDF URLs to their HTML equivalents."""
        import re as _re

        # arxiv: /pdf/ID → /abs/ID (abstract page with full metadata)
        m = _re.match(r"(https?://arxiv\.org)/pdf/(.+?)(?:\.pdf)?$", url)
        if m:
            return f"{m.group(1)}/abs/{m.group(2)}"
        return url

    @staticmethod
    def _fetch_url(url: str, max_chars: int = 6000) -> str:
        """Fetch a URL and return extracted text after checking each redirect."""
        import re as _re

        import httpx

        url = WebSearchTool._normalize_url(url)
        current_url = url.strip()
        # One client preserves cookie scope across redirect hops. Its lifetime
        # is limited to this fetch so unrelated requests do not share cookies.
        with httpx.Client(
            follow_redirects=False,
            timeout=30.0,
            headers={
                "User-Agent": "Mozilla/5.0 (compatible; OpenJarvis/1.0; +https://github.com/openjarvis)"
            },
        ) as client:
            for _ in range(_MAX_FETCH_REDIRECTS + 1):
                ssrf_error = check_ssrf(current_url)
                if ssrf_error:
                    raise ValueError(ssrf_error)
                resp = client.get(current_url)
                if resp.status_code not in _REDIRECT_STATUS_CODES:
                    resp.raise_for_status()
                    break
                location = resp.headers.get("location", "")
                if not location:
                    resp.raise_for_status()
                    break
                current_url = urljoin(str(resp.url), location)
            else:
                raise ValueError(
                    f"URL exceeded the maximum of {_MAX_FETCH_REDIRECTS} redirects"
                )
        content_type = resp.headers.get("content-type", "")
        if "application/pdf" in content_type:
            return (
                "[This URL points to a PDF file which"
                f" cannot be read directly. URL: {url}]"
            )
        html = resp.text
        # Strip script/style tags and their contents
        html = _re.sub(
            r"<(script|style)[^>]*>.*?</\1>",
            "",
            html,
            flags=_re.DOTALL | _re.IGNORECASE,
        )
        # Strip HTML tags
        text = _re.sub(r"<[^>]+>", " ", html)
        # Collapse whitespace
        text = _re.sub(r"\s+", " ", text).strip()
        if len(text) > max_chars:
            text = text[:max_chars] + "\n\n[Content truncated]"
        return text

    def _youcom_request(self, path_is_keyed: bool) -> tuple[str, dict[str, str]]:
        """Pick the You.com endpoint and its headers together.

        The keyless endpoint rejects an API key, and the keyed endpoint
        requires one, so URL and headers must be chosen in one step — a stale
        key sent to the keyless path 401s.
        """
        headers = {"Accept": "application/json", "User-Agent": YOUCOM_USER_AGENT}
        if path_is_keyed:
            headers["X-API-Key"] = self._youcom_api_key or ""
            return YOUCOM_KEYED_SEARCH_URL, headers
        return YOUCOM_KEYLESS_SEARCH_URL, headers

    @staticmethod
    def _format_youcom_results(payload: dict[str, Any]) -> tuple[str, int]:
        """Render a You.com search payload in the shared result format.

        Web results come before news. Snippets carry the page text the agent
        should synthesize from; ``description`` is the fallback when a result
        has none.
        """
        results = payload.get("results") or {}
        parts: list[str] = []
        count = 0
        for section in ("web", "news"):
            for item in results.get(section) or []:
                title = item.get("title") or "Untitled"
                url = item.get("url", "")
                snippets = item.get("snippets") or []
                content = (
                    "\n".join(snippets) if snippets else item.get("description", "")
                )
                parts.append(f"### {title}\nSource: {url}\nSummary: {content}")
                count += 1
        return "\n\n---\n\n".join(parts), count

    def _youcom_search(self, query: str, max_results: int) -> ToolResult:
        """Search via You.com, keyed when a key is set and keyless otherwise."""
        import httpx

        keyed = bool(self._youcom_api_key)
        url, headers = self._youcom_request(keyed)
        try:
            response = httpx.get(
                url,
                params={"query": query, "count": max_results},
                headers=headers,
                timeout=30.0,
            )
            response.raise_for_status()
            payload = response.json()
        except httpx.HTTPStatusError as exc:
            status = exc.response.status_code
            if not keyed and status in _KEYLESS_LIMIT_STATUSES:
                raise _WebSearchEngineError(_KEY_UPGRADE_HINT) from exc
            raise _WebSearchEngineError(
                f"You.com search failed with HTTP {status}"
            ) from exc
        except (httpx.HTTPError, ValueError) as exc:
            raise _WebSearchEngineError(f"You.com search failed: {exc}") from exc

        formatted, count = self._format_youcom_results(payload)
        return ToolResult(
            tool_name="web_search",
            content=formatted or "No results found.",
            success=True,
            metadata={
                "num_results": count,
                "engine": "youcom",
                "youcom_tier": "keyed" if keyed else "keyless",
            },
        )

    def _youcom_extract(self, url: str) -> str | None:
        """Return You.com-extracted markdown for ``url``, or ``None``.

        The Contents API needs a key, so this is an upgrade over the regex HTML
        strip in :meth:`_fetch_url` rather than a replacement for it. Any
        failure returns ``None`` and the caller falls back to the local fetch.
        """
        if not self._youcom_api_key:
            return None

        import httpx

        try:
            response = httpx.post(
                YOUCOM_CONTENTS_URL,
                json={"urls": [url], "formats": ["markdown"]},
                headers={
                    "Accept": "application/json",
                    "Content-Type": "application/json",
                    "User-Agent": YOUCOM_USER_AGENT,
                    "X-API-Key": self._youcom_api_key,
                },
                timeout=30.0,
            )
            response.raise_for_status()
            payload = response.json()
        except (httpx.HTTPError, ValueError) as exc:
            logger.warning(
                "You.com Contents extraction failed for %s (%s); falling back to "
                "local HTML extraction",
                url,
                type(exc).__name__,
            )
            return None

        items = payload if isinstance(payload, list) else payload.get("results") or []
        for item in items:
            markdown = (item or {}).get("markdown")
            if markdown:
                return str(markdown)
        return None

    @staticmethod
    def _format_serply_results(payload: dict[str, Any]) -> tuple[str, int]:
        """Render a Serply search payload in the shared result format.

        Only ``results`` is mapped. The payload also carries ``answers``,
        ``knowledge_graph`` and ``related_questions``, but those are filled for
        some queries and empty for others, so nothing here depends on them.
        """
        parts: list[str] = []
        count = 0
        for item in payload.get("results") or []:
            title = item.get("title") or "Untitled"
            url = item.get("link", "")
            content = item.get("description", "")
            parts.append(f"### {title}\nSource: {url}\nSummary: {content}")
            count += 1
        return "\n\n---\n\n".join(parts), count

    def _serply_search(self, query: str, max_results: int) -> ToolResult:
        """Search via Serply, which proxies Google.

        The query reaches Google as written, so operators survive the round
        trip: ``site:`` restricts the corpus and ``after:``/``before:`` bound
        results by date, neither of which an AI-search index can honour.
        Setting ``SERPLY_PROXY_LOCATION`` to a country code asks for that
        country's result set instead of the API's default region.
        """
        import httpx

        headers = {
            "Accept": "application/json",
            "X-Api-Key": self._serply_api_key or "",
        }
        if self._serply_location:
            headers["X-Proxy-Location"] = self._serply_location
        try:
            response = httpx.get(
                SERPLY_SEARCH_URL,
                params={"q": query, "num": max_results},
                headers=headers,
                timeout=30.0,
            )
            response.raise_for_status()
            payload = response.json()
        except httpx.HTTPStatusError as exc:
            status = exc.response.status_code
            if status == 401:
                raise _WebSearchEngineError(
                    f"Serply rejected the credential in {SERPLY_API_KEY_ENV}"
                ) from exc
            raise _WebSearchEngineError(
                f"Serply search failed with HTTP {status}"
            ) from exc
        except (httpx.HTTPError, ValueError) as exc:
            raise _WebSearchEngineError(f"Serply search failed: {exc}") from exc

        formatted, count = self._format_serply_results(payload)
        metadata: dict[str, Any] = {"num_results": count, "engine": "serply"}
        if self._serply_location:
            metadata["serply_location"] = self._serply_location
        return ToolResult(
            tool_name="web_search",
            content=formatted or "No results found.",
            success=True,
            metadata=metadata,
        )

    def _firecrawl_headers(self) -> dict[str, str]:
        headers = {"Accept": "application/json", "Content-Type": "application/json"}
        if self._firecrawl_api_key:
            headers["Authorization"] = f"Bearer {self._firecrawl_api_key}"
        return headers

    def _firecrawl_endpoint(self, operation: str) -> tuple[str, tuple[str, str] | None]:
        """Keep self-hosted Basic Auth out of URLs exposed by HTTPX logging."""
        import httpx

        url = httpx.URL(f"{self._firecrawl_api_url}/v2/{operation}")
        auth = None
        if url.username or url.password:
            # HTTPX already uses URL userinfo as Basic Auth, overriding a
            # Bearer header when both are present. Preserve that behavior while
            # ensuring its INFO request log contains only the endpoint URL.
            auth = (url.username, url.password)
            url = url.copy_with(username=None, password=None)
        return str(url), auth

    @staticmethod
    def _format_firecrawl_results(payload: dict[str, Any]) -> tuple[str, int]:
        """Render a Firecrawl search payload in the shared result format.

        Only snippets are returned, each description cut to
        :data:`FIRECRAWL_SNIPPET_CHARS`. Search can also scrape every result
        page, but full pages per result would crowd out a small local model's
        context, so URL lookups are the place to read a page in full.
        """
        data = payload.get("data") if isinstance(payload, dict) else None
        if not isinstance(data, dict):
            data = {}
        parts: list[str] = []
        count = 0
        for item in data.get("web") or []:
            title = item.get("title") or "Untitled"
            url = item.get("url", "")
            content = item.get("description") or ""
            if len(content) > FIRECRAWL_SNIPPET_CHARS:
                content = content[:FIRECRAWL_SNIPPET_CHARS].rstrip() + "…"
            parts.append(f"### {title}\nSource: {url}\nSummary: {content}")
            count += 1
        return "\n\n---\n\n".join(parts), count

    def _firecrawl_search(self, query: str, max_results: int) -> ToolResult:
        """Search via Firecrawl, or the instance at ``FIRECRAWL_API_URL``."""
        import httpx

        if not self._firecrawl_api_key and not self._firecrawl_self_hosted:
            raise _WebSearchEngineError(f"{FIRECRAWL_API_KEY_ENV} is not set")
        try:
            endpoint, auth = self._firecrawl_endpoint("search")
            response = httpx.post(
                endpoint,
                json={
                    "query": query,
                    "limit": max_results,
                    "origin": FIRECRAWL_ORIGIN,
                },
                headers=self._firecrawl_headers(),
                auth=auth,
                timeout=30.0,
            )
            response.raise_for_status()
            payload = response.json()
        except httpx.HTTPStatusError as exc:
            status = exc.response.status_code
            if status == 401:
                raise _WebSearchEngineError(
                    f"Firecrawl rejected the credential in {FIRECRAWL_API_KEY_ENV}"
                ) from None
            if status == 402:
                raise _WebSearchEngineError(
                    "Firecrawl credits are exhausted for this key. Check the "
                    f"plan or create a new key at {FIRECRAWL_KEYS_URL}"
                ) from None
            if status == 429:
                raise _WebSearchEngineError(
                    "Firecrawl rate limit reached (HTTP 429)"
                ) from None
            raise _WebSearchEngineError(
                f"Firecrawl search failed with HTTP {status}"
            ) from None
        except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc:
            # Header validation can include the Authorization value; URL
            # parsing errors can include credentials from a self-hosted URL.
            # execute() logs this message and returns it in fallback metadata.
            raise _WebSearchEngineError(
                f"Firecrawl search failed ({type(exc).__name__})"
            ) from None

        formatted, count = self._format_firecrawl_results(payload)
        metadata: dict[str, Any] = {"num_results": count, "engine": "firecrawl"}
        if isinstance(payload, dict) and "creditsUsed" in payload:
            metadata["credits"] = payload["creditsUsed"]
        return ToolResult(
            tool_name="web_search",
            content=formatted or "No results found.",
            success=True,
            metadata=metadata,
        )

    def _firecrawl_extract(self, url: str, max_chars: int = 6000) -> str | None:
        """Return Firecrawl-scraped markdown for ``url``, or ``None``.

        Firecrawl renders JavaScript and parses PDFs, which the regex HTML
        strip in :meth:`_fetch_url` cannot. Output is truncated at the same
        ``max_chars`` as the local path so a long PDF cannot flood the
        context, and PDF parsing stops after :data:`FIRECRAWL_PDF_MAX_PAGES`
        pages since PDFs are billed per page. Any failure returns ``None`` and
        the caller falls back to the local fetch.
        """
        if not self._firecrawl_api_key and not self._firecrawl_self_hosted:
            return None

        import httpx

        try:
            endpoint, auth = self._firecrawl_endpoint("scrape")
            response = httpx.post(
                endpoint,
                json={
                    "url": url,
                    "formats": ["markdown"],
                    "onlyMainContent": True,
                    "parsers": [{"type": "pdf", "maxPages": FIRECRAWL_PDF_MAX_PAGES}],
                    "timeout": FIRECRAWL_SCRAPE_TIMEOUT_MS,
                    "origin": FIRECRAWL_ORIGIN,
                },
                headers=self._firecrawl_headers(),
                auth=auth,
                timeout=60.0,
            )
            response.raise_for_status()
            payload = response.json()
        except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc:
            reason = type(exc).__name__
            if isinstance(exc, httpx.HTTPStatusError):
                reason = f"HTTP {exc.response.status_code}"
            logger.warning(
                "Firecrawl scrape failed (%s); falling back to local HTML extraction",
                reason,
            )
            return None

        data = payload.get("data") if isinstance(payload, dict) else None
        markdown = data.get("markdown") if isinstance(data, dict) else None
        if not markdown:
            return None
        text = str(markdown)
        if len(text) > max_chars:
            text = text[:max_chars] + "\n\n[Content truncated]"
        return text

    def _duckduckgo_search(self, query: str, max_results: int) -> str:
        """Search using DuckDuckGo as fallback."""
        from ddgs import DDGS

        ddgs = DDGS()
        raw_results = list(ddgs.text(query, max_results=max_results))
        results = []
        for r in raw_results:
            title = r.get("title", "Untitled")
            url = r.get("href", "")
            snippet = r.get("body", "")
            results.append(f"### {title}\nSource: {url}\nSummary: {snippet}")

        formatted = "\n\n---\n\n".join(results)
        return formatted

    def execute(self, **params: Any) -> ToolResult:
        query = params.get("query", "")
        if not query:
            return ToolResult(
                tool_name="web_search",
                content="No query provided.",
                success=False,
            )

        # If the query contains a URL, fetch it directly instead of searching
        url = self._extract_url(query) if not self._is_url(query) else query.strip()
        if url:
            if self._resolve_engine() == "firecrawl":
                ssrf_error = check_ssrf(self._normalize_url(url))
                if ssrf_error:
                    return ToolResult(
                        tool_name="web_search",
                        content=f"Failed to fetch URL: {ssrf_error}",
                        success=False,
                    )
                extracted = self._firecrawl_extract(url)
                if extracted:
                    return ToolResult(
                        tool_name="web_search",
                        content=extracted,
                        success=True,
                        metadata={
                            "url": url,
                            "mode": "fetch",
                            "extractor": "firecrawl_scrape",
                        },
                    )
            if self._resolve_engine() == "youcom":
                ssrf_error = check_ssrf(self._normalize_url(url))
                if ssrf_error:
                    return ToolResult(
                        tool_name="web_search",
                        content=f"Failed to fetch URL: {ssrf_error}",
                        success=False,
                    )
                extracted = self._youcom_extract(url)
                if extracted:
                    return ToolResult(
                        tool_name="web_search",
                        content=extracted,
                        success=True,
                        metadata={
                            "url": url,
                            "mode": "fetch",
                            "extractor": "youcom_contents",
                        },
                    )
            try:
                content = self._fetch_url(url)
                return ToolResult(
                    tool_name="web_search",
                    content=content or "No content found at URL.",
                    success=True,
                    metadata={"url": url, "mode": "fetch", "extractor": "local"},
                )
            except Exception as exc:
                return ToolResult(
                    tool_name="web_search",
                    content=f"Failed to fetch URL: {exc}",
                    success=False,
                )

        max_results = params.get("max_results", self._max_results)
        engine = self._resolve_engine()

        if engine == "duckduckgo":
            return self._duckduckgo_result(query, max_results)

        try:
            if engine == "youcom":
                return self._youcom_search(query, max_results)
            if engine == "serply":
                return self._serply_search(query, max_results)
            if engine == "firecrawl":
                return self._firecrawl_search(query, max_results)
            return self._tavily_search(query, max_results)
        except _WebSearchEngineError as exc:
            reason = str(exc)
        except Exception as exc:  # engine SDKs raise their own error types
            reason = f"{type(exc).__name__}: {exc}"

        # Falling back is a degradation in result quality, so say so at WARNING
        # rather than DEBUG — a typo'd key used to look like a quiet install.
        logger.warning(
            "%s engine failed (%s); falling back to DuckDuckGo, which returns "
            "unranked scraped results",
            engine,
            reason,
        )
        return self._duckduckgo_result(
            query, max_results, fallback_from=engine, reason=reason
        )

    def _tavily_search(self, query: str, max_results: int) -> ToolResult:
        """Search via the Tavily API."""
        from tavily import TavilyClient

        client = TavilyClient(api_key=self._api_key)
        response = client.search(
            query,
            max_results=max_results,
            search_depth="advanced",
            include_usage=True,
        )
        results = response.get("results", [])
        formatted_parts = []
        for r in results:
            title = r.get("title", "Untitled")
            url = r.get("url", "")
            content = r.get("content", "") or r.get("snippet", "")
            formatted_parts.append(f"### {title}\nSource: {url}\nSummary: {content}")

        formatted = "\n\n---\n\n".join(formatted_parts)
        return ToolResult(
            tool_name="web_search",
            content=formatted or "No results found.",
            success=True,
            metadata={
                "num_results": len(results),
                "engine": "tavily",
                "credits": (response.get("usage") or {}).get("credits"),
            },
        )

    def _duckduckgo_result(
        self,
        query: str,
        max_results: int,
        *,
        fallback_from: str | None = None,
        reason: str | None = None,
    ) -> ToolResult:
        """Run the DuckDuckGo scrape and record why it was used."""
        metadata: dict[str, Any] = {"engine": "duckduckgo"}
        if fallback_from:
            metadata["fallback_from"] = fallback_from
            metadata["degraded"] = True
            if reason:
                metadata["fallback_reason"] = reason

        try:
            formatted = self._duckduckgo_search(query, max_results)
            return ToolResult(
                tool_name="web_search",
                content=formatted or "No results found.",
                success=True,
                metadata=metadata,
            )
        except ImportError:
            return ToolResult(
                tool_name="web_search",
                content=(
                    "No search engine available: ddgs is not installed."
                    " Install with: pip install ddgs"
                ),
                success=False,
                metadata=metadata,
            )
        except Exception as exc:
            return ToolResult(
                tool_name="web_search",
                content=f"Search error: {exc}",
                success=False,
                metadata=metadata,
            )


__all__ = ["WebSearchTool"]