Skip to content

Automate browser based workflows with AI

Search the web through the search API instead of a browser tab (SKY-17071) (#8708)

Andrew Neilsond79656357806
diff --git a/skyvern/config.py b/skyvern/config.py
index 62f007ff2..a49e02162 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -608,6 +608,9 @@ class Settings(BaseSettings):
YUTORI_API_KEY: str | None = None
SERPAPI_API_KEY: str | None = Field(default=None, repr=False)
EXA_API_KEY: str | None = Field(default=None, repr=False)
+ # Lets the search_web helper and Copilot tool spend the Search block's keys above. Off by default so
+ # keys set only for the Search block are not spent by code blocks and the Copilot.
+ ENABLE_SEARCH_WEB: bool = False
YUTORI_API_BASE: str = "https://api.yutori.com/v1"
YUTORI_MODEL: str = "n1.5-latest"
YUTORI_LLM_KEY: str = "YUTORI_NAVIGATOR"
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index e828a2871..0fe78daf9 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -77,7 +77,6 @@ from skyvern.forge.sdk.services.credentials import AuthenticatorTotpParseResult
from skyvern.forge.sdk.trace import traced
from skyvern.forge.sdk.workflow.models.block import BaseTaskBlock, BlockTypeVar
from skyvern.forge.sdk.workflow.retry_policy import WORKFLOW_WEBHOOK_HTTP_TIMEOUT_SECONDS
-from skyvern.forge.sdk.workflow.web_search import WebSearchProvider
from skyvern.schemas.run_enums import RunEngine, RunType
from skyvern.schemas.workflows import BlockResult, FileStorageType, FileUploadDestination
from skyvern.services.otp_email import EmailOTPSearchError, EmailOTPVerificationContext, build_email_otp_sources
@@ -1259,11 +1258,6 @@ class AgentFunction:
under them; OSS has no runner, so its budget is unknown rather than unlimited."""
return None
- def web_search_provider(self) -> WebSearchProvider | None:
- """No search engine is configured in OSS; reading a provider's result markup is
- deployment configuration. Cloud overrides this with its configured provider."""
- return None
-
def redact_codeblock_parameter_values(self, value: Any, parameters: dict[str, Any]) -> Any:
"""Cloud overrides this with the runner's canonical parameter scrubber."""
return value
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 0929f5c57..e7cbd6cfe 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -1685,28 +1685,25 @@ async def search_web_tool(ctx: RunContextWrapper, query: str, max_results: int =
Use this while scouting -- to find companies, suppliers, listings, or
documentation pages the user described but did not name. ``results`` holds
- up to ``max_results`` entries with ``title``, ``url`` and ``snippet``, each
+ up to ``max_results`` (1 to 100) entries with ``title``, ``url`` and ``snippet``, each
``url`` a direct absolute link to the result site.
The rest of the reply is what the search actually did, so you can tell the
- cases apart yourself: ``extracted_count`` is how many results the page
- carried, ``withheld_count`` how many of those were withheld because their
- destination is not allowed, ``http_status`` how the page was served (null
- here, since this tab does not report one), ``error_kind`` the failure if the
- fetch itself failed, and ``page_title`` the title served. An empty
- ``results`` with a non-zero ``withheld_count`` is a filtered page; with
- ``extracted_count`` zero it is a page carrying no results, which is a
- refusal page as often as a genuine miss -- ``page_title`` usually says
- which. Do not report a failed fetch as "no matches".
-
- This navigates the scouting tab away from whatever page it was on. The
- same search is available inside a code block as
- ``await search_web(query, max_results=10)``, returning the same shape.
+ cases apart yourself: ``extracted_count`` is how many results the search
+ returned, ``withheld_count`` how many of those were withheld because they fall
+ outside the query's ``site:`` filter or their destination is not allowed, ``http_status`` the status of the
+ search API request whose results you got, and ``error_kind`` the failure if the search failed without
+ returning anything. An empty ``results`` with a non-zero ``withheld_count``
+ is a filtered search; with ``extracted_count`` zero and no ``error_kind`` the
+ query found nothing. Do not report a failed search as "no matches".
+
+ This does not touch the scouting tab. The same search is available inside a
+ code block as ``await search_web(query, max_results=10)``, returning the same shape.
"""
authority_error = _authority_tool_error(ctx.context, "search_web")
if authority_error:
return _diagnosis_repair_tool_error(ctx.context, "search_web", authority_error)
- result = await _search_web_impl(ctx.context, query, max_results)
+ result = await _search_web_impl(query, max_results)
return json.dumps(scrub_secrets_from_structure(ctx.context, result))
@@ -2106,13 +2103,11 @@ NATIVE_TOOLS = [
]
-# Native tools that cannot do their job without a browser: they dispatch a run, drive the
-# scouting tab, or read a live page. Membership is by hand because FunctionTool carries no
-# capability metadata; a new tool that touches a browser belongs here.
+# Not advertised without browser authority: these drive a run, the scouting tab or a live page.
+# Listed by hand; FunctionTool has no capability metadata.
BROWSER_BOUND_TOOL_NAMES = BLOCK_RUNNING_TOOLS | frozenset(
{
"discover_workflow_entrypoint",
- "search_web",
"inspect_page_for_composition",
LOCATOR_INSPECTION_TOOL_NAME,
"fill_credential_field",
diff --git a/skyvern/forge/sdk/copilot/tools/browser_code.py b/skyvern/forge/sdk/copilot/tools/browser_code.py
index fdf7e6a52..9a9be1991 100644
--- a/skyvern/forge/sdk/copilot/tools/browser_code.py
+++ b/skyvern/forge/sdk/copilot/tools/browser_code.py
@@ -105,7 +105,7 @@ Browser API (async, Playwright-shaped):
- Navigation: `page.goto(url)`, `page.go_back()`, `page.reload()`. Internal, private-network, and
non-web destinations are refused.
- Frames: `page.frames`, `page.main_frame`, `page.frame_locator(css)`.
-- After a sensitive sign-in on this page, screenshots and `page.evaluate` are refused for the rest of
+- After a sensitive sign-in on this page, screenshots, `page.evaluate` and `search_web` are refused for the rest of
the turn. Reading text still works; that is the way to inspect such a page.
- Tabs and popups: each call starts on the browser's current tab, the one the direct browser tools act on.
`await tabs()` lists open tabs; `await switch_tab(index)` makes that tab `page` for the rest of the call;
@@ -124,6 +124,8 @@ Browser API (async, Playwright-shaped):
image with the same OCR a saved block uses and types the text into the answer field, raising with
nothing typed when no text is read. It requires image OCR enabled for the organization; otherwise it
raises.
+- `await search_web(query, max_results=10)` is the saved block's search helper: it calls the server-side
+ search API, touches no tab, and returns the same result shape as the `search_web` tool.
- workbench-only, not valid in a saved block: `tabs`, `switch_tab`, `click_and_wait_for_popup`,
`click_and_download`, and `files`.
diff --git a/skyvern/forge/sdk/copilot/tools/web_search.py b/skyvern/forge/sdk/copilot/tools/web_search.py
index e972cf145..684038d38 100644
--- a/skyvern/forge/sdk/copilot/tools/web_search.py
+++ b/skyvern/forge/sdk/copilot/tools/web_search.py
@@ -2,40 +2,11 @@ from __future__ import annotations
from typing import Any
-from skyvern.forge import app
-from skyvern.forge.sdk.copilot.context import CopilotContext
-from skyvern.forge.sdk.workflow.web_search import FetchedPage, WebSearchObservation, search_web
+from skyvern.forge.sdk.workflow.web_search import WebSearchObservation, search_web
-from ._shared import _composition_get_html, _discovery_navigate
-
-class ScoutingTabTransport:
- """Fetches on the copilot's scouting tab. It reports no HTTP status, so the observation
- carries `http_status: None` rather than an invented one."""
-
- def __init__(self, copilot_ctx: CopilotContext) -> None:
- self._ctx = copilot_ctx
-
- async def fetch(self, url: str) -> FetchedPage:
- navigation = await _discovery_navigate(self._ctx, url)
- if not navigation.get("ok"):
- # The reason only: a navigation error embeds the URL, which carries the query.
- return FetchedPage(url=url, title="", html="", error_kind="navigation_failed")
- data = navigation.get("data")
- title = str(data.get("title") or "") if isinstance(data, dict) else ""
- html, read_error, truncated, _used_stripped = await _composition_get_html(self._ctx)
- if read_error:
- return FetchedPage(url=url, title=title, html="", error_kind="page_read_failed")
- return FetchedPage(url=url, title=title, html=html, truncated=truncated)
-
-
-async def _search_web_impl(copilot_ctx: CopilotContext, query: str, max_results: int = 10) -> dict[str, Any]:
- observation: WebSearchObservation = await search_web(
- app.AGENT_FUNCTION.web_search_provider(),
- ScoutingTabTransport(copilot_ctx),
- query,
- max_results,
- )
+async def _search_web_impl(query: str, max_results: int = 10) -> dict[str, Any]:
+ observation: WebSearchObservation = await search_web(query, max_results)
if observation["error_kind"] == "not_configured":
# A capability this deployment does not have, not a search that returned nothing. The
# sibling discovery helpers report an unavailable capability the same way.
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index aa5475cf2..dcaee0eff 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -6419,25 +6419,6 @@ def log_owned_page_disposal(
)
-def _bind_code_block_search_web(
- page: Page | RecordingPage | None,
-) -> Callable[[str, int], Awaitable[web_search.WebSearchObservation]]:
- """Closure rather than a `partial`: a keyword default on a `partial` would let block code
- override the page whose browser context the search is fetched through."""
-
- async def search_web(query: str, max_results: int = 10) -> web_search.WebSearchObservation:
- if page is None:
- raise RuntimeError("search_web is only supported while the run browser is open.")
- return await web_search.search_web(
- app.AGENT_FUNCTION.web_search_provider(),
- web_search.RunBrowserTransport(page.context),
- query,
- max_results,
- )
-
- return search_web
-
-
def _link_without_overwrite(source: str, directory: str) -> str:
"""Hard-link `source` into `directory` under its own name, or `name (n).ext` when that is taken."""
stem, suffix = os.path.splitext(os.path.basename(source))
@@ -6605,7 +6586,7 @@ class CodeBlock(Block):
organization_id=None, workflow_run_id=None, download_binding=None
),
"attach_authorized_file": unbound_attach_authorized_file,
- "search_web": _bind_code_block_search_web(None),
+ "search_web": web_search.search_web,
"open_page": _bind_code_block_open_page(None, []),
"clear_browser_data": _code_block_clear_browser_data_builtin,
"publish_file": _code_block_publish_file_builtin,
@@ -6686,7 +6667,6 @@ class CodeBlock(Block):
locate=page._pinned_locator() if isinstance(page, RecordingPage) else None,
registered_downloads=registered_downloads,
)
- safe_vars["search_web"] = _bind_code_block_search_web(page)
safe_vars["open_page"] = _bind_code_block_open_page(
page, opened_pages if opened_pages is not None else [], browser_state
)
diff --git a/skyvern/forge/sdk/workflow/models/web_search_block.py b/skyvern/forge/sdk/workflow/models/web_search_block.py
index db4ed363e..50b393421 100644
--- a/skyvern/forge/sdk/workflow/models/web_search_block.py
+++ b/skyvern/forge/sdk/workflow/models/web_search_block.py
@@ -2,27 +2,22 @@ from __future__ import annotations
import asyncio
import json
-import re
-from collections.abc import Callable, Collection, Iterable
-from dataclasses import dataclass, field
+from collections.abc import Callable, Iterable
from datetime import datetime
-from typing import Any, ClassVar, Literal, TypedDict
-from urllib.parse import SplitResult, parse_qs, urlencode, urlsplit
+from typing import Any, ClassVar, Literal
import jinja2
import structlog
from jsonschema import Draft202012Validator
from jsonschema.exceptions import SchemaError
-from opentelemetry.context import _SUPPRESS_HTTP_INSTRUMENTATION_KEY, attach, detach, set_value
from pydantic import Field, ValidationError, field_validator, model_validator
-from skyvern.config import settings
from skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codes
from skyvern.forge import app
from skyvern.forge.failure_classifier import FailureCategory
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.core import skyvern_context
-from skyvern.forge.sdk.core.aiohttp_helper import aiohttp_request
+from skyvern.forge.sdk.workflow import web_search_client
from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.exceptions import FailedToFormatJinjaStyleParameter, MissingJinjaVariables
from skyvern.forge.sdk.workflow.models.block import (
@@ -36,6 +31,7 @@ from skyvern.forge.sdk.workflow.models.block import (
)
from skyvern.forge.sdk.workflow.models.parameter import PARAMETER_TYPE
from skyvern.forge.sdk.workflow.page_derived_templates import classify_roots
+from skyvern.forge.sdk.workflow.web_search_client import SearchResponse, WebSearchError, redact_keys
from skyvern.schemas.workflows import (
BlockResult,
BlockStatus,
@@ -43,104 +39,34 @@ from skyvern.schemas.workflows import (
_normalize_outcome_error_code,
_validate_no_match_error_code_prompt,
)
-from skyvern.utils.contained_effects import contained_effect
-from skyvern.utils.secret_redaction import redact_secrets_from_text
LOG = structlog.get_logger()
-SearchProvider = Literal["google", "exa"]
-
-def _site_restriction(query: str) -> tuple[str, str] | None:
- if "OR" in query.split() or any(character in query for character in '|"“”()'):
- return None
- tokens = [
- token
- for token in query.split()
- if re.search(r"(?<!\w)site:", token, re.IGNORECASE) and not token.lower().startswith("-site:")
- ]
- if len(tokens) != 1 or not re.fullmatch(r"site:([a-z0-9-]+\.)+[a-z0-9-]+(/\S*)?", tokens[0], re.IGNORECASE):
- return None
- host, separator, path = tokens[0][5:].lower().partition("/")
- return host, separator + path if path else ""
-
-
-def _http_url(link: Any) -> SplitResult | None:
- try:
- parts = urlsplit(link) if isinstance(link, str) else None
- except ValueError:
- return None
- return parts if parts is not None and parts.scheme in {"http", "https"} and parts.hostname else None
-
-
-def _within_site(parts: SplitResult, restriction: tuple[str, str]) -> bool:
- host, path = restriction
- hostname = (parts.hostname or "").removesuffix(".")
- return (hostname == host or hostname.endswith("." + host)) and parts.path.lower().startswith(path)
-
-
-def _all_results_outside_site(items: Any, restriction: tuple[str, str]) -> bool:
- if not isinstance(items, list):
- return False
- urls = [url for item in items if isinstance(item, dict) and (url := _http_url(item.get("link"))) is not None]
- return bool(urls) and not any(_within_site(url, restriction) for url in urls)
-
-
-class WebSearchError(Exception):
- def __init__(self, message: str, category: FailureCategory = FailureCategory.INFRASTRUCTURE_ERROR) -> None:
- super().__init__(message)
- self.category = category
-
-
-class SearchResult(TypedDict):
- title: str
- link: str
- snippet: str
- display_link: str
- position: int
-
-
-@dataclass
-class SearchResponse:
- query: str
- provider: SearchProvider
- results: list[SearchResult] = field(default_factory=list)
- pages: list[dict[str, Any]] = field(default_factory=list)
- prompt_output: Any = None
- withheld_count: int = 0
-
- def output(
- self,
- status: BlockStatus,
- failure_reason: str | None,
- category: FailureCategory | None,
- errors: list[UserDefinedError],
- available_keys: list[str] | None,
- ) -> dict[str, Any]:
- output: dict[str, Any] = {
- "query": self.query,
- "provider": self.provider,
- "results": self.results,
- "total_count": len(self.results),
- "prompt_output": self.prompt_output,
- "raw_response": {"pages": self.pages},
- }
-
- if status != BlockStatus.completed:
- assert failure_reason is not None
- output.update(build_block_failure_output(failure_reason, []))
- output.update(
- status=status.value, errors=[error.model_dump(mode="json") for error in errors], failure_category=None
- )
- if errors:
- output["failure_category"] = user_defined_failure_category(errors[0])
- elif category is not None:
- output["failure_category"] = [
- {"category": category.value, "confidence_float": 1.0, "reasoning": failure_reason}
- ]
- if available_keys:
- output["available_keys"] = available_keys
- return output
+def _block_output(
+ response: SearchResponse,
+ status: BlockStatus,
+ failure_reason: str | None,
+ category: FailureCategory | None,
+ errors: list[UserDefinedError],
+ available_keys: list[str] | None,
+) -> dict[str, Any]:
+ output = response.output()
+ if status != BlockStatus.completed:
+ assert failure_reason is not None
+ output.update(build_block_failure_output(failure_reason, []))
+ output.update(
+ status=status.value, errors=[error.model_dump(mode="json") for error in errors], failure_category=None
+ )
+ if errors:
+ output["failure_category"] = user_defined_failure_category(errors[0])
+ elif category is not None:
+ output["failure_category"] = [
+ {"category": category.value, "confidence_float": 1.0, "reasoning": failure_reason}
+ ]
+ if available_keys:
+ output["available_keys"] = available_keys
+ return output
class WebSearchBlock(Block):
@@ -181,232 +107,6 @@ class WebSearchBlock(Block):
def get_all_parameters(self, workflow_run_id: str) -> list[PARAMETER_TYPE]:
return self.parameters
- @staticmethod
- def _redact_keys(value: Any, secret_values: Collection[str] = ()) -> Any:
- if isinstance(value, str):
- keys = [key for key in (*secret_values, settings.SERPAPI_API_KEY, settings.EXA_API_KEY) if key]
- redacted = redact_secrets_from_text(value, keys)
- normalized = re.sub(r"%[0-9a-fA-F]{2}", lambda match: match[0].upper(), redacted)
- masked = redact_secrets_from_text(normalized, keys)
- return masked if masked != normalized else redacted
- if isinstance(value, dict):
- return {
- WebSearchBlock._redact_keys(key, secret_values): WebSearchBlock._redact_keys(item, secret_values)
- for key, item in value.items()
- }
- if isinstance(value, list):
- return [WebSearchBlock._redact_keys(item, secret_values) for item in value]
- return value
-
- async def _request(
- self, provider: SearchProvider, url: str, payload: dict[str, Any] | None = None
- ) -> dict[str, Any]:
- # SerpAPI authenticates in the URL; the HTTP instrumentor does not redact api_key.
- token = attach(set_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY, True))
- try:
- status, _, body = await aiohttp_request(
- method="GET" if provider == "google" else "POST",
- url=url,
- headers={"x-api-key": settings.EXA_API_KEY or ""} if provider == "exa" else None,
- json_data=payload,
- timeout=30,
- follow_redirects=False,
- )
- except TimeoutError:
- raise TimeoutError(f"{provider.title()} search timed out after 30 seconds.") from None
- except Exception:
- raise WebSearchError(f"{provider.title()} search request failed.") from None
- finally:
- detach(token)
-
- if status in {401, 403}:
- raise WebSearchError(f"{provider.title()} search rejected the platform API key (HTTP {status}).")
- if status == 429:
- raise WebSearchError(f"{provider.title()} search quota or rate limit was exceeded (HTTP 429).")
- if not 200 <= status < 300:
- raise WebSearchError(f"{provider.title()} search failed (HTTP {status}).")
- if not isinstance(body, dict):
- raise WebSearchError(f"{provider.title()} search returned an invalid JSON response.")
- return self._redact_keys(body)
-
- def _append_results(
- self, response: SearchResponse, items: Any, start: int = 0, restriction: tuple[str, str] | None = None
- ) -> None:
- if not isinstance(items, list):
- raise WebSearchError(f"{response.provider.title()} search returned an invalid results list.")
- validated_results: list[SearchResult] = []
- seen = {result["link"] for result in response.results}
- remaining = self.num_results - len(response.results)
- for index, item in enumerate(items):
- if len(validated_results) >= remaining:
- break
- if not isinstance(item, dict):
- raise WebSearchError(f"{response.provider.title()} search returned an invalid result.")
- link = item.get("link") if response.provider == "google" else item.get("url")
- if not isinstance(link, str):
- raise WebSearchError(f"{response.provider.title()} search returned a result without a URL.")
- parsed = _http_url(link)
- if parsed is None:
- raise WebSearchError(f"{response.provider.title()} search returned an invalid result URL.")
- if restriction is not None and not _within_site(parsed, restriction):
- response.withheld_count += 1
- continue
- if link in seen:
- continue
- seen.add(link)
- title = item.get("title")
- snippet = item.get("snippet") if response.provider == "google" else ""
- display_link = item.get("displayed_link")
- validated_results.append(
- SearchResult(
- title=title if isinstance(title, str) else "",
- link=link,
- snippet=snippet if isinstance(snippet, str) else "",
- display_link=display_link if isinstance(display_link, str) else (parsed.hostname or ""),
- position=start + index + 1,
- )
- )
-
- response.results.extend(validated_results)
-
- async def _google_search(self, response: SearchResponse) -> None:
- if not settings.SERPAPI_API_KEY:
- raise WebSearchError("Google search is not configured on this server (SERPAPI_API_KEY is not set).")
- restriction = _site_restriction(response.query)
- refetches = 0
- refetch_stopped_reason = None
- start = 0
- try:
- for _ in range(10):
- params = {"engine": "google", "q": response.query, "start": start, "api_key": settings.SERPAPI_API_KEY}
- body = await self._request("google", f"https://serpapi.com/search.json?{urlencode(params)}")
- response.pages.append(body)
- metadata = body.get("search_metadata")
- if not isinstance(metadata, dict) or metadata.get("status") != "Success":
- raise WebSearchError("Google search did not complete successfully.")
- items = body.get("organic_results", [])
- outside_site = restriction is not None and _all_results_outside_site(items, restriction)
- while outside_site and refetches < 2:
- refetches += 1
- query = urlencode({**params, "no_cache": "true"})
- try:
- fresh_body = await self._request("google", f"https://serpapi.com/search.json?{query}")
- except Exception: # noqa: BLE001
- refetch_stopped_reason = "request_failed"
- break
- metadata = fresh_body.get("search_metadata")
- if not isinstance(metadata, dict) or metadata.get("status") != "Success":
- refetch_stopped_reason = "status_not_success"
- break
- body = fresh_body
- response.pages[-1] = body
- items = body.get("organic_results", [])
- outside_site = restriction is not None and _all_results_outside_site(items, restriction)
- if outside_site and refetches == 2 and refetch_stopped_reason is None:
- refetch_stopped_reason = "budget_spent"
- self._append_results(response, items, start, restriction)
- if outside_site or not items or len(response.results) >= self.num_results:
- return
- pagination = body.get("serpapi_pagination")
- next_page = pagination.get("next") if isinstance(pagination, dict) else None
- if not isinstance(next_page, str):
- return
- try:
- next_start = int(parse_qs(urlsplit(next_page).query)["start"][0])
- except (KeyError, IndexError, ValueError):
- raise WebSearchError("Google search returned invalid pagination metadata.") from None
- if next_start <= start or next_start > 1000:
- raise WebSearchError("Google search returned a non-advancing page offset.")
- start = next_start
- finally:
- if refetches > 0 or response.withheld_count > 0:
- with contained_effect("log Google search site restriction"):
- LOG.info(
- "Google search site restriction applied",
- refetches=refetches,
- withheld_count=response.withheld_count,
- results_returned=len(response.results),
- refetch_stopped_reason=refetch_stopped_reason,
- )
-
- async def _exa_search(self, response: SearchResponse) -> None:
- if not settings.EXA_API_KEY:
- raise WebSearchError("Exa search is not configured on this server (EXA_API_KEY is not set).")
- query = response.query
- payload: dict[str, Any] = {
- "query": query,
- "type": "auto",
- "numResults": self.num_results,
- }
- site_tokens = [token for token in query.split() if re.search(r"(?<!\w)site:", token, re.IGNORECASE)]
- if site_tokens:
- if len(site_tokens) != 1 or not re.fullmatch(
- r"site:([a-z0-9-]+\.)+[a-z0-9-]+", site_tokens[0], re.IGNORECASE
- ):
- raise WebSearchError(
- "Exa supports a single site:domain filter. Select Google for other site expressions.",
- FailureCategory.PARAMETER_BINDING_ERROR,
- )
- payload["includeDomains"] = [site_tokens[0][5:]]
- payload["query"] = query.replace(site_tokens[0], "", 1).strip()
- if not payload["query"]:
- raise WebSearchError(
- "Add search terms after the site:domain filter for Exa.", FailureCategory.PARAMETER_BINDING_ERROR
- )
- body = await self._request("exa", "https://api.exa.ai/search", payload)
- response.pages.append(body)
- if body.get("error"):
- raise WebSearchError("Exa search did not complete successfully.")
- self._append_results(response, body.get("results"))
- if not response.results:
- return
- try:
- contents = await self._request(
- "exa",
- "https://api.exa.ai/contents",
- {
- "urls": [result["link"] for result in response.results],
- "highlights": {"maxCharacters": 1000, "query": payload["query"]},
- "maxAgeHours": -1,
- },
- )
- if contents.get("error"):
- raise WebSearchError("Exa highlights request did not complete successfully.")
- items = contents.get("results")
- if not isinstance(items, list):
- raise WebSearchError("Exa highlights request returned an invalid results list.")
- snippets = {result["link"]: "" for result in response.results}
- unmatched_count = 0
- for item in items:
- if not isinstance(item, dict) or not isinstance(item.get("url"), str):
- raise WebSearchError("Exa highlights request returned an invalid result URL.")
- if item["url"] in snippets:
- highlights = item.get("highlights")
- snippets[item["url"]] = (
- "\n".join(text for text in highlights if isinstance(text, str))
- if isinstance(highlights, list)
- else ""
- )
- else:
- unmatched_count += 1
- except Exception as exc: # noqa: BLE001
- with contained_effect("log Exa highlights failure"):
- LOG.warning(
- "Exa highlights request failed; results keep empty snippets",
- error_type=type(exc).__name__,
- reason=str(exc) if isinstance(exc, (WebSearchError, TimeoutError)) else None,
- )
- return
- for result in response.results:
- result["snippet"] = snippets[result["link"]]
- if unmatched_count:
- with contained_effect("log Exa highlights mismatch"):
- LOG.warning(
- "Exa highlights returned pages that match no search result",
- unmatched_count=unmatched_count,
- result_count=len(response.results),
- )
-
def _render_schema(self, value: Any, context: WorkflowRunContext) -> Any:
if isinstance(value, str):
return self.render_templatable_field("json_schema", value, context)
@@ -589,8 +289,8 @@ class WebSearchBlock(Block):
def sanitize(data: Any) -> Any:
if context is None:
- return self._redact_keys(data)
- return self._redact_keys(context.mask_secrets_in_data(data), self._registered_secret_values(context))
+ return redact_keys(data)
+ return redact_keys(context.mask_secrets_in_data(data), self._registered_secret_values(context))
def template_failure(exc: Exception) -> None:
nonlocal failure_reason, category, status, available_keys
@@ -644,25 +344,16 @@ class WebSearchBlock(Block):
execution_denied = False
prompt_ran = False
if failure_reason is None:
- fallback_reason: str | None = None
try:
- async with asyncio.timeout(180):
- if self.provider == "exa":
- await self._exa_search(response)
- else:
- try:
- await self._google_search(response)
- except (TimeoutError, WebSearchError) as exc:
- if self.provider != "auto" or response.results or not settings.EXA_API_KEY:
- raise
- fallback_reason = str(exc) or "Google search timed out after 30 seconds."
- response.provider = "exa"
- response.pages.clear()
- await self._exa_search(response)
+ async with asyncio.timeout(web_search_client.SEARCH_TIMEOUT_SECONDS):
+ await web_search_client.search(response, self.provider, self.num_results)
except (TimeoutError, WebSearchError) as exc:
- reason = str(exc) or f"{response.provider.title()} search timed out after 180 seconds."
- if fallback_reason:
- reason += f" Exa ran because {fallback_reason.rstrip('.')}."
+ reason = str(exc) or (
+ f"{response.provider.title()} search timed out after "
+ f"{web_search_client.SEARCH_TIMEOUT_SECONDS} seconds."
+ )
+ if response.fallback_reason:
+ reason += f" Exa ran because {response.fallback_reason.rstrip('.')}."
if response.results:
LOG.warning(
"Web search stopped before reaching the requested result count",
@@ -804,13 +495,13 @@ class WebSearchBlock(Block):
if errors and failure_reason is None:
status = BlockStatus.terminated
failure_reason = errors[0].reasoning
- output = sanitize(response.output(status, failure_reason, category, errors, available_keys))
+ output = sanitize(_block_output(response, status, failure_reason, category, errors, available_keys))
failure_reason = sanitize(failure_reason)
except Exception as exc: # noqa: BLE001
status = BlockStatus.failed
failure_reason = f"The Search block stopped on an unexpected error ({type(exc).__name__})."
category = FailureCategory.INFRASTRUCTURE_ERROR
- output = sanitize(response.output(status, failure_reason, category, errors, available_keys))
+ output = sanitize(_block_output(response, status, failure_reason, category, errors, available_keys))
if context is None:
context = self.get_workflow_run_context(workflow_run_id)
diff --git a/skyvern/forge/sdk/workflow/web_search.py b/skyvern/forge/sdk/workflow/web_search.py
index eead0330c..b4666c6cf 100644
--- a/skyvern/forge/sdk/workflow/web_search.py
+++ b/skyvern/forge/sdk/workflow/web_search.py
@@ -1,17 +1,23 @@
from __future__ import annotations
import asyncio
-from dataclasses import dataclass
-from typing import Any, Protocol, TypedDict
+from typing import Any, TypedDict
import structlog
+from skyvern.config import settings
from skyvern.forge.sdk.browser_egress_policy import DestinationBlockedError, classify_url_async
-from skyvern.utils.contained_effects import contained_effect
-from skyvern.webeye.utils.page import mask_otp_values_in_html
+from skyvern.forge.sdk.settings_manager import SettingsManager
+from skyvern.forge.sdk.workflow import web_search_client
+from skyvern.forge.sdk.workflow.web_search_client import SEARCH_TIMEOUT_SECONDS, SearchResponse, WebSearchError
LOG = structlog.get_logger()
+MAX_RESULTS_LIMIT = 100
+# A limit below this still fetches this many, so a withheld result is replaced from the headroom; at
+# or above it nothing extra is fetched and a withheld result costs the caller a slot.
+_MIN_FETCHED_RESULTS = 10
+
class SearchResult(TypedDict):
title: str
@@ -21,8 +27,7 @@ class SearchResult(TypedDict):
class WebSearchObservation(TypedDict):
"""What the search actually produced. Deliberately not a verdict: the caller reads the
- facts and decides. A refused page, an error page and a page with no matches are different
- observations, and no wording heuristic has to tell them apart."""
+ facts and decides."""
provider: str
http_status: int | None
@@ -38,56 +43,32 @@ WEB_SEARCH_HELPER_CONTRACT: dict[str, Any] = {
"call": "await search_web(query, max_results=10)",
"parameters": {
"query": {"accepted_type": "str"},
- "max_results": {"accepted_type": "int", "minimum": 1},
+ "max_results": {"accepted_type": "int", "minimum": 1, "maximum": MAX_RESULTS_LIMIT},
},
"returns": {
"results": ["title", "url", "snippet"],
- "http_status": "status the search page was served with; null when the transport does not report one",
- "error_kind": "exception class when the fetch itself failed, else null",
- "page_title": "title of the page that was served",
- "extracted_count": "results found on the page before any were withheld",
- "withheld_count": "results withheld because their destination is not allowed",
- "capture_truncated": "true when the page was too large to read in full",
+ "http_status": (
+ "status of the search API request whose results are reported; a failed best-effort follow-up "
+ "request does not change it; null when no request completed"
+ ),
+ "error_kind": "exception class when the search failed without returning any result, else null",
+ "page_title": "always empty",
+ "extracted_count": "results the search returned before any were withheld",
+ "withheld_count": (
+ "results withheld because they fall outside the query's site: filter or their destination is not allowed"
+ ),
+ "capture_truncated": "always false",
},
"reading_the_result": (
- "results is what you can use. extracted_count == 0 with a 2xx http_status means the page "
- "carried no results, which is a refusal page as often as it is a genuine miss -- read "
- "page_title. A non-2xx http_status or a non-null error_kind means the fetch failed and "
- "says nothing about whether matches exist. withheld_count > 0 with an empty results list "
- "means the page had results and they were filtered, not that the query found nothing."
+ "results is what you can use. extracted_count == 0 with a null error_kind means the search "
+ "found nothing for this query. A non-null error_kind means the search failed and says nothing "
+ "about whether matches exist. withheld_count > 0 with an empty results list means the search "
+ "had results and they were filtered, not that the query found nothing. Fewer results than "
+ "max_results is not a complete list of what exists."
),
}
-@dataclass(frozen=True)
-class FetchedPage:
- """One page as a transport actually saw it."""
-
- url: str
- title: str
- html: str
- http_status: int | None = None
- truncated: bool = False
- error_kind: str | None = None
-
-
-class SearchTransport(Protocol):
- """Fetches a URL on whichever browser the caller already owns."""
-
- async def fetch(self, url: str) -> FetchedPage: ...
-
-
-class WebSearchProvider(Protocol):
- """A search engine: how to ask it, and how to read its answer. Provider-specific parsing
- lives with the provider so the generic path never guesses at result shape from anchors."""
-
- name: str
-
- def result_page_url(self, query: str) -> str: ...
-
- def extract_results(self, html: str) -> list[SearchResult]: ...
-
-
async def destination_allowed(url: str) -> bool:
"""One result may not veto its siblings: a hostname the resolver cannot even encode is
dropped rather than raised, so a single malformed result does not discard the whole page."""
@@ -104,14 +85,9 @@ async def admit_results(results: list[SearchResult]) -> list[SearchResult]:
return [result for result, allowed in zip(results, verdicts) if allowed]
-def _validated_max_results(value: object) -> int | None:
- """None means "every admitted result": the secure runner cannot carry the caller's limit
- across the operation boundary, so it takes them all and the sandbox applies the limit --
- after admission on both paths, so a withheld result never costs the caller a slot."""
- if value is None:
- return None
- if isinstance(value, bool) or not isinstance(value, int) or value < 1:
- raise ValueError("max_results must be a positive integer")
+def _validated_max_results(value: object) -> int:
+ if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= MAX_RESULTS_LIMIT:
+ raise ValueError(f"max_results must be an integer from 1 to {MAX_RESULTS_LIMIT}")
return value
@@ -121,18 +97,15 @@ def _validated_query(value: object) -> str:
return value
-async def search_web(
- provider: WebSearchProvider | None,
- transport: SearchTransport,
- query: str,
- max_results: int | None = 10,
-) -> WebSearchObservation:
- """Fetch a search page and report what came back. Extraction runs only on a page that was
- actually served, admission runs on every extracted result, and the caller's limit is applied
- last so a withheld result is replaced by the next allowed one rather than lost."""
+async def search_web(query: str, max_results: int = 10) -> WebSearchObservation:
+ """Ask the search API and report what came back. Admission runs on every returned result and
+ the caller's limit is applied after it."""
query = _validated_query(query)
max_results = _validated_max_results(max_results)
- if provider is None:
+ # The deployment's settings (CloudSettings in cloud) own this default; the module-level settings do not.
+ if not SettingsManager.get_settings().ENABLE_SEARCH_WEB or (
+ not settings.SERPAPI_API_KEY and not settings.EXA_API_KEY
+ ):
return WebSearchObservation(
provider="",
http_status=None,
@@ -143,31 +116,38 @@ async def search_web(
withheld_count=0,
capture_truncated=False,
)
- result_page_url = provider.result_page_url(query)
- if not await destination_allowed(result_page_url):
- return WebSearchObservation(
- provider=provider.name,
- http_status=None,
- error_kind="endpoint_blocked",
- page_title="",
- results=[],
- extracted_count=0,
- withheld_count=0,
- capture_truncated=False,
- )
- page = await transport.fetch(result_page_url)
- served = page.error_kind is None and (page.http_status is None or 200 <= page.http_status < 300)
- extracted = provider.extract_results(page.html) if served else []
- admitted = await admit_results(extracted)
+ response = SearchResponse(query=query, provider="google")
+ error_kind = None
+ extracted: list[SearchResult] = []
+ admitted: list[SearchResult] = []
+ try:
+ async with asyncio.timeout(SEARCH_TIMEOUT_SECONDS):
+ await web_search_client.search(
+ response,
+ "auto",
+ max(_MIN_FETCHED_RESULTS, max_results),
+ # One billed Google page per call (plus the client's site: re-fetches); Exa returns
+ # max_results in its one request.
+ max_pages=1,
+ )
+ extracted = [
+ SearchResult(title=result["title"], url=result["link"], snippet=result["snippet"])
+ for result in response.results
+ ]
+ admitted = await admit_results(extracted)
+ except (TimeoutError, WebSearchError) as exc:
+ error_kind = type(exc).__name__
+ extracted, admitted = [], []
+ off_site_count = 0 if error_kind else response.withheld_count
observation = WebSearchObservation(
- provider=provider.name,
- http_status=page.http_status,
- error_kind=page.error_kind,
- page_title=page.title,
- results=admitted if max_results is None else admitted[:max_results],
- extracted_count=len(extracted),
- withheld_count=len(extracted) - len(admitted),
- capture_truncated=page.truncated,
+ provider=response.http_status_provider or response.provider,
+ http_status=response.http_status,
+ error_kind=error_kind,
+ page_title="",
+ results=admitted[:max_results],
+ extracted_count=len(extracted) + off_site_count,
+ withheld_count=len(extracted) - len(admitted) + off_site_count,
+ capture_truncated=False,
)
LOG.info(
"web_search.observed",
@@ -181,57 +161,3 @@ async def search_web(
query_len=len(query),
)
return observation
-
-
-_NAVIGATION_TIMEOUT_MS = 20_000
-
-
-class ResponseLike(Protocol):
- @property
- def status(self) -> int: ...
-
-
-class PageLike(Protocol):
- async def goto(self, url: str, timeout: float) -> ResponseLike | None: ...
-
- async def title(self) -> str: ...
-
- async def content(self) -> str: ...
-
- async def close(self) -> None: ...
-
-
-class BrowserContextLike(Protocol):
- async def new_page(self) -> PageLike: ...
-
-
-class RunBrowserTransport:
- """Fetches on the run's own browser context rather than opening a separate network path, so
- the fetch inherits whatever that context is configured with. It is not itself a guard: the
- endpoint and every returned result are screened by the egress policy in `search_web`."""
-
- def __init__(self, context: BrowserContextLike) -> None:
- self._context = context
-
- async def fetch(self, url: str) -> FetchedPage:
- page = None
- try:
- page = await self._context.new_page()
- response = await page.goto(url, timeout=_NAVIGATION_TIMEOUT_MS)
- return FetchedPage(
- url=url,
- title=await page.title(),
- html=mask_otp_values_in_html(await page.content()),
- http_status=response.status if response is not None else None,
- )
- except Exception as exc:
- # The class name, never the message: a navigation error embeds the target URL,
- # which carries the query, and this is persisted as ordinary block output.
- return FetchedPage(url=url, title="", html="", error_kind=type(exc).__name__)
- finally:
- if page is not None:
- try:
- await page.close()
- except Exception:
- with contained_effect("search_web page close failure"):
- LOG.warning("web_search temporary page close failed")
diff --git a/skyvern/forge/sdk/workflow/web_search_client.py b/skyvern/forge/sdk/workflow/web_search_client.py
new file mode 100644
index 000000000..e2956a519
--- /dev/null
+++ b/skyvern/forge/sdk/workflow/web_search_client.py
@@ -0,0 +1,353 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Collection
+from dataclasses import dataclass, field
+from typing import Any, Literal, TypedDict
+from urllib.parse import SplitResult, parse_qs, urlencode, urlsplit
+
+import structlog
+from opentelemetry.context import _SUPPRESS_HTTP_INSTRUMENTATION_KEY, attach, detach, set_value
+
+from skyvern.config import settings
+from skyvern.forge.failure_classifier import FailureCategory
+from skyvern.forge.sdk.core.aiohttp_helper import aiohttp_request
+from skyvern.utils.contained_effects import contained_effect
+from skyvern.utils.secret_redaction import redact_secrets_from_text
+
+LOG = structlog.get_logger()
+
+SearchProvider = Literal["google", "exa"]
+ProviderChoice = Literal["auto", "google", "exa"]
+SEARCH_TIMEOUT_SECONDS = 180
+
+
+def _site_restriction(query: str) -> tuple[str, str] | None:
+ if "OR" in query.split() or any(character in query for character in '|"“”()'):
+ return None
+ tokens = [
+ token
+ for token in query.split()
+ if re.search(r"(?<!\w)site:", token, re.IGNORECASE) and not token.lower().startswith("-site:")
+ ]
+ if len(tokens) != 1 or not re.fullmatch(r"site:([a-z0-9-]+\.)+[a-z0-9-]+(/\S*)?", tokens[0], re.IGNORECASE):
+ return None
+ host, separator, path = tokens[0][5:].lower().partition("/")
+ return host, separator + path if path else ""
+
+
+def _http_url(link: Any) -> SplitResult | None:
+ try:
+ parts = urlsplit(link) if isinstance(link, str) else None
+ except ValueError:
+ return None
+ return parts if parts is not None and parts.scheme in {"http", "https"} and parts.hostname else None
+
+
+def _within_site(parts: SplitResult, restriction: tuple[str, str]) -> bool:
+ host, path = restriction
+ hostname = (parts.hostname or "").removesuffix(".")
+ return (hostname == host or hostname.endswith("." + host)) and parts.path.lower().startswith(path)
+
+
+def _all_results_outside_site(items: Any, restriction: tuple[str, str]) -> bool:
+ if not isinstance(items, list):
+ return False
+ urls = [url for item in items if isinstance(item, dict) and (url := _http_url(item.get("link"))) is not None]
+ return bool(urls) and not any(_within_site(url, restriction) for url in urls)
+
+
+class WebSearchError(Exception):
+ def __init__(self, message: str, category: FailureCategory = FailureCategory.INFRASTRUCTURE_ERROR) -> None:
+ super().__init__(message)
+ self.category = category
+
+
+class SearchResult(TypedDict):
+ title: str
+ link: str
+ snippet: str
+ display_link: str
+ position: int
+
+
+@dataclass
+class SearchResponse:
+ query: str
+ provider: SearchProvider
+ results: list[SearchResult] = field(default_factory=list)
+ pages: list[dict[str, Any]] = field(default_factory=list)
+ prompt_output: Any = None
+ withheld_count: int = 0
+ # Status and provider of the last API request that completed, set together before the status is
+ # judged, so a request that fails on its status still reports it.
+ http_status: int | None = None
+ http_status_provider: SearchProvider | None = None
+ fallback_reason: str | None = None
+
+ def output(self) -> dict[str, Any]:
+ return {
+ "query": self.query,
+ "provider": self.provider,
+ "results": self.results,
+ "total_count": len(self.results),
+ "prompt_output": self.prompt_output,
+ "raw_response": {"pages": self.pages},
+ }
+
+
+def redact_keys(value: Any, secret_values: Collection[str] = ()) -> Any:
+ if isinstance(value, str):
+ keys = [key for key in (*secret_values, settings.SERPAPI_API_KEY, settings.EXA_API_KEY) if key]
+ redacted = redact_secrets_from_text(value, keys)
+ normalized = re.sub(r"%[0-9a-fA-F]{2}", lambda match: match[0].upper(), redacted)
+ masked = redact_secrets_from_text(normalized, keys)
+ return masked if masked != normalized else redacted
+ if isinstance(value, dict):
+ return {redact_keys(key, secret_values): redact_keys(item, secret_values) for key, item in value.items()}
+ if isinstance(value, list):
+ return [redact_keys(item, secret_values) for item in value]
+ return value
+
+
+async def request(
+ response: SearchResponse, provider: SearchProvider, url: str, payload: dict[str, Any] | None = None
+) -> dict[str, Any]:
+ # SerpAPI authenticates in the URL; the HTTP instrumentor does not redact api_key.
+ token = attach(set_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY, True))
+ try:
+ status, _, body = await aiohttp_request(
+ method="GET" if provider == "google" else "POST",
+ url=url,
+ headers={"x-api-key": settings.EXA_API_KEY or ""} if provider == "exa" else None,
+ json_data=payload,
+ timeout=30,
+ follow_redirects=False,
+ )
+ except TimeoutError:
+ raise TimeoutError(f"{provider.title()} search timed out after 30 seconds.") from None
+ except Exception:
+ raise WebSearchError(f"{provider.title()} search request failed.") from None
+ finally:
+ detach(token)
+
+ response.http_status = status
+ response.http_status_provider = provider
+ if status in {401, 403}:
+ raise WebSearchError(f"{provider.title()} search rejected the platform API key (HTTP {status}).")
+ if status == 429:
+ raise WebSearchError(f"{provider.title()} search quota or rate limit was exceeded (HTTP 429).")
+ if not 200 <= status < 300:
+ raise WebSearchError(f"{provider.title()} search failed (HTTP {status}).")
+ if not isinstance(body, dict):
+ raise WebSearchError(f"{provider.title()} search returned an invalid JSON response.")
+ return redact_keys(body)
+
+
+def append_results(
+ response: SearchResponse,
+ items: Any,
+ num_results: int,
+ start: int = 0,
+ restriction: tuple[str, str] | None = None,
+) -> None:
+ if not isinstance(items, list):
+ raise WebSearchError(f"{response.provider.title()} search returned an invalid results list.")
+ validated_results: list[SearchResult] = []
+ seen = {result["link"] for result in response.results}
+ remaining = num_results - len(response.results)
+ for index, item in enumerate(items):
+ if len(validated_results) >= remaining:
+ break
+ if not isinstance(item, dict):
+ raise WebSearchError(f"{response.provider.title()} search returned an invalid result.")
+ link = item.get("link") if response.provider == "google" else item.get("url")
+ if not isinstance(link, str):
+ raise WebSearchError(f"{response.provider.title()} search returned a result without a URL.")
+ parsed = _http_url(link)
+ if parsed is None:
+ raise WebSearchError(f"{response.provider.title()} search returned an invalid result URL.")
+ if restriction is not None and not _within_site(parsed, restriction):
+ response.withheld_count += 1
+ continue
+ if link in seen:
+ continue
+ seen.add(link)
+ title = item.get("title")
+ snippet = item.get("snippet") if response.provider == "google" else ""
+ display_link = item.get("displayed_link")
+ validated_results.append(
+ SearchResult(
+ title=title if isinstance(title, str) else "",
+ link=link,
+ snippet=snippet if isinstance(snippet, str) else "",
+ display_link=display_link if isinstance(display_link, str) else (parsed.hostname or ""),
+ position=start + index + 1,
+ )
+ )
+
+ response.results.extend(validated_results)
+
+
+async def google_search(response: SearchResponse, num_results: int, max_pages: int = 10) -> None:
+ if not settings.SERPAPI_API_KEY:
+ raise WebSearchError("Google search is not configured on this server (SERPAPI_API_KEY is not set).")
+ restriction = _site_restriction(response.query)
+ refetches = 0
+ refetch_stopped_reason = None
+ start = 0
+ try:
+ for page_index in range(max_pages):
+ params = {"engine": "google", "q": response.query, "start": start, "api_key": settings.SERPAPI_API_KEY}
+ body = await request(response, "google", f"https://serpapi.com/search.json?{urlencode(params)}")
+ response.pages.append(body)
+ metadata = body.get("search_metadata")
+ if not isinstance(metadata, dict) or metadata.get("status") != "Success":
+ raise WebSearchError("Google search did not complete successfully.")
+ items = body.get("organic_results", [])
+ outside_site = restriction is not None and _all_results_outside_site(items, restriction)
+ while outside_site and refetches < 2:
+ refetches += 1
+ query = urlencode({**params, "no_cache": "true"})
+ page_status = (response.http_status, response.http_status_provider)
+ try:
+ fresh_body = await request(response, "google", f"https://serpapi.com/search.json?{query}")
+ except Exception: # noqa: BLE001
+ # The page already in hand is what gets reported, so its status stays the reported one.
+ response.http_status, response.http_status_provider = page_status
+ refetch_stopped_reason = "request_failed"
+ break
+ metadata = fresh_body.get("search_metadata")
+ if not isinstance(metadata, dict) or metadata.get("status") != "Success":
+ refetch_stopped_reason = "status_not_success"
+ break
+ body = fresh_body
+ response.pages[-1] = body
+ items = body.get("organic_results", [])
+ outside_site = restriction is not None and _all_results_outside_site(items, restriction)
+ if outside_site and refetches == 2 and refetch_stopped_reason is None:
+ refetch_stopped_reason = "budget_spent"
+ append_results(response, items, num_results, start, restriction)
+ last_page = page_index == max_pages - 1
+ if last_page or outside_site or not items or len(response.results) >= num_results:
+ return
+ pagination = body.get("serpapi_pagination")
+ next_page = pagination.get("next") if isinstance(pagination, dict) else None
+ if not isinstance(next_page, str):
+ return
+ try:
+ next_start = int(parse_qs(urlsplit(next_page).query)["start"][0])
+ except (KeyError, IndexError, ValueError):
+ raise WebSearchError("Google search returned invalid pagination metadata.") from None
+ if next_start <= start or next_start > 1000:
+ raise WebSearchError("Google search returned a non-advancing page offset.")
+ start = next_start
+ finally:
+ if refetches > 0 or response.withheld_count > 0:
+ with contained_effect("log Google search site restriction"):
+ LOG.info(
+ "Google search site restriction applied",
+ refetches=refetches,
+ withheld_count=response.withheld_count,
+ results_returned=len(response.results),
+ refetch_stopped_reason=refetch_stopped_reason,
+ )
+
+
+async def exa_search(response: SearchResponse, num_results: int) -> None:
+ if not settings.EXA_API_KEY:
+ raise WebSearchError("Exa search is not configured on this server (EXA_API_KEY is not set).")
+ query = response.query
+ payload: dict[str, Any] = {
+ "query": query,
+ "type": "auto",
+ "numResults": num_results,
+ }
+ site_tokens = [token for token in query.split() if re.search(r"(?<!\w)site:", token, re.IGNORECASE)]
+ if site_tokens:
+ if len(site_tokens) != 1 or not re.fullmatch(r"site:([a-z0-9-]+\.)+[a-z0-9-]+", site_tokens[0], re.IGNORECASE):
+ raise WebSearchError(
+ "Exa supports a single site:domain filter. Select Google for other site expressions.",
+ FailureCategory.PARAMETER_BINDING_ERROR,
+ )
+ payload["includeDomains"] = [site_tokens[0][5:]]
+ payload["query"] = query.replace(site_tokens[0], "", 1).strip()
+ if not payload["query"]:
+ raise WebSearchError(
+ "Add search terms after the site:domain filter for Exa.", FailureCategory.PARAMETER_BINDING_ERROR
+ )
+ body = await request(response, "exa", "https://api.exa.ai/search", payload)
+ response.pages.append(body)
+ if body.get("error"):
+ raise WebSearchError("Exa search did not complete successfully.")
+ append_results(response, body.get("results"), num_results)
+ if not response.results:
+ return
+ # The snippet lookup is best-effort; the reported status stays the search's.
+ search_status = (response.http_status, response.http_status_provider)
+ try:
+ contents = await request(
+ response,
+ "exa",
+ "https://api.exa.ai/contents",
+ {
+ "urls": [result["link"] for result in response.results],
+ "highlights": {"maxCharacters": 1000, "query": payload["query"]},
+ "maxAgeHours": -1,
+ },
+ )
+ if contents.get("error"):
+ raise WebSearchError("Exa highlights request did not complete successfully.")
+ items = contents.get("results")
+ if not isinstance(items, list):
+ raise WebSearchError("Exa highlights request returned an invalid results list.")
+ snippets = {result["link"]: "" for result in response.results}
+ unmatched_count = 0
+ for item in items:
+ if not isinstance(item, dict) or not isinstance(item.get("url"), str):
+ raise WebSearchError("Exa highlights request returned an invalid result URL.")
+ if item["url"] in snippets:
+ highlights = item.get("highlights")
+ snippets[item["url"]] = (
+ "\n".join(text for text in highlights if isinstance(text, str))
+ if isinstance(highlights, list)
+ else ""
+ )
+ else:
+ unmatched_count += 1
+ except Exception as exc: # noqa: BLE001
+ with contained_effect("log Exa highlights failure"):
+ LOG.warning(
+ "Exa highlights request failed; results keep empty snippets",
+ error_type=type(exc).__name__,
+ reason=str(exc) if isinstance(exc, (WebSearchError, TimeoutError)) else None,
+ )
+ return
+ finally:
+ response.http_status, response.http_status_provider = search_status
+ for result in response.results:
+ result["snippet"] = snippets[result["link"]]
+ if unmatched_count:
+ with contained_effect("log Exa highlights mismatch"):
+ LOG.warning(
+ "Exa highlights returned pages that match no search result",
+ unmatched_count=unmatched_count,
+ result_count=len(response.results),
+ )
+
+
+async def search(response: SearchResponse, provider: ProviderChoice, num_results: int, max_pages: int = 10) -> None:
+ """Under "auto", a Google failure that returned nothing falls back to Exa; one that already
+ returned results raises with them kept on `response`."""
+ if provider == "exa":
+ await exa_search(response, num_results)
+ return
+ try:
+ await google_search(response, num_results, max_pages)
+ except (TimeoutError, WebSearchError) as exc:
+ if provider != "auto" or response.results or not settings.EXA_API_KEY:
+ raise
+ response.fallback_reason = str(exc) or "Google search timed out after 30 seconds."
+ response.provider = "exa"
+ response.pages.clear()
+ await exa_search(response, num_results)
diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py
index c62c741d4..04e205a4a 100644
--- a/tests/unit/conftest.py
+++ b/tests/unit/conftest.py
@@ -38,6 +38,8 @@ from skyvern.forge.sdk.api import files
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.db.models import Base
from skyvern.forge.sdk.schemas.files import FileInfo
+from skyvern.forge.sdk.settings_manager import SettingsManager
+from skyvern.forge.sdk.workflow import web_search, web_search_client
from skyvern.forge.sdk.workflow.context_manager import WorkflowContextManager
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
from skyvern.webeye.utils import page as page_module
@@ -623,22 +625,62 @@ def fake_api_request_context() -> Callable[[], object]:
return _build
+def serpapi_page(*links: str, next_start: int | None = None) -> dict[str, Any]:
+ page: dict[str, Any] = {
+ "search_metadata": {"status": "Success"},
+ "organic_results": [{"title": f"Title {link}", "link": link, "snippet": f"About {link}"} for link in links],
+ }
+ if next_start is not None:
+ page["serpapi_pagination"] = {"next": f"https://serpapi.com/search.json?start={next_start}"}
+ return page
+
+
+SearchApiReply = tuple[int, object] | BaseException
+
+
+class FakeSearchApi:
+ """Stands in for `aiohttp_request` under the search client: answers each call with the next queued
+ (status, body) reply or raises it, repeating the last reply once the queue runs out."""
+
+ def __init__(self, *replies: SearchApiReply) -> None:
+ self._replies = list(replies)
+ self.urls: list[str] = []
+
+ async def __call__(self, *, url: str, **_kwargs: object) -> tuple[int, dict[str, str], object]:
+ self.urls.append(url)
+ reply = self._replies.pop(0) if len(self._replies) > 1 else self._replies[0]
+ if isinstance(reply, BaseException):
+ raise reply
+ status, body = reply
+ return status, {}, body
+
+
+def arm_search_api(
+ monkeypatch: pytest.MonkeyPatch,
+ *replies: SearchApiReply,
+ serpapi_key: str | None = "serp-test-key",
+ exa_key: str | None = None,
+) -> FakeSearchApi:
+ """Configures the search keys, answers the vendor calls from `replies`, and admits every result
+ destination; a test that screens destinations patches `web_search.classify_url_async` after this."""
+
+ async def allow(_url: str) -> str | None:
+ return None
+
+ api = FakeSearchApi(*replies)
+ monkeypatch.setattr(web_search_client, "aiohttp_request", api)
+ monkeypatch.setattr(web_search_client.settings, "SERPAPI_API_KEY", serpapi_key)
+ monkeypatch.setattr(web_search_client.settings, "EXA_API_KEY", exa_key)
+ monkeypatch.setattr(SettingsManager.get_settings(), "ENABLE_SEARCH_WEB", True)
+ monkeypatch.setattr(web_search, "classify_url_async", allow)
+ return api
+
+
class FakeSearchPage:
- """A tab the block's browser context opens: for a `search_web` call or an `open_page` one. A URL
- ending in ``/refused`` fails to load; a page with no fixed title reports one derived from its URL."""
+ """A tab the block's browser context opens for an `open_page` call. A URL ending in ``/refused``
+ fails to load; the title is derived from the URL."""
- def __init__(
- self,
- html: str,
- page_title: str,
- goto_error: Exception | None,
- http_status: int = 200,
- context: "FakeSearchBrowserContext | None" = None,
- ) -> None:
- self._html = html
- self._page_title = page_title
- self._goto_error = goto_error
- self.http_status = http_status
+ def __init__(self, context: "FakeSearchBrowserContext | None" = None) -> None:
self.context = context
self.url = "about:blank"
self.closed = False
@@ -646,18 +688,16 @@ class FakeSearchPage:
async def goto(self, url: str, timeout: float | None = None, **_kwargs: object) -> SimpleNamespace:
self.requested_url = url
- if self._goto_error is not None:
- raise self._goto_error
if url.endswith("/refused"):
raise PlaywrightError("net::ERR_FAILED")
self.url = url
- return SimpleNamespace(status=self.http_status)
+ return SimpleNamespace(status=200)
async def title(self) -> str:
- return self._page_title or f"title of {self.url}"
+ return f"title of {self.url}"
async def content(self) -> str:
- return self._html
+ return ""
def is_closed(self) -> bool:
return self.closed
@@ -667,10 +707,7 @@ class FakeSearchPage:
class FakeSearchBrowserContext:
- def __init__(
- self, html: str = "", page_title: str = "", goto_error: Exception | None = None, http_status: int = 200
- ) -> None:
- self._page_args = (html, page_title, goto_error, http_status)
+ def __init__(self) -> None:
self.opened: list[FakeSearchPage] = []
@property
@@ -682,7 +719,7 @@ class FakeSearchBrowserContext:
return list(self.opened)
async def new_page(self) -> FakeSearchPage:
- page = FakeSearchPage(*self._page_args, context=self)
+ page = FakeSearchPage(context=self)
self.opened.append(page)
return page
diff --git a/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py b/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py
index b10899370..a52eaa32d 100644
--- a/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py
+++ b/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py
@@ -154,7 +154,7 @@ def test_a_turn_without_browser_authority_advertises_no_browser_tool() -> None:
assert BROWSER_BOUND_TOOL_NAMES.isdisjoint(withheld.ordered_native_names)
assert BROWSER_CODE_TOOL_NAME not in withheld.ordered_native_names
- assert withheld.ordered_native_names
+ assert "search_web" in withheld.ordered_native_names
assert withheld.ordered_mcp_names == (
"get_workflow_knowledge",
"get_block_schema",
diff --git a/tests/unit/test_code_block_sandbox.py b/tests/unit/test_code_block_sandbox.py
index 2fcedd012..4b338a4a5 100644
--- a/tests/unit/test_code_block_sandbox.py
+++ b/tests/unit/test_code_block_sandbox.py
@@ -80,7 +80,13 @@ from skyvern.webeye import dialog_handler
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.browser_artifacts import BrowserArtifacts
from skyvern.webeye.skycdp.errors import CdpError
-from tests.unit.conftest import FakeClearingBrowserContext, FakeSearchBrowserContext, FakeSearchPage
+from tests.unit.conftest import (
+ FakeClearingBrowserContext,
+ FakeSearchBrowserContext,
+ FakeSearchPage,
+ arm_search_api,
+ serpapi_page,
+)
from tests.unit.fake_workflow_run_context import FakeWorkflowRunContext
RAW_DATETIME_TYPES = (stdlib_date, datetime, stdlib_time)
@@ -3647,27 +3653,29 @@ class TestSearchWebHelperBinding:
assert "search_web" in CodeBlock.build_safe_vars()
@pytest.mark.asyncio
- async def test_authored_code_reaches_the_search_helper_through_the_run_browser_context(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- """OSS configures no search provider, so the observation says exactly that rather than
- reporting a search that found nothing."""
- monkeypatch.setattr(
- "skyvern.forge.sdk.workflow.models.block.app.AGENT_FUNCTION.web_search_provider",
- lambda: None,
- )
- context = FakeSearchBrowserContext(html="<html>page</html>", page_title="Search results")
+ async def test_back_to_back_searches_open_no_browser_page(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ async def new_page_refused() -> None:
+ raise AssertionError("search_web opened a browser page")
+
+ arm_search_api(monkeypatch, (200, serpapi_page("https://a.example/", "https://b.example/")))
+ context = FakeSearchBrowserContext()
+ context.new_page = new_page_refused # type: ignore[method-assign]
page = SimpleNamespace(context=context)
block = self._block()
user_function = block.generate_async_user_function(
- 'found = await search_web("construction company", max_results=2)\nreturn found\n',
+ "found = []\n"
+ "for _ in range(4):\n"
+ ' found.append(await search_web("construction company", max_results=2))\n'
+ "return found\n",
page, # type: ignore[arg-type]
)
- result = await user_function()
+ results = await user_function()
- assert result["error_kind"] == "not_configured"
- assert result["results"] == []
+ assert [(found["http_status"], found["error_kind"], len(found["results"])) for found in results] == [
+ (200, None, 2)
+ ] * 4
+ assert context.opened == []
class TestOpenPageHelperBinding:
@@ -3868,7 +3876,7 @@ class TestOpenPageHelperBinding:
return SimpleNamespace(page=self, goto=self.goto, child_frames=[], parent_frame=None)
async def new_page() -> DetailPage:
- page = DetailPage("", "", None, context=context)
+ page = DetailPage(context=context)
context.opened.append(page)
return page
diff --git a/tests/unit/workflow/test_web_search.py b/tests/unit/workflow/test_web_search.py
index 75e82fc40..c706398eb 100644
--- a/tests/unit/workflow/test_web_search.py
+++ b/tests/unit/workflow/test_web_search.py
@@ -1,272 +1,282 @@
+import asyncio
+
import pytest
from skyvern.forge.sdk.browser_egress_policy import DestinationBlockedError
+from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.workflow import web_search
-from skyvern.forge.sdk.workflow.web_search import (
- FetchedPage,
- RunBrowserTransport,
- SearchResult,
- search_web,
-)
-from tests.unit.conftest import FakeSearchBrowserContext
-
-ENDPOINT = "https://search.example.test/html/"
-RESULTS = [
- SearchResult(title="First", url="https://first.example/about", snippet="first snippet"),
- SearchResult(title="Second", url="https://second.example/", snippet="second snippet"),
- SearchResult(title="Third", url="https://third.example/team", snippet="third snippet"),
-]
-
-
-class StubProvider:
- """Stands in for a deployment's configured engine. Extraction is the provider's job, so the
- orchestration is tested against a fixed result set rather than any engine's markup."""
-
- name = "stub"
-
- def __init__(self, results: list[SearchResult] | None = None) -> None:
- self._results = RESULTS if results is None else results
-
- def result_page_url(self, query: str) -> str:
- return f"{ENDPOINT}?q={query}"
-
- def extract_results(self, html: str) -> list[SearchResult]:
- return list(self._results) if html else []
-
-
-class StubTransport:
- def __init__(self, page: FetchedPage) -> None:
- self._page = page
- self.requested_url: str | None = None
+from skyvern.forge.sdk.workflow.web_search import search_web
+from tests.unit.conftest import SearchApiReply, arm_search_api, serpapi_page
- async def fetch(self, url: str) -> FetchedPage:
- self.requested_url = url
- return self._page
+FIRST, SECOND, THIRD = "https://first.example/about", "https://second.example/", "https://third.example/team"
+PAGE = serpapi_page(FIRST, SECOND, THIRD)
-def served(*, status: int | None = 200, truncated: bool = False, title: str = "results") -> StubTransport:
- return StubTransport(
- FetchedPage(url=ENDPOINT, title=title, html="<html>page</html>", http_status=status, truncated=truncated)
- )
-
+async def test_a_served_search_reports_the_api_status_and_its_results(monkeypatch: pytest.MonkeyPatch) -> None:
+ arm_search_api(monkeypatch, (200, PAGE))
[email protected](autouse=True)
-def allow_every_destination(monkeypatch: pytest.MonkeyPatch) -> None:
- async def allow(_url: str) -> str | None:
- return None
+ observed = await search_web("roofing")
- monkeypatch.setattr(web_search, "classify_url_async", allow)
+ assert observed == {
+ "provider": "google",
+ "http_status": 200,
+ "error_kind": None,
+ "page_title": "",
+ "results": [
+ {"title": f"Title {link}", "url": link, "snippet": f"About {link}"} for link in (FIRST, SECOND, THIRD)
+ ],
+ "extracted_count": 3,
+ "withheld_count": 0,
+ "capture_truncated": False,
+ }
@pytest.mark.asyncio
-async def test_a_page_that_was_not_served_extracts_nothing_and_says_why() -> None:
- """An error page carries no results, and reporting that as "no matches" is the false
- negative this helper exists to remove."""
- observed = await search_web(StubProvider(), served(status=503), "anything")
+ ("google_status", "exa_key", "query", "exa_reply"),
+ [
+ (401, None, "roofing", None),
+ (429, "exa-test-key", "roofing site:example.com/path", None),
+ (401, "exa-test-key", "roofing", TimeoutError()),
+ ],
+ ids=["exa-unconfigured", "exa-rejects-site-path", "exa-never-completes"],
+)
+async def test_a_failed_fallback_reports_the_provider_and_status_of_the_last_completed_request(
+ monkeypatch: pytest.MonkeyPatch,
+ google_status: int,
+ exa_key: str | None,
+ query: str,
+ exa_reply: BaseException | None,
+) -> None:
+ replies = [(google_status, {"error": "refused"}), *([exa_reply] if exa_reply else [])]
+ arm_search_api(monkeypatch, *replies, exa_key=exa_key)
- assert observed["http_status"] == 503
+ observed = await search_web(query)
+
+ assert (observed["provider"], observed["http_status"]) == ("google", google_status)
+ assert observed["error_kind"] is not None
assert observed["results"] == []
- assert observed["extracted_count"] == 0
@pytest.mark.asyncio
-async def test_a_transport_failure_reports_its_kind_not_an_empty_result_set() -> None:
- transport = StubTransport(FetchedPage(url=ENDPOINT, title="", html="", error_kind="TimeoutError"))
+async def test_a_malformed_body_reports_the_status_it_was_served_with(monkeypatch: pytest.MonkeyPatch) -> None:
+ arm_search_api(monkeypatch, (200, ["not", "an", "object"]))
- observed = await search_web(StubProvider(), transport, "anything")
+ observed = await search_web("roofing")
- assert observed["error_kind"] == "TimeoutError"
- assert observed["extracted_count"] == 0
+ assert observed["error_kind"] == "WebSearchError"
+ assert observed["http_status"] == 200
@pytest.mark.asyncio
-async def test_a_transport_that_reports_no_status_still_extracts() -> None:
- observed = await search_web(StubProvider(), served(status=None), "anything")
+async def test_a_request_that_never_completes_reports_no_status(monkeypatch: pytest.MonkeyPatch) -> None:
+ arm_search_api(monkeypatch, TimeoutError(), exa_key="exa-test-key")
+
+ observed = await search_web("roofing")
+ assert observed["error_kind"] == "TimeoutError"
assert observed["http_status"] is None
- assert observed["extracted_count"] == 3
@pytest.mark.asyncio
-async def test_a_page_carrying_no_results_is_reported_with_its_title_and_no_verdict() -> None:
- observed = await search_web(StubProvider(results=[]), served(title="Just a moment..."), "anything")
-
- assert observed["results"] == []
- assert observed["extracted_count"] == 0
- assert observed["withheld_count"] == 0
- assert observed["page_title"] == "Just a moment..."
[email protected]("stalls", ["search", "admission"])
+async def test_the_whole_call_is_bounded(monkeypatch: pytest.MonkeyPatch, stalls: str) -> None:
+ async def never_answers(*_args: object, **_kwargs: object) -> None:
+ await asyncio.Event().wait()
+ arm_search_api(monkeypatch, (200, PAGE))
+ if stalls == "search":
+ monkeypatch.setattr(web_search.web_search_client, "aiohttp_request", never_answers)
+ else:
+ monkeypatch.setattr(web_search, "classify_url_async", never_answers)
+ monkeypatch.setattr(web_search, "SEARCH_TIMEOUT_SECONDS", 0.05)
-async def test_a_truncated_capture_is_reported() -> None:
- observed = await search_web(StubProvider(), served(truncated=True), "anything")
+ observed = await asyncio.wait_for(search_web("roofing"), timeout=5)
- assert observed["capture_truncated"] is True
+ assert observed["error_kind"] == "TimeoutError"
+ assert (observed["results"], observed["extracted_count"], observed["withheld_count"]) == ([], 0, 0)
@pytest.mark.asyncio
-async def test_withheld_results_are_counted_rather_than_read_as_an_empty_page(
- monkeypatch: pytest.MonkeyPatch,
+ ("contents_reply", "snippet"),
+ [((200, {"results": [{"url": SECOND, "highlights": ["about second"]}]}), "about second"), ((500, {}), "")],
+ ids=["highlights", "highlights-fail"],
+)
+async def test_google_failing_falls_back_to_exa(
+ monkeypatch: pytest.MonkeyPatch, contents_reply: SearchApiReply, snippet: str
) -> None:
- async def block_every_result(url: str) -> str | None:
- return None if url.startswith(ENDPOINT) else "blocked egress to internal address"
+ exa_body = {"results": [{"url": SECOND, "title": "Second"}]}
+ api = arm_search_api(monkeypatch, (500, {}), (200, exa_body), contents_reply, exa_key="exa-test-key")
- monkeypatch.setattr(web_search, "classify_url_async", block_every_result)
+ observed = await search_web("roofing")
- observed = await search_web(StubProvider(), served(), "anything")
-
- assert observed["results"] == []
- assert observed["extracted_count"] == 3
- assert observed["withheld_count"] == 3
+ assert observed["provider"] == "exa"
+ assert observed["http_status"] == 200
+ assert observed["error_kind"] is None
+ assert observed["results"] == [{"title": "Second", "url": SECOND, "snippet": snippet}]
+ assert api.urls == [api.urls[0], "https://api.exa.ai/search", "https://api.exa.ai/contents"]
@pytest.mark.asyncio
-async def test_the_caller_limit_is_filled_from_later_results_when_an_earlier_one_is_withheld(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """Admission runs before the limit, so a withheld result costs the caller nothing and both
- execution paths return the same set for the same arguments."""
+async def test_exa_alone_serves_a_deployment_without_a_serpapi_key(monkeypatch: pytest.MonkeyPatch) -> None:
+ exa_body = {"results": [{"url": FIRST, "title": "First", "highlights": []}]}
+ arm_search_api(monkeypatch, (200, exa_body), serpapi_key=None, exa_key="exa-test-key")
- async def block_the_first(url: str) -> str | None:
- return "blocked" if url == "https://first.example/about" else None
+ observed = await search_web("roofing")
- monkeypatch.setattr(web_search, "classify_url_async", block_the_first)
+ assert observed["provider"] == "exa"
+ assert [result["url"] for result in observed["results"]] == [FIRST]
- observed = await search_web(StubProvider(), served(), "anything", max_results=2)
- assert [result["url"] for result in observed["results"]] == [
- "https://second.example/",
- "https://third.example/team",
- ]
- assert observed["withheld_count"] == 1
+async def test_a_call_bills_one_google_page_whatever_the_limit(monkeypatch: pytest.MonkeyPatch) -> None:
+ first_page = serpapi_page(*(f"https://site{index}.example/" for index in range(10)), next_start=10)
+ api = arm_search_api(monkeypatch, (200, first_page), (200, first_page))
+ observed = await search_web("roofing", max_results=100)
-async def test_one_unresolvable_result_does_not_discard_its_siblings(monkeypatch: pytest.MonkeyPatch) -> None:
- """A hostname the resolver cannot encode raises UnicodeError, which is not an OSError and
- escapes the shared classifier. It drops that result, not the whole search."""
+ assert len(api.urls) == 1
+ assert len(observed["results"]) == 10
- async def raise_for_the_first(url: str) -> str | None:
- if url == "https://first.example/about":
- raise UnicodeError("label too long")
- return None
- monkeypatch.setattr(web_search, "classify_url_async", raise_for_the_first)
+async def test_off_site_results_count_as_withheld_not_as_nothing_found(monkeypatch: pytest.MonkeyPatch) -> None:
+ off_site = serpapi_page("https://elsewhere.test/a", "https://elsewhere.test/b")
+ arm_search_api(monkeypatch, (200, off_site), (200, off_site), (200, off_site))
- observed = await search_web(StubProvider(), served(), "anything")
+ observed = await search_web("site:example.com roofing")
- assert [result["url"] for result in observed["results"]] == [
- "https://second.example/",
- "https://third.example/team",
- ]
- assert observed["withheld_count"] == 1
+ assert observed["error_kind"] is None
+ assert (observed["results"], observed["extracted_count"], observed["withheld_count"]) == ([], 2, 2)
@pytest.mark.asyncio
-async def test_a_result_the_classifier_rejects_outright_is_withheld_not_raised(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- async def reject(url: str) -> str | None:
- if url.startswith(ENDPOINT):
- return None
- raise DestinationBlockedError("blocked egress: URL has no host")
-
- monkeypatch.setattr(web_search, "classify_url_async", reject)
+async def test_unused_pagination_metadata_cannot_drop_the_page_returned(monkeypatch: pytest.MonkeyPatch) -> None:
+ short_page = serpapi_page("https://site.example/a", next_start=0)
+ arm_search_api(monkeypatch, (200, short_page))
- observed = await search_web(StubProvider(), served(), "anything")
+ observed = await search_web("roofing")
- assert observed["results"] == []
- assert observed["withheld_count"] == 3
+ assert (observed["error_kind"], len(observed["results"])) == (None, 1)
@pytest.mark.asyncio
-async def test_no_limit_returns_every_admitted_result() -> None:
- """The secure runner cannot carry the caller's limit across the operation boundary, so it
- admits them all and the sandbox slices the admitted set."""
- observed = await search_web(StubProvider(), served(), "anything", max_results=None)
+async def test_a_failed_site_refetch_keeps_the_status_of_the_page_returned(monkeypatch: pytest.MonkeyPatch) -> None:
+ mixed = serpapi_page("https://elsewhere.test/a")
+ arm_search_api(monkeypatch, (200, mixed), (429, {}))
+
+ observed = await search_web("site:example.com roofing")
- assert len(observed["results"]) == 3
+ assert (observed["http_status"], observed["error_kind"]) == (200, None)
@pytest.mark.asyncio
-async def test_an_unconfigured_deployment_says_so_rather_than_reporting_an_empty_search() -> None:
- observed = await search_web(None, served(), "anything")
+async def test_search_web_switched_off_makes_no_request(monkeypatch: pytest.MonkeyPatch) -> None:
+ api = arm_search_api(monkeypatch, (200, PAGE))
+ monkeypatch.setattr(SettingsManager.get_settings(), "ENABLE_SEARCH_WEB", False)
+ observed = await search_web("roofing")
+
+ assert api.urls == []
assert observed["error_kind"] == "not_configured"
- assert observed["results"] == []
@pytest.mark.asyncio
[email protected]("query", [None, 7, "", " "])
-async def test_an_unusable_query_is_rejected_before_anything_is_fetched(query: object) -> None:
- transport = served()
-
- with pytest.raises(ValueError):
- await search_web(StubProvider(), transport, query) # type: ignore[arg-type]
+async def test_a_withheld_result_is_replaced_from_the_fetched_headroom(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def block_the_first(url: str) -> str | None:
+ return "blocked" if url == FIRST else None
- assert transport.requested_url is None
+ arm_search_api(monkeypatch, (200, PAGE))
+ monkeypatch.setattr(web_search, "classify_url_async", block_the_first)
+ observed = await search_web("roofing", max_results=2)
[email protected]("max_results", [0, -1, "3", 2.0, True])
-async def test_an_unusable_result_limit_is_rejected(max_results: object) -> None:
- with pytest.raises(ValueError):
- await search_web(StubProvider(), served(), "anything", max_results) # type: ignore[arg-type]
+ assert [result["url"] for result in observed["results"]] == [SECOND, THIRD]
+ assert observed["extracted_count"] == 3
+ assert observed["withheld_count"] == 1
@pytest.mark.asyncio
-async def test_the_provider_builds_the_url_the_transport_fetches() -> None:
- transport = served()
+async def test_withheld_results_are_counted_rather_than_read_as_an_empty_search(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def block_everything(_url: str) -> str | None:
+ return "blocked egress to internal address"
- await search_web(StubProvider(), transport, "roofing")
+ arm_search_api(monkeypatch, (200, PAGE))
+ monkeypatch.setattr(web_search, "classify_url_async", block_everything)
- assert transport.requested_url == f"{ENDPOINT}?q=roofing"
+ observed = await search_web("roofing")
+
+ assert observed["results"] == []
+ assert observed["extracted_count"] == 3
+ assert observed["withheld_count"] == 3
@pytest.mark.asyncio
-async def test_the_run_browser_transport_reports_the_served_status_and_closes_its_tab() -> None:
- context = FakeSearchBrowserContext(html="<html>page</html>", page_title="results", http_status=200)
+ "failure", [UnicodeError("label too long"), DestinationBlockedError("blocked egress: URL has no host")]
+)
+async def test_one_unclassifiable_result_does_not_discard_its_siblings(
+ monkeypatch: pytest.MonkeyPatch, failure: Exception
+) -> None:
+ async def raise_for_the_first(url: str) -> str | None:
+ if url == FIRST:
+ raise failure
+ return None
- page = await RunBrowserTransport(context).fetch(ENDPOINT)
+ arm_search_api(monkeypatch, (200, PAGE))
+ monkeypatch.setattr(web_search, "classify_url_async", raise_for_the_first)
- assert page.http_status == 200
- assert page.title == "results"
- assert context.page.closed is True
+ observed = await search_web("roofing")
+
+ assert [result["url"] for result in observed["results"]] == [SECOND, THIRD]
+ assert observed["withheld_count"] == 1
@pytest.mark.asyncio
-async def test_the_run_browser_transport_reports_a_navigation_failure_without_its_message() -> None:
- context = FakeSearchBrowserContext(goto_error=TimeoutError("navigating to https://engine/?q=secret"))
+async def test_the_api_key_never_reaches_the_observation(monkeypatch: pytest.MonkeyPatch) -> None:
+ page = serpapi_page(FIRST)
+ page["organic_results"][0]["snippet"] = "echoed serp-test-key back"
+ arm_search_api(monkeypatch, (200, page))
- page = await RunBrowserTransport(context).fetch(ENDPOINT)
+ observed = await search_web("roofing")
- assert page.error_kind == "TimeoutError"
- assert "secret" not in repr(page)
- assert context.page.closed is True
+ assert "serp-test-key" not in repr(observed)
@pytest.mark.asyncio
-async def test_a_blocked_endpoint_is_reported_and_never_fetched(monkeypatch: pytest.MonkeyPatch) -> None:
- """The endpoint is screened by the same policy as the results, before any navigation."""
+async def test_an_unconfigured_deployment_says_so_rather_than_reporting_an_empty_search(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ api = arm_search_api(monkeypatch, (200, PAGE), serpapi_key=None, exa_key=None)
- async def block_the_endpoint(url: str) -> str | None:
- return "blocked egress to internal address" if url.startswith(ENDPOINT) else None
+ observed = await search_web("roofing")
- monkeypatch.setattr(web_search, "classify_url_async", block_the_endpoint)
- transport = served()
+ assert observed["error_kind"] == "not_configured"
+ assert observed["results"] == []
+ assert api.urls == []
- observed = await search_web(StubProvider(), transport, "anything")
- assert observed["error_kind"] == "endpoint_blocked"
- assert observed["results"] == []
- assert transport.requested_url is None
[email protected]("query", [None, 7, "", " "])
+async def test_an_unusable_query_is_rejected_before_anything_is_fetched(
+ monkeypatch: pytest.MonkeyPatch, query: object
+) -> None:
+ api = arm_search_api(monkeypatch, (200, PAGE))
+
+ with pytest.raises(ValueError):
+ await search_web(query) # type: ignore[arg-type]
+
+ assert api.urls == []
@pytest.mark.asyncio
-async def test_the_caller_limit_truncates_a_longer_admitted_set() -> None:
- """Pins the inline slice: without it the caller asking for one result gets three."""
- observed = await search_web(StubProvider(), served(), "anything", max_results=1)
[email protected]("max_results", [0, -1, 101, "3", 2.0, True, None])
+async def test_an_unusable_result_limit_is_rejected(monkeypatch: pytest.MonkeyPatch, max_results: object) -> None:
+ arm_search_api(monkeypatch, (200, PAGE))
- assert [result["url"] for result in observed["results"]] == ["https://first.example/about"]
- assert observed["extracted_count"] == 3
- assert observed["withheld_count"] == 0
+ with pytest.raises(ValueError):
+ await search_web("roofing", max_results) # type: ignore[arg-type]
diff --git a/tests/unit/workflow/test_web_search_block.py b/tests/unit/workflow/test_web_search_block.py
index 5b82d5ed4..16e02d9c9 100644
--- a/tests/unit/workflow/test_web_search_block.py
+++ b/tests/unit/workflow/test_web_search_block.py
@@ -8,12 +8,13 @@ import pytest
from pydantic import ValidationError
from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMResponseFormat
+from skyvern.forge.sdk.workflow import web_search_client
from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.models import block as block_module
-from skyvern.forge.sdk.workflow.models import web_search_block as search_module
from skyvern.forge.sdk.workflow.models.block import TextPromptBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
from skyvern.forge.sdk.workflow.models.web_search_block import WebSearchBlock, WebSearchError
+from skyvern.forge.sdk.workflow.web_search_client import SearchResponse
from skyvern.schemas.workflows import BlockStatus, WebSearchBlockYAML
SCHEMA_ECHO = {
@@ -93,8 +94,8 @@ def search_setup(monkeypatch: pytest.MonkeyPatch) -> tuple[WebSearchBlock, Workf
)
handler = AsyncMock()
monkeypatch.setattr(WebSearchBlock, "get_workflow_run_context", staticmethod(lambda _: context))
- monkeypatch.setattr(WebSearchBlock, "_request", AsyncMock(return_value=PROVIDER_PAGE))
- monkeypatch.setattr(search_module.settings, "EXA_API_KEY", "test-provider-key")
+ monkeypatch.setattr(web_search_client, "request", AsyncMock(return_value=PROVIDER_PAGE))
+ monkeypatch.setattr(web_search_client.settings, "EXA_API_KEY", "test-provider-key")
monkeypatch.setattr(TextPromptBlock, "_resolve_default_llm_handler", AsyncMock(return_value=handler))
monkeypatch.setattr(
block_module.LLMAPIHandlerFactory, "get_override_llm_api_handler", lambda llm_key, *, default: default
@@ -116,7 +117,7 @@ async def test_exa_search_keeps_pages_without_cached_content(
payloads: dict[str, dict[str, Any]] = {}
async def request(
- self: WebSearchBlock, provider: str, url: str, payload: dict[str, Any] | None = None
+ _response: SearchResponse, provider: str, url: str, payload: dict[str, Any] | None = None
) -> dict[str, Any]:
assert payload is not None
payloads[url] = payload
@@ -132,7 +133,7 @@ async def test_exa_search_keeps_pages_without_cached_content(
]
}
- monkeypatch.setattr(WebSearchBlock, "_request", request)
+ monkeypatch.setattr(web_search_client, "request", request)
result = await block.execute("workflow-run-test", "block-run-test", "org-test")
assert "contents" not in payloads["https://api.exa.ai/search"]
@@ -280,8 +281,8 @@ async def test_search_completes_with_validated_partial_results(
],
"serpapi_pagination": {"next": "https://serpapi.com/search.json?start=10"},
}
- monkeypatch.setattr(search_module.settings, "SERPAPI_API_KEY", "test-google-key")
- monkeypatch.setattr(WebSearchBlock, "_request", AsyncMock(side_effect=[page, second_page]))
+ monkeypatch.setattr(web_search_client.settings, "SERPAPI_API_KEY", "test-google-key")
+ monkeypatch.setattr(web_search_client, "request", AsyncMock(side_effect=[page, second_page]))
handler.return_value = {"llm_response": "Partial results processed."}
result = await block.execute("workflow-run-test", "block-run-test", "org-test")
@@ -312,8 +313,8 @@ async def test_google_site_filter_refetches_off_site_page_at_most_twice(
"serpapi_pagination": {"next": "https://serpapi.com/search.json?start=10"},
}
request = AsyncMock(side_effect=[off_site_page] * 3)
- monkeypatch.setattr(search_module.settings, "SERPAPI_API_KEY", "test-google-key")
- monkeypatch.setattr(WebSearchBlock, "_request", request)
+ monkeypatch.setattr(web_search_client.settings, "SERPAPI_API_KEY", "test-google-key")
+ monkeypatch.setattr(web_search_client, "request", request)
result = await block.execute("workflow-run-test", "block-run-test", "org-test")
@@ -322,8 +323,8 @@ async def test_google_site_filter_refetches_off_site_page_at_most_twice(
output = result.output_parameter_value
assert output["results"] == []
assert request.await_count == 3
- assert all(call.args[0] == "google" for call in request.await_args_list)
- parameters = [parse_qs(urlsplit(call.args[1]).query) for call in request.await_args_list]
+ assert all(call.args[1] == "google" for call in request.await_args_list)
+ parameters = [parse_qs(urlsplit(call.args[2]).query) for call in request.await_args_list]
assert "no_cache" not in parameters[0]
assert parameters[1] == parameters[2] == {**parameters[0], "no_cache": ["true"]}
assert all(params["start"] == ["0"] for params in parameters)
@@ -341,8 +342,8 @@ async def test_search_first_page_timeout_preserves_fallback(
request = AsyncMock(
side_effect=[TimeoutError("Google search timed out after 30 seconds."), PROVIDER_PAGE, PROVIDER_PAGE]
)
- monkeypatch.setattr(search_module.settings, "SERPAPI_API_KEY", "test-google-key")
- monkeypatch.setattr(WebSearchBlock, "_request", request)
+ monkeypatch.setattr(web_search_client.settings, "SERPAPI_API_KEY", "test-google-key")
+ monkeypatch.setattr(web_search_client, "request", request)
handler.return_value = {"llm_response": "Results processed."}
result = await block.execute("workflow-run-test", "block-run-test", "org-test")
@@ -353,7 +354,40 @@ async def test_search_first_page_timeout_preserves_fallback(
else:
assert result.status == BlockStatus.completed
assert result.output_parameter_value["provider"] == "exa"
- assert request.await_args_list[1].args[0] == "exa"
+ assert request.await_args_list[1].args[1] == "exa"
+
+
+ ("exa_key", "failure_reason"),
+ [
+ ("test-provider-key", "Exa search failed (HTTP 500). Exa ran because Google search failed (HTTP 500)."),
+ (None, "Google search failed (HTTP 500)."),
+ ],
+ ids=["exa-fallback-fails", "no-exa-key"],
+)
+async def test_a_google_failure_under_auto_falls_back_only_to_a_configured_exa(
+ search_setup: tuple[WebSearchBlock, WorkflowRunContext, AsyncMock],
+ monkeypatch: pytest.MonkeyPatch,
+ exa_key: str | None,
+ failure_reason: str,
+) -> None:
+ original, _, _ = search_setup
+ block = original.model_copy(update={"provider": "auto"})
+ request = AsyncMock(
+ side_effect=[
+ WebSearchError("Google search failed (HTTP 500)."),
+ WebSearchError("Exa search failed (HTTP 500)."),
+ ]
+ )
+ monkeypatch.setattr(web_search_client.settings, "SERPAPI_API_KEY", "test-google-key")
+ monkeypatch.setattr(web_search_client.settings, "EXA_API_KEY", exa_key)
+ monkeypatch.setattr(web_search_client, "request", request)
+
+ result = await block.execute("workflow-run-test", "block-run-test", "org-test")
+
+ assert result.status == BlockStatus.failed
+ assert result.failure_reason == failure_reason
@pytest.mark.asyncio
@@ -371,7 +405,7 @@ async def test_search_no_results_detects_legacy_code_after_prompt(
"errors": [{"error_code": "NO_SEARCH_RESULTS", "reasoning": "No results.", "confidence_float": 0.9}],
},
]
- monkeypatch.setattr(WebSearchBlock, "_request", AsyncMock(return_value={"results": []}))
+ monkeypatch.setattr(web_search_client, "request", AsyncMock(return_value={"results": []}))
result = await block.execute("workflow-run-test", "block-run-test", "org-test")
@@ -413,7 +447,7 @@ async def test_search_prompt_match_outcome(
block, context, handler = search_setup
block.no_match_error_code = "NO_MATCHING_RESULT"
block.json_schema = {"type": "array", "items": {"type": "string"}}
- monkeypatch.setattr(WebSearchBlock, "_request", AsyncMock(return_value=page))
+ monkeypatch.setattr(web_search_client, "request", AsyncMock(return_value=page))
handler.side_effect = [answer[0], {"reasoning": "Checked results.", "errors": answer[1]}]
result = await block.execute("workflow-run-test", "block-run-test", "org-test")