Skip to content

Automate browser based workflows with AI

feat: hook local debugger to the cdp-connect remote browser (#6179)

Andrew Neilson83eb2cabe842
diff --git a/skyvern/forge/forge_app.py b/skyvern/forge/forge_app.py
index c239d3af1..e2a36c886 100644
--- a/skyvern/forge/forge_app.py
+++ b/skyvern/forge/forge_app.py
@@ -2,6 +2,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Awaitable, Callable
+import structlog
from anthropic import AsyncAnthropic, AsyncAnthropicBedrock
from fastapi import FastAPI
from openai import AsyncAzureOpenAI, AsyncOpenAI
@@ -46,6 +47,8 @@ from skyvern.webeye.persistent_sessions_manager import PersistentSessionsManager
from skyvern.webeye.real_browser_manager import RealBrowserManager
from skyvern.webeye.scraper.scraper import ScrapeExcludeFunc
+LOG = structlog.get_logger()
+
class ForgeApp:
"""Container for shared Forge services"""
@@ -261,6 +264,13 @@ def create_forge_app() -> ForgeApp:
app.AGENT_FUNCTION = AgentFunction()
app.PERSISTENT_SESSIONS_MANAGER = DefaultPersistentSessionsManager(database=app.DATABASE)
app.PERSISTENT_SESSIONS_MANAGER.watch_session_pool()
+ if settings.BROWSER_TYPE == "cdp-connect" and settings.BROWSER_STREAMING_MODE != "cdp":
+ LOG.warning(
+ "BROWSER_TYPE=cdp-connect is set but BROWSER_STREAMING_MODE is not 'cdp'; "
+ "the debugger live-browser panel will be unavailable. "
+ "Set BROWSER_STREAMING_MODE=cdp to stream the remote browser through this backend.",
+ browser_streaming_mode=settings.BROWSER_STREAMING_MODE,
+ )
app.BROWSER_SESSION_RECORDING_SERVICE = BrowserSessionRecordingService()
app.AZURE_CLIENT_FACTORY = RealAzureClientFactory()
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
index 48f7a286d..9fd9a0c7a 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
@@ -185,9 +185,18 @@ PARAMETERIZED REQUESTS WITHOUT A SAMPLE VALUE:
* Defer to CREDENTIAL HANDLING first. If the request contains a raw credential, follow the credential-safety refusal flow above. If the request requires authentication and you have no `credential_id`, run `list_credentials` (with paging) before asking about a sample value. Only when no credential issue applies does this section take over.
* If the user's request signals a parameterized workflow ("for any X we input/provide", "for each Y", "for every Z", "for a list of Ns") but does NOT include a concrete value to test the workflow against, you MUST:
1. Decide which runtime parameter(s) the workflow needs and pick descriptive keys.
- 2. Respond with `ASK_QUESTION` on this turn — do NOT call any block-running, navigation, or page-exploration tool first. Name the parameter(s) you propose and ask for one sample value (per parameter) you can use to verify the workflow end-to-end. Example: "I'll build a parameterized workflow that takes `applicant_name`. What sample person should I use to test it?"
- 3. Do NOT loop exploring the page or testing with invented placeholders to discover what the parameter values should be — without a real value the form returns no results, every test reads as suspicious-success or null-data, and the budget burns without producing a committed workflow.
- 4. Once the user provides the sample, build the parameterized workflow, test it with the sample passed via `parameters`, and finalize.
+ 2. If the entrypoint URL is unknown (the user named the site but not the page), call `discover_workflow_entrypoint(site_or_url, intent_hint)` ONCE before composing blocks. This is the only browser-touching tool available before composition; direct browser primitives and workflow-mutation tools will be rejected with a steering message until discovery has produced a candidate URL.
+ 3. Once the entrypoint is known (from discovery, the user message, or the current workflow YAML), respond with `ASK_QUESTION` naming the parameter(s) you propose and asking for one sample value per parameter. Example: "I'll build a parameterized workflow that takes `applicant_name`. What sample person should I use to test it?"
+ 4. Do NOT loop exploring the page or testing with invented placeholders to discover what the parameter values should be — without a real value the form returns no results, every test reads as suspicious-success or null-data, and the budget burns without producing a committed workflow.
+ 5. Once the user provides the sample, build the parameterized workflow, test it with the sample passed via `parameters`, and finalize.
+
+BUILD-TIME DISCOVERY:
+* If the latest user message names a website but not a specific URL, call `discover_workflow_entrypoint(site_or_url, intent_hint)` BEFORE writing any blocks. Direct browser tools and workflow-mutation tools are gated off until composition; calling them will return a steering error pointing back to discovery.
+* `site_or_url` accepts: a URL with or without scheme (`example.com/login` is fine), a bare domain (`example.com`), or a single brand word (resolved as `https://www.<word>.com`). English phrases ("the X website") return `failure_reason=could_not_resolve_site_name` — `ASK_QUESTION` for a URL instead.
+* When discovery returns a `candidate_url`, USE IT — call `update_and_run_blocks` immediately with a `goto_url` block whose `url:` is that `candidate_url`. Do NOT `ASK_QUESTION` for confirmation the user already implied by naming the site; confirmation is appropriate only when discovery returned a `failure_reason`.
+* Discovery returns `candidate_url`, `candidate_form_fields`, and an `evidence_trail`. Use `candidate_url` as the `url:` value on a `goto_url` block. Do NOT paste evidence verbatim into workflow YAML.
+* Budget: one successful call per turn, three per chat, eight page hops, sixty seconds. On any `failure_reason` (`step_limit`, `no_candidate`, `could_not_resolve_site_name`, `login_wall`, `anti_bot_wall`, `discovery_budget_exhausted_for_chat`), `ASK_QUESTION` for a specific URL — do not propose a vendor of your own.
+* Discovery navigates and reads pages. It will NOT type, click form buttons, run JavaScript, or submit forms.
WORKFLOW SUBMISSION RULE:
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index a92ef888b..81b33e2a8 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -31,6 +31,7 @@ from pydantic import ValidationError
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.copilot.build_phase import initial_build_phase
from skyvern.forge.sdk.copilot.config import CopilotConfig
from skyvern.forge.sdk.copilot.context import (
COPILOT_RESPONSE_TYPES,
@@ -2081,11 +2082,24 @@ async def _run_copilot_turn_impl(
previous_user_message=previous_user_message,
)
- # Hydrate the per-chat discovery counter from the inbound global_llm_context.
- # `ctx.build_phase` stays at the safe default and `initial_build_phase` is
- # not yet called from the agent loop — that activates in the discovery PR.
+ # Hydrate the per-chat discovery counter from the inbound global_llm_context
+ # and set the initial build phase. Phase is set once per turn by the
+ # orchestrator; transitions happen inside `discover_workflow_entrypoint`
+ # and `update_and_run_blocks`, never from a model emission.
prior_structured_context = StructuredContext.from_json_str(global_llm_context)
ctx.prior_discovery_calls_made = prior_structured_context.discovery_calls_made
+ ctx.build_phase = initial_build_phase(
+ ctx.turn_intent,
+ chat_request.message or "",
+ agent_user_message or "",
+ chat_request.workflow_yaml or "",
+ )
+ LOG.info(
+ "copilot.build_phase_initial",
+ build_phase=ctx.build_phase.value,
+ workflow_permanent_id=chat_request.workflow_permanent_id,
+ prior_discovery_calls_made=ctx.prior_discovery_calls_made,
+ )
# Preflight feasibility classifier — fires on every turn so mid-session pivots
# to impossible targets are caught the same as first-turn structural mismatches.
@@ -2206,6 +2220,9 @@ async def _run_copilot_turn_impl(
allowlist=frozenset(alias_map.values()),
context_provider=lambda: ctx,
)
+ # The discovery walker reaches the connected FastMCP client through
+ # ctx, without exposing private overlay state.
+ ctx.discovery_mcp_server = mcp_server
agent = Agent(
name="workflow-copilot",
instructions=system_prompt,
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index 989b5bc95..2371959f4 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -20,6 +20,7 @@ from mcp.types import (
TextContent,
)
+from skyvern.forge.sdk.copilot.build_phase import _phase_tool_error
from skyvern.forge.sdk.copilot.loop_detection import (
detect_failed_tool_step_loop_for_ctx,
detect_tool_loop,
@@ -200,6 +201,22 @@ class SkyvernOverlayMCPServer(MCPServer):
copilot_ctx = self._context_provider()
overlay = self._overlays.get(tool_name, SchemaOverlay())
+ # Defense-in-depth phase gate for MCP-exposed browser primitives. The
+ # native gate (`_authority_tool_error` in tools.py) covers native
+ # function_tool calls; this branch covers tools dispatched only
+ # through SkyvernOverlayMCPServer, which never pass through the
+ # native gate. The two checks share `_phase_tool_error`, so a tool
+ # gated at both layers reports the same error message.
+ phase_error = _phase_tool_error(copilot_ctx, tool_name)
+ if phase_error:
+ LOG.warning(
+ "Phase-gated MCP tool call rejected",
+ tool_name=tool_name,
+ build_phase=getattr(getattr(copilot_ctx, "build_phase", None), "value", None),
+ )
+ record_tool_step_result_for_ctx(copilot_ctx, tool_name, arguments, {"ok": False, "error": phase_error})
+ return _copilot_to_call_tool_result({"ok": False, "error": phase_error})
+
loop_error = detect_failed_tool_step_loop_for_ctx(copilot_ctx, tool_name, arguments)
if loop_error:
LOG.warning(
@@ -282,6 +299,49 @@ class SkyvernOverlayMCPServer(MCPServer):
enqueue_screenshot_from_result(copilot_ctx, copilot_result)
return _copilot_to_call_tool_result(copilot_result)
+ async def call_internal_tool(
+ self,
+ mcp_tool_name: str,
+ mcp_args: dict[str, Any],
+ ) -> dict[str, Any]:
+ """Raw FastMCP call for internal copilot subsystems (discovery walker).
+
+ Bypasses overlay hooks, loop detection, and screenshot recording —
+ those are model-facing concerns. Still routes through
+ ``ensure_browser_session`` and ``mcp_browser_context`` for session/auth
+ scoping. Mirrors the error-handling block from ``call_tool`` so MCP-
+ side validation or tool errors surface as ``ok=False`` with an
+ extracted error string rather than silently defaulting to
+ ``ok=True``.
+ """
+ if not self._client:
+ return {"ok": False, "error": "MCP client not connected"}
+ ctx = self._context_provider()
+ err = await ensure_browser_session(ctx)
+ if err:
+ return err
+ merged_args = {**mcp_args, "session_id": ctx.browser_session_id}
+ try:
+ async with mcp_browser_context(ctx):
+ raw = await self._client.call_tool(mcp_tool_name, merged_args, raise_on_error=False)
+ except Exception as exc:
+ LOG.warning(
+ "Internal MCP tool call failed",
+ tool=mcp_tool_name,
+ error=str(exc),
+ exc_info=True,
+ )
+ return {"ok": False, "error": f"{mcp_tool_name} failed: {exc}"}
+ raw_mcp = dict(raw.structured_content or {})
+ if raw.is_error:
+ raw_mcp["ok"] = False
+ if not raw.structured_content and raw.content:
+ text_parts = [c.text for c in raw.content if hasattr(c, "text")]
+ raw_mcp["error"] = " ".join(text_parts) if text_parts else "Unknown MCP error"
+ else:
+ raw_mcp["error"] = raw_mcp.get("error") or "Unknown MCP error"
+ return mcp_to_copilot(raw_mcp)
+
async def list_prompts(self) -> ListPromptsResult:
return ListPromptsResult(prompts=[])
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index 7f1b58a27..75ca6f1e7 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -10,11 +10,16 @@ import time
from collections import Counter, defaultdict
from datetime import datetime
from typing import Any, Literal
-from urllib.parse import parse_qsl, urlparse
+from urllib.parse import parse_qsl, urljoin, urlparse
import structlog
import yaml
from agents import ToolGuardrailFunctionOutput, ToolInputGuardrail, ToolInputGuardrailData, function_tool
+
+try:
+ from bs4 import BeautifulSoup # type: ignore[import-not-found]
+except ImportError: # pragma: no cover — bs4 is a transitive dep but discovery degrades gracefully without it.
+ BeautifulSoup = None # type: ignore[assignment, misc]
from agents.run_context import RunContextWrapper
from jinja2.sandbox import SandboxedEnvironment
from pydantic import ValidationError
@@ -25,6 +30,13 @@ from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stamp
from skyvern.forge.sdk.copilot.block_goal_wrapping import wrap_workflow_block_goals
from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
+from skyvern.forge.sdk.copilot.build_phase import (
+ BuildPhase,
+ _phase_tool_error,
+ advance_to_composing,
+ advance_to_discovering,
+ advance_to_testing,
+)
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
DiagnosisRepairContract,
@@ -1127,7 +1139,18 @@ def _authority_tool_error(
)
if request_policy_error is not None and not ignore_request_policy_error:
return request_policy_error
- return turn_intent_error
+ if turn_intent_error is not None:
+ return turn_intent_error
+ phase_error = _phase_tool_error(ctx, tool_name)
+ if phase_error is not None:
+ LOG.info(
+ "copilot authority gate blocked tool",
+ authority_gate_layer="build_phase",
+ blocked_tool=tool_name,
+ build_phase=getattr(getattr(ctx, "build_phase", None), "value", None),
+ safe_reason_code="build_phase_gate",
+ )
+ return phase_error
_PARAMETER_TYPE_PLACEHOLDERS: dict[WorkflowParameterType, Any] = {
@@ -4487,15 +4510,560 @@ async def update_and_run_blocks_tool(
workflow_updated=True,
)
enqueue_screenshot_from_result(copilot_ctx, run_result)
+ if run_result.get("ok"):
+ advance_to_testing(copilot_ctx)
sanitized = sanitize_tool_result_for_llm("run_blocks_and_collect_debug", run_result)
return json.dumps(sanitized)
+# Build-time entrypoint discovery: navigates and reads pages, returns a
+# candidate URL into the agent's context. Never mutates workflow YAML.
+# Available only during INITIAL / DISCOVERING phases.
+
+_DISCOVERY_PER_CHAT_BUDGET = 3
+_DISCOVERY_PER_TURN_BUDGET = 1
+_DISCOVERY_WALL_CLOCK_SECONDS = 60.0
+_DISCOVERY_STEP_CAP = 8
+_DISCOVERY_EVIDENCE_TRAIL_MAX = 8
+_DISCOVERY_CANDIDATE_FORM_FIELDS_MAX = 10
+_DISCOVERY_HTML_BYTES_MAX = 200_000
+
+_DISCOVERY_URL_SCHEME_RE = re.compile(r"^https?://", re.IGNORECASE)
+# host + optional path/query/fragment — handles `example.com/login`,
+# `the-internet.herokuapp.com/tables?x=y`, `site.com?q=x`, `site.com#frag`.
+_DISCOVERY_DOMAIN_WITH_PATH_RE = re.compile(
+ r"^[a-z0-9-]+(\.[a-z]{2,})+([/?#][^\s]*)?$",
+ re.IGNORECASE,
+)
+_DISCOVERY_BARE_WORD_RE = re.compile(r"^[a-z0-9-]{2,32}$", re.IGNORECASE)
+_DISCOVERY_TOKEN_RE = re.compile(r"[a-z0-9]+", re.IGNORECASE)
+_DISCOVERY_LOGIN_TITLE_RE = re.compile(r"\b(sign\s*in|log\s*in|login)\b", re.IGNORECASE)
+_DISCOVERY_PASSWORD_INPUT_RE = re.compile(
+ r"<input[^>]*type\s*=\s*[\"']password[\"']",
+ re.IGNORECASE,
+)
+_DISCOVERY_ANTI_BOT_PATTERNS = (
+ "cloudflare",
+ "just a moment",
+ "captcha",
+ "verify you are human",
+ "access denied",
+ "akamai",
+ "are you a robot",
+)
+
+
+def _resolve_discovery_entry_url(site_or_url: str) -> tuple[str | None, str]:
+ """Resolve the user-supplied site name/URL into a navigable URL.
+
+ Returns ``(resolved_url, kind)`` where ``kind`` is one of:
+ ``url`` / ``domain`` / ``word`` / ``unresolved``.
+ """
+ token = (site_or_url or "").strip()
+ if not token:
+ return None, "unresolved"
+ if _DISCOVERY_URL_SCHEME_RE.match(token):
+ return token, "url"
+ if _DISCOVERY_DOMAIN_WITH_PATH_RE.match(token):
+ return f"https://{token}", "domain"
+ if _DISCOVERY_BARE_WORD_RE.match(token):
+ return f"https://www.{token.lower()}.com", "word"
+ return None, "unresolved"
+
+
+def _discovery_anchor_score(
+ anchor_text: str,
+ anchor_title: str,
+ href_path: str,
+ intent_tokens: set[str],
+) -> int:
+ """Count intent tokens that appear as substrings of the combined anchor text.
+
+ Substring (not exact-token) matching handles ``sort`` ↔ ``sortable`` and
+ ``table`` ↔ ``tables`` without a full stemmer.
+ """
+ if not intent_tokens:
+ return 0
+ combined = f"{anchor_text} {anchor_title} {href_path}".lower()
+ return sum(1 for token in intent_tokens if token in combined)
+
+
+def _discovery_title_score(page_title: str, intent_tokens: set[str]) -> int:
+ if not intent_tokens or not page_title:
+ return 0
+ lowered = page_title.lower()
+ return sum(1 for token in intent_tokens if token in lowered)
+
+
+def _discovery_detect_login_wall(html: str, page_title: str) -> bool:
+ if _DISCOVERY_LOGIN_TITLE_RE.search(page_title or ""):
+ return True
+ return bool(_DISCOVERY_PASSWORD_INPUT_RE.search(html or ""))
+
+
+def _discovery_detect_anti_bot(html: str, page_title: str) -> bool:
+ lowered_title = (page_title or "").lower()
+ lowered_html = (html or "")[:_DISCOVERY_HTML_BYTES_MAX].lower()
+ return any(pat in lowered_title or pat in lowered_html for pat in _DISCOVERY_ANTI_BOT_PATTERNS)
+
+
+def _discovery_build_result(
+ *,
+ candidate_url: str | None,
+ candidate_form_fields: list[dict[str, Any]],
+ evidence_trail: list[dict[str, Any]],
+ confidence: float,
+ failure_reason: str | None,
+ ok: bool = True,
+ error: str | None = None,
+) -> dict[str, Any]:
+ """Shape a `discover_workflow_entrypoint` result envelope.
+
+ Convention: ``ok=True`` for any *completed* walk — including controlled
+ outcomes that report a ``failure_reason`` and ``candidate_url=None``.
+ ``ok=False`` is reserved for actual tool errors (MCP unavailable, browser
+ boot failure, internal exception). Matches the existing copilot
+ ``_request_policy_tool_error`` convention so the eval harness counts a
+ controlled failure as a successful tool call.
+ """
+ return {
+ "ok": ok,
+ "data": {
+ "candidate_url": candidate_url,
+ "candidate_form_fields": candidate_form_fields[:_DISCOVERY_CANDIDATE_FORM_FIELDS_MAX],
+ "evidence_trail": evidence_trail[:_DISCOVERY_EVIDENCE_TRAIL_MAX],
+ "confidence": float(confidence),
+ "failure_reason": failure_reason,
+ },
+ "error": error,
+ }
+
+
+def _discovery_parse_html(html: str) -> tuple[str, list[dict[str, str]], list[dict[str, str]]]:
+ """Parse HTML for page title, link anchors, and form-field metadata.
+
+ Uses BeautifulSoup if available (a transitive Skyvern dep). Falls back to
+ empty results if not — discovery degrades gracefully rather than crashing.
+ """
+ if BeautifulSoup is None:
+ return "", [], []
+
+ try:
+ soup = BeautifulSoup(html or "", "html.parser")
+ except Exception:
+ return "", [], []
+
+ title_text = ""
+ title_tag = soup.find("title")
+ if title_tag and title_tag.string:
+ title_text = title_tag.string.strip()
+ h1_tag = soup.find("h1")
+ if h1_tag:
+ h1_text = h1_tag.get_text(strip=True)
+ if h1_text:
+ title_text = f"{title_text} {h1_text}".strip()
+
+ anchors: list[dict[str, str]] = []
+ for a in soup.find_all("a", href=True):
+ href = a.get("href", "").strip()
+ if not href or href.startswith("#") or href.lower().startswith("javascript:"):
+ continue
+ text = a.get_text(" ", strip=True)
+ anchors.append(
+ {
+ "href": href,
+ "text": text[:240],
+ "title": (a.get("title") or "")[:240],
+ }
+ )
+
+ form_fields: list[dict[str, str]] = []
+ form = soup.find("form")
+ if form is not None:
+ for inp in form.find_all(["input", "select", "textarea"]):
+ field_type = inp.get("type", inp.name) or "text"
+ if field_type.lower() in {"hidden", "submit", "button"}:
+ continue
+ field_name = inp.get("name") or inp.get("id") or ""
+ label_text = ""
+ label_id = inp.get("id")
+ if label_id:
+ label_tag = soup.find("label", attrs={"for": label_id})
+ if label_tag is not None:
+ label_text = label_tag.get_text(" ", strip=True)
+ form_fields.append(
+ {
+ "name": field_name[:120],
+ "label": label_text[:240],
+ "type": str(field_type)[:40],
+ "value_hint": (inp.get("placeholder") or "")[:240],
+ }
+ )
+
+ return title_text, anchors, form_fields
+
+
+def _discovery_resolve_href(base_url: str, href: str) -> str | None:
+ try:
+ absolute = urljoin(base_url, href)
+ except Exception:
+ return None
+ parsed_abs = urlparse(absolute)
+ parsed_base = urlparse(base_url)
+ if parsed_abs.scheme not in {"http", "https"}:
+ return None
+ # Same-origin only — discovery does not follow cross-origin links to keep
+ # the entrypoint search bounded to the user's named site.
+ if parsed_abs.netloc and parsed_base.netloc and parsed_abs.netloc != parsed_base.netloc:
+ return None
+ return absolute
+
+
+# Per-call timeout for each MCP primitive inside the discovery walker. The
+# walker also checks the cumulative 60s wall clock between steps, but without
+# a per-call cap a single hung navigate or get_html could block past the
+# cumulative cap (cumulative is only checked between awaits).
+_DISCOVERY_PER_CALL_TIMEOUT_SECONDS = 20.0
+
+
+async def _discovery_navigate(ctx: CopilotContext, url: str) -> dict[str, Any]:
+ server = getattr(ctx, "discovery_mcp_server", None)
+ if server is None:
+ return {"ok": False, "error": "discovery MCP server not attached to context"}
+ try:
+ return await asyncio.wait_for(
+ server.call_internal_tool("skyvern_navigate", {"url": url}),
+ timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+ )
+ except asyncio.TimeoutError:
+ return {"ok": False, "error": f"skyvern_navigate timed out after {_DISCOVERY_PER_CALL_TIMEOUT_SECONDS:g}s"}
+
+
+async def _discovery_get_html(ctx: CopilotContext) -> dict[str, Any]:
+ """Read the full page body. ``skyvern_get_html`` requires a selector arg;
+ pass ``body`` so the walker receives the full document body. Without this
+ the raw MCP call fails validation since the inspection tool has a
+ required positional ``selector``.
+ """
+ server = getattr(ctx, "discovery_mcp_server", None)
+ if server is None:
+ return {"ok": False, "error": "discovery MCP server not attached to context"}
+ try:
+ return await asyncio.wait_for(
+ server.call_internal_tool("skyvern_get_html", {"selector": "body"}),
+ timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+ )
+ except asyncio.TimeoutError:
+ return {"ok": False, "error": f"skyvern_get_html timed out after {_DISCOVERY_PER_CALL_TIMEOUT_SECONDS:g}s"}
+
+
+def _discovery_extract_html_payload(result: dict[str, Any]) -> str:
+ data = result.get("data")
+ if isinstance(data, dict):
+ for key in ("html", "outer_html", "text", "content"):
+ value = data.get(key)
+ if isinstance(value, str):
+ return value
+ return ""
+
+
+def _discovery_extract_current_url(result: dict[str, Any], fallback: str) -> str:
+ data = result.get("data")
+ if isinstance(data, dict):
+ url = data.get("url") or data.get("current_url")
+ if isinstance(url, str) and url:
+ return url
+ return fallback
+
+
+async def _discovery_walk(
+ ctx: CopilotContext,
+ *,
+ entry_url: str,
+ intent_hint: str,
+) -> dict[str, Any]:
+ """Deterministic anchor-scoring walker. No inner LLM call.
+
+ Reads each visited page, looks for a strong title/H1 match with the
+ intent_hint, surfaces a form if one is present, and otherwise follows
+ the highest-scored same-origin anchor whose text matches the
+ intent_hint. Bounded by step / wall-clock caps.
+ """
+ intent_tokens = set(_DISCOVERY_TOKEN_RE.findall(intent_hint.lower())) if intent_hint else set()
+ evidence_trail: list[dict[str, Any]] = []
+ current_url = entry_url
+ started = ctx.discovery_started_monotonic or time.monotonic()
+
+ for step in range(_DISCOVERY_STEP_CAP):
+ ctx.discovery_step_count = step + 1
+ elapsed = time.monotonic() - started
+ if elapsed > _DISCOVERY_WALL_CLOCK_SECONDS:
+ return _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=evidence_trail,
+ confidence=0.0,
+ failure_reason="wall_clock_limit",
+ )
+
+ nav_result = await _discovery_navigate(ctx, current_url)
+ if not nav_result.get("ok"):
+ evidence_trail.append(
+ {
+ "url": current_url,
+ "page_title": "",
+ "transition_reason": f"navigate_failed: {nav_result.get('error', 'unknown')}"[:240],
+ }
+ )
+ return _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=evidence_trail,
+ confidence=0.0,
+ failure_reason="no_candidate",
+ )
+
+ current_url = _discovery_extract_current_url(nav_result, current_url)
+ html_result = await _discovery_get_html(ctx)
+ html = _discovery_extract_html_payload(html_result) if html_result.get("ok") else ""
+ page_title, anchors, form_fields = _discovery_parse_html(html)
+
+ evidence_trail.append(
+ {
+ "url": current_url,
+ "page_title": page_title[:240],
+ "transition_reason": "initial" if step == 0 else "anchor_match",
+ }
+ )
+
+ if _discovery_detect_anti_bot(html, page_title):
+ return _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=evidence_trail,
+ confidence=0.0,
+ failure_reason="anti_bot_wall",
+ )
+ if _discovery_detect_login_wall(html, page_title):
+ return _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=evidence_trail,
+ confidence=0.0,
+ failure_reason="login_wall",
+ )
+
+ title_score = _discovery_title_score(page_title, intent_tokens)
+
+ if intent_tokens and title_score >= 2:
+ confidence = min(1.0, title_score / max(1, len(intent_tokens)))
+ return _discovery_build_result(
+ candidate_url=current_url,
+ candidate_form_fields=form_fields,
+ evidence_trail=evidence_trail,
+ confidence=confidence,
+ failure_reason=None,
+ )
+
+ if form_fields and (title_score >= 1 or step > 0):
+ confidence = 0.6 if title_score >= 1 else 0.4
+ return _discovery_build_result(
+ candidate_url=current_url,
+ candidate_form_fields=form_fields,
+ evidence_trail=evidence_trail,
+ confidence=confidence,
+ failure_reason=None,
+ )
+
+ if not intent_tokens:
+ return _discovery_build_result(
+ candidate_url=current_url,
+ candidate_form_fields=form_fields,
+ evidence_trail=evidence_trail,
+ confidence=0.3,
+ failure_reason=None,
+ )
+
+ best_score = 0
+ best_href: str | None = None
+ for anchor in anchors:
+ score = _discovery_anchor_score(
+ anchor.get("text", ""),
+ anchor.get("title", ""),
+ anchor.get("href", ""),
+ intent_tokens,
+ )
+ if score > best_score:
+ resolved = _discovery_resolve_href(current_url, anchor.get("href", ""))
+ if resolved is None:
+ continue
+ best_score = score
+ best_href = resolved
+
+ if best_score == 0 or best_href is None:
+ return _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=evidence_trail,
+ confidence=0.0,
+ failure_reason="no_candidate",
+ )
+
+ current_url = best_href
+
+ return _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=evidence_trail,
+ confidence=0.0,
+ failure_reason="step_limit",
+ )
+
+
+async def _discover_workflow_entrypoint_impl(
+ copilot_ctx: Any,
+ site_or_url: str,
+ intent_hint: str,
+) -> dict[str, Any]:
+ """Discovery tool body — separated from the @function_tool wrapper so
+ tests can drive it with a stand-in ctx without the SDK's invocation
+ machinery.
+ """
+ arguments = {"site_or_url": site_or_url, "intent_hint": intent_hint}
+
+ authority_error = _authority_tool_error(copilot_ctx, "discover_workflow_entrypoint")
+ if authority_error:
+ result = {"ok": False, "error": authority_error}
+ record_tool_step_result_for_ctx(copilot_ctx, "discover_workflow_entrypoint", arguments, result)
+ return result
+
+ if copilot_ctx.discovery_calls_this_turn >= _DISCOVERY_PER_TURN_BUDGET:
+ result = _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=list(copilot_ctx.discovery_evidence_trail),
+ confidence=0.0,
+ failure_reason="discovery_already_completed_this_turn",
+ )
+ record_tool_step_result_for_ctx(copilot_ctx, "discover_workflow_entrypoint", arguments, result)
+ return result
+
+ cumulative = copilot_ctx.prior_discovery_calls_made + copilot_ctx.discovery_calls_this_turn
+ if cumulative >= _DISCOVERY_PER_CHAT_BUDGET:
+ result = _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=[],
+ confidence=0.0,
+ failure_reason="discovery_budget_exhausted_for_chat",
+ )
+ record_tool_step_result_for_ctx(copilot_ctx, "discover_workflow_entrypoint", arguments, result)
+ return result
+
+ entry_url, kind = _resolve_discovery_entry_url(site_or_url)
+ if entry_url is None:
+ result = _discovery_build_result(
+ candidate_url=None,
+ candidate_form_fields=[],
+ evidence_trail=[],
+ confidence=0.0,
+ failure_reason="could_not_resolve_site_name",
+ )
+ record_tool_step_result_for_ctx(copilot_ctx, "discover_workflow_entrypoint", arguments, result)
+ return result
+
+ if copilot_ctx.build_phase == BuildPhase.INITIAL:
+ try:
+ advance_to_discovering(copilot_ctx)
+ except ValueError as exc:
+ # Race or unexpected prior advance — proceed without re-transitioning,
+ # but surface the impossible state so it shows up in production logs.
+ LOG.warning(
+ "discover_workflow_entrypoint phase transition to discovering rejected",
+ error=str(exc),
+ build_phase=copilot_ctx.build_phase.value,
+ )
+ copilot_ctx.discovery_calls_this_turn += 1
+
+ with copilot_span(
+ "discover_workflow_entrypoint",
+ data={
+ "site_or_url_kind": kind,
+ "intent_hint_len": len(intent_hint or ""),
+ "phase_entered": copilot_ctx.build_phase.value,
+ },
+ ):
+ try:
+ result = await _discovery_walk(
+ copilot_ctx,
+ entry_url=entry_url,
+ intent_hint=intent_hint or "",
+ )
+ except Exception as exc:
+ LOG.exception("discover_workflow_entrypoint walker raised")
+ result = {
+ "ok": False,
+ "data": {
+ "candidate_url": None,
+ "candidate_form_fields": [],
+ "evidence_trail": [],
+ "confidence": 0.0,
+ "failure_reason": None,
+ },
+ "error": f"discover_workflow_entrypoint failed: {exc}",
+ }
+
+ data_payload = result.get("data") or {}
+ data: dict[str, Any] = data_payload if isinstance(data_payload, dict) else {}
+ copilot_ctx.discovery_evidence_trail = list(data.get("evidence_trail", []))
+ if result.get("ok") and data.get("candidate_url"):
+ try:
+ advance_to_composing(copilot_ctx, reason="discovery_returned_candidate")
+ except ValueError as exc:
+ LOG.warning(
+ "discover_workflow_entrypoint phase transition to composing rejected",
+ error=str(exc),
+ build_phase=copilot_ctx.build_phase.value,
+ )
+
+ record_tool_step_result_for_ctx(copilot_ctx, "discover_workflow_entrypoint", arguments, result)
+ return result
+
+
+@function_tool(name_override="discover_workflow_entrypoint", strict_mode=False)
+async def discover_workflow_entrypoint_tool(
+ ctx: RunContextWrapper,
+ site_or_url: str,
+ intent_hint: str,
+) -> str:
+ """Find the page a new workflow should start at when the user named a site but not the page.
+
+ Use this BEFORE writing blocks when the user named a website (with a URL,
+ a bare domain, or a single brand word) but no specific page. Accepts:
+ a URL with or without scheme (``example.com/login`` is fine), a bare
+ domain (``example.com``), or a single brand word (resolved as
+ ``https://www.<word>.com``). English phrases ("the X website") return
+ ``failure_reason=could_not_resolve_site_name`` — ASK_QUESTION for a URL.
+
+ Returns ``candidate_url`` plus a short ``evidence_trail`` and any
+ ``candidate_form_fields``. Use ``candidate_url`` as the ``url`` value
+ on a ``goto_url`` block. Do NOT paste the evidence into workflow YAML.
+
+ Budget: one successful call per turn, three per chat, eight page hops,
+ sixty seconds. On any ``failure_reason``, ASK_QUESTION for a URL — do not
+ retry. Discovery navigates and reads pages; it will NOT type, click form
+ buttons, run JavaScript, or submit forms.
+ """
+ result = await _discover_workflow_entrypoint_impl(ctx.context, site_or_url, intent_hint)
+ return json.dumps(result)
+
+
NATIVE_TOOLS = [
update_workflow_tool,
list_credentials_tool,
run_blocks_tool,
get_run_results_tool,
update_and_run_blocks_tool,
+ discover_workflow_entrypoint_tool,
]
diff --git a/skyvern/webeye/default_persistent_sessions_manager.py b/skyvern/webeye/default_persistent_sessions_manager.py
index 174cc0f5b..f318d584b 100644
--- a/skyvern/webeye/default_persistent_sessions_manager.py
+++ b/skyvern/webeye/default_persistent_sessions_manager.py
@@ -296,9 +296,12 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):
browser_profile_id=browser_profile_id,
)
- # In local mode, launch the browser immediately for standalone sessions
- # so the screencast/CDP input endpoints can connect.
- if settings.BROWSER_STREAMING_MODE == "cdp" and runnable_id is None:
+ # Launch the browser immediately for standalone sessions so the
+ # screencast/CDP input endpoints can connect. Triggered both by the
+ # in-process CDP streaming mode and by cdp-connect, which forwards to a
+ # remote browser and still needs a registered BrowserState locally.
+ should_launch = settings.BROWSER_STREAMING_MODE == "cdp" or settings.BROWSER_TYPE == "cdp-connect"
+ if should_launch and runnable_id is None:
session_id = session.persistent_browser_session_id
task = asyncio.create_task(
self._launch_browser_for_session(session_id, organization_id, proxy_location, url)