Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-1967): add platform-level JS dialog handler with LLM decision (#5679)
LawyZheng7ccb3a1dc267
diff --git a/skyvern/constants.py b/skyvern/constants.pyindex 837f25528..14f360484 100644--- a/skyvern/constants.py+++ b/skyvern/constants.py@@ -12,6 +12,7 @@ BROWSER_PAGE_CLOSE_TIMEOUT = 5 # 5 secondsBROWSER_CLOSE_TIMEOUT = 180 # 3 minuteBROWSER_DOWNLOAD_MAX_WAIT_TIME = 120 # 2 minuteBROWSER_DOWNLOAD_TIMEOUT = 600 # 10 minute+DIALOG_LLM_TIMEOUT = 10 # 10 secondsDOWNLOAD_FILE_PREFIX = "downloads"SAVE_DOWNLOADED_FILES_TIMEOUT = 180GET_DOWNLOADED_FILES_TIMEOUT = 30diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex c419113f0..e33a7a686 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -511,6 +511,8 @@ class ForgeAgent:context = skyvern_context.ensure_context()context.step_id = step.step_idcontext.task_id = task.task_id+ context.navigation_goal = task.navigation_goal+ context.navigation_payload = task.navigation_payload# do not need to do complete verification when it's a CUA task# 1. CUA executes only one action step by step -- it's pretty less likely to have a hallucination for completion or forget to return a complete@@ -1061,6 +1063,8 @@ class ForgeAgent:context = skyvern_context.ensure_context()context.step_id = Nonecontext.task_id = None+ context.navigation_goal = None+ context.navigation_payload = Noneasync def fail_task(self,diff --git a/skyvern/forge/prompts/skyvern/handle-dialog.j2 b/skyvern/forge/prompts/skyvern/handle-dialog.j2new file mode 100644index 000000000..48b3a31f6--- /dev/null+++ b/skyvern/forge/prompts/skyvern/handle-dialog.j2@@ -0,0 +1,30 @@+A JavaScript dialog has appeared in the browser during automated web navigation.++Dialog type: {{ dialog_type }}+Dialog message: "{{ dialog_message }}"+{% if default_value %}Default value: "{{ default_value }}"{% endif %}+{% if navigation_goal %}+Current task goal: {{ navigation_goal }}+{% endif %}+{% if navigation_payload %}+Task context/payload: {{ navigation_payload }}+{% endif %}++Based on the dialog message and the task goal, decide whether to ACCEPT or DISMISS this dialog.+{% if dialog_type == "prompt" %}+If you decide to accept, also provide the text to enter in the prompt field. If the default value is appropriate, you may use it.+{% endif %}++Guidelines:+- For confirmation dialogs asking to proceed with an action (submit, save, continue), typically ACCEPT.+- For dialogs warning about leaving the page or losing unsaved data, consider the task goal to decide.+- When in doubt, prefer ACCEPT as it is more likely to advance the workflow.++MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments.++Reply in JSON format:+{+ "reasoning": str,+ "action": "accept" | "dismiss",+ "prompt_text": str | null+}diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex 527abe084..8e7003995 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -29,6 +29,8 @@ class SkyvernContext:tz_info: ZoneInfo | None = Nonerun_id: str | None = Nonecopilot_session_id: str | None = None+ navigation_goal: str | None = None+ navigation_payload: dict[str, Any] | list | str | None = Nonetotp_codes: dict[str, str | None] = field(default_factory=dict)log: list[dict] = field(default_factory=list)hashed_href_map: dict[str, str] = field(default_factory=dict)diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.pyindex 913f75c73..e5baf8ef7 100644--- a/skyvern/webeye/browser_factory.py+++ b/skyvern/webeye/browser_factory.py@@ -33,6 +33,7 @@ from skyvern.forge.sdk.core.skyvern_context import current, ensure_contextfrom skyvern.schemas.runs import ProxyLocation, get_tzinfo_from_proxyfrom skyvern.webeye.browser_artifacts import BrowserArtifacts, VideoArtifactfrom skyvern.webeye.cdp_download_interceptor import CDPDownloadInterceptor+from skyvern.webeye.dialog_handler import set_dialog_handlerLOG = structlog.get_logger()@@ -350,6 +351,7 @@ class BrowserContextFactory:if settings.BROWSER_LOGS_ENABLED:set_browser_console_log(browser_context=browser_context, browser_artifacts=browser_artifacts)set_download_file_listener(browser_context=browser_context, **kwargs)+ set_dialog_handler(browser_context=browser_context)proxy_location: ProxyLocation | None = kwargs.get("proxy_location")if proxy_location is not None:diff --git a/skyvern/webeye/dialog_handler.py b/skyvern/webeye/dialog_handler.pynew file mode 100644index 000000000..55b888f4a--- /dev/null+++ b/skyvern/webeye/dialog_handler.py@@ -0,0 +1,134 @@+from __future__ import annotations++import asyncio+import json+import weakref++import structlog+from playwright.async_api import BrowserContext, Dialog, Page++from skyvern.constants import DIALOG_LLM_TIMEOUT+from skyvern.forge import app+from skyvern.forge.prompts import prompt_engine+from skyvern.forge.sdk.core import skyvern_context++LOG = structlog.get_logger()++# Track contexts that already have a dialog handler to avoid duplicate registration+# when the same BrowserContext is returned by CDP reconnect paths.+_registered_contexts: weakref.WeakSet[BrowserContext] = weakref.WeakSet()+++async def _handle_dialog(dialog: Dialog) -> None:+ """Handle a JavaScript dialog (alert/confirm/prompt/beforeunload) using LLM-based decision making.++ For alert and beforeunload dialogs, always accepts without calling the LLM.+ For confirm/prompt dialogs with no task context, auto-accepts (no LLM round-trip needed).+ For confirm/prompt dialogs with task context, calls the secondary LLM handler to decide.+ Falls back to accept on any error (safer than dismiss for form submissions).+ """+ dialog_type = dialog.type+ dialog_message = dialog.message+ default_value = dialog.default_value++ ctx = skyvern_context.current()+ organization_id = ctx.organization_id if ctx else None+ navigation_goal = (ctx.navigation_goal or ctx.prompt) if ctx else None+ navigation_payload = ctx.navigation_payload if ctx else None+ task_id = ctx.task_id if ctx else None+ workflow_run_id = ctx.workflow_run_id if ctx else None++ log = LOG.bind(+ dialog_type=dialog_type,+ dialog_message=dialog_message,+ task_id=task_id,+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ )++ # Alert, beforeunload, or no task context — auto-accept without LLM call+ if dialog_type in ("alert", "beforeunload"):+ log.info("Dialog auto-accepted", dialog_type=dialog_type)+ await dialog.accept()+ return++ if not navigation_goal and not navigation_payload:+ log.info("Dialog auto-accepted (no task context)", dialog_type=dialog_type)+ await dialog.accept(default_value or "")+ return++ # For confirm/prompt with task context, call LLM to decide+ try:+ prompt = prompt_engine.load_prompt(+ "handle-dialog",+ dialog_type=dialog_type,+ dialog_message=dialog_message,+ default_value=default_value,+ navigation_goal=navigation_goal,+ navigation_payload=json.dumps(navigation_payload) if navigation_payload else None,+ )++ # JS dialogs block the page's JS thread while open. We need a hard timeout+ # to ensure the page doesn't stay frozen indefinitely if the LLM call is slow.+ response = await asyncio.wait_for(+ app.SECONDARY_LLM_API_HANDLER(+ prompt=prompt,+ prompt_name="handle-dialog",+ organization_id=organization_id,+ ),+ timeout=DIALOG_LLM_TIMEOUT,+ )++ action = str(response.get("action", "accept")).lower()+ prompt_text = response.get("prompt_text")++ if action not in ("accept", "dismiss"):+ log.warning("Dialog LLM returned unexpected action, defaulting to accept", llm_action=action)+ action = "accept"++ log.info(+ "Dialog handled via LLM",+ action=action,+ has_prompt_text=prompt_text is not None,+ )++ if action == "dismiss":+ await dialog.dismiss()+ else:+ await dialog.accept(prompt_text if prompt_text is not None else (default_value or ""))++ except asyncio.TimeoutError:+ log.warning("Dialog LLM call timed out, falling back to accept")+ await dialog.accept(default_value or "")++ except Exception:+ log.exception("Dialog handler error, falling back to accept")+ await dialog.accept(default_value or "")+++def set_dialog_handler(browser_context: BrowserContext) -> None:+ """Register a dialog handler on all pages in the browser context.++ Hooks into browser_context.on("page", ...) to register the handler+ on every new page, including popups and new tabs. Also registers on+ any pages that already exist in the context.++ Uses a WeakSet to skip registration if the same BrowserContext is+ returned again (e.g., CDP reconnect reusing contexts[0]).++ Playwright-Python schedules async callbacks as tasks internally,+ so registering _handle_dialog directly is GC-safe.+ """+ if browser_context in _registered_contexts:+ return+ _registered_contexts.add(browser_context)++ def _on_page(page: Page) -> None:+ page.on("dialog", _handle_dialog)++ # Register on pages that already exist+ for page in browser_context.pages:+ _on_page(page)++ # Register on future pages+ browser_context.on("page", _on_page)