Skip to content

Automate browser based workflows with AI

feat(SKY-1967): add platform-level JS dialog handler with LLM decision (#5679)

LawyZheng7ccb3a1dc267
diff --git a/skyvern/constants.py b/skyvern/constants.py
index 837f25528..14f360484 100644
--- a/skyvern/constants.py
+++ b/skyvern/constants.py
@@ -12,6 +12,7 @@ BROWSER_PAGE_CLOSE_TIMEOUT = 5 # 5 seconds
BROWSER_CLOSE_TIMEOUT = 180 # 3 minute
BROWSER_DOWNLOAD_MAX_WAIT_TIME = 120 # 2 minute
BROWSER_DOWNLOAD_TIMEOUT = 600 # 10 minute
+DIALOG_LLM_TIMEOUT = 10 # 10 seconds
DOWNLOAD_FILE_PREFIX = "downloads"
SAVE_DOWNLOADED_FILES_TIMEOUT = 180
GET_DOWNLOADED_FILES_TIMEOUT = 30
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index c419113f0..e33a7a686 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -511,6 +511,8 @@ class ForgeAgent:
context = skyvern_context.ensure_context()
context.step_id = step.step_id
context.task_id = task.task_id
+ context.navigation_goal = task.navigation_goal
+ context.navigation_payload = task.navigation_payload
# do not need to do complete verification when it's a CUA task
# 1. CUA executes only one action step by step -- it's pretty less likely to have a hallucination for completion or forget to return a complete
@@ -1061,6 +1063,8 @@ class ForgeAgent:
context = skyvern_context.ensure_context()
context.step_id = None
context.task_id = None
+ context.navigation_goal = None
+ context.navigation_payload = None
async def fail_task(
self,
diff --git a/skyvern/forge/prompts/skyvern/handle-dialog.j2 b/skyvern/forge/prompts/skyvern/handle-dialog.j2
new file mode 100644
index 000000000..48b3a31f6
--- /dev/null
+++ b/skyvern/forge/prompts/skyvern/handle-dialog.j2
@@ -0,0 +1,30 @@
+A JavaScript dialog has appeared in the browser during automated web navigation.
+
+Dialog type: {{ dialog_type }}
+Dialog message: "{{ dialog_message }}"
+{% if default_value %}Default value: "{{ default_value }}"{% endif %}
+{% if navigation_goal %}
+Current task goal: {{ navigation_goal }}
+{% endif %}
+{% if navigation_payload %}
+Task context/payload: {{ navigation_payload }}
+{% endif %}
+
+Based on the dialog message and the task goal, decide whether to ACCEPT or DISMISS this dialog.
+{% if dialog_type == "prompt" %}
+If you decide to accept, also provide the text to enter in the prompt field. If the default value is appropriate, you may use it.
+{% endif %}
+
+Guidelines:
+- For confirmation dialogs asking to proceed with an action (submit, save, continue), typically ACCEPT.
+- For dialogs warning about leaving the page or losing unsaved data, consider the task goal to decide.
+- When in doubt, prefer ACCEPT as it is more likely to advance the workflow.
+
+MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments.
+
+Reply in JSON format:
+{
+ "reasoning": str,
+ "action": "accept" | "dismiss",
+ "prompt_text": str | null
+}
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.py
index 527abe084..8e7003995 100644
--- a/skyvern/forge/sdk/core/skyvern_context.py
+++ b/skyvern/forge/sdk/core/skyvern_context.py
@@ -29,6 +29,8 @@ class SkyvernContext:
tz_info: ZoneInfo | None = None
run_id: str | None = None
copilot_session_id: str | None = None
+ navigation_goal: str | None = None
+ navigation_payload: dict[str, Any] | list | str | None = None
totp_codes: dict[str, str | None] = field(default_factory=dict)
log: list[dict] = field(default_factory=list)
hashed_href_map: dict[str, str] = field(default_factory=dict)
diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.py
index 913f75c73..e5baf8ef7 100644
--- a/skyvern/webeye/browser_factory.py
+++ b/skyvern/webeye/browser_factory.py
@@ -33,6 +33,7 @@ from skyvern.forge.sdk.core.skyvern_context import current, ensure_context
from skyvern.schemas.runs import ProxyLocation, get_tzinfo_from_proxy
from skyvern.webeye.browser_artifacts import BrowserArtifacts, VideoArtifact
from skyvern.webeye.cdp_download_interceptor import CDPDownloadInterceptor
+from skyvern.webeye.dialog_handler import set_dialog_handler
LOG = structlog.get_logger()
@@ -350,6 +351,7 @@ class BrowserContextFactory:
if settings.BROWSER_LOGS_ENABLED:
set_browser_console_log(browser_context=browser_context, browser_artifacts=browser_artifacts)
set_download_file_listener(browser_context=browser_context, **kwargs)
+ set_dialog_handler(browser_context=browser_context)
proxy_location: ProxyLocation | None = kwargs.get("proxy_location")
if proxy_location is not None:
diff --git a/skyvern/webeye/dialog_handler.py b/skyvern/webeye/dialog_handler.py
new file mode 100644
index 000000000..55b888f4a
--- /dev/null
+++ b/skyvern/webeye/dialog_handler.py
@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import weakref
+
+import structlog
+from playwright.async_api import BrowserContext, Dialog, Page
+
+from skyvern.constants import DIALOG_LLM_TIMEOUT
+from skyvern.forge import app
+from skyvern.forge.prompts import prompt_engine
+from skyvern.forge.sdk.core import skyvern_context
+
+LOG = structlog.get_logger()
+
+# Track contexts that already have a dialog handler to avoid duplicate registration
+# when the same BrowserContext is returned by CDP reconnect paths.
+_registered_contexts: weakref.WeakSet[BrowserContext] = weakref.WeakSet()
+
+
+async def _handle_dialog(dialog: Dialog) -> None:
+ """Handle a JavaScript dialog (alert/confirm/prompt/beforeunload) using LLM-based decision making.
+
+ For alert and beforeunload dialogs, always accepts without calling the LLM.
+ For confirm/prompt dialogs with no task context, auto-accepts (no LLM round-trip needed).
+ For confirm/prompt dialogs with task context, calls the secondary LLM handler to decide.
+ Falls back to accept on any error (safer than dismiss for form submissions).
+ """
+ dialog_type = dialog.type
+ dialog_message = dialog.message
+ default_value = dialog.default_value
+
+ ctx = skyvern_context.current()
+ organization_id = ctx.organization_id if ctx else None
+ navigation_goal = (ctx.navigation_goal or ctx.prompt) if ctx else None
+ navigation_payload = ctx.navigation_payload if ctx else None
+ task_id = ctx.task_id if ctx else None
+ workflow_run_id = ctx.workflow_run_id if ctx else None
+
+ log = LOG.bind(
+ dialog_type=dialog_type,
+ dialog_message=dialog_message,
+ task_id=task_id,
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ )
+
+ # Alert, beforeunload, or no task context — auto-accept without LLM call
+ if dialog_type in ("alert", "beforeunload"):
+ log.info("Dialog auto-accepted", dialog_type=dialog_type)
+ await dialog.accept()
+ return
+
+ if not navigation_goal and not navigation_payload:
+ log.info("Dialog auto-accepted (no task context)", dialog_type=dialog_type)
+ await dialog.accept(default_value or "")
+ return
+
+ # For confirm/prompt with task context, call LLM to decide
+ try:
+ prompt = prompt_engine.load_prompt(
+ "handle-dialog",
+ dialog_type=dialog_type,
+ dialog_message=dialog_message,
+ default_value=default_value,
+ navigation_goal=navigation_goal,
+ navigation_payload=json.dumps(navigation_payload) if navigation_payload else None,
+ )
+
+ # JS dialogs block the page's JS thread while open. We need a hard timeout
+ # to ensure the page doesn't stay frozen indefinitely if the LLM call is slow.
+ response = await asyncio.wait_for(
+ app.SECONDARY_LLM_API_HANDLER(
+ prompt=prompt,
+ prompt_name="handle-dialog",
+ organization_id=organization_id,
+ ),
+ timeout=DIALOG_LLM_TIMEOUT,
+ )
+
+ action = str(response.get("action", "accept")).lower()
+ prompt_text = response.get("prompt_text")
+
+ if action not in ("accept", "dismiss"):
+ log.warning("Dialog LLM returned unexpected action, defaulting to accept", llm_action=action)
+ action = "accept"
+
+ log.info(
+ "Dialog handled via LLM",
+ action=action,
+ has_prompt_text=prompt_text is not None,
+ )
+
+ if action == "dismiss":
+ await dialog.dismiss()
+ else:
+ await dialog.accept(prompt_text if prompt_text is not None else (default_value or ""))
+
+ except asyncio.TimeoutError:
+ log.warning("Dialog LLM call timed out, falling back to accept")
+ await dialog.accept(default_value or "")
+
+ except Exception:
+ log.exception("Dialog handler error, falling back to accept")
+ await dialog.accept(default_value or "")
+
+
+def set_dialog_handler(browser_context: BrowserContext) -> None:
+ """Register a dialog handler on all pages in the browser context.
+
+ Hooks into browser_context.on("page", ...) to register the handler
+ on every new page, including popups and new tabs. Also registers on
+ any pages that already exist in the context.
+
+ Uses a WeakSet to skip registration if the same BrowserContext is
+ returned again (e.g., CDP reconnect reusing contexts[0]).
+
+ Playwright-Python schedules async callbacks as tasks internally,
+ so registering _handle_dialog directly is GC-safe.
+ """
+ if browser_context in _registered_contexts:
+ return
+ _registered_contexts.add(browser_context)
+
+ def _on_page(page: Page) -> None:
+ page.on("dialog", _handle_dialog)
+
+ # Register on pages that already exist
+ for page in browser_context.pages:
+ _on_page(page)
+
+ # Register on future pages
+ browser_context.on("page", _on_page)