Skip to content

Automate browser based workflows with AI

fix(SKY-9471): surface browser dialog rejections to agent prompt (#5810)

Aaron Perezffd2cab0e269
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 753065c93..e257afa9f 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -3368,6 +3368,12 @@ class ForgeAgent:
else:
elements_for_prompt = scraped_page.build_element_tree(element_tree_format)
+ # Format-then-clear so a render failure can't drop the signal permanently;
+ # gate on extract-action template since other task types don't render it.
+ recent_dialog_messages_str = (
+ context.format_recent_dialog_messages() if template == EXTRACT_ACTION_TEMPLATE else None
+ )
+
if template == EXTRACT_ACTION_TEMPLATE and cache_enabled:
try:
# Try to load split templates for caching
@@ -3387,6 +3393,7 @@ class ForgeAgent:
"terminate_criterion": task.terminate_criterion.strip() if task.terminate_criterion else None,
"parse_select_feature_enabled": context.enable_parse_select_in_extract,
"has_magic_link_page": context.has_magic_link_page(task.task_id),
+ "recent_dialog_messages_str": recent_dialog_messages_str,
}
cache_variant = self._build_extract_action_cache_variant(
verification_code_check=verification_code_check,
@@ -3438,6 +3445,8 @@ class ForgeAgent:
)
# Map template to prompt_name for logging/caching guards
prompt_name = EXTRACT_ACTION_PROMPT_NAME if template == EXTRACT_ACTION_TEMPLATE else template
+ if recent_dialog_messages_str is not None:
+ context.clear_recent_dialog_messages()
return combined_prompt, use_caching, prompt_name
except Exception as e:
@@ -3462,6 +3471,7 @@ class ForgeAgent:
terminate_criterion=task.terminate_criterion.strip() if task.terminate_criterion else None,
parse_select_feature_enabled=context.enable_parse_select_in_extract,
has_magic_link_page=context.has_magic_link_page(task.task_id),
+ recent_dialog_messages_str=recent_dialog_messages_str,
)
# Map template to prompt_name for logging/caching guards
@@ -3474,6 +3484,9 @@ class ForgeAgent:
_prompt_build_span.set_attribute("prompt_tokens", count_tokens(full_prompt))
_prompt_build_span.set_attribute("use_caching", bool(use_caching))
+ if recent_dialog_messages_str is not None:
+ context.clear_recent_dialog_messages()
+
return full_prompt, use_caching, prompt_name
async def _get_prompt_caching_settings(self, context: SkyvernContext) -> dict[str, bool]:
diff --git a/skyvern/forge/prompts/skyvern/extract-action-dynamic.j2 b/skyvern/forge/prompts/skyvern/extract-action-dynamic.j2
index 59c2165ed..28cd8a825 100644
--- a/skyvern/forge/prompts/skyvern/extract-action-dynamic.j2
+++ b/skyvern/forge/prompts/skyvern/extract-action-dynamic.j2
@@ -1,6 +1,12 @@
```
{{ action_history }}
```
+{% if recent_dialog_messages_str %}
+Browser alert messages raised during the previous step (the agent auto-accepted each one). Use these as additional context: if a message indicates a validation error or that the previous action was rejected, change the value, its format, or the target element before retrying — do NOT retry the same INPUT_TEXT with the same text on the same field. Treat purely informational messages (e.g. "Saved successfully") as confirmation, not as a signal to change anything.
+```
+{{ recent_dialog_messages_str }}
+```
+{% endif %}
{% if complete_criterion %}
Complete criterion:
```
diff --git a/skyvern/forge/prompts/skyvern/extract-action.j2 b/skyvern/forge/prompts/skyvern/extract-action.j2
index 0f6ca6a7e..95b769477 100644
--- a/skyvern/forge/prompts/skyvern/extract-action.j2
+++ b/skyvern/forge/prompts/skyvern/extract-action.j2
@@ -57,6 +57,12 @@ Action history from previous steps: (note: even if the action history suggests g
```
{{ action_history }}
```
+{% if recent_dialog_messages_str %}
+Browser alert messages raised during the previous step (the agent auto-accepted each one). Use these as additional context: if a message indicates a validation error or that the previous action was rejected, change the value, its format, or the target element before retrying — do NOT retry the same INPUT_TEXT with the same text on the same field. Treat purely informational messages (e.g. "Saved successfully") as confirmation, not as a signal to change anything.
+```
+{{ recent_dialog_messages_str }}
+```
+{% endif %}
{% if complete_criterion %}
Complete criterion:
```
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.py
index cd1c66993..6f7eaf960 100644
--- a/skyvern/forge/sdk/core/skyvern_context.py
+++ b/skyvern/forge/sdk/core/skyvern_context.py
@@ -1,7 +1,7 @@
from contextlib import contextmanager
from contextvars import ContextVar, Token
from dataclasses import dataclass, field
-from typing import Any, Iterator
+from typing import Any, Iterator, TypedDict
from zoneinfo import ZoneInfo
import structlog
@@ -11,6 +11,19 @@ from skyvern.config import settings
LOG = structlog.get_logger()
+# Cap on entries kept in `recent_dialog_messages` so a chatty page (e.g. validation
+# alerts firing on every keystroke) cannot inflate the next prompt unboundedly.
+MAX_RECENT_DIALOG_MESSAGES = 5
+# Per-message length cap so a single pathological alert (multi-KB page-stack
+# trace, etc.) cannot dominate the prompt budget.
+MAX_DIALOG_MESSAGE_CHARS = 500
+
+
+class DialogEntry(TypedDict):
+ type: str
+ message: str
+ count: int
+
@dataclass
class SkyvernContext:
@@ -104,6 +117,10 @@ class SkyvernContext:
# preventing repeated injection loops when the captcha solver succeeds but the page doesn't change
proactive_captcha_task_ids: set[str] = field(default_factory=set)
+ # Browser dialogs captured since the last agent prompt build, surfaced into the
+ # next extract-action prompt so the LLM can react to validation rejections.
+ recent_dialog_messages: list[DialogEntry] = field(default_factory=list)
+
def __repr__(self) -> str:
return f"SkyvernContext(request_id={self.request_id}, organization_id={self.organization_id}, task_id={self.task_id}, step_id={self.step_id}, workflow_id={self.workflow_id}, workflow_run_id={self.workflow_run_id}, task_v2_id={self.task_v2_id}, max_steps_override={self.max_steps_override}, run_id={self.run_id}, copilot_session_id={self.copilot_session_id})"
@@ -114,6 +131,34 @@ class SkyvernContext:
if task_id in self.totp_codes:
self.totp_codes.pop(task_id)
+ def record_dialog_message(self, dialog_type: str, dialog_message: str) -> None:
+ """Buffer a dialog with FIFO cap; identical entries bump a count instead of duplicating."""
+ if not dialog_message:
+ return
+ if len(dialog_message) > MAX_DIALOG_MESSAGE_CHARS:
+ dialog_message = dialog_message[:MAX_DIALOG_MESSAGE_CHARS] + "…"
+ for entry in self.recent_dialog_messages:
+ if entry["type"] == dialog_type and entry["message"] == dialog_message:
+ entry["count"] += 1
+ return
+ self.recent_dialog_messages.append({"type": dialog_type, "message": dialog_message, "count": 1})
+ if len(self.recent_dialog_messages) > MAX_RECENT_DIALOG_MESSAGES:
+ del self.recent_dialog_messages[0]
+
+ def format_recent_dialog_messages(self) -> str | None:
+ """Render the buffered dialogs into prompt-ready text without clearing; None when empty."""
+ if not self.recent_dialog_messages:
+ return None
+ lines: list[str] = []
+ for entry in self.recent_dialog_messages:
+ suffix = f" (x{entry['count']})" if entry["count"] > 1 else ""
+ lines.append(f"[{entry['type']}{suffix}] {entry['message']}")
+ return "\n".join(lines)
+
+ def clear_recent_dialog_messages(self) -> None:
+ """Drop the buffered dialogs once the prompt has consumed them."""
+ self.recent_dialog_messages.clear()
+
def add_magic_link_page(self, task_id: str, page: Page) -> None:
self.magic_link_pages[task_id] = page
diff --git a/skyvern/webeye/dialog_handler.py b/skyvern/webeye/dialog_handler.py
index 55b888f4a..8b63cb0cf 100644
--- a/skyvern/webeye/dialog_handler.py
+++ b/skyvern/webeye/dialog_handler.py
@@ -46,6 +46,15 @@ async def _handle_dialog(dialog: Dialog) -> None:
organization_id=organization_id,
)
+ # Record alert only — beforeunload is informational ("Changes you made may not
+ # be saved") and would misfit the "field rejection" prompt copy; confirm/prompt
+ # are handled deliberately by the LLM and would be mislabeled as rejections.
+ if ctx is not None and dialog_type == "alert":
+ try:
+ ctx.record_dialog_message(dialog_type, dialog_message)
+ except Exception:
+ log.exception("Failed to record dialog message into context")
+
# Alert, beforeunload, or no task context — auto-accept without LLM call
if dialog_type in ("alert", "beforeunload"):
log.info("Dialog auto-accepted", dialog_type=dialog_type)
diff --git a/tests/unit/test_dialog_handler_records_to_context.py b/tests/unit/test_dialog_handler_records_to_context.py
new file mode 100644
index 000000000..699cf483c
--- /dev/null
+++ b/tests/unit/test_dialog_handler_records_to_context.py
@@ -0,0 +1,93 @@
+"""Tests for dialog-handler recording into SkyvernContext."""
+
+from collections.abc import Generator
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.webeye import dialog_handler
+
+
+def _make_dialog(dialog_type: str, message: str, default_value: str = "") -> MagicMock:
+ dialog = MagicMock()
+ dialog.type = dialog_type
+ dialog.message = message
+ dialog.default_value = default_value
+ dialog.accept = AsyncMock()
+ dialog.dismiss = AsyncMock()
+ return dialog
+
+
+def isolated_context() -> Generator[SkyvernContext, None, None]:
+ ctx = SkyvernContext(
+ organization_id="o_test",
+ task_id="tsk_test",
+ workflow_run_id="wr_test",
+ )
+ skyvern_context.set(ctx)
+ try:
+ yield ctx
+ finally:
+ skyvern_context.reset()
+
+
+async def test_alert_records_into_context_and_auto_accepts(isolated_context: SkyvernContext) -> None:
+ dialog = _make_dialog("alert", "The value of '47' is invalid.")
+
+ await dialog_handler._handle_dialog(dialog)
+
+ dialog.accept.assert_awaited_once()
+ assert isolated_context.recent_dialog_messages == [
+ {"type": "alert", "message": "The value of '47' is invalid.", "count": 1}
+ ]
+
+
+async def test_repeated_alerts_dedupe_with_count(isolated_context: SkyvernContext) -> None:
+ dialog = _make_dialog("alert", "phone invalid")
+
+ for _ in range(5):
+ await dialog_handler._handle_dialog(dialog)
+
+ assert len(isolated_context.recent_dialog_messages) == 1
+ assert isolated_context.recent_dialog_messages[0]["count"] == 5
+
+
+async def test_no_context_does_not_raise() -> None:
+ skyvern_context.reset()
+ dialog = _make_dialog("alert", "something")
+ await dialog_handler._handle_dialog(dialog)
+ dialog.accept.assert_awaited_once()
+
+
+async def test_record_failure_does_not_break_dialog_acceptance(
+ isolated_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ def boom(*_a: object, **_kw: object) -> None:
+ raise RuntimeError("simulated record failure")
+
+ monkeypatch.setattr(SkyvernContext, "record_dialog_message", boom)
+ dialog = _make_dialog("alert", "anything")
+
+ await dialog_handler._handle_dialog(dialog)
+
+ dialog.accept.assert_awaited_once()
+
+
+async def test_non_alert_dialogs_are_not_recorded(
+ isolated_context: SkyvernContext,
+) -> None:
+ isolated_context.navigation_goal = "test"
+
+ await dialog_handler._handle_dialog(_make_dialog("confirm", "Are you sure?"))
+ await dialog_handler._handle_dialog(_make_dialog("prompt", "Enter your name:"))
+ await dialog_handler._handle_dialog(_make_dialog("beforeunload", "Changes may not be saved."))
+
+ assert isolated_context.recent_dialog_messages == []
diff --git a/tests/unit/test_extract_action_dialog_messages_render.py b/tests/unit/test_extract_action_dialog_messages_render.py
new file mode 100644
index 000000000..5999a20af
--- /dev/null
+++ b/tests/unit/test_extract_action_dialog_messages_render.py
@@ -0,0 +1,62 @@
+"""Render-level tests for the dialog-messages section of extract-action prompts."""
+
+from typing import Any
+
+import pytest
+
+from skyvern.forge.prompts import prompt_engine
+
+_BASE_KWARGS: dict[str, Any] = {
+ "navigation_goal": "fill out the order form",
+ "navigation_payload_str": "{}",
+ "starting_url": "https://example.test/start",
+ "current_url": "https://example.test/form",
+ "data_extraction_goal": None,
+ "action_history": "[]",
+ "error_code_mapping_str": None,
+ "local_datetime": "2026-05-04T17:14:00Z",
+ "verification_code_check": False,
+ "complete_criterion": None,
+ "terminate_criterion": None,
+ "parse_select_feature_enabled": False,
+ "has_magic_link_page": False,
+ "elements": "<html></html>",
+}
+
+
[email protected]("template", ["extract-action", "extract-action-dynamic"])
+def test_renders_dialog_section_when_messages_present(template: str) -> None:
+ rendered = prompt_engine.load_prompt(
+ template,
+ recent_dialog_messages_str=(
+ "[alert (x1031)] The value of '47' is invalid. "
+ "Only 9999999999, 999-999-9999 or (999) 999-9999 are allowed."
+ ),
+ **_BASE_KWARGS,
+ )
+ assert "Browser alert messages raised during the previous step" in rendered
+ assert "is invalid" in rendered
+ assert "999-999-9999" in rendered
+
+
[email protected]("template", ["extract-action", "extract-action-dynamic"])
+def test_omits_dialog_section_when_no_messages(template: str) -> None:
+ rendered = prompt_engine.load_prompt(
+ template,
+ recent_dialog_messages_str=None,
+ **_BASE_KWARGS,
+ )
+ assert "Browser alert messages raised during the previous step" not in rendered
+
+
+def test_static_template_omits_dialog_block() -> None:
+ """The dialog section must live in the dynamic template only; if it leaks into
+ the static prefix the cached static contents would invalidate the Vertex cache
+ every time a new alert fires."""
+ rendered = prompt_engine.load_prompt(
+ "extract-action-static",
+ recent_dialog_messages_str="[alert] should not appear in static",
+ **_BASE_KWARGS,
+ )
+ assert "Browser alert messages" not in rendered
+ assert "should not appear in static" not in rendered
diff --git a/tests/unit/test_skyvern_context_dialog_messages.py b/tests/unit/test_skyvern_context_dialog_messages.py
new file mode 100644
index 000000000..f46a13762
--- /dev/null
+++ b/tests/unit/test_skyvern_context_dialog_messages.py
@@ -0,0 +1,98 @@
+"""Tests for the dialog-message buffer on SkyvernContext."""
+
+from skyvern.forge.sdk.core.skyvern_context import (
+ MAX_DIALOG_MESSAGE_CHARS,
+ MAX_RECENT_DIALOG_MESSAGES,
+ SkyvernContext,
+)
+
+
+class TestRecordDialogMessage:
+ def test_appends_first_message(self) -> None:
+ ctx = SkyvernContext()
+ ctx.record_dialog_message("alert", "phone invalid")
+ assert ctx.recent_dialog_messages == [{"type": "alert", "message": "phone invalid", "count": 1}]
+
+ def test_dedupes_repeated_message_by_bumping_count(self) -> None:
+ ctx = SkyvernContext()
+ for _ in range(2062):
+ ctx.record_dialog_message("alert", "phone invalid")
+ assert len(ctx.recent_dialog_messages) == 1
+ assert ctx.recent_dialog_messages[0]["count"] == 2062
+
+ def test_distinct_messages_kept_separately_with_independent_counts(self) -> None:
+ ctx = SkyvernContext()
+ for _ in range(3):
+ ctx.record_dialog_message("alert", "phone invalid")
+ ctx.record_dialog_message("alert", "name invalid")
+ assert len(ctx.recent_dialog_messages) == 2
+ assert {(e["message"], e["count"]) for e in ctx.recent_dialog_messages} == {
+ ("phone invalid", 3),
+ ("name invalid", 3),
+ }
+
+ def test_distinguishes_by_dialog_type(self) -> None:
+ ctx = SkyvernContext()
+ ctx.record_dialog_message("alert", "are you sure?")
+ ctx.record_dialog_message("confirm", "are you sure?")
+ assert len(ctx.recent_dialog_messages) == 2
+
+ def test_caps_at_max_with_fifo_eviction(self) -> None:
+ ctx = SkyvernContext()
+ for i in range(MAX_RECENT_DIALOG_MESSAGES + 3):
+ ctx.record_dialog_message("alert", f"msg-{i}")
+ assert len(ctx.recent_dialog_messages) == MAX_RECENT_DIALOG_MESSAGES
+ assert ctx.recent_dialog_messages[0]["message"] == "msg-3"
+ assert ctx.recent_dialog_messages[-1]["message"] == f"msg-{MAX_RECENT_DIALOG_MESSAGES + 2}"
+
+ def test_empty_message_is_ignored(self) -> None:
+ ctx = SkyvernContext()
+ ctx.record_dialog_message("alert", "")
+ assert ctx.recent_dialog_messages == []
+
+ def test_long_messages_are_truncated_with_ellipsis(self) -> None:
+ ctx = SkyvernContext()
+ long_msg = "x" * (MAX_DIALOG_MESSAGE_CHARS + 200)
+ ctx.record_dialog_message("alert", long_msg)
+ stored = ctx.recent_dialog_messages[0]["message"]
+ assert len(stored) == MAX_DIALOG_MESSAGE_CHARS + 1
+ assert stored.endswith("…")
+
+
+class TestFormatAndClearRecentDialogMessages:
+ def test_format_returns_text_without_clearing(self) -> None:
+ ctx = SkyvernContext()
+ ctx.record_dialog_message("alert", "phone invalid")
+ rendered = ctx.format_recent_dialog_messages()
+ assert rendered == "[alert] phone invalid"
+ assert len(ctx.recent_dialog_messages) == 1
+
+ def test_format_can_be_called_repeatedly(self) -> None:
+ ctx = SkyvernContext()
+ ctx.record_dialog_message("alert", "msg")
+ a = ctx.format_recent_dialog_messages()
+ b = ctx.format_recent_dialog_messages()
+ assert a == b == "[alert] msg"
+
+ def test_format_returns_none_when_empty(self) -> None:
+ assert SkyvernContext().format_recent_dialog_messages() is None
+
+ def test_format_renders_count_suffix_when_deduped(self) -> None:
+ ctx = SkyvernContext()
+ for _ in range(2062):
+ ctx.record_dialog_message("alert", "phone invalid")
+ assert ctx.format_recent_dialog_messages() == "[alert (x2062)] phone invalid"
+
+ def test_format_renders_multiple_messages_in_insertion_order(self) -> None:
+ ctx = SkyvernContext()
+ ctx.record_dialog_message("alert", "first")
+ ctx.record_dialog_message("alert", "second")
+ ctx.record_dialog_message("alert", "first")
+ assert ctx.format_recent_dialog_messages() == "[alert (x2)] first\n[alert] second"
+
+ def test_clear_empties_buffer(self) -> None:
+ ctx = SkyvernContext()
+ ctx.record_dialog_message("alert", "a")
+ ctx.record_dialog_message("alert", "b")
+ ctx.clear_recent_dialog_messages()
+ assert ctx.recent_dialog_messages == []