Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9471): surface browser dialog rejections to agent prompt (#5810)
Aaron Perezffd2cab0e269
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 753065c93..e257afa9f 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -3368,6 +3368,12 @@ class ForgeAgent:else:elements_for_prompt = scraped_page.build_element_tree(element_tree_format)+ # Format-then-clear so a render failure can't drop the signal permanently;+ # gate on extract-action template since other task types don't render it.+ recent_dialog_messages_str = (+ context.format_recent_dialog_messages() if template == EXTRACT_ACTION_TEMPLATE else None+ )+if template == EXTRACT_ACTION_TEMPLATE and cache_enabled:try:# Try to load split templates for caching@@ -3387,6 +3393,7 @@ class ForgeAgent:"terminate_criterion": task.terminate_criterion.strip() if task.terminate_criterion else None,"parse_select_feature_enabled": context.enable_parse_select_in_extract,"has_magic_link_page": context.has_magic_link_page(task.task_id),+ "recent_dialog_messages_str": recent_dialog_messages_str,}cache_variant = self._build_extract_action_cache_variant(verification_code_check=verification_code_check,@@ -3438,6 +3445,8 @@ class ForgeAgent:)# Map template to prompt_name for logging/caching guardsprompt_name = EXTRACT_ACTION_PROMPT_NAME if template == EXTRACT_ACTION_TEMPLATE else template+ if recent_dialog_messages_str is not None:+ context.clear_recent_dialog_messages()return combined_prompt, use_caching, prompt_nameexcept Exception as e:@@ -3462,6 +3471,7 @@ class ForgeAgent:terminate_criterion=task.terminate_criterion.strip() if task.terminate_criterion else None,parse_select_feature_enabled=context.enable_parse_select_in_extract,has_magic_link_page=context.has_magic_link_page(task.task_id),+ recent_dialog_messages_str=recent_dialog_messages_str,)# Map template to prompt_name for logging/caching guards@@ -3474,6 +3484,9 @@ class ForgeAgent:_prompt_build_span.set_attribute("prompt_tokens", count_tokens(full_prompt))_prompt_build_span.set_attribute("use_caching", bool(use_caching))+ if recent_dialog_messages_str is not None:+ context.clear_recent_dialog_messages()+return full_prompt, use_caching, prompt_nameasync def _get_prompt_caching_settings(self, context: SkyvernContext) -> dict[str, bool]:diff --git a/skyvern/forge/prompts/skyvern/extract-action-dynamic.j2 b/skyvern/forge/prompts/skyvern/extract-action-dynamic.j2index 59c2165ed..28cd8a825 100644--- a/skyvern/forge/prompts/skyvern/extract-action-dynamic.j2+++ b/skyvern/forge/prompts/skyvern/extract-action-dynamic.j2@@ -1,6 +1,12 @@```{{ action_history }}```+{% if recent_dialog_messages_str %}+Browser alert messages raised during the previous step (the agent auto-accepted each one). Use these as additional context: if a message indicates a validation error or that the previous action was rejected, change the value, its format, or the target element before retrying — do NOT retry the same INPUT_TEXT with the same text on the same field. Treat purely informational messages (e.g. "Saved successfully") as confirmation, not as a signal to change anything.+```+{{ recent_dialog_messages_str }}+```+{% endif %}{% if complete_criterion %}Complete criterion:```diff --git a/skyvern/forge/prompts/skyvern/extract-action.j2 b/skyvern/forge/prompts/skyvern/extract-action.j2index 0f6ca6a7e..95b769477 100644--- a/skyvern/forge/prompts/skyvern/extract-action.j2+++ b/skyvern/forge/prompts/skyvern/extract-action.j2@@ -57,6 +57,12 @@ Action history from previous steps: (note: even if the action history suggests g```{{ action_history }}```+{% if recent_dialog_messages_str %}+Browser alert messages raised during the previous step (the agent auto-accepted each one). Use these as additional context: if a message indicates a validation error or that the previous action was rejected, change the value, its format, or the target element before retrying — do NOT retry the same INPUT_TEXT with the same text on the same field. Treat purely informational messages (e.g. "Saved successfully") as confirmation, not as a signal to change anything.+```+{{ recent_dialog_messages_str }}+```+{% endif %}{% if complete_criterion %}Complete criterion:```diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex cd1c66993..6f7eaf960 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -1,7 +1,7 @@from contextlib import contextmanagerfrom contextvars import ContextVar, Tokenfrom dataclasses import dataclass, field-from typing import Any, Iterator+from typing import Any, Iterator, TypedDictfrom zoneinfo import ZoneInfoimport structlog@@ -11,6 +11,19 @@ from skyvern.config import settingsLOG = structlog.get_logger()+# Cap on entries kept in `recent_dialog_messages` so a chatty page (e.g. validation+# alerts firing on every keystroke) cannot inflate the next prompt unboundedly.+MAX_RECENT_DIALOG_MESSAGES = 5+# Per-message length cap so a single pathological alert (multi-KB page-stack+# trace, etc.) cannot dominate the prompt budget.+MAX_DIALOG_MESSAGE_CHARS = 500+++class DialogEntry(TypedDict):+ type: str+ message: str+ count: int+@dataclassclass SkyvernContext:@@ -104,6 +117,10 @@ class SkyvernContext:# preventing repeated injection loops when the captcha solver succeeds but the page doesn't changeproactive_captcha_task_ids: set[str] = field(default_factory=set)+ # Browser dialogs captured since the last agent prompt build, surfaced into the+ # next extract-action prompt so the LLM can react to validation rejections.+ recent_dialog_messages: list[DialogEntry] = field(default_factory=list)+def __repr__(self) -> str:return f"SkyvernContext(request_id={self.request_id}, organization_id={self.organization_id}, task_id={self.task_id}, step_id={self.step_id}, workflow_id={self.workflow_id}, workflow_run_id={self.workflow_run_id}, task_v2_id={self.task_v2_id}, max_steps_override={self.max_steps_override}, run_id={self.run_id}, copilot_session_id={self.copilot_session_id})"@@ -114,6 +131,34 @@ class SkyvernContext:if task_id in self.totp_codes:self.totp_codes.pop(task_id)+ def record_dialog_message(self, dialog_type: str, dialog_message: str) -> None:+ """Buffer a dialog with FIFO cap; identical entries bump a count instead of duplicating."""+ if not dialog_message:+ return+ if len(dialog_message) > MAX_DIALOG_MESSAGE_CHARS:+ dialog_message = dialog_message[:MAX_DIALOG_MESSAGE_CHARS] + "…"+ for entry in self.recent_dialog_messages:+ if entry["type"] == dialog_type and entry["message"] == dialog_message:+ entry["count"] += 1+ return+ self.recent_dialog_messages.append({"type": dialog_type, "message": dialog_message, "count": 1})+ if len(self.recent_dialog_messages) > MAX_RECENT_DIALOG_MESSAGES:+ del self.recent_dialog_messages[0]++ def format_recent_dialog_messages(self) -> str | None:+ """Render the buffered dialogs into prompt-ready text without clearing; None when empty."""+ if not self.recent_dialog_messages:+ return None+ lines: list[str] = []+ for entry in self.recent_dialog_messages:+ suffix = f" (x{entry['count']})" if entry["count"] > 1 else ""+ lines.append(f"[{entry['type']}{suffix}] {entry['message']}")+ return "\n".join(lines)++ def clear_recent_dialog_messages(self) -> None:+ """Drop the buffered dialogs once the prompt has consumed them."""+ self.recent_dialog_messages.clear()+def add_magic_link_page(self, task_id: str, page: Page) -> None:self.magic_link_pages[task_id] = pagediff --git a/skyvern/webeye/dialog_handler.py b/skyvern/webeye/dialog_handler.pyindex 55b888f4a..8b63cb0cf 100644--- a/skyvern/webeye/dialog_handler.py+++ b/skyvern/webeye/dialog_handler.py@@ -46,6 +46,15 @@ async def _handle_dialog(dialog: Dialog) -> None:organization_id=organization_id,)+ # Record alert only — beforeunload is informational ("Changes you made may not+ # be saved") and would misfit the "field rejection" prompt copy; confirm/prompt+ # are handled deliberately by the LLM and would be mislabeled as rejections.+ if ctx is not None and dialog_type == "alert":+ try:+ ctx.record_dialog_message(dialog_type, dialog_message)+ except Exception:+ log.exception("Failed to record dialog message into context")+# Alert, beforeunload, or no task context — auto-accept without LLM callif dialog_type in ("alert", "beforeunload"):log.info("Dialog auto-accepted", dialog_type=dialog_type)diff --git a/tests/unit/test_dialog_handler_records_to_context.py b/tests/unit/test_dialog_handler_records_to_context.pynew file mode 100644index 000000000..699cf483c--- /dev/null+++ b/tests/unit/test_dialog_handler_records_to_context.py@@ -0,0 +1,93 @@+"""Tests for dialog-handler recording into SkyvernContext."""++from collections.abc import Generator+from unittest.mock import AsyncMock, MagicMock++import pytest++from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.webeye import dialog_handler+++def _make_dialog(dialog_type: str, message: str, default_value: str = "") -> MagicMock:+ dialog = MagicMock()+ dialog.type = dialog_type+ dialog.message = message+ dialog.default_value = default_value+ dialog.accept = AsyncMock()+ dialog.dismiss = AsyncMock()+ return dialog+++def isolated_context() -> Generator[SkyvernContext, None, None]:+ ctx = SkyvernContext(+ organization_id="o_test",+ task_id="tsk_test",+ workflow_run_id="wr_test",+ )+ skyvern_context.set(ctx)+ try:+ yield ctx+ finally:+ skyvern_context.reset()+++async def test_alert_records_into_context_and_auto_accepts(isolated_context: SkyvernContext) -> None:+ dialog = _make_dialog("alert", "The value of '47' is invalid.")++ await dialog_handler._handle_dialog(dialog)++ dialog.accept.assert_awaited_once()+ assert isolated_context.recent_dialog_messages == [+ {"type": "alert", "message": "The value of '47' is invalid.", "count": 1}+ ]+++async def test_repeated_alerts_dedupe_with_count(isolated_context: SkyvernContext) -> None:+ dialog = _make_dialog("alert", "phone invalid")++ for _ in range(5):+ await dialog_handler._handle_dialog(dialog)++ assert len(isolated_context.recent_dialog_messages) == 1+ assert isolated_context.recent_dialog_messages[0]["count"] == 5+++async def test_no_context_does_not_raise() -> None:+ skyvern_context.reset()+ dialog = _make_dialog("alert", "something")+ await dialog_handler._handle_dialog(dialog)+ dialog.accept.assert_awaited_once()+++async def test_record_failure_does_not_break_dialog_acceptance(+ isolated_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ def boom(*_a: object, **_kw: object) -> None:+ raise RuntimeError("simulated record failure")++ monkeypatch.setattr(SkyvernContext, "record_dialog_message", boom)+ dialog = _make_dialog("alert", "anything")++ await dialog_handler._handle_dialog(dialog)++ dialog.accept.assert_awaited_once()+++async def test_non_alert_dialogs_are_not_recorded(+ isolated_context: SkyvernContext,+) -> None:+ isolated_context.navigation_goal = "test"++ await dialog_handler._handle_dialog(_make_dialog("confirm", "Are you sure?"))+ await dialog_handler._handle_dialog(_make_dialog("prompt", "Enter your name:"))+ await dialog_handler._handle_dialog(_make_dialog("beforeunload", "Changes may not be saved."))++ assert isolated_context.recent_dialog_messages == []diff --git a/tests/unit/test_extract_action_dialog_messages_render.py b/tests/unit/test_extract_action_dialog_messages_render.pynew file mode 100644index 000000000..5999a20af--- /dev/null+++ b/tests/unit/test_extract_action_dialog_messages_render.py@@ -0,0 +1,62 @@+"""Render-level tests for the dialog-messages section of extract-action prompts."""++from typing import Any++import pytest++from skyvern.forge.prompts import prompt_engine++_BASE_KWARGS: dict[str, Any] = {+ "navigation_goal": "fill out the order form",+ "navigation_payload_str": "{}",+ "starting_url": "https://example.test/start",+ "current_url": "https://example.test/form",+ "data_extraction_goal": None,+ "action_history": "[]",+ "error_code_mapping_str": None,+ "local_datetime": "2026-05-04T17:14:00Z",+ "verification_code_check": False,+ "complete_criterion": None,+ "terminate_criterion": None,+ "parse_select_feature_enabled": False,+ "has_magic_link_page": False,+ "elements": "<html></html>",+}++[email protected]("template", ["extract-action", "extract-action-dynamic"])+def test_renders_dialog_section_when_messages_present(template: str) -> None:+ rendered = prompt_engine.load_prompt(+ template,+ recent_dialog_messages_str=(+ "[alert (x1031)] The value of '47' is invalid. "+ "Only 9999999999, 999-999-9999 or (999) 999-9999 are allowed."+ ),+ **_BASE_KWARGS,+ )+ assert "Browser alert messages raised during the previous step" in rendered+ assert "is invalid" in rendered+ assert "999-999-9999" in rendered++[email protected]("template", ["extract-action", "extract-action-dynamic"])+def test_omits_dialog_section_when_no_messages(template: str) -> None:+ rendered = prompt_engine.load_prompt(+ template,+ recent_dialog_messages_str=None,+ **_BASE_KWARGS,+ )+ assert "Browser alert messages raised during the previous step" not in rendered+++def test_static_template_omits_dialog_block() -> None:+ """The dialog section must live in the dynamic template only; if it leaks into+ the static prefix the cached static contents would invalidate the Vertex cache+ every time a new alert fires."""+ rendered = prompt_engine.load_prompt(+ "extract-action-static",+ recent_dialog_messages_str="[alert] should not appear in static",+ **_BASE_KWARGS,+ )+ assert "Browser alert messages" not in rendered+ assert "should not appear in static" not in rendereddiff --git a/tests/unit/test_skyvern_context_dialog_messages.py b/tests/unit/test_skyvern_context_dialog_messages.pynew file mode 100644index 000000000..f46a13762--- /dev/null+++ b/tests/unit/test_skyvern_context_dialog_messages.py@@ -0,0 +1,98 @@+"""Tests for the dialog-message buffer on SkyvernContext."""++from skyvern.forge.sdk.core.skyvern_context import (+ MAX_DIALOG_MESSAGE_CHARS,+ MAX_RECENT_DIALOG_MESSAGES,+ SkyvernContext,+)+++class TestRecordDialogMessage:+ def test_appends_first_message(self) -> None:+ ctx = SkyvernContext()+ ctx.record_dialog_message("alert", "phone invalid")+ assert ctx.recent_dialog_messages == [{"type": "alert", "message": "phone invalid", "count": 1}]++ def test_dedupes_repeated_message_by_bumping_count(self) -> None:+ ctx = SkyvernContext()+ for _ in range(2062):+ ctx.record_dialog_message("alert", "phone invalid")+ assert len(ctx.recent_dialog_messages) == 1+ assert ctx.recent_dialog_messages[0]["count"] == 2062++ def test_distinct_messages_kept_separately_with_independent_counts(self) -> None:+ ctx = SkyvernContext()+ for _ in range(3):+ ctx.record_dialog_message("alert", "phone invalid")+ ctx.record_dialog_message("alert", "name invalid")+ assert len(ctx.recent_dialog_messages) == 2+ assert {(e["message"], e["count"]) for e in ctx.recent_dialog_messages} == {+ ("phone invalid", 3),+ ("name invalid", 3),+ }++ def test_distinguishes_by_dialog_type(self) -> None:+ ctx = SkyvernContext()+ ctx.record_dialog_message("alert", "are you sure?")+ ctx.record_dialog_message("confirm", "are you sure?")+ assert len(ctx.recent_dialog_messages) == 2++ def test_caps_at_max_with_fifo_eviction(self) -> None:+ ctx = SkyvernContext()+ for i in range(MAX_RECENT_DIALOG_MESSAGES + 3):+ ctx.record_dialog_message("alert", f"msg-{i}")+ assert len(ctx.recent_dialog_messages) == MAX_RECENT_DIALOG_MESSAGES+ assert ctx.recent_dialog_messages[0]["message"] == "msg-3"+ assert ctx.recent_dialog_messages[-1]["message"] == f"msg-{MAX_RECENT_DIALOG_MESSAGES + 2}"++ def test_empty_message_is_ignored(self) -> None:+ ctx = SkyvernContext()+ ctx.record_dialog_message("alert", "")+ assert ctx.recent_dialog_messages == []++ def test_long_messages_are_truncated_with_ellipsis(self) -> None:+ ctx = SkyvernContext()+ long_msg = "x" * (MAX_DIALOG_MESSAGE_CHARS + 200)+ ctx.record_dialog_message("alert", long_msg)+ stored = ctx.recent_dialog_messages[0]["message"]+ assert len(stored) == MAX_DIALOG_MESSAGE_CHARS + 1+ assert stored.endswith("…")+++class TestFormatAndClearRecentDialogMessages:+ def test_format_returns_text_without_clearing(self) -> None:+ ctx = SkyvernContext()+ ctx.record_dialog_message("alert", "phone invalid")+ rendered = ctx.format_recent_dialog_messages()+ assert rendered == "[alert] phone invalid"+ assert len(ctx.recent_dialog_messages) == 1++ def test_format_can_be_called_repeatedly(self) -> None:+ ctx = SkyvernContext()+ ctx.record_dialog_message("alert", "msg")+ a = ctx.format_recent_dialog_messages()+ b = ctx.format_recent_dialog_messages()+ assert a == b == "[alert] msg"++ def test_format_returns_none_when_empty(self) -> None:+ assert SkyvernContext().format_recent_dialog_messages() is None++ def test_format_renders_count_suffix_when_deduped(self) -> None:+ ctx = SkyvernContext()+ for _ in range(2062):+ ctx.record_dialog_message("alert", "phone invalid")+ assert ctx.format_recent_dialog_messages() == "[alert (x2062)] phone invalid"++ def test_format_renders_multiple_messages_in_insertion_order(self) -> None:+ ctx = SkyvernContext()+ ctx.record_dialog_message("alert", "first")+ ctx.record_dialog_message("alert", "second")+ ctx.record_dialog_message("alert", "first")+ assert ctx.format_recent_dialog_messages() == "[alert (x2)] first\n[alert] second"++ def test_clear_empties_buffer(self) -> None:+ ctx = SkyvernContext()+ ctx.record_dialog_message("alert", "a")+ ctx.record_dialog_message("alert", "b")+ ctx.clear_recent_dialog_messages()+ assert ctx.recent_dialog_messages == []