Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11343): stop action batch after in-INPUT_TEXT combobox selection (#6787)
pedrohsdbeebea3c733c2
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex c1742e5f9..b824cd409 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -151,6 +151,7 @@ from skyvern.webeye.actions.actions import (WebAction,)from skyvern.webeye.actions.handler import ActionHandler+from skyvern.webeye.actions.handler_utils import should_stop_batch_after_dropdown_selectfrom skyvern.webeye.actions.models import DetailedAgentStepOutputfrom skyvern.webeye.actions.parse_actions import (parse_actions,@@ -1844,12 +1845,15 @@ class ForgeAgent:# Tell the handler to skip the auto-completion Tab hack when the# next batched action would be broken by a focus change — e.g. a# KEYPRESS Enter or another action on the same element.- if action.action_type == ActionType.INPUT_TEXT and action_idx + 1 < len(action_linked_list):- next_action = action_linked_list[action_idx + 1].action+ if action.action_type == ActionType.INPUT_TEXT:+ next_action = (+ action_linked_list[action_idx + 1].action if action_idx + 1 < len(action_linked_list) else None+ )if isinstance(next_action, KeypressAction) or (isinstance(next_action, WebAction) and next_action.element_id == action.element_id):action.skip_auto_complete_tab = True+ action.stop_batch_after_dropdown_select = should_stop_batch_after_dropdown_select(next_action)results = await ActionHandler.handle_action(scraped_page=scraped_page,diff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.pyindex 02e706f35..f70a90d84 100644--- a/skyvern/webeye/actions/actions.py+++ b/skyvern/webeye/actions/actions.py@@ -162,6 +162,10 @@ class Action(BaseModel):# Pressing Tab would move focus away and break that next action.skip_auto_complete_tab: bool = False+ # Transient (never serialized): set per-step by the agent loop when this INPUT_TEXT's next+ # batched action would clobber an in-action combobox selection.+ stop_batch_after_dropdown_select: bool = Field(default=False, exclude=True)+created_at: datetime | None = Nonemodified_at: datetime | None = Nonecreated_by: str | None = Nonediff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 0eb613c24..a1d7f665a 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -2259,6 +2259,10 @@ async def handle_input_text_action()if select_result and select_result.action_result and select_result.action_result.success:auto_complete_hacky_flag = False+ # A matching option was committed during this INPUT_TEXT. Stop the batch only when+ # the next queued action would clobber it (a trailing Enter/Return); next step re-scrapes.+ if action.stop_batch_after_dropdown_select:+ select_result.action_result.skip_remaining_actions = Truereturn [select_result.action_result]except PlaywrightError as inc_error:# Handle Playwright-specific errors during incremental element processingdiff --git a/skyvern/webeye/actions/handler_utils.py b/skyvern/webeye/actions/handler_utils.pyindex 974be9847..d2c8d6a08 100644--- a/skyvern/webeye/actions/handler_utils.py+++ b/skyvern/webeye/actions/handler_utils.py@@ -8,6 +8,7 @@ from skyvern.config import settingsfrom skyvern.constants import TEXT_PRESS_MAX_LENGTHfrom skyvern.forge.sdk.api.files import download_file as download_file_apifrom skyvern.forge.sdk.event.factory import EventStrategyFactory+from skyvern.webeye.actions.actions import Action, KeypressActionLOG = structlog.get_logger()@@ -39,11 +40,28 @@ async def input_sequentially(locator: Locator, text: str, timeout: float = settiawait EventStrategyFactory.type_text(locator.page, locator, text)+ENTER_KEY_ALIASES = ("enter", "return")+++def keys_include_enter(keys: list[str]) -> bool:+ # "enter" and "return" both execute as the Enter key (see keypress() below).+ return any(key.lower() in ENTER_KEY_ALIASES for key in keys)+++def should_stop_batch_after_dropdown_select(next_action: Action | None) -> bool:+ """Whether a committed in-INPUT_TEXT combobox selection should stop the batch.++ True only when the next batched action would clobber the selection: a trailing Enter/Return+ keypress. Same-element / other follow-ups are intentionally left to run.+ """+ return isinstance(next_action, KeypressAction) and keys_include_enter(next_action.keys)++async def keypress(page: Page, keys: list[str], hold: bool = False, duration: float = 0, repeat: int = 1) -> None:updated_keys = []for key in keys:key_lower_case = key.lower()- if key_lower_case in ("enter", "return"):+ if key_lower_case in ENTER_KEY_ALIASES:updated_keys.append("Enter")elif key_lower_case == "space":updated_keys.append(" ")diff --git a/tests/unit/test_input_text_search_bar_skip.py b/tests/unit/test_input_text_search_bar_skip.pynew file mode 100644index 000000000..c0ce4f19e--- /dev/null+++ b/tests/unit/test_input_text_search_bar_skip.py@@ -0,0 +1,175 @@+"""Regression tests for SKY-11343.++When INPUT_TEXT into a search-bar/combobox commits a selection in-action, a queued+page-level Enter (or a same-element action) would re-open the widget and reset the+selection. The fix stops the batch (skip_remaining_actions) only in that case, gated by+``action.stop_batch_after_dropdown_select`` (set by the agent loop). Plain search inputs+and non-clobbering follow-ups (Tab/Escape/Arrow, different-element actions) are unaffected.+"""++from __future__ import annotations++from datetime import UTC, datetime+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from skyvern.forge.sdk.models import StepStatus+from skyvern.webeye.actions.actions import InputOrSelectContext, InputTextAction, KeypressAction+from skyvern.webeye.actions.handler import handle_input_text_action+from skyvern.webeye.actions.handler_utils import keys_include_enter, should_stop_batch_after_dropdown_select+from skyvern.webeye.actions.responses import ActionSuccess+from tests.unit.helpers import make_organization, make_step, make_task++_NOW = datetime.now(UTC)+_ORG = make_organization(_NOW)+_TASK = make_task(_NOW, _ORG, navigation_payload={}, navigation_goal="Select the account")+_STEP = make_step(_NOW, _TASK, step_id="stp-1", status=StepStatus.created, order=0, output=None)+++# --------------------------------------------------------------------------- #+# keys_include_enter — Enter/Return alias (mirrors keypress() normalization)+# --------------------------------------------------------------------------- #+ "keys,expected",+ [+ (["Enter"], True),+ (["Return"], True),+ (["reTURN"], True),+ (["RETURN"], True),+ (["a", "Enter"], True),+ (["Tab"], False),+ (["Escape"], False),+ (["ArrowDown"], False),+ ([], False),+ ],+)+def test_keys_include_enter(keys: list[str], expected: bool) -> None:+ assert keys_include_enter(keys) is expected+++# --------------------------------------------------------------------------- #+# should_stop_batch_after_dropdown_select — the gate the agent loop computes+# --------------------------------------------------------------------------- #+def _input(element_id: str = "AADC") -> InputTextAction:+ return InputTextAction(element_id=element_id, text="123456", reasoning="type the account number")+++ "next_action,expected",+ [+ (KeypressAction(keys=["Enter"], reasoning="submit"), True),+ (KeypressAction(keys=["Return"], reasoning="submit"), True),+ (KeypressAction(keys=["Tab"], reasoning="next field"), False),+ (KeypressAction(keys=["Escape"], reasoning="close"), False),+ (InputTextAction(element_id="AADC", text="x", reasoning="same element no longer triggers"), False),+ (InputTextAction(element_id="OTHER", text="x", reasoning="different element"), False),+ (None, False),+ ],+)+def test_should_stop_batch_after_dropdown_select(next_action: object, expected: bool) -> None:+ assert should_stop_batch_after_dropdown_select(next_action) is expected+++def test_flag_is_transient_not_serialized() -> None:+ action = _input()+ action.stop_batch_after_dropdown_select = True+ assert "stop_batch_after_dropdown_select" not in action.model_dump()+++# --------------------------------------------------------------------------- #+# handle_input_text_action — gated batch-stop behavior+# --------------------------------------------------------------------------- #+def _mock_search_input() -> MagicMock:+ el = MagicMock()+ el.get_id.return_value = "AADC"+ el.get_tag_name.return_value = "input"+ el.get_frame.return_value = MagicMock()+ locator = MagicMock()+ locator.focus = AsyncMock()+ el.get_locator.return_value = locator+ el.is_disabled = AsyncMock(return_value=False)+ el.get_selectable = AsyncMock(return_value=False)+ el.has_hidden_attr = AsyncMock(return_value=False)+ el.is_readonly = AsyncMock(return_value=False)+ el.get_attr = AsyncMock(return_value=None)+ el.is_spinbtn_input = AsyncMock(return_value=False)+ el.is_editable = AsyncMock(return_value=True)+ el.is_visible = AsyncMock(return_value=True)+ el.is_raw_input = AsyncMock(return_value=False)+ el.is_auto_completion_input = AsyncMock(return_value=False)+ el.find_blocking_element = AsyncMock(return_value=(None, False))+ el.get_element_handler = AsyncMock(return_value=MagicMock())+ el.input_sequentially = AsyncMock()+ el.input_clear = AsyncMock()+ el.scroll_into_view = AsyncMock()+ el.press_key = AsyncMock()+ return el+++async def _run_search_bar_input(stop_flag: bool, incremental: list[dict]) -> list:+ skyvern_el = _mock_search_input()+ dom_instance = MagicMock()+ dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=skyvern_el)++ inc = MagicMock()+ inc.start_listen_dom_increment = AsyncMock()+ inc.stop_listen_dom_increment = AsyncMock()+ inc.get_incremental_element_tree = AsyncMock(return_value=incremental)++ skyvern_frame = MagicMock()+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()++ scraped_page = MagicMock()+ scraped_page.id_to_element_dict = {"AADC": {"tagName": "input"}}++ context = InputOrSelectContext(field="Account", is_search_bar=True, is_location_input=False)+ select_result = MagicMock()+ select_result.action_result = ActionSuccess()++ action = _input()+ action.stop_batch_after_dropdown_select = stop_flag++ with (+ patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_instance),+ patch("skyvern.webeye.actions.handler.SkyvernFrame.create_instance", new=AsyncMock(return_value=skyvern_frame)),+ patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc),+ patch("skyvern.webeye.actions.handler.get_input_value", new=AsyncMock(return_value="")),+ patch(+ "skyvern.webeye.actions.handler.get_actual_value_of_parameter_if_secret_with_task",+ return_value="123456",+ ),+ patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=context)),+ patch("skyvern.webeye.actions.handler._incremental_tree_contains_target_value", return_value=True),+ patch(+ "skyvern.webeye.actions.handler.sequentially_select_from_dropdown",+ new=AsyncMock(return_value=select_result),+ ),+ ):+ return await handle_input_text_action(+ action=action, page=MagicMock(), scraped_page=scraped_page, task=_TASK, step=_STEP+ )+++async def test_selection_with_clobbering_next_stops_batch() -> None:+ """Selection committed + a clobbering next action (flag set) -> skip_remaining_actions=True."""+ results = await _run_search_bar_input(stop_flag=True, incremental=[{"id": "OPT", "text": "Target 123456"}])+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)+ assert results[0].skip_remaining_actions is True+++async def test_selection_without_clobbering_next_preserves_batch() -> None:+ """Selection committed but next action is benign (flag unset) -> batch NOT stopped."""+ results = await _run_search_bar_input(stop_flag=False, incremental=[{"id": "OPT", "text": "Target 123456"}])+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)+ assert not results[0].skip_remaining_actions+++async def test_plain_search_input_does_not_stop_batch() -> None:+ """No dropdown surfaced (plain search box) -> ActionSuccess without skip, so a trailing Enter still fires."""+ results = await _run_search_bar_input(stop_flag=True, incremental=[])+ assert len(results) == 1 and isinstance(results[0], ActionSuccess)+ assert not results[0].skip_remaining_actions