Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Recognize JS-wired submit controls in the explicit-submit click bypass (SKY-15195) (#8459)
LawyZheng287867e72143
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 6f793afce..c9a4949ce 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -908,6 +908,10 @@ class AgentFunction:async def resolve_task_v3_error_code_choice(self, *, task: Task, organization: Organization) -> bool:return False+ # Recognition of submit controls whose submission is wired in JS (rendered type=button); base recognizes none.+ async def is_recognized_submit_control(self, element: SkyvernElement) -> bool:+ return False+async def record_run_duration(self,run_type: str,diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex d31ca1be4..1df8c2de7 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -6418,12 +6418,18 @@ async def handle_sequential_click_with_submit_bypass(submit controls; every other click still runs ``handle_sequential_click_for_dropdown``.``anchor_element`` is the final click target after any disabled-wrapper- retargeting, so the explicit-submit check is evaluated here and never stale.+ retargeting, so the submit check is evaluated here and never stale."""- if await anchor_element.is_explicit_submit():+ is_explicit_submit = await anchor_element.is_explicit_submit()+ # Short-circuit: only consult the deployment recognizer when this is not already an explicit submit.+ recognized_submit_control = not is_explicit_submit and await app.AGENT_FUNCTION.is_recognized_submit_control(+ anchor_element+ )+ if is_explicit_submit or recognized_submit_control:LOG.info("Explicit submit click; bypassing the dropdown sequential-click rescrape",element_id=anchor_element.get_id(),+ recognized_submit_control=recognized_submit_control,)return Nonediff --git a/tests/unit/test_explicit_submit_sequential_click_bypass.py b/tests/unit/test_explicit_submit_sequential_click_bypass.pyindex d34451ece..db0ef0b69 100644--- a/tests/unit/test_explicit_submit_sequential_click_bypass.py+++ b/tests/unit/test_explicit_submit_sequential_click_bypass.py@@ -17,6 +17,7 @@ from unittest.mock import AsyncMock, MagicMockimport pytestimport skyvern.webeye.actions.handler as handler_module+from skyvern.forge.agent_functions import AgentFunctionfrom skyvern.forge.sdk.models import StepStatusfrom skyvern.webeye.actions.actions import ClickActionfrom skyvern.webeye.actions.handler import (@@ -34,6 +35,16 @@ def _el(*, element_id: str, tag_name: str, attributes: dict | None = None) -> Skreturn SkyvernElement(MagicMock(), MagicMock(), static)[email protected](autouse=True)+def _oss_neutral_submit_recognizer(monkeypatch: pytest.MonkeyPatch) -> None:+ # The stub app's AGENT_FUNCTION auto-mocks unset methods as truthy AsyncMocks, which would make+ # the recognizer seam bypass everything. Pin the OSS-neutral default (recognizes nothing) here;+ # tests needing recognition override this method afterwards.+ monkeypatch.setattr(+ handler_module.app.AGENT_FUNCTION, "is_recognized_submit_control", AsyncMock(return_value=False)+ )++class TestSkyvernElementIsExplicitSubmit:@pytest.mark.asyncioasync def test_button_type_submit_is_explicit(self) -> None:@@ -196,6 +207,83 @@ class TestSequentialClickSubmitBypass:assert result is Noneinner.assert_not_called()+ @pytest.mark.asyncio+ async def test_agent_function_recognized_submit_bypasses(self, monkeypatch: pytest.MonkeyPatch) -> None:+ # When the AGENT_FUNCTION seam recognizes the control (cloud recognizes a JS-wired submit),+ # the wrapper bypasses the rescrape exactly like an explicit type=submit. The anchor here is+ # an ordinary type=button with no hooks — only the seam's verdict drives the bypass.+ inner = AsyncMock(return_value=MagicMock(name="sequential_result"))+ monkeypatch.setattr("skyvern.webeye.actions.handler.handle_sequential_click_for_dropdown", inner)+ monkeypatch.setattr(+ handler_module.app.AGENT_FUNCTION, "is_recognized_submit_control", AsyncMock(return_value=True)+ )+ el = _el(element_id="E1", tag_name="button", attributes={"type": "button"})++ result = await handle_sequential_click_with_submit_bypass(**self._kwargs(el))++ assert result is None+ inner.assert_not_called()++ @pytest.mark.asyncio+ async def test_oss_base_recognizes_no_submit_control_and_rescrapes(self, monkeypatch: pytest.MonkeyPatch) -> None:+ # OSS neutrality: bind the REAL base AgentFunction — it recognizes no submit control, so a+ # click runs the normal rescrape. The hook-specific recognizer is a deployment-layer contract+ # tested elsewhere; no submit fingerprint lives in skyvern/ or tests/unit.+ sentinel = MagicMock(name="sequential_result")+ inner = AsyncMock(return_value=sentinel)+ monkeypatch.setattr("skyvern.webeye.actions.handler.handle_sequential_click_for_dropdown", inner)+ monkeypatch.setattr(+ handler_module.app.AGENT_FUNCTION,+ "is_recognized_submit_control",+ AgentFunction().is_recognized_submit_control,+ )+ el = _el(element_id="E1", tag_name="button", attributes={"type": "button"})++ result = await handle_sequential_click_with_submit_bypass(**self._kwargs(el))++ assert result is sentinel+ inner.assert_called_once()++ @pytest.mark.asyncio+ async def test_bypass_log_marks_recognizer_arm(self, monkeypatch: pytest.MonkeyPatch) -> None:+ # Observability: the bypass log must carry recognized_submit_control=True when the deployment+ # recognizer arm (not the legacy explicit type=submit) caused the bypass. Message unchanged.+ monkeypatch.setattr("skyvern.webeye.actions.handler.handle_sequential_click_for_dropdown", AsyncMock())+ monkeypatch.setattr(+ handler_module.app.AGENT_FUNCTION, "is_recognized_submit_control", AsyncMock(return_value=True)+ )+ log_mock = MagicMock()+ monkeypatch.setattr(handler_module, "LOG", log_mock)+ el = _el(element_id="E1", tag_name="button", attributes={"type": "button"})++ result = await handle_sequential_click_with_submit_bypass(**self._kwargs(el))++ assert result is None+ log_mock.info.assert_called_once()+ args, kwargs = log_mock.info.call_args+ assert args[0] == "Explicit submit click; bypassing the dropdown sequential-click rescrape"+ assert kwargs["recognized_submit_control"] is True++ @pytest.mark.asyncio+ async def test_bypass_log_marks_legacy_explicit_submit(self, monkeypatch: pytest.MonkeyPatch) -> None:+ # The legacy explicit type=submit arm logs recognized_submit_control=False, and the recognizer+ # is never consulted (short-circuit preserved). Message unchanged.+ monkeypatch.setattr("skyvern.webeye.actions.handler.handle_sequential_click_for_dropdown", AsyncMock())+ recognizer = AsyncMock(side_effect=AssertionError("recognizer must not be called for explicit submit"))+ monkeypatch.setattr(handler_module.app.AGENT_FUNCTION, "is_recognized_submit_control", recognizer)+ log_mock = MagicMock()+ monkeypatch.setattr(handler_module, "LOG", log_mock)+ el = _el(element_id="E1", tag_name="button", attributes={"type": "submit"})++ result = await handle_sequential_click_with_submit_bypass(**self._kwargs(el))++ assert result is None+ recognizer.assert_not_called()+ log_mock.info.assert_called_once()+ args, kwargs = log_mock.info.call_args+ assert args[0] == "Explicit submit click; bypassing the dropdown sequential-click rescrape"+ assert kwargs["recognized_submit_control"] is False+@pytest.mark.asyncio@pytest.mark.parametrize("padded", [" submit ", "submit ", " submit"])async def test_whitespace_padded_type_invokes_sequential_handler(@@ -275,6 +363,29 @@ class TestHandleClickActionIntegration:assert len(results) == 1assert isinstance(results[-1], ActionSuccess)+ @pytest.mark.asyncio+ async def test_seam_recognized_submit_clicks_and_bypasses_dropdown_rescrape(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ # End-to-end at the handle_click_action entry: when the AGENT_FUNCTION seam recognizes the+ # control (cloud recognizes a JS-wired submit), the click runs and the expensive rescrape is+ # bypassed. The anchor is an ordinary type=button — only the seam verdict drives the bypass.+ submit_el = self._clickable_element(tag_name="button", type_value="button")+ chain_click_mock, sequential_mock, incremental = self._wire_handler(monkeypatch, submit_el)+ monkeypatch.setattr(+ handler_module.app.AGENT_FUNCTION, "is_recognized_submit_control", AsyncMock(return_value=True)+ )++ results = await handle_click_action(+ ClickAction(element_id="E1"), self._page(), MagicMock(), MagicMock(), MagicMock()+ )++ chain_click_mock.assert_awaited_once()+ sequential_mock.assert_not_called()+ incremental.stop_listen_dom_increment.assert_awaited_once()+ assert results == chain_click_mock.return_value+ assert isinstance(results[-1], ActionSuccess)+@pytest.mark.asyncioasync def test_non_submit_button_still_invokes_dropdown_rescrape(self, monkeypatch: pytest.MonkeyPatch) -> None:# Positive control: identical wiring, only tag/type differ — proves the@@ -538,6 +649,7 @@ class TestHandleActionPublicPathFalseClickBypass:app_mock = MagicMock()app_mock.BROWSER_MANAGER.get_for_task.return_value = MagicMock()app_mock.AGENT_FUNCTION.wait_for_challenge_solver = AsyncMock()+ app_mock.AGENT_FUNCTION.is_recognized_submit_control = AsyncMock(return_value=False)app_mock.DATABASE.workflow_params.create_action = AsyncMock(return_value=SimpleNamespace(action_id="a-1"))monkeypatch.setattr(handler_module, "app", app_mock)monkeypatch.setattr(handler_module, "preflight_action", MagicMock(return_value=None))