Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Give type's forced click the same focus fallback as the plain click (SKY-16451) (#8585)
pedrohsdbf4b56073d34c
diff --git a/skyvern/cli/mcp_tools/browser.py b/skyvern/cli/mcp_tools/browser.pyindex c0ebea0d9..307efc74e 100644--- a/skyvern/cli/mcp_tools/browser.py+++ b/skyvern/cli/mcp_tools/browser.py@@ -96,6 +96,7 @@ from skyvern.schemas.action_log import ActionLogOutcome, project_action_eventfrom skyvern.schemas.run_blocks import CredentialTypefrom skyvern.utils.url_validators import validate_fetch_urlfrom skyvern.webeye.actions.handler_utils import strategy_aware_input+from skyvern.webeye.actions.key_names import normalize_key_chordfrom skyvern.webeye.navigation import reported_nav_error_codefrom skyvern.webeye.utils.page import SkyvernFrame@@ -2132,6 +2133,7 @@ async def skyvern_press_key(Use `intent` or `selector` to focus a specific element before pressing.Without either, presses the key on the currently focused element."""+ key = normalize_key_chord(key)selector = _blank_to_none(selector)intent = _blank_to_none(intent)try:diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex 77f6abb38..c6a5b50f2 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -63,6 +63,7 @@ from skyvern.forge.taskv3.loop import ()from skyvern.forge.taskv3.preflight import PREFLIGHT_TOOL_NAMES, preflight_tool_actionfrom skyvern.forge.taskv3.target_label import TARGET_KIND_TOKENS, TARGET_NAME_CAP+from skyvern.webeye.actions.key_names import normalize_key_chordfrom skyvern.webeye.browser_driver_errors import is_driver_timeout_errorfrom skyvern.webeye.browser_state import BLANK_PAGE_URLSfrom skyvern.webeye.utils.page import OTP_INPUT_PRIVACY_JS, OTP_SAFE_FRAGMENT_HTML_JS, mask_otp_values_in_html@@ -9732,6 +9733,24 @@ def build_browser_tools(return Truereturn False+ async def _focus_in_place_of_click(page: Any, selector: str, exc: Exception, *, focus_fallback: bool) -> None:+ # focus() needs no hit target, so it stands in for a click refused only by the viewport check.+ # A widget that hands the caret to another segment would take the keys there, so a caret that+ # does not stay put re-raises the click's error; the caller must still prove the keystrokes landed.+ if not focus_fallback or not _click_blocked_only_by_viewport(exc):+ raise exc+ # A field that commits a picked suggestion holds the raw query until blur, so a read-back could+ # not tell a fill from a query; with no click to reach its rows, keep the error.+ if await _declares_a_list(page, selector):+ raise exc+ try:+ await page.focus(selector, timeout=15000)+ held = await page.evaluate(_ACTIVE_IS_JS, await _probe_arg(page, selector))+ except Exception:+ held = None+ if held is not True:+ raise exc+async def _focus_for_typing(page: Any, selector: str, *, focus_fallback: bool = False) -> tuple[bool, dict[str, Any] | None, bool]:@@ -9753,7 +9772,13 @@ def build_browser_tools(# field that never moved. A navigation clears window, so a token planted on it answers# "is this still the same document" exactly -- the same technique the pre-snapshot uses.await page.evaluate("() => { window.__tv3_doc = 1; }")- await page.click(selector, timeout=15000, force=True)+ try:+ await page.click(selector, timeout=15000, force=True)+ except Exception as exc:+ # Force skips the hit-target check but not the viewport one, which rejects any box of+ # at most one square pixel: a segment input kept sub-pixel under its own display layer.+ await _focus_in_place_of_click(page, selector, exc, focus_fallback=focus_fallback)+ return True, None, Truetry:await page.wait_for_load_state("domcontentloaded", timeout=1000)except Exception:@@ -9780,22 +9805,8 @@ def build_browser_tools(except Exception as exc:# A segmented control can keep its real input off-viewport with tabindex=-1 under an# aria-hidden display layer: the probe finds nothing on top of it, but the click's- # hit-test has no point to land on. focus() needs no hit target. A widget that hands the- # caret to another segment would take the keys there, so a caret that does not stay put- # keeps today's error; the caller must still prove the keystrokes landed.- if not focus_fallback or not _click_blocked_only_by_viewport(exc):- raise- # A field that commits a picked suggestion holds the raw query until blur, so a read-back- # could not tell a fill from a query; with no click to reach its rows, keep the error.- if await _declares_a_list(page, selector):- raise- try:- await page.focus(selector, timeout=15000)- held = await page.evaluate(_ACTIVE_IS_JS, await _probe_arg(page, selector))- except Exception:- held = None- if held is not True:- raise+ # hit-test has no point to land on.+ await _focus_in_place_of_click(page, selector, exc, focus_fallback=focus_fallback)focused_without_click = Truetry:focused = await page.evaluate(_ACTIVE_IS_JS, await _probe_arg(page, selector))@@ -11668,7 +11679,7 @@ def build_browser_tools(page, error = await _resolve_page()if error is not None:return error- key = args["key"]+ key = normalize_key_chord(args["key"])selector = args.get("selector")if selector:ambiguous = await _ambiguous_selector_error(page, selector)@@ -12813,7 +12824,9 @@ def build_browser_tools(),_spec("press_key",- "Press a keyboard key (optionally focused on a selector), e.g. Enter, Escape, Tab.",+ "Press a keyboard key or chord (optionally focused on a selector), e.g. Enter, Escape, Control+a. "+ "Keys reach only the page, never the browser, so browser shortcuts (reload, back, forward) do "+ "nothing; use navigate for those.",_obj({"key": {"type": "string"}, "selector": {"type": "string"}}, ["key"]),press_key,),diff --git a/skyvern/webeye/actions/handler_utils.py b/skyvern/webeye/actions/handler_utils.pyindex 7b4990c1b..8306f1b2e 100644--- a/skyvern/webeye/actions/handler_utils.py+++ b/skyvern/webeye/actions/handler_utils.py@@ -12,6 +12,7 @@ from skyvern.forge.sdk.api.files import resolve_run_download_id, validate_local_from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.event.factory import EventStrategyFactoryfrom skyvern.webeye.actions.actions import Action, KeypressAction+from skyvern.webeye.actions.key_names import ENTER_KEY_ALIASES, normalize_key_nameLOG = structlog.get_logger()@@ -188,9 +189,6 @@ async def input_sequentially(locator: Locator, text: str, timeout: float = setti)-ENTER_KEY_ALIASES = ("enter", "return")--def keys_include_enter(keys: list[str]) -> bool:# "enter" and "return" both execute as the Enter key (see keypress() below).return any(key.lower() in ENTER_KEY_ALIASES for key in keys)@@ -206,48 +204,7 @@ def should_stop_batch_after_dropdown_select(next_action: Action | None) -> bool:async def keypress(page: Page, keys: list[str], hold: bool = False, duration: float = 0, repeat: int = 1) -> None:- updated_keys = []- for key in keys:- key_lower_case = key.lower()- if key_lower_case in ENTER_KEY_ALIASES:- updated_keys.append("Enter")- elif key_lower_case == "space":- updated_keys.append(" ")- elif key_lower_case == "ctrl":- updated_keys.append("Control")- elif key_lower_case == "backspace":- updated_keys.append("Backspace")- elif key_lower_case == "pagedown":- updated_keys.append("PageDown")- elif key_lower_case == "pageup":- updated_keys.append("PageUp")- elif key_lower_case == "tab":- updated_keys.append("Tab")- elif key_lower_case == "shift":- updated_keys.append("Shift")- elif key_lower_case in ("arrowleft", "left"):- updated_keys.append("ArrowLeft")- elif key_lower_case in ("arrowright", "right"):- updated_keys.append("ArrowRight")- elif key_lower_case in ("arrowup", "up"):- updated_keys.append("ArrowUp")- elif key_lower_case in ("arrowdown", "down"):- updated_keys.append("ArrowDown")- elif key_lower_case == "home":- updated_keys.append("Home")- elif key_lower_case == "end":- updated_keys.append("End")- elif key_lower_case == "delete":- updated_keys.append("Delete")- elif key_lower_case == "esc":- updated_keys.append("Escape")- elif key_lower_case == "alt":- updated_keys.append("Alt")- elif key_lower_case.startswith("f") and key_lower_case[1:].isdigit():- # Handle function keys: f1 -> F1, f5 -> F5, etc.- updated_keys.append(key_lower_case.upper())- else:- updated_keys.append(key)+ updated_keys = [normalize_key_name(key) for key in keys]keypress_str = "+".join(updated_keys)n = max(1, repeat)if hold:diff --git a/skyvern/webeye/actions/key_names.py b/skyvern/webeye/actions/key_names.pynew file mode 100644index 000000000..8f747e95b--- /dev/null+++ b/skyvern/webeye/actions/key_names.py@@ -0,0 +1,49 @@+ENTER_KEY_ALIASES = ("enter", "return")++_KEY_ALIASES = {+ "space": " ",+ "ctrl": "Control",+ "backspace": "Backspace",+ "pagedown": "PageDown",+ "pageup": "PageUp",+ "tab": "Tab",+ "shift": "Shift",+ "arrowleft": "ArrowLeft",+ "left": "ArrowLeft",+ "arrowright": "ArrowRight",+ "right": "ArrowRight",+ "arrowup": "ArrowUp",+ "up": "ArrowUp",+ "arrowdown": "ArrowDown",+ "down": "ArrowDown",+ "home": "Home",+ "end": "End",+ "delete": "Delete",+ "esc": "Escape",+ "alt": "Alt",+}+++def normalize_key_name(key: str) -> str:+ """Map a loosely written key name (any case, common short forms) to the name Playwright accepts."""+ key_lower_case = key.lower()+ if key_lower_case in ENTER_KEY_ALIASES:+ return "Enter"+ if key_lower_case in _KEY_ALIASES:+ return _KEY_ALIASES[key_lower_case]+ if key_lower_case.startswith("f") and key_lower_case[1:].isdigit():+ return key_lower_case.upper()+ return key+++def split_key_chord(chord: str) -> list[str]:+ # "+" joins a chord's keys and is also a key itself, so "Control++" is Control then "+".+ if chord == "+":+ return ["+"]+ if chord.endswith("++"):+ return chord[:-2].split("+") + ["+"]+ return chord.split("+")+++def normalize_key_chord(chord: str) -> str:+ return "+".join(normalize_key_name(part) for part in split_key_chord(chord))diff --git a/tests/unit/test_mcp_browser_tools.py b/tests/unit/test_mcp_browser_tools.pyindex 8aceaf4ed..0cbcbee97 100644--- a/tests/unit/test_mcp_browser_tools.py+++ b/tests/unit/test_mcp_browser_tools.py@@ -849,6 +849,23 @@ async def test_skyvern_press_key_selector_only_uses_fast_default_timeout(monkeyppress.assert_awaited_once_with("Enter", timeout=5000)[email protected](("key", "pressed"), [("CTRL+a", "Control+a"), ("left", "ArrowLeft"), ("esc", "Escape")])+async def test_skyvern_press_key_sends_the_driver_the_key_name_it_accepts(+ monkeypatch: pytest.MonkeyPatch, key: str, pressed: str+) -> None:+ # The driver raises "Unknown key" on these spellings, which models write routinely.+ press = AsyncMock()+ page = SimpleNamespace(locator=MagicMock(return_value=SimpleNamespace(press=press)))+ context = BrowserContext(mode="cloud_session", session_id="pbs_test")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, context)))++ result = await mcp_browser.skyvern_press_key(key=key, selector="#field")++ assert result["ok"] is True+ press.assert_awaited_once_with(pressed, timeout=5000)++@pytest.mark.asyncioasync def test_skyvern_press_key_intent_explicit_timeout_is_honored(monkeypatch: pytest.MonkeyPatch) -> None:press = AsyncMock()diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.pyindex f5bd2db11..ce425920f 100644--- a/tests/unit/test_taskv3_tools.py+++ b/tests/unit/test_taskv3_tools.py@@ -7780,6 +7780,31 @@ _SEGMENTED_DATE_EXPOSED_ECHO_HTML = """"""+# SKY-16451: the same control with its display layer painted OVER the inputs, which are kept sub-pixel.+# The probe reads the layer as the field's own skin, so the click is forced -- and a forced click still+# refuses a box of at most one square pixel as "outside of the viewport".+_SEGMENTED_DATE_SKINNED_SUBPIXEL_HTML = """+<div role="group" aria-label="Start date" style="display:flex;width:240px;height:30px">+ <div style="position:relative;width:60px;height:30px">+ <input id="year" type="text" role="spinbutton" aria-label="Year"+ style="position:absolute;left:4px;top:4px;width:1px;height:0.5px;padding:0;border:0;box-sizing:border-box">+ <div id="year-display" aria-hidden="true" style="position:absolute;inset:0;z-index:1;background:#fff">YYYY</div>+ </div>+ <div style="position:relative;width:40px;height:30px">+ <input id="month" type="text" role="spinbutton" aria-label="Month"+ style="position:absolute;left:4px;top:4px;width:1px;height:0.5px;padding:0;border:0;box-sizing:border-box">+ <div aria-hidden="true" style="position:absolute;inset:0;z-index:1;background:#fff">MM</div>+ </div>+</div>+<script>+ const year = document.getElementById("year");+ year.addEventListener("input", () => {+ document.getElementById("year-display").textContent = year.value || "YYYY";+ });+</script>+"""++@_skip_no_browser@pytest.mark.asyncio@pytest.mark.parametrize(@@ -7800,11 +7825,16 @@ _SEGMENTED_DATE_EXPOSED_ECHO_HTML = """],ids=["keys-land-in-sibling-segment", "keys-dropped", "cleared-after-typing", "trimmed"],)-async def test_type_into_an_unclickable_field_never_reports_a_fill_that_did_not_land(misroute: str, text: str) -> None:+ "template", [_SEGMENTED_DATE_HTML, _SEGMENTED_DATE_SKINNED_SUBPIXEL_HTML], ids=["unclickable", "skinned-subpixel"]+)+async def test_type_into_an_unclickable_field_never_reports_a_fill_that_did_not_land(+ misroute: str, text: str, template: str+) -> None:# Reaching the field by focus() alone proves nothing about the keystrokes. A success here would# turn today's loud failure into a date that reads as filled and is not.# A raised error is the loud outcome too: the tool wrapper turns it into a tool error.- html = _SEGMENTED_DATE_HTML + f"<script>{misroute}</script>"+ html = template + f"<script>{misroute}</script>"async with _content_page(html) as page:tools = build_browser_tools(_fixed_page_provider(page))try:@@ -7867,6 +7897,15 @@ _UNCLICKABLE_BARE_TYPEAHEAD_HTML = """</script>"""+# The declared typeahead again, with its display layer over a sub-pixel input so the click is forced.+_SKINNED_SUBPIXEL_TYPEAHEAD_HTML = _UNCLICKABLE_TYPEAHEAD_HTML.replace(+ '<div aria-hidden="true" style="position:absolute;inset:0;background:#fff">City</div>',+ '<div aria-hidden="true" style="position:absolute;inset:0;z-index:1;background:#fff">City</div>',+).replace(+ 'style="position:absolute;left:-500px;top:0;width:200px;height:30px"',+ 'style="position:absolute;left:4px;top:4px;width:1px;height:0.5px;padding:0;border:0;box-sizing:border-box"',+)+# A widget that renders BOTH: an echo of the keystrokes and a real list. Excluding the echo must not# excuse the list._ECHO_SCRIPT = (@@ -7896,6 +7935,12 @@ _ECHO_SCRIPT = (# Rows a pickability test can read nothing off, alone and beside an echo of the keystrokes.(_UNCLICKABLE_BARE_TYPEAHEAD_HTML, "", 0, ""),(_UNCLICKABLE_BARE_TYPEAHEAD_HTML, "", 0, _ECHO_SCRIPT),+ (+ _SKINNED_SUBPIXEL_TYPEAHEAD_HTML,+ 'role="combobox" aria-autocomplete="list" aria-controls="city-list" aria-expanded="false"',+ 2500,+ "",+ ),],ids=["declared-slow-rows",@@ -7904,6 +7949,7 @@ _ECHO_SCRIPT = ("undeclared-rows-beside-an-echo","bare-div-rows","bare-div-rows-beside-an-echo",+ "skinned-subpixel-declared-slow-rows",],)async def test_type_into_an_unclickable_typeahead_never_reports_the_raw_query_as_filled(@@ -7929,8 +7975,13 @@ async def test_type_into_an_unclickable_typeahead_never_reports_the_raw_query_as@pytest.mark.asyncio@pytest.mark.parametrize("html",- [_SEGMENTED_DATE_HTML, _SEGMENTED_DATE_REPLACED_ECHO_HTML, _SEGMENTED_DATE_EXPOSED_ECHO_HTML],- ids=["echo-mutated", "echo-replaced", "echo-exposed"],+ [+ _SEGMENTED_DATE_HTML,+ _SEGMENTED_DATE_REPLACED_ECHO_HTML,+ _SEGMENTED_DATE_EXPOSED_ECHO_HTML,+ _SEGMENTED_DATE_SKINNED_SUBPIXEL_HTML,+ ],+ ids=["echo-mutated", "echo-replaced", "echo-exposed", "skinned-subpixel"],)async def test_type_fills_a_segment_input_the_click_cannot_reach(html: str) -> None:async with _content_page(html) as page:@@ -25456,3 +25507,37 @@ def test_only_an_engine_minted_marker_value_is_folded_at_a_window_head() -> Noneassert reported == closing - offset + 1, (value, offset, reported)else:assert reported == 0, (value, offset, reported)+++_KEY_LOG_HTML = """+<input id="field" value="text">+<script>+ window.__keys = [];+ document.addEventListener("keydown", (e) => {+ window.__keys.push((e.ctrlKey ? "Control+" : "") + (e.altKey ? "Alt+" : "") + e.key);+ });+</script>+"""+++@_skip_no_browser+async def test_press_key_accepts_the_key_names_v1_accepts() -> None:+ # Models write CTRL, ALT, left, HOME...: v1 maps these before pressing, and an unmapped name raises+ # "Unknown key" and burns a round.+ cases = [+ ("CTRL+a", "Control+a"),+ ("ALT+r", "Alt+r"),+ ("left", "ArrowLeft"),+ ("HOME", "Home"),+ ("esc", "Escape"),+ ("TAB", "Tab"),+ ("enter", "Enter"),+ ]+ async with _content_page(_KEY_LOG_HTML) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ for key, expected in cases:+ await page.evaluate("window.__keys = []")+ r = await _tool(tools, "press_key").handler({"key": key, "selector": "#field"})+ assert r.status == "ok", (key, r.content)+ assert expected in await page.evaluate("window.__keys"), key