Skip to content

Automate browser based workflows with AI

Give type's forced click the same focus fallback as the plain click (SKY-16451) (#8585)

pedrohsdbf4b56073d34c
diff --git a/skyvern/cli/mcp_tools/browser.py b/skyvern/cli/mcp_tools/browser.py
index c0ebea0d9..307efc74e 100644
--- a/skyvern/cli/mcp_tools/browser.py
+++ b/skyvern/cli/mcp_tools/browser.py
@@ -96,6 +96,7 @@ from skyvern.schemas.action_log import ActionLogOutcome, project_action_event
from skyvern.schemas.run_blocks import CredentialType
from skyvern.utils.url_validators import validate_fetch_url
from skyvern.webeye.actions.handler_utils import strategy_aware_input
+from skyvern.webeye.actions.key_names import normalize_key_chord
from skyvern.webeye.navigation import reported_nav_error_code
from skyvern.webeye.utils.page import SkyvernFrame
@@ -2132,6 +2133,7 @@ async def skyvern_press_key(
Use `intent` or `selector` to focus a specific element before pressing.
Without either, presses the key on the currently focused element.
"""
+ key = normalize_key_chord(key)
selector = _blank_to_none(selector)
intent = _blank_to_none(intent)
try:
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 77f6abb38..c6a5b50f2 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -63,6 +63,7 @@ from skyvern.forge.taskv3.loop import (
)
from skyvern.forge.taskv3.preflight import PREFLIGHT_TOOL_NAMES, preflight_tool_action
from skyvern.forge.taskv3.target_label import TARGET_KIND_TOKENS, TARGET_NAME_CAP
+from skyvern.webeye.actions.key_names import normalize_key_chord
from skyvern.webeye.browser_driver_errors import is_driver_timeout_error
from skyvern.webeye.browser_state import BLANK_PAGE_URLS
from skyvern.webeye.utils.page import OTP_INPUT_PRIVACY_JS, OTP_SAFE_FRAGMENT_HTML_JS, mask_otp_values_in_html
@@ -9732,6 +9733,24 @@ def build_browser_tools(
return True
return False
+ async def _focus_in_place_of_click(page: Any, selector: str, exc: Exception, *, focus_fallback: bool) -> None:
+ # focus() needs no hit target, so it stands in for a click refused only by the viewport check.
+ # A widget that hands the caret to another segment would take the keys there, so a caret that
+ # does not stay put re-raises the click's error; the caller must still prove the keystrokes landed.
+ if not focus_fallback or not _click_blocked_only_by_viewport(exc):
+ raise exc
+ # A field that commits a picked suggestion holds the raw query until blur, so a read-back could
+ # not tell a fill from a query; with no click to reach its rows, keep the error.
+ if await _declares_a_list(page, selector):
+ raise exc
+ try:
+ await page.focus(selector, timeout=15000)
+ held = await page.evaluate(_ACTIVE_IS_JS, await _probe_arg(page, selector))
+ except Exception:
+ held = None
+ if held is not True:
+ raise exc
+
async def _focus_for_typing(
page: Any, selector: str, *, focus_fallback: bool = False
) -> tuple[bool, dict[str, Any] | None, bool]:
@@ -9753,7 +9772,13 @@ def build_browser_tools(
# field that never moved. A navigation clears window, so a token planted on it answers
# "is this still the same document" exactly -- the same technique the pre-snapshot uses.
await page.evaluate("() => { window.__tv3_doc = 1; }")
- await page.click(selector, timeout=15000, force=True)
+ try:
+ await page.click(selector, timeout=15000, force=True)
+ except Exception as exc:
+ # Force skips the hit-target check but not the viewport one, which rejects any box of
+ # at most one square pixel: a segment input kept sub-pixel under its own display layer.
+ await _focus_in_place_of_click(page, selector, exc, focus_fallback=focus_fallback)
+ return True, None, True
try:
await page.wait_for_load_state("domcontentloaded", timeout=1000)
except Exception:
@@ -9780,22 +9805,8 @@ def build_browser_tools(
except Exception as exc:
# A segmented control can keep its real input off-viewport with tabindex=-1 under an
# aria-hidden display layer: the probe finds nothing on top of it, but the click's
- # hit-test has no point to land on. focus() needs no hit target. A widget that hands the
- # caret to another segment would take the keys there, so a caret that does not stay put
- # keeps today's error; the caller must still prove the keystrokes landed.
- if not focus_fallback or not _click_blocked_only_by_viewport(exc):
- raise
- # A field that commits a picked suggestion holds the raw query until blur, so a read-back
- # could not tell a fill from a query; with no click to reach its rows, keep the error.
- if await _declares_a_list(page, selector):
- raise
- try:
- await page.focus(selector, timeout=15000)
- held = await page.evaluate(_ACTIVE_IS_JS, await _probe_arg(page, selector))
- except Exception:
- held = None
- if held is not True:
- raise
+ # hit-test has no point to land on.
+ await _focus_in_place_of_click(page, selector, exc, focus_fallback=focus_fallback)
focused_without_click = True
try:
focused = await page.evaluate(_ACTIVE_IS_JS, await _probe_arg(page, selector))
@@ -11668,7 +11679,7 @@ def build_browser_tools(
page, error = await _resolve_page()
if error is not None:
return error
- key = args["key"]
+ key = normalize_key_chord(args["key"])
selector = args.get("selector")
if selector:
ambiguous = await _ambiguous_selector_error(page, selector)
@@ -12813,7 +12824,9 @@ def build_browser_tools(
),
_spec(
"press_key",
- "Press a keyboard key (optionally focused on a selector), e.g. Enter, Escape, Tab.",
+ "Press a keyboard key or chord (optionally focused on a selector), e.g. Enter, Escape, Control+a. "
+ "Keys reach only the page, never the browser, so browser shortcuts (reload, back, forward) do "
+ "nothing; use navigate for those.",
_obj({"key": {"type": "string"}, "selector": {"type": "string"}}, ["key"]),
press_key,
),
diff --git a/skyvern/webeye/actions/handler_utils.py b/skyvern/webeye/actions/handler_utils.py
index 7b4990c1b..8306f1b2e 100644
--- a/skyvern/webeye/actions/handler_utils.py
+++ b/skyvern/webeye/actions/handler_utils.py
@@ -12,6 +12,7 @@ from skyvern.forge.sdk.api.files import resolve_run_download_id, validate_local_
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.event.factory import EventStrategyFactory
from skyvern.webeye.actions.actions import Action, KeypressAction
+from skyvern.webeye.actions.key_names import ENTER_KEY_ALIASES, normalize_key_name
LOG = structlog.get_logger()
@@ -188,9 +189,6 @@ async def input_sequentially(locator: Locator, text: str, timeout: float = setti
)
-ENTER_KEY_ALIASES = ("enter", "return")
-
-
def keys_include_enter(keys: list[str]) -> bool:
# "enter" and "return" both execute as the Enter key (see keypress() below).
return any(key.lower() in ENTER_KEY_ALIASES for key in keys)
@@ -206,48 +204,7 @@ def should_stop_batch_after_dropdown_select(next_action: Action | None) -> bool:
async def keypress(page: Page, keys: list[str], hold: bool = False, duration: float = 0, repeat: int = 1) -> None:
- updated_keys = []
- for key in keys:
- key_lower_case = key.lower()
- if key_lower_case in ENTER_KEY_ALIASES:
- updated_keys.append("Enter")
- elif key_lower_case == "space":
- updated_keys.append(" ")
- elif key_lower_case == "ctrl":
- updated_keys.append("Control")
- elif key_lower_case == "backspace":
- updated_keys.append("Backspace")
- elif key_lower_case == "pagedown":
- updated_keys.append("PageDown")
- elif key_lower_case == "pageup":
- updated_keys.append("PageUp")
- elif key_lower_case == "tab":
- updated_keys.append("Tab")
- elif key_lower_case == "shift":
- updated_keys.append("Shift")
- elif key_lower_case in ("arrowleft", "left"):
- updated_keys.append("ArrowLeft")
- elif key_lower_case in ("arrowright", "right"):
- updated_keys.append("ArrowRight")
- elif key_lower_case in ("arrowup", "up"):
- updated_keys.append("ArrowUp")
- elif key_lower_case in ("arrowdown", "down"):
- updated_keys.append("ArrowDown")
- elif key_lower_case == "home":
- updated_keys.append("Home")
- elif key_lower_case == "end":
- updated_keys.append("End")
- elif key_lower_case == "delete":
- updated_keys.append("Delete")
- elif key_lower_case == "esc":
- updated_keys.append("Escape")
- elif key_lower_case == "alt":
- updated_keys.append("Alt")
- elif key_lower_case.startswith("f") and key_lower_case[1:].isdigit():
- # Handle function keys: f1 -> F1, f5 -> F5, etc.
- updated_keys.append(key_lower_case.upper())
- else:
- updated_keys.append(key)
+ updated_keys = [normalize_key_name(key) for key in keys]
keypress_str = "+".join(updated_keys)
n = max(1, repeat)
if hold:
diff --git a/skyvern/webeye/actions/key_names.py b/skyvern/webeye/actions/key_names.py
new file mode 100644
index 000000000..8f747e95b
--- /dev/null
+++ b/skyvern/webeye/actions/key_names.py
@@ -0,0 +1,49 @@
+ENTER_KEY_ALIASES = ("enter", "return")
+
+_KEY_ALIASES = {
+ "space": " ",
+ "ctrl": "Control",
+ "backspace": "Backspace",
+ "pagedown": "PageDown",
+ "pageup": "PageUp",
+ "tab": "Tab",
+ "shift": "Shift",
+ "arrowleft": "ArrowLeft",
+ "left": "ArrowLeft",
+ "arrowright": "ArrowRight",
+ "right": "ArrowRight",
+ "arrowup": "ArrowUp",
+ "up": "ArrowUp",
+ "arrowdown": "ArrowDown",
+ "down": "ArrowDown",
+ "home": "Home",
+ "end": "End",
+ "delete": "Delete",
+ "esc": "Escape",
+ "alt": "Alt",
+}
+
+
+def normalize_key_name(key: str) -> str:
+ """Map a loosely written key name (any case, common short forms) to the name Playwright accepts."""
+ key_lower_case = key.lower()
+ if key_lower_case in ENTER_KEY_ALIASES:
+ return "Enter"
+ if key_lower_case in _KEY_ALIASES:
+ return _KEY_ALIASES[key_lower_case]
+ if key_lower_case.startswith("f") and key_lower_case[1:].isdigit():
+ return key_lower_case.upper()
+ return key
+
+
+def split_key_chord(chord: str) -> list[str]:
+ # "+" joins a chord's keys and is also a key itself, so "Control++" is Control then "+".
+ if chord == "+":
+ return ["+"]
+ if chord.endswith("++"):
+ return chord[:-2].split("+") + ["+"]
+ return chord.split("+")
+
+
+def normalize_key_chord(chord: str) -> str:
+ return "+".join(normalize_key_name(part) for part in split_key_chord(chord))
diff --git a/tests/unit/test_mcp_browser_tools.py b/tests/unit/test_mcp_browser_tools.py
index 8aceaf4ed..0cbcbee97 100644
--- a/tests/unit/test_mcp_browser_tools.py
+++ b/tests/unit/test_mcp_browser_tools.py
@@ -849,6 +849,23 @@ async def test_skyvern_press_key_selector_only_uses_fast_default_timeout(monkeyp
press.assert_awaited_once_with("Enter", timeout=5000)
[email protected](("key", "pressed"), [("CTRL+a", "Control+a"), ("left", "ArrowLeft"), ("esc", "Escape")])
+async def test_skyvern_press_key_sends_the_driver_the_key_name_it_accepts(
+ monkeypatch: pytest.MonkeyPatch, key: str, pressed: str
+) -> None:
+ # The driver raises "Unknown key" on these spellings, which models write routinely.
+ press = AsyncMock()
+ page = SimpleNamespace(locator=MagicMock(return_value=SimpleNamespace(press=press)))
+ context = BrowserContext(mode="cloud_session", session_id="pbs_test")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, context)))
+
+ result = await mcp_browser.skyvern_press_key(key=key, selector="#field")
+
+ assert result["ok"] is True
+ press.assert_awaited_once_with(pressed, timeout=5000)
+
+
@pytest.mark.asyncio
async def test_skyvern_press_key_intent_explicit_timeout_is_honored(monkeypatch: pytest.MonkeyPatch) -> None:
press = AsyncMock()
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index f5bd2db11..ce425920f 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -7780,6 +7780,31 @@ _SEGMENTED_DATE_EXPOSED_ECHO_HTML = """
"""
+# SKY-16451: the same control with its display layer painted OVER the inputs, which are kept sub-pixel.
+# The probe reads the layer as the field's own skin, so the click is forced -- and a forced click still
+# refuses a box of at most one square pixel as "outside of the viewport".
+_SEGMENTED_DATE_SKINNED_SUBPIXEL_HTML = """
+<div role="group" aria-label="Start date" style="display:flex;width:240px;height:30px">
+ <div style="position:relative;width:60px;height:30px">
+ <input id="year" type="text" role="spinbutton" aria-label="Year"
+ style="position:absolute;left:4px;top:4px;width:1px;height:0.5px;padding:0;border:0;box-sizing:border-box">
+ <div id="year-display" aria-hidden="true" style="position:absolute;inset:0;z-index:1;background:#fff">YYYY</div>
+ </div>
+ <div style="position:relative;width:40px;height:30px">
+ <input id="month" type="text" role="spinbutton" aria-label="Month"
+ style="position:absolute;left:4px;top:4px;width:1px;height:0.5px;padding:0;border:0;box-sizing:border-box">
+ <div aria-hidden="true" style="position:absolute;inset:0;z-index:1;background:#fff">MM</div>
+ </div>
+</div>
+<script>
+ const year = document.getElementById("year");
+ year.addEventListener("input", () => {
+ document.getElementById("year-display").textContent = year.value || "YYYY";
+ });
+</script>
+"""
+
+
@_skip_no_browser
@pytest.mark.asyncio
@pytest.mark.parametrize(
@@ -7800,11 +7825,16 @@ _SEGMENTED_DATE_EXPOSED_ECHO_HTML = """
],
ids=["keys-land-in-sibling-segment", "keys-dropped", "cleared-after-typing", "trimmed"],
)
-async def test_type_into_an_unclickable_field_never_reports_a_fill_that_did_not_land(misroute: str, text: str) -> None:
+ "template", [_SEGMENTED_DATE_HTML, _SEGMENTED_DATE_SKINNED_SUBPIXEL_HTML], ids=["unclickable", "skinned-subpixel"]
+)
+async def test_type_into_an_unclickable_field_never_reports_a_fill_that_did_not_land(
+ misroute: str, text: str, template: str
+) -> None:
# Reaching the field by focus() alone proves nothing about the keystrokes. A success here would
# turn today's loud failure into a date that reads as filled and is not.
# A raised error is the loud outcome too: the tool wrapper turns it into a tool error.
- html = _SEGMENTED_DATE_HTML + f"<script>{misroute}</script>"
+ html = template + f"<script>{misroute}</script>"
async with _content_page(html) as page:
tools = build_browser_tools(_fixed_page_provider(page))
try:
@@ -7867,6 +7897,15 @@ _UNCLICKABLE_BARE_TYPEAHEAD_HTML = """
</script>
"""
+# The declared typeahead again, with its display layer over a sub-pixel input so the click is forced.
+_SKINNED_SUBPIXEL_TYPEAHEAD_HTML = _UNCLICKABLE_TYPEAHEAD_HTML.replace(
+ '<div aria-hidden="true" style="position:absolute;inset:0;background:#fff">City</div>',
+ '<div aria-hidden="true" style="position:absolute;inset:0;z-index:1;background:#fff">City</div>',
+).replace(
+ 'style="position:absolute;left:-500px;top:0;width:200px;height:30px"',
+ 'style="position:absolute;left:4px;top:4px;width:1px;height:0.5px;padding:0;border:0;box-sizing:border-box"',
+)
+
# A widget that renders BOTH: an echo of the keystrokes and a real list. Excluding the echo must not
# excuse the list.
_ECHO_SCRIPT = (
@@ -7896,6 +7935,12 @@ _ECHO_SCRIPT = (
# Rows a pickability test can read nothing off, alone and beside an echo of the keystrokes.
(_UNCLICKABLE_BARE_TYPEAHEAD_HTML, "", 0, ""),
(_UNCLICKABLE_BARE_TYPEAHEAD_HTML, "", 0, _ECHO_SCRIPT),
+ (
+ _SKINNED_SUBPIXEL_TYPEAHEAD_HTML,
+ 'role="combobox" aria-autocomplete="list" aria-controls="city-list" aria-expanded="false"',
+ 2500,
+ "",
+ ),
],
ids=[
"declared-slow-rows",
@@ -7904,6 +7949,7 @@ _ECHO_SCRIPT = (
"undeclared-rows-beside-an-echo",
"bare-div-rows",
"bare-div-rows-beside-an-echo",
+ "skinned-subpixel-declared-slow-rows",
],
)
async def test_type_into_an_unclickable_typeahead_never_reports_the_raw_query_as_filled(
@@ -7929,8 +7975,13 @@ async def test_type_into_an_unclickable_typeahead_never_reports_the_raw_query_as
@pytest.mark.asyncio
@pytest.mark.parametrize(
"html",
- [_SEGMENTED_DATE_HTML, _SEGMENTED_DATE_REPLACED_ECHO_HTML, _SEGMENTED_DATE_EXPOSED_ECHO_HTML],
- ids=["echo-mutated", "echo-replaced", "echo-exposed"],
+ [
+ _SEGMENTED_DATE_HTML,
+ _SEGMENTED_DATE_REPLACED_ECHO_HTML,
+ _SEGMENTED_DATE_EXPOSED_ECHO_HTML,
+ _SEGMENTED_DATE_SKINNED_SUBPIXEL_HTML,
+ ],
+ ids=["echo-mutated", "echo-replaced", "echo-exposed", "skinned-subpixel"],
)
async def test_type_fills_a_segment_input_the_click_cannot_reach(html: str) -> None:
async with _content_page(html) as page:
@@ -25456,3 +25507,37 @@ def test_only_an_engine_minted_marker_value_is_folded_at_a_window_head() -> None
assert reported == closing - offset + 1, (value, offset, reported)
else:
assert reported == 0, (value, offset, reported)
+
+
+_KEY_LOG_HTML = """
+<input id="field" value="text">
+<script>
+ window.__keys = [];
+ document.addEventListener("keydown", (e) => {
+ window.__keys.push((e.ctrlKey ? "Control+" : "") + (e.altKey ? "Alt+" : "") + e.key);
+ });
+</script>
+"""
+
+
+@_skip_no_browser
+async def test_press_key_accepts_the_key_names_v1_accepts() -> None:
+ # Models write CTRL, ALT, left, HOME...: v1 maps these before pressing, and an unmapped name raises
+ # "Unknown key" and burns a round.
+ cases = [
+ ("CTRL+a", "Control+a"),
+ ("ALT+r", "Alt+r"),
+ ("left", "ArrowLeft"),
+ ("HOME", "Home"),
+ ("esc", "Escape"),
+ ("TAB", "Tab"),
+ ("enter", "Enter"),
+ ]
+ async with _content_page(_KEY_LOG_HTML) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ for key, expected in cases:
+ await page.evaluate("window.__keys = []")
+ r = await _tool(tools, "press_key").handler({"key": key, "selector": "#field"})
+ assert r.status == "ok", (key, r.content)
+ assert expected in await page.evaluate("window.__keys"), key