Skip to content

Automate browser based workflows with AI

fix(SKY-8778): click-first + re-scrape diff for shadow DOM comboboxes (#5471)

LawyZheng603b6bbb716e
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index f297843eb..4ca7fc487 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -3096,6 +3096,7 @@ async def input_or_auto_complete_input(
fallback_result = await discover_and_select_from_full_dropdown(
context=input_or_select_context,
page=page,
+ scraped_page=scraped_page,
dom=dom,
original_text=text,
skyvern_element=skyvern_element,
@@ -3116,6 +3117,7 @@ async def input_or_auto_complete_input(
async def discover_and_select_from_full_dropdown(
context: InputOrSelectContext,
page: Page,
+ scraped_page: ScrapedPage,
dom: DomUtil,
original_text: str,
skyvern_element: SkyvernElement,
@@ -3123,7 +3125,7 @@ async def discover_and_select_from_full_dropdown(
task: Task,
relevance_threshold: float = 0.6,
) -> ActionResult | None:
- """Fallback for auto-completion: clear input, press ArrowDown to reveal all options,
+ """Fallback for auto-completion: clear input, click/ArrowDown to reveal all options,
then ask LLM to pick the best semantic match from actual dropdown values."""
if not await skyvern_element.is_visible():
return None
@@ -3137,27 +3139,73 @@ async def discover_and_select_from_full_dropdown(
await skyvern_element.scroll_into_view()
await skyvern_element.input_clear()
+ # Try click first to open the dropdown (most combobox components respond to click)
try:
- await skyvern_element.press_key("ArrowDown")
- except TimeoutError:
+ await skyvern_element.get_locator().click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS)
+ except Exception:
LOG.info(
- "Timeout pressing ArrowDown in discover fallback, continuing",
+ "Click failed in discover fallback, continuing to ArrowDown",
element_id=skyvern_element.get_id(),
)
await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=1)
- incremental_element = await incremental_scraped.get_incremental_element_tree(
- clean_and_remove_element_tree_factory(
- task=task,
- step=step,
- check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],
- ),
+ cleanup_func = clean_and_remove_element_tree_factory(
+ task=task,
+ step=step,
+ check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],
)
+ incremental_element = await incremental_scraped.get_incremental_element_tree(cleanup_func)
+ # If click didn't produce options, try ArrowDown as fallback
if not incremental_element:
LOG.info(
- "Discover fallback: no options appeared after ArrowDown",
+ "Discover fallback: no options after click, trying ArrowDown",
+ element_id=skyvern_element.get_id(),
+ )
+ try:
+ await skyvern_element.press_key("ArrowDown")
+ except TimeoutError:
+ LOG.info(
+ "Timeout pressing ArrowDown in discover fallback, continuing",
+ element_id=skyvern_element.get_id(),
+ )
+
+ await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=1)
+ incremental_element = await incremental_scraped.get_incremental_element_tree(cleanup_func)
+
+ # If incremental detection failed (e.g. options in a different shadow root),
+ # try a full page re-scrape diff as last resort
+ if not incremental_element:
+ LOG.info(
+ "Discover fallback: no options from incremental detection, trying re-scrape diff",
+ element_id=skyvern_element.get_id(),
+ )
+ scraped_page_after = await scraped_page.generate_scraped_page_without_screenshots()
+ new_element_ids_from_rescrape = list(
+ set(scraped_page_after.id_to_css_dict.keys()) - set(scraped_page.id_to_css_dict.keys())
+ )
+ if new_element_ids_from_rescrape:
+ # Feed re-scrape results back into incremental_element so the unified
+ # auto-completion-choose-option path below handles them (best-effort,
+ # with relevance_threshold). This avoids select_from_emerging_elements
+ # which uses the more aggressive custom-select prompt.
+ rescrape_elements = [
+ scraped_page_after.id_to_element_dict[eid]
+ for eid in new_element_ids_from_rescrape
+ if eid in scraped_page_after.id_to_element_dict
+ ]
+ if rescrape_elements:
+ LOG.info(
+ "Discover fallback: re-scrape diff found new elements",
+ new_element_count=len(rescrape_elements),
+ )
+ incremental_element = rescrape_elements
+ incremental_scraped.id_to_element_dict.update(scraped_page_after.id_to_element_dict)
+
+ if not incremental_element:
+ LOG.info(
+ "Discover fallback: no options found after all attempts",
element_id=skyvern_element.get_id(),
)
return None
@@ -3200,28 +3248,44 @@ async def discover_and_select_from_full_dropdown(
)
return None
+ discovered_value = json_response.get("value", "")
LOG.info(
- "Discover fallback: found suitable option",
+ "Discover fallback: found suitable option, typing discovered value to trigger auto-completion",
element_id=element_id,
relevance_float=relevance_float,
+ discovered_value=discovered_value,
)
- locator = current_frame.locator(f'[{SKYVERN_ID_ATTR}="{element_id}"]')
- if await locator.count() == 0:
- LOG.warning(
- "Discover fallback: selected element not found in DOM",
- element_id=element_id,
- )
+ if not discovered_value:
+ # FIXME: when element_id is valid and the dropdown is still open (incremental path),
+ # we could try clicking the element directly instead of requiring the value text.
+ # Currently this only affects the re-scrape path where the dropdown is closed.
return None
- selected_element = SkyvernElement(
- locator=locator,
- frame=current_frame,
- static_element=incremental_scraped.id_to_element_dict.get(element_id, {}),
- )
- await selected_element.scroll_into_view()
- await selected_element.click(page=page)
- return ActionSuccess()
+ # Instead of clicking the option directly (dropdown may have closed during re-scrape),
+ # input the discovered value into the combobox. Since it's an exact match, the combobox's
+ # filter will show it as the only option. Then find and click it directly via Playwright.
+ await skyvern_element.input_clear()
+ await skyvern_element.press_fill(discovered_value)
+ await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=1)
+
+ # Select the first matching option via keyboard: ArrowDown highlights it, Enter confirms.
+ # This avoids needing to locate the option element in shadow DOM.
+ try:
+ await skyvern_element.press_key("ArrowDown")
+ await skyvern_element.press_key("Enter")
+ LOG.info(
+ "Discover fallback: selected option via keyboard",
+ discovered_value=discovered_value,
+ )
+ return ActionSuccess()
+ except Exception:
+ LOG.info(
+ "Discover fallback: keyboard selection failed",
+ exc_info=True,
+ discovered_value=discovered_value,
+ )
+ return None
except Exception:
LOG.warning(
diff --git a/tests/unit/test_auto_completion_discover_fallback.py b/tests/unit/test_auto_completion_discover_fallback.py
index 3839d6ab4..bc7a8bc1e 100644
--- a/tests/unit/test_auto_completion_discover_fallback.py
+++ b/tests/unit/test_auto_completion_discover_fallback.py
@@ -13,7 +13,6 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
-from skyvern.constants import SKYVERN_ID_ATTR
from skyvern.forge.sdk.models import StepStatus
from skyvern.webeye.actions.actions import InputOrSelectContext
from skyvern.webeye.actions.handler import (
@@ -68,6 +67,10 @@ def _mock_skyvern_element(frame: MagicMock | None = None) -> MagicMock:
el.is_visible = AsyncMock(return_value=True)
el.get_element_handler = AsyncMock(return_value=MagicMock())
el.click = AsyncMock()
+ # get_locator().click() needs to be async for the discover fallback's click call
+ mock_locator = MagicMock()
+ mock_locator.click = AsyncMock()
+ el.get_locator.return_value = mock_locator
return el
@@ -79,10 +82,12 @@ def _mock_frame(locator_count: int = 1) -> MagicMock:
locator.element_handle = AsyncMock(return_value=MagicMock())
locator.bounding_box = AsyncMock(return_value={"x": 0, "y": 0, "width": 100, "height": 30})
locator.scroll_into_view_if_needed = AsyncMock()
- # SkyvernElement.is_visible() needs locator.is_visible()
locator.is_visible = AsyncMock(return_value=True)
frame.locator.return_value = locator
frame.evaluate = AsyncMock(return_value=None)
+ # locator.first.click needs to be async for option clicking
+ locator.first = MagicMock()
+ locator.first.click = AsyncMock()
return frame
@@ -111,7 +116,7 @@ def _mock_selected_element() -> MagicMock:
@pytest.mark.asyncio
async def test_discover_fallback_succeeds_when_options_appear() -> None:
- """ArrowDown reveals options, LLM picks a match with relevance >= 0.6 → ActionSuccess."""
+ """Click reveals options, LLM picks a match, then types discovered value and clicks matched option."""
frame = _mock_frame(locator_count=1)
skyvern_el = _mock_skyvern_element(frame)
inc_scrape = _mock_incremental_scrape(DROPDOWN_OPTIONS)
@@ -120,11 +125,10 @@ async def test_discover_fallback_succeeds_when_options_appear() -> None:
"auto_completion_attempt": True,
"relevance_float": 0.9,
"id": "OPT3",
+ "value": "I do not wish to disclose",
"reasoning": "'I do not wish to disclose' matches 'Decline to Self Identify'",
}
- mock_selected_el = _mock_selected_element()
-
with (
patch(
"skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
@@ -137,7 +141,6 @@ async def test_discover_fallback_succeeds_when_options_appear() -> None:
patch("skyvern.webeye.actions.handler.app") as mock_app,
patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- patch("skyvern.webeye.actions.handler.SkyvernElement", return_value=mock_selected_el),
):
mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)
mock_prompt.load_prompt.return_value = "mocked prompt"
@@ -146,6 +149,7 @@ async def test_discover_fallback_succeeds_when_options_appear() -> None:
result = await discover_and_select_from_full_dropdown(
context=_make_context(),
page=MagicMock(),
+ scraped_page=MagicMock(),
dom=MagicMock(),
original_text="Decline to Self Identify",
skyvern_element=skyvern_el,
@@ -155,10 +159,9 @@ async def test_discover_fallback_succeeds_when_options_appear() -> None:
assert isinstance(result, ActionSuccess)
skyvern_el.input_clear.assert_awaited()
- skyvern_el.press_key.assert_awaited_with("ArrowDown")
- frame.locator.assert_called_with(f'[{SKYVERN_ID_ATTR}="OPT3"]')
- mock_selected_el.scroll_into_view.assert_awaited()
- mock_selected_el.click.assert_awaited()
+ skyvern_el.get_locator().click.assert_awaited()
+ # Types the discovered value, then finds and clicks matched option via Playwright locator
+ skyvern_el.press_fill.assert_awaited_with("I do not wish to disclose")
inc_scrape.stop_listen_dom_increment.assert_awaited()
@@ -184,6 +187,7 @@ async def test_discover_fallback_returns_none_when_no_options() -> None:
result = await discover_and_select_from_full_dropdown(
context=_make_context(),
page=MagicMock(),
+ scraped_page=MagicMock(),
dom=MagicMock(),
original_text="Decline to Self Identify",
skyvern_element=skyvern_el,
@@ -229,6 +233,7 @@ async def test_discover_fallback_returns_none_when_relevance_too_low() -> None:
result = await discover_and_select_from_full_dropdown(
context=_make_context(),
page=MagicMock(),
+ scraped_page=MagicMock(),
dom=MagicMock(),
original_text="Decline to Self Identify",
skyvern_element=skyvern_el,
@@ -272,6 +277,7 @@ async def test_discover_fallback_returns_none_when_llm_returns_no_id() -> None:
result = await discover_and_select_from_full_dropdown(
context=_make_context(),
page=MagicMock(),
+ scraped_page=MagicMock(),
dom=MagicMock(),
original_text="Decline to Self Identify",
skyvern_element=skyvern_el,
@@ -291,6 +297,7 @@ async def test_discover_fallback_returns_none_when_element_invisible() -> None:
result = await discover_and_select_from_full_dropdown(
context=_make_context(),
page=MagicMock(),
+ scraped_page=MagicMock(),
dom=MagicMock(),
original_text="Decline to Self Identify",
skyvern_element=skyvern_el,
@@ -335,6 +342,7 @@ async def test_discover_fallback_returns_none_when_element_not_in_dom() -> None:
result = await discover_and_select_from_full_dropdown(
context=_make_context(),
page=MagicMock(),
+ scraped_page=MagicMock(),
dom=MagicMock(),
original_text="Decline to Self Identify",
skyvern_element=skyvern_el,
@@ -347,23 +355,37 @@ async def test_discover_fallback_returns_none_when_element_not_in_dom() -> None:
@pytest.mark.asyncio
async def test_discover_fallback_handles_arrow_down_timeout() -> None:
- """If ArrowDown times out, fallback should continue (not crash)."""
+ """If click yields no options and ArrowDown times out, fallback succeeds when options found after ArrowDown."""
from playwright.async_api import TimeoutError as PlaywrightTimeout
frame = _mock_frame(locator_count=1)
skyvern_el = _mock_skyvern_element(frame)
- skyvern_el.press_key = AsyncMock(side_effect=PlaywrightTimeout("timeout"))
- inc_scrape = _mock_incremental_scrape(DROPDOWN_OPTIONS)
+ # ArrowDown for opening dropdown times out, but subsequent keyboard presses succeed
+ call_count = 0
+
+ async def press_key_side_effect(key, **kwargs):
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ raise PlaywrightTimeout("timeout")
+
+ skyvern_el.press_key = AsyncMock(side_effect=press_key_side_effect)
+ # Click yields no options on first call, ArrowDown (despite timeout) yields options on second
+ inc_scrape = _mock_incremental_scrape([])
+ # 3 calls: click→empty, ArrowDown→options, press_fill→options (for select_one_element_by_value)
+ inc_scrape.get_incremental_element_tree = AsyncMock(
+ side_effect=[[], copy.deepcopy(DROPDOWN_OPTIONS), copy.deepcopy(DROPDOWN_OPTIONS)]
+ )
+ inc_scrape.id_to_element_dict = {e["id"]: e for e in DROPDOWN_OPTIONS}
llm_response = {
"auto_completion_attempt": True,
"relevance_float": 0.9,
"id": "OPT3",
+ "value": "I do not wish to disclose",
"reasoning": "Match found",
}
- mock_selected_el = _mock_selected_element()
-
with (
patch(
"skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
@@ -376,7 +398,6 @@ async def test_discover_fallback_handles_arrow_down_timeout() -> None:
patch("skyvern.webeye.actions.handler.app") as mock_app,
patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- patch("skyvern.webeye.actions.handler.SkyvernElement", return_value=mock_selected_el),
):
mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)
mock_prompt.load_prompt.return_value = "mocked prompt"
@@ -385,6 +406,7 @@ async def test_discover_fallback_handles_arrow_down_timeout() -> None:
result = await discover_and_select_from_full_dropdown(
context=_make_context(),
page=MagicMock(),
+ scraped_page=MagicMock(),
dom=MagicMock(),
original_text="Decline to Self Identify",
skyvern_element=skyvern_el,
@@ -396,6 +418,126 @@ async def test_discover_fallback_handles_arrow_down_timeout() -> None:
assert isinstance(result, ActionSuccess)
+# ---------------------------------------------------------------------------
+# Tests for re-scrape diff fallback (Strategy 3)
+# ---------------------------------------------------------------------------
+
+
+async def test_discover_fallback_rescrape_diff_succeeds() -> None:
+ """When click and ArrowDown both return empty, re-scrape diff finds new elements and keyboard selects."""
+ frame = _mock_frame(locator_count=1)
+ skyvern_el = _mock_skyvern_element(frame)
+ # Both click and ArrowDown produce no incremental elements
+ inc_scrape = _mock_incremental_scrape([])
+ inc_scrape.get_incremental_element_tree = AsyncMock(
+ side_effect=[[], [], copy.deepcopy(DROPDOWN_OPTIONS)] # click→empty, ArrowDown→empty, press_fill→options
+ )
+ inc_scrape.id_to_element_dict = {}
+
+ llm_response = {
+ "auto_completion_attempt": True,
+ "relevance_float": 0.95,
+ "id": "OPT3",
+ "value": "I do not wish to disclose",
+ "reasoning": "Best semantic match",
+ }
+
+ # Mock scraped_page for re-scrape diff
+ mock_scraped_page = MagicMock()
+ mock_scraped_page.id_to_css_dict = {"EXISTING1": "css1"} # before scrape
+
+ # After re-scrape: has new elements
+ mock_scraped_after = MagicMock()
+ mock_scraped_after.id_to_css_dict = {
+ "EXISTING1": "css1",
+ "OPT1": "css2",
+ "OPT2": "css3",
+ "OPT3": "css4",
+ }
+ mock_scraped_after.id_to_element_dict = {e["id"]: e for e in DROPDOWN_OPTIONS}
+ mock_scraped_page.generate_scraped_page_without_screenshots = AsyncMock(return_value=mock_scraped_after)
+
+ with (
+ patch(
+ "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
+ new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),
+ ),
+ patch(
+ "skyvern.webeye.actions.handler.IncrementalScrapePage",
+ return_value=inc_scrape,
+ ),
+ patch("skyvern.webeye.actions.handler.app") as mock_app,
+ patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
+ patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
+ ):
+ mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)
+ mock_prompt.load_prompt.return_value = "mocked prompt"
+ mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
+
+ result = await discover_and_select_from_full_dropdown(
+ context=_make_context(),
+ page=MagicMock(),
+ scraped_page=mock_scraped_page,
+ dom=MagicMock(),
+ original_text="Decline to Self Identify",
+ skyvern_element=skyvern_el,
+ step=_STEP,
+ task=_TASK,
+ )
+
+ assert isinstance(result, ActionSuccess)
+ # Verify re-scrape was triggered
+ mock_scraped_page.generate_scraped_page_without_screenshots.assert_awaited_once()
+ # Verify discovered value was typed + keyboard selection
+ skyvern_el.press_fill.assert_awaited_with("I do not wish to disclose")
+ # ArrowDown + Enter for keyboard selection
+ assert skyvern_el.press_key.call_count >= 2
+
+
+async def test_discover_fallback_rescrape_diff_no_new_elements() -> None:
+ """When re-scrape diff finds no new elements, returns None."""
+ skyvern_el = _mock_skyvern_element()
+ inc_scrape = _mock_incremental_scrape([])
+
+ # Mock scraped_page where re-scrape returns same elements (no diff)
+ mock_scraped_page = MagicMock()
+ mock_scraped_page.id_to_css_dict = {"EXISTING1": "css1"}
+ mock_scraped_after = MagicMock()
+ mock_scraped_after.id_to_css_dict = {"EXISTING1": "css1"} # same as before
+ mock_scraped_page.generate_scraped_page_without_screenshots = AsyncMock(return_value=mock_scraped_after)
+
+ with (
+ patch(
+ "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
+ new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),
+ ),
+ patch(
+ "skyvern.webeye.actions.handler.IncrementalScrapePage",
+ return_value=inc_scrape,
+ ),
+ patch("skyvern.webeye.actions.handler.app") as mock_app,
+ ):
+ mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock()
+
+ result = await discover_and_select_from_full_dropdown(
+ context=_make_context(),
+ page=MagicMock(),
+ scraped_page=mock_scraped_page,
+ dom=MagicMock(),
+ original_text="Decline to Self Identify",
+ skyvern_element=skyvern_el,
+ step=_STEP,
+ task=_TASK,
+ )
+
+ assert result is None
+ mock_scraped_page.generate_scraped_page_without_screenshots.assert_awaited_once()
+ # LLM should not be called since no new elements
+ mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_not_called()
+
+
# ---------------------------------------------------------------------------
# Tests for integration with input_or_auto_complete_input
# ---------------------------------------------------------------------------