Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-8778): click-first + re-scrape diff for shadow DOM comboboxes (#5471)
LawyZheng603b6bbb716e
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex f297843eb..4ca7fc487 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -3096,6 +3096,7 @@ async def input_or_auto_complete_input(fallback_result = await discover_and_select_from_full_dropdown(context=input_or_select_context,page=page,+ scraped_page=scraped_page,dom=dom,original_text=text,skyvern_element=skyvern_element,@@ -3116,6 +3117,7 @@ async def input_or_auto_complete_input(async def discover_and_select_from_full_dropdown(context: InputOrSelectContext,page: Page,+ scraped_page: ScrapedPage,dom: DomUtil,original_text: str,skyvern_element: SkyvernElement,@@ -3123,7 +3125,7 @@ async def discover_and_select_from_full_dropdown(task: Task,relevance_threshold: float = 0.6,) -> ActionResult | None:- """Fallback for auto-completion: clear input, press ArrowDown to reveal all options,+ """Fallback for auto-completion: clear input, click/ArrowDown to reveal all options,then ask LLM to pick the best semantic match from actual dropdown values."""if not await skyvern_element.is_visible():return None@@ -3137,27 +3139,73 @@ async def discover_and_select_from_full_dropdown(await skyvern_element.scroll_into_view()await skyvern_element.input_clear()+ # Try click first to open the dropdown (most combobox components respond to click)try:- await skyvern_element.press_key("ArrowDown")- except TimeoutError:+ await skyvern_element.get_locator().click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS)+ except Exception:LOG.info(- "Timeout pressing ArrowDown in discover fallback, continuing",+ "Click failed in discover fallback, continuing to ArrowDown",element_id=skyvern_element.get_id(),)await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=1)- incremental_element = await incremental_scraped.get_incremental_element_tree(- clean_and_remove_element_tree_factory(- task=task,- step=step,- check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],- ),+ cleanup_func = clean_and_remove_element_tree_factory(+ task=task,+ step=step,+ check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],)+ incremental_element = await incremental_scraped.get_incremental_element_tree(cleanup_func)+ # If click didn't produce options, try ArrowDown as fallbackif not incremental_element:LOG.info(- "Discover fallback: no options appeared after ArrowDown",+ "Discover fallback: no options after click, trying ArrowDown",+ element_id=skyvern_element.get_id(),+ )+ try:+ await skyvern_element.press_key("ArrowDown")+ except TimeoutError:+ LOG.info(+ "Timeout pressing ArrowDown in discover fallback, continuing",+ element_id=skyvern_element.get_id(),+ )++ await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=1)+ incremental_element = await incremental_scraped.get_incremental_element_tree(cleanup_func)++ # If incremental detection failed (e.g. options in a different shadow root),+ # try a full page re-scrape diff as last resort+ if not incremental_element:+ LOG.info(+ "Discover fallback: no options from incremental detection, trying re-scrape diff",+ element_id=skyvern_element.get_id(),+ )+ scraped_page_after = await scraped_page.generate_scraped_page_without_screenshots()+ new_element_ids_from_rescrape = list(+ set(scraped_page_after.id_to_css_dict.keys()) - set(scraped_page.id_to_css_dict.keys())+ )+ if new_element_ids_from_rescrape:+ # Feed re-scrape results back into incremental_element so the unified+ # auto-completion-choose-option path below handles them (best-effort,+ # with relevance_threshold). This avoids select_from_emerging_elements+ # which uses the more aggressive custom-select prompt.+ rescrape_elements = [+ scraped_page_after.id_to_element_dict[eid]+ for eid in new_element_ids_from_rescrape+ if eid in scraped_page_after.id_to_element_dict+ ]+ if rescrape_elements:+ LOG.info(+ "Discover fallback: re-scrape diff found new elements",+ new_element_count=len(rescrape_elements),+ )+ incremental_element = rescrape_elements+ incremental_scraped.id_to_element_dict.update(scraped_page_after.id_to_element_dict)++ if not incremental_element:+ LOG.info(+ "Discover fallback: no options found after all attempts",element_id=skyvern_element.get_id(),)return None@@ -3200,28 +3248,44 @@ async def discover_and_select_from_full_dropdown()return None+ discovered_value = json_response.get("value", "")LOG.info(- "Discover fallback: found suitable option",+ "Discover fallback: found suitable option, typing discovered value to trigger auto-completion",element_id=element_id,relevance_float=relevance_float,+ discovered_value=discovered_value,)- locator = current_frame.locator(f'[{SKYVERN_ID_ATTR}="{element_id}"]')- if await locator.count() == 0:- LOG.warning(- "Discover fallback: selected element not found in DOM",- element_id=element_id,- )+ if not discovered_value:+ # FIXME: when element_id is valid and the dropdown is still open (incremental path),+ # we could try clicking the element directly instead of requiring the value text.+ # Currently this only affects the re-scrape path where the dropdown is closed.return None- selected_element = SkyvernElement(- locator=locator,- frame=current_frame,- static_element=incremental_scraped.id_to_element_dict.get(element_id, {}),- )- await selected_element.scroll_into_view()- await selected_element.click(page=page)- return ActionSuccess()+ # Instead of clicking the option directly (dropdown may have closed during re-scrape),+ # input the discovered value into the combobox. Since it's an exact match, the combobox's+ # filter will show it as the only option. Then find and click it directly via Playwright.+ await skyvern_element.input_clear()+ await skyvern_element.press_fill(discovered_value)+ await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=1)++ # Select the first matching option via keyboard: ArrowDown highlights it, Enter confirms.+ # This avoids needing to locate the option element in shadow DOM.+ try:+ await skyvern_element.press_key("ArrowDown")+ await skyvern_element.press_key("Enter")+ LOG.info(+ "Discover fallback: selected option via keyboard",+ discovered_value=discovered_value,+ )+ return ActionSuccess()+ except Exception:+ LOG.info(+ "Discover fallback: keyboard selection failed",+ exc_info=True,+ discovered_value=discovered_value,+ )+ return Noneexcept Exception:LOG.warning(diff --git a/tests/unit/test_auto_completion_discover_fallback.py b/tests/unit/test_auto_completion_discover_fallback.pyindex 3839d6ab4..bc7a8bc1e 100644--- a/tests/unit/test_auto_completion_discover_fallback.py+++ b/tests/unit/test_auto_completion_discover_fallback.py@@ -13,7 +13,6 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytest-from skyvern.constants import SKYVERN_ID_ATTRfrom skyvern.forge.sdk.models import StepStatusfrom skyvern.webeye.actions.actions import InputOrSelectContextfrom skyvern.webeye.actions.handler import (@@ -68,6 +67,10 @@ def _mock_skyvern_element(frame: MagicMock | None = None) -> MagicMock:el.is_visible = AsyncMock(return_value=True)el.get_element_handler = AsyncMock(return_value=MagicMock())el.click = AsyncMock()+ # get_locator().click() needs to be async for the discover fallback's click call+ mock_locator = MagicMock()+ mock_locator.click = AsyncMock()+ el.get_locator.return_value = mock_locatorreturn el@@ -79,10 +82,12 @@ def _mock_frame(locator_count: int = 1) -> MagicMock:locator.element_handle = AsyncMock(return_value=MagicMock())locator.bounding_box = AsyncMock(return_value={"x": 0, "y": 0, "width": 100, "height": 30})locator.scroll_into_view_if_needed = AsyncMock()- # SkyvernElement.is_visible() needs locator.is_visible()locator.is_visible = AsyncMock(return_value=True)frame.locator.return_value = locatorframe.evaluate = AsyncMock(return_value=None)+ # locator.first.click needs to be async for option clicking+ locator.first = MagicMock()+ locator.first.click = AsyncMock()return frame@@ -111,7 +116,7 @@ def _mock_selected_element() -> MagicMock:@pytest.mark.asyncioasync def test_discover_fallback_succeeds_when_options_appear() -> None:- """ArrowDown reveals options, LLM picks a match with relevance >= 0.6 → ActionSuccess."""+ """Click reveals options, LLM picks a match, then types discovered value and clicks matched option."""frame = _mock_frame(locator_count=1)skyvern_el = _mock_skyvern_element(frame)inc_scrape = _mock_incremental_scrape(DROPDOWN_OPTIONS)@@ -120,11 +125,10 @@ async def test_discover_fallback_succeeds_when_options_appear() -> None:"auto_completion_attempt": True,"relevance_float": 0.9,"id": "OPT3",+ "value": "I do not wish to disclose","reasoning": "'I do not wish to disclose' matches 'Decline to Self Identify'",}- mock_selected_el = _mock_selected_element()-with (patch("skyvern.webeye.actions.handler.SkyvernFrame.create_instance",@@ -137,7 +141,6 @@ async def test_discover_fallback_succeeds_when_options_appear() -> None:patch("skyvern.webeye.actions.handler.app") as mock_app,patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- patch("skyvern.webeye.actions.handler.SkyvernElement", return_value=mock_selected_el),):mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)mock_prompt.load_prompt.return_value = "mocked prompt"@@ -146,6 +149,7 @@ async def test_discover_fallback_succeeds_when_options_appear() -> None:result = await discover_and_select_from_full_dropdown(context=_make_context(),page=MagicMock(),+ scraped_page=MagicMock(),dom=MagicMock(),original_text="Decline to Self Identify",skyvern_element=skyvern_el,@@ -155,10 +159,9 @@ async def test_discover_fallback_succeeds_when_options_appear() -> None:assert isinstance(result, ActionSuccess)skyvern_el.input_clear.assert_awaited()- skyvern_el.press_key.assert_awaited_with("ArrowDown")- frame.locator.assert_called_with(f'[{SKYVERN_ID_ATTR}="OPT3"]')- mock_selected_el.scroll_into_view.assert_awaited()- mock_selected_el.click.assert_awaited()+ skyvern_el.get_locator().click.assert_awaited()+ # Types the discovered value, then finds and clicks matched option via Playwright locator+ skyvern_el.press_fill.assert_awaited_with("I do not wish to disclose")inc_scrape.stop_listen_dom_increment.assert_awaited()@@ -184,6 +187,7 @@ async def test_discover_fallback_returns_none_when_no_options() -> None:result = await discover_and_select_from_full_dropdown(context=_make_context(),page=MagicMock(),+ scraped_page=MagicMock(),dom=MagicMock(),original_text="Decline to Self Identify",skyvern_element=skyvern_el,@@ -229,6 +233,7 @@ async def test_discover_fallback_returns_none_when_relevance_too_low() -> None:result = await discover_and_select_from_full_dropdown(context=_make_context(),page=MagicMock(),+ scraped_page=MagicMock(),dom=MagicMock(),original_text="Decline to Self Identify",skyvern_element=skyvern_el,@@ -272,6 +277,7 @@ async def test_discover_fallback_returns_none_when_llm_returns_no_id() -> None:result = await discover_and_select_from_full_dropdown(context=_make_context(),page=MagicMock(),+ scraped_page=MagicMock(),dom=MagicMock(),original_text="Decline to Self Identify",skyvern_element=skyvern_el,@@ -291,6 +297,7 @@ async def test_discover_fallback_returns_none_when_element_invisible() -> None:result = await discover_and_select_from_full_dropdown(context=_make_context(),page=MagicMock(),+ scraped_page=MagicMock(),dom=MagicMock(),original_text="Decline to Self Identify",skyvern_element=skyvern_el,@@ -335,6 +342,7 @@ async def test_discover_fallback_returns_none_when_element_not_in_dom() -> None:result = await discover_and_select_from_full_dropdown(context=_make_context(),page=MagicMock(),+ scraped_page=MagicMock(),dom=MagicMock(),original_text="Decline to Self Identify",skyvern_element=skyvern_el,@@ -347,23 +355,37 @@ async def test_discover_fallback_returns_none_when_element_not_in_dom() -> None:@pytest.mark.asyncioasync def test_discover_fallback_handles_arrow_down_timeout() -> None:- """If ArrowDown times out, fallback should continue (not crash)."""+ """If click yields no options and ArrowDown times out, fallback succeeds when options found after ArrowDown."""from playwright.async_api import TimeoutError as PlaywrightTimeoutframe = _mock_frame(locator_count=1)skyvern_el = _mock_skyvern_element(frame)- skyvern_el.press_key = AsyncMock(side_effect=PlaywrightTimeout("timeout"))- inc_scrape = _mock_incremental_scrape(DROPDOWN_OPTIONS)+ # ArrowDown for opening dropdown times out, but subsequent keyboard presses succeed+ call_count = 0++ async def press_key_side_effect(key, **kwargs):+ nonlocal call_count+ call_count += 1+ if call_count == 1:+ raise PlaywrightTimeout("timeout")++ skyvern_el.press_key = AsyncMock(side_effect=press_key_side_effect)+ # Click yields no options on first call, ArrowDown (despite timeout) yields options on second+ inc_scrape = _mock_incremental_scrape([])+ # 3 calls: click→empty, ArrowDown→options, press_fill→options (for select_one_element_by_value)+ inc_scrape.get_incremental_element_tree = AsyncMock(+ side_effect=[[], copy.deepcopy(DROPDOWN_OPTIONS), copy.deepcopy(DROPDOWN_OPTIONS)]+ )+ inc_scrape.id_to_element_dict = {e["id"]: e for e in DROPDOWN_OPTIONS}llm_response = {"auto_completion_attempt": True,"relevance_float": 0.9,"id": "OPT3",+ "value": "I do not wish to disclose","reasoning": "Match found",}- mock_selected_el = _mock_selected_element()-with (patch("skyvern.webeye.actions.handler.SkyvernFrame.create_instance",@@ -376,7 +398,6 @@ async def test_discover_fallback_handles_arrow_down_timeout() -> None:patch("skyvern.webeye.actions.handler.app") as mock_app,patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,- patch("skyvern.webeye.actions.handler.SkyvernElement", return_value=mock_selected_el),):mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)mock_prompt.load_prompt.return_value = "mocked prompt"@@ -385,6 +406,7 @@ async def test_discover_fallback_handles_arrow_down_timeout() -> None:result = await discover_and_select_from_full_dropdown(context=_make_context(),page=MagicMock(),+ scraped_page=MagicMock(),dom=MagicMock(),original_text="Decline to Self Identify",skyvern_element=skyvern_el,@@ -396,6 +418,126 @@ async def test_discover_fallback_handles_arrow_down_timeout() -> None:assert isinstance(result, ActionSuccess)+# ---------------------------------------------------------------------------+# Tests for re-scrape diff fallback (Strategy 3)+# ---------------------------------------------------------------------------+++async def test_discover_fallback_rescrape_diff_succeeds() -> None:+ """When click and ArrowDown both return empty, re-scrape diff finds new elements and keyboard selects."""+ frame = _mock_frame(locator_count=1)+ skyvern_el = _mock_skyvern_element(frame)+ # Both click and ArrowDown produce no incremental elements+ inc_scrape = _mock_incremental_scrape([])+ inc_scrape.get_incremental_element_tree = AsyncMock(+ side_effect=[[], [], copy.deepcopy(DROPDOWN_OPTIONS)] # click→empty, ArrowDown→empty, press_fill→options+ )+ inc_scrape.id_to_element_dict = {}++ llm_response = {+ "auto_completion_attempt": True,+ "relevance_float": 0.95,+ "id": "OPT3",+ "value": "I do not wish to disclose",+ "reasoning": "Best semantic match",+ }++ # Mock scraped_page for re-scrape diff+ mock_scraped_page = MagicMock()+ mock_scraped_page.id_to_css_dict = {"EXISTING1": "css1"} # before scrape++ # After re-scrape: has new elements+ mock_scraped_after = MagicMock()+ mock_scraped_after.id_to_css_dict = {+ "EXISTING1": "css1",+ "OPT1": "css2",+ "OPT2": "css3",+ "OPT3": "css4",+ }+ mock_scraped_after.id_to_element_dict = {e["id"]: e for e in DROPDOWN_OPTIONS}+ mock_scraped_page.generate_scraped_page_without_screenshots = AsyncMock(return_value=mock_scraped_after)++ with (+ patch(+ "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",+ new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),+ ),+ patch(+ "skyvern.webeye.actions.handler.IncrementalScrapePage",+ return_value=inc_scrape,+ ),+ patch("skyvern.webeye.actions.handler.app") as mock_app,+ patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,+ patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,+ ):+ mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)+ mock_prompt.load_prompt.return_value = "mocked prompt"+ mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)++ result = await discover_and_select_from_full_dropdown(+ context=_make_context(),+ page=MagicMock(),+ scraped_page=mock_scraped_page,+ dom=MagicMock(),+ original_text="Decline to Self Identify",+ skyvern_element=skyvern_el,+ step=_STEP,+ task=_TASK,+ )++ assert isinstance(result, ActionSuccess)+ # Verify re-scrape was triggered+ mock_scraped_page.generate_scraped_page_without_screenshots.assert_awaited_once()+ # Verify discovered value was typed + keyboard selection+ skyvern_el.press_fill.assert_awaited_with("I do not wish to disclose")+ # ArrowDown + Enter for keyboard selection+ assert skyvern_el.press_key.call_count >= 2+++async def test_discover_fallback_rescrape_diff_no_new_elements() -> None:+ """When re-scrape diff finds no new elements, returns None."""+ skyvern_el = _mock_skyvern_element()+ inc_scrape = _mock_incremental_scrape([])++ # Mock scraped_page where re-scrape returns same elements (no diff)+ mock_scraped_page = MagicMock()+ mock_scraped_page.id_to_css_dict = {"EXISTING1": "css1"}+ mock_scraped_after = MagicMock()+ mock_scraped_after.id_to_css_dict = {"EXISTING1": "css1"} # same as before+ mock_scraped_page.generate_scraped_page_without_screenshots = AsyncMock(return_value=mock_scraped_after)++ with (+ patch(+ "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",+ new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),+ ),+ patch(+ "skyvern.webeye.actions.handler.IncrementalScrapePage",+ return_value=inc_scrape,+ ),+ patch("skyvern.webeye.actions.handler.app") as mock_app,+ ):+ mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock()++ result = await discover_and_select_from_full_dropdown(+ context=_make_context(),+ page=MagicMock(),+ scraped_page=mock_scraped_page,+ dom=MagicMock(),+ original_text="Decline to Self Identify",+ skyvern_element=skyvern_el,+ step=_STEP,+ task=_TASK,+ )++ assert result is None+ mock_scraped_page.generate_scraped_page_without_screenshots.assert_awaited_once()+ # LLM should not be called since no new elements+ mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_not_called()++# ---------------------------------------------------------------------------# Tests for integration with input_or_auto_complete_input# ---------------------------------------------------------------------------