Skip to content

Automate browser based workflows with AI

fix dropdown menu detect bug (#874)

LawyZhengc27c2a55f98a
diff --git a/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 b/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2
index ea3a7da94..12b8d9a2d 100644
--- a/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2
+++ b/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2
@@ -3,6 +3,7 @@ There is a screenshot from a part of a web HTML page. Help me confirm if it is a
An open dropdown menu can be defined as:
- At least one option is visible in the screenshot.
- Do not consider it an open dropdown menu if the only visible option displays a message like "No results" or "No match".
+ - Do not consider it an open dropdown menu if the only visible element displays a placeholder like "Please select", "-", or "Select...".
MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index f3979815a..4c4862585 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -417,12 +417,12 @@ async def handle_input_text_action(
# check if it's selectable
if skyvern_element.get_tag_name() == InteractiveElement.INPUT and not await skyvern_element.is_raw_input():
await skyvern_element.scroll_into_view()
- select_action = SelectOptionAction(
- reasoning=action.reasoning,
- element_id=skyvern_element.get_id(),
- option=SelectOption(label=text),
- )
if skyvern_element.get_selectable():
+ select_action = SelectOptionAction(
+ reasoning=action.reasoning,
+ element_id=skyvern_element.get_id(),
+ option=SelectOption(label=text),
+ )
LOG.info(
"Input element is selectable, doing select actions",
task_id=task.task_id,
@@ -434,8 +434,7 @@ async def handle_input_text_action(
# press arrowdown to watch if there's any options popping up
await incremental_scraped.start_listen_dom_increment()
- await skyvern_element.get_locator().focus(timeout=timeout)
- await skyvern_element.get_locator().press("ArrowDown", timeout=timeout)
+ await skyvern_element.press_key("ArrowDown")
await asyncio.sleep(5)
incremental_element = await incremental_scraped.get_incremental_element_tree(
@@ -726,46 +725,25 @@ async def handle_select_option_action(
timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame())
incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame)
- dropdown_element: SkyvernElement | None = None
is_open = False
suggested_value: str | None = None
results: list[ActionResult] = []
try:
await incremental_scraped.start_listen_dom_increment()
- await skyvern_element.focus()
- # wait 2s for options to load
- await asyncio.sleep(2)
+ await skyvern_element.scroll_into_view()
- incremental_element = await incremental_scraped.get_incremental_element_tree(
- clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
- )
- if len(incremental_element) > 0:
+ try:
+ await skyvern_element.get_locator().click(timeout=timeout)
+ except Exception:
LOG.info(
- "Focus trigger DOM element change, confirm if it's a dropdown showing up",
+ "fail to open dropdown by clicking, try to press ArrowDown to open",
+ element_id=skyvern_element.get_id(),
task_id=task.task_id,
step_id=step.step_id,
)
- dropdown_element = await locate_dropdown_menu(
- incremental_scraped=incremental_scraped,
- step=step,
- task=task,
- )
- if dropdown_element is not None:
- is_open = True
-
- if not is_open:
- try:
- await skyvern_element.get_locator().click(timeout=timeout)
- except Exception:
- LOG.info(
- "fail to open dropdown by clicking, try to press ArrowDown to open",
- element_id=skyvern_element.get_id(),
- task_id=task.task_id,
- step_id=step.step_id,
- )
- await skyvern_element.focus()
- await skyvern_element.press_key("ArrowDown")
+ await skyvern_element.scroll_into_view()
+ await skyvern_element.press_key("ArrowDown")
# wait 5s for options to load
await asyncio.sleep(5)
@@ -786,7 +764,6 @@ async def handle_select_option_action(
incremental_scraped=incremental_scraped,
step=step,
task=task,
- dropdown_menu_element=dropdown_element,
force_select=True,
)
# force_select won't return None result
@@ -805,7 +782,6 @@ async def handle_select_option_action(
await skyvern_element.coordinate_click(page=page)
await skyvern_element.press_key("Escape")
is_open = False
- dropdown_element = None
await skyvern_element.blur()
await incremental_scraped.stop_listen_dom_increment()
@@ -819,41 +795,21 @@ async def handle_select_option_action(
try:
await incremental_scraped.start_listen_dom_increment()
timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
- await skyvern_element.focus()
- # wait 2s for options to load
- await asyncio.sleep(2)
+ await skyvern_element.scroll_into_view()
- incremental_element = await incremental_scraped.get_incremental_element_tree(
- clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
- )
- if len(incremental_element) > 0:
+ try:
+ await skyvern_element.get_locator().click(timeout=timeout)
+ except Exception:
LOG.info(
- "Focus trigger DOM element change, confirm if it's a dropdown showing up",
+ "fail to open dropdown by clicking, try to press arrow down to open",
+ element_id=skyvern_element.get_id(),
task_id=task.task_id,
step_id=step.step_id,
)
- dropdown_element = await locate_dropdown_menu(
- incremental_scraped=incremental_scraped,
- step=step,
- task=task,
- )
- if dropdown_element is not None:
- is_open = True
-
- if not is_open:
- try:
- await skyvern_element.get_locator().click(timeout=timeout)
- except Exception:
- LOG.info(
- "fail to open dropdown by clicking, try to press arrow down to open",
- element_id=skyvern_element.get_id(),
- task_id=task.task_id,
- step_id=step.step_id,
- )
- await skyvern_element.focus()
- await skyvern_element.press_key("ArrowDown")
- await asyncio.sleep(5)
- is_open = True
+ await skyvern_element.scroll_into_view()
+ await skyvern_element.press_key("ArrowDown")
+ await asyncio.sleep(5)
+ is_open = True
result = await select_from_dropdown_by_value(
value=suggested_value,
@@ -863,7 +819,6 @@ async def handle_select_option_action(
incremental_scraped=incremental_scraped,
task=task,
step=step,
- dropdown_menu_element=dropdown_element,
)
results.append(result)
return results