Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix dropdown menu detect bug (#874)
LawyZhengc27c2a55f98a
diff --git a/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 b/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2index ea3a7da94..12b8d9a2d 100644--- a/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2+++ b/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2@@ -3,6 +3,7 @@ There is a screenshot from a part of a web HTML page. Help me confirm if it is aAn open dropdown menu can be defined as:- At least one option is visible in the screenshot.- Do not consider it an open dropdown menu if the only visible option displays a message like "No results" or "No match".+ - Do not consider it an open dropdown menu if the only visible element displays a placeholder like "Please select", "-", or "Select...".MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex f3979815a..4c4862585 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -417,12 +417,12 @@ async def handle_input_text_action(# check if it's selectableif skyvern_element.get_tag_name() == InteractiveElement.INPUT and not await skyvern_element.is_raw_input():await skyvern_element.scroll_into_view()- select_action = SelectOptionAction(- reasoning=action.reasoning,- element_id=skyvern_element.get_id(),- option=SelectOption(label=text),- )if skyvern_element.get_selectable():+ select_action = SelectOptionAction(+ reasoning=action.reasoning,+ element_id=skyvern_element.get_id(),+ option=SelectOption(label=text),+ )LOG.info("Input element is selectable, doing select actions",task_id=task.task_id,@@ -434,8 +434,7 @@ async def handle_input_text_action(# press arrowdown to watch if there's any options popping upawait incremental_scraped.start_listen_dom_increment()- await skyvern_element.get_locator().focus(timeout=timeout)- await skyvern_element.get_locator().press("ArrowDown", timeout=timeout)+ await skyvern_element.press_key("ArrowDown")await asyncio.sleep(5)incremental_element = await incremental_scraped.get_incremental_element_tree(@@ -726,46 +725,25 @@ async def handle_select_option_action(timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MSskyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame())incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame)- dropdown_element: SkyvernElement | None = Noneis_open = Falsesuggested_value: str | None = Noneresults: list[ActionResult] = []try:await incremental_scraped.start_listen_dom_increment()- await skyvern_element.focus()- # wait 2s for options to load- await asyncio.sleep(2)+ await skyvern_element.scroll_into_view()- incremental_element = await incremental_scraped.get_incremental_element_tree(- clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),- )- if len(incremental_element) > 0:+ try:+ await skyvern_element.get_locator().click(timeout=timeout)+ except Exception:LOG.info(- "Focus trigger DOM element change, confirm if it's a dropdown showing up",+ "fail to open dropdown by clicking, try to press ArrowDown to open",+ element_id=skyvern_element.get_id(),task_id=task.task_id,step_id=step.step_id,)- dropdown_element = await locate_dropdown_menu(- incremental_scraped=incremental_scraped,- step=step,- task=task,- )- if dropdown_element is not None:- is_open = True-- if not is_open:- try:- await skyvern_element.get_locator().click(timeout=timeout)- except Exception:- LOG.info(- "fail to open dropdown by clicking, try to press ArrowDown to open",- element_id=skyvern_element.get_id(),- task_id=task.task_id,- step_id=step.step_id,- )- await skyvern_element.focus()- await skyvern_element.press_key("ArrowDown")+ await skyvern_element.scroll_into_view()+ await skyvern_element.press_key("ArrowDown")# wait 5s for options to loadawait asyncio.sleep(5)@@ -786,7 +764,6 @@ async def handle_select_option_action(incremental_scraped=incremental_scraped,step=step,task=task,- dropdown_menu_element=dropdown_element,force_select=True,)# force_select won't return None result@@ -805,7 +782,6 @@ async def handle_select_option_action(await skyvern_element.coordinate_click(page=page)await skyvern_element.press_key("Escape")is_open = False- dropdown_element = Noneawait skyvern_element.blur()await incremental_scraped.stop_listen_dom_increment()@@ -819,41 +795,21 @@ async def handle_select_option_action(try:await incremental_scraped.start_listen_dom_increment()timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS- await skyvern_element.focus()- # wait 2s for options to load- await asyncio.sleep(2)+ await skyvern_element.scroll_into_view()- incremental_element = await incremental_scraped.get_incremental_element_tree(- clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),- )- if len(incremental_element) > 0:+ try:+ await skyvern_element.get_locator().click(timeout=timeout)+ except Exception:LOG.info(- "Focus trigger DOM element change, confirm if it's a dropdown showing up",+ "fail to open dropdown by clicking, try to press arrow down to open",+ element_id=skyvern_element.get_id(),task_id=task.task_id,step_id=step.step_id,)- dropdown_element = await locate_dropdown_menu(- incremental_scraped=incremental_scraped,- step=step,- task=task,- )- if dropdown_element is not None:- is_open = True-- if not is_open:- try:- await skyvern_element.get_locator().click(timeout=timeout)- except Exception:- LOG.info(- "fail to open dropdown by clicking, try to press arrow down to open",- element_id=skyvern_element.get_id(),- task_id=task.task_id,- step_id=step.step_id,- )- await skyvern_element.focus()- await skyvern_element.press_key("ArrowDown")- await asyncio.sleep(5)- is_open = True+ await skyvern_element.scroll_into_view()+ await skyvern_element.press_key("ArrowDown")+ await asyncio.sleep(5)+ is_open = Trueresult = await select_from_dropdown_by_value(value=suggested_value,@@ -863,7 +819,6 @@ async def handle_select_option_action(incremental_scraped=incremental_scraped,task=task,step=step,- dropdown_menu_element=dropdown_element,)results.append(result)return results