Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix sequential click bug (#2494)
Shuchang Zheng91a666f70572
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 4788f0332..6f3adafb8 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -597,6 +597,7 @@ async def handle_click_action(workflow_run_id=task.workflow_run_id,)else:+ incremental_scraped: IncrementalScrapePage | None = Nonetry:skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame())incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame)@@ -616,63 +617,87 @@ async def handle_click_action(if results and not isinstance(results[-1], ActionSuccess):return results- if await incremental_scraped.get_incremental_elements_num() == 0:+ try:+ if sequential_click_result := await handle_sequential_click_for_dropdown(+ action=action,+ anchor_element=skyvern_element,+ dom=dom,+ page=page,+ scraped_page=scraped_page,+ incremental_scraped=incremental_scraped,+ task=task,+ step=step,+ ):+ results.append(sequential_click_result)+ return results++ except Exception:+ LOG.warning(+ "Failed to do sequential logic for the click action, skipping",+ exc_info=True,+ step_id=step.step_id,+ task_id=task.task_id,+ element_id=skyvern_element.get_id(),+ )return results- incremental_elements = await incremental_scraped.get_incremental_element_tree(- clean_and_remove_element_tree_factory(- task=task, step=step, check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)]- ),- )+ finally:+ if incremental_scraped:+ await incremental_scraped.stop_listen_dom_increment()- if len(incremental_elements) == 0:- return results+ return results- LOG.info("Detected new element after clicking", action=action)- dropdown_menu_element = await locate_dropdown_menu(- current_anchor_element=skyvern_element,- incremental_scraped=incremental_scraped,- step=step,- task=task,- )- if dropdown_menu_element is None:- return results+async def handle_sequential_click_for_dropdown(+ action: actions.ClickAction,+ anchor_element: SkyvernElement,+ dom: DomUtil,+ page: Page,+ scraped_page: ScrapedPage,+ incremental_scraped: IncrementalScrapePage,+ task: Task,+ step: Step,+) -> ActionResult | None:+ if await incremental_scraped.get_incremental_elements_num() == 0:+ return None- LOG.info(- "Found the dropdown menu element after clicking, triggering the sequential click logic",- step_id=step.step_id,- task_id=task.task_id,- element_id=dropdown_menu_element.get_id(),- )+ incremental_elements = await incremental_scraped.get_incremental_element_tree(+ clean_and_remove_element_tree_factory(+ task=task, step=step, check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)]+ ),+ )- action_result = await select_from_emerging_elements(- current_element_id=skyvern_element.get_id(),- options=CustomSelectPromptOptions(- field_information=action.intention if action.intention else action.reasoning,- ), # FIXME: need a better options data- page=page,- scraped_page=scraped_page,- step=step,- task=task,- )+ if len(incremental_elements) == 0:+ return None- results.append(action_result)- return results+ LOG.info("Detected new element after clicking", action=action)+ dropdown_menu_element = await locate_dropdown_menu(+ current_anchor_element=anchor_element,+ incremental_scraped=incremental_scraped,+ step=step,+ task=task,+ )- except NoIncrementalElementFoundForCustomSelection:- LOG.info(- "No incremental element found, skip the sequential click logic",- step_id=step.step_id,- task_id=task.task_id,- element_id=skyvern_element.get_id(),- )- return results+ if dropdown_menu_element is None:+ return None- finally:- await incremental_scraped.stop_listen_dom_increment()+ LOG.info(+ "Found the dropdown menu element after clicking, triggering the sequential click logic",+ step_id=step.step_id,+ task_id=task.task_id,+ element_id=dropdown_menu_element.get_id(),+ )- return results+ return await select_from_emerging_elements(+ current_element_id=anchor_element.get_id(),+ options=CustomSelectPromptOptions(+ field_information=action.intention if action.intention else action.reasoning,+ ), # FIXME: need a better options data+ page=page,+ scraped_page=scraped_page,+ step=step,+ task=task,+ )async def handle_click_to_download_file_action(