Skip to content

Automate browser based workflows with AI

select option on click (#2391)

Shuchang Zheng24a73b7af08e
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index e7b0bc532..4006e30ed 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -479,6 +479,7 @@ async def handle_click_action(
task: Task,
step: Step,
) -> list[ActionResult]:
+ original_url = page.url
if action.x is not None and action.y is not None:
# Find the element at the clicked location using JavaScript evaluation
element_id = await page.evaluate(
@@ -592,14 +593,80 @@ async def handle_click_action(
workflow_run_id=task.workflow_run_id,
)
else:
- results = await chain_click(
- task,
- scraped_page,
- page,
- action,
- skyvern_element,
- timeout=settings.BROWSER_ACTION_TIMEOUT_MS,
- )
+ try:
+ skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame())
+ incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame)
+ await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())
+
+ results = await chain_click(
+ task,
+ scraped_page,
+ page,
+ action,
+ skyvern_element,
+ timeout=settings.BROWSER_ACTION_TIMEOUT_MS,
+ )
+ if page.url != original_url:
+ return results
+
+ if results and not isinstance(results[-1], ActionSuccess):
+ return results
+
+ if await incremental_scraped.get_incremental_elements_num() == 0:
+ return results
+
+ incremental_elements = await incremental_scraped.get_incremental_element_tree(
+ clean_and_remove_element_tree_factory(
+ task=task, step=step, check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)]
+ ),
+ )
+
+ if len(incremental_elements) == 0:
+ return results
+
+ LOG.info("Detected new element after clicking", action=action)
+ dropdown_menu_element = await locate_dropdown_menu(
+ current_anchor_element=skyvern_element,
+ incremental_scraped=incremental_scraped,
+ step=step,
+ task=task,
+ )
+
+ if dropdown_menu_element is None:
+ return results
+
+ LOG.info(
+ "Found the dropdown menu element after clicking, triggering the sequential click logic",
+ step_id=step.step_id,
+ task_id=task.task_id,
+ element_id=dropdown_menu_element.get_id(),
+ )
+
+ action_result = await select_from_emerging_elements(
+ current_element_id=skyvern_element.get_id(),
+ options=CustomSelectPromptOptions(
+ field_information=action.intention if action.intention else action.reasoning,
+ ), # FIXME: need a better options data
+ page=page,
+ scraped_page=scraped_page,
+ step=step,
+ task=task,
+ )
+
+ results.append(action_result)
+ return results
+
+ except NoIncrementalElementFoundForCustomSelection:
+ LOG.info(
+ "No incremental element found, skip the sequential click logic",
+ step_id=step.step_id,
+ task_id=task.task_id,
+ element_id=skyvern_element.get_id(),
+ )
+ return results
+
+ finally:
+ await incremental_scraped.stop_listen_dom_increment()
return results
@@ -1344,10 +1411,18 @@ async def handle_select_option_action(
)
if len(incremental_element) == 0:
+ LOG.info(
+ "No incremental elements detected by MutationObserver, using re-scraping the page to find the match element"
+ )
results.append(
await select_from_emerging_elements(
- action=action,
- input_or_select_context=input_or_select_context,
+ current_element_id=skyvern_element.get_id(),
+ options=CustomSelectPromptOptions(
+ is_date_related=input_or_select_context.is_date_related or False,
+ field_information=input_or_select_context.intention or input_or_select_context.field or "",
+ required_field=input_or_select_context.is_required or False,
+ target_value=action.option.label or action.option.value or "",
+ ),
page=page,
scraped_page=scraped_page,
task=task,
@@ -2458,18 +2533,36 @@ def build_sequential_select_history(history_list: list[CustomSingleSelectResult]
return result
+class CustomSelectPromptOptions(BaseModel):
+ """
+ This is the options for the custom select prompt.
+ It's used to generate the prompt for the custom select action.
+ is_date_related: whether the field is date related
+ required_field: whether the field is required
+ field_information: the description about the field, could be field name, action intention, action reasoning about the field, etc.
+ target_value: the target value of the field (generated by the LLM in the main prompt).
+ """
+
+ is_date_related: bool = False
+ required_field: bool = False
+ field_information: str = ""
+ target_value: str | None = None
+
+
async def select_from_emerging_elements(
- action: SelectOptionAction,
- input_or_select_context: InputOrSelectContext,
+ current_element_id: str,
+ options: CustomSelectPromptOptions,
page: Page,
scraped_page: ScrapedPage,
step: Step,
task: Task,
) -> ActionResult:
+ """
+ This is the function to select an element from the new showing elements.
+ Currently mainly used for the dropdown menu selection.
+ """
+
# TODO: support to handle the case when options are loaded by scroll
- LOG.info(
- "No incremental elements detected by MutationObserver, using re-scraping the page to find the match element"
- )
scraped_page_after_open = await scraped_page.generate_scraped_page_without_screenshots()
new_element_ids = set(scraped_page_after_open.id_to_css_dict.keys()) - set(scraped_page.id_to_css_dict.keys())
@@ -2481,18 +2574,16 @@ async def select_from_emerging_elements(
]
if len(new_interactable_element_ids) == 0:
- raise NoIncrementalElementFoundForCustomSelection(element_id=action.element_id)
+ raise NoIncrementalElementFoundForCustomSelection(element_id=current_element_id)
prompt = load_prompt_with_elements(
scraped_page=scraped_page_after_open,
prompt_engine=prompt_engine,
template_name="custom-select",
- is_date_related=input_or_select_context.is_date_related,
- field_information=input_or_select_context.field
- if not input_or_select_context.intention
- else input_or_select_context.intention,
- required_field=input_or_select_context.is_required,
- target_value=action.option.label,
+ is_date_related=options.is_date_related,
+ field_information=options.field_information,
+ required_field=options.required_field,
+ target_value=options.target_value,
navigation_goal=task.navigation_goal,
new_elements_ids=new_interactable_element_ids,
navigation_payload_str=json.dumps(task.navigation_payload),
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index a74fd21d1..1163d68d1 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -738,6 +738,11 @@ class IncrementalScrapePage:
)
async def get_incremental_elements_num(self) -> int:
+ # check if the DOM has navigated away or refreshed
+ js_script = "() => window.globalOneTimeIncrementElements === undefined"
+ if await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script):
+ return 0
+
js_script = "() => window.globalOneTimeIncrementElements.length"
return await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script)