Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
multi-selection history (#813)
LawyZheng512d0c1a885f
diff --git a/skyvern/forge/prompts/skyvern/custom-select.j2 b/skyvern/forge/prompts/skyvern/custom-select.j2index 6c15af0e8..0585407d1 100644--- a/skyvern/forge/prompts/skyvern/custom-select.j2+++ b/skyvern/forge/prompts/skyvern/custom-select.j2@@ -5,7 +5,8 @@ You can identify the matching element based on the following guidelines:2. If no option is a perfect match, and there is a fallback option such as "Others" or "None of the above" in the DOM elements, you can consider it a match.3. If a field is required, do not leave it blank.4. If a field is required, do not select a placeholder value, such as "Please select", "-", or "Select...".- 5. Exclude loading indicators like "loading more results" as valid options.+ 5. Exclude loading indicators like "loading more results" as valid options.{% if select_history %}+ 6. The selection history displays the previously selected values for the multi-level selection. Continue to complete the entire selection process.{% endif %}MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.Each interactable element is tagged with an ID.@@ -42,4 +43,10 @@ User details:HTML elements:```{{ elements }}-```\ No newline at end of file+```+{% if select_history %}+Select History:+```+{{ select_history }}+```+{% endif %}\ No newline at end of filediff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex d7fc4501a..f5809064f 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -83,6 +83,7 @@ COMMON_INPUT_TAGS = {"input", "textarea", "select"}class CustomSingleSelectResult:def __init__(self, skyvern_frame: SkyvernFrame) -> None:+ self.reasoning: str | None = Noneself.action_result: ActionResult | None = Noneself.value: str | None = Noneself.dropdown_menu: SkyvernElement | None = None@@ -1339,7 +1340,7 @@ async def sequentially_select_from_dropdown(# TODO: only suport the third-level dropdown selection nowMAX_SELECT_DEPTH = 3values: list[str | None] = []- single_select_result = CustomSingleSelectResult(skyvern_frame=skyvern_frame)+ select_history: list[CustomSingleSelectResult] = []check_exist_funcs: list[CheckExistIDFunc] = [dom.check_id_in_dom]for i in range(MAX_SELECT_DEPTH):@@ -1352,9 +1353,11 @@ async def sequentially_select_from_dropdown(check_exist_funcs=check_exist_funcs,step=step,task=task,+ select_history=select_history,force_select=force_select,should_relevant=should_relevant,)+ select_history.append(single_select_result)values.append(single_select_result.value)# wait 1s until DOM finished updatingawait asyncio.sleep(1)@@ -1399,7 +1402,21 @@ async def sequentially_select_from_dropdown()return single_select_result.action_result, values[-1] if len(values) > 0 else None- return single_select_result.action_result, values[-1] if len(values) > 0 else None+ return select_history[-1].action_result if len(select_history) > 0 else None, values[-1] if len(+ values+ ) > 0 else None+++def build_sequential_select_history(history_list: list[CustomSingleSelectResult]) -> list[dict[str, Any]]:+ result = [+ {+ "reasoning": select_result.reasoning,+ "value": select_result.value,+ "result": "success" if isinstance(select_result.action_result, ActionSuccess) else "failed",+ }+ for select_result in history_list+ ]+ return resultasync def select_from_dropdown(@@ -1411,6 +1428,7 @@ async def select_from_dropdown(check_exist_funcs: list[CheckExistIDFunc],step: Step,task: Task,+ select_history: list[CustomSingleSelectResult] | None = None,force_select: bool = False,should_relevant: bool = True,) -> CustomSingleSelectResult:@@ -1421,6 +1439,7 @@ async def select_from_dropdown(1. force_select is false and no dropdown menu popped2. force_select is false and match value is not relevant to the target value"""+ select_history = [] if select_history is None else select_historysingle_select_result = CustomSingleSelectResult(skyvern_frame=skyvern_frame)timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS@@ -1471,6 +1490,7 @@ async def select_from_dropdown(navigation_goal=task.navigation_goal,navigation_payload_str=json.dumps(task.navigation_payload),elements=html,+ select_history=json.dumps(build_sequential_select_history(select_history)) if select_history else "",)LOG.info(@@ -1481,6 +1501,8 @@ async def select_from_dropdown(json_response = await llm_handler(prompt=prompt, step=step)value: str | None = json_response.get("value", None)single_select_result.value = value+ select_reason: str | None = json_response.get("reasoning", None)+ single_select_result.reasoning = select_reasonLOG.info("LLM response for the matched element",@@ -1657,6 +1679,31 @@ async def locate_dropdown_menu()continue+ ul_or_listbox_element_id = await head_element.find_children_element_id_by_callback(+ cb=is_ul_or_listbox_element_factory(incremental_scraped=incremental_scraped, task=task, step=step),+ )++ if ul_or_listbox_element_id:+ try:+ await SkyvernElement.create_from_incremental(incremental_scraped, ul_or_listbox_element_id)+ LOG.info(+ "Confirm it's an opened dropdown menu since it includes <ul> or <role='listbox'>",+ step_id=step.step_id,+ task_id=task.task_id,+ element_id=element_id,+ )+ return await SkyvernElement.create_from_incremental(+ incre_page=incremental_scraped, element_id=element_id+ )+ except Exception:+ LOG.debug(+ "Failed to get <ul> or <role='listbox'> element in the incremental page",+ element_id=element_id,+ step_id=step.step_id,+ task_id=task.task_id,+ exc_info=True,+ )+screenshot = await head_element.get_locator().screenshot(timeout=SettingsManager.get_settings().BROWSER_SCREENSHOT_TIMEOUT_MS)