Skip to content

Automate browser based workflows with AI

multi-selection history (#813)

LawyZheng512d0c1a885f
diff --git a/skyvern/forge/prompts/skyvern/custom-select.j2 b/skyvern/forge/prompts/skyvern/custom-select.j2
index 6c15af0e8..0585407d1 100644
--- a/skyvern/forge/prompts/skyvern/custom-select.j2
+++ b/skyvern/forge/prompts/skyvern/custom-select.j2
@@ -5,7 +5,8 @@ You can identify the matching element based on the following guidelines:
2. If no option is a perfect match, and there is a fallback option such as "Others" or "None of the above" in the DOM elements, you can consider it a match.
3. If a field is required, do not leave it blank.
4. If a field is required, do not select a placeholder value, such as "Please select", "-", or "Select...".
- 5. Exclude loading indicators like "loading more results" as valid options.
+ 5. Exclude loading indicators like "loading more results" as valid options.{% if select_history %}
+ 6. The selection history displays the previously selected values for the multi-level selection. Continue to complete the entire selection process.{% endif %}
MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
Each interactable element is tagged with an ID.
@@ -42,4 +43,10 @@ User details:
HTML elements:
```
{{ elements }}
-```
\ No newline at end of file
+```
+{% if select_history %}
+Select History:
+```
+{{ select_history }}
+```
+{% endif %}
\ No newline at end of file
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index d7fc4501a..f5809064f 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -83,6 +83,7 @@ COMMON_INPUT_TAGS = {"input", "textarea", "select"}
class CustomSingleSelectResult:
def __init__(self, skyvern_frame: SkyvernFrame) -> None:
+ self.reasoning: str | None = None
self.action_result: ActionResult | None = None
self.value: str | None = None
self.dropdown_menu: SkyvernElement | None = None
@@ -1339,7 +1340,7 @@ async def sequentially_select_from_dropdown(
# TODO: only suport the third-level dropdown selection now
MAX_SELECT_DEPTH = 3
values: list[str | None] = []
- single_select_result = CustomSingleSelectResult(skyvern_frame=skyvern_frame)
+ select_history: list[CustomSingleSelectResult] = []
check_exist_funcs: list[CheckExistIDFunc] = [dom.check_id_in_dom]
for i in range(MAX_SELECT_DEPTH):
@@ -1352,9 +1353,11 @@ async def sequentially_select_from_dropdown(
check_exist_funcs=check_exist_funcs,
step=step,
task=task,
+ select_history=select_history,
force_select=force_select,
should_relevant=should_relevant,
)
+ select_history.append(single_select_result)
values.append(single_select_result.value)
# wait 1s until DOM finished updating
await asyncio.sleep(1)
@@ -1399,7 +1402,21 @@ async def sequentially_select_from_dropdown(
)
return single_select_result.action_result, values[-1] if len(values) > 0 else None
- return single_select_result.action_result, values[-1] if len(values) > 0 else None
+ return select_history[-1].action_result if len(select_history) > 0 else None, values[-1] if len(
+ values
+ ) > 0 else None
+
+
+def build_sequential_select_history(history_list: list[CustomSingleSelectResult]) -> list[dict[str, Any]]:
+ result = [
+ {
+ "reasoning": select_result.reasoning,
+ "value": select_result.value,
+ "result": "success" if isinstance(select_result.action_result, ActionSuccess) else "failed",
+ }
+ for select_result in history_list
+ ]
+ return result
async def select_from_dropdown(
@@ -1411,6 +1428,7 @@ async def select_from_dropdown(
check_exist_funcs: list[CheckExistIDFunc],
step: Step,
task: Task,
+ select_history: list[CustomSingleSelectResult] | None = None,
force_select: bool = False,
should_relevant: bool = True,
) -> CustomSingleSelectResult:
@@ -1421,6 +1439,7 @@ async def select_from_dropdown(
1. force_select is false and no dropdown menu popped
2. force_select is false and match value is not relevant to the target value
"""
+ select_history = [] if select_history is None else select_history
single_select_result = CustomSingleSelectResult(skyvern_frame=skyvern_frame)
timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
@@ -1471,6 +1490,7 @@ async def select_from_dropdown(
navigation_goal=task.navigation_goal,
navigation_payload_str=json.dumps(task.navigation_payload),
elements=html,
+ select_history=json.dumps(build_sequential_select_history(select_history)) if select_history else "",
)
LOG.info(
@@ -1481,6 +1501,8 @@ async def select_from_dropdown(
json_response = await llm_handler(prompt=prompt, step=step)
value: str | None = json_response.get("value", None)
single_select_result.value = value
+ select_reason: str | None = json_response.get("reasoning", None)
+ single_select_result.reasoning = select_reason
LOG.info(
"LLM response for the matched element",
@@ -1657,6 +1679,31 @@ async def locate_dropdown_menu(
)
continue
+ ul_or_listbox_element_id = await head_element.find_children_element_id_by_callback(
+ cb=is_ul_or_listbox_element_factory(incremental_scraped=incremental_scraped, task=task, step=step),
+ )
+
+ if ul_or_listbox_element_id:
+ try:
+ await SkyvernElement.create_from_incremental(incremental_scraped, ul_or_listbox_element_id)
+ LOG.info(
+ "Confirm it's an opened dropdown menu since it includes <ul> or <role='listbox'>",
+ step_id=step.step_id,
+ task_id=task.task_id,
+ element_id=element_id,
+ )
+ return await SkyvernElement.create_from_incremental(
+ incre_page=incremental_scraped, element_id=element_id
+ )
+ except Exception:
+ LOG.debug(
+ "Failed to get <ul> or <role='listbox'> element in the incremental page",
+ element_id=element_id,
+ step_id=step.step_id,
+ task_id=task.task_id,
+ exc_info=True,
+ )
+
screenshot = await head_element.get_locator().screenshot(
timeout=SettingsManager.get_settings().BROWSER_SCREENSHOT_TIMEOUT_MS
)