Skip to content

Automate browser based workflows with AI

fix custom selection bugs (#800)

LawyZhengb12f09c53539
diff --git a/skyvern/forge/prompts/skyvern/auto-completion-choose-option.j2 b/skyvern/forge/prompts/skyvern/auto-completion-choose-option.j2
index c3f208101..396f3c5ef 100644
--- a/skyvern/forge/prompts/skyvern/auto-completion-choose-option.j2
+++ b/skyvern/forge/prompts/skyvern/auto-completion-choose-option.j2
@@ -1,14 +1,14 @@
There is an input element on an HTML page. Based on the context and information provided, you have two goals:
- Confirm if an auto-completion attempt appears after the user inputs the current value.
- - If auto-completion suggestions appear, assist the user in selecting the most appropriate element based on the user’s goal, details, and the context.
+ - If auto-completion suggestions appear, assist the user in selecting the most appropriate element based on the user's goal, details, and the context.
You can confirm an auto-completion attempt based on the following rules:
- Several auto-completion suggestions appear for the input value.
- - Although messages like “No results” and “No match” mean no option was matched, they still indicate an attempt to generate auto-completion suggestions.
+ - Although messages like "No results" and "No match" mean no option was matched, they still indicate an attempt to generate auto-completion suggestions.
You must identify a potential auto-completion suggestion based on the following rules:
- - The option must be an element with an ID from the provided “HTML elements”. Do not create or assume options outside of these elements.
- - The content of the option must be meaningful. Do not consider non-message indicators like “No results” or “No match” as valid options.
+ - The option must be an element with an ID from the provided "HTML elements". Do not create or assume options outside of these elements.
+ - The content of the option must be meaningful. Do not consider non-message indicators like "No results" or "No match" as valid options.
MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
Each interactable element is tagged with an ID.
diff --git a/skyvern/forge/prompts/skyvern/custom-select.j2 b/skyvern/forge/prompts/skyvern/custom-select.j2
index 3b0cd2bf9..34f108d89 100644
--- a/skyvern/forge/prompts/skyvern/custom-select.j2
+++ b/skyvern/forge/prompts/skyvern/custom-select.j2
@@ -2,10 +2,10 @@ You are performing a selection action on an HTML page. Assist the user in select
You can identify the matching element based on the following guidelines:
1. Select the most suitable element based on the user goal, user details, and the context.
- 2. If no option is a perfect match, choose a fallback option such as “Others” or “None of the above”.
+ 2. If no option is a perfect match, and there is a fallback option such as "Others" or "None of the above" in the DOM elements, you can consider it a match.
3. If a field is required, do not leave it blank.
- 4. If a field is required, do not select a placeholder value, such as “Please select”, “-”, or “Select…”.
- 5. Exclude loading indicators like “loading more results” as valid options.
+ 4. If a field is required, do not select a placeholder value, such as "Please select", "-", or "Select…".
+ 5. Exclude loading indicators like "loading more results" as valid options.
MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
Each interactable element is tagged with an ID.
diff --git a/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 b/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2
index cf6f99009..4bbaef11e 100644
--- a/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2
+++ b/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2
@@ -2,7 +2,7 @@ There is a screenshot from a part of a web HTML page. Help me confirm if it is a
An open dropdown menu can be defined as:
- At least one option is visible in the screenshot.
- - Do not consider it an open dropdown menu if the only visible option displays a message like “No results” or “No match”.
+ - Do not consider it an open dropdown menu if the only visible option displays a message like "No results" or "No match".
MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 6c38df5f2..a632a79ff 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -82,10 +82,11 @@ COMMON_INPUT_TAGS = {"input", "textarea", "select"}
class CustomSingleSelectResult:
- def __init__(self) -> None:
+ def __init__(self, skyvern_frame: SkyvernFrame) -> None:
self.action_result: ActionResult | None = None
self.value: str | None = None
self.dropdown_menu: SkyvernElement | None = None
+ self.skyvern_frame = skyvern_frame
async def is_done(self) -> bool:
# check if the dropdown menu is still on the page
@@ -100,7 +101,7 @@ class CustomSingleSelectResult:
if await self.dropdown_menu.get_locator().count() == 0:
return True
- return False
+ return not await self.skyvern_frame.get_element_visible(await self.dropdown_menu.get_element_handler())
def is_ul_or_listbox_element_factory(
@@ -1336,7 +1337,7 @@ async def sequentially_select_from_dropdown(
# TODO: only suport the third-level dropdown selection now
MAX_SELECT_DEPTH = 3
values: list[str | None] = []
- single_select_result = CustomSingleSelectResult()
+ single_select_result = CustomSingleSelectResult(skyvern_frame=skyvern_frame)
check_exist_funcs: list[CheckExistIDFunc] = [dom.check_id_in_dom]
for i in range(MAX_SELECT_DEPTH):
@@ -1418,7 +1419,7 @@ async def select_from_dropdown(
1. force_select is false and no dropdown menu popped
2. force_select is false and match value is not relevant to the target value
"""
- single_select_result = CustomSingleSelectResult()
+ single_select_result = CustomSingleSelectResult(skyvern_frame=skyvern_frame)
timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index c0d6dc7e3..1e2708d28 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -1798,7 +1798,23 @@ function getIncrementElements() {
const treeList = window.globalDomDepthMap.get(depth);
const removeDupAndConcatChildren = (element) => {
- const children = element.children;
+ let children = element.children;
+ for (let i = 0; i < children.length; i++) {
+ const child = children[i];
+ const domElement = document.querySelector(`[unique_id="${child.id}"]`);
+ // if the element is still on the page, we rebuild the element to update the information
+ if (domElement) {
+ let newChild = buildElementObject(
+ "",
+ domElement,
+ child.interactable,
+ child.purgeable,
+ );
+ newChild.children = child.children;
+ children[i] = newChild;
+ }
+ }
+
if (idToElement.has(element.id)) {
element = idToElement.get(element.id);
for (let i = 0; i < children.length; i++) {
@@ -1815,7 +1831,22 @@ function getIncrementElements() {
}
};
- for (const treeHeadElement of treeList) {
+ for (let treeHeadElement of treeList) {
+ const domElement = document.querySelector(
+ `[unique_id="${treeHeadElement.id}"]`,
+ );
+ // if the element is still on the page, we rebuild the element to update the information
+ if (domElement) {
+ let newHead = buildElementObject(
+ "",
+ domElement,
+ treeHeadElement.interactable,
+ treeHeadElement.purgeable,
+ );
+ newHead.children = treeHeadElement.children;
+ treeHeadElement = newHead;
+ }
+
// check if the element is existed
if (!idToElement.has(treeHeadElement.id)) {
cleanedTreeList.push(treeHeadElement);