Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix selection issues (#1515)
LawyZhengc6140fa405f4
diff --git a/skyvern/constants.py b/skyvern/constants.pyindex a523ce9e4..fb644d7c8 100644--- a/skyvern/constants.py+++ b/skyvern/constants.py@@ -16,6 +16,7 @@ SAVE_DOWNLOADED_FILES_TIMEOUT = 180GET_DOWNLOADED_FILES_TIMEOUT = 30NAVIGATION_MAX_RETRY_TIME = 5AUTO_COMPLETION_POTENTIAL_VALUES_COUNT = 5+DROPDOWN_MENU_MAX_DISTANCE = 100# reserved fields for navigation payloadSPECIAL_FIELD_VERIFICATION_CODE = "verification_code"diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 5e2f1e1d0..a0d8740e9 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -267,6 +267,16 @@ async def _convert_css_shape_to_string(try:LOG.debug("call LLM to convert css shape to string shape", element_id=element_id)+ if not await locater.is_visible(timeout=settings.BROWSER_ACTION_TIMEOUT_MS):+ LOG.info(+ "element is not visible on the page, going to abort conversion",+ task_id=task_id,+ step_id=step_id,+ element_id=element_id,+ key=shape_key,+ )+ return None+screenshot = await locater.screenshot(timeout=settings.BROWSER_SCREENSHOT_TIMEOUT_MS)prompt = prompt_engine.load_prompt("css-shape-convert")diff --git a/skyvern/forge/prompts/skyvern/custom-select.j2 b/skyvern/forge/prompts/skyvern/custom-select.j2index 98cadabb6..f73f10a74 100644--- a/skyvern/forge/prompts/skyvern/custom-select.j2+++ b/skyvern/forge/prompts/skyvern/custom-select.j2@@ -2,7 +2,7 @@ You are performing a {{ "multi-level selection" if select_history else "selectioYou can identify the matching element based on the following guidelines:1. Select the most suitable element based on the user goal, user details, and the context.- 2. If no option is a perfect match, and there is a fallback option such as "Others" or "None of the above" in the DOM elements, you can consider it a match.+ 2. If none of the options perfectly match, and there is no search box for input, but there is a fallback option such as "Others" or "None of the above" in the DOM elements, you can consider it a match.3. If a field is required, do not leave it blank.4. If a field is required, do not select a placeholder value, such as "Please select", "-", or "Select...".5. Exclude loading indicators like "loading more results" as valid options.{% if select_history %}diff --git a/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 b/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2index 9e147fb16..f90ecb4d3 100644--- a/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2+++ b/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2@@ -2,7 +2,7 @@ There is a screenshot from a part of a web HTML page. Help me confirm if it is aAn open dropdown menu can be defined as:- At least one option is visible in the screenshot.- - A calendar view or date picker could be considered as an open dropdown menu.+ - A calendar view could be considered as an open dropdown menu. But DO NOT consider an calendar icon as the dropdown menu.- Do not consider it an open dropdown menu if the only visible option displays a message like "No results" or "No match".- Do not consider it an open dropdown menu if the only visible element displays a placeholder like "Please select", "-", or "Select...".diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex e516c7be3..ea0f40599 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -17,6 +17,7 @@ from skyvern.config import settingsfrom skyvern.constants import (AUTO_COMPLETION_POTENTIAL_VALUES_COUNT,BROWSER_DOWNLOAD_TIMEOUT,+ DROPDOWN_MENU_MAX_DISTANCE,REPO_ROOT_DIR,SKYVERN_ID_ATTR,)@@ -579,6 +580,16 @@ async def handle_input_text_action(# press arrowdown to watch if there's any options popping upawait incremental_scraped.start_listen_dom_increment()+ try:+ await skyvern_element.input_clear()+ except Exception:+ LOG.info(+ "Failed to clear up the input, but continue to input",+ task_id=task.task_id,+ step_id=step.step_id,+ element_id=skyvern_element.get_id(),+ )+try:await skyvern_element.press_key("ArrowDown")except TimeoutError:@@ -613,6 +624,7 @@ async def handle_input_text_action(action=select_action,page=page,dom=dom,+ skyvern_element=skyvern_element,skyvern_frame=skyvern_frame,incremental_scraped=incremental_scraped,step=step,@@ -641,14 +653,29 @@ async def handle_input_text_action()except Exception:- await skyvern_element.scroll_into_view()LOG.warning("Failed to do custom selection transformed from input action, continue to input text",exc_info=True,task_id=task.task_id,step_id=step.step_id,)+ await skyvern_element.scroll_into_view()finally:+ blocking_element, exist = await skyvern_element.find_blocking_element(+ dom=dom, incremental_page=incremental_scraped+ )+ if blocking_element and exist:+ LOG.info(+ "Find a blocking element to the current element, going to blur the blocking element first",+ task_id=task.task_id,+ step_id=step.step_id,+ blocking_element=blocking_element.get_locator(),+ )+ if await blocking_element.get_locator().count():+ await blocking_element.press_key("Escape")+ if await blocking_element.get_locator().count():+ await blocking_element.blur()+await skyvern_element.press_key("Escape")await skyvern_element.blur()await incremental_scraped.stop_listen_dom_increment()@@ -672,6 +699,25 @@ async def handle_input_text_action(await skyvern_element.press_fill(text=text)return [ActionSuccess()]+ # wait 2s for blocking element to show up+ await asyncio.sleep(2)+ try:+ blocking_element, exist = await skyvern_element.find_blocking_element(+ dom=dom, incremental_page=incremental_scraped+ )+ if blocking_element and exist:+ LOG.warning(+ "Find a blocking element to the current element, going to input on the blocking element",+ )+ skyvern_element = blocking_element+ except Exception:+ LOG.info(+ "Failed to find the blocking element, continue with the orignal element",+ exc_info=True,+ task_id=task.task_id,+ step_id=step.step_id,+ )+try:# TODO: not sure if this case will trigger auto-completionif tag_name not in COMMON_INPUT_TAGS:@@ -1032,6 +1078,7 @@ async def handle_select_option_action(action=action,page=page,dom=dom,+ skyvern_element=skyvern_element,skyvern_frame=skyvern_frame,incremental_scraped=incremental_scraped,step=step,@@ -1087,6 +1134,7 @@ async def handle_select_option_action(value=suggested_value,page=page,dom=dom,+ skyvern_element=skyvern_element,skyvern_frame=skyvern_frame,incremental_scraped=incremental_scraped,task=task,@@ -1765,6 +1813,7 @@ async def sequentially_select_from_dropdown(action: SelectOptionAction,page: Page,dom: DomUtil,+ skyvern_element: SkyvernElement,skyvern_frame: SkyvernFrame,incremental_scraped: IncrementalScrapePage,step: Step,@@ -1812,6 +1861,7 @@ async def sequentially_select_from_dropdown(single_select_result = await select_from_dropdown(context=input_or_select_context,page=page,+ skyvern_element=skyvern_element,skyvern_frame=skyvern_frame,incremental_scraped=incremental_scraped,check_exist_funcs=check_exist_funcs,@@ -1887,6 +1937,7 @@ def build_sequential_select_history(history_list: list[CustomSingleSelectResult]async def select_from_dropdown(context: InputOrSelectContext,page: Page,+ skyvern_element: SkyvernElement,skyvern_frame: SkyvernFrame,incremental_scraped: IncrementalScrapePage,check_exist_funcs: list[CheckExistIDFunc],@@ -1911,6 +1962,7 @@ async def select_from_dropdown(if dropdown_menu_element is None:dropdown_menu_element = await locate_dropdown_menu(+ current_anchor_element=skyvern_element,incremental_scraped=incremental_scraped,step=step,task=task,@@ -2059,6 +2111,7 @@ async def select_from_dropdown(async def select_from_dropdown_by_value(value: str,page: Page,+ skyvern_element: SkyvernElement,skyvern_frame: SkyvernFrame,dom: DomUtil,incremental_scraped: IncrementalScrapePage,@@ -2078,6 +2131,7 @@ async def select_from_dropdown_by_value(if dropdown_menu_element is None:dropdown_menu_element = await locate_dropdown_menu(+ current_anchor_element=skyvern_element,incremental_scraped=incremental_scraped,step=step,task=task,@@ -2131,6 +2185,7 @@ async def select_from_dropdown_by_value(async def locate_dropdown_menu(+ current_anchor_element: SkyvernElement,incremental_scraped: IncrementalScrapePage,step: Step,task: Task,@@ -2164,6 +2219,30 @@ async def locate_dropdown_menu()continue+ try:+ if not await head_element.is_next_to_element(+ target_locator=current_anchor_element.get_locator(),+ max_x_distance=DROPDOWN_MENU_MAX_DISTANCE,+ max_y_distance=DROPDOWN_MENU_MAX_DISTANCE,+ ):+ LOG.debug(+ "Skip the element since it's too far away from the anchor element",+ step_id=step.step_id,+ task_id=task.task_id,+ element_id=element_id,+ )+ continue++ except Exception:+ LOG.info(+ "Failed to calculate the distance between the elements",+ element_id=element_id,+ step_id=step.step_id,+ task_id=task.task_id,+ exc_info=True,+ )+ continue+if not await skyvern_frame.get_element_visible(await head_element.get_element_handler()):LOG.debug("Skip the element since it's invisible",diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex f8453c001..0d8f2ec73 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -770,6 +770,21 @@ const isComboboxDropdown = (element) => {return role && haspopup && controls && readonly;};+const isDivComboboxDropdown = (element) => {+ const tagName = element.tagName.toLowerCase();+ if (tagName !== "div") {+ return false;+ }+ const role = element.getAttribute("role")+ ? element.getAttribute("role").toLowerCase()+ : "";+ const haspopup = element.getAttribute("aria-haspopup")+ ? element.getAttribute("aria-haspopup").toLowerCase()+ : "";+ const controls = element.hasAttribute("aria-controls");+ return role === "combobox" && controls && haspopup;+};+const isDropdownButton = (element) => {const tagName = element.tagName.toLowerCase();const type = element.getAttribute("type")@@ -1182,6 +1197,7 @@ function buildElementObject(frame, element, interactable, purgeable = false) {elementTagNameLower === "svg" || element.closest("svg") !== null,isSelectable:elementTagNameLower === "select" ||+ isDivComboboxDropdown(element) ||isDropdownButton(element) ||isAngularDropdown(element) ||isSelect2Dropdown(element) ||@@ -2083,9 +2099,12 @@ if (window.globalObserverForDOMIncrement === undefined) {}if (mutation.attributeName === "class") {const node = mutation.target;+ if (node.nodeType === Node.TEXT_NODE) continue;+ if (node.tagName.toLowerCase() === "body") continue;+ if (!mutation.oldValue) continue;if (- !mutation.oldValue ||- !isClassNameIncludesHidden(mutation.oldValue)+ !isClassNameIncludesHidden(mutation.oldValue) &&+ !node.hasAttribute("data-menu-uid") // google framework use this to trace dropdown menu)continue;const newStyle = getElementComputedStyle(node);diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex c44d27a87..93169803e 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -541,6 +541,12 @@ class IncrementalScrapePage:self.element_tree_trimmed: list[dict] = list()self.skyvern_frame = skyvern_frame+ def check_id_in_page(self, element_id: str) -> bool:+ css_selector = self.id_to_css_dict.get(element_id, "")+ if css_selector:+ return True+ return False+async def get_incremental_element_tree(self,cleanup_element_tree: CleanupElementTreeFunc,diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex 7eb06477b..a06b8c890 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -318,12 +318,21 @@ class SkyvernElement:assert handler is not Nonereturn handler- async def find_blocking_element(self, dom: DomUtil) -> tuple[SkyvernElement | None, bool]:+ async def find_blocking_element(+ self, dom: DomUtil, incremental_page: IncrementalScrapePage | None = None+ ) -> tuple[SkyvernElement | None, bool]:skyvern_frame = await SkyvernFrame.create_instance(self.get_frame())blocking_element_id, blocked = await skyvern_frame.get_blocking_element_id(await self.get_element_handler())if not blocking_element_id:return None, blocked- return await dom.get_skyvern_element_by_id(blocking_element_id), blocked++ if dom.check_id_in_dom(blocking_element_id):+ return await dom.get_skyvern_element_by_id(blocking_element_id), blocked++ if incremental_page and incremental_page.check_id_in_page(blocking_element_id):+ return await SkyvernElement.create_from_incremental(incremental_page, blocking_element_id), blocked++ return None, blockedasync def find_element_in_label_children(self, dom: DomUtil, element_type: InteractiveElement@@ -590,10 +599,34 @@ class SkyvernElement:await self.focus(timeout=timeout)await asyncio.sleep(2) # wait for scrolling into the target- async def calculate_vertical_distance_to(+ async def calculate_min_y_distance_to(+ self,+ target_locator: Locator,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> float:+ self_rect = await self.get_locator().bounding_box(timeout=timeout)+ if self_rect is None:+ raise Exception("Can't Skyvern element rect")++ target_rect = await target_locator.bounding_box(timeout=timeout)+ if self_rect is None or target_rect is None:+ raise Exception("Can't get the target element rect")++ y_1 = self_rect["y"] + self_rect["height"] - target_rect["y"]+ y_2 = self_rect["y"] - (target_rect["y"] + target_rect["height"])++ # if y1 * y2 <= 0, it means the two elements are overlapping+ if y_1 * y_2 <= 0:+ return 0++ return min(+ abs(y_1),+ abs(y_2),+ )++ async def calculate_min_x_distance_to(self,target_locator: Locator,- mode: typing.Literal["inner", "outer"],timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,) -> float:self_rect = await self.get_locator().bounding_box(timeout=timeout)@@ -604,10 +637,32 @@ class SkyvernElement:if self_rect is None or target_rect is None:raise Exception("Can't get the target element rect")- if mode == "inner":- return abs(self_rect["y"] + self_rect["height"] - target_rect["y"])- else:- return abs(self_rect["y"] - (target_rect["y"] + target_rect["height"]))+ x_1 = self_rect["x"] + self_rect["width"] - target_rect["x"]+ x_2 = self_rect["x"] - (target_rect["x"] + target_rect["width"])++ # if x1 * x2 <= 0, it means the two elements are overlapping+ if x_1 * x_2 <= 0:+ return 0++ return min(+ abs(x_1),+ abs(x_2),+ )++ async def is_next_to_element(+ self,+ target_locator: Locator,+ max_x_distance: float = 0,+ max_y_distance: float = 0,+ timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS,+ ) -> bool:+ if max_x_distance > 0 and await self.calculate_min_x_distance_to(target_locator, timeout) > max_x_distance:+ return False++ if max_y_distance > 0 and await self.calculate_min_y_distance_to(target_locator, timeout) > max_y_distance:+ return False++ return Trueclass DomUtil: