Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
remove useless select support legacy (#863)
LawyZhengfb56cba6ba81
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.pyindex 8bf3a75de..58e082867 100644--- a/skyvern/exceptions.py+++ b/skyvern/exceptions.py@@ -313,21 +313,6 @@ class ElementIsNotLabel(SkyvernException):super().__init__(f"<{tag_name}> element is not <label>")-class ElementIsNotSelect2Dropdown(SkyvernException):- def __init__(self, element_id: str, element: dict):- super().__init__(f"element[{element}] is not select2 dropdown. element_id={element_id}")---class ElementIsNotReactSelectDropdown(SkyvernException):- def __init__(self, element_id: str, element: dict):- super().__init__(f"element[{element}] is not react select dropdown. element_id={element_id}")---class ElementIsNotComboboxDropdown(SkyvernException):- def __init__(self, element_id: str, element: dict):- super().__init__(f"element[{element}] is not combobox dropdown. element_id={element_id}")--class NoneFrameError(SkyvernException):def __init__(self, frame_id: str):super().__init__(f"frame content is none. frame_id={frame_id}")@@ -411,23 +396,6 @@ class NoSelectableElementFound(SkyvernException):super().__init__(f"No selectable elements found in the children list. element_id={element_id}")-class NoDropdownAnchorErr(SkyvernException):- def __init__(self, dropdowm_type: str, element_id: str):- super().__init__(f"No {dropdowm_type} dropdown found. element_id={element_id}")---class MultipleDropdownAnchorErr(SkyvernException):- def __init__(self, dropdowm_type: str, element_id: str):- super().__init__(f"Multiple {dropdowm_type} dropdown found. element_id={element_id}")---class FailedToGetCurrentValueOfDropdown(SkyvernException):- def __init__(self, dropdowm_type: str, element_id: str, fail_reason: str):- super().__init__(- f"Failed to get current value of {dropdowm_type} dropdown. element_id={element_id}, failure_reason={fail_reason}"- )--class HttpException(SkyvernException):def __init__(self, status_code: int, url: str, msg: str | None = None) -> None:super().__init__(f"HTTP Exception, status_code={status_code}, url={url}" + (f", msg={msg}" if msg else ""))diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex 3307a0736..b6b418793 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -745,151 +745,6 @@ function getSelectOptions(element) {return [selectOptions, removeMultipleSpaces(selectedOption.textContent)];}-function getListboxOptions(element) {- // get all the elements with role="option" under the element- var optionElements = element.querySelectorAll('[role="option"]');- let selectOptions = [];- for (var i = 0; i < optionElements.length; i++) {- let ele = optionElements[i];-- selectOptions.push({- optionIndex: i,- text: removeMultipleSpaces(getVisibleText(ele)),- });- }- return selectOptions;-}--async function getSelect2OptionElements(element) {- let optionList = [];- const document = element.getRootNode();-- while (true) {- oldOptionCount = optionList.length;- let newOptionList = document.querySelectorAll("[id='select2-drop'] ul li");- if (newOptionList.length === oldOptionCount) {- console.log("no more options loaded, wait 5s to query again");- // sometimes need more time to load the options, so sleep 10s and try again- await globalSleep(5000); // wait 5s- newOptionList = document.querySelectorAll("[id='select2-drop'] ul li");- console.log(newOptionList.length, " options found, after 5s");- }-- optionList = newOptionList;- if (optionList.length === 0 || optionList.length === oldOptionCount) {- break;- }-- lastOption = optionList[optionList.length - 1];- if (!lastOption.className.toString().includes("select2-more-results")) {- break;- }- lastOption.scrollIntoView();- }-- return optionList;-}--async function getSelect2Options(element) {- const optionList = await getSelect2OptionElements(element);-- let selectOptions = [];- for (let i = 0; i < optionList.length; i++) {- let ele = optionList[i];- if (ele.className.toString().includes("select2-more-results")) {- continue;- }-- selectOptions.push({- optionIndex: i,- text: removeMultipleSpaces(ele.textContent),- });- }-- return selectOptions;-}--async function getReactSelectOptionElements(element) {- var scrollLeft = window.scrollX;- var scrollTop = window.scrollY;-- let optionList = [];- // wait for 2s until the element is updated with `aria-controls`- console.log("wait 2s for the dropdown being updated.");- await globalSleep(2000);-- dropdownId = element.getAttribute("aria-controls");- if (!dropdownId) {- return optionList;- }-- const document = element.getRootNode();- dropdownDiv = document.querySelector(`div[id="${dropdownId}"]`);- let previousOptionCount = null;-- while (true) {- // sometimes need more time to load the options- console.log("wait 5s to load all options");- await globalSleep(5000); // wait 5s- optionList = dropdownDiv.querySelectorAll("div[class*='select__option']");- if (optionList.length === 0) {- break;- }-- if (- previousOptionCount !== null &&- previousOptionCount == optionList.length- ) {- break;- }- previousOptionCount = optionList.length;-- lastOption = optionList[optionList.length - 1];- lastOption.scrollIntoView({ behavior: "instant" });-- lastOption.dispatchEvent(- new WheelEvent("wheel", {- bubbles: true,- cancelable: true,- deltaX: 0,- deltaY: -20,- deltaZ: 0,- }),- );- lastOption.dispatchEvent(- new WheelEvent("wheel", {- bubbles: true,- cancelable: true,- deltaX: 0,- deltaY: 20,- deltaZ: 0,- }),- );- }-- // scroll back to the original place- window.scroll({- top: scrollTop,- left: scrollLeft,- behavior: "instant",- });- return optionList;-}--async function getReactSelectOptions(element) {- const optionList = await getReactSelectOptionElements(element);-- let selectOptions = [];- for (let i = 0; i < optionList.length; i++) {- let ele = optionList[i];- selectOptions.push({- optionIndex: i,- text: removeMultipleSpaces(ele.textContent),- });- }- return selectOptions;-}-function getDOMElementBySkyvenElement(elementObj) {// if element has shadowHost set, we need to find the shadowHost element first then find the elementif (elementObj.shadowHost) {@@ -1674,10 +1529,6 @@ function scrollToElementTop(element) {});}-async function globalSleep(ms) {- return new Promise((resolve) => setTimeout(resolve, ms));-}-// Helper method for debuggingfunction findNodeById(arr, targetId, path = []) {for (let i = 0; i < arr.length; i++) {diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex 0cb0f3b46..460a3bf94 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -3,7 +3,6 @@ from __future__ import annotationsimport asyncioimport copyimport typing-from abc import ABC, abstractmethodfrom enum import StrEnumfrom random import uniform@@ -12,25 +11,18 @@ from playwright.async_api import ElementHandle, Frame, FrameLocator, Locator, Pafrom skyvern.constants import SKYVERN_ID_ATTRfrom skyvern.exceptions import (- ElementIsNotComboboxDropdown,ElementIsNotLabel,- ElementIsNotReactSelectDropdown,- ElementIsNotSelect2Dropdown,- FailedToGetCurrentValueOfDropdown,MissingElement,MissingElementDict,MissingElementInCSSMap,MissingElementInIframe,- MultipleDropdownAnchorErr,MultipleElementsFound,- NoDropdownAnchorErr,NoElementBoudingBox,NoneFrameError,SkyvernException,)from skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.webeye.scraper.scraper import IncrementalScrapePage, ScrapedPage, json_to_html, trim_element-from skyvern.webeye.utils.page import SkyvernFrameLOG = structlog.get_logger()@@ -276,27 +268,6 @@ class SkyvernElement:assert handler is not Nonereturn handler- async def get_select2_dropdown(self) -> Select2Dropdown:- if not await self.is_select2_dropdown():- raise ElementIsNotSelect2Dropdown(self.get_id(), self.__static_element)-- frame = await SkyvernFrame.create_instance(self.get_frame())- return Select2Dropdown(frame, self)-- async def get_react_select_dropdown(self) -> ReactSelectDropdown:- if not await self.is_react_select_dropdown():- raise ElementIsNotReactSelectDropdown(self.get_id(), self.__static_element)-- frame = await SkyvernFrame.create_instance(self.get_frame())- return ReactSelectDropdown(frame, self)-- async def get_combobox_dropdown(self) -> ComboboxDropdown:- if not await self.is_combobox_dropdown():- raise ElementIsNotComboboxDropdown(self.get_id(), self.__static_element)-- frame = await SkyvernFrame.create_instance(self.get_frame())- return ComboboxDropdown(frame, self)-def find_element_id_in_label_children(self, element_type: InteractiveElement) -> str | None:tag_name = self.get_tag_name()if tag_name != "label":@@ -540,212 +511,3 @@ class DomUtil:raise MultipleElementsFound(num=num_elements, selector=css, element_id=element_id)return SkyvernElement(locator, frame_content, element)---class AbstractSelectDropdown(ABC):- @abstractmethod- def name(self) -> str:- pass-- @abstractmethod- async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:- pass-- @abstractmethod- async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:- pass-- @abstractmethod- async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:- pass-- @abstractmethod- async def get_options(- self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS- ) -> typing.List[SkyvernOptionType]:- pass-- @abstractmethod- async def select_by_index(- self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS- ) -> None:- pass---class Select2Dropdown(AbstractSelectDropdown):- def __init__(self, skyvern_frame: SkyvernFrame, skyvern_element: SkyvernElement) -> None:- self.skyvern_element = skyvern_element- self.skyvern_frame = skyvern_frame-- async def __find_anchor(self, timeout: float) -> Locator:- locator = self.skyvern_element.get_frame().locator("[id='select2-drop']")- await locator.wait_for(state="visible", timeout=timeout)- cnt = await locator.count()- if cnt == 0:- raise NoDropdownAnchorErr(self.name(), self.skyvern_element.get_id())- if cnt > 1:- raise MultipleDropdownAnchorErr(self.name(), self.skyvern_element.get_id())- return locator-- def name(self) -> str:- return "select2"-- async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:- await self.skyvern_element.get_locator().click(timeout=timeout)- await self.__find_anchor(timeout=timeout)-- async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:- anchor = await self.__find_anchor(timeout=timeout)- await anchor.press("Escape", timeout=timeout)-- async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:- tag_name = self.skyvern_element.get_tag_name()- if tag_name == "input":- # TODO: this is multiple options case, we haven't fully supported it yet.- return ""-- # check SkyvernElement.is_select2_dropdown() method, only <a> and <span> element left- # we should make sure the locator is on <a>, so we're able to find the [class="select2-chosen"] child- locator = self.skyvern_element.get_locator()- if tag_name == "span":- locator = locator.locator("..")- elif tag_name == "a":- pass- else:- raise FailedToGetCurrentValueOfDropdown(- self.name(), self.skyvern_element.get_id(), "invalid element of select2"- )-- try:- return await locator.locator("span[class='select2-chosen']").text_content(timeout=timeout)- except Exception as e:- raise FailedToGetCurrentValueOfDropdown(self.name(), self.skyvern_element.get_id(), repr(e))-- async def get_options(- self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS- ) -> typing.List[SkyvernOptionType]:- anchor = await self.__find_anchor(timeout=timeout)- element_handler = await anchor.element_handle(timeout=timeout)- options = await self.skyvern_frame.get_select2_options(element_handler)- return typing.cast(typing.List[SkyvernOptionType], options)-- async def select_by_index(- self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS- ) -> None:- anchor = await self.__find_anchor(timeout=timeout)- options = anchor.locator("ul").locator("li")- await options.nth(index).click(timeout=timeout)---class ReactSelectDropdown(AbstractSelectDropdown):- def __init__(self, skyvern_frame: SkyvernFrame, skyvern_element: SkyvernElement) -> None:- self.skyvern_element = skyvern_element- self.skyvern_frame = skyvern_frame-- def __find_input_locator(self) -> Locator:- tag_name = self.skyvern_element.get_tag_name()- locator = self.skyvern_element.get_locator()-- if tag_name == InteractiveElement.BUTTON:- return locator.locator("..").locator("..").locator("input[class*='select__input']")-- return locator-- async def __find_anchor(self, timeout: float) -> Locator:- input_locator = self.__find_input_locator()- anchor_id = await input_locator.get_attribute("aria-controls", timeout=timeout)-- locator = self.skyvern_element.get_frame().locator(f"div[id='{anchor_id}']")- await locator.wait_for(state="visible", timeout=timeout)- cnt = await locator.count()- if cnt == 0:- raise NoDropdownAnchorErr(self.name(), self.skyvern_element.get_id())- if cnt > 1:- raise MultipleDropdownAnchorErr(self.name(), self.skyvern_element.get_id())- return locator-- def name(self) -> str:- return "react-select"-- async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:- await self.skyvern_element.get_locator().focus(timeout=timeout)- await self.skyvern_element.get_locator().press(key="ArrowDown", timeout=timeout)- await self.__find_anchor(timeout=timeout)-- async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:- await self.__find_anchor(timeout=timeout)- await self.skyvern_element.get_locator().press(key="Escape", timeout=timeout)-- async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:- input_locator = self.__find_input_locator()- # TODO: only support single value now- value_locator = input_locator.locator("..").locator("..").locator("div[class*='select__single-value']")- if await value_locator.count() == 0:- return ""- try:- return await value_locator.text_content(timeout=timeout)- except Exception as e:- raise FailedToGetCurrentValueOfDropdown(self.name(), self.skyvern_element.get_id(), repr(e))-- async def get_options(- self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS- ) -> typing.List[SkyvernOptionType]:- input_locator = self.__find_input_locator()- element_handler = await input_locator.element_handle(timeout=timeout)- options = await self.skyvern_frame.get_react_select_options(element_handler)- return typing.cast(typing.List[SkyvernOptionType], options)-- async def select_by_index(- self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS- ) -> None:- anchor = await self.__find_anchor(timeout=timeout)- options = anchor.locator("div[class*='select__option']")- await options.nth(index).click(timeout=timeout)---class ComboboxDropdown(AbstractSelectDropdown):- def __init__(self, skyvern_frame: SkyvernFrame, skyvern_element: SkyvernElement) -> None:- self.skyvern_element = skyvern_element- self.skyvern_frame = skyvern_frame-- async def __find_anchor(self, timeout: float) -> Locator:- control_id = await self.skyvern_element.get_attr("aria-controls", timeout=timeout)- locator = self.skyvern_element.get_frame().locator(f"[id='{control_id}']")- await locator.wait_for(state="visible", timeout=timeout)- cnt = await locator.count()- if cnt == 0:- raise NoDropdownAnchorErr(self.name(), self.skyvern_element.get_id())- if cnt > 1:- raise MultipleDropdownAnchorErr(self.name(), self.skyvern_element.get_id())- return locator-- def name(self) -> str:- return "combobox"-- async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:- await self.skyvern_element.get_locator().click(timeout=timeout)- await self.__find_anchor(timeout=timeout)-- async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:- await self.skyvern_element.get_locator().press("Tab", timeout=timeout)-- async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:- try:- return await self.skyvern_element.get_attr("value", dynamic=True, timeout=timeout)- except Exception as e:- raise FailedToGetCurrentValueOfDropdown(self.name(), self.skyvern_element.get_id(), repr(e))-- async def get_options(- self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS- ) -> typing.List[SkyvernOptionType]:- anchor = await self.__find_anchor(timeout=timeout)- element_handler = await anchor.element_handle()- options = await self.skyvern_frame.get_combobox_options(element_handler)- return typing.cast(typing.List[SkyvernOptionType], options)-- async def select_by_index(- self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS- ) -> None:- anchor = await self.__find_anchor(timeout=timeout)- options = anchor.locator("li[role='option']")- await options.nth(index).click(timeout=timeout)diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex 1665093f1..a1415aa19 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport asyncioimport time-from typing import Any, Dict, List+from typing import Dict, Listimport structlogfrom playwright._impl._errors import TimeoutError@@ -168,21 +168,6 @@ class SkyvernFrame:js_script = "(element) => scrollToElementTop(element)"return await self.frame.evaluate(js_script, element)- async def get_select2_options(self, element: ElementHandle) -> List[Dict[str, Any]]:- await self.frame.evaluate(JS_FUNCTION_DEFS)- js_script = "async (element) => await getSelect2Options(element)"- return await self.frame.evaluate(js_script, element)-- async def get_react_select_options(self, element: ElementHandle) -> List[Dict[str, Any]]:- await self.frame.evaluate(JS_FUNCTION_DEFS)- js_script = "async (element) => await getReactSelectOptions(element)"- return await self.frame.evaluate(js_script, element)-- async def get_combobox_options(self, element: ElementHandle) -> List[Dict[str, Any]]:- await self.frame.evaluate(JS_FUNCTION_DEFS)- js_script = "async (element) => await getListboxOptions(element)"- return await self.frame.evaluate(js_script, element)-async def parse_element_from_html(self, frame: str, element: ElementHandle, interactable: bool) -> Dict:js_script = "([frame, element, interactable]) => buildElementObject(frame, element, interactable)"return await self.frame.evaluate(js_script, [frame, element, interactable])