Skip to content

Automate browser based workflows with AI

remove useless select support legacy (#863)

LawyZhengfb56cba6ba81
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index 8bf3a75de..58e082867 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -313,21 +313,6 @@ class ElementIsNotLabel(SkyvernException):
super().__init__(f"<{tag_name}> element is not <label>")
-class ElementIsNotSelect2Dropdown(SkyvernException):
- def __init__(self, element_id: str, element: dict):
- super().__init__(f"element[{element}] is not select2 dropdown. element_id={element_id}")
-
-
-class ElementIsNotReactSelectDropdown(SkyvernException):
- def __init__(self, element_id: str, element: dict):
- super().__init__(f"element[{element}] is not react select dropdown. element_id={element_id}")
-
-
-class ElementIsNotComboboxDropdown(SkyvernException):
- def __init__(self, element_id: str, element: dict):
- super().__init__(f"element[{element}] is not combobox dropdown. element_id={element_id}")
-
-
class NoneFrameError(SkyvernException):
def __init__(self, frame_id: str):
super().__init__(f"frame content is none. frame_id={frame_id}")
@@ -411,23 +396,6 @@ class NoSelectableElementFound(SkyvernException):
super().__init__(f"No selectable elements found in the children list. element_id={element_id}")
-class NoDropdownAnchorErr(SkyvernException):
- def __init__(self, dropdowm_type: str, element_id: str):
- super().__init__(f"No {dropdowm_type} dropdown found. element_id={element_id}")
-
-
-class MultipleDropdownAnchorErr(SkyvernException):
- def __init__(self, dropdowm_type: str, element_id: str):
- super().__init__(f"Multiple {dropdowm_type} dropdown found. element_id={element_id}")
-
-
-class FailedToGetCurrentValueOfDropdown(SkyvernException):
- def __init__(self, dropdowm_type: str, element_id: str, fail_reason: str):
- super().__init__(
- f"Failed to get current value of {dropdowm_type} dropdown. element_id={element_id}, failure_reason={fail_reason}"
- )
-
-
class HttpException(SkyvernException):
def __init__(self, status_code: int, url: str, msg: str | None = None) -> None:
super().__init__(f"HTTP Exception, status_code={status_code}, url={url}" + (f", msg={msg}" if msg else ""))
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index 3307a0736..b6b418793 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -745,151 +745,6 @@ function getSelectOptions(element) {
return [selectOptions, removeMultipleSpaces(selectedOption.textContent)];
}
-function getListboxOptions(element) {
- // get all the elements with role="option" under the element
- var optionElements = element.querySelectorAll('[role="option"]');
- let selectOptions = [];
- for (var i = 0; i < optionElements.length; i++) {
- let ele = optionElements[i];
-
- selectOptions.push({
- optionIndex: i,
- text: removeMultipleSpaces(getVisibleText(ele)),
- });
- }
- return selectOptions;
-}
-
-async function getSelect2OptionElements(element) {
- let optionList = [];
- const document = element.getRootNode();
-
- while (true) {
- oldOptionCount = optionList.length;
- let newOptionList = document.querySelectorAll("[id='select2-drop'] ul li");
- if (newOptionList.length === oldOptionCount) {
- console.log("no more options loaded, wait 5s to query again");
- // sometimes need more time to load the options, so sleep 10s and try again
- await globalSleep(5000); // wait 5s
- newOptionList = document.querySelectorAll("[id='select2-drop'] ul li");
- console.log(newOptionList.length, " options found, after 5s");
- }
-
- optionList = newOptionList;
- if (optionList.length === 0 || optionList.length === oldOptionCount) {
- break;
- }
-
- lastOption = optionList[optionList.length - 1];
- if (!lastOption.className.toString().includes("select2-more-results")) {
- break;
- }
- lastOption.scrollIntoView();
- }
-
- return optionList;
-}
-
-async function getSelect2Options(element) {
- const optionList = await getSelect2OptionElements(element);
-
- let selectOptions = [];
- for (let i = 0; i < optionList.length; i++) {
- let ele = optionList[i];
- if (ele.className.toString().includes("select2-more-results")) {
- continue;
- }
-
- selectOptions.push({
- optionIndex: i,
- text: removeMultipleSpaces(ele.textContent),
- });
- }
-
- return selectOptions;
-}
-
-async function getReactSelectOptionElements(element) {
- var scrollLeft = window.scrollX;
- var scrollTop = window.scrollY;
-
- let optionList = [];
- // wait for 2s until the element is updated with `aria-controls`
- console.log("wait 2s for the dropdown being updated.");
- await globalSleep(2000);
-
- dropdownId = element.getAttribute("aria-controls");
- if (!dropdownId) {
- return optionList;
- }
-
- const document = element.getRootNode();
- dropdownDiv = document.querySelector(`div[id="${dropdownId}"]`);
- let previousOptionCount = null;
-
- while (true) {
- // sometimes need more time to load the options
- console.log("wait 5s to load all options");
- await globalSleep(5000); // wait 5s
- optionList = dropdownDiv.querySelectorAll("div[class*='select__option']");
- if (optionList.length === 0) {
- break;
- }
-
- if (
- previousOptionCount !== null &&
- previousOptionCount == optionList.length
- ) {
- break;
- }
- previousOptionCount = optionList.length;
-
- lastOption = optionList[optionList.length - 1];
- lastOption.scrollIntoView({ behavior: "instant" });
-
- lastOption.dispatchEvent(
- new WheelEvent("wheel", {
- bubbles: true,
- cancelable: true,
- deltaX: 0,
- deltaY: -20,
- deltaZ: 0,
- }),
- );
- lastOption.dispatchEvent(
- new WheelEvent("wheel", {
- bubbles: true,
- cancelable: true,
- deltaX: 0,
- deltaY: 20,
- deltaZ: 0,
- }),
- );
- }
-
- // scroll back to the original place
- window.scroll({
- top: scrollTop,
- left: scrollLeft,
- behavior: "instant",
- });
- return optionList;
-}
-
-async function getReactSelectOptions(element) {
- const optionList = await getReactSelectOptionElements(element);
-
- let selectOptions = [];
- for (let i = 0; i < optionList.length; i++) {
- let ele = optionList[i];
- selectOptions.push({
- optionIndex: i,
- text: removeMultipleSpaces(ele.textContent),
- });
- }
- return selectOptions;
-}
-
function getDOMElementBySkyvenElement(elementObj) {
// if element has shadowHost set, we need to find the shadowHost element first then find the element
if (elementObj.shadowHost) {
@@ -1674,10 +1529,6 @@ function scrollToElementTop(element) {
});
}
-async function globalSleep(ms) {
- return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
// Helper method for debugging
function findNodeById(arr, targetId, path = []) {
for (let i = 0; i < arr.length; i++) {
diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py
index 0cb0f3b46..460a3bf94 100644
--- a/skyvern/webeye/utils/dom.py
+++ b/skyvern/webeye/utils/dom.py
@@ -3,7 +3,6 @@ from __future__ import annotations
import asyncio
import copy
import typing
-from abc import ABC, abstractmethod
from enum import StrEnum
from random import uniform
@@ -12,25 +11,18 @@ from playwright.async_api import ElementHandle, Frame, FrameLocator, Locator, Pa
from skyvern.constants import SKYVERN_ID_ATTR
from skyvern.exceptions import (
- ElementIsNotComboboxDropdown,
ElementIsNotLabel,
- ElementIsNotReactSelectDropdown,
- ElementIsNotSelect2Dropdown,
- FailedToGetCurrentValueOfDropdown,
MissingElement,
MissingElementDict,
MissingElementInCSSMap,
MissingElementInIframe,
- MultipleDropdownAnchorErr,
MultipleElementsFound,
- NoDropdownAnchorErr,
NoElementBoudingBox,
NoneFrameError,
SkyvernException,
)
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.webeye.scraper.scraper import IncrementalScrapePage, ScrapedPage, json_to_html, trim_element
-from skyvern.webeye.utils.page import SkyvernFrame
LOG = structlog.get_logger()
@@ -276,27 +268,6 @@ class SkyvernElement:
assert handler is not None
return handler
- async def get_select2_dropdown(self) -> Select2Dropdown:
- if not await self.is_select2_dropdown():
- raise ElementIsNotSelect2Dropdown(self.get_id(), self.__static_element)
-
- frame = await SkyvernFrame.create_instance(self.get_frame())
- return Select2Dropdown(frame, self)
-
- async def get_react_select_dropdown(self) -> ReactSelectDropdown:
- if not await self.is_react_select_dropdown():
- raise ElementIsNotReactSelectDropdown(self.get_id(), self.__static_element)
-
- frame = await SkyvernFrame.create_instance(self.get_frame())
- return ReactSelectDropdown(frame, self)
-
- async def get_combobox_dropdown(self) -> ComboboxDropdown:
- if not await self.is_combobox_dropdown():
- raise ElementIsNotComboboxDropdown(self.get_id(), self.__static_element)
-
- frame = await SkyvernFrame.create_instance(self.get_frame())
- return ComboboxDropdown(frame, self)
-
def find_element_id_in_label_children(self, element_type: InteractiveElement) -> str | None:
tag_name = self.get_tag_name()
if tag_name != "label":
@@ -540,212 +511,3 @@ class DomUtil:
raise MultipleElementsFound(num=num_elements, selector=css, element_id=element_id)
return SkyvernElement(locator, frame_content, element)
-
-
-class AbstractSelectDropdown(ABC):
- @abstractmethod
- def name(self) -> str:
- pass
-
- @abstractmethod
- async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
- pass
-
- @abstractmethod
- async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
- pass
-
- @abstractmethod
- async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:
- pass
-
- @abstractmethod
- async def get_options(
- self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
- ) -> typing.List[SkyvernOptionType]:
- pass
-
- @abstractmethod
- async def select_by_index(
- self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
- ) -> None:
- pass
-
-
-class Select2Dropdown(AbstractSelectDropdown):
- def __init__(self, skyvern_frame: SkyvernFrame, skyvern_element: SkyvernElement) -> None:
- self.skyvern_element = skyvern_element
- self.skyvern_frame = skyvern_frame
-
- async def __find_anchor(self, timeout: float) -> Locator:
- locator = self.skyvern_element.get_frame().locator("[id='select2-drop']")
- await locator.wait_for(state="visible", timeout=timeout)
- cnt = await locator.count()
- if cnt == 0:
- raise NoDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
- if cnt > 1:
- raise MultipleDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
- return locator
-
- def name(self) -> str:
- return "select2"
-
- async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
- await self.skyvern_element.get_locator().click(timeout=timeout)
- await self.__find_anchor(timeout=timeout)
-
- async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
- anchor = await self.__find_anchor(timeout=timeout)
- await anchor.press("Escape", timeout=timeout)
-
- async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:
- tag_name = self.skyvern_element.get_tag_name()
- if tag_name == "input":
- # TODO: this is multiple options case, we haven't fully supported it yet.
- return ""
-
- # check SkyvernElement.is_select2_dropdown() method, only <a> and <span> element left
- # we should make sure the locator is on <a>, so we're able to find the [class="select2-chosen"] child
- locator = self.skyvern_element.get_locator()
- if tag_name == "span":
- locator = locator.locator("..")
- elif tag_name == "a":
- pass
- else:
- raise FailedToGetCurrentValueOfDropdown(
- self.name(), self.skyvern_element.get_id(), "invalid element of select2"
- )
-
- try:
- return await locator.locator("span[class='select2-chosen']").text_content(timeout=timeout)
- except Exception as e:
- raise FailedToGetCurrentValueOfDropdown(self.name(), self.skyvern_element.get_id(), repr(e))
-
- async def get_options(
- self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
- ) -> typing.List[SkyvernOptionType]:
- anchor = await self.__find_anchor(timeout=timeout)
- element_handler = await anchor.element_handle(timeout=timeout)
- options = await self.skyvern_frame.get_select2_options(element_handler)
- return typing.cast(typing.List[SkyvernOptionType], options)
-
- async def select_by_index(
- self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
- ) -> None:
- anchor = await self.__find_anchor(timeout=timeout)
- options = anchor.locator("ul").locator("li")
- await options.nth(index).click(timeout=timeout)
-
-
-class ReactSelectDropdown(AbstractSelectDropdown):
- def __init__(self, skyvern_frame: SkyvernFrame, skyvern_element: SkyvernElement) -> None:
- self.skyvern_element = skyvern_element
- self.skyvern_frame = skyvern_frame
-
- def __find_input_locator(self) -> Locator:
- tag_name = self.skyvern_element.get_tag_name()
- locator = self.skyvern_element.get_locator()
-
- if tag_name == InteractiveElement.BUTTON:
- return locator.locator("..").locator("..").locator("input[class*='select__input']")
-
- return locator
-
- async def __find_anchor(self, timeout: float) -> Locator:
- input_locator = self.__find_input_locator()
- anchor_id = await input_locator.get_attribute("aria-controls", timeout=timeout)
-
- locator = self.skyvern_element.get_frame().locator(f"div[id='{anchor_id}']")
- await locator.wait_for(state="visible", timeout=timeout)
- cnt = await locator.count()
- if cnt == 0:
- raise NoDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
- if cnt > 1:
- raise MultipleDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
- return locator
-
- def name(self) -> str:
- return "react-select"
-
- async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
- await self.skyvern_element.get_locator().focus(timeout=timeout)
- await self.skyvern_element.get_locator().press(key="ArrowDown", timeout=timeout)
- await self.__find_anchor(timeout=timeout)
-
- async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
- await self.__find_anchor(timeout=timeout)
- await self.skyvern_element.get_locator().press(key="Escape", timeout=timeout)
-
- async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:
- input_locator = self.__find_input_locator()
- # TODO: only support single value now
- value_locator = input_locator.locator("..").locator("..").locator("div[class*='select__single-value']")
- if await value_locator.count() == 0:
- return ""
- try:
- return await value_locator.text_content(timeout=timeout)
- except Exception as e:
- raise FailedToGetCurrentValueOfDropdown(self.name(), self.skyvern_element.get_id(), repr(e))
-
- async def get_options(
- self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
- ) -> typing.List[SkyvernOptionType]:
- input_locator = self.__find_input_locator()
- element_handler = await input_locator.element_handle(timeout=timeout)
- options = await self.skyvern_frame.get_react_select_options(element_handler)
- return typing.cast(typing.List[SkyvernOptionType], options)
-
- async def select_by_index(
- self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
- ) -> None:
- anchor = await self.__find_anchor(timeout=timeout)
- options = anchor.locator("div[class*='select__option']")
- await options.nth(index).click(timeout=timeout)
-
-
-class ComboboxDropdown(AbstractSelectDropdown):
- def __init__(self, skyvern_frame: SkyvernFrame, skyvern_element: SkyvernElement) -> None:
- self.skyvern_element = skyvern_element
- self.skyvern_frame = skyvern_frame
-
- async def __find_anchor(self, timeout: float) -> Locator:
- control_id = await self.skyvern_element.get_attr("aria-controls", timeout=timeout)
- locator = self.skyvern_element.get_frame().locator(f"[id='{control_id}']")
- await locator.wait_for(state="visible", timeout=timeout)
- cnt = await locator.count()
- if cnt == 0:
- raise NoDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
- if cnt > 1:
- raise MultipleDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
- return locator
-
- def name(self) -> str:
- return "combobox"
-
- async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
- await self.skyvern_element.get_locator().click(timeout=timeout)
- await self.__find_anchor(timeout=timeout)
-
- async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
- await self.skyvern_element.get_locator().press("Tab", timeout=timeout)
-
- async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:
- try:
- return await self.skyvern_element.get_attr("value", dynamic=True, timeout=timeout)
- except Exception as e:
- raise FailedToGetCurrentValueOfDropdown(self.name(), self.skyvern_element.get_id(), repr(e))
-
- async def get_options(
- self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
- ) -> typing.List[SkyvernOptionType]:
- anchor = await self.__find_anchor(timeout=timeout)
- element_handler = await anchor.element_handle()
- options = await self.skyvern_frame.get_combobox_options(element_handler)
- return typing.cast(typing.List[SkyvernOptionType], options)
-
- async def select_by_index(
- self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
- ) -> None:
- anchor = await self.__find_anchor(timeout=timeout)
- options = anchor.locator("li[role='option']")
- await options.nth(index).click(timeout=timeout)
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index 1665093f1..a1415aa19 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import asyncio
import time
-from typing import Any, Dict, List
+from typing import Dict, List
import structlog
from playwright._impl._errors import TimeoutError
@@ -168,21 +168,6 @@ class SkyvernFrame:
js_script = "(element) => scrollToElementTop(element)"
return await self.frame.evaluate(js_script, element)
- async def get_select2_options(self, element: ElementHandle) -> List[Dict[str, Any]]:
- await self.frame.evaluate(JS_FUNCTION_DEFS)
- js_script = "async (element) => await getSelect2Options(element)"
- return await self.frame.evaluate(js_script, element)
-
- async def get_react_select_options(self, element: ElementHandle) -> List[Dict[str, Any]]:
- await self.frame.evaluate(JS_FUNCTION_DEFS)
- js_script = "async (element) => await getReactSelectOptions(element)"
- return await self.frame.evaluate(js_script, element)
-
- async def get_combobox_options(self, element: ElementHandle) -> List[Dict[str, Any]]:
- await self.frame.evaluate(JS_FUNCTION_DEFS)
- js_script = "async (element) => await getListboxOptions(element)"
- return await self.frame.evaluate(js_script, element)
-
async def parse_element_from_html(self, frame: str, element: ElementHandle, interactable: bool) -> Dict:
js_script = "([frame, element, interactable]) => buildElementObject(frame, element, interactable)"
return await self.frame.evaluate(js_script, [frame, element, interactable])