Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
wrap page evaluate with timeout (#1133)
LawyZheng8b53bc4257ae
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex adef5b617..69779b107 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -777,17 +777,6 @@ class ForgeAgent:action=action,action_result=results,)- # if the action triggered javascript calls- # this action should be the last action this round and do not take more actions.- # for now, we're being optimistic and assuming that- # js call doesn't have impact on the following actions- if results[-1].javascript_triggered:- LOG.info(- "Action triggered javascript. Stop executing reamaining actions.",- action=action,- )- # stop executing the rest actions- breakelif results and not results[-1].success and not results[-1].stop_execution_on_failure:LOG.warning("Action failed, but not stopping execution",@@ -1198,7 +1187,9 @@ class ForgeAgent:navigation_goal = task.navigation_goalstarting_url = task.urlpage = await browser_state.get_working_page()- current_url = await page.evaluate("() => document.location.href") if page else starting_url+ current_url = (+ await SkyvernFrame.evaluate(frame=page, expression="() => document.location.href") if page else starting_url+ )final_navigation_payload = self._build_navigation_payload(task, expire_verification_code=expire_verification_code)diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex a7b675d01..57d1fc2c8 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -613,7 +613,7 @@ async def handle_upload_file_action(locator = skyvern_element.locatorfile_path = await download_file(file_url)- is_file_input = await is_file_input_element(locator)+ is_file_input = await skyvern_element.is_file_input()if is_file_input:LOG.info("Taking UploadFileAction. Found file input tag", action=action)@@ -1103,24 +1103,14 @@ async def chain_click(Clicks on an element identified by the css and its parent if failed.:param css: css of the element to click"""- javascript_triggered = await is_javascript_triggered(scraped_page, page, locator)try:await locator.click(timeout=timeout)LOG.info("Chain click: main element click succeeded", action=action, locator=locator)- return [- ActionSuccess(- javascript_triggered=javascript_triggered,- )- ]+ return [ActionSuccess()]except Exception:- action_results: list[ActionResult] = [- ActionFailure(- FailToClick(action.element_id),- javascript_triggered=javascript_triggered,- )- ]+ action_results: list[ActionResult] = [ActionFailure(FailToClick(action.element_id))]if skyvern_element.get_tag_name() == "label":LOG.info(@@ -1137,12 +1127,7 @@ async def chain_click(action_results.append(ActionSuccess())return action_resultsexcept Exception:- action_results.append(- ActionFailure(- FailToClick(action.element_id, anchor="for"),- javascript_triggered=javascript_triggered,- )- )+ action_results.append(ActionFailure(FailToClick(action.element_id, anchor="for")))if skyvern_element.get_tag_name() == InteractiveElement.INPUT:LOG.info(@@ -1158,10 +1143,6 @@ async def chain_click(try:parent_locator = locator.locator("..")-- parent_javascript_triggered = await is_javascript_triggered(scraped_page, page, parent_locator)- javascript_triggered = javascript_triggered or parent_javascript_triggered-await parent_locator.click(timeout=timeout)LOG.info(@@ -1169,12 +1150,7 @@ async def chain_click(action=action,parent_locator=parent_locator,)- action_results.append(- ActionSuccess(- javascript_triggered=javascript_triggered,- interacted_with_parent=True,- )- )+ action_results.append(ActionSuccess(interacted_with_parent=True))except Exception:LOG.warning("Failed to click parent element",@@ -1185,7 +1161,6 @@ async def chain_click(action_results.append(ActionFailure(FailToClick(action.element_id, anchor="parent"),- javascript_triggered=javascript_triggered,interacted_with_parent=True,))@@ -2223,48 +2198,9 @@ def get_checkbox_id_in_label_children(scraped_page: ScrapedPage, element_id: strreturn None-@deprecated("This function is deprecated. It was used for select2 dropdown, but we don't use it anymore.")-async def is_javascript_triggered(scraped_page: ScrapedPage, page: Page, locator: Locator) -> bool:- element = locator.first-- tag_name = await element.evaluate("e => e.tagName")- if tag_name.lower() == "a":- href = await element.evaluate("e => e.href")- if href.lower().startswith("javascript:"):- LOG.info("Found javascript call in anchor tag, marking step as completed. Dropping remaining actions")- return True- return False---async def get_tag_name_lowercase(locator: Locator) -> str | None:- element = locator.first- if element:- tag_name = await element.evaluate("e => e.tagName")- return tag_name.lower()- return None---async def is_file_input_element(locator: Locator) -> bool:- element = locator.first- if element:- tag_name = await element.evaluate("el => el.tagName")- type_name = await element.evaluate("el => el.type")- return tag_name.lower() == "input" and type_name == "file"- return False---async def is_input_element(locator: Locator) -> bool:- element = locator.first- if element:- tag_name = await element.evaluate("el => el.tagName")- return tag_name.lower() == "input"- return False--async def click_sibling_of_input(locator: Locator,timeout: int,- javascript_triggered: bool = False,) -> ActionResult:try:input_element = locator.first@@ -2278,18 +2214,16 @@ async def click_sibling_of_input("Successfully clicked sibling label of input element",sibling_label_css=sibling_label_css,)- return ActionSuccess(javascript_triggered=javascript_triggered, interacted_with_sibling=True)+ return ActionSuccess(interacted_with_sibling=True)# Should never get herereturn ActionFailure(exception=Exception("Failed while trying to click sibling of input element"),- javascript_triggered=javascript_triggered,interacted_with_sibling=True,)except Exception:LOG.warning("Failed to click sibling label of input element", exc_info=True)return ActionFailure(exception=Exception("Failed while trying to click sibling of input element"),- javascript_triggered=javascript_triggered,)diff --git a/skyvern/webeye/actions/responses.py b/skyvern/webeye/actions/responses.pyindex 773674442..6895a4eed 100644--- a/skyvern/webeye/actions/responses.py+++ b/skyvern/webeye/actions/responses.py@@ -13,7 +13,6 @@ class ActionResult(BaseModel):data: dict[str, Any] | list | str | None = Nonestep_retry_number: int | None = Nonestep_order: int | None = None- javascript_triggered: bool = Falsedownload_triggered: bool | None = None# None is used for old data so that we can differentiate between old and new data which only has booleaninteracted_with_sibling: bool | None = None@@ -30,8 +29,6 @@ class ActionResult(BaseModel):results.append(f"step_order={self.step_order}")if self.step_retry_number:results.append(f"step_retry_number={self.step_retry_number}")- if self.javascript_triggered:- results.append(f"javascript_triggered={self.javascript_triggered}")if self.download_triggered is not None:results.append(f"download_triggered={self.download_triggered}")if self.interacted_with_sibling is not None:@@ -49,7 +46,6 @@ class ActionSuccess(ActionResult):def __init__(self,data: dict[str, Any] | list | str | None = None,- javascript_triggered: bool = False,download_triggered: bool | None = None,interacted_with_sibling: bool = False,interacted_with_parent: bool = False,@@ -57,7 +53,6 @@ class ActionSuccess(ActionResult):super().__init__(success=True,data=data,- javascript_triggered=javascript_triggered,download_triggered=download_triggered,interacted_with_sibling=interacted_with_sibling,interacted_with_parent=interacted_with_parent,@@ -69,7 +64,6 @@ class ActionFailure(ActionResult):self,exception: Exception,stop_execution_on_failure: bool = True,- javascript_triggered: bool = False,download_triggered: bool | None = None,interacted_with_sibling: bool = False,interacted_with_parent: bool = False,@@ -79,7 +73,6 @@ class ActionFailure(ActionResult):exception_type=type(exception).__name__,stop_execution_on_failure=stop_execution_on_failure,exception_message=remove_whitespace(str(exception)),- javascript_triggered=javascript_triggered,download_triggered=download_triggered,interacted_with_sibling=interacted_with_sibling,interacted_with_parent=interacted_with_parent,@@ -91,14 +84,12 @@ class ActionFailure(ActionResult):class ActionAbort(ActionResult):def __init__(self,- javascript_triggered: bool = False,download_triggered: bool | None = None,interacted_with_sibling: bool = False,interacted_with_parent: bool = False,):super().__init__(success=True,- javascript_triggered=javascript_triggered,download_triggered=download_triggered,interacted_with_sibling=interacted_with_sibling,interacted_with_parent=interacted_with_parent,diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.pyindex 501ffcff1..f456f545c 100644--- a/skyvern/webeye/browser_factory.py+++ b/skyvern/webeye/browser_factory.py@@ -419,7 +419,7 @@ class BrowserState:async def stop_page_loading(self) -> None:page = await self.__assert_page()try:- await page.evaluate("window.stop()")+ await SkyvernFrame.evaluate(frame=page, expression="window.stop()")except Exception as e:LOG.exception(f"Error while stop loading the page: {repr(e)}")raise FailedToStopLoadingPage(url=page.url, error_message=repr(e))diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 4b4411ad2..2200a2c2e 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -284,7 +284,7 @@ async def get_frame_text(iframe: Frame) -> str:js_script = "() => document.body.innerText"try:- text = await iframe.evaluate(js_script)+ text = await SkyvernFrame.evaluate(frame=iframe, expression=js_script)except Exception:LOG.warning("failed to get text from iframe",@@ -417,8 +417,10 @@ async def get_interactable_element_tree_in_frame(frame_js_script = f"() => buildTreeFromBody('{unique_id}')"- await frame.evaluate(JS_FUNCTION_DEFS)- frame_elements, frame_element_tree = await frame.evaluate(frame_js_script)+ await SkyvernFrame.evaluate(frame=frame, expression=JS_FUNCTION_DEFS)+ frame_elements, frame_element_tree = await SkyvernFrame.evaluate(+ frame=frame, expression=frame_js_script, timeout_ms=60 * 1000+ )if len(frame.child_frames) > 0:frame_elements, frame_element_tree = await get_interactable_element_tree_in_frame(@@ -450,9 +452,11 @@ async def get_interactable_element_tree(:param page: Page instance to get the element tree from.:return: Tuple containing the element tree and a map of element IDs to elements."""- await page.evaluate(JS_FUNCTION_DEFS)+ await SkyvernFrame.evaluate(frame=page, expression=JS_FUNCTION_DEFS)main_frame_js_script = "() => buildTreeFromBody()"- elements, element_tree = await page.evaluate(main_frame_js_script)+ elements, element_tree = await SkyvernFrame.evaluate(+ frame=page, expression=main_frame_js_script, timeout_ms=60 * 1000+ )if len(page.main_frame.child_frames) > 0:elements, element_tree = await get_interactable_element_tree_in_frame(@@ -481,7 +485,9 @@ class IncrementalScrapePage:frame = self.skyvern_frame.get_frame()js_script = "() => getIncrementElements()"- incremental_elements, incremental_tree = await frame.evaluate(js_script)+ incremental_elements, incremental_tree = await SkyvernFrame.evaluate(+ frame=frame, expression=js_script, timeout_ms=60 * 1000+ )# we listen the incremental elements seperated by frames, so all elements will be in the same SkyvernFrameself.id_to_css_dict, self.id_to_element_dict, _, _, _ = build_element_dict(incremental_elements)@@ -497,15 +503,15 @@ class IncrementalScrapePage:async def start_listen_dom_increment(self) -> None:js_script = "() => startGlobalIncrementalObserver()"- await self.skyvern_frame.get_frame().evaluate(js_script)+ await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script)async def stop_listen_dom_increment(self) -> None:js_script = "() => stopGlobalIncrementalObserver()"- await self.skyvern_frame.get_frame().evaluate(js_script)+ await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script)async def get_incremental_elements_num(self) -> int:js_script = "() => window.globalOneTimeIncrementElements.length"- return await self.skyvern_frame.get_frame().evaluate(js_script)+ return await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script)async def __validate_element_by_value(self, value: str, element: dict) -> tuple[Locator | None, bool]:"""diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex 3ef74c770..6673b48b4 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -213,6 +213,9 @@ class SkyvernElement:return False+ async def is_file_input(self) -> bool:+ return self.get_tag_name() == InteractiveElement.INPUT and await self.get_attr("type") == "file"+def is_interactable(self) -> bool:return self.__static_element.get("interactable", False)@@ -507,7 +510,9 @@ class SkyvernElement:await page.mouse.click(click_x, click_y)async def blur(self) -> None:- await self.get_frame().evaluate("(element) => element.blur()", await self.get_element_handler())+ await SkyvernFrame.evaluate(+ frame=self.get_frame(), expression="(element) => element.blur()", arg=await self.get_element_handler()+ )async def scroll_into_view(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:element_handler = await self.get_element_handler(timeout=timeout)diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex 3d28e9e57..b91e0ae77 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport asyncioimport time-from typing import Dict, List+from typing import Any, Dict, Listimport structlogfrom playwright._impl._errors import TimeoutError@@ -32,6 +32,20 @@ JS_FUNCTION_DEFS = load_js_script()class SkyvernFrame:+ @staticmethod+ async def evaluate(+ frame: Page | Frame,+ expression: str,+ arg: Any | None = None,+ timeout_ms: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,+ ) -> Any:+ try:+ async with asyncio.timeout(timeout_ms / 1000):+ return await frame.evaluate(expression=expression, arg=arg)+ except asyncio.TimeoutError:+ LOG.exception("Timeout to evaluate expression", expression=expression)+ raise TimeoutError("timeout to evaluate expression")+@staticmethodasync def take_screenshot(page: Page,@@ -123,7 +137,7 @@ class SkyvernFrame:@classmethodasync def create_instance(cls, frame: Page | Frame) -> SkyvernFrame:instance = cls(frame=frame)- await instance.frame.evaluate(JS_FUNCTION_DEFS)+ await cls.evaluate(frame=instance.frame, expression=JS_FUNCTION_DEFS)return instancedef __init__(self, frame: Page | Frame) -> None:@@ -138,35 +152,35 @@ class SkyvernFrame:async def get_scroll_x_y(self) -> tuple[int, int]:js_script = "() => getScrollXY()"- return await self.frame.evaluate(js_script)+ return await self.evaluate(frame=self.frame, expression=js_script)async def scroll_to_x_y(self, x: int, y: int) -> None:js_script = "([x, y]) => scrollToXY(x, y)"- return await self.frame.evaluate(js_script, [x, y])+ return await self.evaluate(frame=self.frame, expression=js_script, arg=[x, y])async def scroll_to_element_bottom(self, element: ElementHandle, page_by_page: bool = False) -> None:js_script = "([element, page_by_page]) => scrollToElementBottom(element, page_by_page)"- return await self.frame.evaluate(js_script, [element, page_by_page])+ return await self.evaluate(frame=self.frame, expression=js_script, arg=[element, page_by_page])async def scroll_to_element_top(self, element: ElementHandle) -> None:js_script = "(element) => scrollToElementTop(element)"- return await self.frame.evaluate(js_script, element)+ return await self.evaluate(frame=self.frame, expression=js_script, arg=element)async def parse_element_from_html(self, frame: str, element: ElementHandle, interactable: bool) -> Dict:js_script = "([frame, element, interactable]) => buildElementObject(frame, element, interactable)"- return await self.frame.evaluate(js_script, [frame, element, interactable])+ return await self.evaluate(frame=self.frame, expression=js_script, arg=[frame, element, interactable])async def get_element_scrollable(self, element: ElementHandle) -> bool:js_script = "(element) => isScrollable(element)"- return await self.frame.evaluate(js_script, element)+ return await self.evaluate(frame=self.frame, expression=js_script, arg=element)async def get_element_visible(self, element: ElementHandle) -> bool:js_script = "(element) => isElementVisible(element) && !isHidden(element)"- return await self.frame.evaluate(js_script, element)+ return await self.evaluate(frame=self.frame, expression=js_script, arg=element)async def get_disabled_from_style(self, element: ElementHandle) -> bool:js_script = "(element) => checkDisabledFromStyle(element)"- return await self.frame.evaluate(js_script, element)+ return await self.evaluate(frame=self.frame, expression=js_script, arg=element)async def scroll_to_top(self, draw_boxes: bool) -> float:"""@@ -176,7 +190,7 @@ class SkyvernFrame::return: Screenshot of the page."""js_script = f"() => scrollToTop({str(draw_boxes).lower()})"- scroll_y_px = await self.frame.evaluate(js_script)+ scroll_y_px = await self.evaluate(frame=self.frame, expression=js_script)return scroll_y_pxasync def scroll_to_next_page(self, draw_boxes: bool) -> float:@@ -187,7 +201,7 @@ class SkyvernFrame::return: Screenshot of the page."""js_script = f"() => scrollToNextPage({str(draw_boxes).lower()})"- scroll_y_px = await self.frame.evaluate(js_script)+ scroll_y_px = await self.evaluate(frame=self.frame, expression=js_script)return scroll_y_pxasync def remove_bounding_boxes(self) -> None:@@ -196,12 +210,12 @@ class SkyvernFrame::param page: Page instance to remove the bounding boxes from."""js_script = "() => removeBoundingBoxes()"- await self.frame.evaluate(js_script)+ await self.evaluate(frame=self.frame, expression=js_script)async def build_elements_and_draw_bounding_boxes(self) -> None:js_script = "() => buildElementsAndDrawBoundingBoxes()"- await self.frame.evaluate(js_script)+ await self.evaluate(frame=self.frame, expression=js_script)async def is_window_scrollable(self) -> bool:js_script = "() => isWindowScrollable()"- return await self.frame.evaluate(js_script)+ return await self.evaluate(frame=self.frame, expression=js_script)