Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
make scraping timeout configurable (#2991)
LawyZhengfcd22017b799
diff --git a/skyvern/config.py b/skyvern/config.pyindex 0fa94e436..f4bebfa9e 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -23,6 +23,7 @@ class Settings(BaseSettings):BROWSER_ACTION_TIMEOUT_MS: int = 5000BROWSER_SCREENSHOT_TIMEOUT_MS: int = 20000BROWSER_LOADING_TIMEOUT_MS: int = 90000+ BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS: int = 60 * 1000 # 1 minuteOPTION_LOADING_TIMEOUT_MS: int = 600000MAX_STEPS_PER_RUN: int = 10MAX_STEPS_PER_TASK_V2: int = 25diff --git a/skyvern/constants.py b/skyvern/constants.pyindex 54f63131d..283e2179e 100644--- a/skyvern/constants.py+++ b/skyvern/constants.py@@ -8,7 +8,6 @@ REPO_ROOT_DIR = SKYVERN_DIR.parentINPUT_TEXT_TIMEOUT = 120000 # 2 minutesPAGE_CONTENT_TIMEOUT = 300 # 5 mins-BUILDING_ELEMENT_TREE_TIMEOUT_MS = 60 * 1000 # 1 minuteBROWSER_CLOSE_TIMEOUT = 180 # 3 minuteBROWSER_DOWNLOAD_MAX_WAIT_TIME = 1200 # 20 minuteBROWSER_DOWNLOAD_TIMEOUT = 600 # 10 minutediff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex f6cba3104..7b77eaf20 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -12,10 +12,11 @@ from playwright.async_api import ElementHandle, Frame, Locator, Pagefrom pydantic import BaseModel, PrivateAttrfrom skyvern.config import settings-from skyvern.constants import BUILDING_ELEMENT_TREE_TIMEOUT_MS, DEFAULT_MAX_TOKENS, SKYVERN_DIR, SKYVERN_ID_ATTR+from skyvern.constants import DEFAULT_MAX_TOKENS, SKYVERN_DIR, SKYVERN_ID_ATTRfrom skyvern.exceptions import FailedToTakeScreenshot, ScrapingFailed, UnknownElementTreeFormatfrom skyvern.forge.sdk.api.crypto import calculate_sha256from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.trace import TraceManagerfrom skyvern.utils.image_resizer import Resolutionfrom skyvern.utils.token_counter import count_tokens@@ -765,7 +766,9 @@ class IncrementalScrapePage(ElementTreeBuilder):returnjs_script = "async () => await stopGlobalIncrementalObserver()"await SkyvernFrame.evaluate(- frame=self.skyvern_frame.get_frame(), expression=js_script, timeout_ms=BUILDING_ELEMENT_TREE_TIMEOUT_MS+ frame=self.skyvern_frame.get_frame(),+ expression=js_script,+ timeout_ms=SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,)async def get_incremental_elements_num(self) -> int:diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex f9e885c34..319632097 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -11,7 +11,7 @@ from PIL import Imagefrom playwright._impl._errors import TimeoutErrorfrom playwright.async_api import ElementHandle, Frame, Page-from skyvern.constants import BUILDING_ELEMENT_TREE_TIMEOUT_MS, PAGE_CONTENT_TIMEOUT, SKYVERN_DIR+from skyvern.constants import PAGE_CONTENT_TIMEOUT, SKYVERN_DIRfrom skyvern.exceptions import FailedToTakeScreenshotfrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.trace import TraceManager@@ -376,7 +376,7 @@ class SkyvernFrame:scroll_y_px = await self.evaluate(frame=self.frame,expression=js_script,- timeout_ms=BUILDING_ELEMENT_TREE_TIMEOUT_MS,+ timeout_ms=SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,arg=[draw_boxes, frame, frame_index],)return scroll_y_px@@ -394,7 +394,7 @@ class SkyvernFrame:scroll_y_px = await self.evaluate(frame=self.frame,expression=js_script,- timeout_ms=BUILDING_ELEMENT_TREE_TIMEOUT_MS,+ timeout_ms=SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,arg=[draw_boxes, frame, frame_index, need_overlap],)return scroll_y_px@@ -405,14 +405,18 @@ class SkyvernFrame::param page: Page instance to remove the bounding boxes from."""js_script = "() => removeBoundingBoxes()"- await self.evaluate(frame=self.frame, expression=js_script, timeout_ms=BUILDING_ELEMENT_TREE_TIMEOUT_MS)+ await self.evaluate(+ frame=self.frame,+ expression=js_script,+ timeout_ms=SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,+ )async def build_elements_and_draw_bounding_boxes(self, frame: str, frame_index: int) -> None:js_script = "async ([frame, frame_index]) => await buildElementsAndDrawBoundingBoxes(frame, frame_index)"await self.evaluate(frame=self.frame,expression=js_script,- timeout_ms=BUILDING_ELEMENT_TREE_TIMEOUT_MS,+ timeout_ms=SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,arg=[frame, frame_index],)@@ -446,7 +450,10 @@ class SkyvernFrame:@TraceManager.traced_async()async def build_tree_from_body(- self, frame_name: str | None, frame_index: int, timeout_ms: float = BUILDING_ELEMENT_TREE_TIMEOUT_MS+ self,+ frame_name: str | None,+ frame_index: int,+ timeout_ms: float = SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,) -> tuple[list[dict], list[dict]]:js_script = "async ([frame_name, frame_index]) => await buildTreeFromBody(frame_name, frame_index)"return await self.evaluate(@@ -455,7 +462,9 @@ class SkyvernFrame:@TraceManager.traced_async()async def get_incremental_element_tree(- self, wait_until_finished: bool = True, timeout_ms: float = BUILDING_ELEMENT_TREE_TIMEOUT_MS+ self,+ wait_until_finished: bool = True,+ timeout_ms: float = SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,) -> tuple[list[dict], list[dict]]:js_script = "async ([wait_until_finished]) => await getIncrementElements(wait_until_finished)"return await self.evaluate(