Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
add support_empty_page and wait_seconds to the scrape_website interface (#3181)
Shuchang Zheng434bbff459f1
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 1f10e8e98..08e509c68 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -405,6 +405,8 @@ async def scrape_website(draw_boxes: bool = True,max_screenshot_number: int = settings.MAX_NUM_SCREENSHOTS,scroll: bool = True,+ support_empty_page: bool = False,+ wait_seconds: float = 3,) -> ScrapedPage:"""************************************************************************************************@@ -439,6 +441,8 @@ async def scrape_website(draw_boxes=draw_boxes,max_screenshot_number=max_screenshot_number,scroll=scroll,+ support_empty_page=support_empty_page,+ wait_seconds=wait_seconds,)except ScrapingFailedBlankPage:raise@@ -517,6 +521,8 @@ async def scrape_web_unsafe(draw_boxes: bool = True,max_screenshot_number: int = settings.MAX_NUM_SCREENSHOTS,scroll: bool = True,+ support_empty_page: bool = False,+ wait_seconds: float = 3,) -> ScrapedPage:"""Asynchronous function that performs web scraping without any built-in error handling. This function is intended@@ -538,11 +544,11 @@ async def scrape_web_unsafe(# This also solves the issue where we can't scroll due to a popup.(e.g. geico first popup on the homepage after# clicking start my quote)url = page.url- if url == "about:blank":+ if url == "about:blank" and not support_empty_page:raise ScrapingFailedBlankPage()- LOG.info("Waiting for 3 seconds before scraping the website.")- await asyncio.sleep(3)+ LOG.info(f"Waiting for {wait_seconds} seconds before scraping the website.")+ await asyncio.sleep(wait_seconds)elements, element_tree = await get_interactable_element_tree(page, scrape_exclude)element_tree = await cleanup_element_tree(page, url, copy.deepcopy(element_tree))@@ -569,7 +575,7 @@ async def scrape_web_unsafe()# if there are no elements, fail the scraping- if not elements:+ if not elements and not support_empty_page:raise Exception("No elements found on the page")text_content = await get_frame_text(page.main_frame)