Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
better failure reason for blank page (#3049)
LawyZhengecc0e2e17ddd
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.pyindex 4759f8608..acd9a0343 100644--- a/skyvern/exceptions.py+++ b/skyvern/exceptions.py@@ -269,9 +269,9 @@ class ScrapingFailed(SkyvernException):super().__init__("Scraping failed.")-class ScrapingFailedNoUrl(ScrapingFailed):+class ScrapingFailedBlankPage(ScrapingFailed):def __init__(self) -> None:- super().__init__(reason="A URL is missing. Please ensure there is a URL for Skyvern to work with.")+ super().__init__(reason="It's a blank page. Please ensure there is a non-blank page for Skyvern to work with.")class WorkflowRunContextNotInitialized(SkyvernException):diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 282e2a759..ff1fe431d 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -1883,7 +1883,7 @@ class ForgeAgent:step_id=step.step_id,exc_info=True,)- raise ScrapingFailed()+ raise eif scraped_page is None:raise EmptyScrapePage()diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex f6459d5ed..33edd0b43 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -13,7 +13,7 @@ from pydantic import BaseModel, PrivateAttrfrom skyvern.config import settingsfrom skyvern.constants import DEFAULT_MAX_TOKENS, SKYVERN_DIR, SKYVERN_ID_ATTR-from skyvern.exceptions import FailedToTakeScreenshot, ScrapingFailed, UnknownElementTreeFormat+from skyvern.exceptions import FailedToTakeScreenshot, ScrapingFailed, ScrapingFailedBlankPage, UnknownElementTreeFormatfrom skyvern.forge.sdk.api.crypto import calculate_sha256from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.settings_manager import SettingsManager@@ -428,11 +428,6 @@ async def scrape_website(:raises Exception: When scraping fails after maximum retries."""- # TODO(jdo) why is this a problem?- # ref: https://skyvern.slack.com/archives/C074UNDSRJM/p1752771256298149- # if not url.strip():- # raise ScrapingFailedNoUrl()-try:num_retry += 1return await scrape_web_unsafe(@@ -445,6 +440,8 @@ async def scrape_website(max_screenshot_number=max_screenshot_number,scroll=scroll,)+ except ScrapingFailedBlankPage:+ raiseexcept Exception as e:# NOTE: MAX_SCRAPING_RETRIES is set to 0 in both staging and productionif num_retry > settings.MAX_SCRAPING_RETRIES:@@ -540,6 +537,9 @@ async def scrape_web_unsafe(# We check if the scroll_y_px_old is the same as scroll_y_px to determine if we have reached the end of the page.# This also solves the issue where we can't scroll due to a popup.(e.g. geico first popup on the homepage after# clicking start my quote)+ url = page.url+ if url == "about:blank":+ raise ScrapingFailedBlankPage()LOG.info("Waiting for 3 seconds before scraping the website.")await asyncio.sleep(3)@@ -598,7 +598,7 @@ async def scrape_web_unsafe(element_tree=element_tree,element_tree_trimmed=element_tree_trimmed,screenshots=screenshots,- url=page.url,+ url=url,html=html,extracted_text=text_content,window_dimension=window_dimension,