Skip to content

Automate browser based workflows with AI

fix(SKY-10476): add waitUntil degradation to extraction scrape reload (#6304)

LawyZheng3a0d67ae1483
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 0ba9cd00a..e13ee0641 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -48,6 +48,7 @@ from skyvern.exceptions import (
FailedToGetTOTPVerificationCode,
FailedToNavigateToUrl,
FailedToParseActionInstruction,
+ FailedToReloadPage,
FailedToSendWebhook,
FailedToTakeScreenshot,
InvalidTaskStatusTransition,
@@ -3138,7 +3139,7 @@ class ForgeAgent:
await browser_state.stop_page_loading()
elif scrape_type == ScrapeType.RELOAD:
LOG.info("Try to reload the page before scraping")
- await browser_state.reload_page()
+ await browser_state.reload_page(degradation=True)
max_screenshot_number = settings.MAX_NUM_SCREENSHOTS
draw_boxes = True
@@ -3287,7 +3288,7 @@ class ForgeAgent:
engine=engine,
)
break
- except (FailedToTakeScreenshot, ScrapingFailed) as e:
+ except (FailedToTakeScreenshot, ScrapingFailed, FailedToReloadPage) as e:
if idx < len(SCRAPE_TYPE_ORDER) - 1:
LOG.warning(
"Scrape attempt failed, will retry with next strategy",
diff --git a/skyvern/webeye/browser_state.py b/skyvern/webeye/browser_state.py
index 78d14dbcc..fe55e0b55 100644
--- a/skyvern/webeye/browser_state.py
+++ b/skyvern/webeye/browser_state.py
@@ -72,7 +72,7 @@ class BrowserState(Protocol):
async def new_page(self) -> Page: ...
- async def reload_page(self) -> None: ...
+ async def reload_page(self, degradation: bool = False) -> None: ...
async def close(self, close_browser_on_completion: bool = True) -> None: ...
diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.py
index d3d8db1f8..4cd04c143 100644
--- a/skyvern/webeye/real_browser_state.py
+++ b/skyvern/webeye/real_browser_state.py
@@ -346,23 +346,50 @@ class RealBrowserState(BrowserState):
raise EmptyBrowserContext()
return await self.browser_context.new_page()
- async def reload_page(self) -> None:
+ async def reload_page(self, degradation: bool = False) -> None:
page = await self.__assert_page()
+ url = page.url
- LOG.info("Reload page", url=page.url)
- try:
- start_time = time.time()
- await page.reload(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)
- end_time = time.time()
- LOG.info(
- "Page loading time",
- loading_time=end_time - start_time,
- )
- await self._wait_for_settle()
- await self._wait_for_challenge_solver(page=page)
- except Exception as e:
- LOG.exception(f"Error while reload url: {repr(e)}")
- raise FailedToReloadPage(url=page.url, error_message=repr(e))
+ if not degradation:
+ LOG.info("Reload page", url=url)
+ try:
+ start_time = time.time()
+ await page.reload(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)
+ LOG.info("Page loading time", loading_time=time.time() - start_time)
+ await self._wait_for_settle()
+ await self._wait_for_challenge_solver(page=page)
+ except Exception as e:
+ LOG.exception("Error while reload url", error=repr(e))
+ raise FailedToReloadPage(url=url, error_message=repr(e))
+ return
+
+ strategies: list[str] = ["load", "domcontentloaded", "commit"]
+ for i, strategy in enumerate(strategies):
+ try:
+ LOG.info("Reload page", url=url, wait_until=strategy, degradation_attempt=i)
+ start_time = time.time()
+ await page.reload(timeout=settings.BROWSER_LOADING_TIMEOUT_MS, wait_until=strategy)
+ LOG.info(
+ "Page loading time",
+ loading_time=time.time() - start_time,
+ wait_until=strategy,
+ degraded=i > 0,
+ )
+ await self._wait_for_settle()
+ await self._wait_for_challenge_solver(page=page)
+ return
+ except Exception as e:
+ if i < len(strategies) - 1:
+ LOG.warning(
+ "Reload timed out, degrading wait strategy",
+ url=url,
+ wait_until=strategy,
+ next_strategy=strategies[i + 1],
+ error=repr(e),
+ )
+ continue
+ LOG.exception("Error while reload url after degradation", error=repr(e))
+ raise FailedToReloadPage(url=url, error_message=repr(e))
async def scrape_website(
self,
diff --git a/tests/unit/test_real_browser_state_reload.py b/tests/unit/test_real_browser_state_reload.py
new file mode 100644
index 000000000..b8041fcd9
--- /dev/null
+++ b/tests/unit/test_real_browser_state_reload.py
@@ -0,0 +1,192 @@
+"""Unit tests for RealBrowserState.reload_page degradation and scoped usage.
+
+Covers SKY-10476: extraction scrape reload must degrade through
+load → domcontentloaded → commit instead of hard-failing on SPA pages.
+Degradation is scoped to extraction tasks only.
+"""
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+
+from skyvern.constants import ScrapeType
+from skyvern.exceptions import FailedToReloadPage
+from skyvern.forge.agent import ForgeAgent
+from skyvern.webeye.real_browser_state import RealBrowserState
+
+_AGENT_MODULE = "skyvern.forge.agent"
+
+
+def browser_state() -> RealBrowserState:
+ state = RealBrowserState.__new__(RealBrowserState)
+ return state
+
+
+def _make_page(reload_side_effect=None) -> MagicMock:
+ page = MagicMock()
+ page.url = "https://example.test/spa"
+ page.reload = AsyncMock(side_effect=reload_side_effect)
+ return page
+
+
+async def test_reload_page_default_raises_on_timeout(browser_state: RealBrowserState) -> None:
+ """Default reload_page (no degradation) raises FailedToReloadPage on timeout — existing behavior."""
+ page = _make_page(PlaywrightTimeoutError("Page.reload: Timeout 60000ms exceeded"))
+
+ with patch.object(browser_state, "_RealBrowserState__assert_page", return_value=page):
+ with pytest.raises(FailedToReloadPage):
+ await browser_state.reload_page()
+
+ page.reload.assert_called_once()
+ call_kwargs = page.reload.call_args
+ assert "wait_until" not in (call_kwargs.kwargs or {})
+
+
+async def test_reload_page_default_succeeds_unchanged(browser_state: RealBrowserState) -> None:
+ """Default reload_page succeeds without passing wait_until — no behavior change."""
+ page = _make_page()
+ browser_state._wait_for_settle = AsyncMock()
+ browser_state._wait_for_challenge_solver = AsyncMock()
+
+ with patch.object(browser_state, "_RealBrowserState__assert_page", return_value=page):
+ await browser_state.reload_page()
+
+ page.reload.assert_called_once()
+ call_kwargs = page.reload.call_args
+ assert "wait_until" not in (call_kwargs.kwargs or {})
+
+
+async def test_reload_page_degradation_succeeds_on_domcontentloaded(browser_state: RealBrowserState) -> None:
+ """Degradation mode: load times out, domcontentloaded succeeds."""
+ strategies_tried: list[str] = []
+
+ async def fake_reload(timeout: int, wait_until: str = "load") -> None:
+ strategies_tried.append(wait_until)
+ if wait_until == "load":
+ raise PlaywrightTimeoutError("Page.reload: Timeout 60000ms exceeded")
+
+ page = _make_page(fake_reload)
+ browser_state._wait_for_settle = AsyncMock()
+ browser_state._wait_for_challenge_solver = AsyncMock()
+
+ with patch.object(browser_state, "_RealBrowserState__assert_page", return_value=page):
+ await browser_state.reload_page(degradation=True)
+
+ assert strategies_tried == ["load", "domcontentloaded"]
+
+
+async def test_reload_page_degradation_succeeds_on_commit(browser_state: RealBrowserState) -> None:
+ """Degradation mode: load and domcontentloaded time out, commit succeeds."""
+ strategies_tried: list[str] = []
+
+ async def fake_reload(timeout: int, wait_until: str = "load") -> None:
+ strategies_tried.append(wait_until)
+ if wait_until in ("load", "domcontentloaded"):
+ raise PlaywrightTimeoutError(f"Page.reload: Timeout 60000ms exceeded ({wait_until})")
+
+ page = _make_page(fake_reload)
+ browser_state._wait_for_settle = AsyncMock()
+ browser_state._wait_for_challenge_solver = AsyncMock()
+
+ with patch.object(browser_state, "_RealBrowserState__assert_page", return_value=page):
+ await browser_state.reload_page(degradation=True)
+
+ assert strategies_tried == ["load", "domcontentloaded", "commit"]
+
+
+async def test_reload_page_degradation_raises_when_all_strategies_fail(browser_state: RealBrowserState) -> None:
+ """Degradation mode: all strategies fail, raises FailedToReloadPage."""
+
+ async def always_timeout(timeout: int, wait_until: str = "load") -> None:
+ raise PlaywrightTimeoutError(f"Page.reload: Timeout 60000ms exceeded ({wait_until})")
+
+ page = _make_page(always_timeout)
+
+ with patch.object(browser_state, "_RealBrowserState__assert_page", return_value=page):
+ with pytest.raises(FailedToReloadPage):
+ await browser_state.reload_page(degradation=True)
+
+ assert page.reload.call_count == 3
+
+
+async def test_reload_page_degradation_succeeds_on_first_try(browser_state: RealBrowserState) -> None:
+ """Degradation mode: load succeeds immediately, no degradation needed."""
+ strategies_tried: list[str] = []
+
+ async def fake_reload(timeout: int, wait_until: str = "load") -> None:
+ strategies_tried.append(wait_until)
+
+ page = _make_page(fake_reload)
+ browser_state._wait_for_settle = AsyncMock()
+ browser_state._wait_for_challenge_solver = AsyncMock()
+
+ with patch.object(browser_state, "_RealBrowserState__assert_page", return_value=page):
+ await browser_state.reload_page(degradation=True)
+
+ assert strategies_tried == ["load"]
+
+
+# --- Scrape retry integration: RELOAD always uses degradation ---
+
+
+def _make_agent() -> ForgeAgent:
+ return ForgeAgent.__new__(ForgeAgent)
+
+
+def _make_browser_state_mock() -> MagicMock:
+ bs = MagicMock()
+ bs.reload_page = AsyncMock()
+ bs.scrape_website = AsyncMock(return_value=MagicMock())
+ return bs
+
+
+async def test_scrape_with_type_reload_uses_degradation() -> None:
+ """Scrape retry RELOAD always passes degradation=True."""
+ agent = _make_agent()
+ bs = _make_browser_state_mock()
+ task = MagicMock()
+ task.url = "https://example.test"
+ step = MagicMock()
+ mock_app = MagicMock()
+
+ with patch(f"{_AGENT_MODULE}.app", mock_app):
+ await agent._scrape_with_type(
+ task=task,
+ step=step,
+ browser_state=bs,
+ scrape_type=ScrapeType.RELOAD,
+ engine=MagicMock(),
+ )
+
+ bs.reload_page.assert_called_once_with(degradation=True)
+
+
+async def test_scrape_with_type_normal_no_reload_call() -> None:
+ """NORMAL scrape type does not call reload_page at all."""
+ agent = _make_agent()
+ bs = _make_browser_state_mock()
+ task = MagicMock()
+ task.url = "https://example.test"
+ step = MagicMock()
+ mock_app = MagicMock()
+
+ with patch(f"{_AGENT_MODULE}.app", mock_app):
+ await agent._scrape_with_type(
+ task=task,
+ step=step,
+ browser_state=bs,
+ scrape_type=ScrapeType.NORMAL,
+ engine=MagicMock(),
+ )
+
+ bs.reload_page.assert_not_called()