Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-8818: progressive wait_until for file_download navigation + low max_steps observability (#5431)
Aaron Perez5750f736de2a
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex f0168143c..5479633a4 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -122,6 +122,39 @@ from skyvern.webeye.utils.page import SkyvernFrameLOG = structlog.get_logger()++# SKY-8818: observability threshold for under-configured file_download blocks.+# Warning fires when `max_steps_per_run` is set below this value. Not a behavior change —+# purely a Datadog-searchable signal (`log_code=file_download_low_max_steps`).+MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD = 5+++def warn_if_file_download_max_steps_low(+ block: BaseTaskBlock,+ workflow_run_id: str | None = None,+) -> None:+ """Emit a structured warning if a file_download block has an under-configured step budget.++ A `max_steps_per_run` of None means "use org default", which is not a misconfiguration.+ Only configured values strictly below MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD warn.+ """+ if block.block_type != BlockType.FILE_DOWNLOAD:+ return+ configured = block.max_steps_per_run+ if configured is None:+ return+ if configured >= MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD:+ return+ LOG.warning(+ "file_download block configured with low max_steps_per_run",+ log_code="file_download_low_max_steps",+ block_label=block.label,+ max_steps_per_run=configured,+ recommended_minimum=MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD,+ workflow_run_id=workflow_run_id,+ )++if settings.WORKFLOW_TEMPLATING_STRICTNESS == "strict":jinja_sandbox_env = SandboxedEnvironment(undefined=StrictUndefined)else:@@ -870,6 +903,14 @@ class BaseTaskBlock(Block):organization_id=organization_id,)+ # SKY-8818: observability + wait_until override. Computed ONCE per block+ # execution — hoisted outside the retry loop so the Datadog signal counts+ # block runs, not retries, and `_navigate_wait_until` is a pure function of+ # self.block_type (which does not change between retries).+ warn_if_file_download_max_steps_low(self, workflow_run_id=workflow_run_id)+ _is_file_download = self.block_type == BlockType.FILE_DOWNLOAD+ _navigate_wait_until = "domcontentloaded" if _is_file_download else "load"+# TODO (kerem) we should always retry on terminated. We should make a distinction between retriable and# non-retryable terminationswhile will_retry:@@ -899,9 +940,13 @@ class BaseTaskBlock(Block):if is_first_task:# the first task block will create the browser state and do the navigationtry:+ # SKY-8818: for file_download blocks, skip the browser factory's built-in+ # goto (which uses wait_until='load' and stalls on slow subresources) and+ # let the about:blank fallback below handle navigation with our override.+ _bm_url = None if _is_file_download else self.urlbrowser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(workflow_run=workflow_run,- url=self.url,+ url=_bm_url,browser_session_id=browser_session_id,browser_profile_id=workflow_run.browser_profile_id,)@@ -912,8 +957,21 @@ class BaseTaskBlock(Block):workflow_run_id=workflow_run.workflow_run_id,)raise MissingBrowserStatePage(workflow_run_id=workflow_run.workflow_run_id)- if working_page.url == "about:blank" and self.url:- await browser_state.navigate_to_url(page=working_page, url=self.url)+ # SKY-8818: for file_download we passed url=None above so the factory+ # skipped its built-in goto. We must therefore navigate explicitly to+ # self.url — not just when the page is about:blank, but whenever the+ # working page is not already on the target URL (e.g. persistent+ # browser sessions that carry state from a prior block).+ if self.url:+ _needs_navigation = working_page.url == "about:blank" or (+ _is_file_download and working_page.url.rstrip("/") != self.url.rstrip("/")+ )+ if _needs_navigation:+ await browser_state.navigate_to_url(+ page=working_page,+ url=self.url,+ wait_until=_navigate_wait_until,+ )# When a browser profile is loaded, wait for the page to fully settle# so that cookie-based authentication can redirect or restore the session@@ -990,7 +1048,13 @@ class BaseTaskBlock(Block):step_id=step.step_id,)try:- await browser_state.navigate_to_url(page=working_page, url=self.url)+ # SKY-8818: use the hoisted wait_until override so file_download+ # pages with slow subresources can still resolve via domcontentloaded.+ await browser_state.navigate_to_url(+ page=working_page,+ url=self.url,+ wait_until=_navigate_wait_until,+ )except Exception as e:await self._handle_task_failure_with_error_detection(task=task,diff --git a/skyvern/webeye/browser_state.py b/skyvern/webeye/browser_state.pyindex 68aeb4923..c1ee2d80e 100644--- a/skyvern/webeye/browser_state.py+++ b/skyvern/webeye/browser_state.py@@ -38,7 +38,13 @@ class BrowserState(Protocol):async def set_working_page(self, page: Page | None, index: int = 0) -> None: ...- async def navigate_to_url(self, page: Page, url: str, retry_times: int = NAVIGATION_MAX_RETRY_TIME) -> None: ...+ async def navigate_to_url(+ self,+ page: Page,+ url: str,+ retry_times: int = NAVIGATION_MAX_RETRY_TIME,+ wait_until: str = "load",+ ) -> None: ...async def get_or_create_page(self,diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.pyindex 3a2c34faa..ee32c40b3 100644--- a/skyvern/webeye/real_browser_state.py+++ b/skyvern/webeye/real_browser_state.py@@ -130,21 +130,54 @@ class RealBrowserState(BrowserState):total_wait_ms += random.randint(0, SETTLE_JITTER_MS)await asyncio.sleep(total_wait_ms / 1000)- async def navigate_to_url(self, page: Page, url: str, retry_times: int = NAVIGATION_MAX_RETRY_TIME) -> None:+ async def navigate_to_url(+ self,+ page: Page,+ url: str,+ retry_times: int = NAVIGATION_MAX_RETRY_TIME,+ wait_until: str = "load",+ ) -> None:+ # SKY-8818: progressive wait_until degradation. Many pages never fire+ # `load` because a subresource stalls; degrading to `domcontentloaded` and+ # then `commit` lets navigation succeed once the DOM or response is ready.+ # Monotonic: a retry never upgrades to a STRONGER wait state than the caller asked for.+ _degradation_map: dict[str, list[str]] = {+ "load": ["load", "domcontentloaded", "commit"],+ "domcontentloaded": ["domcontentloaded", "commit"],+ "commit": ["commit"],+ }+ degradation = _degradation_map.get(wait_until, [wait_until])+try:for retry_time in range(retry_times):- LOG.info("Trying to navigate to url", url=url, retry_time=retry_time)+ strategy = degradation[min(retry_time, len(degradation) - 1)]+ LOG.info(+ "Trying to navigate to url",+ url=url,+ retry_time=retry_time,+ wait_until=strategy,+ )try:start_time = time.time()- await page.goto(url, timeout=settings.BROWSER_LOADING_TIMEOUT_MS)+ await page.goto(+ url,+ timeout=settings.BROWSER_LOADING_TIMEOUT_MS,+ wait_until=strategy,+ )end_time = time.time()LOG.info("Page loading time",loading_time=end_time - start_time,url=url,+ wait_until=strategy,)await self._wait_for_settle()- LOG.info("Successfully navigated to url", url=url, retry_time=retry_time)+ LOG.info(+ "Successfully navigated to url",+ url=url,+ retry_time=retry_time,+ wait_until=strategy,+ )returnexcept Exception as e:@@ -156,6 +189,7 @@ class RealBrowserState(BrowserState):exc_info=True,url=url,retry_time=retry_time,+ wait_until=strategy,)# Wait for 1 seconds before retryingawait asyncio.sleep(1)diff --git a/tests/unit/test_file_download_block_warning.py b/tests/unit/test_file_download_block_warning.pynew file mode 100644index 000000000..648176ef0--- /dev/null+++ b/tests/unit/test_file_download_block_warning.py@@ -0,0 +1,57 @@+"""Unit tests for SKY-8818 observability warning on under-configured file_download blocks.++When a FileDownloadBlock is configured with `max_steps_per_run < MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD`,+the block must emit a structured LOG.warning with `log_code='file_download_low_max_steps'` at block+start, so Datadog can surface under-configured workflows.+"""++from unittest.mock import patch++import pytest++from skyvern.forge.sdk.workflow.models.block import (+ MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD,+ FileDownloadBlock,+ warn_if_file_download_max_steps_low,+)+++def _make_block(max_steps: int | None) -> FileDownloadBlock:+ return FileDownloadBlock.model_construct(+ label="download_files",+ max_steps_per_run=max_steps,+ )+++def test_threshold_constant_is_five() -> None:+ """Anchor the threshold so tuning is a deliberate change."""+ assert MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD == 5++[email protected]("configured", [1, 2, 3, 4])+def test_warning_fires_below_threshold(configured: int) -> None:+ block = _make_block(configured)+ with patch("skyvern.forge.sdk.workflow.models.block.LOG") as mock_log:+ warn_if_file_download_max_steps_low(block, workflow_run_id="wr_test")+ assert mock_log.warning.call_count == 1+ _, kwargs = mock_log.warning.call_args+ assert kwargs.get("log_code") == "file_download_low_max_steps"+ assert kwargs.get("max_steps_per_run") == configured+ assert kwargs.get("recommended_minimum") == MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD+ assert kwargs.get("workflow_run_id") == "wr_test"++[email protected]("configured", [5, 6, 10, 25])+def test_warning_does_not_fire_at_or_above_threshold(configured: int) -> None:+ block = _make_block(configured)+ with patch("skyvern.forge.sdk.workflow.models.block.LOG") as mock_log:+ warn_if_file_download_max_steps_low(block, workflow_run_id="wr_test")+ assert mock_log.warning.call_count == 0+++def test_warning_does_not_fire_when_max_steps_unset() -> None:+ """max_steps_per_run=None means the org-level default applies — not a misconfiguration signal."""+ block = _make_block(None)+ with patch("skyvern.forge.sdk.workflow.models.block.LOG") as mock_log:+ warn_if_file_download_max_steps_low(block, workflow_run_id="wr_test")+ assert mock_log.warning.call_count == 0diff --git a/tests/unit/test_real_browser_state_navigate.py b/tests/unit/test_real_browser_state_navigate.pynew file mode 100644index 000000000..965222962--- /dev/null+++ b/tests/unit/test_real_browser_state_navigate.py@@ -0,0 +1,112 @@+"""Unit tests for skyvern.webeye.real_browser_state.navigate_to_url.++Covers SKY-8818: pages whose subresources never finish loading must still+succeed if the DOM has parsed, via progressive wait_until degradation.+"""++from unittest.mock import AsyncMock, MagicMock++import pytest+from playwright.async_api import TimeoutError as PlaywrightTimeoutError++from skyvern.exceptions import FailedToNavigateToUrl+from skyvern.webeye.real_browser_state import RealBrowserState+++def browser_state() -> RealBrowserState:+ # Bypass __init__; navigate_to_url only uses `self` for LOG context and _wait_for_settle.+ state = RealBrowserState.__new__(RealBrowserState)+ return state+++async def test_navigate_to_url_progresses_from_load_to_domcontentloaded(+ browser_state: RealBrowserState,+) -> None:+ """If wait_until='load' times out but 'domcontentloaded' succeeds, we succeed."""+ page = MagicMock()+ calls: list[str] = []++ async def fake_goto(url: str, timeout: int, wait_until: str = "load") -> None:+ calls.append(wait_until)+ if wait_until == "load":+ raise PlaywrightTimeoutError("Page.goto: Timeout 60000ms exceeded (load)")+ return None++ page.goto = AsyncMock(side_effect=fake_goto)++ await browser_state.navigate_to_url(+ page=page,+ url="https://example.test/slow-subresources",+ wait_until="load",+ )++ assert "load" in calls+ assert "domcontentloaded" in calls+ assert calls.index("domcontentloaded") > calls.index("load")+++async def test_navigate_to_url_raises_when_all_strategies_fail(+ browser_state: RealBrowserState,+) -> None:+ """If every wait_until strategy times out, raise FailedToNavigateToUrl."""+ page = MagicMock()++ async def always_timeout(url: str, timeout: int, wait_until: str = "load") -> None:+ raise PlaywrightTimeoutError(f"Page.goto: Timeout 60000ms exceeded ({wait_until})")++ page.goto = AsyncMock(side_effect=always_timeout)++ with pytest.raises(FailedToNavigateToUrl):+ await browser_state.navigate_to_url(+ page=page,+ url="https://example.test/fully-dead",+ wait_until="load",+ )+++async def test_navigate_to_url_honors_caller_supplied_wait_until_on_first_try(+ browser_state: RealBrowserState,+) -> None:+ """FileDownloadBlock passes wait_until='domcontentloaded' — it must be honored on first try."""+ page = MagicMock()+ calls: list[str] = []++ async def fake_goto(url: str, timeout: int, wait_until: str = "load") -> None:+ calls.append(wait_until)+ return None++ page.goto = AsyncMock(side_effect=fake_goto)++ await browser_state.navigate_to_url(+ page=page,+ url="https://example.test/fast",+ wait_until="domcontentloaded",+ )++ assert calls == ["domcontentloaded"]+++async def test_navigate_to_url_succeeds_on_first_try_with_default_load(+ browser_state: RealBrowserState,+) -> None:+ """Existing callers that use default wait_until='load' must keep working untouched."""+ page = MagicMock()+ calls: list[str] = []++ async def fake_goto(url: str, timeout: int, wait_until: str = "load") -> None:+ calls.append(wait_until)+ return None++ page.goto = AsyncMock(side_effect=fake_goto)++ await browser_state.navigate_to_url(+ page=page,+ url="https://example.test/fast-load",+ )++ assert calls == ["load"]