Skip to content

Automate browser based workflows with AI

SKY-8818: progressive wait_until for file_download navigation + low max_steps observability (#5431)

Aaron Perez5750f736de2a
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index f0168143c..5479633a4 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -122,6 +122,39 @@ from skyvern.webeye.utils.page import SkyvernFrame
LOG = structlog.get_logger()
+
+# SKY-8818: observability threshold for under-configured file_download blocks.
+# Warning fires when `max_steps_per_run` is set below this value. Not a behavior change —
+# purely a Datadog-searchable signal (`log_code=file_download_low_max_steps`).
+MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD = 5
+
+
+def warn_if_file_download_max_steps_low(
+ block: BaseTaskBlock,
+ workflow_run_id: str | None = None,
+) -> None:
+ """Emit a structured warning if a file_download block has an under-configured step budget.
+
+ A `max_steps_per_run` of None means "use org default", which is not a misconfiguration.
+ Only configured values strictly below MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD warn.
+ """
+ if block.block_type != BlockType.FILE_DOWNLOAD:
+ return
+ configured = block.max_steps_per_run
+ if configured is None:
+ return
+ if configured >= MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD:
+ return
+ LOG.warning(
+ "file_download block configured with low max_steps_per_run",
+ log_code="file_download_low_max_steps",
+ block_label=block.label,
+ max_steps_per_run=configured,
+ recommended_minimum=MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD,
+ workflow_run_id=workflow_run_id,
+ )
+
+
if settings.WORKFLOW_TEMPLATING_STRICTNESS == "strict":
jinja_sandbox_env = SandboxedEnvironment(undefined=StrictUndefined)
else:
@@ -870,6 +903,14 @@ class BaseTaskBlock(Block):
organization_id=organization_id,
)
+ # SKY-8818: observability + wait_until override. Computed ONCE per block
+ # execution — hoisted outside the retry loop so the Datadog signal counts
+ # block runs, not retries, and `_navigate_wait_until` is a pure function of
+ # self.block_type (which does not change between retries).
+ warn_if_file_download_max_steps_low(self, workflow_run_id=workflow_run_id)
+ _is_file_download = self.block_type == BlockType.FILE_DOWNLOAD
+ _navigate_wait_until = "domcontentloaded" if _is_file_download else "load"
+
# TODO (kerem) we should always retry on terminated. We should make a distinction between retriable and
# non-retryable terminations
while will_retry:
@@ -899,9 +940,13 @@ class BaseTaskBlock(Block):
if is_first_task:
# the first task block will create the browser state and do the navigation
try:
+ # SKY-8818: for file_download blocks, skip the browser factory's built-in
+ # goto (which uses wait_until='load' and stalls on slow subresources) and
+ # let the about:blank fallback below handle navigation with our override.
+ _bm_url = None if _is_file_download else self.url
browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(
workflow_run=workflow_run,
- url=self.url,
+ url=_bm_url,
browser_session_id=browser_session_id,
browser_profile_id=workflow_run.browser_profile_id,
)
@@ -912,8 +957,21 @@ class BaseTaskBlock(Block):
workflow_run_id=workflow_run.workflow_run_id,
)
raise MissingBrowserStatePage(workflow_run_id=workflow_run.workflow_run_id)
- if working_page.url == "about:blank" and self.url:
- await browser_state.navigate_to_url(page=working_page, url=self.url)
+ # SKY-8818: for file_download we passed url=None above so the factory
+ # skipped its built-in goto. We must therefore navigate explicitly to
+ # self.url — not just when the page is about:blank, but whenever the
+ # working page is not already on the target URL (e.g. persistent
+ # browser sessions that carry state from a prior block).
+ if self.url:
+ _needs_navigation = working_page.url == "about:blank" or (
+ _is_file_download and working_page.url.rstrip("/") != self.url.rstrip("/")
+ )
+ if _needs_navigation:
+ await browser_state.navigate_to_url(
+ page=working_page,
+ url=self.url,
+ wait_until=_navigate_wait_until,
+ )
# When a browser profile is loaded, wait for the page to fully settle
# so that cookie-based authentication can redirect or restore the session
@@ -990,7 +1048,13 @@ class BaseTaskBlock(Block):
step_id=step.step_id,
)
try:
- await browser_state.navigate_to_url(page=working_page, url=self.url)
+ # SKY-8818: use the hoisted wait_until override so file_download
+ # pages with slow subresources can still resolve via domcontentloaded.
+ await browser_state.navigate_to_url(
+ page=working_page,
+ url=self.url,
+ wait_until=_navigate_wait_until,
+ )
except Exception as e:
await self._handle_task_failure_with_error_detection(
task=task,
diff --git a/skyvern/webeye/browser_state.py b/skyvern/webeye/browser_state.py
index 68aeb4923..c1ee2d80e 100644
--- a/skyvern/webeye/browser_state.py
+++ b/skyvern/webeye/browser_state.py
@@ -38,7 +38,13 @@ class BrowserState(Protocol):
async def set_working_page(self, page: Page | None, index: int = 0) -> None: ...
- async def navigate_to_url(self, page: Page, url: str, retry_times: int = NAVIGATION_MAX_RETRY_TIME) -> None: ...
+ async def navigate_to_url(
+ self,
+ page: Page,
+ url: str,
+ retry_times: int = NAVIGATION_MAX_RETRY_TIME,
+ wait_until: str = "load",
+ ) -> None: ...
async def get_or_create_page(
self,
diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.py
index 3a2c34faa..ee32c40b3 100644
--- a/skyvern/webeye/real_browser_state.py
+++ b/skyvern/webeye/real_browser_state.py
@@ -130,21 +130,54 @@ class RealBrowserState(BrowserState):
total_wait_ms += random.randint(0, SETTLE_JITTER_MS)
await asyncio.sleep(total_wait_ms / 1000)
- async def navigate_to_url(self, page: Page, url: str, retry_times: int = NAVIGATION_MAX_RETRY_TIME) -> None:
+ async def navigate_to_url(
+ self,
+ page: Page,
+ url: str,
+ retry_times: int = NAVIGATION_MAX_RETRY_TIME,
+ wait_until: str = "load",
+ ) -> None:
+ # SKY-8818: progressive wait_until degradation. Many pages never fire
+ # `load` because a subresource stalls; degrading to `domcontentloaded` and
+ # then `commit` lets navigation succeed once the DOM or response is ready.
+ # Monotonic: a retry never upgrades to a STRONGER wait state than the caller asked for.
+ _degradation_map: dict[str, list[str]] = {
+ "load": ["load", "domcontentloaded", "commit"],
+ "domcontentloaded": ["domcontentloaded", "commit"],
+ "commit": ["commit"],
+ }
+ degradation = _degradation_map.get(wait_until, [wait_until])
+
try:
for retry_time in range(retry_times):
- LOG.info("Trying to navigate to url", url=url, retry_time=retry_time)
+ strategy = degradation[min(retry_time, len(degradation) - 1)]
+ LOG.info(
+ "Trying to navigate to url",
+ url=url,
+ retry_time=retry_time,
+ wait_until=strategy,
+ )
try:
start_time = time.time()
- await page.goto(url, timeout=settings.BROWSER_LOADING_TIMEOUT_MS)
+ await page.goto(
+ url,
+ timeout=settings.BROWSER_LOADING_TIMEOUT_MS,
+ wait_until=strategy,
+ )
end_time = time.time()
LOG.info(
"Page loading time",
loading_time=end_time - start_time,
url=url,
+ wait_until=strategy,
)
await self._wait_for_settle()
- LOG.info("Successfully navigated to url", url=url, retry_time=retry_time)
+ LOG.info(
+ "Successfully navigated to url",
+ url=url,
+ retry_time=retry_time,
+ wait_until=strategy,
+ )
return
except Exception as e:
@@ -156,6 +189,7 @@ class RealBrowserState(BrowserState):
exc_info=True,
url=url,
retry_time=retry_time,
+ wait_until=strategy,
)
# Wait for 1 seconds before retrying
await asyncio.sleep(1)
diff --git a/tests/unit/test_file_download_block_warning.py b/tests/unit/test_file_download_block_warning.py
new file mode 100644
index 000000000..648176ef0
--- /dev/null
+++ b/tests/unit/test_file_download_block_warning.py
@@ -0,0 +1,57 @@
+"""Unit tests for SKY-8818 observability warning on under-configured file_download blocks.
+
+When a FileDownloadBlock is configured with `max_steps_per_run < MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD`,
+the block must emit a structured LOG.warning with `log_code='file_download_low_max_steps'` at block
+start, so Datadog can surface under-configured workflows.
+"""
+
+from unittest.mock import patch
+
+import pytest
+
+from skyvern.forge.sdk.workflow.models.block import (
+ MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD,
+ FileDownloadBlock,
+ warn_if_file_download_max_steps_low,
+)
+
+
+def _make_block(max_steps: int | None) -> FileDownloadBlock:
+ return FileDownloadBlock.model_construct(
+ label="download_files",
+ max_steps_per_run=max_steps,
+ )
+
+
+def test_threshold_constant_is_five() -> None:
+ """Anchor the threshold so tuning is a deliberate change."""
+ assert MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD == 5
+
+
[email protected]("configured", [1, 2, 3, 4])
+def test_warning_fires_below_threshold(configured: int) -> None:
+ block = _make_block(configured)
+ with patch("skyvern.forge.sdk.workflow.models.block.LOG") as mock_log:
+ warn_if_file_download_max_steps_low(block, workflow_run_id="wr_test")
+ assert mock_log.warning.call_count == 1
+ _, kwargs = mock_log.warning.call_args
+ assert kwargs.get("log_code") == "file_download_low_max_steps"
+ assert kwargs.get("max_steps_per_run") == configured
+ assert kwargs.get("recommended_minimum") == MAX_STEPS_DOWNLOAD_WARNING_THRESHOLD
+ assert kwargs.get("workflow_run_id") == "wr_test"
+
+
[email protected]("configured", [5, 6, 10, 25])
+def test_warning_does_not_fire_at_or_above_threshold(configured: int) -> None:
+ block = _make_block(configured)
+ with patch("skyvern.forge.sdk.workflow.models.block.LOG") as mock_log:
+ warn_if_file_download_max_steps_low(block, workflow_run_id="wr_test")
+ assert mock_log.warning.call_count == 0
+
+
+def test_warning_does_not_fire_when_max_steps_unset() -> None:
+ """max_steps_per_run=None means the org-level default applies — not a misconfiguration signal."""
+ block = _make_block(None)
+ with patch("skyvern.forge.sdk.workflow.models.block.LOG") as mock_log:
+ warn_if_file_download_max_steps_low(block, workflow_run_id="wr_test")
+ assert mock_log.warning.call_count == 0
diff --git a/tests/unit/test_real_browser_state_navigate.py b/tests/unit/test_real_browser_state_navigate.py
new file mode 100644
index 000000000..965222962
--- /dev/null
+++ b/tests/unit/test_real_browser_state_navigate.py
@@ -0,0 +1,112 @@
+"""Unit tests for skyvern.webeye.real_browser_state.navigate_to_url.
+
+Covers SKY-8818: pages whose subresources never finish loading must still
+succeed if the DOM has parsed, via progressive wait_until degradation.
+"""
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+
+from skyvern.exceptions import FailedToNavigateToUrl
+from skyvern.webeye.real_browser_state import RealBrowserState
+
+
+def browser_state() -> RealBrowserState:
+ # Bypass __init__; navigate_to_url only uses `self` for LOG context and _wait_for_settle.
+ state = RealBrowserState.__new__(RealBrowserState)
+ return state
+
+
+async def test_navigate_to_url_progresses_from_load_to_domcontentloaded(
+ browser_state: RealBrowserState,
+) -> None:
+ """If wait_until='load' times out but 'domcontentloaded' succeeds, we succeed."""
+ page = MagicMock()
+ calls: list[str] = []
+
+ async def fake_goto(url: str, timeout: int, wait_until: str = "load") -> None:
+ calls.append(wait_until)
+ if wait_until == "load":
+ raise PlaywrightTimeoutError("Page.goto: Timeout 60000ms exceeded (load)")
+ return None
+
+ page.goto = AsyncMock(side_effect=fake_goto)
+
+ await browser_state.navigate_to_url(
+ page=page,
+ url="https://example.test/slow-subresources",
+ wait_until="load",
+ )
+
+ assert "load" in calls
+ assert "domcontentloaded" in calls
+ assert calls.index("domcontentloaded") > calls.index("load")
+
+
+async def test_navigate_to_url_raises_when_all_strategies_fail(
+ browser_state: RealBrowserState,
+) -> None:
+ """If every wait_until strategy times out, raise FailedToNavigateToUrl."""
+ page = MagicMock()
+
+ async def always_timeout(url: str, timeout: int, wait_until: str = "load") -> None:
+ raise PlaywrightTimeoutError(f"Page.goto: Timeout 60000ms exceeded ({wait_until})")
+
+ page.goto = AsyncMock(side_effect=always_timeout)
+
+ with pytest.raises(FailedToNavigateToUrl):
+ await browser_state.navigate_to_url(
+ page=page,
+ url="https://example.test/fully-dead",
+ wait_until="load",
+ )
+
+
+async def test_navigate_to_url_honors_caller_supplied_wait_until_on_first_try(
+ browser_state: RealBrowserState,
+) -> None:
+ """FileDownloadBlock passes wait_until='domcontentloaded' — it must be honored on first try."""
+ page = MagicMock()
+ calls: list[str] = []
+
+ async def fake_goto(url: str, timeout: int, wait_until: str = "load") -> None:
+ calls.append(wait_until)
+ return None
+
+ page.goto = AsyncMock(side_effect=fake_goto)
+
+ await browser_state.navigate_to_url(
+ page=page,
+ url="https://example.test/fast",
+ wait_until="domcontentloaded",
+ )
+
+ assert calls == ["domcontentloaded"]
+
+
+async def test_navigate_to_url_succeeds_on_first_try_with_default_load(
+ browser_state: RealBrowserState,
+) -> None:
+ """Existing callers that use default wait_until='load' must keep working untouched."""
+ page = MagicMock()
+ calls: list[str] = []
+
+ async def fake_goto(url: str, timeout: int, wait_until: str = "load") -> None:
+ calls.append(wait_until)
+ return None
+
+ page.goto = AsyncMock(side_effect=fake_goto)
+
+ await browser_state.navigate_to_url(
+ page=page,
+ url="https://example.test/fast-load",
+ )
+
+ assert calls == ["load"]