Skip to content

Automate browser based workflows with AI

Fail the run at the first unexpected browser connection loss (SKY-18726) (#9022)

LawyZhenge9477af39eea
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.py
index bee83c74c..63b7baec4 100644
--- a/skyvern/core/script_generations/real_skyvern_page_ai.py
+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py
@@ -59,6 +59,7 @@ from skyvern.webeye.actions.handler import (
)
from skyvern.webeye.actions.parse_actions import parse_actions
from skyvern.webeye.actions.responses import ActionAbort, ActionResult
+from skyvern.webeye.browser_state import raise_if_browser_connection_lost
from skyvern.webeye.scraper.scraped_page import ScrapedPage
jinja_sandbox_env = SandboxedEnvironment()
@@ -191,9 +192,16 @@ class RealSkyvernPageAi(SkyvernPageAi):
self.page = page
self.current_label: str | None = None
+ def _raise_if_browser_connection_lost(self) -> None:
+ context = skyvern_context.current()
+ raise_if_browser_connection_lost(
+ self.scraped_page._browser_state, workflow_run_id=context.workflow_run_id if context else None
+ )
+
async def _refresh_scraped_page(
self, take_screenshots: bool = True, max_retries: int = SKYVERN_PAGE_MAX_SCRAPING_RETRIES
) -> None:
+ self._raise_if_browser_connection_lost()
self.scraped_page = await self.scraped_page.generate_scraped_page(
take_screenshots=take_screenshots, max_retries=max_retries
)
@@ -372,6 +380,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
exc_info=True,
)
+ self._raise_if_browser_connection_lost()
if selector:
locator = self.page.locator(selector)
await locator.click(timeout=timeout)
@@ -395,6 +404,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
v3_parent_episode_id: str | None = None,
) -> str:
"""Input text into an element using AI to determine the value."""
+ self._raise_if_browser_connection_lost()
sensitive_value = value if value_is_sensitive else None
# v3 mid-run hook — see ai_click for the contract.
@@ -534,6 +544,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
except Exception:
LOG.exception("Failed to adapt value for input text action", selector=selector)
+ self._raise_if_browser_connection_lost()
if action and organization_id and task and step:
result = await handle_input_text_action_direct(action, self.page, self.scraped_page, task, step)
if result and result[-1].success is False:
@@ -658,6 +669,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
except Exception:
LOG.exception(f"Failed to adapt value for upload file action on selector={selector}, file={files}")
+ self._raise_if_browser_connection_lost()
if action and original_files and action.file_url and action.file_url != original_files:
LOG.warning(
"LLM returned a different file url than the user provided, using the original",
@@ -757,6 +769,7 @@ class RealSkyvernPageAi(SkyvernPageAi):
exc_info=True,
)
+ self._raise_if_browser_connection_lost()
if not selector:
raise SkyvernActionFailed(
f"Select option failed and no fallback selector available for intention: {intention}"
diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.py
index 26ab7cc69..8589273de 100644
--- a/skyvern/core/script_generations/script_skyvern_page.py
+++ b/skyvern/core/script_generations/script_skyvern_page.py
@@ -5,7 +5,7 @@ import inspect
import os
import re
import weakref
-from datetime import datetime
+from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Callable
@@ -27,8 +27,10 @@ from skyvern.core.script_generations.skyvern_page import (
from skyvern.core.script_generations.skyvern_page_ai import SkyvernPageAi
from skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codes
from skyvern.exceptions import (
+ BROWSER_CONNECTION_LOST_ERROR_CODE,
BrowserSessionSwitchNotAllowed,
IllegitCompleteScriptTermination,
+ MissingBrowserState,
ScriptTerminationException,
WorkflowRunNotFound,
)
@@ -74,7 +76,7 @@ from skyvern.webeye.actions.handler import (
)
from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess
from skyvern.webeye.browser_engine import BrowserEngineSelection
-from skyvern.webeye.browser_state import BrowserState
+from skyvern.webeye.browser_state import BrowserState, get_browser_connection_loss, raise_if_browser_connection_lost
from skyvern.webeye.navigation import default_navigation_settle, navigate_with_retry, redact_url_secrets
from skyvern.webeye.scraper.scraped_page import ScrapedPage
from skyvern.webeye.utils.challenge_solve_record import calling_solve_client
@@ -145,6 +147,12 @@ class ScriptSkyvernPage(SkyvernPage):
_set_scraped_page(self, scraped_page)
self._record = recorder or (lambda ac: None)
+ def _raise_if_browser_connection_lost(self) -> None:
+ context = skyvern_context.current()
+ raise_if_browser_connection_lost(
+ _get_scraped_page(self)._browser_state, workflow_run_id=context.workflow_run_id if context else None
+ )
+
@classmethod
async def _get_or_create_browser_state(
cls, browser_session_id: str | None = None, url: str | None = None
@@ -286,6 +294,7 @@ class ScriptSkyvernPage(SkyvernPage):
Auto-creates action records in DB before action execution
and screenshot artifacts after action execution.
"""
+ self._raise_if_browser_connection_lost()
prompt = kwargs.get("prompt", "")
@@ -338,6 +347,7 @@ class ScriptSkyvernPage(SkyvernPage):
# Wait for page to be ready before executing action
# This helps prevent issues where cached actions execute before the page is fully loaded
await self._wait_for_page_ready_before_action()
+ self._raise_if_browser_connection_lost()
# NOTE: _ensure_element_ids_on_page() removed from here.
# unique_id attrs are only needed by the AI fallback path, which
# already calls _refresh_scraped_page() → build_tree_from_body()
@@ -1115,6 +1125,16 @@ class ScriptSkyvernPage(SkyvernPage):
last_error: Exception | None = None
_get_scraped_page(self)._browser_state.begin_navigation()
for attempt in range(max_retries):
+ # The URL check and the retry backoff both await, and a loss latched meanwhile must not reach a dispatch.
+ connection_loss = get_browser_connection_loss(_get_scraped_page(self)._browser_state)
+ if connection_loss is not None:
+ last_error = last_error or MissingBrowserState(
+ workflow_run_id=context.workflow_run_id if context else None,
+ diagnostic=connection_loss,
+ detected_at=datetime.now(UTC),
+ failure_reason=BROWSER_CONNECTION_LOST_ERROR_CODE,
+ )
+ break
try:
await self.page.goto(url, timeout=timeout, **kwargs)
await _get_scraped_page(self)._browser_state.record_navigation_result(None, url=url)
@@ -1123,7 +1143,10 @@ class ScriptSkyvernPage(SkyvernPage):
return
except Exception as e:
last_error = e
- if attempt >= max_retries - 1:
+ if (
+ attempt >= max_retries - 1
+ or get_browser_connection_loss(_get_scraped_page(self)._browser_state) is not None
+ ):
break
LOG.warning(
"Navigation attempt failed, retrying",
@@ -1188,6 +1211,7 @@ class ScriptSkyvernPage(SkyvernPage):
retry_times=NAVIGATION_MAX_RETRY_TIME,
settle=default_navigation_settle,
log_url=redact_url_secrets(polled.value),
+ stop_retrying=lambda: get_browser_connection_loss(_get_scraped_page(self)._browser_state) is not None,
)
@action_wrap(ActionType.SOLVE_CAPTCHA)
diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.py
index f6f5e3634..fa4ede074 100644
--- a/skyvern/core/script_generations/skyvern_page.py
+++ b/skyvern/core/script_generations/skyvern_page.py
@@ -286,6 +286,10 @@ class SkyvernPage(Page):
return ResolvedSensitiveValue(totp_code[digit_index])
return ""
+ def _raise_if_browser_connection_lost(self) -> None:
+ # Only a run-bound page knows its run's browser; a standalone library page has no loss to observe.
+ return None
+
@staticmethod
def _track_ai_call() -> None:
"""Increment the script LLM call counter for cost-cap tracking."""
@@ -346,6 +350,7 @@ class SkyvernPage(Page):
retry_interval=retry_interval,
)
await asyncio.sleep(retry_interval)
+ self._raise_if_browser_connection_lost()
# Re-acquire locator in case the DOM was replaced entirely
locator = self._locator_scope.locator(selector).first
else:
@@ -511,6 +516,7 @@ class SkyvernPage(Page):
except StaleFrameSelectionError:
raise
except Exception as e:
+ self._raise_if_browser_connection_lost()
if is_post_dispatch_click_timeout(e, self.engine_selection):
LOG.info(
"CSS selector click dispatched; navigation-wait timed out — skipping fallback",
@@ -533,6 +539,7 @@ class SkyvernPage(Page):
try:
await self.page.keyboard.press("Escape")
await asyncio.sleep(0.3)
+ self._raise_if_browser_connection_lost()
locator = self._locator_scope.locator(selector).first
# The primary click may have dispatched and reached the desired state
# before raising, in which case retrying would toggle it back off.
@@ -545,6 +552,7 @@ class SkyvernPage(Page):
)
return selector
except Exception as retry_error:
+ self._raise_if_browser_connection_lost()
if is_post_dispatch_click_timeout(retry_error, self.engine_selection):
LOG.info(
"CSS selector click dispatched after dismissing overlay; "
@@ -1917,6 +1925,7 @@ class SkyvernPage(Page):
return True
for idx, value in sorted(mapping.items()):
+ self._raise_if_browser_connection_lost()
if idx >= len(form_fields) or value is None:
continue
totp_placeholder = value if isinstance(value, str) and is_unresolved_totp_value(value) else None
@@ -2202,6 +2211,7 @@ class SkyvernPage(Page):
break
if uploaded:
break
+ self._raise_if_browser_connection_lost()
async def validate_mapping(
self,
@@ -2405,6 +2415,7 @@ class SkyvernPage(Page):
if page_num > 0:
await asyncio.sleep(1)
+ self._raise_if_browser_connection_lost()
form_fields = await self.extract_form_fields()
# Filter to fillable fields — includes standard inputs AND custom widgets
@@ -2602,6 +2613,7 @@ class SkyvernPage(Page):
except Exception:
pass
+ self._raise_if_browser_connection_lost()
elapsed = time.monotonic() - start_time
LOG.info(
"fill_multipage_form: completed",
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index 8cf8e8d24..71526ba10 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -551,6 +551,9 @@ def _browser_state_diagnostic_suffix(
return suffix
+BROWSER_CONNECTION_LOST_ERROR_CODE = "browser_connection_lost"
+
+
class MissingBrowserState(SkyvernException):
def __init__(
self,
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 01cbde791..417b9d217 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -52,6 +52,7 @@ from skyvern.errors.errors import (
resolve_error_code_mapping_key,
)
from skyvern.exceptions import (
+ BROWSER_CONNECTION_LOST_ERROR_CODE,
BrowserSessionAlreadyOccupiedError,
BrowserSessionClosed,
BrowserSessionDegraded,
@@ -347,7 +348,7 @@ from skyvern.webeye.browser_errors import (
is_context_lost_message,
is_target_closed_message,
)
-from skyvern.webeye.browser_state import BrowserState, get_browser_state_diagnostic
+from skyvern.webeye.browser_state import BrowserState, get_browser_connection_loss, get_browser_state_diagnostic
from skyvern.webeye.cdp_download_interceptor import (
consume_unsolicited_download_error_for_context,
download_filename_from_suffix,
@@ -3449,7 +3450,11 @@ class ForgeAgent:
block_type=block_type,
)
completion_rejection: str | None = None
- if outcome.status == "completed":
+ if outcome.status == "completed" and get_browser_connection_loss(browser_state) is not None:
+ # The gates below fail open on a page that is gone, so a loss after the last action must fail here.
+ completion_rejection = f"the run lost its browser connection ({BROWSER_CONNECTION_LOST_ERROR_CODE})"
+ LOG.warning("task_v3 completion rejected after a browser connection loss", task_id=task.task_id)
+ elif outcome.status == "completed":
# The deployment gate for a completion the finish tool could not put to it (no headroom left to
# act on a refusal, a re-ask conversion, or a gate error). The loop has returned, so a veto fails.
gate_page = None
@@ -5508,6 +5513,8 @@ class ForgeAgent:
)
if canceled is not None:
return actions, canceled
+ if get_browser_connection_loss(browser_state) is not None:
+ return actions, await self._fail_step_unless_run_canceled(task, step, detailed_agent_step_output)
context = skyvern_context.ensure_context()
if context.refresh_working_page:
@@ -5818,6 +5825,8 @@ class ForgeAgent:
return actions, await self._fail_step_unless_run_canceled(task, step, detailed_agent_step_output)
await artifact_tracker.drain()
+ if get_browser_connection_loss(browser_state) is not None:
+ return actions, await self._fail_step_unless_run_canceled(task, step, detailed_agent_step_output)
# A cancel landing during the last action would otherwise reach extraction and completion.
return actions, await self._stop_step_if_run_canceled(task, step, detailed_agent_step_output, skipped_actions=0)
diff --git a/skyvern/forge/sdk/workflow/credential_fallback.py b/skyvern/forge/sdk/workflow/credential_fallback.py
index 09d23b23e..9f908d8e0 100644
--- a/skyvern/forge/sdk/workflow/credential_fallback.py
+++ b/skyvern/forge/sdk/workflow/credential_fallback.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import structlog
from sqlalchemy.exc import IntegrityError
-from skyvern.exceptions import WorkflowNotFound
+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE, WorkflowNotFound
from skyvern.forge import app
from skyvern.forge.failure_classifier import classify_from_failure_reason
from skyvern.forge.sdk.core import skyvern_context
@@ -179,6 +179,12 @@ async def _maybe_start_credential_fallback_retry(workflow_run: WorkflowRun, orga
]
if not triggered_parameters:
return None
+ # Another credential cannot help a run whose browser went away under it.
+ run_blocks = await app.DATABASE.observer.get_workflow_run_blocks(
+ workflow_run_id=workflow_run.workflow_run_id, organization_id=organization_id
+ )
+ if any(BROWSER_CONNECTION_LOST_ERROR_CODE in run_block.error_codes for run_block in run_blocks):
+ return None
all_fallback_ids = list(
dict.fromkeys(
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index ade78a30d..1ed36bd0f 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -29,6 +29,7 @@ import uuid
import zipfile
from collections import defaultdict, deque
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
+from contextvars import ContextVar
from dataclasses import dataclass, replace
from datetime import UTC, date, datetime, time
from email.message import EmailMessage
@@ -94,6 +95,7 @@ from skyvern.constants import (
)
from skyvern.errors.errors import UserDefinedError
from skyvern.exceptions import (
+ BROWSER_CONNECTION_LOST_ERROR_CODE,
AzureConfigurationError,
BlockedHost,
BlockedNavigationDestination,
@@ -363,7 +365,12 @@ from skyvern.webeye.browser_driver_errors import is_driver_error
from skyvern.webeye.browser_engine import is_any_engine_error
from skyvern.webeye.browser_factory import rebind_download_dir
from skyvern.webeye.browser_object_predicates import is_page_like
-from skyvern.webeye.browser_state import BrowserState, get_browser_state_diagnostic
+from skyvern.webeye.browser_state import (
+ BrowserState,
+ get_browser_connection_loss,
+ get_browser_state_diagnostic,
+ raise_if_browser_connection_lost,
+)
from skyvern.webeye.cdp_download_interceptor import (
ORIGINAL_FILENAME_MARKER,
ORIGINAL_FILENAME_TEMPLATE_VARIABLE,
@@ -1318,6 +1325,14 @@ class Block(BaseModel, abc.ABC):
# stopped it — reusing it makes the block's first page.goto raise "Connection closed
# while reading from the driver". Rebuild a fresh connection to the same browser first.
if browser_state is not None and not browser_state.is_connected():
+ connection_loss = get_browser_connection_loss(browser_state)
+ if connection_loss is not None:
+ raise MissingBrowserState(
+ workflow_run_id=workflow_run_id,
+ diagnostic=connection_loss,
+ detected_at=datetime.now(UTC),
+ failure_reason=BROWSER_CONNECTION_LOST_ERROR_CODE,
+ )
workflow_run = await app.WORKFLOW_SERVICE.get_workflow_run(
workflow_run_id=workflow_run_id,
organization_id=organization_id,
@@ -1862,6 +1877,11 @@ class Block(BaseModel, abc.ABC):
organization_id=organization_id,
can_continue_after_failure=False,
)
+ # A native loss can latch during the setup awaits above; only the finally route runs after one.
+ if not _runs_after_browser_loss(workflow_run_id):
+ raise_if_browser_connection_lost(
+ app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id), workflow_run_id=workflow_run_id
+ )
LOG.info(
"Executing block",
@@ -1884,6 +1904,9 @@ class Block(BaseModel, abc.ABC):
await self._invalidate_stale_output_on_failure(
workflow_run_id, current_index, include_missing_value_guard=False
)
+ result = await self._with_browser_connection_loss(
+ result, workflow_run_id, workflow_run_block_id, organization_id
+ )
return result
except Exception as e:
can_continue_after_failure: bool | None = None
@@ -1953,6 +1976,14 @@ class Block(BaseModel, abc.ABC):
nav_run_context = None
if nav_run_context is not None and not is_registered_secret(e.nav_error_code, nav_run_context):
error_codes = [*error_codes, e.nav_error_code]
+ # The run's browser is known lost, so this failure is terminal whatever exception surfaced it. Inside the
+ # finally block only a refused browser use is; the body's loss does not end its ordinary failures.
+ browser_lost = _run_lost_its_browser(workflow_run_id) and (
+ not _runs_after_browser_loss(workflow_run_id)
+ or isinstance(e, (MissingBrowserState, MissingBrowserStatePage))
+ )
+ if browser_lost:
+ error_codes = [*error_codes, BROWSER_CONNECTION_LOST_ERROR_CODE]
return await self.build_block_result(
success=False,
@@ -1961,12 +1992,38 @@ class Block(BaseModel, abc.ABC):
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
error_codes=error_codes or None,
- can_continue_after_failure=can_continue_after_failure,
+ can_continue_after_failure=False if browser_lost else can_continue_after_failure,
)
finally:
if download_timeout_context is not None:
download_timeout_context.download_timeout = prev_download_timeout
+ async def _with_browser_connection_loss(
+ self,
+ result: BlockResult,
+ workflow_run_id: str,
+ workflow_run_block_id: str | None,
+ organization_id: str | None,
+ ) -> BlockResult:
+ # A block that returns its outcome instead of raising it never reaches the exception handler above. After the
+ # loss any unsuccessful outcome but a cancellation fails the run, so a terminated task cannot let it continue.
+ if (
+ _runs_after_browser_loss(workflow_run_id)
+ or not _run_lost_its_browser(workflow_run_id)
+ or result.success
+ or result.status == BlockStatus.canceled
+ ):
+ return result
+ error_codes = list(dict.fromkeys([*result.error_codes, BROWSER_CONNECTION_LOST_ERROR_CODE]))
+ if workflow_run_block_id and (result.status != BlockStatus.failed or error_codes != result.error_codes):
+ await app.DATABASE.observer.update_workflow_run_block(
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ status=BlockStatus.failed,
+ error_codes=error_codes,
+ )
+ return replace(result, status=BlockStatus.failed, error_codes=error_codes, can_continue_after_failure=False)
+
@abc.abstractmethod
def get_all_parameters(
self,
@@ -1975,6 +2032,30 @@ class Block(BaseModel, abc.ABC):
pass
+# The finally block's dispatch task copies this context. It names the run whose finally block it is, because a
+# synchronous child workflow started there runs inline in the same context and keeps its own body guards.
+_finally_after_browser_loss_run_id: ContextVar[str | None] = ContextVar(
+ "finally_after_browser_loss_run_id", default=None
+)
+
+
+def running_after_browser_loss(workflow_run_id: str) -> Iterator[None]:
+ token = _finally_after_browser_loss_run_id.set(workflow_run_id)
+ try:
+ yield
+ finally:
+ _finally_after_browser_loss_run_id.reset(token)
+
+
+def _runs_after_browser_loss(workflow_run_id: str) -> bool:
+ return _finally_after_browser_loss_run_id.get() == workflow_run_id
+
+
+def _run_lost_its_browser(workflow_run_id: str) -> bool:
+ return get_browser_connection_loss(app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)) is not None
+
+
def _is_retry_blocking_anti_bot_entry(category: dict) -> bool:
return (
category.get("category") == "ANTI_BOT_DETECTION"
@@ -2469,8 +2550,9 @@ class BaseTaskBlock(Block):
# TODO (kerem) we should always retry on terminated. We should make a distinction between retriable and
# non-retryable terminations
+ next_task_order: tuple[int, int] | None = None
while will_retry:
- task_order, task_retry = await self.get_task_order(workflow_run_id, current_retry)
+ task_order, task_retry = next_task_order or await self.get_task_order(workflow_run_id, current_retry)
is_first_task = task_order == 0
task, step = await app.agent.create_task_and_step_from_block(
task_block=self,
@@ -2834,7 +2916,6 @@ class BaseTaskBlock(Block):
failure_category=updated_task.failure_category,
)
will_retry = False
- retry_message = f", retrying task {current_retry}/{self.max_retries}" if will_retry else ""
downloaded_files = None
try:
async with asyncio.timeout(GET_DOWNLOADED_FILES_TIMEOUT):
@@ -2864,6 +2945,13 @@ class BaseTaskBlock(Block):
workflow_run_id=workflow_run_id,
organization_id=workflow_run.organization_id,
)
+ if will_retry:
+ next_task_order = await self.get_task_order(workflow_run_id, current_retry)
+ # The retry is admitted only after every await above, so a loss during one ends the block here.
+ browser_lost = _run_lost_its_browser(workflow_run_id)
+ if browser_lost:
+ will_retry = False
+ retry_message = f", retrying task {current_retry}/{self.max_retries}" if will_retry else ""
task_output = TaskOutput.from_task(
updated_task,
@@ -2887,6 +2975,9 @@ class BaseTaskBlock(Block):
await self.record_output_parameter_value(
workflow_run_context, workflow_run_id, output_parameter_value
)
+ error_codes = _recorded_task_nav_error_codes(updated_task.task_id, workflow_run_context) or []
+ if browser_lost:
+ error_codes.append(BROWSER_CONNECTION_LOST_ERROR_CODE)
return await self.build_block_result(
success=False,
failure_reason=(
@@ -2897,7 +2988,8 @@ class BaseTaskBlock(Block):
status=block_status_mapping[updated_task.status],
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
- error_codes=_recorded_task_nav_error_codes(updated_task.task_id, workflow_run_context),
+ error_codes=error_codes or None,
+ can_continue_after_failure=False if browser_lost else None,
)
await self.record_output_parameter_value(workflow_run_context, workflow_run_id)
@@ -5937,6 +6029,7 @@ def _bind_code_block_magic_link(
retry_times=NAVIGATION_MAX_RETRY_TIME,
settle=default_navigation_settle,
log_url=redact_url_secrets(link),
+ stop_retrying=lambda: workflow_run_id is not None and _run_lost_its_browser(workflow_run_id),
)
return magic_link
@@ -6579,6 +6672,7 @@ def _bind_code_block_open_page(
open_tabs = await open_page_tab_count(_raw_code_block_page(page), browser_state)
if open_page_cap_reached(open_tabs):
raise RuntimeError(open_page_cap_error(open_tabs))
+ raise_if_browser_connection_lost(browser_state)
opened: Page | RecordingPage
if isinstance(page, RecordingPage):
opened = page._wrap_page(await page._underlying_page.context.new_page())
@@ -9247,6 +9341,13 @@ async def wrapper({default_args}):
final_url=staged_failure_url,
)
+ if (
+ get_browser_connection_loss(
+ browser_state or app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id=workflow_run_id)
+ )
+ is not None
+ ):
+ return await _finalize_heal_result(None)
if not classification.healable:
return await _finalize_heal_result(None)
if not await self._ai_fallback_enabled(workflow_run_context):
@@ -9418,13 +9519,16 @@ async def wrapper({default_args}):
) -> BlockResult:
propagated_error: BaseException
try:
- return await self._execute(
+ result = await self._execute(
workflow_run_id,
workflow_run_block_id,
organization_id=organization_id,
browser_session_id=browser_session_id,
**kwargs,
)
+ return await self._with_browser_connection_loss(
+ result, workflow_run_id, workflow_run_block_id, organization_id
+ )
except BaseException as exc:
if isinstance(exc, (MissingBrowserState, MissingBrowserStatePage)):
propagated_error = exc.with_traceback(None)
@@ -16870,13 +16974,20 @@ class TaskV2Block(Block):
"workflow_screenshot_artifact_ids": [a.artifact_id for a in workflow_screenshot_artifacts],
}
await self.record_output_parameter_value(workflow_run_context, workflow_run_id, task_v2_output)
+ # An explicit persistent session registers the child's browser under the child run only.
+ browser_lost = not success and (
+ _run_lost_its_browser(workflow_run_id)
+ or (task_v2_workflow_run_id is not None and _run_lost_its_browser(task_v2_workflow_run_id))
+ )
return await self.build_block_result(
- success=success or self.continue_on_failure,
+ success=success or (self.continue_on_failure and not browser_lost),
failure_reason=failure_reason,
output_parameter_value=task_v2_output,
status=block_status,
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
+ error_codes=[BROWSER_CONNECTION_LOST_ERROR_CODE] if browser_lost else None,
+ can_continue_after_failure=False if browser_lost else None,
)
diff --git a/skyvern/forge/sdk/workflow/retry_policy.py b/skyvern/forge/sdk/workflow/retry_policy.py
index a0080e378..d697cfa25 100644
--- a/skyvern/forge/sdk/workflow/retry_policy.py
+++ b/skyvern/forge/sdk/workflow/retry_policy.py
@@ -10,7 +10,7 @@ import structlog
from sqlalchemy import update
from sqlalchemy.exc import IntegrityError
-from skyvern.exceptions import WorkflowNotFound, WorkflowNotFoundForWorkflowRun
+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE, WorkflowNotFound, WorkflowNotFoundForWorkflowRun
from skyvern.forge import app
from skyvern.forge.sdk.cache import extraction_cache
from skyvern.forge.sdk.db.datetime_utils import naive_utc_now, to_naive_utc
@@ -121,6 +121,8 @@ def evaluate_retry_policy(
) -> tuple[bool, str]:
if status is WorkflowRunStatus.canceled:
return False, "canceled"
+ if BROWSER_CONNECTION_LOST_ERROR_CODE in error_codes:
+ return False, BROWSER_CONNECTION_LOST_ERROR_CODE
if retries_used >= policy.max_retries:
return False, "budget_exhausted"
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 6434c7738..5484aa537 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -52,6 +52,7 @@ from skyvern.constants import (
)
from skyvern.errors.errors import drop_unsolicited_download_errors
from skyvern.exceptions import (
+ BROWSER_CONNECTION_LOST_ERROR_CODE,
BlockedHost,
BlockNotFound,
BrowserActionPolicyNotEnforceable,
@@ -68,6 +69,7 @@ from skyvern.exceptions import (
InProcessScriptExecutionDenied,
InvalidCredentialId,
InvalidWorkflowParameter,
+ MissingBrowserState,
MissingValueForParameter,
ScriptTerminationException,
SequentialCredentialLimitExceeded,
@@ -226,6 +228,7 @@ from skyvern.forge.sdk.workflow.models.block import (
get_all_blocks,
pinned_v1_block_count,
resolve_conditional_merge_edges,
+ running_after_browser_loss,
takes_default_engine,
v3_ab_ineligibility_reason,
)
@@ -420,7 +423,7 @@ from skyvern.utils.url_validators import validate_webhook_url
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import Action, InputTextAction, VerificationCodeAction
from skyvern.webeye.actions.responses import ActionResult
-from skyvern.webeye.browser_state import BrowserState
+from skyvern.webeye.browser_state import BrowserState, get_browser_connection_loss
from skyvern.webeye.persistent_session_errors import BrowserSessionCreditAdmissionRefusal
from skyvern.webeye.persistent_sessions_manager import BROWSER_RETIREMENT_DENIED_NOTE
from skyvern.webeye.profile_cookie_merge import cookie_delta, seed_cookie_values, union_cookies_into_profile_dir
@@ -8567,10 +8570,9 @@ class WorkflowService:
page_text_snapshot = None
working_page = None
try:
- browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(
- workflow_run=workflow_run,
- )
- working_page = await browser_state.get_working_page()
+ # A diagnostic read: acquiring here would open a browser or take over one another run holds.
+ browser_state = app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id=workflow_run.workflow_run_id)
+ working_page = await browser_state.get_working_page() if browser_state else None
if working_page:
page_url = working_page.url
page_text_snapshot = (await working_page.inner_text("body"))[:5000]
@@ -9047,6 +9049,15 @@ class WorkflowService:
)
return workflow_run, blocks_to_update, workflow_run_block_result, True, branch_metadata
+ # A unit that completed after the run lost its browser natively cannot complete the body.
+ if (
+ workflow_run_block_result.success
+ and get_browser_connection_loss(app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)) is not None
+ ):
+ workflow_run_block_result = await self._browser_lost_script_block_result(
+ block, workflow_run_id, organization_id, attempt, parent_workflow_run_block_id
+ )
+
self._track_blocks_for_script_generation(
workflow=workflow,
workflow_run=workflow_run,
@@ -9496,6 +9507,58 @@ class WorkflowService:
script_exception=script_exception,
)
+ async def _browser_lost_script_block_result(
+ self,
+ block: BlockTypeVar,
+ workflow_run_id: str,
+ organization_id: str,
+ attempt: ScriptBlockAttempt,
+ parent_workflow_run_block_id: str | None = None,
+ ) -> BlockResult:
+ attempt_number = app.WORKFLOW_CONTEXT_MANAGER.get_attempt_number(workflow_run_id)
+ workflow_run_blocks = await app.DATABASE.observer.get_workflow_run_blocks(
+ workflow_run_id=workflow_run_id, organization_id=organization_id
+ )
+ matching_blocks = [
+ b
+ for b in workflow_run_blocks
+ if b.label == block.label
+ and (b.attempt_number or 1) == attempt_number
+ and b.parent_workflow_run_block_id == parent_workflow_run_block_id
+ ]
+ latest_block = max(matching_blocks, key=lambda b: b.created_at) if matching_blocks else None
+ if attempt.block_result is not None:
+ latest_block = next(
+ (b for b in matching_blocks if b.workflow_run_block_id == attempt.block_result.workflow_run_block_id),
+ latest_block,
+ )
+ if latest_block is None:
+ latest_block = await app.DATABASE.observer.create_workflow_run_block(
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ attempt_number=attempt_number,
+ parent_workflow_run_block_id=parent_workflow_run_block_id,
+ label=block.label,
+ block_type=block.block_type,
+ continue_on_failure=block.continue_on_failure,
+ )
+ failure_reason = latest_block.failure_reason
+ if failure_reason is None and attempt.script_exception is not None:
+ failure_reason = str(attempt.script_exception)
+ if failure_reason is None:
+ failure_reason = get_user_facing_exception_message(MissingBrowserState(workflow_run_id=workflow_run_id))
+ error_codes = latest_block.error_codes
+ return await block.build_block_result(
+ success=False,
+ failure_reason=failure_reason,
+ output_parameter_value=latest_block.output,
+ status=BlockStatus.failed,
+ workflow_run_block_id=latest_block.workflow_run_block_id,
+ organization_id=organization_id,
+ error_codes=list(dict.fromkeys([*error_codes, BROWSER_CONNECTION_LOST_ERROR_CODE])),
+ can_continue_after_failure=False,
+ )
+
async def _execute_block_via_agent_if_allowed(
self,
*,
@@ -9509,6 +9572,19 @@ class WorkflowService:
script_blocks_by_label: dict[str, Any],
attempt: ScriptBlockAttempt,
) -> tuple[BlockResult | None, bool] | WorkflowRunDispatchStopped:
+ if (
+ attempt.valid_to_run_code
+ and get_browser_connection_loss(app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)) is not None
+ ):
+ LOG.warning(
+ "Browser connection lost: skipping agent fallback for the failed script block", block_label=block.label
+ )
+ return (
+ await self._browser_lost_script_block_result(
+ block, workflow_run_id, organization_id, attempt, parent_workflow_run_block_id
+ ),
+ False,
+ )
workflow_run_block_result = attempt.block_result
# Check if this block is designated as requires_agent by the script reviewer.
# These blocks must execute via agent even when ai_fallback=False.
@@ -10687,14 +10763,15 @@ class WorkflowService:
await app.WORKFLOW_CONTEXT_MANAGER.register_block_parameters_for_workflow_run(
workflow_run.workflow_run_id, parameters, organization
)
- finally_execution = await self._dispatch_workflow_run_block(
- workflow_run.workflow_run_id,
- lambda: block.execute_safe(
- workflow_run_id=workflow_run.workflow_run_id,
- organization_id=organization.organization_id,
- browser_session_id=browser_session_id,
- ),
- )
+ with running_after_browser_loss(workflow_run.workflow_run_id):
+ finally_execution = await self._dispatch_workflow_run_block(
+ workflow_run.workflow_run_id,
+ lambda: block.execute_safe(
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=organization.organization_id,
+ browser_session_id=browser_session_id,
+ ),
+ )
if isinstance(finally_execution, WorkflowRunDispatchStopped):
return finally_execution
block_result = finally_execution
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index 4806dbcf3..15237afc2 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -30,10 +30,12 @@ from skyvern.core.script_generations.generate_script import _build_block_fn, cre
from skyvern.core.script_generations.script_skyvern_page import script_run_context_manager
from skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codes
from skyvern.exceptions import (
+ BROWSER_CONNECTION_LOST_ERROR_CODE,
CachedDownloadError,
DownloadSaveIncompleteError,
IllegitCompleteScriptTermination,
InProcessScriptExecutionDenied,
+ MissingBrowserState,
ScriptNotFound,
ScriptTerminationException,
StepTerminationError,
@@ -131,6 +133,7 @@ from skyvern.utils.templating import reject_jinja_transformations_on_variable
from skyvern.utils.url_validators import validate_fetch_url
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import Action, DecisiveAction, reasoning_is_turn_scoped
+from skyvern.webeye.browser_state import get_browser_connection_loss, raise_if_browser_connection_lost
from skyvern.webeye.cdp_download_interceptor import (
ORIGINAL_FILENAME_MARKER,
ORIGINAL_FILENAME_TEMPLATE_VARIABLE,
@@ -1126,6 +1129,15 @@ async def _update_workflow_block(
if run_lease_tasks_enabled() and await run_lease_lost(context.workflow_run_id):
return
+ # User-defined codes stay on the task row (run details read both rows), but the lost-browser code has no
+ # task record, so a failed row carries it for workflow retry and credential fallback.
+ browser_connection_lost = (
+ status == BlockStatus.failed
+ and get_browser_connection_loss(app.BROWSER_MANAGER.get_for_workflow_run(context.workflow_run_id))
+ is not None
+ )
+ if browser_connection_lost:
+ error_codes = list(dict.fromkeys([*(error_codes or []), BROWSER_CONNECTION_LOST_ERROR_CODE]))
updated_block = await app.DATABASE.observer.update_workflow_run_block(
workflow_run_block_id=workflow_run_block_id,
organization_id=context.organization_id if context else None,
@@ -1133,6 +1145,7 @@ async def _update_workflow_block(
failure_reason=failure_reason,
output=final_output,
ai_fallback_triggered=ai_fallback_triggered,
+ error_codes=[BROWSER_CONNECTION_LOST_ERROR_CODE] if browser_connection_lost else None,
)
# The row carries the block label even when the caller passed none (a cached wait block).
@@ -1175,7 +1188,36 @@ async def _update_workflow_block(
)
+def _raise_if_browser_connection_lost(outcome: BlockResult | BlockStatus | None) -> None:
+ # Generated statements and loop iterations run in sequence, so a child that failed after the run lost its
+ # browser stops the script here; its outcome is already published.
+ context = skyvern_context.current()
+ if not context or not context.workflow_run_id:
+ return
+ connection_loss = get_browser_connection_loss(app.BROWSER_MANAGER.get_for_workflow_run(context.workflow_run_id))
+ if connection_loss is None:
+ return
+ status = outcome.status if isinstance(outcome, BlockResult) else outcome
+ if status == BlockStatus.failed:
+ raise MissingBrowserState(
+ workflow_run_id=context.workflow_run_id,
+ diagnostic=connection_loss,
+ detected_at=datetime.now(timezone.utc),
+ failure_reason=BROWSER_CONNECTION_LOST_ERROR_CODE,
+ )
+
+
+def _refuse_statement_after_browser_connection_loss() -> None:
+ # A loss can latch between statements, with no browser call in flight; the next statement must not start.
+ context = skyvern_context.current()
+ if context and context.workflow_run_id:
+ raise_if_browser_connection_lost(
+ app.BROWSER_MANAGER.get_for_workflow_run(context.workflow_run_id), workflow_run_id=context.workflow_run_id
+ )
+
+
async def _run_cached_function(cached_fn: Callable) -> Any:
+ _refuse_statement_after_browser_connection_loss()
run_context = script_run_context_manager.ensure_run_context()
return await cached_fn(page=run_context.page, context=run_context)
@@ -1531,7 +1573,7 @@ async def _fallback_to_ai_run(
include_action_history_in_verification: bool = False,
error: Exception | None = None,
workflow_run_block_id: str | None = None,
-) -> None:
+) -> BlockStatus | None:
context = skyvern_context.current()
if not (
context
@@ -1541,7 +1583,7 @@ async def _fallback_to_ai_run(
and context.task_id
and context.step_id
):
- return
+ return None
organization_id = context.organization_id
workflow_id = context.workflow_id
workflow_run_id = context.workflow_run_id
@@ -1564,6 +1606,21 @@ async def _fallback_to_ai_run(
organization_id=organization_id,
status=StepStatus.failed,
)
+ if get_browser_connection_loss(app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)) is not None:
+ LOG.warning("Browser connection lost: script failure will not be retried by agent", cache_key=cache_key)
+ if workflow_run_block_id:
+ await _update_workflow_block(
+ workflow_run_block_id,
+ BlockStatus.failed,
+ task_id=task_id,
+ task_status=TaskStatus.failed,
+ failure_reason=str(error),
+ step_id=script_step_id,
+ step_status=StepStatus.failed,
+ label=cache_key,
+ error_codes=[BROWSER_CONNECTION_LOST_ERROR_CODE],
+ )
+ return BlockStatus.failed
# 2. run execute_step
organization = await app.DATABASE.organizations.get_organization(organization_id=organization_id)
if not organization:
@@ -1575,12 +1632,12 @@ async def _fallback_to_ai_run(
workflow_id=context.workflow_id, organization_id=organization_id
)
if not workflow:
- return
+ return None
workflow_run = await app.DATABASE.workflow_runs.get_workflow_run(
workflow_run_id=workflow_run_id, organization_id=organization_id
)
if not workflow_run:
- return
+ return None
# Use workflow_run.ai_fallback if explicitly set, otherwise fall back to workflow.ai_fallback
effective_ai_fallback = (
workflow_run.ai_fallback if workflow_run.ai_fallback is not None else workflow.ai_fallback
@@ -1644,7 +1701,7 @@ async def _fallback_to_ai_run(
label=cache_key,
error_codes=error_codes,
)
- return
+ return BlockStatus.failed
# Record a fallback episode for adaptive caching (code_v2).
# This must happen BEFORE the AI step runs so we capture the page state
@@ -1660,10 +1717,9 @@ async def _fallback_to_ai_run(
page_text_snapshot = None
working_page = None
try:
- browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(
- workflow_run=workflow_run,
- )
- working_page = await browser_state.get_working_page()
+ # A diagnostic read: acquiring here would open a browser or take over one another run holds.
+ browser_state = app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id=workflow_run_id)
+ working_page = await browser_state.get_working_page() if browser_state else None
if working_page:
page_url = working_page.url
page_text_snapshot = (await working_page.inner_text("body", timeout=5000))[:5000]
@@ -1822,6 +1878,7 @@ async def _fallback_to_ai_run(
)
# Update block status to completed if workflow block was created
+ fallback_status: BlockStatus | None = None
if workflow_run_block_id:
# refresh the task
failure_reason = None
@@ -1835,9 +1892,10 @@ async def _fallback_to_ai_run(
for error in task.errors or []
if isinstance(error, dict) and isinstance(error.get("error_code"), str)
]
+ fallback_status = BlockStatus(task.status.value)
await _update_workflow_block(
workflow_run_block_id,
- BlockStatus(task.status.value),
+ fallback_status,
task_id=task_id,
failure_reason=failure_reason,
label=cache_key,
@@ -1932,6 +1990,7 @@ async def _fallback_to_ai_run(
episode_id=fallback_episode_id,
exc_info=True,
)
+ return fallback_status
except Exception as e:
LOG.warning("Failed to fallback to AI run", cache_key=cache_key, exc_info=True)
# Update block status to failed if workflow block was created
@@ -2370,6 +2429,7 @@ async def run_task(
context: skyvern_context.SkyvernContext | None = None
if cache_key and cached_fn:
+ _refuse_statement_after_browser_connection_loss()
# Auto-create workflow block run and task if workflow_run_id is available.
# Use `label` (the workflow block label) for the block run so the
# framework can match it, and `cache_key` to look up the cached function.
@@ -2416,7 +2476,7 @@ async def run_task(
raise
except Exception as e:
LOG.warning("Failed to run task block. Falling back to AI run.", exc_info=True)
- await _fallback_to_ai_run(
+ fallback_status = await _fallback_to_ai_run(
block_type=BlockType.NAVIGATION,
cache_key=cache_key,
prompt=prompt,
@@ -2428,6 +2488,7 @@ async def run_task(
workflow_run_block_id=workflow_run_block_id,
error_code_mapping=error_code_mapping,
)
+ _raise_if_browser_connection_lost(fallback_status)
return None
finally:
# clear the prompt in the RunContext
@@ -2458,6 +2519,7 @@ async def run_task(
label,
output_parameter=block_validation_output.output_parameter,
)
+ _raise_if_browser_connection_lost(block_output)
return block_output.output_parameter_value
@@ -2566,6 +2628,7 @@ async def download(
cached_fn = script_run_context_manager.get_cached_fn(cache_key)
context: skyvern_context.SkyvernContext | None
if cache_key and cached_fn:
+ _refuse_statement_after_browser_connection_loss()
# Auto-create workflow block run and task if workflow_run_id is available
workflow_run_block_id, task_id, step_id = await _create_workflow_block_run_and_task(
block_type=BlockType.FILE_DOWNLOAD,
@@ -3009,7 +3072,7 @@ async def download(
except OSError:
continue
fallback_file_signatures_before[file_path] = (file_stat.st_mtime_ns, file_stat.st_size)
- await _fallback_to_ai_run(
+ fallback_status = await _fallback_to_ai_run(
block_type=BlockType.FILE_DOWNLOAD,
cache_key=cache_key,
prompt=navigation_prompt,
@@ -3021,6 +3084,7 @@ async def download(
workflow_run_block_id=workflow_run_block_id,
error_code_mapping=error_code_mapping,
)
+ _raise_if_browser_connection_lost(fallback_status)
if file_download_block is not None and storage_type is not None:
fallback_files_after = current_attempt_local_files(fallback_download_dir)
fallback_downloaded_files = []
@@ -3098,6 +3162,7 @@ async def download(
browser_session_id=block_validation_output.browser_session_id,
)
if not download_result.success:
+ _raise_if_browser_connection_lost(download_result)
# Halt the script on a failed download/delivery instead of ignoring the result, matching
# the cached path; otherwise later generated statements would run after the failure.
raise ScriptTerminationException(download_result.failure_reason or "File download block failed")
@@ -3119,6 +3184,7 @@ async def action(
cache_key = cache_key or label
cached_fn = script_run_context_manager.get_cached_fn(cache_key)
if cache_key and cached_fn:
+ _refuse_statement_after_browser_connection_loss()
# Auto-create workflow block run and task if workflow_run_id is available
workflow_run_block_id, task_id, step_id = await _create_workflow_block_run_and_task(
block_type=BlockType.ACTION,
@@ -3154,7 +3220,7 @@ async def action(
raise
except Exception as e:
LOG.warning("Failed to run action block. Falling back to AI run.", exc_info=True)
- await _fallback_to_ai_run(
+ fallback_status = await _fallback_to_ai_run(
block_type=BlockType.ACTION,
cache_key=cache_key,
prompt=prompt,
@@ -3167,6 +3233,7 @@ async def action(
workflow_run_block_id=workflow_run_block_id,
error_code_mapping=error_code_mapping,
)
+ _raise_if_browser_connection_lost(fallback_status)
finally:
context.prompt = None
_clear_cached_block_overrides(cache_key)
@@ -3185,12 +3252,13 @@ async def action(
model=model,
download_suffix=download_suffix,
)
- await action_block.execute_safe(
+ action_result = await action_block.execute_safe(
workflow_run_id=block_validation_output.workflow_run_id,
parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,
organization_id=block_validation_output.organization_id,
browser_session_id=block_validation_output.browser_session_id,
)
+ _raise_if_browser_connection_lost(action_result)
async def login(
@@ -3208,6 +3276,7 @@ async def login(
cache_key = cache_key or label
cached_fn = script_run_context_manager.get_cached_fn(cache_key)
if cache_key and cached_fn:
+ _refuse_statement_after_browser_connection_loss()
# Auto-create workflow block run and task if workflow_run_id is available
# render template with label
prompt = _render_template_with_label(prompt, cache_key) if prompt else prompt
@@ -3248,7 +3317,7 @@ async def login(
raise
except Exception as e:
LOG.exception("Failed to run login block")
- await _fallback_to_ai_run(
+ fallback_status = await _fallback_to_ai_run(
block_type=BlockType.LOGIN,
cache_key=cache_key,
prompt=prompt,
@@ -3260,6 +3329,7 @@ async def login(
workflow_run_block_id=workflow_run_block_id,
error_code_mapping=error_code_mapping,
)
+ _raise_if_browser_connection_lost(fallback_status)
finally:
context.prompt = None
_clear_cached_block_overrides(cache_key)
@@ -3277,12 +3347,13 @@ async def login(
model=model,
)
login_block._built_by_script = True
- await login_block.execute_safe(
+ login_result = await login_block.execute_safe(
workflow_run_id=block_validation_output.workflow_run_id,
parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,
organization_id=block_validation_output.organization_id,
browser_session_id=block_validation_output.browser_session_id,
)
+ _raise_if_browser_connection_lost(login_result)
async def extract(
@@ -3300,6 +3371,7 @@ async def extract(
cache_key = cache_key or label
cached_fn = script_run_context_manager.get_cached_fn(cache_key)
if cache_key and cached_fn:
+ _refuse_statement_after_browser_connection_loss()
# Auto-create workflow block run and task if workflow_run_id is available
workflow_run_block_id, task_id, step_id = await _create_workflow_block_run_and_task(
block_type=BlockType.EXTRACTION,
@@ -3368,6 +3440,7 @@ async def extract(
label,
output_parameter=block_validation_output.output_parameter,
)
+ _raise_if_browser_connection_lost(block_result)
return block_result.output_parameter_value
@@ -3382,6 +3455,7 @@ async def conditional(
cached_fn = script_run_context_manager.get_cached_fn(label)
if not cached_fn:
raise Exception(f"No cached function for conditional block '{label}'")
+ _refuse_statement_after_browser_connection_loss()
# Create workflow run block entry (no task needed for conditional blocks)
workflow_run_block_id, _, _ = await _create_workflow_block_run_and_task(
@@ -3400,6 +3474,7 @@ async def conditional(
run_context.parameters[key] = value
try:
+ _refuse_statement_after_browser_connection_loss()
result = await cached_fn(page=run_context.page, context=run_context)
if not isinstance(result, dict) or "next_block_label" not in result:
raise Exception(f"Conditional function '{label}' must return dict with 'next_block_label', got: {result}")
@@ -3445,6 +3520,7 @@ async def validate(
result = await execute_validation(complete_criterion, terminate_criterion, error_code_mapping, label, model)
if result.status == BlockStatus.terminated:
raise ScriptTerminationException(result.failure_reason)
+ _raise_if_browser_connection_lost(result)
async def execute_validation(
@@ -3477,10 +3553,12 @@ async def execute_validation(
label,
output_parameter=block_validation_output.output_parameter,
)
+ _raise_if_browser_connection_lost(result)
return result
async def wait(seconds: int, label: str | None = None) -> None:
+ _refuse_statement_after_browser_connection_loss()
# Auto-create workflow block run if workflow_run_id is available (wait block doesn't create tasks)
workflow_run_block_id, _, _ = await _create_workflow_block_run_and_task(block_type=BlockType.WAIT, label=label)
@@ -3747,6 +3825,7 @@ class BlockValidationOutput:
async def _validate_and_get_output_parameter(
label: str | None = None, parameter_keys: list[str] | None = None
) -> BlockValidationOutput:
+ _refuse_statement_after_browser_connection_loss()
context = skyvern_context.ensure_context()
workflow_id = context.workflow_id
workflow_run_id = context.workflow_run_id
@@ -3819,6 +3898,7 @@ async def run_code(
label,
output_parameter=block_validation_output.output_parameter,
)
+ _raise_if_browser_connection_lost(block_result)
return cast(dict[str, Any], block_result.output_parameter_value)
@@ -3912,12 +3992,13 @@ async def upload_file(
prompt=prompt,
path=path,
)
- await file_upload_block.execute_safe(
+ upload_result = await file_upload_block.execute_safe(
workflow_run_id=block_validation_output.workflow_run_id,
parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,
organization_id=block_validation_output.organization_id,
browser_session_id=block_validation_output.browser_session_id,
)
+ _raise_if_browser_connection_lost(upload_result)
async def _send_email_via_gmail(block_validation_output: BlockValidationOutput, credential_id: str | None) -> None:
@@ -3947,6 +4028,7 @@ async def _send_email_via_gmail(block_validation_output: BlockValidationOutput,
block_validation_output.label,
output_parameter=block_validation_output.output_parameter,
)
+ _raise_if_browser_connection_lost(result)
if not result.success and not send_email_block.continue_on_failure:
raise Exception(result.failure_reason or "Gmail send failed")
@@ -4038,12 +4120,13 @@ async def send_email(
output_parameter=block_validation_output.output_parameter,
parameters=block_validation_output.input_parameters,
)
- await send_email_block.execute_safe(
+ send_email_result = await send_email_block.execute_safe(
workflow_run_id=block_validation_output.workflow_run_id,
parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,
organization_id=block_validation_output.organization_id,
browser_session_id=block_validation_output.browser_session_id,
)
+ _raise_if_browser_connection_lost(send_email_result)
async def parse_pdf(
@@ -4061,12 +4144,13 @@ async def parse_pdf(
output_parameter=block_validation_output.output_parameter,
parameters=block_validation_output.input_parameters,
)
- await pdf_parser_block.execute_safe(
+ parse_pdf_result = await pdf_parser_block.execute_safe(
workflow_run_id=block_validation_output.workflow_run_id,
parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,
organization_id=block_validation_output.organization_id,
browser_session_id=block_validation_output.browser_session_id,
)
+ _raise_if_browser_connection_lost(parse_pdf_result)
async def parse_file(
@@ -4094,12 +4178,13 @@ async def parse_file(
parameters=block_validation_output.input_parameters,
model=model,
)
- await file_parser_block.execute_safe(
+ parse_file_result = await file_parser_block.execute_safe(
workflow_run_id=block_validation_output.workflow_run_id,
parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,
organization_id=block_validation_output.organization_id,
browser_session_id=block_validation_output.browser_session_id,
)
+ _raise_if_browser_connection_lost(parse_file_result)
async def http_request(
@@ -4128,12 +4213,13 @@ async def http_request(
output_parameter=block_validation_output.output_parameter,
parameters=block_validation_output.input_parameters,
)
- await http_request_block.execute_safe(
+ http_request_result = await http_request_block.execute_safe(
workflow_run_id=block_validation_output.workflow_run_id,
parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,
organization_id=block_validation_output.organization_id,
browser_session_id=block_validation_output.browser_session_id,
)
+ _raise_if_browser_connection_lost(http_request_result)
async def goto(
@@ -4150,7 +4236,7 @@ async def goto(
output_parameter=block_validation_output.output_parameter,
parameters=block_validation_output.input_parameters,
)
- await goto_url_block.execute_safe(
+ goto_result = await goto_url_block.execute_safe(
workflow_run_id=block_validation_output.workflow_run_id,
parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,
organization_id=block_validation_output.organization_id,
@@ -4164,6 +4250,8 @@ async def goto(
candidate_url = await asyncio.to_thread(validate_fetch_url, candidate_url)
run_context = script_run_context_manager.ensure_run_context()
await run_context.page.goto(candidate_url)
+ else:
+ _raise_if_browser_connection_lost(goto_result)
async def trigger_workflow(
@@ -4207,6 +4295,7 @@ async def trigger_workflow(
label,
output_parameter=block_validation_output.output_parameter,
)
+ _raise_if_browser_connection_lost(result)
return result.output_parameter_value
@@ -4238,6 +4327,7 @@ async def prompt(
label,
output_parameter=block_validation_output.output_parameter,
)
+ _raise_if_browser_connection_lost(result)
return result.output_parameter_value
@@ -4280,6 +4370,7 @@ async def loop(
complete_if_empty: bool = False,
label: str | None = None,
) -> AsyncGenerator[SkyvernLoopItem, None]:
+ _refuse_statement_after_browser_connection_loss()
workflow_run_block_id, _, _ = await _create_workflow_block_run_and_task(block_type=BlockType.FOR_LOOP, label=label)
# process values:
loop_variable_reference = None
@@ -4384,6 +4475,7 @@ async def loop(
workflow_run_id=workflow_run_id,
loop_index=index,
)
+ _refuse_statement_after_browser_connection_loss()
# register current_value, current_item and current_index in workflow run context
# Block metadata for template context (user-facing fields only)
@@ -4408,6 +4500,7 @@ async def loop(
# Build the SkyvernLoopItem for this loop
yield SkyvernLoopItem(index, value)
+ _refuse_statement_after_browser_connection_loss()
# build success output
if workflow_run_block_id:
await _update_workflow_block(
@@ -4457,6 +4550,7 @@ async def while_loop(
*,
criteria_type: str | None = None,
) -> AsyncGenerator[SkyvernLoopItem, None]:
+ _refuse_statement_after_browser_connection_loss()
workflow_run_block_id, _, _ = await _create_workflow_block_run_and_task(
block_type=BlockType.WHILE_LOOP,
label=label,
@@ -4480,6 +4574,7 @@ async def while_loop(
loop_idx = 0
try:
while True:
+ _refuse_statement_after_browser_connection_loss()
condition_metadata: BlockMetadata = {
"current_index": loop_idx,
"current_value": None,
@@ -4506,6 +4601,7 @@ async def while_loop(
)
raise Exception(f"Failed to evaluate while-loop condition: {exc}") from exc
+ _refuse_statement_after_browser_connection_loss()
if not should_continue:
break
@@ -4539,6 +4635,7 @@ async def while_loop(
workflow_run_id=workflow_run_id,
loop_index=loop_idx,
)
+ _refuse_statement_after_browser_connection_loss()
loop_metadata: BlockMetadata = {
"current_index": loop_idx,
diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.py
index 0655ec62f..aad05eb1f 100644
--- a/skyvern/services/task_v2_service.py
+++ b/skyvern/services/task_v2_service.py
@@ -15,12 +15,15 @@ from sqlalchemy.exc import OperationalError
from skyvern.config import settings
from skyvern.constants import MINI_GOAL_TEMPLATE
from skyvern.exceptions import (
+ BROWSER_CONNECTION_LOST_ERROR_CODE,
BrowserSessionNotFound,
FailedToSendWebhook,
+ MissingBrowserState,
ScreenshotTargetClosed,
TaskTerminationError,
TaskV2NotFound,
UrlGenerationFailure,
+ get_user_facing_exception_message,
)
from skyvern.forge import app
from skyvern.forge.failure_classifier import bounded_failure_categories, classify_from_failure_reason
@@ -120,7 +123,7 @@ from skyvern.utils.prompt_engine import load_prompt_with_elements
from skyvern.utils.strings import generate_random_string
from skyvern.utils.url_validators import validate_fetch_url
from skyvern.webeye.actions.actions import ActionType
-from skyvern.webeye.browser_state import BrowserState
+from skyvern.webeye.browser_state import BrowserState, get_browser_connection_loss
from skyvern.webeye.scraper.scraped_page import ScrapedPage
from skyvern.webeye.utils.page import SkyvernFrame, build_open_tabs_context, capture_open_tab_screenshots
@@ -1126,6 +1129,10 @@ async def run_task_v2_helper(
iteration=i,
workflow_run_id=workflow_run_id,
)
+ lost_task_v2 = await _fail_task_v2_after_browser_loss(task_v2_id, workflow_run_id, organization_id)
+ if lost_task_v2 is not None:
+ task_v2 = lost_task_v2
+ break
await _persist_completion_tab_screenshots(browser_state=browser_state, thought=thought)
task_v2 = await _summarize_task_v2(
task_v2=task_v2,
@@ -1133,7 +1140,7 @@ async def run_task_v2_helper(
context=context,
screenshots=scraped_page.screenshots,
)
- if task_v2.run_with == "code":
+ if task_v2.run_with == "code" and task_v2.status == TaskV2Status.completed:
await app.WORKFLOW_SERVICE.generate_script_if_needed(
workflow=workflow,
workflow_run=workflow_run,
@@ -1385,6 +1392,12 @@ async def run_task_v2_helper(
)
except Exception:
LOG.warning("Failed to build open-tabs context for completion check", exc_info=True)
+ lost_task_v2 = await _fail_task_v2_after_browser_loss(
+ task_v2_id, workflow_run_id, organization_id, completed_block=(block, block_result)
+ )
+ if lost_task_v2 is not None:
+ task_v2 = lost_task_v2
+ break
task_v2_completion_prompt = prompt_engine.load_prompt(
"task_v2_check_completion",
@@ -1438,14 +1451,21 @@ async def run_task_v2_helper(
workflow_run_id=workflow_run_id,
completion_resp=completion_resp,
)
+ lost_task_v2 = await _fail_task_v2_after_browser_loss(
+ task_v2_id, workflow_run_id, organization_id, completed_block=(block, block_result)
+ )
+ if lost_task_v2 is not None:
+ task_v2 = lost_task_v2
+ break
await _persist_completion_tab_screenshots(browser_state=browser_state, thought=thought)
task_v2 = await _summarize_task_v2(
task_v2=task_v2,
task_history=task_history,
context=context,
screenshots=completion_screenshots,
+ completed_block=(block, block_result),
)
- if task_v2.run_with == "code":
+ if task_v2.run_with == "code" and task_v2.status == TaskV2Status.completed:
await app.WORKFLOW_SERVICE.generate_script_if_needed(
workflow=workflow,
workflow_run=workflow_run,
@@ -1586,7 +1606,12 @@ async def handle_block_result(
block_type_var=block.block_type,
block_label=block.label,
)
- if block.continue_on_failure and not is_last_block:
+ if BROWSER_CONNECTION_LOST_ERROR_CODE in block_result.error_codes:
+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(
+ workflow_run_id=workflow_run_id, failure_reason=block_result.failure_reason
+ )
+ await _end_task_v2_after_browser_loss(task_v2_id, workflow_run_id, workflow_run.organization_id)
+ elif block.continue_on_failure and not is_last_block:
LOG.warning(
f"Block with type {block.block_type} failed but will continue executing the workflow run {workflow_run_id}",
block_type=block.block_type,
@@ -1623,6 +1648,57 @@ async def handle_block_result(
)
+async def _fail_task_v2_after_browser_loss(
+ task_v2_id: str,
+ workflow_run_id: str,
+ organization_id: str,
+ completed_block: tuple[BlockTypeVar, BlockResult] | None = None,
+) -> TaskV2 | None:
+ # A success observed after the run lost its browser natively cannot complete the task.
+ if get_browser_connection_loss(app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)) is None:
+ return None
+ LOG.warning("Browser connection lost: task v2 stops without a completion decision", workflow_run_id=workflow_run_id)
+ failure_reason = get_user_facing_exception_message(MissingBrowserState(workflow_run_id=workflow_run_id))
+ if completed_block is not None:
+ block, block_result = completed_block
+ await block.build_block_result(
+ success=False,
+ failure_reason=failure_reason,
+ status=BlockStatus.failed,
+ workflow_run_block_id=block_result.workflow_run_block_id,
+ organization_id=organization_id,
+ error_codes=list(dict.fromkeys([*block_result.error_codes, BROWSER_CONNECTION_LOST_ERROR_CODE])),
+ can_continue_after_failure=False,
+ )
+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(
+ workflow_run_id=workflow_run_id, failure_reason=failure_reason
+ )
+ return await _end_task_v2_after_browser_loss(task_v2_id, workflow_run_id, organization_id)
+
+
+async def _end_task_v2_after_browser_loss(task_v2_id: str, workflow_run_id: str, organization_id: str) -> TaskV2:
+ current = await app.DATABASE.observer.get_task_v2(task_v2_id, organization_id=organization_id)
+ if current is not None and current.status.is_final():
+ # The writer that ended the task first, such as a cancellation, already sent its terminal callback.
+ return current
+ workflow_run = await app.WORKFLOW_SERVICE.get_workflow_run(
+ workflow_run_id=workflow_run_id, organization_id=organization_id
+ )
+ won = False
+ if workflow_run.status.is_final():
+ won = await app.DATABASE.observer.update_task_v2_status_if_not_final(
+ task_v2_id=task_v2_id,
+ organization_id=organization_id,
+ status=TaskV2Status(workflow_run.status),
+ )
+ task_v2 = await app.DATABASE.observer.get_task_v2(task_v2_id, organization_id=organization_id)
+ if task_v2 is None:
+ raise TaskV2NotFound(task_v2_id=task_v2_id)
+ if won:
+ await send_task_v2_webhook(task_v2)
+ return task_v2
+
+
async def _set_up_workflow_context(workflow: Workflow, workflow_run_id: str, organization: Organization) -> None:
"""
TODO: see if we could remove this function as we can just set an empty workflow context
@@ -2819,6 +2895,7 @@ async def _summarize_task_v2(
task_history: list[dict],
context: SkyvernContext,
screenshots: list[bytes] | None = None,
+ completed_block: tuple[BlockTypeVar, BlockResult] | None = None,
) -> TaskV2:
summary_description, summarized_output = await _generate_task_v2_deliverable(
task_v2=task_v2,
@@ -2827,6 +2904,12 @@ async def _summarize_task_v2(
screenshots=screenshots,
is_partial=False,
)
+ if task_v2.workflow_run_id is not None:
+ lost_task_v2 = await _fail_task_v2_after_browser_loss(
+ task_v2.observer_cruise_id, task_v2.workflow_run_id, task_v2.organization_id, completed_block
+ )
+ if lost_task_v2 is not None:
+ return lost_task_v2
return await mark_task_v2_as_completed(
task_v2_id=task_v2.observer_cruise_id,
workflow_run_id=task_v2.workflow_run_id,
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 6b99ac73e..f71a9e84d 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -52,6 +52,7 @@ from skyvern.constants import (
from skyvern.core.script_generations.fuzzy_matcher import match_option_exact_or_stem
from skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codes
from skyvern.exceptions import (
+ BROWSER_CONNECTION_LOST_ERROR_CODE,
ActionExecutionTimeout,
BlockedHost,
CaptchaSolveError,
@@ -79,6 +80,7 @@ from skyvern.exceptions import (
InteractWithDisabledElement,
InteractWithDropdownContainer,
InvalidElementForTextInput,
+ MissingBrowserState,
MissingElement,
MissingElementDict,
MissingElementInCSSMap,
@@ -223,7 +225,7 @@ from skyvern.webeye.browser_factory import (
resolve_artifact_path,
was_download_cancelled_by_skyvern,
)
-from skyvern.webeye.browser_state import BLANK_PAGE_URLS, BrowserState
+from skyvern.webeye.browser_state import BLANK_PAGE_URLS, BrowserState, get_browser_connection_loss
from skyvern.webeye.cdp_download_interceptor import (
BROWSER_DOWNLOAD_EVENT_ADMISSION_GRACE_SECONDS,
DOWNLOAD_MIME_TYPES,
@@ -2437,6 +2439,15 @@ async def _recover_download_page(
timeout_seconds: float,
recovery_site: str,
) -> Page | None:
+ connection_loss = get_browser_connection_loss(browser_state)
+ if connection_loss is not None:
+ raise MissingBrowserState(
+ task_id=task.task_id,
+ workflow_run_id=task.workflow_run_id,
+ diagnostic=connection_loss,
+ detected_at=datetime.now(UTC),
+ failure_reason=BROWSER_CONNECTION_LOST_ERROR_CODE,
+ )
LOG.warning(
"Working page closed during download action; recreating it before continuing",
workflow_run_id=task.workflow_run_id,
@@ -2509,6 +2520,9 @@ async def _recover_download_page(
recovery_site=recovery_site,
exc_info=True,
)
+ # A connection lost while recovery was in flight ends the action, so the caller never starts waiting.
+ if get_browser_connection_loss(browser_state) is not None:
+ raise
return None
return recovered_page
diff --git a/skyvern/webeye/browser_state.py b/skyvern/webeye/browser_state.py
index f18e016f2..452007549 100644
--- a/skyvern/webeye/browser_state.py
+++ b/skyvern/webeye/browser_state.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+from datetime import UTC, datetime
from typing import TYPE_CHECKING, Awaitable, Callable, Literal, Protocol
from urllib.parse import urlparse
@@ -7,7 +8,7 @@ from playwright.async_api import BrowserContext, Page, Playwright
from skyvern.config import settings
from skyvern.constants import NAVIGATION_MAX_RETRY_TIME
-from skyvern.exceptions import BrowserStateDiagnostic
+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE, BrowserStateDiagnostic, MissingBrowserState
from skyvern.schemas.runs import ProxyLocationInput
from skyvern.webeye.browser_artifacts import BrowserArtifacts
from skyvern.webeye.browser_factory import BrowserCleanupFunc
@@ -86,6 +87,11 @@ class BrowserState(Protocol):
def get_browser_state_diagnostic(self) -> BrowserStateDiagnostic | None: ...
+ def get_browser_connection_loss(self) -> BrowserStateDiagnostic | None: ...
+
+ @property
+ def active_run_owner(self) -> str | None: ...
+
async def reconnect(
self,
proxy_location: ProxyLocationInput = None,
@@ -194,3 +200,26 @@ def get_browser_state_diagnostic(browser_state: BrowserState | None) -> BrowserS
getter = getattr(browser_state, "get_browser_state_diagnostic", None)
diagnostic = getter() if callable(getter) else None
return diagnostic if isinstance(diagnostic, BrowserStateDiagnostic) else None
+
+
+def get_browser_connection_loss(browser_state: BrowserState | None) -> BrowserStateDiagnostic | None:
+ """Read the terminal loss fact without requiring every test double to implement it."""
+ if browser_state is None:
+ return None
+ getter = getattr(browser_state, "get_browser_connection_loss", None)
+ loss = getter() if callable(getter) else None
+ return loss if isinstance(loss, BrowserStateDiagnostic) else None
+
+
+def raise_if_browser_connection_lost(
+ browser_state: BrowserState | None, *, task_id: str | None = None, workflow_run_id: str | None = None
+) -> None:
+ connection_loss = get_browser_connection_loss(browser_state)
+ if connection_loss is not None:
+ raise MissingBrowserState(
+ task_id=task_id,
+ workflow_run_id=workflow_run_id,
+ diagnostic=connection_loss,
+ detected_at=datetime.now(UTC),
+ failure_reason=BROWSER_CONNECTION_LOST_ERROR_CODE,
+ )
diff --git a/skyvern/webeye/navigation.py b/skyvern/webeye/navigation.py
index d5d35899c..124895b2f 100644
--- a/skyvern/webeye/navigation.py
+++ b/skyvern/webeye/navigation.py
@@ -280,6 +280,7 @@ async def navigate_with_retry(
wait_until: Literal["load", "domcontentloaded", "commit"] = "load",
sleep: SleepFunc | None = None,
log_url: str | None = None,
+ stop_retrying: Callable[[], bool] | None = None,
) -> int | None:
# Returns the final navigation response's HTTP status when one is available (None when the
# navigation produced no response object). Additive: every caller today ignores the return; the
@@ -312,6 +313,11 @@ async def navigate_with_retry(
url=display_url, host=error.host, error_message=_NO_ADDRESS_RECORD_REFUSAL
) from None
+ # The destination check above can wait on DNS, and the caller's stop (the run's lost browser connection) may
+ # have latched meanwhile; nothing is dispatched then.
+ if stop_retrying is not None and stop_retrying():
+ raise FailedToNavigateToUrl(url=display_url, error_message="the browser connection was lost before navigation")
+
for attempt in range(retry_times):
strategy = degradation[min(attempt, len(degradation) - 1)]
LOG.info("Trying to navigate to url", url=display_url, retry_time=attempt, wait_until=strategy)
@@ -376,7 +382,21 @@ async def navigate_with_retry(
nav_error_code=driver_nav_error_code(error_str),
) from (None if redacting else error)
- if attempt >= retry_times - 1:
+ # The caller's typed stop ends retries too, rechecked after the backoff so a loss latched while sleeping
+ # never reaches another dispatch.
+ stopped = stop_retrying is not None and stop_retrying()
+ if attempt < retry_times - 1 and not stopped:
+ LOG.warning(
+ "Error while navigating to url, retrying",
+ exc_info=not redacting,
+ url=display_url,
+ retry_time=attempt,
+ wait_until=strategy,
+ error=safe_error_str,
+ )
+ await sleep(1)
+ stopped = stop_retrying is not None and stop_retrying()
+ if attempt >= retry_times - 1 or stopped:
# Terminal navigation failure is re-raised as FailedToNavigateToUrl and surfaced
# via the run's failure_reason; logging at error here double-logs an external
# (site-unreachable) outcome. Keep the traceback, drop the error level.
@@ -393,15 +413,5 @@ async def navigate_with_retry(
nav_error_code=driver_nav_error_code(error_str),
) from (None if redacting else error)
- LOG.warning(
- "Error while navigating to url, retrying",
- exc_info=not redacting,
- url=display_url,
- retry_time=attempt,
- wait_until=strategy,
- error=safe_error_str,
- )
- await sleep(1)
-
# Reached only when retry_times <= 0 (the loop never ran); a real navigation returns or raises inside.
return None
diff --git a/skyvern/webeye/real_browser_manager.py b/skyvern/webeye/real_browser_manager.py
index fa0053ac2..851bf117b 100644
--- a/skyvern/webeye/real_browser_manager.py
+++ b/skyvern/webeye/real_browser_manager.py
@@ -66,7 +66,7 @@ from skyvern.webeye.browser_settings_receipts import (
measure_timezone_receipt,
record_session_timezone_receipt,
)
-from skyvern.webeye.browser_state import BrowserState
+from skyvern.webeye.browser_state import BrowserState, raise_if_browser_connection_lost
from skyvern.webeye.cdp_frame_publisher import (
CDPFramePublisher,
stream_key_for_task,
@@ -301,11 +301,15 @@ async def _on_browser_state_acquired(
task_id: str | None = None,
browser_session_id: str | None = None,
acquire: bool = True,
+ handoff: bool = False,
) -> BrowserState:
browser_context = browser_state.browser_context
if browser_context is not None:
_retain_run_dialog_answers(browser_state, workflow_run_id, parent_workflow_run_id, live_run_ids)
await app.AGENT_FUNCTION.on_browser_context_acquired(browser_context, workflow_run_id)
+ if handoff:
+ # The run still bound to this browser owns any loss latched up to here; rebinding would hide it.
+ raise_if_browser_connection_lost(browser_state, workflow_run_id=workflow_run_id)
if acquire:
browser_state.bind_runtime_event_context(
BrowserRuntimeLogContext.for_run(
@@ -1050,6 +1054,9 @@ class RealBrowserManager(BrowserManager):
lost_external_browser = False
if browser_state is not None:
if browser_session_id and not browser_state.is_connected():
+ raise_if_browser_connection_lost(
+ browser_state, workflow_run_id=task.workflow_run_id, task_id=task.task_id
+ )
lost_external_browser = is_external_browser_state(browser_state)
LOG.warning(
"Cached persistent-session browser state for task is disconnected; reconnecting",
@@ -1295,6 +1302,7 @@ class RealBrowserManager(BrowserManager):
lost_external_browser = False
if browser_state:
if browser_session_id and not browser_state.is_connected():
+ raise_if_browser_connection_lost(browser_state, workflow_run_id=workflow_run_id)
lost_external_browser = is_external_browser_state(browser_state)
LOG.warning(
"Cached persistent-session browser state for workflow run is disconnected; reconnecting",
@@ -1313,6 +1321,7 @@ class RealBrowserManager(BrowserManager):
self._live_run_ids_sharing(browser_state),
browser_session_id=browser_session_id,
acquire=acquire,
+ handoff=browser_state.active_run_owner not in (None, workflow_run_id),
)
# When an explicit browser_session_id is provided (e.g. from a workflow
@@ -1325,6 +1334,8 @@ class RealBrowserManager(BrowserManager):
workflow_run_id=workflow_run_id, parent_workflow_run_id=parent_workflow_run_id
)
if browser_state:
+ if browser_state.active_run_owner is not None:
+ raise_if_browser_connection_lost(browser_state, workflow_run_id=workflow_run_id)
# A parent_workflow_run_id entry is shared by every child run of the same parent.
# Independent children dispatched to one long-lived worker (e.g. a sequential
# fan-out) can therefore find an earlier, already-completed sibling's torn-down
@@ -1354,6 +1365,8 @@ class RealBrowserManager(BrowserManager):
browser_profile_id=browser_profile_id,
)
working_page = await browser_state.get_working_page()
+ if browser_state.active_run_owner is not None:
+ raise_if_browser_connection_lost(browser_state, workflow_run_id=workflow_run_id)
if working_page is not None:
# always keep the browser state for the workflow run and the parent workflow run synced
self.pages[workflow_run_id] = browser_state
@@ -1370,13 +1383,19 @@ class RealBrowserManager(BrowserManager):
await _record_run_timezone_receipt(
workflow_run, browser_state, browser_session_id=None, inherited=True
)
- return await _on_browser_state_acquired(
- browser_state,
- workflow_run_id,
- parent_workflow_run_id,
- self._live_run_ids_sharing(browser_state),
- browser_session_id=browser_session_id,
- )
+ try:
+ return await _on_browser_state_acquired(
+ browser_state,
+ workflow_run_id,
+ parent_workflow_run_id,
+ self._live_run_ids_sharing(browser_state),
+ browser_session_id=browser_session_id,
+ handoff=browser_state.active_run_owner is not None,
+ )
+ except MissingBrowserState:
+ if self.pages.get(workflow_run_id) is browser_state:
+ self.pages.pop(workflow_run_id, None)
+ raise
# The inherited state is genuinely torn down (disconnected and page-less).
# Drop the stale entry and fall through to create a fresh browser for this run.
LOG.warning(
diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.py
index 8f8ec5434..2a78a0bc6 100644
--- a/skyvern/webeye/real_browser_state.py
+++ b/skyvern/webeye/real_browser_state.py
@@ -23,12 +23,14 @@ from skyvern.constants import (
NAVIGATION_MAX_RETRY_TIME,
)
from skyvern.exceptions import (
+ BROWSER_CONNECTION_LOST_ERROR_CODE,
BrowserStateDiagnostic,
EmptyBrowserContext,
ExternalBrowserUnavailable,
FailedToNavigateToUrl,
FailedToReloadPage,
FailedToStopLoadingPage,
+ MissingBrowserState,
MissingBrowserStatePage,
UnresolvableNavigationHost,
)
@@ -428,6 +430,9 @@ class RealBrowserState(BrowserState):
# A release retry after a transient DB failure must not repeat local CDP teardown.
self._remote_driver_detached = False
self._browser_state_diagnostic: BrowserStateDiagnostic | None = None
+ # The run that saw this browser end unexpectedly, and how. Unlike the diagnostic it survives a context
+ # rebuild, so that run never gets the browser back; a later run that reuses this state does not see it.
+ self._connection_loss: tuple[str | None, BrowserStateDiagnostic] | None = None
# HTTP status of the most recent navigate_to_url (None until one runs, or when it produced no
# response). Read by the Task V3 loop to classify a dead/removed starting URL; v1 ignores it.
self.last_navigation_status: int | None = None
@@ -573,10 +578,17 @@ class RealBrowserState(BrowserState):
# A tab can die on its own (a download-turned-navigation, a renderer crash) while the
# context survives. Recover here rather than in each consumer: every caller that needs a
# page treats "no page" as fatal, so one of them recovering only relocates the failure.
- if self.browser_context is None or self.external_browser or not self.is_connected():
+ if (
+ self.browser_context is None
+ or self.external_browser
+ or not self.is_connected()
+ or self.get_browser_connection_loss() is not None
+ ):
return None
async with self._reopen_working_page_lock:
already_reopened = await self.get_working_page()
+ if self.get_browser_connection_loss() is not None:
+ return None
if already_reopened is not None:
await self._close_crashed_targets()
return already_reopened
@@ -704,6 +716,7 @@ class RealBrowserState(BrowserState):
reconcile_persistent_init_scripts: bool = False,
sessionless_init_script_registrations: Collection[Any] = (),
) -> None:
+ self._refuse_after_connection_loss()
if self.browser_context is None:
if self.external_browser:
raise ExternalBrowserUnavailable.lost()
@@ -790,6 +803,7 @@ class RealBrowserState(BrowserState):
if pages:
page = pages[-1]
use_existing_page = True
+ self._refuse_after_connection_loss()
if page is None:
page = await self.browser_context.new_page()
@@ -894,6 +908,7 @@ class RealBrowserState(BrowserState):
retry_times: int = NAVIGATION_MAX_RETRY_TIME,
wait_until: Literal["load", "domcontentloaded", "commit"] = "load",
) -> None:
+ self._refuse_after_connection_loss()
landed_url: str | None = None
async def goto(strategy: str) -> object:
@@ -916,6 +931,7 @@ class RealBrowserState(BrowserState):
retry_times=retry_times,
settle=self._wait_for_settle,
wait_until=wait_until,
+ stop_retrying=lambda: self.get_browser_connection_loss() is not None,
)
except Exception as navigation_error:
await self.record_navigation_result(navigation_error)
@@ -1386,15 +1402,7 @@ class RealBrowserState(BrowserState):
observation_source: Literal["liveness_probe", "browser_event", "driver_event"] = "liveness_probe",
disconnect_evidence: DisconnectEvidence = "liveness_state",
) -> None:
- if self._browser_state_diagnostic is not None or not self._ever_connected:
- return
- pending = self._deferred_runtime_end
- if (
- self._runtime_events_deferred
- and pending is not None
- and pending.context is self.browser_context
- and pending.driver is self.pw
- ):
+ if not self._ever_connected:
return
observation = self._make_runtime_end_observation(
reason,
@@ -1406,6 +1414,18 @@ class RealBrowserState(BrowserState):
observation_source=observation_source,
disconnect_evidence=disconnect_evidence,
)
+ # Before the first-diagnostic dedupe: a probe failure may already have latched the diagnostic.
+ self._latch_connection_loss(observation)
+ if self._browser_state_diagnostic is not None:
+ return
+ pending = self._deferred_runtime_end
+ if (
+ self._runtime_events_deferred
+ and pending is not None
+ and pending.context is self.browser_context
+ and pending.driver is self.pw
+ ):
+ return
if self._runtime_events_deferred:
self._deferred_runtime_end = observation
return
@@ -1515,6 +1535,47 @@ class RealBrowserState(BrowserState):
def get_browser_state_diagnostic(self) -> BrowserStateDiagnostic | None:
return self._browser_state_diagnostic
+ def _run_owner(self) -> str | None:
+ return self._runtime_event_context.workflow_run_id or self._runtime_event_context.task_id
+
+ def _latch_connection_loss(self, observation: _RuntimeEndObservation) -> None:
+ owner = self._run_owner()
+ # No run owns an unbound or not-yet-published state, so its loss is not a run's terminal fact.
+ if (
+ owner is None
+ or self._runtime_events_deferred
+ or observation.source != "browser_event"
+ or observation.expected
+ or observation.run_phase != "active"
+ or observation.context is None
+ or observation.context is not self.browser_context
+ or observation.driver is not self.pw
+ or (self._connection_loss is not None and self._connection_loss[0] == owner)
+ ):
+ return
+ self._connection_loss = (owner, observation.diagnostic)
+
+ def get_browser_connection_loss(self) -> BrowserStateDiagnostic | None:
+ if self._connection_loss is None or self._connection_loss[0] != self._run_owner():
+ return None
+ return self._connection_loss[1]
+
+ @property
+ def active_run_owner(self) -> str | None:
+ return None if self._run_released else self._run_owner()
+
+ def _refuse_after_connection_loss(self) -> None:
+ # Reconnect, context rebuild and navigation for the owning run stop here, before any side effect.
+ connection_loss = self.get_browser_connection_loss()
+ if connection_loss is not None:
+ raise MissingBrowserState(
+ task_id=self._runtime_event_context.task_id,
+ workflow_run_id=self._runtime_event_context.workflow_run_id,
+ diagnostic=connection_loss,
+ detected_at=datetime.now(UTC),
+ failure_reason=BROWSER_CONNECTION_LOST_ERROR_CODE,
+ )
+
def _connection_status(self) -> tuple[bool, str | None]:
stopped = browser_context_stopped_reason(self.browser_context)
return stopped is None, stopped
@@ -1540,6 +1601,7 @@ class RealBrowserState(BrowserState):
browser_session_id: str | None = None,
stale_context_is_unusable: bool = False,
) -> None:
+ self._refuse_after_connection_loss()
# Rebuild through a fresh Playwright driver only after the old connection is known dead or
# an explicitly unusable connection has completed bounded shutdown.
stale_pw = self.pw
@@ -1888,6 +1950,7 @@ class RealBrowserState(BrowserState):
raise FailedToStopLoadingPage(url=page.url, error_message=repr(e))
async def new_page(self) -> Page:
+ self._refuse_after_connection_loss()
if self.browser_context is None:
raise EmptyBrowserContext()
return await self.browser_context.new_page()
@@ -1938,7 +2001,8 @@ class RealBrowserState(BrowserState):
return
except Exception as e:
self._record_reload_timeout(e)
- if i < len(strategies) - 1:
+ # A reload that failed after the owning run lost its browser is final; degrading would send another.
+ if i < len(strategies) - 1 and self.get_browser_connection_loss() is None:
LOG.warning(
"Reload timed out, degrading wait strategy",
url=url,
diff --git a/tests/unit/_task_block_fakes.py b/tests/unit/_task_block_fakes.py
index f7d72815f..9062d1d05 100644
--- a/tests/unit/_task_block_fakes.py
+++ b/tests/unit/_task_block_fakes.py
@@ -10,6 +10,8 @@ from unittest.mock import AsyncMock, MagicMock, patch
from skyvern.forge.sdk.schemas.tasks import TaskStatus
from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
+from skyvern.webeye.browser_runtime_events import BrowserRuntimeLogContext
+from skyvern.webeye.real_browser_state import RealBrowserState
def output_parameter(key: str = "task_output") -> OutputParameter:
@@ -38,6 +40,21 @@ def _workflow_run_context(values: dict[str, Any] | None = None) -> WorkflowRunCo
return ctx
+def run_bound_browser_state(workflow_run_id: str) -> tuple[RealBrowserState, MagicMock]:
+ """A run-bound browser state and its still-open current context."""
+ context = MagicMock(browser=MagicMock())
+ state = RealBrowserState(pw=MagicMock(), browser_context=context)
+ state.bind_runtime_event_context(BrowserRuntimeLogContext(workflow_run_id=workflow_run_id))
+ return state, context
+
+
+def lost_browser_state(workflow_run_id: str) -> RealBrowserState:
+ """A run-bound browser state whose current context has just closed unexpectedly."""
+ state, context = run_bound_browser_state(workflow_run_id)
+ state._on_browser_context_closed(context)
+ return state
+
+
@dataclass(frozen=True)
class BlockExecuteDeps:
task: SimpleNamespace
diff --git a/tests/unit/test_agent_step_cancellation.py b/tests/unit/test_agent_step_cancellation.py
index f3939e89e..a50dde3dc 100644
--- a/tests/unit/test_agent_step_cancellation.py
+++ b/tests/unit/test_agent_step_cancellation.py
@@ -38,6 +38,7 @@ from skyvern.services.run_cancellation import RunCancellation
from skyvern.webeye.actions.actions import ClickAction, CompleteAction, ExtractAction
from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess
from skyvern.webeye.scraper.scraped_page import ScrapedPage
+from tests.unit._task_block_fakes import lost_browser_state
from tests.unit.helpers import make_browser_state, make_organization, make_step, make_task
from tests.unit.scoped_asyncio import ScopedAsyncio
@@ -344,6 +345,48 @@ async def test_canceled_run_stops_before_the_next_action(
harness.agent.check_user_goal_complete.assert_not_awaited()
+ ("action_count", "browser_lost", "run_canceled", "expected_handled"),
+ [
+ pytest.param(2, True, False, 1, id="lost-during-a-continuable-failure"),
+ pytest.param(1, True, False, 1, id="lost-during-the-last-action"),
+ pytest.param(2, False, False, 2, id="ordinary-continuable-failure"),
+ pytest.param(2, True, True, 1, id="cancel-takes-precedence-over-the-loss"),
+ ],
+)
+async def test_lost_browser_fails_the_step_before_its_next_action(
+ monkeypatch: pytest.MonkeyPatch, action_count: int, browser_lost: bool, run_canceled: bool, expected_handled: int
+) -> None:
+ harness = _make_harness(monkeypatch, navigation_goal="Submit the application")
+ actions = [
+ _click_action(harness.task, harness.step, element_id=f"node-{i}", action_order=i) for i in range(action_count)
+ ]
+ monkeypatch.setattr("skyvern.forge.agent.parse_actions", lambda *_, **__: actions)
+ harness.browser_state.must_get_working_page = AsyncMock(return_value=harness.page)
+ canceled = _cancel_when_flagged(monkeypatch, harness)
+ connection_loss = lost_browser_state("workflow-1").get_browser_connection_loss()
+ harness.browser_state.get_browser_connection_loss = MagicMock(return_value=None)
+
+ async def _handle_action(*_args, **_kwargs) -> list[ActionResult]:
+ if browser_lost:
+ harness.browser_state.get_browser_connection_loss.return_value = connection_loss
+ canceled["yet"] = run_canceled
+ return [ActionFailure(exception=RuntimeError("Target closed"), stop_execution_on_failure=False)]
+
+ handle_action_mock = AsyncMock(side_effect=_handle_action)
+ monkeypatch.setattr("skyvern.forge.agent.ActionHandler.handle_action", handle_action_mock)
+
+ step, _ = await harness.run()
+
+ assert handle_action_mock.await_count == expected_handled
+ if browser_lost:
+ assert step.status == (StepStatus.canceled if run_canceled else StepStatus.failed)
+ harness.agent.check_user_goal_complete.assert_not_awaited()
+ else:
+ assert step.status != StepStatus.failed
+
+
@pytest.mark.asyncio
async def test_canceled_run_stops_after_the_implicit_extraction(monkeypatch: pytest.MonkeyPatch) -> None:
"""A cancel landing during the extraction that follows a complete action still stops the step."""
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index fb6d4de68..191c2d17e 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -24,7 +24,12 @@ from structlog.testing import capture_logs
from skyvern.config import Settings, settings
from skyvern.errors.errors import UserDefinedError
-from skyvern.exceptions import CompletionGateTerminationError, MissingBrowserStatePage, StepTerminationError
+from skyvern.exceptions import (
+ BROWSER_CONNECTION_LOST_ERROR_CODE,
+ CompletionGateTerminationError,
+ MissingBrowserStatePage,
+ StepTerminationError,
+)
from skyvern.forge import agent as agent_module
from skyvern.forge import app
from skyvern.forge.agent import ForgeAgent
@@ -121,6 +126,7 @@ from tests.unit._download_provider_fakes import (
_secret_bearing_validation_error,
_StallingActionDownloadObservation,
)
+from tests.unit._task_block_fakes import run_bound_browser_state
from tests.unit.helpers import (
make_action_row,
make_browser_state,
@@ -130,6 +136,7 @@ from tests.unit.helpers import (
)
from tests.unit.scoped_asyncio import ScopedAsyncio
from tests.unit.test_taskv3_loop import _ScriptedCaller
+from tests.unit.test_taskv3_tools import _FakePage
async def _run_execute_task_v3(
@@ -4173,6 +4180,172 @@ async def test_a_fifth_veto_on_a_reask_conversion_restores_the_models_own_verdic
assert task.failure_reason == "the site offers no way to continue"
+class _ModelSubstitute(_ScriptedCaller):
+ """A deterministic stand-in for the Task V3 model; no LLM is called."""
+
+ def __init__(self, script: list[list[tuple[str, dict[str, Any]]]]) -> None:
+ super().__init__(script)
+ self.original_llm_key = self.llm_key
+ self.llm_config = LLMConfig("scripted/model-substitute", [], True, False)
+
+ async def call(self, *, step: Step | None = None, **kwargs: Any) -> dict[str, Any]:
+ return await super().call(**kwargs)
+
+ def emit_retry_chain_exhausted(self, error: BaseException, prompt_name: str | None) -> None:
+ raise AssertionError(f"the model substitute never exhausts a retry chain: {error!r}")
+
+
+class _PageWithEventAfterFill(_FakePage):
+ def __init__(self, after_fill: Callable[[], None]) -> None:
+ super().__init__()
+ self._after_fill = after_fill
+
+ async def fill(self, selector: str, text: str, timeout: int | None = None) -> None:
+ await super().fill(selector, text, timeout=timeout)
+ self._after_fill()
+
+
+ ("event", "status"),
+ [
+ ("none", TaskStatus.completed),
+ ("expected_close", TaskStatus.completed),
+ ("native_close", TaskStatus.failed),
+ ("cancel_and_native_close", TaskStatus.canceled),
+ ],
+)
+async def test_execute_task_v3_native_browser_loss_after_the_last_action_fails_the_completion(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, event: str, status: TaskStatus
+) -> None:
+ # The real loop, finish tool, completion gate and post-loop write; only the model and the page are fakes.
+ agent = ForgeAgent()
+ now = datetime.now(UTC)
+ organization = make_organization(now)
+ task = make_task(
+ now,
+ organization,
+ workflow_run_id="wr_v3_browser_loss",
+ navigation_goal="Fill in the first name",
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+ step = make_step(now, task, step_id="step-v3", status=StepStatus.created, order=0, output=None)
+ state, context = run_bound_browser_state("wr_v3_browser_loss")
+ canceled = False
+
+ def _after_last_action() -> None:
+ nonlocal canceled
+ if event == "expected_close":
+ state._expect_runtime_end("normal_close", context)
+ if event == "cancel_and_native_close":
+ canceled = True
+ if event != "none":
+ # Playwright closes a closed context's pages and stops listing them.
+ page._closed = True
+ context.pages = []
+ state._on_browser_context_closed(context)
+
+ page = _PageWithEventAfterFill(_after_last_action)
+ page.url = task.url
+ context.pages = [page]
+ context._skyvern_cdp_download_interceptor = None
+ await state.set_working_page(page)
+ monkeypatch.setattr(state, "take_post_action_screenshot", AsyncMock(return_value=b"png-bytes"))
+
+ # The settle check may hold a finish twice before it accepts one.
+ finish = [("finish", {"status": "completed", "reason": "the first name is filled in"})]
+ model = _ModelSubstitute([[("observe", {})], [("type", {"selector": "#first", "text": "John"})], *[finish] * 3])
+ monkeypatch.setattr("skyvern.forge.agent.LLMCaller", lambda **_kwargs: model)
+ run_context = WorkflowRunContext(
+ workflow_title="wf",
+ workflow_id="w_test",
+ workflow_permanent_id="wpid_test",
+ workflow_run_id="wr_v3_browser_loss",
+ aws_client=MagicMock(),
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.has_workflow_run_context", lambda *_a, **_k: True
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context", lambda *_a, **_k: run_context
+ )
+
+ async def _get_task(**_kwargs: Any) -> SimpleNamespace:
+ return SimpleNamespace(status=TaskStatus.canceled if canceled else TaskStatus.running)
+
+ monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.get_task", AsyncMock(side_effect=_get_task))
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.DATABASE.workflow_runs.get_workflow_run",
+ AsyncMock(return_value=SimpleNamespace(status=WorkflowRunStatus.running, parent_workflow_run_id=None)),
+ )
+ monkeypatch.setattr("skyvern.forge.sdk.api.files.get_download_dir", lambda *_a, **_k: str(tmp_path))
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.ARTIFACT_MANAGER.create_artifact", AsyncMock(return_value="artifact-1")
+ )
+ persisted_actions: list[Any] = []
+
+ async def _create_action(action: Any) -> Any:
+ persisted_actions.append(action)
+ return action
+
+ monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.workflow_params.create_action", _create_action)
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.DATABASE.observer.get_workflow_run_block_by_task_id", AsyncMock(return_value=None)
+ )
+ monkeypatch.setattr("skyvern.services.otp_service.has_credential_totp_candidate", lambda *_a, **_k: False)
+ monkeypatch.setattr("skyvern.forge.agent.app.AGENT_FUNCTION.post_step_execution", AsyncMock())
+
+ async def fake_update_step(step: Step, status: StepStatus | None = None, **_kwargs: Any) -> Step:
+ if status is not None:
+ step.status = status
+ return step
+
+ async def fake_update_task(task: Any, status: Any = None, **kwargs: Any) -> Any:
+ task.status = status
+ task.failure_reason = kwargs.get("failure_reason")
+ return task
+
+ agent.update_step = AsyncMock(side_effect=fake_update_step)
+ agent.update_task = AsyncMock(side_effect=fake_update_task)
+ agent.clean_up_task = AsyncMock()
+
+ skyvern_context.set(
+ SkyvernContext(
+ task_id=task.task_id,
+ step_id=step.step_id,
+ organization_id=task.organization_id,
+ workflow_run_id=task.workflow_run_id,
+ )
+ )
+ try:
+ await agent._execute_task_v3(
+ task=task,
+ step=step,
+ browser_state=state,
+ organization=organization,
+ api_key=None,
+ close_browser_on_completion=True,
+ browser_session_id=None,
+ task_block=_make_block(NavigationBlock, navigation_goal="Fill in the first name"),
+ )
+ finally:
+ skyvern_context.reset()
+
+ assert ("fill", {"selector": "#first", "text": "John"}) in page.calls
+ assert task.status == status
+ if status == TaskStatus.canceled:
+ return
+ decision = persisted_actions[-1]
+ assert decision.description.endswith("finish completed")
+ if status == TaskStatus.failed:
+ assert decision.status == ActionStatus.failed
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in task.failure_reason
+ else:
+ assert decision.status == ActionStatus.completed
+ assert task.failure_reason is None
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
("task_type", "custom_key", "criteria"),
diff --git a/tests/unit/test_ai_click_empty_actions.py b/tests/unit/test_ai_click_empty_actions.py
index 31bf37bfc..231058906 100644
--- a/tests/unit/test_ai_click_empty_actions.py
+++ b/tests/unit/test_ai_click_empty_actions.py
@@ -8,7 +8,7 @@ skipped the select entirely but returned the requested value as if it had succee
from __future__ import annotations
-from contextlib import contextmanager
+from contextlib import contextmanager, nullcontext
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -426,3 +426,61 @@ class TestAiClickHandlerRefusal:
result = await self._ai_click(mock_page, mock_scraped_page, mock_context, mock_app, failure)
assert result == "xpath=//div[@id='opt']"
mock_page.locator.return_value.click.assert_awaited_once()
+
+
+class TestAiFallbackAfterBrowserLoss:
+ """A cached run that lost its browser starts no AI work and retries nothing with the raw selector."""
+
+ _CALLS = {
+ "click": (lambda ai: ai.ai_click(selector="#go", intention="Go"), "#go"),
+ "input_text": (lambda ai: ai.ai_input_text("#name", "Noor", "Name", failed_selector="#name"), "Noor"),
+ "upload_file": (
+ lambda ai: ai.ai_upload_file("#cv", "https://example.com/cv.pdf", "CV"),
+ "https://example.com/cv.pdf",
+ ),
+ "select_option": (lambda ai: ai.ai_select_option("#color", "Blue", "Pick blue"), "Blue"),
+ }
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("lost", ["before_the_call", "during_the_llm_call", "never"])
+ @pytest.mark.parametrize("method", sorted(_CALLS))
+ async def test_ai_fallback_dispatches_nothing_after_native_browser_loss(
+ self, mock_page, mock_scraped_page, mock_context, mock_app, method: str, lost: str
+ ):
+ from skyvern.exceptions import MissingBrowserState
+ from tests.unit._task_block_fakes import run_bound_browser_state
+
+ browser_state, browser_context = run_bound_browser_state("wr_ai")
+ mock_scraped_page._browser_state = browser_state
+ mock_context.workflow_run_id = "wr_ai"
+ mock_context.script_run_parameters = None
+ page_ai = RealSkyvernPageAi(mock_scraped_page, mock_page)
+ call, expected = self._CALLS[method]
+
+ async def llm(**_: object) -> dict:
+ if lost == "during_the_llm_call":
+ browser_state._on_browser_context_closed(browser_context)
+ return {"actions": []}
+
+ raw_input = AsyncMock()
+ v3_hook = AsyncMock(return_value=(None, False))
+ if lost == "before_the_call":
+ browser_state._on_browser_context_closed(browser_context)
+ with (
+ patch.object(page_ai, "_maybe_run_v3_midrun", v3_hook),
+ patch(f"{MODULE}.skyvern_context.ensure_context", return_value=mock_context),
+ patch(f"{MODULE}.skyvern_context.current", return_value=mock_context),
+ patch(f"{MODULE}.app", mock_app),
+ patch(f"{MODULE}.prompt_engine.load_prompt", return_value="mock_prompt"),
+ patch(f"{MODULE}._resolve_assist_llm_handler", AsyncMock(return_value=AsyncMock(side_effect=llm))),
+ patch(f"{MODULE}.handler_utils.input_sequentially", raw_input),
+ pytest.raises(MissingBrowserState) if lost != "never" else nullcontext(),
+ ):
+ assert await call(page_ai) == expected
+
+ raw_locator = mock_page.locator.return_value
+ raw_dispatches = raw_locator.click.await_count + raw_locator.select_option.await_count + raw_input.await_count
+ assert raw_dispatches == (1 if lost == "never" and method != "upload_file" else 0)
+ assert mock_scraped_page.generate_scraped_page.await_count == (0 if lost == "before_the_call" else 1)
+ if method == "input_text":
+ assert v3_hook.await_count == (0 if lost == "before_the_call" else 1)
diff --git a/tests/unit/test_code_block_downloads.py b/tests/unit/test_code_block_downloads.py
index f4e478070..7d01d3a06 100644
--- a/tests/unit/test_code_block_downloads.py
+++ b/tests/unit/test_code_block_downloads.py
@@ -269,6 +269,7 @@ def _fake_storage_app(
get_downloaded_file_signature_aliases=lambda _: [],
list_downloading_files_in_browser_session=in_flight or AsyncMock(return_value=[]),
),
+ BROWSER_MANAGER=SimpleNamespace(get_for_workflow_run=lambda *_args, **_kwargs: None),
AGENT_FUNCTION=SimpleNamespace(
validate_code_block=AsyncMock(),
validate_block_execution=AsyncMock(),
diff --git a/tests/unit/test_code_block_sandbox.py b/tests/unit/test_code_block_sandbox.py
index 49617e7e3..0f5ade4e2 100644
--- a/tests/unit/test_code_block_sandbox.py
+++ b/tests/unit/test_code_block_sandbox.py
@@ -13,7 +13,7 @@ import operator
import re
import warnings
from collections.abc import AsyncIterator, Callable, Mapping
-from contextlib import asynccontextmanager
+from contextlib import asynccontextmanager, nullcontext
from datetime import UTC
from datetime import date as stdlib_date
from datetime import datetime
@@ -82,6 +82,7 @@ from skyvern.webeye import dialog_handler
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.browser_artifacts import BrowserArtifacts
from skyvern.webeye.skycdp.errors import CdpError
+from tests.unit._task_block_fakes import run_bound_browser_state
from tests.unit.conftest import (
FakeClearingBrowserContext,
FakeSearchBrowserContext,
@@ -3117,6 +3118,46 @@ class TestCodeBlockMagicLink:
assert navigations == [link]
assert fills == []
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "native_close", [True, False], ids=["native_close_during_goto", "ordinary_navigation_error"]
+ )
+ async def test_link_navigation_stops_retrying_after_native_browser_loss(
+ self, monkeypatch: pytest.MonkeyPatch, native_close: bool
+ ) -> None:
+ from playwright.async_api import Error as PlaywrightError
+
+ from skyvern.constants import NAVIGATION_MAX_RETRY_TIME
+ from skyvern.exceptions import FailedToNavigateToUrl
+ from skyvern.forge.sdk.schemas.totp_codes import OTPType
+ from skyvern.forge.sdk.workflow.models import block as block_module
+ from skyvern.forge.sdk.workflow.models.block import _bind_code_block_magic_link
+ from skyvern.services.otp_service import OTPValue
+ from skyvern.webeye import navigation as navigation_module
+ from tests.unit.scoped_asyncio import ScopedAsyncio
+
+ link = "https://93.184.216.34/signin?token=x"
+ self._patch_poll(monkeypatch, _build_wrc_with_identifier(), OTPValue(value=link, type=OTPType.MAGIC_LINK))
+ browser_state, context = run_bound_browser_state(_WORKFLOW_RUN_ID)
+ monkeypatch.setattr(block_module.app.BROWSER_MANAGER, "get_for_workflow_run", lambda *_, **__: browser_state)
+ monkeypatch.setattr(navigation_module, "asyncio", ScopedAsyncio(sleep=AsyncMock()))
+ navigations: list[str] = []
+ page = _fake_code_block_page()
+
+ async def goto(url: str, **kwargs: object) -> None:
+ navigations.append(url)
+ if native_close:
+ browser_state._on_browser_context_closed(context)
+ raise PlaywrightError("Target page, context or browser has been closed")
+ raise PlaywrightError("Timeout 30000ms exceeded.")
+
+ page.goto = goto
+
+ with pytest.raises(FailedToNavigateToUrl):
+ await _bind_code_block_magic_link(_CREDENTIAL_KEY, _ORG_ID, _WORKFLOW_RUN_ID)(page)
+
+ assert navigations == [link] * (1 if native_close else NAVIGATION_MAX_RETRY_TIME)
+
@pytest.mark.asyncio
@pytest.mark.usefixtures("public_dns")
async def test_anchors_created_after_at_call_time_not_run_start(self, monkeypatch: pytest.MonkeyPatch) -> None:
@@ -3843,6 +3884,27 @@ class TestOpenPageHelperBinding:
await without_state(page, "https://example.test/detail/1")
assert len(context.opened) == cap
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("native_close", [True, False], ids=["lost_during_the_tab_count", "browser_alive"])
+ async def test_open_page_opens_no_tab_after_a_loss_during_its_tab_count(self, native_close: bool) -> None:
+ from skyvern.exceptions import MissingBrowserState
+
+ context = FakeSearchBrowserContext()
+ page = SimpleNamespace(context=context, url="https://example.test/listing")
+ browser_state, browser_context = run_bound_browser_state("wr_open_page")
+
+ async def count_tabs(**_: object) -> list[object]:
+ if native_close:
+ browser_state._on_browser_context_closed(browser_context)
+ return []
+
+ browser_state.list_valid_pages = count_tabs # type: ignore[method-assign]
+ open_page = _bind_code_block_open_page(page, [], browser_state) # type: ignore[arg-type]
+
+ with pytest.raises(MissingBrowserState) if native_close else nullcontext():
+ await open_page(page, "https://example.test/detail/1")
+ assert len(context.opened) == (0 if native_close else 1)
+
@pytest.mark.asyncio
async def test_open_page_treats_a_non_positive_cap_as_unlimited(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(settings, "BROWSER_MAX_PAGES_NUMBER", 0)
diff --git a/tests/unit/test_copilot_failure_publication.py b/tests/unit/test_copilot_failure_publication.py
index 4e268a095..057bb130b 100644
--- a/tests/unit/test_copilot_failure_publication.py
+++ b/tests/unit/test_copilot_failure_publication.py
@@ -1365,7 +1365,7 @@ async def test_successor_execute_safe_settles_capture_before_first_execution_mut
result = await _block().execute_safe(workflow_run_id="wr_test", organization_id="o_test")
assert result.success is True
- assert events == ["capture_settled", "browser_read"]
+ assert events[0] == "capture_settled" and set(events[1:]) == {"browser_read"}
@pytest.mark.asyncio
@@ -1434,7 +1434,9 @@ async def test_capture_settlement_failure_uses_protected_block_failure_path(monk
assert result.failure_reason == "CodeBlock execution failed."
assert build_result.await_args.kwargs["workflow_run_block_id"] == "wrb_successor"
- browser_lookup.assert_not_called()
+ # The failure handler may read the run's browser-loss fact from the registry; the block itself
+ # must still never reach the browser (its pre-execution screenshot) after a failed settlement.
+ browser_lookup.return_value.take_fullpage_screenshot.assert_not_called()
@pytest.mark.asyncio
diff --git a/tests/unit/test_credential_fallback.py b/tests/unit/test_credential_fallback.py
index bc43ff03f..de4e595b1 100644
--- a/tests/unit/test_credential_fallback.py
+++ b/tests/unit/test_credential_fallback.py
@@ -273,6 +273,7 @@ async def _run_fallback_retry(
flag_error: Exception | None = None,
run_tags: dict[str, str] | None = None,
run_group_item: object | None = None,
+ block_error_codes: list[str] | None = None,
) -> tuple[str | None, AsyncMock, MagicMock]:
captured: dict[str, object] = {}
@@ -301,6 +302,9 @@ async def _run_fallback_retry(
mock_app.AGENT_FUNCTION.is_block_scoped_workflow_run = AsyncMock(return_value=block_scoped)
mock_app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers = _strip_marker_header
mock_app.DATABASE.debug.has_block_run_for_workflow_run = AsyncMock(return_value=False)
+ mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(
+ return_value=[SimpleNamespace(error_codes=block_error_codes or [])]
+ )
mock_app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id = AsyncMock(return_value=run_group_item)
mock_app.WORKFLOW_SERVICE.get_workflow = AsyncMock(return_value=_workflow(parameters))
if retried_by_results is not None:
@@ -335,6 +339,19 @@ async def _run_fallback_retry(
return result, run_workflow_mock, mock_app
+async def test_any_failure_fallback_is_not_started_after_the_run_lost_its_browser() -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
+ login = _credential_parameter(fallback_credential_ids=["cred_fb1"], fallback_trigger=ANY_FAILURE)
+
+ result, run_workflow_mock, _ = await _run_fallback_retry(
+ workflow_run=workflow_run, parameters=[login], block_error_codes=["browser_connection_lost"]
+ )
+
+ assert result is None
+ run_workflow_mock.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_fallback_attempt_one_uses_first_fallback_and_preserves_other_pins() -> None:
workflow_run = _workflow_run(
diff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.py
index f0c7c2425..c588d1ce1 100644
--- a/tests/unit/test_download_file_action_handler.py
+++ b/tests/unit/test_download_file_action_handler.py
@@ -19,10 +19,11 @@ from structlog.testing import capture_logs
from skyvern.config import settings
from skyvern.constants import BROWSER_DOWNLOAD_TIMEOUT, BROWSER_DOWNLOADING_SUFFIX
from skyvern.errors.errors import UserDefinedError
-from skyvern.exceptions import BlockedHost
+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE, BlockedHost, MissingBrowserState
from skyvern.forge.agent import ForgeAgent
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.models import StepStatus
+from skyvern.webeye import real_browser_state as real_browser_state_module
from skyvern.webeye.actions.actions import (
ActionStatus,
ClickAction,
@@ -49,6 +50,7 @@ from skyvern.webeye.actions.handler import (
_record_action_owned_popup_delta,
_recover_adopted_session_blob_pdf_iframe,
_recover_blocked_inline_pdf_download,
+ _recover_download_page,
_remove_download_listener,
handle_download_file_action,
)
@@ -56,7 +58,9 @@ from skyvern.webeye.actions.responses import ActionAbort, ActionFailure, ActionS
from skyvern.webeye.browser_factory import (
mark_download_cancelled_by_skyvern,
)
+from skyvern.webeye.browser_runtime_events import BrowserRuntimeLogContext
from skyvern.webeye.cdp_download_interceptor import CDPDownloadInterceptor
+from skyvern.webeye.real_browser_state import RealBrowserState
from skyvern.webeye.scraper.scraped_page import ScrapedPage
from skyvern.webeye.utils.page import BlobActionFreshness
from tests.unit._download_provider_fakes import (
@@ -1561,6 +1565,156 @@ async def test_handle_action_reconnects_context_closed_during_download_wait(
assert results[-1].skip_remaining_actions is True
[email protected]("recovery_site", ["post_click", "post_wait"])
[email protected]("native_close", [True, False], ids=["native_context_close", "page_only_close"])
+async def test_download_page_recovery_refuses_a_browser_whose_context_closed_natively(
+ recovery_site: str, native_close: bool
+) -> None:
+ now = datetime.now(UTC)
+ organization = make_organization(now)
+ task, step, page, _, scraped_page, action = _make_download_click_context(
+ now=now,
+ organization=organization,
+ page_url="https://example.com/downloads",
+ )
+ if recovery_site == "post_click":
+ task = task.model_copy(update={"download_timeout": 0.01})
+ page.expose_binding = AsyncMock()
+ page.evaluate = AsyncMock(return_value=[])
+ context = page.context
+ context._skyvern_cdp_download_active = False
+ context._skyvern_cdp_download_interceptor = None
+ browser_state = RealBrowserState(pw=MagicMock(), browser_context=context)
+ browser_state.bind_runtime_event_context(
+ BrowserRuntimeLogContext(workflow_run_id=task.workflow_run_id, task_id=task.task_id)
+ )
+ recovered_page = MagicMock(url=page.url, context=context)
+ recovered_page.is_closed.return_value = False
+ recovered_page.evaluate = AsyncMock(return_value=[])
+ recovered_page.expose_binding = AsyncMock()
+ page_closed = False
+ page.is_closed.side_effect = lambda: page_closed
+
+ async def get_working_page(**_: object) -> object | None:
+ return None if page_closed else page
+
+ browser_state.get_working_page = AsyncMock(side_effect=get_working_page) # type: ignore[method-assign]
+ browser_state.list_valid_pages = AsyncMock(return_value=[page]) # type: ignore[method-assign]
+ browser_state.new_page = AsyncMock(return_value=recovered_page) # type: ignore[method-assign]
+ browser_state.reconnect = AsyncMock() # type: ignore[method-assign]
+ browser_state.navigate_to_url = AsyncMock() # type: ignore[method-assign]
+ browser_state.set_active_page = AsyncMock() # type: ignore[method-assign]
+
+ calls_at_close: int | None = None
+
+ def close_working_page() -> None:
+ nonlocal page_closed, calls_at_close
+ page_closed = True
+ calls_at_close = list_calls
+ if native_close:
+ browser_state._on_browser_context_closed(context)
+
+ async def click(*_args: object, **_kwargs: object) -> list[ActionSuccess]:
+ if recovery_site == "post_click":
+ close_working_page()
+ return [ActionSuccess()]
+
+ list_calls = 0
+
+ def list_files(_path: Path | str, *, attempt_started_at: datetime | None = None) -> list[str]:
+ nonlocal list_calls
+ list_calls += 1
+ if recovery_site == "post_wait" and list_calls == 2:
+ close_working_page()
+ return []
+
+ xhr_capture = MagicMock(has_in_flight_requests=False)
+ xhr_capture.drain = AsyncMock(return_value=False)
+ mock_app = MagicMock()
+ mock_app.BROWSER_MANAGER.get_for_task.return_value = browser_state
+ mock_app.DATABASE.workflow_params.create_action = AsyncMock(return_value=action)
+ mock_app.STORAGE = MagicMock()
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ with (
+ patch.object(ActionHandler, "_handle_action", side_effect=click),
+ patch("skyvern.webeye.actions.handler.BROWSER_DOWNLOAD_NO_SIGNAL_GRACE_TIME", 0),
+ patch("skyvern.webeye.actions.handler.get_download_dir", return_value=temp_dir),
+ patch("skyvern.webeye.actions.handler.list_files_in_directory", side_effect=list_files),
+ patch("skyvern.webeye.actions.handler.ScopedXhrDownloadCapture", return_value=xhr_capture),
+ patch("skyvern.webeye.actions.handler._recover_blocked_inline_pdf_download", new=AsyncMock()),
+ patch("skyvern.webeye.actions.handler.skyvern_context.current", return_value=None),
+ patch("skyvern.webeye.actions.handler.app", mock_app),
+ ):
+ handle = ActionHandler.handle_action(
+ scraped_page=scraped_page, task=task, step=step, page=page, action=action
+ )
+ if native_close:
+ with pytest.raises(MissingBrowserState) as refusal:
+ await handle
+ else:
+ await handle
+
+ if native_close:
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in str(refusal.value)
+ browser_state.new_page.assert_not_awaited()
+ browser_state.reconnect.assert_not_awaited()
+ browser_state.navigate_to_url.assert_not_awaited()
+ if recovery_site == "post_click":
+ # The download wait never starts polling after the refusal.
+ assert list_calls == calls_at_close
+ else:
+ browser_state.new_page.assert_awaited_once_with()
+ browser_state.reconnect.assert_not_awaited()
+ browser_state.navigate_to_url.assert_awaited_once_with(page=recovered_page, url="https://example.com/downloads")
+
+
+async def test_download_page_recovery_propagates_a_loss_observed_while_new_page_is_pending(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ now = datetime.now(UTC)
+ task, _, page, _, _, _ = _make_download_click_context(
+ now=now, organization=make_organization(now), page_url="https://example.com/downloads"
+ )
+ context = page.context
+ context._skyvern_cdp_download_interceptor = None
+ browser_state = RealBrowserState(pw=MagicMock(), browser_context=context)
+ browser_state.bind_runtime_event_context(
+ BrowserRuntimeLogContext(workflow_run_id=task.workflow_run_id, task_id=task.task_id)
+ )
+
+ async def new_page() -> object:
+ # The context-close event lands while newPage is pending, after the recovery entry check passed.
+ browser_state._on_browser_context_closed(context)
+ raise RuntimeError("Target page, context or browser has been closed")
+
+ driver_starts: list[bool] = []
+
+ class CountingPlaywright:
+ async def start(self) -> object:
+ driver_starts.append(True)
+ raise RuntimeError("driver start reached")
+
+ monkeypatch.setattr(real_browser_state_module, "async_playwright", lambda: CountingPlaywright())
+ create_context = AsyncMock(side_effect=RuntimeError("context factory reached"))
+ monkeypatch.setattr(real_browser_state_module.BrowserContextFactory, "create_browser_context", create_context)
+ browser_state.new_page = AsyncMock(side_effect=new_page) # type: ignore[method-assign]
+ browser_state.navigate_to_url = AsyncMock() # type: ignore[method-assign]
+
+ with pytest.raises(MissingBrowserState) as refusal:
+ await _recover_download_page(
+ browser_state, task, "https://example.com/downloads", 5.0, recovery_site="post_click"
+ )
+
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in str(refusal.value)
+ browser_state.new_page.assert_awaited_once_with()
+ assert driver_starts == []
+ create_context.assert_not_awaited()
+ browser_state.navigate_to_url.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_handle_action_restores_blank_working_page_when_download_not_triggered() -> None:
now = datetime.now(UTC)
diff --git a/tests/unit/test_print_page_block.py b/tests/unit/test_print_page_block.py
index bec52c733..e43f6af3a 100644
--- a/tests/unit/test_print_page_block.py
+++ b/tests/unit/test_print_page_block.py
@@ -4,18 +4,21 @@ recovering the print when the renderer crashes part-way through it (SKY-17342)."
import asyncio
from datetime import UTC, datetime
from types import SimpleNamespace
-from unittest.mock import AsyncMock
+from unittest.mock import AsyncMock, patch
import pytest
from playwright.async_api import Error as PlaywrightError
-from skyvern.exceptions import MissingBrowserStatePage
+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE, MissingBrowserStatePage
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.workflow.models import block as block_module
-from skyvern.forge.sdk.workflow.models.block import PrintPageBlock
+from skyvern.forge.sdk.workflow.models.block import Block, PrintPageBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
+from skyvern.schemas.workflows import BlockStatus
+from tests.unit._task_block_fakes import run_bound_browser_state
+from tests.unit.test_block_description_caching import _setup_mocks
def _output_parameter(key: str) -> OutputParameter:
@@ -653,3 +656,51 @@ async def test_print_page_block_fails_rather_than_printing_a_blank_replacement_t
assert result.success is False
assert "Printing failed" in result.failure_reason
blank_page.pdf.assert_not_awaited()
+
+
[email protected]("native_close", [True, False], ids=["native_close_during_print", "ordinary_print_failure"])
+async def test_print_page_failure_after_native_browser_loss_cannot_be_continued(
+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str, native_close: bool
+) -> None:
+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))
+ browser_state, context = run_bound_browser_state("wr_1")
+ page = SimpleNamespace(url="https://example.com/listing")
+
+ async def print_pdf(**_: object) -> bytes:
+ if native_close:
+ browser_state._on_browser_context_closed(context)
+ raise PlaywrightError("Target page, context or browser has been closed")
+ raise PlaywrightError("Page.pdf: some other failure")
+
+ page.pdf = print_pdf
+ browser_state.get_working_page = AsyncMock(return_value=page) # type: ignore[method-assign]
+ browser_state.list_valid_pages = AsyncMock(return_value=[page]) # type: ignore[method-assign]
+ browser_state.take_fullpage_screenshot = AsyncMock(return_value=None) # type: ignore[method-assign]
+ block = PrintPageBlock(label="print", output_parameter=_output_parameter("print_out"), continue_on_failure=True)
+
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,
+ patch.object(Block, "_generate_workflow_run_block_description", new_callable=AsyncMock),
+ patch.object(Block, "_invalidate_stale_output_on_failure", new_callable=AsyncMock),
+ ):
+ _setup_mocks(mock_app)
+ mock_app.BROWSER_MANAGER.get_for_workflow_run.return_value = browser_state
+ monkeypatch.setattr(PrintPageBlock, "get_or_create_browser_state", AsyncMock(return_value=browser_state))
+
+ result = await block.execute_safe(workflow_run_id="wr_1", organization_id="o_1")
+ persisted_codes = {
+ code
+ for update in mock_app.DATABASE.observer.update_workflow_run_block.await_args_list
+ for code in update.kwargs.get("error_codes") or []
+ }
+
+ assert result.status == BlockStatus.failed
+ assert result.failure_reason.startswith("Failed to generate PDF")
+ if native_close:
+ assert result.can_continue_after_failure is False
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in result.error_codes
+ assert persisted_codes == {BROWSER_CONNECTION_LOST_ERROR_CODE}
+ else:
+ assert result.can_continue_after_failure is True
+ assert persisted_codes == set()
diff --git a/tests/unit/test_real_browser_manager.py b/tests/unit/test_real_browser_manager.py
index 43aec0b05..9cd67ecfa 100644
--- a/tests/unit/test_real_browser_manager.py
+++ b/tests/unit/test_real_browser_manager.py
@@ -8,7 +8,7 @@ on every call and re-invoke navigate_to_url() on every step.
import asyncio
from collections.abc import Iterator
-from contextlib import contextmanager
+from contextlib import contextmanager, nullcontext
from dataclasses import replace
from itertools import pairwise
from types import SimpleNamespace
@@ -19,8 +19,10 @@ import pytest
from structlog.testing import capture_logs
from skyvern.exceptions import (
+ BROWSER_CONNECTION_LOST_ERROR_CODE,
BrowserSessionAlreadyOccupiedError,
FailedToNavigateToUrl,
+ MissingBrowserState,
MissingBrowserStateForBrowserSession,
)
from skyvern.forge import app as forge_app
@@ -49,6 +51,7 @@ from skyvern.webeye.browser_runtime_events import BrowserRuntimeLogContext
from skyvern.webeye.real_browser_manager import RealBrowserManager, _PersistentSessionLease
from skyvern.webeye.real_browser_state import RealBrowserState
from skyvern.webeye.utils.page import SkyvernFrame
+from tests.unit._task_block_fakes import lost_browser_state, run_bound_browser_state
from tests.unit.forge_log_capture import capture_runtime_logs
@@ -4214,3 +4217,171 @@ async def test_the_receipt_accepts_the_browser_name_for_the_requested_zone_and_n
)
assert receipt.status == expected_status
+
+
+async def test_persistent_session_lost_mid_run_is_not_reconnected_for_that_run() -> None:
+ manager = RealBrowserManager()
+ state = lost_browser_state("wfr_pbs")
+ manager.pages["wfr_pbs"] = state
+
+ with (
+ patch.object(manager, "_create_browser_state", new=AsyncMock()) as mock_create,
+ pytest.raises(MissingBrowserState) as exc_info,
+ ):
+ await manager.get_or_create_for_workflow_run(
+ workflow_run=make_workflow_run("wfr_pbs"), url=None, browser_session_id="pbs_1"
+ )
+
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in str(exc_info.value)
+ mock_create.assert_not_awaited()
+ assert manager.pages["wfr_pbs"] is state
+
+
[email protected]("loss", ["before_inheritance", "during_the_liveness_probe", "probe_failure_only"])
+async def test_child_run_does_not_get_a_fresh_browser_after_the_parent_lost_its_browser(loss: str) -> None:
+ manager = RealBrowserManager()
+ parent_state, context = run_bound_browser_state("wfr_parent")
+ context._impl_obj = SimpleNamespace(_closed=False, _close_was_called=False, _connection=None)
+ context.browser.is_connected.return_value = True
+
+ def probe() -> None:
+ if loss == "during_the_liveness_probe":
+ parent_state._on_browser_context_closed(context)
+ raise TargetClosedError("Target page, context or browser has been closed")
+
+ context.cookies = AsyncMock(side_effect=probe)
+ if loss == "before_inheritance":
+ parent_state._on_browser_context_closed(context)
+ manager.pages["wfr_parent"] = parent_state
+ fresh_state = MagicMock()
+ fresh_state.get_or_create_page = AsyncMock()
+
+ with patch.object(manager, "_create_browser_state", new=AsyncMock(return_value=fresh_state)) as mock_create:
+ acquire = manager.get_or_create_for_workflow_run(
+ workflow_run=make_workflow_run("wfr_child", parent_workflow_run_id="wfr_parent"),
+ url="https://example.com",
+ browser_session_id=None,
+ )
+ if loss == "probe_failure_only":
+ assert await acquire is fresh_state
+ assert manager.pages["wfr_parent"] is fresh_state
+ return
+ with pytest.raises(MissingBrowserState, match=BROWSER_CONNECTION_LOST_ERROR_CODE):
+ await acquire
+
+ assert context.cookies.await_count == (0 if loss == "before_inheritance" else 1)
+ mock_create.assert_not_awaited()
+ assert manager.pages["wfr_parent"] is parent_state
+ assert parent_state.get_browser_connection_loss() is not None
+ assert "wfr_child" not in manager.pages
+
+
[email protected]("seam", ["frame_publisher", "timezone_receipt", "acquisition_hook", None])
+async def test_child_does_not_take_over_a_parent_browser_lost_during_the_handoff(seam: str | None) -> None:
+ manager = RealBrowserManager()
+ parent_state, context = run_bound_browser_state("wfr_parent")
+ parent_state.get_working_page = AsyncMock(return_value=MagicMock())
+ manager.pages["wfr_parent"] = parent_state
+
+ def lose_at(name: str) -> None:
+ if seam == name:
+ parent_state._on_browser_context_closed(context)
+
+ with (
+ patch.object(manager, "_start_frame_publisher", AsyncMock(side_effect=lambda **_: lose_at("frame_publisher"))),
+ patch.object(
+ real_browser_manager,
+ "_record_run_timezone_receipt",
+ AsyncMock(side_effect=lambda *_, **__: lose_at("timezone_receipt")),
+ ),
+ patch.object(
+ real_browser_manager.app.AGENT_FUNCTION,
+ "on_browser_context_acquired",
+ AsyncMock(side_effect=lambda *_: lose_at("acquisition_hook")),
+ ),
+ patch.object(manager, "_create_browser_state", new=AsyncMock()) as mock_create,
+ pytest.raises(MissingBrowserState, match=BROWSER_CONNECTION_LOST_ERROR_CODE) if seam else nullcontext(),
+ ):
+ await manager.get_or_create_for_workflow_run(
+ workflow_run=make_workflow_run("wfr_child", parent_workflow_run_id="wfr_parent"), browser_session_id=None
+ )
+
+ mock_create.assert_not_awaited()
+ assert manager.pages["wfr_parent"] is parent_state
+ assert manager.pages.get("wfr_child") is (None if seam else parent_state)
+ assert parent_state.runtime_event_context.workflow_run_id == ("wfr_parent" if seam else "wfr_child")
+ assert (parent_state.get_browser_connection_loss() is not None) == bool(seam)
+
+
[email protected]("bound_to", ["finished_child_lost", "finished_child_healthy", "own_run_lost"])
+async def test_run_does_not_take_over_a_shared_browser_its_finished_child_lost(bound_to: str) -> None:
+ manager = RealBrowserManager()
+ owner = "wfr_parent" if bound_to == "own_run_lost" else "wfr_child"
+ state, context = run_bound_browser_state(owner)
+ if bound_to != "finished_child_healthy":
+ state._on_browser_context_closed(context)
+ # A finished child that shared the parent's browser leaves it bound to itself; only the parent key remains.
+ manager.pages["wfr_parent"] = state
+
+ with (
+ patch.object(manager, "_create_browser_state", new=AsyncMock()) as mock_create,
+ pytest.raises(MissingBrowserState, match=BROWSER_CONNECTION_LOST_ERROR_CODE)
+ if bound_to == "finished_child_lost"
+ else nullcontext(),
+ ):
+ assert await manager.get_or_create_for_workflow_run(make_workflow_run("wfr_parent")) is state
+
+ mock_create.assert_not_awaited()
+ assert manager.pages == {"wfr_parent": state}
+ assert state.runtime_event_context.workflow_run_id == (
+ "wfr_child" if bound_to == "finished_child_lost" else "wfr_parent"
+ )
+ assert (state.get_browser_connection_loss() is not None) == (bound_to != "finished_child_healthy")
+
+
+async def test_released_owners_loss_does_not_refuse_an_independent_later_owner() -> None:
+ manager = RealBrowserManager()
+ state = lost_browser_state("wfr_earlier")
+ state.mark_run_released()
+ manager.pages["wfr_later"] = state
+
+ assert (
+ await manager.get_or_create_for_workflow_run(make_workflow_run("wfr_later"), browser_session_id=None) is state
+ )
+ assert state.runtime_event_context.workflow_run_id == "wfr_later"
+ assert state.get_browser_connection_loss() is None
+
+
[email protected]("released", [False, True])
+async def test_parent_keyed_loss_only_blocks_an_active_owner(released: bool, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(forge_app.AGENT_FUNCTION, "on_browser_run_released", MagicMock())
+ manager = RealBrowserManager()
+ state = lost_browser_state("wfr_sibling")
+ if released:
+ state.mark_run_released()
+ manager.pages["wfr_parent"] = state
+ fresh_state, _ = run_bound_browser_state("wfr_child")
+ fresh_state.get_or_create_page = AsyncMock()
+ state.get_or_create_page = AsyncMock(side_effect=AssertionError("stale browser must not be reconnected"))
+
+ with patch.object(manager, "_create_browser_state", new=AsyncMock(return_value=fresh_state)) as create:
+ acquire = manager.get_or_create_for_workflow_run(
+ make_workflow_run("wfr_child", parent_workflow_run_id="wfr_parent")
+ )
+ if released:
+ assert await acquire is fresh_state
+ assert manager.pages["wfr_parent"] is fresh_state
+ assert manager.pages["wfr_child"] is fresh_state
+ create.assert_awaited_once()
+ else:
+ with pytest.raises(MissingBrowserState, match=BROWSER_CONNECTION_LOST_ERROR_CODE):
+ await acquire
+ assert manager.pages == {"wfr_parent": state}
+ create.assert_not_awaited()
+ state.get_or_create_page.assert_not_awaited()
diff --git a/tests/unit/test_real_browser_state_arm.py b/tests/unit/test_real_browser_state_arm.py
index 5032c281a..b3bcdcc85 100644
--- a/tests/unit/test_real_browser_state_arm.py
+++ b/tests/unit/test_real_browser_state_arm.py
@@ -27,6 +27,7 @@ def _state(recorder: object, working_page: object) -> RealBrowserState:
st.browser_context = MagicMock() # non-None -> creation branch skipped
st.browser_context.pages = []
st.external_browser = False
+ st._connection_loss = None
st.browser_artifacts = SimpleNamespace(_display_recorder=recorder, remote_browser_session_id=None)
st.get_working_page = AsyncMock(return_value=working_page)
st.set_working_page = AsyncMock()
diff --git a/tests/unit/test_real_browser_state_navigate.py b/tests/unit/test_real_browser_state_navigate.py
index 4c8d48e4d..b9c97e7e6 100644
--- a/tests/unit/test_real_browser_state_navigate.py
+++ b/tests/unit/test_real_browser_state_navigate.py
@@ -30,8 +30,10 @@ from skyvern.webeye.real_browser_state import RealBrowserState, _same_page_ignor
def browser_state(monkeypatch: pytest.MonkeyPatch) -> RealBrowserState:
retry_sleep = AsyncMock()
monkeypatch.setattr(real_browser_state, "navigate_with_retry", partial(navigate_with_retry, sleep=retry_sleep))
- # Bypass __init__; navigate_to_url only uses `self` for LOG context and _wait_for_settle.
+ # Bypass __init__; navigate_to_url only uses `self` for LOG context, _wait_for_settle and the
+ # run's browser-loss fact (none here).
state = RealBrowserState.__new__(RealBrowserState)
+ state._connection_loss = None
monkeypatch.setattr(state, "_wait_for_settle", AsyncMock())
return state
diff --git a/tests/unit/test_real_browser_state_reload.py b/tests/unit/test_real_browser_state_reload.py
index 04d5181d0..f4cd580fc 100644
--- a/tests/unit/test_real_browser_state_reload.py
+++ b/tests/unit/test_real_browser_state_reload.py
@@ -24,6 +24,7 @@ _AGENT_MODULE = "skyvern.forge.agent"
def browser_state() -> RealBrowserState:
state = RealBrowserState.__new__(RealBrowserState)
state.engine_selection = None
+ state._connection_loss = None
return state
diff --git a/tests/unit/test_script_service_block_outcome.py b/tests/unit/test_script_service_block_outcome.py
index 6456c5d65..38225adc7 100644
--- a/tests/unit/test_script_service_block_outcome.py
+++ b/tests/unit/test_script_service_block_outcome.py
@@ -1,17 +1,475 @@
"""Outcome records from cached blocks and loops."""
+import contextlib
+from collections.abc import AsyncIterator, Awaitable, Callable, Iterator
+from datetime import datetime, timezone
from types import SimpleNamespace
-from unittest.mock import AsyncMock, MagicMock
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from playwright.async_api import Error as PlaywrightError
+from skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAi
+from skyvern.core.script_generations.script_skyvern_page import ScriptRunContextManager, ScriptSkyvernPage
+from skyvern.core.script_generations.skyvern_page import RunContext
+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE, MissingBrowserState, ScriptTerminationException
from skyvern.forge import app
from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.schemas.tasks import TaskStatus
from skyvern.forge.sdk.workflow.context_manager import BlockOutcome, WorkflowRunContext
-from skyvern.forge.sdk.workflow.models.block import ForLoopBlock
-from skyvern.schemas.workflows import BlockStatus
+from skyvern.forge.sdk.workflow.models.block import (
+ ActionBlock,
+ Block,
+ ExtractionBlock,
+ FileDownloadBlock,
+ ForLoopBlock,
+ HttpRequestBlock,
+ LoginBlock,
+ NavigationBlock,
+ TextPromptBlock,
+ UrlBlock,
+ ValidationBlock,
+ WhileLoopBlock,
+)
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
+from skyvern.schemas.workflows import BlockResult, BlockStatus, BlockType
from skyvern.services import script_service
+from tests.unit._task_block_fakes import run_bound_browser_state
from tests.unit.conftest import make_block_output_parameter
+from tests.unit.helpers import make_organization, make_task
+from tests.unit.scoped_asyncio import ScopedAsyncio
+
+
+class _BlockRows:
+ """The workflow_run_block rows a cached script run writes, by id."""
+
+ def __init__(self) -> None:
+ self.rows: dict[str, dict[str, Any]] = {}
+
+ def create(self, label: str | None) -> str:
+ workflow_run_block_id = f"wrb_{len(self.rows)}"
+ self.rows[workflow_run_block_id] = {
+ "label": label,
+ "status": BlockStatus.running,
+ "error_codes": None,
+ "output": None,
+ }
+ return workflow_run_block_id
+
+ async def update(self, workflow_run_block_id: str, **fields: Any) -> SimpleNamespace:
+ row = self.rows[workflow_run_block_id]
+ for key in ("status", "output"):
+ if fields.get(key):
+ row[key] = fields[key]
+ if fields.get("error_codes") is not None:
+ row["error_codes"] = fields["error_codes"]
+ return SimpleNamespace(label=row["label"], error_codes=row["error_codes"])
+
+ def labelled(self, label: str) -> list[dict[str, Any]]:
+ return [row for row in self.rows.values() if row["label"] == label]
+
+
+def cached_run(monkeypatch: pytest.MonkeyPatch) -> Iterator[SimpleNamespace]:
+ """A cached script run over the real helpers, with only the repository and storage edges faked."""
+ now = datetime.now(timezone.utc)
+ organization = make_organization(now)
+ rows = _BlockRows()
+ browser_state, browser_context = run_bound_browser_state("wr_cached")
+ workflow_context = WorkflowRunContext(
+ workflow_title="test",
+ workflow_id="wf",
+ workflow_permanent_id="wpid",
+ workflow_run_id="wr_cached",
+ aws_client=AsyncMock(),
+ )
+ run_context = skyvern_context.SkyvernContext(
+ organization_id="o_cached", workflow_id="wf", workflow_run_id="wr_cached", script_mode=True
+ )
+ manager = ScriptRunContextManager()
+ manager.set_run_context(RunContext(parameters={}, page=MagicMock()))
+ workflow = SimpleNamespace(ai_fallback=False, workflow_definition=SimpleNamespace(blocks=[]))
+ billed = AsyncMock()
+
+ async def create_block(
+ block_type: BlockType, label: str | None = None, **_: object
+ ) -> tuple[str, str | None, str | None]:
+ workflow_run_block_id = rows.create(label)
+ if block_type in (BlockType.FOR_LOOP, BlockType.WAIT):
+ return workflow_run_block_id, None, None
+ run_context.task_id = f"tsk_{workflow_run_block_id}"
+ run_context.step_id = f"stp_{workflow_run_block_id}"
+ return workflow_run_block_id, run_context.task_id, run_context.step_id
+
+ async def update_task(task_id: str, status: TaskStatus, failure_reason: str | None = None, **_: object) -> Any:
+ return make_task(now, organization, task_id=task_id, status=status, failure_reason=failure_reason)
+
+ monkeypatch.setattr(script_service, "script_run_context_manager", manager)
+ monkeypatch.setattr(script_service, "_create_workflow_block_run_and_task", create_block)
+ monkeypatch.setattr(
+ script_service,
+ "_validate_and_get_output_parameter",
+ AsyncMock(
+ return_value=script_service.BlockValidationOutput(
+ context=run_context,
+ label="rows",
+ output_parameter=make_block_output_parameter("rows_output"),
+ input_parameters=[],
+ workflow=workflow,
+ workflow_id="wf",
+ workflow_run_id="wr_cached",
+ organization_id="o_cached",
+ )
+ ),
+ )
+ monkeypatch.setattr(script_service, "run_lease_lost", AsyncMock(return_value=False))
+ monkeypatch.setattr(script_service, "_record_output_parameter_value", AsyncMock())
+ monkeypatch.setattr(
+ app.WORKFLOW_CONTEXT_MANAGER, "get_workflow_run_context", MagicMock(return_value=workflow_context)
+ )
+ monkeypatch.setattr(app.BROWSER_MANAGER, "get_for_workflow_run", MagicMock(return_value=browser_state))
+ monkeypatch.setattr(app.DATABASE.observer, "update_workflow_run_block", rows.update)
+ monkeypatch.setattr(app.DATABASE.tasks, "update_step", AsyncMock(return_value=SimpleNamespace(order=0)))
+ monkeypatch.setattr(app.DATABASE.tasks, "update_task", update_task)
+ monkeypatch.setattr(app.DATABASE.tasks, "get_step", AsyncMock(return_value=SimpleNamespace(step_id="stp")))
+ monkeypatch.setattr(app.DATABASE.tasks, "get_task", AsyncMock(return_value=make_task(now, organization)))
+ monkeypatch.setattr(app.DATABASE.organizations, "get_organization", AsyncMock(return_value=organization))
+ monkeypatch.setattr(app.DATABASE.workflows, "get_workflow", AsyncMock(return_value=workflow))
+ monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run", AsyncMock(return_value=workflow))
+ monkeypatch.setattr(app.AGENT_FUNCTION, "post_cache_step_execution", billed)
+ monkeypatch.setattr(app.ARTIFACT_MANAGER, "flush_step_archive", AsyncMock())
+ monkeypatch.setattr(app.STORAGE, "get_current_attempt_downloaded_files", AsyncMock(return_value=[]))
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "get_recent_task_screenshot_artifacts", AsyncMock(return_value=[]))
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "get_recent_workflow_screenshot_artifacts", AsyncMock(return_value=[]))
+ with skyvern_context.scoped(run_context):
+ yield SimpleNamespace(
+ rows=rows,
+ manager=manager,
+ browser_state=browser_state,
+ browser_context=browser_context,
+ billed=billed,
+ workflow_context=workflow_context,
+ run_context=run_context,
+ )
+
+
[email protected]("child", ["native_loss", "ordinary_error", "healthy"])
[email protected]("helper", ["run_task", "action", "login"])
+async def test_cached_loop_stops_at_the_child_that_lost_its_browser(
+ cached_run: SimpleNamespace, helper: str, child: str
+) -> None:
+ cached_helper = getattr(script_service, helper)
+ opened: list[int] = []
+
+ async def open_page(page: object, context: object) -> dict[str, str]:
+ opened.append(len(opened))
+ if child == "native_loss":
+ cached_run.browser_state._on_browser_context_closed(cached_run.browser_context)
+ raise PlaywrightError("Target page, context or browser has been closed")
+ if child == "ordinary_error":
+ raise PlaywrightError("Timeout 30000ms exceeded.")
+ return {"opened": "yes"}
+
+ cached_run.manager.set_cached_fn("open_page", open_page)
+ after_child: list[int] = []
+
+ async def generated_loop() -> None:
+ # Generated code leaves an abandoned loop to the garbage collector; closing it here keeps that late
+ # finalizer off the test's closing event loop.
+ async with contextlib.aclosing(script_service.loop([1, 2], label="rows")) as rows:
+ async for _ in rows:
+ await cached_helper(prompt="Open the page", label="open_page", cache_key="open_page")
+ after_child.append(len(after_child))
+ await script_service.wait(0, label="pause")
+
+ if child == "native_loss":
+ with pytest.raises(MissingBrowserState):
+ await generated_loop()
+ else:
+ await generated_loop()
+
+ open_rows = cached_run.rows.labelled("open_page")
+ (loop_row,) = cached_run.rows.labelled("rows")
+ if child == "native_loss":
+ assert opened == [0]
+ assert after_child == []
+ assert cached_run.rows.labelled("pause") == []
+ assert loop_row["status"] != BlockStatus.completed
+ assert [row["status"] for row in open_rows] == [BlockStatus.failed]
+ assert open_rows[0]["error_codes"] == [BROWSER_CONNECTION_LOST_ERROR_CODE]
+ assert open_rows[0]["output"]
+ cached_run.billed.assert_awaited_once()
+ assert cached_run.workflow_context.get_block_outcome("open_page").error_codes == [
+ BROWSER_CONNECTION_LOST_ERROR_CODE
+ ]
+ assert cached_run.run_context.prompt is None
+ else:
+ expected = BlockStatus.failed if child == "ordinary_error" else BlockStatus.completed
+ assert opened == [0, 1]
+ assert len(cached_run.rows.labelled("pause")) == 2
+ assert loop_row["status"] == BlockStatus.completed
+ assert [row["status"] for row in open_rows] == [expected, expected]
+ assert [row["error_codes"] for row in open_rows] == [None, None]
+
+
+_UNCACHED_CHILDREN: dict[str, tuple[type[Block], Callable[[], Awaitable[object]]]] = {
+ "run_task": (NavigationBlock, lambda: script_service.run_task(prompt="Open the page", label="open_page")),
+ "goto": (UrlBlock, lambda: script_service.goto("https://93.184.216.34/next", label="go")),
+ "action": (ActionBlock, lambda: script_service.action(prompt="Click it", label="click")),
+ "login": (LoginBlock, lambda: script_service.login(prompt="Sign in", label="sign_in")),
+ "extract": (ExtractionBlock, lambda: script_service.extract(prompt="Read it", label="read")),
+ "prompt": (TextPromptBlock, lambda: script_service.prompt(prompt="Summarize", label="summary")),
+}
+
+
[email protected]("browser_lost", [True, False], ids=["browser_lost", "browser_alive"])
[email protected]("helper", sorted(_UNCACHED_CHILDREN))
+async def test_uncached_child_that_failed_after_browser_loss_ends_the_script(
+ cached_run: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, helper: str, browser_lost: bool
+) -> None:
+ block_type, run_child = _UNCACHED_CHILDREN[helper]
+ failed = BlockResult(
+ success=False,
+ output_parameter=make_block_output_parameter("child_output"),
+ status=BlockStatus.failed,
+ failure_reason="Failed",
+ )
+ monkeypatch.setattr(block_type, "execute_safe", AsyncMock(return_value=failed))
+ fallback_page = cached_run.manager.ensure_run_context().page
+ fallback_page.goto = AsyncMock()
+ if browser_lost:
+ cached_run.browser_state._on_browser_context_closed(cached_run.browser_context)
+
+ if browser_lost:
+ with pytest.raises(MissingBrowserState):
+ await run_child()
+ else:
+ await run_child()
+
+ fallback_page.goto.assert_not_awaited()
+
+
+ ("kind", "seam", "iterations", "conditions"),
+ [
+ ("for", "before_the_loop", [], 0),
+ ("while", "before_the_loop", [], 0),
+ ("for", "during_the_baseline", [], 0),
+ ("while", "during_the_baseline", [], 1),
+ ("while", "during_the_condition", [], 1),
+ ("for", "in_a_body", [0], 0),
+ ("while", "in_a_body", [0], 1),
+ ("for", "in_the_final_body", [0, 1], 0),
+ ("for", "browser_alive", [0, 1], 0),
+ ("while", "browser_alive", [0, 1], 3),
+ ],
+)
+async def test_no_loop_starts_iterates_or_completes_after_its_run_lost_the_browser(
+ cached_run: SimpleNamespace,
+ monkeypatch: pytest.MonkeyPatch,
+ kind: str,
+ seam: str,
+ iterations: list[int],
+ conditions: int,
+) -> None:
+ def lose() -> None:
+ cached_run.browser_state._on_browser_context_closed(cached_run.browser_context)
+
+ evaluated: list[bool] = []
+
+ async def evaluate_condition(*_: object, **__: object) -> bool:
+ evaluated.append(seam != "during_the_condition" and len(evaluated) < 2)
+ if seam == "during_the_condition":
+ lose()
+ return evaluated[-1]
+
+ async def baseline(**_: object) -> list[object]:
+ if seam == "during_the_baseline":
+ lose()
+ return []
+
+ monkeypatch.setattr(WhileLoopBlock, "_evaluate_condition", evaluate_condition)
+ monkeypatch.setattr(app.STORAGE, "get_current_attempt_downloaded_files", baseline)
+ if seam == "before_the_loop":
+ lose()
+ if kind == "for":
+ rows = script_service.loop([1, 2], label="rows")
+ else:
+ rows = script_service.while_loop("{{ true }}", label="rows")
+ seen: list[int] = []
+ lost = seam != "browser_alive"
+
+ with pytest.raises(MissingBrowserState) if lost else contextlib.nullcontext():
+ async with contextlib.aclosing(rows) as items:
+ async for _ in items:
+ seen.append(len(seen))
+ cached_run.run_context.loop_output_values[-1].append(f"row {len(seen)}")
+ if seam == "in_a_body" or (seam == "in_the_final_body" and len(seen) == 2):
+ lose()
+
+ assert seen == iterations
+ assert len(evaluated) == conditions
+ loop_rows = cached_run.rows.labelled("rows")
+ if seam == "before_the_loop":
+ assert loop_rows == []
+ return
+ (loop_row,) = loop_rows
+ assert loop_row["output"] == ([[f"row {n + 1}"] for n in iterations] or None)
+ assert loop_row["status"] == (BlockStatus.failed if lost else BlockStatus.completed)
+ assert loop_row["error_codes"] == ([BROWSER_CONNECTION_LOST_ERROR_CODE] if lost else None)
+
+
+ ("helper", "seam"),
+ [
+ (helper, seam)
+ for helper in ("conditional", "run_task", "action", "login", "extract", "download")
+ for seam in ("lost_during_the_wait", "lost_during_setup", "browser_alive")
+ if (helper, seam) != ("download", "browser_alive")
+ ],
+)
+async def test_no_cached_statement_runs_after_its_run_lost_the_browser(
+ cached_run: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, helper: str, seam: str
+) -> None:
+ def lose() -> None:
+ cached_run.browser_state._on_browser_context_closed(cached_run.browser_context)
+
+ async def sleep(_: float) -> None:
+ if seam == "lost_during_the_wait":
+ lose()
+
+ create_row = script_service._create_workflow_block_run_and_task
+
+ async def create_row_then_lose(*args: Any, **kwargs: Any) -> tuple[str, str | None, str | None]:
+ created = await create_row(*args, **kwargs)
+ if seam == "lost_during_setup" and kwargs.get("label") == helper:
+ lose()
+ return created
+
+ evaluated: list[str] = []
+
+ async def cached_statement(page: object, context: object) -> dict[str, Any]:
+ evaluated.append(helper)
+ return {"next_block_label": None, "branch_index": 0}
+
+ monkeypatch.setattr(script_service, "asyncio", ScopedAsyncio(sleep=sleep))
+ monkeypatch.setattr(script_service, "_create_workflow_block_run_and_task", create_row_then_lose)
+ monkeypatch.setattr(script_service, "CACHED_DOWNLOAD_NO_FILE_GRACE_SECONDS", 0)
+ cached_run.manager.set_cached_fn(helper, cached_statement)
+ statements: list[str] = []
+
+ async def generated_function() -> None:
+ await script_service.wait(1, label="pause")
+ if helper == "conditional":
+ await script_service.conditional(label=helper)
+ else:
+ await getattr(script_service, helper)(prompt="Read the page", label=helper, cache_key=helper)
+ statements.append("after")
+
+ lost = seam != "browser_alive"
+ with pytest.raises(MissingBrowserState) if lost else contextlib.nullcontext():
+ await generated_function()
+
+ assert evaluated == ([] if lost else [helper])
+ assert statements == ([] if lost else ["after"])
+ rows = [(row["status"], row["error_codes"]) for row in cached_run.rows.labelled(helper)]
+ if seam == "lost_during_the_wait":
+ assert rows == []
+ elif seam == "lost_during_setup":
+ assert rows == [(BlockStatus.failed, [BROWSER_CONNECTION_LOST_ERROR_CODE])]
+ else:
+ assert rows == [(BlockStatus.completed, None)]
+
+
[email protected]("lost", [True, False], ids=["lost_during_setup", "browser_alive"])
+async def test_cached_request_is_not_sent_after_a_loss_during_its_setup(
+ cached_run: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, lost: bool
+) -> None:
+ validated = await script_service._validate_and_get_output_parameter()
+
+ async def validate_then_lose(*_: object) -> script_service.BlockValidationOutput:
+ if lost:
+ cached_run.browser_state._on_browser_context_closed(cached_run.browser_context)
+ return validated
+
+ @contextlib.asynccontextmanager
+ async def admit(_: str) -> AsyncIterator[SimpleNamespace]:
+ yield SimpleNamespace(status=WorkflowRunStatus.running)
+
+ sent = AsyncMock(return_value=BlockResult(success=True, output_parameter=validated.output_parameter))
+ monkeypatch.setattr(script_service, "_validate_and_get_output_parameter", validate_then_lose)
+ monkeypatch.setattr(
+ app.DATABASE.observer,
+ "create_workflow_run_block",
+ AsyncMock(side_effect=lambda **kw: SimpleNamespace(workflow_run_block_id=cached_run.rows.create(kw["label"]))),
+ )
+ monkeypatch.setattr(app.DATABASE.workflow_runs, "admit_workflow_run_block_dispatch", admit)
+ monkeypatch.setattr(HttpRequestBlock, "execute", sent)
+
+ with pytest.raises(MissingBrowserState) if lost else contextlib.nullcontext():
+ await script_service.http_request(method="POST", url="https://example.com/orders", label="rows")
+
+ assert sent.await_count == (0 if lost else 1)
+ assert [row["error_codes"] for row in cached_run.rows.labelled("rows")] == [
+ [BROWSER_CONNECTION_LOST_ERROR_CODE] if lost else None
+ ]
+
+
[email protected]("browser_lost", [True, False], ids=["browser_lost_during_validation", "ordinary_failure"])
+async def test_cached_page_validate_after_browser_loss_ends_the_script(
+ cached_run: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, browser_lost: bool
+) -> None:
+ async def validate_page(*_args: object, **_kwargs: object) -> BlockResult:
+ if browser_lost:
+ cached_run.browser_state._on_browser_context_closed(cached_run.browser_context)
+ return BlockResult(
+ success=False,
+ output_parameter=make_block_output_parameter("check_output"),
+ status=BlockStatus.failed,
+ failure_reason="The order is not placed",
+ )
+
+ monkeypatch.setattr(ValidationBlock, "execute_safe", validate_page)
+ scraped_page = SimpleNamespace(_browser_state=cached_run.browser_state)
+ raw_page = SimpleNamespace(url="https://example.com/checkout")
+ with patch("skyvern.core.script_generations.skyvern_page.Page.__init__", return_value=None):
+ page = ScriptSkyvernPage(scraped_page=scraped_page, page=raw_page, ai=RealSkyvernPageAi(scraped_page, raw_page))
+ statements: list[object] = []
+
+ async def generated_function() -> None:
+ statements.append(await page.validate("Is the order placed?"))
+ statements.append("next statement")
+
+ with pytest.raises(MissingBrowserState) if browser_lost else contextlib.nullcontext():
+ await generated_function()
+
+ assert statements == ([] if browser_lost else [False, "next statement"])
+
+
[email protected]("browser_lost", [True, False], ids=["browser_lost", "browser_alive"])
+async def test_uncached_download_failure_after_browser_loss_raises_the_typed_refusal(
+ cached_run: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, browser_lost: bool
+) -> None:
+ failed = BlockResult(
+ success=False,
+ output_parameter=make_block_output_parameter("download_output"),
+ status=BlockStatus.failed,
+ failure_reason="Failed",
+ )
+ monkeypatch.setattr(FileDownloadBlock, "execute_safe", AsyncMock(return_value=failed))
+ if browser_lost:
+ cached_run.browser_state._on_browser_context_closed(cached_run.browser_context)
+
+ with pytest.raises(MissingBrowserState if browser_lost else ScriptTerminationException):
+ await script_service.download(prompt="Download the invoice", label="invoice")
@pytest.mark.asyncio
@@ -105,7 +563,7 @@ async def test_cached_outcome_survives_output_parameter_failure(monkeypatch: pyt
assert context.get_block_outcome("cached") == BlockOutcome(
status=BlockStatus.failed, error_codes=["picked"], failure_reason="Failed"
)
- assert "error_codes" not in app.DATABASE.observer.update_workflow_run_block.await_args.kwargs
+ assert app.DATABASE.observer.update_workflow_run_block.await_args.kwargs["error_codes"] is None
@pytest.mark.asyncio
diff --git a/tests/unit/test_script_service_fallback_engine.py b/tests/unit/test_script_service_fallback_engine.py
index 1bc0be89a..591931152 100644
--- a/tests/unit/test_script_service_fallback_engine.py
+++ b/tests/unit/test_script_service_fallback_engine.py
@@ -32,6 +32,8 @@ from skyvern.schemas.workflows import BlockType, WorkflowDefinitionYAML
from skyvern.services import script_service
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import Action, ActionStatus
+from skyvern.webeye.real_browser_manager import RealBrowserManager
+from tests.unit._task_block_fakes import lost_browser_state, run_bound_browser_state
from tests.unit._workflow_block_engine_fakes import FakeExperimentationProvider, resolve_arm
MODULE = "skyvern.services.script_service"
@@ -129,6 +131,71 @@ async def test_fallback_inherits_engine_from_run_bound_definition() -> None:
assert _fallback_task_block(app).engine == RunEngine.skyvern_v3
+async def test_fallback_does_not_run_the_agent_after_the_browser_was_lost() -> None:
+ app = _make_app(_make_workflow([_make_task_block("my_block")]))
+ app.BROWSER_MANAGER.get_for_workflow_run = MagicMock(return_value=lost_browser_state("wr_test"))
+ update_block = AsyncMock()
+ with (
+ patch(f"{MODULE}.app", app),
+ patch(f"{MODULE}.skyvern_context.current", return_value=_make_context()),
+ patch(f"{MODULE}._update_workflow_block", update_block),
+ ):
+ await script_service._fallback_to_ai_run(
+ block_type=BlockType.NAVIGATION,
+ cache_key="my_block",
+ prompt="do the thing",
+ error=RuntimeError("Target page, context or browser has been closed"),
+ workflow_run_block_id="wrb_script",
+ )
+
+ app.agent.execute_step.assert_not_awaited()
+ assert update_block.await_args.kwargs["error_codes"] == ["browser_connection_lost"]
+
+
[email protected]("held_by", [None, "wr_child"])
+async def test_adaptive_fallback_capture_does_not_acquire_a_browser(held_by: str | None) -> None:
+ workflow = _make_workflow([_make_task_block("my_block")]).model_copy(update={"run_with": "code", "code_version": 2})
+ app = _make_app(workflow)
+ app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(
+ return_value=MagicMock(
+ workflow_run_id="wr_test",
+ organization_id="o_test",
+ parent_workflow_run_id=None,
+ browser_profile_id=None,
+ ai_fallback=True,
+ run_with=None,
+ )
+ )
+ app.DATABASE.scripts.create_fallback_episode = AsyncMock(return_value=SimpleNamespace(episode_id="ep_test"))
+ app.DATABASE.tasks.create_step = AsyncMock(side_effect=RuntimeError("stop after the capture"))
+ manager = RealBrowserManager()
+ if held_by is not None:
+ # A finished synchronous child still holds the run's shared browser.
+ state, _ = run_bound_browser_state(held_by)
+ manager.pages["wr_test"] = state
+ pages_before = dict(manager.pages)
+ app.BROWSER_MANAGER = manager
+ context = _make_context()
+ context.workflow_permanent_id = "wpid_test"
+ with (
+ patch(f"{MODULE}.app", app),
+ patch(f"{MODULE}.skyvern_context.current", return_value=context),
+ patch.object(manager, "_create_browser_state", new=AsyncMock()) as mock_create,
+ pytest.raises(RuntimeError, match="stop after the capture"),
+ ):
+ await script_service._fallback_to_ai_run(
+ block_type=BlockType.NAVIGATION, cache_key="my_block", prompt="do the thing", error=RuntimeError("failed")
+ )
+
+ app.DATABASE.scripts.create_fallback_episode.assert_awaited_once()
+ mock_create.assert_not_awaited()
+ assert manager.pages == pages_before
+ if held_by is not None:
+ assert manager.pages["wr_test"].runtime_event_context.workflow_run_id == held_by
+
+
@pytest.mark.asyncio
async def test_fallback_keeps_default_engine_when_block_engine_is_default() -> None:
workflow = _make_workflow([_make_task_block("my_block", engine=RunEngine.skyvern_v1)])
diff --git a/tests/unit/test_script_skyvern_page.py b/tests/unit/test_script_skyvern_page.py
index 765e16949..e34cf23ea 100644
--- a/tests/unit/test_script_skyvern_page.py
+++ b/tests/unit/test_script_skyvern_page.py
@@ -21,6 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
+from playwright.async_api import Error as PlaywrightError
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
import skyvern.forge.api_app as api_app_module
@@ -30,12 +31,16 @@ import skyvern.forge.sdk.artifact.storage.local as local_storage_module
import skyvern.forge.sdk.workflow.service as workflow_service_module
import skyvern.library.embedded_server_factory as embedded_server_module
from skyvern.config import settings
+from skyvern.constants import NAVIGATION_MAX_RETRY_TIME
+from skyvern.core.script_generations import skyvern_page as skyvern_page_module
from skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAi
from skyvern.core.script_generations.script_skyvern_page import ScriptSkyvernPage
from skyvern.core.script_generations.skyvern_page import ResolvedSensitiveValue, SkyvernPage
from skyvern.errors.errors import UserDefinedError
from skyvern.exceptions import (
+ FailedToNavigateToUrl,
IllegitCompleteScriptTermination,
+ MissingBrowserState,
NoTOTPSecretFound,
ScriptTerminationException,
SkyvernActionFailed,
@@ -54,10 +59,14 @@ from skyvern.forge.sdk.workflow.service import WorkflowService
from skyvern.library.skyvern import Skyvern
from skyvern.library.skyvern_browser_page import SkyvernBrowserPage
from skyvern.services import script_service
+from skyvern.webeye import navigation as navigation_module
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.handler import ActionHandler
from skyvern.webeye.actions.responses import ActionAbort, ActionFailure, ActionSuccess
from skyvern.webeye.skycdp.facade.page import Frame as CdpFrame
+from skyvern.webeye.utils.page import SkyvernFrame
+from tests.unit._task_block_fakes import run_bound_browser_state
+from tests.unit.scoped_asyncio import ScopedAsyncio
class _KeyboardStub:
@@ -176,6 +185,51 @@ async def test_script_page_reports_deferred_navigation_failure(mock_scraped_page
mock_scraped_page._browser_state.record_navigation_result.assert_awaited_once_with(error)
+ ("seam", "dispatches", "backoffs"),
+ [
+ ("native_close_during_goto", 1, 0),
+ ("ordinary_navigation_error", 5, 4),
+ ("native_close_during_backoff", 1, 1),
+ ("lost_before_first_dispatch", 0, 0),
+ ],
+)
+async def test_script_page_stops_retrying_navigation_after_native_browser_loss(
+ seam: str, dispatches: int, backoffs: int, mock_ai, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from skyvern.core.script_generations import script_skyvern_page as module
+
+ browser_state, context = run_bound_browser_state("wr_script")
+ context._skyvern_cdp_download_interceptor = None
+ scraped_page = SimpleNamespace(_browser_state=browser_state)
+ page, raw_page = _build_navigation_page(ScriptSkyvernPage, scraped_page, mock_ai)
+
+ async def goto(url: str, **_: object) -> None:
+ if seam == "native_close_during_goto":
+ browser_state._on_browser_context_closed(context)
+ raise PlaywrightError("Target page, context or browser has been closed")
+ raise PlaywrightError("Timeout 30000ms exceeded.")
+
+ async def lose_browser_while_waiting(_: float) -> None:
+ if seam == "native_close_during_backoff":
+ browser_state._on_browser_context_closed(context)
+
+ raw_page.goto = AsyncMock(side_effect=goto)
+ backoff = AsyncMock(side_effect=lose_browser_while_waiting)
+ monkeypatch.setattr(module, "asyncio", ScopedAsyncio(sleep=backoff))
+ if seam == "lost_before_first_dispatch":
+ browser_state._on_browser_context_closed(context)
+
+ with pytest.raises(MissingBrowserState if seam == "lost_before_first_dispatch" else PlaywrightError) as error:
+ await page.goto("https://93.184.216.34/path", max_retries=5)
+
+ assert raw_page.goto.await_count == dispatches
+ assert backoff.await_count == backoffs
+ if seam == "native_close_during_backoff":
+ assert "Timeout 30000ms exceeded." in str(error.value)
+
+
@pytest.mark.asyncio
async def test_script_service_goto_fallback_rejects_blocked_destination(
monkeypatch: pytest.MonkeyPatch,
@@ -1643,6 +1697,7 @@ async def test_input_text_rejects_raw_totp_markers_in_direct_mode(mock_scraped_p
@pytest.mark.asyncio
async def test_real_page_ai_rejects_raw_totp_placeholder_before_fallback() -> None:
page_ai = RealSkyvernPageAi.__new__(RealSkyvernPageAi)
+ page_ai.scraped_page = SimpleNamespace(_browser_state=None)
page_ai._maybe_run_v3_midrun = AsyncMock(return_value=(None, False))
with (
@@ -2975,6 +3030,183 @@ async def test_post_dispatch_timeout_after_escape_skips_ai_fallback(mock_ai, eng
mock_ai.ai_click.assert_not_awaited()
+_TARGET_CLOSED = "Target page, context or browser has been closed"
+
+
+def _run_bound_script_page(ai: object, locators: dict[str, object]) -> tuple[ScriptSkyvernPage, object, MagicMock]:
+ """A cached-script page whose scraped page carries a real run-bound browser state."""
+ browser_state, browser_context = run_bound_browser_state("wr_script")
+ raw_page = create_mock_page()
+ raw_page.keyboard = MagicMock(press=AsyncMock())
+ with patch("skyvern.core.script_generations.skyvern_page.Page.__init__", return_value=None):
+ script_page = ScriptSkyvernPage(
+ scraped_page=SimpleNamespace(_browser_state=browser_state), page=raw_page, ai=ai
+ )
+ script_page._working_frame = MagicMock(page=raw_page, locator=MagicMock(side_effect=lambda s, **_: locators[s]))
+ return script_page, browser_state, browser_context
+
+
+ # expected: page-ready waits, selector waits, clicks, Escape presses, AI fallbacks
+ ("seam", "expected"),
+ [
+ ("lost_before_the_action", [0, 0, 0, 0, 0]),
+ ("lost_during_the_page_ready_wait", [1, 0, 0, 0, 0]),
+ ("lost_during_the_selector_backoff", [1, 1, 0, 0, 0]),
+ ("lost_with_the_click_failure", [1, 1, 1, 0, 0]),
+ ("lost_with_a_post_dispatch_click_timeout", [1, 1, 1, 0, 0]),
+ ("lost_during_the_escape_pause", [1, 1, 1, 1, 0]),
+ ("lost_during_the_reclick", [1, 1, 2, 1, 0]),
+ ("ordinary_selector_timeout", [1, 3, 1, 1, 1]),
+ ("ordinary_post_dispatch_click_timeout", [1, 1, 1, 0, 0]),
+ ],
+)
+async def test_cached_click_dispatches_nothing_after_native_browser_loss(
+ monkeypatch: pytest.MonkeyPatch, seam: str, expected: list[int]
+) -> None:
+ ai = _AiStub()
+ ai.ai_click = AsyncMock(return_value="xpath=//button")
+ locator = MagicMock()
+ locator.first = locator
+ page, browser_state, browser_context = _run_bound_script_page(ai, {"#submit": locator})
+ calls: list[str] = []
+ dispatched = PlaywrightTimeoutError("click action done; waiting for scheduled navigations to finish")
+ intercepted = PlaywrightTimeoutError("<div class='overlay'></div> intercepts pointer events")
+ timed_out = PlaywrightTimeoutError("Timeout 5000ms exceeded.")
+ # Per click: (whether the native loss latches during it, what it raises).
+ click_outcomes = {
+ "lost_with_the_click_failure": [(True, PlaywrightError(_TARGET_CLOSED))],
+ "lost_with_a_post_dispatch_click_timeout": [(True, dispatched)],
+ "lost_during_the_escape_pause": [(False, intercepted)],
+ "lost_during_the_reclick": [(False, intercepted), (True, PlaywrightError(_TARGET_CLOSED))],
+ "ordinary_selector_timeout": [(False, timed_out)],
+ "ordinary_post_dispatch_click_timeout": [(False, dispatched)],
+ }
+
+ def lose_browser() -> None:
+ browser_state._on_browser_context_closed(browser_context)
+
+ async def wait_for(*, state: str, **_: object) -> None:
+ if state == "attached":
+ calls.append("attached")
+ if browser_state.get_browser_connection_loss() is not None:
+ raise PlaywrightError(_TARGET_CLOSED)
+ if seam in ("lost_during_the_selector_backoff", "ordinary_selector_timeout"):
+ raise timed_out
+
+ async def click(**_: object) -> None:
+ calls.append("click")
+ if browser_state.get_browser_connection_loss() is not None:
+ raise PlaywrightError(_TARGET_CLOSED)
+ latches, error = click_outcomes[seam][calls.count("click") - 1]
+ if latches:
+ lose_browser()
+ raise error
+
+ async def pause(seconds: float) -> None:
+ if (seam, seconds) in (("lost_during_the_selector_backoff", 1.0), ("lost_during_the_escape_pause", 0.3)):
+ lose_browser()
+
+ async def wait_for_page_ready() -> None:
+ calls.append("page_ready")
+ if seam == "lost_during_the_page_ready_wait":
+ lose_browser()
+
+ locator.wait_for = AsyncMock(side_effect=wait_for)
+ locator.scroll_into_view_if_needed = AsyncMock()
+ locator.click = AsyncMock(side_effect=click)
+ monkeypatch.setattr(page, "_wait_for_page_ready_before_action", wait_for_page_ready)
+ monkeypatch.setattr(skyvern_page_module, "asyncio", ScopedAsyncio(sleep=pause))
+ if seam == "lost_before_the_action":
+ lose_browser()
+
+ with skyvern_context.scoped(SkyvernContext(code_version=2)):
+ if seam.startswith("lost"):
+ with pytest.raises(MissingBrowserState):
+ await page.click("#submit", prompt="Click submit")
+ else:
+ result = await page.click("#submit", prompt="Click submit")
+ assert result == ("xpath=//button" if seam == "ordinary_selector_timeout" else "#submit")
+
+ escapes, ai_clicks = page.page.keyboard.press.await_count, ai.ai_click.await_count
+ assert [*(calls.count(call) for call in ("page_ready", "attached", "click")), escapes, ai_clicks] == expected
+
+
+ ("browser_lost", "mapping"),
+ [(True, {0: "Noor", 1: "Blue"}), (True, {0: "Noor"}), (False, {0: "Noor", 1: "Blue"})],
+ ids=["lost_before_the_next_field", "lost_on_the_last_field", "ordinary_field_failure"],
+)
+async def test_fill_from_mapping_stops_at_the_field_after_native_browser_loss(
+ browser_lost: bool, mapping: dict[int, str]
+) -> None:
+ ai = _AiStub()
+ ai.ai_input_text = AsyncMock(return_value="Noor")
+ name, color = MagicMock(), MagicMock(select_option=AsyncMock())
+ name.first = name
+ page, browser_state, browser_context = _run_bound_script_page(ai, {"#name": name, "#color": color})
+
+ async def fill(_value: str, **_: object) -> None:
+ if browser_lost:
+ browser_state._on_browser_context_closed(browser_context)
+ raise PlaywrightError(_TARGET_CLOSED)
+ raise RuntimeError("element is not attached to the DOM")
+
+ name.fill = AsyncMock(side_effect=fill)
+ page._wait_for_page_ready_before_action = AsyncMock() # type: ignore[method-assign]
+ form_fields = [
+ {"selector": "#name", "type": "text", "tag": "input", "label": "Name"},
+ {"selector": "#color", "type": "select-one", "tag": "select", "label": "Color"},
+ ]
+
+ with pytest.raises(MissingBrowserState) if browser_lost else contextlib.nullcontext():
+ await page.fill_from_mapping(form_fields, mapping)
+
+ assert color.select_option.await_count == ai.ai_input_text.await_count == (0 if browser_lost else 1)
+
+
+ "seam", ["lost_during_the_next_click", "lost_during_the_page_transition", "ordinary_last_page"]
+)
+async def test_fill_multipage_form_reports_no_pages_after_native_browser_loss(
+ monkeypatch: pytest.MonkeyPatch, seam: str
+) -> None:
+ page, browser_state, browser_context = _run_bound_script_page(_AiStub(), {})
+ pages = [[{"selector": f"#{name}", "type": "text", "tag": "input", "label": name}] for name in ("name", "email")]
+ extract_form_fields = AsyncMock(side_effect=[pages[0], pages[0], pages[1], pages[1]])
+
+ async def click_next(**_: object) -> None:
+ if seam == "lost_during_the_next_click":
+ browser_state._on_browser_context_closed(browser_context)
+ if seam != "lost_during_the_page_transition":
+ raise RuntimeError("no next button")
+
+ async def pause(seconds: float) -> None:
+ if seam == "lost_during_the_page_transition" and seconds == 1:
+ browser_state._on_browser_context_closed(browser_context)
+
+ monkeypatch.setattr(page, "extract_form_fields", extract_form_fields)
+ monkeypatch.setattr(page, "dynamic_field_map", AsyncMock(return_value={0: "Noor"}))
+ monkeypatch.setattr(page, "fill_from_mapping", AsyncMock())
+ monkeypatch.setattr(page, "click", AsyncMock(side_effect=click_next))
+ monkeypatch.setattr(page.page, "evaluate", AsyncMock(return_value=[]), raising=False)
+ ready_frame = SimpleNamespace(wait_for_page_ready=AsyncMock())
+ monkeypatch.setattr(SkyvernFrame, "create_instance", AsyncMock(return_value=ready_frame))
+ monkeypatch.setattr(skyvern_page_module, "asyncio", ScopedAsyncio(sleep=pause))
+
+ if seam == "ordinary_last_page":
+ assert await page.fill_multipage_form({"name": "Noor"}, max_pages=2) == 1
+ else:
+ with pytest.raises(MissingBrowserState):
+ await page.fill_multipage_form({"name": "Noor"}, max_pages=2)
+
+ assert extract_form_fields.await_count == 2
+ assert page.dynamic_field_map.await_count == 1
+
+
@pytest.mark.asyncio
async def test_get_or_create_browser_state_forwards_script_id_to_manager(monkeypatch):
# MUST_FIX 1: the production script caller must forward the active run's script_id so the
@@ -3155,6 +3387,7 @@ class TestScriptSkyvernPageMagicLink:
page = SimpleNamespace(goto=fake_goto, fill=fake_fill)
script_page = ScriptSkyvernPage.__new__(ScriptSkyvernPage)
script_page.page = page
+ module._set_scraped_page(script_page, SimpleNamespace(_browser_state=None))
return script_page, state
@staticmethod
@@ -3192,6 +3425,39 @@ class TestScriptSkyvernPageMagicLink:
expected_anchor = datetime.now(timezone.utc).replace(tzinfo=None) - MAGIC_LINK_ANCHOR_GRACE
assert abs((anchor - expected_anchor).total_seconds()) < 30
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "native_close", [True, False], ids=["native_close_during_goto", "ordinary_navigation_error"]
+ )
+ async def test_link_navigation_stops_retrying_after_native_browser_loss(self, monkeypatch, native_close) -> None:
+ from skyvern.core.script_generations import script_skyvern_page as module
+ from skyvern.forge.sdk.schemas.totp_codes import OTPType
+ from skyvern.services.otp_service import OTPValue
+
+ link = "https://portal.example.com/verify?token=x"
+ script_page, state = self._install(monkeypatch, OTPValue(value=link, type=OTPType.MAGIC_LINK))
+ browser_state, context = run_bound_browser_state("wr_1")
+ module._set_scraped_page(script_page, SimpleNamespace(_browser_state=browser_state))
+ monkeypatch.setattr(navigation_module, "asyncio", ScopedAsyncio(sleep=AsyncMock()))
+
+ async def goto(url, **kwargs):
+ state["navigations"].append(url)
+ if native_close:
+ browser_state._on_browser_context_closed(context)
+ raise PlaywrightError("Target page, context or browser has been closed")
+ raise PlaywrightError("Timeout 30000ms exceeded.")
+
+ script_page.page.goto = goto
+
+ ctx = self._with_context()
+ try:
+ with pytest.raises(FailedToNavigateToUrl):
+ await script_page.magic_link(totp_identifier="[email protected]")
+ finally:
+ ctx.reset()
+
+ assert state["navigations"] == [link] * (1 if native_close else NAVIGATION_MAX_RETRY_TIME)
+
@pytest.mark.asyncio
async def test_a_returned_one_time_code_is_reported_not_navigated_to(self, monkeypatch) -> None:
"""A configured webhook answers without honouring expected_otp_type."""
diff --git a/tests/unit/test_send_email_block.py b/tests/unit/test_send_email_block.py
index f22c34497..74923432b 100644
--- a/tests/unit/test_send_email_block.py
+++ b/tests/unit/test_send_email_block.py
@@ -19,6 +19,7 @@ from __future__ import annotations
import ast
import asyncio
+import contextlib
import logging
import os
import pickle
@@ -49,7 +50,7 @@ from sqlalchemy.orm import undefer
from skyvern.config import settings
from skyvern.core.script_generations.generate_script import _build_send_email_statement
-from skyvern.exceptions import BlockedHost, UnresolvableHost
+from skyvern.exceptions import BlockedHost, MissingBrowserState, UnresolvableHost
from skyvern.forge import app
from skyvern.forge.sdk.api import email as email_api
from skyvern.forge.sdk.api.email import InvalidEmailRecipient, send, validate_recipients
@@ -105,6 +106,7 @@ from skyvern.schemas.workflows import (
)
from skyvern.services import script_service
from skyvern.services.email import gmail as gmail_service
+from tests.unit._task_block_fakes import run_bound_browser_state
from tests.unit.scoped_asyncio import ScopedAsyncio
@@ -1868,6 +1870,48 @@ async def test_cached_nested_loop_records_the_inner_loop_in_each_outer_iteration
] == [[("inner_output", "one", len(inner_items))], [("inner_output", "two", len(inner_items))]]
+ "seam", ["lost_during_the_send", "lost_before_the_send", "lost_before_a_wait", "ordinary_send_failure"]
+)
+async def test_cached_script_runs_no_statement_after_its_browser_was_lost(
+ gmail_env: GmailEnv, monkeypatch: pytest.MonkeyPatch, seam: str
+) -> None:
+ await _cached_run(gmail_env, "wr_lost", _gmail_block(continue_on_failure=True))
+ browser_state, browser_context = run_bound_browser_state("wr_lost")
+ monkeypatch.setattr(app.BROWSER_MANAGER, "get_for_workflow_run", lambda *_args, **_kwargs: browser_state)
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ if seam == "lost_during_the_send":
+ browser_state._on_browser_context_closed(browser_context)
+ return _accepted(request) if seam == "lost_before_the_send" else _rejected(request)
+
+ gmail_env.respond = respond
+ if seam.startswith("lost_before"):
+ browser_state._on_browser_context_closed(browser_context)
+ statements: list[str] = []
+
+ async def generated_function() -> None:
+ if seam == "lost_before_a_wait":
+ await script_service.wait(0, label="pause")
+ else:
+ await script_service.send_email(transport="gmail", credential_id=GMAIL_CREDENTIAL, label="send_email")
+ statements.append("next statement")
+
+ try:
+ with contextlib.nullcontext() if seam == "ordinary_send_failure" else pytest.raises(MissingBrowserState):
+ await generated_function()
+ finally:
+ skyvern_context.reset()
+
+ started = seam.startswith("lost_before")
+ assert statements == (["next statement"] if seam == "ordinary_send_failure" else [])
+ assert len(gmail_env.requests) == (0 if started else 1)
+ assert len([row for row in await gmail_env.rows(WorkflowRunBlockModel) if row.workflow_run_id == "wr_lost"]) == (
+ 0 if started else 1
+ )
+
+
def test_gmail_send_block_carries_the_same_fields_through_yaml_conversion_reload_and_codegen() -> None:
authored = {
"transport": "gmail",
diff --git a/tests/unit/test_task_v2_completion_gate.py b/tests/unit/test_task_v2_completion_gate.py
index 77b56fc4e..36710e62c 100644
--- a/tests/unit/test_task_v2_completion_gate.py
+++ b/tests/unit/test_task_v2_completion_gate.py
@@ -13,12 +13,16 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from skyvern.config import Settings
+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE
from skyvern.forge.sdk.prompting import PromptEngine
from skyvern.forge.sdk.schemas.task_v2 import TaskV2Status
from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition, WorkflowRunStatus
from skyvern.forge.sdk.workflow.service import WorkflowService
+from skyvern.schemas.workflows import BlockStatus
from skyvern.services import planner_levers, task_v2_service
from skyvern.services.task_v2_service import _should_run_post_block_completion_check
+from skyvern.webeye.real_browser_manager import RealBrowserManager
+from tests.unit._task_block_fakes import run_bound_browser_state
class _TaskV2WiringObserved(Exception):
@@ -434,6 +438,251 @@ async def test_missing_task_type_failure_finalizes_with_organization_scope(
workflow_service.mark_workflow_run_as_failed.assert_awaited_once()
+ ("achieved_by", "lost_during"),
+ [
+ ("completion_check", "completion_scrape"),
+ ("completion_check", "completion_check"),
+ ("planning", "planning"),
+ ("completion_check", "completion_screenshots"),
+ ("completion_check", "summary"),
+ ("planning", "summary"),
+ ("completion_check", None),
+ ("completion_check", "expected_teardown"),
+ ("completion_check", "run_canceled"),
+ ("planning", "task_canceled"),
+ ("completion_check", "run_write_refused"),
+ ("block", "block"),
+ ("block", "run_canceled"),
+ ("block", "task_canceled"),
+ ],
+)
+async def test_task_v2_cannot_complete_after_its_browser_was_lost(
+ monkeypatch: pytest.MonkeyPatch, achieved_by: str, lost_during: str | None
+) -> None:
+ now = datetime.now(UTC)
+ organization = SimpleNamespace(organization_id="org_test", max_steps_per_run=None)
+ task_v2 = MagicMock(
+ observer_cruise_id="task_v2_test",
+ status=TaskV2Status.queued,
+ prompt="finish the task",
+ workflow_system_prompt=None,
+ run_with="code",
+ started_at=now,
+ created_at=now,
+ )
+ task_v2.url, task_v2.workflow_run_id = "https://example.test", "wr_test"
+ workflow = MagicMock(
+ workflow_id="wf_test",
+ workflow_permanent_id="wp_test",
+ title="Test",
+ workflow_definition=WorkflowDefinition(parameters=[], blocks=[]),
+ )
+ runs = {
+ status: MagicMock(
+ workflow_run_id="wr_test",
+ workflow_id="wf_test",
+ status=status,
+ parent_workflow_run_id=None,
+ browser_profile_id=None,
+ )
+ for status in (
+ WorkflowRunStatus.queued,
+ WorkflowRunStatus.running,
+ WorkflowRunStatus.failed,
+ WorkflowRunStatus.canceled,
+ )
+ }
+ run_status = [WorkflowRunStatus.queued]
+
+ async def get_workflow_run(*_args: object, **_kwargs: object) -> MagicMock:
+ run = runs[run_status[0]]
+ if run_status[0] == WorkflowRunStatus.queued:
+ run_status[0] = WorkflowRunStatus.running
+ return run
+
+ async def fail_run_if_not_final(**_kwargs: object) -> None:
+ # A fenced write leaves the run with the owner that refused it.
+ if lost_during != "run_write_refused" and not run_status[0].is_final():
+ run_status[0] = WorkflowRunStatus.failed
+
+ async def cancel_run(*_args: object, **_kwargs: object) -> None:
+ run_status[0] = WorkflowRunStatus.canceled
+
+ mark_failed = AsyncMock(side_effect=fail_run_if_not_final)
+ lost_in_block = achieved_by == "block"
+ # A block whose browser is lost while it runs returns its failure with the code from the block's own seam.
+ block_result = MagicMock(
+ failure_reason="The browser connection was lost." if lost_in_block else None,
+ output_parameter_value={},
+ success=not lost_in_block,
+ status=BlockStatus.failed if lost_in_block else BlockStatus.completed,
+ error_codes=[BROWSER_CONNECTION_LOST_ERROR_CODE] if lost_in_block else [],
+ workflow_run_block_id="wrb_navigate",
+ )
+
+ async def execute_safe(**_kwargs: object) -> MagicMock:
+ if lost_in_block:
+ await lose_browser_after_any_cancellation()
+ return block_result
+
+ block = SimpleNamespace(
+ execute_safe=AsyncMock(side_effect=execute_safe),
+ build_block_result=AsyncMock(),
+ block_type="navigation",
+ label="navigate",
+ continue_on_failure=False,
+ )
+ state, browser_context = run_bound_browser_state("wr_test")
+ state.get_working_page = AsyncMock(return_value=MagicMock()) # type: ignore[method-assign]
+ state.validate_browser_context = AsyncMock(return_value=True) # type: ignore[method-assign]
+ scrapes: list[str] = []
+
+ async def scrape_website(**_kwargs: object) -> SimpleNamespace:
+ scrapes.append("scrape")
+ # The planner reads the page first, the completion check second.
+ if lost_during == "completion_scrape" and len(scrapes) == 2:
+ state._on_browser_context_closed(browser_context)
+ raise RuntimeError("Target page, context or browser has been closed")
+ return SimpleNamespace(screenshots=[])
+
+ state.scrape_website = scrape_website # type: ignore[method-assign]
+ manager = RealBrowserManager()
+ manager.pages["wr_test"] = state
+
+ async def update_task_v2(*_args: object, **kwargs: object) -> MagicMock:
+ if status := kwargs.get("status"):
+ task_v2.status = status
+ return task_v2
+
+ async def update_task_v2_if_not_final(*args: object, **kwargs: object) -> bool:
+ if task_v2.status.is_final():
+ return False
+ await update_task_v2(*args, **kwargs)
+ return True
+
+ def close_browser(point: str) -> None:
+ if lost_during == point:
+ state._on_browser_context_closed(browser_context)
+ elif lost_during == "expected_teardown" and point == "summary":
+ state.mark_run_released()
+ state._on_browser_context_closed(browser_context)
+
+ async def screenshots(**_kwargs: object) -> None:
+ close_browser("completion_screenshots")
+
+ async def lose_browser_after_any_cancellation() -> None:
+ # The browser is lost while the deciding unit runs, after any cancellation that lands during it.
+ if lost_during == "task_canceled":
+ await task_v2_service.mark_task_v2_as_canceled(
+ "task_v2_test", workflow_run_id="wr_test", organization_id="org_test"
+ )
+ elif lost_during == "run_canceled":
+ await cancel_run()
+ state._on_browser_context_closed(browser_context)
+
+ async def llm(*, prompt_name: str, **_kwargs: object) -> dict[str, Any]:
+ point = {"task_v2_check_completion": "completion_check", "task_v2_summary": "summary"}.get(
+ prompt_name, "planning"
+ )
+ if lost_during in ("run_canceled", "task_canceled", "run_write_refused") and point == achieved_by:
+ await lose_browser_after_any_cancellation()
+ close_browser(point)
+ if point == "summary":
+ return {"description": "done", "output": {"done": True}}
+ if point == "completion_check" or achieved_by == "planning":
+ return {"user_goal_achieved": True, "should_terminate": False, "thoughts": "done"}
+ return {"user_goal_achieved": False, "should_terminate": False, "plan": "continue", "task_type": "navigate"}
+
+ llm_handler = AsyncMock(side_effect=llm)
+ test_app = MagicMock()
+ test_app.scrape_exclude = []
+ test_app.BROWSER_MANAGER = manager
+ test_app.WORKFLOW_SERVICE.get_workflow_run = AsyncMock(side_effect=get_workflow_run)
+ test_app.WORKFLOW_SERVICE.get_workflow = AsyncMock(return_value=workflow)
+ test_app.WORKFLOW_SERVICE.mark_workflow_run_as_running = AsyncMock()
+ test_app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final = mark_failed
+ test_app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled = AsyncMock(side_effect=cancel_run)
+ test_app.WORKFLOW_SERVICE.mark_workflow_run_as_completed = mark_completed = AsyncMock()
+ test_app.WORKFLOW_SERVICE.generate_script_if_needed = finalize_script = AsyncMock()
+ test_app.WORKFLOW_SERVICE.create_workflow_from_request = AsyncMock(return_value=workflow)
+ test_app.AGENT_FUNCTION.has_code_block_access = AsyncMock(return_value=False)
+ test_app.AGENT_FUNCTION.validate_task_execution = AsyncMock()
+ test_app.AGENT_FUNCTION.cleanup_element_tree_factory = MagicMock(return_value=MagicMock())
+ test_app.DATABASE.observer.get_task_v2 = AsyncMock(return_value=task_v2)
+ test_app.DATABASE.observer.update_task_v2 = AsyncMock(side_effect=update_task_v2)
+ test_app.DATABASE.observer.update_task_v2_status_if_not_final = AsyncMock(side_effect=update_task_v2_if_not_final)
+ test_app.DATABASE.observer.create_thought = AsyncMock(return_value=SimpleNamespace(observer_thought_id="thought"))
+ test_app.DATABASE.observer.update_thought = AsyncMock()
+ context = SimpleNamespace(run_id=None, root_workflow_run_id=None, tz_info=timezone.utc)
+ monkeypatch.setattr(task_v2_service, "app", test_app)
+ monkeypatch.setattr(
+ planner_levers.app.EXPERIMENTATION_PROVIDER, "is_feature_enabled_cached", AsyncMock(return_value=False)
+ )
+ monkeypatch.setattr(planner_levers.settings, "TASK_V2_CONVERGE_PCT", 0)
+ monkeypatch.setattr(planner_levers.settings, "TASK_V2_CARRY_SUBGOALS", False)
+ monkeypatch.setattr(planner_levers.settings, "TASK_V2_SKIP_COMPLETION_CHECK_AFTER_NAVIGATE", False)
+ monkeypatch.setattr(task_v2_service.skyvern_context, "ensure_context", lambda: context)
+ monkeypatch.setattr(task_v2_service, "initialize_task_v2_metadata", AsyncMock(return_value=task_v2))
+ monkeypatch.setattr(task_v2_service, "_set_up_workflow_context", AsyncMock())
+ monkeypatch.setattr(task_v2_service, "_resolve_max_iterations", lambda _override: 1)
+ monkeypatch.setattr(task_v2_service, "_is_planner_mini_goal_improvements_enabled", AsyncMock(return_value=False))
+ monkeypatch.setattr(task_v2_service, "build_open_tabs_context", AsyncMock(return_value=None))
+ monkeypatch.setattr(task_v2_service, "load_prompt_with_elements", MagicMock(return_value="prompt"))
+ monkeypatch.setattr(task_v2_service, "_get_task_v2_llm_api_handler", lambda _task_v2: llm_handler)
+ monkeypatch.setattr(task_v2_service.SkyvernFrame, "get_url", AsyncMock(return_value=task_v2.url))
+ monkeypatch.setattr(task_v2_service, "_generate_navigation_task", AsyncMock(return_value=(block, [], [])))
+ monkeypatch.setattr(task_v2_service, "WorkflowDefinitionYAML", MagicMock())
+ monkeypatch.setattr(task_v2_service, "WorkflowCreateYAMLRequest", MagicMock())
+ monkeypatch.setattr(task_v2_service, "runtime_proxy_location", lambda _location: None)
+ monkeypatch.setattr(task_v2_service, "_get_extracted_data_from_block_result", lambda *_args, **_kwargs: None)
+ monkeypatch.setattr(task_v2_service, "_persist_completion_tab_screenshots", screenshots)
+ callbacks: list[TaskV2Status] = []
+
+ async def send_webhook(task: MagicMock, **_kwargs: object) -> None:
+ callbacks.append(task.status)
+
+ monkeypatch.setattr(task_v2_service, "send_task_v2_webhook", send_webhook)
+
+ await task_v2_service.run_task_v2_helper(organization, task_v2, max_iterations_override=1)
+
+ prompts = [call.kwargs["prompt_name"] for call in llm_handler.await_args_list]
+ checked = achieved_by == "completion_check" and lost_during != "completion_scrape"
+ assert prompts.count("task_v2_check_completion") == checked
+ assert prompts.count("task_v2_summary") == (
+ lost_during in (None, "expected_teardown", "summary", "completion_screenshots")
+ )
+ # A cached script is finalized only from a TaskV2 that completed.
+ assert finalize_script.await_count == (lost_during in (None, "expected_teardown"))
+ if lost_during in (None, "expected_teardown"):
+ assert task_v2.status == TaskV2Status.completed
+ mark_completed.assert_awaited_once_with("wr_test")
+ assert callbacks == [TaskV2Status.completed]
+ mark_failed.assert_not_awaited()
+ return
+ if lost_during == "run_write_refused":
+ # The task did not end, so there is no terminal state to report.
+ assert task_v2.status == TaskV2Status.running
+ assert callbacks == []
+ else:
+ # The terminal callback reports the state the task actually ended in, exactly once; a cancellation wins.
+ ended = TaskV2Status.canceled if lost_during.endswith("canceled") else TaskV2Status.failed
+ assert task_v2.status == ended
+ assert callbacks == [ended]
+ mark_failed.assert_awaited_once()
+ mark_completed.assert_not_awaited()
+ if achieved_by == "planning":
+ block.execute_safe.assert_not_awaited()
+ elif lost_in_block:
+ # The block's own row already carries its failure and code, and the run keeps the block's reason.
+ block.build_block_result.assert_not_awaited()
+ assert mark_failed.await_args.kwargs["failure_reason"] == block_result.failure_reason
+ else:
+ assert block.build_block_result.await_args.kwargs["status"] == BlockStatus.failed
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in block.build_block_result.await_args.kwargs["error_codes"]
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"provider_enabled,settings_default,expected",
diff --git a/tests/unit/test_task_v2_failure_deliverable.py b/tests/unit/test_task_v2_failure_deliverable.py
index adae26e44..3acb17fc4 100644
--- a/tests/unit/test_task_v2_failure_deliverable.py
+++ b/tests/unit/test_task_v2_failure_deliverable.py
@@ -3,12 +3,14 @@ mark_task_v2_as_* terminal transitions refuse to overwrite a status that is alre
from __future__ import annotations
-from unittest.mock import AsyncMock
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
import pytest
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.schemas.task_v2 import TaskV2Status
+from skyvern.schemas.workflows import BlockResult, BlockStatus, BlockType
from skyvern.services import task_v2_service
@@ -217,3 +219,35 @@ async def test_mark_task_v2_as_failed_redrive_keeps_termination_metadata(monkeyp
terminate_run.assert_awaited_once_with(
"wr_test", "late failure", failure_category=[{"category": "unreachable_goal"}]
)
+
+
+ ("error_codes", "fails_run"), [(["browser_connection_lost"], True), ([], False)], ids=["browser_lost", "other"]
+)
+async def test_failed_block_stops_the_planner_only_when_the_browser_was_lost(
+ monkeypatch: pytest.MonkeyPatch, error_codes: list[str], fails_run: bool
+) -> None:
+ mark_failed = AsyncMock()
+ monkeypatch.setattr(task_v2_service.app.WORKFLOW_SERVICE, "mark_workflow_run_as_failed_if_not_final", mark_failed)
+ monkeypatch.setattr(task_v2_service.app.WORKFLOW_SERVICE, "get_workflow_run", AsyncMock())
+ block = SimpleNamespace(block_type=BlockType.NAVIGATION, label="nav", continue_on_failure=True)
+ result = BlockResult(
+ success=False,
+ output_parameter=MagicMock(),
+ status=BlockStatus.failed,
+ failure_reason="Browser state page is missing.",
+ error_codes=error_codes,
+ can_continue_after_failure=not error_codes,
+ )
+
+ await task_v2_service.handle_block_result(
+ "tsk_v2_test",
+ block, # type: ignore[arg-type]
+ result,
+ MagicMock(),
+ SimpleNamespace(workflow_run_id="wr_test", organization_id="o_test"), # type: ignore[arg-type]
+ is_last_block=False,
+ )
+
+ assert mark_failed.await_count == (1 if fails_run else 0)
diff --git a/tests/unit/test_taskv2_block_browser_address_propagation.py b/tests/unit/test_taskv2_block_browser_address_propagation.py
index dc618d07c..7f7781d72 100644
--- a/tests/unit/test_taskv2_block_browser_address_propagation.py
+++ b/tests/unit/test_taskv2_block_browser_address_propagation.py
@@ -8,12 +8,19 @@ from unittest.mock import AsyncMock
import pytest
+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.schemas.task_v2 import TaskV2Status
from skyvern.forge.sdk.workflow.models import block as block_module
from skyvern.forge.sdk.workflow.models.block import TaskV2Block
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
+from skyvern.forge.sdk.workflow.retry_policy import evaluate_retry_policy
+from skyvern.schemas.workflows import WorkflowRetryPolicy
+from skyvern.webeye.real_browser_manager import RealBrowserManager
+from skyvern.webeye.real_browser_state import RealBrowserState
+from tests.unit._task_block_fakes import run_bound_browser_state
@pytest.fixture(autouse=True)
@@ -218,3 +225,108 @@ async def test_taskv2_block_settles_the_child_capture_when_the_runner_raises(
)
assert settled == ["wr_child"]
+
+
+def _child_browser_state(*, lost: bool) -> RealBrowserState:
+ state, context = run_bound_browser_state("wr_child")
+ if lost:
+ state._on_browser_context_closed(context)
+ return state
+
+
[email protected]("mapping", ["persistent_session_child_only", "shared_with_parent"])
[email protected]("continue_on_failure", [True, False])
[email protected]("child_lost_browser", [True, False], ids=["child_lost_browser", "ordinary_child_failure"])
+async def test_taskv2_block_reports_a_child_browser_loss_from_the_child_run_entry(
+ monkeypatch: pytest.MonkeyPatch, mapping: str, continue_on_failure: bool, child_lost_browser: bool
+) -> None:
+ skyvern_context.set(SkyvernContext(organization_id="org_1", workflow_run_id="wr_parent", run_id="wr_parent"))
+ child_state = _child_browser_state(lost=child_lost_browser)
+ browser_manager = RealBrowserManager()
+ # An explicit persistent session registers only the child run; a shared browser is also aliased to the parent.
+ browser_manager.pages = (
+ {"wr_child": child_state}
+ if mapping == "persistent_session_child_only"
+ else {"wr_child": child_state, "wr_parent": child_state}
+ )
+ workflow_run = SimpleNamespace(
+ proxy_location=None,
+ max_screenshot_scrolls=None,
+ browser_address=None,
+ extra_http_headers=None,
+ cdp_connect_headers=None,
+ failure_reason="Browser state page is missing.",
+ )
+ update_block = AsyncMock()
+ fake_app = SimpleNamespace(
+ DATABASE=SimpleNamespace(
+ organizations=SimpleNamespace(get_organization=AsyncMock(return_value=SimpleNamespace())),
+ workflow_runs=SimpleNamespace(
+ get_workflow_run=AsyncMock(return_value=workflow_run), update_workflow_run=AsyncMock()
+ ),
+ observer=SimpleNamespace(update_task_v2=AsyncMock(), update_workflow_run_block=update_block),
+ ),
+ BROWSER_MANAGER=browser_manager,
+ WORKFLOW_SERVICE=SimpleNamespace(
+ get_recent_task_screenshot_artifacts=AsyncMock(return_value=[]),
+ get_recent_workflow_screenshot_artifacts=AsyncMock(return_value=[]),
+ ),
+ STORAGE=SimpleNamespace(
+ get_current_attempt_downloaded_files=AsyncMock(return_value=[]),
+ get_downloaded_file_signature_aliases=lambda _: [],
+ ),
+ WORKFLOW_CONTEXT_MANAGER=SimpleNamespace(has_workflow_run_context=lambda _run_id: False),
+ )
+ monkeypatch.setattr(block_module, "app", fake_app)
+
+ from skyvern.services import task_v2_service
+
+ child = SimpleNamespace(
+ observer_cruise_id="tsk_v2_1",
+ workflow_run_id="wr_child",
+ output=None,
+ status=TaskV2Status.failed,
+ summary=None,
+ failure_category=None,
+ )
+ monkeypatch.setattr(task_v2_service, "initialize_task_v2", AsyncMock(return_value=child))
+ monkeypatch.setattr(task_v2_service, "run_task_v2", AsyncMock(return_value=child))
+ monkeypatch.setattr(
+ TaskV2Block,
+ "get_workflow_run_context",
+ lambda self, workflow_run_id: SimpleNamespace(credential_totp_identifiers={}),
+ )
+ monkeypatch.setattr(TaskV2Block, "format_potential_template_parameters", lambda self, _: None)
+ monkeypatch.setattr(TaskV2Block, "record_output_parameter_value", AsyncMock())
+
+ block = TaskV2Block(
+ label="task1",
+ output_parameter=_output_parameter("task1_output"),
+ prompt="do the thing",
+ url="https://example.com",
+ continue_on_failure=continue_on_failure,
+ )
+
+ result = await block.execute(workflow_run_id="wr_parent", workflow_run_block_id="wrb_1", organization_id="org_1")
+
+ persisted_codes = {
+ code
+ for call in update_block.await_args_list
+ if call.kwargs.get("workflow_run_block_id") == "wrb_1"
+ for code in call.kwargs.get("error_codes") or []
+ }
+ retry_policy = WorkflowRetryPolicy.model_validate({"max_retries": 2, "retry_on": [{"status": "failed"}]})
+ retry = evaluate_retry_policy(retry_policy, WorkflowRunStatus.failed, persisted_codes, 0)
+ if child_lost_browser:
+ assert result.success is False
+ assert result.can_continue_after_failure is False
+ assert result.error_codes == [BROWSER_CONNECTION_LOST_ERROR_CODE]
+ assert persisted_codes == {BROWSER_CONNECTION_LOST_ERROR_CODE}
+ assert retry == (False, BROWSER_CONNECTION_LOST_ERROR_CODE)
+ else:
+ assert result.success is continue_on_failure
+ assert result.can_continue_after_failure is True
+ assert result.error_codes == []
+ assert persisted_codes == set()
+ assert retry == (True, "matched")
diff --git a/tests/unit/test_workflow_webhook_delivery.py b/tests/unit/test_workflow_webhook_delivery.py
index 1062690d9..3a348838b 100644
--- a/tests/unit/test_workflow_webhook_delivery.py
+++ b/tests/unit/test_workflow_webhook_delivery.py
@@ -1588,6 +1588,8 @@ async def test_final_release_retries_keep_own_claim_and_completed_hook(
("failed", "failed", ["A", "B"], {"C"}, 0, (False, "no_match")),
("failed", "failed", ["A"], set(), 0, (False, "no_match")),
("completed", "failed", None, set(), 0, (False, "no_match")),
+ ("failed", "failed", None, {"browser_connection_lost"}, 0, (False, "browser_connection_lost")),
+ ("failed", "failed", [], {"ANY", "browser_connection_lost"}, 0, (False, "browser_connection_lost")),
],
)
def test_evaluate_retry_policy_decisions(status, rule_status, rule_codes, errors, retries_used, expected) -> None:
diff --git a/tests/unit/workflow/test_block_browser_state_recovery.py b/tests/unit/workflow/test_block_browser_state_recovery.py
index 98aae1f77..34023226e 100644
--- a/tests/unit/workflow/test_block_browser_state_recovery.py
+++ b/tests/unit/workflow/test_block_browser_state_recovery.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
+import contextlib
import json
import socket
import subprocess
@@ -11,29 +12,49 @@ from collections.abc import Awaitable
from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import SimpleNamespace
-from unittest.mock import AsyncMock, MagicMock, PropertyMock, call
+from unittest.mock import AsyncMock, MagicMock, PropertyMock, call, patch
import pytest
+from playwright.async_api import Error as PlaywrightError
from playwright.async_api import async_playwright
from structlog.testing import capture_logs
+from skyvern.constants import NAVIGATION_MAX_RETRY_TIME
from skyvern.exceptions import (
+ BROWSER_CONNECTION_LOST_ERROR_CODE,
BrowserStateDiagnostic,
+ FailedToNavigateToUrl,
+ FailedToReloadPage,
MissingBrowserState,
MissingBrowserStatePage,
get_user_facing_exception_message,
)
from skyvern.forge import app
+from skyvern.forge.agent import ForgeAgent
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
-from skyvern.forge.sdk.workflow.models.block import CodeBlock
+from skyvern.forge.sdk.models import StepStatus
+from skyvern.forge.sdk.schemas.tasks import TaskStatus
+from skyvern.forge.sdk.workflow.models import block as block_module
+from skyvern.forge.sdk.workflow.models.block import Block, CodeBlock, TaskBlock, UrlBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
+from skyvern.forge.sdk.workflow.retry_policy import evaluate_retry_policy
+from skyvern.forge.sdk.workflow.service import WorkflowService
+from skyvern.schemas.workflows import BlockResult, BlockStatus, WorkflowRetryPolicy
+from skyvern.webeye import navigation as navigation_module
from skyvern.webeye import real_browser_state as real_browser_state_module
from skyvern.webeye.browser_artifacts import BrowserArtifacts
from skyvern.webeye.browser_runtime_events import BrowserRuntimeLogContext
from skyvern.webeye.driver_connection import close_driver_connection_on_transport_loss
+from skyvern.webeye.real_browser_manager import RealBrowserManager
from skyvern.webeye.real_browser_state import RealBrowserState, expect_process_driver_teardown
+from tests.unit._task_block_fakes import lost_browser_state, mock_block_execute_deps, output_parameter
+from tests.unit.force_stub_app import admit_block_dispatch
from tests.unit.forge_log_capture import capture_runtime_logs
+from tests.unit.helpers import make_organization, make_step, make_task
+from tests.unit.scoped_asyncio import ScopedAsyncio
+from tests.unit.test_block_description_caching import _setup_mocks
def _has_playwright_browser() -> bool:
@@ -342,6 +363,730 @@ def test_browser_disconnect_event_latches_event_observation() -> None:
assert diagnostic.observation_source == "browser_event"
+def _bound_state(run_id: str = "wr_test") -> tuple[RealBrowserState, MagicMock, MagicMock]:
+ browser = MagicMock()
+ browser.is_connected = MagicMock(return_value=True)
+ context = MagicMock(browser=browser)
+ context._impl_obj = MagicMock(_close_was_called=False, _closed=False, _connection=MagicMock(_closed_error=None))
+ state = RealBrowserState(pw=MagicMock(), browser_context=context)
+ state.bind_runtime_event_context(BrowserRuntimeLogContext(workflow_run_id=run_id))
+ return state, context, browser
+
+
+def _probe_timeout(state: RealBrowserState, context: MagicMock, browser: MagicMock) -> None:
+ state.record_connection_probe_failure(context, state.pw, timed_out=True)
+
+
+ ("events", "terminal"),
+ [
+ pytest.param([lambda s, c, b: s._on_browser_context_closed(c)], True, id="native_context_close"),
+ pytest.param([lambda s, c, b: s._on_browser_disconnected(b)], True, id="native_browser_disconnect"),
+ pytest.param(
+ [_probe_timeout, lambda s, c, b: s._on_browser_context_closed(c)], True, id="probe_then_native_close"
+ ),
+ pytest.param(
+ [lambda s, c, b: s._on_driver_transport_lost(s.pw, c), lambda s, c, b: s._on_browser_disconnected(b)],
+ True,
+ id="driver_loss_then_native_disconnect",
+ ),
+ pytest.param([lambda s, c, b: s._on_driver_transport_lost(s.pw, c)], False, id="driver_loss_only"),
+ pytest.param([_probe_timeout], False, id="probe_timeout_only"),
+ pytest.param([lambda s, c, b: s.record_connection_probe_failure(c, s.pw)], False, id="probe_failure_only"),
+ pytest.param(
+ [
+ lambda s, c, b: s._expect_runtime_end("normal_close", c),
+ lambda s, c, b: s._on_browser_context_closed(c),
+ ],
+ False,
+ id="expected_close",
+ ),
+ pytest.param(
+ [
+ lambda s, c, b: s._expect_runtime_end("driver_replacement", c),
+ lambda s, c, b: s._on_browser_context_closed(c),
+ ],
+ False,
+ id="expected_driver_replacement",
+ ),
+ pytest.param(
+ [lambda s, c, b: s.mark_run_released(), lambda s, c, b: s._on_browser_context_closed(c)],
+ False,
+ id="after_release",
+ ),
+ ],
+)
+def test_browser_connection_loss_is_only_an_unexpected_active_native_event(
+ events: list, terminal: bool, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_browser_run_released", MagicMock())
+ state, context, browser = _bound_state()
+
+ for event in events:
+ event(state, context, browser)
+
+ loss = state.get_browser_connection_loss()
+ if terminal:
+ assert loss is not None
+ assert loss.observation_source == "browser_event"
+ else:
+ assert loss is None
+
+
+def test_browser_connection_loss_ignores_a_stale_generation() -> None:
+ state, old_context, old_browser = _bound_state()
+ new_context = MagicMock(browser=MagicMock())
+ state.browser_context = new_context
+ state._register_disconnect_listeners(new_context)
+
+ state._on_browser_context_closed(old_context)
+ state._on_browser_disconnected(old_browser)
+
+ assert state.get_browser_connection_loss() is None
+
+
+def test_browser_connection_loss_belongs_to_the_run_that_observed_it() -> None:
+ state, context, _ = _bound_state("wr_first")
+ state._on_browser_context_closed(context)
+ assert state.get_browser_connection_loss() is not None
+
+ state.bind_runtime_event_context(BrowserRuntimeLogContext(workflow_run_id="wr_next"))
+
+ assert state.get_browser_connection_loss() is None
+
+
+async def test_lost_browser_is_not_reconnected_for_the_same_run(monkeypatch: pytest.MonkeyPatch) -> None:
+ state, _, browser = _bound_state()
+ browser.is_connected.return_value = False
+ state._on_browser_disconnected(browser)
+ state.reconnect = AsyncMock() # type: ignore[method-assign]
+ monkeypatch.setattr(app.BROWSER_MANAGER, "get_for_workflow_run", MagicMock(return_value=state))
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "get_workflow_run", AsyncMock(return_value=_FakeWorkflowRun()))
+
+ with pytest.raises(MissingBrowserState) as exc_info:
+ await _make_code_block().get_or_create_browser_state(workflow_run_id="wr_test", organization_id="o_test")
+
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in str(exc_info.value)
+ state.reconnect.assert_not_awaited()
+
+
[email protected]("browser_lost", [True, False], ids=["browser_lost", "browser_alive"])
+async def test_block_that_failed_on_a_lost_browser_cannot_be_continued(browser_lost: bool) -> None:
+ block = _make_code_block()
+ block.continue_on_failure = True
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,
+ patch.object(CodeBlock, "execute", AsyncMock(side_effect=MissingBrowserStatePage(workflow_run_id="wr_test"))),
+ patch.object(Block, "_generate_workflow_run_block_description", new_callable=AsyncMock),
+ ):
+ _setup_mocks(mock_app)
+ mock_app.BROWSER_MANAGER.get_for_workflow_run.return_value = (
+ lost_browser_state("wr_test") if browser_lost else None
+ )
+
+ result = await block.execute_safe(workflow_run_id="wr_test")
+
+ assert result.success is False
+ assert result.can_continue_after_failure is not browser_lost
+ assert (BROWSER_CONNECTION_LOST_ERROR_CODE in result.error_codes) is browser_lost
+
+
+async def test_task_block_is_not_retried_after_its_browser_was_lost() -> None:
+ block = TaskBlock(
+ label="nav",
+ url="https://example.com",
+ navigation_goal="go",
+ output_parameter=output_parameter(),
+ max_retries=2,
+ )
+ state, context, _ = _bound_state("wr_task_block_test")
+
+ async def _lose_browser(**_: object) -> None:
+ state._on_browser_context_closed(context)
+
+ now = datetime.now(timezone.utc)
+ failed_task = make_task(now, make_organization(now), status=TaskStatus.failed, failure_reason="page is missing")
+ with (
+ skyvern_context.scoped(SkyvernContext()),
+ mock_block_execute_deps(working_page_url="https://example.com") as deps,
+ ):
+ deps.browser_manager.get_for_workflow_run = MagicMock(return_value=state)
+ deps.agent.execute_step = AsyncMock(side_effect=_lose_browser)
+ deps.tasks_db.get_task = AsyncMock(return_value=failed_task)
+ with (
+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock),
+ patch(
+ "skyvern.forge.sdk.workflow.models.block.app.STORAGE",
+ new=MagicMock(
+ get_current_attempt_downloaded_files=AsyncMock(return_value=[]),
+ get_downloaded_file_signature_aliases=MagicMock(return_value=()),
+ ),
+ create=True,
+ ),
+ patch(
+ "skyvern.forge.sdk.workflow.models.block.app.WORKFLOW_SERVICE.get_recent_task_screenshot_artifacts",
+ new=AsyncMock(return_value=[]),
+ ),
+ patch(
+ "skyvern.forge.sdk.workflow.models.block.app.WORKFLOW_SERVICE.get_recent_workflow_screenshot_artifacts",
+ new=AsyncMock(return_value=[]),
+ ),
+ ):
+ result = await block.execute(
+ workflow_run_id="wr_task_block_test", workflow_run_block_id="wrb_test", organization_id="o_test"
+ )
+
+ assert deps.agent.execute_step.await_count == 1
+ assert result.success is False
+ assert result.can_continue_after_failure is False
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in result.error_codes
+
+
[email protected]("case", ["lost_during_downloaded_files", "lost_during_task_order", "ordinary_failure"])
+async def test_task_block_admits_no_retry_after_a_loss_during_its_failure_bookkeeping(case: str) -> None:
+ block = TaskBlock(
+ label="nav",
+ url="https://example.com",
+ navigation_goal="go",
+ output_parameter=output_parameter(),
+ max_retries=1,
+ )
+ state, context, _ = _bound_state("wr_task_block_test")
+ now = datetime.now(timezone.utc)
+ organization = make_organization(now)
+ first = make_task(now, organization, task_id="tsk_first", status=TaskStatus.failed, failure_reason="No such button")
+ second = make_task(now, organization, task_id="tsk_second", status=TaskStatus.completed)
+ order_reads: list[str] = []
+
+ async def downloaded_files(**_: object) -> list:
+ if case == "lost_during_downloaded_files":
+ state._on_browser_context_closed(context)
+ return []
+
+ async def last_task_for_run(**_: object) -> object:
+ # The first read orders the first attempt, the second orders its retry.
+ order_reads.append("read")
+ if case == "lost_during_task_order" and len(order_reads) == 2:
+ state._on_browser_context_closed(context)
+ return None if len(order_reads) == 1 else first
+
+ with (
+ skyvern_context.scoped(SkyvernContext()),
+ mock_block_execute_deps(working_page_url="https://example.com") as deps,
+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock),
+ ):
+ deps.browser_manager.get_for_workflow_run = MagicMock(return_value=state)
+ deps.tasks_db.get_last_task_for_workflow_run = AsyncMock(side_effect=last_task_for_run)
+ deps.tasks_db.get_task = AsyncMock(side_effect=[first, second])
+ deps.agent.create_task_and_step_from_block = AsyncMock(
+ side_effect=[(first, SimpleNamespace(step_id="stp_first")), (second, SimpleNamespace(step_id="stp_second"))]
+ )
+ block_module.app.STORAGE = MagicMock(
+ get_current_attempt_downloaded_files=AsyncMock(side_effect=downloaded_files),
+ get_downloaded_file_signature_aliases=MagicMock(return_value=()),
+ )
+ block_module.app.WORKFLOW_SERVICE.get_recent_task_screenshot_artifacts = AsyncMock(return_value=[])
+ block_module.app.WORKFLOW_SERVICE.get_recent_workflow_screenshot_artifacts = AsyncMock(return_value=[])
+
+ result = await block.execute(
+ workflow_run_id="wr_task_block_test", workflow_run_block_id="wrb_test", organization_id="o_test"
+ )
+
+ linked_tasks = [
+ update.kwargs["task_id"]
+ for update in deps.observer_db.update_workflow_run_block.await_args_list
+ if "task_id" in update.kwargs
+ ]
+ if case == "ordinary_failure":
+ assert linked_tasks == ["tsk_first", "tsk_second"]
+ assert result.success is True
+ return
+ # The loss lands after the retry decision was first made; no second attempt is created or linked.
+ assert deps.agent.create_task_and_step_from_block.await_count == 1
+ assert linked_tasks == ["tsk_first"]
+ assert result.status == BlockStatus.failed
+ assert result.failure_reason == "No such button"
+ assert isinstance(result.output_parameter_value, dict)
+ assert result.output_parameter_value["task_id"] == "tsk_first"
+ assert result.can_continue_after_failure is False
+ assert result.error_codes == [BROWSER_CONNECTION_LOST_ERROR_CODE]
+
+
+ "case",
+ ["lost_during_downloaded_files", "lost_during_screenshot_query", "healthy_termination", "canceled_after_loss"],
+)
+async def test_a_terminated_task_cannot_carry_the_run_on_after_a_loss_during_its_artifact_reads(
+ monkeypatch: pytest.MonkeyPatch, case: str
+) -> None:
+ block = TaskBlock(
+ label="nav",
+ url="https://example.com",
+ navigation_goal="go",
+ output_parameter=output_parameter(),
+ continue_on_failure=True,
+ )
+ state, context, _ = _bound_state("wr_task_block_test")
+ state.take_fullpage_screenshot = AsyncMock(return_value=None) # type: ignore[method-assign]
+ now = datetime.now(timezone.utc)
+ status = TaskStatus.canceled if case == "canceled_after_loss" else TaskStatus.terminated
+ finished = make_task(now, make_organization(now), task_id="tsk_nav", status=status, failure_reason="Blocked page")
+
+ async def execute_step(**_: object) -> None:
+ if case == "canceled_after_loss":
+ state._on_browser_context_closed(context)
+
+ async def downloaded_files(**_: object) -> list:
+ if case == "lost_during_downloaded_files":
+ state._on_browser_context_closed(context)
+ return []
+
+ async def task_screenshots(**_: object) -> list:
+ if case == "lost_during_screenshot_query":
+ state._on_browser_context_closed(context)
+ return []
+
+ mark_failed = AsyncMock(return_value=MagicMock())
+ mark_canceled = AsyncMock(return_value=MagicMock())
+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", mark_failed)
+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_canceled", mark_canceled)
+
+ with (
+ skyvern_context.scoped(SkyvernContext()),
+ mock_block_execute_deps(working_page_url="https://example.com") as deps,
+ patch.object(Block, "_generate_workflow_run_block_description", new_callable=AsyncMock),
+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock),
+ ):
+ deps.browser_manager.get_for_workflow_run = MagicMock(return_value=state)
+ deps.agent.execute_step = AsyncMock(side_effect=execute_step)
+ deps.tasks_db.get_task = AsyncMock(return_value=finished)
+ deps.observer_db.create_workflow_run_block = AsyncMock(
+ return_value=SimpleNamespace(workflow_run_block_id="wrb_nav")
+ )
+ block_module.app.DATABASE.workflow_runs.admit_workflow_run_block_dispatch = admit_block_dispatch()
+ block_module.app.STORAGE = MagicMock(
+ get_current_attempt_downloaded_files=AsyncMock(side_effect=downloaded_files),
+ get_downloaded_file_signature_aliases=MagicMock(return_value=()),
+ )
+ block_module.app.WORKFLOW_SERVICE.get_recent_task_screenshot_artifacts = AsyncMock(side_effect=task_screenshots)
+ block_module.app.WORKFLOW_SERVICE.get_recent_workflow_screenshot_artifacts = AsyncMock(return_value=[])
+
+ result = await block.execute_safe(workflow_run_id="wr_task_block_test", organization_id="o_test")
+ row: dict[str, object] = {}
+ for update in deps.observer_db.update_workflow_run_block.await_args_list:
+ row.update({key: value for key, value in update.kwargs.items() if value is not None})
+
+ _, should_stop = await WorkflowService()._handle_block_result_status(
+ block=block,
+ block_idx=0,
+ blocks_cnt=1,
+ block_result=result,
+ workflow_run=MagicMock(),
+ workflow_run_id="wr_task_block_test",
+ )
+
+ if case == "canceled_after_loss":
+ # A cancellation keeps its outcome; the loss does not turn it into a failure.
+ assert result.status == row["status"] == BlockStatus.canceled
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE not in result.error_codes
+ assert should_stop is True
+ mark_canceled.assert_awaited_once()
+ mark_failed.assert_not_awaited()
+ return
+ output = row["output"]
+ assert isinstance(output, dict)
+ assert output["task_id"] == "tsk_nav"
+ assert result.output_parameter_value == output
+ assert result.failure_reason == "Blocked page"
+ if case == "healthy_termination":
+ assert result.status == row["status"] == BlockStatus.terminated
+ assert result.can_continue_after_failure is True
+ assert should_stop is False
+ mark_failed.assert_not_awaited()
+ return
+ assert result.status == row["status"] == BlockStatus.failed
+ assert result.error_codes == row["error_codes"] == [BROWSER_CONNECTION_LOST_ERROR_CODE]
+ assert result.can_continue_after_failure is False
+ assert should_stop is True
+ mark_failed.assert_awaited_once()
+
+
+async def test_a_code_block_success_after_a_loss_is_left_to_the_run_completion_gate() -> None:
+ block = _make_code_block()
+ succeeded = BlockResult(
+ success=True,
+ output_parameter=block.output_parameter,
+ status=BlockStatus.completed,
+ workflow_run_block_id="wrb_code",
+ )
+ with (
+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,
+ patch.object(CodeBlock, "_execute", AsyncMock(return_value=succeeded)),
+ ):
+ mock_app.BROWSER_MANAGER.get_for_workflow_run.return_value = lost_browser_state("wr_test")
+ mock_app.DATABASE.observer.update_workflow_run_block = AsyncMock()
+
+ result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_code")
+
+ assert result == succeeded
+ mock_app.DATABASE.observer.update_workflow_run_block.assert_not_awaited()
+
+
+async def test_task_block_does_not_take_over_a_browser_its_finished_child_lost() -> None:
+ block = TaskBlock(
+ label="nav",
+ url="https://example.com",
+ navigation_goal="go",
+ output_parameter=output_parameter(),
+ max_retries=1,
+ )
+ # A synchronous child shared this run's browser and finished; the browser stays bound to the child.
+ state, context, _ = _bound_state("wr_child")
+ state.reconnect = AsyncMock() # type: ignore[method-assign]
+ manager = RealBrowserManager()
+ manager.pages["wr_task_block_test"] = state
+ now = datetime.now(timezone.utc)
+ organization = make_organization(now)
+ task = make_task(now, organization, workflow_run_id="wr_task_block_test")
+ step = make_step(now, task, step_id="stp_test", status=StepStatus.created, order=0, output=None)
+ workflow_run = MagicMock(workflow_run_id="wr_task_block_test", parent_workflow_run_id=None, browser_profile_id=None)
+
+ async def execute_step(**_: object) -> None:
+ # The loss lands after the block checked its browser and before the step acquires it.
+ state._on_browser_context_closed(context)
+ # The real step fails its task on either refusal and returns.
+ with contextlib.suppress(MissingBrowserState, MissingBrowserStatePage):
+ await ForgeAgent().initialize_execution_state(task, step, workflow_run)
+
+ with (
+ skyvern_context.scoped(SkyvernContext()),
+ mock_block_execute_deps(working_page_url="https://example.com") as deps,
+ patch.object(object.__getattribute__(app, "_inst"), "BROWSER_MANAGER", manager),
+ patch.object(manager, "_create_browser_state", new=AsyncMock()) as mock_create,
+ ):
+ deps.browser_manager.get_for_workflow_run = MagicMock(return_value=state)
+ deps.agent.create_task_and_step_from_block = AsyncMock(return_value=(task, step))
+ deps.agent.execute_step = AsyncMock(side_effect=execute_step)
+ deps.tasks_db.get_task = AsyncMock(return_value=task.model_copy(update={"status": TaskStatus.failed}))
+ with (
+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock),
+ patch(
+ "skyvern.forge.sdk.workflow.models.block.app.STORAGE",
+ new=MagicMock(
+ get_current_attempt_downloaded_files=AsyncMock(return_value=[]),
+ get_downloaded_file_signature_aliases=MagicMock(return_value=()),
+ ),
+ create=True,
+ ),
+ patch(
+ "skyvern.forge.sdk.workflow.models.block.app.WORKFLOW_SERVICE.get_recent_task_screenshot_artifacts",
+ new=AsyncMock(return_value=[]),
+ ),
+ patch(
+ "skyvern.forge.sdk.workflow.models.block.app.WORKFLOW_SERVICE.get_recent_workflow_screenshot_artifacts",
+ new=AsyncMock(return_value=[]),
+ ),
+ ):
+ result = await block.execute(
+ workflow_run_id="wr_task_block_test", workflow_run_block_id="wrb_test", organization_id="o_test"
+ )
+
+ assert deps.agent.execute_step.await_count == 1
+ state.reconnect.assert_not_awaited()
+ mock_create.assert_not_awaited()
+ assert state.runtime_event_context.workflow_run_id == "wr_child"
+ assert result.can_continue_after_failure is False
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in result.error_codes
+
+
[email protected]("owner", ["observing_run", "later_run"])
+async def test_lost_browser_refuses_state_mutations_for_its_owner_before_side_effects(
+ monkeypatch: pytest.MonkeyPatch, owner: str
+) -> None:
+ state, context, _ = _bound_state("wr_test")
+ state._on_browser_context_closed(context)
+ if owner == "later_run":
+ state.bind_runtime_event_context(BrowserRuntimeLogContext(workflow_run_id="wr_next"))
+ refused = owner == "observing_run"
+ driver_starts: list[bool] = []
+
+ class CountingPlaywright:
+ async def start(self) -> object:
+ driver_starts.append(True)
+ raise RuntimeError("driver start reached")
+
+ monkeypatch.setattr(real_browser_state_module, "async_playwright", lambda: CountingPlaywright())
+ create_context = AsyncMock(side_effect=RuntimeError("context factory reached"))
+ monkeypatch.setattr(real_browser_state_module.BrowserContextFactory, "create_browser_context", create_context)
+ monkeypatch.setattr(navigation_module, "asyncio", ScopedAsyncio(sleep=AsyncMock()))
+ context.new_page = AsyncMock(side_effect=RuntimeError("new page reached"))
+ context._skyvern_cdp_download_interceptor = None
+ page = MagicMock()
+ page.goto = AsyncMock(side_effect=PlaywrightError("Target page, context or browser has been closed"))
+
+ with pytest.raises(MissingBrowserStatePage):
+ await state.must_get_working_page()
+ with pytest.raises(MissingBrowserState if refused else FailedToNavigateToUrl):
+ await state.navigate_to_url(page=page, url="https://93.184.216.34/next")
+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed")) # type: ignore[method-assign]
+ with pytest.raises((MissingBrowserState, RuntimeError)) as reconnect_error:
+ await state.reconnect(browser_address="ws://remote-browser")
+ state.browser_context = None
+ with pytest.raises((MissingBrowserState, RuntimeError)):
+ await state.check_and_fix_state()
+
+ assert context.new_page.await_count == (0 if refused else 1)
+ assert page.goto.await_count == (0 if refused else NAVIGATION_MAX_RETRY_TIME)
+ assert len(driver_starts) == (0 if refused else 1)
+ assert create_context.await_count == (0 if refused else 1)
+ assert (BROWSER_CONNECTION_LOST_ERROR_CODE in str(reconnect_error.value)) is refused
+
+
+def _page_opening_context(context: MagicMock) -> None:
+ """The run's retained context object: it opens tabs as Playwright does, so any attempt is counted."""
+ context.pages = []
+ context._skyvern_cdp_download_interceptor = None
+ replacement = MagicMock(url="about:blank", close=AsyncMock(), bring_to_front=AsyncMock())
+ context.new_page = _replacement_page_opener(context, replacement, [])
+
+
[email protected]("owner", ["observing_run", "later_run", "page_only_close"])
+async def test_retained_closed_context_opens_no_page_for_the_run_that_lost_it(owner: str) -> None:
+ state, context, _ = _bound_state("wr_test")
+ _page_opening_context(context)
+ if owner != "page_only_close":
+ state._on_browser_context_closed(context)
+ if owner == "later_run":
+ state.bind_runtime_event_context(BrowserRuntimeLogContext(workflow_run_id="wr_next"))
+ refused = owner == "observing_run"
+
+ for open_page in (state.new_page, state.check_and_fix_state):
+ with (
+ pytest.raises(MissingBrowserState, match=BROWSER_CONNECTION_LOST_ERROR_CODE)
+ if refused
+ else (contextlib.nullcontext())
+ ):
+ await open_page()
+
+ assert context.new_page.await_count == (0 if refused else 2)
+
+
[email protected]("existing_page", [False, True], ids=["opens_a_page", "reuses_a_page"])
[email protected]("native_close", [True, False], ids=["lost_during_page_selection", "browser_alive"])
+async def test_missing_page_is_neither_opened_nor_reused_after_a_loss_during_page_selection(
+ monkeypatch: pytest.MonkeyPatch, existing_page: bool, native_close: bool
+) -> None:
+ state, context, _ = _bound_state("wr_test")
+ _page_opening_context(context)
+ remote_page = MagicMock(url="https://example.test/app")
+ context.pages = [remote_page]
+ state.browser_artifacts = BrowserArtifacts(remote_browser_session_id="pbs_remote")
+
+ async def select_pages(*_args: object, **_kwargs: object) -> list[MagicMock]:
+ if native_close:
+ state._on_browser_context_closed(context)
+ return [remote_page] if existing_page else []
+
+ monkeypatch.setattr(state, "list_valid_pages", select_pages)
+ monkeypatch.setattr(state, "set_working_page", set_working_page := AsyncMock())
+ monkeypatch.setattr(state, "_close_all_other_pages", AsyncMock())
+
+ with pytest.raises(MissingBrowserState) if native_close else contextlib.nullcontext():
+ await state.check_and_fix_state()
+
+ assert context.new_page.await_count == (0 if native_close or existing_page else 1)
+ assert set_working_page.await_count == (0 if native_close else 1)
+
+
[email protected]("native_close", [True, False], ids=["lost_during_destination_check", "browser_alive"])
+async def test_new_tab_opens_no_page_after_a_loss_during_its_destination_check(
+ monkeypatch: pytest.MonkeyPatch, native_close: bool
+) -> None:
+ from skyvern.webeye.actions import actions as action_types
+ from skyvern.webeye.actions import handler as handler_module
+ from skyvern.webeye.actions.handler import handle_new_tab_action
+ from skyvern.webeye.actions.responses import ActionSuccess
+
+ state, context, _ = _bound_state("wr_tab")
+ _page_opening_context(context)
+
+ async def check_destination(validate: object, url: str) -> str:
+ if native_close:
+ state._on_browser_context_closed(context)
+ return url
+
+ monkeypatch.setattr(handler_module, "asyncio", ScopedAsyncio(to_thread=check_destination))
+ monkeypatch.setattr(app.BROWSER_MANAGER, "get_for_task", lambda *_args, **_kwargs: state)
+ monkeypatch.setattr(state, "navigate_to_url", AsyncMock())
+ monkeypatch.setattr(state, "set_active_page", AsyncMock())
+ task = SimpleNamespace(task_id="tsk_tab", workflow_run_id="wr_tab")
+ action = action_types.NewTabAction(url="https://93.184.216.34/next")
+
+ with pytest.raises(MissingBrowserState) if native_close else contextlib.nullcontext():
+ results = await handle_new_tab_action(action, MagicMock(), MagicMock(), task, MagicMock())
+ assert isinstance(results[-1], ActionSuccess)
+
+ assert context.new_page.await_count == (0 if native_close else 1)
+
+
[email protected]("native_close", [True, False], ids=["native_close_during_goto", "ordinary_navigation_error"])
+async def test_url_block_that_loses_its_browser_mid_navigation_stops_the_run(
+ monkeypatch: pytest.MonkeyPatch, native_close: bool
+) -> None:
+ block = UrlBlock(
+ label="go", url="https://93.184.216.34/next", output_parameter=output_parameter(), continue_on_failure=True
+ )
+ state, context, _ = _bound_state("wr_task_block_test")
+ gotos: list[str] = []
+
+ async def goto(url: str, **_: object) -> None:
+ gotos.append(url)
+ if native_close:
+ state._on_browser_context_closed(context)
+ raise PlaywrightError("Target page, context or browser has been closed")
+ raise PlaywrightError("Timeout 30000ms exceeded.")
+
+ page = MagicMock(url="https://93.184.216.34/start")
+ page.goto = goto
+ state.get_working_page = AsyncMock(return_value=page) # type: ignore[method-assign]
+ state.take_fullpage_screenshot = AsyncMock(return_value=None) # type: ignore[method-assign]
+ monkeypatch.setattr(navigation_module, "asyncio", ScopedAsyncio(sleep=AsyncMock()))
+ now = datetime.now(timezone.utc)
+ previous_task = make_task(now, make_organization(now), status=TaskStatus.completed)
+ mark_failed = AsyncMock(return_value=MagicMock())
+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", mark_failed)
+
+ with (
+ skyvern_context.scoped(SkyvernContext()),
+ mock_block_execute_deps(working_page_url="https://93.184.216.34/start") as deps,
+ patch.object(Block, "_generate_workflow_run_block_description", new_callable=AsyncMock),
+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock),
+ ):
+ deps.browser_manager.get_for_workflow_run = MagicMock(return_value=state)
+ deps.tasks_db.get_last_task_for_workflow_run = AsyncMock(return_value=previous_task)
+ deps.observer_db.create_workflow_run_block = AsyncMock(
+ return_value=SimpleNamespace(workflow_run_block_id="wrb_go")
+ )
+ block_module.app.DATABASE.workflow_runs.admit_workflow_run_block_dispatch = admit_block_dispatch()
+
+ result = await block.execute_safe(workflow_run_id="wr_task_block_test", organization_id="o_test")
+ persisted_codes = {
+ code
+ for update in deps.observer_db.update_workflow_run_block.await_args_list
+ if update.kwargs.get("workflow_run_block_id") == "wrb_go"
+ for code in update.kwargs.get("error_codes") or []
+ }
+
+ _, should_stop = await WorkflowService()._handle_block_result_status(
+ block=block,
+ block_idx=0,
+ blocks_cnt=3,
+ block_result=result,
+ workflow_run=MagicMock(),
+ workflow_run_id="wr_task_block_test",
+ )
+ retry_policy = WorkflowRetryPolicy.model_validate({"max_retries": 2, "retry_on": [{"status": "failed"}]})
+ retry = evaluate_retry_policy(retry_policy, WorkflowRunStatus.failed, persisted_codes, 0)
+
+ assert result.status == BlockStatus.failed
+ if native_close:
+ assert gotos == [block.url]
+ assert result.can_continue_after_failure is False
+ assert persisted_codes == {BROWSER_CONNECTION_LOST_ERROR_CODE}
+ assert should_stop is True
+ assert retry == (False, BROWSER_CONNECTION_LOST_ERROR_CODE)
+ else:
+ assert gotos == [block.url] * NAVIGATION_MAX_RETRY_TIME
+ assert result.can_continue_after_failure is True
+ assert persisted_codes == set()
+ assert should_stop is False
+ assert retry == (True, "matched")
+
+
+ ("seam", "dispatches", "backoffs"),
+ [
+ ("native_close_during_goto", 1, 0),
+ ("native_close_during_backoff", 1, 1),
+ ("native_close_during_destination_check", 0, 0),
+ ("probe_timeout_during_backoff", NAVIGATION_MAX_RETRY_TIME, NAVIGATION_MAX_RETRY_TIME - 1),
+ ("cancelled_during_backoff", 1, 1),
+ ],
+)
+async def test_navigation_is_not_dispatched_again_after_its_browser_is_lost_while_it_waits(
+ monkeypatch: pytest.MonkeyPatch, seam: str, dispatches: int, backoffs: int
+) -> None:
+ state, context, browser = _bound_state("wr_test")
+ context._skyvern_cdp_download_interceptor = None
+ page = MagicMock()
+
+ async def goto(*_: object, **__: object) -> None:
+ if seam == "native_close_during_goto":
+ state._on_browser_context_closed(context)
+ raise PlaywrightError("net::ERR_CONNECTION_RESET")
+
+ async def lose_browser_while_waiting(_: float) -> None:
+ if seam == "native_close_during_backoff":
+ state._on_browser_context_closed(context)
+ elif seam == "probe_timeout_during_backoff":
+ _probe_timeout(state, context, browser)
+ elif seam == "cancelled_during_backoff":
+ raise asyncio.CancelledError
+
+ async def destination_check(check: object, *args: object) -> object:
+ if seam == "native_close_during_destination_check":
+ state._on_browser_context_closed(context)
+ return check(*args) # type: ignore[operator]
+
+ page.goto = AsyncMock(side_effect=goto)
+ backoff = AsyncMock(side_effect=lose_browser_while_waiting)
+ monkeypatch.setattr(navigation_module, "asyncio", ScopedAsyncio(sleep=backoff, to_thread=destination_check))
+
+ with pytest.raises(
+ asyncio.CancelledError if seam == "cancelled_during_backoff" else FailedToNavigateToUrl
+ ) as error:
+ await state.navigate_to_url(page=page, url="https://93.184.216.34/next")
+
+ assert page.goto.await_count == dispatches
+ assert backoff.await_count == backoffs
+ if seam == "native_close_during_backoff":
+ assert "ERR_CONNECTION_RESET" in str(error.value)
+
+
[email protected]("native_close", [True, False], ids=["native_close_during_reload", "ordinary_reload_failure"])
+async def test_reload_degradation_stops_after_its_browser_is_lost(native_close: bool) -> None:
+ state, context, _ = _bound_state("wr_test")
+ page = MagicMock(url="https://93.184.216.34/page")
+
+ async def reload(**_: object) -> None:
+ if native_close:
+ state._on_browser_context_closed(context)
+ raise PlaywrightError("Target page, context or browser has been closed")
+ raise PlaywrightError("net::ERR_CONNECTION_RESET")
+
+ page.reload = AsyncMock(side_effect=reload)
+ state.get_working_page = AsyncMock(return_value=page) # type: ignore[method-assign]
+
+ with pytest.raises(FailedToReloadPage):
+ await state.reload_page(degradation=True)
+
+ assert page.reload.await_count == (1 if native_close else 3)
+
+
def test_is_connected_false_when_context_close_was_called() -> None:
browser = MagicMock()
browser.is_connected = MagicMock(return_value=True)
@@ -2624,6 +3369,37 @@ async def test_a_caller_recovering_during_the_reap_gets_the_reaper_replacement()
assert context.new_page.await_count == 1
[email protected]("native_close", [True, False], ids=["lost_during_the_first_replacement", "page_only_crash"])
+async def test_queued_crash_recoveries_open_no_page_after_the_first_lost_the_browser(native_close: bool) -> None:
+ gate = asyncio.Event()
+ crashed, sibling, replacement = _crashable_page(), _crashable_page(), _crashable_page()
+ crashed.url = sibling.url = replacement.url = "about:blank"
+ context = _crashable_context(crashed, sibling)
+ state = RealBrowserState(pw=MagicMock(), browser_context=context, page=crashed)
+ state.bind_runtime_event_context(BrowserRuntimeLogContext(workflow_run_id="wr_test"))
+
+ async def open_page() -> MagicMock:
+ await gate.wait()
+ if native_close:
+ state._on_browser_context_closed(context)
+ raise PlaywrightError("Target page, context or browser has been closed")
+ context.pages.append(replacement)
+ return replacement
+
+ context.new_page = AsyncMock(side_effect=open_page)
+ state._on_page_crashed(crashed)
+ state._on_page_crashed(sibling)
+ await asyncio.sleep(0)
+ gate.set()
+ await _reap_crashed_pages(state)
+
+ assert context.new_page.await_count == 1
+ async with state._reopen_working_page_lock:
+ with pytest.raises(MissingBrowserStatePage) if native_close else contextlib.nullcontext():
+ assert await asyncio.wait_for(state.must_get_working_page(), 1) is replacement
+
+
@pytest.mark.asyncio
async def test_a_crashed_tab_with_a_live_sibling_closes_without_opening_another() -> None:
crashed = _crashable_page()
@@ -3547,6 +4323,16 @@ async def test_reconnect_scheduled_stop_observes_loss_before_installing_intent(
)
monkeypatch.setattr(state, "get_working_page", AsyncMock(return_value=MagicMock()))
monkeypatch.setattr(state, "check_and_fix_state", finish_setup)
+ if signal == "context_then_driver" and not guarded:
+ # The owning run's current context closes natively before the swap, so the run never gets a replacement.
+ with skyvern_context.scoped(SkyvernContext(workflow_run_id="ambient-run")):
+ with pytest.raises(MissingBrowserState, match=BROWSER_CONNECTION_LOST_ERROR_CODE):
+ await state.reconnect(stale_context_is_unusable=True)
+ assert state.browser_context is old
+ assert state.pw is old_pw
+ replacement.pw.stop.assert_awaited()
+ assert state.get_browser_connection_loss() is not None
+ return
with skyvern_context.scoped(SkyvernContext(workflow_run_id="ambient-run")), capture_logs() as logs:
await state.reconnect(stale_context_is_unusable=True)
await delivered.wait()
diff --git a/tests/unit/workflow/test_code_block_escalation.py b/tests/unit/workflow/test_code_block_escalation.py
index 2b0e67c5f..4ac872ea8 100644
--- a/tests/unit/workflow/test_code_block_escalation.py
+++ b/tests/unit/workflow/test_code_block_escalation.py
@@ -22,7 +22,7 @@ from structlog.testing import capture_logs
import skyvern.webeye.navigation as navigation_module
from skyvern.errors.errors import UserDefinedError
-from skyvern.exceptions import NO_ADDRESS_RECORD_NAV_ERROR_CODE
+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE, NO_ADDRESS_RECORD_NAV_ERROR_CODE
from skyvern.forge import app
from skyvern.forge.agent_functions import CodeBlockEngineFailure, CodeBlockEngineResult
from skyvern.forge.sdk.api.llm.schema_validator import validate_and_fill_extraction_result, validate_schema
@@ -61,7 +61,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (
WorkflowParameterType,
)
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun
-from skyvern.forge.sdk.workflow.service import _merge_workflow_run_errors
+from skyvern.forge.sdk.workflow.service import WorkflowService, _merge_workflow_run_errors
from skyvern.forge.taskv3.goal_composition import (
CodeProgressRecord,
GoalDirectives,
@@ -74,7 +74,9 @@ from skyvern.schemas.self_heal import HealClassification, HealSkipReason, HealSt
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import Action
from skyvern.webeye.browser_artifacts import BrowserArtifacts
+from skyvern.webeye.browser_runtime_events import BrowserRuntimeLogContext
from skyvern.webeye.navigation import clear_task_nav_error_code, record_task_nav_error_code
+from skyvern.webeye.real_browser_state import RealBrowserState
SECRET_VALUE = "hunter2-super-secret"
SHORT_SECRET_VALUE = "abc"
@@ -3835,6 +3837,124 @@ async def test_legacy_heal_success_skips_failed_write_and_ends_completed(
assert FakeRecorder.instances[0].finalized_success is True
[email protected]("engine", ["inline", "secure"])
[email protected]("browser_lost", [True, False], ids=["native_context_close", "ordinary_failure"])
+async def test_failure_after_native_browser_loss_stops_the_run_instead_of_healing(
+ monkeypatch: pytest.MonkeyPatch,
+ ai_fallback_flag: Callable[[str | None], None],
+ engine: str,
+ browser_lost: bool,
+) -> None:
+ ai_fallback_flag("o_test")
+ state = _install_db_fakes(monkeypatch, final_status=TaskStatus.completed)
+ block = _make_code_block(
+ steps=[CodeBlockStep(description="open the report", line_start=1, line_end=1)],
+ code="await page.click('#go')",
+ )
+ block.continue_on_failure = True
+ context = _make_context()
+ native_context = MagicMock(browser=MagicMock())
+ browser_state = RealBrowserState(pw=MagicMock(), browser_context=native_context)
+ browser_state.bind_runtime_event_context(BrowserRuntimeLogContext(workflow_run_id="wr_test"))
+ browser_state.get_working_page = AsyncMock(return_value=MagicMock()) # type: ignore[method-assign]
+ _patch_execute_chokepoint_environment(
+ monkeypatch, context=context, fake_browser_state=browser_state, use_codeblock_runner=engine == "secure"
+ )
+ closed = PlaywrightError("Target page, context or browser has been closed")
+
+ def close_in_flight() -> None:
+ if browser_lost:
+ browser_state._on_browser_context_closed(native_context)
+
+ async def run_on_secure_runner(**_: Any) -> CodeBlockEngineResult | None:
+ if engine != "secure":
+ return None
+ close_in_flight()
+ return CodeBlockEngineResult(
+ block_result=None,
+ failure=CodeBlockEngineFailure(
+ error_code="user_code_error",
+ safe_message=None,
+ failure_reason="CodeBlock failed while running user code.",
+ exception_class="playwright._impl._errors.Error",
+ failing_line=1,
+ healability_hint=True,
+ ),
+ )
+
+ class ClickInFlightRecorder(FakeRecorder):
+ def __init__(self, **kwargs: Any) -> None:
+ super().__init__(**kwargs)
+
+ async def click(*_: Any, **__: Any) -> None:
+ close_in_flight()
+ raise closed
+
+ self.recording_page.click = click
+
+ monkeypatch.setattr(app.AGENT_FUNCTION, "execute_code_block_override", AsyncMock(side_effect=run_on_secure_runner))
+ ClickInFlightRecorder.reset(last_recorded_exception=closed)
+ monkeypatch.setattr("skyvern.forge.sdk.workflow.models.block.CodeBlockActionRecording", ClickInFlightRecorder)
+ monkeypatch.setattr(CodeBlock, "_capture_failure_evidence", AsyncMock(return_value=None))
+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", AsyncMock(return_value=None))
+
+ async def heal_and_fall_through(**kwargs: Any) -> BlockResult:
+ return await block.build_block_result(
+ success=True,
+ failure_reason=None,
+ output_parameter_value={"task_id": "tsk_escalation", "status": "completed"},
+ status=BlockStatus.completed,
+ workflow_run_block_id=kwargs["workflow_run_block_id"],
+ organization_id=kwargs["organization_id"],
+ )
+
+ heal = AsyncMock(side_effect=heal_and_fall_through)
+ monkeypatch.setattr(block, "_attempt_self_heal", heal)
+ mark_failed = AsyncMock(return_value=MagicMock())
+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", mark_failed)
+
+ with skyvern_context.scoped(SkyvernContext(organization_id="o_test", workflow_run_id="wr_test")):
+ result = await block.execute(
+ workflow_run_id="wr_test",
+ workflow_run_block_id="wrb_test",
+ organization_id="o_test",
+ browser_session_id="pbs_test",
+ )
+ _, should_stop = await WorkflowService()._handle_block_result_status(
+ block=block,
+ block_idx=0,
+ blocks_cnt=2,
+ block_result=result,
+ workflow_run=MagicMock(),
+ workflow_run_id="wr_test",
+ )
+
+ persisted_codes = [
+ code
+ for update in state["workflow_run_block_updates"]
+ if update.get("workflow_run_block_id") == "wrb_test"
+ for code in update.get("error_codes") or []
+ ]
+ if browser_lost:
+ heal.assert_not_awaited()
+ assert result.success is False
+ assert result.status == BlockStatus.failed
+ assert result.failure_reason
+ assert result.can_continue_after_failure is False
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in result.error_codes
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in persisted_codes
+ assert _statuses_for_block(state, "wrb_test")[-1] == BlockStatus.failed
+ assert ClickInFlightRecorder.instances[0].finalized_success is False
+ assert should_stop is True
+ mark_failed.assert_awaited_once()
+ else:
+ heal.assert_awaited_once()
+ assert result.success is True
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE not in persisted_codes
+ assert should_stop is False
+
+
@pytest.mark.asyncio
async def test_heal_output_write_failure_does_not_finalize_completed(
monkeypatch: pytest.MonkeyPatch, ai_fallback_flag: Callable[[str | None], None]
diff --git a/tests/unit/workflow/test_execute_single_block_phases.py b/tests/unit/workflow/test_execute_single_block_phases.py
index cb3fbcc74..36aec9658 100644
--- a/tests/unit/workflow/test_execute_single_block_phases.py
+++ b/tests/unit/workflow/test_execute_single_block_phases.py
@@ -19,19 +19,27 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from structlog.testing import capture_logs
-from skyvern.exceptions import MissingBrowserState, MissingBrowserStatePage, StepTerminationError
+from skyvern.exceptions import (
+ BROWSER_CONNECTION_LOST_ERROR_CODE,
+ MissingBrowserState,
+ MissingBrowserStatePage,
+ StepTerminationError,
+)
from skyvern.forge import app
from skyvern.forge.sdk.db.enums import BrowserSeedSource
from skyvern.forge.sdk.workflow.context_manager import BlockOutcome, WorkflowRunContext
+from skyvern.forge.sdk.workflow.models import block as block_module
from skyvern.forge.sdk.workflow.models.block import (
Block,
BranchCondition,
CodeBlock,
ConditionalBlock,
ForLoopBlock,
+ HttpRequestBlock,
JinjaBranchCriteria,
LoginBlock,
NavigationBlock,
+ PrintPageBlock,
WaitBlock,
)
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
@@ -47,7 +55,10 @@ from skyvern.services import script_service
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import Action
from skyvern.webeye.browser_artifacts import BrowserArtifacts
+from skyvern.webeye.real_browser_manager import RealBrowserManager
+from tests.unit._task_block_fakes import lost_browser_state, run_bound_browser_state
from tests.unit.fake_workflow_run_context import FakeWorkflowRunContext
+from tests.unit.scoped_asyncio import ScopedAsyncio
def _output_parameter(key: str) -> OutputParameter:
@@ -177,6 +188,7 @@ def _stub_run_refresh(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "register_block_parameters_for_workflow_run", AsyncMock())
# get_all_parameters needs a sync context object; the stub app's auto-AsyncMock returns a coroutine.
monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "get_workflow_run_context", MagicMock(return_value=MagicMock()))
+ monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "get_attempt_number", MagicMock(return_value=1))
@asynccontextmanager
async def admit_dispatch(_: str) -> AsyncIterator[MagicMock]:
@@ -378,6 +390,7 @@ def _wire_real_execute_safe(
*,
execute: AsyncMock | None = None,
pause_point: str | None = None,
+ browser_state: Any = None,
) -> _PausableAwait:
pause = _PausableAwait()
@@ -394,7 +407,7 @@ def _wire_real_execute_safe(
context = MagicMock()
context.cancel_failure_evidence_capture = pause if pause_point == "capture" else AsyncMock()
monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "get_workflow_run_context", MagicMock(return_value=context))
- browser_state = MagicMock()
+ browser_state = browser_state or MagicMock()
browser_state.take_fullpage_screenshot = pause if pause_point == "screenshot" else AsyncMock(return_value=b"png")
monkeypatch.setattr(app.BROWSER_MANAGER, "get_for_workflow_run", MagicMock(return_value=browser_state))
monkeypatch.setattr(
@@ -785,6 +798,430 @@ async def test_refused_script_step_fails_the_run_without_ai_fallback(monkeypatch
assert should_stop is True
+async def test_script_failure_after_browser_loss_is_not_retried_by_the_agent(monkeypatch: pytest.MonkeyPatch) -> None:
+ service = WorkflowService()
+ block = _navigation_block("cached_nav")
+ execute_safe = AsyncMock()
+ monkeypatch.setattr(NavigationBlock, "execute_safe", execute_safe)
+ monkeypatch.setattr(NavigationBlock, "_apply_workflow_system_prompt", lambda self, ctx: None)
+ monkeypatch.setattr(
+ app.BROWSER_MANAGER, "get_for_workflow_run", MagicMock(return_value=lost_browser_state("wr_test"))
+ )
+ script_row = SimpleNamespace(
+ label="cached_nav",
+ attempt_number=1,
+ parent_workflow_run_block_id=None,
+ created_at=datetime.now(UTC),
+ status=BlockStatus.failed,
+ failure_reason="Browser state page is missing.",
+ output=None,
+ error_codes=[],
+ workflow_run_block_id="wrb_script_1",
+ )
+ monkeypatch.setattr(app.DATABASE.observer, "get_workflow_run_blocks", AsyncMock(return_value=[script_row]))
+ update_row = AsyncMock()
+ monkeypatch.setattr(app.DATABASE.observer, "update_workflow_run_block", update_row)
+ mark_failed = AsyncMock(return_value=MagicMock())
+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", mark_failed)
+
+ async def run_cached_task() -> None:
+ raise MissingBrowserStatePage(workflow_run_id="wr_test")
+
+ _, _, block_result, should_stop, _ = await _run_single_block(
+ service,
+ block,
+ is_script_run=True,
+ script_blocks_by_label={"cached_nav": _script_block("cached_nav", "await run_cached_task()")},
+ loaded_script_module=SimpleNamespace(run_cached_task=run_cached_task),
+ )
+
+ execute_safe.assert_not_awaited()
+ mark_failed.assert_awaited_once()
+ assert should_stop is True
+ assert block_result is not None
+ assert block_result.can_continue_after_failure is False
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in block_result.error_codes
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in update_row.await_args.kwargs["error_codes"]
+
+
+async def test_unexpected_browser_loss_lookup_error_stops_script_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
+ block = _navigation_block("cached_nav")
+ execute_safe = AsyncMock()
+ monkeypatch.setattr(NavigationBlock, "execute_safe", execute_safe)
+ monkeypatch.setattr(
+ app.BROWSER_MANAGER, "get_for_workflow_run", MagicMock(side_effect=RuntimeError("loss lookup failed"))
+ )
+ monkeypatch.setattr(app.DATABASE.observer, "get_workflow_run_blocks", AsyncMock(return_value=[]))
+ mark_failed = AsyncMock(return_value=_workflow_run())
+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", mark_failed)
+
+ _, _, _, should_stop, _ = await _run_single_block(
+ WorkflowService(),
+ block,
+ is_script_run=True,
+ script_blocks_by_label={"cached_nav": _script_block("cached_nav", "None")},
+ loaded_script_module=SimpleNamespace(),
+ )
+
+ assert should_stop
+ assert "loss lookup failed" in mark_failed.await_args.kwargs["failure_reason"]
+ execute_safe.assert_not_awaited()
+
+
+async def test_adaptive_fallback_does_not_take_over_a_browser_the_finished_child_lost(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ manager = RealBrowserManager()
+ # A synchronous child shared this run's browser, finished, and stays bound to it; its loss is latched.
+ state, context = run_bound_browser_state("wr_child")
+ state._on_browser_context_closed(context)
+ manager.pages["wr_test"] = state
+ monkeypatch.setattr(object.__getattribute__(app, "_inst"), "BROWSER_MANAGER", manager)
+ block = _navigation_block("cached_nav")
+ execute_safe = AsyncMock(return_value=_completed_result(block))
+ monkeypatch.setattr(NavigationBlock, "execute_safe", execute_safe)
+ monkeypatch.setattr(NavigationBlock, "_apply_workflow_system_prompt", lambda self, ctx: None)
+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=MagicMock()))
+ script_row = SimpleNamespace(
+ label="cached_nav",
+ attempt_number=1,
+ parent_workflow_run_block_id=None,
+ created_at=datetime.now(UTC),
+ status=BlockStatus.failed,
+ failure_reason="Browser state for workflow_run_id=wr_test is missing.",
+ output=None,
+ error_codes=[],
+ workflow_run_block_id="wrb_script_1",
+ )
+ monkeypatch.setattr(app.DATABASE.observer, "get_workflow_run_blocks", AsyncMock(return_value=[script_row]))
+ monkeypatch.setattr(app.DATABASE.observer, "update_workflow_run_block", AsyncMock())
+ monkeypatch.setattr(
+ WorkflowService, "mark_workflow_run_as_failed_if_not_final", AsyncMock(return_value=MagicMock())
+ )
+ workflow_run = _workflow_run()
+ workflow_run.parent_workflow_run_id = workflow_run.browser_profile_id = None
+
+ async def run_cached_task() -> None:
+ raise MissingBrowserState(workflow_run_id="wr_test", failure_reason=BROWSER_CONNECTION_LOST_ERROR_CODE)
+
+ _, _, block_result, should_stop, _ = await _run_single_block(
+ WorkflowService(),
+ block,
+ workflow=_adaptive_workflow(),
+ workflow_run=workflow_run,
+ is_script_run=True,
+ script_blocks_by_label={"cached_nav": _script_block("cached_nav", "await run_cached_task()")},
+ loaded_script_module=SimpleNamespace(run_cached_task=run_cached_task),
+ )
+
+ execute_safe.assert_not_awaited()
+ assert should_stop is True
+ assert block_result.can_continue_after_failure is False
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in block_result.error_codes
+ assert state.runtime_event_context.workflow_run_id == "wr_child"
+ assert state.get_browser_connection_loss() is not None
+
+
[email protected]("held_by", [None, "wr_child"])
+async def test_adaptive_fallback_capture_does_not_acquire_a_browser(
+ monkeypatch: pytest.MonkeyPatch, held_by: str | None
+) -> None:
+ manager = RealBrowserManager()
+ if held_by is not None:
+ # A finished synchronous child still holds the run's shared browser.
+ state, _ = run_bound_browser_state(held_by)
+ manager.pages["wr_test"] = state
+ pages_before = dict(manager.pages)
+ monkeypatch.setattr(object.__getattribute__(app, "_inst"), "BROWSER_MANAGER", manager)
+ record_episode = AsyncMock(return_value=SimpleNamespace(episode_id="ep_test"))
+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", record_episode)
+ workflow_run = _workflow_run()
+ workflow_run.parent_workflow_run_id = workflow_run.browser_profile_id = None
+
+ with patch.object(manager, "_create_browser_state", new=AsyncMock()) as mock_create:
+ episode_id, _ = await WorkflowService()._record_fallback_episode(
+ workflow_run=workflow_run,
+ workflow=_adaptive_workflow(),
+ block=_navigation_block("cached_nav"),
+ organization_id="org_test",
+ workflow_run_id="wr_test",
+ error_message="script failed",
+ )
+
+ mock_create.assert_not_awaited()
+ assert episode_id == "ep_test"
+ assert manager.pages == pages_before
+ if held_by is not None:
+ assert manager.pages["wr_test"].runtime_event_context.workflow_run_id == held_by
+
+
+ ("case", "rows_after"),
+ [
+ ("lost_during_the_wait", ["pause=failed!", "notify=completed"]),
+ ("lost_during_the_final_wait", ["pause=failed!", "notify=completed"]),
+ ("lost_during_the_request_setup", ["pause=completed", "request=failed!", "notify=completed"]),
+ ("healthy", ["pause=completed", "request=completed", "notify=completed"]),
+ ("ordinary_error", ["pause=completed", "request=failed", "notify=completed"]),
+ ],
+)
+async def test_no_business_block_runs_and_the_body_cannot_complete_after_a_native_loss(
+ monkeypatch: pytest.MonkeyPatch, case: str, rows_after: list[str]
+) -> None:
+ state, context = run_bound_browser_state("wr_test")
+ _wire_real_execute_safe(monkeypatch, [WorkflowRunStatus.running], browser_state=state)
+ rows: dict[str, SimpleNamespace] = {}
+ sent: list[str] = []
+
+ async def create_row(*, label: str, **_: Any) -> SimpleNamespace:
+ if case == "lost_during_the_request_setup" and label == "request":
+ state._on_browser_context_closed(context)
+ rows[label] = SimpleNamespace(
+ label=label,
+ workflow_run_block_id=label,
+ created_at=0,
+ failure_reason=None,
+ attempt_number=1,
+ parent_workflow_run_block_id=None,
+ output=None,
+ )
+ rows[label].status, rows[label].error_codes = None, []
+ return rows[label]
+
+ async def update_row(*, workflow_run_block_id: str, status: Any = None, error_codes: Any = None, **_: Any) -> None:
+ row = rows[workflow_run_block_id]
+ row.status, row.error_codes = status or row.status, error_codes or row.error_codes
+
+ async def sleep(_: float) -> None:
+ if case.endswith("wait"):
+ state._on_browser_context_closed(context)
+
+ async def send(self: HttpRequestBlock, workflow_run_id: str, workflow_run_block_id: str, **_: Any) -> BlockResult:
+ sent.append(self.label)
+ if case == "ordinary_error" and self.label == "request":
+ raise RuntimeError("Cannot connect to host")
+ return await self.build_block_result(
+ True, None, status=BlockStatus.completed, workflow_run_block_id=workflow_run_block_id
+ )
+
+ monkeypatch.setattr(app.DATABASE.observer, "create_workflow_run_block", create_row)
+ monkeypatch.setattr(app.DATABASE.observer, "update_workflow_run_block", update_row)
+ monkeypatch.setattr(
+ app.DATABASE.observer, "get_workflow_run_blocks", AsyncMock(side_effect=lambda **_: [*rows.values()])
+ )
+ monkeypatch.setattr(block_module, "asyncio", ScopedAsyncio(sleep=sleep))
+ monkeypatch.setattr(WaitBlock, "record_output_parameter_value", AsyncMock())
+ monkeypatch.setattr(HttpRequestBlock, "execute", send)
+ mark_failed = AsyncMock(return_value=_workflow_run())
+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", mark_failed)
+ service = WorkflowService()
+ wait = WaitBlock(label="pause", wait_sec=1, output_parameter=_output_parameter("pause_output"))
+ request, notify = (HttpRequestBlock(label=n, output_parameter=_output_parameter(n)) for n in ("request", "notify"))
+ workflow = _workflow()
+ workflow.workflow_definition.finally_block_label = "notify"
+ workflow.workflow_definition.blocks = [wait, request, notify]
+
+ for block in [wait] if case == "lost_during_the_final_wait" else [wait, request]:
+ _, _, result, should_stop, _ = await _run_single_block(service, block, workflow=workflow)
+ if should_stop:
+ break
+ await service._execute_finally_block_if_configured(workflow, _workflow_run(), MagicMock(), None)
+
+ lost = case.startswith("lost")
+ assert sent == ([] if lost else ["request"]) + ["notify"]
+ assert [
+ f"{label}={row.status.value}{'!' * (BROWSER_CONNECTION_LOST_ERROR_CODE in row.error_codes)}"
+ for label, row in rows.items()
+ ] == rows_after
+ assert result.can_continue_after_failure is not lost
+ assert (result.failure_reason is None) == (case == "healthy")
+ assert mark_failed.await_count == (0 if case == "healthy" else 1)
+
+
+ "case",
+ [
+ "finally_notification",
+ "finally_first_fails_then_continues",
+ "finally_first_raises_then_continues",
+ "finally_first_fails_without_continue",
+ "finally_browser_child",
+ "body_loop",
+ "body_after_finally",
+ ],
+)
+async def test_a_finally_loop_still_sends_its_notification_after_a_native_loss(
+ monkeypatch: pytest.MonkeyPatch, case: str
+) -> None:
+ state = lost_browser_state("wr_test")
+ state.reconnect = AsyncMock() # type: ignore[method-assign]
+ _wire_real_execute_safe(monkeypatch, [WorkflowRunStatus.running], browser_state=state)
+ acquire = AsyncMock()
+ monkeypatch.setattr(app.BROWSER_MANAGER, "get_or_create_for_workflow_run", acquire)
+ sent: list[str] = []
+
+ async def create_row(*, label: str, **_: Any) -> SimpleNamespace:
+ return SimpleNamespace(label=label, workflow_run_block_id=f"wrb_{label}")
+
+ async def send(self: HttpRequestBlock, workflow_run_id: str, workflow_run_block_id: str, **_: Any) -> BlockResult:
+ sent.append(self.label)
+ # An ordinary request failure, unrelated to the browser the body lost.
+ failed = self.label == "notify" and case.startswith("finally_first")
+ if failed and "raises" in case:
+ raise RuntimeError("Cannot connect to host")
+ return await self.build_block_result(
+ not failed,
+ "HTTP 500" if failed else None,
+ status=BlockStatus.failed if failed else BlockStatus.completed,
+ workflow_run_block_id=workflow_run_block_id,
+ )
+
+ update_row = AsyncMock()
+ monkeypatch.setattr(app.DATABASE.observer, "create_workflow_run_block", create_row)
+ monkeypatch.setattr(app.DATABASE.observer, "update_workflow_run_block", update_row)
+ monkeypatch.setattr(HttpRequestBlock, "execute", send)
+ monkeypatch.setattr(ForLoopBlock, "get_loop_over_parameter_values", AsyncMock(return_value=["[email protected]"]))
+ for block_type in (ForLoopBlock, HttpRequestBlock, PrintPageBlock):
+ monkeypatch.setattr(block_type, "record_output_parameter_value", AsyncMock())
+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", AsyncMock(return_value=None))
+ loop_blocks: list[Any] = [
+ HttpRequestBlock(
+ label="notify",
+ output_parameter=_output_parameter("notify_output"),
+ continue_on_failure=case.endswith("continues"),
+ )
+ ]
+ if case.startswith("finally_first"):
+ loop_blocks.append(HttpRequestBlock(label="notify_again", output_parameter=_output_parameter("again_output")))
+ if case == "finally_browser_child":
+ loop_blocks.append(PrintPageBlock(label="snapshot", output_parameter=_output_parameter("snapshot_output")))
+ loop = ForLoopBlock(
+ label="notify_all", output_parameter=_output_parameter("notify_all_output"), loop_blocks=loop_blocks
+ )
+ workflow = _workflow()
+ workflow.workflow_definition.finally_block_label = "notify_all"
+ workflow.workflow_definition.blocks = [loop]
+
+ if case != "body_loop":
+ finally_execution = await WorkflowService()._execute_finally_block_if_configured(
+ workflow, _workflow_run(), MagicMock(), None
+ )
+ assert isinstance(finally_execution, tuple)
+ _, result = finally_execution
+ if case.startswith("body"):
+ # The finally scope ends with the finally block, so later work in the same task is still refused.
+ _, _, result, _, _ = await _run_single_block(WorkflowService(), loop, workflow=workflow)
+
+ continued = case.endswith("continues")
+ expected_sent = [] if case == "body_loop" else ["notify", "notify_again"] if continued else ["notify"]
+ assert sent == expected_sent
+ if case == "finally_notification" or continued:
+ assert result.status == BlockStatus.completed
+ else:
+ assert result.status == BlockStatus.failed
+ coded_rows = [
+ call.kwargs["workflow_run_block_id"]
+ for call in update_row.await_args_list
+ if BROWSER_CONNECTION_LOST_ERROR_CODE in (call.kwargs.get("error_codes") or [])
+ ]
+ if case.startswith("body"):
+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in result.error_codes
+ else:
+ # Inside the finally block only the refused browser child carries the body's loss, and it ends the loop.
+ assert coded_rows == (["wrb_snapshot"] if case == "finally_browser_child" else [])
+ assert (result.can_continue_after_failure is False) == (case == "finally_browser_child")
+ state.reconnect.assert_not_awaited()
+ acquire.assert_not_awaited()
+
+
[email protected]("child_lost", [False, True], ids=["healthy_child", "child_lost"])
+async def test_a_child_run_started_from_a_finally_block_keeps_its_own_body_guards(
+ monkeypatch: pytest.MonkeyPatch, child_lost: bool
+) -> None:
+ parent_state = lost_browser_state("wr_test")
+ child_state, child_context = run_bound_browser_state("wr_child")
+ _wire_real_execute_safe(monkeypatch, [WorkflowRunStatus.running])
+ states = {"wr_test": parent_state, "wr_child": child_state}
+ monkeypatch.setattr(
+ app.BROWSER_MANAGER, "get_for_workflow_run", lambda workflow_run_id, **_: states[workflow_run_id]
+ )
+ sent: list[str] = []
+ child_results: dict[str, BlockResult] = {}
+ permission_after_child: list[str | None] = []
+
+ async def create_row(*, label: str, **_: Any) -> SimpleNamespace:
+ return SimpleNamespace(label=label, workflow_run_block_id=f"wrb_{label}")
+
+ def child_block(label: str) -> HttpRequestBlock:
+ return HttpRequestBlock(label=label, output_parameter=_output_parameter(f"{label}_output"))
+
+ async def run_child_inline() -> None:
+ # Stand-in for a synchronous WorkflowTriggerBlock: the child run executes inline, in this same context.
+ for label in ("child_first", "child_email"):
+ child_results[label] = await child_block(label).execute_safe(
+ workflow_run_id="wr_child", organization_id="org_test"
+ )
+ if child_lost and label == "child_first":
+ child_state._on_browser_context_closed(child_context)
+ with block_module.running_after_browser_loss("wr_child"):
+ child_results["child_notify"] = await child_block("child_notify").execute_safe(
+ workflow_run_id="wr_child", organization_id="org_test"
+ )
+ with pytest.raises(RuntimeError), block_module.running_after_browser_loss("wr_nested"):
+ raise RuntimeError("a nested run's finally ends")
+ permission_after_child.append(block_module._finally_after_browser_loss_run_id.get())
+ permission_after_child.append(block_module._finally_after_browser_loss_run_id.get())
+
+ async def send(self: HttpRequestBlock, workflow_run_id: str, workflow_run_block_id: str, **_: Any) -> BlockResult:
+ sent.append(self.label)
+ if self.label == "trigger":
+ await run_child_inline()
+ return await self.build_block_result(
+ True, None, status=BlockStatus.completed, workflow_run_block_id=workflow_run_block_id
+ )
+
+ update_row = AsyncMock()
+ monkeypatch.setattr(app.DATABASE.observer, "create_workflow_run_block", create_row)
+ monkeypatch.setattr(app.DATABASE.observer, "update_workflow_run_block", update_row)
+ monkeypatch.setattr(HttpRequestBlock, "execute", send)
+ monkeypatch.setattr(ForLoopBlock, "get_loop_over_parameter_values", AsyncMock(return_value=["[email protected]"]))
+ for block_type in (ForLoopBlock, HttpRequestBlock):
+ monkeypatch.setattr(block_type, "record_output_parameter_value", AsyncMock())
+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", AsyncMock(return_value=None))
+ loop = ForLoopBlock(
+ label="finally_steps",
+ output_parameter=_output_parameter("finally_steps_output"),
+ loop_blocks=[child_block("trigger"), child_block("parent_notify")],
+ )
+ workflow = _workflow()
+ workflow.workflow_definition.finally_block_label = "finally_steps"
+ workflow.workflow_definition.blocks = [loop]
+
+ finally_execution = await WorkflowService()._execute_finally_block_if_configured(
+ workflow, _workflow_run(), MagicMock(), None
+ )
+
+ assert isinstance(finally_execution, tuple)
+ # The child's own loss refuses its next body effect; its own finally block and the parent's finally still run.
+ child_sent = ["child_first", "child_notify"] if child_lost else ["child_first", "child_email", "child_notify"]
+ assert sent == ["trigger", *child_sent, "parent_notify"]
+ email = child_results["child_email"]
+ assert email.success is not child_lost
+ if child_lost:
+ assert email.status == BlockStatus.failed and BROWSER_CONNECTION_LOST_ERROR_CODE in email.error_codes
+ assert email.can_continue_after_failure is False
+ assert child_results["child_notify"].success
+ assert permission_after_child == ["wr_child", "wr_test"]
+ assert block_module._finally_after_browser_loss_run_id.get() is None
+
+
@pytest.mark.asyncio
async def test_script_success_skips_agent_execution(monkeypatch: pytest.MonkeyPatch) -> None:
service = WorkflowService()