Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Fail the run at the first unexpected browser connection loss (SKY-18726) (#9022)
LawyZhenge9477af39eea
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.pyindex bee83c74c..63b7baec4 100644--- a/skyvern/core/script_generations/real_skyvern_page_ai.py+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py@@ -59,6 +59,7 @@ from skyvern.webeye.actions.handler import ()from skyvern.webeye.actions.parse_actions import parse_actionsfrom skyvern.webeye.actions.responses import ActionAbort, ActionResult+from skyvern.webeye.browser_state import raise_if_browser_connection_lostfrom skyvern.webeye.scraper.scraped_page import ScrapedPagejinja_sandbox_env = SandboxedEnvironment()@@ -191,9 +192,16 @@ class RealSkyvernPageAi(SkyvernPageAi):self.page = pageself.current_label: str | None = None+ def _raise_if_browser_connection_lost(self) -> None:+ context = skyvern_context.current()+ raise_if_browser_connection_lost(+ self.scraped_page._browser_state, workflow_run_id=context.workflow_run_id if context else None+ )+async def _refresh_scraped_page(self, take_screenshots: bool = True, max_retries: int = SKYVERN_PAGE_MAX_SCRAPING_RETRIES) -> None:+ self._raise_if_browser_connection_lost()self.scraped_page = await self.scraped_page.generate_scraped_page(take_screenshots=take_screenshots, max_retries=max_retries)@@ -372,6 +380,7 @@ class RealSkyvernPageAi(SkyvernPageAi):exc_info=True,)+ self._raise_if_browser_connection_lost()if selector:locator = self.page.locator(selector)await locator.click(timeout=timeout)@@ -395,6 +404,7 @@ class RealSkyvernPageAi(SkyvernPageAi):v3_parent_episode_id: str | None = None,) -> str:"""Input text into an element using AI to determine the value."""+ self._raise_if_browser_connection_lost()sensitive_value = value if value_is_sensitive else None# v3 mid-run hook — see ai_click for the contract.@@ -534,6 +544,7 @@ class RealSkyvernPageAi(SkyvernPageAi):except Exception:LOG.exception("Failed to adapt value for input text action", selector=selector)+ self._raise_if_browser_connection_lost()if action and organization_id and task and step:result = await handle_input_text_action_direct(action, self.page, self.scraped_page, task, step)if result and result[-1].success is False:@@ -658,6 +669,7 @@ class RealSkyvernPageAi(SkyvernPageAi):except Exception:LOG.exception(f"Failed to adapt value for upload file action on selector={selector}, file={files}")+ self._raise_if_browser_connection_lost()if action and original_files and action.file_url and action.file_url != original_files:LOG.warning("LLM returned a different file url than the user provided, using the original",@@ -757,6 +769,7 @@ class RealSkyvernPageAi(SkyvernPageAi):exc_info=True,)+ self._raise_if_browser_connection_lost()if not selector:raise SkyvernActionFailed(f"Select option failed and no fallback selector available for intention: {intention}"diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.pyindex 26ab7cc69..8589273de 100644--- a/skyvern/core/script_generations/script_skyvern_page.py+++ b/skyvern/core/script_generations/script_skyvern_page.py@@ -5,7 +5,7 @@ import inspectimport osimport reimport weakref-from datetime import datetime+from datetime import UTC, datetimefrom pathlib import Pathfrom typing import Any, Callable@@ -27,8 +27,10 @@ from skyvern.core.script_generations.skyvern_page import (from skyvern.core.script_generations.skyvern_page_ai import SkyvernPageAifrom skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codesfrom skyvern.exceptions import (+ BROWSER_CONNECTION_LOST_ERROR_CODE,BrowserSessionSwitchNotAllowed,IllegitCompleteScriptTermination,+ MissingBrowserState,ScriptTerminationException,WorkflowRunNotFound,)@@ -74,7 +76,7 @@ from skyvern.webeye.actions.handler import ()from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccessfrom skyvern.webeye.browser_engine import BrowserEngineSelection-from skyvern.webeye.browser_state import BrowserState+from skyvern.webeye.browser_state import BrowserState, get_browser_connection_loss, raise_if_browser_connection_lostfrom skyvern.webeye.navigation import default_navigation_settle, navigate_with_retry, redact_url_secretsfrom skyvern.webeye.scraper.scraped_page import ScrapedPagefrom skyvern.webeye.utils.challenge_solve_record import calling_solve_client@@ -145,6 +147,12 @@ class ScriptSkyvernPage(SkyvernPage):_set_scraped_page(self, scraped_page)self._record = recorder or (lambda ac: None)+ def _raise_if_browser_connection_lost(self) -> None:+ context = skyvern_context.current()+ raise_if_browser_connection_lost(+ _get_scraped_page(self)._browser_state, workflow_run_id=context.workflow_run_id if context else None+ )+@classmethodasync def _get_or_create_browser_state(cls, browser_session_id: str | None = None, url: str | None = None@@ -286,6 +294,7 @@ class ScriptSkyvernPage(SkyvernPage):Auto-creates action records in DB before action executionand screenshot artifacts after action execution."""+ self._raise_if_browser_connection_lost()prompt = kwargs.get("prompt", "")@@ -338,6 +347,7 @@ class ScriptSkyvernPage(SkyvernPage):# Wait for page to be ready before executing action# This helps prevent issues where cached actions execute before the page is fully loadedawait self._wait_for_page_ready_before_action()+ self._raise_if_browser_connection_lost()# NOTE: _ensure_element_ids_on_page() removed from here.# unique_id attrs are only needed by the AI fallback path, which# already calls _refresh_scraped_page() → build_tree_from_body()@@ -1115,6 +1125,16 @@ class ScriptSkyvernPage(SkyvernPage):last_error: Exception | None = None_get_scraped_page(self)._browser_state.begin_navigation()for attempt in range(max_retries):+ # The URL check and the retry backoff both await, and a loss latched meanwhile must not reach a dispatch.+ connection_loss = get_browser_connection_loss(_get_scraped_page(self)._browser_state)+ if connection_loss is not None:+ last_error = last_error or MissingBrowserState(+ workflow_run_id=context.workflow_run_id if context else None,+ diagnostic=connection_loss,+ detected_at=datetime.now(UTC),+ failure_reason=BROWSER_CONNECTION_LOST_ERROR_CODE,+ )+ breaktry:await self.page.goto(url, timeout=timeout, **kwargs)await _get_scraped_page(self)._browser_state.record_navigation_result(None, url=url)@@ -1123,7 +1143,10 @@ class ScriptSkyvernPage(SkyvernPage):returnexcept Exception as e:last_error = e- if attempt >= max_retries - 1:+ if (+ attempt >= max_retries - 1+ or get_browser_connection_loss(_get_scraped_page(self)._browser_state) is not None+ ):breakLOG.warning("Navigation attempt failed, retrying",@@ -1188,6 +1211,7 @@ class ScriptSkyvernPage(SkyvernPage):retry_times=NAVIGATION_MAX_RETRY_TIME,settle=default_navigation_settle,log_url=redact_url_secrets(polled.value),+ stop_retrying=lambda: get_browser_connection_loss(_get_scraped_page(self)._browser_state) is not None,)@action_wrap(ActionType.SOLVE_CAPTCHA)diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex f6f5e3634..fa4ede074 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -286,6 +286,10 @@ class SkyvernPage(Page):return ResolvedSensitiveValue(totp_code[digit_index])return ""+ def _raise_if_browser_connection_lost(self) -> None:+ # Only a run-bound page knows its run's browser; a standalone library page has no loss to observe.+ return None+@staticmethoddef _track_ai_call() -> None:"""Increment the script LLM call counter for cost-cap tracking."""@@ -346,6 +350,7 @@ class SkyvernPage(Page):retry_interval=retry_interval,)await asyncio.sleep(retry_interval)+ self._raise_if_browser_connection_lost()# Re-acquire locator in case the DOM was replaced entirelylocator = self._locator_scope.locator(selector).firstelse:@@ -511,6 +516,7 @@ class SkyvernPage(Page):except StaleFrameSelectionError:raiseexcept Exception as e:+ self._raise_if_browser_connection_lost()if is_post_dispatch_click_timeout(e, self.engine_selection):LOG.info("CSS selector click dispatched; navigation-wait timed out — skipping fallback",@@ -533,6 +539,7 @@ class SkyvernPage(Page):try:await self.page.keyboard.press("Escape")await asyncio.sleep(0.3)+ self._raise_if_browser_connection_lost()locator = self._locator_scope.locator(selector).first# The primary click may have dispatched and reached the desired state# before raising, in which case retrying would toggle it back off.@@ -545,6 +552,7 @@ class SkyvernPage(Page):)return selectorexcept Exception as retry_error:+ self._raise_if_browser_connection_lost()if is_post_dispatch_click_timeout(retry_error, self.engine_selection):LOG.info("CSS selector click dispatched after dismissing overlay; "@@ -1917,6 +1925,7 @@ class SkyvernPage(Page):return Truefor idx, value in sorted(mapping.items()):+ self._raise_if_browser_connection_lost()if idx >= len(form_fields) or value is None:continuetotp_placeholder = value if isinstance(value, str) and is_unresolved_totp_value(value) else None@@ -2202,6 +2211,7 @@ class SkyvernPage(Page):breakif uploaded:break+ self._raise_if_browser_connection_lost()async def validate_mapping(self,@@ -2405,6 +2415,7 @@ class SkyvernPage(Page):if page_num > 0:await asyncio.sleep(1)+ self._raise_if_browser_connection_lost()form_fields = await self.extract_form_fields()# Filter to fillable fields — includes standard inputs AND custom widgets@@ -2602,6 +2613,7 @@ class SkyvernPage(Page):except Exception:pass+ self._raise_if_browser_connection_lost()elapsed = time.monotonic() - start_timeLOG.info("fill_multipage_form: completed",diff --git a/skyvern/exceptions.py b/skyvern/exceptions.pyindex 8cf8e8d24..71526ba10 100644--- a/skyvern/exceptions.py+++ b/skyvern/exceptions.py@@ -551,6 +551,9 @@ def _browser_state_diagnostic_suffix(return suffix+BROWSER_CONNECTION_LOST_ERROR_CODE = "browser_connection_lost"++class MissingBrowserState(SkyvernException):def __init__(self,diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 01cbde791..417b9d217 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -52,6 +52,7 @@ from skyvern.errors.errors import (resolve_error_code_mapping_key,)from skyvern.exceptions import (+ BROWSER_CONNECTION_LOST_ERROR_CODE,BrowserSessionAlreadyOccupiedError,BrowserSessionClosed,BrowserSessionDegraded,@@ -347,7 +348,7 @@ from skyvern.webeye.browser_errors import (is_context_lost_message,is_target_closed_message,)-from skyvern.webeye.browser_state import BrowserState, get_browser_state_diagnostic+from skyvern.webeye.browser_state import BrowserState, get_browser_connection_loss, get_browser_state_diagnosticfrom skyvern.webeye.cdp_download_interceptor import (consume_unsolicited_download_error_for_context,download_filename_from_suffix,@@ -3449,7 +3450,11 @@ class ForgeAgent:block_type=block_type,)completion_rejection: str | None = None- if outcome.status == "completed":+ if outcome.status == "completed" and get_browser_connection_loss(browser_state) is not None:+ # The gates below fail open on a page that is gone, so a loss after the last action must fail here.+ completion_rejection = f"the run lost its browser connection ({BROWSER_CONNECTION_LOST_ERROR_CODE})"+ LOG.warning("task_v3 completion rejected after a browser connection loss", task_id=task.task_id)+ elif outcome.status == "completed":# The deployment gate for a completion the finish tool could not put to it (no headroom left to# act on a refusal, a re-ask conversion, or a gate error). The loop has returned, so a veto fails.gate_page = None@@ -5508,6 +5513,8 @@ class ForgeAgent:)if canceled is not None:return actions, canceled+ if get_browser_connection_loss(browser_state) is not None:+ return actions, await self._fail_step_unless_run_canceled(task, step, detailed_agent_step_output)context = skyvern_context.ensure_context()if context.refresh_working_page:@@ -5818,6 +5825,8 @@ class ForgeAgent:return actions, await self._fail_step_unless_run_canceled(task, step, detailed_agent_step_output)await artifact_tracker.drain()+ if get_browser_connection_loss(browser_state) is not None:+ return actions, await self._fail_step_unless_run_canceled(task, step, detailed_agent_step_output)# A cancel landing during the last action would otherwise reach extraction and completion.return actions, await self._stop_step_if_run_canceled(task, step, detailed_agent_step_output, skipped_actions=0)diff --git a/skyvern/forge/sdk/workflow/credential_fallback.py b/skyvern/forge/sdk/workflow/credential_fallback.pyindex 09d23b23e..9f908d8e0 100644--- a/skyvern/forge/sdk/workflow/credential_fallback.py+++ b/skyvern/forge/sdk/workflow/credential_fallback.py@@ -3,7 +3,7 @@ from __future__ import annotationsimport structlogfrom sqlalchemy.exc import IntegrityError-from skyvern.exceptions import WorkflowNotFound+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE, WorkflowNotFoundfrom skyvern.forge import appfrom skyvern.forge.failure_classifier import classify_from_failure_reasonfrom skyvern.forge.sdk.core import skyvern_context@@ -179,6 +179,12 @@ async def _maybe_start_credential_fallback_retry(workflow_run: WorkflowRun, orga]if not triggered_parameters:return None+ # Another credential cannot help a run whose browser went away under it.+ run_blocks = await app.DATABASE.observer.get_workflow_run_blocks(+ workflow_run_id=workflow_run.workflow_run_id, organization_id=organization_id+ )+ if any(BROWSER_CONNECTION_LOST_ERROR_CODE in run_block.error_codes for run_block in run_blocks):+ return Noneall_fallback_ids = list(dict.fromkeys(diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex ade78a30d..1ed36bd0f 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -29,6 +29,7 @@ import uuidimport zipfilefrom collections import defaultdict, dequefrom collections.abc import AsyncIterator, Iterator, Mapping, Sequence+from contextvars import ContextVarfrom dataclasses import dataclass, replacefrom datetime import UTC, date, datetime, timefrom email.message import EmailMessage@@ -94,6 +95,7 @@ from skyvern.constants import ()from skyvern.errors.errors import UserDefinedErrorfrom skyvern.exceptions import (+ BROWSER_CONNECTION_LOST_ERROR_CODE,AzureConfigurationError,BlockedHost,BlockedNavigationDestination,@@ -363,7 +365,12 @@ from skyvern.webeye.browser_driver_errors import is_driver_errorfrom skyvern.webeye.browser_engine import is_any_engine_errorfrom skyvern.webeye.browser_factory import rebind_download_dirfrom skyvern.webeye.browser_object_predicates import is_page_like-from skyvern.webeye.browser_state import BrowserState, get_browser_state_diagnostic+from skyvern.webeye.browser_state import (+ BrowserState,+ get_browser_connection_loss,+ get_browser_state_diagnostic,+ raise_if_browser_connection_lost,+)from skyvern.webeye.cdp_download_interceptor import (ORIGINAL_FILENAME_MARKER,ORIGINAL_FILENAME_TEMPLATE_VARIABLE,@@ -1318,6 +1325,14 @@ class Block(BaseModel, abc.ABC):# stopped it — reusing it makes the block's first page.goto raise "Connection closed# while reading from the driver". Rebuild a fresh connection to the same browser first.if browser_state is not None and not browser_state.is_connected():+ connection_loss = get_browser_connection_loss(browser_state)+ if connection_loss is not None:+ raise MissingBrowserState(+ workflow_run_id=workflow_run_id,+ diagnostic=connection_loss,+ detected_at=datetime.now(UTC),+ failure_reason=BROWSER_CONNECTION_LOST_ERROR_CODE,+ )workflow_run = await app.WORKFLOW_SERVICE.get_workflow_run(workflow_run_id=workflow_run_id,organization_id=organization_id,@@ -1862,6 +1877,11 @@ class Block(BaseModel, abc.ABC):organization_id=organization_id,can_continue_after_failure=False,)+ # A native loss can latch during the setup awaits above; only the finally route runs after one.+ if not _runs_after_browser_loss(workflow_run_id):+ raise_if_browser_connection_lost(+ app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id), workflow_run_id=workflow_run_id+ )LOG.info("Executing block",@@ -1884,6 +1904,9 @@ class Block(BaseModel, abc.ABC):await self._invalidate_stale_output_on_failure(workflow_run_id, current_index, include_missing_value_guard=False)+ result = await self._with_browser_connection_loss(+ result, workflow_run_id, workflow_run_block_id, organization_id+ )return resultexcept Exception as e:can_continue_after_failure: bool | None = None@@ -1953,6 +1976,14 @@ class Block(BaseModel, abc.ABC):nav_run_context = Noneif nav_run_context is not None and not is_registered_secret(e.nav_error_code, nav_run_context):error_codes = [*error_codes, e.nav_error_code]+ # The run's browser is known lost, so this failure is terminal whatever exception surfaced it. Inside the+ # finally block only a refused browser use is; the body's loss does not end its ordinary failures.+ browser_lost = _run_lost_its_browser(workflow_run_id) and (+ not _runs_after_browser_loss(workflow_run_id)+ or isinstance(e, (MissingBrowserState, MissingBrowserStatePage))+ )+ if browser_lost:+ error_codes = [*error_codes, BROWSER_CONNECTION_LOST_ERROR_CODE]return await self.build_block_result(success=False,@@ -1961,12 +1992,38 @@ class Block(BaseModel, abc.ABC):workflow_run_block_id=workflow_run_block_id,organization_id=organization_id,error_codes=error_codes or None,- can_continue_after_failure=can_continue_after_failure,+ can_continue_after_failure=False if browser_lost else can_continue_after_failure,)finally:if download_timeout_context is not None:download_timeout_context.download_timeout = prev_download_timeout+ async def _with_browser_connection_loss(+ self,+ result: BlockResult,+ workflow_run_id: str,+ workflow_run_block_id: str | None,+ organization_id: str | None,+ ) -> BlockResult:+ # A block that returns its outcome instead of raising it never reaches the exception handler above. After the+ # loss any unsuccessful outcome but a cancellation fails the run, so a terminated task cannot let it continue.+ if (+ _runs_after_browser_loss(workflow_run_id)+ or not _run_lost_its_browser(workflow_run_id)+ or result.success+ or result.status == BlockStatus.canceled+ ):+ return result+ error_codes = list(dict.fromkeys([*result.error_codes, BROWSER_CONNECTION_LOST_ERROR_CODE]))+ if workflow_run_block_id and (result.status != BlockStatus.failed or error_codes != result.error_codes):+ await app.DATABASE.observer.update_workflow_run_block(+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ status=BlockStatus.failed,+ error_codes=error_codes,+ )+ return replace(result, status=BlockStatus.failed, error_codes=error_codes, can_continue_after_failure=False)+@abc.abstractmethoddef get_all_parameters(self,@@ -1975,6 +2032,30 @@ class Block(BaseModel, abc.ABC):pass+# The finally block's dispatch task copies this context. It names the run whose finally block it is, because a+# synchronous child workflow started there runs inline in the same context and keeps its own body guards.+_finally_after_browser_loss_run_id: ContextVar[str | None] = ContextVar(+ "finally_after_browser_loss_run_id", default=None+)+++def running_after_browser_loss(workflow_run_id: str) -> Iterator[None]:+ token = _finally_after_browser_loss_run_id.set(workflow_run_id)+ try:+ yield+ finally:+ _finally_after_browser_loss_run_id.reset(token)+++def _runs_after_browser_loss(workflow_run_id: str) -> bool:+ return _finally_after_browser_loss_run_id.get() == workflow_run_id+++def _run_lost_its_browser(workflow_run_id: str) -> bool:+ return get_browser_connection_loss(app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)) is not None++def _is_retry_blocking_anti_bot_entry(category: dict) -> bool:return (category.get("category") == "ANTI_BOT_DETECTION"@@ -2469,8 +2550,9 @@ class BaseTaskBlock(Block):# TODO (kerem) we should always retry on terminated. We should make a distinction between retriable and# non-retryable terminations+ next_task_order: tuple[int, int] | None = Nonewhile will_retry:- task_order, task_retry = await self.get_task_order(workflow_run_id, current_retry)+ task_order, task_retry = next_task_order or await self.get_task_order(workflow_run_id, current_retry)is_first_task = task_order == 0task, step = await app.agent.create_task_and_step_from_block(task_block=self,@@ -2834,7 +2916,6 @@ class BaseTaskBlock(Block):failure_category=updated_task.failure_category,)will_retry = False- retry_message = f", retrying task {current_retry}/{self.max_retries}" if will_retry else ""downloaded_files = Nonetry:async with asyncio.timeout(GET_DOWNLOADED_FILES_TIMEOUT):@@ -2864,6 +2945,13 @@ class BaseTaskBlock(Block):workflow_run_id=workflow_run_id,organization_id=workflow_run.organization_id,)+ if will_retry:+ next_task_order = await self.get_task_order(workflow_run_id, current_retry)+ # The retry is admitted only after every await above, so a loss during one ends the block here.+ browser_lost = _run_lost_its_browser(workflow_run_id)+ if browser_lost:+ will_retry = False+ retry_message = f", retrying task {current_retry}/{self.max_retries}" if will_retry else ""task_output = TaskOutput.from_task(updated_task,@@ -2887,6 +2975,9 @@ class BaseTaskBlock(Block):await self.record_output_parameter_value(workflow_run_context, workflow_run_id, output_parameter_value)+ error_codes = _recorded_task_nav_error_codes(updated_task.task_id, workflow_run_context) or []+ if browser_lost:+ error_codes.append(BROWSER_CONNECTION_LOST_ERROR_CODE)return await self.build_block_result(success=False,failure_reason=(@@ -2897,7 +2988,8 @@ class BaseTaskBlock(Block):status=block_status_mapping[updated_task.status],workflow_run_block_id=workflow_run_block_id,organization_id=organization_id,- error_codes=_recorded_task_nav_error_codes(updated_task.task_id, workflow_run_context),+ error_codes=error_codes or None,+ can_continue_after_failure=False if browser_lost else None,)await self.record_output_parameter_value(workflow_run_context, workflow_run_id)@@ -5937,6 +6029,7 @@ def _bind_code_block_magic_link(retry_times=NAVIGATION_MAX_RETRY_TIME,settle=default_navigation_settle,log_url=redact_url_secrets(link),+ stop_retrying=lambda: workflow_run_id is not None and _run_lost_its_browser(workflow_run_id),)return magic_link@@ -6579,6 +6672,7 @@ def _bind_code_block_open_page(open_tabs = await open_page_tab_count(_raw_code_block_page(page), browser_state)if open_page_cap_reached(open_tabs):raise RuntimeError(open_page_cap_error(open_tabs))+ raise_if_browser_connection_lost(browser_state)opened: Page | RecordingPageif isinstance(page, RecordingPage):opened = page._wrap_page(await page._underlying_page.context.new_page())@@ -9247,6 +9341,13 @@ async def wrapper({default_args}):final_url=staged_failure_url,)+ if (+ get_browser_connection_loss(+ browser_state or app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id=workflow_run_id)+ )+ is not None+ ):+ return await _finalize_heal_result(None)if not classification.healable:return await _finalize_heal_result(None)if not await self._ai_fallback_enabled(workflow_run_context):@@ -9418,13 +9519,16 @@ async def wrapper({default_args}):) -> BlockResult:propagated_error: BaseExceptiontry:- return await self._execute(+ result = await self._execute(workflow_run_id,workflow_run_block_id,organization_id=organization_id,browser_session_id=browser_session_id,**kwargs,)+ return await self._with_browser_connection_loss(+ result, workflow_run_id, workflow_run_block_id, organization_id+ )except BaseException as exc:if isinstance(exc, (MissingBrowserState, MissingBrowserStatePage)):propagated_error = exc.with_traceback(None)@@ -16870,13 +16974,20 @@ class TaskV2Block(Block):"workflow_screenshot_artifact_ids": [a.artifact_id for a in workflow_screenshot_artifacts],}await self.record_output_parameter_value(workflow_run_context, workflow_run_id, task_v2_output)+ # An explicit persistent session registers the child's browser under the child run only.+ browser_lost = not success and (+ _run_lost_its_browser(workflow_run_id)+ or (task_v2_workflow_run_id is not None and _run_lost_its_browser(task_v2_workflow_run_id))+ )return await self.build_block_result(- success=success or self.continue_on_failure,+ success=success or (self.continue_on_failure and not browser_lost),failure_reason=failure_reason,output_parameter_value=task_v2_output,status=block_status,workflow_run_block_id=workflow_run_block_id,organization_id=organization_id,+ error_codes=[BROWSER_CONNECTION_LOST_ERROR_CODE] if browser_lost else None,+ can_continue_after_failure=False if browser_lost else None,)diff --git a/skyvern/forge/sdk/workflow/retry_policy.py b/skyvern/forge/sdk/workflow/retry_policy.pyindex a0080e378..d697cfa25 100644--- a/skyvern/forge/sdk/workflow/retry_policy.py+++ b/skyvern/forge/sdk/workflow/retry_policy.py@@ -10,7 +10,7 @@ import structlogfrom sqlalchemy import updatefrom sqlalchemy.exc import IntegrityError-from skyvern.exceptions import WorkflowNotFound, WorkflowNotFoundForWorkflowRun+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE, WorkflowNotFound, WorkflowNotFoundForWorkflowRunfrom skyvern.forge import appfrom skyvern.forge.sdk.cache import extraction_cachefrom skyvern.forge.sdk.db.datetime_utils import naive_utc_now, to_naive_utc@@ -121,6 +121,8 @@ def evaluate_retry_policy() -> tuple[bool, str]:if status is WorkflowRunStatus.canceled:return False, "canceled"+ if BROWSER_CONNECTION_LOST_ERROR_CODE in error_codes:+ return False, BROWSER_CONNECTION_LOST_ERROR_CODEif retries_used >= policy.max_retries:return False, "budget_exhausted"diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 6434c7738..5484aa537 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -52,6 +52,7 @@ from skyvern.constants import ()from skyvern.errors.errors import drop_unsolicited_download_errorsfrom skyvern.exceptions import (+ BROWSER_CONNECTION_LOST_ERROR_CODE,BlockedHost,BlockNotFound,BrowserActionPolicyNotEnforceable,@@ -68,6 +69,7 @@ from skyvern.exceptions import (InProcessScriptExecutionDenied,InvalidCredentialId,InvalidWorkflowParameter,+ MissingBrowserState,MissingValueForParameter,ScriptTerminationException,SequentialCredentialLimitExceeded,@@ -226,6 +228,7 @@ from skyvern.forge.sdk.workflow.models.block import (get_all_blocks,pinned_v1_block_count,resolve_conditional_merge_edges,+ running_after_browser_loss,takes_default_engine,v3_ab_ineligibility_reason,)@@ -420,7 +423,7 @@ from skyvern.utils.url_validators import validate_webhook_urlfrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.actions.actions import Action, InputTextAction, VerificationCodeActionfrom skyvern.webeye.actions.responses import ActionResult-from skyvern.webeye.browser_state import BrowserState+from skyvern.webeye.browser_state import BrowserState, get_browser_connection_lossfrom skyvern.webeye.persistent_session_errors import BrowserSessionCreditAdmissionRefusalfrom skyvern.webeye.persistent_sessions_manager import BROWSER_RETIREMENT_DENIED_NOTEfrom skyvern.webeye.profile_cookie_merge import cookie_delta, seed_cookie_values, union_cookies_into_profile_dir@@ -8567,10 +8570,9 @@ class WorkflowService:page_text_snapshot = Noneworking_page = Nonetry:- browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(- workflow_run=workflow_run,- )- working_page = await browser_state.get_working_page()+ # A diagnostic read: acquiring here would open a browser or take over one another run holds.+ browser_state = app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id=workflow_run.workflow_run_id)+ working_page = await browser_state.get_working_page() if browser_state else Noneif working_page:page_url = working_page.urlpage_text_snapshot = (await working_page.inner_text("body"))[:5000]@@ -9047,6 +9049,15 @@ class WorkflowService:)return workflow_run, blocks_to_update, workflow_run_block_result, True, branch_metadata+ # A unit that completed after the run lost its browser natively cannot complete the body.+ if (+ workflow_run_block_result.success+ and get_browser_connection_loss(app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)) is not None+ ):+ workflow_run_block_result = await self._browser_lost_script_block_result(+ block, workflow_run_id, organization_id, attempt, parent_workflow_run_block_id+ )+self._track_blocks_for_script_generation(workflow=workflow,workflow_run=workflow_run,@@ -9496,6 +9507,58 @@ class WorkflowService:script_exception=script_exception,)+ async def _browser_lost_script_block_result(+ self,+ block: BlockTypeVar,+ workflow_run_id: str,+ organization_id: str,+ attempt: ScriptBlockAttempt,+ parent_workflow_run_block_id: str | None = None,+ ) -> BlockResult:+ attempt_number = app.WORKFLOW_CONTEXT_MANAGER.get_attempt_number(workflow_run_id)+ workflow_run_blocks = await app.DATABASE.observer.get_workflow_run_blocks(+ workflow_run_id=workflow_run_id, organization_id=organization_id+ )+ matching_blocks = [+ b+ for b in workflow_run_blocks+ if b.label == block.label+ and (b.attempt_number or 1) == attempt_number+ and b.parent_workflow_run_block_id == parent_workflow_run_block_id+ ]+ latest_block = max(matching_blocks, key=lambda b: b.created_at) if matching_blocks else None+ if attempt.block_result is not None:+ latest_block = next(+ (b for b in matching_blocks if b.workflow_run_block_id == attempt.block_result.workflow_run_block_id),+ latest_block,+ )+ if latest_block is None:+ latest_block = await app.DATABASE.observer.create_workflow_run_block(+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ attempt_number=attempt_number,+ parent_workflow_run_block_id=parent_workflow_run_block_id,+ label=block.label,+ block_type=block.block_type,+ continue_on_failure=block.continue_on_failure,+ )+ failure_reason = latest_block.failure_reason+ if failure_reason is None and attempt.script_exception is not None:+ failure_reason = str(attempt.script_exception)+ if failure_reason is None:+ failure_reason = get_user_facing_exception_message(MissingBrowserState(workflow_run_id=workflow_run_id))+ error_codes = latest_block.error_codes+ return await block.build_block_result(+ success=False,+ failure_reason=failure_reason,+ output_parameter_value=latest_block.output,+ status=BlockStatus.failed,+ workflow_run_block_id=latest_block.workflow_run_block_id,+ organization_id=organization_id,+ error_codes=list(dict.fromkeys([*error_codes, BROWSER_CONNECTION_LOST_ERROR_CODE])),+ can_continue_after_failure=False,+ )+async def _execute_block_via_agent_if_allowed(self,*,@@ -9509,6 +9572,19 @@ class WorkflowService:script_blocks_by_label: dict[str, Any],attempt: ScriptBlockAttempt,) -> tuple[BlockResult | None, bool] | WorkflowRunDispatchStopped:+ if (+ attempt.valid_to_run_code+ and get_browser_connection_loss(app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)) is not None+ ):+ LOG.warning(+ "Browser connection lost: skipping agent fallback for the failed script block", block_label=block.label+ )+ return (+ await self._browser_lost_script_block_result(+ block, workflow_run_id, organization_id, attempt, parent_workflow_run_block_id+ ),+ False,+ )workflow_run_block_result = attempt.block_result# Check if this block is designated as requires_agent by the script reviewer.# These blocks must execute via agent even when ai_fallback=False.@@ -10687,14 +10763,15 @@ class WorkflowService:await app.WORKFLOW_CONTEXT_MANAGER.register_block_parameters_for_workflow_run(workflow_run.workflow_run_id, parameters, organization)- finally_execution = await self._dispatch_workflow_run_block(- workflow_run.workflow_run_id,- lambda: block.execute_safe(- workflow_run_id=workflow_run.workflow_run_id,- organization_id=organization.organization_id,- browser_session_id=browser_session_id,- ),- )+ with running_after_browser_loss(workflow_run.workflow_run_id):+ finally_execution = await self._dispatch_workflow_run_block(+ workflow_run.workflow_run_id,+ lambda: block.execute_safe(+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=organization.organization_id,+ browser_session_id=browser_session_id,+ ),+ )if isinstance(finally_execution, WorkflowRunDispatchStopped):return finally_executionblock_result = finally_executiondiff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 4806dbcf3..15237afc2 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -30,10 +30,12 @@ from skyvern.core.script_generations.generate_script import _build_block_fn, crefrom skyvern.core.script_generations.script_skyvern_page import script_run_context_managerfrom skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codesfrom skyvern.exceptions import (+ BROWSER_CONNECTION_LOST_ERROR_CODE,CachedDownloadError,DownloadSaveIncompleteError,IllegitCompleteScriptTermination,InProcessScriptExecutionDenied,+ MissingBrowserState,ScriptNotFound,ScriptTerminationException,StepTerminationError,@@ -131,6 +133,7 @@ from skyvern.utils.templating import reject_jinja_transformations_on_variablefrom skyvern.utils.url_validators import validate_fetch_urlfrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.actions.actions import Action, DecisiveAction, reasoning_is_turn_scoped+from skyvern.webeye.browser_state import get_browser_connection_loss, raise_if_browser_connection_lostfrom skyvern.webeye.cdp_download_interceptor import (ORIGINAL_FILENAME_MARKER,ORIGINAL_FILENAME_TEMPLATE_VARIABLE,@@ -1126,6 +1129,15 @@ async def _update_workflow_block(if run_lease_tasks_enabled() and await run_lease_lost(context.workflow_run_id):return+ # User-defined codes stay on the task row (run details read both rows), but the lost-browser code has no+ # task record, so a failed row carries it for workflow retry and credential fallback.+ browser_connection_lost = (+ status == BlockStatus.failed+ and get_browser_connection_loss(app.BROWSER_MANAGER.get_for_workflow_run(context.workflow_run_id))+ is not None+ )+ if browser_connection_lost:+ error_codes = list(dict.fromkeys([*(error_codes or []), BROWSER_CONNECTION_LOST_ERROR_CODE]))updated_block = await app.DATABASE.observer.update_workflow_run_block(workflow_run_block_id=workflow_run_block_id,organization_id=context.organization_id if context else None,@@ -1133,6 +1145,7 @@ async def _update_workflow_block(failure_reason=failure_reason,output=final_output,ai_fallback_triggered=ai_fallback_triggered,+ error_codes=[BROWSER_CONNECTION_LOST_ERROR_CODE] if browser_connection_lost else None,)# The row carries the block label even when the caller passed none (a cached wait block).@@ -1175,7 +1188,36 @@ async def _update_workflow_block()+def _raise_if_browser_connection_lost(outcome: BlockResult | BlockStatus | None) -> None:+ # Generated statements and loop iterations run in sequence, so a child that failed after the run lost its+ # browser stops the script here; its outcome is already published.+ context = skyvern_context.current()+ if not context or not context.workflow_run_id:+ return+ connection_loss = get_browser_connection_loss(app.BROWSER_MANAGER.get_for_workflow_run(context.workflow_run_id))+ if connection_loss is None:+ return+ status = outcome.status if isinstance(outcome, BlockResult) else outcome+ if status == BlockStatus.failed:+ raise MissingBrowserState(+ workflow_run_id=context.workflow_run_id,+ diagnostic=connection_loss,+ detected_at=datetime.now(timezone.utc),+ failure_reason=BROWSER_CONNECTION_LOST_ERROR_CODE,+ )+++def _refuse_statement_after_browser_connection_loss() -> None:+ # A loss can latch between statements, with no browser call in flight; the next statement must not start.+ context = skyvern_context.current()+ if context and context.workflow_run_id:+ raise_if_browser_connection_lost(+ app.BROWSER_MANAGER.get_for_workflow_run(context.workflow_run_id), workflow_run_id=context.workflow_run_id+ )++async def _run_cached_function(cached_fn: Callable) -> Any:+ _refuse_statement_after_browser_connection_loss()run_context = script_run_context_manager.ensure_run_context()return await cached_fn(page=run_context.page, context=run_context)@@ -1531,7 +1573,7 @@ async def _fallback_to_ai_run(include_action_history_in_verification: bool = False,error: Exception | None = None,workflow_run_block_id: str | None = None,-) -> None:+) -> BlockStatus | None:context = skyvern_context.current()if not (context@@ -1541,7 +1583,7 @@ async def _fallback_to_ai_run(and context.task_idand context.step_id):- return+ return Noneorganization_id = context.organization_idworkflow_id = context.workflow_idworkflow_run_id = context.workflow_run_id@@ -1564,6 +1606,21 @@ async def _fallback_to_ai_run(organization_id=organization_id,status=StepStatus.failed,)+ if get_browser_connection_loss(app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)) is not None:+ LOG.warning("Browser connection lost: script failure will not be retried by agent", cache_key=cache_key)+ if workflow_run_block_id:+ await _update_workflow_block(+ workflow_run_block_id,+ BlockStatus.failed,+ task_id=task_id,+ task_status=TaskStatus.failed,+ failure_reason=str(error),+ step_id=script_step_id,+ step_status=StepStatus.failed,+ label=cache_key,+ error_codes=[BROWSER_CONNECTION_LOST_ERROR_CODE],+ )+ return BlockStatus.failed# 2. run execute_steporganization = await app.DATABASE.organizations.get_organization(organization_id=organization_id)if not organization:@@ -1575,12 +1632,12 @@ async def _fallback_to_ai_run(workflow_id=context.workflow_id, organization_id=organization_id)if not workflow:- return+ return Noneworkflow_run = await app.DATABASE.workflow_runs.get_workflow_run(workflow_run_id=workflow_run_id, organization_id=organization_id)if not workflow_run:- return+ return None# Use workflow_run.ai_fallback if explicitly set, otherwise fall back to workflow.ai_fallbackeffective_ai_fallback = (workflow_run.ai_fallback if workflow_run.ai_fallback is not None else workflow.ai_fallback@@ -1644,7 +1701,7 @@ async def _fallback_to_ai_run(label=cache_key,error_codes=error_codes,)- return+ return BlockStatus.failed# Record a fallback episode for adaptive caching (code_v2).# This must happen BEFORE the AI step runs so we capture the page state@@ -1660,10 +1717,9 @@ async def _fallback_to_ai_run(page_text_snapshot = Noneworking_page = Nonetry:- browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(- workflow_run=workflow_run,- )- working_page = await browser_state.get_working_page()+ # A diagnostic read: acquiring here would open a browser or take over one another run holds.+ browser_state = app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id=workflow_run_id)+ working_page = await browser_state.get_working_page() if browser_state else Noneif working_page:page_url = working_page.urlpage_text_snapshot = (await working_page.inner_text("body", timeout=5000))[:5000]@@ -1822,6 +1878,7 @@ async def _fallback_to_ai_run()# Update block status to completed if workflow block was created+ fallback_status: BlockStatus | None = Noneif workflow_run_block_id:# refresh the taskfailure_reason = None@@ -1835,9 +1892,10 @@ async def _fallback_to_ai_run(for error in task.errors or []if isinstance(error, dict) and isinstance(error.get("error_code"), str)]+ fallback_status = BlockStatus(task.status.value)await _update_workflow_block(workflow_run_block_id,- BlockStatus(task.status.value),+ fallback_status,task_id=task_id,failure_reason=failure_reason,label=cache_key,@@ -1932,6 +1990,7 @@ async def _fallback_to_ai_run(episode_id=fallback_episode_id,exc_info=True,)+ return fallback_statusexcept Exception as e:LOG.warning("Failed to fallback to AI run", cache_key=cache_key, exc_info=True)# Update block status to failed if workflow block was created@@ -2370,6 +2429,7 @@ async def run_task(context: skyvern_context.SkyvernContext | None = Noneif cache_key and cached_fn:+ _refuse_statement_after_browser_connection_loss()# Auto-create workflow block run and task if workflow_run_id is available.# Use `label` (the workflow block label) for the block run so the# framework can match it, and `cache_key` to look up the cached function.@@ -2416,7 +2476,7 @@ async def run_task(raiseexcept Exception as e:LOG.warning("Failed to run task block. Falling back to AI run.", exc_info=True)- await _fallback_to_ai_run(+ fallback_status = await _fallback_to_ai_run(block_type=BlockType.NAVIGATION,cache_key=cache_key,prompt=prompt,@@ -2428,6 +2488,7 @@ async def run_task(workflow_run_block_id=workflow_run_block_id,error_code_mapping=error_code_mapping,)+ _raise_if_browser_connection_lost(fallback_status)return Nonefinally:# clear the prompt in the RunContext@@ -2458,6 +2519,7 @@ async def run_task(label,output_parameter=block_validation_output.output_parameter,)+ _raise_if_browser_connection_lost(block_output)return block_output.output_parameter_value@@ -2566,6 +2628,7 @@ async def download(cached_fn = script_run_context_manager.get_cached_fn(cache_key)context: skyvern_context.SkyvernContext | Noneif cache_key and cached_fn:+ _refuse_statement_after_browser_connection_loss()# Auto-create workflow block run and task if workflow_run_id is availableworkflow_run_block_id, task_id, step_id = await _create_workflow_block_run_and_task(block_type=BlockType.FILE_DOWNLOAD,@@ -3009,7 +3072,7 @@ async def download(except OSError:continuefallback_file_signatures_before[file_path] = (file_stat.st_mtime_ns, file_stat.st_size)- await _fallback_to_ai_run(+ fallback_status = await _fallback_to_ai_run(block_type=BlockType.FILE_DOWNLOAD,cache_key=cache_key,prompt=navigation_prompt,@@ -3021,6 +3084,7 @@ async def download(workflow_run_block_id=workflow_run_block_id,error_code_mapping=error_code_mapping,)+ _raise_if_browser_connection_lost(fallback_status)if file_download_block is not None and storage_type is not None:fallback_files_after = current_attempt_local_files(fallback_download_dir)fallback_downloaded_files = []@@ -3098,6 +3162,7 @@ async def download(browser_session_id=block_validation_output.browser_session_id,)if not download_result.success:+ _raise_if_browser_connection_lost(download_result)# Halt the script on a failed download/delivery instead of ignoring the result, matching# the cached path; otherwise later generated statements would run after the failure.raise ScriptTerminationException(download_result.failure_reason or "File download block failed")@@ -3119,6 +3184,7 @@ async def action(cache_key = cache_key or labelcached_fn = script_run_context_manager.get_cached_fn(cache_key)if cache_key and cached_fn:+ _refuse_statement_after_browser_connection_loss()# Auto-create workflow block run and task if workflow_run_id is availableworkflow_run_block_id, task_id, step_id = await _create_workflow_block_run_and_task(block_type=BlockType.ACTION,@@ -3154,7 +3220,7 @@ async def action(raiseexcept Exception as e:LOG.warning("Failed to run action block. Falling back to AI run.", exc_info=True)- await _fallback_to_ai_run(+ fallback_status = await _fallback_to_ai_run(block_type=BlockType.ACTION,cache_key=cache_key,prompt=prompt,@@ -3167,6 +3233,7 @@ async def action(workflow_run_block_id=workflow_run_block_id,error_code_mapping=error_code_mapping,)+ _raise_if_browser_connection_lost(fallback_status)finally:context.prompt = None_clear_cached_block_overrides(cache_key)@@ -3185,12 +3252,13 @@ async def action(model=model,download_suffix=download_suffix,)- await action_block.execute_safe(+ action_result = await action_block.execute_safe(workflow_run_id=block_validation_output.workflow_run_id,parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,organization_id=block_validation_output.organization_id,browser_session_id=block_validation_output.browser_session_id,)+ _raise_if_browser_connection_lost(action_result)async def login(@@ -3208,6 +3276,7 @@ async def login(cache_key = cache_key or labelcached_fn = script_run_context_manager.get_cached_fn(cache_key)if cache_key and cached_fn:+ _refuse_statement_after_browser_connection_loss()# Auto-create workflow block run and task if workflow_run_id is available# render template with labelprompt = _render_template_with_label(prompt, cache_key) if prompt else prompt@@ -3248,7 +3317,7 @@ async def login(raiseexcept Exception as e:LOG.exception("Failed to run login block")- await _fallback_to_ai_run(+ fallback_status = await _fallback_to_ai_run(block_type=BlockType.LOGIN,cache_key=cache_key,prompt=prompt,@@ -3260,6 +3329,7 @@ async def login(workflow_run_block_id=workflow_run_block_id,error_code_mapping=error_code_mapping,)+ _raise_if_browser_connection_lost(fallback_status)finally:context.prompt = None_clear_cached_block_overrides(cache_key)@@ -3277,12 +3347,13 @@ async def login(model=model,)login_block._built_by_script = True- await login_block.execute_safe(+ login_result = await login_block.execute_safe(workflow_run_id=block_validation_output.workflow_run_id,parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,organization_id=block_validation_output.organization_id,browser_session_id=block_validation_output.browser_session_id,)+ _raise_if_browser_connection_lost(login_result)async def extract(@@ -3300,6 +3371,7 @@ async def extract(cache_key = cache_key or labelcached_fn = script_run_context_manager.get_cached_fn(cache_key)if cache_key and cached_fn:+ _refuse_statement_after_browser_connection_loss()# Auto-create workflow block run and task if workflow_run_id is availableworkflow_run_block_id, task_id, step_id = await _create_workflow_block_run_and_task(block_type=BlockType.EXTRACTION,@@ -3368,6 +3440,7 @@ async def extract(label,output_parameter=block_validation_output.output_parameter,)+ _raise_if_browser_connection_lost(block_result)return block_result.output_parameter_value@@ -3382,6 +3455,7 @@ async def conditional(cached_fn = script_run_context_manager.get_cached_fn(label)if not cached_fn:raise Exception(f"No cached function for conditional block '{label}'")+ _refuse_statement_after_browser_connection_loss()# Create workflow run block entry (no task needed for conditional blocks)workflow_run_block_id, _, _ = await _create_workflow_block_run_and_task(@@ -3400,6 +3474,7 @@ async def conditional(run_context.parameters[key] = valuetry:+ _refuse_statement_after_browser_connection_loss()result = await cached_fn(page=run_context.page, context=run_context)if not isinstance(result, dict) or "next_block_label" not in result:raise Exception(f"Conditional function '{label}' must return dict with 'next_block_label', got: {result}")@@ -3445,6 +3520,7 @@ async def validate(result = await execute_validation(complete_criterion, terminate_criterion, error_code_mapping, label, model)if result.status == BlockStatus.terminated:raise ScriptTerminationException(result.failure_reason)+ _raise_if_browser_connection_lost(result)async def execute_validation(@@ -3477,10 +3553,12 @@ async def execute_validation(label,output_parameter=block_validation_output.output_parameter,)+ _raise_if_browser_connection_lost(result)return resultasync def wait(seconds: int, label: str | None = None) -> None:+ _refuse_statement_after_browser_connection_loss()# Auto-create workflow block run if workflow_run_id is available (wait block doesn't create tasks)workflow_run_block_id, _, _ = await _create_workflow_block_run_and_task(block_type=BlockType.WAIT, label=label)@@ -3747,6 +3825,7 @@ class BlockValidationOutput:async def _validate_and_get_output_parameter(label: str | None = None, parameter_keys: list[str] | None = None) -> BlockValidationOutput:+ _refuse_statement_after_browser_connection_loss()context = skyvern_context.ensure_context()workflow_id = context.workflow_idworkflow_run_id = context.workflow_run_id@@ -3819,6 +3898,7 @@ async def run_code(label,output_parameter=block_validation_output.output_parameter,)+ _raise_if_browser_connection_lost(block_result)return cast(dict[str, Any], block_result.output_parameter_value)@@ -3912,12 +3992,13 @@ async def upload_file(prompt=prompt,path=path,)- await file_upload_block.execute_safe(+ upload_result = await file_upload_block.execute_safe(workflow_run_id=block_validation_output.workflow_run_id,parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,organization_id=block_validation_output.organization_id,browser_session_id=block_validation_output.browser_session_id,)+ _raise_if_browser_connection_lost(upload_result)async def _send_email_via_gmail(block_validation_output: BlockValidationOutput, credential_id: str | None) -> None:@@ -3947,6 +4028,7 @@ async def _send_email_via_gmail(block_validation_output: BlockValidationOutput,block_validation_output.label,output_parameter=block_validation_output.output_parameter,)+ _raise_if_browser_connection_lost(result)if not result.success and not send_email_block.continue_on_failure:raise Exception(result.failure_reason or "Gmail send failed")@@ -4038,12 +4120,13 @@ async def send_email(output_parameter=block_validation_output.output_parameter,parameters=block_validation_output.input_parameters,)- await send_email_block.execute_safe(+ send_email_result = await send_email_block.execute_safe(workflow_run_id=block_validation_output.workflow_run_id,parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,organization_id=block_validation_output.organization_id,browser_session_id=block_validation_output.browser_session_id,)+ _raise_if_browser_connection_lost(send_email_result)async def parse_pdf(@@ -4061,12 +4144,13 @@ async def parse_pdf(output_parameter=block_validation_output.output_parameter,parameters=block_validation_output.input_parameters,)- await pdf_parser_block.execute_safe(+ parse_pdf_result = await pdf_parser_block.execute_safe(workflow_run_id=block_validation_output.workflow_run_id,parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,organization_id=block_validation_output.organization_id,browser_session_id=block_validation_output.browser_session_id,)+ _raise_if_browser_connection_lost(parse_pdf_result)async def parse_file(@@ -4094,12 +4178,13 @@ async def parse_file(parameters=block_validation_output.input_parameters,model=model,)- await file_parser_block.execute_safe(+ parse_file_result = await file_parser_block.execute_safe(workflow_run_id=block_validation_output.workflow_run_id,parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,organization_id=block_validation_output.organization_id,browser_session_id=block_validation_output.browser_session_id,)+ _raise_if_browser_connection_lost(parse_file_result)async def http_request(@@ -4128,12 +4213,13 @@ async def http_request(output_parameter=block_validation_output.output_parameter,parameters=block_validation_output.input_parameters,)- await http_request_block.execute_safe(+ http_request_result = await http_request_block.execute_safe(workflow_run_id=block_validation_output.workflow_run_id,parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,organization_id=block_validation_output.organization_id,browser_session_id=block_validation_output.browser_session_id,)+ _raise_if_browser_connection_lost(http_request_result)async def goto(@@ -4150,7 +4236,7 @@ async def goto(output_parameter=block_validation_output.output_parameter,parameters=block_validation_output.input_parameters,)- await goto_url_block.execute_safe(+ goto_result = await goto_url_block.execute_safe(workflow_run_id=block_validation_output.workflow_run_id,parent_workflow_run_block_id=block_validation_output.context.parent_workflow_run_block_id,organization_id=block_validation_output.organization_id,@@ -4164,6 +4250,8 @@ async def goto(candidate_url = await asyncio.to_thread(validate_fetch_url, candidate_url)run_context = script_run_context_manager.ensure_run_context()await run_context.page.goto(candidate_url)+ else:+ _raise_if_browser_connection_lost(goto_result)async def trigger_workflow(@@ -4207,6 +4295,7 @@ async def trigger_workflow(label,output_parameter=block_validation_output.output_parameter,)+ _raise_if_browser_connection_lost(result)return result.output_parameter_value@@ -4238,6 +4327,7 @@ async def prompt(label,output_parameter=block_validation_output.output_parameter,)+ _raise_if_browser_connection_lost(result)return result.output_parameter_value@@ -4280,6 +4370,7 @@ async def loop(complete_if_empty: bool = False,label: str | None = None,) -> AsyncGenerator[SkyvernLoopItem, None]:+ _refuse_statement_after_browser_connection_loss()workflow_run_block_id, _, _ = await _create_workflow_block_run_and_task(block_type=BlockType.FOR_LOOP, label=label)# process values:loop_variable_reference = None@@ -4384,6 +4475,7 @@ async def loop(workflow_run_id=workflow_run_id,loop_index=index,)+ _refuse_statement_after_browser_connection_loss()# register current_value, current_item and current_index in workflow run context# Block metadata for template context (user-facing fields only)@@ -4408,6 +4500,7 @@ async def loop(# Build the SkyvernLoopItem for this loopyield SkyvernLoopItem(index, value)+ _refuse_statement_after_browser_connection_loss()# build success outputif workflow_run_block_id:await _update_workflow_block(@@ -4457,6 +4550,7 @@ async def while_loop(*,criteria_type: str | None = None,) -> AsyncGenerator[SkyvernLoopItem, None]:+ _refuse_statement_after_browser_connection_loss()workflow_run_block_id, _, _ = await _create_workflow_block_run_and_task(block_type=BlockType.WHILE_LOOP,label=label,@@ -4480,6 +4574,7 @@ async def while_loop(loop_idx = 0try:while True:+ _refuse_statement_after_browser_connection_loss()condition_metadata: BlockMetadata = {"current_index": loop_idx,"current_value": None,@@ -4506,6 +4601,7 @@ async def while_loop()raise Exception(f"Failed to evaluate while-loop condition: {exc}") from exc+ _refuse_statement_after_browser_connection_loss()if not should_continue:break@@ -4539,6 +4635,7 @@ async def while_loop(workflow_run_id=workflow_run_id,loop_index=loop_idx,)+ _refuse_statement_after_browser_connection_loss()loop_metadata: BlockMetadata = {"current_index": loop_idx,diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.pyindex 0655ec62f..aad05eb1f 100644--- a/skyvern/services/task_v2_service.py+++ b/skyvern/services/task_v2_service.py@@ -15,12 +15,15 @@ from sqlalchemy.exc import OperationalErrorfrom skyvern.config import settingsfrom skyvern.constants import MINI_GOAL_TEMPLATEfrom skyvern.exceptions import (+ BROWSER_CONNECTION_LOST_ERROR_CODE,BrowserSessionNotFound,FailedToSendWebhook,+ MissingBrowserState,ScreenshotTargetClosed,TaskTerminationError,TaskV2NotFound,UrlGenerationFailure,+ get_user_facing_exception_message,)from skyvern.forge import appfrom skyvern.forge.failure_classifier import bounded_failure_categories, classify_from_failure_reason@@ -120,7 +123,7 @@ from skyvern.utils.prompt_engine import load_prompt_with_elementsfrom skyvern.utils.strings import generate_random_stringfrom skyvern.utils.url_validators import validate_fetch_urlfrom skyvern.webeye.actions.actions import ActionType-from skyvern.webeye.browser_state import BrowserState+from skyvern.webeye.browser_state import BrowserState, get_browser_connection_lossfrom skyvern.webeye.scraper.scraped_page import ScrapedPagefrom skyvern.webeye.utils.page import SkyvernFrame, build_open_tabs_context, capture_open_tab_screenshots@@ -1126,6 +1129,10 @@ async def run_task_v2_helper(iteration=i,workflow_run_id=workflow_run_id,)+ lost_task_v2 = await _fail_task_v2_after_browser_loss(task_v2_id, workflow_run_id, organization_id)+ if lost_task_v2 is not None:+ task_v2 = lost_task_v2+ breakawait _persist_completion_tab_screenshots(browser_state=browser_state, thought=thought)task_v2 = await _summarize_task_v2(task_v2=task_v2,@@ -1133,7 +1140,7 @@ async def run_task_v2_helper(context=context,screenshots=scraped_page.screenshots,)- if task_v2.run_with == "code":+ if task_v2.run_with == "code" and task_v2.status == TaskV2Status.completed:await app.WORKFLOW_SERVICE.generate_script_if_needed(workflow=workflow,workflow_run=workflow_run,@@ -1385,6 +1392,12 @@ async def run_task_v2_helper()except Exception:LOG.warning("Failed to build open-tabs context for completion check", exc_info=True)+ lost_task_v2 = await _fail_task_v2_after_browser_loss(+ task_v2_id, workflow_run_id, organization_id, completed_block=(block, block_result)+ )+ if lost_task_v2 is not None:+ task_v2 = lost_task_v2+ breaktask_v2_completion_prompt = prompt_engine.load_prompt("task_v2_check_completion",@@ -1438,14 +1451,21 @@ async def run_task_v2_helper(workflow_run_id=workflow_run_id,completion_resp=completion_resp,)+ lost_task_v2 = await _fail_task_v2_after_browser_loss(+ task_v2_id, workflow_run_id, organization_id, completed_block=(block, block_result)+ )+ if lost_task_v2 is not None:+ task_v2 = lost_task_v2+ breakawait _persist_completion_tab_screenshots(browser_state=browser_state, thought=thought)task_v2 = await _summarize_task_v2(task_v2=task_v2,task_history=task_history,context=context,screenshots=completion_screenshots,+ completed_block=(block, block_result),)- if task_v2.run_with == "code":+ if task_v2.run_with == "code" and task_v2.status == TaskV2Status.completed:await app.WORKFLOW_SERVICE.generate_script_if_needed(workflow=workflow,workflow_run=workflow_run,@@ -1586,7 +1606,12 @@ async def handle_block_result(block_type_var=block.block_type,block_label=block.label,)- if block.continue_on_failure and not is_last_block:+ if BROWSER_CONNECTION_LOST_ERROR_CODE in block_result.error_codes:+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(+ workflow_run_id=workflow_run_id, failure_reason=block_result.failure_reason+ )+ await _end_task_v2_after_browser_loss(task_v2_id, workflow_run_id, workflow_run.organization_id)+ elif block.continue_on_failure and not is_last_block:LOG.warning(f"Block with type {block.block_type} failed but will continue executing the workflow run {workflow_run_id}",block_type=block.block_type,@@ -1623,6 +1648,57 @@ async def handle_block_result()+async def _fail_task_v2_after_browser_loss(+ task_v2_id: str,+ workflow_run_id: str,+ organization_id: str,+ completed_block: tuple[BlockTypeVar, BlockResult] | None = None,+) -> TaskV2 | None:+ # A success observed after the run lost its browser natively cannot complete the task.+ if get_browser_connection_loss(app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)) is None:+ return None+ LOG.warning("Browser connection lost: task v2 stops without a completion decision", workflow_run_id=workflow_run_id)+ failure_reason = get_user_facing_exception_message(MissingBrowserState(workflow_run_id=workflow_run_id))+ if completed_block is not None:+ block, block_result = completed_block+ await block.build_block_result(+ success=False,+ failure_reason=failure_reason,+ status=BlockStatus.failed,+ workflow_run_block_id=block_result.workflow_run_block_id,+ organization_id=organization_id,+ error_codes=list(dict.fromkeys([*block_result.error_codes, BROWSER_CONNECTION_LOST_ERROR_CODE])),+ can_continue_after_failure=False,+ )+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final(+ workflow_run_id=workflow_run_id, failure_reason=failure_reason+ )+ return await _end_task_v2_after_browser_loss(task_v2_id, workflow_run_id, organization_id)+++async def _end_task_v2_after_browser_loss(task_v2_id: str, workflow_run_id: str, organization_id: str) -> TaskV2:+ current = await app.DATABASE.observer.get_task_v2(task_v2_id, organization_id=organization_id)+ if current is not None and current.status.is_final():+ # The writer that ended the task first, such as a cancellation, already sent its terminal callback.+ return current+ workflow_run = await app.WORKFLOW_SERVICE.get_workflow_run(+ workflow_run_id=workflow_run_id, organization_id=organization_id+ )+ won = False+ if workflow_run.status.is_final():+ won = await app.DATABASE.observer.update_task_v2_status_if_not_final(+ task_v2_id=task_v2_id,+ organization_id=organization_id,+ status=TaskV2Status(workflow_run.status),+ )+ task_v2 = await app.DATABASE.observer.get_task_v2(task_v2_id, organization_id=organization_id)+ if task_v2 is None:+ raise TaskV2NotFound(task_v2_id=task_v2_id)+ if won:+ await send_task_v2_webhook(task_v2)+ return task_v2++async def _set_up_workflow_context(workflow: Workflow, workflow_run_id: str, organization: Organization) -> None:"""TODO: see if we could remove this function as we can just set an empty workflow context@@ -2819,6 +2895,7 @@ async def _summarize_task_v2(task_history: list[dict],context: SkyvernContext,screenshots: list[bytes] | None = None,+ completed_block: tuple[BlockTypeVar, BlockResult] | None = None,) -> TaskV2:summary_description, summarized_output = await _generate_task_v2_deliverable(task_v2=task_v2,@@ -2827,6 +2904,12 @@ async def _summarize_task_v2(screenshots=screenshots,is_partial=False,)+ if task_v2.workflow_run_id is not None:+ lost_task_v2 = await _fail_task_v2_after_browser_loss(+ task_v2.observer_cruise_id, task_v2.workflow_run_id, task_v2.organization_id, completed_block+ )+ if lost_task_v2 is not None:+ return lost_task_v2return await mark_task_v2_as_completed(task_v2_id=task_v2.observer_cruise_id,workflow_run_id=task_v2.workflow_run_id,diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex 6b99ac73e..f71a9e84d 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -52,6 +52,7 @@ from skyvern.constants import (from skyvern.core.script_generations.fuzzy_matcher import match_option_exact_or_stemfrom skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codesfrom skyvern.exceptions import (+ BROWSER_CONNECTION_LOST_ERROR_CODE,ActionExecutionTimeout,BlockedHost,CaptchaSolveError,@@ -79,6 +80,7 @@ from skyvern.exceptions import (InteractWithDisabledElement,InteractWithDropdownContainer,InvalidElementForTextInput,+ MissingBrowserState,MissingElement,MissingElementDict,MissingElementInCSSMap,@@ -223,7 +225,7 @@ from skyvern.webeye.browser_factory import (resolve_artifact_path,was_download_cancelled_by_skyvern,)-from skyvern.webeye.browser_state import BLANK_PAGE_URLS, BrowserState+from skyvern.webeye.browser_state import BLANK_PAGE_URLS, BrowserState, get_browser_connection_lossfrom skyvern.webeye.cdp_download_interceptor import (BROWSER_DOWNLOAD_EVENT_ADMISSION_GRACE_SECONDS,DOWNLOAD_MIME_TYPES,@@ -2437,6 +2439,15 @@ async def _recover_download_page(timeout_seconds: float,recovery_site: str,) -> Page | None:+ connection_loss = get_browser_connection_loss(browser_state)+ if connection_loss is not None:+ raise MissingBrowserState(+ task_id=task.task_id,+ workflow_run_id=task.workflow_run_id,+ diagnostic=connection_loss,+ detected_at=datetime.now(UTC),+ failure_reason=BROWSER_CONNECTION_LOST_ERROR_CODE,+ )LOG.warning("Working page closed during download action; recreating it before continuing",workflow_run_id=task.workflow_run_id,@@ -2509,6 +2520,9 @@ async def _recover_download_page(recovery_site=recovery_site,exc_info=True,)+ # A connection lost while recovery was in flight ends the action, so the caller never starts waiting.+ if get_browser_connection_loss(browser_state) is not None:+ raisereturn Nonereturn recovered_pagediff --git a/skyvern/webeye/browser_state.py b/skyvern/webeye/browser_state.pyindex f18e016f2..452007549 100644--- a/skyvern/webeye/browser_state.py+++ b/skyvern/webeye/browser_state.py@@ -1,5 +1,6 @@from __future__ import annotations+from datetime import UTC, datetimefrom typing import TYPE_CHECKING, Awaitable, Callable, Literal, Protocolfrom urllib.parse import urlparse@@ -7,7 +8,7 @@ from playwright.async_api import BrowserContext, Page, Playwrightfrom skyvern.config import settingsfrom skyvern.constants import NAVIGATION_MAX_RETRY_TIME-from skyvern.exceptions import BrowserStateDiagnostic+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE, BrowserStateDiagnostic, MissingBrowserStatefrom skyvern.schemas.runs import ProxyLocationInputfrom skyvern.webeye.browser_artifacts import BrowserArtifactsfrom skyvern.webeye.browser_factory import BrowserCleanupFunc@@ -86,6 +87,11 @@ class BrowserState(Protocol):def get_browser_state_diagnostic(self) -> BrowserStateDiagnostic | None: ...+ def get_browser_connection_loss(self) -> BrowserStateDiagnostic | None: ...++ @property+ def active_run_owner(self) -> str | None: ...+async def reconnect(self,proxy_location: ProxyLocationInput = None,@@ -194,3 +200,26 @@ def get_browser_state_diagnostic(browser_state: BrowserState | None) -> BrowserSgetter = getattr(browser_state, "get_browser_state_diagnostic", None)diagnostic = getter() if callable(getter) else Nonereturn diagnostic if isinstance(diagnostic, BrowserStateDiagnostic) else None+++def get_browser_connection_loss(browser_state: BrowserState | None) -> BrowserStateDiagnostic | None:+ """Read the terminal loss fact without requiring every test double to implement it."""+ if browser_state is None:+ return None+ getter = getattr(browser_state, "get_browser_connection_loss", None)+ loss = getter() if callable(getter) else None+ return loss if isinstance(loss, BrowserStateDiagnostic) else None+++def raise_if_browser_connection_lost(+ browser_state: BrowserState | None, *, task_id: str | None = None, workflow_run_id: str | None = None+) -> None:+ connection_loss = get_browser_connection_loss(browser_state)+ if connection_loss is not None:+ raise MissingBrowserState(+ task_id=task_id,+ workflow_run_id=workflow_run_id,+ diagnostic=connection_loss,+ detected_at=datetime.now(UTC),+ failure_reason=BROWSER_CONNECTION_LOST_ERROR_CODE,+ )diff --git a/skyvern/webeye/navigation.py b/skyvern/webeye/navigation.pyindex d5d35899c..124895b2f 100644--- a/skyvern/webeye/navigation.py+++ b/skyvern/webeye/navigation.py@@ -280,6 +280,7 @@ async def navigate_with_retry(wait_until: Literal["load", "domcontentloaded", "commit"] = "load",sleep: SleepFunc | None = None,log_url: str | None = None,+ stop_retrying: Callable[[], bool] | None = None,) -> int | None:# Returns the final navigation response's HTTP status when one is available (None when the# navigation produced no response object). Additive: every caller today ignores the return; the@@ -312,6 +313,11 @@ async def navigate_with_retry(url=display_url, host=error.host, error_message=_NO_ADDRESS_RECORD_REFUSAL) from None+ # The destination check above can wait on DNS, and the caller's stop (the run's lost browser connection) may+ # have latched meanwhile; nothing is dispatched then.+ if stop_retrying is not None and stop_retrying():+ raise FailedToNavigateToUrl(url=display_url, error_message="the browser connection was lost before navigation")+for attempt in range(retry_times):strategy = degradation[min(attempt, len(degradation) - 1)]LOG.info("Trying to navigate to url", url=display_url, retry_time=attempt, wait_until=strategy)@@ -376,7 +382,21 @@ async def navigate_with_retry(nav_error_code=driver_nav_error_code(error_str),) from (None if redacting else error)- if attempt >= retry_times - 1:+ # The caller's typed stop ends retries too, rechecked after the backoff so a loss latched while sleeping+ # never reaches another dispatch.+ stopped = stop_retrying is not None and stop_retrying()+ if attempt < retry_times - 1 and not stopped:+ LOG.warning(+ "Error while navigating to url, retrying",+ exc_info=not redacting,+ url=display_url,+ retry_time=attempt,+ wait_until=strategy,+ error=safe_error_str,+ )+ await sleep(1)+ stopped = stop_retrying is not None and stop_retrying()+ if attempt >= retry_times - 1 or stopped:# Terminal navigation failure is re-raised as FailedToNavigateToUrl and surfaced# via the run's failure_reason; logging at error here double-logs an external# (site-unreachable) outcome. Keep the traceback, drop the error level.@@ -393,15 +413,5 @@ async def navigate_with_retry(nav_error_code=driver_nav_error_code(error_str),) from (None if redacting else error)- LOG.warning(- "Error while navigating to url, retrying",- exc_info=not redacting,- url=display_url,- retry_time=attempt,- wait_until=strategy,- error=safe_error_str,- )- await sleep(1)-# Reached only when retry_times <= 0 (the loop never ran); a real navigation returns or raises inside.return Nonediff --git a/skyvern/webeye/real_browser_manager.py b/skyvern/webeye/real_browser_manager.pyindex fa0053ac2..851bf117b 100644--- a/skyvern/webeye/real_browser_manager.py+++ b/skyvern/webeye/real_browser_manager.py@@ -66,7 +66,7 @@ from skyvern.webeye.browser_settings_receipts import (measure_timezone_receipt,record_session_timezone_receipt,)-from skyvern.webeye.browser_state import BrowserState+from skyvern.webeye.browser_state import BrowserState, raise_if_browser_connection_lostfrom skyvern.webeye.cdp_frame_publisher import (CDPFramePublisher,stream_key_for_task,@@ -301,11 +301,15 @@ async def _on_browser_state_acquired(task_id: str | None = None,browser_session_id: str | None = None,acquire: bool = True,+ handoff: bool = False,) -> BrowserState:browser_context = browser_state.browser_contextif browser_context is not None:_retain_run_dialog_answers(browser_state, workflow_run_id, parent_workflow_run_id, live_run_ids)await app.AGENT_FUNCTION.on_browser_context_acquired(browser_context, workflow_run_id)+ if handoff:+ # The run still bound to this browser owns any loss latched up to here; rebinding would hide it.+ raise_if_browser_connection_lost(browser_state, workflow_run_id=workflow_run_id)if acquire:browser_state.bind_runtime_event_context(BrowserRuntimeLogContext.for_run(@@ -1050,6 +1054,9 @@ class RealBrowserManager(BrowserManager):lost_external_browser = Falseif browser_state is not None:if browser_session_id and not browser_state.is_connected():+ raise_if_browser_connection_lost(+ browser_state, workflow_run_id=task.workflow_run_id, task_id=task.task_id+ )lost_external_browser = is_external_browser_state(browser_state)LOG.warning("Cached persistent-session browser state for task is disconnected; reconnecting",@@ -1295,6 +1302,7 @@ class RealBrowserManager(BrowserManager):lost_external_browser = Falseif browser_state:if browser_session_id and not browser_state.is_connected():+ raise_if_browser_connection_lost(browser_state, workflow_run_id=workflow_run_id)lost_external_browser = is_external_browser_state(browser_state)LOG.warning("Cached persistent-session browser state for workflow run is disconnected; reconnecting",@@ -1313,6 +1321,7 @@ class RealBrowserManager(BrowserManager):self._live_run_ids_sharing(browser_state),browser_session_id=browser_session_id,acquire=acquire,+ handoff=browser_state.active_run_owner not in (None, workflow_run_id),)# When an explicit browser_session_id is provided (e.g. from a workflow@@ -1325,6 +1334,8 @@ class RealBrowserManager(BrowserManager):workflow_run_id=workflow_run_id, parent_workflow_run_id=parent_workflow_run_id)if browser_state:+ if browser_state.active_run_owner is not None:+ raise_if_browser_connection_lost(browser_state, workflow_run_id=workflow_run_id)# A parent_workflow_run_id entry is shared by every child run of the same parent.# Independent children dispatched to one long-lived worker (e.g. a sequential# fan-out) can therefore find an earlier, already-completed sibling's torn-down@@ -1354,6 +1365,8 @@ class RealBrowserManager(BrowserManager):browser_profile_id=browser_profile_id,)working_page = await browser_state.get_working_page()+ if browser_state.active_run_owner is not None:+ raise_if_browser_connection_lost(browser_state, workflow_run_id=workflow_run_id)if working_page is not None:# always keep the browser state for the workflow run and the parent workflow run syncedself.pages[workflow_run_id] = browser_state@@ -1370,13 +1383,19 @@ class RealBrowserManager(BrowserManager):await _record_run_timezone_receipt(workflow_run, browser_state, browser_session_id=None, inherited=True)- return await _on_browser_state_acquired(- browser_state,- workflow_run_id,- parent_workflow_run_id,- self._live_run_ids_sharing(browser_state),- browser_session_id=browser_session_id,- )+ try:+ return await _on_browser_state_acquired(+ browser_state,+ workflow_run_id,+ parent_workflow_run_id,+ self._live_run_ids_sharing(browser_state),+ browser_session_id=browser_session_id,+ handoff=browser_state.active_run_owner is not None,+ )+ except MissingBrowserState:+ if self.pages.get(workflow_run_id) is browser_state:+ self.pages.pop(workflow_run_id, None)+ raise# The inherited state is genuinely torn down (disconnected and page-less).# Drop the stale entry and fall through to create a fresh browser for this run.LOG.warning(diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.pyindex 8f8ec5434..2a78a0bc6 100644--- a/skyvern/webeye/real_browser_state.py+++ b/skyvern/webeye/real_browser_state.py@@ -23,12 +23,14 @@ from skyvern.constants import (NAVIGATION_MAX_RETRY_TIME,)from skyvern.exceptions import (+ BROWSER_CONNECTION_LOST_ERROR_CODE,BrowserStateDiagnostic,EmptyBrowserContext,ExternalBrowserUnavailable,FailedToNavigateToUrl,FailedToReloadPage,FailedToStopLoadingPage,+ MissingBrowserState,MissingBrowserStatePage,UnresolvableNavigationHost,)@@ -428,6 +430,9 @@ class RealBrowserState(BrowserState):# A release retry after a transient DB failure must not repeat local CDP teardown.self._remote_driver_detached = Falseself._browser_state_diagnostic: BrowserStateDiagnostic | None = None+ # The run that saw this browser end unexpectedly, and how. Unlike the diagnostic it survives a context+ # rebuild, so that run never gets the browser back; a later run that reuses this state does not see it.+ self._connection_loss: tuple[str | None, BrowserStateDiagnostic] | None = None# HTTP status of the most recent navigate_to_url (None until one runs, or when it produced no# response). Read by the Task V3 loop to classify a dead/removed starting URL; v1 ignores it.self.last_navigation_status: int | None = None@@ -573,10 +578,17 @@ class RealBrowserState(BrowserState):# A tab can die on its own (a download-turned-navigation, a renderer crash) while the# context survives. Recover here rather than in each consumer: every caller that needs a# page treats "no page" as fatal, so one of them recovering only relocates the failure.- if self.browser_context is None or self.external_browser or not self.is_connected():+ if (+ self.browser_context is None+ or self.external_browser+ or not self.is_connected()+ or self.get_browser_connection_loss() is not None+ ):return Noneasync with self._reopen_working_page_lock:already_reopened = await self.get_working_page()+ if self.get_browser_connection_loss() is not None:+ return Noneif already_reopened is not None:await self._close_crashed_targets()return already_reopened@@ -704,6 +716,7 @@ class RealBrowserState(BrowserState):reconcile_persistent_init_scripts: bool = False,sessionless_init_script_registrations: Collection[Any] = (),) -> None:+ self._refuse_after_connection_loss()if self.browser_context is None:if self.external_browser:raise ExternalBrowserUnavailable.lost()@@ -790,6 +803,7 @@ class RealBrowserState(BrowserState):if pages:page = pages[-1]use_existing_page = True+ self._refuse_after_connection_loss()if page is None:page = await self.browser_context.new_page()@@ -894,6 +908,7 @@ class RealBrowserState(BrowserState):retry_times: int = NAVIGATION_MAX_RETRY_TIME,wait_until: Literal["load", "domcontentloaded", "commit"] = "load",) -> None:+ self._refuse_after_connection_loss()landed_url: str | None = Noneasync def goto(strategy: str) -> object:@@ -916,6 +931,7 @@ class RealBrowserState(BrowserState):retry_times=retry_times,settle=self._wait_for_settle,wait_until=wait_until,+ stop_retrying=lambda: self.get_browser_connection_loss() is not None,)except Exception as navigation_error:await self.record_navigation_result(navigation_error)@@ -1386,15 +1402,7 @@ class RealBrowserState(BrowserState):observation_source: Literal["liveness_probe", "browser_event", "driver_event"] = "liveness_probe",disconnect_evidence: DisconnectEvidence = "liveness_state",) -> None:- if self._browser_state_diagnostic is not None or not self._ever_connected:- return- pending = self._deferred_runtime_end- if (- self._runtime_events_deferred- and pending is not None- and pending.context is self.browser_context- and pending.driver is self.pw- ):+ if not self._ever_connected:returnobservation = self._make_runtime_end_observation(reason,@@ -1406,6 +1414,18 @@ class RealBrowserState(BrowserState):observation_source=observation_source,disconnect_evidence=disconnect_evidence,)+ # Before the first-diagnostic dedupe: a probe failure may already have latched the diagnostic.+ self._latch_connection_loss(observation)+ if self._browser_state_diagnostic is not None:+ return+ pending = self._deferred_runtime_end+ if (+ self._runtime_events_deferred+ and pending is not None+ and pending.context is self.browser_context+ and pending.driver is self.pw+ ):+ returnif self._runtime_events_deferred:self._deferred_runtime_end = observationreturn@@ -1515,6 +1535,47 @@ class RealBrowserState(BrowserState):def get_browser_state_diagnostic(self) -> BrowserStateDiagnostic | None:return self._browser_state_diagnostic+ def _run_owner(self) -> str | None:+ return self._runtime_event_context.workflow_run_id or self._runtime_event_context.task_id++ def _latch_connection_loss(self, observation: _RuntimeEndObservation) -> None:+ owner = self._run_owner()+ # No run owns an unbound or not-yet-published state, so its loss is not a run's terminal fact.+ if (+ owner is None+ or self._runtime_events_deferred+ or observation.source != "browser_event"+ or observation.expected+ or observation.run_phase != "active"+ or observation.context is None+ or observation.context is not self.browser_context+ or observation.driver is not self.pw+ or (self._connection_loss is not None and self._connection_loss[0] == owner)+ ):+ return+ self._connection_loss = (owner, observation.diagnostic)++ def get_browser_connection_loss(self) -> BrowserStateDiagnostic | None:+ if self._connection_loss is None or self._connection_loss[0] != self._run_owner():+ return None+ return self._connection_loss[1]++ @property+ def active_run_owner(self) -> str | None:+ return None if self._run_released else self._run_owner()++ def _refuse_after_connection_loss(self) -> None:+ # Reconnect, context rebuild and navigation for the owning run stop here, before any side effect.+ connection_loss = self.get_browser_connection_loss()+ if connection_loss is not None:+ raise MissingBrowserState(+ task_id=self._runtime_event_context.task_id,+ workflow_run_id=self._runtime_event_context.workflow_run_id,+ diagnostic=connection_loss,+ detected_at=datetime.now(UTC),+ failure_reason=BROWSER_CONNECTION_LOST_ERROR_CODE,+ )+def _connection_status(self) -> tuple[bool, str | None]:stopped = browser_context_stopped_reason(self.browser_context)return stopped is None, stopped@@ -1540,6 +1601,7 @@ class RealBrowserState(BrowserState):browser_session_id: str | None = None,stale_context_is_unusable: bool = False,) -> None:+ self._refuse_after_connection_loss()# Rebuild through a fresh Playwright driver only after the old connection is known dead or# an explicitly unusable connection has completed bounded shutdown.stale_pw = self.pw@@ -1888,6 +1950,7 @@ class RealBrowserState(BrowserState):raise FailedToStopLoadingPage(url=page.url, error_message=repr(e))async def new_page(self) -> Page:+ self._refuse_after_connection_loss()if self.browser_context is None:raise EmptyBrowserContext()return await self.browser_context.new_page()@@ -1938,7 +2001,8 @@ class RealBrowserState(BrowserState):returnexcept Exception as e:self._record_reload_timeout(e)- if i < len(strategies) - 1:+ # A reload that failed after the owning run lost its browser is final; degrading would send another.+ if i < len(strategies) - 1 and self.get_browser_connection_loss() is None:LOG.warning("Reload timed out, degrading wait strategy",url=url,diff --git a/tests/unit/_task_block_fakes.py b/tests/unit/_task_block_fakes.pyindex f7d72815f..9062d1d05 100644--- a/tests/unit/_task_block_fakes.py+++ b/tests/unit/_task_block_fakes.py@@ -10,6 +10,8 @@ from unittest.mock import AsyncMock, MagicMock, patchfrom skyvern.forge.sdk.schemas.tasks import TaskStatusfrom skyvern.forge.sdk.workflow.context_manager import WorkflowRunContextfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType+from skyvern.webeye.browser_runtime_events import BrowserRuntimeLogContext+from skyvern.webeye.real_browser_state import RealBrowserStatedef output_parameter(key: str = "task_output") -> OutputParameter:@@ -38,6 +40,21 @@ def _workflow_run_context(values: dict[str, Any] | None = None) -> WorkflowRunCoreturn ctx+def run_bound_browser_state(workflow_run_id: str) -> tuple[RealBrowserState, MagicMock]:+ """A run-bound browser state and its still-open current context."""+ context = MagicMock(browser=MagicMock())+ state = RealBrowserState(pw=MagicMock(), browser_context=context)+ state.bind_runtime_event_context(BrowserRuntimeLogContext(workflow_run_id=workflow_run_id))+ return state, context+++def lost_browser_state(workflow_run_id: str) -> RealBrowserState:+ """A run-bound browser state whose current context has just closed unexpectedly."""+ state, context = run_bound_browser_state(workflow_run_id)+ state._on_browser_context_closed(context)+ return state++@dataclass(frozen=True)class BlockExecuteDeps:task: SimpleNamespacediff --git a/tests/unit/test_agent_step_cancellation.py b/tests/unit/test_agent_step_cancellation.pyindex f3939e89e..a50dde3dc 100644--- a/tests/unit/test_agent_step_cancellation.py+++ b/tests/unit/test_agent_step_cancellation.py@@ -38,6 +38,7 @@ from skyvern.services.run_cancellation import RunCancellationfrom skyvern.webeye.actions.actions import ClickAction, CompleteAction, ExtractActionfrom skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccessfrom skyvern.webeye.scraper.scraped_page import ScrapedPage+from tests.unit._task_block_fakes import lost_browser_statefrom tests.unit.helpers import make_browser_state, make_organization, make_step, make_taskfrom tests.unit.scoped_asyncio import ScopedAsyncio@@ -344,6 +345,48 @@ async def test_canceled_run_stops_before_the_next_action(harness.agent.check_user_goal_complete.assert_not_awaited()+ ("action_count", "browser_lost", "run_canceled", "expected_handled"),+ [+ pytest.param(2, True, False, 1, id="lost-during-a-continuable-failure"),+ pytest.param(1, True, False, 1, id="lost-during-the-last-action"),+ pytest.param(2, False, False, 2, id="ordinary-continuable-failure"),+ pytest.param(2, True, True, 1, id="cancel-takes-precedence-over-the-loss"),+ ],+)+async def test_lost_browser_fails_the_step_before_its_next_action(+ monkeypatch: pytest.MonkeyPatch, action_count: int, browser_lost: bool, run_canceled: bool, expected_handled: int+) -> None:+ harness = _make_harness(monkeypatch, navigation_goal="Submit the application")+ actions = [+ _click_action(harness.task, harness.step, element_id=f"node-{i}", action_order=i) for i in range(action_count)+ ]+ monkeypatch.setattr("skyvern.forge.agent.parse_actions", lambda *_, **__: actions)+ harness.browser_state.must_get_working_page = AsyncMock(return_value=harness.page)+ canceled = _cancel_when_flagged(monkeypatch, harness)+ connection_loss = lost_browser_state("workflow-1").get_browser_connection_loss()+ harness.browser_state.get_browser_connection_loss = MagicMock(return_value=None)++ async def _handle_action(*_args, **_kwargs) -> list[ActionResult]:+ if browser_lost:+ harness.browser_state.get_browser_connection_loss.return_value = connection_loss+ canceled["yet"] = run_canceled+ return [ActionFailure(exception=RuntimeError("Target closed"), stop_execution_on_failure=False)]++ handle_action_mock = AsyncMock(side_effect=_handle_action)+ monkeypatch.setattr("skyvern.forge.agent.ActionHandler.handle_action", handle_action_mock)++ step, _ = await harness.run()++ assert handle_action_mock.await_count == expected_handled+ if browser_lost:+ assert step.status == (StepStatus.canceled if run_canceled else StepStatus.failed)+ harness.agent.check_user_goal_complete.assert_not_awaited()+ else:+ assert step.status != StepStatus.failed++@pytest.mark.asyncioasync def test_canceled_run_stops_after_the_implicit_extraction(monkeypatch: pytest.MonkeyPatch) -> None:"""A cancel landing during the extraction that follows a complete action still stops the step."""diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex fb6d4de68..191c2d17e 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -24,7 +24,12 @@ from structlog.testing import capture_logsfrom skyvern.config import Settings, settingsfrom skyvern.errors.errors import UserDefinedError-from skyvern.exceptions import CompletionGateTerminationError, MissingBrowserStatePage, StepTerminationError+from skyvern.exceptions import (+ BROWSER_CONNECTION_LOST_ERROR_CODE,+ CompletionGateTerminationError,+ MissingBrowserStatePage,+ StepTerminationError,+)from skyvern.forge import agent as agent_modulefrom skyvern.forge import appfrom skyvern.forge.agent import ForgeAgent@@ -121,6 +126,7 @@ from tests.unit._download_provider_fakes import (_secret_bearing_validation_error,_StallingActionDownloadObservation,)+from tests.unit._task_block_fakes import run_bound_browser_statefrom tests.unit.helpers import (make_action_row,make_browser_state,@@ -130,6 +136,7 @@ from tests.unit.helpers import ()from tests.unit.scoped_asyncio import ScopedAsynciofrom tests.unit.test_taskv3_loop import _ScriptedCaller+from tests.unit.test_taskv3_tools import _FakePageasync def _run_execute_task_v3(@@ -4173,6 +4180,172 @@ async def test_a_fifth_veto_on_a_reask_conversion_restores_the_models_own_verdicassert task.failure_reason == "the site offers no way to continue"+class _ModelSubstitute(_ScriptedCaller):+ """A deterministic stand-in for the Task V3 model; no LLM is called."""++ def __init__(self, script: list[list[tuple[str, dict[str, Any]]]]) -> None:+ super().__init__(script)+ self.original_llm_key = self.llm_key+ self.llm_config = LLMConfig("scripted/model-substitute", [], True, False)++ async def call(self, *, step: Step | None = None, **kwargs: Any) -> dict[str, Any]:+ return await super().call(**kwargs)++ def emit_retry_chain_exhausted(self, error: BaseException, prompt_name: str | None) -> None:+ raise AssertionError(f"the model substitute never exhausts a retry chain: {error!r}")+++class _PageWithEventAfterFill(_FakePage):+ def __init__(self, after_fill: Callable[[], None]) -> None:+ super().__init__()+ self._after_fill = after_fill++ async def fill(self, selector: str, text: str, timeout: int | None = None) -> None:+ await super().fill(selector, text, timeout=timeout)+ self._after_fill()+++ ("event", "status"),+ [+ ("none", TaskStatus.completed),+ ("expected_close", TaskStatus.completed),+ ("native_close", TaskStatus.failed),+ ("cancel_and_native_close", TaskStatus.canceled),+ ],+)+async def test_execute_task_v3_native_browser_loss_after_the_last_action_fails_the_completion(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, event: str, status: TaskStatus+) -> None:+ # The real loop, finish tool, completion gate and post-loop write; only the model and the page are fakes.+ agent = ForgeAgent()+ now = datetime.now(UTC)+ organization = make_organization(now)+ task = make_task(+ now,+ organization,+ workflow_run_id="wr_v3_browser_loss",+ navigation_goal="Fill in the first name",+ data_extraction_goal=None,+ extracted_information_schema=None,+ )+ step = make_step(now, task, step_id="step-v3", status=StepStatus.created, order=0, output=None)+ state, context = run_bound_browser_state("wr_v3_browser_loss")+ canceled = False++ def _after_last_action() -> None:+ nonlocal canceled+ if event == "expected_close":+ state._expect_runtime_end("normal_close", context)+ if event == "cancel_and_native_close":+ canceled = True+ if event != "none":+ # Playwright closes a closed context's pages and stops listing them.+ page._closed = True+ context.pages = []+ state._on_browser_context_closed(context)++ page = _PageWithEventAfterFill(_after_last_action)+ page.url = task.url+ context.pages = [page]+ context._skyvern_cdp_download_interceptor = None+ await state.set_working_page(page)+ monkeypatch.setattr(state, "take_post_action_screenshot", AsyncMock(return_value=b"png-bytes"))++ # The settle check may hold a finish twice before it accepts one.+ finish = [("finish", {"status": "completed", "reason": "the first name is filled in"})]+ model = _ModelSubstitute([[("observe", {})], [("type", {"selector": "#first", "text": "John"})], *[finish] * 3])+ monkeypatch.setattr("skyvern.forge.agent.LLMCaller", lambda **_kwargs: model)+ run_context = WorkflowRunContext(+ workflow_title="wf",+ workflow_id="w_test",+ workflow_permanent_id="wpid_test",+ workflow_run_id="wr_v3_browser_loss",+ aws_client=MagicMock(),+ )+ monkeypatch.setattr(+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.has_workflow_run_context", lambda *_a, **_k: True+ )+ monkeypatch.setattr(+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context", lambda *_a, **_k: run_context+ )++ async def _get_task(**_kwargs: Any) -> SimpleNamespace:+ return SimpleNamespace(status=TaskStatus.canceled if canceled else TaskStatus.running)++ monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.get_task", AsyncMock(side_effect=_get_task))+ monkeypatch.setattr(+ "skyvern.forge.agent.app.DATABASE.workflow_runs.get_workflow_run",+ AsyncMock(return_value=SimpleNamespace(status=WorkflowRunStatus.running, parent_workflow_run_id=None)),+ )+ monkeypatch.setattr("skyvern.forge.sdk.api.files.get_download_dir", lambda *_a, **_k: str(tmp_path))+ monkeypatch.setattr(+ "skyvern.forge.agent.app.ARTIFACT_MANAGER.create_artifact", AsyncMock(return_value="artifact-1")+ )+ persisted_actions: list[Any] = []++ async def _create_action(action: Any) -> Any:+ persisted_actions.append(action)+ return action++ monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.workflow_params.create_action", _create_action)+ monkeypatch.setattr(+ "skyvern.forge.agent.app.DATABASE.observer.get_workflow_run_block_by_task_id", AsyncMock(return_value=None)+ )+ monkeypatch.setattr("skyvern.services.otp_service.has_credential_totp_candidate", lambda *_a, **_k: False)+ monkeypatch.setattr("skyvern.forge.agent.app.AGENT_FUNCTION.post_step_execution", AsyncMock())++ async def fake_update_step(step: Step, status: StepStatus | None = None, **_kwargs: Any) -> Step:+ if status is not None:+ step.status = status+ return step++ async def fake_update_task(task: Any, status: Any = None, **kwargs: Any) -> Any:+ task.status = status+ task.failure_reason = kwargs.get("failure_reason")+ return task++ agent.update_step = AsyncMock(side_effect=fake_update_step)+ agent.update_task = AsyncMock(side_effect=fake_update_task)+ agent.clean_up_task = AsyncMock()++ skyvern_context.set(+ SkyvernContext(+ task_id=task.task_id,+ step_id=step.step_id,+ organization_id=task.organization_id,+ workflow_run_id=task.workflow_run_id,+ )+ )+ try:+ await agent._execute_task_v3(+ task=task,+ step=step,+ browser_state=state,+ organization=organization,+ api_key=None,+ close_browser_on_completion=True,+ browser_session_id=None,+ task_block=_make_block(NavigationBlock, navigation_goal="Fill in the first name"),+ )+ finally:+ skyvern_context.reset()++ assert ("fill", {"selector": "#first", "text": "John"}) in page.calls+ assert task.status == status+ if status == TaskStatus.canceled:+ return+ decision = persisted_actions[-1]+ assert decision.description.endswith("finish completed")+ if status == TaskStatus.failed:+ assert decision.status == ActionStatus.failed+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in task.failure_reason+ else:+ assert decision.status == ActionStatus.completed+ assert task.failure_reason is None++@pytest.mark.asyncio@pytest.mark.parametrize(("task_type", "custom_key", "criteria"),diff --git a/tests/unit/test_ai_click_empty_actions.py b/tests/unit/test_ai_click_empty_actions.pyindex 31bf37bfc..231058906 100644--- a/tests/unit/test_ai_click_empty_actions.py+++ b/tests/unit/test_ai_click_empty_actions.py@@ -8,7 +8,7 @@ skipped the select entirely but returned the requested value as if it had succeefrom __future__ import annotations-from contextlib import contextmanager+from contextlib import contextmanager, nullcontextfrom unittest.mock import AsyncMock, MagicMock, patchimport pytest@@ -426,3 +426,61 @@ class TestAiClickHandlerRefusal:result = await self._ai_click(mock_page, mock_scraped_page, mock_context, mock_app, failure)assert result == "xpath=//div[@id='opt']"mock_page.locator.return_value.click.assert_awaited_once()+++class TestAiFallbackAfterBrowserLoss:+ """A cached run that lost its browser starts no AI work and retries nothing with the raw selector."""++ _CALLS = {+ "click": (lambda ai: ai.ai_click(selector="#go", intention="Go"), "#go"),+ "input_text": (lambda ai: ai.ai_input_text("#name", "Noor", "Name", failed_selector="#name"), "Noor"),+ "upload_file": (+ lambda ai: ai.ai_upload_file("#cv", "https://example.com/cv.pdf", "CV"),+ "https://example.com/cv.pdf",+ ),+ "select_option": (lambda ai: ai.ai_select_option("#color", "Blue", "Pick blue"), "Blue"),+ }++ @pytest.mark.asyncio+ @pytest.mark.parametrize("lost", ["before_the_call", "during_the_llm_call", "never"])+ @pytest.mark.parametrize("method", sorted(_CALLS))+ async def test_ai_fallback_dispatches_nothing_after_native_browser_loss(+ self, mock_page, mock_scraped_page, mock_context, mock_app, method: str, lost: str+ ):+ from skyvern.exceptions import MissingBrowserState+ from tests.unit._task_block_fakes import run_bound_browser_state++ browser_state, browser_context = run_bound_browser_state("wr_ai")+ mock_scraped_page._browser_state = browser_state+ mock_context.workflow_run_id = "wr_ai"+ mock_context.script_run_parameters = None+ page_ai = RealSkyvernPageAi(mock_scraped_page, mock_page)+ call, expected = self._CALLS[method]++ async def llm(**_: object) -> dict:+ if lost == "during_the_llm_call":+ browser_state._on_browser_context_closed(browser_context)+ return {"actions": []}++ raw_input = AsyncMock()+ v3_hook = AsyncMock(return_value=(None, False))+ if lost == "before_the_call":+ browser_state._on_browser_context_closed(browser_context)+ with (+ patch.object(page_ai, "_maybe_run_v3_midrun", v3_hook),+ patch(f"{MODULE}.skyvern_context.ensure_context", return_value=mock_context),+ patch(f"{MODULE}.skyvern_context.current", return_value=mock_context),+ patch(f"{MODULE}.app", mock_app),+ patch(f"{MODULE}.prompt_engine.load_prompt", return_value="mock_prompt"),+ patch(f"{MODULE}._resolve_assist_llm_handler", AsyncMock(return_value=AsyncMock(side_effect=llm))),+ patch(f"{MODULE}.handler_utils.input_sequentially", raw_input),+ pytest.raises(MissingBrowserState) if lost != "never" else nullcontext(),+ ):+ assert await call(page_ai) == expected++ raw_locator = mock_page.locator.return_value+ raw_dispatches = raw_locator.click.await_count + raw_locator.select_option.await_count + raw_input.await_count+ assert raw_dispatches == (1 if lost == "never" and method != "upload_file" else 0)+ assert mock_scraped_page.generate_scraped_page.await_count == (0 if lost == "before_the_call" else 1)+ if method == "input_text":+ assert v3_hook.await_count == (0 if lost == "before_the_call" else 1)diff --git a/tests/unit/test_code_block_downloads.py b/tests/unit/test_code_block_downloads.pyindex f4e478070..7d01d3a06 100644--- a/tests/unit/test_code_block_downloads.py+++ b/tests/unit/test_code_block_downloads.py@@ -269,6 +269,7 @@ def _fake_storage_app(get_downloaded_file_signature_aliases=lambda _: [],list_downloading_files_in_browser_session=in_flight or AsyncMock(return_value=[]),),+ BROWSER_MANAGER=SimpleNamespace(get_for_workflow_run=lambda *_args, **_kwargs: None),AGENT_FUNCTION=SimpleNamespace(validate_code_block=AsyncMock(),validate_block_execution=AsyncMock(),diff --git a/tests/unit/test_code_block_sandbox.py b/tests/unit/test_code_block_sandbox.pyindex 49617e7e3..0f5ade4e2 100644--- a/tests/unit/test_code_block_sandbox.py+++ b/tests/unit/test_code_block_sandbox.py@@ -13,7 +13,7 @@ import operatorimport reimport warningsfrom collections.abc import AsyncIterator, Callable, Mapping-from contextlib import asynccontextmanager+from contextlib import asynccontextmanager, nullcontextfrom datetime import UTCfrom datetime import date as stdlib_datefrom datetime import datetime@@ -82,6 +82,7 @@ from skyvern.webeye import dialog_handlerfrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.browser_artifacts import BrowserArtifactsfrom skyvern.webeye.skycdp.errors import CdpError+from tests.unit._task_block_fakes import run_bound_browser_statefrom tests.unit.conftest import (FakeClearingBrowserContext,FakeSearchBrowserContext,@@ -3117,6 +3118,46 @@ class TestCodeBlockMagicLink:assert navigations == [link]assert fills == []+ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "native_close", [True, False], ids=["native_close_during_goto", "ordinary_navigation_error"]+ )+ async def test_link_navigation_stops_retrying_after_native_browser_loss(+ self, monkeypatch: pytest.MonkeyPatch, native_close: bool+ ) -> None:+ from playwright.async_api import Error as PlaywrightError++ from skyvern.constants import NAVIGATION_MAX_RETRY_TIME+ from skyvern.exceptions import FailedToNavigateToUrl+ from skyvern.forge.sdk.schemas.totp_codes import OTPType+ from skyvern.forge.sdk.workflow.models import block as block_module+ from skyvern.forge.sdk.workflow.models.block import _bind_code_block_magic_link+ from skyvern.services.otp_service import OTPValue+ from skyvern.webeye import navigation as navigation_module+ from tests.unit.scoped_asyncio import ScopedAsyncio++ link = "https://93.184.216.34/signin?token=x"+ self._patch_poll(monkeypatch, _build_wrc_with_identifier(), OTPValue(value=link, type=OTPType.MAGIC_LINK))+ browser_state, context = run_bound_browser_state(_WORKFLOW_RUN_ID)+ monkeypatch.setattr(block_module.app.BROWSER_MANAGER, "get_for_workflow_run", lambda *_, **__: browser_state)+ monkeypatch.setattr(navigation_module, "asyncio", ScopedAsyncio(sleep=AsyncMock()))+ navigations: list[str] = []+ page = _fake_code_block_page()++ async def goto(url: str, **kwargs: object) -> None:+ navigations.append(url)+ if native_close:+ browser_state._on_browser_context_closed(context)+ raise PlaywrightError("Target page, context or browser has been closed")+ raise PlaywrightError("Timeout 30000ms exceeded.")++ page.goto = goto++ with pytest.raises(FailedToNavigateToUrl):+ await _bind_code_block_magic_link(_CREDENTIAL_KEY, _ORG_ID, _WORKFLOW_RUN_ID)(page)++ assert navigations == [link] * (1 if native_close else NAVIGATION_MAX_RETRY_TIME)+@pytest.mark.asyncio@pytest.mark.usefixtures("public_dns")async def test_anchors_created_after_at_call_time_not_run_start(self, monkeypatch: pytest.MonkeyPatch) -> None:@@ -3843,6 +3884,27 @@ class TestOpenPageHelperBinding:await without_state(page, "https://example.test/detail/1")assert len(context.opened) == cap+ @pytest.mark.asyncio+ @pytest.mark.parametrize("native_close", [True, False], ids=["lost_during_the_tab_count", "browser_alive"])+ async def test_open_page_opens_no_tab_after_a_loss_during_its_tab_count(self, native_close: bool) -> None:+ from skyvern.exceptions import MissingBrowserState++ context = FakeSearchBrowserContext()+ page = SimpleNamespace(context=context, url="https://example.test/listing")+ browser_state, browser_context = run_bound_browser_state("wr_open_page")++ async def count_tabs(**_: object) -> list[object]:+ if native_close:+ browser_state._on_browser_context_closed(browser_context)+ return []++ browser_state.list_valid_pages = count_tabs # type: ignore[method-assign]+ open_page = _bind_code_block_open_page(page, [], browser_state) # type: ignore[arg-type]++ with pytest.raises(MissingBrowserState) if native_close else nullcontext():+ await open_page(page, "https://example.test/detail/1")+ assert len(context.opened) == (0 if native_close else 1)+@pytest.mark.asyncioasync def test_open_page_treats_a_non_positive_cap_as_unlimited(self, monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr(settings, "BROWSER_MAX_PAGES_NUMBER", 0)diff --git a/tests/unit/test_copilot_failure_publication.py b/tests/unit/test_copilot_failure_publication.pyindex 4e268a095..057bb130b 100644--- a/tests/unit/test_copilot_failure_publication.py+++ b/tests/unit/test_copilot_failure_publication.py@@ -1365,7 +1365,7 @@ async def test_successor_execute_safe_settles_capture_before_first_execution_mutresult = await _block().execute_safe(workflow_run_id="wr_test", organization_id="o_test")assert result.success is True- assert events == ["capture_settled", "browser_read"]+ assert events[0] == "capture_settled" and set(events[1:]) == {"browser_read"}@pytest.mark.asyncio@@ -1434,7 +1434,9 @@ async def test_capture_settlement_failure_uses_protected_block_failure_path(monkassert result.failure_reason == "CodeBlock execution failed."assert build_result.await_args.kwargs["workflow_run_block_id"] == "wrb_successor"- browser_lookup.assert_not_called()+ # The failure handler may read the run's browser-loss fact from the registry; the block itself+ # must still never reach the browser (its pre-execution screenshot) after a failed settlement.+ browser_lookup.return_value.take_fullpage_screenshot.assert_not_called()@pytest.mark.asynciodiff --git a/tests/unit/test_credential_fallback.py b/tests/unit/test_credential_fallback.pyindex bc43ff03f..de4e595b1 100644--- a/tests/unit/test_credential_fallback.py+++ b/tests/unit/test_credential_fallback.py@@ -273,6 +273,7 @@ async def _run_fallback_retry(flag_error: Exception | None = None,run_tags: dict[str, str] | None = None,run_group_item: object | None = None,+ block_error_codes: list[str] | None = None,) -> tuple[str | None, AsyncMock, MagicMock]:captured: dict[str, object] = {}@@ -301,6 +302,9 @@ async def _run_fallback_retry(mock_app.AGENT_FUNCTION.is_block_scoped_workflow_run = AsyncMock(return_value=block_scoped)mock_app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers = _strip_marker_headermock_app.DATABASE.debug.has_block_run_for_workflow_run = AsyncMock(return_value=False)+ mock_app.DATABASE.observer.get_workflow_run_blocks = AsyncMock(+ return_value=[SimpleNamespace(error_codes=block_error_codes or [])]+ )mock_app.DATABASE.workflow_run_groups.get_item_by_workflow_run_id = AsyncMock(return_value=run_group_item)mock_app.WORKFLOW_SERVICE.get_workflow = AsyncMock(return_value=_workflow(parameters))if retried_by_results is not None:@@ -335,6 +339,19 @@ async def _run_fallback_retry(return result, run_workflow_mock, mock_app+async def test_any_failure_fallback_is_not_started_after_the_run_lost_its_browser() -> None:+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])+ login = _credential_parameter(fallback_credential_ids=["cred_fb1"], fallback_trigger=ANY_FAILURE)++ result, run_workflow_mock, _ = await _run_fallback_retry(+ workflow_run=workflow_run, parameters=[login], block_error_codes=["browser_connection_lost"]+ )++ assert result is None+ run_workflow_mock.assert_not_awaited()++@pytest.mark.asyncioasync def test_fallback_attempt_one_uses_first_fallback_and_preserves_other_pins() -> None:workflow_run = _workflow_run(diff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.pyindex f0c7c2425..c588d1ce1 100644--- a/tests/unit/test_download_file_action_handler.py+++ b/tests/unit/test_download_file_action_handler.py@@ -19,10 +19,11 @@ from structlog.testing import capture_logsfrom skyvern.config import settingsfrom skyvern.constants import BROWSER_DOWNLOAD_TIMEOUT, BROWSER_DOWNLOADING_SUFFIXfrom skyvern.errors.errors import UserDefinedError-from skyvern.exceptions import BlockedHost+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE, BlockedHost, MissingBrowserStatefrom skyvern.forge.agent import ForgeAgentfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.models import StepStatus+from skyvern.webeye import real_browser_state as real_browser_state_modulefrom skyvern.webeye.actions.actions import (ActionStatus,ClickAction,@@ -49,6 +50,7 @@ from skyvern.webeye.actions.handler import (_record_action_owned_popup_delta,_recover_adopted_session_blob_pdf_iframe,_recover_blocked_inline_pdf_download,+ _recover_download_page,_remove_download_listener,handle_download_file_action,)@@ -56,7 +58,9 @@ from skyvern.webeye.actions.responses import ActionAbort, ActionFailure, ActionSfrom skyvern.webeye.browser_factory import (mark_download_cancelled_by_skyvern,)+from skyvern.webeye.browser_runtime_events import BrowserRuntimeLogContextfrom skyvern.webeye.cdp_download_interceptor import CDPDownloadInterceptor+from skyvern.webeye.real_browser_state import RealBrowserStatefrom skyvern.webeye.scraper.scraped_page import ScrapedPagefrom skyvern.webeye.utils.page import BlobActionFreshnessfrom tests.unit._download_provider_fakes import (@@ -1561,6 +1565,156 @@ async def test_handle_action_reconnects_context_closed_during_download_wait(assert results[-1].skip_remaining_actions is True[email protected]("recovery_site", ["post_click", "post_wait"])[email protected]("native_close", [True, False], ids=["native_context_close", "page_only_close"])+async def test_download_page_recovery_refuses_a_browser_whose_context_closed_natively(+ recovery_site: str, native_close: bool+) -> None:+ now = datetime.now(UTC)+ organization = make_organization(now)+ task, step, page, _, scraped_page, action = _make_download_click_context(+ now=now,+ organization=organization,+ page_url="https://example.com/downloads",+ )+ if recovery_site == "post_click":+ task = task.model_copy(update={"download_timeout": 0.01})+ page.expose_binding = AsyncMock()+ page.evaluate = AsyncMock(return_value=[])+ context = page.context+ context._skyvern_cdp_download_active = False+ context._skyvern_cdp_download_interceptor = None+ browser_state = RealBrowserState(pw=MagicMock(), browser_context=context)+ browser_state.bind_runtime_event_context(+ BrowserRuntimeLogContext(workflow_run_id=task.workflow_run_id, task_id=task.task_id)+ )+ recovered_page = MagicMock(url=page.url, context=context)+ recovered_page.is_closed.return_value = False+ recovered_page.evaluate = AsyncMock(return_value=[])+ recovered_page.expose_binding = AsyncMock()+ page_closed = False+ page.is_closed.side_effect = lambda: page_closed++ async def get_working_page(**_: object) -> object | None:+ return None if page_closed else page++ browser_state.get_working_page = AsyncMock(side_effect=get_working_page) # type: ignore[method-assign]+ browser_state.list_valid_pages = AsyncMock(return_value=[page]) # type: ignore[method-assign]+ browser_state.new_page = AsyncMock(return_value=recovered_page) # type: ignore[method-assign]+ browser_state.reconnect = AsyncMock() # type: ignore[method-assign]+ browser_state.navigate_to_url = AsyncMock() # type: ignore[method-assign]+ browser_state.set_active_page = AsyncMock() # type: ignore[method-assign]++ calls_at_close: int | None = None++ def close_working_page() -> None:+ nonlocal page_closed, calls_at_close+ page_closed = True+ calls_at_close = list_calls+ if native_close:+ browser_state._on_browser_context_closed(context)++ async def click(*_args: object, **_kwargs: object) -> list[ActionSuccess]:+ if recovery_site == "post_click":+ close_working_page()+ return [ActionSuccess()]++ list_calls = 0++ def list_files(_path: Path | str, *, attempt_started_at: datetime | None = None) -> list[str]:+ nonlocal list_calls+ list_calls += 1+ if recovery_site == "post_wait" and list_calls == 2:+ close_working_page()+ return []++ xhr_capture = MagicMock(has_in_flight_requests=False)+ xhr_capture.drain = AsyncMock(return_value=False)+ mock_app = MagicMock()+ mock_app.BROWSER_MANAGER.get_for_task.return_value = browser_state+ mock_app.DATABASE.workflow_params.create_action = AsyncMock(return_value=action)+ mock_app.STORAGE = MagicMock()++ with tempfile.TemporaryDirectory() as temp_dir:+ with (+ patch.object(ActionHandler, "_handle_action", side_effect=click),+ patch("skyvern.webeye.actions.handler.BROWSER_DOWNLOAD_NO_SIGNAL_GRACE_TIME", 0),+ patch("skyvern.webeye.actions.handler.get_download_dir", return_value=temp_dir),+ patch("skyvern.webeye.actions.handler.list_files_in_directory", side_effect=list_files),+ patch("skyvern.webeye.actions.handler.ScopedXhrDownloadCapture", return_value=xhr_capture),+ patch("skyvern.webeye.actions.handler._recover_blocked_inline_pdf_download", new=AsyncMock()),+ patch("skyvern.webeye.actions.handler.skyvern_context.current", return_value=None),+ patch("skyvern.webeye.actions.handler.app", mock_app),+ ):+ handle = ActionHandler.handle_action(+ scraped_page=scraped_page, task=task, step=step, page=page, action=action+ )+ if native_close:+ with pytest.raises(MissingBrowserState) as refusal:+ await handle+ else:+ await handle++ if native_close:+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in str(refusal.value)+ browser_state.new_page.assert_not_awaited()+ browser_state.reconnect.assert_not_awaited()+ browser_state.navigate_to_url.assert_not_awaited()+ if recovery_site == "post_click":+ # The download wait never starts polling after the refusal.+ assert list_calls == calls_at_close+ else:+ browser_state.new_page.assert_awaited_once_with()+ browser_state.reconnect.assert_not_awaited()+ browser_state.navigate_to_url.assert_awaited_once_with(page=recovered_page, url="https://example.com/downloads")+++async def test_download_page_recovery_propagates_a_loss_observed_while_new_page_is_pending(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ now = datetime.now(UTC)+ task, _, page, _, _, _ = _make_download_click_context(+ now=now, organization=make_organization(now), page_url="https://example.com/downloads"+ )+ context = page.context+ context._skyvern_cdp_download_interceptor = None+ browser_state = RealBrowserState(pw=MagicMock(), browser_context=context)+ browser_state.bind_runtime_event_context(+ BrowserRuntimeLogContext(workflow_run_id=task.workflow_run_id, task_id=task.task_id)+ )++ async def new_page() -> object:+ # The context-close event lands while newPage is pending, after the recovery entry check passed.+ browser_state._on_browser_context_closed(context)+ raise RuntimeError("Target page, context or browser has been closed")++ driver_starts: list[bool] = []++ class CountingPlaywright:+ async def start(self) -> object:+ driver_starts.append(True)+ raise RuntimeError("driver start reached")++ monkeypatch.setattr(real_browser_state_module, "async_playwright", lambda: CountingPlaywright())+ create_context = AsyncMock(side_effect=RuntimeError("context factory reached"))+ monkeypatch.setattr(real_browser_state_module.BrowserContextFactory, "create_browser_context", create_context)+ browser_state.new_page = AsyncMock(side_effect=new_page) # type: ignore[method-assign]+ browser_state.navigate_to_url = AsyncMock() # type: ignore[method-assign]++ with pytest.raises(MissingBrowserState) as refusal:+ await _recover_download_page(+ browser_state, task, "https://example.com/downloads", 5.0, recovery_site="post_click"+ )++ assert BROWSER_CONNECTION_LOST_ERROR_CODE in str(refusal.value)+ browser_state.new_page.assert_awaited_once_with()+ assert driver_starts == []+ create_context.assert_not_awaited()+ browser_state.navigate_to_url.assert_not_awaited()++@pytest.mark.asyncioasync def test_handle_action_restores_blank_working_page_when_download_not_triggered() -> None:now = datetime.now(UTC)diff --git a/tests/unit/test_print_page_block.py b/tests/unit/test_print_page_block.pyindex bec52c733..e43f6af3a 100644--- a/tests/unit/test_print_page_block.py+++ b/tests/unit/test_print_page_block.py@@ -4,18 +4,21 @@ recovering the print when the renderer crashes part-way through it (SKY-17342)."import asynciofrom datetime import UTC, datetimefrom types import SimpleNamespace-from unittest.mock import AsyncMock+from unittest.mock import AsyncMock, patchimport pytestfrom playwright.async_api import Error as PlaywrightError-from skyvern.exceptions import MissingBrowserStatePage+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE, MissingBrowserStatePagefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.schemas.files import FileInfofrom skyvern.forge.sdk.workflow.models import block as block_module-from skyvern.forge.sdk.workflow.models.block import PrintPageBlock+from skyvern.forge.sdk.workflow.models.block import Block, PrintPageBlockfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType+from skyvern.schemas.workflows import BlockStatus+from tests.unit._task_block_fakes import run_bound_browser_state+from tests.unit.test_block_description_caching import _setup_mocksdef _output_parameter(key: str) -> OutputParameter:@@ -653,3 +656,51 @@ async def test_print_page_block_fails_rather_than_printing_a_blank_replacement_tassert result.success is Falseassert "Printing failed" in result.failure_reasonblank_page.pdf.assert_not_awaited()++[email protected]("native_close", [True, False], ids=["native_close_during_print", "ordinary_print_failure"])+async def test_print_page_failure_after_native_browser_loss_cannot_be_continued(+ monkeypatch: pytest.MonkeyPatch, _isolated_download_path: str, native_close: bool+) -> None:+ skyvern_context.set(SkyvernContext(organization_id="o_1", workflow_run_id="wr_1", run_id="wr_1"))+ browser_state, context = run_bound_browser_state("wr_1")+ page = SimpleNamespace(url="https://example.com/listing")++ async def print_pdf(**_: object) -> bytes:+ if native_close:+ browser_state._on_browser_context_closed(context)+ raise PlaywrightError("Target page, context or browser has been closed")+ raise PlaywrightError("Page.pdf: some other failure")++ page.pdf = print_pdf+ browser_state.get_working_page = AsyncMock(return_value=page) # type: ignore[method-assign]+ browser_state.list_valid_pages = AsyncMock(return_value=[page]) # type: ignore[method-assign]+ browser_state.take_fullpage_screenshot = AsyncMock(return_value=None) # type: ignore[method-assign]+ block = PrintPageBlock(label="print", output_parameter=_output_parameter("print_out"), continue_on_failure=True)++ with (+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,+ patch.object(Block, "_generate_workflow_run_block_description", new_callable=AsyncMock),+ patch.object(Block, "_invalidate_stale_output_on_failure", new_callable=AsyncMock),+ ):+ _setup_mocks(mock_app)+ mock_app.BROWSER_MANAGER.get_for_workflow_run.return_value = browser_state+ monkeypatch.setattr(PrintPageBlock, "get_or_create_browser_state", AsyncMock(return_value=browser_state))++ result = await block.execute_safe(workflow_run_id="wr_1", organization_id="o_1")+ persisted_codes = {+ code+ for update in mock_app.DATABASE.observer.update_workflow_run_block.await_args_list+ for code in update.kwargs.get("error_codes") or []+ }++ assert result.status == BlockStatus.failed+ assert result.failure_reason.startswith("Failed to generate PDF")+ if native_close:+ assert result.can_continue_after_failure is False+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in result.error_codes+ assert persisted_codes == {BROWSER_CONNECTION_LOST_ERROR_CODE}+ else:+ assert result.can_continue_after_failure is True+ assert persisted_codes == set()diff --git a/tests/unit/test_real_browser_manager.py b/tests/unit/test_real_browser_manager.pyindex 43aec0b05..9cd67ecfa 100644--- a/tests/unit/test_real_browser_manager.py+++ b/tests/unit/test_real_browser_manager.py@@ -8,7 +8,7 @@ on every call and re-invoke navigate_to_url() on every step.import asynciofrom collections.abc import Iterator-from contextlib import contextmanager+from contextlib import contextmanager, nullcontextfrom dataclasses import replacefrom itertools import pairwisefrom types import SimpleNamespace@@ -19,8 +19,10 @@ import pytestfrom structlog.testing import capture_logsfrom skyvern.exceptions import (+ BROWSER_CONNECTION_LOST_ERROR_CODE,BrowserSessionAlreadyOccupiedError,FailedToNavigateToUrl,+ MissingBrowserState,MissingBrowserStateForBrowserSession,)from skyvern.forge import app as forge_app@@ -49,6 +51,7 @@ from skyvern.webeye.browser_runtime_events import BrowserRuntimeLogContextfrom skyvern.webeye.real_browser_manager import RealBrowserManager, _PersistentSessionLeasefrom skyvern.webeye.real_browser_state import RealBrowserStatefrom skyvern.webeye.utils.page import SkyvernFrame+from tests.unit._task_block_fakes import lost_browser_state, run_bound_browser_statefrom tests.unit.forge_log_capture import capture_runtime_logs@@ -4214,3 +4217,171 @@ async def test_the_receipt_accepts_the_browser_name_for_the_requested_zone_and_n)assert receipt.status == expected_status+++async def test_persistent_session_lost_mid_run_is_not_reconnected_for_that_run() -> None:+ manager = RealBrowserManager()+ state = lost_browser_state("wfr_pbs")+ manager.pages["wfr_pbs"] = state++ with (+ patch.object(manager, "_create_browser_state", new=AsyncMock()) as mock_create,+ pytest.raises(MissingBrowserState) as exc_info,+ ):+ await manager.get_or_create_for_workflow_run(+ workflow_run=make_workflow_run("wfr_pbs"), url=None, browser_session_id="pbs_1"+ )++ assert BROWSER_CONNECTION_LOST_ERROR_CODE in str(exc_info.value)+ mock_create.assert_not_awaited()+ assert manager.pages["wfr_pbs"] is state++[email protected]("loss", ["before_inheritance", "during_the_liveness_probe", "probe_failure_only"])+async def test_child_run_does_not_get_a_fresh_browser_after_the_parent_lost_its_browser(loss: str) -> None:+ manager = RealBrowserManager()+ parent_state, context = run_bound_browser_state("wfr_parent")+ context._impl_obj = SimpleNamespace(_closed=False, _close_was_called=False, _connection=None)+ context.browser.is_connected.return_value = True++ def probe() -> None:+ if loss == "during_the_liveness_probe":+ parent_state._on_browser_context_closed(context)+ raise TargetClosedError("Target page, context or browser has been closed")++ context.cookies = AsyncMock(side_effect=probe)+ if loss == "before_inheritance":+ parent_state._on_browser_context_closed(context)+ manager.pages["wfr_parent"] = parent_state+ fresh_state = MagicMock()+ fresh_state.get_or_create_page = AsyncMock()++ with patch.object(manager, "_create_browser_state", new=AsyncMock(return_value=fresh_state)) as mock_create:+ acquire = manager.get_or_create_for_workflow_run(+ workflow_run=make_workflow_run("wfr_child", parent_workflow_run_id="wfr_parent"),+ url="https://example.com",+ browser_session_id=None,+ )+ if loss == "probe_failure_only":+ assert await acquire is fresh_state+ assert manager.pages["wfr_parent"] is fresh_state+ return+ with pytest.raises(MissingBrowserState, match=BROWSER_CONNECTION_LOST_ERROR_CODE):+ await acquire++ assert context.cookies.await_count == (0 if loss == "before_inheritance" else 1)+ mock_create.assert_not_awaited()+ assert manager.pages["wfr_parent"] is parent_state+ assert parent_state.get_browser_connection_loss() is not None+ assert "wfr_child" not in manager.pages++[email protected]("seam", ["frame_publisher", "timezone_receipt", "acquisition_hook", None])+async def test_child_does_not_take_over_a_parent_browser_lost_during_the_handoff(seam: str | None) -> None:+ manager = RealBrowserManager()+ parent_state, context = run_bound_browser_state("wfr_parent")+ parent_state.get_working_page = AsyncMock(return_value=MagicMock())+ manager.pages["wfr_parent"] = parent_state++ def lose_at(name: str) -> None:+ if seam == name:+ parent_state._on_browser_context_closed(context)++ with (+ patch.object(manager, "_start_frame_publisher", AsyncMock(side_effect=lambda **_: lose_at("frame_publisher"))),+ patch.object(+ real_browser_manager,+ "_record_run_timezone_receipt",+ AsyncMock(side_effect=lambda *_, **__: lose_at("timezone_receipt")),+ ),+ patch.object(+ real_browser_manager.app.AGENT_FUNCTION,+ "on_browser_context_acquired",+ AsyncMock(side_effect=lambda *_: lose_at("acquisition_hook")),+ ),+ patch.object(manager, "_create_browser_state", new=AsyncMock()) as mock_create,+ pytest.raises(MissingBrowserState, match=BROWSER_CONNECTION_LOST_ERROR_CODE) if seam else nullcontext(),+ ):+ await manager.get_or_create_for_workflow_run(+ workflow_run=make_workflow_run("wfr_child", parent_workflow_run_id="wfr_parent"), browser_session_id=None+ )++ mock_create.assert_not_awaited()+ assert manager.pages["wfr_parent"] is parent_state+ assert manager.pages.get("wfr_child") is (None if seam else parent_state)+ assert parent_state.runtime_event_context.workflow_run_id == ("wfr_parent" if seam else "wfr_child")+ assert (parent_state.get_browser_connection_loss() is not None) == bool(seam)++[email protected]("bound_to", ["finished_child_lost", "finished_child_healthy", "own_run_lost"])+async def test_run_does_not_take_over_a_shared_browser_its_finished_child_lost(bound_to: str) -> None:+ manager = RealBrowserManager()+ owner = "wfr_parent" if bound_to == "own_run_lost" else "wfr_child"+ state, context = run_bound_browser_state(owner)+ if bound_to != "finished_child_healthy":+ state._on_browser_context_closed(context)+ # A finished child that shared the parent's browser leaves it bound to itself; only the parent key remains.+ manager.pages["wfr_parent"] = state++ with (+ patch.object(manager, "_create_browser_state", new=AsyncMock()) as mock_create,+ pytest.raises(MissingBrowserState, match=BROWSER_CONNECTION_LOST_ERROR_CODE)+ if bound_to == "finished_child_lost"+ else nullcontext(),+ ):+ assert await manager.get_or_create_for_workflow_run(make_workflow_run("wfr_parent")) is state++ mock_create.assert_not_awaited()+ assert manager.pages == {"wfr_parent": state}+ assert state.runtime_event_context.workflow_run_id == (+ "wfr_child" if bound_to == "finished_child_lost" else "wfr_parent"+ )+ assert (state.get_browser_connection_loss() is not None) == (bound_to != "finished_child_healthy")+++async def test_released_owners_loss_does_not_refuse_an_independent_later_owner() -> None:+ manager = RealBrowserManager()+ state = lost_browser_state("wfr_earlier")+ state.mark_run_released()+ manager.pages["wfr_later"] = state++ assert (+ await manager.get_or_create_for_workflow_run(make_workflow_run("wfr_later"), browser_session_id=None) is state+ )+ assert state.runtime_event_context.workflow_run_id == "wfr_later"+ assert state.get_browser_connection_loss() is None++[email protected]("released", [False, True])+async def test_parent_keyed_loss_only_blocks_an_active_owner(released: bool, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(forge_app.AGENT_FUNCTION, "on_browser_run_released", MagicMock())+ manager = RealBrowserManager()+ state = lost_browser_state("wfr_sibling")+ if released:+ state.mark_run_released()+ manager.pages["wfr_parent"] = state+ fresh_state, _ = run_bound_browser_state("wfr_child")+ fresh_state.get_or_create_page = AsyncMock()+ state.get_or_create_page = AsyncMock(side_effect=AssertionError("stale browser must not be reconnected"))++ with patch.object(manager, "_create_browser_state", new=AsyncMock(return_value=fresh_state)) as create:+ acquire = manager.get_or_create_for_workflow_run(+ make_workflow_run("wfr_child", parent_workflow_run_id="wfr_parent")+ )+ if released:+ assert await acquire is fresh_state+ assert manager.pages["wfr_parent"] is fresh_state+ assert manager.pages["wfr_child"] is fresh_state+ create.assert_awaited_once()+ else:+ with pytest.raises(MissingBrowserState, match=BROWSER_CONNECTION_LOST_ERROR_CODE):+ await acquire+ assert manager.pages == {"wfr_parent": state}+ create.assert_not_awaited()+ state.get_or_create_page.assert_not_awaited()diff --git a/tests/unit/test_real_browser_state_arm.py b/tests/unit/test_real_browser_state_arm.pyindex 5032c281a..b3bcdcc85 100644--- a/tests/unit/test_real_browser_state_arm.py+++ b/tests/unit/test_real_browser_state_arm.py@@ -27,6 +27,7 @@ def _state(recorder: object, working_page: object) -> RealBrowserState:st.browser_context = MagicMock() # non-None -> creation branch skippedst.browser_context.pages = []st.external_browser = False+ st._connection_loss = Nonest.browser_artifacts = SimpleNamespace(_display_recorder=recorder, remote_browser_session_id=None)st.get_working_page = AsyncMock(return_value=working_page)st.set_working_page = AsyncMock()diff --git a/tests/unit/test_real_browser_state_navigate.py b/tests/unit/test_real_browser_state_navigate.pyindex 4c8d48e4d..b9c97e7e6 100644--- a/tests/unit/test_real_browser_state_navigate.py+++ b/tests/unit/test_real_browser_state_navigate.py@@ -30,8 +30,10 @@ from skyvern.webeye.real_browser_state import RealBrowserState, _same_page_ignordef browser_state(monkeypatch: pytest.MonkeyPatch) -> RealBrowserState:retry_sleep = AsyncMock()monkeypatch.setattr(real_browser_state, "navigate_with_retry", partial(navigate_with_retry, sleep=retry_sleep))- # Bypass __init__; navigate_to_url only uses `self` for LOG context and _wait_for_settle.+ # Bypass __init__; navigate_to_url only uses `self` for LOG context, _wait_for_settle and the+ # run's browser-loss fact (none here).state = RealBrowserState.__new__(RealBrowserState)+ state._connection_loss = Nonemonkeypatch.setattr(state, "_wait_for_settle", AsyncMock())return statediff --git a/tests/unit/test_real_browser_state_reload.py b/tests/unit/test_real_browser_state_reload.pyindex 04d5181d0..f4cd580fc 100644--- a/tests/unit/test_real_browser_state_reload.py+++ b/tests/unit/test_real_browser_state_reload.py@@ -24,6 +24,7 @@ _AGENT_MODULE = "skyvern.forge.agent"def browser_state() -> RealBrowserState:state = RealBrowserState.__new__(RealBrowserState)state.engine_selection = None+ state._connection_loss = Nonereturn statediff --git a/tests/unit/test_script_service_block_outcome.py b/tests/unit/test_script_service_block_outcome.pyindex 6456c5d65..38225adc7 100644--- a/tests/unit/test_script_service_block_outcome.py+++ b/tests/unit/test_script_service_block_outcome.py@@ -1,17 +1,475 @@"""Outcome records from cached blocks and loops."""+import contextlib+from collections.abc import AsyncIterator, Awaitable, Callable, Iterator+from datetime import datetime, timezonefrom types import SimpleNamespace-from unittest.mock import AsyncMock, MagicMock+from typing import Any+from unittest.mock import AsyncMock, MagicMock, patchimport pytest+from playwright.async_api import Error as PlaywrightError+from skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAi+from skyvern.core.script_generations.script_skyvern_page import ScriptRunContextManager, ScriptSkyvernPage+from skyvern.core.script_generations.skyvern_page import RunContext+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE, MissingBrowserState, ScriptTerminationExceptionfrom skyvern.forge import appfrom skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.schemas.tasks import TaskStatusfrom skyvern.forge.sdk.workflow.context_manager import BlockOutcome, WorkflowRunContext-from skyvern.forge.sdk.workflow.models.block import ForLoopBlock-from skyvern.schemas.workflows import BlockStatus+from skyvern.forge.sdk.workflow.models.block import (+ ActionBlock,+ Block,+ ExtractionBlock,+ FileDownloadBlock,+ ForLoopBlock,+ HttpRequestBlock,+ LoginBlock,+ NavigationBlock,+ TextPromptBlock,+ UrlBlock,+ ValidationBlock,+ WhileLoopBlock,+)+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus+from skyvern.schemas.workflows import BlockResult, BlockStatus, BlockTypefrom skyvern.services import script_service+from tests.unit._task_block_fakes import run_bound_browser_statefrom tests.unit.conftest import make_block_output_parameter+from tests.unit.helpers import make_organization, make_task+from tests.unit.scoped_asyncio import ScopedAsyncio+++class _BlockRows:+ """The workflow_run_block rows a cached script run writes, by id."""++ def __init__(self) -> None:+ self.rows: dict[str, dict[str, Any]] = {}++ def create(self, label: str | None) -> str:+ workflow_run_block_id = f"wrb_{len(self.rows)}"+ self.rows[workflow_run_block_id] = {+ "label": label,+ "status": BlockStatus.running,+ "error_codes": None,+ "output": None,+ }+ return workflow_run_block_id++ async def update(self, workflow_run_block_id: str, **fields: Any) -> SimpleNamespace:+ row = self.rows[workflow_run_block_id]+ for key in ("status", "output"):+ if fields.get(key):+ row[key] = fields[key]+ if fields.get("error_codes") is not None:+ row["error_codes"] = fields["error_codes"]+ return SimpleNamespace(label=row["label"], error_codes=row["error_codes"])++ def labelled(self, label: str) -> list[dict[str, Any]]:+ return [row for row in self.rows.values() if row["label"] == label]+++def cached_run(monkeypatch: pytest.MonkeyPatch) -> Iterator[SimpleNamespace]:+ """A cached script run over the real helpers, with only the repository and storage edges faked."""+ now = datetime.now(timezone.utc)+ organization = make_organization(now)+ rows = _BlockRows()+ browser_state, browser_context = run_bound_browser_state("wr_cached")+ workflow_context = WorkflowRunContext(+ workflow_title="test",+ workflow_id="wf",+ workflow_permanent_id="wpid",+ workflow_run_id="wr_cached",+ aws_client=AsyncMock(),+ )+ run_context = skyvern_context.SkyvernContext(+ organization_id="o_cached", workflow_id="wf", workflow_run_id="wr_cached", script_mode=True+ )+ manager = ScriptRunContextManager()+ manager.set_run_context(RunContext(parameters={}, page=MagicMock()))+ workflow = SimpleNamespace(ai_fallback=False, workflow_definition=SimpleNamespace(blocks=[]))+ billed = AsyncMock()++ async def create_block(+ block_type: BlockType, label: str | None = None, **_: object+ ) -> tuple[str, str | None, str | None]:+ workflow_run_block_id = rows.create(label)+ if block_type in (BlockType.FOR_LOOP, BlockType.WAIT):+ return workflow_run_block_id, None, None+ run_context.task_id = f"tsk_{workflow_run_block_id}"+ run_context.step_id = f"stp_{workflow_run_block_id}"+ return workflow_run_block_id, run_context.task_id, run_context.step_id++ async def update_task(task_id: str, status: TaskStatus, failure_reason: str | None = None, **_: object) -> Any:+ return make_task(now, organization, task_id=task_id, status=status, failure_reason=failure_reason)++ monkeypatch.setattr(script_service, "script_run_context_manager", manager)+ monkeypatch.setattr(script_service, "_create_workflow_block_run_and_task", create_block)+ monkeypatch.setattr(+ script_service,+ "_validate_and_get_output_parameter",+ AsyncMock(+ return_value=script_service.BlockValidationOutput(+ context=run_context,+ label="rows",+ output_parameter=make_block_output_parameter("rows_output"),+ input_parameters=[],+ workflow=workflow,+ workflow_id="wf",+ workflow_run_id="wr_cached",+ organization_id="o_cached",+ )+ ),+ )+ monkeypatch.setattr(script_service, "run_lease_lost", AsyncMock(return_value=False))+ monkeypatch.setattr(script_service, "_record_output_parameter_value", AsyncMock())+ monkeypatch.setattr(+ app.WORKFLOW_CONTEXT_MANAGER, "get_workflow_run_context", MagicMock(return_value=workflow_context)+ )+ monkeypatch.setattr(app.BROWSER_MANAGER, "get_for_workflow_run", MagicMock(return_value=browser_state))+ monkeypatch.setattr(app.DATABASE.observer, "update_workflow_run_block", rows.update)+ monkeypatch.setattr(app.DATABASE.tasks, "update_step", AsyncMock(return_value=SimpleNamespace(order=0)))+ monkeypatch.setattr(app.DATABASE.tasks, "update_task", update_task)+ monkeypatch.setattr(app.DATABASE.tasks, "get_step", AsyncMock(return_value=SimpleNamespace(step_id="stp")))+ monkeypatch.setattr(app.DATABASE.tasks, "get_task", AsyncMock(return_value=make_task(now, organization)))+ monkeypatch.setattr(app.DATABASE.organizations, "get_organization", AsyncMock(return_value=organization))+ monkeypatch.setattr(app.DATABASE.workflows, "get_workflow", AsyncMock(return_value=workflow))+ monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run", AsyncMock(return_value=workflow))+ monkeypatch.setattr(app.AGENT_FUNCTION, "post_cache_step_execution", billed)+ monkeypatch.setattr(app.ARTIFACT_MANAGER, "flush_step_archive", AsyncMock())+ monkeypatch.setattr(app.STORAGE, "get_current_attempt_downloaded_files", AsyncMock(return_value=[]))+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "get_recent_task_screenshot_artifacts", AsyncMock(return_value=[]))+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "get_recent_workflow_screenshot_artifacts", AsyncMock(return_value=[]))+ with skyvern_context.scoped(run_context):+ yield SimpleNamespace(+ rows=rows,+ manager=manager,+ browser_state=browser_state,+ browser_context=browser_context,+ billed=billed,+ workflow_context=workflow_context,+ run_context=run_context,+ )++[email protected]("child", ["native_loss", "ordinary_error", "healthy"])[email protected]("helper", ["run_task", "action", "login"])+async def test_cached_loop_stops_at_the_child_that_lost_its_browser(+ cached_run: SimpleNamespace, helper: str, child: str+) -> None:+ cached_helper = getattr(script_service, helper)+ opened: list[int] = []++ async def open_page(page: object, context: object) -> dict[str, str]:+ opened.append(len(opened))+ if child == "native_loss":+ cached_run.browser_state._on_browser_context_closed(cached_run.browser_context)+ raise PlaywrightError("Target page, context or browser has been closed")+ if child == "ordinary_error":+ raise PlaywrightError("Timeout 30000ms exceeded.")+ return {"opened": "yes"}++ cached_run.manager.set_cached_fn("open_page", open_page)+ after_child: list[int] = []++ async def generated_loop() -> None:+ # Generated code leaves an abandoned loop to the garbage collector; closing it here keeps that late+ # finalizer off the test's closing event loop.+ async with contextlib.aclosing(script_service.loop([1, 2], label="rows")) as rows:+ async for _ in rows:+ await cached_helper(prompt="Open the page", label="open_page", cache_key="open_page")+ after_child.append(len(after_child))+ await script_service.wait(0, label="pause")++ if child == "native_loss":+ with pytest.raises(MissingBrowserState):+ await generated_loop()+ else:+ await generated_loop()++ open_rows = cached_run.rows.labelled("open_page")+ (loop_row,) = cached_run.rows.labelled("rows")+ if child == "native_loss":+ assert opened == [0]+ assert after_child == []+ assert cached_run.rows.labelled("pause") == []+ assert loop_row["status"] != BlockStatus.completed+ assert [row["status"] for row in open_rows] == [BlockStatus.failed]+ assert open_rows[0]["error_codes"] == [BROWSER_CONNECTION_LOST_ERROR_CODE]+ assert open_rows[0]["output"]+ cached_run.billed.assert_awaited_once()+ assert cached_run.workflow_context.get_block_outcome("open_page").error_codes == [+ BROWSER_CONNECTION_LOST_ERROR_CODE+ ]+ assert cached_run.run_context.prompt is None+ else:+ expected = BlockStatus.failed if child == "ordinary_error" else BlockStatus.completed+ assert opened == [0, 1]+ assert len(cached_run.rows.labelled("pause")) == 2+ assert loop_row["status"] == BlockStatus.completed+ assert [row["status"] for row in open_rows] == [expected, expected]+ assert [row["error_codes"] for row in open_rows] == [None, None]+++_UNCACHED_CHILDREN: dict[str, tuple[type[Block], Callable[[], Awaitable[object]]]] = {+ "run_task": (NavigationBlock, lambda: script_service.run_task(prompt="Open the page", label="open_page")),+ "goto": (UrlBlock, lambda: script_service.goto("https://93.184.216.34/next", label="go")),+ "action": (ActionBlock, lambda: script_service.action(prompt="Click it", label="click")),+ "login": (LoginBlock, lambda: script_service.login(prompt="Sign in", label="sign_in")),+ "extract": (ExtractionBlock, lambda: script_service.extract(prompt="Read it", label="read")),+ "prompt": (TextPromptBlock, lambda: script_service.prompt(prompt="Summarize", label="summary")),+}++[email protected]("browser_lost", [True, False], ids=["browser_lost", "browser_alive"])[email protected]("helper", sorted(_UNCACHED_CHILDREN))+async def test_uncached_child_that_failed_after_browser_loss_ends_the_script(+ cached_run: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, helper: str, browser_lost: bool+) -> None:+ block_type, run_child = _UNCACHED_CHILDREN[helper]+ failed = BlockResult(+ success=False,+ output_parameter=make_block_output_parameter("child_output"),+ status=BlockStatus.failed,+ failure_reason="Failed",+ )+ monkeypatch.setattr(block_type, "execute_safe", AsyncMock(return_value=failed))+ fallback_page = cached_run.manager.ensure_run_context().page+ fallback_page.goto = AsyncMock()+ if browser_lost:+ cached_run.browser_state._on_browser_context_closed(cached_run.browser_context)++ if browser_lost:+ with pytest.raises(MissingBrowserState):+ await run_child()+ else:+ await run_child()++ fallback_page.goto.assert_not_awaited()+++ ("kind", "seam", "iterations", "conditions"),+ [+ ("for", "before_the_loop", [], 0),+ ("while", "before_the_loop", [], 0),+ ("for", "during_the_baseline", [], 0),+ ("while", "during_the_baseline", [], 1),+ ("while", "during_the_condition", [], 1),+ ("for", "in_a_body", [0], 0),+ ("while", "in_a_body", [0], 1),+ ("for", "in_the_final_body", [0, 1], 0),+ ("for", "browser_alive", [0, 1], 0),+ ("while", "browser_alive", [0, 1], 3),+ ],+)+async def test_no_loop_starts_iterates_or_completes_after_its_run_lost_the_browser(+ cached_run: SimpleNamespace,+ monkeypatch: pytest.MonkeyPatch,+ kind: str,+ seam: str,+ iterations: list[int],+ conditions: int,+) -> None:+ def lose() -> None:+ cached_run.browser_state._on_browser_context_closed(cached_run.browser_context)++ evaluated: list[bool] = []++ async def evaluate_condition(*_: object, **__: object) -> bool:+ evaluated.append(seam != "during_the_condition" and len(evaluated) < 2)+ if seam == "during_the_condition":+ lose()+ return evaluated[-1]++ async def baseline(**_: object) -> list[object]:+ if seam == "during_the_baseline":+ lose()+ return []++ monkeypatch.setattr(WhileLoopBlock, "_evaluate_condition", evaluate_condition)+ monkeypatch.setattr(app.STORAGE, "get_current_attempt_downloaded_files", baseline)+ if seam == "before_the_loop":+ lose()+ if kind == "for":+ rows = script_service.loop([1, 2], label="rows")+ else:+ rows = script_service.while_loop("{{ true }}", label="rows")+ seen: list[int] = []+ lost = seam != "browser_alive"++ with pytest.raises(MissingBrowserState) if lost else contextlib.nullcontext():+ async with contextlib.aclosing(rows) as items:+ async for _ in items:+ seen.append(len(seen))+ cached_run.run_context.loop_output_values[-1].append(f"row {len(seen)}")+ if seam == "in_a_body" or (seam == "in_the_final_body" and len(seen) == 2):+ lose()++ assert seen == iterations+ assert len(evaluated) == conditions+ loop_rows = cached_run.rows.labelled("rows")+ if seam == "before_the_loop":+ assert loop_rows == []+ return+ (loop_row,) = loop_rows+ assert loop_row["output"] == ([[f"row {n + 1}"] for n in iterations] or None)+ assert loop_row["status"] == (BlockStatus.failed if lost else BlockStatus.completed)+ assert loop_row["error_codes"] == ([BROWSER_CONNECTION_LOST_ERROR_CODE] if lost else None)+++ ("helper", "seam"),+ [+ (helper, seam)+ for helper in ("conditional", "run_task", "action", "login", "extract", "download")+ for seam in ("lost_during_the_wait", "lost_during_setup", "browser_alive")+ if (helper, seam) != ("download", "browser_alive")+ ],+)+async def test_no_cached_statement_runs_after_its_run_lost_the_browser(+ cached_run: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, helper: str, seam: str+) -> None:+ def lose() -> None:+ cached_run.browser_state._on_browser_context_closed(cached_run.browser_context)++ async def sleep(_: float) -> None:+ if seam == "lost_during_the_wait":+ lose()++ create_row = script_service._create_workflow_block_run_and_task++ async def create_row_then_lose(*args: Any, **kwargs: Any) -> tuple[str, str | None, str | None]:+ created = await create_row(*args, **kwargs)+ if seam == "lost_during_setup" and kwargs.get("label") == helper:+ lose()+ return created++ evaluated: list[str] = []++ async def cached_statement(page: object, context: object) -> dict[str, Any]:+ evaluated.append(helper)+ return {"next_block_label": None, "branch_index": 0}++ monkeypatch.setattr(script_service, "asyncio", ScopedAsyncio(sleep=sleep))+ monkeypatch.setattr(script_service, "_create_workflow_block_run_and_task", create_row_then_lose)+ monkeypatch.setattr(script_service, "CACHED_DOWNLOAD_NO_FILE_GRACE_SECONDS", 0)+ cached_run.manager.set_cached_fn(helper, cached_statement)+ statements: list[str] = []++ async def generated_function() -> None:+ await script_service.wait(1, label="pause")+ if helper == "conditional":+ await script_service.conditional(label=helper)+ else:+ await getattr(script_service, helper)(prompt="Read the page", label=helper, cache_key=helper)+ statements.append("after")++ lost = seam != "browser_alive"+ with pytest.raises(MissingBrowserState) if lost else contextlib.nullcontext():+ await generated_function()++ assert evaluated == ([] if lost else [helper])+ assert statements == ([] if lost else ["after"])+ rows = [(row["status"], row["error_codes"]) for row in cached_run.rows.labelled(helper)]+ if seam == "lost_during_the_wait":+ assert rows == []+ elif seam == "lost_during_setup":+ assert rows == [(BlockStatus.failed, [BROWSER_CONNECTION_LOST_ERROR_CODE])]+ else:+ assert rows == [(BlockStatus.completed, None)]++[email protected]("lost", [True, False], ids=["lost_during_setup", "browser_alive"])+async def test_cached_request_is_not_sent_after_a_loss_during_its_setup(+ cached_run: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, lost: bool+) -> None:+ validated = await script_service._validate_and_get_output_parameter()++ async def validate_then_lose(*_: object) -> script_service.BlockValidationOutput:+ if lost:+ cached_run.browser_state._on_browser_context_closed(cached_run.browser_context)+ return validated++ @contextlib.asynccontextmanager+ async def admit(_: str) -> AsyncIterator[SimpleNamespace]:+ yield SimpleNamespace(status=WorkflowRunStatus.running)++ sent = AsyncMock(return_value=BlockResult(success=True, output_parameter=validated.output_parameter))+ monkeypatch.setattr(script_service, "_validate_and_get_output_parameter", validate_then_lose)+ monkeypatch.setattr(+ app.DATABASE.observer,+ "create_workflow_run_block",+ AsyncMock(side_effect=lambda **kw: SimpleNamespace(workflow_run_block_id=cached_run.rows.create(kw["label"]))),+ )+ monkeypatch.setattr(app.DATABASE.workflow_runs, "admit_workflow_run_block_dispatch", admit)+ monkeypatch.setattr(HttpRequestBlock, "execute", sent)++ with pytest.raises(MissingBrowserState) if lost else contextlib.nullcontext():+ await script_service.http_request(method="POST", url="https://example.com/orders", label="rows")++ assert sent.await_count == (0 if lost else 1)+ assert [row["error_codes"] for row in cached_run.rows.labelled("rows")] == [+ [BROWSER_CONNECTION_LOST_ERROR_CODE] if lost else None+ ]++[email protected]("browser_lost", [True, False], ids=["browser_lost_during_validation", "ordinary_failure"])+async def test_cached_page_validate_after_browser_loss_ends_the_script(+ cached_run: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, browser_lost: bool+) -> None:+ async def validate_page(*_args: object, **_kwargs: object) -> BlockResult:+ if browser_lost:+ cached_run.browser_state._on_browser_context_closed(cached_run.browser_context)+ return BlockResult(+ success=False,+ output_parameter=make_block_output_parameter("check_output"),+ status=BlockStatus.failed,+ failure_reason="The order is not placed",+ )++ monkeypatch.setattr(ValidationBlock, "execute_safe", validate_page)+ scraped_page = SimpleNamespace(_browser_state=cached_run.browser_state)+ raw_page = SimpleNamespace(url="https://example.com/checkout")+ with patch("skyvern.core.script_generations.skyvern_page.Page.__init__", return_value=None):+ page = ScriptSkyvernPage(scraped_page=scraped_page, page=raw_page, ai=RealSkyvernPageAi(scraped_page, raw_page))+ statements: list[object] = []++ async def generated_function() -> None:+ statements.append(await page.validate("Is the order placed?"))+ statements.append("next statement")++ with pytest.raises(MissingBrowserState) if browser_lost else contextlib.nullcontext():+ await generated_function()++ assert statements == ([] if browser_lost else [False, "next statement"])++[email protected]("browser_lost", [True, False], ids=["browser_lost", "browser_alive"])+async def test_uncached_download_failure_after_browser_loss_raises_the_typed_refusal(+ cached_run: SimpleNamespace, monkeypatch: pytest.MonkeyPatch, browser_lost: bool+) -> None:+ failed = BlockResult(+ success=False,+ output_parameter=make_block_output_parameter("download_output"),+ status=BlockStatus.failed,+ failure_reason="Failed",+ )+ monkeypatch.setattr(FileDownloadBlock, "execute_safe", AsyncMock(return_value=failed))+ if browser_lost:+ cached_run.browser_state._on_browser_context_closed(cached_run.browser_context)++ with pytest.raises(MissingBrowserState if browser_lost else ScriptTerminationException):+ await script_service.download(prompt="Download the invoice", label="invoice")@pytest.mark.asyncio@@ -105,7 +563,7 @@ async def test_cached_outcome_survives_output_parameter_failure(monkeypatch: pytassert context.get_block_outcome("cached") == BlockOutcome(status=BlockStatus.failed, error_codes=["picked"], failure_reason="Failed")- assert "error_codes" not in app.DATABASE.observer.update_workflow_run_block.await_args.kwargs+ assert app.DATABASE.observer.update_workflow_run_block.await_args.kwargs["error_codes"] is None@pytest.mark.asynciodiff --git a/tests/unit/test_script_service_fallback_engine.py b/tests/unit/test_script_service_fallback_engine.pyindex 1bc0be89a..591931152 100644--- a/tests/unit/test_script_service_fallback_engine.py+++ b/tests/unit/test_script_service_fallback_engine.py@@ -32,6 +32,8 @@ from skyvern.schemas.workflows import BlockType, WorkflowDefinitionYAMLfrom skyvern.services import script_servicefrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.actions.actions import Action, ActionStatus+from skyvern.webeye.real_browser_manager import RealBrowserManager+from tests.unit._task_block_fakes import lost_browser_state, run_bound_browser_statefrom tests.unit._workflow_block_engine_fakes import FakeExperimentationProvider, resolve_armMODULE = "skyvern.services.script_service"@@ -129,6 +131,71 @@ async def test_fallback_inherits_engine_from_run_bound_definition() -> None:assert _fallback_task_block(app).engine == RunEngine.skyvern_v3+async def test_fallback_does_not_run_the_agent_after_the_browser_was_lost() -> None:+ app = _make_app(_make_workflow([_make_task_block("my_block")]))+ app.BROWSER_MANAGER.get_for_workflow_run = MagicMock(return_value=lost_browser_state("wr_test"))+ update_block = AsyncMock()+ with (+ patch(f"{MODULE}.app", app),+ patch(f"{MODULE}.skyvern_context.current", return_value=_make_context()),+ patch(f"{MODULE}._update_workflow_block", update_block),+ ):+ await script_service._fallback_to_ai_run(+ block_type=BlockType.NAVIGATION,+ cache_key="my_block",+ prompt="do the thing",+ error=RuntimeError("Target page, context or browser has been closed"),+ workflow_run_block_id="wrb_script",+ )++ app.agent.execute_step.assert_not_awaited()+ assert update_block.await_args.kwargs["error_codes"] == ["browser_connection_lost"]++[email protected]("held_by", [None, "wr_child"])+async def test_adaptive_fallback_capture_does_not_acquire_a_browser(held_by: str | None) -> None:+ workflow = _make_workflow([_make_task_block("my_block")]).model_copy(update={"run_with": "code", "code_version": 2})+ app = _make_app(workflow)+ app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(+ return_value=MagicMock(+ workflow_run_id="wr_test",+ organization_id="o_test",+ parent_workflow_run_id=None,+ browser_profile_id=None,+ ai_fallback=True,+ run_with=None,+ )+ )+ app.DATABASE.scripts.create_fallback_episode = AsyncMock(return_value=SimpleNamespace(episode_id="ep_test"))+ app.DATABASE.tasks.create_step = AsyncMock(side_effect=RuntimeError("stop after the capture"))+ manager = RealBrowserManager()+ if held_by is not None:+ # A finished synchronous child still holds the run's shared browser.+ state, _ = run_bound_browser_state(held_by)+ manager.pages["wr_test"] = state+ pages_before = dict(manager.pages)+ app.BROWSER_MANAGER = manager+ context = _make_context()+ context.workflow_permanent_id = "wpid_test"+ with (+ patch(f"{MODULE}.app", app),+ patch(f"{MODULE}.skyvern_context.current", return_value=context),+ patch.object(manager, "_create_browser_state", new=AsyncMock()) as mock_create,+ pytest.raises(RuntimeError, match="stop after the capture"),+ ):+ await script_service._fallback_to_ai_run(+ block_type=BlockType.NAVIGATION, cache_key="my_block", prompt="do the thing", error=RuntimeError("failed")+ )++ app.DATABASE.scripts.create_fallback_episode.assert_awaited_once()+ mock_create.assert_not_awaited()+ assert manager.pages == pages_before+ if held_by is not None:+ assert manager.pages["wr_test"].runtime_event_context.workflow_run_id == held_by++@pytest.mark.asyncioasync def test_fallback_keeps_default_engine_when_block_engine_is_default() -> None:workflow = _make_workflow([_make_task_block("my_block", engine=RunEngine.skyvern_v1)])diff --git a/tests/unit/test_script_skyvern_page.py b/tests/unit/test_script_skyvern_page.pyindex 765e16949..e34cf23ea 100644--- a/tests/unit/test_script_skyvern_page.py+++ b/tests/unit/test_script_skyvern_page.py@@ -21,6 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport httpximport pytest+from playwright.async_api import Error as PlaywrightErrorfrom playwright.async_api import TimeoutError as PlaywrightTimeoutErrorimport skyvern.forge.api_app as api_app_module@@ -30,12 +31,16 @@ import skyvern.forge.sdk.artifact.storage.local as local_storage_moduleimport skyvern.forge.sdk.workflow.service as workflow_service_moduleimport skyvern.library.embedded_server_factory as embedded_server_modulefrom skyvern.config import settings+from skyvern.constants import NAVIGATION_MAX_RETRY_TIME+from skyvern.core.script_generations import skyvern_page as skyvern_page_modulefrom skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAifrom skyvern.core.script_generations.script_skyvern_page import ScriptSkyvernPagefrom skyvern.core.script_generations.skyvern_page import ResolvedSensitiveValue, SkyvernPagefrom skyvern.errors.errors import UserDefinedErrorfrom skyvern.exceptions import (+ FailedToNavigateToUrl,IllegitCompleteScriptTermination,+ MissingBrowserState,NoTOTPSecretFound,ScriptTerminationException,SkyvernActionFailed,@@ -54,10 +59,14 @@ from skyvern.forge.sdk.workflow.service import WorkflowServicefrom skyvern.library.skyvern import Skyvernfrom skyvern.library.skyvern_browser_page import SkyvernBrowserPagefrom skyvern.services import script_service+from skyvern.webeye import navigation as navigation_modulefrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.actions.handler import ActionHandlerfrom skyvern.webeye.actions.responses import ActionAbort, ActionFailure, ActionSuccessfrom skyvern.webeye.skycdp.facade.page import Frame as CdpFrame+from skyvern.webeye.utils.page import SkyvernFrame+from tests.unit._task_block_fakes import run_bound_browser_state+from tests.unit.scoped_asyncio import ScopedAsyncioclass _KeyboardStub:@@ -176,6 +185,51 @@ async def test_script_page_reports_deferred_navigation_failure(mock_scraped_pagemock_scraped_page._browser_state.record_navigation_result.assert_awaited_once_with(error)+ ("seam", "dispatches", "backoffs"),+ [+ ("native_close_during_goto", 1, 0),+ ("ordinary_navigation_error", 5, 4),+ ("native_close_during_backoff", 1, 1),+ ("lost_before_first_dispatch", 0, 0),+ ],+)+async def test_script_page_stops_retrying_navigation_after_native_browser_loss(+ seam: str, dispatches: int, backoffs: int, mock_ai, monkeypatch: pytest.MonkeyPatch+) -> None:+ from skyvern.core.script_generations import script_skyvern_page as module++ browser_state, context = run_bound_browser_state("wr_script")+ context._skyvern_cdp_download_interceptor = None+ scraped_page = SimpleNamespace(_browser_state=browser_state)+ page, raw_page = _build_navigation_page(ScriptSkyvernPage, scraped_page, mock_ai)++ async def goto(url: str, **_: object) -> None:+ if seam == "native_close_during_goto":+ browser_state._on_browser_context_closed(context)+ raise PlaywrightError("Target page, context or browser has been closed")+ raise PlaywrightError("Timeout 30000ms exceeded.")++ async def lose_browser_while_waiting(_: float) -> None:+ if seam == "native_close_during_backoff":+ browser_state._on_browser_context_closed(context)++ raw_page.goto = AsyncMock(side_effect=goto)+ backoff = AsyncMock(side_effect=lose_browser_while_waiting)+ monkeypatch.setattr(module, "asyncio", ScopedAsyncio(sleep=backoff))+ if seam == "lost_before_first_dispatch":+ browser_state._on_browser_context_closed(context)++ with pytest.raises(MissingBrowserState if seam == "lost_before_first_dispatch" else PlaywrightError) as error:+ await page.goto("https://93.184.216.34/path", max_retries=5)++ assert raw_page.goto.await_count == dispatches+ assert backoff.await_count == backoffs+ if seam == "native_close_during_backoff":+ assert "Timeout 30000ms exceeded." in str(error.value)++@pytest.mark.asyncioasync def test_script_service_goto_fallback_rejects_blocked_destination(monkeypatch: pytest.MonkeyPatch,@@ -1643,6 +1697,7 @@ async def test_input_text_rejects_raw_totp_markers_in_direct_mode(mock_scraped_p@pytest.mark.asyncioasync def test_real_page_ai_rejects_raw_totp_placeholder_before_fallback() -> None:page_ai = RealSkyvernPageAi.__new__(RealSkyvernPageAi)+ page_ai.scraped_page = SimpleNamespace(_browser_state=None)page_ai._maybe_run_v3_midrun = AsyncMock(return_value=(None, False))with (@@ -2975,6 +3030,183 @@ async def test_post_dispatch_timeout_after_escape_skips_ai_fallback(mock_ai, engmock_ai.ai_click.assert_not_awaited()+_TARGET_CLOSED = "Target page, context or browser has been closed"+++def _run_bound_script_page(ai: object, locators: dict[str, object]) -> tuple[ScriptSkyvernPage, object, MagicMock]:+ """A cached-script page whose scraped page carries a real run-bound browser state."""+ browser_state, browser_context = run_bound_browser_state("wr_script")+ raw_page = create_mock_page()+ raw_page.keyboard = MagicMock(press=AsyncMock())+ with patch("skyvern.core.script_generations.skyvern_page.Page.__init__", return_value=None):+ script_page = ScriptSkyvernPage(+ scraped_page=SimpleNamespace(_browser_state=browser_state), page=raw_page, ai=ai+ )+ script_page._working_frame = MagicMock(page=raw_page, locator=MagicMock(side_effect=lambda s, **_: locators[s]))+ return script_page, browser_state, browser_context+++ # expected: page-ready waits, selector waits, clicks, Escape presses, AI fallbacks+ ("seam", "expected"),+ [+ ("lost_before_the_action", [0, 0, 0, 0, 0]),+ ("lost_during_the_page_ready_wait", [1, 0, 0, 0, 0]),+ ("lost_during_the_selector_backoff", [1, 1, 0, 0, 0]),+ ("lost_with_the_click_failure", [1, 1, 1, 0, 0]),+ ("lost_with_a_post_dispatch_click_timeout", [1, 1, 1, 0, 0]),+ ("lost_during_the_escape_pause", [1, 1, 1, 1, 0]),+ ("lost_during_the_reclick", [1, 1, 2, 1, 0]),+ ("ordinary_selector_timeout", [1, 3, 1, 1, 1]),+ ("ordinary_post_dispatch_click_timeout", [1, 1, 1, 0, 0]),+ ],+)+async def test_cached_click_dispatches_nothing_after_native_browser_loss(+ monkeypatch: pytest.MonkeyPatch, seam: str, expected: list[int]+) -> None:+ ai = _AiStub()+ ai.ai_click = AsyncMock(return_value="xpath=//button")+ locator = MagicMock()+ locator.first = locator+ page, browser_state, browser_context = _run_bound_script_page(ai, {"#submit": locator})+ calls: list[str] = []+ dispatched = PlaywrightTimeoutError("click action done; waiting for scheduled navigations to finish")+ intercepted = PlaywrightTimeoutError("<div class='overlay'></div> intercepts pointer events")+ timed_out = PlaywrightTimeoutError("Timeout 5000ms exceeded.")+ # Per click: (whether the native loss latches during it, what it raises).+ click_outcomes = {+ "lost_with_the_click_failure": [(True, PlaywrightError(_TARGET_CLOSED))],+ "lost_with_a_post_dispatch_click_timeout": [(True, dispatched)],+ "lost_during_the_escape_pause": [(False, intercepted)],+ "lost_during_the_reclick": [(False, intercepted), (True, PlaywrightError(_TARGET_CLOSED))],+ "ordinary_selector_timeout": [(False, timed_out)],+ "ordinary_post_dispatch_click_timeout": [(False, dispatched)],+ }++ def lose_browser() -> None:+ browser_state._on_browser_context_closed(browser_context)++ async def wait_for(*, state: str, **_: object) -> None:+ if state == "attached":+ calls.append("attached")+ if browser_state.get_browser_connection_loss() is not None:+ raise PlaywrightError(_TARGET_CLOSED)+ if seam in ("lost_during_the_selector_backoff", "ordinary_selector_timeout"):+ raise timed_out++ async def click(**_: object) -> None:+ calls.append("click")+ if browser_state.get_browser_connection_loss() is not None:+ raise PlaywrightError(_TARGET_CLOSED)+ latches, error = click_outcomes[seam][calls.count("click") - 1]+ if latches:+ lose_browser()+ raise error++ async def pause(seconds: float) -> None:+ if (seam, seconds) in (("lost_during_the_selector_backoff", 1.0), ("lost_during_the_escape_pause", 0.3)):+ lose_browser()++ async def wait_for_page_ready() -> None:+ calls.append("page_ready")+ if seam == "lost_during_the_page_ready_wait":+ lose_browser()++ locator.wait_for = AsyncMock(side_effect=wait_for)+ locator.scroll_into_view_if_needed = AsyncMock()+ locator.click = AsyncMock(side_effect=click)+ monkeypatch.setattr(page, "_wait_for_page_ready_before_action", wait_for_page_ready)+ monkeypatch.setattr(skyvern_page_module, "asyncio", ScopedAsyncio(sleep=pause))+ if seam == "lost_before_the_action":+ lose_browser()++ with skyvern_context.scoped(SkyvernContext(code_version=2)):+ if seam.startswith("lost"):+ with pytest.raises(MissingBrowserState):+ await page.click("#submit", prompt="Click submit")+ else:+ result = await page.click("#submit", prompt="Click submit")+ assert result == ("xpath=//button" if seam == "ordinary_selector_timeout" else "#submit")++ escapes, ai_clicks = page.page.keyboard.press.await_count, ai.ai_click.await_count+ assert [*(calls.count(call) for call in ("page_ready", "attached", "click")), escapes, ai_clicks] == expected+++ ("browser_lost", "mapping"),+ [(True, {0: "Noor", 1: "Blue"}), (True, {0: "Noor"}), (False, {0: "Noor", 1: "Blue"})],+ ids=["lost_before_the_next_field", "lost_on_the_last_field", "ordinary_field_failure"],+)+async def test_fill_from_mapping_stops_at_the_field_after_native_browser_loss(+ browser_lost: bool, mapping: dict[int, str]+) -> None:+ ai = _AiStub()+ ai.ai_input_text = AsyncMock(return_value="Noor")+ name, color = MagicMock(), MagicMock(select_option=AsyncMock())+ name.first = name+ page, browser_state, browser_context = _run_bound_script_page(ai, {"#name": name, "#color": color})++ async def fill(_value: str, **_: object) -> None:+ if browser_lost:+ browser_state._on_browser_context_closed(browser_context)+ raise PlaywrightError(_TARGET_CLOSED)+ raise RuntimeError("element is not attached to the DOM")++ name.fill = AsyncMock(side_effect=fill)+ page._wait_for_page_ready_before_action = AsyncMock() # type: ignore[method-assign]+ form_fields = [+ {"selector": "#name", "type": "text", "tag": "input", "label": "Name"},+ {"selector": "#color", "type": "select-one", "tag": "select", "label": "Color"},+ ]++ with pytest.raises(MissingBrowserState) if browser_lost else contextlib.nullcontext():+ await page.fill_from_mapping(form_fields, mapping)++ assert color.select_option.await_count == ai.ai_input_text.await_count == (0 if browser_lost else 1)+++ "seam", ["lost_during_the_next_click", "lost_during_the_page_transition", "ordinary_last_page"]+)+async def test_fill_multipage_form_reports_no_pages_after_native_browser_loss(+ monkeypatch: pytest.MonkeyPatch, seam: str+) -> None:+ page, browser_state, browser_context = _run_bound_script_page(_AiStub(), {})+ pages = [[{"selector": f"#{name}", "type": "text", "tag": "input", "label": name}] for name in ("name", "email")]+ extract_form_fields = AsyncMock(side_effect=[pages[0], pages[0], pages[1], pages[1]])++ async def click_next(**_: object) -> None:+ if seam == "lost_during_the_next_click":+ browser_state._on_browser_context_closed(browser_context)+ if seam != "lost_during_the_page_transition":+ raise RuntimeError("no next button")++ async def pause(seconds: float) -> None:+ if seam == "lost_during_the_page_transition" and seconds == 1:+ browser_state._on_browser_context_closed(browser_context)++ monkeypatch.setattr(page, "extract_form_fields", extract_form_fields)+ monkeypatch.setattr(page, "dynamic_field_map", AsyncMock(return_value={0: "Noor"}))+ monkeypatch.setattr(page, "fill_from_mapping", AsyncMock())+ monkeypatch.setattr(page, "click", AsyncMock(side_effect=click_next))+ monkeypatch.setattr(page.page, "evaluate", AsyncMock(return_value=[]), raising=False)+ ready_frame = SimpleNamespace(wait_for_page_ready=AsyncMock())+ monkeypatch.setattr(SkyvernFrame, "create_instance", AsyncMock(return_value=ready_frame))+ monkeypatch.setattr(skyvern_page_module, "asyncio", ScopedAsyncio(sleep=pause))++ if seam == "ordinary_last_page":+ assert await page.fill_multipage_form({"name": "Noor"}, max_pages=2) == 1+ else:+ with pytest.raises(MissingBrowserState):+ await page.fill_multipage_form({"name": "Noor"}, max_pages=2)++ assert extract_form_fields.await_count == 2+ assert page.dynamic_field_map.await_count == 1++@pytest.mark.asyncioasync def test_get_or_create_browser_state_forwards_script_id_to_manager(monkeypatch):# MUST_FIX 1: the production script caller must forward the active run's script_id so the@@ -3155,6 +3387,7 @@ class TestScriptSkyvernPageMagicLink:page = SimpleNamespace(goto=fake_goto, fill=fake_fill)script_page = ScriptSkyvernPage.__new__(ScriptSkyvernPage)script_page.page = page+ module._set_scraped_page(script_page, SimpleNamespace(_browser_state=None))return script_page, state@staticmethod@@ -3192,6 +3425,39 @@ class TestScriptSkyvernPageMagicLink:expected_anchor = datetime.now(timezone.utc).replace(tzinfo=None) - MAGIC_LINK_ANCHOR_GRACEassert abs((anchor - expected_anchor).total_seconds()) < 30+ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "native_close", [True, False], ids=["native_close_during_goto", "ordinary_navigation_error"]+ )+ async def test_link_navigation_stops_retrying_after_native_browser_loss(self, monkeypatch, native_close) -> None:+ from skyvern.core.script_generations import script_skyvern_page as module+ from skyvern.forge.sdk.schemas.totp_codes import OTPType+ from skyvern.services.otp_service import OTPValue++ link = "https://portal.example.com/verify?token=x"+ script_page, state = self._install(monkeypatch, OTPValue(value=link, type=OTPType.MAGIC_LINK))+ browser_state, context = run_bound_browser_state("wr_1")+ module._set_scraped_page(script_page, SimpleNamespace(_browser_state=browser_state))+ monkeypatch.setattr(navigation_module, "asyncio", ScopedAsyncio(sleep=AsyncMock()))++ async def goto(url, **kwargs):+ state["navigations"].append(url)+ if native_close:+ browser_state._on_browser_context_closed(context)+ raise PlaywrightError("Target page, context or browser has been closed")+ raise PlaywrightError("Timeout 30000ms exceeded.")++ script_page.page.goto = goto++ ctx = self._with_context()+ try:+ with pytest.raises(FailedToNavigateToUrl):+ await script_page.magic_link(totp_identifier="[email protected]")+ finally:+ ctx.reset()++ assert state["navigations"] == [link] * (1 if native_close else NAVIGATION_MAX_RETRY_TIME)+@pytest.mark.asyncioasync def test_a_returned_one_time_code_is_reported_not_navigated_to(self, monkeypatch) -> None:"""A configured webhook answers without honouring expected_otp_type."""diff --git a/tests/unit/test_send_email_block.py b/tests/unit/test_send_email_block.pyindex f22c34497..74923432b 100644--- a/tests/unit/test_send_email_block.py+++ b/tests/unit/test_send_email_block.py@@ -19,6 +19,7 @@ from __future__ import annotationsimport astimport asyncio+import contextlibimport loggingimport osimport pickle@@ -49,7 +50,7 @@ from sqlalchemy.orm import undeferfrom skyvern.config import settingsfrom skyvern.core.script_generations.generate_script import _build_send_email_statement-from skyvern.exceptions import BlockedHost, UnresolvableHost+from skyvern.exceptions import BlockedHost, MissingBrowserState, UnresolvableHostfrom skyvern.forge import appfrom skyvern.forge.sdk.api import email as email_apifrom skyvern.forge.sdk.api.email import InvalidEmailRecipient, send, validate_recipients@@ -105,6 +106,7 @@ from skyvern.schemas.workflows import ()from skyvern.services import script_servicefrom skyvern.services.email import gmail as gmail_service+from tests.unit._task_block_fakes import run_bound_browser_statefrom tests.unit.scoped_asyncio import ScopedAsyncio@@ -1868,6 +1870,48 @@ async def test_cached_nested_loop_records_the_inner_loop_in_each_outer_iteration] == [[("inner_output", "one", len(inner_items))], [("inner_output", "two", len(inner_items))]]+ "seam", ["lost_during_the_send", "lost_before_the_send", "lost_before_a_wait", "ordinary_send_failure"]+)+async def test_cached_script_runs_no_statement_after_its_browser_was_lost(+ gmail_env: GmailEnv, monkeypatch: pytest.MonkeyPatch, seam: str+) -> None:+ await _cached_run(gmail_env, "wr_lost", _gmail_block(continue_on_failure=True))+ browser_state, browser_context = run_bound_browser_state("wr_lost")+ monkeypatch.setattr(app.BROWSER_MANAGER, "get_for_workflow_run", lambda *_args, **_kwargs: browser_state)++ def respond(request: httpx.Request) -> httpx.Response:+ if seam == "lost_during_the_send":+ browser_state._on_browser_context_closed(browser_context)+ return _accepted(request) if seam == "lost_before_the_send" else _rejected(request)++ gmail_env.respond = respond+ if seam.startswith("lost_before"):+ browser_state._on_browser_context_closed(browser_context)+ statements: list[str] = []++ async def generated_function() -> None:+ if seam == "lost_before_a_wait":+ await script_service.wait(0, label="pause")+ else:+ await script_service.send_email(transport="gmail", credential_id=GMAIL_CREDENTIAL, label="send_email")+ statements.append("next statement")++ try:+ with contextlib.nullcontext() if seam == "ordinary_send_failure" else pytest.raises(MissingBrowserState):+ await generated_function()+ finally:+ skyvern_context.reset()++ started = seam.startswith("lost_before")+ assert statements == (["next statement"] if seam == "ordinary_send_failure" else [])+ assert len(gmail_env.requests) == (0 if started else 1)+ assert len([row for row in await gmail_env.rows(WorkflowRunBlockModel) if row.workflow_run_id == "wr_lost"]) == (+ 0 if started else 1+ )++def test_gmail_send_block_carries_the_same_fields_through_yaml_conversion_reload_and_codegen() -> None:authored = {"transport": "gmail",diff --git a/tests/unit/test_task_v2_completion_gate.py b/tests/unit/test_task_v2_completion_gate.pyindex 77b56fc4e..36710e62c 100644--- a/tests/unit/test_task_v2_completion_gate.py+++ b/tests/unit/test_task_v2_completion_gate.py@@ -13,12 +13,16 @@ from unittest.mock import AsyncMock, MagicMockimport pytestfrom skyvern.config import Settings+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODEfrom skyvern.forge.sdk.prompting import PromptEnginefrom skyvern.forge.sdk.schemas.task_v2 import TaskV2Statusfrom skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition, WorkflowRunStatusfrom skyvern.forge.sdk.workflow.service import WorkflowService+from skyvern.schemas.workflows import BlockStatusfrom skyvern.services import planner_levers, task_v2_servicefrom skyvern.services.task_v2_service import _should_run_post_block_completion_check+from skyvern.webeye.real_browser_manager import RealBrowserManager+from tests.unit._task_block_fakes import run_bound_browser_stateclass _TaskV2WiringObserved(Exception):@@ -434,6 +438,251 @@ async def test_missing_task_type_failure_finalizes_with_organization_scope(workflow_service.mark_workflow_run_as_failed.assert_awaited_once()+ ("achieved_by", "lost_during"),+ [+ ("completion_check", "completion_scrape"),+ ("completion_check", "completion_check"),+ ("planning", "planning"),+ ("completion_check", "completion_screenshots"),+ ("completion_check", "summary"),+ ("planning", "summary"),+ ("completion_check", None),+ ("completion_check", "expected_teardown"),+ ("completion_check", "run_canceled"),+ ("planning", "task_canceled"),+ ("completion_check", "run_write_refused"),+ ("block", "block"),+ ("block", "run_canceled"),+ ("block", "task_canceled"),+ ],+)+async def test_task_v2_cannot_complete_after_its_browser_was_lost(+ monkeypatch: pytest.MonkeyPatch, achieved_by: str, lost_during: str | None+) -> None:+ now = datetime.now(UTC)+ organization = SimpleNamespace(organization_id="org_test", max_steps_per_run=None)+ task_v2 = MagicMock(+ observer_cruise_id="task_v2_test",+ status=TaskV2Status.queued,+ prompt="finish the task",+ workflow_system_prompt=None,+ run_with="code",+ started_at=now,+ created_at=now,+ )+ task_v2.url, task_v2.workflow_run_id = "https://example.test", "wr_test"+ workflow = MagicMock(+ workflow_id="wf_test",+ workflow_permanent_id="wp_test",+ title="Test",+ workflow_definition=WorkflowDefinition(parameters=[], blocks=[]),+ )+ runs = {+ status: MagicMock(+ workflow_run_id="wr_test",+ workflow_id="wf_test",+ status=status,+ parent_workflow_run_id=None,+ browser_profile_id=None,+ )+ for status in (+ WorkflowRunStatus.queued,+ WorkflowRunStatus.running,+ WorkflowRunStatus.failed,+ WorkflowRunStatus.canceled,+ )+ }+ run_status = [WorkflowRunStatus.queued]++ async def get_workflow_run(*_args: object, **_kwargs: object) -> MagicMock:+ run = runs[run_status[0]]+ if run_status[0] == WorkflowRunStatus.queued:+ run_status[0] = WorkflowRunStatus.running+ return run++ async def fail_run_if_not_final(**_kwargs: object) -> None:+ # A fenced write leaves the run with the owner that refused it.+ if lost_during != "run_write_refused" and not run_status[0].is_final():+ run_status[0] = WorkflowRunStatus.failed++ async def cancel_run(*_args: object, **_kwargs: object) -> None:+ run_status[0] = WorkflowRunStatus.canceled++ mark_failed = AsyncMock(side_effect=fail_run_if_not_final)+ lost_in_block = achieved_by == "block"+ # A block whose browser is lost while it runs returns its failure with the code from the block's own seam.+ block_result = MagicMock(+ failure_reason="The browser connection was lost." if lost_in_block else None,+ output_parameter_value={},+ success=not lost_in_block,+ status=BlockStatus.failed if lost_in_block else BlockStatus.completed,+ error_codes=[BROWSER_CONNECTION_LOST_ERROR_CODE] if lost_in_block else [],+ workflow_run_block_id="wrb_navigate",+ )++ async def execute_safe(**_kwargs: object) -> MagicMock:+ if lost_in_block:+ await lose_browser_after_any_cancellation()+ return block_result++ block = SimpleNamespace(+ execute_safe=AsyncMock(side_effect=execute_safe),+ build_block_result=AsyncMock(),+ block_type="navigation",+ label="navigate",+ continue_on_failure=False,+ )+ state, browser_context = run_bound_browser_state("wr_test")+ state.get_working_page = AsyncMock(return_value=MagicMock()) # type: ignore[method-assign]+ state.validate_browser_context = AsyncMock(return_value=True) # type: ignore[method-assign]+ scrapes: list[str] = []++ async def scrape_website(**_kwargs: object) -> SimpleNamespace:+ scrapes.append("scrape")+ # The planner reads the page first, the completion check second.+ if lost_during == "completion_scrape" and len(scrapes) == 2:+ state._on_browser_context_closed(browser_context)+ raise RuntimeError("Target page, context or browser has been closed")+ return SimpleNamespace(screenshots=[])++ state.scrape_website = scrape_website # type: ignore[method-assign]+ manager = RealBrowserManager()+ manager.pages["wr_test"] = state++ async def update_task_v2(*_args: object, **kwargs: object) -> MagicMock:+ if status := kwargs.get("status"):+ task_v2.status = status+ return task_v2++ async def update_task_v2_if_not_final(*args: object, **kwargs: object) -> bool:+ if task_v2.status.is_final():+ return False+ await update_task_v2(*args, **kwargs)+ return True++ def close_browser(point: str) -> None:+ if lost_during == point:+ state._on_browser_context_closed(browser_context)+ elif lost_during == "expected_teardown" and point == "summary":+ state.mark_run_released()+ state._on_browser_context_closed(browser_context)++ async def screenshots(**_kwargs: object) -> None:+ close_browser("completion_screenshots")++ async def lose_browser_after_any_cancellation() -> None:+ # The browser is lost while the deciding unit runs, after any cancellation that lands during it.+ if lost_during == "task_canceled":+ await task_v2_service.mark_task_v2_as_canceled(+ "task_v2_test", workflow_run_id="wr_test", organization_id="org_test"+ )+ elif lost_during == "run_canceled":+ await cancel_run()+ state._on_browser_context_closed(browser_context)++ async def llm(*, prompt_name: str, **_kwargs: object) -> dict[str, Any]:+ point = {"task_v2_check_completion": "completion_check", "task_v2_summary": "summary"}.get(+ prompt_name, "planning"+ )+ if lost_during in ("run_canceled", "task_canceled", "run_write_refused") and point == achieved_by:+ await lose_browser_after_any_cancellation()+ close_browser(point)+ if point == "summary":+ return {"description": "done", "output": {"done": True}}+ if point == "completion_check" or achieved_by == "planning":+ return {"user_goal_achieved": True, "should_terminate": False, "thoughts": "done"}+ return {"user_goal_achieved": False, "should_terminate": False, "plan": "continue", "task_type": "navigate"}++ llm_handler = AsyncMock(side_effect=llm)+ test_app = MagicMock()+ test_app.scrape_exclude = []+ test_app.BROWSER_MANAGER = manager+ test_app.WORKFLOW_SERVICE.get_workflow_run = AsyncMock(side_effect=get_workflow_run)+ test_app.WORKFLOW_SERVICE.get_workflow = AsyncMock(return_value=workflow)+ test_app.WORKFLOW_SERVICE.mark_workflow_run_as_running = AsyncMock()+ test_app.WORKFLOW_SERVICE.mark_workflow_run_as_failed_if_not_final = mark_failed+ test_app.WORKFLOW_SERVICE.mark_workflow_run_as_canceled = AsyncMock(side_effect=cancel_run)+ test_app.WORKFLOW_SERVICE.mark_workflow_run_as_completed = mark_completed = AsyncMock()+ test_app.WORKFLOW_SERVICE.generate_script_if_needed = finalize_script = AsyncMock()+ test_app.WORKFLOW_SERVICE.create_workflow_from_request = AsyncMock(return_value=workflow)+ test_app.AGENT_FUNCTION.has_code_block_access = AsyncMock(return_value=False)+ test_app.AGENT_FUNCTION.validate_task_execution = AsyncMock()+ test_app.AGENT_FUNCTION.cleanup_element_tree_factory = MagicMock(return_value=MagicMock())+ test_app.DATABASE.observer.get_task_v2 = AsyncMock(return_value=task_v2)+ test_app.DATABASE.observer.update_task_v2 = AsyncMock(side_effect=update_task_v2)+ test_app.DATABASE.observer.update_task_v2_status_if_not_final = AsyncMock(side_effect=update_task_v2_if_not_final)+ test_app.DATABASE.observer.create_thought = AsyncMock(return_value=SimpleNamespace(observer_thought_id="thought"))+ test_app.DATABASE.observer.update_thought = AsyncMock()+ context = SimpleNamespace(run_id=None, root_workflow_run_id=None, tz_info=timezone.utc)+ monkeypatch.setattr(task_v2_service, "app", test_app)+ monkeypatch.setattr(+ planner_levers.app.EXPERIMENTATION_PROVIDER, "is_feature_enabled_cached", AsyncMock(return_value=False)+ )+ monkeypatch.setattr(planner_levers.settings, "TASK_V2_CONVERGE_PCT", 0)+ monkeypatch.setattr(planner_levers.settings, "TASK_V2_CARRY_SUBGOALS", False)+ monkeypatch.setattr(planner_levers.settings, "TASK_V2_SKIP_COMPLETION_CHECK_AFTER_NAVIGATE", False)+ monkeypatch.setattr(task_v2_service.skyvern_context, "ensure_context", lambda: context)+ monkeypatch.setattr(task_v2_service, "initialize_task_v2_metadata", AsyncMock(return_value=task_v2))+ monkeypatch.setattr(task_v2_service, "_set_up_workflow_context", AsyncMock())+ monkeypatch.setattr(task_v2_service, "_resolve_max_iterations", lambda _override: 1)+ monkeypatch.setattr(task_v2_service, "_is_planner_mini_goal_improvements_enabled", AsyncMock(return_value=False))+ monkeypatch.setattr(task_v2_service, "build_open_tabs_context", AsyncMock(return_value=None))+ monkeypatch.setattr(task_v2_service, "load_prompt_with_elements", MagicMock(return_value="prompt"))+ monkeypatch.setattr(task_v2_service, "_get_task_v2_llm_api_handler", lambda _task_v2: llm_handler)+ monkeypatch.setattr(task_v2_service.SkyvernFrame, "get_url", AsyncMock(return_value=task_v2.url))+ monkeypatch.setattr(task_v2_service, "_generate_navigation_task", AsyncMock(return_value=(block, [], [])))+ monkeypatch.setattr(task_v2_service, "WorkflowDefinitionYAML", MagicMock())+ monkeypatch.setattr(task_v2_service, "WorkflowCreateYAMLRequest", MagicMock())+ monkeypatch.setattr(task_v2_service, "runtime_proxy_location", lambda _location: None)+ monkeypatch.setattr(task_v2_service, "_get_extracted_data_from_block_result", lambda *_args, **_kwargs: None)+ monkeypatch.setattr(task_v2_service, "_persist_completion_tab_screenshots", screenshots)+ callbacks: list[TaskV2Status] = []++ async def send_webhook(task: MagicMock, **_kwargs: object) -> None:+ callbacks.append(task.status)++ monkeypatch.setattr(task_v2_service, "send_task_v2_webhook", send_webhook)++ await task_v2_service.run_task_v2_helper(organization, task_v2, max_iterations_override=1)++ prompts = [call.kwargs["prompt_name"] for call in llm_handler.await_args_list]+ checked = achieved_by == "completion_check" and lost_during != "completion_scrape"+ assert prompts.count("task_v2_check_completion") == checked+ assert prompts.count("task_v2_summary") == (+ lost_during in (None, "expected_teardown", "summary", "completion_screenshots")+ )+ # A cached script is finalized only from a TaskV2 that completed.+ assert finalize_script.await_count == (lost_during in (None, "expected_teardown"))+ if lost_during in (None, "expected_teardown"):+ assert task_v2.status == TaskV2Status.completed+ mark_completed.assert_awaited_once_with("wr_test")+ assert callbacks == [TaskV2Status.completed]+ mark_failed.assert_not_awaited()+ return+ if lost_during == "run_write_refused":+ # The task did not end, so there is no terminal state to report.+ assert task_v2.status == TaskV2Status.running+ assert callbacks == []+ else:+ # The terminal callback reports the state the task actually ended in, exactly once; a cancellation wins.+ ended = TaskV2Status.canceled if lost_during.endswith("canceled") else TaskV2Status.failed+ assert task_v2.status == ended+ assert callbacks == [ended]+ mark_failed.assert_awaited_once()+ mark_completed.assert_not_awaited()+ if achieved_by == "planning":+ block.execute_safe.assert_not_awaited()+ elif lost_in_block:+ # The block's own row already carries its failure and code, and the run keeps the block's reason.+ block.build_block_result.assert_not_awaited()+ assert mark_failed.await_args.kwargs["failure_reason"] == block_result.failure_reason+ else:+ assert block.build_block_result.await_args.kwargs["status"] == BlockStatus.failed+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in block.build_block_result.await_args.kwargs["error_codes"]++@pytest.mark.asyncio@pytest.mark.parametrize("provider_enabled,settings_default,expected",diff --git a/tests/unit/test_task_v2_failure_deliverable.py b/tests/unit/test_task_v2_failure_deliverable.pyindex adae26e44..3acb17fc4 100644--- a/tests/unit/test_task_v2_failure_deliverable.py+++ b/tests/unit/test_task_v2_failure_deliverable.py@@ -3,12 +3,14 @@ mark_task_v2_as_* terminal transitions refuse to overwrite a status that is alrefrom __future__ import annotations-from unittest.mock import AsyncMock+from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMockimport pytestfrom skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.schemas.task_v2 import TaskV2Status+from skyvern.schemas.workflows import BlockResult, BlockStatus, BlockTypefrom skyvern.services import task_v2_service@@ -217,3 +219,35 @@ async def test_mark_task_v2_as_failed_redrive_keeps_termination_metadata(monkeypterminate_run.assert_awaited_once_with("wr_test", "late failure", failure_category=[{"category": "unreachable_goal"}])+++ ("error_codes", "fails_run"), [(["browser_connection_lost"], True), ([], False)], ids=["browser_lost", "other"]+)+async def test_failed_block_stops_the_planner_only_when_the_browser_was_lost(+ monkeypatch: pytest.MonkeyPatch, error_codes: list[str], fails_run: bool+) -> None:+ mark_failed = AsyncMock()+ monkeypatch.setattr(task_v2_service.app.WORKFLOW_SERVICE, "mark_workflow_run_as_failed_if_not_final", mark_failed)+ monkeypatch.setattr(task_v2_service.app.WORKFLOW_SERVICE, "get_workflow_run", AsyncMock())+ block = SimpleNamespace(block_type=BlockType.NAVIGATION, label="nav", continue_on_failure=True)+ result = BlockResult(+ success=False,+ output_parameter=MagicMock(),+ status=BlockStatus.failed,+ failure_reason="Browser state page is missing.",+ error_codes=error_codes,+ can_continue_after_failure=not error_codes,+ )++ await task_v2_service.handle_block_result(+ "tsk_v2_test",+ block, # type: ignore[arg-type]+ result,+ MagicMock(),+ SimpleNamespace(workflow_run_id="wr_test", organization_id="o_test"), # type: ignore[arg-type]+ is_last_block=False,+ )++ assert mark_failed.await_count == (1 if fails_run else 0)diff --git a/tests/unit/test_taskv2_block_browser_address_propagation.py b/tests/unit/test_taskv2_block_browser_address_propagation.pyindex dc618d07c..7f7781d72 100644--- a/tests/unit/test_taskv2_block_browser_address_propagation.py+++ b/tests/unit/test_taskv2_block_browser_address_propagation.py@@ -8,12 +8,19 @@ from unittest.mock import AsyncMockimport pytest+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODEfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.schemas.task_v2 import TaskV2Statusfrom skyvern.forge.sdk.workflow.models import block as block_modulefrom skyvern.forge.sdk.workflow.models.block import TaskV2Blockfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus+from skyvern.forge.sdk.workflow.retry_policy import evaluate_retry_policy+from skyvern.schemas.workflows import WorkflowRetryPolicy+from skyvern.webeye.real_browser_manager import RealBrowserManager+from skyvern.webeye.real_browser_state import RealBrowserState+from tests.unit._task_block_fakes import run_bound_browser_state@pytest.fixture(autouse=True)@@ -218,3 +225,108 @@ async def test_taskv2_block_settles_the_child_capture_when_the_runner_raises()assert settled == ["wr_child"]+++def _child_browser_state(*, lost: bool) -> RealBrowserState:+ state, context = run_bound_browser_state("wr_child")+ if lost:+ state._on_browser_context_closed(context)+ return state++[email protected]("mapping", ["persistent_session_child_only", "shared_with_parent"])[email protected]("continue_on_failure", [True, False])[email protected]("child_lost_browser", [True, False], ids=["child_lost_browser", "ordinary_child_failure"])+async def test_taskv2_block_reports_a_child_browser_loss_from_the_child_run_entry(+ monkeypatch: pytest.MonkeyPatch, mapping: str, continue_on_failure: bool, child_lost_browser: bool+) -> None:+ skyvern_context.set(SkyvernContext(organization_id="org_1", workflow_run_id="wr_parent", run_id="wr_parent"))+ child_state = _child_browser_state(lost=child_lost_browser)+ browser_manager = RealBrowserManager()+ # An explicit persistent session registers only the child run; a shared browser is also aliased to the parent.+ browser_manager.pages = (+ {"wr_child": child_state}+ if mapping == "persistent_session_child_only"+ else {"wr_child": child_state, "wr_parent": child_state}+ )+ workflow_run = SimpleNamespace(+ proxy_location=None,+ max_screenshot_scrolls=None,+ browser_address=None,+ extra_http_headers=None,+ cdp_connect_headers=None,+ failure_reason="Browser state page is missing.",+ )+ update_block = AsyncMock()+ fake_app = SimpleNamespace(+ DATABASE=SimpleNamespace(+ organizations=SimpleNamespace(get_organization=AsyncMock(return_value=SimpleNamespace())),+ workflow_runs=SimpleNamespace(+ get_workflow_run=AsyncMock(return_value=workflow_run), update_workflow_run=AsyncMock()+ ),+ observer=SimpleNamespace(update_task_v2=AsyncMock(), update_workflow_run_block=update_block),+ ),+ BROWSER_MANAGER=browser_manager,+ WORKFLOW_SERVICE=SimpleNamespace(+ get_recent_task_screenshot_artifacts=AsyncMock(return_value=[]),+ get_recent_workflow_screenshot_artifacts=AsyncMock(return_value=[]),+ ),+ STORAGE=SimpleNamespace(+ get_current_attempt_downloaded_files=AsyncMock(return_value=[]),+ get_downloaded_file_signature_aliases=lambda _: [],+ ),+ WORKFLOW_CONTEXT_MANAGER=SimpleNamespace(has_workflow_run_context=lambda _run_id: False),+ )+ monkeypatch.setattr(block_module, "app", fake_app)++ from skyvern.services import task_v2_service++ child = SimpleNamespace(+ observer_cruise_id="tsk_v2_1",+ workflow_run_id="wr_child",+ output=None,+ status=TaskV2Status.failed,+ summary=None,+ failure_category=None,+ )+ monkeypatch.setattr(task_v2_service, "initialize_task_v2", AsyncMock(return_value=child))+ monkeypatch.setattr(task_v2_service, "run_task_v2", AsyncMock(return_value=child))+ monkeypatch.setattr(+ TaskV2Block,+ "get_workflow_run_context",+ lambda self, workflow_run_id: SimpleNamespace(credential_totp_identifiers={}),+ )+ monkeypatch.setattr(TaskV2Block, "format_potential_template_parameters", lambda self, _: None)+ monkeypatch.setattr(TaskV2Block, "record_output_parameter_value", AsyncMock())++ block = TaskV2Block(+ label="task1",+ output_parameter=_output_parameter("task1_output"),+ prompt="do the thing",+ url="https://example.com",+ continue_on_failure=continue_on_failure,+ )++ result = await block.execute(workflow_run_id="wr_parent", workflow_run_block_id="wrb_1", organization_id="org_1")++ persisted_codes = {+ code+ for call in update_block.await_args_list+ if call.kwargs.get("workflow_run_block_id") == "wrb_1"+ for code in call.kwargs.get("error_codes") or []+ }+ retry_policy = WorkflowRetryPolicy.model_validate({"max_retries": 2, "retry_on": [{"status": "failed"}]})+ retry = evaluate_retry_policy(retry_policy, WorkflowRunStatus.failed, persisted_codes, 0)+ if child_lost_browser:+ assert result.success is False+ assert result.can_continue_after_failure is False+ assert result.error_codes == [BROWSER_CONNECTION_LOST_ERROR_CODE]+ assert persisted_codes == {BROWSER_CONNECTION_LOST_ERROR_CODE}+ assert retry == (False, BROWSER_CONNECTION_LOST_ERROR_CODE)+ else:+ assert result.success is continue_on_failure+ assert result.can_continue_after_failure is True+ assert result.error_codes == []+ assert persisted_codes == set()+ assert retry == (True, "matched")diff --git a/tests/unit/test_workflow_webhook_delivery.py b/tests/unit/test_workflow_webhook_delivery.pyindex 1062690d9..3a348838b 100644--- a/tests/unit/test_workflow_webhook_delivery.py+++ b/tests/unit/test_workflow_webhook_delivery.py@@ -1588,6 +1588,8 @@ async def test_final_release_retries_keep_own_claim_and_completed_hook(("failed", "failed", ["A", "B"], {"C"}, 0, (False, "no_match")),("failed", "failed", ["A"], set(), 0, (False, "no_match")),("completed", "failed", None, set(), 0, (False, "no_match")),+ ("failed", "failed", None, {"browser_connection_lost"}, 0, (False, "browser_connection_lost")),+ ("failed", "failed", [], {"ANY", "browser_connection_lost"}, 0, (False, "browser_connection_lost")),],)def test_evaluate_retry_policy_decisions(status, rule_status, rule_codes, errors, retries_used, expected) -> None:diff --git a/tests/unit/workflow/test_block_browser_state_recovery.py b/tests/unit/workflow/test_block_browser_state_recovery.pyindex 98aae1f77..34023226e 100644--- a/tests/unit/workflow/test_block_browser_state_recovery.py+++ b/tests/unit/workflow/test_block_browser_state_recovery.py@@ -3,6 +3,7 @@from __future__ import annotationsimport asyncio+import contextlibimport jsonimport socketimport subprocess@@ -11,29 +12,49 @@ from collections.abc import Awaitablefrom datetime import datetime, timedelta, timezonefrom pathlib import Pathfrom types import SimpleNamespace-from unittest.mock import AsyncMock, MagicMock, PropertyMock, call+from unittest.mock import AsyncMock, MagicMock, PropertyMock, call, patchimport pytest+from playwright.async_api import Error as PlaywrightErrorfrom playwright.async_api import async_playwrightfrom structlog.testing import capture_logs+from skyvern.constants import NAVIGATION_MAX_RETRY_TIMEfrom skyvern.exceptions import (+ BROWSER_CONNECTION_LOST_ERROR_CODE,BrowserStateDiagnostic,+ FailedToNavigateToUrl,+ FailedToReloadPage,MissingBrowserState,MissingBrowserStatePage,get_user_facing_exception_message,)from skyvern.forge import app+from skyvern.forge.agent import ForgeAgentfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext-from skyvern.forge.sdk.workflow.models.block import CodeBlock+from skyvern.forge.sdk.models import StepStatus+from skyvern.forge.sdk.schemas.tasks import TaskStatus+from skyvern.forge.sdk.workflow.models import block as block_module+from skyvern.forge.sdk.workflow.models.block import Block, CodeBlock, TaskBlock, UrlBlockfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus+from skyvern.forge.sdk.workflow.retry_policy import evaluate_retry_policy+from skyvern.forge.sdk.workflow.service import WorkflowService+from skyvern.schemas.workflows import BlockResult, BlockStatus, WorkflowRetryPolicy+from skyvern.webeye import navigation as navigation_modulefrom skyvern.webeye import real_browser_state as real_browser_state_modulefrom skyvern.webeye.browser_artifacts import BrowserArtifactsfrom skyvern.webeye.browser_runtime_events import BrowserRuntimeLogContextfrom skyvern.webeye.driver_connection import close_driver_connection_on_transport_loss+from skyvern.webeye.real_browser_manager import RealBrowserManagerfrom skyvern.webeye.real_browser_state import RealBrowserState, expect_process_driver_teardown+from tests.unit._task_block_fakes import lost_browser_state, mock_block_execute_deps, output_parameter+from tests.unit.force_stub_app import admit_block_dispatchfrom tests.unit.forge_log_capture import capture_runtime_logs+from tests.unit.helpers import make_organization, make_step, make_task+from tests.unit.scoped_asyncio import ScopedAsyncio+from tests.unit.test_block_description_caching import _setup_mocksdef _has_playwright_browser() -> bool:@@ -342,6 +363,730 @@ def test_browser_disconnect_event_latches_event_observation() -> None:assert diagnostic.observation_source == "browser_event"+def _bound_state(run_id: str = "wr_test") -> tuple[RealBrowserState, MagicMock, MagicMock]:+ browser = MagicMock()+ browser.is_connected = MagicMock(return_value=True)+ context = MagicMock(browser=browser)+ context._impl_obj = MagicMock(_close_was_called=False, _closed=False, _connection=MagicMock(_closed_error=None))+ state = RealBrowserState(pw=MagicMock(), browser_context=context)+ state.bind_runtime_event_context(BrowserRuntimeLogContext(workflow_run_id=run_id))+ return state, context, browser+++def _probe_timeout(state: RealBrowserState, context: MagicMock, browser: MagicMock) -> None:+ state.record_connection_probe_failure(context, state.pw, timed_out=True)+++ ("events", "terminal"),+ [+ pytest.param([lambda s, c, b: s._on_browser_context_closed(c)], True, id="native_context_close"),+ pytest.param([lambda s, c, b: s._on_browser_disconnected(b)], True, id="native_browser_disconnect"),+ pytest.param(+ [_probe_timeout, lambda s, c, b: s._on_browser_context_closed(c)], True, id="probe_then_native_close"+ ),+ pytest.param(+ [lambda s, c, b: s._on_driver_transport_lost(s.pw, c), lambda s, c, b: s._on_browser_disconnected(b)],+ True,+ id="driver_loss_then_native_disconnect",+ ),+ pytest.param([lambda s, c, b: s._on_driver_transport_lost(s.pw, c)], False, id="driver_loss_only"),+ pytest.param([_probe_timeout], False, id="probe_timeout_only"),+ pytest.param([lambda s, c, b: s.record_connection_probe_failure(c, s.pw)], False, id="probe_failure_only"),+ pytest.param(+ [+ lambda s, c, b: s._expect_runtime_end("normal_close", c),+ lambda s, c, b: s._on_browser_context_closed(c),+ ],+ False,+ id="expected_close",+ ),+ pytest.param(+ [+ lambda s, c, b: s._expect_runtime_end("driver_replacement", c),+ lambda s, c, b: s._on_browser_context_closed(c),+ ],+ False,+ id="expected_driver_replacement",+ ),+ pytest.param(+ [lambda s, c, b: s.mark_run_released(), lambda s, c, b: s._on_browser_context_closed(c)],+ False,+ id="after_release",+ ),+ ],+)+def test_browser_connection_loss_is_only_an_unexpected_active_native_event(+ events: list, terminal: bool, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setattr(app.AGENT_FUNCTION, "on_browser_run_released", MagicMock())+ state, context, browser = _bound_state()++ for event in events:+ event(state, context, browser)++ loss = state.get_browser_connection_loss()+ if terminal:+ assert loss is not None+ assert loss.observation_source == "browser_event"+ else:+ assert loss is None+++def test_browser_connection_loss_ignores_a_stale_generation() -> None:+ state, old_context, old_browser = _bound_state()+ new_context = MagicMock(browser=MagicMock())+ state.browser_context = new_context+ state._register_disconnect_listeners(new_context)++ state._on_browser_context_closed(old_context)+ state._on_browser_disconnected(old_browser)++ assert state.get_browser_connection_loss() is None+++def test_browser_connection_loss_belongs_to_the_run_that_observed_it() -> None:+ state, context, _ = _bound_state("wr_first")+ state._on_browser_context_closed(context)+ assert state.get_browser_connection_loss() is not None++ state.bind_runtime_event_context(BrowserRuntimeLogContext(workflow_run_id="wr_next"))++ assert state.get_browser_connection_loss() is None+++async def test_lost_browser_is_not_reconnected_for_the_same_run(monkeypatch: pytest.MonkeyPatch) -> None:+ state, _, browser = _bound_state()+ browser.is_connected.return_value = False+ state._on_browser_disconnected(browser)+ state.reconnect = AsyncMock() # type: ignore[method-assign]+ monkeypatch.setattr(app.BROWSER_MANAGER, "get_for_workflow_run", MagicMock(return_value=state))+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "get_workflow_run", AsyncMock(return_value=_FakeWorkflowRun()))++ with pytest.raises(MissingBrowserState) as exc_info:+ await _make_code_block().get_or_create_browser_state(workflow_run_id="wr_test", organization_id="o_test")++ assert BROWSER_CONNECTION_LOST_ERROR_CODE in str(exc_info.value)+ state.reconnect.assert_not_awaited()++[email protected]("browser_lost", [True, False], ids=["browser_lost", "browser_alive"])+async def test_block_that_failed_on_a_lost_browser_cannot_be_continued(browser_lost: bool) -> None:+ block = _make_code_block()+ block.continue_on_failure = True+ with (+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,+ patch.object(CodeBlock, "execute", AsyncMock(side_effect=MissingBrowserStatePage(workflow_run_id="wr_test"))),+ patch.object(Block, "_generate_workflow_run_block_description", new_callable=AsyncMock),+ ):+ _setup_mocks(mock_app)+ mock_app.BROWSER_MANAGER.get_for_workflow_run.return_value = (+ lost_browser_state("wr_test") if browser_lost else None+ )++ result = await block.execute_safe(workflow_run_id="wr_test")++ assert result.success is False+ assert result.can_continue_after_failure is not browser_lost+ assert (BROWSER_CONNECTION_LOST_ERROR_CODE in result.error_codes) is browser_lost+++async def test_task_block_is_not_retried_after_its_browser_was_lost() -> None:+ block = TaskBlock(+ label="nav",+ url="https://example.com",+ navigation_goal="go",+ output_parameter=output_parameter(),+ max_retries=2,+ )+ state, context, _ = _bound_state("wr_task_block_test")++ async def _lose_browser(**_: object) -> None:+ state._on_browser_context_closed(context)++ now = datetime.now(timezone.utc)+ failed_task = make_task(now, make_organization(now), status=TaskStatus.failed, failure_reason="page is missing")+ with (+ skyvern_context.scoped(SkyvernContext()),+ mock_block_execute_deps(working_page_url="https://example.com") as deps,+ ):+ deps.browser_manager.get_for_workflow_run = MagicMock(return_value=state)+ deps.agent.execute_step = AsyncMock(side_effect=_lose_browser)+ deps.tasks_db.get_task = AsyncMock(return_value=failed_task)+ with (+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock),+ patch(+ "skyvern.forge.sdk.workflow.models.block.app.STORAGE",+ new=MagicMock(+ get_current_attempt_downloaded_files=AsyncMock(return_value=[]),+ get_downloaded_file_signature_aliases=MagicMock(return_value=()),+ ),+ create=True,+ ),+ patch(+ "skyvern.forge.sdk.workflow.models.block.app.WORKFLOW_SERVICE.get_recent_task_screenshot_artifacts",+ new=AsyncMock(return_value=[]),+ ),+ patch(+ "skyvern.forge.sdk.workflow.models.block.app.WORKFLOW_SERVICE.get_recent_workflow_screenshot_artifacts",+ new=AsyncMock(return_value=[]),+ ),+ ):+ result = await block.execute(+ workflow_run_id="wr_task_block_test", workflow_run_block_id="wrb_test", organization_id="o_test"+ )++ assert deps.agent.execute_step.await_count == 1+ assert result.success is False+ assert result.can_continue_after_failure is False+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in result.error_codes++[email protected]("case", ["lost_during_downloaded_files", "lost_during_task_order", "ordinary_failure"])+async def test_task_block_admits_no_retry_after_a_loss_during_its_failure_bookkeeping(case: str) -> None:+ block = TaskBlock(+ label="nav",+ url="https://example.com",+ navigation_goal="go",+ output_parameter=output_parameter(),+ max_retries=1,+ )+ state, context, _ = _bound_state("wr_task_block_test")+ now = datetime.now(timezone.utc)+ organization = make_organization(now)+ first = make_task(now, organization, task_id="tsk_first", status=TaskStatus.failed, failure_reason="No such button")+ second = make_task(now, organization, task_id="tsk_second", status=TaskStatus.completed)+ order_reads: list[str] = []++ async def downloaded_files(**_: object) -> list:+ if case == "lost_during_downloaded_files":+ state._on_browser_context_closed(context)+ return []++ async def last_task_for_run(**_: object) -> object:+ # The first read orders the first attempt, the second orders its retry.+ order_reads.append("read")+ if case == "lost_during_task_order" and len(order_reads) == 2:+ state._on_browser_context_closed(context)+ return None if len(order_reads) == 1 else first++ with (+ skyvern_context.scoped(SkyvernContext()),+ mock_block_execute_deps(working_page_url="https://example.com") as deps,+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock),+ ):+ deps.browser_manager.get_for_workflow_run = MagicMock(return_value=state)+ deps.tasks_db.get_last_task_for_workflow_run = AsyncMock(side_effect=last_task_for_run)+ deps.tasks_db.get_task = AsyncMock(side_effect=[first, second])+ deps.agent.create_task_and_step_from_block = AsyncMock(+ side_effect=[(first, SimpleNamespace(step_id="stp_first")), (second, SimpleNamespace(step_id="stp_second"))]+ )+ block_module.app.STORAGE = MagicMock(+ get_current_attempt_downloaded_files=AsyncMock(side_effect=downloaded_files),+ get_downloaded_file_signature_aliases=MagicMock(return_value=()),+ )+ block_module.app.WORKFLOW_SERVICE.get_recent_task_screenshot_artifacts = AsyncMock(return_value=[])+ block_module.app.WORKFLOW_SERVICE.get_recent_workflow_screenshot_artifacts = AsyncMock(return_value=[])++ result = await block.execute(+ workflow_run_id="wr_task_block_test", workflow_run_block_id="wrb_test", organization_id="o_test"+ )++ linked_tasks = [+ update.kwargs["task_id"]+ for update in deps.observer_db.update_workflow_run_block.await_args_list+ if "task_id" in update.kwargs+ ]+ if case == "ordinary_failure":+ assert linked_tasks == ["tsk_first", "tsk_second"]+ assert result.success is True+ return+ # The loss lands after the retry decision was first made; no second attempt is created or linked.+ assert deps.agent.create_task_and_step_from_block.await_count == 1+ assert linked_tasks == ["tsk_first"]+ assert result.status == BlockStatus.failed+ assert result.failure_reason == "No such button"+ assert isinstance(result.output_parameter_value, dict)+ assert result.output_parameter_value["task_id"] == "tsk_first"+ assert result.can_continue_after_failure is False+ assert result.error_codes == [BROWSER_CONNECTION_LOST_ERROR_CODE]+++ "case",+ ["lost_during_downloaded_files", "lost_during_screenshot_query", "healthy_termination", "canceled_after_loss"],+)+async def test_a_terminated_task_cannot_carry_the_run_on_after_a_loss_during_its_artifact_reads(+ monkeypatch: pytest.MonkeyPatch, case: str+) -> None:+ block = TaskBlock(+ label="nav",+ url="https://example.com",+ navigation_goal="go",+ output_parameter=output_parameter(),+ continue_on_failure=True,+ )+ state, context, _ = _bound_state("wr_task_block_test")+ state.take_fullpage_screenshot = AsyncMock(return_value=None) # type: ignore[method-assign]+ now = datetime.now(timezone.utc)+ status = TaskStatus.canceled if case == "canceled_after_loss" else TaskStatus.terminated+ finished = make_task(now, make_organization(now), task_id="tsk_nav", status=status, failure_reason="Blocked page")++ async def execute_step(**_: object) -> None:+ if case == "canceled_after_loss":+ state._on_browser_context_closed(context)++ async def downloaded_files(**_: object) -> list:+ if case == "lost_during_downloaded_files":+ state._on_browser_context_closed(context)+ return []++ async def task_screenshots(**_: object) -> list:+ if case == "lost_during_screenshot_query":+ state._on_browser_context_closed(context)+ return []++ mark_failed = AsyncMock(return_value=MagicMock())+ mark_canceled = AsyncMock(return_value=MagicMock())+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", mark_failed)+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_canceled", mark_canceled)++ with (+ skyvern_context.scoped(SkyvernContext()),+ mock_block_execute_deps(working_page_url="https://example.com") as deps,+ patch.object(Block, "_generate_workflow_run_block_description", new_callable=AsyncMock),+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock),+ ):+ deps.browser_manager.get_for_workflow_run = MagicMock(return_value=state)+ deps.agent.execute_step = AsyncMock(side_effect=execute_step)+ deps.tasks_db.get_task = AsyncMock(return_value=finished)+ deps.observer_db.create_workflow_run_block = AsyncMock(+ return_value=SimpleNamespace(workflow_run_block_id="wrb_nav")+ )+ block_module.app.DATABASE.workflow_runs.admit_workflow_run_block_dispatch = admit_block_dispatch()+ block_module.app.STORAGE = MagicMock(+ get_current_attempt_downloaded_files=AsyncMock(side_effect=downloaded_files),+ get_downloaded_file_signature_aliases=MagicMock(return_value=()),+ )+ block_module.app.WORKFLOW_SERVICE.get_recent_task_screenshot_artifacts = AsyncMock(side_effect=task_screenshots)+ block_module.app.WORKFLOW_SERVICE.get_recent_workflow_screenshot_artifacts = AsyncMock(return_value=[])++ result = await block.execute_safe(workflow_run_id="wr_task_block_test", organization_id="o_test")+ row: dict[str, object] = {}+ for update in deps.observer_db.update_workflow_run_block.await_args_list:+ row.update({key: value for key, value in update.kwargs.items() if value is not None})++ _, should_stop = await WorkflowService()._handle_block_result_status(+ block=block,+ block_idx=0,+ blocks_cnt=1,+ block_result=result,+ workflow_run=MagicMock(),+ workflow_run_id="wr_task_block_test",+ )++ if case == "canceled_after_loss":+ # A cancellation keeps its outcome; the loss does not turn it into a failure.+ assert result.status == row["status"] == BlockStatus.canceled+ assert BROWSER_CONNECTION_LOST_ERROR_CODE not in result.error_codes+ assert should_stop is True+ mark_canceled.assert_awaited_once()+ mark_failed.assert_not_awaited()+ return+ output = row["output"]+ assert isinstance(output, dict)+ assert output["task_id"] == "tsk_nav"+ assert result.output_parameter_value == output+ assert result.failure_reason == "Blocked page"+ if case == "healthy_termination":+ assert result.status == row["status"] == BlockStatus.terminated+ assert result.can_continue_after_failure is True+ assert should_stop is False+ mark_failed.assert_not_awaited()+ return+ assert result.status == row["status"] == BlockStatus.failed+ assert result.error_codes == row["error_codes"] == [BROWSER_CONNECTION_LOST_ERROR_CODE]+ assert result.can_continue_after_failure is False+ assert should_stop is True+ mark_failed.assert_awaited_once()+++async def test_a_code_block_success_after_a_loss_is_left_to_the_run_completion_gate() -> None:+ block = _make_code_block()+ succeeded = BlockResult(+ success=True,+ output_parameter=block.output_parameter,+ status=BlockStatus.completed,+ workflow_run_block_id="wrb_code",+ )+ with (+ patch("skyvern.forge.sdk.workflow.models.block.app") as mock_app,+ patch.object(CodeBlock, "_execute", AsyncMock(return_value=succeeded)),+ ):+ mock_app.BROWSER_MANAGER.get_for_workflow_run.return_value = lost_browser_state("wr_test")+ mock_app.DATABASE.observer.update_workflow_run_block = AsyncMock()++ result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_code")++ assert result == succeeded+ mock_app.DATABASE.observer.update_workflow_run_block.assert_not_awaited()+++async def test_task_block_does_not_take_over_a_browser_its_finished_child_lost() -> None:+ block = TaskBlock(+ label="nav",+ url="https://example.com",+ navigation_goal="go",+ output_parameter=output_parameter(),+ max_retries=1,+ )+ # A synchronous child shared this run's browser and finished; the browser stays bound to the child.+ state, context, _ = _bound_state("wr_child")+ state.reconnect = AsyncMock() # type: ignore[method-assign]+ manager = RealBrowserManager()+ manager.pages["wr_task_block_test"] = state+ now = datetime.now(timezone.utc)+ organization = make_organization(now)+ task = make_task(now, organization, workflow_run_id="wr_task_block_test")+ step = make_step(now, task, step_id="stp_test", status=StepStatus.created, order=0, output=None)+ workflow_run = MagicMock(workflow_run_id="wr_task_block_test", parent_workflow_run_id=None, browser_profile_id=None)++ async def execute_step(**_: object) -> None:+ # The loss lands after the block checked its browser and before the step acquires it.+ state._on_browser_context_closed(context)+ # The real step fails its task on either refusal and returns.+ with contextlib.suppress(MissingBrowserState, MissingBrowserStatePage):+ await ForgeAgent().initialize_execution_state(task, step, workflow_run)++ with (+ skyvern_context.scoped(SkyvernContext()),+ mock_block_execute_deps(working_page_url="https://example.com") as deps,+ patch.object(object.__getattribute__(app, "_inst"), "BROWSER_MANAGER", manager),+ patch.object(manager, "_create_browser_state", new=AsyncMock()) as mock_create,+ ):+ deps.browser_manager.get_for_workflow_run = MagicMock(return_value=state)+ deps.agent.create_task_and_step_from_block = AsyncMock(return_value=(task, step))+ deps.agent.execute_step = AsyncMock(side_effect=execute_step)+ deps.tasks_db.get_task = AsyncMock(return_value=task.model_copy(update={"status": TaskStatus.failed}))+ with (+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock),+ patch(+ "skyvern.forge.sdk.workflow.models.block.app.STORAGE",+ new=MagicMock(+ get_current_attempt_downloaded_files=AsyncMock(return_value=[]),+ get_downloaded_file_signature_aliases=MagicMock(return_value=()),+ ),+ create=True,+ ),+ patch(+ "skyvern.forge.sdk.workflow.models.block.app.WORKFLOW_SERVICE.get_recent_task_screenshot_artifacts",+ new=AsyncMock(return_value=[]),+ ),+ patch(+ "skyvern.forge.sdk.workflow.models.block.app.WORKFLOW_SERVICE.get_recent_workflow_screenshot_artifacts",+ new=AsyncMock(return_value=[]),+ ),+ ):+ result = await block.execute(+ workflow_run_id="wr_task_block_test", workflow_run_block_id="wrb_test", organization_id="o_test"+ )++ assert deps.agent.execute_step.await_count == 1+ state.reconnect.assert_not_awaited()+ mock_create.assert_not_awaited()+ assert state.runtime_event_context.workflow_run_id == "wr_child"+ assert result.can_continue_after_failure is False+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in result.error_codes++[email protected]("owner", ["observing_run", "later_run"])+async def test_lost_browser_refuses_state_mutations_for_its_owner_before_side_effects(+ monkeypatch: pytest.MonkeyPatch, owner: str+) -> None:+ state, context, _ = _bound_state("wr_test")+ state._on_browser_context_closed(context)+ if owner == "later_run":+ state.bind_runtime_event_context(BrowserRuntimeLogContext(workflow_run_id="wr_next"))+ refused = owner == "observing_run"+ driver_starts: list[bool] = []++ class CountingPlaywright:+ async def start(self) -> object:+ driver_starts.append(True)+ raise RuntimeError("driver start reached")++ monkeypatch.setattr(real_browser_state_module, "async_playwright", lambda: CountingPlaywright())+ create_context = AsyncMock(side_effect=RuntimeError("context factory reached"))+ monkeypatch.setattr(real_browser_state_module.BrowserContextFactory, "create_browser_context", create_context)+ monkeypatch.setattr(navigation_module, "asyncio", ScopedAsyncio(sleep=AsyncMock()))+ context.new_page = AsyncMock(side_effect=RuntimeError("new page reached"))+ context._skyvern_cdp_download_interceptor = None+ page = MagicMock()+ page.goto = AsyncMock(side_effect=PlaywrightError("Target page, context or browser has been closed"))++ with pytest.raises(MissingBrowserStatePage):+ await state.must_get_working_page()+ with pytest.raises(MissingBrowserState if refused else FailedToNavigateToUrl):+ await state.navigate_to_url(page=page, url="https://93.184.216.34/next")+ state._connection_status = MagicMock(return_value=(False, "playwright_driver_connection_closed")) # type: ignore[method-assign]+ with pytest.raises((MissingBrowserState, RuntimeError)) as reconnect_error:+ await state.reconnect(browser_address="ws://remote-browser")+ state.browser_context = None+ with pytest.raises((MissingBrowserState, RuntimeError)):+ await state.check_and_fix_state()++ assert context.new_page.await_count == (0 if refused else 1)+ assert page.goto.await_count == (0 if refused else NAVIGATION_MAX_RETRY_TIME)+ assert len(driver_starts) == (0 if refused else 1)+ assert create_context.await_count == (0 if refused else 1)+ assert (BROWSER_CONNECTION_LOST_ERROR_CODE in str(reconnect_error.value)) is refused+++def _page_opening_context(context: MagicMock) -> None:+ """The run's retained context object: it opens tabs as Playwright does, so any attempt is counted."""+ context.pages = []+ context._skyvern_cdp_download_interceptor = None+ replacement = MagicMock(url="about:blank", close=AsyncMock(), bring_to_front=AsyncMock())+ context.new_page = _replacement_page_opener(context, replacement, [])++[email protected]("owner", ["observing_run", "later_run", "page_only_close"])+async def test_retained_closed_context_opens_no_page_for_the_run_that_lost_it(owner: str) -> None:+ state, context, _ = _bound_state("wr_test")+ _page_opening_context(context)+ if owner != "page_only_close":+ state._on_browser_context_closed(context)+ if owner == "later_run":+ state.bind_runtime_event_context(BrowserRuntimeLogContext(workflow_run_id="wr_next"))+ refused = owner == "observing_run"++ for open_page in (state.new_page, state.check_and_fix_state):+ with (+ pytest.raises(MissingBrowserState, match=BROWSER_CONNECTION_LOST_ERROR_CODE)+ if refused+ else (contextlib.nullcontext())+ ):+ await open_page()++ assert context.new_page.await_count == (0 if refused else 2)++[email protected]("existing_page", [False, True], ids=["opens_a_page", "reuses_a_page"])[email protected]("native_close", [True, False], ids=["lost_during_page_selection", "browser_alive"])+async def test_missing_page_is_neither_opened_nor_reused_after_a_loss_during_page_selection(+ monkeypatch: pytest.MonkeyPatch, existing_page: bool, native_close: bool+) -> None:+ state, context, _ = _bound_state("wr_test")+ _page_opening_context(context)+ remote_page = MagicMock(url="https://example.test/app")+ context.pages = [remote_page]+ state.browser_artifacts = BrowserArtifacts(remote_browser_session_id="pbs_remote")++ async def select_pages(*_args: object, **_kwargs: object) -> list[MagicMock]:+ if native_close:+ state._on_browser_context_closed(context)+ return [remote_page] if existing_page else []++ monkeypatch.setattr(state, "list_valid_pages", select_pages)+ monkeypatch.setattr(state, "set_working_page", set_working_page := AsyncMock())+ monkeypatch.setattr(state, "_close_all_other_pages", AsyncMock())++ with pytest.raises(MissingBrowserState) if native_close else contextlib.nullcontext():+ await state.check_and_fix_state()++ assert context.new_page.await_count == (0 if native_close or existing_page else 1)+ assert set_working_page.await_count == (0 if native_close else 1)++[email protected]("native_close", [True, False], ids=["lost_during_destination_check", "browser_alive"])+async def test_new_tab_opens_no_page_after_a_loss_during_its_destination_check(+ monkeypatch: pytest.MonkeyPatch, native_close: bool+) -> None:+ from skyvern.webeye.actions import actions as action_types+ from skyvern.webeye.actions import handler as handler_module+ from skyvern.webeye.actions.handler import handle_new_tab_action+ from skyvern.webeye.actions.responses import ActionSuccess++ state, context, _ = _bound_state("wr_tab")+ _page_opening_context(context)++ async def check_destination(validate: object, url: str) -> str:+ if native_close:+ state._on_browser_context_closed(context)+ return url++ monkeypatch.setattr(handler_module, "asyncio", ScopedAsyncio(to_thread=check_destination))+ monkeypatch.setattr(app.BROWSER_MANAGER, "get_for_task", lambda *_args, **_kwargs: state)+ monkeypatch.setattr(state, "navigate_to_url", AsyncMock())+ monkeypatch.setattr(state, "set_active_page", AsyncMock())+ task = SimpleNamespace(task_id="tsk_tab", workflow_run_id="wr_tab")+ action = action_types.NewTabAction(url="https://93.184.216.34/next")++ with pytest.raises(MissingBrowserState) if native_close else contextlib.nullcontext():+ results = await handle_new_tab_action(action, MagicMock(), MagicMock(), task, MagicMock())+ assert isinstance(results[-1], ActionSuccess)++ assert context.new_page.await_count == (0 if native_close else 1)++[email protected]("native_close", [True, False], ids=["native_close_during_goto", "ordinary_navigation_error"])+async def test_url_block_that_loses_its_browser_mid_navigation_stops_the_run(+ monkeypatch: pytest.MonkeyPatch, native_close: bool+) -> None:+ block = UrlBlock(+ label="go", url="https://93.184.216.34/next", output_parameter=output_parameter(), continue_on_failure=True+ )+ state, context, _ = _bound_state("wr_task_block_test")+ gotos: list[str] = []++ async def goto(url: str, **_: object) -> None:+ gotos.append(url)+ if native_close:+ state._on_browser_context_closed(context)+ raise PlaywrightError("Target page, context or browser has been closed")+ raise PlaywrightError("Timeout 30000ms exceeded.")++ page = MagicMock(url="https://93.184.216.34/start")+ page.goto = goto+ state.get_working_page = AsyncMock(return_value=page) # type: ignore[method-assign]+ state.take_fullpage_screenshot = AsyncMock(return_value=None) # type: ignore[method-assign]+ monkeypatch.setattr(navigation_module, "asyncio", ScopedAsyncio(sleep=AsyncMock()))+ now = datetime.now(timezone.utc)+ previous_task = make_task(now, make_organization(now), status=TaskStatus.completed)+ mark_failed = AsyncMock(return_value=MagicMock())+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", mark_failed)++ with (+ skyvern_context.scoped(SkyvernContext()),+ mock_block_execute_deps(working_page_url="https://93.184.216.34/start") as deps,+ patch.object(Block, "_generate_workflow_run_block_description", new_callable=AsyncMock),+ patch.object(Block, "record_output_parameter_value", new_callable=AsyncMock),+ ):+ deps.browser_manager.get_for_workflow_run = MagicMock(return_value=state)+ deps.tasks_db.get_last_task_for_workflow_run = AsyncMock(return_value=previous_task)+ deps.observer_db.create_workflow_run_block = AsyncMock(+ return_value=SimpleNamespace(workflow_run_block_id="wrb_go")+ )+ block_module.app.DATABASE.workflow_runs.admit_workflow_run_block_dispatch = admit_block_dispatch()++ result = await block.execute_safe(workflow_run_id="wr_task_block_test", organization_id="o_test")+ persisted_codes = {+ code+ for update in deps.observer_db.update_workflow_run_block.await_args_list+ if update.kwargs.get("workflow_run_block_id") == "wrb_go"+ for code in update.kwargs.get("error_codes") or []+ }++ _, should_stop = await WorkflowService()._handle_block_result_status(+ block=block,+ block_idx=0,+ blocks_cnt=3,+ block_result=result,+ workflow_run=MagicMock(),+ workflow_run_id="wr_task_block_test",+ )+ retry_policy = WorkflowRetryPolicy.model_validate({"max_retries": 2, "retry_on": [{"status": "failed"}]})+ retry = evaluate_retry_policy(retry_policy, WorkflowRunStatus.failed, persisted_codes, 0)++ assert result.status == BlockStatus.failed+ if native_close:+ assert gotos == [block.url]+ assert result.can_continue_after_failure is False+ assert persisted_codes == {BROWSER_CONNECTION_LOST_ERROR_CODE}+ assert should_stop is True+ assert retry == (False, BROWSER_CONNECTION_LOST_ERROR_CODE)+ else:+ assert gotos == [block.url] * NAVIGATION_MAX_RETRY_TIME+ assert result.can_continue_after_failure is True+ assert persisted_codes == set()+ assert should_stop is False+ assert retry == (True, "matched")+++ ("seam", "dispatches", "backoffs"),+ [+ ("native_close_during_goto", 1, 0),+ ("native_close_during_backoff", 1, 1),+ ("native_close_during_destination_check", 0, 0),+ ("probe_timeout_during_backoff", NAVIGATION_MAX_RETRY_TIME, NAVIGATION_MAX_RETRY_TIME - 1),+ ("cancelled_during_backoff", 1, 1),+ ],+)+async def test_navigation_is_not_dispatched_again_after_its_browser_is_lost_while_it_waits(+ monkeypatch: pytest.MonkeyPatch, seam: str, dispatches: int, backoffs: int+) -> None:+ state, context, browser = _bound_state("wr_test")+ context._skyvern_cdp_download_interceptor = None+ page = MagicMock()++ async def goto(*_: object, **__: object) -> None:+ if seam == "native_close_during_goto":+ state._on_browser_context_closed(context)+ raise PlaywrightError("net::ERR_CONNECTION_RESET")++ async def lose_browser_while_waiting(_: float) -> None:+ if seam == "native_close_during_backoff":+ state._on_browser_context_closed(context)+ elif seam == "probe_timeout_during_backoff":+ _probe_timeout(state, context, browser)+ elif seam == "cancelled_during_backoff":+ raise asyncio.CancelledError++ async def destination_check(check: object, *args: object) -> object:+ if seam == "native_close_during_destination_check":+ state._on_browser_context_closed(context)+ return check(*args) # type: ignore[operator]++ page.goto = AsyncMock(side_effect=goto)+ backoff = AsyncMock(side_effect=lose_browser_while_waiting)+ monkeypatch.setattr(navigation_module, "asyncio", ScopedAsyncio(sleep=backoff, to_thread=destination_check))++ with pytest.raises(+ asyncio.CancelledError if seam == "cancelled_during_backoff" else FailedToNavigateToUrl+ ) as error:+ await state.navigate_to_url(page=page, url="https://93.184.216.34/next")++ assert page.goto.await_count == dispatches+ assert backoff.await_count == backoffs+ if seam == "native_close_during_backoff":+ assert "ERR_CONNECTION_RESET" in str(error.value)++[email protected]("native_close", [True, False], ids=["native_close_during_reload", "ordinary_reload_failure"])+async def test_reload_degradation_stops_after_its_browser_is_lost(native_close: bool) -> None:+ state, context, _ = _bound_state("wr_test")+ page = MagicMock(url="https://93.184.216.34/page")++ async def reload(**_: object) -> None:+ if native_close:+ state._on_browser_context_closed(context)+ raise PlaywrightError("Target page, context or browser has been closed")+ raise PlaywrightError("net::ERR_CONNECTION_RESET")++ page.reload = AsyncMock(side_effect=reload)+ state.get_working_page = AsyncMock(return_value=page) # type: ignore[method-assign]++ with pytest.raises(FailedToReloadPage):+ await state.reload_page(degradation=True)++ assert page.reload.await_count == (1 if native_close else 3)++def test_is_connected_false_when_context_close_was_called() -> None:browser = MagicMock()browser.is_connected = MagicMock(return_value=True)@@ -2624,6 +3369,37 @@ async def test_a_caller_recovering_during_the_reap_gets_the_reaper_replacement()assert context.new_page.await_count == 1[email protected]("native_close", [True, False], ids=["lost_during_the_first_replacement", "page_only_crash"])+async def test_queued_crash_recoveries_open_no_page_after_the_first_lost_the_browser(native_close: bool) -> None:+ gate = asyncio.Event()+ crashed, sibling, replacement = _crashable_page(), _crashable_page(), _crashable_page()+ crashed.url = sibling.url = replacement.url = "about:blank"+ context = _crashable_context(crashed, sibling)+ state = RealBrowserState(pw=MagicMock(), browser_context=context, page=crashed)+ state.bind_runtime_event_context(BrowserRuntimeLogContext(workflow_run_id="wr_test"))++ async def open_page() -> MagicMock:+ await gate.wait()+ if native_close:+ state._on_browser_context_closed(context)+ raise PlaywrightError("Target page, context or browser has been closed")+ context.pages.append(replacement)+ return replacement++ context.new_page = AsyncMock(side_effect=open_page)+ state._on_page_crashed(crashed)+ state._on_page_crashed(sibling)+ await asyncio.sleep(0)+ gate.set()+ await _reap_crashed_pages(state)++ assert context.new_page.await_count == 1+ async with state._reopen_working_page_lock:+ with pytest.raises(MissingBrowserStatePage) if native_close else contextlib.nullcontext():+ assert await asyncio.wait_for(state.must_get_working_page(), 1) is replacement++@pytest.mark.asyncioasync def test_a_crashed_tab_with_a_live_sibling_closes_without_opening_another() -> None:crashed = _crashable_page()@@ -3547,6 +4323,16 @@ async def test_reconnect_scheduled_stop_observes_loss_before_installing_intent()monkeypatch.setattr(state, "get_working_page", AsyncMock(return_value=MagicMock()))monkeypatch.setattr(state, "check_and_fix_state", finish_setup)+ if signal == "context_then_driver" and not guarded:+ # The owning run's current context closes natively before the swap, so the run never gets a replacement.+ with skyvern_context.scoped(SkyvernContext(workflow_run_id="ambient-run")):+ with pytest.raises(MissingBrowserState, match=BROWSER_CONNECTION_LOST_ERROR_CODE):+ await state.reconnect(stale_context_is_unusable=True)+ assert state.browser_context is old+ assert state.pw is old_pw+ replacement.pw.stop.assert_awaited()+ assert state.get_browser_connection_loss() is not None+ returnwith skyvern_context.scoped(SkyvernContext(workflow_run_id="ambient-run")), capture_logs() as logs:await state.reconnect(stale_context_is_unusable=True)await delivered.wait()diff --git a/tests/unit/workflow/test_code_block_escalation.py b/tests/unit/workflow/test_code_block_escalation.pyindex 2b0e67c5f..4ac872ea8 100644--- a/tests/unit/workflow/test_code_block_escalation.py+++ b/tests/unit/workflow/test_code_block_escalation.py@@ -22,7 +22,7 @@ from structlog.testing import capture_logsimport skyvern.webeye.navigation as navigation_modulefrom skyvern.errors.errors import UserDefinedError-from skyvern.exceptions import NO_ADDRESS_RECORD_NAV_ERROR_CODE+from skyvern.exceptions import BROWSER_CONNECTION_LOST_ERROR_CODE, NO_ADDRESS_RECORD_NAV_ERROR_CODEfrom skyvern.forge import appfrom skyvern.forge.agent_functions import CodeBlockEngineFailure, CodeBlockEngineResultfrom skyvern.forge.sdk.api.llm.schema_validator import validate_and_fill_extraction_result, validate_schema@@ -61,7 +61,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (WorkflowParameterType,)from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun-from skyvern.forge.sdk.workflow.service import _merge_workflow_run_errors+from skyvern.forge.sdk.workflow.service import WorkflowService, _merge_workflow_run_errorsfrom skyvern.forge.taskv3.goal_composition import (CodeProgressRecord,GoalDirectives,@@ -74,7 +74,9 @@ from skyvern.schemas.self_heal import HealClassification, HealSkipReason, HealStfrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.actions.actions import Actionfrom skyvern.webeye.browser_artifacts import BrowserArtifacts+from skyvern.webeye.browser_runtime_events import BrowserRuntimeLogContextfrom skyvern.webeye.navigation import clear_task_nav_error_code, record_task_nav_error_code+from skyvern.webeye.real_browser_state import RealBrowserStateSECRET_VALUE = "hunter2-super-secret"SHORT_SECRET_VALUE = "abc"@@ -3835,6 +3837,124 @@ async def test_legacy_heal_success_skips_failed_write_and_ends_completed(assert FakeRecorder.instances[0].finalized_success is True[email protected]("engine", ["inline", "secure"])[email protected]("browser_lost", [True, False], ids=["native_context_close", "ordinary_failure"])+async def test_failure_after_native_browser_loss_stops_the_run_instead_of_healing(+ monkeypatch: pytest.MonkeyPatch,+ ai_fallback_flag: Callable[[str | None], None],+ engine: str,+ browser_lost: bool,+) -> None:+ ai_fallback_flag("o_test")+ state = _install_db_fakes(monkeypatch, final_status=TaskStatus.completed)+ block = _make_code_block(+ steps=[CodeBlockStep(description="open the report", line_start=1, line_end=1)],+ code="await page.click('#go')",+ )+ block.continue_on_failure = True+ context = _make_context()+ native_context = MagicMock(browser=MagicMock())+ browser_state = RealBrowserState(pw=MagicMock(), browser_context=native_context)+ browser_state.bind_runtime_event_context(BrowserRuntimeLogContext(workflow_run_id="wr_test"))+ browser_state.get_working_page = AsyncMock(return_value=MagicMock()) # type: ignore[method-assign]+ _patch_execute_chokepoint_environment(+ monkeypatch, context=context, fake_browser_state=browser_state, use_codeblock_runner=engine == "secure"+ )+ closed = PlaywrightError("Target page, context or browser has been closed")++ def close_in_flight() -> None:+ if browser_lost:+ browser_state._on_browser_context_closed(native_context)++ async def run_on_secure_runner(**_: Any) -> CodeBlockEngineResult | None:+ if engine != "secure":+ return None+ close_in_flight()+ return CodeBlockEngineResult(+ block_result=None,+ failure=CodeBlockEngineFailure(+ error_code="user_code_error",+ safe_message=None,+ failure_reason="CodeBlock failed while running user code.",+ exception_class="playwright._impl._errors.Error",+ failing_line=1,+ healability_hint=True,+ ),+ )++ class ClickInFlightRecorder(FakeRecorder):+ def __init__(self, **kwargs: Any) -> None:+ super().__init__(**kwargs)++ async def click(*_: Any, **__: Any) -> None:+ close_in_flight()+ raise closed++ self.recording_page.click = click++ monkeypatch.setattr(app.AGENT_FUNCTION, "execute_code_block_override", AsyncMock(side_effect=run_on_secure_runner))+ ClickInFlightRecorder.reset(last_recorded_exception=closed)+ monkeypatch.setattr("skyvern.forge.sdk.workflow.models.block.CodeBlockActionRecording", ClickInFlightRecorder)+ monkeypatch.setattr(CodeBlock, "_capture_failure_evidence", AsyncMock(return_value=None))+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", AsyncMock(return_value=None))++ async def heal_and_fall_through(**kwargs: Any) -> BlockResult:+ return await block.build_block_result(+ success=True,+ failure_reason=None,+ output_parameter_value={"task_id": "tsk_escalation", "status": "completed"},+ status=BlockStatus.completed,+ workflow_run_block_id=kwargs["workflow_run_block_id"],+ organization_id=kwargs["organization_id"],+ )++ heal = AsyncMock(side_effect=heal_and_fall_through)+ monkeypatch.setattr(block, "_attempt_self_heal", heal)+ mark_failed = AsyncMock(return_value=MagicMock())+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", mark_failed)++ with skyvern_context.scoped(SkyvernContext(organization_id="o_test", workflow_run_id="wr_test")):+ result = await block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="o_test",+ browser_session_id="pbs_test",+ )+ _, should_stop = await WorkflowService()._handle_block_result_status(+ block=block,+ block_idx=0,+ blocks_cnt=2,+ block_result=result,+ workflow_run=MagicMock(),+ workflow_run_id="wr_test",+ )++ persisted_codes = [+ code+ for update in state["workflow_run_block_updates"]+ if update.get("workflow_run_block_id") == "wrb_test"+ for code in update.get("error_codes") or []+ ]+ if browser_lost:+ heal.assert_not_awaited()+ assert result.success is False+ assert result.status == BlockStatus.failed+ assert result.failure_reason+ assert result.can_continue_after_failure is False+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in result.error_codes+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in persisted_codes+ assert _statuses_for_block(state, "wrb_test")[-1] == BlockStatus.failed+ assert ClickInFlightRecorder.instances[0].finalized_success is False+ assert should_stop is True+ mark_failed.assert_awaited_once()+ else:+ heal.assert_awaited_once()+ assert result.success is True+ assert BROWSER_CONNECTION_LOST_ERROR_CODE not in persisted_codes+ assert should_stop is False++@pytest.mark.asyncioasync def test_heal_output_write_failure_does_not_finalize_completed(monkeypatch: pytest.MonkeyPatch, ai_fallback_flag: Callable[[str | None], None]diff --git a/tests/unit/workflow/test_execute_single_block_phases.py b/tests/unit/workflow/test_execute_single_block_phases.pyindex cb3fbcc74..36aec9658 100644--- a/tests/unit/workflow/test_execute_single_block_phases.py+++ b/tests/unit/workflow/test_execute_single_block_phases.py@@ -19,19 +19,27 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom structlog.testing import capture_logs-from skyvern.exceptions import MissingBrowserState, MissingBrowserStatePage, StepTerminationError+from skyvern.exceptions import (+ BROWSER_CONNECTION_LOST_ERROR_CODE,+ MissingBrowserState,+ MissingBrowserStatePage,+ StepTerminationError,+)from skyvern.forge import appfrom skyvern.forge.sdk.db.enums import BrowserSeedSourcefrom skyvern.forge.sdk.workflow.context_manager import BlockOutcome, WorkflowRunContext+from skyvern.forge.sdk.workflow.models import block as block_modulefrom skyvern.forge.sdk.workflow.models.block import (Block,BranchCondition,CodeBlock,ConditionalBlock,ForLoopBlock,+ HttpRequestBlock,JinjaBranchCriteria,LoginBlock,NavigationBlock,+ PrintPageBlock,WaitBlock,)from skyvern.forge.sdk.workflow.models.parameter import OutputParameter@@ -47,7 +55,10 @@ from skyvern.services import script_servicefrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.actions.actions import Actionfrom skyvern.webeye.browser_artifacts import BrowserArtifacts+from skyvern.webeye.real_browser_manager import RealBrowserManager+from tests.unit._task_block_fakes import lost_browser_state, run_bound_browser_statefrom tests.unit.fake_workflow_run_context import FakeWorkflowRunContext+from tests.unit.scoped_asyncio import ScopedAsynciodef _output_parameter(key: str) -> OutputParameter:@@ -177,6 +188,7 @@ def _stub_run_refresh(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "register_block_parameters_for_workflow_run", AsyncMock())# get_all_parameters needs a sync context object; the stub app's auto-AsyncMock returns a coroutine.monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "get_workflow_run_context", MagicMock(return_value=MagicMock()))+ monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "get_attempt_number", MagicMock(return_value=1))@asynccontextmanagerasync def admit_dispatch(_: str) -> AsyncIterator[MagicMock]:@@ -378,6 +390,7 @@ def _wire_real_execute_safe(*,execute: AsyncMock | None = None,pause_point: str | None = None,+ browser_state: Any = None,) -> _PausableAwait:pause = _PausableAwait()@@ -394,7 +407,7 @@ def _wire_real_execute_safe(context = MagicMock()context.cancel_failure_evidence_capture = pause if pause_point == "capture" else AsyncMock()monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "get_workflow_run_context", MagicMock(return_value=context))- browser_state = MagicMock()+ browser_state = browser_state or MagicMock()browser_state.take_fullpage_screenshot = pause if pause_point == "screenshot" else AsyncMock(return_value=b"png")monkeypatch.setattr(app.BROWSER_MANAGER, "get_for_workflow_run", MagicMock(return_value=browser_state))monkeypatch.setattr(@@ -785,6 +798,430 @@ async def test_refused_script_step_fails_the_run_without_ai_fallback(monkeypatchassert should_stop is True+async def test_script_failure_after_browser_loss_is_not_retried_by_the_agent(monkeypatch: pytest.MonkeyPatch) -> None:+ service = WorkflowService()+ block = _navigation_block("cached_nav")+ execute_safe = AsyncMock()+ monkeypatch.setattr(NavigationBlock, "execute_safe", execute_safe)+ monkeypatch.setattr(NavigationBlock, "_apply_workflow_system_prompt", lambda self, ctx: None)+ monkeypatch.setattr(+ app.BROWSER_MANAGER, "get_for_workflow_run", MagicMock(return_value=lost_browser_state("wr_test"))+ )+ script_row = SimpleNamespace(+ label="cached_nav",+ attempt_number=1,+ parent_workflow_run_block_id=None,+ created_at=datetime.now(UTC),+ status=BlockStatus.failed,+ failure_reason="Browser state page is missing.",+ output=None,+ error_codes=[],+ workflow_run_block_id="wrb_script_1",+ )+ monkeypatch.setattr(app.DATABASE.observer, "get_workflow_run_blocks", AsyncMock(return_value=[script_row]))+ update_row = AsyncMock()+ monkeypatch.setattr(app.DATABASE.observer, "update_workflow_run_block", update_row)+ mark_failed = AsyncMock(return_value=MagicMock())+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", mark_failed)++ async def run_cached_task() -> None:+ raise MissingBrowserStatePage(workflow_run_id="wr_test")++ _, _, block_result, should_stop, _ = await _run_single_block(+ service,+ block,+ is_script_run=True,+ script_blocks_by_label={"cached_nav": _script_block("cached_nav", "await run_cached_task()")},+ loaded_script_module=SimpleNamespace(run_cached_task=run_cached_task),+ )++ execute_safe.assert_not_awaited()+ mark_failed.assert_awaited_once()+ assert should_stop is True+ assert block_result is not None+ assert block_result.can_continue_after_failure is False+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in block_result.error_codes+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in update_row.await_args.kwargs["error_codes"]+++async def test_unexpected_browser_loss_lookup_error_stops_script_fallback(monkeypatch: pytest.MonkeyPatch) -> None:+ block = _navigation_block("cached_nav")+ execute_safe = AsyncMock()+ monkeypatch.setattr(NavigationBlock, "execute_safe", execute_safe)+ monkeypatch.setattr(+ app.BROWSER_MANAGER, "get_for_workflow_run", MagicMock(side_effect=RuntimeError("loss lookup failed"))+ )+ monkeypatch.setattr(app.DATABASE.observer, "get_workflow_run_blocks", AsyncMock(return_value=[]))+ mark_failed = AsyncMock(return_value=_workflow_run())+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", mark_failed)++ _, _, _, should_stop, _ = await _run_single_block(+ WorkflowService(),+ block,+ is_script_run=True,+ script_blocks_by_label={"cached_nav": _script_block("cached_nav", "None")},+ loaded_script_module=SimpleNamespace(),+ )++ assert should_stop+ assert "loss lookup failed" in mark_failed.await_args.kwargs["failure_reason"]+ execute_safe.assert_not_awaited()+++async def test_adaptive_fallback_does_not_take_over_a_browser_the_finished_child_lost(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ manager = RealBrowserManager()+ # A synchronous child shared this run's browser, finished, and stays bound to it; its loss is latched.+ state, context = run_bound_browser_state("wr_child")+ state._on_browser_context_closed(context)+ manager.pages["wr_test"] = state+ monkeypatch.setattr(object.__getattribute__(app, "_inst"), "BROWSER_MANAGER", manager)+ block = _navigation_block("cached_nav")+ execute_safe = AsyncMock(return_value=_completed_result(block))+ monkeypatch.setattr(NavigationBlock, "execute_safe", execute_safe)+ monkeypatch.setattr(NavigationBlock, "_apply_workflow_system_prompt", lambda self, ctx: None)+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", AsyncMock(return_value=MagicMock()))+ script_row = SimpleNamespace(+ label="cached_nav",+ attempt_number=1,+ parent_workflow_run_block_id=None,+ created_at=datetime.now(UTC),+ status=BlockStatus.failed,+ failure_reason="Browser state for workflow_run_id=wr_test is missing.",+ output=None,+ error_codes=[],+ workflow_run_block_id="wrb_script_1",+ )+ monkeypatch.setattr(app.DATABASE.observer, "get_workflow_run_blocks", AsyncMock(return_value=[script_row]))+ monkeypatch.setattr(app.DATABASE.observer, "update_workflow_run_block", AsyncMock())+ monkeypatch.setattr(+ WorkflowService, "mark_workflow_run_as_failed_if_not_final", AsyncMock(return_value=MagicMock())+ )+ workflow_run = _workflow_run()+ workflow_run.parent_workflow_run_id = workflow_run.browser_profile_id = None++ async def run_cached_task() -> None:+ raise MissingBrowserState(workflow_run_id="wr_test", failure_reason=BROWSER_CONNECTION_LOST_ERROR_CODE)++ _, _, block_result, should_stop, _ = await _run_single_block(+ WorkflowService(),+ block,+ workflow=_adaptive_workflow(),+ workflow_run=workflow_run,+ is_script_run=True,+ script_blocks_by_label={"cached_nav": _script_block("cached_nav", "await run_cached_task()")},+ loaded_script_module=SimpleNamespace(run_cached_task=run_cached_task),+ )++ execute_safe.assert_not_awaited()+ assert should_stop is True+ assert block_result.can_continue_after_failure is False+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in block_result.error_codes+ assert state.runtime_event_context.workflow_run_id == "wr_child"+ assert state.get_browser_connection_loss() is not None++[email protected]("held_by", [None, "wr_child"])+async def test_adaptive_fallback_capture_does_not_acquire_a_browser(+ monkeypatch: pytest.MonkeyPatch, held_by: str | None+) -> None:+ manager = RealBrowserManager()+ if held_by is not None:+ # A finished synchronous child still holds the run's shared browser.+ state, _ = run_bound_browser_state(held_by)+ manager.pages["wr_test"] = state+ pages_before = dict(manager.pages)+ monkeypatch.setattr(object.__getattribute__(app, "_inst"), "BROWSER_MANAGER", manager)+ record_episode = AsyncMock(return_value=SimpleNamespace(episode_id="ep_test"))+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", record_episode)+ workflow_run = _workflow_run()+ workflow_run.parent_workflow_run_id = workflow_run.browser_profile_id = None++ with patch.object(manager, "_create_browser_state", new=AsyncMock()) as mock_create:+ episode_id, _ = await WorkflowService()._record_fallback_episode(+ workflow_run=workflow_run,+ workflow=_adaptive_workflow(),+ block=_navigation_block("cached_nav"),+ organization_id="org_test",+ workflow_run_id="wr_test",+ error_message="script failed",+ )++ mock_create.assert_not_awaited()+ assert episode_id == "ep_test"+ assert manager.pages == pages_before+ if held_by is not None:+ assert manager.pages["wr_test"].runtime_event_context.workflow_run_id == held_by+++ ("case", "rows_after"),+ [+ ("lost_during_the_wait", ["pause=failed!", "notify=completed"]),+ ("lost_during_the_final_wait", ["pause=failed!", "notify=completed"]),+ ("lost_during_the_request_setup", ["pause=completed", "request=failed!", "notify=completed"]),+ ("healthy", ["pause=completed", "request=completed", "notify=completed"]),+ ("ordinary_error", ["pause=completed", "request=failed", "notify=completed"]),+ ],+)+async def test_no_business_block_runs_and_the_body_cannot_complete_after_a_native_loss(+ monkeypatch: pytest.MonkeyPatch, case: str, rows_after: list[str]+) -> None:+ state, context = run_bound_browser_state("wr_test")+ _wire_real_execute_safe(monkeypatch, [WorkflowRunStatus.running], browser_state=state)+ rows: dict[str, SimpleNamespace] = {}+ sent: list[str] = []++ async def create_row(*, label: str, **_: Any) -> SimpleNamespace:+ if case == "lost_during_the_request_setup" and label == "request":+ state._on_browser_context_closed(context)+ rows[label] = SimpleNamespace(+ label=label,+ workflow_run_block_id=label,+ created_at=0,+ failure_reason=None,+ attempt_number=1,+ parent_workflow_run_block_id=None,+ output=None,+ )+ rows[label].status, rows[label].error_codes = None, []+ return rows[label]++ async def update_row(*, workflow_run_block_id: str, status: Any = None, error_codes: Any = None, **_: Any) -> None:+ row = rows[workflow_run_block_id]+ row.status, row.error_codes = status or row.status, error_codes or row.error_codes++ async def sleep(_: float) -> None:+ if case.endswith("wait"):+ state._on_browser_context_closed(context)++ async def send(self: HttpRequestBlock, workflow_run_id: str, workflow_run_block_id: str, **_: Any) -> BlockResult:+ sent.append(self.label)+ if case == "ordinary_error" and self.label == "request":+ raise RuntimeError("Cannot connect to host")+ return await self.build_block_result(+ True, None, status=BlockStatus.completed, workflow_run_block_id=workflow_run_block_id+ )++ monkeypatch.setattr(app.DATABASE.observer, "create_workflow_run_block", create_row)+ monkeypatch.setattr(app.DATABASE.observer, "update_workflow_run_block", update_row)+ monkeypatch.setattr(+ app.DATABASE.observer, "get_workflow_run_blocks", AsyncMock(side_effect=lambda **_: [*rows.values()])+ )+ monkeypatch.setattr(block_module, "asyncio", ScopedAsyncio(sleep=sleep))+ monkeypatch.setattr(WaitBlock, "record_output_parameter_value", AsyncMock())+ monkeypatch.setattr(HttpRequestBlock, "execute", send)+ mark_failed = AsyncMock(return_value=_workflow_run())+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", mark_failed)+ service = WorkflowService()+ wait = WaitBlock(label="pause", wait_sec=1, output_parameter=_output_parameter("pause_output"))+ request, notify = (HttpRequestBlock(label=n, output_parameter=_output_parameter(n)) for n in ("request", "notify"))+ workflow = _workflow()+ workflow.workflow_definition.finally_block_label = "notify"+ workflow.workflow_definition.blocks = [wait, request, notify]++ for block in [wait] if case == "lost_during_the_final_wait" else [wait, request]:+ _, _, result, should_stop, _ = await _run_single_block(service, block, workflow=workflow)+ if should_stop:+ break+ await service._execute_finally_block_if_configured(workflow, _workflow_run(), MagicMock(), None)++ lost = case.startswith("lost")+ assert sent == ([] if lost else ["request"]) + ["notify"]+ assert [+ f"{label}={row.status.value}{'!' * (BROWSER_CONNECTION_LOST_ERROR_CODE in row.error_codes)}"+ for label, row in rows.items()+ ] == rows_after+ assert result.can_continue_after_failure is not lost+ assert (result.failure_reason is None) == (case == "healthy")+ assert mark_failed.await_count == (0 if case == "healthy" else 1)+++ "case",+ [+ "finally_notification",+ "finally_first_fails_then_continues",+ "finally_first_raises_then_continues",+ "finally_first_fails_without_continue",+ "finally_browser_child",+ "body_loop",+ "body_after_finally",+ ],+)+async def test_a_finally_loop_still_sends_its_notification_after_a_native_loss(+ monkeypatch: pytest.MonkeyPatch, case: str+) -> None:+ state = lost_browser_state("wr_test")+ state.reconnect = AsyncMock() # type: ignore[method-assign]+ _wire_real_execute_safe(monkeypatch, [WorkflowRunStatus.running], browser_state=state)+ acquire = AsyncMock()+ monkeypatch.setattr(app.BROWSER_MANAGER, "get_or_create_for_workflow_run", acquire)+ sent: list[str] = []++ async def create_row(*, label: str, **_: Any) -> SimpleNamespace:+ return SimpleNamespace(label=label, workflow_run_block_id=f"wrb_{label}")++ async def send(self: HttpRequestBlock, workflow_run_id: str, workflow_run_block_id: str, **_: Any) -> BlockResult:+ sent.append(self.label)+ # An ordinary request failure, unrelated to the browser the body lost.+ failed = self.label == "notify" and case.startswith("finally_first")+ if failed and "raises" in case:+ raise RuntimeError("Cannot connect to host")+ return await self.build_block_result(+ not failed,+ "HTTP 500" if failed else None,+ status=BlockStatus.failed if failed else BlockStatus.completed,+ workflow_run_block_id=workflow_run_block_id,+ )++ update_row = AsyncMock()+ monkeypatch.setattr(app.DATABASE.observer, "create_workflow_run_block", create_row)+ monkeypatch.setattr(app.DATABASE.observer, "update_workflow_run_block", update_row)+ monkeypatch.setattr(HttpRequestBlock, "execute", send)+ monkeypatch.setattr(ForLoopBlock, "get_loop_over_parameter_values", AsyncMock(return_value=["[email protected]"]))+ for block_type in (ForLoopBlock, HttpRequestBlock, PrintPageBlock):+ monkeypatch.setattr(block_type, "record_output_parameter_value", AsyncMock())+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", AsyncMock(return_value=None))+ loop_blocks: list[Any] = [+ HttpRequestBlock(+ label="notify",+ output_parameter=_output_parameter("notify_output"),+ continue_on_failure=case.endswith("continues"),+ )+ ]+ if case.startswith("finally_first"):+ loop_blocks.append(HttpRequestBlock(label="notify_again", output_parameter=_output_parameter("again_output")))+ if case == "finally_browser_child":+ loop_blocks.append(PrintPageBlock(label="snapshot", output_parameter=_output_parameter("snapshot_output")))+ loop = ForLoopBlock(+ label="notify_all", output_parameter=_output_parameter("notify_all_output"), loop_blocks=loop_blocks+ )+ workflow = _workflow()+ workflow.workflow_definition.finally_block_label = "notify_all"+ workflow.workflow_definition.blocks = [loop]++ if case != "body_loop":+ finally_execution = await WorkflowService()._execute_finally_block_if_configured(+ workflow, _workflow_run(), MagicMock(), None+ )+ assert isinstance(finally_execution, tuple)+ _, result = finally_execution+ if case.startswith("body"):+ # The finally scope ends with the finally block, so later work in the same task is still refused.+ _, _, result, _, _ = await _run_single_block(WorkflowService(), loop, workflow=workflow)++ continued = case.endswith("continues")+ expected_sent = [] if case == "body_loop" else ["notify", "notify_again"] if continued else ["notify"]+ assert sent == expected_sent+ if case == "finally_notification" or continued:+ assert result.status == BlockStatus.completed+ else:+ assert result.status == BlockStatus.failed+ coded_rows = [+ call.kwargs["workflow_run_block_id"]+ for call in update_row.await_args_list+ if BROWSER_CONNECTION_LOST_ERROR_CODE in (call.kwargs.get("error_codes") or [])+ ]+ if case.startswith("body"):+ assert BROWSER_CONNECTION_LOST_ERROR_CODE in result.error_codes+ else:+ # Inside the finally block only the refused browser child carries the body's loss, and it ends the loop.+ assert coded_rows == (["wrb_snapshot"] if case == "finally_browser_child" else [])+ assert (result.can_continue_after_failure is False) == (case == "finally_browser_child")+ state.reconnect.assert_not_awaited()+ acquire.assert_not_awaited()++[email protected]("child_lost", [False, True], ids=["healthy_child", "child_lost"])+async def test_a_child_run_started_from_a_finally_block_keeps_its_own_body_guards(+ monkeypatch: pytest.MonkeyPatch, child_lost: bool+) -> None:+ parent_state = lost_browser_state("wr_test")+ child_state, child_context = run_bound_browser_state("wr_child")+ _wire_real_execute_safe(monkeypatch, [WorkflowRunStatus.running])+ states = {"wr_test": parent_state, "wr_child": child_state}+ monkeypatch.setattr(+ app.BROWSER_MANAGER, "get_for_workflow_run", lambda workflow_run_id, **_: states[workflow_run_id]+ )+ sent: list[str] = []+ child_results: dict[str, BlockResult] = {}+ permission_after_child: list[str | None] = []++ async def create_row(*, label: str, **_: Any) -> SimpleNamespace:+ return SimpleNamespace(label=label, workflow_run_block_id=f"wrb_{label}")++ def child_block(label: str) -> HttpRequestBlock:+ return HttpRequestBlock(label=label, output_parameter=_output_parameter(f"{label}_output"))++ async def run_child_inline() -> None:+ # Stand-in for a synchronous WorkflowTriggerBlock: the child run executes inline, in this same context.+ for label in ("child_first", "child_email"):+ child_results[label] = await child_block(label).execute_safe(+ workflow_run_id="wr_child", organization_id="org_test"+ )+ if child_lost and label == "child_first":+ child_state._on_browser_context_closed(child_context)+ with block_module.running_after_browser_loss("wr_child"):+ child_results["child_notify"] = await child_block("child_notify").execute_safe(+ workflow_run_id="wr_child", organization_id="org_test"+ )+ with pytest.raises(RuntimeError), block_module.running_after_browser_loss("wr_nested"):+ raise RuntimeError("a nested run's finally ends")+ permission_after_child.append(block_module._finally_after_browser_loss_run_id.get())+ permission_after_child.append(block_module._finally_after_browser_loss_run_id.get())++ async def send(self: HttpRequestBlock, workflow_run_id: str, workflow_run_block_id: str, **_: Any) -> BlockResult:+ sent.append(self.label)+ if self.label == "trigger":+ await run_child_inline()+ return await self.build_block_result(+ True, None, status=BlockStatus.completed, workflow_run_block_id=workflow_run_block_id+ )++ update_row = AsyncMock()+ monkeypatch.setattr(app.DATABASE.observer, "create_workflow_run_block", create_row)+ monkeypatch.setattr(app.DATABASE.observer, "update_workflow_run_block", update_row)+ monkeypatch.setattr(HttpRequestBlock, "execute", send)+ monkeypatch.setattr(ForLoopBlock, "get_loop_over_parameter_values", AsyncMock(return_value=["[email protected]"]))+ for block_type in (ForLoopBlock, HttpRequestBlock):+ monkeypatch.setattr(block_type, "record_output_parameter_value", AsyncMock())+ monkeypatch.setattr(WorkflowService, "mark_workflow_run_as_failed_if_not_final", AsyncMock(return_value=None))+ loop = ForLoopBlock(+ label="finally_steps",+ output_parameter=_output_parameter("finally_steps_output"),+ loop_blocks=[child_block("trigger"), child_block("parent_notify")],+ )+ workflow = _workflow()+ workflow.workflow_definition.finally_block_label = "finally_steps"+ workflow.workflow_definition.blocks = [loop]++ finally_execution = await WorkflowService()._execute_finally_block_if_configured(+ workflow, _workflow_run(), MagicMock(), None+ )++ assert isinstance(finally_execution, tuple)+ # The child's own loss refuses its next body effect; its own finally block and the parent's finally still run.+ child_sent = ["child_first", "child_notify"] if child_lost else ["child_first", "child_email", "child_notify"]+ assert sent == ["trigger", *child_sent, "parent_notify"]+ email = child_results["child_email"]+ assert email.success is not child_lost+ if child_lost:+ assert email.status == BlockStatus.failed and BROWSER_CONNECTION_LOST_ERROR_CODE in email.error_codes+ assert email.can_continue_after_failure is False+ assert child_results["child_notify"].success+ assert permission_after_child == ["wr_child", "wr_test"]+ assert block_module._finally_after_browser_loss_run_id.get() is None++@pytest.mark.asyncioasync def test_script_success_skips_agent_execution(monkeypatch: pytest.MonkeyPatch) -> None:service = WorkflowService()