Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11072): reconnect a dead reused browser state before block run (#6647)
Aaron Perezd04b5a650bb1
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 5598b9c44..a6ff3f9f7 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -525,6 +525,58 @@ class Block(BaseModel, abc.ABC):else:browser_state = app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)+ # A reused browser state (a persistent debug session shared with the copilot, or a+ # cached workflow-run browser) can have a dead Playwright driver after a prior owner+ # stopped it — reusing it makes the block's first page.goto raise "Connection closed+ # while reading from the driver". Rebuild a fresh connection to the same browser first.+ if browser_state is not None and not browser_state.is_connected():+ workflow_run = await app.WORKFLOW_SERVICE.get_workflow_run(+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ )+ # Reconnect to the session's own remote browser, never the run's (possibly pooled)+ # browser_address. get_browser_address_if_ready resolves a dead session to None+ # instead of blocking on the workflow query, so a missing address aborts fast.+ if browser_session_id and organization_id:+ try:+ browser_address = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_address_if_ready(+ session_id=browser_session_id, organization_id=organization_id+ )+ except Exception:+ browser_address = None+ if not browser_address:+ LOG.warning(+ "No session browser address available to reconnect; aborting browser setup",+ workflow_run_id=workflow_run_id,+ browser_session_id=browser_session_id,+ )+ return None+ else:+ browser_address = workflow_run.browser_address+ try:+ await browser_state.reconnect(+ proxy_location=workflow_run.proxy_location,+ workflow_run_id=workflow_run_id,+ workflow_permanent_id=workflow_run.workflow_permanent_id,+ organization_id=workflow_run.organization_id,+ extra_http_headers=workflow_run.extra_http_headers,+ cdp_connect_headers=workflow_run.cdp_connect_headers,+ browser_address=browser_address,+ browser_profile_id=workflow_run.browser_profile_id,+ )+ LOG.info(+ "Rebuilt a disconnected browser state before block execution",+ workflow_run_id=workflow_run_id,+ browser_session_id=browser_session_id,+ )+ except Exception:+ LOG.exception(+ "Failed to rebuild disconnected browser state",+ workflow_run_id=workflow_run_id,+ browser_session_id=browser_session_id,+ )+ return None+if not browser_state:workflow_run = await app.WORKFLOW_SERVICE.get_workflow_run(workflow_run_id=workflow_run_id,diff --git a/skyvern/webeye/browser_state.py b/skyvern/webeye/browser_state.pyindex bda49f059..b906ca1b9 100644--- a/skyvern/webeye/browser_state.py+++ b/skyvern/webeye/browser_state.py@@ -33,6 +33,20 @@ class BrowserState(Protocol):browser_profile_id: str | None = None,) -> None: ...+ def is_connected(self) -> bool: ...++ async def reconnect(+ self,+ proxy_location: ProxyLocationInput = None,+ workflow_run_id: str | None = None,+ workflow_permanent_id: str | None = None,+ organization_id: str | None = None,+ extra_http_headers: dict[str, str] | None = None,+ cdp_connect_headers: dict[str, str] | None = None,+ browser_address: str | None = None,+ browser_profile_id: str | None = None,+ ) -> None: ...+async def get_working_page(self) -> Page | None: ...async def must_get_working_page(self) -> Page: ...diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.pyindex 962892c5e..376d9da32 100644--- a/skyvern/webeye/real_browser_state.py+++ b/skyvern/webeye/real_browser_state.py@@ -7,7 +7,7 @@ from typing import Literalfrom urllib.parse import urlparseimport structlog-from playwright.async_api import BrowserContext, Page, Playwright+from playwright.async_api import BrowserContext, Page, Playwright, async_playwrightfrom skyvern.config import settingsfrom skyvern.constants import BROWSER_CLOSE_TIMEOUT, BROWSER_PAGE_CLOSE_TIMEOUT, NAVIGATION_MAX_RETRY_TIME@@ -362,6 +362,69 @@ class RealBrowserState(BrowserState):page = await self.__assert_page()return page+ def is_connected(self) -> bool:+ # A reused browser state (e.g. a persistent debug session) can have a stopped driver+ # after a prior owner's cleanup; page.goto then raises "Connection closed while reading+ # from the driver". A bare pw.stop() leaves browser.is_connected() stale and never flips+ # _close_was_called, so also inspect the shared driver Connection's closed-error.+ context = self.browser_context+ if context is None:+ return False+ impl = getattr(context, "_impl_obj", None)+ if getattr(impl, "_close_was_called", False) is True or getattr(impl, "_closed", False) is True:+ return False+ connection = getattr(impl, "_connection", None)+ if getattr(connection, "_closed_error", None) is not None:+ return False+ browser = getattr(context, "browser", None)+ if browser is None:+ return True+ try:+ return browser.is_connected()+ except Exception:+ return False++ async def reconnect(+ self,+ proxy_location: ProxyLocationInput = None,+ workflow_run_id: str | None = None,+ workflow_permanent_id: str | None = None,+ organization_id: str | None = None,+ extra_http_headers: dict[str, str] | None = None,+ cdp_connect_headers: dict[str, str] | None = None,+ browser_address: str | None = None,+ browser_profile_id: str | None = None,+ ) -> None:+ # The old driver pipe is gone, so check_and_fix_state must not reuse self.pw; start a+ # fresh Playwright driver and reconnect to the same (still-alive) remote browser.+ stale_pw = self.pw+ self.browser_context = None+ await self.set_working_page(None)+ self.pw = await async_playwright().start()+ try:+ await self.check_and_fix_state(+ proxy_location=proxy_location,+ workflow_run_id=workflow_run_id,+ workflow_permanent_id=workflow_permanent_id,+ organization_id=organization_id,+ extra_http_headers=extra_http_headers,+ cdp_connect_headers=cdp_connect_headers,+ browser_address=browser_address,+ browser_profile_id=browser_profile_id,+ )+ except Exception:+ # The caller abandons this state on failure, so stop the just-started driver too or it leaks.+ try:+ await self.pw.stop()+ except Exception:+ LOG.debug("Failed to stop the new Playwright driver after a failed reconnect", exc_info=True)+ raise+ finally:+ try:+ await stale_pw.stop()+ except Exception:+ LOG.debug("Failed to stop the stale Playwright driver during reconnect", exc_info=True)+async def close_current_open_page(self) -> bool:try:async with asyncio.timeout(BROWSER_CLOSE_TIMEOUT):diff --git a/tests/unit/workflow/test_block_browser_state_recovery.py b/tests/unit/workflow/test_block_browser_state_recovery.pynew file mode 100644index 000000000..a4302aa9a--- /dev/null+++ b/tests/unit/workflow/test_block_browser_state_recovery.py@@ -0,0 +1,327 @@+"""Liveness validation + reconnect of a reused browser state whose driver was already stopped."""++from __future__ import annotations++import asyncio+import json+import socket+import subprocess+import urllib.request+from datetime import datetime, timezone+from pathlib import Path+from unittest.mock import AsyncMock, MagicMock++import pytest+from playwright.async_api import async_playwright++from skyvern.forge import app+from skyvern.forge.sdk.workflow.models.block import CodeBlock+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType+from skyvern.webeye.real_browser_state import RealBrowserState+++def _has_playwright_browser() -> bool:+ """Check that Playwright's chromium binary exists for the current installed version."""+ try:+ from playwright.sync_api import sync_playwright # noqa: PLC0415++ with sync_playwright() as p:+ return Path(p.chromium.executable_path).exists()+ except Exception:+ return False+++_skip_no_browser = pytest.mark.skipif(+ not _has_playwright_browser(),+ reason="Requires Playwright browsers installed (run: playwright install chromium)",+)+++def _make_code_block() -> CodeBlock:+ now = datetime.now(timezone.utc)+ output_parameter = OutputParameter(+ parameter_type=ParameterType.OUTPUT,+ key="code_output",+ description="test output",+ output_parameter_id="op_code",+ workflow_id="w_test",+ created_at=now,+ modified_at=now,+ )+ return CodeBlock(label="code_1", code="value = 'ok'", output_parameter=output_parameter)+++class _FakeWorkflowRun:+ workflow_run_id = "wr_test"+ organization_id = "o_test"+ workflow_permanent_id = "wpid_test"+ proxy_location = None+ extra_http_headers: dict[str, str] | None = None+ cdp_connect_headers: dict[str, str] | None = None+ browser_address = "ws://remote-browser"+ browser_profile_id = None+ parent_workflow_run_id = None+++async def test_reused_persistent_session_is_reconnected_when_disconnected(monkeypatch: pytest.MonkeyPatch) -> None:+ block = _make_code_block()++ fake_state = MagicMock()+ fake_state.is_connected = MagicMock(return_value=False)+ fake_state.reconnect = AsyncMock(return_value=None)++ monkeypatch.setattr(+ app.PERSISTENT_SESSIONS_MANAGER, "get_browser_state", AsyncMock(return_value=fake_state), raising=False+ )+ monkeypatch.setattr(+ app.PERSISTENT_SESSIONS_MANAGER,+ "get_browser_address_if_ready",+ AsyncMock(return_value="ws://session-browser"),+ raising=False,+ )+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "get_workflow_run", AsyncMock(return_value=_FakeWorkflowRun()))++ result = await block.get_or_create_browser_state(+ workflow_run_id="wr_test", organization_id="o_test", browser_session_id="pbs_1"+ )++ assert result is fake_state+ fake_state.reconnect.assert_awaited_once()+ # The session's own remote browser is the reconnect target, not the run's pooled address.+ assert fake_state.reconnect.await_args.kwargs["browser_address"] == "ws://session-browser"+ # CDP handshake headers must survive the rebuild or remote browsers needing them fail to reattach.+ assert "cdp_connect_headers" in fake_state.reconnect.await_args.kwargs+++async def test_disconnected_session_without_resolvable_address_aborts(monkeypatch: pytest.MonkeyPatch) -> None:+ block = _make_code_block()++ fake_state = MagicMock()+ fake_state.is_connected = MagicMock(return_value=False)+ fake_state.reconnect = AsyncMock(return_value=None)++ # The run still carries a (pooled) address, but a session-backed block must never reconnect+ # to it — only the session's own browser is correct, so an unresolved session address is fatal.+ class _PooledAddressRun(_FakeWorkflowRun):+ browser_address = "ws://pooled-wrong-browser"++ monkeypatch.setattr(+ app.PERSISTENT_SESSIONS_MANAGER, "get_browser_state", AsyncMock(return_value=fake_state), raising=False+ )+ monkeypatch.setattr(+ app.PERSISTENT_SESSIONS_MANAGER,+ "get_browser_address_if_ready",+ AsyncMock(return_value=None),+ raising=False,+ )+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "get_workflow_run", AsyncMock(return_value=_PooledAddressRun()))++ result = await block.get_or_create_browser_state(+ workflow_run_id="wr_test", organization_id="o_test", browser_session_id="pbs_1"+ )++ assert result is None+ fake_state.reconnect.assert_not_awaited()+++async def test_connected_reused_session_is_not_reconnected(monkeypatch: pytest.MonkeyPatch) -> None:+ block = _make_code_block()++ fake_state = MagicMock()+ fake_state.is_connected = MagicMock(return_value=True)+ fake_state.reconnect = AsyncMock(return_value=None)++ monkeypatch.setattr(+ app.PERSISTENT_SESSIONS_MANAGER, "get_browser_state", AsyncMock(return_value=fake_state), raising=False+ )+ get_run = AsyncMock(return_value=_FakeWorkflowRun())+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "get_workflow_run", get_run)++ result = await block.get_or_create_browser_state(+ workflow_run_id="wr_test", organization_id="o_test", browser_session_id="pbs_1"+ )++ assert result is fake_state+ fake_state.reconnect.assert_not_awaited()+ # A healthy reused state never needs the workflow run looked up for a rebuild.+ get_run.assert_not_awaited()+++async def test_reconnect_failure_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:+ block = _make_code_block()++ fake_state = MagicMock()+ fake_state.is_connected = MagicMock(return_value=False)+ fake_state.reconnect = AsyncMock(side_effect=RuntimeError("driver gone"))++ monkeypatch.setattr(+ app.PERSISTENT_SESSIONS_MANAGER, "get_browser_state", AsyncMock(return_value=fake_state), raising=False+ )+ monkeypatch.setattr(+ app.PERSISTENT_SESSIONS_MANAGER,+ "get_browser_address_if_ready",+ AsyncMock(return_value="ws://session-browser"),+ raising=False,+ )+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "get_workflow_run", AsyncMock(return_value=_FakeWorkflowRun()))++ result = await block.get_or_create_browser_state(+ workflow_run_id="wr_test", organization_id="o_test", browser_session_id="pbs_1"+ )++ assert result is None+++def _state_with_context(context: object | None) -> RealBrowserState:+ return RealBrowserState(pw=MagicMock(), browser_context=context) # type: ignore[arg-type]+++def test_is_connected_false_when_no_context() -> None:+ assert _state_with_context(None).is_connected() is False+++def test_is_connected_true_when_browser_connected() -> None:+ browser = MagicMock()+ browser.is_connected = MagicMock(return_value=True)+ context = MagicMock()+ context.browser = browser+ context._impl_obj = MagicMock(_close_was_called=False, _closed=False, _connection=MagicMock(_closed_error=None))+ assert _state_with_context(context).is_connected() is True+++def test_is_connected_false_when_browser_disconnected() -> None:+ browser = MagicMock()+ browser.is_connected = MagicMock(return_value=False)+ context = MagicMock()+ context.browser = browser+ context._impl_obj = MagicMock(_close_was_called=False, _closed=False, _connection=MagicMock(_closed_error=None))+ assert _state_with_context(context).is_connected() is False+++def test_is_connected_false_when_context_close_was_called() -> None:+ browser = MagicMock()+ browser.is_connected = MagicMock(return_value=True)+ context = MagicMock()+ context.browser = browser+ context._impl_obj = MagicMock(_close_was_called=True, _closed=False)+ assert _state_with_context(context).is_connected() is False+++def test_is_connected_false_when_driver_connection_closed() -> None:+ # A bare pw.stop() leaves browser.is_connected() True and _close_was_called False, but the+ # shared driver Connection records a closed-error — that is the only reliable dead-driver signal.+ browser = MagicMock()+ browser.is_connected = MagicMock(return_value=True)+ context = MagicMock()+ context.browser = browser+ context._impl_obj = MagicMock(+ _close_was_called=False,+ _closed=False,+ _connection=MagicMock(_closed_error=RuntimeError("Target page, context or browser has been closed")),+ )+ assert _state_with_context(context).is_connected() is False+++def _free_port() -> int:+ with socket.socket() as sock:+ sock.bind(("127.0.0.1", 0))+ return int(sock.getsockname()[1])+++@_skip_no_browser+async def test_is_connected_false_after_real_driver_stop(tmp_path: Path) -> None:+ # The real reused-dead-session repro: connect_over_cdp, then a bare pw.stop() with no graceful+ # context.close(). browser.is_connected() stays True, so the probe must fall through to the+ # driver Connection's closed-error to report the dead state and trigger a reconnect.+ launcher = await async_playwright().start()+ chromium_path = launcher.chromium.executable_path+ await launcher.stop()++ port = _free_port()+ proc = subprocess.Popen(+ [+ chromium_path,+ "--headless=new",+ f"--remote-debugging-port={port}",+ f"--user-data-dir={tmp_path}",+ "--no-first-run",+ "--no-default-browser-check",+ ],+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,+ )+ try:+ ws_url: str | None = None+ for _ in range(100):+ try:+ with urllib.request.urlopen(f"http://127.0.0.1:{port}/json/version") as resp:+ ws_url = json.loads(resp.read())["webSocketDebuggerUrl"]+ break+ except Exception:+ await asyncio.sleep(0.1)+ assert ws_url is not None, "chromium CDP endpoint never came up"++ pw = await async_playwright().start()+ browser = await pw.chromium.connect_over_cdp(ws_url)+ context = browser.contexts[0] if browser.contexts else await browser.new_context()+ state = RealBrowserState(pw=pw, browser_context=context)++ assert state.is_connected() is True++ await pw.stop()++ assert state.is_connected() is False+ finally:+ proc.kill()+++async def test_reconnect_starts_fresh_driver_and_stops_stale_one(monkeypatch: pytest.MonkeyPatch) -> None:+ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(return_value=None)+ fresh_pw = MagicMock()++ class _FakeAsyncPlaywright:+ async def start(self) -> object:+ return fresh_pw++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())++ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())+ check_and_fix = AsyncMock(return_value=None)+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)++ await state.reconnect(browser_address="ws://remote-browser")++ assert state.pw is fresh_pw+ stale_pw.stop.assert_awaited_once()+ assert check_and_fix.await_args.kwargs["browser_address"] == "ws://remote-browser"+++async def test_reconnect_stops_fresh_driver_when_state_rebuild_fails(monkeypatch: pytest.MonkeyPatch) -> None:+ stale_pw = MagicMock()+ stale_pw.stop = AsyncMock(return_value=None)+ fresh_pw = MagicMock()+ fresh_pw.stop = AsyncMock(return_value=None)++ class _FakeAsyncPlaywright:+ async def start(self) -> object:+ return fresh_pw++ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())++ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())+ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock(side_effect=RuntimeError("cdp handshake failed")))++ with pytest.raises(RuntimeError, match="cdp handshake failed"):+ await state.reconnect(browser_address="ws://remote-browser")++ # A failed rebuild must stop both drivers so it never orphans the freshly started one.+ fresh_pw.stop.assert_awaited_once()+ stale_pw.stop.assert_awaited_once()