Skip to content

Automate browser based workflows with AI

fix(SKY-11072): reconnect a dead reused browser state before block run (#6647)

Aaron Perezd04b5a650bb1
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 5598b9c44..a6ff3f9f7 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -525,6 +525,58 @@ class Block(BaseModel, abc.ABC):
else:
browser_state = app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)
+ # A reused browser state (a persistent debug session shared with the copilot, or a
+ # cached workflow-run browser) can have a dead Playwright driver after a prior owner
+ # stopped it — reusing it makes the block's first page.goto raise "Connection closed
+ # while reading from the driver". Rebuild a fresh connection to the same browser first.
+ if browser_state is not None and not browser_state.is_connected():
+ workflow_run = await app.WORKFLOW_SERVICE.get_workflow_run(
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ )
+ # Reconnect to the session's own remote browser, never the run's (possibly pooled)
+ # browser_address. get_browser_address_if_ready resolves a dead session to None
+ # instead of blocking on the workflow query, so a missing address aborts fast.
+ if browser_session_id and organization_id:
+ try:
+ browser_address = await app.PERSISTENT_SESSIONS_MANAGER.get_browser_address_if_ready(
+ session_id=browser_session_id, organization_id=organization_id
+ )
+ except Exception:
+ browser_address = None
+ if not browser_address:
+ LOG.warning(
+ "No session browser address available to reconnect; aborting browser setup",
+ workflow_run_id=workflow_run_id,
+ browser_session_id=browser_session_id,
+ )
+ return None
+ else:
+ browser_address = workflow_run.browser_address
+ try:
+ await browser_state.reconnect(
+ proxy_location=workflow_run.proxy_location,
+ workflow_run_id=workflow_run_id,
+ workflow_permanent_id=workflow_run.workflow_permanent_id,
+ organization_id=workflow_run.organization_id,
+ extra_http_headers=workflow_run.extra_http_headers,
+ cdp_connect_headers=workflow_run.cdp_connect_headers,
+ browser_address=browser_address,
+ browser_profile_id=workflow_run.browser_profile_id,
+ )
+ LOG.info(
+ "Rebuilt a disconnected browser state before block execution",
+ workflow_run_id=workflow_run_id,
+ browser_session_id=browser_session_id,
+ )
+ except Exception:
+ LOG.exception(
+ "Failed to rebuild disconnected browser state",
+ workflow_run_id=workflow_run_id,
+ browser_session_id=browser_session_id,
+ )
+ return None
+
if not browser_state:
workflow_run = await app.WORKFLOW_SERVICE.get_workflow_run(
workflow_run_id=workflow_run_id,
diff --git a/skyvern/webeye/browser_state.py b/skyvern/webeye/browser_state.py
index bda49f059..b906ca1b9 100644
--- a/skyvern/webeye/browser_state.py
+++ b/skyvern/webeye/browser_state.py
@@ -33,6 +33,20 @@ class BrowserState(Protocol):
browser_profile_id: str | None = None,
) -> None: ...
+ def is_connected(self) -> bool: ...
+
+ async def reconnect(
+ self,
+ proxy_location: ProxyLocationInput = None,
+ workflow_run_id: str | None = None,
+ workflow_permanent_id: str | None = None,
+ organization_id: str | None = None,
+ extra_http_headers: dict[str, str] | None = None,
+ cdp_connect_headers: dict[str, str] | None = None,
+ browser_address: str | None = None,
+ browser_profile_id: str | None = None,
+ ) -> None: ...
+
async def get_working_page(self) -> Page | None: ...
async def must_get_working_page(self) -> Page: ...
diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.py
index 962892c5e..376d9da32 100644
--- a/skyvern/webeye/real_browser_state.py
+++ b/skyvern/webeye/real_browser_state.py
@@ -7,7 +7,7 @@ from typing import Literal
from urllib.parse import urlparse
import structlog
-from playwright.async_api import BrowserContext, Page, Playwright
+from playwright.async_api import BrowserContext, Page, Playwright, async_playwright
from skyvern.config import settings
from skyvern.constants import BROWSER_CLOSE_TIMEOUT, BROWSER_PAGE_CLOSE_TIMEOUT, NAVIGATION_MAX_RETRY_TIME
@@ -362,6 +362,69 @@ class RealBrowserState(BrowserState):
page = await self.__assert_page()
return page
+ def is_connected(self) -> bool:
+ # A reused browser state (e.g. a persistent debug session) can have a stopped driver
+ # after a prior owner's cleanup; page.goto then raises "Connection closed while reading
+ # from the driver". A bare pw.stop() leaves browser.is_connected() stale and never flips
+ # _close_was_called, so also inspect the shared driver Connection's closed-error.
+ context = self.browser_context
+ if context is None:
+ return False
+ impl = getattr(context, "_impl_obj", None)
+ if getattr(impl, "_close_was_called", False) is True or getattr(impl, "_closed", False) is True:
+ return False
+ connection = getattr(impl, "_connection", None)
+ if getattr(connection, "_closed_error", None) is not None:
+ return False
+ browser = getattr(context, "browser", None)
+ if browser is None:
+ return True
+ try:
+ return browser.is_connected()
+ except Exception:
+ return False
+
+ async def reconnect(
+ self,
+ proxy_location: ProxyLocationInput = None,
+ workflow_run_id: str | None = None,
+ workflow_permanent_id: str | None = None,
+ organization_id: str | None = None,
+ extra_http_headers: dict[str, str] | None = None,
+ cdp_connect_headers: dict[str, str] | None = None,
+ browser_address: str | None = None,
+ browser_profile_id: str | None = None,
+ ) -> None:
+ # The old driver pipe is gone, so check_and_fix_state must not reuse self.pw; start a
+ # fresh Playwright driver and reconnect to the same (still-alive) remote browser.
+ stale_pw = self.pw
+ self.browser_context = None
+ await self.set_working_page(None)
+ self.pw = await async_playwright().start()
+ try:
+ await self.check_and_fix_state(
+ proxy_location=proxy_location,
+ workflow_run_id=workflow_run_id,
+ workflow_permanent_id=workflow_permanent_id,
+ organization_id=organization_id,
+ extra_http_headers=extra_http_headers,
+ cdp_connect_headers=cdp_connect_headers,
+ browser_address=browser_address,
+ browser_profile_id=browser_profile_id,
+ )
+ except Exception:
+ # The caller abandons this state on failure, so stop the just-started driver too or it leaks.
+ try:
+ await self.pw.stop()
+ except Exception:
+ LOG.debug("Failed to stop the new Playwright driver after a failed reconnect", exc_info=True)
+ raise
+ finally:
+ try:
+ await stale_pw.stop()
+ except Exception:
+ LOG.debug("Failed to stop the stale Playwright driver during reconnect", exc_info=True)
+
async def close_current_open_page(self) -> bool:
try:
async with asyncio.timeout(BROWSER_CLOSE_TIMEOUT):
diff --git a/tests/unit/workflow/test_block_browser_state_recovery.py b/tests/unit/workflow/test_block_browser_state_recovery.py
new file mode 100644
index 000000000..a4302aa9a
--- /dev/null
+++ b/tests/unit/workflow/test_block_browser_state_recovery.py
@@ -0,0 +1,327 @@
+"""Liveness validation + reconnect of a reused browser state whose driver was already stopped."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import socket
+import subprocess
+import urllib.request
+from datetime import datetime, timezone
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from playwright.async_api import async_playwright
+
+from skyvern.forge import app
+from skyvern.forge.sdk.workflow.models.block import CodeBlock
+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
+from skyvern.webeye.real_browser_state import RealBrowserState
+
+
+def _has_playwright_browser() -> bool:
+ """Check that Playwright's chromium binary exists for the current installed version."""
+ try:
+ from playwright.sync_api import sync_playwright # noqa: PLC0415
+
+ with sync_playwright() as p:
+ return Path(p.chromium.executable_path).exists()
+ except Exception:
+ return False
+
+
+_skip_no_browser = pytest.mark.skipif(
+ not _has_playwright_browser(),
+ reason="Requires Playwright browsers installed (run: playwright install chromium)",
+)
+
+
+def _make_code_block() -> CodeBlock:
+ now = datetime.now(timezone.utc)
+ output_parameter = OutputParameter(
+ parameter_type=ParameterType.OUTPUT,
+ key="code_output",
+ description="test output",
+ output_parameter_id="op_code",
+ workflow_id="w_test",
+ created_at=now,
+ modified_at=now,
+ )
+ return CodeBlock(label="code_1", code="value = 'ok'", output_parameter=output_parameter)
+
+
+class _FakeWorkflowRun:
+ workflow_run_id = "wr_test"
+ organization_id = "o_test"
+ workflow_permanent_id = "wpid_test"
+ proxy_location = None
+ extra_http_headers: dict[str, str] | None = None
+ cdp_connect_headers: dict[str, str] | None = None
+ browser_address = "ws://remote-browser"
+ browser_profile_id = None
+ parent_workflow_run_id = None
+
+
+async def test_reused_persistent_session_is_reconnected_when_disconnected(monkeypatch: pytest.MonkeyPatch) -> None:
+ block = _make_code_block()
+
+ fake_state = MagicMock()
+ fake_state.is_connected = MagicMock(return_value=False)
+ fake_state.reconnect = AsyncMock(return_value=None)
+
+ monkeypatch.setattr(
+ app.PERSISTENT_SESSIONS_MANAGER, "get_browser_state", AsyncMock(return_value=fake_state), raising=False
+ )
+ monkeypatch.setattr(
+ app.PERSISTENT_SESSIONS_MANAGER,
+ "get_browser_address_if_ready",
+ AsyncMock(return_value="ws://session-browser"),
+ raising=False,
+ )
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "get_workflow_run", AsyncMock(return_value=_FakeWorkflowRun()))
+
+ result = await block.get_or_create_browser_state(
+ workflow_run_id="wr_test", organization_id="o_test", browser_session_id="pbs_1"
+ )
+
+ assert result is fake_state
+ fake_state.reconnect.assert_awaited_once()
+ # The session's own remote browser is the reconnect target, not the run's pooled address.
+ assert fake_state.reconnect.await_args.kwargs["browser_address"] == "ws://session-browser"
+ # CDP handshake headers must survive the rebuild or remote browsers needing them fail to reattach.
+ assert "cdp_connect_headers" in fake_state.reconnect.await_args.kwargs
+
+
+async def test_disconnected_session_without_resolvable_address_aborts(monkeypatch: pytest.MonkeyPatch) -> None:
+ block = _make_code_block()
+
+ fake_state = MagicMock()
+ fake_state.is_connected = MagicMock(return_value=False)
+ fake_state.reconnect = AsyncMock(return_value=None)
+
+ # The run still carries a (pooled) address, but a session-backed block must never reconnect
+ # to it — only the session's own browser is correct, so an unresolved session address is fatal.
+ class _PooledAddressRun(_FakeWorkflowRun):
+ browser_address = "ws://pooled-wrong-browser"
+
+ monkeypatch.setattr(
+ app.PERSISTENT_SESSIONS_MANAGER, "get_browser_state", AsyncMock(return_value=fake_state), raising=False
+ )
+ monkeypatch.setattr(
+ app.PERSISTENT_SESSIONS_MANAGER,
+ "get_browser_address_if_ready",
+ AsyncMock(return_value=None),
+ raising=False,
+ )
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "get_workflow_run", AsyncMock(return_value=_PooledAddressRun()))
+
+ result = await block.get_or_create_browser_state(
+ workflow_run_id="wr_test", organization_id="o_test", browser_session_id="pbs_1"
+ )
+
+ assert result is None
+ fake_state.reconnect.assert_not_awaited()
+
+
+async def test_connected_reused_session_is_not_reconnected(monkeypatch: pytest.MonkeyPatch) -> None:
+ block = _make_code_block()
+
+ fake_state = MagicMock()
+ fake_state.is_connected = MagicMock(return_value=True)
+ fake_state.reconnect = AsyncMock(return_value=None)
+
+ monkeypatch.setattr(
+ app.PERSISTENT_SESSIONS_MANAGER, "get_browser_state", AsyncMock(return_value=fake_state), raising=False
+ )
+ get_run = AsyncMock(return_value=_FakeWorkflowRun())
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "get_workflow_run", get_run)
+
+ result = await block.get_or_create_browser_state(
+ workflow_run_id="wr_test", organization_id="o_test", browser_session_id="pbs_1"
+ )
+
+ assert result is fake_state
+ fake_state.reconnect.assert_not_awaited()
+ # A healthy reused state never needs the workflow run looked up for a rebuild.
+ get_run.assert_not_awaited()
+
+
+async def test_reconnect_failure_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
+ block = _make_code_block()
+
+ fake_state = MagicMock()
+ fake_state.is_connected = MagicMock(return_value=False)
+ fake_state.reconnect = AsyncMock(side_effect=RuntimeError("driver gone"))
+
+ monkeypatch.setattr(
+ app.PERSISTENT_SESSIONS_MANAGER, "get_browser_state", AsyncMock(return_value=fake_state), raising=False
+ )
+ monkeypatch.setattr(
+ app.PERSISTENT_SESSIONS_MANAGER,
+ "get_browser_address_if_ready",
+ AsyncMock(return_value="ws://session-browser"),
+ raising=False,
+ )
+ monkeypatch.setattr(app.WORKFLOW_SERVICE, "get_workflow_run", AsyncMock(return_value=_FakeWorkflowRun()))
+
+ result = await block.get_or_create_browser_state(
+ workflow_run_id="wr_test", organization_id="o_test", browser_session_id="pbs_1"
+ )
+
+ assert result is None
+
+
+def _state_with_context(context: object | None) -> RealBrowserState:
+ return RealBrowserState(pw=MagicMock(), browser_context=context) # type: ignore[arg-type]
+
+
+def test_is_connected_false_when_no_context() -> None:
+ assert _state_with_context(None).is_connected() is False
+
+
+def test_is_connected_true_when_browser_connected() -> None:
+ browser = MagicMock()
+ browser.is_connected = MagicMock(return_value=True)
+ context = MagicMock()
+ context.browser = browser
+ context._impl_obj = MagicMock(_close_was_called=False, _closed=False, _connection=MagicMock(_closed_error=None))
+ assert _state_with_context(context).is_connected() is True
+
+
+def test_is_connected_false_when_browser_disconnected() -> None:
+ browser = MagicMock()
+ browser.is_connected = MagicMock(return_value=False)
+ context = MagicMock()
+ context.browser = browser
+ context._impl_obj = MagicMock(_close_was_called=False, _closed=False, _connection=MagicMock(_closed_error=None))
+ assert _state_with_context(context).is_connected() is False
+
+
+def test_is_connected_false_when_context_close_was_called() -> None:
+ browser = MagicMock()
+ browser.is_connected = MagicMock(return_value=True)
+ context = MagicMock()
+ context.browser = browser
+ context._impl_obj = MagicMock(_close_was_called=True, _closed=False)
+ assert _state_with_context(context).is_connected() is False
+
+
+def test_is_connected_false_when_driver_connection_closed() -> None:
+ # A bare pw.stop() leaves browser.is_connected() True and _close_was_called False, but the
+ # shared driver Connection records a closed-error — that is the only reliable dead-driver signal.
+ browser = MagicMock()
+ browser.is_connected = MagicMock(return_value=True)
+ context = MagicMock()
+ context.browser = browser
+ context._impl_obj = MagicMock(
+ _close_was_called=False,
+ _closed=False,
+ _connection=MagicMock(_closed_error=RuntimeError("Target page, context or browser has been closed")),
+ )
+ assert _state_with_context(context).is_connected() is False
+
+
+def _free_port() -> int:
+ with socket.socket() as sock:
+ sock.bind(("127.0.0.1", 0))
+ return int(sock.getsockname()[1])
+
+
+@_skip_no_browser
+async def test_is_connected_false_after_real_driver_stop(tmp_path: Path) -> None:
+ # The real reused-dead-session repro: connect_over_cdp, then a bare pw.stop() with no graceful
+ # context.close(). browser.is_connected() stays True, so the probe must fall through to the
+ # driver Connection's closed-error to report the dead state and trigger a reconnect.
+ launcher = await async_playwright().start()
+ chromium_path = launcher.chromium.executable_path
+ await launcher.stop()
+
+ port = _free_port()
+ proc = subprocess.Popen(
+ [
+ chromium_path,
+ "--headless=new",
+ f"--remote-debugging-port={port}",
+ f"--user-data-dir={tmp_path}",
+ "--no-first-run",
+ "--no-default-browser-check",
+ ],
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+ try:
+ ws_url: str | None = None
+ for _ in range(100):
+ try:
+ with urllib.request.urlopen(f"http://127.0.0.1:{port}/json/version") as resp:
+ ws_url = json.loads(resp.read())["webSocketDebuggerUrl"]
+ break
+ except Exception:
+ await asyncio.sleep(0.1)
+ assert ws_url is not None, "chromium CDP endpoint never came up"
+
+ pw = await async_playwright().start()
+ browser = await pw.chromium.connect_over_cdp(ws_url)
+ context = browser.contexts[0] if browser.contexts else await browser.new_context()
+ state = RealBrowserState(pw=pw, browser_context=context)
+
+ assert state.is_connected() is True
+
+ await pw.stop()
+
+ assert state.is_connected() is False
+ finally:
+ proc.kill()
+
+
+async def test_reconnect_starts_fresh_driver_and_stops_stale_one(monkeypatch: pytest.MonkeyPatch) -> None:
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(return_value=None)
+ fresh_pw = MagicMock()
+
+ class _FakeAsyncPlaywright:
+ async def start(self) -> object:
+ return fresh_pw
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+
+ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
+ check_and_fix = AsyncMock(return_value=None)
+ monkeypatch.setattr(state, "check_and_fix_state", check_and_fix)
+
+ await state.reconnect(browser_address="ws://remote-browser")
+
+ assert state.pw is fresh_pw
+ stale_pw.stop.assert_awaited_once()
+ assert check_and_fix.await_args.kwargs["browser_address"] == "ws://remote-browser"
+
+
+async def test_reconnect_stops_fresh_driver_when_state_rebuild_fails(monkeypatch: pytest.MonkeyPatch) -> None:
+ stale_pw = MagicMock()
+ stale_pw.stop = AsyncMock(return_value=None)
+ fresh_pw = MagicMock()
+ fresh_pw.stop = AsyncMock(return_value=None)
+
+ class _FakeAsyncPlaywright:
+ async def start(self) -> object:
+ return fresh_pw
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.async_playwright", lambda: _FakeAsyncPlaywright())
+
+ state = RealBrowserState(pw=stale_pw, browser_context=MagicMock())
+ monkeypatch.setattr(state, "check_and_fix_state", AsyncMock(side_effect=RuntimeError("cdp handshake failed")))
+
+ with pytest.raises(RuntimeError, match="cdp handshake failed"):
+ await state.reconnect(browser_address="ws://remote-browser")
+
+ # A failed rebuild must stop both drivers so it never orphans the freshly started one.
+ fresh_pw.stop.assert_awaited_once()
+ stale_pw.stop.assert_awaited_once()