Skip to content

Automate browser based workflows with AI

fix(SKY-11215): bind live screencast to the run's current browser state (#6865)

Aaron Perez56e31ce2ca4b
diff --git a/skyvern/forge/sdk/routes/streaming/cdp_input.py b/skyvern/forge/sdk/routes/streaming/cdp_input.py
index e4cb21bf2..4673f1a70 100644
--- a/skyvern/forge/sdk/routes/streaming/cdp_input.py
+++ b/skyvern/forge/sdk/routes/streaming/cdp_input.py
@@ -22,7 +22,7 @@ from skyvern.forge.sdk.routes.streaming.registries import (
stream_ref_dec,
stream_ref_inc,
)
-from skyvern.forge.sdk.routes.streaming.screencast import wait_for_browser_state
+from skyvern.forge.sdk.routes.streaming.screencast import _resolve_working_page, wait_for_browser_state
from skyvern.forge.sdk.schemas.persistent_browser_sessions import is_final_status
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
from skyvern.webeye.browser_state import BrowserState
@@ -59,13 +59,19 @@ class ActivePageCdpInputSession:
def __init__(
self,
browser_state: BrowserState,
- log_id_key: str,
- log_id_value: str,
+ entity_id: str,
+ entity_type: str,
+ workflow_run_id: str | None = None,
+ organization_id: str | None = None,
refresh_interval: float = ACTIVE_PAGE_INPUT_REFRESH_INTERVAL,
) -> None:
self.browser_state = browser_state
- self.log_id_key = log_id_key
- self.log_id_value = log_id_value
+ self.entity_id = entity_id
+ self.entity_type = entity_type
+ self.workflow_run_id = workflow_run_id
+ self.organization_id = organization_id
+ self.log_id_key = f"{entity_type}_id"
+ self.log_id_value = entity_id
self.refresh_interval = refresh_interval
self.cdp_session: CDPSession | None = None
self.page: object | None = None
@@ -77,7 +83,14 @@ class ActivePageCdpInputSession:
if not force_refresh and now < self.next_refresh_at:
return None if self.page_resolution_failed else self.cdp_session
- page = await self.browser_state.get_working_page()
+ page = await _resolve_working_page(
+ self.browser_state,
+ self.entity_id,
+ self.entity_type,
+ self.workflow_run_id,
+ self.organization_id,
+ fall_back_to_captured=self.cdp_session is None,
+ )
if page is None:
self.page_resolution_failed = True
self.next_refresh_at = now + self.refresh_interval
@@ -381,13 +394,7 @@ async def cdp_input_stream(
await websocket.close(code=4408, reason="browser_state_timeout")
return
- page = await browser_state.get_working_page()
- if page is None:
- LOG.warning("CDP input: no working page", workflow_run_id=workflow_run_id)
- await websocket.close(code=4410, reason="no_working_page")
- return
-
- input_session = ActivePageCdpInputSession(browser_state, "workflow_run_id", workflow_run_id)
+ input_session = ActivePageCdpInputSession(browser_state, workflow_run_id, "workflow_run")
cdp_session = await input_session.get_session(force_refresh=True)
if cdp_session is None:
LOG.warning("CDP input: no working page", workflow_run_id=workflow_run_id)
@@ -465,13 +472,9 @@ async def cdp_input_browser_session_stream(
await websocket.close(code=4408, reason="browser_state_timeout")
return
- page = await browser_state.get_working_page()
- if page is None:
- LOG.warning("CDP input: no working page", browser_session_id=browser_session_id)
- await websocket.close(code=4410, reason="no_working_page")
- return
-
- input_session = ActivePageCdpInputSession(browser_state, "browser_session_id", browser_session_id)
+ input_session = ActivePageCdpInputSession(
+ browser_state, browser_session_id, "browser_session", organization_id=organization_id
+ )
if await input_session.get_session(force_refresh=True) is None:
LOG.warning("CDP input: no working page", browser_session_id=browser_session_id)
await websocket.close(code=4410, reason="no_working_page")
diff --git a/skyvern/forge/sdk/routes/streaming/screencast.py b/skyvern/forge/sdk/routes/streaming/screencast.py
index 639f7d3f4..4e8480307 100644
--- a/skyvern/forge/sdk/routes/streaming/screencast.py
+++ b/skyvern/forge/sdk/routes/streaming/screencast.py
@@ -65,12 +65,44 @@ async def _resolve_browser_state(
return None
+async def _resolve_working_page(
+ browser_state: BrowserState,
+ entity_id: str,
+ entity_type: str,
+ workflow_run_id: str | None = None,
+ organization_id: str | None = None,
+ fall_back_to_captured: bool = True,
+) -> object | None:
+ # Re-resolve each poll so the screencast follows a BrowserState the run swaps in post-connect (skyvern#6703).
+ try:
+ state = await _resolve_browser_state(entity_id, entity_type, workflow_run_id, organization_id)
+ except Exception:
+ state = None
+ if state is not None:
+ try:
+ page = await state.get_working_page()
+ except Exception:
+ page = None
+ if page is not None:
+ return page
+ # Captured fallback (validated by wait_for_browser_state) only before the first rebind; afterwards a transient
+ # failure returns None so the caller keeps the live page instead of downgrading to the initial about:blank.
+ if not fall_back_to_captured:
+ return None
+ try:
+ return await browser_state.get_working_page()
+ except Exception:
+ return None
+
+
async def start_screencast_loop(
websocket: WebSocket,
browser_state: BrowserState,
entity_id: str,
entity_type: str,
check_finalized: Callable[[], Awaitable[bool]],
+ workflow_run_id: str | None = None,
+ organization_id: str | None = None,
) -> None:
id_key = f"{entity_type}_id"
cdp_session: CDPSession | None = None
@@ -208,12 +240,11 @@ async def start_screencast_loop(
while True:
data = await frame_queue.get()
current_url = ""
- try:
- page = await browser_state.get_working_page()
- if page is not None:
- current_url = page.url
- except Exception:
- pass
+ if attached_page is not None:
+ try:
+ current_url = getattr(attached_page, "url", "") or ""
+ except Exception:
+ pass
try:
await websocket.send_json(
{
@@ -247,7 +278,14 @@ async def start_screencast_loop(
while True:
await asyncio.sleep(ACTIVE_PAGE_POLL_INTERVAL)
try:
- page = await browser_state.get_working_page()
+ page = await _resolve_working_page(
+ browser_state,
+ entity_id,
+ entity_type,
+ workflow_run_id,
+ organization_id,
+ fall_back_to_captured=False,
+ )
if page is not None and page is not attached_page:
await _attach_to_page(page)
except Exception:
@@ -259,7 +297,7 @@ async def start_screencast_loop(
)
try:
- page = await browser_state.get_working_page()
+ page = await _resolve_working_page(browser_state, entity_id, entity_type, workflow_run_id, organization_id)
if page is None:
raise RuntimeError("No working page available for screencast")
diff --git a/skyvern/forge/sdk/routes/streaming/screenshot.py b/skyvern/forge/sdk/routes/streaming/screenshot.py
index b820daf20..0876c9473 100644
--- a/skyvern/forge/sdk/routes/streaming/screenshot.py
+++ b/skyvern/forge/sdk/routes/streaming/screenshot.py
@@ -380,6 +380,8 @@ async def _run_local_screencast(
entity_id=entity_id,
entity_type=entity_type,
check_finalized=check_finalized,
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
)
final_status = await get_current_status()
diff --git a/tests/unit_tests/test_streaming_screenshot_local.py b/tests/unit_tests/test_streaming_screenshot_local.py
index 834bcca52..688eee287 100644
--- a/tests/unit_tests/test_streaming_screenshot_local.py
+++ b/tests/unit_tests/test_streaming_screenshot_local.py
@@ -43,6 +43,8 @@ async def test_run_local_screencast_happy_path(monkeypatch: pytest.MonkeyPatch)
entity_id="task_123",
entity_type="task",
check_finalized=check_finalized,
+ workflow_run_id="wr_123",
+ organization_id=None,
)
get_current_status.assert_awaited_once()
send_status_mock.assert_awaited_once_with(websocket, "task_id", "task_123", "completed")