Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11215): bind live screencast to the run's current browser state (#6865)
Aaron Perez56e31ce2ca4b
diff --git a/skyvern/forge/sdk/routes/streaming/cdp_input.py b/skyvern/forge/sdk/routes/streaming/cdp_input.pyindex e4cb21bf2..4673f1a70 100644--- a/skyvern/forge/sdk/routes/streaming/cdp_input.py+++ b/skyvern/forge/sdk/routes/streaming/cdp_input.py@@ -22,7 +22,7 @@ from skyvern.forge.sdk.routes.streaming.registries import (stream_ref_dec,stream_ref_inc,)-from skyvern.forge.sdk.routes.streaming.screencast import wait_for_browser_state+from skyvern.forge.sdk.routes.streaming.screencast import _resolve_working_page, wait_for_browser_statefrom skyvern.forge.sdk.schemas.persistent_browser_sessions import is_final_statusfrom skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatusfrom skyvern.webeye.browser_state import BrowserState@@ -59,13 +59,19 @@ class ActivePageCdpInputSession:def __init__(self,browser_state: BrowserState,- log_id_key: str,- log_id_value: str,+ entity_id: str,+ entity_type: str,+ workflow_run_id: str | None = None,+ organization_id: str | None = None,refresh_interval: float = ACTIVE_PAGE_INPUT_REFRESH_INTERVAL,) -> None:self.browser_state = browser_state- self.log_id_key = log_id_key- self.log_id_value = log_id_value+ self.entity_id = entity_id+ self.entity_type = entity_type+ self.workflow_run_id = workflow_run_id+ self.organization_id = organization_id+ self.log_id_key = f"{entity_type}_id"+ self.log_id_value = entity_idself.refresh_interval = refresh_intervalself.cdp_session: CDPSession | None = Noneself.page: object | None = None@@ -77,7 +83,14 @@ class ActivePageCdpInputSession:if not force_refresh and now < self.next_refresh_at:return None if self.page_resolution_failed else self.cdp_session- page = await self.browser_state.get_working_page()+ page = await _resolve_working_page(+ self.browser_state,+ self.entity_id,+ self.entity_type,+ self.workflow_run_id,+ self.organization_id,+ fall_back_to_captured=self.cdp_session is None,+ )if page is None:self.page_resolution_failed = Trueself.next_refresh_at = now + self.refresh_interval@@ -381,13 +394,7 @@ async def cdp_input_stream(await websocket.close(code=4408, reason="browser_state_timeout")return- page = await browser_state.get_working_page()- if page is None:- LOG.warning("CDP input: no working page", workflow_run_id=workflow_run_id)- await websocket.close(code=4410, reason="no_working_page")- return-- input_session = ActivePageCdpInputSession(browser_state, "workflow_run_id", workflow_run_id)+ input_session = ActivePageCdpInputSession(browser_state, workflow_run_id, "workflow_run")cdp_session = await input_session.get_session(force_refresh=True)if cdp_session is None:LOG.warning("CDP input: no working page", workflow_run_id=workflow_run_id)@@ -465,13 +472,9 @@ async def cdp_input_browser_session_stream(await websocket.close(code=4408, reason="browser_state_timeout")return- page = await browser_state.get_working_page()- if page is None:- LOG.warning("CDP input: no working page", browser_session_id=browser_session_id)- await websocket.close(code=4410, reason="no_working_page")- return-- input_session = ActivePageCdpInputSession(browser_state, "browser_session_id", browser_session_id)+ input_session = ActivePageCdpInputSession(+ browser_state, browser_session_id, "browser_session", organization_id=organization_id+ )if await input_session.get_session(force_refresh=True) is None:LOG.warning("CDP input: no working page", browser_session_id=browser_session_id)await websocket.close(code=4410, reason="no_working_page")diff --git a/skyvern/forge/sdk/routes/streaming/screencast.py b/skyvern/forge/sdk/routes/streaming/screencast.pyindex 639f7d3f4..4e8480307 100644--- a/skyvern/forge/sdk/routes/streaming/screencast.py+++ b/skyvern/forge/sdk/routes/streaming/screencast.py@@ -65,12 +65,44 @@ async def _resolve_browser_state(return None+async def _resolve_working_page(+ browser_state: BrowserState,+ entity_id: str,+ entity_type: str,+ workflow_run_id: str | None = None,+ organization_id: str | None = None,+ fall_back_to_captured: bool = True,+) -> object | None:+ # Re-resolve each poll so the screencast follows a BrowserState the run swaps in post-connect (skyvern#6703).+ try:+ state = await _resolve_browser_state(entity_id, entity_type, workflow_run_id, organization_id)+ except Exception:+ state = None+ if state is not None:+ try:+ page = await state.get_working_page()+ except Exception:+ page = None+ if page is not None:+ return page+ # Captured fallback (validated by wait_for_browser_state) only before the first rebind; afterwards a transient+ # failure returns None so the caller keeps the live page instead of downgrading to the initial about:blank.+ if not fall_back_to_captured:+ return None+ try:+ return await browser_state.get_working_page()+ except Exception:+ return None++async def start_screencast_loop(websocket: WebSocket,browser_state: BrowserState,entity_id: str,entity_type: str,check_finalized: Callable[[], Awaitable[bool]],+ workflow_run_id: str | None = None,+ organization_id: str | None = None,) -> None:id_key = f"{entity_type}_id"cdp_session: CDPSession | None = None@@ -208,12 +240,11 @@ async def start_screencast_loop(while True:data = await frame_queue.get()current_url = ""- try:- page = await browser_state.get_working_page()- if page is not None:- current_url = page.url- except Exception:- pass+ if attached_page is not None:+ try:+ current_url = getattr(attached_page, "url", "") or ""+ except Exception:+ passtry:await websocket.send_json({@@ -247,7 +278,14 @@ async def start_screencast_loop(while True:await asyncio.sleep(ACTIVE_PAGE_POLL_INTERVAL)try:- page = await browser_state.get_working_page()+ page = await _resolve_working_page(+ browser_state,+ entity_id,+ entity_type,+ workflow_run_id,+ organization_id,+ fall_back_to_captured=False,+ )if page is not None and page is not attached_page:await _attach_to_page(page)except Exception:@@ -259,7 +297,7 @@ async def start_screencast_loop()try:- page = await browser_state.get_working_page()+ page = await _resolve_working_page(browser_state, entity_id, entity_type, workflow_run_id, organization_id)if page is None:raise RuntimeError("No working page available for screencast")diff --git a/skyvern/forge/sdk/routes/streaming/screenshot.py b/skyvern/forge/sdk/routes/streaming/screenshot.pyindex b820daf20..0876c9473 100644--- a/skyvern/forge/sdk/routes/streaming/screenshot.py+++ b/skyvern/forge/sdk/routes/streaming/screenshot.py@@ -380,6 +380,8 @@ async def _run_local_screencast(entity_id=entity_id,entity_type=entity_type,check_finalized=check_finalized,+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,)final_status = await get_current_status()diff --git a/tests/unit_tests/test_streaming_screenshot_local.py b/tests/unit_tests/test_streaming_screenshot_local.pyindex 834bcca52..688eee287 100644--- a/tests/unit_tests/test_streaming_screenshot_local.py+++ b/tests/unit_tests/test_streaming_screenshot_local.py@@ -43,6 +43,8 @@ async def test_run_local_screencast_happy_path(monkeypatch: pytest.MonkeyPatch)entity_id="task_123",entity_type="task",check_finalized=check_finalized,+ workflow_run_id="wr_123",+ organization_id=None,)get_current_status.assert_awaited_once()send_status_mock.assert_awaited_once_with(websocket, "task_id", "task_123", "completed")