Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9974): capture popup page recordings via context-level listener (#6078)
LawyZheng2b75c67b1556
diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.pyindex b13589305..277f3e0e1 100644--- a/skyvern/webeye/browser_factory.py+++ b/skyvern/webeye/browser_factory.py@@ -118,6 +118,37 @@ def set_browser_console_log(browser_context: BrowserContext, browser_artifacts:browser_context.on("console", browser_console_log)+def set_popup_video_listener(browser_context: BrowserContext, browser_artifacts: BrowserArtifacts) -> None:+ tracked_paths: set[str] = set()++ async def _on_page(page: Page) -> None:+ try:+ if not page.video:+ return+ async with asyncio.timeout(settings.BROWSER_ACTION_TIMEOUT_MS / 1000):+ raw_path = await page.video.path()+ if raw_path is None:+ return+ video_path = str(raw_path)+ # After the await, another handler may have already registered this path+ if video_path in tracked_paths:+ return+ for va in browser_artifacts.video_artifacts:+ if va.video_path == video_path:+ tracked_paths.add(video_path)+ return+ tracked_paths.add(video_path)+ browser_artifacts.video_artifacts.append(VideoArtifact(video_path=video_path))+ except Exception:+ LOG.warning("Failed to register popup page video", exc_info=True)++ browser_context.on("page", _on_page)++ # Register pages that already exist (e.g. the initial page from launch_persistent_context)+ for page in browser_context.pages:+ asyncio.ensure_future(_on_page(page))++def set_download_file_listener(browser_context: BrowserContext, download_timeout: float | None = None, **kwargs: Any) -> None:@@ -355,6 +386,7 @@ class BrowserContextFactory:browser_context, browser_artifacts, cleanup_func = await creator(playwright, **kwargs)if settings.BROWSER_LOGS_ENABLED:set_browser_console_log(browser_context=browser_context, browser_artifacts=browser_artifacts)+ set_popup_video_listener(browser_context=browser_context, browser_artifacts=browser_artifacts)set_download_file_listener(browser_context=browser_context, **kwargs)set_dialog_handler(browser_context=browser_context)diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.pyindex 567daca49..d81478616 100644--- a/skyvern/webeye/real_browser_state.py+++ b/skyvern/webeye/real_browser_state.py@@ -21,7 +21,7 @@ from skyvern.exceptions import (from skyvern.forge import appfrom skyvern.forge.sdk.trace import tracedfrom skyvern.schemas.runs import ProxyLocationInput-from skyvern.webeye.browser_artifacts import BrowserArtifacts, VideoArtifact+from skyvern.webeye.browser_artifacts import BrowserArtifactsfrom skyvern.webeye.browser_factory import BrowserCleanupFunc, BrowserContextFactoryfrom skyvern.webeye.browser_state import BrowserStatefrom skyvern.webeye.navigation import is_permanent_navigation_error, navigate_with_retry@@ -241,33 +241,6 @@ class RealBrowserState(BrowserState):async def set_working_page(self, page: Page | None, index: int = 0) -> None:self.__page = page- if page is None:- return- if len(self.browser_artifacts.video_artifacts) > index:- if self.browser_artifacts.video_artifacts[index].video_path is None:- try:- async with asyncio.timeout(settings.BROWSER_ACTION_TIMEOUT_MS / 1000):- if page.video:- self.browser_artifacts.video_artifacts[index].video_path = await page.video.path()- except asyncio.TimeoutError:- LOG.info("Timeout to get the page video, skip the exception")- except Exception:- LOG.exception("Error while getting the page video", exc_info=True)- return-- target_length = index + 1- self.browser_artifacts.video_artifacts.extend(- [VideoArtifact()] * (target_length - len(self.browser_artifacts.video_artifacts))- )- try:- async with asyncio.timeout(settings.BROWSER_ACTION_TIMEOUT_MS / 1000):- if page.video:- self.browser_artifacts.video_artifacts[index].video_path = await page.video.path()- except asyncio.TimeoutError:- LOG.info("Timeout to get the page video, skip the exception")- except Exception:- LOG.exception("Error while getting the page video", exc_info=True)- returnasync def get_or_create_page(self,diff --git a/tests/unit/test_real_browser_manager.py b/tests/unit/test_real_browser_manager.pyindex 41895b04c..41a6001af 100644--- a/tests/unit/test_real_browser_manager.py+++ b/tests/unit/test_real_browser_manager.py@@ -10,7 +10,10 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytest+from skyvern.webeye.browser_artifacts import BrowserArtifacts, VideoArtifact+from skyvern.webeye.browser_factory import set_popup_video_listenerfrom skyvern.webeye.real_browser_manager import RealBrowserManager+from skyvern.webeye.real_browser_state import RealBrowserStatedef make_workflow_run(@@ -312,3 +315,110 @@ async def test_get_video_artifacts_finalize_false_skips_ffmpeg(tmp_path) -> Nonem.assert_not_awaited()assert artifacts[0].video_data == b"partial-webm-bytes"+++def _make_page_mock(video_path: str | None) -> MagicMock:+ page = MagicMock()+ if video_path is None:+ page.video = None+ else:+ page.video = MagicMock()+ page.video.path = AsyncMock(return_value=video_path)+ return page+++async def test_popup_video_listener_picks_up_popup_page() -> None:+ """set_popup_video_listener registers popup video paths on the page event."""++ artifacts = BrowserArtifacts(video_artifacts=[VideoArtifact(video_path="/tmp/videos/main.webm")])+ browser_context = MagicMock()+ set_popup_video_listener(browser_context=browser_context, browser_artifacts=artifacts)++ handler = browser_context.on.call_args[0][1]+ popup = _make_page_mock("/tmp/videos/popup.webm")+ await handler(popup)++ paths = [va.video_path for va in artifacts.video_artifacts]+ assert paths == ["/tmp/videos/main.webm", "/tmp/videos/popup.webm"]+++async def test_popup_video_listener_deduplicates() -> None:+ """Already-tracked pages are not added twice."""++ artifacts = BrowserArtifacts(video_artifacts=[VideoArtifact(video_path="/tmp/videos/main.webm")])+ browser_context = MagicMock()+ set_popup_video_listener(browser_context=browser_context, browser_artifacts=artifacts)++ handler = browser_context.on.call_args[0][1]+ page = _make_page_mock("/tmp/videos/main.webm")+ await handler(page)++ assert len(artifacts.video_artifacts) == 1+++async def test_popup_video_listener_skips_pages_without_video() -> None:+ """Pages with no video (e.g. about:blank) are silently skipped."""++ artifacts = BrowserArtifacts()+ browser_context = MagicMock()+ set_popup_video_listener(browser_context=browser_context, browser_artifacts=artifacts)++ handler = browser_context.on.call_args[0][1]+ await handler(_make_page_mock(None))++ assert len(artifacts.video_artifacts) == 0+++async def test_popup_video_listener_multiple_popups() -> None:+ """Multiple popup pages from loop iterations are all captured."""++ artifacts = BrowserArtifacts(video_artifacts=[VideoArtifact(video_path="/tmp/videos/main.webm")])+ browser_context = MagicMock()+ set_popup_video_listener(browser_context=browser_context, browser_artifacts=artifacts)++ handler = browser_context.on.call_args[0][1]+ for name in ["popup1", "popup2", "popup3"]:+ await handler(_make_page_mock(f"/tmp/videos/{name}.webm"))++ paths = [va.video_path for va in artifacts.video_artifacts]+ assert paths == [+ "/tmp/videos/main.webm",+ "/tmp/videos/popup1.webm",+ "/tmp/videos/popup2.webm",+ "/tmp/videos/popup3.webm",+ ]+++async def test_set_working_page_does_not_touch_video_artifacts() -> None:+ """set_working_page only sets the working page; video tracking is handled by the listener."""++ artifacts = BrowserArtifacts()+ state = RealBrowserState(pw=MagicMock(), browser_context=MagicMock(), browser_artifacts=artifacts)++ page = _make_page_mock("/tmp/v/page.webm")+ await state.set_working_page(page, index=0)++ assert len(artifacts.video_artifacts) == 0+++async def test_popup_video_listener_registers_pre_existing_pages() -> None:+ """Pages that already exist when the listener is registered are captured."""+ import asyncio++ artifacts = BrowserArtifacts()+ initial_page = _make_page_mock("/tmp/videos/initial.webm")+ browser_context = MagicMock()+ browser_context.pages = [initial_page]+ set_popup_video_listener(browser_context=browser_context, browser_artifacts=artifacts)++ # Let the ensure_future tasks run+ await asyncio.sleep(0)++ paths = [va.video_path for va in artifacts.video_artifacts]+ assert paths == ["/tmp/videos/initial.webm"]