Skip to content

Automate browser based workflows with AI

fix(SKY-9974): capture popup page recordings via context-level listener (#6078)

LawyZheng2b75c67b1556
diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.py
index b13589305..277f3e0e1 100644
--- a/skyvern/webeye/browser_factory.py
+++ b/skyvern/webeye/browser_factory.py
@@ -118,6 +118,37 @@ def set_browser_console_log(browser_context: BrowserContext, browser_artifacts:
browser_context.on("console", browser_console_log)
+def set_popup_video_listener(browser_context: BrowserContext, browser_artifacts: BrowserArtifacts) -> None:
+ tracked_paths: set[str] = set()
+
+ async def _on_page(page: Page) -> None:
+ try:
+ if not page.video:
+ return
+ async with asyncio.timeout(settings.BROWSER_ACTION_TIMEOUT_MS / 1000):
+ raw_path = await page.video.path()
+ if raw_path is None:
+ return
+ video_path = str(raw_path)
+ # After the await, another handler may have already registered this path
+ if video_path in tracked_paths:
+ return
+ for va in browser_artifacts.video_artifacts:
+ if va.video_path == video_path:
+ tracked_paths.add(video_path)
+ return
+ tracked_paths.add(video_path)
+ browser_artifacts.video_artifacts.append(VideoArtifact(video_path=video_path))
+ except Exception:
+ LOG.warning("Failed to register popup page video", exc_info=True)
+
+ browser_context.on("page", _on_page)
+
+ # Register pages that already exist (e.g. the initial page from launch_persistent_context)
+ for page in browser_context.pages:
+ asyncio.ensure_future(_on_page(page))
+
+
def set_download_file_listener(
browser_context: BrowserContext, download_timeout: float | None = None, **kwargs: Any
) -> None:
@@ -355,6 +386,7 @@ class BrowserContextFactory:
browser_context, browser_artifacts, cleanup_func = await creator(playwright, **kwargs)
if settings.BROWSER_LOGS_ENABLED:
set_browser_console_log(browser_context=browser_context, browser_artifacts=browser_artifacts)
+ set_popup_video_listener(browser_context=browser_context, browser_artifacts=browser_artifacts)
set_download_file_listener(browser_context=browser_context, **kwargs)
set_dialog_handler(browser_context=browser_context)
diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.py
index 567daca49..d81478616 100644
--- a/skyvern/webeye/real_browser_state.py
+++ b/skyvern/webeye/real_browser_state.py
@@ -21,7 +21,7 @@ from skyvern.exceptions import (
from skyvern.forge import app
from skyvern.forge.sdk.trace import traced
from skyvern.schemas.runs import ProxyLocationInput
-from skyvern.webeye.browser_artifacts import BrowserArtifacts, VideoArtifact
+from skyvern.webeye.browser_artifacts import BrowserArtifacts
from skyvern.webeye.browser_factory import BrowserCleanupFunc, BrowserContextFactory
from skyvern.webeye.browser_state import BrowserState
from skyvern.webeye.navigation import is_permanent_navigation_error, navigate_with_retry
@@ -241,33 +241,6 @@ class RealBrowserState(BrowserState):
async def set_working_page(self, page: Page | None, index: int = 0) -> None:
self.__page = page
- if page is None:
- return
- if len(self.browser_artifacts.video_artifacts) > index:
- if self.browser_artifacts.video_artifacts[index].video_path is None:
- try:
- async with asyncio.timeout(settings.BROWSER_ACTION_TIMEOUT_MS / 1000):
- if page.video:
- self.browser_artifacts.video_artifacts[index].video_path = await page.video.path()
- except asyncio.TimeoutError:
- LOG.info("Timeout to get the page video, skip the exception")
- except Exception:
- LOG.exception("Error while getting the page video", exc_info=True)
- return
-
- target_length = index + 1
- self.browser_artifacts.video_artifacts.extend(
- [VideoArtifact()] * (target_length - len(self.browser_artifacts.video_artifacts))
- )
- try:
- async with asyncio.timeout(settings.BROWSER_ACTION_TIMEOUT_MS / 1000):
- if page.video:
- self.browser_artifacts.video_artifacts[index].video_path = await page.video.path()
- except asyncio.TimeoutError:
- LOG.info("Timeout to get the page video, skip the exception")
- except Exception:
- LOG.exception("Error while getting the page video", exc_info=True)
- return
async def get_or_create_page(
self,
diff --git a/tests/unit/test_real_browser_manager.py b/tests/unit/test_real_browser_manager.py
index 41895b04c..41a6001af 100644
--- a/tests/unit/test_real_browser_manager.py
+++ b/tests/unit/test_real_browser_manager.py
@@ -10,7 +10,10 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from skyvern.webeye.browser_artifacts import BrowserArtifacts, VideoArtifact
+from skyvern.webeye.browser_factory import set_popup_video_listener
from skyvern.webeye.real_browser_manager import RealBrowserManager
+from skyvern.webeye.real_browser_state import RealBrowserState
def make_workflow_run(
@@ -312,3 +315,110 @@ async def test_get_video_artifacts_finalize_false_skips_ffmpeg(tmp_path) -> None
m.assert_not_awaited()
assert artifacts[0].video_data == b"partial-webm-bytes"
+
+
+def _make_page_mock(video_path: str | None) -> MagicMock:
+ page = MagicMock()
+ if video_path is None:
+ page.video = None
+ else:
+ page.video = MagicMock()
+ page.video.path = AsyncMock(return_value=video_path)
+ return page
+
+
+async def test_popup_video_listener_picks_up_popup_page() -> None:
+ """set_popup_video_listener registers popup video paths on the page event."""
+
+ artifacts = BrowserArtifacts(video_artifacts=[VideoArtifact(video_path="/tmp/videos/main.webm")])
+ browser_context = MagicMock()
+ set_popup_video_listener(browser_context=browser_context, browser_artifacts=artifacts)
+
+ handler = browser_context.on.call_args[0][1]
+ popup = _make_page_mock("/tmp/videos/popup.webm")
+ await handler(popup)
+
+ paths = [va.video_path for va in artifacts.video_artifacts]
+ assert paths == ["/tmp/videos/main.webm", "/tmp/videos/popup.webm"]
+
+
+async def test_popup_video_listener_deduplicates() -> None:
+ """Already-tracked pages are not added twice."""
+
+ artifacts = BrowserArtifacts(video_artifacts=[VideoArtifact(video_path="/tmp/videos/main.webm")])
+ browser_context = MagicMock()
+ set_popup_video_listener(browser_context=browser_context, browser_artifacts=artifacts)
+
+ handler = browser_context.on.call_args[0][1]
+ page = _make_page_mock("/tmp/videos/main.webm")
+ await handler(page)
+
+ assert len(artifacts.video_artifacts) == 1
+
+
+async def test_popup_video_listener_skips_pages_without_video() -> None:
+ """Pages with no video (e.g. about:blank) are silently skipped."""
+
+ artifacts = BrowserArtifacts()
+ browser_context = MagicMock()
+ set_popup_video_listener(browser_context=browser_context, browser_artifacts=artifacts)
+
+ handler = browser_context.on.call_args[0][1]
+ await handler(_make_page_mock(None))
+
+ assert len(artifacts.video_artifacts) == 0
+
+
+async def test_popup_video_listener_multiple_popups() -> None:
+ """Multiple popup pages from loop iterations are all captured."""
+
+ artifacts = BrowserArtifacts(video_artifacts=[VideoArtifact(video_path="/tmp/videos/main.webm")])
+ browser_context = MagicMock()
+ set_popup_video_listener(browser_context=browser_context, browser_artifacts=artifacts)
+
+ handler = browser_context.on.call_args[0][1]
+ for name in ["popup1", "popup2", "popup3"]:
+ await handler(_make_page_mock(f"/tmp/videos/{name}.webm"))
+
+ paths = [va.video_path for va in artifacts.video_artifacts]
+ assert paths == [
+ "/tmp/videos/main.webm",
+ "/tmp/videos/popup1.webm",
+ "/tmp/videos/popup2.webm",
+ "/tmp/videos/popup3.webm",
+ ]
+
+
+async def test_set_working_page_does_not_touch_video_artifacts() -> None:
+ """set_working_page only sets the working page; video tracking is handled by the listener."""
+
+ artifacts = BrowserArtifacts()
+ state = RealBrowserState(pw=MagicMock(), browser_context=MagicMock(), browser_artifacts=artifacts)
+
+ page = _make_page_mock("/tmp/v/page.webm")
+ await state.set_working_page(page, index=0)
+
+ assert len(artifacts.video_artifacts) == 0
+
+
+async def test_popup_video_listener_registers_pre_existing_pages() -> None:
+ """Pages that already exist when the listener is registered are captured."""
+ import asyncio
+
+ artifacts = BrowserArtifacts()
+ initial_page = _make_page_mock("/tmp/videos/initial.webm")
+ browser_context = MagicMock()
+ browser_context.pages = [initial_page]
+ set_popup_video_listener(browser_context=browser_context, browser_artifacts=artifacts)
+
+ # Let the ensure_future tasks run
+ await asyncio.sleep(0)
+
+ paths = [va.video_path for va in artifacts.video_artifacts]
+ assert paths == ["/tmp/videos/initial.webm"]