Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

support multi-tab recording

EnzoFanAccount7bae4e09c061
diff --git a/browser_use/browser/video_recorder.py b/browser_use/browser/video_recorder.py
index 86dfcc998..37c4d1ab6 100644
--- a/browser_use/browser/video_recorder.py
+++ b/browser_use/browser/video_recorder.py
@@ -1,9 +1,9 @@
"""Video Recording Service for Browser Use Sessions."""
import base64
+import io
import logging
import math
-import subprocess
from pathlib import Path
from typing import Optional
@@ -11,9 +11,9 @@ from browser_use.browser.profile import ViewportSize
try:
import imageio.v2 as iio # type: ignore[import-not-found]
- import imageio_ffmpeg # type: ignore[import-not-found]
import numpy as np # type: ignore[import-not-found]
from imageio.core.format import Format # type: ignore[import-not-found]
+ from PIL import Image
IMAGEIO_AVAILABLE = True
except ImportError:
@@ -98,46 +98,25 @@ class VideoRecorderService:
try:
frame_bytes = base64.b64decode(frame_data_b64)
- # Build a filter chain for ffmpeg:
- # 1. scale: Resizes the frame to the user-specified dimensions.
- # 2. pad: Adds black bars to meet codec's macro-block requirements,
- # centering the original content.
- vf_chain = (
- f'scale={self.size["width"]}:{self.size["height"]},'
- f'pad={self.padded_size["width"]}:{self.padded_size["height"]}:(ow-iw)/2:(oh-ih)/2:color=black'
- )
-
- output_pix_fmt = 'rgb24'
- command = [
- imageio_ffmpeg.get_ffmpeg_exe(),
- '-f',
- 'image2pipe', # Input format from a pipe
- '-c:v',
- 'png', # Specify input codec is PNG
- '-i',
- '-', # Input from stdin
- '-vf',
- vf_chain, # Video filter for resizing and padding
- '-f',
- 'rawvideo', # Output format is raw video
- '-pix_fmt',
- output_pix_fmt, # Output pixel format
- '-', # Output to stdout
- ]
-
- # Execute ffmpeg as a subprocess
- proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- out, err = proc.communicate(input=frame_bytes)
-
- if proc.returncode != 0:
- err_msg = err.decode(errors='ignore').strip()
- if 'deprecated pixel format used' not in err_msg.lower():
- raise OSError(f'ffmpeg error during resizing/padding: {err_msg}')
- else:
- logger.debug(f'ffmpeg warning during resizing/padding: {err_msg}')
-
- # Convert the raw output bytes to a numpy array with the padded dimensions
- img_array = np.frombuffer(out, dtype=np.uint8).reshape((self.padded_size['height'], self.padded_size['width'], 3))
+ # Use PIL to handle image processing in memory - much faster than spawning ffmpeg subprocess per frame
+ with Image.open(io.BytesIO(frame_bytes)) as img:
+ # 1. Resize if needed to target viewport size
+ if img.size != (self.size['width'], self.size['height']):
+ # Use BICUBIC as it's faster than LANCZOS and good enough for screen recordings
+ img = img.resize((self.size['width'], self.size['height']), Image.Resampling.BICUBIC)
+
+ # 2. Handle Padding (Macro block alignment for codecs)
+ # Check if padding is actually needed
+ if self.padded_size['width'] != self.size['width'] or self.padded_size['height'] != self.size['height']:
+ new_img = Image.new('RGB', (self.padded_size['width'], self.padded_size['height']), (0, 0, 0))
+ # Center the image
+ x_offset = (self.padded_size['width'] - self.size['width']) // 2
+ y_offset = (self.padded_size['height'] - self.size['height']) // 2
+ new_img.paste(img, (x_offset, y_offset))
+ img = new_img
+
+ # 3. Convert to numpy array for imageio
+ img_array = np.array(img)
self._writer.append_data(img_array)
except Exception as e:
diff --git a/browser_use/browser/watchdogs/recording_watchdog.py b/browser_use/browser/watchdogs/recording_watchdog.py
index 6c09abbf3..a9b74f9e4 100644
--- a/browser_use/browser/watchdogs/recording_watchdog.py
+++ b/browser_use/browser/watchdogs/recording_watchdog.py
@@ -2,13 +2,14 @@
import asyncio
from pathlib import Path
-from typing import ClassVar
+from typing import Any, ClassVar
from bubus import BaseEvent
from cdp_use.cdp.page.events import ScreencastFrameEvent
+from pydantic import PrivateAttr
from uuid_extensions import uuid7str
-from browser_use.browser.events import BrowserConnectedEvent, BrowserStopEvent
+from browser_use.browser.events import AgentFocusChangedEvent, BrowserConnectedEvent, BrowserStopEvent
from browser_use.browser.profile import ViewportSize
from browser_use.browser.video_recorder import VideoRecorderService
from browser_use.browser.watchdog_base import BaseWatchdog
@@ -20,10 +21,12 @@ class RecordingWatchdog(BaseWatchdog):
Manages video recording of a browser session using CDP screencasting.
"""
- LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [BrowserConnectedEvent, BrowserStopEvent]
+ LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [BrowserConnectedEvent, BrowserStopEvent, AgentFocusChangedEvent]
EMITS: ClassVar[list[type[BaseEvent]]] = []
- _recorder: VideoRecorderService | None = None
+ _recorder: VideoRecorderService | None = PrivateAttr(default=None)
+ _current_session_id: str | None = PrivateAttr(default=None)
+ _screencast_params: dict[str, Any] | None = PrivateAttr(default=None)
async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
"""
@@ -56,24 +59,58 @@ class RecordingWatchdog(BaseWatchdog):
self.browser_session.cdp_client.register.Page.screencastFrame(self.on_screencastFrame)
+ self._screencast_params = {
+ 'format': 'png',
+ 'quality': 90,
+ 'maxWidth': size['width'],
+ 'maxHeight': size['height'],
+ 'everyNthFrame': 1,
+ }
+
+ await self._start_screencast()
+
+ async def on_AgentFocusChangedEvent(self, event: AgentFocusChangedEvent) -> None:
+ """
+ Switches video recording to the new tab.
+ """
+ if self._recorder:
+ self.logger.debug(f'Agent focus changed to {event.target_id}, switching screencast...')
+ await self._start_screencast()
+
+ async def _start_screencast(self) -> None:
+ """Starts screencast on the currently focused tab."""
+ if not self._recorder or not self._screencast_params:
+ return
+
try:
+ # Get the current session (for the focused target)
cdp_session = await self.browser_session.get_or_create_cdp_session()
+
+ # If we are already recording this session, do nothing
+ if self._current_session_id == cdp_session.session_id:
+ return
+
+ # Stop recording on the previous session
+ if self._current_session_id:
+ try:
+ # Use the root client to stop screencast on the specific session
+ await self.browser_session.cdp_client.send.Page.stopScreencast(session_id=self._current_session_id)
+ except Exception as e:
+ # It's possible the session is already closed
+ self.logger.debug(f'Failed to stop screencast on old session {self._current_session_id}: {e}')
+
+ self._current_session_id = cdp_session.session_id
+
+ # Start recording on the new session
await cdp_session.cdp_client.send.Page.startScreencast(
- params={
- 'format': 'png',
- 'quality': 90,
- 'maxWidth': size['width'],
- 'maxHeight': size['height'],
- 'everyNthFrame': 1,
- },
+ params=self._screencast_params, # type: ignore
session_id=cdp_session.session_id,
)
- self.logger.info(f'📹 Started video recording to {output_path}')
+ self.logger.info(f'📹 Started/Switched video recording to target {cdp_session.target_id}')
except Exception as e:
- self.logger.error(f'Failed to start screencast via CDP: {e}')
- if self._recorder:
- self._recorder.stop_and_save()
- self._recorder = None
+ self.logger.error(f'Failed to switch screencast via CDP: {e}')
+ # If we fail to start on the new tab, we reset current session id
+ self._current_session_id = None
async def _get_current_viewport_size(self) -> ViewportSize | None:
"""Gets the current viewport size directly from the browser via CDP."""
@@ -98,6 +135,10 @@ class RecordingWatchdog(BaseWatchdog):
"""
Synchronous handler for incoming screencast frames.
"""
+ # Only process frames from the current session we intend to record
+ # This handles race conditions where old session might still send frames before stop completes
+ if self._current_session_id and session_id != self._current_session_id:
+ return
if not self._recorder:
return
@@ -127,6 +168,8 @@ class RecordingWatchdog(BaseWatchdog):
if self._recorder:
recorder = self._recorder
self._recorder = None
+ self._current_session_id = None
+ self._screencast_params = None
self.logger.debug('Stopping video recording and saving file...')
loop = asyncio.get_event_loop()