Skip to content

Agent Zero AI framework

Slim browser viewer frame transport

Alessandro91b3bc9613c7
diff --git a/docs/AGENTS.md b/docs/AGENTS.md
index 3f3002b01..c81fb55bd 100644
--- a/docs/AGENTS.md
+++ b/docs/AGENTS.md
@@ -9,6 +9,7 @@
- `README.md`, `quickstart.md`, `guides/`, and `setup/` cover user-facing setup and workflows.
- `developer/` covers compact developer references and source handoffs.
+- `plans/` covers implementation plans, migration notes, and staged technical roadmaps.
- `res/` contains documentation images and other documentation assets.
## Local Contracts
diff --git a/plugins/_browser/AGENTS.md b/plugins/_browser/AGENTS.md
index 15a7ac85c..e3f2a137e 100644
--- a/plugins/_browser/AGENTS.md
+++ b/plugins/_browser/AGENTS.md
@@ -19,6 +19,7 @@
- Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.
- Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation.
- Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport.
+- Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback.
- Keep narrow WebUI Browser controls usable by grouping navigation with Annotate/settings above a full-width address bar.
- Browser URL-intent handling must only claim web URL schemes and leave custom Agent Zero schemes to their owning surfaces.
- Prefer DOM/CDP browser actions with refs, selectors, frame-chain refs, and screenshots over viewport coordinate input. Coordinates remain a visual fallback.
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.py
index ec5e137b3..827e91df4 100644
--- a/plugins/_browser/api/ws_browser.py
+++ b/plugins/_browser/api/ws_browser.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import base64
import contextlib
import time
from typing import Any, ClassVar
@@ -15,7 +16,8 @@ FRAME_READ_TIMEOUT_SECONDS = 0.5
FRAME_RETRY_DELAY_SECONDS = 0.5
FRAME_STATE_REFRESH_SECONDS = 0.75
SNAPSHOT_STATE_POLL_SECONDS = 0.75
-SCREENCAST_QUALITY = 92
+SCREENCAST_STREAM_QUALITY = 80
+SCREENSHOT_QUALITY = 92
VIEWER_TRANSPORT_SCREENCAST = "screencast"
VIEWER_TRANSPORT_SNAPSHOT = "snapshot"
VIEWER_TRANSPORTS = {VIEWER_TRANSPORT_SCREENCAST, VIEWER_TRANSPORT_SNAPSHOT}
@@ -97,10 +99,21 @@ class WsBrowser(WsHandler):
existing.cancel()
viewer_id = str(data.get("viewer_id") or "")
viewer_transport = self._viewer_transport(data)
+ binary_frames = self._bool(data.get("binary_frames", data.get("binaryFrames")))
+ slim_frames = self._bool(data.get("slim_frames", data.get("slimFrames", binary_frames)))
+ capture_scale = self._capture_scale_from_data(data)
snapshot = None
if runtime:
if viewer_transport == VIEWER_TRANSPORT_SCREENCAST:
- stream_task = self._stream_frames(sid, context_id, active_id, viewer_id)
+ stream_task = self._stream_frames(
+ sid,
+ context_id,
+ active_id,
+ viewer_id,
+ binary_frames=binary_frames,
+ slim_frames=slim_frames,
+ capture_scale=capture_scale,
+ )
else:
stream_task = self._stream_state(sid, context_id, active_id, viewer_id)
self._streams[stream_key] = asyncio.create_task(stream_task)
@@ -115,6 +128,8 @@ class WsBrowser(WsHandler):
"all_browsers": True,
"viewer_id": viewer_id,
"viewer_transport": viewer_transport,
+ "binary_frames": binary_frames,
+ "slim_frames": slim_frames,
}
def _unsubscribe(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult:
@@ -157,9 +172,9 @@ class WsBrowser(WsHandler):
snapshot = None
if active_id:
try:
- quality = int(data.get("quality") or SCREENCAST_QUALITY)
+ quality = int(data.get("quality") or SCREENSHOT_QUALITY)
except (TypeError, ValueError):
- quality = SCREENCAST_QUALITY
+ quality = SCREENSHOT_QUALITY
with contextlib.suppress(Exception):
snapshot = await runtime.call(
"screenshot",
@@ -347,7 +362,7 @@ class WsBrowser(WsHandler):
if not browser_id:
return None
with contextlib.suppress(Exception):
- return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)
+ return await runtime.call("screenshot", browser_id, quality=SCREENSHOT_QUALITY)
return None
async def _snapshot_for_browser(
@@ -358,7 +373,7 @@ class WsBrowser(WsHandler):
if not browser_id:
return None
with contextlib.suppress(Exception):
- return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)
+ return await runtime.call("screenshot", browser_id, quality=SCREENSHOT_QUALITY)
return None
async def _all_browser_tabs(self) -> list[dict[str, Any]]:
@@ -377,6 +392,10 @@ class WsBrowser(WsHandler):
context_id: str,
browser_id: int | str | None,
viewer_id: str = "",
+ *,
+ binary_frames: bool = False,
+ slim_frames: bool = False,
+ capture_scale: float = 1.0,
) -> None:
runtime = None
stream_id = None
@@ -397,12 +416,14 @@ class WsBrowser(WsHandler):
browsers = listing.get("browsers") or []
active_id = self._active_browser_id(listing, browser_id)
if not active_id:
- await self._emit_empty_frame(
+ await self._emit_viewer_state(
sid,
context_id,
+ active_id,
browsers=browsers,
viewer_id=viewer_id,
- frame_source=VIEWER_TRANSPORT_SCREENCAST,
+ state=None,
+ viewer_transport=VIEWER_TRANSPORT_SCREENCAST,
)
await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)
continue
@@ -410,29 +431,26 @@ class WsBrowser(WsHandler):
screencast = await runtime.call(
"start_screencast",
active_id,
- quality=SCREENCAST_QUALITY,
+ quality=SCREENCAST_STREAM_QUALITY,
every_nth_frame=1,
+ capture_scale=capture_scale,
)
stream_id = screencast["stream_id"]
active_id = screencast["browser_id"]
state = screencast.get("state")
- await self.emit_to(
+ await self._emit_viewer_state(
sid,
- "browser_viewer_frame",
- {
- "context_id": context_id,
- "viewer_id": viewer_id,
- "browser_id": active_id,
- "browsers": browsers,
- "image": "",
- "mime": "",
- "state": state,
- "frame_source": "state",
- "viewer_transport": VIEWER_TRANSPORT_SCREENCAST,
- },
+ context_id,
+ active_id,
+ browsers=browsers,
+ viewer_id=viewer_id,
+ state=state,
+ viewer_transport=VIEWER_TRANSPORT_SCREENCAST,
)
last_state_refresh = 0.0
+ last_state_signature = self._state_signature(active_id, browsers)
+ frame_sequence = 0
while True:
now = time.monotonic()
if now - last_state_refresh >= FRAME_STATE_REFRESH_SECONDS:
@@ -442,6 +460,18 @@ class WsBrowser(WsHandler):
if str(active_id) not in browser_ids:
break
state = self._state_for_browser(browsers, active_id, state)
+ state_signature = self._state_signature(active_id, browsers)
+ if state_signature != last_state_signature:
+ await self._emit_viewer_state(
+ sid,
+ context_id,
+ active_id,
+ browsers=browsers,
+ viewer_id=viewer_id,
+ state=state,
+ viewer_transport=VIEWER_TRANSPORT_SCREENCAST,
+ )
+ last_state_signature = state_signature
last_state_refresh = now
try:
@@ -455,14 +485,19 @@ class WsBrowser(WsHandler):
except TimeoutError:
continue
- frame["context_id"] = context_id
- frame["viewer_id"] = viewer_id
- frame["browser_id"] = active_id
- frame["browsers"] = browsers
- frame["state"] = state
- frame["frame_source"] = VIEWER_TRANSPORT_SCREENCAST
- frame["viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST
- await self.emit_to(sid, "browser_viewer_frame", frame)
+ frame_sequence += 1
+ payload = self._frame_payload(
+ frame,
+ context_id=context_id,
+ viewer_id=viewer_id,
+ browser_id=active_id,
+ sequence=frame_sequence,
+ binary_frames=binary_frames,
+ )
+ if not slim_frames:
+ payload["browsers"] = browsers
+ payload["state"] = state
+ await self._emit_to_connected_viewer(sid, "browser_viewer_frame", payload)
except asyncio.CancelledError:
raise
except Exception:
@@ -515,20 +550,14 @@ class WsBrowser(WsHandler):
),
)
if signature != last_signature:
- await self.emit_to(
+ await self._emit_viewer_state(
sid,
- "browser_viewer_frame",
- {
- "context_id": context_id,
- "viewer_id": viewer_id,
- "browser_id": active_id,
- "browsers": browsers,
- "image": "",
- "mime": "",
- "state": state,
- "frame_source": VIEWER_TRANSPORT_SNAPSHOT,
- "viewer_transport": VIEWER_TRANSPORT_SNAPSHOT,
- },
+ context_id,
+ active_id,
+ browsers=browsers,
+ viewer_id=viewer_id,
+ state=state,
+ viewer_transport=VIEWER_TRANSPORT_SNAPSHOT,
)
last_signature = signature
await asyncio.sleep(SNAPSHOT_STATE_POLL_SECONDS)
@@ -567,6 +596,105 @@ class WsBrowser(WsHandler):
return browser
return current_state
+ @staticmethod
+ def _state_signature(
+ active_id: int | str | None,
+ browsers: list[dict[str, Any]],
+ ) -> tuple[str, tuple[tuple[str, str, str, str, bool], ...]]:
+ return (
+ str(active_id or ""),
+ tuple(
+ (
+ str(browser.get("context_id") or ""),
+ str(browser.get("id") or ""),
+ str(browser.get("currentUrl") or ""),
+ str(browser.get("title") or ""),
+ bool(browser.get("loading")),
+ )
+ for browser in browsers
+ ),
+ )
+
+ @staticmethod
+ def _frame_payload(
+ frame: dict[str, Any],
+ *,
+ context_id: str,
+ viewer_id: str,
+ browser_id: int | str,
+ sequence: int,
+ binary_frames: bool,
+ ) -> dict[str, Any]:
+ image = str(frame.get("image") or "")
+ payload: dict[str, Any] = {
+ "context_id": context_id,
+ "viewer_id": viewer_id,
+ "browser_id": browser_id,
+ "seq": sequence,
+ "mime": frame.get("mime") or "image/jpeg",
+ "frame_source": VIEWER_TRANSPORT_SCREENCAST,
+ "viewer_transport": VIEWER_TRANSPORT_SCREENCAST,
+ }
+ dimensions = WsBrowser._frame_dimensions(frame.get("metadata"))
+ if dimensions:
+ payload.update(dimensions)
+ if binary_frames:
+ try:
+ payload["image"] = base64.b64decode(image, validate=False)
+ payload["encoding"] = "binary"
+ except Exception:
+ payload["image"] = image
+ payload["encoding"] = "base64"
+ else:
+ payload["image"] = image
+ payload["encoding"] = "base64"
+ return payload
+
+ @staticmethod
+ def _frame_dimensions(metadata: Any) -> dict[str, int]:
+ if not isinstance(metadata, dict):
+ return {}
+ for width_key, height_key in (
+ ("expectedWidth", "expectedHeight"),
+ ("deviceWidth", "deviceHeight"),
+ ("jpegWidth", "jpegHeight"),
+ ):
+ try:
+ width = int(metadata.get(width_key) or 0)
+ height = int(metadata.get(height_key) or 0)
+ except (TypeError, ValueError):
+ continue
+ if width > 0 and height > 0:
+ return {"width": width, "height": height}
+ return {}
+
+ async def _emit_viewer_state(
+ self,
+ sid: str,
+ context_id: str,
+ browser_id: int | str | None,
+ *,
+ browsers: list[dict[str, Any]] | None = None,
+ viewer_id: str = "",
+ state: dict[str, Any] | None = None,
+ viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT,
+ ) -> None:
+ await self._emit_to_connected_viewer(
+ sid,
+ "browser_viewer_state",
+ {
+ "context_id": context_id,
+ "active_browser_context_id": context_id,
+ "viewer_id": viewer_id,
+ "browser_id": browser_id,
+ "active_browser_id": browser_id,
+ "browsers": browsers or [],
+ "state": state,
+ "all_browsers": False,
+ "viewer_transport": viewer_transport,
+ },
+ )
+
async def _emit_empty_frame(
self,
sid: str,
@@ -576,7 +704,7 @@ class WsBrowser(WsHandler):
viewer_id: str = "",
frame_source: str = "",
) -> None:
- await self.emit_to(
+ await self._emit_to_connected_viewer(
sid,
"browser_viewer_frame",
{
@@ -592,6 +720,20 @@ class WsBrowser(WsHandler):
},
)
+ async def _emit_to_connected_viewer(
+ self,
+ sid: str,
+ event: str,
+ data: dict[str, Any],
+ ) -> None:
+ manager = getattr(self, "_manager", None)
+ if manager is not None:
+ with manager.lock:
+ connected = (getattr(self, "namespace", "/ws"), sid) in manager.connections
+ if not connected:
+ raise asyncio.CancelledError()
+ await self.emit_to(sid, event, data)
+
@staticmethod
def _viewer_transport(data: dict[str, Any]) -> str:
raw = (
@@ -623,6 +765,20 @@ class WsBrowser(WsHandler):
"height": max(200, min(4096, height)),
}
+ @staticmethod
+ def _capture_scale_from_data(data: dict[str, Any]) -> float:
+ try:
+ scale = float(
+ data.get("device_pixel_ratio")
+ or data.get("devicePixelRatio")
+ or data.get("pixel_ratio")
+ or data.get("pixelRatio")
+ or 1
+ )
+ except (TypeError, ValueError):
+ return 1.0
+ return max(1.0, min(2.0, scale))
+
@staticmethod
def _context_id(data: dict[str, Any]) -> str:
return str(data.get("context_id") or data.get("context") or "").strip()
diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py
index e7455d81a..94524071d 100644
--- a/plugins/_browser/helpers/runtime.py
+++ b/plugins/_browser/helpers/runtime.py
@@ -329,10 +329,14 @@ class _BrowserScreencast:
quality: int,
every_nth_frame: int,
viewport: dict[str, int],
+ capture_scale: float = 1.0,
) -> None:
self.session.on("Page.screencastFrame", self._on_frame)
width = max(320, min(4096, int(viewport.get("width") or DEFAULT_VIEWPORT["width"])))
height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"])))
+ scale = max(1.0, min(2.0, float(capture_scale or 1.0)))
+ max_width = max(320, min(SCREENCAST_MAX_WIDTH, int(round(width * scale))))
+ max_height = max(200, min(SCREENCAST_MAX_HEIGHT, int(round(height * scale))))
self._expected_width = width
self._expected_height = height
with contextlib.suppress(Exception):
@@ -343,8 +347,8 @@ class _BrowserScreencast:
{
"format": "jpeg",
"quality": max(20, min(95, int(quality))),
- "maxWidth": SCREENCAST_MAX_WIDTH,
- "maxHeight": SCREENCAST_MAX_HEIGHT,
+ "maxWidth": max_width,
+ "maxHeight": max_height,
"everyNthFrame": max(1, int(every_nth_frame)),
},
)
@@ -1616,6 +1620,7 @@ class _BrowserRuntimeCore:
*,
quality: int = 78,
every_nth_frame: int = 1,
+ capture_scale: float = 1.0,
) -> dict[str, Any]:
await self.ensure_started()
resolved_id = self._resolve_browser_id(browser_id)
@@ -1634,6 +1639,7 @@ class _BrowserRuntimeCore:
quality=quality,
every_nth_frame=every_nth_frame,
viewport=page.viewport_size or DEFAULT_VIEWPORT,
+ capture_scale=capture_scale,
)
except Exception:
self.screencasts.pop(stream_id, None)
diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js
index 6e70a1891..a4b3d4525 100644
--- a/plugins/_browser/webui/browser-store.js
+++ b/plugins/_browser/webui/browser-store.js
@@ -34,6 +34,8 @@ const ANNOTATION_DOM_LIMIT = 1200;
const ANNOTATION_TRAY_MARGIN = 10;
const BROWSER_VISUAL_SHORTCUT_KEYS = new Set(["a", "c", "insert", "v", "x", "y", "z"]);
const LOCAL_EDITABLE_SELECTOR = "input, textarea, select, [contenteditable]";
+const BROWSER_BINARY_FRAMES_SUPPORTED = typeof Blob === "function"
+ && typeof globalThis.URL?.createObjectURL === "function";
function makeViewerToken() {
return globalThis.crypto?.randomUUID?.()
@@ -114,6 +116,31 @@ function loadFrameDimensions(src) {
});
}
+function frameImageSource(data = {}) {
+ const image = data?.image;
+ if (!image) return null;
+ const mime = data.mime || "image/jpeg";
+ const isArrayBuffer = typeof ArrayBuffer !== "undefined" && image instanceof ArrayBuffer;
+ const isView = typeof ArrayBuffer !== "undefined" && ArrayBuffer.isView?.(image);
+ const isBlob = typeof Blob !== "undefined" && image instanceof Blob;
+ if (data.encoding === "binary" || isArrayBuffer || isView || isBlob) {
+ if (!BROWSER_BINARY_FRAMES_SUPPORTED) return null;
+ const blob = isBlob ? image : new Blob([image], { type: mime });
+ const src = globalThis.URL.createObjectURL(blob);
+ return {
+ src,
+ objectUrl: src,
+ cleanup: () => globalThis.URL.revokeObjectURL(src),
+ };
+ }
+ if (typeof image !== "string") return null;
+ return {
+ src: `data:${mime};base64,${image}`,
+ objectUrl: "",
+ cleanup: null,
+ };
+}
+
const model = {
loading: true,
error: "",
@@ -146,6 +173,7 @@ const model = {
_lastFrameDimensions: null,
_pendingFrameSrc: "",
_pendingFrameOptions: null,
+ _frameObjectUrl: "",
_frameRenderHandle: null,
_frameRenderCancel: null,
_frameRenderSequence: 0,
@@ -442,7 +470,7 @@ const model = {
this.setActiveBrowserId(null);
this.address = "";
this.frameState = null;
- this.frameSrc = "";
+ this.clearFrameSrc();
if (this.contextId) {
await this.connectViewer();
}
@@ -833,7 +861,7 @@ const model = {
resetRenderedFrame() {
this.cancelFrameRender();
- this.frameSrc = "";
+ this.clearFrameSrc();
this._lastFrameDimensions = null;
this._lastFrameAt = 0;
},
@@ -915,6 +943,25 @@ const model = {
return this.viewerTransport === BROWSER_VIEWER_TRANSPORT_SCREENCAST;
},
+ supportsBinaryFrames() {
+ return BROWSER_BINARY_FRAMES_SUPPORTED;
+ },
+
+ captureDevicePixelRatio() {
+ const value = Number(globalThis.devicePixelRatio || 1);
+ if (!Number.isFinite(value) || value <= 1) return 1;
+ return Math.min(2, value);
+ },
+
+ frameDimensionsFromData(data = null) {
+ const width = Number(data?.width || 0);
+ const height = Number(data?.height || 0);
+ if (Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0) {
+ return { width, height };
+ }
+ return this.frameDimensionsFromMetadata(data?.metadata);
+ },
+
frameDimensionsFromMetadata(metadata = null) {
if (!metadata || typeof metadata !== "object") return null;
const width = Number(metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth || 0);
@@ -986,6 +1033,9 @@ const model = {
viewer_id: viewerToken,
create_browser: Boolean(options.createBrowser || options.create_browser),
viewer_transport: this.requestedViewerTransport(),
+ binary_frames: this.supportsBinaryFrames(),
+ slim_frames: true,
+ device_pixel_ratio: this.captureDevicePixelRatio(),
viewport_width: initialViewport?.width,
viewport_height: initialViewport?.height,
},
@@ -1014,9 +1064,9 @@ const model = {
data.active_browser_context_id || contextId,
);
this.applySnapshot(data.snapshot);
- this.connected = true;
- this.browserInstallExpected = false;
- },
+ this.connected = true;
+ this.browserInstallExpected = false;
+ },
async _bindSocketEvents() {
if (!this._frameOff) {
@@ -1028,7 +1078,9 @@ const model = {
}
const incomingContextId = this.normalizeContextId(data.context_id || this.contextId);
const incomingBrowserId = this.normalizeBrowserId(data.browser_id || data.state?.id);
- this.applyBrowserListing(data.browsers || [], incomingContextId, { replaceContext: true });
+ if (Array.isArray(data.browsers)) {
+ this.applyBrowserListing(data.browsers, incomingContextId, { replaceContext: true });
+ }
if (incomingBrowserId && !this.activeBrowserId) {
this.setActiveBrowserId(incomingBrowserId, incomingContextId);
}
@@ -1046,11 +1098,15 @@ const model = {
this.address = data.state.currentUrl;
}
if (data.image) {
+ const frameImage = frameImageSource(data);
+ if (!frameImage?.src) return;
const frameBrowserId = incomingBrowserId || this.activeBrowserId;
- this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`, {
+ this.queueFrameRender(frameImage.src, {
browserId: frameBrowserId,
contextId: incomingContextId,
- dimensions: this.frameDimensionsFromMetadata(data.metadata),
+ dimensions: this.frameDimensionsFromData(data),
+ objectUrl: frameImage.objectUrl,
+ cleanup: frameImage.cleanup,
onAccepted: () => {
if (
this.sameBrowserId(this.switchingBrowserId, frameBrowserId)
@@ -1063,13 +1119,13 @@ const model = {
});
} else if (!data.state) {
this.cancelFrameRender();
- this.frameSrc = "";
+ this.clearFrameSrc();
}
if (!data.image && !data.state) {
if (!this.activeBrowserId) {
this.setActiveBrowserId(null, "");
this.frameState = null;
- this.frameSrc = "";
+ this.clearFrameSrc();
}
}
this._lastFrameAt = Date.now();
@@ -1077,15 +1133,20 @@ const model = {
await websocket.on("browser_viewer_frame", frameHandler);
this._frameOff = () => websocket.off("browser_viewer_frame", frameHandler);
}
- if (!this._stateOff) {
- const stateHandler = ({ data }) => {
- if (data?.context_id !== this.contextId) return;
- if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
- if (data?.viewer_transport) {
- this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
- }
+ if (!this._stateOff) {
+ const stateHandler = ({ data }) => {
+ if (data?.context_id !== this.contextId) return;
+ if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
+ if (data?.viewer_transport) {
+ this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
+ }
const commandContextId = this.normalizeContextId(data.active_browser_context_id || data.context_id || this.contextId);
- this.applyBrowserListing(data.browsers || [], commandContextId, { replaceAll: Boolean(data.all_browsers) });
+ if (Array.isArray(data.browsers)) {
+ this.applyBrowserListing(data.browsers, commandContextId, {
+ replaceAll: Boolean(data.all_browsers),
+ replaceContext: !data.all_browsers,
+ });
+ }
const command = String(data.command || "").toLowerCase();
const commandBrowserId = this.normalizeBrowserId(data.browser_id);
const result = data.result || {};
@@ -1102,23 +1163,40 @@ const model = {
|| this.activeBrowserId
|| this.firstBrowserId(resultContextId)
);
- if (
- !this.activeBrowserId
- || command === "open"
- || command === "close"
- || this.sameBrowserTab(commandBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)
- ) {
- this.setActiveBrowserId(preferredBrowserId, resultContextId);
- }
- this.applyActiveFrameState(resultState || this.browserById(this.activeBrowserId, this.activeBrowserContextId));
- this.applySnapshot(data.snapshot);
- };
+ const stateBrowserId = this.normalizeBrowserId(data.active_browser_id || data.browser_id || data.state?.id);
+ if (
+ stateBrowserId
+ && (
+ !this.activeBrowserId
+ || this.sameBrowserTab(stateBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)
+ )
+ ) {
+ this.setActiveBrowserId(stateBrowserId, commandContextId);
+ }
+ if (
+ !this.activeBrowserId
+ || command === "open"
+ || command === "close"
+ || this.sameBrowserTab(commandBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)
+ ) {
+ this.setActiveBrowserId(preferredBrowserId, resultContextId);
+ }
+ this.applyActiveFrameState(
+ resultState
+ || data.state
+ || this.browserById(this.activeBrowserId, this.activeBrowserContextId)
+ );
+ this.applySnapshot(data.snapshot);
+ };
await websocket.on("browser_viewer_state", stateHandler);
this._stateOff = () => websocket.off("browser_viewer_state", stateHandler);
}
},
queueFrameRender(frameSrc, options = {}) {
+ if (this._pendingFrameSrc) {
+ this.releasePendingFrame();
+ }
this._pendingFrameSrc = frameSrc;
this._pendingFrameOptions = options || null;
if (this._frameRenderHandle) return;
@@ -1148,22 +1226,26 @@ const model = {
async renderDecodedFrame(frameSrc, options = {}, sequence = 0, surfaceSequence = this._surfaceOpenSequence) {
if (!frameSrc) {
if (sequence === this._frameRenderSequence) {
- this.frameSrc = "";
+ this.clearFrameSrc();
}
return;
}
const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc);
if (sequence !== this._frameRenderSequence || surfaceSequence !== this._surfaceOpenSequence) {
+ options?.cleanup?.();
return;
}
const viewport = this.currentViewportSize() || this._lastViewport;
if (!this.frameMatchesViewport(dimensions, viewport)) {
this.requestViewportSyncAfterRejectedFrame();
if (!this.shouldAcceptMismatchedFrame(dimensions)) {
+ options?.cleanup?.();
return;
}
}
+ this.releaseRenderedFrameUrl(frameSrc);
this.frameSrc = frameSrc;
+ this._frameObjectUrl = options?.objectUrl || "";
this._lastFrameDimensions = dimensions;
this._lastFrameAt = Date.now();
options?.onAccepted?.();
@@ -1262,9 +1344,26 @@ const model = {
}
this._frameRenderHandle = null;
this._frameRenderCancel = null;
+ this.releasePendingFrame();
+ this._frameRenderSequence += 1;
+ },
+
+ releasePendingFrame() {
+ this._pendingFrameOptions?.cleanup?.();
this._pendingFrameSrc = "";
this._pendingFrameOptions = null;
- this._frameRenderSequence += 1;
+ },
+
+ releaseRenderedFrameUrl(nextSrc = "") {
+ if (this._frameObjectUrl && this._frameObjectUrl !== nextSrc) {
+ globalThis.URL?.revokeObjectURL?.(this._frameObjectUrl);
+ this._frameObjectUrl = "";
+ }
+ },
+
+ clearFrameSrc() {
+ this.releaseRenderedFrameUrl("");
+ this.frameSrc = "";
},
beginCommand() {
@@ -1329,7 +1428,7 @@ const model = {
);
if (!this.activeBrowserId) {
this.frameState = null;
- this.frameSrc = "";
+ this.clearFrameSrc();
}
if (result.state?.currentUrl || result.currentUrl) {
this.address = result.state?.currentUrl || result.currentUrl;
@@ -1438,7 +1537,7 @@ const model = {
this.error = "";
this.switchingBrowserId = targetId;
this.cancelFrameRender();
- this.frameSrc = "";
+ this.clearFrameSrc();
this.frameState = browser || null;
if (!this.addressFocused && browser?.currentUrl) {
this.address = browser.currentUrl;
@@ -1643,35 +1742,35 @@ const model = {
}
},
- applySnapshot(snapshot = null) {
- if (!snapshot?.image) return;
- const snapshotId = this.normalizeBrowserId(snapshot.browser_id || snapshot.state?.id);
+ applySnapshot(snapshot = null) {
+ if (!snapshot?.image) return;
+ const snapshotId = this.normalizeBrowserId(snapshot.browser_id || snapshot.state?.id);
const snapshotContextId = this.normalizeContextId(snapshot.context_id || snapshot.state?.context_id || this.activeBrowserContextId);
- if (
+ if (
snapshotId
&& this.activeBrowserId
&& !this.sameBrowserTab(snapshotId, snapshotContextId, this.activeBrowserId, this.activeBrowserContextId)
) {
- return;
- }
- if (snapshot.state) {
- this.applyActiveFrameState(snapshot.state);
- }
- const frameBrowserId = snapshotId || this.activeBrowserId;
- this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {
- browserId: frameBrowserId,
+ return;
+ }
+ if (snapshot.state) {
+ this.applyActiveFrameState(snapshot.state);
+ }
+ const frameBrowserId = snapshotId || this.activeBrowserId;
+ this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {
+ browserId: frameBrowserId,
contextId: snapshotContextId,
- onAccepted: () => {
- if (
+ onAccepted: () => {
+ if (
this.sameBrowserId(this.switchingBrowserId, frameBrowserId)
&& this.normalizeContextId(this.activeBrowserContextId) === snapshotContextId
) {
- this.switchingBrowserId = null;
- }
- this._surfaceSwitching = false;
- },
- });
- },
+ this.switchingBrowserId = null;
+ }
+ this._surfaceSwitching = false;
+ },
+ });
+ },
isSwitchingBrowser() {
return Boolean(
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index eef2e141b..7503efeb3 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -1158,7 +1158,8 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "def _viewer_transport(data: dict[str, Any])" in ws_browser
assert "return VIEWER_TRANSPORT_SNAPSHOT" in ws_browser
assert "self._stream_state" in ws_browser
- assert "SCREENCAST_QUALITY = 92" in ws_browser
+ assert "SCREENCAST_STREAM_QUALITY = 80" in ws_browser
+ assert "SCREENSHOT_QUALITY = 92" in ws_browser
assert "initial_viewport = self._viewport_from_data(data)" in ws_browser
assert '"set_viewport"' in ws_browser
assert "start_screencast" in ws_browser
@@ -1169,7 +1170,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "stop_screencast" in ws_browser
assert "viewer_transport == VIEWER_TRANSPORT_SCREENCAST" in ws_browser
assert '"viewer_transport": viewer_transport' in ws_browser
- assert '"viewer_transport": VIEWER_TRANSPORT_SNAPSHOT' in ws_browser
+ assert "viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT" in ws_browser
assert '"Page.startScreencast"' in runtime
assert '"Page.screencastFrame"' in runtime
assert '"Page.screencastFrameAck"' in runtime
@@ -1180,6 +1181,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "await self._stop_screencasts_for_browser(resolved_id)" in runtime
assert "queueFrameRender" in browser_store
assert "requestAnimationFrame" in browser_store
+ assert "function frameImageSource(data = {})" in browser_store
assert 'const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";' in browser_store
assert 'const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";' in browser_store
assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST" in browser_store
@@ -1187,11 +1189,17 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "requestedViewerTransport()" in browser_store
assert "normalizeViewerTransport(value = \"\")" in browser_store
assert "usesScreencastTransport()" in browser_store
+ assert "supportsBinaryFrames()" in browser_store
+ assert "captureDevicePixelRatio()" in browser_store
+ assert "frameDimensionsFromData(data = null)" in browser_store
assert "frameDimensionsFromMetadata(metadata = null)" in browser_store
assert "metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth" in browser_store
- assert "dimensions: this.frameDimensionsFromMetadata(data.metadata)" in browser_store
+ assert "dimensions: this.frameDimensionsFromData(data)" in browser_store
assert "const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc)" in browser_store
assert "viewer_transport: this.requestedViewerTransport()" in browser_store
+ assert "binary_frames: this.supportsBinaryFrames()" in browser_store
+ assert "slim_frames: true" in browser_store
+ assert "device_pixel_ratio: this.captureDevicePixelRatio()" in browser_store
assert "viewport_width: initialViewport?.width" in browser_store
assert "viewport_height: initialViewport?.height" in browser_store
assert "restart_stream: restartStream && this.usesScreencastTransport()" in browser_store
@@ -1213,6 +1221,13 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "this.applySnapshot(data.snapshot);" in browser_store
assert "else if (!data.state)" in browser_store
assert '"snapshot": snapshot' in ws_browser
+ assert '"binary_frames": binary_frames' in ws_browser
+ assert '"slim_frames": slim_frames' in ws_browser
+ assert "capture_scale=capture_scale" in ws_browser
+ assert "base64.b64decode(image, validate=False)" in ws_browser
+ assert '"encoding"] = "binary"' in ws_browser
+ assert "def _frame_payload(" in ws_browser
+ assert "def _emit_viewer_state(" in ws_browser
assert 'const BROWSER_SNAPSHOT_META_KEY = "browser_snapshot";' in browser_tool_handler
assert "staticScreenshotUri(kvps)" in browser_tool_handler
assert "/components/modals/image-viewer/image-viewer-store.js" in browser_tool_handler
@@ -1223,9 +1238,9 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "delete displayKvps[BROWSER_SNAPSHOT_META_KEY];" in browser_tool_handler
assert "startBrowserScreenshotPreview(button, image, resolveBrowserPayload)" in browser_tool_handler
assert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browser
- assert '"frame_source": "state"' in ws_browser
- assert '"frame_source"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser
- assert '"viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser
+ assert '"browser_viewer_state"' in ws_browser
+ assert '"frame_source": VIEWER_TRANSPORT_SCREENCAST' in ws_browser
+ assert '"viewer_transport": VIEWER_TRANSPORT_SCREENCAST' in ws_browser
assert "fallback_screenshot" not in ws_browser
assert "canvas_wheel_screenshot" not in ws_browser
assert "surface_mode: this._mode" not in browser_store
@@ -1234,6 +1249,29 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "image-rendering: auto;" in main_html
+def test_browser_viewer_frame_payload_supports_binary_slim_frames():
+ payload = ws_browser_module.WsBrowser._frame_payload(
+ {
+ "image": SMALL_JPEG_10X10,
+ "mime": "image/jpeg",
+ "metadata": {"expectedWidth": 900, "expectedHeight": 600},
+ },
+ context_id="ctx",
+ viewer_id="viewer",
+ browser_id=1,
+ sequence=3,
+ binary_frames=True,
+ )
+
+ assert payload["encoding"] == "binary"
+ assert payload["image"] == __import__("base64").b64decode(SMALL_JPEG_10X10)
+ assert payload["width"] == 900
+ assert payload["height"] == 600
+ assert payload["seq"] == 3
+ assert "browsers" not in payload
+ assert "state" not in payload
+
+
def test_browser_navigation_errors_stay_inside_native_browser_page():
runtime = (
PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"
@@ -1504,7 +1542,12 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
mime="image/jpeg",
)
- await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 1118, "height": 662})
+ await screencast.start(
+ quality=92,
+ every_nth_frame=1,
+ viewport={"width": 1118, "height": 662},
+ capture_scale=2,
+ )
session.handlers["Page.screencastFrame"](
{"data": first_image, "metadata": {"deviceWidth": 10}, "sessionId": 1}
)
@@ -1563,6 +1606,10 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
for index, (method, _params) in enumerate(session.sent)
if method == "Page.startScreencast"
)
+ start_params = session.sent[start_index][1]
+ assert start_params["quality"] == 92
+ assert start_params["maxWidth"] == 2236
+ assert start_params["maxHeight"] == 1324
cdp_viewport_indices = [
index
for index, (method, _params) in enumerate(session.sent)
@@ -2287,7 +2334,7 @@ async def test_browser_viewer_subscribe_returns_initial_snapshot(monkeypatch):
assert result["active_browser_id"] == 1
assert result["snapshot"]["image"] == "jpeg-data"
- assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENCAST_QUALITY}) in calls
+ assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENSHOT_QUALITY}) in calls
await handler.on_disconnect("sid-snapshot")