Agent Zero AI framework
Slim browser viewer frame transport
Alessandro91b3bc9613c7
diff --git a/docs/AGENTS.md b/docs/AGENTS.mdindex 3f3002b01..c81fb55bd 100644--- a/docs/AGENTS.md+++ b/docs/AGENTS.md@@ -9,6 +9,7 @@- `README.md`, `quickstart.md`, `guides/`, and `setup/` cover user-facing setup and workflows.- `developer/` covers compact developer references and source handoffs.+- `plans/` covers implementation plans, migration notes, and staged technical roadmaps.- `res/` contains documentation images and other documentation assets.## Local Contractsdiff --git a/plugins/_browser/AGENTS.md b/plugins/_browser/AGENTS.mdindex 15a7ac85c..e3f2a137e 100644--- a/plugins/_browser/AGENTS.md+++ b/plugins/_browser/AGENTS.md@@ -19,6 +19,7 @@- Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.- Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation.- Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport.+- Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback.- Keep narrow WebUI Browser controls usable by grouping navigation with Annotate/settings above a full-width address bar.- Browser URL-intent handling must only claim web URL schemes and leave custom Agent Zero schemes to their owning surfaces.- Prefer DOM/CDP browser actions with refs, selectors, frame-chain refs, and screenshots over viewport coordinate input. Coordinates remain a visual fallback.diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.pyindex ec5e137b3..827e91df4 100644--- a/plugins/_browser/api/ws_browser.py+++ b/plugins/_browser/api/ws_browser.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import base64import contextlibimport timefrom typing import Any, ClassVar@@ -15,7 +16,8 @@ FRAME_READ_TIMEOUT_SECONDS = 0.5FRAME_RETRY_DELAY_SECONDS = 0.5FRAME_STATE_REFRESH_SECONDS = 0.75SNAPSHOT_STATE_POLL_SECONDS = 0.75-SCREENCAST_QUALITY = 92+SCREENCAST_STREAM_QUALITY = 80+SCREENSHOT_QUALITY = 92VIEWER_TRANSPORT_SCREENCAST = "screencast"VIEWER_TRANSPORT_SNAPSHOT = "snapshot"VIEWER_TRANSPORTS = {VIEWER_TRANSPORT_SCREENCAST, VIEWER_TRANSPORT_SNAPSHOT}@@ -97,10 +99,21 @@ class WsBrowser(WsHandler):existing.cancel()viewer_id = str(data.get("viewer_id") or "")viewer_transport = self._viewer_transport(data)+ binary_frames = self._bool(data.get("binary_frames", data.get("binaryFrames")))+ slim_frames = self._bool(data.get("slim_frames", data.get("slimFrames", binary_frames)))+ capture_scale = self._capture_scale_from_data(data)snapshot = Noneif runtime:if viewer_transport == VIEWER_TRANSPORT_SCREENCAST:- stream_task = self._stream_frames(sid, context_id, active_id, viewer_id)+ stream_task = self._stream_frames(+ sid,+ context_id,+ active_id,+ viewer_id,+ binary_frames=binary_frames,+ slim_frames=slim_frames,+ capture_scale=capture_scale,+ )else:stream_task = self._stream_state(sid, context_id, active_id, viewer_id)self._streams[stream_key] = asyncio.create_task(stream_task)@@ -115,6 +128,8 @@ class WsBrowser(WsHandler):"all_browsers": True,"viewer_id": viewer_id,"viewer_transport": viewer_transport,+ "binary_frames": binary_frames,+ "slim_frames": slim_frames,}def _unsubscribe(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult:@@ -157,9 +172,9 @@ class WsBrowser(WsHandler):snapshot = Noneif active_id:try:- quality = int(data.get("quality") or SCREENCAST_QUALITY)+ quality = int(data.get("quality") or SCREENSHOT_QUALITY)except (TypeError, ValueError):- quality = SCREENCAST_QUALITY+ quality = SCREENSHOT_QUALITYwith contextlib.suppress(Exception):snapshot = await runtime.call("screenshot",@@ -347,7 +362,7 @@ class WsBrowser(WsHandler):if not browser_id:return Nonewith contextlib.suppress(Exception):- return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)+ return await runtime.call("screenshot", browser_id, quality=SCREENSHOT_QUALITY)return Noneasync def _snapshot_for_browser(@@ -358,7 +373,7 @@ class WsBrowser(WsHandler):if not browser_id:return Nonewith contextlib.suppress(Exception):- return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)+ return await runtime.call("screenshot", browser_id, quality=SCREENSHOT_QUALITY)return Noneasync def _all_browser_tabs(self) -> list[dict[str, Any]]:@@ -377,6 +392,10 @@ class WsBrowser(WsHandler):context_id: str,browser_id: int | str | None,viewer_id: str = "",+ *,+ binary_frames: bool = False,+ slim_frames: bool = False,+ capture_scale: float = 1.0,) -> None:runtime = Nonestream_id = None@@ -397,12 +416,14 @@ class WsBrowser(WsHandler):browsers = listing.get("browsers") or []active_id = self._active_browser_id(listing, browser_id)if not active_id:- await self._emit_empty_frame(+ await self._emit_viewer_state(sid,context_id,+ active_id,browsers=browsers,viewer_id=viewer_id,- frame_source=VIEWER_TRANSPORT_SCREENCAST,+ state=None,+ viewer_transport=VIEWER_TRANSPORT_SCREENCAST,)await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)continue@@ -410,29 +431,26 @@ class WsBrowser(WsHandler):screencast = await runtime.call("start_screencast",active_id,- quality=SCREENCAST_QUALITY,+ quality=SCREENCAST_STREAM_QUALITY,every_nth_frame=1,+ capture_scale=capture_scale,)stream_id = screencast["stream_id"]active_id = screencast["browser_id"]state = screencast.get("state")- await self.emit_to(+ await self._emit_viewer_state(sid,- "browser_viewer_frame",- {- "context_id": context_id,- "viewer_id": viewer_id,- "browser_id": active_id,- "browsers": browsers,- "image": "",- "mime": "",- "state": state,- "frame_source": "state",- "viewer_transport": VIEWER_TRANSPORT_SCREENCAST,- },+ context_id,+ active_id,+ browsers=browsers,+ viewer_id=viewer_id,+ state=state,+ viewer_transport=VIEWER_TRANSPORT_SCREENCAST,)last_state_refresh = 0.0+ last_state_signature = self._state_signature(active_id, browsers)+ frame_sequence = 0while True:now = time.monotonic()if now - last_state_refresh >= FRAME_STATE_REFRESH_SECONDS:@@ -442,6 +460,18 @@ class WsBrowser(WsHandler):if str(active_id) not in browser_ids:breakstate = self._state_for_browser(browsers, active_id, state)+ state_signature = self._state_signature(active_id, browsers)+ if state_signature != last_state_signature:+ await self._emit_viewer_state(+ sid,+ context_id,+ active_id,+ browsers=browsers,+ viewer_id=viewer_id,+ state=state,+ viewer_transport=VIEWER_TRANSPORT_SCREENCAST,+ )+ last_state_signature = state_signaturelast_state_refresh = nowtry:@@ -455,14 +485,19 @@ class WsBrowser(WsHandler):except TimeoutError:continue- frame["context_id"] = context_id- frame["viewer_id"] = viewer_id- frame["browser_id"] = active_id- frame["browsers"] = browsers- frame["state"] = state- frame["frame_source"] = VIEWER_TRANSPORT_SCREENCAST- frame["viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST- await self.emit_to(sid, "browser_viewer_frame", frame)+ frame_sequence += 1+ payload = self._frame_payload(+ frame,+ context_id=context_id,+ viewer_id=viewer_id,+ browser_id=active_id,+ sequence=frame_sequence,+ binary_frames=binary_frames,+ )+ if not slim_frames:+ payload["browsers"] = browsers+ payload["state"] = state+ await self._emit_to_connected_viewer(sid, "browser_viewer_frame", payload)except asyncio.CancelledError:raiseexcept Exception:@@ -515,20 +550,14 @@ class WsBrowser(WsHandler):),)if signature != last_signature:- await self.emit_to(+ await self._emit_viewer_state(sid,- "browser_viewer_frame",- {- "context_id": context_id,- "viewer_id": viewer_id,- "browser_id": active_id,- "browsers": browsers,- "image": "",- "mime": "",- "state": state,- "frame_source": VIEWER_TRANSPORT_SNAPSHOT,- "viewer_transport": VIEWER_TRANSPORT_SNAPSHOT,- },+ context_id,+ active_id,+ browsers=browsers,+ viewer_id=viewer_id,+ state=state,+ viewer_transport=VIEWER_TRANSPORT_SNAPSHOT,)last_signature = signatureawait asyncio.sleep(SNAPSHOT_STATE_POLL_SECONDS)@@ -567,6 +596,105 @@ class WsBrowser(WsHandler):return browserreturn current_state+ @staticmethod+ def _state_signature(+ active_id: int | str | None,+ browsers: list[dict[str, Any]],+ ) -> tuple[str, tuple[tuple[str, str, str, str, bool], ...]]:+ return (+ str(active_id or ""),+ tuple(+ (+ str(browser.get("context_id") or ""),+ str(browser.get("id") or ""),+ str(browser.get("currentUrl") or ""),+ str(browser.get("title") or ""),+ bool(browser.get("loading")),+ )+ for browser in browsers+ ),+ )++ @staticmethod+ def _frame_payload(+ frame: dict[str, Any],+ *,+ context_id: str,+ viewer_id: str,+ browser_id: int | str,+ sequence: int,+ binary_frames: bool,+ ) -> dict[str, Any]:+ image = str(frame.get("image") or "")+ payload: dict[str, Any] = {+ "context_id": context_id,+ "viewer_id": viewer_id,+ "browser_id": browser_id,+ "seq": sequence,+ "mime": frame.get("mime") or "image/jpeg",+ "frame_source": VIEWER_TRANSPORT_SCREENCAST,+ "viewer_transport": VIEWER_TRANSPORT_SCREENCAST,+ }+ dimensions = WsBrowser._frame_dimensions(frame.get("metadata"))+ if dimensions:+ payload.update(dimensions)+ if binary_frames:+ try:+ payload["image"] = base64.b64decode(image, validate=False)+ payload["encoding"] = "binary"+ except Exception:+ payload["image"] = image+ payload["encoding"] = "base64"+ else:+ payload["image"] = image+ payload["encoding"] = "base64"+ return payload++ @staticmethod+ def _frame_dimensions(metadata: Any) -> dict[str, int]:+ if not isinstance(metadata, dict):+ return {}+ for width_key, height_key in (+ ("expectedWidth", "expectedHeight"),+ ("deviceWidth", "deviceHeight"),+ ("jpegWidth", "jpegHeight"),+ ):+ try:+ width = int(metadata.get(width_key) or 0)+ height = int(metadata.get(height_key) or 0)+ except (TypeError, ValueError):+ continue+ if width > 0 and height > 0:+ return {"width": width, "height": height}+ return {}++ async def _emit_viewer_state(+ self,+ sid: str,+ context_id: str,+ browser_id: int | str | None,+ *,+ browsers: list[dict[str, Any]] | None = None,+ viewer_id: str = "",+ state: dict[str, Any] | None = None,+ viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT,+ ) -> None:+ await self._emit_to_connected_viewer(+ sid,+ "browser_viewer_state",+ {+ "context_id": context_id,+ "active_browser_context_id": context_id,+ "viewer_id": viewer_id,+ "browser_id": browser_id,+ "active_browser_id": browser_id,+ "browsers": browsers or [],+ "state": state,+ "all_browsers": False,+ "viewer_transport": viewer_transport,+ },+ )+async def _emit_empty_frame(self,sid: str,@@ -576,7 +704,7 @@ class WsBrowser(WsHandler):viewer_id: str = "",frame_source: str = "",) -> None:- await self.emit_to(+ await self._emit_to_connected_viewer(sid,"browser_viewer_frame",{@@ -592,6 +720,20 @@ class WsBrowser(WsHandler):},)+ async def _emit_to_connected_viewer(+ self,+ sid: str,+ event: str,+ data: dict[str, Any],+ ) -> None:+ manager = getattr(self, "_manager", None)+ if manager is not None:+ with manager.lock:+ connected = (getattr(self, "namespace", "/ws"), sid) in manager.connections+ if not connected:+ raise asyncio.CancelledError()+ await self.emit_to(sid, event, data)+@staticmethoddef _viewer_transport(data: dict[str, Any]) -> str:raw = (@@ -623,6 +765,20 @@ class WsBrowser(WsHandler):"height": max(200, min(4096, height)),}+ @staticmethod+ def _capture_scale_from_data(data: dict[str, Any]) -> float:+ try:+ scale = float(+ data.get("device_pixel_ratio")+ or data.get("devicePixelRatio")+ or data.get("pixel_ratio")+ or data.get("pixelRatio")+ or 1+ )+ except (TypeError, ValueError):+ return 1.0+ return max(1.0, min(2.0, scale))+@staticmethoddef _context_id(data: dict[str, Any]) -> str:return str(data.get("context_id") or data.get("context") or "").strip()diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.pyindex e7455d81a..94524071d 100644--- a/plugins/_browser/helpers/runtime.py+++ b/plugins/_browser/helpers/runtime.py@@ -329,10 +329,14 @@ class _BrowserScreencast:quality: int,every_nth_frame: int,viewport: dict[str, int],+ capture_scale: float = 1.0,) -> None:self.session.on("Page.screencastFrame", self._on_frame)width = max(320, min(4096, int(viewport.get("width") or DEFAULT_VIEWPORT["width"])))height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"])))+ scale = max(1.0, min(2.0, float(capture_scale or 1.0)))+ max_width = max(320, min(SCREENCAST_MAX_WIDTH, int(round(width * scale))))+ max_height = max(200, min(SCREENCAST_MAX_HEIGHT, int(round(height * scale))))self._expected_width = widthself._expected_height = heightwith contextlib.suppress(Exception):@@ -343,8 +347,8 @@ class _BrowserScreencast:{"format": "jpeg","quality": max(20, min(95, int(quality))),- "maxWidth": SCREENCAST_MAX_WIDTH,- "maxHeight": SCREENCAST_MAX_HEIGHT,+ "maxWidth": max_width,+ "maxHeight": max_height,"everyNthFrame": max(1, int(every_nth_frame)),},)@@ -1616,6 +1620,7 @@ class _BrowserRuntimeCore:*,quality: int = 78,every_nth_frame: int = 1,+ capture_scale: float = 1.0,) -> dict[str, Any]:await self.ensure_started()resolved_id = self._resolve_browser_id(browser_id)@@ -1634,6 +1639,7 @@ class _BrowserRuntimeCore:quality=quality,every_nth_frame=every_nth_frame,viewport=page.viewport_size or DEFAULT_VIEWPORT,+ capture_scale=capture_scale,)except Exception:self.screencasts.pop(stream_id, None)diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.jsindex 6e70a1891..a4b3d4525 100644--- a/plugins/_browser/webui/browser-store.js+++ b/plugins/_browser/webui/browser-store.js@@ -34,6 +34,8 @@ const ANNOTATION_DOM_LIMIT = 1200;const ANNOTATION_TRAY_MARGIN = 10;const BROWSER_VISUAL_SHORTCUT_KEYS = new Set(["a", "c", "insert", "v", "x", "y", "z"]);const LOCAL_EDITABLE_SELECTOR = "input, textarea, select, [contenteditable]";+const BROWSER_BINARY_FRAMES_SUPPORTED = typeof Blob === "function"+ && typeof globalThis.URL?.createObjectURL === "function";function makeViewerToken() {return globalThis.crypto?.randomUUID?.()@@ -114,6 +116,31 @@ function loadFrameDimensions(src) {});}+function frameImageSource(data = {}) {+ const image = data?.image;+ if (!image) return null;+ const mime = data.mime || "image/jpeg";+ const isArrayBuffer = typeof ArrayBuffer !== "undefined" && image instanceof ArrayBuffer;+ const isView = typeof ArrayBuffer !== "undefined" && ArrayBuffer.isView?.(image);+ const isBlob = typeof Blob !== "undefined" && image instanceof Blob;+ if (data.encoding === "binary" || isArrayBuffer || isView || isBlob) {+ if (!BROWSER_BINARY_FRAMES_SUPPORTED) return null;+ const blob = isBlob ? image : new Blob([image], { type: mime });+ const src = globalThis.URL.createObjectURL(blob);+ return {+ src,+ objectUrl: src,+ cleanup: () => globalThis.URL.revokeObjectURL(src),+ };+ }+ if (typeof image !== "string") return null;+ return {+ src: `data:${mime};base64,${image}`,+ objectUrl: "",+ cleanup: null,+ };+}+const model = {loading: true,error: "",@@ -146,6 +173,7 @@ const model = {_lastFrameDimensions: null,_pendingFrameSrc: "",_pendingFrameOptions: null,+ _frameObjectUrl: "",_frameRenderHandle: null,_frameRenderCancel: null,_frameRenderSequence: 0,@@ -442,7 +470,7 @@ const model = {this.setActiveBrowserId(null);this.address = "";this.frameState = null;- this.frameSrc = "";+ this.clearFrameSrc();if (this.contextId) {await this.connectViewer();}@@ -833,7 +861,7 @@ const model = {resetRenderedFrame() {this.cancelFrameRender();- this.frameSrc = "";+ this.clearFrameSrc();this._lastFrameDimensions = null;this._lastFrameAt = 0;},@@ -915,6 +943,25 @@ const model = {return this.viewerTransport === BROWSER_VIEWER_TRANSPORT_SCREENCAST;},+ supportsBinaryFrames() {+ return BROWSER_BINARY_FRAMES_SUPPORTED;+ },++ captureDevicePixelRatio() {+ const value = Number(globalThis.devicePixelRatio || 1);+ if (!Number.isFinite(value) || value <= 1) return 1;+ return Math.min(2, value);+ },++ frameDimensionsFromData(data = null) {+ const width = Number(data?.width || 0);+ const height = Number(data?.height || 0);+ if (Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0) {+ return { width, height };+ }+ return this.frameDimensionsFromMetadata(data?.metadata);+ },+frameDimensionsFromMetadata(metadata = null) {if (!metadata || typeof metadata !== "object") return null;const width = Number(metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth || 0);@@ -986,6 +1033,9 @@ const model = {viewer_id: viewerToken,create_browser: Boolean(options.createBrowser || options.create_browser),viewer_transport: this.requestedViewerTransport(),+ binary_frames: this.supportsBinaryFrames(),+ slim_frames: true,+ device_pixel_ratio: this.captureDevicePixelRatio(),viewport_width: initialViewport?.width,viewport_height: initialViewport?.height,},@@ -1014,9 +1064,9 @@ const model = {data.active_browser_context_id || contextId,);this.applySnapshot(data.snapshot);- this.connected = true;- this.browserInstallExpected = false;- },+ this.connected = true;+ this.browserInstallExpected = false;+ },async _bindSocketEvents() {if (!this._frameOff) {@@ -1028,7 +1078,9 @@ const model = {}const incomingContextId = this.normalizeContextId(data.context_id || this.contextId);const incomingBrowserId = this.normalizeBrowserId(data.browser_id || data.state?.id);- this.applyBrowserListing(data.browsers || [], incomingContextId, { replaceContext: true });+ if (Array.isArray(data.browsers)) {+ this.applyBrowserListing(data.browsers, incomingContextId, { replaceContext: true });+ }if (incomingBrowserId && !this.activeBrowserId) {this.setActiveBrowserId(incomingBrowserId, incomingContextId);}@@ -1046,11 +1098,15 @@ const model = {this.address = data.state.currentUrl;}if (data.image) {+ const frameImage = frameImageSource(data);+ if (!frameImage?.src) return;const frameBrowserId = incomingBrowserId || this.activeBrowserId;- this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`, {+ this.queueFrameRender(frameImage.src, {browserId: frameBrowserId,contextId: incomingContextId,- dimensions: this.frameDimensionsFromMetadata(data.metadata),+ dimensions: this.frameDimensionsFromData(data),+ objectUrl: frameImage.objectUrl,+ cleanup: frameImage.cleanup,onAccepted: () => {if (this.sameBrowserId(this.switchingBrowserId, frameBrowserId)@@ -1063,13 +1119,13 @@ const model = {});} else if (!data.state) {this.cancelFrameRender();- this.frameSrc = "";+ this.clearFrameSrc();}if (!data.image && !data.state) {if (!this.activeBrowserId) {this.setActiveBrowserId(null, "");this.frameState = null;- this.frameSrc = "";+ this.clearFrameSrc();}}this._lastFrameAt = Date.now();@@ -1077,15 +1133,20 @@ const model = {await websocket.on("browser_viewer_frame", frameHandler);this._frameOff = () => websocket.off("browser_viewer_frame", frameHandler);}- if (!this._stateOff) {- const stateHandler = ({ data }) => {- if (data?.context_id !== this.contextId) return;- if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;- if (data?.viewer_transport) {- this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);- }+ if (!this._stateOff) {+ const stateHandler = ({ data }) => {+ if (data?.context_id !== this.contextId) return;+ if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;+ if (data?.viewer_transport) {+ this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);+ }const commandContextId = this.normalizeContextId(data.active_browser_context_id || data.context_id || this.contextId);- this.applyBrowserListing(data.browsers || [], commandContextId, { replaceAll: Boolean(data.all_browsers) });+ if (Array.isArray(data.browsers)) {+ this.applyBrowserListing(data.browsers, commandContextId, {+ replaceAll: Boolean(data.all_browsers),+ replaceContext: !data.all_browsers,+ });+ }const command = String(data.command || "").toLowerCase();const commandBrowserId = this.normalizeBrowserId(data.browser_id);const result = data.result || {};@@ -1102,23 +1163,40 @@ const model = {|| this.activeBrowserId|| this.firstBrowserId(resultContextId));- if (- !this.activeBrowserId- || command === "open"- || command === "close"- || this.sameBrowserTab(commandBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)- ) {- this.setActiveBrowserId(preferredBrowserId, resultContextId);- }- this.applyActiveFrameState(resultState || this.browserById(this.activeBrowserId, this.activeBrowserContextId));- this.applySnapshot(data.snapshot);- };+ const stateBrowserId = this.normalizeBrowserId(data.active_browser_id || data.browser_id || data.state?.id);+ if (+ stateBrowserId+ && (+ !this.activeBrowserId+ || this.sameBrowserTab(stateBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)+ )+ ) {+ this.setActiveBrowserId(stateBrowserId, commandContextId);+ }+ if (+ !this.activeBrowserId+ || command === "open"+ || command === "close"+ || this.sameBrowserTab(commandBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)+ ) {+ this.setActiveBrowserId(preferredBrowserId, resultContextId);+ }+ this.applyActiveFrameState(+ resultState+ || data.state+ || this.browserById(this.activeBrowserId, this.activeBrowserContextId)+ );+ this.applySnapshot(data.snapshot);+ };await websocket.on("browser_viewer_state", stateHandler);this._stateOff = () => websocket.off("browser_viewer_state", stateHandler);}},queueFrameRender(frameSrc, options = {}) {+ if (this._pendingFrameSrc) {+ this.releasePendingFrame();+ }this._pendingFrameSrc = frameSrc;this._pendingFrameOptions = options || null;if (this._frameRenderHandle) return;@@ -1148,22 +1226,26 @@ const model = {async renderDecodedFrame(frameSrc, options = {}, sequence = 0, surfaceSequence = this._surfaceOpenSequence) {if (!frameSrc) {if (sequence === this._frameRenderSequence) {- this.frameSrc = "";+ this.clearFrameSrc();}return;}const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc);if (sequence !== this._frameRenderSequence || surfaceSequence !== this._surfaceOpenSequence) {+ options?.cleanup?.();return;}const viewport = this.currentViewportSize() || this._lastViewport;if (!this.frameMatchesViewport(dimensions, viewport)) {this.requestViewportSyncAfterRejectedFrame();if (!this.shouldAcceptMismatchedFrame(dimensions)) {+ options?.cleanup?.();return;}}+ this.releaseRenderedFrameUrl(frameSrc);this.frameSrc = frameSrc;+ this._frameObjectUrl = options?.objectUrl || "";this._lastFrameDimensions = dimensions;this._lastFrameAt = Date.now();options?.onAccepted?.();@@ -1262,9 +1344,26 @@ const model = {}this._frameRenderHandle = null;this._frameRenderCancel = null;+ this.releasePendingFrame();+ this._frameRenderSequence += 1;+ },++ releasePendingFrame() {+ this._pendingFrameOptions?.cleanup?.();this._pendingFrameSrc = "";this._pendingFrameOptions = null;- this._frameRenderSequence += 1;+ },++ releaseRenderedFrameUrl(nextSrc = "") {+ if (this._frameObjectUrl && this._frameObjectUrl !== nextSrc) {+ globalThis.URL?.revokeObjectURL?.(this._frameObjectUrl);+ this._frameObjectUrl = "";+ }+ },++ clearFrameSrc() {+ this.releaseRenderedFrameUrl("");+ this.frameSrc = "";},beginCommand() {@@ -1329,7 +1428,7 @@ const model = {);if (!this.activeBrowserId) {this.frameState = null;- this.frameSrc = "";+ this.clearFrameSrc();}if (result.state?.currentUrl || result.currentUrl) {this.address = result.state?.currentUrl || result.currentUrl;@@ -1438,7 +1537,7 @@ const model = {this.error = "";this.switchingBrowserId = targetId;this.cancelFrameRender();- this.frameSrc = "";+ this.clearFrameSrc();this.frameState = browser || null;if (!this.addressFocused && browser?.currentUrl) {this.address = browser.currentUrl;@@ -1643,35 +1742,35 @@ const model = {}},- applySnapshot(snapshot = null) {- if (!snapshot?.image) return;- const snapshotId = this.normalizeBrowserId(snapshot.browser_id || snapshot.state?.id);+ applySnapshot(snapshot = null) {+ if (!snapshot?.image) return;+ const snapshotId = this.normalizeBrowserId(snapshot.browser_id || snapshot.state?.id);const snapshotContextId = this.normalizeContextId(snapshot.context_id || snapshot.state?.context_id || this.activeBrowserContextId);- if (+ if (snapshotId&& this.activeBrowserId&& !this.sameBrowserTab(snapshotId, snapshotContextId, this.activeBrowserId, this.activeBrowserContextId)) {- return;- }- if (snapshot.state) {- this.applyActiveFrameState(snapshot.state);- }- const frameBrowserId = snapshotId || this.activeBrowserId;- this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {- browserId: frameBrowserId,+ return;+ }+ if (snapshot.state) {+ this.applyActiveFrameState(snapshot.state);+ }+ const frameBrowserId = snapshotId || this.activeBrowserId;+ this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {+ browserId: frameBrowserId,contextId: snapshotContextId,- onAccepted: () => {- if (+ onAccepted: () => {+ if (this.sameBrowserId(this.switchingBrowserId, frameBrowserId)&& this.normalizeContextId(this.activeBrowserContextId) === snapshotContextId) {- this.switchingBrowserId = null;- }- this._surfaceSwitching = false;- },- });- },+ this.switchingBrowserId = null;+ }+ this._surfaceSwitching = false;+ },+ });+ },isSwitchingBrowser() {return Boolean(diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.pyindex eef2e141b..7503efeb3 100644--- a/tests/test_browser_agent_regressions.py+++ b/tests/test_browser_agent_regressions.py@@ -1158,7 +1158,8 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():assert "def _viewer_transport(data: dict[str, Any])" in ws_browserassert "return VIEWER_TRANSPORT_SNAPSHOT" in ws_browserassert "self._stream_state" in ws_browser- assert "SCREENCAST_QUALITY = 92" in ws_browser+ assert "SCREENCAST_STREAM_QUALITY = 80" in ws_browser+ assert "SCREENSHOT_QUALITY = 92" in ws_browserassert "initial_viewport = self._viewport_from_data(data)" in ws_browserassert '"set_viewport"' in ws_browserassert "start_screencast" in ws_browser@@ -1169,7 +1170,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():assert "stop_screencast" in ws_browserassert "viewer_transport == VIEWER_TRANSPORT_SCREENCAST" in ws_browserassert '"viewer_transport": viewer_transport' in ws_browser- assert '"viewer_transport": VIEWER_TRANSPORT_SNAPSHOT' in ws_browser+ assert "viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT" in ws_browserassert '"Page.startScreencast"' in runtimeassert '"Page.screencastFrame"' in runtimeassert '"Page.screencastFrameAck"' in runtime@@ -1180,6 +1181,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():assert "await self._stop_screencasts_for_browser(resolved_id)" in runtimeassert "queueFrameRender" in browser_storeassert "requestAnimationFrame" in browser_store+ assert "function frameImageSource(data = {})" in browser_storeassert 'const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";' in browser_storeassert 'const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";' in browser_storeassert "viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST" in browser_store@@ -1187,11 +1189,17 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():assert "requestedViewerTransport()" in browser_storeassert "normalizeViewerTransport(value = \"\")" in browser_storeassert "usesScreencastTransport()" in browser_store+ assert "supportsBinaryFrames()" in browser_store+ assert "captureDevicePixelRatio()" in browser_store+ assert "frameDimensionsFromData(data = null)" in browser_storeassert "frameDimensionsFromMetadata(metadata = null)" in browser_storeassert "metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth" in browser_store- assert "dimensions: this.frameDimensionsFromMetadata(data.metadata)" in browser_store+ assert "dimensions: this.frameDimensionsFromData(data)" in browser_storeassert "const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc)" in browser_storeassert "viewer_transport: this.requestedViewerTransport()" in browser_store+ assert "binary_frames: this.supportsBinaryFrames()" in browser_store+ assert "slim_frames: true" in browser_store+ assert "device_pixel_ratio: this.captureDevicePixelRatio()" in browser_storeassert "viewport_width: initialViewport?.width" in browser_storeassert "viewport_height: initialViewport?.height" in browser_storeassert "restart_stream: restartStream && this.usesScreencastTransport()" in browser_store@@ -1213,6 +1221,13 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():assert "this.applySnapshot(data.snapshot);" in browser_storeassert "else if (!data.state)" in browser_storeassert '"snapshot": snapshot' in ws_browser+ assert '"binary_frames": binary_frames' in ws_browser+ assert '"slim_frames": slim_frames' in ws_browser+ assert "capture_scale=capture_scale" in ws_browser+ assert "base64.b64decode(image, validate=False)" in ws_browser+ assert '"encoding"] = "binary"' in ws_browser+ assert "def _frame_payload(" in ws_browser+ assert "def _emit_viewer_state(" in ws_browserassert 'const BROWSER_SNAPSHOT_META_KEY = "browser_snapshot";' in browser_tool_handlerassert "staticScreenshotUri(kvps)" in browser_tool_handlerassert "/components/modals/image-viewer/image-viewer-store.js" in browser_tool_handler@@ -1223,9 +1238,9 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():assert "delete displayKvps[BROWSER_SNAPSHOT_META_KEY];" in browser_tool_handlerassert "startBrowserScreenshotPreview(button, image, resolveBrowserPayload)" in browser_tool_handlerassert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browser- assert '"frame_source": "state"' in ws_browser- assert '"frame_source"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser- assert '"viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser+ assert '"browser_viewer_state"' in ws_browser+ assert '"frame_source": VIEWER_TRANSPORT_SCREENCAST' in ws_browser+ assert '"viewer_transport": VIEWER_TRANSPORT_SCREENCAST' in ws_browserassert "fallback_screenshot" not in ws_browserassert "canvas_wheel_screenshot" not in ws_browserassert "surface_mode: this._mode" not in browser_store@@ -1234,6 +1249,29 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():assert "image-rendering: auto;" in main_html+def test_browser_viewer_frame_payload_supports_binary_slim_frames():+ payload = ws_browser_module.WsBrowser._frame_payload(+ {+ "image": SMALL_JPEG_10X10,+ "mime": "image/jpeg",+ "metadata": {"expectedWidth": 900, "expectedHeight": 600},+ },+ context_id="ctx",+ viewer_id="viewer",+ browser_id=1,+ sequence=3,+ binary_frames=True,+ )++ assert payload["encoding"] == "binary"+ assert payload["image"] == __import__("base64").b64decode(SMALL_JPEG_10X10)+ assert payload["width"] == 900+ assert payload["height"] == 600+ assert payload["seq"] == 3+ assert "browsers" not in payload+ assert "state" not in payload++def test_browser_navigation_errors_stay_inside_native_browser_page():runtime = (PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"@@ -1504,7 +1542,12 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():mime="image/jpeg",)- await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 1118, "height": 662})+ await screencast.start(+ quality=92,+ every_nth_frame=1,+ viewport={"width": 1118, "height": 662},+ capture_scale=2,+ )session.handlers["Page.screencastFrame"]({"data": first_image, "metadata": {"deviceWidth": 10}, "sessionId": 1})@@ -1563,6 +1606,10 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():for index, (method, _params) in enumerate(session.sent)if method == "Page.startScreencast")+ start_params = session.sent[start_index][1]+ assert start_params["quality"] == 92+ assert start_params["maxWidth"] == 2236+ assert start_params["maxHeight"] == 1324cdp_viewport_indices = [indexfor index, (method, _params) in enumerate(session.sent)@@ -2287,7 +2334,7 @@ async def test_browser_viewer_subscribe_returns_initial_snapshot(monkeypatch):assert result["active_browser_id"] == 1assert result["snapshot"]["image"] == "jpeg-data"- assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENCAST_QUALITY}) in calls+ assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENSHOT_QUALITY}) in callsawait handler.on_disconnect("sid-snapshot")