Skip to content

Agent Zero AI framework

Smooth browser viewer rendering and frame push

Alessandro69fb0fca2aa3
diff --git a/plugins/_browser/AGENTS.md b/plugins/_browser/AGENTS.md
index e80bbe016..6a434bbd5 100644
--- a/plugins/_browser/AGENTS.md
+++ b/plugins/_browser/AGENTS.md
@@ -19,6 +19,8 @@
- Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.
- Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation.
- Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport.
+- Paint live screencast frames through the Browser panel canvas/ImageBitmap path when available; keep the `<img>`/data URL path for snapshots and fallback rendering.
+- Push internal screencast frames from the runtime to the WebSocket consumer after subscription; keep `read/pop_screencast_frame` as fallback/tooling APIs, not the WebUI hot path.
- Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback. Do not let the WebUI advertise binary frames unless its Socket.IO client reconstructs attachments as real `Blob`, `ArrayBuffer`, or typed-array values.
- Keep narrow WebUI Browser controls usable by grouping navigation with Annotate/settings above a full-width address bar.
- Browser URL-intent handling must only claim web URL schemes and leave custom Agent Zero schemes to their owning surfaces.
@@ -36,6 +38,7 @@
## Verification
- Smoke-test browser launch, navigation, DOM capture, and WebUI viewer after runtime changes.
+- For viewer render-path changes, verify the live Browser panel paints a screencast frame on canvas with `frameSrc` empty and snapshots still falling back to the image path.
- Run browser prompt/skill regression tests after changing browser prompt or Browser plugin skills.
## Child DOX Index
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.py
index 827e91df4..566ec10b8 100644
--- a/plugins/_browser/api/ws_browser.py
+++ b/plugins/_browser/api/ws_browser.py
@@ -451,7 +451,37 @@ class WsBrowser(WsHandler):
last_state_refresh = 0.0
last_state_signature = self._state_signature(active_id, browsers)
frame_sequence = 0
+ server_loop = asyncio.get_running_loop()
+ stop_event = asyncio.Event()
+
+ async def emit_frame(frame: dict[str, Any]) -> None:
+ nonlocal frame_sequence
+ try:
+ frame_sequence += 1
+ payload = self._frame_payload(
+ frame,
+ context_id=context_id,
+ viewer_id=viewer_id,
+ browser_id=active_id,
+ sequence=frame_sequence,
+ binary_frames=binary_frames,
+ )
+ if not slim_frames:
+ payload["browsers"] = browsers
+ payload["state"] = state
+ await self._emit_to_connected_viewer(sid, "browser_viewer_frame", payload)
+ except BaseException:
+ stop_event.set()
+ raise
+
+ def frame_consumer(frame: dict[str, Any]):
+ return asyncio.run_coroutine_threadsafe(emit_frame(frame), server_loop)
+
+ await runtime.call("attach_screencast_consumer", stream_id, frame_consumer)
+
while True:
+ if stop_event.is_set():
+ break
now = time.monotonic()
if now - last_state_refresh >= FRAME_STATE_REFRESH_SECONDS:
listing = await runtime.call("list")
@@ -475,29 +505,10 @@ class WsBrowser(WsHandler):
last_state_refresh = now
try:
- frame = await runtime.call(
- "read_screencast_frame",
- stream_id,
- timeout=FRAME_READ_TIMEOUT_SECONDS,
- )
- except KeyError:
+ await asyncio.wait_for(stop_event.wait(), timeout=FRAME_READ_TIMEOUT_SECONDS)
break
except TimeoutError:
continue
-
- frame_sequence += 1
- payload = self._frame_payload(
- frame,
- context_id=context_id,
- viewer_id=viewer_id,
- browser_id=active_id,
- sequence=frame_sequence,
- binary_frames=binary_frames,
- )
- if not slim_frames:
- payload["browsers"] = browsers
- payload["state"] = state
- await self._emit_to_connected_viewer(sid, "browser_viewer_frame", payload)
except asyncio.CancelledError:
raise
except Exception:
diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py
index 94524071d..6031a7f5f 100644
--- a/plugins/_browser/helpers/runtime.py
+++ b/plugins/_browser/helpers/runtime.py
@@ -317,6 +317,7 @@ class _BrowserScreencast:
self.browser_id = browser_id
self.session = session
self.mime = mime
+ self.frame_consumer: Any | None = None
self.queue = asyncio.Queue(maxsize=1)
self.stopped = False
self._ack_tasks: set[asyncio.Task] = set()
@@ -398,6 +399,12 @@ class _BrowserScreencast:
raise RuntimeError("Browser screencast stopped.")
return frame
+ async def attach_consumer(self, frame_consumer: Any) -> None:
+ self.frame_consumer = frame_consumer
+ frame = await self.pop_frame()
+ if frame:
+ await self._deliver_frame(frame)
+
async def stop(self) -> None:
if self.stopped:
return
@@ -423,6 +430,7 @@ class _BrowserScreencast:
task.add_done_callback(self._ack_tasks.discard)
async def _handle_frame(self, params: dict[str, Any]) -> None:
+ stop_after_ack = False
try:
data = params.get("data") or ""
if data:
@@ -432,7 +440,7 @@ class _BrowserScreencast:
metadata["jpegWidth"], metadata["jpegHeight"] = size
metadata["expectedWidth"] = self._expected_width
metadata["expectedHeight"] = self._expected_height
- self._queue_latest(
+ await self._deliver_frame(
{
"browser_id": self.browser_id,
"mime": self.mime,
@@ -440,6 +448,13 @@ class _BrowserScreencast:
"metadata": metadata,
}
)
+ except asyncio.CancelledError:
+ stop_after_ack = True
+ except Exception:
+ if self.frame_consumer:
+ stop_after_ack = True
+ else:
+ raise
finally:
session_id = params.get("sessionId")
if session_id is not None and not self.stopped:
@@ -448,6 +463,16 @@ class _BrowserScreencast:
"Page.screencastFrameAck",
{"sessionId": int(session_id)},
)
+ if stop_after_ack:
+ self.stopped = True
+
+ async def _deliver_frame(self, frame: dict[str, Any]) -> None:
+ if not self.frame_consumer:
+ self._queue_latest(frame)
+ return
+ future = self.frame_consumer(frame)
+ if future is not None:
+ await asyncio.wrap_future(future)
def _queue_latest(self, frame: dict[str, Any]) -> None:
self._drop_queued_frames()
@@ -1669,6 +1694,12 @@ class _BrowserRuntimeCore:
raise KeyError("Browser screencast is not active.")
return await screencast.pop_frame()
+ async def attach_screencast_consumer(self, stream_id: str, frame_consumer: Any) -> None:
+ screencast = self.screencasts.get(str(stream_id or ""))
+ if not screencast:
+ raise KeyError("Browser screencast is not active.")
+ await screencast.attach_consumer(frame_consumer)
+
async def stop_screencast(self, stream_id: str) -> None:
screencast = self.screencasts.pop(str(stream_id or ""), None)
if screencast:
diff --git a/plugins/_browser/webui/browser-panel.html b/plugins/_browser/webui/browser-panel.html
index 5692dd063..1abcde86f 100644
--- a/plugins/_browser/webui/browser-panel.html
+++ b/plugins/_browser/webui/browser-panel.html
@@ -187,12 +187,17 @@
<div class="browser-stage" tabindex="0" @click="$el.focus()"
:class="{ 'is-annotating': $store.browserPage.annotating }"
@wheel.prevent="$store.browserPage.handleStageWheel($event)">
+ <canvas class="browser-frame browser-frame-canvas"
+ x-show="$store.browserPage.frameCanvasReady"
+ x-init="$store.browserPage.attachFrameCanvas($el)"
+ @click="$store.browserPage.sendMouse('click', $event)"
+ @mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)"></canvas>
<template x-if="$store.browserPage.frameSrc">
- <img class="browser-frame" :src="$store.browserPage.frameSrc"
+ <img class="browser-frame browser-frame-image" :src="$store.browserPage.frameSrc"
@click="$store.browserPage.sendMouse('click', $event)"
@mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)" draggable="false" />
</template>
- <template x-if="$store.browserPage.annotating && $store.browserPage.frameSrc">
+ <template x-if="$store.browserPage.annotating && $store.browserPage.hasFrame()">
<div class="browser-annotation-layer"
:class="{ 'is-busy': $store.browserPage.annotationBusy }"
@pointerdown.stop.prevent="$store.browserPage.startAnnotationSelection($event)"
@@ -267,7 +272,7 @@
@click="$store.browserPage.sendAnnotationsToChat()">Send now</button>
</div>
</div>
- <template x-if="!$store.browserPage.frameSrc && !$store.browserPage.isBusy()">
+ <template x-if="!$store.browserPage.hasFrame() && !$store.browserPage.isBusy()">
<div class="browser-empty">
<span class="material-symbols-outlined">captive_portal</span>
<button class="btn btn-field" @click="$store.browserPage.command('open')">Open
diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js
index c2fdcd84b..1b8bb9e26 100644
--- a/plugins/_browser/webui/browser-store.js
+++ b/plugins/_browser/webui/browser-store.js
@@ -37,6 +37,7 @@ const LOCAL_EDITABLE_SELECTOR = "input, textarea, select, [contenteditable]";
const BROWSER_BINARY_FRAME_REQUESTS_ENABLED = false;
const BROWSER_BINARY_PAYLOADS_SUPPORTED = typeof Blob === "function"
&& typeof globalThis.URL?.createObjectURL === "function";
+const BROWSER_CANVAS_FRAMES_SUPPORTED = typeof globalThis.createImageBitmap === "function";
function makeViewerToken() {
return globalThis.crypto?.randomUUID?.()
@@ -130,6 +131,7 @@ function frameImageSource(data = {}) {
const src = globalThis.URL.createObjectURL(blob);
return {
src,
+ blob,
objectUrl: src,
cleanup: () => globalThis.URL.revokeObjectURL(src),
};
@@ -143,6 +145,16 @@ function frameImageSource(data = {}) {
};
}
+async function loadFrameBitmap(src, options = {}) {
+ if (!BROWSER_CANVAS_FRAMES_SUPPORTED || !src) return null;
+ try {
+ const blob = options.blob || await fetch(src).then((response) => response.blob());
+ return await globalThis.createImageBitmap(blob);
+ } catch {
+ return null;
+ }
+}
+
const model = {
loading: true,
error: "",
@@ -153,6 +165,7 @@ const model = {
activeBrowserContextId: "",
address: "",
frameSrc: "",
+ frameCanvasReady: false,
frameState: null,
viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST,
liveScreencastEnabled: true,
@@ -179,6 +192,7 @@ const model = {
_frameRenderHandle: null,
_frameRenderCancel: null,
_frameRenderSequence: 0,
+ _frameCanvas: null,
_floatingCleanup: null,
_stageElement: null,
_stageResizeObserver: null,
@@ -473,6 +487,7 @@ const model = {
this.address = "";
this.frameState = null;
this.clearFrameSrc();
+ this.clearFrameCanvas();
if (this.contextId) {
await this.connectViewer();
}
@@ -784,11 +799,13 @@ const model = {
},
releaseSurfaceBindings() {
+ this.freezeCanvasFrameToImage();
this._floatingCleanup?.();
this._floatingCleanup = null;
this._stageResizeObserver?.disconnect?.();
this._stageResizeObserver = null;
this._stageElement = null;
+ this._frameCanvas = null;
},
isCanvasSurfaceVisible(element = null) {
@@ -843,7 +860,7 @@ const model = {
this._surfaceMounted = true;
this._surfaceOpenedAt = Date.now();
this._lastViewportKey = "";
- if (this.frameSrc && !targetChanged) {
+ if (this.hasFrame() && !targetChanged) {
this._surfaceSwitching = false;
this.switchingBrowserId = null;
return;
@@ -864,12 +881,13 @@ const model = {
resetRenderedFrame() {
this.cancelFrameRender();
this.clearFrameSrc();
+ this.clearFrameCanvas();
this._lastFrameDimensions = null;
this._lastFrameAt = 0;
},
resetRenderedFrameIfViewportChanged(viewport = null, requestedBrowserId = null, requestedContextId = "") {
- if (!viewport || !this.frameSrc || !this._lastViewport) return;
+ if (!viewport || !this.hasFrame() || !this._lastViewport) return;
const targetBrowserId = requestedBrowserId || this.activeBrowserId || this.firstBrowserId();
const targetContextId = this.normalizeContextId(requestedContextId || this.contextIdForBrowserId(targetBrowserId) || this.activeBrowserContextId);
if (!this.sameBrowserTab(this._lastViewport.browserId, this._lastViewport.contextId, targetBrowserId, targetContextId)) return;
@@ -1107,7 +1125,9 @@ const model = {
browserId: frameBrowserId,
contextId: incomingContextId,
dimensions: this.frameDimensionsFromData(data),
+ blob: frameImage.blob,
objectUrl: frameImage.objectUrl,
+ useCanvas: true,
cleanup: frameImage.cleanup,
onAccepted: () => {
if (
@@ -1122,12 +1142,14 @@ const model = {
} else if (!data.state) {
this.cancelFrameRender();
this.clearFrameSrc();
+ this.clearFrameCanvas();
}
if (!data.image && !data.state) {
if (!this.activeBrowserId) {
this.setActiveBrowserId(null, "");
this.frameState = null;
this.clearFrameSrc();
+ this.clearFrameCanvas();
}
}
this._lastFrameAt = Date.now();
@@ -1229,11 +1251,21 @@ const model = {
if (!frameSrc) {
if (sequence === this._frameRenderSequence) {
this.clearFrameSrc();
+ this.clearFrameCanvas();
}
return;
}
- const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc);
+ let bitmap = null;
+ let dimensions = options?.dimensions || null;
+ if (options?.useCanvas && this.canUseCanvasFrames()) {
+ bitmap = await loadFrameBitmap(frameSrc, options);
+ if (bitmap) {
+ dimensions ||= { width: bitmap.width || 0, height: bitmap.height || 0 };
+ }
+ }
+ dimensions ||= await loadFrameDimensions(frameSrc);
if (sequence !== this._frameRenderSequence || surfaceSequence !== this._surfaceOpenSequence) {
+ bitmap?.close?.();
options?.cleanup?.();
return;
}
@@ -1241,13 +1273,21 @@ const model = {
if (!this.frameMatchesViewport(dimensions, viewport)) {
this.requestViewportSyncAfterRejectedFrame();
if (!this.shouldAcceptMismatchedFrame(dimensions)) {
+ bitmap?.close?.();
options?.cleanup?.();
return;
}
}
- this.releaseRenderedFrameUrl(frameSrc);
- this.frameSrc = frameSrc;
- this._frameObjectUrl = options?.objectUrl || "";
+ if (bitmap && this.paintFrameBitmap(bitmap)) {
+ this.clearFrameSrc();
+ options?.cleanup?.();
+ } else {
+ this.clearFrameCanvas();
+ this.releaseRenderedFrameUrl(frameSrc);
+ this.frameSrc = frameSrc;
+ this._frameObjectUrl = options?.objectUrl || "";
+ }
+ bitmap?.close?.();
this._lastFrameDimensions = dimensions;
this._lastFrameAt = Date.now();
options?.onAccepted?.();
@@ -1259,7 +1299,7 @@ const model = {
return Boolean(
dimensions?.width
&& dimensions?.height
- && (!this.frameSrc || this._surfaceSwitching || this.isSwitchingBrowser())
+ && (!this.hasFrame() || this._surfaceSwitching || this.isSwitchingBrowser())
);
},
@@ -1330,7 +1370,7 @@ const model = {
clearRenderedFrameIfViewportChanged() {
const viewport = this.currentViewportSize();
- if (!this.frameSrc || !this._lastFrameDimensions || !viewport) return;
+ if (!this.hasFrame() || !this._lastFrameDimensions || !viewport) return;
if (this.frameMatchesViewport(this._lastFrameDimensions, viewport)) return;
this.cancelFrameRender();
this.resetViewportTracking();
@@ -1368,6 +1408,55 @@ const model = {
this.frameSrc = "";
},
+ attachFrameCanvas(canvas = null) {
+ this._frameCanvas = canvas || null;
+ },
+
+ canUseCanvasFrames() {
+ return Boolean(BROWSER_CANVAS_FRAMES_SUPPORTED && this._frameCanvas?.getContext);
+ },
+
+ hasFrame() {
+ return Boolean(this.frameSrc || this.frameCanvasReady);
+ },
+
+ paintFrameBitmap(bitmap) {
+ const canvas = this._frameCanvas;
+ if (!canvas || !bitmap?.width || !bitmap?.height) return false;
+ canvas.width = bitmap.width;
+ canvas.height = bitmap.height;
+ const context = canvas.getContext("2d");
+ if (!context) return false;
+ context.drawImage(bitmap, 0, 0);
+ this.frameCanvasReady = true;
+ return true;
+ },
+
+ clearFrameCanvas() {
+ const canvas = this._frameCanvas;
+ if (canvas?.width && canvas?.height) {
+ canvas.getContext("2d")?.clearRect(0, 0, canvas.width, canvas.height);
+ }
+ this.frameCanvasReady = false;
+ },
+
+ freezeCanvasFrameToImage() {
+ if (!this.frameCanvasReady || !this._frameCanvas) return;
+ try {
+ this.frameSrc = this._frameCanvas.toDataURL("image/jpeg", 0.86);
+ } catch {
+ this.frameSrc = "";
+ }
+ this.clearFrameCanvas();
+ },
+
+ frameElement() {
+ if (this.frameCanvasReady && this._frameCanvas) {
+ return this._frameCanvas;
+ }
+ return this._stageElement?.querySelector?.(".browser-frame-image") || null;
+ },
+
beginCommand() {
this._commandInFlightCount += 1;
this.commandInFlight = true;
@@ -1431,6 +1520,7 @@ const model = {
if (!this.activeBrowserId) {
this.frameState = null;
this.clearFrameSrc();
+ this.clearFrameCanvas();
}
if (result.state?.currentUrl || result.currentUrl) {
this.address = result.state?.currentUrl || result.currentUrl;
@@ -1540,6 +1630,7 @@ const model = {
this.switchingBrowserId = targetId;
this.cancelFrameRender();
this.clearFrameSrc();
+ this.clearFrameCanvas();
this.frameState = browser || null;
if (!this.addressFocused && browser?.currentUrl) {
this.address = browser.currentUrl;
@@ -1811,8 +1902,8 @@ const model = {
const target = element || event?.currentTarget;
if (!target) return null;
const rect = target.getBoundingClientRect();
- const naturalWidth = target.naturalWidth || rect.width;
- const naturalHeight = target.naturalHeight || rect.height;
+ const naturalWidth = target.naturalWidth || target.width || rect.width;
+ const naturalHeight = target.naturalHeight || target.height || rect.height;
let contentLeft = rect.left;
let contentTop = rect.top;
let contentWidth = rect.width;
@@ -1963,7 +2054,7 @@ const model = {
},
canAnnotate() {
- return Boolean(this.activeBrowserId && this.frameSrc && !this.isBusy());
+ return Boolean(this.activeBrowserId && this.hasFrame() && !this.isBusy());
},
activeAnnotationUrl() {
@@ -2124,8 +2215,7 @@ const model = {
},
stagePointForEvent(event) {
- const image = this._stageElement?.querySelector?.(".browser-frame") || null;
- return this.pointerCoordinatesFor(event, image);
+ return this.pointerCoordinatesFor(event, this.frameElement());
},
normalizeAnnotationRect(start = {}, end = {}) {
@@ -2538,8 +2628,7 @@ const model = {
async sendWheel(event) {
const contextId = this.normalizeContextId(this.activeBrowserContextId || this.contextId);
if (!contextId || !this.activeBrowserId || !event) return;
- const image = event.currentTarget?.querySelector?.(".browser-frame") || event.target?.closest?.(".browser-frame");
- const pointer = this.pointerCoordinatesFor(event, image);
+ const pointer = this.pointerCoordinatesFor(event, this.frameElement());
if (!pointer) return;
const payload = {
context_id: contextId,
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index a67de6054..c11ab4da3 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -1,4 +1,5 @@
import asyncio
+import concurrent.futures
import json
import re
import sys
@@ -609,7 +610,7 @@ def test_browser_canvas_startup_waits_for_raw_viewport_settle():
assert "isCanvasSurfaceVisible(element)" in js
assert "scheduleViewportSyncForSurface" in js
assert "const targetChanged = Boolean(" in js
- assert "if (this.frameSrc && !targetChanged)" in js
+ assert "if (this.hasFrame() && !targetChanged)" in js
assert "this.cancelFrameRender();" in js
assert "this.resetRenderedFrame();" in js
@@ -626,7 +627,7 @@ def test_browser_surface_handoffs_keep_existing_frame_until_replacement_arrives(
clear_block = js[clear_start: js.index("beginCommand()", clear_start)]
assert "modeChanged" not in prepare_block
- assert "if (this.frameSrc && !targetChanged)" in prepare_block
+ assert "if (this.hasFrame() && !targetChanged)" in prepare_block
assert "this.resetRenderedFrame();" in prepare_block
assert "this.cancelFrameRender();" in viewport_block
assert "this.resetRenderedFrame();" not in viewport_block
@@ -674,12 +675,12 @@ def test_browser_canvas_nudges_width_after_first_accepted_frame():
assert 'canvas.activeSurfaceId !== "browser"' in js
assert "canvas.setWidth?.(nudgedWidth, { persist: false })" in js
assert "this.queueViewportSync(true)" in js
- frame_assignment_index = js.index("this.frameSrc = frameSrc;")
+ frame_accept_index = js.index("this._lastFrameDimensions = dimensions;")
frame_nudge_schedule_index = js.index(
"this.scheduleCanvasWidthNudgeAfterFirstFrame();",
- frame_assignment_index,
+ frame_accept_index,
)
- assert frame_assignment_index < frame_nudge_schedule_index
+ assert frame_accept_index < frame_nudge_schedule_index
def test_browser_canvas_restarts_stream_after_page_navigation():
@@ -1163,7 +1164,10 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "initial_viewport = self._viewport_from_data(data)" in ws_browser
assert '"set_viewport"' in ws_browser
assert "start_screencast" in ws_browser
- assert "read_screencast_frame" in ws_browser
+ assert '"attach_screencast_consumer"' in ws_browser
+ assert "asyncio.run_coroutine_threadsafe" in ws_browser
+ assert 'runtime.call(\n "read_screencast_frame"' not in ws_browser
+ assert "read_screencast_frame" in runtime
assert "FRAME_READ_TIMEOUT_SECONDS" in ws_browser
assert "FRAME_IDLE_POLL_SECONDS" not in ws_browser
assert "except TimeoutError:" in ws_browser
@@ -1174,6 +1178,10 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert '"Page.startScreencast"' in runtime
assert '"Page.screencastFrame"' in runtime
assert '"Page.screencastFrameAck"' in runtime
+ assert "async def attach_screencast_consumer" in runtime
+ assert "async def attach_consumer" in runtime
+ assert "await self._deliver_frame(" in runtime
+ assert "await asyncio.wrap_future(future)" in runtime
assert '"Page.stopScreencast"' in runtime
assert '"Emulation.setDeviceMetricsOverride"' in runtime
assert '"Emulation.setVisibleSize"' in runtime
@@ -1184,7 +1192,10 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "function frameImageSource(data = {})" in browser_store
assert "const BROWSER_BINARY_FRAME_REQUESTS_ENABLED = false;" in browser_store
assert "const BROWSER_BINARY_PAYLOADS_SUPPORTED = typeof Blob" in browser_store
+ assert "const BROWSER_CANVAS_FRAMES_SUPPORTED = typeof globalThis.createImageBitmap" in browser_store
assert "if (data.encoding === \"binary\") return null;" in browser_store
+ assert "async function loadFrameBitmap(src, options = {})" in browser_store
+ assert "return await globalThis.createImageBitmap(blob);" in browser_store
assert 'const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";' in browser_store
assert 'const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";' in browser_store
assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST" in browser_store
@@ -1198,7 +1209,16 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "frameDimensionsFromMetadata(metadata = null)" in browser_store
assert "metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth" in browser_store
assert "dimensions: this.frameDimensionsFromData(data)" in browser_store
- assert "const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc)" in browser_store
+ assert "useCanvas: true" in browser_store
+ assert "let dimensions = options?.dimensions || null" in browser_store
+ assert "dimensions ||= await loadFrameDimensions(frameSrc)" in browser_store
+ assert "paintFrameBitmap(bitmap)" in browser_store
+ assert "freezeCanvasFrameToImage()" in browser_store
+ assert 'this._frameCanvas.toDataURL("image/jpeg", 0.86)' in browser_store
+ assert "this.frameSrc = frameSrc;" in browser_store
+ assert "hasFrame()" in browser_store
+ assert "frameCanvasReady: false" in browser_store
+ assert "attachFrameCanvas(canvas = null)" in browser_store
assert "viewer_transport: this.requestedViewerTransport()" in browser_store
assert "binary_frames: this.supportsBinaryFrames()" in browser_store
assert "slim_frames: true" in browser_store
@@ -1247,6 +1267,10 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "fallback_screenshot" not in ws_browser
assert "canvas_wheel_screenshot" not in ws_browser
assert "surface_mode: this._mode" not in browser_store
+ assert '<canvas class="browser-frame browser-frame-canvas"' in main_html
+ assert 'x-init="$store.browserPage.attachFrameCanvas($el)"' in main_html
+ assert '<img class="browser-frame browser-frame-image"' in main_html
+ assert "$store.browserPage.hasFrame()" in main_html
assert "overflow: hidden;" in main_html
assert "object-fit: contain;" in main_html
assert "image-rendering: auto;" in main_html
@@ -1628,6 +1652,93 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
assert session.detached is True
+async def test_browser_screencast_acks_after_consumer_settles():
+ class FakeSession:
+ def __init__(self):
+ self.handlers = {}
+ self.sent = []
+
+ def on(self, event, handler):
+ self.handlers[event] = handler
+
+ async def send(self, method, params=None):
+ self.sent.append((method, params or {}))
+
+ async def detach(self):
+ pass
+
+ session = FakeSession()
+ delivered = []
+ delivery = concurrent.futures.Future()
+ screencast = _BrowserScreencast(
+ stream_id="stream",
+ browser_id=7,
+ session=session,
+ mime="image/jpeg",
+ )
+ screencast.frame_consumer = lambda frame: delivered.append(frame) or delivery
+
+ await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 640, "height": 480})
+ session.handlers["Page.screencastFrame"](
+ {"data": SMALL_JPEG_10X10, "metadata": {}, "sessionId": 21}
+ )
+ await asyncio.sleep(0)
+
+ assert delivered
+ assert ("Page.screencastFrameAck", {"sessionId": 21}) not in session.sent
+
+ delivery.set_result(None)
+ await asyncio.sleep(0)
+ await asyncio.sleep(0)
+
+ assert ("Page.screencastFrameAck", {"sessionId": 21}) in session.sent
+
+ await screencast.stop()
+
+
+async def test_browser_screencast_attach_consumer_flushes_queued_frame():
+ class FakeSession:
+ def __init__(self):
+ self.handlers = {}
+ self.sent = []
+
+ def on(self, event, handler):
+ self.handlers[event] = handler
+
+ async def send(self, method, params=None):
+ self.sent.append((method, params or {}))
+
+ async def detach(self):
+ pass
+
+ session = FakeSession()
+ delivered = []
+ delivery = concurrent.futures.Future()
+ delivery.set_result(None)
+ screencast = _BrowserScreencast(
+ stream_id="stream",
+ browser_id=7,
+ session=session,
+ mime="image/jpeg",
+ )
+
+ await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 640, "height": 480})
+ session.handlers["Page.screencastFrame"](
+ {"data": SMALL_JPEG_10X10, "metadata": {}, "sessionId": 31}
+ )
+ await asyncio.sleep(0)
+
+ await screencast.attach_consumer(lambda frame: delivered.append(frame) or delivery)
+
+ assert delivered
+ assert delivered[0]["image"] == SMALL_JPEG_10X10
+ assert await screencast.pop_frame() is None
+
+ await screencast.stop()
+
+
@pytest.mark.anyio
async def test_browser_screencast_passes_wrong_viewport_frames_to_frontend_validator():
class FakeSession: