Skip to content

Agent Zero AI framework

Fix Browser viewer input after stream restarts

Alessandro9bfa3b5c81d7
diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py
index 8f27cdc73..994c6f04e 100644
--- a/plugins/_browser/helpers/runtime.py
+++ b/plugins/_browser/helpers/runtime.py
@@ -411,8 +411,11 @@ class _BrowserScreencast:
async def stop(self) -> None:
if self._closed:
return
+ was_stopped = self.stopped
self._closed = True
self.stopped = True
+ if not was_stopped:
+ self._notify_stopped()
self._drop_queued_frames()
with contextlib.suppress(asyncio.QueueFull):
self.queue.put_nowait(None)
diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js
index 6e9a779a8..bf2a68411 100644
--- a/plugins/_browser/webui/browser-store.js
+++ b/plugins/_browser/webui/browser-store.js
@@ -1412,8 +1412,15 @@ const model = {
this._frameCanvas = canvas || null;
},
+ currentFrameCanvas() {
+ const stageCanvas = this._stageElement?.querySelector?.(".browser-frame-canvas");
+ if (stageCanvas?.isConnected) return stageCanvas;
+ if (this._frameCanvas?.isConnected) return this._frameCanvas;
+ return null;
+ },
+
canUseCanvasFrames() {
- return Boolean(BROWSER_CANVAS_FRAMES_SUPPORTED && this._frameCanvas?.getContext);
+ return Boolean(BROWSER_CANVAS_FRAMES_SUPPORTED && this.currentFrameCanvas()?.getContext);
},
hasFrame() {
@@ -1421,7 +1428,7 @@ const model = {
},
paintFrameBitmap(bitmap) {
- const canvas = this._frameCanvas;
+ const canvas = this.currentFrameCanvas();
if (!canvas || !bitmap?.width || !bitmap?.height) return false;
if (canvas.width !== bitmap.width) canvas.width = bitmap.width;
if (canvas.height !== bitmap.height) canvas.height = bitmap.height;
@@ -1433,7 +1440,7 @@ const model = {
},
clearFrameCanvas() {
- const canvas = this._frameCanvas;
+ const canvas = this.currentFrameCanvas();
if (canvas?.width && canvas?.height) {
canvas.getContext("2d")?.clearRect(0, 0, canvas.width, canvas.height);
}
@@ -1441,9 +1448,10 @@ const model = {
},
freezeCanvasFrameToImage() {
- if (!this.frameCanvasReady || !this._frameCanvas) return;
+ const canvas = this.currentFrameCanvas();
+ if (!this.frameCanvasReady || !canvas) return;
try {
- this.frameSrc = this._frameCanvas.toDataURL("image/jpeg", 0.86);
+ this.frameSrc = canvas.toDataURL("image/jpeg", 0.86);
} catch {
this.frameSrc = "";
}
@@ -1451,8 +1459,9 @@ const model = {
},
frameElement() {
- if (this.frameCanvasReady && this._frameCanvas) {
- return this._frameCanvas;
+ if (this.frameCanvasReady) {
+ const canvas = this.currentFrameCanvas();
+ if (canvas) return canvas;
}
return this._stageElement?.querySelector?.(".browser-frame-image") || null;
},
@@ -2616,7 +2625,9 @@ const model = {
const response = await websocket.request("browser_viewer_input", payload, { timeoutMs: 10000 });
const data = firstOk(response);
this.applyActiveFrameState(data.state);
- this.applySnapshot(data.snapshot);
+ if (!this.frameCanvasReady || !this.usesScreencastTransport()) {
+ this.applySnapshot(data.snapshot);
+ }
} catch (error) {
this.error = error instanceof Error ? error.message : String(error);
}
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index 67ee8c9ba..825fcd382 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -1218,7 +1218,9 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "paintFrameBitmap(bitmap)" in browser_store
assert "if (canvas.width !== bitmap.width) canvas.width = bitmap.width;" in browser_store
assert "freezeCanvasFrameToImage()" in browser_store
- assert 'this._frameCanvas.toDataURL("image/jpeg", 0.86)' in browser_store
+ assert "currentFrameCanvas()" in browser_store
+ assert 'this._stageElement?.querySelector?.(".browser-frame-canvas")' in browser_store
+ assert 'canvas.toDataURL("image/jpeg", 0.86)' in browser_store
assert "this.frameSrc = frameSrc;" in browser_store
assert "hasFrame()" in browser_store
assert "frameCanvasReady: false" in browser_store
@@ -1246,6 +1248,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "shouldAcceptMismatchedFrame(dimensions = null)" in browser_store
assert "requestViewportSyncAfterRejectedFrame()" in browser_store
assert "this.applySnapshot(data.snapshot);" in browser_store
+ assert "if (!this.frameCanvasReady || !this.usesScreencastTransport())" in browser_store
assert "else if (!data.state)" in browser_store
assert '"snapshot": snapshot' in ws_browser
assert '"binary_frames": binary_frames' in ws_browser
@@ -1755,6 +1758,40 @@ async def test_browser_screencast_notifies_consumer_when_frame_task_stops_before
assert session.detached is True
+async def test_browser_screencast_stop_notifies_consumer_once():
+ class FakeSession:
+ def __init__(self):
+ self.handlers = {}
+ self.sent = []
+
+ def on(self, event, handler):
+ self.handlers[event] = handler
+
+ async def send(self, method, params=None):
+ self.sent.append((method, params or {}))
+
+ async def detach(self):
+ pass
+
+ session = FakeSession()
+ stopped = []
+ screencast = _BrowserScreencast(
+ stream_id="stream",
+ browser_id=7,
+ session=session,
+ mime="image/jpeg",
+ )
+
+ await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 640, "height": 480})
+ await screencast.attach_consumer(lambda frame: None, lambda: stopped.append(True))
+ await screencast.stop()
+ await screencast.stop()
+
+ assert stopped == [True]
+ assert ("Page.stopScreencast", {}) in session.sent
+
+
@pytest.mark.anyio
async def test_browser_screencast_attach_consumer_flushes_queued_frame():
class FakeSession: