Skip to content

Agent Zero AI framework

Stabilize browser viewer viewport rendering

Alessandrodecb05a6823b
diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py
index db459de91..3111b3068 100644
--- a/plugins/_browser/helpers/runtime.py
+++ b/plugins/_browser/helpers/runtime.py
@@ -111,7 +111,6 @@ class _BrowserScreencast:
self._ack_tasks: set[asyncio.Task] = set()
self._expected_width = 0
self._expected_height = 0
- self._dimension_mismatches = 0
async def start(
self,
@@ -125,7 +124,6 @@ class _BrowserScreencast:
height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"])))
self._expected_width = width
self._expected_height = height
- self._dimension_mismatches = 0
with contextlib.suppress(Exception):
await self.session.send("Page.enable")
await self.session.send(
@@ -229,10 +227,12 @@ class _BrowserScreencast:
if not size:
return True
width, height = size
- if abs(width - self._expected_width) <= 2 and abs(height - self._expected_height) <= 2:
+ if (
+ abs(width - self._expected_width) <= VIEWPORT_SIZE_TOLERANCE
+ and abs(height - self._expected_height) <= VIEWPORT_SIZE_TOLERANCE
+ ):
return True
- self._dimension_mismatches += 1
- return self._dimension_mismatches > 10
+ return False
@staticmethod
def _jpeg_size(data: str) -> tuple[int, int] | None:
diff --git a/plugins/_browser/webui/browser-panel.html b/plugins/_browser/webui/browser-panel.html
index e10baaaf5..3ff50ef5d 100644
--- a/plugins/_browser/webui/browser-panel.html
+++ b/plugins/_browser/webui/browser-panel.html
@@ -620,7 +620,7 @@
align-items: end;
gap: 6px;
min-width: 0;
- padding-bottom: 1px;
+ padding-bottom: 7px;
}
.browser-annotate-toggle {
diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js
index b7db50928..bb12b161e 100644
--- a/plugins/_browser/webui/browser-store.js
+++ b/plugins/_browser/webui/browser-store.js
@@ -13,8 +13,9 @@ const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;
const BROWSER_CONFIG_REFRESH_MS = 15000;
const VIEWPORT_SYNC_DEBOUNCE_MS = 220;
const VIEWPORT_SYNC_SIZE_TOLERANCE = 4;
-const CANVAS_VIEWPORT_SETTLE_MS = 260;
-const SURFACE_VIEWPORT_STABLE_FRAMES = 2;
+const CANVAS_VIEWPORT_SETTLE_MS = 520;
+const SURFACE_VIEWPORT_STABLE_FRAMES = 4;
+const FRAME_REJECT_SYNC_COOLDOWN_MS = 600;
const ANNOTATION_DRAG_THRESHOLD = 6;
const ANNOTATION_MAX_COMMENTS = 24;
const ANNOTATION_DOM_LIMIT = 1200;
@@ -55,6 +56,34 @@ function nextAnimationFrame() {
});
}
+function loadFrameDimensions(src) {
+ return new Promise((resolve) => {
+ if (!src) {
+ resolve(null);
+ return;
+ }
+
+ const image = new Image();
+ let settled = false;
+ const finish = (dimensions) => {
+ if (settled) return;
+ settled = true;
+ resolve(dimensions);
+ };
+
+ image.onload = () => finish({
+ width: image.naturalWidth || 0,
+ height: image.naturalHeight || 0,
+ });
+ image.onerror = () => finish(null);
+ image.src = src;
+
+ if (image.complete) {
+ image.onload();
+ }
+ });
+}
+
const model = {
loading: true,
error: "",
@@ -79,13 +108,17 @@ const model = {
_frameOff: null,
_stateOff: null,
_lastFrameAt: 0,
+ _lastFrameDimensions: null,
_pendingFrameSrc: "",
+ _pendingFrameOptions: null,
_frameRenderHandle: null,
_frameRenderCancel: null,
+ _frameRenderSequence: 0,
_floatingCleanup: null,
_stageElement: null,
_stageResizeObserver: null,
_viewportSyncTimer: null,
+ _lastFrameRejectSyncAt: 0,
_lastViewportKey: "",
_lastViewport: null,
_annotationPointer: null,
@@ -466,6 +499,7 @@ const model = {
resetRenderedFrame() {
this.cancelFrameRender();
this.frameSrc = "";
+ this._lastFrameDimensions = null;
this._lastFrameAt = 0;
},
@@ -599,11 +633,16 @@ const model = {
this.address = data.state.currentUrl;
}
if (data.image) {
- this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`);
- if (this.sameBrowserId(this.switchingBrowserId, incomingBrowserId || this.activeBrowserId)) {
- this.switchingBrowserId = null;
- }
- this._surfaceSwitching = false;
+ const frameBrowserId = incomingBrowserId || this.activeBrowserId;
+ this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`, {
+ browserId: frameBrowserId,
+ onAccepted: () => {
+ if (this.sameBrowserId(this.switchingBrowserId, frameBrowserId)) {
+ this.switchingBrowserId = null;
+ }
+ this._surfaceSwitching = false;
+ },
+ });
} else {
this.cancelFrameRender();
if (!data.state) {
@@ -654,8 +693,9 @@ const model = {
}
},
- queueFrameRender(frameSrc) {
+ queueFrameRender(frameSrc, options = {}) {
this._pendingFrameSrc = frameSrc;
+ this._pendingFrameOptions = options || null;
if (this._frameRenderHandle) return;
const schedule = globalThis.requestAnimationFrame?.bind(globalThis);
if (schedule) {
@@ -670,8 +710,59 @@ const model = {
flushFrameRender() {
this._frameRenderHandle = null;
this._frameRenderCancel = null;
- this.frameSrc = this._pendingFrameSrc || "";
+ const frameSrc = this._pendingFrameSrc || "";
+ const options = this._pendingFrameOptions || {};
this._pendingFrameSrc = "";
+ this._pendingFrameOptions = null;
+ const sequence = this._frameRenderSequence + 1;
+ this._frameRenderSequence = sequence;
+ void this.renderDecodedFrame(frameSrc, options, sequence);
+ },
+
+ async renderDecodedFrame(frameSrc, options = {}, sequence = 0) {
+ if (!frameSrc) {
+ if (sequence === this._frameRenderSequence) {
+ this.frameSrc = "";
+ }
+ return;
+ }
+ const dimensions = await loadFrameDimensions(frameSrc);
+ if (sequence !== this._frameRenderSequence) return;
+ const viewport = this.currentViewportSize() || this._lastViewport;
+ if (!this.frameMatchesViewport(dimensions, viewport)) {
+ this.requestViewportSyncAfterRejectedFrame();
+ return;
+ }
+ this.frameSrc = frameSrc;
+ this._lastFrameDimensions = dimensions;
+ this._lastFrameAt = Date.now();
+ options?.onAccepted?.();
+ },
+
+ frameMatchesViewport(dimensions = null, viewport = null) {
+ if (!dimensions?.width || !dimensions?.height || !viewport?.width || !viewport?.height) {
+ return false;
+ }
+ return Math.abs(Number(dimensions.width) - Number(viewport.width)) <= VIEWPORT_SYNC_SIZE_TOLERANCE
+ && Math.abs(Number(dimensions.height) - Number(viewport.height)) <= VIEWPORT_SYNC_SIZE_TOLERANCE;
+ },
+
+ requestViewportSyncAfterRejectedFrame() {
+ const now = Date.now();
+ if (now - this._lastFrameRejectSyncAt < FRAME_REJECT_SYNC_COOLDOWN_MS) return;
+ this._lastFrameRejectSyncAt = now;
+ this.queueViewportSync(true);
+ },
+
+ clearRenderedFrameIfViewportChanged() {
+ const viewport = this.currentViewportSize();
+ if (!this.frameSrc || !this._lastFrameDimensions || !viewport) return;
+ if (this.frameMatchesViewport(this._lastFrameDimensions, viewport)) return;
+ this.resetRenderedFrame();
+ if (this.activeBrowserId) {
+ this._surfaceSwitching = true;
+ this.switchingBrowserId = this.activeBrowserId;
+ }
},
cancelFrameRender() {
@@ -681,6 +772,8 @@ const model = {
this._frameRenderHandle = null;
this._frameRenderCancel = null;
this._pendingFrameSrc = "";
+ this._pendingFrameOptions = null;
+ this._frameRenderSequence += 1;
},
async command(command, extra = {}) {
@@ -865,11 +958,16 @@ const model = {
if (snapshot.state) {
this.applyActiveFrameState(snapshot.state);
}
- this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`);
- if (this.sameBrowserId(this.switchingBrowserId, snapshotId || this.activeBrowserId)) {
- this.switchingBrowserId = null;
- }
- this._surfaceSwitching = false;
+ const frameBrowserId = snapshotId || this.activeBrowserId;
+ this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {
+ browserId: frameBrowserId,
+ onAccepted: () => {
+ if (this.sameBrowserId(this.switchingBrowserId, frameBrowserId)) {
+ this.switchingBrowserId = null;
+ }
+ this._surfaceSwitching = false;
+ },
+ });
},
isSwitchingBrowser() {
@@ -1371,6 +1469,7 @@ const model = {
},
queueViewportSync(force = false) {
+ this.clearRenderedFrameIfViewportChanged();
if (this._viewportSyncTimer) {
globalThis.clearTimeout(this._viewportSyncTimer);
}
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index bcbec0f67..f4bc81b17 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -110,6 +110,21 @@ import plugins._browser.tools.browser as browser_tool_module
import plugins._browser.api.ws_browser as ws_browser_module
+SMALL_JPEG_10X10 = (
+ "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL"
+ "DBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/"
+ "2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy"
+ "MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAKAAoDASIAAhEBAxEB/8QAFQAB"
+ "AAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhADE"
+ "AAAAKf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAA"
+ "AAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECA"
+ "QE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Aqf/xAAUEAEAAAAAAA"
+ "AAAAAAAAAAAAAA/9oACAEBAAE/ISf/2gAMAwEAAgADAAAAEP/EABQRAQAAAAAAAAA"
+ "AAAAAAAAAAP/aAAgBAwEBPxAk/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEB"
+ "PxAk/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxAn/9k="
+)
+
+
def test_browser_url_normalization_matches_address_bar_hosts():
assert normalize_url("localhost:3000") == "http://localhost:3000/"
assert normalize_url("127.0.0.1:8000/path") == "http://127.0.0.1:8000/path"
@@ -365,7 +380,8 @@ def test_browser_canvas_startup_waits_for_raw_viewport_settle():
encoding="utf-8"
)
- assert "const CANVAS_VIEWPORT_SETTLE_MS = 260;" in js
+ assert "const CANVAS_VIEWPORT_SETTLE_MS = 520;" in js
+ assert "const SURFACE_VIEWPORT_STABLE_FRAMES = 4;" in js
assert "surfaceViewportMeasurement()" in js
assert "rawWidth" in js
assert "rawHeight" in js
@@ -454,6 +470,9 @@ def test_browser_viewer_uses_cdp_screencast_transport():
assert "viewport_width: initialViewport?.width" in browser_store
assert "viewport_height: initialViewport?.height" in browser_store
assert "this.frameState = data.state || null" not in browser_store
+ assert "function loadFrameDimensions(src)" in browser_store
+ assert "frameMatchesViewport(dimensions = null, viewport = null)" in browser_store
+ assert "requestViewportSyncAfterRejectedFrame()" in browser_store
assert "overflow: hidden;" in main_html
assert "object-fit: fill;" in main_html
assert "image-rendering: auto;" in main_html
@@ -504,19 +523,7 @@ def test_browser_runtime_and_content_helper_expose_annotation_target():
@pytest.mark.anyio
async def test_browser_screencast_acknowledges_and_drops_stale_frames():
- first_image = (
- "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL"
- "DBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/"
- "2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy"
- "MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAKAAoDASIAAhEBAxEB/8QAFQAB"
- "AAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhADE"
- "AAAAKf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAA"
- "AAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECA"
- "QE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Aqf/xAAUEAEAAAAAAA"
- "AAAAAAAAAAAAAA/9oACAEBAAE/ISf/2gAMAwEAAgADAAAAEP/EABQRAQAAAAAAAAA"
- "AAAAAAAAAAP/aAAgBAwEBPxAk/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEB"
- "PxAk/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxAn/9k="
- )
+ first_image = SMALL_JPEG_10X10
class FakeSession:
def __init__(self):
@@ -572,6 +579,43 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
assert session.detached is True
+async def test_browser_screencast_keeps_rejecting_wrong_viewport_frames():
+ class FakeSession:
+ def __init__(self):
+ self.handlers = {}
+ self.sent = []
+
+ def on(self, event, handler):
+ self.handlers[event] = handler
+
+ async def send(self, method, params=None):
+ self.sent.append((method, params or {}))
+
+ async def detach(self):
+ pass
+
+ session = FakeSession()
+ screencast = _BrowserScreencast(
+ stream_id="stream",
+ browser_id=7,
+ session=session,
+ mime="image/jpeg",
+ )
+
+ await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 1118, "height": 662})
+ for session_id in range(1, 14):
+ session.handlers["Page.screencastFrame"](
+ {"data": SMALL_JPEG_10X10, "metadata": {}, "sessionId": session_id}
+ )
+ await asyncio.sleep(0)
+
+ assert await screencast.pop_frame() is None
+ assert ("Page.screencastFrameAck", {"sessionId": 13}) in session.sent
+
+ await screencast.stop()
+
+
def test_browser_docker_installs_full_chromium_to_persistent_cache():
script = (
PROJECT_ROOT / "docker" / "run" / "fs" / "ins" / "install_playwright.sh"