Agent Zero AI framework
Stabilize browser viewer viewport rendering
Alessandrodecb05a6823b
diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.pyindex db459de91..3111b3068 100644--- a/plugins/_browser/helpers/runtime.py+++ b/plugins/_browser/helpers/runtime.py@@ -111,7 +111,6 @@ class _BrowserScreencast:self._ack_tasks: set[asyncio.Task] = set()self._expected_width = 0self._expected_height = 0- self._dimension_mismatches = 0async def start(self,@@ -125,7 +124,6 @@ class _BrowserScreencast:height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"])))self._expected_width = widthself._expected_height = height- self._dimension_mismatches = 0with contextlib.suppress(Exception):await self.session.send("Page.enable")await self.session.send(@@ -229,10 +227,12 @@ class _BrowserScreencast:if not size:return Truewidth, height = size- if abs(width - self._expected_width) <= 2 and abs(height - self._expected_height) <= 2:+ if (+ abs(width - self._expected_width) <= VIEWPORT_SIZE_TOLERANCE+ and abs(height - self._expected_height) <= VIEWPORT_SIZE_TOLERANCE+ ):return True- self._dimension_mismatches += 1- return self._dimension_mismatches > 10+ return False@staticmethoddef _jpeg_size(data: str) -> tuple[int, int] | None:diff --git a/plugins/_browser/webui/browser-panel.html b/plugins/_browser/webui/browser-panel.htmlindex e10baaaf5..3ff50ef5d 100644--- a/plugins/_browser/webui/browser-panel.html+++ b/plugins/_browser/webui/browser-panel.html@@ -620,7 +620,7 @@align-items: end;gap: 6px;min-width: 0;- padding-bottom: 1px;+ padding-bottom: 7px;}.browser-annotate-toggle {diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.jsindex b7db50928..bb12b161e 100644--- a/plugins/_browser/webui/browser-store.js+++ b/plugins/_browser/webui/browser-store.js@@ -13,8 +13,9 @@ const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;const BROWSER_CONFIG_REFRESH_MS = 15000;const VIEWPORT_SYNC_DEBOUNCE_MS = 220;const VIEWPORT_SYNC_SIZE_TOLERANCE = 4;-const CANVAS_VIEWPORT_SETTLE_MS = 260;-const SURFACE_VIEWPORT_STABLE_FRAMES = 2;+const CANVAS_VIEWPORT_SETTLE_MS = 520;+const SURFACE_VIEWPORT_STABLE_FRAMES = 4;+const FRAME_REJECT_SYNC_COOLDOWN_MS = 600;const ANNOTATION_DRAG_THRESHOLD = 6;const ANNOTATION_MAX_COMMENTS = 24;const ANNOTATION_DOM_LIMIT = 1200;@@ -55,6 +56,34 @@ function nextAnimationFrame() {});}+function loadFrameDimensions(src) {+ return new Promise((resolve) => {+ if (!src) {+ resolve(null);+ return;+ }++ const image = new Image();+ let settled = false;+ const finish = (dimensions) => {+ if (settled) return;+ settled = true;+ resolve(dimensions);+ };++ image.onload = () => finish({+ width: image.naturalWidth || 0,+ height: image.naturalHeight || 0,+ });+ image.onerror = () => finish(null);+ image.src = src;++ if (image.complete) {+ image.onload();+ }+ });+}+const model = {loading: true,error: "",@@ -79,13 +108,17 @@ const model = {_frameOff: null,_stateOff: null,_lastFrameAt: 0,+ _lastFrameDimensions: null,_pendingFrameSrc: "",+ _pendingFrameOptions: null,_frameRenderHandle: null,_frameRenderCancel: null,+ _frameRenderSequence: 0,_floatingCleanup: null,_stageElement: null,_stageResizeObserver: null,_viewportSyncTimer: null,+ _lastFrameRejectSyncAt: 0,_lastViewportKey: "",_lastViewport: null,_annotationPointer: null,@@ -466,6 +499,7 @@ const model = {resetRenderedFrame() {this.cancelFrameRender();this.frameSrc = "";+ this._lastFrameDimensions = null;this._lastFrameAt = 0;},@@ -599,11 +633,16 @@ const model = {this.address = data.state.currentUrl;}if (data.image) {- this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`);- if (this.sameBrowserId(this.switchingBrowserId, incomingBrowserId || this.activeBrowserId)) {- this.switchingBrowserId = null;- }- this._surfaceSwitching = false;+ const frameBrowserId = incomingBrowserId || this.activeBrowserId;+ this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`, {+ browserId: frameBrowserId,+ onAccepted: () => {+ if (this.sameBrowserId(this.switchingBrowserId, frameBrowserId)) {+ this.switchingBrowserId = null;+ }+ this._surfaceSwitching = false;+ },+ });} else {this.cancelFrameRender();if (!data.state) {@@ -654,8 +693,9 @@ const model = {}},- queueFrameRender(frameSrc) {+ queueFrameRender(frameSrc, options = {}) {this._pendingFrameSrc = frameSrc;+ this._pendingFrameOptions = options || null;if (this._frameRenderHandle) return;const schedule = globalThis.requestAnimationFrame?.bind(globalThis);if (schedule) {@@ -670,8 +710,59 @@ const model = {flushFrameRender() {this._frameRenderHandle = null;this._frameRenderCancel = null;- this.frameSrc = this._pendingFrameSrc || "";+ const frameSrc = this._pendingFrameSrc || "";+ const options = this._pendingFrameOptions || {};this._pendingFrameSrc = "";+ this._pendingFrameOptions = null;+ const sequence = this._frameRenderSequence + 1;+ this._frameRenderSequence = sequence;+ void this.renderDecodedFrame(frameSrc, options, sequence);+ },++ async renderDecodedFrame(frameSrc, options = {}, sequence = 0) {+ if (!frameSrc) {+ if (sequence === this._frameRenderSequence) {+ this.frameSrc = "";+ }+ return;+ }+ const dimensions = await loadFrameDimensions(frameSrc);+ if (sequence !== this._frameRenderSequence) return;+ const viewport = this.currentViewportSize() || this._lastViewport;+ if (!this.frameMatchesViewport(dimensions, viewport)) {+ this.requestViewportSyncAfterRejectedFrame();+ return;+ }+ this.frameSrc = frameSrc;+ this._lastFrameDimensions = dimensions;+ this._lastFrameAt = Date.now();+ options?.onAccepted?.();+ },++ frameMatchesViewport(dimensions = null, viewport = null) {+ if (!dimensions?.width || !dimensions?.height || !viewport?.width || !viewport?.height) {+ return false;+ }+ return Math.abs(Number(dimensions.width) - Number(viewport.width)) <= VIEWPORT_SYNC_SIZE_TOLERANCE+ && Math.abs(Number(dimensions.height) - Number(viewport.height)) <= VIEWPORT_SYNC_SIZE_TOLERANCE;+ },++ requestViewportSyncAfterRejectedFrame() {+ const now = Date.now();+ if (now - this._lastFrameRejectSyncAt < FRAME_REJECT_SYNC_COOLDOWN_MS) return;+ this._lastFrameRejectSyncAt = now;+ this.queueViewportSync(true);+ },++ clearRenderedFrameIfViewportChanged() {+ const viewport = this.currentViewportSize();+ if (!this.frameSrc || !this._lastFrameDimensions || !viewport) return;+ if (this.frameMatchesViewport(this._lastFrameDimensions, viewport)) return;+ this.resetRenderedFrame();+ if (this.activeBrowserId) {+ this._surfaceSwitching = true;+ this.switchingBrowserId = this.activeBrowserId;+ }},cancelFrameRender() {@@ -681,6 +772,8 @@ const model = {this._frameRenderHandle = null;this._frameRenderCancel = null;this._pendingFrameSrc = "";+ this._pendingFrameOptions = null;+ this._frameRenderSequence += 1;},async command(command, extra = {}) {@@ -865,11 +958,16 @@ const model = {if (snapshot.state) {this.applyActiveFrameState(snapshot.state);}- this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`);- if (this.sameBrowserId(this.switchingBrowserId, snapshotId || this.activeBrowserId)) {- this.switchingBrowserId = null;- }- this._surfaceSwitching = false;+ const frameBrowserId = snapshotId || this.activeBrowserId;+ this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {+ browserId: frameBrowserId,+ onAccepted: () => {+ if (this.sameBrowserId(this.switchingBrowserId, frameBrowserId)) {+ this.switchingBrowserId = null;+ }+ this._surfaceSwitching = false;+ },+ });},isSwitchingBrowser() {@@ -1371,6 +1469,7 @@ const model = {},queueViewportSync(force = false) {+ this.clearRenderedFrameIfViewportChanged();if (this._viewportSyncTimer) {globalThis.clearTimeout(this._viewportSyncTimer);}diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.pyindex bcbec0f67..f4bc81b17 100644--- a/tests/test_browser_agent_regressions.py+++ b/tests/test_browser_agent_regressions.py@@ -110,6 +110,21 @@ import plugins._browser.tools.browser as browser_tool_moduleimport plugins._browser.api.ws_browser as ws_browser_module+SMALL_JPEG_10X10 = (+ "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL"+ "DBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/"+ "2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy"+ "MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAKAAoDASIAAhEBAxEB/8QAFQAB"+ "AAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhADE"+ "AAAAKf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAA"+ "AAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECA"+ "QE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Aqf/xAAUEAEAAAAAAA"+ "AAAAAAAAAAAAAA/9oACAEBAAE/ISf/2gAMAwEAAgADAAAAEP/EABQRAQAAAAAAAAA"+ "AAAAAAAAAAP/aAAgBAwEBPxAk/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEB"+ "PxAk/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxAn/9k="+)++def test_browser_url_normalization_matches_address_bar_hosts():assert normalize_url("localhost:3000") == "http://localhost:3000/"assert normalize_url("127.0.0.1:8000/path") == "http://127.0.0.1:8000/path"@@ -365,7 +380,8 @@ def test_browser_canvas_startup_waits_for_raw_viewport_settle():encoding="utf-8")- assert "const CANVAS_VIEWPORT_SETTLE_MS = 260;" in js+ assert "const CANVAS_VIEWPORT_SETTLE_MS = 520;" in js+ assert "const SURFACE_VIEWPORT_STABLE_FRAMES = 4;" in jsassert "surfaceViewportMeasurement()" in jsassert "rawWidth" in jsassert "rawHeight" in js@@ -454,6 +470,9 @@ def test_browser_viewer_uses_cdp_screencast_transport():assert "viewport_width: initialViewport?.width" in browser_storeassert "viewport_height: initialViewport?.height" in browser_storeassert "this.frameState = data.state || null" not in browser_store+ assert "function loadFrameDimensions(src)" in browser_store+ assert "frameMatchesViewport(dimensions = null, viewport = null)" in browser_store+ assert "requestViewportSyncAfterRejectedFrame()" in browser_storeassert "overflow: hidden;" in main_htmlassert "object-fit: fill;" in main_htmlassert "image-rendering: auto;" in main_html@@ -504,19 +523,7 @@ def test_browser_runtime_and_content_helper_expose_annotation_target():@pytest.mark.anyioasync def test_browser_screencast_acknowledges_and_drops_stale_frames():- first_image = (- "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL"- "DBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/"- "2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy"- "MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAKAAoDASIAAhEBAxEB/8QAFQAB"- "AAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhADE"- "AAAAKf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAA"- "AAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECA"- "QE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Aqf/xAAUEAEAAAAAAA"- "AAAAAAAAAAAAAA/9oACAEBAAE/ISf/2gAMAwEAAgADAAAAEP/EABQRAQAAAAAAAAA"- "AAAAAAAAAAP/aAAgBAwEBPxAk/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEB"- "PxAk/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxAn/9k="- )+ first_image = SMALL_JPEG_10X10class FakeSession:def __init__(self):@@ -572,6 +579,43 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():assert session.detached is True+async def test_browser_screencast_keeps_rejecting_wrong_viewport_frames():+ class FakeSession:+ def __init__(self):+ self.handlers = {}+ self.sent = []++ def on(self, event, handler):+ self.handlers[event] = handler++ async def send(self, method, params=None):+ self.sent.append((method, params or {}))++ async def detach(self):+ pass++ session = FakeSession()+ screencast = _BrowserScreencast(+ stream_id="stream",+ browser_id=7,+ session=session,+ mime="image/jpeg",+ )++ await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 1118, "height": 662})+ for session_id in range(1, 14):+ session.handlers["Page.screencastFrame"](+ {"data": SMALL_JPEG_10X10, "metadata": {}, "sessionId": session_id}+ )+ await asyncio.sleep(0)++ assert await screencast.pop_frame() is None+ assert ("Page.screencastFrameAck", {"sessionId": 13}) in session.sent++ await screencast.stop()++def test_browser_docker_installs_full_chromium_to_persistent_cache():script = (PROJECT_ROOT / "docker" / "run" / "fs" / "ins" / "install_playwright.sh"