Agent Zero AI framework
Stabilize browser canvas screencast lifecycle
Alessandro9ec070793df5
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.pyindex 8430f7e7a..a48a67b07 100644--- a/plugins/_browser/api/ws_browser.py+++ b/plugins/_browser/api/ws_browser.py@@ -14,7 +14,6 @@ from plugins._browser.helpers.runtime import get_runtimeFRAME_IDLE_POLL_SECONDS = 0.05FRAME_RETRY_DELAY_SECONDS = 0.5FRAME_STATE_REFRESH_SECONDS = 0.75-FRAME_FALLBACK_SCREENSHOT_SECONDS = 1.2SCREENCAST_QUALITY = 92@@ -195,6 +194,7 @@ class WsBrowser(WsHandler):browser_id,int(data.get("width") or 0),int(data.get("height") or 0),+ restart_screencast=bool(data.get("restart_stream")),)elif input_type == "wheel":result = await runtime.call(@@ -300,12 +300,11 @@ class WsBrowser(WsHandler):"image": "","mime": "","state": state,+ "frame_source": "state",},)last_state_refresh = 0.0- last_image_at = 0.0- stream_started_at = time.monotonic()while True:now = time.monotonic()if now - last_state_refresh >= FRAME_STATE_REFRESH_SECONDS:@@ -322,26 +321,6 @@ class WsBrowser(WsHandler):except KeyError:breakif frame is None:- if (- now - stream_started_at >= FRAME_FALLBACK_SCREENSHOT_SECONDS- and (- last_image_at == 0.0- or now - last_image_at >= FRAME_FALLBACK_SCREENSHOT_SECONDS- )- ):- with contextlib.suppress(Exception):- frame = await runtime.call(- "screenshot",- active_id,- quality=SCREENCAST_QUALITY,- )- frame["context_id"] = context_id- frame["viewer_id"] = viewer_id- frame["browser_id"] = active_id- frame["browsers"] = browsers- frame["state"] = frame.get("state") or state- await self.emit_to(sid, "browser_viewer_frame", frame)- last_image_at = time.monotonic()await asyncio.sleep(FRAME_IDLE_POLL_SECONDS)continue@@ -350,9 +329,8 @@ class WsBrowser(WsHandler):frame["browser_id"] = active_idframe["browsers"] = browsersframe["state"] = state+ frame["frame_source"] = "screencast"await self.emit_to(sid, "browser_viewer_frame", frame)- if frame.get("image"):- last_image_at = time.monotonic()except asyncio.CancelledError:raiseexcept Exception:diff --git a/plugins/_browser/extensions/webui/chat-input-bottom-actions-start/browser-button.html b/plugins/_browser/extensions/webui/chat-input-bottom-actions-start/browser-button.htmlindex f35e8e491..fbcb6491b 100644--- a/plugins/_browser/extensions/webui/chat-input-bottom-actions-start/browser-button.html+++ b/plugins/_browser/extensions/webui/chat-input-bottom-actions-start/browser-button.html@@ -1,11 +1,11 @@<buttontype="button"class="text-button browser-chat-action"- title="Show or hide Browser"- aria-label="Show or hide Browser"+ title="Open Browser"+ aria-label="Open Browser"data-bs-placement="top"data-bs-trigger="hover"- @click="$store.rightCanvas ? $store.rightCanvas.toggle('browser') : (window.toggleModal ? window.toggleModal('/plugins/_browser/webui/main.html') : window.openModal('/plugins/_browser/webui/main.html'))"+ @click="$store.rightCanvas ? $store.rightCanvas.open('browser') : (window.ensureModalOpen ? window.ensureModalOpen('/plugins/_browser/webui/main.html') : (window.openModal && window.openModal('/plugins/_browser/webui/main.html')))"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" width="14" height="14" aria-hidden="true"><rect x="3" y="4" width="18" height="16" rx="2"></rect>diff --git a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.jsindex 78c608945..9d427ea23 100644--- a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js+++ b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js@@ -4,6 +4,8 @@ import {} from "/components/messages/action-buttons/simple-action-buttons.js";import { store as stepDetailStore } from "/components/modals/process-step-detail/step-detail-store.js";import { store as speechStore } from "/components/chat/speech/speech-store.js";+import { store as rightCanvasStore } from "/components/canvas/right-canvas-store.js";+import { store as browserStore } from "/plugins/_browser/webui/browser-store.js";import {buildDetailPayload,cleanStepTitle,@@ -21,25 +23,24 @@ export default async function registerBrowserToolHandler(extData) {}async function openBrowserCanvas(payload = {}) {- const canvas = globalThis.Alpine?.store?.("rightCanvas")- || (await import("/components/canvas/right-canvas-store.js")).store;- if (canvas) {- await canvas.open("browser", payload);+ if (rightCanvasStore?.open) {+ await rightCanvasStore.open("browser", payload);return;}+if (window.ensureModalOpen) {await window.ensureModalOpen(BROWSER_MODAL);return;}- await window.openModal?.(BROWSER_MODAL);+ if (window.openModal) {+ await window.openModal(BROWSER_MODAL);+ }}async function browserAllowsToolAutofocus() {try {- const browser = globalThis.Alpine?.store?.("browserPage")- || (await import("/plugins/_browser/webui/browser-store.js")).store;- if (browser?.allowsToolAutofocus) {- return await browser.allowsToolAutofocus();+ if (browserStore.allowsToolAutofocus) {+ return await browserStore.allowsToolAutofocus();}} catch (error) {console.warn("Browser autofocus setting could not be checked", error);diff --git a/plugins/_browser/extensions/webui/right_canvas_register_surfaces/register-browser.js b/plugins/_browser/extensions/webui/right_canvas_register_surfaces/register-browser.jsindex 8b513bcda..4d9a86eb2 100644--- a/plugins/_browser/extensions/webui/right_canvas_register_surfaces/register-browser.js+++ b/plugins/_browser/extensions/webui/right_canvas_register_surfaces/register-browser.js@@ -1,3 +1,5 @@+import { store as browserStore } from "/plugins/_browser/webui/browser-store.js";+function waitForElement(selector, timeoutMs = 3000) {const found = document.querySelector(selector);if (found) return Promise.resolve(found);@@ -17,6 +19,56 @@ function waitForElement(selector, timeoutMs = 3000) {});}+function nextAnimationFrame() {+ return new Promise((resolve) => {+ const schedule = globalThis.requestAnimationFrame || ((callback) => globalThis.setTimeout(callback, 16));+ schedule(() => resolve());+ });+}++function isVisibleCanvasPanel(panel) {+ if (!panel?.isConnected) return false;+ const surface = panel.closest(".browser-canvas-surface");+ const stage = panel.querySelector(".browser-stage") || panel;+ const surfaceStyle = surface ? globalThis.getComputedStyle?.(surface) : null;+ const panelStyle = globalThis.getComputedStyle?.(panel);+ if (surfaceStyle?.display === "none" || surfaceStyle?.visibility === "hidden") return false;+ if (panelStyle?.display === "none" || panelStyle?.visibility === "hidden") return false;+ const rect = stage.getBoundingClientRect?.();+ return Boolean(rect && Math.round(rect.width || 0) >= 80 && Math.round(rect.height || 0) >= 80);+}++async function waitForVisibleCanvasPanel(selector, timeoutMs = 3000) {+ const deadline = Date.now() + timeoutMs;+ let stableKey = "";+ let stableCount = 0;++ while (Date.now() <= deadline) {+ const panel = document.querySelector(selector);+ const visible = isVisibleCanvasPanel(panel);+ if (visible) {+ const stage = panel.querySelector(".browser-stage") || panel;+ const rect = stage.getBoundingClientRect();+ const key = `${Math.round(rect.width || 0)}x${Math.round(rect.height || 0)}`;+ if (key === stableKey) {+ stableCount += 1;+ if (stableCount >= 2) {+ return panel;+ }+ } else {+ stableKey = key;+ stableCount = 0;+ }+ } else {+ stableKey = "";+ stableCount = 0;+ }+ await nextAnimationFrame();+ }++ return document.querySelector(selector);+}+export default async function registerBrowserSurface(canvas) {canvas.registerSurface({id: "browser",@@ -24,9 +76,19 @@ export default async function registerBrowserSurface(canvas) {icon: "language",order: 10,modalPath: "/plugins/_browser/webui/main.html",+ beginDockHandoff() {+ browserStore.beginSurfaceHandoff?.();+ },+ finishDockHandoff() {+ browserStore.finishSurfaceHandoff?.();+ },+ cancelDockHandoff() {+ browserStore.cancelSurfaceHandoff?.();+ },async open(payload = {}) {- const panel = await waitForElement('[data-surface-id="browser"] .browser-panel');- const browser = globalThis.Alpine?.store?.("browserPage");+ await waitForElement('[data-surface-id="browser"] .browser-panel');+ const panel = await waitForVisibleCanvasPanel('[data-surface-id="browser"] .browser-panel');+ const browser = browserStore;if (panel && browser?.onOpen) {await browser.onOpen(panel, {mode: "canvas",@@ -35,8 +97,7 @@ export default async function registerBrowserSurface(canvas) {}},async close() {- const browser = globalThis.Alpine?.store?.("browserPage");- await browser?.cleanup?.();+ await browserStore.cleanup?.();},});}diff --git a/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js b/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.jsindex 58f18e875..55a2d9a8a 100644--- a/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js+++ b/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js@@ -1,3 +1,6 @@+import { store as rightCanvasStore } from "/components/canvas/right-canvas-store.js";+import { store as browserStore } from "/plugins/_browser/webui/browser-store.js";+const AUTO_OPEN_WINDOW_MS = 10 * 60 * 1000;const BROWSER_MODAL = "/plugins/_browser/webui/main.html";const autoOpenedBrowsers = new Set();@@ -135,10 +138,8 @@ function hasOpened(key, persistedKey) {}async function openBrowserCanvas(payload = {}) {- const canvas = globalThis.Alpine?.store?.("rightCanvas")- || (await import("/components/canvas/right-canvas-store.js")).store;- if (canvas) {- await canvas.open("browser", payload);+ if (rightCanvasStore?.open) {+ await rightCanvasStore.open("browser", payload);return;}@@ -146,15 +147,15 @@ async function openBrowserCanvas(payload = {}) {await window.ensureModalOpen(BROWSER_MODAL);return;}- await window.openModal?.(BROWSER_MODAL);+ if (window.openModal) {+ await window.openModal(BROWSER_MODAL);+ }}async function browserAllowsToolAutofocus() {try {- const browser = globalThis.Alpine?.store?.("browserPage")- || (await import("/plugins/_browser/webui/browser-store.js")).store;- if (browser?.allowsToolAutofocus) {- return await browser.allowsToolAutofocus();+ if (browserStore.allowsToolAutofocus) {+ return await browserStore.allowsToolAutofocus();}} catch (error) {console.warn("Browser autofocus setting could not be checked", error);diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.pyindex 3111b3068..7c7115435 100644--- a/plugins/_browser/helpers/runtime.py+++ b/plugins/_browser/helpers/runtime.py@@ -36,6 +36,7 @@ CHROME_SINGLETON_FILES = ("SingletonLock", "SingletonCookie", "SingletonSocket")SCREENCAST_MAX_WIDTH = 4096SCREENCAST_MAX_HEIGHT = 4096VIEWPORT_SIZE_TOLERANCE = 4+VIEWPORT_REMOUNT_PAUSE_SECONDS = 0.05_SPECIAL_SCHEME_RE = re.compile(r"^(?:about|blob|data|file|mailto|tel):", re.I)_URL_SCHEME_RE = re.compile(r"^[a-z][a-z\d+\-.]*://", re.I)@@ -51,6 +52,18 @@ _TYPED_HOST_RE = re.compile(_SAFE_CONTEXT_RE = re.compile(r"[^a-zA-Z0-9_.-]+")+def _nudged_viewport(viewport: dict[str, int]) -> dict[str, int]:+ width = int(viewport["width"])+ height = int(viewport["height"])+ if width < 4096:+ return {"width": width + 1, "height": height}+ if width > 320:+ return {"width": width - 1, "height": height}+ if height < 4096:+ return {"width": width, "height": height + 1}+ return {"width": width, "height": height - 1}++def normalize_url(value: str) -> str:raw = str(value or "").strip()if not raw:@@ -126,6 +139,29 @@ class _BrowserScreencast:self._expected_height = heightwith contextlib.suppress(Exception):await self.session.send("Page.enable")+ await self._apply_cdp_viewport_with_remount({"width": width, "height": height})+ await self.session.send(+ "Page.startScreencast",+ {+ "format": "jpeg",+ "quality": max(20, min(95, int(quality))),+ "maxWidth": SCREENCAST_MAX_WIDTH,+ "maxHeight": SCREENCAST_MAX_HEIGHT,+ "everyNthFrame": max(1, int(every_nth_frame)),+ },+ )++ async def _apply_cdp_viewport_with_remount(self, viewport: dict[str, int]) -> None:+ await self._apply_cdp_viewport(viewport)+ await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)+ await self._apply_cdp_viewport(_nudged_viewport(viewport))+ await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)+ await self._apply_cdp_viewport(viewport)+ await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)++ async def _apply_cdp_viewport(self, viewport: dict[str, int]) -> None:+ width = max(320, min(4096, int(viewport.get("width") or DEFAULT_VIEWPORT["width"])))+ height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"])))await self.session.send("Emulation.setDeviceMetricsOverride",{@@ -144,16 +180,6 @@ class _BrowserScreencast:"height": height,},)- await self.session.send(- "Page.startScreencast",- {- "format": "jpeg",- "quality": max(20, min(95, int(quality))),- "maxWidth": SCREENCAST_MAX_WIDTH,- "maxHeight": SCREENCAST_MAX_HEIGHT,- "everyNthFrame": max(1, int(every_nth_frame)),- },- )async def next_frame(self, timeout: float = 1.0) -> dict[str, Any]:frame = await asyncio.wait_for(self.queue.get(), timeout=max(0.1, float(timeout)))@@ -197,13 +223,19 @@ class _BrowserScreencast:async def _handle_frame(self, params: dict[str, Any]) -> None:try:data = params.get("data") or ""- if data and self._frame_matches_viewport(data):+ if data:+ metadata = dict(params.get("metadata") or {})+ size = self._jpeg_size(data)+ if size:+ metadata["jpegWidth"], metadata["jpegHeight"] = size+ metadata["expectedWidth"] = self._expected_width+ metadata["expectedHeight"] = self._expected_heightself._queue_latest({"browser_id": self.browser_id,"mime": self.mime,"image": data,- "metadata": params.get("metadata") or {},+ "metadata": metadata,})finally:@@ -220,20 +252,6 @@ class _BrowserScreencast:with contextlib.suppress(asyncio.QueueFull):self.queue.put_nowait(frame)- def _frame_matches_viewport(self, data: str) -> bool:- if not self._expected_width or not self._expected_height:- return True- size = self._jpeg_size(data)- if not size:- return True- width, height = size- if (- abs(width - self._expected_width) <= VIEWPORT_SIZE_TOLERANCE- and abs(height - self._expected_height) <= VIEWPORT_SIZE_TOLERANCE- ):- return True- return False-@staticmethoddef _jpeg_size(data: str) -> tuple[int, int] | None:try:@@ -709,6 +727,7 @@ class _BrowserRuntimeCore:browser_id: int | str | None,width: int,height: int,+ restart_screencast: bool = False,) -> dict[str, Any]:await self.ensure_started()resolved_id = self._resolve_browser_id(browser_id)@@ -724,13 +743,33 @@ class _BrowserRuntimeCore:or abs(int(current_viewport.get("height") or 0) - viewport["height"])> VIEWPORT_SIZE_TOLERANCE)- if changed:- await page.set_viewport_size(viewport)+ should_remount_viewport = changed or restart_screencast+ if should_remount_viewport:await self._stop_screencasts_for_browser(resolved_id)+ if changed:+ await self._apply_viewport_with_remount(page, viewport)+ elif restart_screencast:+ await self._remount_viewport(page, viewport)+ if should_remount_viewport:await self._settle(page, short=True)self.last_interacted_browser_id = resolved_idreturn {"state": await self._state(resolved_id), "viewport": viewport}+ async def _apply_viewport_with_remount(self, page: Any, viewport: dict[str, int]) -> None:+ await page.set_viewport_size(viewport)+ await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)+ await self._remount_viewport(page, viewport)++ async def _remount_viewport(self, page: Any, viewport: dict[str, int]) -> None:+ nudged_viewport = self._nudged_viewport(viewport)+ await page.set_viewport_size(nudged_viewport)+ await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)+ await page.set_viewport_size(viewport)++ @staticmethod+ def _nudged_viewport(viewport: dict[str, int]) -> dict[str, int]:+ return _nudged_viewport(viewport)+async def mouse(self,browser_id: int | str | None,@@ -838,15 +877,19 @@ class _BrowserRuntimeCore:return {"action": action or {}, "state": await self._state(resolved_id)}async def _goto(self, page: Any, url: str) -> None:+ from playwright.async_api import Error as PlaywrightErrorfrom playwright.async_api import TimeoutError as PlaywrightTimeoutErrortry:await page.goto(url, wait_until="domcontentloaded", timeout=30000)except PlaywrightTimeoutError:PrintStyle.warning(f"Browser navigation timed out after DOM handoff: {url}")+ except PlaywrightError as exc:+ PrintStyle.warning(f"Browser navigation showed a native error page for {url}: {exc}")await self._settle(page)async def _settle(self, page: Any, short: bool = False) -> None:+ from playwright.async_api import Error as PlaywrightErrorfrom playwright.async_api import TimeoutError as PlaywrightTimeoutErrortry:@@ -854,7 +897,7 @@ class _BrowserRuntimeCore:"domcontentloaded",timeout=1000 if short else 5000,)- except PlaywrightTimeoutError:+ except (PlaywrightError, PlaywrightTimeoutError):passawait asyncio.sleep(0.1 if short else 0.35)diff --git a/plugins/_browser/webui/browser-panel.html b/plugins/_browser/webui/browser-panel.htmlindex 3ff50ef5d..aa1bdb0e3 100644--- a/plugins/_browser/webui/browser-panel.html+++ b/plugins/_browser/webui/browser-panel.html@@ -27,8 +27,9 @@<button type="button" class="browser-tab-close":title="'Close ' + $store.browserPage.browserTabLabel(browser)":aria-label="'Close ' + $store.browserPage.browserTabLabel(browser)"- :disabled="$store.browserPage.isBusy()"- @click.stop="$confirmClick($event, () => $store.browserPage.command('close', { browser_id: browser.id }))">+ :disabled="$store.browserPage.isClosingBrowser(browser.id)"+ @pointerdown.stop+ @click.stop="$store.browserPage.closeBrowser(browser.id)"><span class="material-symbols-outlined">close</span></button></div>@@ -349,8 +350,8 @@--browser-tab-hover-border: color-mix(in srgb, var(--color-border) 78%, transparent);--browser-control-size: 34px;--browser-address-height: 34px;- --browser-tab-height: 34px;- --browser-tab-close-size: 28px;+ --browser-tab-height: 36px;+ --browser-tab-close-size: 32px;--browser-control-radius: 0.55rem;box-sizing: border-box;display: flex;@@ -592,7 +593,7 @@}.browser-tab-close .material-symbols-outlined {- font-size: 0.82rem;+ font-size: 1rem;line-height: 1;}diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.jsindex bb12b161e..482fe4a5a 100644--- a/plugins/_browser/webui/browser-store.js+++ b/plugins/_browser/webui/browser-store.js@@ -1,8 +1,11 @@import { createStore } from "/js/AlpineStore.js";import { callJsonApi } from "/js/api.js";import { getNamespacedClient } from "/js/websocket.js";+import { getContext, setContext } from "/index.js";import { store as chatInputStore } from "/components/chat/input/input-store.js";import { store as pluginSettingsStore } from "/components/plugins/plugin-settings-store.js";+import { store as chatsStore } from "/components/sidebar/chats/chats-store.js";+import { store as rightCanvasStore } from "/components/canvas/right-canvas-store.js";const websocket = getNamespacedClient("/ws");websocket.addHandlers(["ws_webui"]);@@ -15,6 +18,7 @@ const VIEWPORT_SYNC_DEBOUNCE_MS = 220;const VIEWPORT_SYNC_SIZE_TOLERANCE = 4;const CANVAS_VIEWPORT_SETTLE_MS = 520;const SURFACE_VIEWPORT_STABLE_FRAMES = 4;+const SURFACE_VIEWPORT_MAX_WAIT_MS = 1200;const FRAME_REJECT_SYNC_COOLDOWN_MS = 600;const ANNOTATION_DRAG_THRESHOLD = 6;const ANNOTATION_MAX_COMMENTS = 24;@@ -126,7 +130,15 @@ const model = {_mode: "",_surfaceMounted: false,_surfaceSwitching: false,+ _surfaceHandoff: false,+ _surfaceHandoffTimer: null,_surfaceOpenedAt: 0,+ _surfaceOpenSequence: 0,+ _canvasSurfaceReadySequence: 0,+ _canvasFirstFrameAcceptedSequence: 0,+ _canvasFirstFrameNudgeSequence: 0,+ _openPromise: null,+ _openSignature: "",_connectSequence: 0,_viewerToken: "",_contextCreatePromise: null,@@ -146,6 +158,8 @@ const model = {browserInstallExpected: false,defaultHomepage: "about:blank",autofocusActivePage: true,+ _commandInFlightCount: 0,+ _closingBrowserIds: {},_configLoadedAt: 0,_configRefreshPromise: null,@@ -234,8 +248,7 @@ const model = {resolveContextId() {const urlContext = new URLSearchParams(globalThis.location?.search || "").get("ctxid");- const selectedChat = globalThis.Alpine?.store?.("chats")?.selected;- return globalThis.getContext?.() || urlContext || selectedChat || "";+ return getContext() || urlContext || chatsStore.selected || "";},async ensureContextId() {@@ -269,12 +282,8 @@ const model = {throw new Error(response?.error || "Could not create a chat for Browser.");}- if (typeof globalThis.setContext === "function") {- globalThis.setContext(contextId);- } else {- const chatsStore = globalThis.Alpine?.store?.("chats");- chatsStore?.setSelected?.(contextId);- }+ setContext(contextId);+ chatsStore.setSelected?.(contextId);return contextId;},@@ -451,10 +460,47 @@ const model = {},async onOpen(element = null, options = {}) {+ const requestedBrowserId = this.normalizeBrowserId(+ options.requestedBrowserId ?? options.browserId ?? options.browser_id,+ );+ const nextMode = options?.mode === "modal" ? "modal" : "canvas";+ if (nextMode === "canvas" && !this.isCanvasSurfaceVisible(element)) {+ return;+ }+ const openSignature = this.surfaceOpenSignature(element, nextMode, requestedBrowserId);+ if (this._openPromise && this._openSignature === openSignature) {+ return await this._openPromise;+ }+ const promise = this.openSurface(element, {+ ...options,+ requestedBrowserId,+ nextMode,+ });+ this._openPromise = promise;+ this._openSignature = openSignature;+ try {+ return await promise;+ } finally {+ if (this._openPromise === promise) {+ this._openPromise = null;+ this._openSignature = "";+ }+ }+ },++ async openSurface(element = null, options = {}) {this.loading = true;this.error = "";- const requestedBrowserId = this.normalizeBrowserId(options.browserId ?? options.browser_id);- const nextMode = options?.mode === "modal" ? "modal" : "canvas";+ const requestedBrowserId = this.normalizeBrowserId(+ options.requestedBrowserId ?? options.browserId ?? options.browser_id,+ );+ const nextMode = options?.nextMode || (options?.mode === "modal" ? "modal" : "canvas");+ if (nextMode === "canvas" && !this.isCanvasSurfaceVisible(element)) {+ this.loading = false;+ return;+ }+ const surfaceSequence = this._surfaceOpenSequence + 1;+ this._surfaceOpenSequence = surfaceSequence;this.prepareSurfaceOpen(nextMode, requestedBrowserId);if (nextMode === "modal") {this.setupFloatingModal(element);@@ -463,16 +509,95 @@ const model = {}try {await this.ensureContextId();+ if (!this.isCurrentSurfaceOpen(surfaceSequence)) return;await this.refreshStatus();- const viewport = await this.waitForSurfaceViewport();+ if (!this.isCurrentSurfaceOpen(surfaceSequence)) return;+ const viewport = await this.waitForSurfaceViewport({ sequence: surfaceSequence });+ if (!this.isCurrentSurfaceOpen(surfaceSequence)) return;+ if (nextMode === "canvas" && !viewport) return;this.resetRenderedFrameIfViewportChanged(viewport, requestedBrowserId);await this.connectViewer({ browserId: requestedBrowserId, initialViewport: viewport });- await this.syncViewportAfterSurfaceOpen();+ if (!this.isCurrentSurfaceOpen(surfaceSequence)) return;+ await this.syncViewportAfterSurfaceOpen(surfaceSequence);} catch (error) {- this.error = error instanceof Error ? error.message : String(error);+ if (this.isCurrentSurfaceOpen(surfaceSequence)) {+ this.error = error instanceof Error ? error.message : String(error);+ }} finally {- this.loading = false;+ if (this.isCurrentSurfaceOpen(surfaceSequence)) {+ this.loading = false;+ if (this._mode === "canvas") {+ this._canvasSurfaceReadySequence = surfaceSequence;+ this.scheduleCanvasWidthNudgeAfterFirstFrame();+ }+ }+ }+ },++ surfaceOpenSignature(element = null, mode = "", browserId = null) {+ const root = element || globalThis.document?.querySelector(".browser-panel");+ if (root && !root.__browserSurfaceOpenId) {+ root.__browserSurfaceOpenId = makeViewerToken();}+ return [+ mode || "",+ this.normalizeBrowserId(browserId) || "",+ root?.__browserSurfaceOpenId || "",+ ].join(":");+ },++ isCurrentSurfaceOpen(sequence) {+ return this._surfaceMounted && sequence === this._surfaceOpenSequence;+ },++ beginSurfaceHandoff() {+ if (this._surfaceHandoffTimer) {+ globalThis.clearTimeout(this._surfaceHandoffTimer);+ }+ this._surfaceHandoff = true;+ this._surfaceHandoffTimer = globalThis.setTimeout(() => {+ this._surfaceHandoff = false;+ this._surfaceHandoffTimer = null;+ }, 3000);+ },++ finishSurfaceHandoff() {+ if (this._surfaceHandoffTimer) {+ globalThis.clearTimeout(this._surfaceHandoffTimer);+ this._surfaceHandoffTimer = null;+ }+ this._surfaceHandoff = false;+ },++ cancelSurfaceHandoff() {+ if (this._surfaceHandoffTimer) {+ globalThis.clearTimeout(this._surfaceHandoffTimer);+ this._surfaceHandoffTimer = null;+ }+ this._surfaceHandoff = false;+ },++ releaseSurfaceBindings() {+ this._floatingCleanup?.();+ this._floatingCleanup = null;+ this._stageResizeObserver?.disconnect?.();+ this._stageResizeObserver = null;+ this._stageElement = null;+ },++ isCanvasSurfaceVisible(element = null) {+ const root = element+ || globalThis.document?.querySelector?.(".browser-canvas-surface .browser-panel")+ || globalThis.document?.querySelector?.(".browser-panel");+ if (!root?.isConnected) return false;+ const surface = root.closest?.(".browser-canvas-surface");+ const stage = root.querySelector?.(".browser-stage") || root;+ const surfaceStyle = surface ? globalThis.getComputedStyle?.(surface) : null;+ const rootStyle = globalThis.getComputedStyle?.(root);+ if (surfaceStyle?.display === "none" || surfaceStyle?.visibility === "hidden") return false;+ if (rootStyle?.display === "none" || rootStyle?.visibility === "hidden") return false;+ const rect = stage.getBoundingClientRect?.();+ return Boolean(rect && Math.round(rect.width || 0) >= 80 && Math.round(rect.height || 0) >= 80);},prepareSurfaceOpen(nextMode, requestedBrowserId = null) {@@ -517,11 +642,16 @@ const model = {this.switchingBrowserId = targetBrowserId;},- async waitForSurfaceViewport() {+ async waitForSurfaceViewport(options = {}) {+ const sequence = Number(options.sequence || 0);+ const startedAt = Date.now();let lastKey = "";let stableCount = 0;- for (let index = 0; index < 24; index += 1) {+ while (Date.now() - startedAt <= SURFACE_VIEWPORT_MAX_WAIT_MS) {await nextAnimationFrame();+ if (sequence && !this.isCurrentSurfaceOpen(sequence)) {+ return null;+ }const viewport = this.surfaceViewportMeasurement();if (!viewport) continue;const key = `${viewport.rawWidth}x${viewport.rawHeight}`;@@ -538,16 +668,29 @@ const model = {lastKey = key;}}- return this.currentViewportSize();+ const fallbackViewport = this.currentViewportSize();+ return fallbackViewport;},- async syncViewportAfterSurfaceOpen() {+ async syncViewportAfterSurfaceOpen(sequence = this._surfaceOpenSequence) {if (!this.connected || !this.activeBrowserId) return;- await this.waitForSurfaceViewport();- await this.syncViewport(true);+ await this.waitForSurfaceViewport({ sequence });+ if (!this.isCurrentSurfaceOpen(sequence)) {+ return;+ }+ await this.syncViewport(true, { restartStream: this._mode === "canvas" });if (this._mode !== "canvas") return;- globalThis.setTimeout?.(() => this.queueViewportSync(true), 240);- globalThis.setTimeout?.(() => this.queueViewportSync(true), 420);+ this.scheduleViewportSyncForSurface(sequence, 240);+ this.scheduleViewportSyncForSurface(sequence, 520);+ },++ scheduleViewportSyncForSurface(sequence, delayMs = 0) {+ globalThis.setTimeout?.(() => {+ if (!this.isCurrentSurfaceOpen(sequence) || this._mode !== "canvas") {+ return;+ }+ this.queueViewportSync(true);+ }, delayMs);},async connectViewer(options = {}) {@@ -715,11 +858,12 @@ const model = {this._pendingFrameSrc = "";this._pendingFrameOptions = null;const sequence = this._frameRenderSequence + 1;+ const surfaceSequence = this._surfaceOpenSequence;this._frameRenderSequence = sequence;- void this.renderDecodedFrame(frameSrc, options, sequence);+ void this.renderDecodedFrame(frameSrc, options, sequence, surfaceSequence);},- async renderDecodedFrame(frameSrc, options = {}, sequence = 0) {+ async renderDecodedFrame(frameSrc, options = {}, sequence = 0, surfaceSequence = this._surfaceOpenSequence) {if (!frameSrc) {if (sequence === this._frameRenderSequence) {this.frameSrc = "";@@ -727,7 +871,9 @@ const model = {return;}const dimensions = await loadFrameDimensions(frameSrc);- if (sequence !== this._frameRenderSequence) return;+ if (sequence !== this._frameRenderSequence || surfaceSequence !== this._surfaceOpenSequence) {+ return;+ }const viewport = this.currentViewportSize() || this._lastViewport;if (!this.frameMatchesViewport(dimensions, viewport)) {this.requestViewportSyncAfterRejectedFrame();@@ -737,6 +883,56 @@ const model = {this._lastFrameDimensions = dimensions;this._lastFrameAt = Date.now();options?.onAccepted?.();+ this._canvasFirstFrameAcceptedSequence = surfaceSequence;+ this.scheduleCanvasWidthNudgeAfterFirstFrame();+ },++ scheduleCanvasWidthNudgeAfterFirstFrame() {+ const surfaceSequence = this._surfaceOpenSequence;+ if (this._mode !== "canvas" || !this.isCurrentSurfaceOpen(surfaceSequence) || !this.activeBrowserId) {+ return;+ }+ if (this._canvasFirstFrameNudgeSequence === surfaceSequence) {+ return;+ }+ if (+ this._canvasSurfaceReadySequence !== surfaceSequence+ || this._canvasFirstFrameAcceptedSequence !== surfaceSequence+ ) {+ return;+ }+ this._canvasFirstFrameNudgeSequence = surfaceSequence;++ void (async () => {+ await nextAnimationFrame();+ await nextAnimationFrame();+ if (!this.isCurrentSurfaceOpen(surfaceSequence) || this._mode !== "canvas") {+ return;+ }+ this.forceRightCanvasWidthNudge();+ })();+ },++ forceRightCanvasWidthNudge() {+ const canvas = rightCanvasStore;+ if (!canvas || canvas.isMobileMode || !canvas.isOpen || canvas.activeSurfaceId !== "browser") {+ return;+ }++ const currentWidth = Number(canvas.width || 0);+ if (!Number.isFinite(currentWidth) || currentWidth <= 0) {+ return;+ }+ const maxWidth = Number(canvas.maxWidth?.() || currentWidth);+ const minWidth = Number(canvas.minWidth || 420);+ const direction = currentWidth < maxWidth ? 1 : -1;+ const nudgedWidth = currentWidth + direction;+ if (nudgedWidth < minWidth || nudgedWidth > maxWidth || nudgedWidth === currentWidth) {+ return;+ }++ canvas.setWidth?.(nudgedWidth, { persist: false });+ this.queueViewportSync(true);},frameMatchesViewport(dimensions = null, viewport = null) {@@ -749,7 +945,9 @@ const model = {requestViewportSyncAfterRejectedFrame() {const now = Date.now();- if (now - this._lastFrameRejectSyncAt < FRAME_REJECT_SYNC_COOLDOWN_MS) return;+ if (now - this._lastFrameRejectSyncAt < FRAME_REJECT_SYNC_COOLDOWN_MS) {+ return;+ }this._lastFrameRejectSyncAt = now;this.queueViewportSync(true);},@@ -776,11 +974,22 @@ const model = {this._frameRenderSequence += 1;},+ beginCommand() {+ this._commandInFlightCount += 1;+ this.commandInFlight = true;+ },++ finishCommand() {+ this._commandInFlightCount = Math.max(0, this._commandInFlightCount - 1);+ this.commandInFlight = this._commandInFlightCount > 0;+ },+async command(command, extra = {}) {this.error = "";this.annotationError = "";- this.commandInFlight = true;+ this.beginCommand();const previousActiveBrowserId = this.activeBrowserId;+ const commandName = String(command || "").toLowerCase();try {await this.ensureContextId();const response = await websocket.request(@@ -813,19 +1022,33 @@ const model = {this.address = result.state?.currentUrl || result.currentUrl;}this.applySnapshot(data.snapshot);- if (["navigate", "back", "forward", "reload", "close"].includes(String(command || "").toLowerCase())) {+ if (["navigate", "back", "forward", "reload", "close"].includes(commandName)) {this.clearAnnotationsForBrowser(previousActiveBrowserId);this.cancelAnnotationDraft();}const activeChanged = this.activeBrowserId && this.activeBrowserId !== previousActiveBrowserId;- if ((command === "open" || command === "close" || activeChanged) && this.contextId && this.activeBrowserId) {+ if ((commandName === "open" || commandName === "close" || activeChanged) && this.contextId && this.activeBrowserId) {await this.connectViewer({ browserId: this.activeBrowserId });- }- } catch (error) {+ } else if (["navigate", "back", "forward", "reload"].includes(commandName)) {+ await this.restartCanvasStreamAfterPageChange();+ }+ } catch (error) {this.error = error instanceof Error ? error.message : String(error);} finally {- this.commandInFlight = false;+ this.finishCommand();+ }+ },++ async restartCanvasStreamAfterPageChange() {+ const surfaceSequence = this._surfaceOpenSequence;+ if (this._mode !== "canvas" || !this.isCurrentSurfaceOpen(surfaceSequence) || !this.activeBrowserId) {+ return;}+ await this.waitForSurfaceViewport({ sequence: surfaceSequence });+ if (this._mode !== "canvas" || !this.isCurrentSurfaceOpen(surfaceSequence) || !this.activeBrowserId) {+ return;+ }+ await this.syncViewport(true, { restartStream: true });},async go() {@@ -886,6 +1109,35 @@ const model = {await this.command("open");},+ isClosingBrowser(id) {+ const browserId = this.normalizeBrowserId(id);+ return Boolean(browserId && this._closingBrowserIds[String(browserId)]);+ },++ markBrowserClosing(id, closing = true) {+ const browserId = this.normalizeBrowserId(id);+ if (!browserId) return;+ const key = String(browserId);+ const nextClosing = { ...this._closingBrowserIds };+ if (closing) {+ nextClosing[key] = true;+ } else {+ delete nextClosing[key];+ }+ this._closingBrowserIds = nextClosing;+ },++ async closeBrowser(id) {+ const browserId = this.normalizeBrowserId(id);+ if (!browserId || this.isClosingBrowser(browserId)) return;+ this.markBrowserClosing(browserId, true);+ try {+ await this.command("close", { browser_id: browserId });+ } finally {+ this.markBrowserClosing(browserId, false);+ }+ },+isActiveBrowser(browser) {return Number(browser?.id) === Number(this.activeBrowserId);},@@ -1479,37 +1731,46 @@ const model = {}, force ? 0 : VIEWPORT_SYNC_DEBOUNCE_MS);},- async syncViewport(force = false) {- if (!this.contextId || !this.activeBrowserId) return;- const viewport = this.currentViewportSize();- if (!viewport) return;- const key = `${this.activeBrowserId}:${viewport.width}x${viewport.height}`;- if (- this._lastViewportKey === key- || (- !force- && this._lastViewport- && this.sameBrowserId(this._lastViewport.browserId, this.activeBrowserId)- && Math.abs(this._lastViewport.width - viewport.width) <= VIEWPORT_SYNC_SIZE_TOLERANCE- && Math.abs(this._lastViewport.height - viewport.height) <= VIEWPORT_SYNC_SIZE_TOLERANCE- )- ) return;- try {- await websocket.emit("browser_viewer_input", {- context_id: this.contextId,- browser_id: this.activeBrowserId,- viewer_id: this._viewerToken,- input_type: "viewport",- width: viewport.width,- height: viewport.height,- });- this._lastViewportKey = key;- this._lastViewport = {- browserId: this.activeBrowserId,- width: viewport.width,- height: viewport.height,- };- } catch (error) {+ async syncViewport(force = false, options = {}) {+ const restartStream = Boolean(options.restartStream);+ if (!this.contextId || !this.activeBrowserId) {+ return;+ }+ const viewport = this.currentViewportSize();+ if (!viewport) {+ return;+ }+ const key = `${this.activeBrowserId}:${viewport.width}x${viewport.height}`;+ if (+ (!restartStream && this._lastViewportKey === key)+ || (+ !force+ && !restartStream+ && this._lastViewport+ && this.sameBrowserId(this._lastViewport.browserId, this.activeBrowserId)+ && Math.abs(this._lastViewport.width - viewport.width) <= VIEWPORT_SYNC_SIZE_TOLERANCE+ && Math.abs(this._lastViewport.height - viewport.height) <= VIEWPORT_SYNC_SIZE_TOLERANCE+ )+ ) {+ return;+ }+ try {+ await websocket.emit("browser_viewer_input", {+ context_id: this.contextId,+ browser_id: this.activeBrowserId,+ viewer_id: this._viewerToken,+ input_type: "viewport",+ width: viewport.width,+ height: viewport.height,+ restart_stream: restartStream,+ });+ this._lastViewportKey = key;+ this._lastViewport = {+ browserId: this.activeBrowserId,+ width: viewport.width,+ height: viewport.height,+ };+ } catch (error) {this._lastViewportKey = "";this._lastViewport = null;console.warn("Browser viewport sync failed", error);@@ -1545,7 +1806,7 @@ const model = {await websocket.emit("browser_viewer_input", payload);},- async sendWheel(event) {+ async sendWheel(event) {if (!this.activeBrowserId || !event) return;const image = event.currentTarget?.querySelector?.(".browser-frame") || event.target?.closest?.(".browser-frame");const pointer = this.pointerCoordinatesFor(event, image);@@ -1560,12 +1821,12 @@ const model = {delta_x: Number(event.deltaX || 0),delta_y: Number(event.deltaY || 0),};- try {- await websocket.emit("browser_viewer_input", payload);- } catch (error) {- this.error = error instanceof Error ? error.message : String(error);- }- },+ try {+ await websocket.emit("browser_viewer_input", payload);+ } catch (error) {+ this.error = error instanceof Error ? error.message : String(error);+ }+ },async sendKey(event) {if (this.annotating) return;@@ -1585,12 +1846,22 @@ const model = {},async cleanup() {+ if (this._surfaceHandoff) {+ this.releaseSurfaceBindings();+ this.extensionMenuOpen = false;+ return;+ }+ this._surfaceOpenSequence += 1;+ this._openPromise = null;+ this._openSignature = "";this._connectSequence += 1;this._viewerToken = "";this.switchingBrowserId = null;this._surfaceMounted = false;this._surfaceSwitching = false;this.commandInFlight = false;+ this._commandInFlightCount = 0;+ this._closingBrowserIds = {};this.annotating = false;this.annotationBusy = false;this.annotationError = "";@@ -1606,11 +1877,7 @@ const model = {this._frameOff = null;this._stateOff = null;this.resetRenderedFrame();- this._floatingCleanup?.();- this._floatingCleanup = null;- this._stageResizeObserver?.disconnect?.();- this._stageResizeObserver = null;- this._stageElement = null;+ this.releaseSurfaceBindings();if (this._viewportSyncTimer) {globalThis.clearTimeout(this._viewportSyncTimer);this._viewportSyncTimer = null;@@ -1681,11 +1948,17 @@ const model = {resizeObserver.observe(inner);if (stage) {this._stageResizeObserver?.disconnect?.();- this._stageResizeObserver = new ResizeObserver(() => this.queueViewportSync());+ this._stageResizeObserver = new ResizeObserver(() => {+ this.queueViewportSync();+ });this._stageResizeObserver.observe(stage);}}- globalThis.requestAnimationFrame(() => this.queueViewportSync(true));+ const surfaceSequence = this._surfaceOpenSequence;+ globalThis.requestAnimationFrame(() => {+ if (!this.isCurrentSurfaceOpen(surfaceSequence)) return;+ this.queueViewportSync(true);+ });const onPointerMove = (event) => {if (!drag) return;@@ -1735,6 +2008,7 @@ const model = {},setupCanvasSurface(element = null) {+ const surfaceSequence = this._surfaceOpenSequence;this._floatingCleanup?.();this._floatingCleanup = null;this._stageResizeObserver?.disconnect?.();@@ -1742,10 +2016,15 @@ const model = {const stage = root?.querySelector?.(".browser-stage");this._stageElement = stage || null;if (stage && globalThis.ResizeObserver) {- this._stageResizeObserver = new ResizeObserver(() => this.queueViewportSync());+ this._stageResizeObserver = new ResizeObserver(() => {+ this.queueViewportSync();+ });this._stageResizeObserver.observe(stage);}- globalThis.requestAnimationFrame?.(() => this.queueViewportSync(true));+ globalThis.requestAnimationFrame?.(() => {+ if (!this.isCurrentSurfaceOpen(surfaceSequence) || this._mode !== "canvas") return;+ this.queueViewportSync(true);+ });},get activeTitle() {diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.pyindex f4bc81b17..14ba37a9c 100644--- a/tests/test_browser_agent_regressions.py+++ b/tests/test_browser_agent_regressions.py@@ -370,7 +370,9 @@ def test_browser_viewer_creates_chat_when_no_context_is_selected():assert "async ensureContextId()" in jsassert "async createChatContextForBrowser()" in jsassert 'callJsonApi("/chat_create"' in js- assert "globalThis.setContext(contextId)" in js+ assert 'import { getContext, setContext } from "/index.js";' in js+ assert "setContext(contextId)" in js+ assert "chatsStore.setSelected?.(contextId)" in jsassert "await this.ensureContextId();" in jsassert "No active chat context is selected." not in js@@ -382,11 +384,206 @@ def test_browser_canvas_startup_waits_for_raw_viewport_settle():assert "const CANVAS_VIEWPORT_SETTLE_MS = 520;" in jsassert "const SURFACE_VIEWPORT_STABLE_FRAMES = 4;" in js+ assert "const SURFACE_VIEWPORT_MAX_WAIT_MS = 1200;" in jsassert "surfaceViewportMeasurement()" in jsassert "rawWidth" in jsassert "rawHeight" in jsassert "const key = `${viewport.rawWidth}x${viewport.rawHeight}`;" in jsassert "Date.now() - this._surfaceOpenedAt >= CANVAS_VIEWPORT_SETTLE_MS" in js+ assert "while (Date.now() - startedAt <= SURFACE_VIEWPORT_MAX_WAIT_MS)" in js+ assert "this._openPromise && this._openSignature === openSignature" in js+ assert "isCurrentSurfaceOpen(surfaceSequence)" in js+ assert "isCanvasSurfaceVisible(element)" in js+ assert "scheduleViewportSyncForSurface" in js+++def test_browser_canvas_surface_open_waits_for_visible_panel():+ js = (+ PROJECT_ROOT+ / "plugins"+ / "_browser"+ / "extensions"+ / "webui"+ / "right_canvas_register_surfaces"+ / "register-browser.js"+ ).read_text(encoding="utf-8")++ assert "waitForVisibleCanvasPanel" in js+ assert "isVisibleCanvasPanel(panel)" in js+ assert 'panel.closest(".browser-canvas-surface")' in js+ assert 'panel.querySelector(".browser-stage")' in js+ assert "getBoundingClientRect" in js+ assert "stableCount >= 2" in js+ assert "forceCanvasWidthNudgeAfterBrowserMount" not in js+++def test_browser_canvas_nudges_width_after_first_accepted_frame():+ js = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(+ encoding="utf-8"+ )++ assert "_canvasSurfaceReadySequence" in js+ assert "_canvasFirstFrameAcceptedSequence" in js+ assert "_canvasFirstFrameNudgeSequence" in js+ assert "scheduleCanvasWidthNudgeAfterFirstFrame()" in js+ assert "this._canvasSurfaceReadySequence = surfaceSequence;" in js+ assert "const surfaceSequence = this._surfaceOpenSequence;" in js+ assert "this._canvasFirstFrameAcceptedSequence = surfaceSequence;" in js+ assert "forceRightCanvasWidthNudge()" in js+ assert "await nextAnimationFrame();" in js+ assert "globalThis.Alpine" not in js+ assert 'import { store as rightCanvasStore } from "/components/canvas/right-canvas-store.js";' in js+ assert "const canvas = rightCanvasStore;" in js+ assert 'canvas.activeSurfaceId !== "browser"' in js+ assert "canvas.setWidth?.(nudgedWidth, { persist: false })" in js+ assert "this.queueViewportSync(true)" in js+ frame_assignment_index = js.index("this.frameSrc = frameSrc;")+ frame_nudge_schedule_index = js.index(+ "this.scheduleCanvasWidthNudgeAfterFirstFrame();",+ frame_assignment_index,+ )+ assert frame_assignment_index < frame_nudge_schedule_index+++def test_browser_canvas_restarts_stream_after_page_navigation():+ js = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(+ encoding="utf-8"+ )++ assert "async restartCanvasStreamAfterPageChange()" in js+ assert '["navigate", "back", "forward", "reload"].includes(commandName)' in js+ assert "await this.restartCanvasStreamAfterPageChange();" in js+ assert "await this.waitForSurfaceViewport({ sequence: surfaceSequence });" in js+ assert "await this.syncViewport(true, { restartStream: true });" in js+ reconnect_index = js.index("await this.connectViewer({ browserId: this.activeBrowserId });")+ restart_index = js.index("await this.restartCanvasStreamAfterPageChange();")+ assert reconnect_index < restart_index+++def test_browser_entry_points_prefer_canvas_and_modal_dock_handoff():+ button_html = (+ PROJECT_ROOT+ / "plugins"+ / "_browser"+ / "extensions"+ / "webui"+ / "chat-input-bottom-actions-start"+ / "browser-button.html"+ ).read_text(encoding="utf-8")+ tool_handler = (+ PROJECT_ROOT+ / "plugins"+ / "_browser"+ / "extensions"+ / "webui"+ / "get_tool_message_handler"+ / "browser-tool-handler.js"+ ).read_text(encoding="utf-8")+ after_loop_handler = (+ PROJECT_ROOT+ / "plugins"+ / "_browser"+ / "extensions"+ / "webui"+ / "set_messages_after_loop"+ / "auto-open-browser-results.js"+ ).read_text(encoding="utf-8")+ register_js = (+ PROJECT_ROOT+ / "plugins"+ / "_browser"+ / "extensions"+ / "webui"+ / "right_canvas_register_surfaces"+ / "register-browser.js"+ ).read_text(encoding="utf-8")+ browser_store = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(+ encoding="utf-8"+ )+ canvas_store = (PROJECT_ROOT / "webui" / "components" / "canvas" / "right-canvas-store.js").read_text(+ encoding="utf-8"+ )+ modals_js = (PROJECT_ROOT / "webui" / "js" / "modals.js").read_text(encoding="utf-8")++ assert "Open Browser" in button_html+ assert "$store.rightCanvas ? $store.rightCanvas.open('browser')" in button_html+ assert "window.ensureModalOpen ? window.ensureModalOpen('/plugins/_browser/webui/main.html')" in button_html+ assert "$store.rightCanvas.toggle('browser')" not in button_html+ assert 'defaultOpenMode: "modal"' not in register_js+ assert "beginDockHandoff()" in register_js+ assert "beginSurfaceHandoff" in register_js+ assert "finishDockHandoff()" in register_js+ assert "cancelDockHandoff()" in register_js+ assert "defaultOpenMode" not in canvas_store+ assert "await surface.beginDockHandoff?.(payload)" in canvas_store+ assert "await this.closeDockSourceModal(payload, modalPath)" in canvas_store+ assert "delete openPayload.closeSourceModal" in canvas_store+ assert "await surface.finishDockHandoff?.({ ...openPayload, opened })" in canvas_store+ assert "await surface.cancelDockHandoff?.(payload)" in canvas_store+ assert "async closeDockSourceModal" in canvas_store++ assert "sourceModalPath: modal.path" in modals_js+ assert "closeSourceModal: async () =>" in modals_js+ assert "const closed = await closeModal(modal.path)" in modals_js+ assert "const fallbackClosed = await closeModal()" in modals_js+ assert "button.disabled = true" in modals_js++ assert "beginSurfaceHandoff()" in browser_store+ assert "finishSurfaceHandoff()" in browser_store+ assert "cancelSurfaceHandoff()" in browser_store+ assert "releaseSurfaceBindings()" in browser_store+ assert "this.releaseSurfaceBindings();" in browser_store++ for js in (tool_handler, after_loop_handler):+ assert "async function openBrowserCanvas" in js+ assert "openBrowserModal" not in js+ assert js.index('await rightCanvasStore.open("browser", payload);') < js.index("if (window.ensureModalOpen)")++ assert "function autoOpenBrowserCanvas" in tool_handler++ for js in (tool_handler, after_loop_handler, register_js, browser_store, modals_js):+ assert "globalThis.Alpine" not in js+ assert "Alpine?.store" not in js+ assert "Alpine.store" not in js+++def test_browser_canvas_uses_plain_panel_without_debug_probe():+ panel_html = (+ PROJECT_ROOT+ / "plugins"+ / "_browser"+ / "extensions"+ / "webui"+ / "right-canvas-panels"+ / "browser-panel.html"+ ).read_text(encoding="utf-8")+ browser_store = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(+ encoding="utf-8"+ )+ register_js = (+ PROJECT_ROOT+ / "plugins"+ / "_browser"+ / "extensions"+ / "webui"+ / "right_canvas_register_surfaces"+ / "register-browser.js"+ ).read_text(encoding="utf-8")+ canvas_store = (PROJECT_ROOT / "webui" / "components" / "canvas" / "right-canvas-store.js").read_text(+ encoding="utf-8"+ )+ modals_js = (PROJECT_ROOT / "webui" / "js" / "modals.js").read_text(encoding="utf-8")++ assert 'x-component path="/plugins/_browser/webui/browser-panel.html" mode="canvas"' in panel_html+ assert "browserCanvasRemountKey" not in panel_html+ assert "remountBrowserCanvasOnce" not in panel_html+ assert "data-remount-slot" not in panel_html++ for js in (browser_store, register_js, canvas_store, modals_js):+ assert "__a0BrowserDebug" not in js+ assert "browserDebug" not in js+ assert "emitBrowserDebug" not in js+ assert ".debug(" not in jsdef test_browser_ui_spinners_have_browser_local_animation():@@ -436,6 +633,29 @@ def test_browser_viewer_uses_tabs_for_session_switching():assert "Using ${this.mainModelSummary}" in browser_store+def test_browser_tabs_close_without_confirmation_or_busy_lock():+ main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(+ encoding="utf-8"+ )+ browser_store = (+ PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js"+ ).read_text(encoding="utf-8")+ close_start = main_html.index('class="browser-tab-close"')+ close_end = main_html.index("</button>", close_start)+ close_markup = main_html[close_start:close_end]++ assert '@click.stop="$store.browserPage.closeBrowser(browser.id)"' in main_html+ assert "@pointerdown.stop" in close_markup+ assert "$confirmClick" not in main_html+ assert "isBusy()" not in close_markup+ assert ':disabled="$store.browserPage.isClosingBrowser(browser.id)"' in close_markup+ assert "--browser-tab-close-size: 32px;" in main_html+ assert "async closeBrowser(id)" in browser_store+ assert "isClosingBrowser(id)" in browser_store+ assert "_closingBrowserIds" in browser_store+ assert "_commandInFlightCount" in browser_store++def test_browser_viewer_uses_cdp_screencast_transport():ws_browser = (PROJECT_ROOT / "plugins" / "_browser" / "api" / "ws_browser.py").read_text(encoding="utf-8"@@ -469,15 +689,44 @@ def test_browser_viewer_uses_cdp_screencast_transport():assert "requestAnimationFrame" in browser_storeassert "viewport_width: initialViewport?.width" in browser_storeassert "viewport_height: initialViewport?.height" in browser_store+ assert "restart_stream: restartStream" in browser_store+ assert 'restart_screencast=bool(data.get("restart_stream"))' in ws_browser+ assert "restart_screencast: bool = False" in runtime+ assert "should_remount_viewport = changed or restart_screencast" in runtime+ assert "VIEWPORT_REMOUNT_PAUSE_SECONDS = 0.05" in runtime+ assert "await self._apply_cdp_viewport_with_remount" in runtime+ assert "await self._apply_viewport_with_remount(page, viewport)" in runtime+ assert "await self._remount_viewport(page, viewport)" in runtime+ assert "await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)" in runtime+ assert "def _nudged_viewport(viewport: dict[str, int])" in runtime+ assert 'await this.syncViewport(true, { restartStream: this._mode === "canvas" });' in browser_storeassert "this.frameState = data.state || null" not in browser_storeassert "function loadFrameDimensions(src)" in browser_storeassert "frameMatchesViewport(dimensions = null, viewport = null)" in browser_storeassert "requestViewportSyncAfterRejectedFrame()" in browser_store+ assert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browser+ assert '"frame_source": "state"' in ws_browser+ assert '"frame_source"] = "screencast"' in ws_browser+ assert "fallback_screenshot" not in ws_browser+ assert "canvas_wheel_screenshot" not in ws_browser+ assert "surface_mode: this._mode" not in browser_storeassert "overflow: hidden;" in main_htmlassert "object-fit: fill;" in main_htmlassert "image-rendering: auto;" in main_html+def test_browser_navigation_errors_stay_inside_native_browser_page():+ runtime = (+ PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"+ ).read_text(encoding="utf-8")++ assert "Error as PlaywrightError" in runtime+ assert "except PlaywrightError as exc:" in runtime+ assert "Browser navigation showed a native error page" in runtime+ assert "await self._settle(page)" in runtime+ assert "except (PlaywrightError, PlaywrightTimeoutError):" in runtime++def test_browser_annotate_mode_ui_and_prompt_hooks():panel_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html"@@ -562,14 +811,57 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():assert frame["browser_id"] == 7assert frame["image"] == "second"assert frame["metadata"]["deviceWidth"] == 200- assert ("Emulation.setDeviceMetricsOverride", {- "width": 1118,- "height": 662,- "deviceScaleFactor": 1,- "mobile": False,- "dontSetVisibleSize": True,- }) in session.sent- assert ("Emulation.setVisibleSize", {"width": 1118, "height": 662}) in session.sent+ assert frame["metadata"]["expectedWidth"] == 1118+ assert frame["metadata"]["expectedHeight"] == 662+ metrics_calls = [+ params+ for method, params in session.sent+ if method == "Emulation.setDeviceMetricsOverride"+ ]+ visible_calls = [+ params+ for method, params in session.sent+ if method == "Emulation.setVisibleSize"+ ]+ assert metrics_calls[:3] == [+ {+ "width": 1118,+ "height": 662,+ "deviceScaleFactor": 1,+ "mobile": False,+ "dontSetVisibleSize": True,+ },+ {+ "width": 1119,+ "height": 662,+ "deviceScaleFactor": 1,+ "mobile": False,+ "dontSetVisibleSize": True,+ },+ {+ "width": 1118,+ "height": 662,+ "deviceScaleFactor": 1,+ "mobile": False,+ "dontSetVisibleSize": True,+ },+ ]+ assert visible_calls[:3] == [+ {"width": 1118, "height": 662},+ {"width": 1119, "height": 662},+ {"width": 1118, "height": 662},+ ]+ start_index = next(+ index+ for index, (method, _params) in enumerate(session.sent)+ if method == "Page.startScreencast"+ )+ cdp_viewport_indices = [+ index+ for index, (method, _params) in enumerate(session.sent)+ if method.startswith("Emulation.")+ ]+ assert max(cdp_viewport_indices) < start_indexassert ("Page.screencastFrameAck", {"sessionId": 1}) in session.sentassert ("Page.screencastFrameAck", {"sessionId": 2}) in session.sent@@ -580,7 +872,7 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():@pytest.mark.anyio-async def test_browser_screencast_keeps_rejecting_wrong_viewport_frames():+async def test_browser_screencast_passes_wrong_viewport_frames_to_frontend_validator():class FakeSession:def __init__(self):self.handlers = {}@@ -610,7 +902,14 @@ async def test_browser_screencast_keeps_rejecting_wrong_viewport_frames():)await asyncio.sleep(0)- assert await screencast.pop_frame() is None+ frame = await screencast.pop_frame()++ assert frame is not None+ assert frame["image"] == SMALL_JPEG_10X10+ assert frame["metadata"]["jpegWidth"] == 10+ assert frame["metadata"]["jpegHeight"] == 10+ assert frame["metadata"]["expectedWidth"] == 1118+ assert frame["metadata"]["expectedHeight"] == 662assert ("Page.screencastFrameAck", {"sessionId": 13}) in session.sentawait screencast.stop()@@ -819,6 +1118,7 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):"input_type": "viewport","width": 1280,"height": 720,+ "restart_stream": True,},"sid-1",)@@ -827,7 +1127,96 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):"state": {"ok": True, "method": "set_viewport", "args": (7, 1280, 720)},"snapshot": None,}- assert calls == [("set_viewport", (7, 1280, 720), {})]+ assert calls == [+ ("set_viewport", (7, 1280, 720), {"restart_screencast": True})+ ]+++async def test_browser_runtime_remounts_same_viewport_when_restarting_screencast():+ viewport_calls = []+ stopped = []+ settled = []++ class FakePage:+ viewport_size = {"width": 1280, "height": 720}++ async def set_viewport_size(self, viewport):+ viewport_calls.append(dict(viewport))++ core = _BrowserRuntimeCore("ctx")+ core.context = object()+ core.pages[7] = browser_runtime_module.BrowserPage(id=7, page=FakePage())++ async def fake_stop_screencasts(browser_id):+ stopped.append(browser_id)++ async def fake_settle(page, short=False):+ settled.append(short)++ async def fake_state(browser_id):+ return {"id": browser_id}++ core._stop_screencasts_for_browser = fake_stop_screencasts+ core._settle = fake_settle+ core._state = fake_state++ result = await core.set_viewport(7, 1280, 720, restart_screencast=True)++ assert result == {+ "state": {"id": 7},+ "viewport": {"width": 1280, "height": 720},+ }+ assert viewport_calls == [+ {"width": 1281, "height": 720},+ {"width": 1280, "height": 720},+ ]+ assert stopped == [7]+ assert settled == [True]+++async def test_browser_runtime_remounts_initial_changed_viewport():+ calls = []+ stopped = []+ settled = []++ class FakePage:+ viewport_size = {"width": 1024, "height": 768}++ async def set_viewport_size(self, viewport):+ calls.append(("viewport", dict(viewport)))++ core = _BrowserRuntimeCore("ctx")+ core.context = object()+ core.pages[7] = browser_runtime_module.BrowserPage(id=7, page=FakePage())++ async def fake_stop_screencasts(browser_id):+ stopped.append(browser_id)++ async def fake_settle(page, short=False):+ settled.append(short)++ async def fake_state(browser_id):+ return {"id": browser_id}++ core._stop_screencasts_for_browser = fake_stop_screencasts+ core._settle = fake_settle+ core._state = fake_state++ result = await core.set_viewport(7, 672, 789)++ assert result == {+ "state": {"id": 7},+ "viewport": {"width": 672, "height": 789},+ }+ assert calls == [+ ("viewport", {"width": 672, "height": 789}),+ ("viewport", {"width": 673, "height": 789}),+ ("viewport", {"width": 672, "height": 789}),+ ]+ assert stopped == [7]+ assert settled == [True]@pytest.mark.anyiodiff --git a/webui/components/canvas/right-canvas-store.js b/webui/components/canvas/right-canvas-store.jsindex 49c1f819b..15b174a90 100644--- a/webui/components/canvas/right-canvas-store.js+++ b/webui/components/canvas/right-canvas-store.js@@ -98,7 +98,9 @@ const model = {async open(surfaceId = "", payload = {}) {const targetId = surfaceId || this.activeSurfaceId || this.panelSurfaces[0]?.id || "";const surface = this.getSurface(targetId);- if (!surface) return false;+ if (!surface) {+ return false;+ }if (typeof surface.canOpen === "function" && surface.canOpen(payload) === false) {return false;}@@ -140,12 +142,48 @@ const model = {async dockSurface(surfaceId, payload = {}) {const surface = this.getSurface(surfaceId);- if (!surface) return false;+ if (!surface) {+ return false;+ }const modalPath = payload.modalPath || surface.modalPath || "";- if (modalPath && globalThis.isModalOpen?.(modalPath)) {- await globalThis.closeModal?.(modalPath);+ let handoffStarted = false;+ try {+ await surface.beginDockHandoff?.(payload);+ handoffStarted = true;++ const closed = await this.closeDockSourceModal(payload, modalPath);+ if (closed === false) {+ await surface.cancelDockHandoff?.(payload);+ return false;+ }++ const openPayload = { ...payload, source: "modal" };+ delete openPayload.closeSourceModal;+ const opened = await this.open(surfaceId, openPayload);+ await surface.finishDockHandoff?.({ ...openPayload, opened });+ return opened;+ } catch (error) {+ if (handoffStarted) {+ await surface.cancelDockHandoff?.(payload);+ }+ console.error(`Canvas surface ${surfaceId} failed to dock`, error);+ return false;+ }+ },++ async closeDockSourceModal(payload = {}, modalPath = "") {+ if (typeof payload.closeSourceModal === "function") {+ return (await payload.closeSourceModal()) !== false;}- return await this.open(surfaceId, { ...payload, source: "modal" });++ const sourceModalPath = payload.sourceModalPath || modalPath;+ if (sourceModalPath && globalThis.isModalOpen?.(sourceModalPath)) {+ return (await globalThis.closeModal?.(sourceModalPath)) !== false;+ }+ if (modalPath && modalPath !== sourceModalPath && globalThis.isModalOpen?.(modalPath)) {+ return (await globalThis.closeModal?.(modalPath)) !== false;+ }+ return true;},async undockSurface(surfaceId = "", payload = {}) {diff --git a/webui/components/canvas/right-canvas.css b/webui/components/canvas/right-canvas.cssindex 67248a7c9..ed5767f19 100644--- a/webui/components/canvas/right-canvas.css+++ b/webui/components/canvas/right-canvas.css@@ -181,11 +181,9 @@ body.right-canvas-resizing {align-items: center;gap: 7px;min-width: 0;- height: 34px;padding: 0 10px;- border-radius: 7px 7px 0 0;background: transparent;- opacity: 0.7;+ opacity: 0.4;white-space: nowrap;transition: border-color 0.16s ease, background-color 0.16s ease, opacity 0.16s ease;}@@ -193,8 +191,6 @@ body.right-canvas-resizing {.right-canvas-tab:hover,.right-canvas-tab.is-active {opacity: 1;- border-color: var(--right-canvas-border);- background: color-mix(in srgb, var(--color-panel) 72%, transparent);}.right-canvas-tab .material-symbols-outlined {diff --git a/webui/js/modals.js b/webui/js/modals.jsindex 8b4e944dc..a7a2b5b36 100644--- a/webui/js/modals.js+++ b/webui/js/modals.js@@ -1,6 +1,7 @@// Import the component loader and page utilitiesimport { importComponent } from "/js/components.js";import { callJsExtensions } from "/js/extensions.js";+import { store as rightCanvasStore } from "/components/canvas/right-canvas-store.js";// Modal functionalityconst modalStack = [];@@ -193,12 +194,26 @@ function configureModalDockButton(modal, doc) {button.setAttribute("aria-label", metadata.title);button.innerHTML = `<span class="material-symbols-outlined" aria-hidden="true">${metadata.icon}</span>`;button.addEventListener("click", async () => {- const canvas = globalThis.Alpine?.store?.("rightCanvas")- || (await import("/components/canvas/right-canvas-store.js")).store;- await canvas?.dockSurface?.(metadata.surfaceId, {- modalPath: metadata.modalPath,- source: "modal",- });+ if (button.disabled) return;+ button.disabled = true;+ try {+ await rightCanvasStore.dockSurface?.(metadata.surfaceId, {+ modalPath: metadata.modalPath,+ sourceModalPath: modal.path,+ source: "modal",+ closeSourceModal: async () => {+ const closed = await closeModal(modal.path);+ if (closed === false) return false;+ if (document.contains(modal.element)) {+ const fallbackClosed = await closeModal();+ if (fallbackClosed === false) return false;+ }+ return !document.contains(modal.element);+ },+ });+ } finally {+ if (document.contains(button)) button.disabled = false;+ }});modal.close?.insertAdjacentElement("beforebegin", button);