Skip to content

Agent Zero AI framework

Stabilize browser canvas screencast lifecycle

Alessandro9ec070793df5
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.py
index 8430f7e7a..a48a67b07 100644
--- a/plugins/_browser/api/ws_browser.py
+++ b/plugins/_browser/api/ws_browser.py
@@ -14,7 +14,6 @@ from plugins._browser.helpers.runtime import get_runtime
FRAME_IDLE_POLL_SECONDS = 0.05
FRAME_RETRY_DELAY_SECONDS = 0.5
FRAME_STATE_REFRESH_SECONDS = 0.75
-FRAME_FALLBACK_SCREENSHOT_SECONDS = 1.2
SCREENCAST_QUALITY = 92
@@ -195,6 +194,7 @@ class WsBrowser(WsHandler):
browser_id,
int(data.get("width") or 0),
int(data.get("height") or 0),
+ restart_screencast=bool(data.get("restart_stream")),
)
elif input_type == "wheel":
result = await runtime.call(
@@ -300,12 +300,11 @@ class WsBrowser(WsHandler):
"image": "",
"mime": "",
"state": state,
+ "frame_source": "state",
},
)
last_state_refresh = 0.0
- last_image_at = 0.0
- stream_started_at = time.monotonic()
while True:
now = time.monotonic()
if now - last_state_refresh >= FRAME_STATE_REFRESH_SECONDS:
@@ -322,26 +321,6 @@ class WsBrowser(WsHandler):
except KeyError:
break
if frame is None:
- if (
- now - stream_started_at >= FRAME_FALLBACK_SCREENSHOT_SECONDS
- and (
- last_image_at == 0.0
- or now - last_image_at >= FRAME_FALLBACK_SCREENSHOT_SECONDS
- )
- ):
- with contextlib.suppress(Exception):
- frame = await runtime.call(
- "screenshot",
- active_id,
- quality=SCREENCAST_QUALITY,
- )
- frame["context_id"] = context_id
- frame["viewer_id"] = viewer_id
- frame["browser_id"] = active_id
- frame["browsers"] = browsers
- frame["state"] = frame.get("state") or state
- await self.emit_to(sid, "browser_viewer_frame", frame)
- last_image_at = time.monotonic()
await asyncio.sleep(FRAME_IDLE_POLL_SECONDS)
continue
@@ -350,9 +329,8 @@ class WsBrowser(WsHandler):
frame["browser_id"] = active_id
frame["browsers"] = browsers
frame["state"] = state
+ frame["frame_source"] = "screencast"
await self.emit_to(sid, "browser_viewer_frame", frame)
- if frame.get("image"):
- last_image_at = time.monotonic()
except asyncio.CancelledError:
raise
except Exception:
diff --git a/plugins/_browser/extensions/webui/chat-input-bottom-actions-start/browser-button.html b/plugins/_browser/extensions/webui/chat-input-bottom-actions-start/browser-button.html
index f35e8e491..fbcb6491b 100644
--- a/plugins/_browser/extensions/webui/chat-input-bottom-actions-start/browser-button.html
+++ b/plugins/_browser/extensions/webui/chat-input-bottom-actions-start/browser-button.html
@@ -1,11 +1,11 @@
<button
type="button"
class="text-button browser-chat-action"
- title="Show or hide Browser"
- aria-label="Show or hide Browser"
+ title="Open Browser"
+ aria-label="Open Browser"
data-bs-placement="top"
data-bs-trigger="hover"
- @click="$store.rightCanvas ? $store.rightCanvas.toggle('browser') : (window.toggleModal ? window.toggleModal('/plugins/_browser/webui/main.html') : window.openModal('/plugins/_browser/webui/main.html'))"
+ @click="$store.rightCanvas ? $store.rightCanvas.open('browser') : (window.ensureModalOpen ? window.ensureModalOpen('/plugins/_browser/webui/main.html') : (window.openModal && window.openModal('/plugins/_browser/webui/main.html')))"
>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" width="14" height="14" aria-hidden="true">
<rect x="3" y="4" width="18" height="16" rx="2"></rect>
diff --git a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
index 78c608945..9d427ea23 100644
--- a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
+++ b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
@@ -4,6 +4,8 @@ import {
} from "/components/messages/action-buttons/simple-action-buttons.js";
import { store as stepDetailStore } from "/components/modals/process-step-detail/step-detail-store.js";
import { store as speechStore } from "/components/chat/speech/speech-store.js";
+import { store as rightCanvasStore } from "/components/canvas/right-canvas-store.js";
+import { store as browserStore } from "/plugins/_browser/webui/browser-store.js";
import {
buildDetailPayload,
cleanStepTitle,
@@ -21,25 +23,24 @@ export default async function registerBrowserToolHandler(extData) {
}
async function openBrowserCanvas(payload = {}) {
- const canvas = globalThis.Alpine?.store?.("rightCanvas")
- || (await import("/components/canvas/right-canvas-store.js")).store;
- if (canvas) {
- await canvas.open("browser", payload);
+ if (rightCanvasStore?.open) {
+ await rightCanvasStore.open("browser", payload);
return;
}
+
if (window.ensureModalOpen) {
await window.ensureModalOpen(BROWSER_MODAL);
return;
}
- await window.openModal?.(BROWSER_MODAL);
+ if (window.openModal) {
+ await window.openModal(BROWSER_MODAL);
+ }
}
async function browserAllowsToolAutofocus() {
try {
- const browser = globalThis.Alpine?.store?.("browserPage")
- || (await import("/plugins/_browser/webui/browser-store.js")).store;
- if (browser?.allowsToolAutofocus) {
- return await browser.allowsToolAutofocus();
+ if (browserStore.allowsToolAutofocus) {
+ return await browserStore.allowsToolAutofocus();
}
} catch (error) {
console.warn("Browser autofocus setting could not be checked", error);
diff --git a/plugins/_browser/extensions/webui/right_canvas_register_surfaces/register-browser.js b/plugins/_browser/extensions/webui/right_canvas_register_surfaces/register-browser.js
index 8b513bcda..4d9a86eb2 100644
--- a/plugins/_browser/extensions/webui/right_canvas_register_surfaces/register-browser.js
+++ b/plugins/_browser/extensions/webui/right_canvas_register_surfaces/register-browser.js
@@ -1,3 +1,5 @@
+import { store as browserStore } from "/plugins/_browser/webui/browser-store.js";
+
function waitForElement(selector, timeoutMs = 3000) {
const found = document.querySelector(selector);
if (found) return Promise.resolve(found);
@@ -17,6 +19,56 @@ function waitForElement(selector, timeoutMs = 3000) {
});
}
+function nextAnimationFrame() {
+ return new Promise((resolve) => {
+ const schedule = globalThis.requestAnimationFrame || ((callback) => globalThis.setTimeout(callback, 16));
+ schedule(() => resolve());
+ });
+}
+
+function isVisibleCanvasPanel(panel) {
+ if (!panel?.isConnected) return false;
+ const surface = panel.closest(".browser-canvas-surface");
+ const stage = panel.querySelector(".browser-stage") || panel;
+ const surfaceStyle = surface ? globalThis.getComputedStyle?.(surface) : null;
+ const panelStyle = globalThis.getComputedStyle?.(panel);
+ if (surfaceStyle?.display === "none" || surfaceStyle?.visibility === "hidden") return false;
+ if (panelStyle?.display === "none" || panelStyle?.visibility === "hidden") return false;
+ const rect = stage.getBoundingClientRect?.();
+ return Boolean(rect && Math.round(rect.width || 0) >= 80 && Math.round(rect.height || 0) >= 80);
+}
+
+async function waitForVisibleCanvasPanel(selector, timeoutMs = 3000) {
+ const deadline = Date.now() + timeoutMs;
+ let stableKey = "";
+ let stableCount = 0;
+
+ while (Date.now() <= deadline) {
+ const panel = document.querySelector(selector);
+ const visible = isVisibleCanvasPanel(panel);
+ if (visible) {
+ const stage = panel.querySelector(".browser-stage") || panel;
+ const rect = stage.getBoundingClientRect();
+ const key = `${Math.round(rect.width || 0)}x${Math.round(rect.height || 0)}`;
+ if (key === stableKey) {
+ stableCount += 1;
+ if (stableCount >= 2) {
+ return panel;
+ }
+ } else {
+ stableKey = key;
+ stableCount = 0;
+ }
+ } else {
+ stableKey = "";
+ stableCount = 0;
+ }
+ await nextAnimationFrame();
+ }
+
+ return document.querySelector(selector);
+}
+
export default async function registerBrowserSurface(canvas) {
canvas.registerSurface({
id: "browser",
@@ -24,9 +76,19 @@ export default async function registerBrowserSurface(canvas) {
icon: "language",
order: 10,
modalPath: "/plugins/_browser/webui/main.html",
+ beginDockHandoff() {
+ browserStore.beginSurfaceHandoff?.();
+ },
+ finishDockHandoff() {
+ browserStore.finishSurfaceHandoff?.();
+ },
+ cancelDockHandoff() {
+ browserStore.cancelSurfaceHandoff?.();
+ },
async open(payload = {}) {
- const panel = await waitForElement('[data-surface-id="browser"] .browser-panel');
- const browser = globalThis.Alpine?.store?.("browserPage");
+ await waitForElement('[data-surface-id="browser"] .browser-panel');
+ const panel = await waitForVisibleCanvasPanel('[data-surface-id="browser"] .browser-panel');
+ const browser = browserStore;
if (panel && browser?.onOpen) {
await browser.onOpen(panel, {
mode: "canvas",
@@ -35,8 +97,7 @@ export default async function registerBrowserSurface(canvas) {
}
},
async close() {
- const browser = globalThis.Alpine?.store?.("browserPage");
- await browser?.cleanup?.();
+ await browserStore.cleanup?.();
},
});
}
diff --git a/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js b/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js
index 58f18e875..55a2d9a8a 100644
--- a/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js
+++ b/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js
@@ -1,3 +1,6 @@
+import { store as rightCanvasStore } from "/components/canvas/right-canvas-store.js";
+import { store as browserStore } from "/plugins/_browser/webui/browser-store.js";
+
const AUTO_OPEN_WINDOW_MS = 10 * 60 * 1000;
const BROWSER_MODAL = "/plugins/_browser/webui/main.html";
const autoOpenedBrowsers = new Set();
@@ -135,10 +138,8 @@ function hasOpened(key, persistedKey) {
}
async function openBrowserCanvas(payload = {}) {
- const canvas = globalThis.Alpine?.store?.("rightCanvas")
- || (await import("/components/canvas/right-canvas-store.js")).store;
- if (canvas) {
- await canvas.open("browser", payload);
+ if (rightCanvasStore?.open) {
+ await rightCanvasStore.open("browser", payload);
return;
}
@@ -146,15 +147,15 @@ async function openBrowserCanvas(payload = {}) {
await window.ensureModalOpen(BROWSER_MODAL);
return;
}
- await window.openModal?.(BROWSER_MODAL);
+ if (window.openModal) {
+ await window.openModal(BROWSER_MODAL);
+ }
}
async function browserAllowsToolAutofocus() {
try {
- const browser = globalThis.Alpine?.store?.("browserPage")
- || (await import("/plugins/_browser/webui/browser-store.js")).store;
- if (browser?.allowsToolAutofocus) {
- return await browser.allowsToolAutofocus();
+ if (browserStore.allowsToolAutofocus) {
+ return await browserStore.allowsToolAutofocus();
}
} catch (error) {
console.warn("Browser autofocus setting could not be checked", error);
diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py
index 3111b3068..7c7115435 100644
--- a/plugins/_browser/helpers/runtime.py
+++ b/plugins/_browser/helpers/runtime.py
@@ -36,6 +36,7 @@ CHROME_SINGLETON_FILES = ("SingletonLock", "SingletonCookie", "SingletonSocket")
SCREENCAST_MAX_WIDTH = 4096
SCREENCAST_MAX_HEIGHT = 4096
VIEWPORT_SIZE_TOLERANCE = 4
+VIEWPORT_REMOUNT_PAUSE_SECONDS = 0.05
_SPECIAL_SCHEME_RE = re.compile(r"^(?:about|blob|data|file|mailto|tel):", re.I)
_URL_SCHEME_RE = re.compile(r"^[a-z][a-z\d+\-.]*://", re.I)
@@ -51,6 +52,18 @@ _TYPED_HOST_RE = re.compile(
_SAFE_CONTEXT_RE = re.compile(r"[^a-zA-Z0-9_.-]+")
+def _nudged_viewport(viewport: dict[str, int]) -> dict[str, int]:
+ width = int(viewport["width"])
+ height = int(viewport["height"])
+ if width < 4096:
+ return {"width": width + 1, "height": height}
+ if width > 320:
+ return {"width": width - 1, "height": height}
+ if height < 4096:
+ return {"width": width, "height": height + 1}
+ return {"width": width, "height": height - 1}
+
+
def normalize_url(value: str) -> str:
raw = str(value or "").strip()
if not raw:
@@ -126,6 +139,29 @@ class _BrowserScreencast:
self._expected_height = height
with contextlib.suppress(Exception):
await self.session.send("Page.enable")
+ await self._apply_cdp_viewport_with_remount({"width": width, "height": height})
+ await self.session.send(
+ "Page.startScreencast",
+ {
+ "format": "jpeg",
+ "quality": max(20, min(95, int(quality))),
+ "maxWidth": SCREENCAST_MAX_WIDTH,
+ "maxHeight": SCREENCAST_MAX_HEIGHT,
+ "everyNthFrame": max(1, int(every_nth_frame)),
+ },
+ )
+
+ async def _apply_cdp_viewport_with_remount(self, viewport: dict[str, int]) -> None:
+ await self._apply_cdp_viewport(viewport)
+ await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)
+ await self._apply_cdp_viewport(_nudged_viewport(viewport))
+ await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)
+ await self._apply_cdp_viewport(viewport)
+ await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)
+
+ async def _apply_cdp_viewport(self, viewport: dict[str, int]) -> None:
+ width = max(320, min(4096, int(viewport.get("width") or DEFAULT_VIEWPORT["width"])))
+ height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"])))
await self.session.send(
"Emulation.setDeviceMetricsOverride",
{
@@ -144,16 +180,6 @@ class _BrowserScreencast:
"height": height,
},
)
- await self.session.send(
- "Page.startScreencast",
- {
- "format": "jpeg",
- "quality": max(20, min(95, int(quality))),
- "maxWidth": SCREENCAST_MAX_WIDTH,
- "maxHeight": SCREENCAST_MAX_HEIGHT,
- "everyNthFrame": max(1, int(every_nth_frame)),
- },
- )
async def next_frame(self, timeout: float = 1.0) -> dict[str, Any]:
frame = await asyncio.wait_for(self.queue.get(), timeout=max(0.1, float(timeout)))
@@ -197,13 +223,19 @@ class _BrowserScreencast:
async def _handle_frame(self, params: dict[str, Any]) -> None:
try:
data = params.get("data") or ""
- if data and self._frame_matches_viewport(data):
+ if data:
+ metadata = dict(params.get("metadata") or {})
+ size = self._jpeg_size(data)
+ if size:
+ metadata["jpegWidth"], metadata["jpegHeight"] = size
+ metadata["expectedWidth"] = self._expected_width
+ metadata["expectedHeight"] = self._expected_height
self._queue_latest(
{
"browser_id": self.browser_id,
"mime": self.mime,
"image": data,
- "metadata": params.get("metadata") or {},
+ "metadata": metadata,
}
)
finally:
@@ -220,20 +252,6 @@ class _BrowserScreencast:
with contextlib.suppress(asyncio.QueueFull):
self.queue.put_nowait(frame)
- def _frame_matches_viewport(self, data: str) -> bool:
- if not self._expected_width or not self._expected_height:
- return True
- size = self._jpeg_size(data)
- if not size:
- return True
- width, height = size
- if (
- abs(width - self._expected_width) <= VIEWPORT_SIZE_TOLERANCE
- and abs(height - self._expected_height) <= VIEWPORT_SIZE_TOLERANCE
- ):
- return True
- return False
-
@staticmethod
def _jpeg_size(data: str) -> tuple[int, int] | None:
try:
@@ -709,6 +727,7 @@ class _BrowserRuntimeCore:
browser_id: int | str | None,
width: int,
height: int,
+ restart_screencast: bool = False,
) -> dict[str, Any]:
await self.ensure_started()
resolved_id = self._resolve_browser_id(browser_id)
@@ -724,13 +743,33 @@ class _BrowserRuntimeCore:
or abs(int(current_viewport.get("height") or 0) - viewport["height"])
> VIEWPORT_SIZE_TOLERANCE
)
- if changed:
- await page.set_viewport_size(viewport)
+ should_remount_viewport = changed or restart_screencast
+ if should_remount_viewport:
await self._stop_screencasts_for_browser(resolved_id)
+ if changed:
+ await self._apply_viewport_with_remount(page, viewport)
+ elif restart_screencast:
+ await self._remount_viewport(page, viewport)
+ if should_remount_viewport:
await self._settle(page, short=True)
self.last_interacted_browser_id = resolved_id
return {"state": await self._state(resolved_id), "viewport": viewport}
+ async def _apply_viewport_with_remount(self, page: Any, viewport: dict[str, int]) -> None:
+ await page.set_viewport_size(viewport)
+ await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)
+ await self._remount_viewport(page, viewport)
+
+ async def _remount_viewport(self, page: Any, viewport: dict[str, int]) -> None:
+ nudged_viewport = self._nudged_viewport(viewport)
+ await page.set_viewport_size(nudged_viewport)
+ await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)
+ await page.set_viewport_size(viewport)
+
+ @staticmethod
+ def _nudged_viewport(viewport: dict[str, int]) -> dict[str, int]:
+ return _nudged_viewport(viewport)
+
async def mouse(
self,
browser_id: int | str | None,
@@ -838,15 +877,19 @@ class _BrowserRuntimeCore:
return {"action": action or {}, "state": await self._state(resolved_id)}
async def _goto(self, page: Any, url: str) -> None:
+ from playwright.async_api import Error as PlaywrightError
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
try:
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
except PlaywrightTimeoutError:
PrintStyle.warning(f"Browser navigation timed out after DOM handoff: {url}")
+ except PlaywrightError as exc:
+ PrintStyle.warning(f"Browser navigation showed a native error page for {url}: {exc}")
await self._settle(page)
async def _settle(self, page: Any, short: bool = False) -> None:
+ from playwright.async_api import Error as PlaywrightError
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
try:
@@ -854,7 +897,7 @@ class _BrowserRuntimeCore:
"domcontentloaded",
timeout=1000 if short else 5000,
)
- except PlaywrightTimeoutError:
+ except (PlaywrightError, PlaywrightTimeoutError):
pass
await asyncio.sleep(0.1 if short else 0.35)
diff --git a/plugins/_browser/webui/browser-panel.html b/plugins/_browser/webui/browser-panel.html
index 3ff50ef5d..aa1bdb0e3 100644
--- a/plugins/_browser/webui/browser-panel.html
+++ b/plugins/_browser/webui/browser-panel.html
@@ -27,8 +27,9 @@
<button type="button" class="browser-tab-close"
:title="'Close ' + $store.browserPage.browserTabLabel(browser)"
:aria-label="'Close ' + $store.browserPage.browserTabLabel(browser)"
- :disabled="$store.browserPage.isBusy()"
- @click.stop="$confirmClick($event, () => $store.browserPage.command('close', { browser_id: browser.id }))">
+ :disabled="$store.browserPage.isClosingBrowser(browser.id)"
+ @pointerdown.stop
+ @click.stop="$store.browserPage.closeBrowser(browser.id)">
<span class="material-symbols-outlined">close</span>
</button>
</div>
@@ -349,8 +350,8 @@
--browser-tab-hover-border: color-mix(in srgb, var(--color-border) 78%, transparent);
--browser-control-size: 34px;
--browser-address-height: 34px;
- --browser-tab-height: 34px;
- --browser-tab-close-size: 28px;
+ --browser-tab-height: 36px;
+ --browser-tab-close-size: 32px;
--browser-control-radius: 0.55rem;
box-sizing: border-box;
display: flex;
@@ -592,7 +593,7 @@
}
.browser-tab-close .material-symbols-outlined {
- font-size: 0.82rem;
+ font-size: 1rem;
line-height: 1;
}
diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js
index bb12b161e..482fe4a5a 100644
--- a/plugins/_browser/webui/browser-store.js
+++ b/plugins/_browser/webui/browser-store.js
@@ -1,8 +1,11 @@
import { createStore } from "/js/AlpineStore.js";
import { callJsonApi } from "/js/api.js";
import { getNamespacedClient } from "/js/websocket.js";
+import { getContext, setContext } from "/index.js";
import { store as chatInputStore } from "/components/chat/input/input-store.js";
import { store as pluginSettingsStore } from "/components/plugins/plugin-settings-store.js";
+import { store as chatsStore } from "/components/sidebar/chats/chats-store.js";
+import { store as rightCanvasStore } from "/components/canvas/right-canvas-store.js";
const websocket = getNamespacedClient("/ws");
websocket.addHandlers(["ws_webui"]);
@@ -15,6 +18,7 @@ const VIEWPORT_SYNC_DEBOUNCE_MS = 220;
const VIEWPORT_SYNC_SIZE_TOLERANCE = 4;
const CANVAS_VIEWPORT_SETTLE_MS = 520;
const SURFACE_VIEWPORT_STABLE_FRAMES = 4;
+const SURFACE_VIEWPORT_MAX_WAIT_MS = 1200;
const FRAME_REJECT_SYNC_COOLDOWN_MS = 600;
const ANNOTATION_DRAG_THRESHOLD = 6;
const ANNOTATION_MAX_COMMENTS = 24;
@@ -126,7 +130,15 @@ const model = {
_mode: "",
_surfaceMounted: false,
_surfaceSwitching: false,
+ _surfaceHandoff: false,
+ _surfaceHandoffTimer: null,
_surfaceOpenedAt: 0,
+ _surfaceOpenSequence: 0,
+ _canvasSurfaceReadySequence: 0,
+ _canvasFirstFrameAcceptedSequence: 0,
+ _canvasFirstFrameNudgeSequence: 0,
+ _openPromise: null,
+ _openSignature: "",
_connectSequence: 0,
_viewerToken: "",
_contextCreatePromise: null,
@@ -146,6 +158,8 @@ const model = {
browserInstallExpected: false,
defaultHomepage: "about:blank",
autofocusActivePage: true,
+ _commandInFlightCount: 0,
+ _closingBrowserIds: {},
_configLoadedAt: 0,
_configRefreshPromise: null,
@@ -234,8 +248,7 @@ const model = {
resolveContextId() {
const urlContext = new URLSearchParams(globalThis.location?.search || "").get("ctxid");
- const selectedChat = globalThis.Alpine?.store?.("chats")?.selected;
- return globalThis.getContext?.() || urlContext || selectedChat || "";
+ return getContext() || urlContext || chatsStore.selected || "";
},
async ensureContextId() {
@@ -269,12 +282,8 @@ const model = {
throw new Error(response?.error || "Could not create a chat for Browser.");
}
- if (typeof globalThis.setContext === "function") {
- globalThis.setContext(contextId);
- } else {
- const chatsStore = globalThis.Alpine?.store?.("chats");
- chatsStore?.setSelected?.(contextId);
- }
+ setContext(contextId);
+ chatsStore.setSelected?.(contextId);
return contextId;
},
@@ -451,10 +460,47 @@ const model = {
},
async onOpen(element = null, options = {}) {
+ const requestedBrowserId = this.normalizeBrowserId(
+ options.requestedBrowserId ?? options.browserId ?? options.browser_id,
+ );
+ const nextMode = options?.mode === "modal" ? "modal" : "canvas";
+ if (nextMode === "canvas" && !this.isCanvasSurfaceVisible(element)) {
+ return;
+ }
+ const openSignature = this.surfaceOpenSignature(element, nextMode, requestedBrowserId);
+ if (this._openPromise && this._openSignature === openSignature) {
+ return await this._openPromise;
+ }
+ const promise = this.openSurface(element, {
+ ...options,
+ requestedBrowserId,
+ nextMode,
+ });
+ this._openPromise = promise;
+ this._openSignature = openSignature;
+ try {
+ return await promise;
+ } finally {
+ if (this._openPromise === promise) {
+ this._openPromise = null;
+ this._openSignature = "";
+ }
+ }
+ },
+
+ async openSurface(element = null, options = {}) {
this.loading = true;
this.error = "";
- const requestedBrowserId = this.normalizeBrowserId(options.browserId ?? options.browser_id);
- const nextMode = options?.mode === "modal" ? "modal" : "canvas";
+ const requestedBrowserId = this.normalizeBrowserId(
+ options.requestedBrowserId ?? options.browserId ?? options.browser_id,
+ );
+ const nextMode = options?.nextMode || (options?.mode === "modal" ? "modal" : "canvas");
+ if (nextMode === "canvas" && !this.isCanvasSurfaceVisible(element)) {
+ this.loading = false;
+ return;
+ }
+ const surfaceSequence = this._surfaceOpenSequence + 1;
+ this._surfaceOpenSequence = surfaceSequence;
this.prepareSurfaceOpen(nextMode, requestedBrowserId);
if (nextMode === "modal") {
this.setupFloatingModal(element);
@@ -463,16 +509,95 @@ const model = {
}
try {
await this.ensureContextId();
+ if (!this.isCurrentSurfaceOpen(surfaceSequence)) return;
await this.refreshStatus();
- const viewport = await this.waitForSurfaceViewport();
+ if (!this.isCurrentSurfaceOpen(surfaceSequence)) return;
+ const viewport = await this.waitForSurfaceViewport({ sequence: surfaceSequence });
+ if (!this.isCurrentSurfaceOpen(surfaceSequence)) return;
+ if (nextMode === "canvas" && !viewport) return;
this.resetRenderedFrameIfViewportChanged(viewport, requestedBrowserId);
await this.connectViewer({ browserId: requestedBrowserId, initialViewport: viewport });
- await this.syncViewportAfterSurfaceOpen();
+ if (!this.isCurrentSurfaceOpen(surfaceSequence)) return;
+ await this.syncViewportAfterSurfaceOpen(surfaceSequence);
} catch (error) {
- this.error = error instanceof Error ? error.message : String(error);
+ if (this.isCurrentSurfaceOpen(surfaceSequence)) {
+ this.error = error instanceof Error ? error.message : String(error);
+ }
} finally {
- this.loading = false;
+ if (this.isCurrentSurfaceOpen(surfaceSequence)) {
+ this.loading = false;
+ if (this._mode === "canvas") {
+ this._canvasSurfaceReadySequence = surfaceSequence;
+ this.scheduleCanvasWidthNudgeAfterFirstFrame();
+ }
+ }
+ }
+ },
+
+ surfaceOpenSignature(element = null, mode = "", browserId = null) {
+ const root = element || globalThis.document?.querySelector(".browser-panel");
+ if (root && !root.__browserSurfaceOpenId) {
+ root.__browserSurfaceOpenId = makeViewerToken();
}
+ return [
+ mode || "",
+ this.normalizeBrowserId(browserId) || "",
+ root?.__browserSurfaceOpenId || "",
+ ].join(":");
+ },
+
+ isCurrentSurfaceOpen(sequence) {
+ return this._surfaceMounted && sequence === this._surfaceOpenSequence;
+ },
+
+ beginSurfaceHandoff() {
+ if (this._surfaceHandoffTimer) {
+ globalThis.clearTimeout(this._surfaceHandoffTimer);
+ }
+ this._surfaceHandoff = true;
+ this._surfaceHandoffTimer = globalThis.setTimeout(() => {
+ this._surfaceHandoff = false;
+ this._surfaceHandoffTimer = null;
+ }, 3000);
+ },
+
+ finishSurfaceHandoff() {
+ if (this._surfaceHandoffTimer) {
+ globalThis.clearTimeout(this._surfaceHandoffTimer);
+ this._surfaceHandoffTimer = null;
+ }
+ this._surfaceHandoff = false;
+ },
+
+ cancelSurfaceHandoff() {
+ if (this._surfaceHandoffTimer) {
+ globalThis.clearTimeout(this._surfaceHandoffTimer);
+ this._surfaceHandoffTimer = null;
+ }
+ this._surfaceHandoff = false;
+ },
+
+ releaseSurfaceBindings() {
+ this._floatingCleanup?.();
+ this._floatingCleanup = null;
+ this._stageResizeObserver?.disconnect?.();
+ this._stageResizeObserver = null;
+ this._stageElement = null;
+ },
+
+ isCanvasSurfaceVisible(element = null) {
+ const root = element
+ || globalThis.document?.querySelector?.(".browser-canvas-surface .browser-panel")
+ || globalThis.document?.querySelector?.(".browser-panel");
+ if (!root?.isConnected) return false;
+ const surface = root.closest?.(".browser-canvas-surface");
+ const stage = root.querySelector?.(".browser-stage") || root;
+ const surfaceStyle = surface ? globalThis.getComputedStyle?.(surface) : null;
+ const rootStyle = globalThis.getComputedStyle?.(root);
+ if (surfaceStyle?.display === "none" || surfaceStyle?.visibility === "hidden") return false;
+ if (rootStyle?.display === "none" || rootStyle?.visibility === "hidden") return false;
+ const rect = stage.getBoundingClientRect?.();
+ return Boolean(rect && Math.round(rect.width || 0) >= 80 && Math.round(rect.height || 0) >= 80);
},
prepareSurfaceOpen(nextMode, requestedBrowserId = null) {
@@ -517,11 +642,16 @@ const model = {
this.switchingBrowserId = targetBrowserId;
},
- async waitForSurfaceViewport() {
+ async waitForSurfaceViewport(options = {}) {
+ const sequence = Number(options.sequence || 0);
+ const startedAt = Date.now();
let lastKey = "";
let stableCount = 0;
- for (let index = 0; index < 24; index += 1) {
+ while (Date.now() - startedAt <= SURFACE_VIEWPORT_MAX_WAIT_MS) {
await nextAnimationFrame();
+ if (sequence && !this.isCurrentSurfaceOpen(sequence)) {
+ return null;
+ }
const viewport = this.surfaceViewportMeasurement();
if (!viewport) continue;
const key = `${viewport.rawWidth}x${viewport.rawHeight}`;
@@ -538,16 +668,29 @@ const model = {
lastKey = key;
}
}
- return this.currentViewportSize();
+ const fallbackViewport = this.currentViewportSize();
+ return fallbackViewport;
},
- async syncViewportAfterSurfaceOpen() {
+ async syncViewportAfterSurfaceOpen(sequence = this._surfaceOpenSequence) {
if (!this.connected || !this.activeBrowserId) return;
- await this.waitForSurfaceViewport();
- await this.syncViewport(true);
+ await this.waitForSurfaceViewport({ sequence });
+ if (!this.isCurrentSurfaceOpen(sequence)) {
+ return;
+ }
+ await this.syncViewport(true, { restartStream: this._mode === "canvas" });
if (this._mode !== "canvas") return;
- globalThis.setTimeout?.(() => this.queueViewportSync(true), 240);
- globalThis.setTimeout?.(() => this.queueViewportSync(true), 420);
+ this.scheduleViewportSyncForSurface(sequence, 240);
+ this.scheduleViewportSyncForSurface(sequence, 520);
+ },
+
+ scheduleViewportSyncForSurface(sequence, delayMs = 0) {
+ globalThis.setTimeout?.(() => {
+ if (!this.isCurrentSurfaceOpen(sequence) || this._mode !== "canvas") {
+ return;
+ }
+ this.queueViewportSync(true);
+ }, delayMs);
},
async connectViewer(options = {}) {
@@ -715,11 +858,12 @@ const model = {
this._pendingFrameSrc = "";
this._pendingFrameOptions = null;
const sequence = this._frameRenderSequence + 1;
+ const surfaceSequence = this._surfaceOpenSequence;
this._frameRenderSequence = sequence;
- void this.renderDecodedFrame(frameSrc, options, sequence);
+ void this.renderDecodedFrame(frameSrc, options, sequence, surfaceSequence);
},
- async renderDecodedFrame(frameSrc, options = {}, sequence = 0) {
+ async renderDecodedFrame(frameSrc, options = {}, sequence = 0, surfaceSequence = this._surfaceOpenSequence) {
if (!frameSrc) {
if (sequence === this._frameRenderSequence) {
this.frameSrc = "";
@@ -727,7 +871,9 @@ const model = {
return;
}
const dimensions = await loadFrameDimensions(frameSrc);
- if (sequence !== this._frameRenderSequence) return;
+ if (sequence !== this._frameRenderSequence || surfaceSequence !== this._surfaceOpenSequence) {
+ return;
+ }
const viewport = this.currentViewportSize() || this._lastViewport;
if (!this.frameMatchesViewport(dimensions, viewport)) {
this.requestViewportSyncAfterRejectedFrame();
@@ -737,6 +883,56 @@ const model = {
this._lastFrameDimensions = dimensions;
this._lastFrameAt = Date.now();
options?.onAccepted?.();
+ this._canvasFirstFrameAcceptedSequence = surfaceSequence;
+ this.scheduleCanvasWidthNudgeAfterFirstFrame();
+ },
+
+ scheduleCanvasWidthNudgeAfterFirstFrame() {
+ const surfaceSequence = this._surfaceOpenSequence;
+ if (this._mode !== "canvas" || !this.isCurrentSurfaceOpen(surfaceSequence) || !this.activeBrowserId) {
+ return;
+ }
+ if (this._canvasFirstFrameNudgeSequence === surfaceSequence) {
+ return;
+ }
+ if (
+ this._canvasSurfaceReadySequence !== surfaceSequence
+ || this._canvasFirstFrameAcceptedSequence !== surfaceSequence
+ ) {
+ return;
+ }
+ this._canvasFirstFrameNudgeSequence = surfaceSequence;
+
+ void (async () => {
+ await nextAnimationFrame();
+ await nextAnimationFrame();
+ if (!this.isCurrentSurfaceOpen(surfaceSequence) || this._mode !== "canvas") {
+ return;
+ }
+ this.forceRightCanvasWidthNudge();
+ })();
+ },
+
+ forceRightCanvasWidthNudge() {
+ const canvas = rightCanvasStore;
+ if (!canvas || canvas.isMobileMode || !canvas.isOpen || canvas.activeSurfaceId !== "browser") {
+ return;
+ }
+
+ const currentWidth = Number(canvas.width || 0);
+ if (!Number.isFinite(currentWidth) || currentWidth <= 0) {
+ return;
+ }
+ const maxWidth = Number(canvas.maxWidth?.() || currentWidth);
+ const minWidth = Number(canvas.minWidth || 420);
+ const direction = currentWidth < maxWidth ? 1 : -1;
+ const nudgedWidth = currentWidth + direction;
+ if (nudgedWidth < minWidth || nudgedWidth > maxWidth || nudgedWidth === currentWidth) {
+ return;
+ }
+
+ canvas.setWidth?.(nudgedWidth, { persist: false });
+ this.queueViewportSync(true);
},
frameMatchesViewport(dimensions = null, viewport = null) {
@@ -749,7 +945,9 @@ const model = {
requestViewportSyncAfterRejectedFrame() {
const now = Date.now();
- if (now - this._lastFrameRejectSyncAt < FRAME_REJECT_SYNC_COOLDOWN_MS) return;
+ if (now - this._lastFrameRejectSyncAt < FRAME_REJECT_SYNC_COOLDOWN_MS) {
+ return;
+ }
this._lastFrameRejectSyncAt = now;
this.queueViewportSync(true);
},
@@ -776,11 +974,22 @@ const model = {
this._frameRenderSequence += 1;
},
+ beginCommand() {
+ this._commandInFlightCount += 1;
+ this.commandInFlight = true;
+ },
+
+ finishCommand() {
+ this._commandInFlightCount = Math.max(0, this._commandInFlightCount - 1);
+ this.commandInFlight = this._commandInFlightCount > 0;
+ },
+
async command(command, extra = {}) {
this.error = "";
this.annotationError = "";
- this.commandInFlight = true;
+ this.beginCommand();
const previousActiveBrowserId = this.activeBrowserId;
+ const commandName = String(command || "").toLowerCase();
try {
await this.ensureContextId();
const response = await websocket.request(
@@ -813,19 +1022,33 @@ const model = {
this.address = result.state?.currentUrl || result.currentUrl;
}
this.applySnapshot(data.snapshot);
- if (["navigate", "back", "forward", "reload", "close"].includes(String(command || "").toLowerCase())) {
+ if (["navigate", "back", "forward", "reload", "close"].includes(commandName)) {
this.clearAnnotationsForBrowser(previousActiveBrowserId);
this.cancelAnnotationDraft();
}
const activeChanged = this.activeBrowserId && this.activeBrowserId !== previousActiveBrowserId;
- if ((command === "open" || command === "close" || activeChanged) && this.contextId && this.activeBrowserId) {
+ if ((commandName === "open" || commandName === "close" || activeChanged) && this.contextId && this.activeBrowserId) {
await this.connectViewer({ browserId: this.activeBrowserId });
- }
- } catch (error) {
+ } else if (["navigate", "back", "forward", "reload"].includes(commandName)) {
+ await this.restartCanvasStreamAfterPageChange();
+ }
+ } catch (error) {
this.error = error instanceof Error ? error.message : String(error);
} finally {
- this.commandInFlight = false;
+ this.finishCommand();
+ }
+ },
+
+ async restartCanvasStreamAfterPageChange() {
+ const surfaceSequence = this._surfaceOpenSequence;
+ if (this._mode !== "canvas" || !this.isCurrentSurfaceOpen(surfaceSequence) || !this.activeBrowserId) {
+ return;
}
+ await this.waitForSurfaceViewport({ sequence: surfaceSequence });
+ if (this._mode !== "canvas" || !this.isCurrentSurfaceOpen(surfaceSequence) || !this.activeBrowserId) {
+ return;
+ }
+ await this.syncViewport(true, { restartStream: true });
},
async go() {
@@ -886,6 +1109,35 @@ const model = {
await this.command("open");
},
+ isClosingBrowser(id) {
+ const browserId = this.normalizeBrowserId(id);
+ return Boolean(browserId && this._closingBrowserIds[String(browserId)]);
+ },
+
+ markBrowserClosing(id, closing = true) {
+ const browserId = this.normalizeBrowserId(id);
+ if (!browserId) return;
+ const key = String(browserId);
+ const nextClosing = { ...this._closingBrowserIds };
+ if (closing) {
+ nextClosing[key] = true;
+ } else {
+ delete nextClosing[key];
+ }
+ this._closingBrowserIds = nextClosing;
+ },
+
+ async closeBrowser(id) {
+ const browserId = this.normalizeBrowserId(id);
+ if (!browserId || this.isClosingBrowser(browserId)) return;
+ this.markBrowserClosing(browserId, true);
+ try {
+ await this.command("close", { browser_id: browserId });
+ } finally {
+ this.markBrowserClosing(browserId, false);
+ }
+ },
+
isActiveBrowser(browser) {
return Number(browser?.id) === Number(this.activeBrowserId);
},
@@ -1479,37 +1731,46 @@ const model = {
}, force ? 0 : VIEWPORT_SYNC_DEBOUNCE_MS);
},
- async syncViewport(force = false) {
- if (!this.contextId || !this.activeBrowserId) return;
- const viewport = this.currentViewportSize();
- if (!viewport) return;
- const key = `${this.activeBrowserId}:${viewport.width}x${viewport.height}`;
- if (
- this._lastViewportKey === key
- || (
- !force
- && this._lastViewport
- && this.sameBrowserId(this._lastViewport.browserId, this.activeBrowserId)
- && Math.abs(this._lastViewport.width - viewport.width) <= VIEWPORT_SYNC_SIZE_TOLERANCE
- && Math.abs(this._lastViewport.height - viewport.height) <= VIEWPORT_SYNC_SIZE_TOLERANCE
- )
- ) return;
- try {
- await websocket.emit("browser_viewer_input", {
- context_id: this.contextId,
- browser_id: this.activeBrowserId,
- viewer_id: this._viewerToken,
- input_type: "viewport",
- width: viewport.width,
- height: viewport.height,
- });
- this._lastViewportKey = key;
- this._lastViewport = {
- browserId: this.activeBrowserId,
- width: viewport.width,
- height: viewport.height,
- };
- } catch (error) {
+ async syncViewport(force = false, options = {}) {
+ const restartStream = Boolean(options.restartStream);
+ if (!this.contextId || !this.activeBrowserId) {
+ return;
+ }
+ const viewport = this.currentViewportSize();
+ if (!viewport) {
+ return;
+ }
+ const key = `${this.activeBrowserId}:${viewport.width}x${viewport.height}`;
+ if (
+ (!restartStream && this._lastViewportKey === key)
+ || (
+ !force
+ && !restartStream
+ && this._lastViewport
+ && this.sameBrowserId(this._lastViewport.browserId, this.activeBrowserId)
+ && Math.abs(this._lastViewport.width - viewport.width) <= VIEWPORT_SYNC_SIZE_TOLERANCE
+ && Math.abs(this._lastViewport.height - viewport.height) <= VIEWPORT_SYNC_SIZE_TOLERANCE
+ )
+ ) {
+ return;
+ }
+ try {
+ await websocket.emit("browser_viewer_input", {
+ context_id: this.contextId,
+ browser_id: this.activeBrowserId,
+ viewer_id: this._viewerToken,
+ input_type: "viewport",
+ width: viewport.width,
+ height: viewport.height,
+ restart_stream: restartStream,
+ });
+ this._lastViewportKey = key;
+ this._lastViewport = {
+ browserId: this.activeBrowserId,
+ width: viewport.width,
+ height: viewport.height,
+ };
+ } catch (error) {
this._lastViewportKey = "";
this._lastViewport = null;
console.warn("Browser viewport sync failed", error);
@@ -1545,7 +1806,7 @@ const model = {
await websocket.emit("browser_viewer_input", payload);
},
- async sendWheel(event) {
+ async sendWheel(event) {
if (!this.activeBrowserId || !event) return;
const image = event.currentTarget?.querySelector?.(".browser-frame") || event.target?.closest?.(".browser-frame");
const pointer = this.pointerCoordinatesFor(event, image);
@@ -1560,12 +1821,12 @@ const model = {
delta_x: Number(event.deltaX || 0),
delta_y: Number(event.deltaY || 0),
};
- try {
- await websocket.emit("browser_viewer_input", payload);
- } catch (error) {
- this.error = error instanceof Error ? error.message : String(error);
- }
- },
+ try {
+ await websocket.emit("browser_viewer_input", payload);
+ } catch (error) {
+ this.error = error instanceof Error ? error.message : String(error);
+ }
+ },
async sendKey(event) {
if (this.annotating) return;
@@ -1585,12 +1846,22 @@ const model = {
},
async cleanup() {
+ if (this._surfaceHandoff) {
+ this.releaseSurfaceBindings();
+ this.extensionMenuOpen = false;
+ return;
+ }
+ this._surfaceOpenSequence += 1;
+ this._openPromise = null;
+ this._openSignature = "";
this._connectSequence += 1;
this._viewerToken = "";
this.switchingBrowserId = null;
this._surfaceMounted = false;
this._surfaceSwitching = false;
this.commandInFlight = false;
+ this._commandInFlightCount = 0;
+ this._closingBrowserIds = {};
this.annotating = false;
this.annotationBusy = false;
this.annotationError = "";
@@ -1606,11 +1877,7 @@ const model = {
this._frameOff = null;
this._stateOff = null;
this.resetRenderedFrame();
- this._floatingCleanup?.();
- this._floatingCleanup = null;
- this._stageResizeObserver?.disconnect?.();
- this._stageResizeObserver = null;
- this._stageElement = null;
+ this.releaseSurfaceBindings();
if (this._viewportSyncTimer) {
globalThis.clearTimeout(this._viewportSyncTimer);
this._viewportSyncTimer = null;
@@ -1681,11 +1948,17 @@ const model = {
resizeObserver.observe(inner);
if (stage) {
this._stageResizeObserver?.disconnect?.();
- this._stageResizeObserver = new ResizeObserver(() => this.queueViewportSync());
+ this._stageResizeObserver = new ResizeObserver(() => {
+ this.queueViewportSync();
+ });
this._stageResizeObserver.observe(stage);
}
}
- globalThis.requestAnimationFrame(() => this.queueViewportSync(true));
+ const surfaceSequence = this._surfaceOpenSequence;
+ globalThis.requestAnimationFrame(() => {
+ if (!this.isCurrentSurfaceOpen(surfaceSequence)) return;
+ this.queueViewportSync(true);
+ });
const onPointerMove = (event) => {
if (!drag) return;
@@ -1735,6 +2008,7 @@ const model = {
},
setupCanvasSurface(element = null) {
+ const surfaceSequence = this._surfaceOpenSequence;
this._floatingCleanup?.();
this._floatingCleanup = null;
this._stageResizeObserver?.disconnect?.();
@@ -1742,10 +2016,15 @@ const model = {
const stage = root?.querySelector?.(".browser-stage");
this._stageElement = stage || null;
if (stage && globalThis.ResizeObserver) {
- this._stageResizeObserver = new ResizeObserver(() => this.queueViewportSync());
+ this._stageResizeObserver = new ResizeObserver(() => {
+ this.queueViewportSync();
+ });
this._stageResizeObserver.observe(stage);
}
- globalThis.requestAnimationFrame?.(() => this.queueViewportSync(true));
+ globalThis.requestAnimationFrame?.(() => {
+ if (!this.isCurrentSurfaceOpen(surfaceSequence) || this._mode !== "canvas") return;
+ this.queueViewportSync(true);
+ });
},
get activeTitle() {
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index f4bc81b17..14ba37a9c 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -370,7 +370,9 @@ def test_browser_viewer_creates_chat_when_no_context_is_selected():
assert "async ensureContextId()" in js
assert "async createChatContextForBrowser()" in js
assert 'callJsonApi("/chat_create"' in js
- assert "globalThis.setContext(contextId)" in js
+ assert 'import { getContext, setContext } from "/index.js";' in js
+ assert "setContext(contextId)" in js
+ assert "chatsStore.setSelected?.(contextId)" in js
assert "await this.ensureContextId();" in js
assert "No active chat context is selected." not in js
@@ -382,11 +384,206 @@ def test_browser_canvas_startup_waits_for_raw_viewport_settle():
assert "const CANVAS_VIEWPORT_SETTLE_MS = 520;" in js
assert "const SURFACE_VIEWPORT_STABLE_FRAMES = 4;" in js
+ assert "const SURFACE_VIEWPORT_MAX_WAIT_MS = 1200;" in js
assert "surfaceViewportMeasurement()" in js
assert "rawWidth" in js
assert "rawHeight" in js
assert "const key = `${viewport.rawWidth}x${viewport.rawHeight}`;" in js
assert "Date.now() - this._surfaceOpenedAt >= CANVAS_VIEWPORT_SETTLE_MS" in js
+ assert "while (Date.now() - startedAt <= SURFACE_VIEWPORT_MAX_WAIT_MS)" in js
+ assert "this._openPromise && this._openSignature === openSignature" in js
+ assert "isCurrentSurfaceOpen(surfaceSequence)" in js
+ assert "isCanvasSurfaceVisible(element)" in js
+ assert "scheduleViewportSyncForSurface" in js
+
+
+def test_browser_canvas_surface_open_waits_for_visible_panel():
+ js = (
+ PROJECT_ROOT
+ / "plugins"
+ / "_browser"
+ / "extensions"
+ / "webui"
+ / "right_canvas_register_surfaces"
+ / "register-browser.js"
+ ).read_text(encoding="utf-8")
+
+ assert "waitForVisibleCanvasPanel" in js
+ assert "isVisibleCanvasPanel(panel)" in js
+ assert 'panel.closest(".browser-canvas-surface")' in js
+ assert 'panel.querySelector(".browser-stage")' in js
+ assert "getBoundingClientRect" in js
+ assert "stableCount >= 2" in js
+ assert "forceCanvasWidthNudgeAfterBrowserMount" not in js
+
+
+def test_browser_canvas_nudges_width_after_first_accepted_frame():
+ js = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(
+ encoding="utf-8"
+ )
+
+ assert "_canvasSurfaceReadySequence" in js
+ assert "_canvasFirstFrameAcceptedSequence" in js
+ assert "_canvasFirstFrameNudgeSequence" in js
+ assert "scheduleCanvasWidthNudgeAfterFirstFrame()" in js
+ assert "this._canvasSurfaceReadySequence = surfaceSequence;" in js
+ assert "const surfaceSequence = this._surfaceOpenSequence;" in js
+ assert "this._canvasFirstFrameAcceptedSequence = surfaceSequence;" in js
+ assert "forceRightCanvasWidthNudge()" in js
+ assert "await nextAnimationFrame();" in js
+ assert "globalThis.Alpine" not in js
+ assert 'import { store as rightCanvasStore } from "/components/canvas/right-canvas-store.js";' in js
+ assert "const canvas = rightCanvasStore;" in js
+ assert 'canvas.activeSurfaceId !== "browser"' in js
+ assert "canvas.setWidth?.(nudgedWidth, { persist: false })" in js
+ assert "this.queueViewportSync(true)" in js
+ frame_assignment_index = js.index("this.frameSrc = frameSrc;")
+ frame_nudge_schedule_index = js.index(
+ "this.scheduleCanvasWidthNudgeAfterFirstFrame();",
+ frame_assignment_index,
+ )
+ assert frame_assignment_index < frame_nudge_schedule_index
+
+
+def test_browser_canvas_restarts_stream_after_page_navigation():
+ js = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(
+ encoding="utf-8"
+ )
+
+ assert "async restartCanvasStreamAfterPageChange()" in js
+ assert '["navigate", "back", "forward", "reload"].includes(commandName)' in js
+ assert "await this.restartCanvasStreamAfterPageChange();" in js
+ assert "await this.waitForSurfaceViewport({ sequence: surfaceSequence });" in js
+ assert "await this.syncViewport(true, { restartStream: true });" in js
+ reconnect_index = js.index("await this.connectViewer({ browserId: this.activeBrowserId });")
+ restart_index = js.index("await this.restartCanvasStreamAfterPageChange();")
+ assert reconnect_index < restart_index
+
+
+def test_browser_entry_points_prefer_canvas_and_modal_dock_handoff():
+ button_html = (
+ PROJECT_ROOT
+ / "plugins"
+ / "_browser"
+ / "extensions"
+ / "webui"
+ / "chat-input-bottom-actions-start"
+ / "browser-button.html"
+ ).read_text(encoding="utf-8")
+ tool_handler = (
+ PROJECT_ROOT
+ / "plugins"
+ / "_browser"
+ / "extensions"
+ / "webui"
+ / "get_tool_message_handler"
+ / "browser-tool-handler.js"
+ ).read_text(encoding="utf-8")
+ after_loop_handler = (
+ PROJECT_ROOT
+ / "plugins"
+ / "_browser"
+ / "extensions"
+ / "webui"
+ / "set_messages_after_loop"
+ / "auto-open-browser-results.js"
+ ).read_text(encoding="utf-8")
+ register_js = (
+ PROJECT_ROOT
+ / "plugins"
+ / "_browser"
+ / "extensions"
+ / "webui"
+ / "right_canvas_register_surfaces"
+ / "register-browser.js"
+ ).read_text(encoding="utf-8")
+ browser_store = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(
+ encoding="utf-8"
+ )
+ canvas_store = (PROJECT_ROOT / "webui" / "components" / "canvas" / "right-canvas-store.js").read_text(
+ encoding="utf-8"
+ )
+ modals_js = (PROJECT_ROOT / "webui" / "js" / "modals.js").read_text(encoding="utf-8")
+
+ assert "Open Browser" in button_html
+ assert "$store.rightCanvas ? $store.rightCanvas.open('browser')" in button_html
+ assert "window.ensureModalOpen ? window.ensureModalOpen('/plugins/_browser/webui/main.html')" in button_html
+ assert "$store.rightCanvas.toggle('browser')" not in button_html
+ assert 'defaultOpenMode: "modal"' not in register_js
+ assert "beginDockHandoff()" in register_js
+ assert "beginSurfaceHandoff" in register_js
+ assert "finishDockHandoff()" in register_js
+ assert "cancelDockHandoff()" in register_js
+ assert "defaultOpenMode" not in canvas_store
+ assert "await surface.beginDockHandoff?.(payload)" in canvas_store
+ assert "await this.closeDockSourceModal(payload, modalPath)" in canvas_store
+ assert "delete openPayload.closeSourceModal" in canvas_store
+ assert "await surface.finishDockHandoff?.({ ...openPayload, opened })" in canvas_store
+ assert "await surface.cancelDockHandoff?.(payload)" in canvas_store
+ assert "async closeDockSourceModal" in canvas_store
+
+ assert "sourceModalPath: modal.path" in modals_js
+ assert "closeSourceModal: async () =>" in modals_js
+ assert "const closed = await closeModal(modal.path)" in modals_js
+ assert "const fallbackClosed = await closeModal()" in modals_js
+ assert "button.disabled = true" in modals_js
+
+ assert "beginSurfaceHandoff()" in browser_store
+ assert "finishSurfaceHandoff()" in browser_store
+ assert "cancelSurfaceHandoff()" in browser_store
+ assert "releaseSurfaceBindings()" in browser_store
+ assert "this.releaseSurfaceBindings();" in browser_store
+
+ for js in (tool_handler, after_loop_handler):
+ assert "async function openBrowserCanvas" in js
+ assert "openBrowserModal" not in js
+ assert js.index('await rightCanvasStore.open("browser", payload);') < js.index("if (window.ensureModalOpen)")
+
+ assert "function autoOpenBrowserCanvas" in tool_handler
+
+ for js in (tool_handler, after_loop_handler, register_js, browser_store, modals_js):
+ assert "globalThis.Alpine" not in js
+ assert "Alpine?.store" not in js
+ assert "Alpine.store" not in js
+
+
+def test_browser_canvas_uses_plain_panel_without_debug_probe():
+ panel_html = (
+ PROJECT_ROOT
+ / "plugins"
+ / "_browser"
+ / "extensions"
+ / "webui"
+ / "right-canvas-panels"
+ / "browser-panel.html"
+ ).read_text(encoding="utf-8")
+ browser_store = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(
+ encoding="utf-8"
+ )
+ register_js = (
+ PROJECT_ROOT
+ / "plugins"
+ / "_browser"
+ / "extensions"
+ / "webui"
+ / "right_canvas_register_surfaces"
+ / "register-browser.js"
+ ).read_text(encoding="utf-8")
+ canvas_store = (PROJECT_ROOT / "webui" / "components" / "canvas" / "right-canvas-store.js").read_text(
+ encoding="utf-8"
+ )
+ modals_js = (PROJECT_ROOT / "webui" / "js" / "modals.js").read_text(encoding="utf-8")
+
+ assert 'x-component path="/plugins/_browser/webui/browser-panel.html" mode="canvas"' in panel_html
+ assert "browserCanvasRemountKey" not in panel_html
+ assert "remountBrowserCanvasOnce" not in panel_html
+ assert "data-remount-slot" not in panel_html
+
+ for js in (browser_store, register_js, canvas_store, modals_js):
+ assert "__a0BrowserDebug" not in js
+ assert "browserDebug" not in js
+ assert "emitBrowserDebug" not in js
+ assert ".debug(" not in js
def test_browser_ui_spinners_have_browser_local_animation():
@@ -436,6 +633,29 @@ def test_browser_viewer_uses_tabs_for_session_switching():
assert "Using ${this.mainModelSummary}" in browser_store
+def test_browser_tabs_close_without_confirmation_or_busy_lock():
+ main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(
+ encoding="utf-8"
+ )
+ browser_store = (
+ PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js"
+ ).read_text(encoding="utf-8")
+ close_start = main_html.index('class="browser-tab-close"')
+ close_end = main_html.index("</button>", close_start)
+ close_markup = main_html[close_start:close_end]
+
+ assert '@click.stop="$store.browserPage.closeBrowser(browser.id)"' in main_html
+ assert "@pointerdown.stop" in close_markup
+ assert "$confirmClick" not in main_html
+ assert "isBusy()" not in close_markup
+ assert ':disabled="$store.browserPage.isClosingBrowser(browser.id)"' in close_markup
+ assert "--browser-tab-close-size: 32px;" in main_html
+ assert "async closeBrowser(id)" in browser_store
+ assert "isClosingBrowser(id)" in browser_store
+ assert "_closingBrowserIds" in browser_store
+ assert "_commandInFlightCount" in browser_store
+
+
def test_browser_viewer_uses_cdp_screencast_transport():
ws_browser = (PROJECT_ROOT / "plugins" / "_browser" / "api" / "ws_browser.py").read_text(
encoding="utf-8"
@@ -469,15 +689,44 @@ def test_browser_viewer_uses_cdp_screencast_transport():
assert "requestAnimationFrame" in browser_store
assert "viewport_width: initialViewport?.width" in browser_store
assert "viewport_height: initialViewport?.height" in browser_store
+ assert "restart_stream: restartStream" in browser_store
+ assert 'restart_screencast=bool(data.get("restart_stream"))' in ws_browser
+ assert "restart_screencast: bool = False" in runtime
+ assert "should_remount_viewport = changed or restart_screencast" in runtime
+ assert "VIEWPORT_REMOUNT_PAUSE_SECONDS = 0.05" in runtime
+ assert "await self._apply_cdp_viewport_with_remount" in runtime
+ assert "await self._apply_viewport_with_remount(page, viewport)" in runtime
+ assert "await self._remount_viewport(page, viewport)" in runtime
+ assert "await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)" in runtime
+ assert "def _nudged_viewport(viewport: dict[str, int])" in runtime
+ assert 'await this.syncViewport(true, { restartStream: this._mode === "canvas" });' in browser_store
assert "this.frameState = data.state || null" not in browser_store
assert "function loadFrameDimensions(src)" in browser_store
assert "frameMatchesViewport(dimensions = null, viewport = null)" in browser_store
assert "requestViewportSyncAfterRejectedFrame()" in browser_store
+ assert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browser
+ assert '"frame_source": "state"' in ws_browser
+ assert '"frame_source"] = "screencast"' in ws_browser
+ assert "fallback_screenshot" not in ws_browser
+ assert "canvas_wheel_screenshot" not in ws_browser
+ assert "surface_mode: this._mode" not in browser_store
assert "overflow: hidden;" in main_html
assert "object-fit: fill;" in main_html
assert "image-rendering: auto;" in main_html
+def test_browser_navigation_errors_stay_inside_native_browser_page():
+ runtime = (
+ PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"
+ ).read_text(encoding="utf-8")
+
+ assert "Error as PlaywrightError" in runtime
+ assert "except PlaywrightError as exc:" in runtime
+ assert "Browser navigation showed a native error page" in runtime
+ assert "await self._settle(page)" in runtime
+ assert "except (PlaywrightError, PlaywrightTimeoutError):" in runtime
+
+
def test_browser_annotate_mode_ui_and_prompt_hooks():
panel_html = (
PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html"
@@ -562,14 +811,57 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
assert frame["browser_id"] == 7
assert frame["image"] == "second"
assert frame["metadata"]["deviceWidth"] == 200
- assert ("Emulation.setDeviceMetricsOverride", {
- "width": 1118,
- "height": 662,
- "deviceScaleFactor": 1,
- "mobile": False,
- "dontSetVisibleSize": True,
- }) in session.sent
- assert ("Emulation.setVisibleSize", {"width": 1118, "height": 662}) in session.sent
+ assert frame["metadata"]["expectedWidth"] == 1118
+ assert frame["metadata"]["expectedHeight"] == 662
+ metrics_calls = [
+ params
+ for method, params in session.sent
+ if method == "Emulation.setDeviceMetricsOverride"
+ ]
+ visible_calls = [
+ params
+ for method, params in session.sent
+ if method == "Emulation.setVisibleSize"
+ ]
+ assert metrics_calls[:3] == [
+ {
+ "width": 1118,
+ "height": 662,
+ "deviceScaleFactor": 1,
+ "mobile": False,
+ "dontSetVisibleSize": True,
+ },
+ {
+ "width": 1119,
+ "height": 662,
+ "deviceScaleFactor": 1,
+ "mobile": False,
+ "dontSetVisibleSize": True,
+ },
+ {
+ "width": 1118,
+ "height": 662,
+ "deviceScaleFactor": 1,
+ "mobile": False,
+ "dontSetVisibleSize": True,
+ },
+ ]
+ assert visible_calls[:3] == [
+ {"width": 1118, "height": 662},
+ {"width": 1119, "height": 662},
+ {"width": 1118, "height": 662},
+ ]
+ start_index = next(
+ index
+ for index, (method, _params) in enumerate(session.sent)
+ if method == "Page.startScreencast"
+ )
+ cdp_viewport_indices = [
+ index
+ for index, (method, _params) in enumerate(session.sent)
+ if method.startswith("Emulation.")
+ ]
+ assert max(cdp_viewport_indices) < start_index
assert ("Page.screencastFrameAck", {"sessionId": 1}) in session.sent
assert ("Page.screencastFrameAck", {"sessionId": 2}) in session.sent
@@ -580,7 +872,7 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
@pytest.mark.anyio
-async def test_browser_screencast_keeps_rejecting_wrong_viewport_frames():
+async def test_browser_screencast_passes_wrong_viewport_frames_to_frontend_validator():
class FakeSession:
def __init__(self):
self.handlers = {}
@@ -610,7 +902,14 @@ async def test_browser_screencast_keeps_rejecting_wrong_viewport_frames():
)
await asyncio.sleep(0)
- assert await screencast.pop_frame() is None
+ frame = await screencast.pop_frame()
+
+ assert frame is not None
+ assert frame["image"] == SMALL_JPEG_10X10
+ assert frame["metadata"]["jpegWidth"] == 10
+ assert frame["metadata"]["jpegHeight"] == 10
+ assert frame["metadata"]["expectedWidth"] == 1118
+ assert frame["metadata"]["expectedHeight"] == 662
assert ("Page.screencastFrameAck", {"sessionId": 13}) in session.sent
await screencast.stop()
@@ -819,6 +1118,7 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):
"input_type": "viewport",
"width": 1280,
"height": 720,
+ "restart_stream": True,
},
"sid-1",
)
@@ -827,7 +1127,96 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):
"state": {"ok": True, "method": "set_viewport", "args": (7, 1280, 720)},
"snapshot": None,
}
- assert calls == [("set_viewport", (7, 1280, 720), {})]
+ assert calls == [
+ ("set_viewport", (7, 1280, 720), {"restart_screencast": True})
+ ]
+
+
+async def test_browser_runtime_remounts_same_viewport_when_restarting_screencast():
+ viewport_calls = []
+ stopped = []
+ settled = []
+
+ class FakePage:
+ viewport_size = {"width": 1280, "height": 720}
+
+ async def set_viewport_size(self, viewport):
+ viewport_calls.append(dict(viewport))
+
+ core = _BrowserRuntimeCore("ctx")
+ core.context = object()
+ core.pages[7] = browser_runtime_module.BrowserPage(id=7, page=FakePage())
+
+ async def fake_stop_screencasts(browser_id):
+ stopped.append(browser_id)
+
+ async def fake_settle(page, short=False):
+ settled.append(short)
+
+ async def fake_state(browser_id):
+ return {"id": browser_id}
+
+ core._stop_screencasts_for_browser = fake_stop_screencasts
+ core._settle = fake_settle
+ core._state = fake_state
+
+ result = await core.set_viewport(7, 1280, 720, restart_screencast=True)
+
+ assert result == {
+ "state": {"id": 7},
+ "viewport": {"width": 1280, "height": 720},
+ }
+ assert viewport_calls == [
+ {"width": 1281, "height": 720},
+ {"width": 1280, "height": 720},
+ ]
+ assert stopped == [7]
+ assert settled == [True]
+
+
+async def test_browser_runtime_remounts_initial_changed_viewport():
+ calls = []
+ stopped = []
+ settled = []
+
+ class FakePage:
+ viewport_size = {"width": 1024, "height": 768}
+
+ async def set_viewport_size(self, viewport):
+ calls.append(("viewport", dict(viewport)))
+
+ core = _BrowserRuntimeCore("ctx")
+ core.context = object()
+ core.pages[7] = browser_runtime_module.BrowserPage(id=7, page=FakePage())
+
+ async def fake_stop_screencasts(browser_id):
+ stopped.append(browser_id)
+
+ async def fake_settle(page, short=False):
+ settled.append(short)
+
+ async def fake_state(browser_id):
+ return {"id": browser_id}
+
+ core._stop_screencasts_for_browser = fake_stop_screencasts
+ core._settle = fake_settle
+ core._state = fake_state
+
+ result = await core.set_viewport(7, 672, 789)
+
+ assert result == {
+ "state": {"id": 7},
+ "viewport": {"width": 672, "height": 789},
+ }
+ assert calls == [
+ ("viewport", {"width": 672, "height": 789}),
+ ("viewport", {"width": 673, "height": 789}),
+ ("viewport", {"width": 672, "height": 789}),
+ ]
+ assert stopped == [7]
+ assert settled == [True]
@pytest.mark.anyio
diff --git a/webui/components/canvas/right-canvas-store.js b/webui/components/canvas/right-canvas-store.js
index 49c1f819b..15b174a90 100644
--- a/webui/components/canvas/right-canvas-store.js
+++ b/webui/components/canvas/right-canvas-store.js
@@ -98,7 +98,9 @@ const model = {
async open(surfaceId = "", payload = {}) {
const targetId = surfaceId || this.activeSurfaceId || this.panelSurfaces[0]?.id || "";
const surface = this.getSurface(targetId);
- if (!surface) return false;
+ if (!surface) {
+ return false;
+ }
if (typeof surface.canOpen === "function" && surface.canOpen(payload) === false) {
return false;
}
@@ -140,12 +142,48 @@ const model = {
async dockSurface(surfaceId, payload = {}) {
const surface = this.getSurface(surfaceId);
- if (!surface) return false;
+ if (!surface) {
+ return false;
+ }
const modalPath = payload.modalPath || surface.modalPath || "";
- if (modalPath && globalThis.isModalOpen?.(modalPath)) {
- await globalThis.closeModal?.(modalPath);
+ let handoffStarted = false;
+ try {
+ await surface.beginDockHandoff?.(payload);
+ handoffStarted = true;
+
+ const closed = await this.closeDockSourceModal(payload, modalPath);
+ if (closed === false) {
+ await surface.cancelDockHandoff?.(payload);
+ return false;
+ }
+
+ const openPayload = { ...payload, source: "modal" };
+ delete openPayload.closeSourceModal;
+ const opened = await this.open(surfaceId, openPayload);
+ await surface.finishDockHandoff?.({ ...openPayload, opened });
+ return opened;
+ } catch (error) {
+ if (handoffStarted) {
+ await surface.cancelDockHandoff?.(payload);
+ }
+ console.error(`Canvas surface ${surfaceId} failed to dock`, error);
+ return false;
+ }
+ },
+
+ async closeDockSourceModal(payload = {}, modalPath = "") {
+ if (typeof payload.closeSourceModal === "function") {
+ return (await payload.closeSourceModal()) !== false;
}
- return await this.open(surfaceId, { ...payload, source: "modal" });
+
+ const sourceModalPath = payload.sourceModalPath || modalPath;
+ if (sourceModalPath && globalThis.isModalOpen?.(sourceModalPath)) {
+ return (await globalThis.closeModal?.(sourceModalPath)) !== false;
+ }
+ if (modalPath && modalPath !== sourceModalPath && globalThis.isModalOpen?.(modalPath)) {
+ return (await globalThis.closeModal?.(modalPath)) !== false;
+ }
+ return true;
},
async undockSurface(surfaceId = "", payload = {}) {
diff --git a/webui/components/canvas/right-canvas.css b/webui/components/canvas/right-canvas.css
index 67248a7c9..ed5767f19 100644
--- a/webui/components/canvas/right-canvas.css
+++ b/webui/components/canvas/right-canvas.css
@@ -181,11 +181,9 @@ body.right-canvas-resizing {
align-items: center;
gap: 7px;
min-width: 0;
- height: 34px;
padding: 0 10px;
- border-radius: 7px 7px 0 0;
background: transparent;
- opacity: 0.7;
+ opacity: 0.4;
white-space: nowrap;
transition: border-color 0.16s ease, background-color 0.16s ease, opacity 0.16s ease;
}
@@ -193,8 +191,6 @@ body.right-canvas-resizing {
.right-canvas-tab:hover,
.right-canvas-tab.is-active {
opacity: 1;
- border-color: var(--right-canvas-border);
- background: color-mix(in srgb, var(--color-panel) 72%, transparent);
}
.right-canvas-tab .material-symbols-outlined {
diff --git a/webui/js/modals.js b/webui/js/modals.js
index 8b4e944dc..a7a2b5b36 100644
--- a/webui/js/modals.js
+++ b/webui/js/modals.js
@@ -1,6 +1,7 @@
// Import the component loader and page utilities
import { importComponent } from "/js/components.js";
import { callJsExtensions } from "/js/extensions.js";
+import { store as rightCanvasStore } from "/components/canvas/right-canvas-store.js";
// Modal functionality
const modalStack = [];
@@ -193,12 +194,26 @@ function configureModalDockButton(modal, doc) {
button.setAttribute("aria-label", metadata.title);
button.innerHTML = `<span class="material-symbols-outlined" aria-hidden="true">${metadata.icon}</span>`;
button.addEventListener("click", async () => {
- const canvas = globalThis.Alpine?.store?.("rightCanvas")
- || (await import("/components/canvas/right-canvas-store.js")).store;
- await canvas?.dockSurface?.(metadata.surfaceId, {
- modalPath: metadata.modalPath,
- source: "modal",
- });
+ if (button.disabled) return;
+ button.disabled = true;
+ try {
+ await rightCanvasStore.dockSurface?.(metadata.surfaceId, {
+ modalPath: metadata.modalPath,
+ sourceModalPath: modal.path,
+ source: "modal",
+ closeSourceModal: async () => {
+ const closed = await closeModal(modal.path);
+ if (closed === false) return false;
+ if (document.contains(modal.element)) {
+ const fallbackClosed = await closeModal();
+ if (fallbackClosed === false) return false;
+ }
+ return !document.contains(modal.element);
+ },
+ });
+ } finally {
+ if (document.contains(button)) button.disabled = false;
+ }
});
modal.close?.insertAdjacentElement("beforebegin", button);