Agent Zero AI framework
Add browser screenshot previews to tool messages
Alessandroc2fb2c3c94e1
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.pyindex da48ab3dd..454f5cbf8 100644--- a/plugins/_browser/api/ws_browser.py+++ b/plugins/_browser/api/ws_browser.py@@ -38,6 +38,8 @@ class WsBrowser(WsHandler):return await self._subscribe(data, sid)if event == "browser_viewer_unsubscribe":return self._unsubscribe(data, sid)+ if event == "browser_viewer_snapshot":+ return await self._snapshot(data)if event == "browser_viewer_sessions":return await self._sessions(data)if event == "browser_viewer_command":@@ -115,6 +117,49 @@ class WsBrowser(WsHandler):"all_browsers": True,}+ async def _snapshot(self, data: dict[str, Any]) -> dict[str, Any] | WsResult:+ context_id = self._context_id(data)+ if not context_id:+ return self._error("MISSING_CONTEXT", "context_id is required", data)+ if not AgentContext.get(context_id):+ return self._error("CONTEXT_NOT_FOUND", f"Context '{context_id}' was not found", data)++ runtime = await get_runtime(context_id, create=False)+ if not runtime:+ return {+ "context_id": context_id,+ "active_browser_context_id": context_id,+ "active_browser_id": None,+ "snapshot": None,+ "browsers": await self._all_browser_tabs(),+ "all_browsers": True,+ }++ listing = await runtime.call("list")+ browsers = listing.get("browsers") or []+ active_id = self._active_browser_id(listing, data.get("browser_id"))+ snapshot = None+ if active_id:+ try:+ quality = int(data.get("quality") or SCREENCAST_QUALITY)+ except (TypeError, ValueError):+ quality = SCREENCAST_QUALITY+ with contextlib.suppress(Exception):+ snapshot = await runtime.call(+ "screenshot",+ active_id,+ quality=quality,+ )++ return {+ "context_id": context_id,+ "active_browser_context_id": context_id,+ "active_browser_id": active_id,+ "snapshot": snapshot,+ "browsers": await self._all_browser_tabs(),+ "all_browsers": True,+ }+async def _command(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult:context_id = self._context_id(data)if not context_id:diff --git a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.jsindex b8c0d3e33..f20f1b159 100644--- a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js+++ b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js@@ -6,6 +6,7 @@ import { store as stepDetailStore } from "/components/modals/process-step-detailimport { store as speechStore } from "/components/chat/speech/speech-store.js";import { store as rightCanvasStore } from "/components/canvas/right-canvas-store.js";import { store as browserStore } from "/plugins/_browser/webui/browser-store.js";+import { getNamespacedClient } from "/js/websocket.js";import {buildDetailPayload,cleanStepTitle,@@ -13,8 +14,19 @@ import {} from "/js/messages.js";const BROWSER_MODAL = "/plugins/_browser/webui/main.html";+const BROWSER_SCREENSHOT_KVP_KEY = "Screenshot";+const BROWSER_SCREENSHOT_STYLE_ID = "a0-browser-screenshot-kvp-style";const AUTO_OPEN_WINDOW_MS = 10 * 60 * 1000;+const PREVIEW_REFRESH_MS = 2500;+const PREVIEW_RETRY_MS = 5000;+const PREVIEW_SNAPSHOT_TIMEOUT_MS = 10000;+const PREVIEW_QUALITY = 62;+const PREVIEW_FRAME_LIMIT = 16;+const NO_SCREENSHOT_ACTIONS = new Set(["close", "close_all"]);const syncedBrowserCanvases = new Set();+const liveScreenshotFrames = new Map();+const websocket = getNamespacedClient("/ws");+websocket.addHandlers(["ws_webui"]);export default async function registerBrowserToolHandler(extData) {if (extData?.tool_name === "browser") {@@ -58,21 +70,37 @@ function parseBrowserResult(content) {}}+function normalizeBrowserAction(kvps = {}) {+ return String(kvps.action || "").trim().toLowerCase().replace("-", "_");+}+function browserIdFromResult(result = {}, kvps = {}) {+ const browsers = Array.isArray(result.browsers) ? result.browsers : [];+ const lastInteractedId = result.last_interacted_browser_id;+ const listedBrowser = lastInteractedId+ ? browsers.find((browser) => String(browser?.id) === String(lastInteractedId))+ : browsers[0];return (result.id|| result.browser_id|| result.state?.id|| result.last_interacted_browser_id+ || listedBrowser?.id|| kvps.browser_id|| null);}function browserContextIdFromResult(result = {}, kvps = {}) {+ const browserId = browserIdFromResult(result, kvps);+ const browsers = Array.isArray(result.browsers) ? result.browsers : [];+ const listedBrowser = browserId+ ? browsers.find((browser) => String(browser?.id) === String(browserId))+ : browsers[0];return (result.context_id|| result.state?.context_id+ || listedBrowser?.context_id|| kvps.context_id|| kvps.contextId|| null@@ -132,6 +160,290 @@ function syncOpenBrowserCanvas(args, result) {});}+function currentBrowserContextId(result = {}, kvps = {}) {+ return (+ browserContextIdFromResult(result, kvps)+ || browserStore?.resolveContextId?.()+ || ""+ );+}++function buildBrowserCanvasPayload(result = {}, kvps = {}, source = "tool-kvp") {+ const browserId = browserIdFromResult(result, kvps);+ const contextId = currentBrowserContextId(result, kvps);+ if (!browserId && !contextId) return null;+ return {+ browserId: browserId || null,+ contextId,+ source,+ };+}++function makeViewerToken() {+ return globalThis.crypto?.randomUUID?.()+ || `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;+}++function snapshotToObjectUrl(snapshot) {+ if (!snapshot?.image) return null;+ const binary = globalThis.atob(snapshot.image);+ const chunks = [];+ const chunkSize = 8192;+ for (let offset = 0; offset < binary.length; offset += chunkSize) {+ const slice = binary.slice(offset, offset + chunkSize);+ const bytes = new Uint8Array(slice.length);+ for (let index = 0; index < slice.length; index += 1) {+ bytes[index] = slice.charCodeAt(index);+ }+ chunks.push(bytes);+ }+ return URL.createObjectURL(new Blob(chunks, { type: snapshot.mime || "image/jpeg" }));+}++function releaseLiveScreenshotFrame(viewerId) {+ const release = liveScreenshotFrames.get(viewerId);+ if (!release) return;+ liveScreenshotFrames.delete(viewerId);+ release();+}++function rememberLiveScreenshotFrame(viewerId, release) {+ liveScreenshotFrames.delete(viewerId);+ liveScreenshotFrames.set(viewerId, release);++ while (liveScreenshotFrames.size > PREVIEW_FRAME_LIMIT) {+ releaseLiveScreenshotFrame(liveScreenshotFrames.keys().next().value);+ }+}++function firstOk(response) {+ const result = response?.results?.find((item) => item?.ok);+ if (result) return result.data || {};+ const error = response?.results?.find((item) => !item?.ok)?.error;+ if (error) throw new Error(error.error || error.code || "Browser request failed");+ return {};+}++function browserResultFromRenderedStep(step) {+ if (!step) return {};+ const contentText = step.querySelector(".process-step-detail-content")?.textContent || "";+ return parseBrowserResult(contentText);+}++function shouldRenderBrowserScreenshotKvp(result = {}, kvps = {}) {+ if (buildBrowserCanvasPayload(result, kvps)) return true;+ const action = normalizeBrowserAction(kvps);+ return Boolean(action && !NO_SCREENSHOT_ACTIONS.has(action));+}++function ensureBrowserScreenshotStyles() {+ if (document.getElementById(BROWSER_SCREENSHOT_STYLE_ID)) return;+ const style = document.createElement("style");+ style.id = BROWSER_SCREENSHOT_STYLE_ID;+ style.textContent = `+ .browser-screenshot-kvp-button {+ padding: 0;+ width: min(12rem, 100%);+ aspect-ratio: 16 / 10;+ position: relative;+ display: inline-flex;+ align-items: center;+ justify-content: center;+ border: 1px solid color-mix(in srgb, var(--color-border) 74%, transparent);+ border-radius: 7px;+ background: color-mix(in srgb, var(--color-panel) 88%, var(--color-chat-background));+ color: var(--color-message-text);+ cursor: pointer;+ overflow: hidden;+ box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.08);+ transition: border-color 0.16s ease, color 0.16s ease, transform 0.16s ease, background 0.16s ease;+ }++ .browser-screenshot-kvp-button:hover {+ border-color: color-mix(in srgb, var(--color-primary) 68%, var(--color-border));+ color: var(--color-text);+ transform: translateY(-1px);+ }++ .browser-screenshot-kvp-button:focus-visible {+ outline: 2px solid color-mix(in srgb, var(--color-primary) 72%, white);+ outline-offset: 2px;+ }++ .browser-screenshot-kvp-image {+ width: 100%;+ height: 100%;+ display: block;+ object-fit: cover;+ object-position: top left;+ background: var(--color-chat-background);+ }++ .browser-screenshot-kvp-button:not(.has-frame) .browser-screenshot-kvp-image {+ display: none;+ }++ .browser-screenshot-kvp-placeholder {+ position: absolute;+ inset: 0;+ display: flex;+ align-items: center;+ justify-content: center;+ background:+ linear-gradient(90deg, transparent, rgba(255, 255, 255, 0.06), transparent),+ color-mix(in srgb, var(--color-panel) 84%, black 16%);+ background-size: 200% 100%, auto;+ animation: browser-screenshot-kvp-pulse 1.6s ease-in-out infinite;+ }++ .browser-screenshot-kvp-button.has-frame .browser-screenshot-kvp-placeholder {+ display: none;+ }++ .browser-screenshot-kvp-button .material-symbols-outlined {+ font-size: 1.15rem;+ line-height: 1;+ font-variation-settings: 'FILL' 0, 'wght' 420, 'GRAD' 0, 'opsz' 20;+ }++ @keyframes browser-screenshot-kvp-pulse {+ 0% { background-position: 160% 0, 0 0; }+ 100% { background-position: -160% 0, 0 0; }+ }+ `;+ document.head.appendChild(style);+}++function renderBrowserScreenshotKvp(kvpsTable, resolveBrowserPayload, label) {+ if (!kvpsTable || typeof resolveBrowserPayload !== "function") return;+ ensureBrowserScreenshotStyles();++ const rows = Array.from(kvpsTable.querySelectorAll(".kvps-row"));+ const row = rows.find((candidate) => {+ const keyText = candidate.querySelector(".kvps-key")?.textContent || "";+ return keyText.trim().toLowerCase() === BROWSER_SCREENSHOT_KVP_KEY.toLowerCase();+ });+ const cell = row?.querySelector(".kvps-val") || row?.cells?.[1];+ if (!cell) return;++ cell.__browserScreenshotCleanup?.();++ const button = document.createElement("button");+ button.type = "button";+ button.className = "browser-screenshot-kvp-button";+ button.title = label;+ button.setAttribute("aria-label", label);+ button.innerHTML = `+ <img class="browser-screenshot-kvp-image" alt="" draggable="false">+ <span class="browser-screenshot-kvp-placeholder" aria-hidden="true">+ <span class="material-symbols-outlined">image_search</span>+ </span>+ `;+ button.addEventListener("click", async (event) => {+ event.preventDefault();+ event.stopPropagation();+ const canvasPayload = resolveBrowserPayload();+ if (!canvasPayload) return;+ await openBrowserCanvas(canvasPayload);+ });++ cell.textContent = "";+ cell.appendChild(button);+ cell.__browserScreenshotCleanup = startBrowserScreenshotPreview(+ button,+ button.querySelector(".browser-screenshot-kvp-image"),+ resolveBrowserPayload,+ );+}++function startBrowserScreenshotPreview(button, image, resolveBrowserPayload) {+ const viewerId = makeViewerToken();+ let stopped = false;+ let timerId = null;+ let refreshInFlight = false;+ let frameUrl = null;++ const releaseFrame = () => {+ liveScreenshotFrames.delete(viewerId);+ if (frameUrl) {+ URL.revokeObjectURL(frameUrl);+ frameUrl = null;+ }+ image.removeAttribute("src");+ button.classList.remove("has-frame");+ };++ const setFrame = (snapshot) => {+ const nextUrl = snapshotToObjectUrl(snapshot);+ if (!nextUrl) {+ releaseFrame();+ return false;+ }+ if (frameUrl) {+ URL.revokeObjectURL(frameUrl);+ }+ frameUrl = nextUrl;+ image.src = frameUrl;+ button.classList.add("has-frame");+ rememberLiveScreenshotFrame(viewerId, releaseFrame);+ return true;+ };++ const cleanup = () => {+ stopped = true;+ if (timerId) {+ globalThis.clearTimeout(timerId);+ timerId = null;+ }+ releaseFrame();+ };++ const schedule = (delay = PREVIEW_REFRESH_MS) => {+ if (stopped || timerId) return;+ timerId = globalThis.setTimeout(refresh, delay);+ };++ const refresh = async () => {+ timerId = null;+ if (stopped || refreshInFlight) return;+ if (!button.isConnected) {+ cleanup();+ return;+ }++ const payload = resolveBrowserPayload();+ if (!payload?.contextId) {+ schedule(PREVIEW_RETRY_MS);+ return;+ }++ refreshInFlight = true;+ try {+ const response = await websocket.request(+ "browser_viewer_snapshot",+ {+ context_id: payload.contextId,+ browser_id: payload.browserId,+ viewer_id: viewerId,+ quality: PREVIEW_QUALITY,+ },+ { timeoutMs: PREVIEW_SNAPSHOT_TIMEOUT_MS },+ );+ const data = firstOk(response);+ const hasFrame = setFrame(data.snapshot);+ schedule(hasFrame ? PREVIEW_REFRESH_MS : PREVIEW_RETRY_MS);+ } catch (_error) {+ schedule(PREVIEW_RETRY_MS);+ } finally {+ refreshInFlight = false;+ }+ };++ schedule(0);++ return cleanup;+}+function drawBrowserTool({id,type,@@ -150,14 +462,25 @@ function drawBrowserTool({].filter(Boolean);const contentText = String(content ?? "");const browserResult = parseBrowserResult(contentText);+ const browserId = browserIdFromResult(browserResult, kvps);+ const browserCanvasPayload = buildBrowserCanvasPayload(browserResult, kvps);+ const browserPreviewLabel = browserId+ ? `Open Browser canvas for Browser ${browserId}`+ : "Open Browser canvas from screenshot";+ if (shouldRenderBrowserScreenshotKvp(browserResult, kvps)) {+ displayKvps[BROWSER_SCREENSHOT_KVP_KEY] = "";+ }const browserButton = createActionButton("visibility","Browser",- () => openBrowserCanvas({- browserId: browserIdFromResult(browserResult, kvps),- contextId: browserContextIdFromResult(browserResult, kvps),- source: "tool",- }),+ () => openBrowserCanvas(+ buildBrowserCanvasPayload(browserResult, kvps, "tool")+ || {+ browserId,+ contextId: browserContextIdFromResult(browserResult, kvps),+ source: "tool",+ },+ ),);browserButton.setAttribute("title", "Open Browser");browserButton.setAttribute("aria-label", "Open Browser");@@ -187,6 +510,15 @@ function drawBrowserTool({actionButtons: actionButtons.filter(Boolean),log: args,});+ renderBrowserScreenshotKvp(+ result.kvpsTable,+ () => (+ buildBrowserCanvasPayload(browserResultFromRenderedStep(result.step), kvps)+ || browserCanvasPayload+ || buildBrowserCanvasPayload({}, kvps)+ ),+ browserPreviewLabel,+ );syncOpenBrowserCanvas(args, browserResult);return result;}