Agent Zero AI framework
fix(canvas): keep browser and office surfaces opt-in
Alessandroce7ec3cb4c08
diff --git a/plugins/_browser/default_config.yaml b/plugins/_browser/default_config.yamlindex b40a95e65..b78e07845 100644--- a/plugins/_browser/default_config.yaml+++ b/plugins/_browser/default_config.yaml@@ -5,7 +5,7 @@ extension_paths: []# Page opened by new Browser sessions when no URL is provided.default_homepage: "about:blank"-# Focus Browser canvas pages automatically when agent Browser tool results arrive.+# When the Browser canvas is already open, keep it synced to agent Browser tool results.autofocus_active_page: true# Optional _model_config preset used by Browser-owned model helpers.diff --git a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.jsindex 99b9e3d88..b25a7b3f9 100644--- a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js+++ b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js@@ -14,7 +14,7 @@ import {const BROWSER_MODAL = "/plugins/_browser/webui/main.html";const AUTO_OPEN_WINDOW_MS = 10 * 60 * 1000;-const autoOpenedBrowsers = new Set();+const syncedBrowserCanvases = new Set();export default async function registerBrowserToolHandler(extData) {if (extData?.tool_name === "browser") {@@ -37,17 +37,6 @@ async function openBrowserCanvas(payload = {}) {}}-async function browserAllowsToolAutofocus() {- try {- if (browserStore.allowsToolAutofocus) {- return await browserStore.allowsToolAutofocus();- }- } catch (error) {- console.warn("Browser autofocus setting could not be checked", error);- }- return true;-}-function parseBrowserResult(content) {if (!content || typeof content !== "string") return {};try {@@ -86,28 +75,47 @@ function isFreshToolMessage(timestamp) {return Math.abs(Date.now() - messageMs) <= AUTO_OPEN_WINDOW_MS;}-function shouldAutoOpenBrowser(args, result) {+async function browserAllowsToolAutofocus() {+ try {+ if (browserStore.allowsToolAutofocus) {+ return await browserStore.allowsToolAutofocus();+ }+ } catch (error) {+ console.warn("Browser autofocus setting could not be checked", error);+ }+ return true;+}++function isBrowserCanvasAlreadyOpen() {+ return Boolean(+ rightCanvasStore?.isOpen+ && rightCanvasStore?.activeSurfaceId === "browser"+ && !rightCanvasStore?.isMobileMode,+ );+}++function shouldSyncOpenBrowserCanvas(args, result) {+ if (!isBrowserCanvasAlreadyOpen()) return false;if (!isFreshToolMessage(args?.timestamp)) return false;const action = String(args?.kvps?.action || "").trim().toLowerCase().replace("-", "_");if (["list", "content", "detail", "close", "close_all"].includes(action)) return false;return Boolean(browserIdFromResult(result, args?.kvps || {}) || action === "open" || action === "navigate");}-function autoOpenBrowserCanvas(args, result) {- if (!shouldAutoOpenBrowser(args, result)) return;+function syncOpenBrowserCanvas(args, result) {+ if (!shouldSyncOpenBrowserCanvas(args, result)) return;const kvps = args?.kvps || {};const browserId = browserIdFromResult(result, kvps);const key = `${args.id || ""}:${kvps.action || ""}:${browserId || ""}:${result.currentUrl || result.state?.currentUrl || kvps.url || ""}`;- const persistedKey = `a0.browser.autoOpened.${key}`;- if (autoOpenedBrowsers.has(key) || sessionStorage.getItem(persistedKey)) return;- autoOpenedBrowsers.add(key);- sessionStorage.setItem(persistedKey, "1");+ if (syncedBrowserCanvases.has(key)) return;+ syncedBrowserCanvases.add(key);requestAnimationFrame(async () => {+ if (!isBrowserCanvasAlreadyOpen()) return;if (!(await browserAllowsToolAutofocus())) return;- void openBrowserCanvas({+ void rightCanvasStore.open("browser", {browserId,contextId: browserContextIdFromResult(result, kvps),- source: "tool",+ source: "tool-sync",});});}@@ -167,6 +175,6 @@ function drawBrowserTool({actionButtons: actionButtons.filter(Boolean),log: args,});- autoOpenBrowserCanvas(args, browserResult);+ syncOpenBrowserCanvas(args, browserResult);return result;}diff --git a/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js b/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.jsindex 1f99d3732..bd6c7e1fb 100644--- a/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js+++ b/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js@@ -2,18 +2,18 @@ import { store as rightCanvasStore } from "/components/canvas/right-canvas-storeimport { store as browserStore } from "/plugins/_browser/webui/browser-store.js";const AUTO_OPEN_WINDOW_MS = 10 * 60 * 1000;-const BROWSER_MODAL = "/plugins/_browser/webui/main.html";-const autoOpenedBrowsers = new Set();+const syncedBrowserCanvases = new Set();-export default async function autoOpenBrowserResults(context) {+export default async function syncBrowserResultsIntoOpenCanvas(context) {if (!context?.results?.length || context.historyEmpty) return;+ if (!isBrowserCanvasAlreadyOpen()) return;for (const { args } of context.results) {const payload = getToolResultPayload(args);if (getToolName(payload) !== "browser") continue;const result = parseMaybeJson(payload.tool_result) || {};- if (!shouldAutoOpen(args, payload, result)) continue;+ if (!shouldSyncOpenBrowserCanvas(args, payload, result)) continue;const browserId = getBrowserId(payload, result);const contextId = getBrowserContextId(payload, result);@@ -22,12 +22,13 @@ export default async function autoOpenBrowserResults(context) {browserId || "",result.currentUrl || result.state?.currentUrl || payload.url || "",].join(":");- const persistedKey = `a0.browser.autoOpened.${key}`;+ const persistedKey = `a0.browser.synced.${key}`;if (hasOpened(key, persistedKey)) continue;requestAnimationFrame(async () => {+ if (!isBrowserCanvasAlreadyOpen()) return;if (!(await browserAllowsToolAutofocus())) return;- void openBrowserCanvas({ browserId, contextId, source: "tool-result" });+ void syncOpenBrowserCanvas({ browserId, contextId, source: "tool-result-sync" });});}}@@ -79,7 +80,8 @@ function parseMaybeJson(value) {}}-function shouldAutoOpen(args = {}, payload = {}, result = {}) {+function shouldSyncOpenBrowserCanvas(args = {}, payload = {}, result = {}) {+ if (!isBrowserCanvasAlreadyOpen()) return false;if (!isFresh(args.timestamp, payload.last_modified || result.last_modified)) return false;const action = String(payload.action || "").trim().toLowerCase().replace("-", "_");@@ -135,32 +137,30 @@ function toMs(value) {}function hasOpened(key, persistedKey) {- if (autoOpenedBrowsers.has(key)) return true;- autoOpenedBrowsers.add(key);+ if (syncedBrowserCanvases.has(key)) return true;+ syncedBrowserCanvases.add(key);try {if (sessionStorage.getItem(persistedKey)) return true;sessionStorage.setItem(persistedKey, "1");} catch {- // Best-effort persistence; the in-memory guard still prevents repeat opens.+ // Best-effort persistence; the in-memory guard still prevents repeat syncs.}return false;}-async function openBrowserCanvas(payload = {}) {- if (rightCanvasStore?.open) {- await rightCanvasStore.open("browser", payload);- return;- }+async function syncOpenBrowserCanvas(payload = {}) {+ if (!isBrowserCanvasAlreadyOpen()) return;+ await rightCanvasStore.open("browser", payload);+}- if (window.ensureModalOpen) {- await window.ensureModalOpen(BROWSER_MODAL);- return;- }- if (window.openModal) {- await window.openModal(BROWSER_MODAL);- }+function isBrowserCanvasAlreadyOpen() {+ return Boolean(+ rightCanvasStore?.isOpen+ && rightCanvasStore?.activeSurfaceId === "browser"+ && !rightCanvasStore?.isMobileMode,+ );}async function browserAllowsToolAutofocus() {diff --git a/plugins/_browser/prompts/agent.system.tool.browser.md b/plugins/_browser/prompts/agent.system.tool.browser.mdindex b0112ae25..b58b67f9d 100644--- a/plugins/_browser/prompts/agent.system.tool.browser.md+++ b/plugins/_browser/prompts/agent.system.tool.browser.md@@ -1,9 +1,11 @@### browser-direct Playwright browser control with visible WebUI viewer+direct Playwright browser control with optional visible WebUI vieweruse for web browsing, page inspection, forms, downloads, and browser-only tasksstate stays open per chat contextrefs come from content as typed markers: [link 3], [button 6], [image 1], [input text 8]+Browser tool actions must not open the right canvas automatically. Use the tool headlessly unless the user opens the Browser canvas or explicitly asks for a visible browser view; if the Browser canvas is already open, it may reflect the active page.+actions: open list state navigate back forward reload content detail click type submit type_submit scroll evaluate close close_allcommon args: action browser_id url ref text selector selectors scriptdiff --git a/plugins/_browser/webui/config.html b/plugins/_browser/webui/config.htmlindex d28203b95..e1166313c 100644--- a/plugins/_browser/webui/config.html+++ b/plugins/_browser/webui/config.html@@ -18,7 +18,7 @@<div class="browser-config-card"><div class="section-title">Browsing</div><div class="section-description">- Set how new Browser sessions start and whether agent activity should pull focus.+ Set how new Browser sessions start and how an already-open Browser canvas follows agent activity.</div><label class="browser-config-field">@@ -34,7 +34,7 @@<label class="browser-config-switch-row"><span class="browser-config-switch-copy"><span class="browser-config-field-label">Autofocus active page</span>- <span class="browser-config-field-help">Focus pages opened or changed by Browser tool results.</span>+ <span class="browser-config-field-help">Update the visible Browser canvas for pages opened or changed by Browser tool results.</span></span><span class="browser-config-toggle-with-label"><span class="browser-config-toggle-label" x-text="$store.browserConfig.autofocusLabel()"></span>diff --git a/plugins/_office/skills/linux-desktop/SKILL.md b/plugins/_office/skills/linux-desktop/SKILL.mdindex 6e4fca1f2..dd601df9b 100644--- a/plugins/_office/skills/linux-desktop/SKILL.md+++ b/plugins/_office/skills/linux-desktop/SKILL.md@@ -19,15 +19,17 @@ allowed_tools:# Linux Desktop Interface-Use the Desktop as a full Linux GUI when the user needs a visual workflow, an installed desktop app, or manual layout polish that is awkward through structured file edits alone. Use structured tools first for deterministic content changes, then use the Desktop for inspection, GUI-only actions, and final visual confirmation.+Use the Desktop as a full Linux GUI when the user explicitly needs a visual workflow, an installed desktop app, or manual layout polish that is awkward through structured file edits alone. The Desktop is opt-in: do not launch it just because the user asks for a document. Use structured tools first for deterministic content changes, then use the Desktop for inspection, GUI-only actions, and final visual confirmation.## Operating Model1. Prefer `document_artifact` for creating, reading, and editing Markdown, DOCX, XLSX, and PPTX files.-2. Open the Desktop only when the user asks for the Desktop, a GUI app, or visual confirmation.-3. Launch common apps from the Desktop icons, the header buttons, or `scripts/desktopctl.sh`.-4. Use the external Agent Zero Browser for web browsing. Do not launch an operating-system browser in this version.-5. Verify GUI work by observing the desktop state, checking window titles, and saving the file before reporting success.+2. Treat Markdown as first-class. For writing, notes, reports, and drafts with no explicit binary Office requirement, create Markdown and use the custom Markdown editor when the user opens the canvas.+3. Open the Desktop only when the user asks for the Desktop, a GUI app, binary Office visual work, or visual confirmation.+4. Never open the Desktop/canvas automatically from a tool result if the user has not opened it.+5. Launch common apps from the Desktop icons, the header buttons, or `scripts/desktopctl.sh`.+6. Use the external Agent Zero Browser for web browsing. Do not launch an operating-system browser in this version.+7. Verify GUI work by observing the desktop state, checking window titles, and saving the file before reporting success.## Control Flowdiff --git a/plugins/_office/skills/office-artifacts/SKILL.md b/plugins/_office/skills/office-artifacts/SKILL.mdindex 386add51a..a990e99fd 100644--- a/plugins/_office/skills/office-artifacts/SKILL.md+++ b/plugins/_office/skills/office-artifacts/SKILL.md@@ -20,7 +20,9 @@ allowed_tools:# Document Artifacts-Use `document_artifact` for substantial deliverables that should remain editable in the canvas. Markdown is the default document format. Use DOCX only when the user explicitly asks for it or needs a Word-compatible binary file.+Use `document_artifact` for substantial deliverables that should remain editable in the custom document canvas. Markdown is the first-class document format and the default for writing, notes, reports, briefs, and drafts. Use DOCX, XLSX, or PPTX only when the user explicitly asks for that binary format, provides an existing file in that format, or needs a Word/Excel/PowerPoint-compatible artifact.++The canvas is user-owned UI. Creating, reading, or editing an artifact must save the file and update its state, but it must not open the canvas automatically if the user has not opened it. Provide an explicit document action/button path for the user instead.## Workflow@@ -126,6 +128,9 @@ Arguments:- Prefer `file_id` from canvas context or prior tool output; use `path` when that is all you have.- Use `read` before editing unless the current saved content is already known.+- For document-style requests with no requested binary format, create Markdown and let the custom Markdown editor be the primary interactive surface.+- Treat Desktop and LibreOffice as opt-in visual tools for explicit GUI requests, binary Office formats, or final layout inspection.+- Never open the canvas automatically from a tool result. If the user has not opened the canvas, leave the saved artifact available through the normal UI affordance.- Do not create ODT, ODS, or ODP in this pass; return a clear unsupported response if asked.- Use native `create_chart` for embedded spreadsheet charts. Reach for Python/code execution only when the requested chart behavior is not supported by the tool.- Use `edit` for precise saved changes; use the visual document canvas for human/manual layout polish.diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.pyindex 0205f673b..56d9d95b0 100644--- a/tests/test_browser_agent_regressions.py+++ b/tests/test_browser_agent_regressions.py@@ -539,12 +539,22 @@ def test_browser_entry_points_prefer_canvas_and_modal_dock_handoff():assert "releaseSurfaceBindings()" in browser_storeassert "this.releaseSurfaceBindings();" in browser_store+ assert "async function openBrowserCanvas" in tool_handler+ assert 'await rightCanvasStore.open("browser", payload);' in tool_handler+ assert "window.ensureModalOpen" in tool_handler+ assert "window.openModal" in tool_handler+ assert "function syncOpenBrowserCanvas" in tool_handler+ assert "async function syncOpenBrowserCanvas" in after_loop_handler+ assert "syncBrowserResultsIntoOpenCanvas" in after_loop_handler+ assert "window.ensureModalOpen" not in after_loop_handler+ assert "window.openModal" not in after_loop_handler+for js in (tool_handler, after_loop_handler):- assert "async function openBrowserCanvas" in jsassert "openBrowserModal" not in js- assert js.index('await rightCanvasStore.open("browser", payload);') < js.index("if (window.ensureModalOpen)")-- assert "function autoOpenBrowserCanvas" in tool_handler+ assert "isBrowserCanvasAlreadyOpen" in js+ assert "rightCanvasStore?.isOpen" in js+ assert 'rightCanvasStore?.activeSurfaceId === "browser"' in js+ assert "autoOpenBrowserCanvas" not in jsfor js in (tool_handler, after_loop_handler, register_js, browser_store, modals_js):assert "globalThis.Alpine" not in js@@ -552,6 +562,23 @@ def test_browser_entry_points_prefer_canvas_and_modal_dock_handoff():assert "Alpine.store" not in js+def test_browser_tool_does_not_auto_open_canvas_policy_is_documented():+ prompt = (+ PROJECT_ROOT / "plugins" / "_browser" / "prompts" / "agent.system.tool.browser.md"+ ).read_text(encoding="utf-8")+ config = (PROJECT_ROOT / "plugins" / "_browser" / "default_config.yaml").read_text(+ encoding="utf-8"+ )+ config_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "config.html").read_text(+ encoding="utf-8"+ )++ assert "must not open the right canvas automatically" in prompt+ assert "Use the tool headlessly unless the user opens the Browser canvas" in prompt+ assert "already open" in config+ assert "already-open Browser canvas" in config_html++def test_browser_canvas_uses_plain_panel_without_debug_probe():panel_html = (PROJECT_ROOTdiff --git a/tests/test_office_canvas_setup.py b/tests/test_office_canvas_setup.pyindex 5f1435956..b6475afc0 100644--- a/tests/test_office_canvas_setup.py+++ b/tests/test_office_canvas_setup.py@@ -304,3 +304,19 @@ def test_right_canvas_requires_explicit_open_and_is_absent_on_mobile():assert "display: none !important" in canvas_cssassert "autoOpenOfficeCanvas" not in handlerassert "requestAnimationFrame" not in after_loop+++def test_office_skills_preserve_markdown_first_and_opt_in_desktop_policy():+ office_skill = (+ PROJECT_ROOT / "plugins" / "_office" / "skills" / "office-artifacts" / "SKILL.md"+ ).read_text(encoding="utf-8")+ desktop_skill = (+ PROJECT_ROOT / "plugins" / "_office" / "skills" / "linux-desktop" / "SKILL.md"+ ).read_text(encoding="utf-8")++ assert "Markdown is the first-class document format" in office_skill+ assert "custom document canvas" in office_skill+ assert "must not open the canvas automatically" in office_skill+ assert "The Desktop is opt-in" in desktop_skill+ assert "custom Markdown editor" in desktop_skill+ assert "Never open the Desktop/canvas automatically" in desktop_skill