Agent Zero AI framework
Harden browser multi-tab focus handling
Alessandro12b96ae41e07
diff --git a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.jsindex 117a2e64b..b8c0d3e33 100644--- a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js+++ b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js@@ -14,7 +14,7 @@ import {const BROWSER_MODAL = "/plugins/_browser/webui/main.html";const AUTO_OPEN_WINDOW_MS = 10 * 60 * 1000;-const autoOpenedBrowsers = new Set();+const syncedBrowserCanvases = new Set();export default async function registerBrowserToolHandler(extData) {if (extData?.tool_name === "browser") {@@ -86,8 +86,17 @@ function isFreshToolMessage(timestamp) {return Math.abs(Date.now() - messageMs) <= AUTO_OPEN_WINDOW_MS;}-// Allowlist: only these actions cause the viewer to follow. Background work-// (evaluate, click, type, key_chord, mouse, multi, ...) does not steal focus.+function isBrowserCanvasAlreadyOpen() {+ return Boolean(+ rightCanvasStore?.isOpen+ && rightCanvasStore?.activeSurfaceId === "browser"+ && !rightCanvasStore?.isMobileMode,+ );+}++// Allowlist: only these actions sync an already-open viewer to the target tab.+// Background work (evaluate, click, type, key_chord, mouse, multi, ...) does+// not steal focus.const FOCUS_ACTIONS = new Set(["open","navigate",@@ -97,28 +106,28 @@ const FOCUS_ACTIONS = new Set(["focus",]);-function shouldAutoOpenBrowser(args, result) {+function shouldSyncOpenBrowserCanvas(args, result) {+ if (!isBrowserCanvasAlreadyOpen()) return false;if (!isFreshToolMessage(args?.timestamp)) return false;const action = String(args?.kvps?.action || "").trim().toLowerCase().replace("-", "_");if (!FOCUS_ACTIONS.has(action)) return false;return Boolean(browserIdFromResult(result, args?.kvps || {}));}-function autoOpenBrowserCanvas(args, result) {- if (!shouldAutoOpenBrowser(args, result)) return;+function syncOpenBrowserCanvas(args, result) {+ if (!shouldSyncOpenBrowserCanvas(args, result)) return;const kvps = args?.kvps || {};const browserId = browserIdFromResult(result, kvps);const key = `${args.id || ""}:${kvps.action || ""}:${browserId || ""}:${result.currentUrl || result.state?.currentUrl || kvps.url || ""}`;- const persistedKey = `a0.browser.autoOpened.${key}`;- if (autoOpenedBrowsers.has(key) || sessionStorage.getItem(persistedKey)) return;- autoOpenedBrowsers.add(key);- sessionStorage.setItem(persistedKey, "1");+ if (syncedBrowserCanvases.has(key)) return;+ syncedBrowserCanvases.add(key);requestAnimationFrame(async () => {+ if (!isBrowserCanvasAlreadyOpen()) return;if (!(await browserAllowsToolAutofocus())) return;- void openBrowserCanvas({+ void rightCanvasStore.open("browser", {browserId,contextId: browserContextIdFromResult(result, kvps),- source: "tool",+ source: "tool-sync",});});}@@ -178,6 +187,6 @@ function drawBrowserTool({actionButtons: actionButtons.filter(Boolean),log: args,});- autoOpenBrowserCanvas(args, browserResult);+ syncOpenBrowserCanvas(args, browserResult);return result;}diff --git a/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js b/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.jsindex b9cd53646..bc296f0a5 100644--- a/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js+++ b/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js@@ -2,18 +2,18 @@ import { store as rightCanvasStore } from "/components/canvas/right-canvas-storeimport { store as browserStore } from "/plugins/_browser/webui/browser-store.js";const AUTO_OPEN_WINDOW_MS = 10 * 60 * 1000;-const BROWSER_MODAL = "/plugins/_browser/webui/main.html";-const autoOpenedBrowsers = new Set();+const syncedBrowserCanvases = new Set();-export default async function autoOpenBrowserResults(context) {+export default async function syncBrowserResultsIntoOpenCanvas(context) {if (!context?.results?.length || context.historyEmpty) return;+ if (!isBrowserCanvasAlreadyOpen()) return;for (const { args } of context.results) {const payload = getToolResultPayload(args);if (getToolName(payload) !== "browser") continue;const result = parseMaybeJson(payload.tool_result) || {};- if (!shouldAutoOpen(args, payload, result)) continue;+ if (!shouldSyncOpenBrowserCanvas(args, payload, result)) continue;const browserId = getBrowserId(payload, result);const contextId = getBrowserContextId(payload, result);@@ -22,12 +22,13 @@ export default async function autoOpenBrowserResults(context) {browserId || "",result.currentUrl || result.state?.currentUrl || payload.url || "",].join(":");- const persistedKey = `a0.browser.autoOpened.${key}`;+ const persistedKey = `a0.browser.synced.${key}`;if (hasOpened(key, persistedKey)) continue;requestAnimationFrame(async () => {+ if (!isBrowserCanvasAlreadyOpen()) return;if (!(await browserAllowsToolAutofocus())) return;- void openBrowserCanvas({ browserId, contextId, source: "tool-result" });+ void syncOpenBrowserCanvas({ browserId, contextId, source: "tool-result-sync" });});}}@@ -79,9 +80,9 @@ function parseMaybeJson(value) {}}-// Actions that should bring the viewer to the targeted tab. Everything else-// (read, click, type, evaluate, key_chord, mouse, multi, ...) leaves the-// viewer where it is so cross-tab work doesn't steal user focus.+// Actions that should sync an already-open viewer to the targeted tab.+// Everything else (read, click, type, evaluate, key_chord, mouse, multi, ...)+// leaves the viewer where it is so cross-tab work doesn't steal user focus.const FOCUS_ACTIONS = new Set(["open","navigate",@@ -91,7 +92,8 @@ const FOCUS_ACTIONS = new Set(["focus",]);-function shouldAutoOpen(args = {}, payload = {}, result = {}) {+function shouldSyncOpenBrowserCanvas(args = {}, payload = {}, result = {}) {+ if (!isBrowserCanvasAlreadyOpen()) return false;if (!isFresh(args.timestamp, payload.last_modified || result.last_modified)) return false;const action = String(payload.action || "").trim().toLowerCase().replace("-", "_");if (!FOCUS_ACTIONS.has(action)) return false;@@ -139,32 +141,30 @@ function toMs(value) {}function hasOpened(key, persistedKey) {- if (autoOpenedBrowsers.has(key)) return true;- autoOpenedBrowsers.add(key);+ if (syncedBrowserCanvases.has(key)) return true;+ syncedBrowserCanvases.add(key);try {if (sessionStorage.getItem(persistedKey)) return true;sessionStorage.setItem(persistedKey, "1");} catch {- // Best-effort persistence; the in-memory guard still prevents repeat opens.+ // Best-effort persistence; the in-memory guard still prevents repeat syncs.}return false;}-async function openBrowserCanvas(payload = {}) {- if (rightCanvasStore?.open) {- await rightCanvasStore.open("browser", payload);- return;- }+async function syncOpenBrowserCanvas(payload = {}) {+ if (!isBrowserCanvasAlreadyOpen()) return;+ await rightCanvasStore.open("browser", payload);+}- if (window.ensureModalOpen) {- await window.ensureModalOpen(BROWSER_MODAL);- return;- }- if (window.openModal) {- await window.openModal(BROWSER_MODAL);- }+function isBrowserCanvasAlreadyOpen() {+ return Boolean(+ rightCanvasStore?.isOpen+ && rightCanvasStore?.activeSurfaceId === "browser"+ && !rightCanvasStore?.isMobileMode,+ );}async function browserAllowsToolAutofocus() {diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.pyindex 975904e35..00a3ad3b3 100644--- a/plugins/_browser/helpers/runtime.py+++ b/plugins/_browser/helpers/runtime.py@@ -369,6 +369,49 @@ class _BrowserRuntimeCore:if current is None or current == resolved_id:self.last_interacted_browser_id = int(resolved_id)+ def _background_focus_target(+ self,+ previous_focus: int | None,+ fallback_id: int,+ ) -> int | None:+ if previous_focus in self.pages:+ return int(previous_focus)+ if fallback_id in self.pages:+ return int(fallback_id)+ return next(iter(sorted(self.pages)), None)++ def _normalize_modifiers(self, modifiers: list[str] | str | None) -> list[str] | None:+ if modifiers is None:+ return None+ if isinstance(modifiers, str):+ raw = [modifiers]+ elif isinstance(modifiers, list):+ raw = modifiers+ else:+ raise ValueError("modifiers must be a string or list")+ normalized = [str(modifier).strip() for modifier in raw if str(modifier).strip()]+ if not normalized:+ return None+ bad = set(normalized) - self._VALID_MODIFIERS+ if bad:+ raise ValueError(+ f"unsupported modifiers: {sorted(bad)}; allowed: {sorted(self._VALID_MODIFIERS)}"+ )+ return normalized++ @staticmethod+ def _multi_group_key(call: dict[str, Any]) -> Any:+ value = call.get("browser_id")+ if value is None or str(value).strip() == "":+ return None+ raw = str(value).strip()+ if raw.startswith("browser-"):+ raw = raw.split("-", 1)[1]+ try:+ return int(raw)+ except ValueError:+ return raw+@propertydef profile_dir(self) -> Path:return Path(files.get_abs_path("tmp/browser/sessions", self.safe_context_id))@@ -560,7 +603,7 @@ class _BrowserRuntimeCore:for idx, call in enumerate(calls):if not isinstance(call, dict):raise ValueError(f"calls[{idx}] is not an object")- key = call.get("browser_id")+ key = self._multi_group_key(call)groups.setdefault(key, []).append((idx, call))results: list[dict[str, Any] | None] = [None] * len(calls)@@ -615,7 +658,7 @@ class _BrowserRuntimeCore:raise ValueError("click requires ref")return await self.click(bid, ref,- modifiers=call.get("modifiers"),+ modifiers=self._normalize_modifiers(call.get("modifiers")),focus_popup=call.get("focus_popup"),)if action == "type":@@ -650,7 +693,7 @@ class _BrowserRuntimeCore:bid, call.get("event_type") or "click",float(call.get("x") or 0), float(call.get("y") or 0),button=call.get("button") or "left",- modifiers=call.get("modifiers"),+ modifiers=self._normalize_modifiers(call.get("modifiers")),)if action == "close":return await self.close_browser(bid)@@ -760,9 +803,10 @@ class _BrowserRuntimeCore:self,browser_id: int | str | None,reference_id: int | str,- modifiers: list[str] | None = None,+ modifiers: list[str] | str | None = None,focus_popup: bool | None = None,) -> dict[str, Any]:+ modifiers = self._normalize_modifiers(modifiers)if modifiers:return await self._modifier_click(browser_id, reference_id, modifiers, focus_popup)return await self._reference_action("click", browser_id, reference_id)@@ -774,13 +818,9 @@ class _BrowserRuntimeCore:modifiers: list[str],focus_popup: bool | None,) -> dict[str, Any]:- bad = set(modifiers) - self._VALID_MODIFIERS- if bad:- raise ValueError(- f"unsupported modifiers: {sorted(bad)}; allowed: {sorted(self._VALID_MODIFIERS)}"- )await self.ensure_started()resolved_id = self._resolve_browser_id(browser_id)+ previous_focus = self.last_interacted_browser_idpage = self._page(resolved_id)await self._ensure_content_helper(page)@@ -843,17 +883,24 @@ class _BrowserRuntimeCore:waiter.cancel()if opened_id is not None and background:- self._background_popup_pages.add(opened_id)if self.last_interacted_browser_id == opened_id:- # Force focus back to origin tab — popup hook had promoted.- self.last_interacted_browser_id = int(resolved_id)+ # Force focus back to the tab that was active before the+ # background click; the popup hook may have promoted.+ self.last_interacted_browser_id = self._background_focus_target(+ previous_focus,+ resolved_id,+ )finally:if waiter in self._pending_popups:self._pending_popups.remove(waiter)if background:- # Background-mode click: explicitly keep focus on origin.- self.last_interacted_browser_id = int(resolved_id)+ # Background-mode click: preserve the pre-click focus even when+ # the clicked tab itself was not active.+ self.last_interacted_browser_id = self._background_focus_target(+ previous_focus,+ resolved_id,+ )return {"action": {"ref": reference_id,@@ -1062,17 +1109,13 @@ class _BrowserRuntimeCore:x: float,y: float,button: str = "left",- modifiers: list[str] | None = None,+ modifiers: list[str] | str | None = None,) -> dict[str, Any]:event_type_lower = str(event_type or "click").lower()+ modifiers = self._normalize_modifiers(modifiers)if modifiers:if event_type_lower != "click":raise ValueError("modifiers are only valid for event_type='click'")- bad = set(modifiers) - self._VALID_MODIFIERS- if bad:- raise ValueError(- f"unsupported modifiers: {sorted(bad)}; allowed: {sorted(self._VALID_MODIFIERS)}"- )await self.ensure_started()resolved_id = self._resolve_browser_id(browser_id)page = self._page(resolved_id)@@ -1342,7 +1385,7 @@ class _BrowserRuntimeCore:async def _ensure_content_helper(self, page: Any) -> None:has_helper = await page.evaluate(- "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture && globalThis.__spaceBrowserPageContent__?.annotate)"+ "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture && globalThis.__spaceBrowserPageContent__?.annotate && globalThis.__spaceBrowserPageContent__?.boundingBoxFor)")if has_helper:returndiff --git a/plugins/_browser/prompts/agent.system.tool.browser.md b/plugins/_browser/prompts/agent.system.tool.browser.mdindex e8c271792..17d14a489 100644--- a/plugins/_browser/prompts/agent.system.tool.browser.md+++ b/plugins/_browser/prompts/agent.system.tool.browser.md@@ -1,9 +1,11 @@### browser-direct Playwright browser control with visible WebUI viewer+direct Playwright browser control with optional visible WebUI vieweruse for web browsing, page inspection, forms, downloads, and browser-only tasksstate stays open per chat contextrefs come from content as typed markers: [link 3], [button 6], [image 1], [input text 8]+Browser tool actions must not open the right canvas automatically. Use the tool headlessly unless the user opens the Browser canvas or explicitly asks for a visible browser view; if the Browser canvas is already open, it may reflect the active page.+actions: open list state set_active navigate back forward reload content detail click type submit type_submit scroll evaluate key_chord mouse multi close close_allcommon args: action browser_id url ref text selector selectors script modifiers keys include_content focus_popup event_type x y button callsdiff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.pyindex 0205f673b..79fe36ec9 100644--- a/tests/test_browser_agent_regressions.py+++ b/tests/test_browser_agent_regressions.py@@ -539,12 +539,25 @@ def test_browser_entry_points_prefer_canvas_and_modal_dock_handoff():assert "releaseSurfaceBindings()" in browser_storeassert "this.releaseSurfaceBindings();" in browser_store- for js in (tool_handler, after_loop_handler):- assert "async function openBrowserCanvas" in js- assert "openBrowserModal" not in js- assert js.index('await rightCanvasStore.open("browser", payload);') < js.index("if (window.ensureModalOpen)")+ assert "async function openBrowserCanvas" in tool_handler+ assert "openBrowserModal" not in tool_handler+ assert tool_handler.index('await rightCanvasStore.open("browser", payload);') < tool_handler.index("if (window.ensureModalOpen)")+ assert "function syncOpenBrowserCanvas" in tool_handler+ assert "function autoOpenBrowserCanvas" not in tool_handler+ assert "function shouldSyncOpenBrowserCanvas" in tool_handler+ assert "function isBrowserCanvasAlreadyOpen" in tool_handler++ assert "async function syncOpenBrowserCanvas" in after_loop_handler+ assert "syncBrowserResultsIntoOpenCanvas" in after_loop_handler+ assert "openBrowserCanvas" not in after_loop_handler+ assert "BROWSER_MODAL" not in after_loop_handler- assert "function autoOpenBrowserCanvas" in tool_handler+ for js in (tool_handler, after_loop_handler):+ assert "syncedBrowserCanvases" in js+ assert "autoOpenedBrowsers" not in js+ assert 'rightCanvasStore?.activeSurfaceId === "browser"' in js+ assert "const FOCUS_ACTIONS = new Set" in js+ assert "FOCUS_ACTIONS.has(action)" in jsfor js in (tool_handler, after_loop_handler, register_js, browser_store, modals_js):assert "globalThis.Alpine" not in js@@ -552,6 +565,16 @@ def test_browser_entry_points_prefer_canvas_and_modal_dock_handoff():assert "Alpine.store" not in js+def test_browser_tool_does_not_auto_open_canvas_policy_is_documented():+ prompt = (+ PROJECT_ROOT / "plugins" / "_browser" / "prompts" / "agent.system.tool.browser.md"+ ).read_text(encoding="utf-8")++ assert "optional visible WebUI viewer" in prompt+ assert "must not open the right canvas automatically" in prompt+ assert "Use the tool headlessly unless the user opens the Browser canvas" in prompt++def test_browser_canvas_uses_plain_panel_without_debug_probe():panel_html = (PROJECT_ROOT@@ -787,6 +810,14 @@ def test_browser_runtime_and_content_helper_expose_annotation_target():assert "password" in helper+def test_browser_runtime_requires_current_content_helper_for_modifier_clicks():+ runtime = (+ PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"+ ).read_text(encoding="utf-8")++ assert "__spaceBrowserPageContent__?.boundingBoxFor" in runtime++@pytest.mark.anyioasync def test_browser_screencast_acknowledges_and_drops_stale_frames():first_image = SMALL_JPEG_10X10@@ -1449,6 +1480,33 @@ async def test_browser_viewer_annotation_dispatches_runtime(monkeypatch):assert calls == [("annotation_target", (4, payload), {})]+def test_browser_runtime_normalizes_multi_group_ids_and_modifiers():+ core = _BrowserRuntimeCore("ctx")++ assert core._multi_group_key({"browser_id": 7}) == 7+ assert core._multi_group_key({"browser_id": "7"}) == 7+ assert core._multi_group_key({"browser_id": "browser-7"}) == 7+ assert core._multi_group_key({"browser_id": ""}) is None+ assert core._normalize_modifiers("Control") == ["Control"]+ assert core._normalize_modifiers(["Control", " Shift "]) == ["Control", "Shift"]+ assert core._normalize_modifiers([]) is None++ with pytest.raises(ValueError):+ core._normalize_modifiers("Ctrl")+++def test_browser_runtime_background_focus_restores_previous_active_tab():+ core = _BrowserRuntimeCore("ctx")+ core.pages[1] = browser_runtime_module.BrowserPage(id=1, page=object())+ core.pages[2] = browser_runtime_module.BrowserPage(id=2, page=object())++ assert core._background_focus_target(previous_focus=1, fallback_id=2) == 1++ core.pages.pop(1)++ assert core._background_focus_target(previous_focus=1, fallback_id=2) == 2++def test_browser_cleanup_extensions_follow_extensible_path_layout():extension = __import__("helpers.extension", fromlist=["_get_extension_classes"])remove_classes = extension._get_extension_classes( # type: ignore[attr-defined]