Skip to content

Agent Zero AI framework

Harden browser multi-tab focus handling

Alessandro12b96ae41e07
diff --git a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
index 117a2e64b..b8c0d3e33 100644
--- a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
+++ b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
@@ -14,7 +14,7 @@ import {
const BROWSER_MODAL = "/plugins/_browser/webui/main.html";
const AUTO_OPEN_WINDOW_MS = 10 * 60 * 1000;
-const autoOpenedBrowsers = new Set();
+const syncedBrowserCanvases = new Set();
export default async function registerBrowserToolHandler(extData) {
if (extData?.tool_name === "browser") {
@@ -86,8 +86,17 @@ function isFreshToolMessage(timestamp) {
return Math.abs(Date.now() - messageMs) <= AUTO_OPEN_WINDOW_MS;
}
-// Allowlist: only these actions cause the viewer to follow. Background work
-// (evaluate, click, type, key_chord, mouse, multi, ...) does not steal focus.
+function isBrowserCanvasAlreadyOpen() {
+ return Boolean(
+ rightCanvasStore?.isOpen
+ && rightCanvasStore?.activeSurfaceId === "browser"
+ && !rightCanvasStore?.isMobileMode,
+ );
+}
+
+// Allowlist: only these actions sync an already-open viewer to the target tab.
+// Background work (evaluate, click, type, key_chord, mouse, multi, ...) does
+// not steal focus.
const FOCUS_ACTIONS = new Set([
"open",
"navigate",
@@ -97,28 +106,28 @@ const FOCUS_ACTIONS = new Set([
"focus",
]);
-function shouldAutoOpenBrowser(args, result) {
+function shouldSyncOpenBrowserCanvas(args, result) {
+ if (!isBrowserCanvasAlreadyOpen()) return false;
if (!isFreshToolMessage(args?.timestamp)) return false;
const action = String(args?.kvps?.action || "").trim().toLowerCase().replace("-", "_");
if (!FOCUS_ACTIONS.has(action)) return false;
return Boolean(browserIdFromResult(result, args?.kvps || {}));
}
-function autoOpenBrowserCanvas(args, result) {
- if (!shouldAutoOpenBrowser(args, result)) return;
+function syncOpenBrowserCanvas(args, result) {
+ if (!shouldSyncOpenBrowserCanvas(args, result)) return;
const kvps = args?.kvps || {};
const browserId = browserIdFromResult(result, kvps);
const key = `${args.id || ""}:${kvps.action || ""}:${browserId || ""}:${result.currentUrl || result.state?.currentUrl || kvps.url || ""}`;
- const persistedKey = `a0.browser.autoOpened.${key}`;
- if (autoOpenedBrowsers.has(key) || sessionStorage.getItem(persistedKey)) return;
- autoOpenedBrowsers.add(key);
- sessionStorage.setItem(persistedKey, "1");
+ if (syncedBrowserCanvases.has(key)) return;
+ syncedBrowserCanvases.add(key);
requestAnimationFrame(async () => {
+ if (!isBrowserCanvasAlreadyOpen()) return;
if (!(await browserAllowsToolAutofocus())) return;
- void openBrowserCanvas({
+ void rightCanvasStore.open("browser", {
browserId,
contextId: browserContextIdFromResult(result, kvps),
- source: "tool",
+ source: "tool-sync",
});
});
}
@@ -178,6 +187,6 @@ function drawBrowserTool({
actionButtons: actionButtons.filter(Boolean),
log: args,
});
- autoOpenBrowserCanvas(args, browserResult);
+ syncOpenBrowserCanvas(args, browserResult);
return result;
}
diff --git a/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js b/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js
index b9cd53646..bc296f0a5 100644
--- a/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js
+++ b/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js
@@ -2,18 +2,18 @@ import { store as rightCanvasStore } from "/components/canvas/right-canvas-store
import { store as browserStore } from "/plugins/_browser/webui/browser-store.js";
const AUTO_OPEN_WINDOW_MS = 10 * 60 * 1000;
-const BROWSER_MODAL = "/plugins/_browser/webui/main.html";
-const autoOpenedBrowsers = new Set();
+const syncedBrowserCanvases = new Set();
-export default async function autoOpenBrowserResults(context) {
+export default async function syncBrowserResultsIntoOpenCanvas(context) {
if (!context?.results?.length || context.historyEmpty) return;
+ if (!isBrowserCanvasAlreadyOpen()) return;
for (const { args } of context.results) {
const payload = getToolResultPayload(args);
if (getToolName(payload) !== "browser") continue;
const result = parseMaybeJson(payload.tool_result) || {};
- if (!shouldAutoOpen(args, payload, result)) continue;
+ if (!shouldSyncOpenBrowserCanvas(args, payload, result)) continue;
const browserId = getBrowserId(payload, result);
const contextId = getBrowserContextId(payload, result);
@@ -22,12 +22,13 @@ export default async function autoOpenBrowserResults(context) {
browserId || "",
result.currentUrl || result.state?.currentUrl || payload.url || "",
].join(":");
- const persistedKey = `a0.browser.autoOpened.${key}`;
+ const persistedKey = `a0.browser.synced.${key}`;
if (hasOpened(key, persistedKey)) continue;
requestAnimationFrame(async () => {
+ if (!isBrowserCanvasAlreadyOpen()) return;
if (!(await browserAllowsToolAutofocus())) return;
- void openBrowserCanvas({ browserId, contextId, source: "tool-result" });
+ void syncOpenBrowserCanvas({ browserId, contextId, source: "tool-result-sync" });
});
}
}
@@ -79,9 +80,9 @@ function parseMaybeJson(value) {
}
}
-// Actions that should bring the viewer to the targeted tab. Everything else
-// (read, click, type, evaluate, key_chord, mouse, multi, ...) leaves the
-// viewer where it is so cross-tab work doesn't steal user focus.
+// Actions that should sync an already-open viewer to the targeted tab.
+// Everything else (read, click, type, evaluate, key_chord, mouse, multi, ...)
+// leaves the viewer where it is so cross-tab work doesn't steal user focus.
const FOCUS_ACTIONS = new Set([
"open",
"navigate",
@@ -91,7 +92,8 @@ const FOCUS_ACTIONS = new Set([
"focus",
]);
-function shouldAutoOpen(args = {}, payload = {}, result = {}) {
+function shouldSyncOpenBrowserCanvas(args = {}, payload = {}, result = {}) {
+ if (!isBrowserCanvasAlreadyOpen()) return false;
if (!isFresh(args.timestamp, payload.last_modified || result.last_modified)) return false;
const action = String(payload.action || "").trim().toLowerCase().replace("-", "_");
if (!FOCUS_ACTIONS.has(action)) return false;
@@ -139,32 +141,30 @@ function toMs(value) {
}
function hasOpened(key, persistedKey) {
- if (autoOpenedBrowsers.has(key)) return true;
- autoOpenedBrowsers.add(key);
+ if (syncedBrowserCanvases.has(key)) return true;
+ syncedBrowserCanvases.add(key);
try {
if (sessionStorage.getItem(persistedKey)) return true;
sessionStorage.setItem(persistedKey, "1");
} catch {
- // Best-effort persistence; the in-memory guard still prevents repeat opens.
+ // Best-effort persistence; the in-memory guard still prevents repeat syncs.
}
return false;
}
-async function openBrowserCanvas(payload = {}) {
- if (rightCanvasStore?.open) {
- await rightCanvasStore.open("browser", payload);
- return;
- }
+async function syncOpenBrowserCanvas(payload = {}) {
+ if (!isBrowserCanvasAlreadyOpen()) return;
+ await rightCanvasStore.open("browser", payload);
+}
- if (window.ensureModalOpen) {
- await window.ensureModalOpen(BROWSER_MODAL);
- return;
- }
- if (window.openModal) {
- await window.openModal(BROWSER_MODAL);
- }
+function isBrowserCanvasAlreadyOpen() {
+ return Boolean(
+ rightCanvasStore?.isOpen
+ && rightCanvasStore?.activeSurfaceId === "browser"
+ && !rightCanvasStore?.isMobileMode,
+ );
}
async function browserAllowsToolAutofocus() {
diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py
index 975904e35..00a3ad3b3 100644
--- a/plugins/_browser/helpers/runtime.py
+++ b/plugins/_browser/helpers/runtime.py
@@ -369,6 +369,49 @@ class _BrowserRuntimeCore:
if current is None or current == resolved_id:
self.last_interacted_browser_id = int(resolved_id)
+ def _background_focus_target(
+ self,
+ previous_focus: int | None,
+ fallback_id: int,
+ ) -> int | None:
+ if previous_focus in self.pages:
+ return int(previous_focus)
+ if fallback_id in self.pages:
+ return int(fallback_id)
+ return next(iter(sorted(self.pages)), None)
+
+ def _normalize_modifiers(self, modifiers: list[str] | str | None) -> list[str] | None:
+ if modifiers is None:
+ return None
+ if isinstance(modifiers, str):
+ raw = [modifiers]
+ elif isinstance(modifiers, list):
+ raw = modifiers
+ else:
+ raise ValueError("modifiers must be a string or list")
+ normalized = [str(modifier).strip() for modifier in raw if str(modifier).strip()]
+ if not normalized:
+ return None
+ bad = set(normalized) - self._VALID_MODIFIERS
+ if bad:
+ raise ValueError(
+ f"unsupported modifiers: {sorted(bad)}; allowed: {sorted(self._VALID_MODIFIERS)}"
+ )
+ return normalized
+
+ @staticmethod
+ def _multi_group_key(call: dict[str, Any]) -> Any:
+ value = call.get("browser_id")
+ if value is None or str(value).strip() == "":
+ return None
+ raw = str(value).strip()
+ if raw.startswith("browser-"):
+ raw = raw.split("-", 1)[1]
+ try:
+ return int(raw)
+ except ValueError:
+ return raw
+
@property
def profile_dir(self) -> Path:
return Path(files.get_abs_path("tmp/browser/sessions", self.safe_context_id))
@@ -560,7 +603,7 @@ class _BrowserRuntimeCore:
for idx, call in enumerate(calls):
if not isinstance(call, dict):
raise ValueError(f"calls[{idx}] is not an object")
- key = call.get("browser_id")
+ key = self._multi_group_key(call)
groups.setdefault(key, []).append((idx, call))
results: list[dict[str, Any] | None] = [None] * len(calls)
@@ -615,7 +658,7 @@ class _BrowserRuntimeCore:
raise ValueError("click requires ref")
return await self.click(
bid, ref,
- modifiers=call.get("modifiers"),
+ modifiers=self._normalize_modifiers(call.get("modifiers")),
focus_popup=call.get("focus_popup"),
)
if action == "type":
@@ -650,7 +693,7 @@ class _BrowserRuntimeCore:
bid, call.get("event_type") or "click",
float(call.get("x") or 0), float(call.get("y") or 0),
button=call.get("button") or "left",
- modifiers=call.get("modifiers"),
+ modifiers=self._normalize_modifiers(call.get("modifiers")),
)
if action == "close":
return await self.close_browser(bid)
@@ -760,9 +803,10 @@ class _BrowserRuntimeCore:
self,
browser_id: int | str | None,
reference_id: int | str,
- modifiers: list[str] | None = None,
+ modifiers: list[str] | str | None = None,
focus_popup: bool | None = None,
) -> dict[str, Any]:
+ modifiers = self._normalize_modifiers(modifiers)
if modifiers:
return await self._modifier_click(browser_id, reference_id, modifiers, focus_popup)
return await self._reference_action("click", browser_id, reference_id)
@@ -774,13 +818,9 @@ class _BrowserRuntimeCore:
modifiers: list[str],
focus_popup: bool | None,
) -> dict[str, Any]:
- bad = set(modifiers) - self._VALID_MODIFIERS
- if bad:
- raise ValueError(
- f"unsupported modifiers: {sorted(bad)}; allowed: {sorted(self._VALID_MODIFIERS)}"
- )
await self.ensure_started()
resolved_id = self._resolve_browser_id(browser_id)
+ previous_focus = self.last_interacted_browser_id
page = self._page(resolved_id)
await self._ensure_content_helper(page)
@@ -843,17 +883,24 @@ class _BrowserRuntimeCore:
waiter.cancel()
if opened_id is not None and background:
- self._background_popup_pages.add(opened_id)
if self.last_interacted_browser_id == opened_id:
- # Force focus back to origin tab — popup hook had promoted.
- self.last_interacted_browser_id = int(resolved_id)
+ # Force focus back to the tab that was active before the
+ # background click; the popup hook may have promoted.
+ self.last_interacted_browser_id = self._background_focus_target(
+ previous_focus,
+ resolved_id,
+ )
finally:
if waiter in self._pending_popups:
self._pending_popups.remove(waiter)
if background:
- # Background-mode click: explicitly keep focus on origin.
- self.last_interacted_browser_id = int(resolved_id)
+ # Background-mode click: preserve the pre-click focus even when
+ # the clicked tab itself was not active.
+ self.last_interacted_browser_id = self._background_focus_target(
+ previous_focus,
+ resolved_id,
+ )
return {
"action": {
"ref": reference_id,
@@ -1062,17 +1109,13 @@ class _BrowserRuntimeCore:
x: float,
y: float,
button: str = "left",
- modifiers: list[str] | None = None,
+ modifiers: list[str] | str | None = None,
) -> dict[str, Any]:
event_type_lower = str(event_type or "click").lower()
+ modifiers = self._normalize_modifiers(modifiers)
if modifiers:
if event_type_lower != "click":
raise ValueError("modifiers are only valid for event_type='click'")
- bad = set(modifiers) - self._VALID_MODIFIERS
- if bad:
- raise ValueError(
- f"unsupported modifiers: {sorted(bad)}; allowed: {sorted(self._VALID_MODIFIERS)}"
- )
await self.ensure_started()
resolved_id = self._resolve_browser_id(browser_id)
page = self._page(resolved_id)
@@ -1342,7 +1385,7 @@ class _BrowserRuntimeCore:
async def _ensure_content_helper(self, page: Any) -> None:
has_helper = await page.evaluate(
- "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture && globalThis.__spaceBrowserPageContent__?.annotate)"
+ "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture && globalThis.__spaceBrowserPageContent__?.annotate && globalThis.__spaceBrowserPageContent__?.boundingBoxFor)"
)
if has_helper:
return
diff --git a/plugins/_browser/prompts/agent.system.tool.browser.md b/plugins/_browser/prompts/agent.system.tool.browser.md
index e8c271792..17d14a489 100644
--- a/plugins/_browser/prompts/agent.system.tool.browser.md
+++ b/plugins/_browser/prompts/agent.system.tool.browser.md
@@ -1,9 +1,11 @@
### browser
-direct Playwright browser control with visible WebUI viewer
+direct Playwright browser control with optional visible WebUI viewer
use for web browsing, page inspection, forms, downloads, and browser-only tasks
state stays open per chat context
refs come from content as typed markers: [link 3], [button 6], [image 1], [input text 8]
+Browser tool actions must not open the right canvas automatically. Use the tool headlessly unless the user opens the Browser canvas or explicitly asks for a visible browser view; if the Browser canvas is already open, it may reflect the active page.
+
actions: open list state set_active navigate back forward reload content detail click type submit type_submit scroll evaluate key_chord mouse multi close close_all
common args: action browser_id url ref text selector selectors script modifiers keys include_content focus_popup event_type x y button calls
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index 0205f673b..79fe36ec9 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -539,12 +539,25 @@ def test_browser_entry_points_prefer_canvas_and_modal_dock_handoff():
assert "releaseSurfaceBindings()" in browser_store
assert "this.releaseSurfaceBindings();" in browser_store
- for js in (tool_handler, after_loop_handler):
- assert "async function openBrowserCanvas" in js
- assert "openBrowserModal" not in js
- assert js.index('await rightCanvasStore.open("browser", payload);') < js.index("if (window.ensureModalOpen)")
+ assert "async function openBrowserCanvas" in tool_handler
+ assert "openBrowserModal" not in tool_handler
+ assert tool_handler.index('await rightCanvasStore.open("browser", payload);') < tool_handler.index("if (window.ensureModalOpen)")
+ assert "function syncOpenBrowserCanvas" in tool_handler
+ assert "function autoOpenBrowserCanvas" not in tool_handler
+ assert "function shouldSyncOpenBrowserCanvas" in tool_handler
+ assert "function isBrowserCanvasAlreadyOpen" in tool_handler
+
+ assert "async function syncOpenBrowserCanvas" in after_loop_handler
+ assert "syncBrowserResultsIntoOpenCanvas" in after_loop_handler
+ assert "openBrowserCanvas" not in after_loop_handler
+ assert "BROWSER_MODAL" not in after_loop_handler
- assert "function autoOpenBrowserCanvas" in tool_handler
+ for js in (tool_handler, after_loop_handler):
+ assert "syncedBrowserCanvases" in js
+ assert "autoOpenedBrowsers" not in js
+ assert 'rightCanvasStore?.activeSurfaceId === "browser"' in js
+ assert "const FOCUS_ACTIONS = new Set" in js
+ assert "FOCUS_ACTIONS.has(action)" in js
for js in (tool_handler, after_loop_handler, register_js, browser_store, modals_js):
assert "globalThis.Alpine" not in js
@@ -552,6 +565,16 @@ def test_browser_entry_points_prefer_canvas_and_modal_dock_handoff():
assert "Alpine.store" not in js
+def test_browser_tool_does_not_auto_open_canvas_policy_is_documented():
+ prompt = (
+ PROJECT_ROOT / "plugins" / "_browser" / "prompts" / "agent.system.tool.browser.md"
+ ).read_text(encoding="utf-8")
+
+ assert "optional visible WebUI viewer" in prompt
+ assert "must not open the right canvas automatically" in prompt
+ assert "Use the tool headlessly unless the user opens the Browser canvas" in prompt
+
+
def test_browser_canvas_uses_plain_panel_without_debug_probe():
panel_html = (
PROJECT_ROOT
@@ -787,6 +810,14 @@ def test_browser_runtime_and_content_helper_expose_annotation_target():
assert "password" in helper
+def test_browser_runtime_requires_current_content_helper_for_modifier_clicks():
+ runtime = (
+ PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"
+ ).read_text(encoding="utf-8")
+
+ assert "__spaceBrowserPageContent__?.boundingBoxFor" in runtime
+
+
@pytest.mark.anyio
async def test_browser_screencast_acknowledges_and_drops_stale_frames():
first_image = SMALL_JPEG_10X10
@@ -1449,6 +1480,33 @@ async def test_browser_viewer_annotation_dispatches_runtime(monkeypatch):
assert calls == [("annotation_target", (4, payload), {})]
+def test_browser_runtime_normalizes_multi_group_ids_and_modifiers():
+ core = _BrowserRuntimeCore("ctx")
+
+ assert core._multi_group_key({"browser_id": 7}) == 7
+ assert core._multi_group_key({"browser_id": "7"}) == 7
+ assert core._multi_group_key({"browser_id": "browser-7"}) == 7
+ assert core._multi_group_key({"browser_id": ""}) is None
+ assert core._normalize_modifiers("Control") == ["Control"]
+ assert core._normalize_modifiers(["Control", " Shift "]) == ["Control", "Shift"]
+ assert core._normalize_modifiers([]) is None
+
+ with pytest.raises(ValueError):
+ core._normalize_modifiers("Ctrl")
+
+
+def test_browser_runtime_background_focus_restores_previous_active_tab():
+ core = _BrowserRuntimeCore("ctx")
+ core.pages[1] = browser_runtime_module.BrowserPage(id=1, page=object())
+ core.pages[2] = browser_runtime_module.BrowserPage(id=2, page=object())
+
+ assert core._background_focus_target(previous_focus=1, fallback_id=2) == 1
+
+ core.pages.pop(1)
+
+ assert core._background_focus_target(previous_focus=1, fallback_id=2) == 2
+
+
def test_browser_cleanup_extensions_follow_extensible_path_layout():
extension = __import__("helpers.extension", fromlist=["_get_extension_classes"])
remove_classes = extension._get_extension_classes( # type: ignore[attr-defined]