Agent Zero AI framework
feat(browser): multi-tab awareness + modifier-key click
TerminallyLazy5012dd3128aa
diff --git a/plugins/_browser/assets/browser-page-content.js b/plugins/_browser/assets/browser-page-content.jsindex edf9d9403..246fe6d07 100644--- a/plugins/_browser/assets/browser-page-content.js+++ b/plugins/_browser/assets/browser-page-content.js@@ -1,7 +1,7 @@(() => {const GLOBAL_KEY = "__spaceBrowserPageContent__";const DOM_HELPER_KEY = "__spaceBrowserDomHelper__";- const VERSION = "7";+ const VERSION = "9";const BLOCK_TAGS = new Set(["ADDRESS","ARTICLE",@@ -2026,6 +2026,76 @@return entry;}+ function computeStableSelector(el) {+ if (!el || el.nodeType !== 1) return null;+ const doc = el.ownerDocument || document;+ if (el.id && /^[A-Za-z_][\w-]*$/.test(el.id)) {+ const sel = "#" + (typeof CSS !== "undefined" && CSS.escape ? CSS.escape(el.id) : el.id);+ try {+ if (doc.querySelectorAll(sel).length === 1) return sel;+ } catch (_) {}+ }+ const parts = [];+ let node = el;+ while (node && node.nodeType === 1 && node !== doc.documentElement) {+ let part = node.tagName.toLowerCase();+ if (node.id && /^[A-Za-z_][\w-]*$/.test(node.id)) {+ const idSel = "#" + (typeof CSS !== "undefined" && CSS.escape ? CSS.escape(node.id) : node.id);+ try {+ if (doc.querySelectorAll(idSel).length === 1) {+ parts.unshift(idSel);+ break;+ }+ } catch (_) {}+ }+ const parent = node.parentElement;+ if (parent) {+ const sibs = parent.children;+ let idx = 0;+ let sameTag = 0;+ for (let i = 0; i < sibs.length; i++) {+ if (sibs[i].tagName === node.tagName) {+ sameTag++;+ if (sibs[i] === node) idx = sameTag;+ }+ }+ if (sameTag > 1) part += ":nth-of-type(" + idx + ")";+ }+ parts.unshift(part);+ node = parent;+ }+ const sel = parts.join(" > ");+ if (!sel) return null;+ try {+ if (doc.querySelectorAll(sel).length === 1) return sel;+ } catch (_) {}+ return null;+ }++ function boundingBoxFor(referenceId) {+ const entry = requireReferenceEntry(referenceId, {+ actionLabel: "boundingBox",+ requireConnected: false+ });+ if (entry.helperBacked || !entry.element) return null;+ const el = entry.element;+ if (typeof el.getBoundingClientRect !== "function") return null;+ try {+ el.scrollIntoView({ block: "center", inline: "center", behavior: "instant" });+ } catch (_) {}+ const r = el.getBoundingClientRect();+ const selector = computeStableSelector(el);+ const hasBox = r && r.width > 0 && r.height > 0;+ if (!hasBox && !selector) return null;+ return {+ x: hasBox ? r.left : 0,+ y: hasBox ? r.top : 0,+ width: hasBox ? r.width : 0,+ height: hasBox ? r.height : 0,+ selector: selector || null+ };+ }+function refreshReferenceEntry(entry) {if (!entry || entry.helperBacked || !entry.element) {return entry;@@ -3253,6 +3323,7 @@typeSubmit(referenceId, value) {return typeAndSubmit(referenceId, value);},+ boundingBoxFor,version: VERSION};})();diff --git a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.jsindex 99b9e3d88..117a2e64b 100644--- a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js+++ b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js@@ -86,11 +86,22 @@ function isFreshToolMessage(timestamp) {return Math.abs(Date.now() - messageMs) <= AUTO_OPEN_WINDOW_MS;}+// Allowlist: only these actions cause the viewer to follow. Background work+// (evaluate, click, type, key_chord, mouse, multi, ...) does not steal focus.+const FOCUS_ACTIONS = new Set([+ "open",+ "navigate",+ "set_active",+ "setactive",+ "activate",+ "focus",+]);+function shouldAutoOpenBrowser(args, result) {if (!isFreshToolMessage(args?.timestamp)) return false;const action = String(args?.kvps?.action || "").trim().toLowerCase().replace("-", "_");- if (["list", "content", "detail", "close", "close_all"].includes(action)) return false;- return Boolean(browserIdFromResult(result, args?.kvps || {}) || action === "open" || action === "navigate");+ if (!FOCUS_ACTIONS.has(action)) return false;+ return Boolean(browserIdFromResult(result, args?.kvps || {}));}function autoOpenBrowserCanvas(args, result) {diff --git a/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js b/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.jsindex 1f99d3732..b9cd53646 100644--- a/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js+++ b/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js@@ -79,19 +79,23 @@ function parseMaybeJson(value) {}}+// Actions that should bring the viewer to the targeted tab. Everything else+// (read, click, type, evaluate, key_chord, mouse, multi, ...) leaves the+// viewer where it is so cross-tab work doesn't steal user focus.+const FOCUS_ACTIONS = new Set([+ "open",+ "navigate",+ "set_active",+ "setactive",+ "activate",+ "focus",+]);+function shouldAutoOpen(args = {}, payload = {}, result = {}) {if (!isFresh(args.timestamp, payload.last_modified || result.last_modified)) return false;-const action = String(payload.action || "").trim().toLowerCase().replace("-", "_");- if (["list", "content", "detail", "close", "close_all"].includes(action)) return false;-- return Boolean(- getBrowserId(payload, result)- || action === "open"- || action === "navigate"- || result.currentUrl- || result.state?.currentUrl,- );+ if (!FOCUS_ACTIONS.has(action)) return false;+ return Boolean(getBrowserId(payload, result) || result.currentUrl || result.state?.currentUrl);}function getBrowserId(payload = {}, result = {}) {diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.pyindex 1f0b56a8c..975904e35 100644--- a/plugins/_browser/helpers/runtime.py+++ b/plugins/_browser/helpers/runtime.py@@ -337,6 +337,9 @@ class BrowserRuntime:class _BrowserRuntimeCore:+ _VALID_MODIFIERS = {"Control", "Shift", "Alt", "Meta"}+ _POPUP_WAIT_SECONDS = 2.0+def __init__(self, context_id: str):self.context_id = context_idself.safe_context_id = _safe_context_id(context_id)@@ -348,6 +351,23 @@ class _BrowserRuntimeCore:self.last_interacted_browser_id: int | None = Noneself._content_helper_source: str | None = Noneself._start_lock: asyncio.Lock | None = None+ self._registry_lock: asyncio.Lock | None = None+ self._closing = False+ self._pending_popups: list[asyncio.Future[int]] = []+ self._background_popup_pages: set[int] = set()++ def _ensure_registry_lock(self) -> asyncio.Lock:+ if self._registry_lock is None:+ self._registry_lock = asyncio.Lock()+ return self._registry_lock++ def _maybe_promote(self, resolved_id: int) -> None:+ # Promote only if the target IS the current active tab or no tab is+ # active yet. Cross-tab work on a backgrounded tab does not steal+ # viewer focus.+ current = self.last_interacted_browser_id+ if current is None or current == resolved_id:+ self.last_interacted_browser_id = int(resolved_id)@propertydef profile_dir(self) -> Path:@@ -411,6 +431,7 @@ class _BrowserRuntimeCore:raiseself.context.set_default_timeout(30000)self.context.set_default_navigation_timeout(30000)+ self.context.on("page", self._on_new_page_sync)await self.context.add_init_script(self._shadow_dom_script())await self.context.add_init_script(path=str(CONTENT_HELPER_PATH))@@ -421,7 +442,7 @@ class _BrowserRuntimeCore:except Exception:passcontinue- self._register_page(page)+ await self._register_page(page)def _release_orphaned_profile_singleton(self) -> None:lock_path = self.profile_dir / "SingletonLock"@@ -486,7 +507,7 @@ class _BrowserRuntimeCore:async def open(self, url: str = "") -> dict[str, Any]:await self.ensure_started()page = await self.context.new_page()- browser_page = self._register_page(page)+ browser_page = await self._register_page(page)self.last_interacted_browser_id = browser_page.idtarget_url = self._initial_url(url)if target_url and target_url != "about:blank":@@ -501,13 +522,149 @@ class _BrowserRuntimeCore:return raw_urlreturn str(get_browser_config().get(DEFAULT_HOMEPAGE_KEY) or "about:blank").strip() or "about:blank"- async def list(self) -> dict[str, Any]:+ async def list(self, include_content: bool = False) -> dict[str, Any]:await self.ensure_started()+ ids = sorted(self.pages)+ if not ids:+ return {+ "browsers": [],+ "last_interacted_browser_id": self.last_interacted_browser_id,+ }+ states_task = asyncio.gather(*(self._state(bid) for bid in ids))+ if include_content:+ contents_task = asyncio.gather(+ *(self.content(bid) for bid in ids),+ return_exceptions=True,+ )+ states, contents = await asyncio.gather(states_task, contents_task)+ out: list[dict[str, Any]] = []+ for idx, bid in enumerate(ids):+ entry = states[idx]+ c = contents[idx]+ if isinstance(c, Exception):+ entry["content_error"] = str(c)+ else:+ entry["content"] = c+ out.append(entry)+ else:+ out = await states_taskreturn {- "browsers": [await self._state(browser_id) for browser_id in sorted(self.pages)],+ "browsers": out,"last_interacted_browser_id": self.last_interacted_browser_id,}+ async def multi(self, calls: list[dict[str, Any]]) -> list[dict[str, Any]]:+ if not isinstance(calls, list) or not calls:+ raise ValueError("multi requires a non-empty list of calls")+ groups: dict[Any, list[tuple[int, dict[str, Any]]]] = {}+ for idx, call in enumerate(calls):+ if not isinstance(call, dict):+ raise ValueError(f"calls[{idx}] is not an object")+ key = call.get("browser_id")+ groups.setdefault(key, []).append((idx, call))++ results: list[dict[str, Any] | None] = [None] * len(calls)++ async def run_group(group: list[tuple[int, dict[str, Any]]]) -> None:+ for idx, call in group:+ try:+ out = await self._dispatch_call(call)+ results[idx] = {"ok": True, "result": out}+ except Exception as exc:+ results[idx] = {"ok": False, "error": str(exc)}++ await asyncio.gather(*(run_group(g) for g in groups.values()))+ return [r if r is not None else {"ok": False, "error": "missing"} for r in results]++ async def _dispatch_call(self, call: dict[str, Any]) -> Any:+ action = str(call.get("action") or "").strip().lower().replace("-", "_")+ bid = call.get("browser_id")+ if action == "open":+ return await self.open(call.get("url") or "")+ if action == "list":+ return await self.list(include_content=bool(call.get("include_content")))+ if action == "state":+ return await self.state(bid)+ if action in {"set_active", "setactive", "activate", "focus"}:+ return await self.set_active(bid)+ if action == "navigate":+ return await self.navigate(bid, call.get("url") or "")+ if action == "back":+ return await self.back(bid)+ if action == "forward":+ return await self.forward(bid)+ if action == "reload":+ return await self.reload(bid)+ if action == "content":+ payload = None+ sels = call.get("selectors")+ sel = call.get("selector")+ if sels:+ payload = {"selectors": sels}+ elif sel:+ payload = {"selector": sel}+ return await self.content(bid, payload)+ if action == "detail":+ ref = call.get("ref")+ if ref is None:+ raise ValueError("detail requires ref")+ return await self.detail(bid, ref)+ if action == "click":+ ref = call.get("ref")+ if ref is None:+ raise ValueError("click requires ref")+ return await self.click(+ bid, ref,+ modifiers=call.get("modifiers"),+ focus_popup=call.get("focus_popup"),+ )+ if action == "type":+ ref = call.get("ref")+ if ref is None:+ raise ValueError("type requires ref")+ return await self.type(bid, ref, call.get("text") or "")+ if action == "submit":+ ref = call.get("ref")+ if ref is None:+ raise ValueError("submit requires ref")+ return await self.submit(bid, ref)+ if action in {"type_submit", "typesubmit"}:+ ref = call.get("ref")+ if ref is None:+ raise ValueError("type_submit requires ref")+ return await self.type_submit(bid, ref, call.get("text") or "")+ if action == "scroll":+ ref = call.get("ref")+ if ref is None:+ raise ValueError("scroll requires ref")+ return await self.scroll(bid, ref)+ if action == "evaluate":+ return await self.evaluate(bid, call.get("script") or "")+ if action in {"key_chord", "keychord"}:+ keys = call.get("keys") or []+ if not keys:+ raise ValueError("key_chord requires non-empty keys")+ return await self.key_chord(bid, list(keys))+ if action == "mouse":+ return await self.mouse(+ bid, call.get("event_type") or "click",+ float(call.get("x") or 0), float(call.get("y") or 0),+ button=call.get("button") or "left",+ modifiers=call.get("modifiers"),+ )+ if action == "close":+ return await self.close_browser(bid)+ if action == "close_all":+ return await self.close_all_browsers()+ raise ValueError(f"unknown action: {action}")++ async def set_active(self, browser_id: int | str | None) -> dict[str, Any]:+ await self.ensure_started()+ resolved_id = self._resolve_browser_id(browser_id)+ # Explicit focus change — bypass _maybe_promote.+ self.last_interacted_browser_id = int(resolved_id)+ return await self._state(resolved_id)+async def state(self, browser_id: int | str | None = None) -> dict[str, Any]:await self.ensure_started()return await self._state(self._resolve_browser_id(browser_id))@@ -517,7 +674,7 @@ class _BrowserRuntimeCore:resolved_id = self._resolve_browser_id(browser_id)page = self._page(resolved_id)await self._goto(page, normalize_url(url))- self.last_interacted_browser_id = resolved_id+ self._maybe_promote(resolved_id)return await self._state(resolved_id)async def back(self, browser_id: int | str | None = None) -> dict[str, Any]:@@ -526,7 +683,7 @@ class _BrowserRuntimeCore:page = self._page(resolved_id)await page.go_back(wait_until="domcontentloaded", timeout=10000)await self._settle(page)- self.last_interacted_browser_id = resolved_id+ self._maybe_promote(resolved_id)return await self._state(resolved_id)async def forward(self, browser_id: int | str | None = None) -> dict[str, Any]:@@ -535,7 +692,7 @@ class _BrowserRuntimeCore:page = self._page(resolved_id)await page.go_forward(wait_until="domcontentloaded", timeout=10000)await self._settle(page)- self.last_interacted_browser_id = resolved_id+ self._maybe_promote(resolved_id)return await self._state(resolved_id)async def reload(self, browser_id: int | str | None = None) -> dict[str, Any]:@@ -544,7 +701,7 @@ class _BrowserRuntimeCore:page = self._page(resolved_id)await page.reload(wait_until="domcontentloaded", timeout=15000)await self._settle(page)- self.last_interacted_browser_id = resolved_id+ self._maybe_promote(resolved_id)return await self._state(resolved_id)async def content(@@ -560,7 +717,7 @@ class _BrowserRuntimeCore:"(payload) => globalThis.__spaceBrowserPageContent__.capture(payload || null)",payload or None,)- self.last_interacted_browser_id = resolved_id+ self._maybe_promote(resolved_id)return result or {}async def detail(self, browser_id: int | str | None, reference_id: int | str) -> dict[str, Any]:@@ -572,7 +729,7 @@ class _BrowserRuntimeCore:"(ref) => globalThis.__spaceBrowserPageContent__.detail(ref)",reference_id,)- self.last_interacted_browser_id = resolved_id+ self._maybe_promote(resolved_id)return result or {}async def annotation_target(@@ -588,7 +745,7 @@ class _BrowserRuntimeCore:"(payload) => globalThis.__spaceBrowserPageContent__.annotate(payload || null)",payload or None,)- self.last_interacted_browser_id = resolved_id+ self._maybe_promote(resolved_id)return result or {}async def evaluate(self, browser_id: int | str | None, script: str) -> dict[str, Any]:@@ -596,12 +753,140 @@ class _BrowserRuntimeCore:resolved_id = self._resolve_browser_id(browser_id)page = self._page(resolved_id)result = await page.evaluate(str(script or "undefined"))- self.last_interacted_browser_id = resolved_id+ self._maybe_promote(resolved_id)return {"result": result, "state": await self._state(resolved_id)}- async def click(self, browser_id: int | str | None, reference_id: int | str) -> dict[str, Any]:+ async def click(+ self,+ browser_id: int | str | None,+ reference_id: int | str,+ modifiers: list[str] | None = None,+ focus_popup: bool | None = None,+ ) -> dict[str, Any]:+ if modifiers:+ return await self._modifier_click(browser_id, reference_id, modifiers, focus_popup)return await self._reference_action("click", browser_id, reference_id)+ async def _modifier_click(+ self,+ browser_id: int | str | None,+ reference_id: int | str,+ modifiers: list[str],+ focus_popup: bool | None,+ ) -> dict[str, Any]:+ bad = set(modifiers) - self._VALID_MODIFIERS+ if bad:+ raise ValueError(+ f"unsupported modifiers: {sorted(bad)}; allowed: {sorted(self._VALID_MODIFIERS)}"+ )+ await self.ensure_started()+ resolved_id = self._resolve_browser_id(browser_id)+ page = self._page(resolved_id)+ await self._ensure_content_helper(page)++ box = await page.evaluate(+ "(ref) => globalThis.__spaceBrowserPageContent__.boundingBoxFor(ref)",+ reference_id,+ )++ background = focus_popup is False or (+ focus_popup is None and bool({"Control", "Meta"} & set(modifiers))+ )++ loop = asyncio.get_running_loop()+ waiter: asyncio.Future[int] = loop.create_future()+ self._pending_popups.append(waiter)++ warning: str | None = None+ opened_id: int | None = None+ try:+ box_has_geometry = bool(box and box.get("width") and box.get("height"))+ box_selector = box.get("selector") if box else None+ if box_has_geometry:+ cx = box["x"] + box["width"] / 2+ cy = box["y"] + box["height"] / 2+ # Mouse.click does not accept modifiers; hold them via keyboard.+ pressed: list[str] = []+ try:+ for mod in modifiers:+ await page.keyboard.down(mod)+ pressed.append(mod)+ await page.mouse.click(cx, cy)+ finally:+ for mod in reversed(pressed):+ with contextlib.suppress(Exception):+ await page.keyboard.up(mod)+ await self._settle(page, short=False)+ elif box_selector:+ try:+ await page.locator(box_selector).click(+ modifiers=list(modifiers), force=True, timeout=5000+ )+ await self._settle(page, short=False)+ except Exception as exc:+ await self._reference_action("click", browser_id, reference_id)+ warning = f"modifiers ignored: locator click failed ({exc})"+ else:+ await self._reference_action("click", browser_id, reference_id)+ warning = "modifiers ignored: target geometry unavailable"++ try:+ opened_id = await asyncio.wait_for(+ asyncio.shield(waiter), timeout=self._POPUP_WAIT_SECONDS+ )+ except asyncio.TimeoutError:+ opened_id = None+ finally:+ if waiter in self._pending_popups:+ self._pending_popups.remove(waiter)+ if not waiter.done():+ waiter.cancel()++ if opened_id is not None and background:+ self._background_popup_pages.add(opened_id)+ if self.last_interacted_browser_id == opened_id:+ # Force focus back to origin tab — popup hook had promoted.+ self.last_interacted_browser_id = int(resolved_id)+ finally:+ if waiter in self._pending_popups:+ self._pending_popups.remove(waiter)++ if background:+ # Background-mode click: explicitly keep focus on origin.+ self.last_interacted_browser_id = int(resolved_id)+ return {+ "action": {+ "ref": reference_id,+ "modifiers": list(modifiers),+ "opened_browser_ids": [opened_id] if opened_id is not None else [],+ **({"warning": warning} if warning else {}),+ },+ "state": await self._state(resolved_id),+ }++ async def key_chord(+ self,+ browser_id: int | str | None,+ keys: list[str],+ ) -> dict[str, Any]:+ if not keys:+ raise ValueError("key_chord requires at least one key")+ await self.ensure_started()+ resolved_id = self._resolve_browser_id(browser_id)+ page = self._page(resolved_id)+ pressed: list[str] = []+ try:+ for k in keys:+ await page.keyboard.down(k)+ pressed.append(k)+ finally:+ for k in reversed(pressed):+ with contextlib.suppress(Exception):+ await page.keyboard.up(k)+ await self._settle(page, short=True)+ self._maybe_promote(resolved_id)+ return await self._state(resolved_id)+async def submit(self, browser_id: int | str | None, reference_id: int | str) -> dict[str, Any]:return await self._reference_action("submit", browser_id, reference_id)@@ -693,7 +978,7 @@ class _BrowserRuntimeCore:self.screencasts.pop(stream_id, None)await screencast.stop()raise- self.last_interacted_browser_id = resolved_id+ self._maybe_promote(resolved_id)return {"stream_id": stream_id,"browser_id": resolved_id,@@ -752,7 +1037,7 @@ class _BrowserRuntimeCore:await self._remount_viewport(page, viewport)if should_remount_viewport:await self._settle(page, short=True)- self.last_interacted_browser_id = resolved_id+ self._maybe_promote(resolved_id)return {"state": await self._state(resolved_id), "viewport": viewport}async def _apply_viewport_with_remount(self, page: Any, viewport: dict[str, int]) -> None:@@ -777,21 +1062,40 @@ class _BrowserRuntimeCore:x: float,y: float,button: str = "left",+ modifiers: list[str] | None = None,) -> dict[str, Any]:+ event_type_lower = str(event_type or "click").lower()+ if modifiers:+ if event_type_lower != "click":+ raise ValueError("modifiers are only valid for event_type='click'")+ bad = set(modifiers) - self._VALID_MODIFIERS+ if bad:+ raise ValueError(+ f"unsupported modifiers: {sorted(bad)}; allowed: {sorted(self._VALID_MODIFIERS)}"+ )await self.ensure_started()resolved_id = self._resolve_browser_id(browser_id)page = self._page(resolved_id)- event_type = str(event_type or "click").lower()- if event_type == "move":+ if event_type_lower == "move":await page.mouse.move(float(x), float(y))- elif event_type == "down":+ elif event_type_lower == "down":await page.mouse.down(button=button)- elif event_type == "up":+ elif event_type_lower == "up":await page.mouse.up(button=button)else:- await page.mouse.click(float(x), float(y), button=button)+ pressed: list[str] = []+ try:+ if modifiers:+ for mod in modifiers:+ await page.keyboard.down(mod)+ pressed.append(mod)+ await page.mouse.click(float(x), float(y), button=button)+ finally:+ for mod in reversed(pressed):+ with contextlib.suppress(Exception):+ await page.keyboard.up(mod)await self._settle(page, short=True)- self.last_interacted_browser_id = resolved_id+ self._maybe_promote(resolved_id)return await self._state(resolved_id)async def wheel(@@ -807,7 +1111,7 @@ class _BrowserRuntimeCore:page = self._page(resolved_id)await page.mouse.move(float(x), float(y))await page.mouse.wheel(float(delta_x), float(delta_y))- self.last_interacted_browser_id = resolved_id+ self._maybe_promote(resolved_id)return await self._state(resolved_id)async def keyboard(@@ -825,10 +1129,16 @@ class _BrowserRuntimeCore:elif key:await page.keyboard.press(str(key))await self._settle(page, short=True)- self.last_interacted_browser_id = resolved_id+ self._maybe_promote(resolved_id)return await self._state(resolved_id)async def close(self, delete_profile: bool = False) -> None:+ self._closing = True+ for waiter in self._pending_popups:+ if not waiter.done():+ waiter.set_exception(RuntimeError("Browser runtime is closing."))+ self._pending_popups.clear()+ self._background_popup_pages.clear()await self._stop_all_screencasts()for browser_id in list(self.pages):try:@@ -873,7 +1183,7 @@ class _BrowserRuntimeCore:{"method": helper_method, "ref": reference_id, "text": text},)await self._settle(page, short=False)- self.last_interacted_browser_id = resolved_id+ self._maybe_promote(resolved_id)return {"action": action or {}, "state": await self._state(resolved_id)}async def _goto(self, page: Any, url: str) -> None:@@ -924,7 +1234,7 @@ class _BrowserRuntimeCore:"loading": False,}- def _register_page(self, page: Any) -> BrowserPage:+ def _register_page_locked(self, page: Any) -> BrowserPage:existing = self._browser_id_for_page(page)if existing is not None:return self.pages[existing]@@ -934,11 +1244,65 @@ class _BrowserRuntimeCore:self.pages[browser_id] = browser_pagedef on_close() -> None:- self.pages.pop(browser_id, None)+ try:+ asyncio.create_task(self._unregister_page_async(browser_id))+ except RuntimeError:+ # No running loop (e.g., during shutdown). Best-effort sync pop.+ self.pages.pop(browser_id, None)page.on("close", on_close)return browser_page+ async def _register_page(self, page: Any) -> BrowserPage:+ lock = self._ensure_registry_lock()+ async with lock:+ return self._register_page_locked(page)++ async def _unregister_page_async(self, browser_id: int) -> None:+ try:+ lock = self._ensure_registry_lock()+ async with lock:+ self.pages.pop(browser_id, None)+ if self.last_interacted_browser_id == browser_id:+ self.last_interacted_browser_id = next(iter(sorted(self.pages)), None)+ self._background_popup_pages.discard(browser_id)+ except Exception as exc:+ PrintStyle.warning(f"Page unregister failed: {exc}")++ def _on_new_page_sync(self, page: Any) -> None:+ if self._closing or self.context is None:+ return+ try:+ asyncio.create_task(self._on_new_page_async(page))+ except RuntimeError:+ return++ async def _on_new_page_async(self, page: Any) -> None:+ try:+ with contextlib.suppress(Exception):+ await page.wait_for_load_state("domcontentloaded", timeout=2000)+ if self._closing or page.is_closed():+ return+ lock = self._ensure_registry_lock()+ async with lock:+ if self._closing:+ return+ if self._browser_id_for_page(page) is not None:+ return+ browser_page = self._register_page_locked(page)+ new_id = browser_page.id+ while self._pending_popups:+ waiter = self._pending_popups.pop(0)+ if not waiter.done():+ waiter.set_result(new_id)+ break+ if new_id not in self._background_popup_pages:+ self.last_interacted_browser_id = new_id+ else:+ self._background_popup_pages.discard(new_id)+ except Exception as exc:+ PrintStyle.warning(f"Popup registration failed: {exc}")+def _browser_id_for_page(self, page: Any) -> int | None:for browser_id, browser_page in self.pages.items():if browser_page.page == page:diff --git a/plugins/_browser/prompts/agent.system.tool.browser.md b/plugins/_browser/prompts/agent.system.tool.browser.mdindex b0112ae25..e8c271792 100644--- a/plugins/_browser/prompts/agent.system.tool.browser.md+++ b/plugins/_browser/prompts/agent.system.tool.browser.md@@ -4,8 +4,8 @@ use for web browsing, page inspection, forms, downloads, and browser-only tasksstate stays open per chat contextrefs come from content as typed markers: [link 3], [button 6], [image 1], [input text 8]-actions: open list state navigate back forward reload content detail click type submit type_submit scroll evaluate close close_all-common args: action browser_id url ref text selector selectors script+actions: open list state set_active navigate back forward reload content detail click type submit type_submit scroll evaluate key_chord mouse multi close close_all+common args: action browser_id url ref text selector selectors script modifiers keys include_content focus_popup event_type x y button callsworkflow:- open creates a new browser and returns id/state@@ -13,7 +13,38 @@ workflow:- detail inspects one ref, including link/image/input/button metadata- click/type/type_submit/submit/scroll use refs from latest content capture and return {action,state}- navigate/back/forward/reload return fresh state-- list shows open browsers+- list shows open browsers; pass include_content: true for one-call bulk read++modifier clicks:+- click accepts modifiers like ["Control"], ["Shift"], ["Alt"], ["Meta"]+- ctrl/meta-click opens link in new tab in background (Chrome rule)+- override with focus_popup: true (focus follows new tab) or false (always background)+- the new tab id is reported in action.opened_browser_ids; list shows all tabs++popup awareness:+- tabs opened by site (window.open, target=_blank, ctrl-click) auto-register+- list returns every tab; last_interacted_browser_id tracks current focus++background work (do not steal focus):+- operations on a non-active tab (read, click, type, evaluate, etc.) target that tab WITHOUT moving focus+- last_interacted_browser_id (and the WebUI viewer that follows it) only changes on:+ - open (new tab created)+ - explicit set_active action+ - action on the already-active tab+ - chrome popup-focus rule (plain click on target=_blank → follow; ctrl-click → stay)+- to switch focus deliberately: {"action":"set_active","browser_id":N}++key_chord:+- presses keys in order, releases in reverse; safe across exceptions+- example: {"action":"key_chord","keys":["Control","a"]} selects all++multi (parallel batch):+- run many actions concurrently across tabs in one tool call+- pass calls: array of action objects (each has its own action+args)+- different browser_ids run in parallel; same browser_id runs in submit order+- returns array of {"ok":true,"result":...} or {"ok":false,"error":"..."} matching input order+- ideal for: scrape N tabs at once, fan-out reads, parallel evaluate+- avoid mutating same tab twice in one batch unless serial order is intendedexamples:~~~json@@ -46,3 +77,50 @@ examples:}}~~~++~~~json+{+ "tool_name": "browser",+ "tool_args": {+ "action": "click",+ "browser_id": 1,+ "ref": 3,+ "modifiers": ["Control"]+ }+}+~~~++~~~json+{+ "tool_name": "browser",+ "tool_args": {+ "action": "key_chord",+ "browser_id": 1,+ "keys": ["Control", "a"]+ }+}+~~~++~~~json+{+ "tool_name": "browser",+ "tool_args": {+ "action": "list",+ "include_content": true+ }+}+~~~++~~~json+{+ "tool_name": "browser",+ "tool_args": {+ "action": "multi",+ "calls": [+ {"action": "content", "browser_id": 1},+ {"action": "content", "browser_id": 2},+ {"action": "evaluate", "browser_id": 3, "script": "document.title"}+ ]+ }+}+~~~diff --git a/plugins/_browser/tools/browser.py b/plugins/_browser/tools/browser.pyindex 8ea6001e2..309428935 100644--- a/plugins/_browser/tools/browser.py+++ b/plugins/_browser/tools/browser.py@@ -18,18 +18,34 @@ class Browser(Tool):selector: str = "",selectors: list[str] | None = None,script: str = "",+ modifiers: list[str] | str | None = None,+ keys: list[str] | None = None,+ include_content: bool = False,+ focus_popup: bool | None = None,+ event_type: str = "",+ x: float = 0.0,+ y: float = 0.0,+ button: str = "left",+ calls: list[dict[str, Any]] | None = None,**kwargs: Any,) -> Response:action = str(action or self.method or "state").strip().lower().replace("-", "_")runtime = await get_runtime(self.agent.context.id)+ if isinstance(modifiers, str):+ modifiers = [modifiers] if modifiers else None+ elif isinstance(modifiers, list) and not modifiers:+ modifiers = None+try:if action == "open":result = await runtime.call("open", url or "")elif action == "list":- result = await runtime.call("list")+ result = await runtime.call("list", include_content=bool(include_content))elif action == "state":result = await runtime.call("state", browser_id)+ elif action in {"set_active", "setactive", "activate", "focus"}:+ result = await runtime.call("set_active", browser_id)elif action == "navigate":result = await runtime.call("navigate", browser_id, url)elif action == "back":@@ -44,7 +60,13 @@ class Browser(Tool):elif action == "detail":result = await runtime.call("detail", browser_id, self._require_ref(ref))elif action == "click":- result = await runtime.call("click", browser_id, self._require_ref(ref))+ if modifiers:+ result = await runtime.call(+ "click", browser_id, self._require_ref(ref),+ modifiers=modifiers, focus_popup=focus_popup,+ )+ else:+ result = await runtime.call("click", browser_id, self._require_ref(ref))elif action == "type":result = await runtime.call("type", browser_id, self._require_ref(ref), text)elif action == "submit":@@ -60,6 +82,19 @@ class Browser(Tool):result = await runtime.call("scroll", browser_id, self._require_ref(ref))elif action == "evaluate":result = await runtime.call("evaluate", browser_id, script)+ elif action in {"key_chord", "keychord"}:+ if not keys:+ raise ValueError("key_chord requires non-empty 'keys' list")+ result = await runtime.call("key_chord", browser_id, list(keys))+ elif action == "mouse":+ result = await runtime.call(+ "mouse", browser_id, event_type or "click", x, y,+ button=button or "left", modifiers=modifiers,+ )+ elif action == "multi":+ if not calls:+ raise ValueError("multi requires non-empty 'calls' list")+ result = await runtime.call("multi", list(calls))elif action == "close":result = await runtime.call("close_browser", browser_id)elif action == "close_all":