Skip to content

Agent Zero AI framework

feat(browser): multi-tab awareness + modifier-key click

TerminallyLazy5012dd3128aa
diff --git a/plugins/_browser/assets/browser-page-content.js b/plugins/_browser/assets/browser-page-content.js
index edf9d9403..246fe6d07 100644
--- a/plugins/_browser/assets/browser-page-content.js
+++ b/plugins/_browser/assets/browser-page-content.js
@@ -1,7 +1,7 @@
(() => {
const GLOBAL_KEY = "__spaceBrowserPageContent__";
const DOM_HELPER_KEY = "__spaceBrowserDomHelper__";
- const VERSION = "7";
+ const VERSION = "9";
const BLOCK_TAGS = new Set([
"ADDRESS",
"ARTICLE",
@@ -2026,6 +2026,76 @@
return entry;
}
+ function computeStableSelector(el) {
+ if (!el || el.nodeType !== 1) return null;
+ const doc = el.ownerDocument || document;
+ if (el.id && /^[A-Za-z_][\w-]*$/.test(el.id)) {
+ const sel = "#" + (typeof CSS !== "undefined" && CSS.escape ? CSS.escape(el.id) : el.id);
+ try {
+ if (doc.querySelectorAll(sel).length === 1) return sel;
+ } catch (_) {}
+ }
+ const parts = [];
+ let node = el;
+ while (node && node.nodeType === 1 && node !== doc.documentElement) {
+ let part = node.tagName.toLowerCase();
+ if (node.id && /^[A-Za-z_][\w-]*$/.test(node.id)) {
+ const idSel = "#" + (typeof CSS !== "undefined" && CSS.escape ? CSS.escape(node.id) : node.id);
+ try {
+ if (doc.querySelectorAll(idSel).length === 1) {
+ parts.unshift(idSel);
+ break;
+ }
+ } catch (_) {}
+ }
+ const parent = node.parentElement;
+ if (parent) {
+ const sibs = parent.children;
+ let idx = 0;
+ let sameTag = 0;
+ for (let i = 0; i < sibs.length; i++) {
+ if (sibs[i].tagName === node.tagName) {
+ sameTag++;
+ if (sibs[i] === node) idx = sameTag;
+ }
+ }
+ if (sameTag > 1) part += ":nth-of-type(" + idx + ")";
+ }
+ parts.unshift(part);
+ node = parent;
+ }
+ const sel = parts.join(" > ");
+ if (!sel) return null;
+ try {
+ if (doc.querySelectorAll(sel).length === 1) return sel;
+ } catch (_) {}
+ return null;
+ }
+
+ function boundingBoxFor(referenceId) {
+ const entry = requireReferenceEntry(referenceId, {
+ actionLabel: "boundingBox",
+ requireConnected: false
+ });
+ if (entry.helperBacked || !entry.element) return null;
+ const el = entry.element;
+ if (typeof el.getBoundingClientRect !== "function") return null;
+ try {
+ el.scrollIntoView({ block: "center", inline: "center", behavior: "instant" });
+ } catch (_) {}
+ const r = el.getBoundingClientRect();
+ const selector = computeStableSelector(el);
+ const hasBox = r && r.width > 0 && r.height > 0;
+ if (!hasBox && !selector) return null;
+ return {
+ x: hasBox ? r.left : 0,
+ y: hasBox ? r.top : 0,
+ width: hasBox ? r.width : 0,
+ height: hasBox ? r.height : 0,
+ selector: selector || null
+ };
+ }
+
function refreshReferenceEntry(entry) {
if (!entry || entry.helperBacked || !entry.element) {
return entry;
@@ -3253,6 +3323,7 @@
typeSubmit(referenceId, value) {
return typeAndSubmit(referenceId, value);
},
+ boundingBoxFor,
version: VERSION
};
})();
diff --git a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
index 99b9e3d88..117a2e64b 100644
--- a/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
+++ b/plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
@@ -86,11 +86,22 @@ function isFreshToolMessage(timestamp) {
return Math.abs(Date.now() - messageMs) <= AUTO_OPEN_WINDOW_MS;
}
+// Allowlist: only these actions cause the viewer to follow. Background work
+// (evaluate, click, type, key_chord, mouse, multi, ...) does not steal focus.
+const FOCUS_ACTIONS = new Set([
+ "open",
+ "navigate",
+ "set_active",
+ "setactive",
+ "activate",
+ "focus",
+]);
+
function shouldAutoOpenBrowser(args, result) {
if (!isFreshToolMessage(args?.timestamp)) return false;
const action = String(args?.kvps?.action || "").trim().toLowerCase().replace("-", "_");
- if (["list", "content", "detail", "close", "close_all"].includes(action)) return false;
- return Boolean(browserIdFromResult(result, args?.kvps || {}) || action === "open" || action === "navigate");
+ if (!FOCUS_ACTIONS.has(action)) return false;
+ return Boolean(browserIdFromResult(result, args?.kvps || {}));
}
function autoOpenBrowserCanvas(args, result) {
diff --git a/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js b/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js
index 1f99d3732..b9cd53646 100644
--- a/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js
+++ b/plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js
@@ -79,19 +79,23 @@ function parseMaybeJson(value) {
}
}
+// Actions that should bring the viewer to the targeted tab. Everything else
+// (read, click, type, evaluate, key_chord, mouse, multi, ...) leaves the
+// viewer where it is so cross-tab work doesn't steal user focus.
+const FOCUS_ACTIONS = new Set([
+ "open",
+ "navigate",
+ "set_active",
+ "setactive",
+ "activate",
+ "focus",
+]);
+
function shouldAutoOpen(args = {}, payload = {}, result = {}) {
if (!isFresh(args.timestamp, payload.last_modified || result.last_modified)) return false;
-
const action = String(payload.action || "").trim().toLowerCase().replace("-", "_");
- if (["list", "content", "detail", "close", "close_all"].includes(action)) return false;
-
- return Boolean(
- getBrowserId(payload, result)
- || action === "open"
- || action === "navigate"
- || result.currentUrl
- || result.state?.currentUrl,
- );
+ if (!FOCUS_ACTIONS.has(action)) return false;
+ return Boolean(getBrowserId(payload, result) || result.currentUrl || result.state?.currentUrl);
}
function getBrowserId(payload = {}, result = {}) {
diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py
index 1f0b56a8c..975904e35 100644
--- a/plugins/_browser/helpers/runtime.py
+++ b/plugins/_browser/helpers/runtime.py
@@ -337,6 +337,9 @@ class BrowserRuntime:
class _BrowserRuntimeCore:
+ _VALID_MODIFIERS = {"Control", "Shift", "Alt", "Meta"}
+ _POPUP_WAIT_SECONDS = 2.0
+
def __init__(self, context_id: str):
self.context_id = context_id
self.safe_context_id = _safe_context_id(context_id)
@@ -348,6 +351,23 @@ class _BrowserRuntimeCore:
self.last_interacted_browser_id: int | None = None
self._content_helper_source: str | None = None
self._start_lock: asyncio.Lock | None = None
+ self._registry_lock: asyncio.Lock | None = None
+ self._closing = False
+ self._pending_popups: list[asyncio.Future[int]] = []
+ self._background_popup_pages: set[int] = set()
+
+ def _ensure_registry_lock(self) -> asyncio.Lock:
+ if self._registry_lock is None:
+ self._registry_lock = asyncio.Lock()
+ return self._registry_lock
+
+ def _maybe_promote(self, resolved_id: int) -> None:
+ # Promote only if the target IS the current active tab or no tab is
+ # active yet. Cross-tab work on a backgrounded tab does not steal
+ # viewer focus.
+ current = self.last_interacted_browser_id
+ if current is None or current == resolved_id:
+ self.last_interacted_browser_id = int(resolved_id)
@property
def profile_dir(self) -> Path:
@@ -411,6 +431,7 @@ class _BrowserRuntimeCore:
raise
self.context.set_default_timeout(30000)
self.context.set_default_navigation_timeout(30000)
+ self.context.on("page", self._on_new_page_sync)
await self.context.add_init_script(self._shadow_dom_script())
await self.context.add_init_script(path=str(CONTENT_HELPER_PATH))
@@ -421,7 +442,7 @@ class _BrowserRuntimeCore:
except Exception:
pass
continue
- self._register_page(page)
+ await self._register_page(page)
def _release_orphaned_profile_singleton(self) -> None:
lock_path = self.profile_dir / "SingletonLock"
@@ -486,7 +507,7 @@ class _BrowserRuntimeCore:
async def open(self, url: str = "") -> dict[str, Any]:
await self.ensure_started()
page = await self.context.new_page()
- browser_page = self._register_page(page)
+ browser_page = await self._register_page(page)
self.last_interacted_browser_id = browser_page.id
target_url = self._initial_url(url)
if target_url and target_url != "about:blank":
@@ -501,13 +522,149 @@ class _BrowserRuntimeCore:
return raw_url
return str(get_browser_config().get(DEFAULT_HOMEPAGE_KEY) or "about:blank").strip() or "about:blank"
- async def list(self) -> dict[str, Any]:
+ async def list(self, include_content: bool = False) -> dict[str, Any]:
await self.ensure_started()
+ ids = sorted(self.pages)
+ if not ids:
+ return {
+ "browsers": [],
+ "last_interacted_browser_id": self.last_interacted_browser_id,
+ }
+ states_task = asyncio.gather(*(self._state(bid) for bid in ids))
+ if include_content:
+ contents_task = asyncio.gather(
+ *(self.content(bid) for bid in ids),
+ return_exceptions=True,
+ )
+ states, contents = await asyncio.gather(states_task, contents_task)
+ out: list[dict[str, Any]] = []
+ for idx, bid in enumerate(ids):
+ entry = states[idx]
+ c = contents[idx]
+ if isinstance(c, Exception):
+ entry["content_error"] = str(c)
+ else:
+ entry["content"] = c
+ out.append(entry)
+ else:
+ out = await states_task
return {
- "browsers": [await self._state(browser_id) for browser_id in sorted(self.pages)],
+ "browsers": out,
"last_interacted_browser_id": self.last_interacted_browser_id,
}
+ async def multi(self, calls: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ if not isinstance(calls, list) or not calls:
+ raise ValueError("multi requires a non-empty list of calls")
+ groups: dict[Any, list[tuple[int, dict[str, Any]]]] = {}
+ for idx, call in enumerate(calls):
+ if not isinstance(call, dict):
+ raise ValueError(f"calls[{idx}] is not an object")
+ key = call.get("browser_id")
+ groups.setdefault(key, []).append((idx, call))
+
+ results: list[dict[str, Any] | None] = [None] * len(calls)
+
+ async def run_group(group: list[tuple[int, dict[str, Any]]]) -> None:
+ for idx, call in group:
+ try:
+ out = await self._dispatch_call(call)
+ results[idx] = {"ok": True, "result": out}
+ except Exception as exc:
+ results[idx] = {"ok": False, "error": str(exc)}
+
+ await asyncio.gather(*(run_group(g) for g in groups.values()))
+ return [r if r is not None else {"ok": False, "error": "missing"} for r in results]
+
+ async def _dispatch_call(self, call: dict[str, Any]) -> Any:
+ action = str(call.get("action") or "").strip().lower().replace("-", "_")
+ bid = call.get("browser_id")
+ if action == "open":
+ return await self.open(call.get("url") or "")
+ if action == "list":
+ return await self.list(include_content=bool(call.get("include_content")))
+ if action == "state":
+ return await self.state(bid)
+ if action in {"set_active", "setactive", "activate", "focus"}:
+ return await self.set_active(bid)
+ if action == "navigate":
+ return await self.navigate(bid, call.get("url") or "")
+ if action == "back":
+ return await self.back(bid)
+ if action == "forward":
+ return await self.forward(bid)
+ if action == "reload":
+ return await self.reload(bid)
+ if action == "content":
+ payload = None
+ sels = call.get("selectors")
+ sel = call.get("selector")
+ if sels:
+ payload = {"selectors": sels}
+ elif sel:
+ payload = {"selector": sel}
+ return await self.content(bid, payload)
+ if action == "detail":
+ ref = call.get("ref")
+ if ref is None:
+ raise ValueError("detail requires ref")
+ return await self.detail(bid, ref)
+ if action == "click":
+ ref = call.get("ref")
+ if ref is None:
+ raise ValueError("click requires ref")
+ return await self.click(
+ bid, ref,
+ modifiers=call.get("modifiers"),
+ focus_popup=call.get("focus_popup"),
+ )
+ if action == "type":
+ ref = call.get("ref")
+ if ref is None:
+ raise ValueError("type requires ref")
+ return await self.type(bid, ref, call.get("text") or "")
+ if action == "submit":
+ ref = call.get("ref")
+ if ref is None:
+ raise ValueError("submit requires ref")
+ return await self.submit(bid, ref)
+ if action in {"type_submit", "typesubmit"}:
+ ref = call.get("ref")
+ if ref is None:
+ raise ValueError("type_submit requires ref")
+ return await self.type_submit(bid, ref, call.get("text") or "")
+ if action == "scroll":
+ ref = call.get("ref")
+ if ref is None:
+ raise ValueError("scroll requires ref")
+ return await self.scroll(bid, ref)
+ if action == "evaluate":
+ return await self.evaluate(bid, call.get("script") or "")
+ if action in {"key_chord", "keychord"}:
+ keys = call.get("keys") or []
+ if not keys:
+ raise ValueError("key_chord requires non-empty keys")
+ return await self.key_chord(bid, list(keys))
+ if action == "mouse":
+ return await self.mouse(
+ bid, call.get("event_type") or "click",
+ float(call.get("x") or 0), float(call.get("y") or 0),
+ button=call.get("button") or "left",
+ modifiers=call.get("modifiers"),
+ )
+ if action == "close":
+ return await self.close_browser(bid)
+ if action == "close_all":
+ return await self.close_all_browsers()
+ raise ValueError(f"unknown action: {action}")
+
+ async def set_active(self, browser_id: int | str | None) -> dict[str, Any]:
+ await self.ensure_started()
+ resolved_id = self._resolve_browser_id(browser_id)
+ # Explicit focus change — bypass _maybe_promote.
+ self.last_interacted_browser_id = int(resolved_id)
+ return await self._state(resolved_id)
+
async def state(self, browser_id: int | str | None = None) -> dict[str, Any]:
await self.ensure_started()
return await self._state(self._resolve_browser_id(browser_id))
@@ -517,7 +674,7 @@ class _BrowserRuntimeCore:
resolved_id = self._resolve_browser_id(browser_id)
page = self._page(resolved_id)
await self._goto(page, normalize_url(url))
- self.last_interacted_browser_id = resolved_id
+ self._maybe_promote(resolved_id)
return await self._state(resolved_id)
async def back(self, browser_id: int | str | None = None) -> dict[str, Any]:
@@ -526,7 +683,7 @@ class _BrowserRuntimeCore:
page = self._page(resolved_id)
await page.go_back(wait_until="domcontentloaded", timeout=10000)
await self._settle(page)
- self.last_interacted_browser_id = resolved_id
+ self._maybe_promote(resolved_id)
return await self._state(resolved_id)
async def forward(self, browser_id: int | str | None = None) -> dict[str, Any]:
@@ -535,7 +692,7 @@ class _BrowserRuntimeCore:
page = self._page(resolved_id)
await page.go_forward(wait_until="domcontentloaded", timeout=10000)
await self._settle(page)
- self.last_interacted_browser_id = resolved_id
+ self._maybe_promote(resolved_id)
return await self._state(resolved_id)
async def reload(self, browser_id: int | str | None = None) -> dict[str, Any]:
@@ -544,7 +701,7 @@ class _BrowserRuntimeCore:
page = self._page(resolved_id)
await page.reload(wait_until="domcontentloaded", timeout=15000)
await self._settle(page)
- self.last_interacted_browser_id = resolved_id
+ self._maybe_promote(resolved_id)
return await self._state(resolved_id)
async def content(
@@ -560,7 +717,7 @@ class _BrowserRuntimeCore:
"(payload) => globalThis.__spaceBrowserPageContent__.capture(payload || null)",
payload or None,
)
- self.last_interacted_browser_id = resolved_id
+ self._maybe_promote(resolved_id)
return result or {}
async def detail(self, browser_id: int | str | None, reference_id: int | str) -> dict[str, Any]:
@@ -572,7 +729,7 @@ class _BrowserRuntimeCore:
"(ref) => globalThis.__spaceBrowserPageContent__.detail(ref)",
reference_id,
)
- self.last_interacted_browser_id = resolved_id
+ self._maybe_promote(resolved_id)
return result or {}
async def annotation_target(
@@ -588,7 +745,7 @@ class _BrowserRuntimeCore:
"(payload) => globalThis.__spaceBrowserPageContent__.annotate(payload || null)",
payload or None,
)
- self.last_interacted_browser_id = resolved_id
+ self._maybe_promote(resolved_id)
return result or {}
async def evaluate(self, browser_id: int | str | None, script: str) -> dict[str, Any]:
@@ -596,12 +753,140 @@ class _BrowserRuntimeCore:
resolved_id = self._resolve_browser_id(browser_id)
page = self._page(resolved_id)
result = await page.evaluate(str(script or "undefined"))
- self.last_interacted_browser_id = resolved_id
+ self._maybe_promote(resolved_id)
return {"result": result, "state": await self._state(resolved_id)}
- async def click(self, browser_id: int | str | None, reference_id: int | str) -> dict[str, Any]:
+ async def click(
+ self,
+ browser_id: int | str | None,
+ reference_id: int | str,
+ modifiers: list[str] | None = None,
+ focus_popup: bool | None = None,
+ ) -> dict[str, Any]:
+ if modifiers:
+ return await self._modifier_click(browser_id, reference_id, modifiers, focus_popup)
return await self._reference_action("click", browser_id, reference_id)
+ async def _modifier_click(
+ self,
+ browser_id: int | str | None,
+ reference_id: int | str,
+ modifiers: list[str],
+ focus_popup: bool | None,
+ ) -> dict[str, Any]:
+ bad = set(modifiers) - self._VALID_MODIFIERS
+ if bad:
+ raise ValueError(
+ f"unsupported modifiers: {sorted(bad)}; allowed: {sorted(self._VALID_MODIFIERS)}"
+ )
+ await self.ensure_started()
+ resolved_id = self._resolve_browser_id(browser_id)
+ page = self._page(resolved_id)
+ await self._ensure_content_helper(page)
+
+ box = await page.evaluate(
+ "(ref) => globalThis.__spaceBrowserPageContent__.boundingBoxFor(ref)",
+ reference_id,
+ )
+
+ background = focus_popup is False or (
+ focus_popup is None and bool({"Control", "Meta"} & set(modifiers))
+ )
+
+ loop = asyncio.get_running_loop()
+ waiter: asyncio.Future[int] = loop.create_future()
+ self._pending_popups.append(waiter)
+
+ warning: str | None = None
+ opened_id: int | None = None
+ try:
+ box_has_geometry = bool(box and box.get("width") and box.get("height"))
+ box_selector = box.get("selector") if box else None
+ if box_has_geometry:
+ cx = box["x"] + box["width"] / 2
+ cy = box["y"] + box["height"] / 2
+ # Mouse.click does not accept modifiers; hold them via keyboard.
+ pressed: list[str] = []
+ try:
+ for mod in modifiers:
+ await page.keyboard.down(mod)
+ pressed.append(mod)
+ await page.mouse.click(cx, cy)
+ finally:
+ for mod in reversed(pressed):
+ with contextlib.suppress(Exception):
+ await page.keyboard.up(mod)
+ await self._settle(page, short=False)
+ elif box_selector:
+ try:
+ await page.locator(box_selector).click(
+ modifiers=list(modifiers), force=True, timeout=5000
+ )
+ await self._settle(page, short=False)
+ except Exception as exc:
+ await self._reference_action("click", browser_id, reference_id)
+ warning = f"modifiers ignored: locator click failed ({exc})"
+ else:
+ await self._reference_action("click", browser_id, reference_id)
+ warning = "modifiers ignored: target geometry unavailable"
+
+ try:
+ opened_id = await asyncio.wait_for(
+ asyncio.shield(waiter), timeout=self._POPUP_WAIT_SECONDS
+ )
+ except asyncio.TimeoutError:
+ opened_id = None
+ finally:
+ if waiter in self._pending_popups:
+ self._pending_popups.remove(waiter)
+ if not waiter.done():
+ waiter.cancel()
+
+ if opened_id is not None and background:
+ self._background_popup_pages.add(opened_id)
+ if self.last_interacted_browser_id == opened_id:
+ # Force focus back to origin tab — popup hook had promoted.
+ self.last_interacted_browser_id = int(resolved_id)
+ finally:
+ if waiter in self._pending_popups:
+ self._pending_popups.remove(waiter)
+
+ if background:
+ # Background-mode click: explicitly keep focus on origin.
+ self.last_interacted_browser_id = int(resolved_id)
+ return {
+ "action": {
+ "ref": reference_id,
+ "modifiers": list(modifiers),
+ "opened_browser_ids": [opened_id] if opened_id is not None else [],
+ **({"warning": warning} if warning else {}),
+ },
+ "state": await self._state(resolved_id),
+ }
+
+ async def key_chord(
+ self,
+ browser_id: int | str | None,
+ keys: list[str],
+ ) -> dict[str, Any]:
+ if not keys:
+ raise ValueError("key_chord requires at least one key")
+ await self.ensure_started()
+ resolved_id = self._resolve_browser_id(browser_id)
+ page = self._page(resolved_id)
+ pressed: list[str] = []
+ try:
+ for k in keys:
+ await page.keyboard.down(k)
+ pressed.append(k)
+ finally:
+ for k in reversed(pressed):
+ with contextlib.suppress(Exception):
+ await page.keyboard.up(k)
+ await self._settle(page, short=True)
+ self._maybe_promote(resolved_id)
+ return await self._state(resolved_id)
+
async def submit(self, browser_id: int | str | None, reference_id: int | str) -> dict[str, Any]:
return await self._reference_action("submit", browser_id, reference_id)
@@ -693,7 +978,7 @@ class _BrowserRuntimeCore:
self.screencasts.pop(stream_id, None)
await screencast.stop()
raise
- self.last_interacted_browser_id = resolved_id
+ self._maybe_promote(resolved_id)
return {
"stream_id": stream_id,
"browser_id": resolved_id,
@@ -752,7 +1037,7 @@ class _BrowserRuntimeCore:
await self._remount_viewport(page, viewport)
if should_remount_viewport:
await self._settle(page, short=True)
- self.last_interacted_browser_id = resolved_id
+ self._maybe_promote(resolved_id)
return {"state": await self._state(resolved_id), "viewport": viewport}
async def _apply_viewport_with_remount(self, page: Any, viewport: dict[str, int]) -> None:
@@ -777,21 +1062,40 @@ class _BrowserRuntimeCore:
x: float,
y: float,
button: str = "left",
+ modifiers: list[str] | None = None,
) -> dict[str, Any]:
+ event_type_lower = str(event_type or "click").lower()
+ if modifiers:
+ if event_type_lower != "click":
+ raise ValueError("modifiers are only valid for event_type='click'")
+ bad = set(modifiers) - self._VALID_MODIFIERS
+ if bad:
+ raise ValueError(
+ f"unsupported modifiers: {sorted(bad)}; allowed: {sorted(self._VALID_MODIFIERS)}"
+ )
await self.ensure_started()
resolved_id = self._resolve_browser_id(browser_id)
page = self._page(resolved_id)
- event_type = str(event_type or "click").lower()
- if event_type == "move":
+ if event_type_lower == "move":
await page.mouse.move(float(x), float(y))
- elif event_type == "down":
+ elif event_type_lower == "down":
await page.mouse.down(button=button)
- elif event_type == "up":
+ elif event_type_lower == "up":
await page.mouse.up(button=button)
else:
- await page.mouse.click(float(x), float(y), button=button)
+ pressed: list[str] = []
+ try:
+ if modifiers:
+ for mod in modifiers:
+ await page.keyboard.down(mod)
+ pressed.append(mod)
+ await page.mouse.click(float(x), float(y), button=button)
+ finally:
+ for mod in reversed(pressed):
+ with contextlib.suppress(Exception):
+ await page.keyboard.up(mod)
await self._settle(page, short=True)
- self.last_interacted_browser_id = resolved_id
+ self._maybe_promote(resolved_id)
return await self._state(resolved_id)
async def wheel(
@@ -807,7 +1111,7 @@ class _BrowserRuntimeCore:
page = self._page(resolved_id)
await page.mouse.move(float(x), float(y))
await page.mouse.wheel(float(delta_x), float(delta_y))
- self.last_interacted_browser_id = resolved_id
+ self._maybe_promote(resolved_id)
return await self._state(resolved_id)
async def keyboard(
@@ -825,10 +1129,16 @@ class _BrowserRuntimeCore:
elif key:
await page.keyboard.press(str(key))
await self._settle(page, short=True)
- self.last_interacted_browser_id = resolved_id
+ self._maybe_promote(resolved_id)
return await self._state(resolved_id)
async def close(self, delete_profile: bool = False) -> None:
+ self._closing = True
+ for waiter in self._pending_popups:
+ if not waiter.done():
+ waiter.set_exception(RuntimeError("Browser runtime is closing."))
+ self._pending_popups.clear()
+ self._background_popup_pages.clear()
await self._stop_all_screencasts()
for browser_id in list(self.pages):
try:
@@ -873,7 +1183,7 @@ class _BrowserRuntimeCore:
{"method": helper_method, "ref": reference_id, "text": text},
)
await self._settle(page, short=False)
- self.last_interacted_browser_id = resolved_id
+ self._maybe_promote(resolved_id)
return {"action": action or {}, "state": await self._state(resolved_id)}
async def _goto(self, page: Any, url: str) -> None:
@@ -924,7 +1234,7 @@ class _BrowserRuntimeCore:
"loading": False,
}
- def _register_page(self, page: Any) -> BrowserPage:
+ def _register_page_locked(self, page: Any) -> BrowserPage:
existing = self._browser_id_for_page(page)
if existing is not None:
return self.pages[existing]
@@ -934,11 +1244,65 @@ class _BrowserRuntimeCore:
self.pages[browser_id] = browser_page
def on_close() -> None:
- self.pages.pop(browser_id, None)
+ try:
+ asyncio.create_task(self._unregister_page_async(browser_id))
+ except RuntimeError:
+ # No running loop (e.g., during shutdown). Best-effort sync pop.
+ self.pages.pop(browser_id, None)
page.on("close", on_close)
return browser_page
+ async def _register_page(self, page: Any) -> BrowserPage:
+ lock = self._ensure_registry_lock()
+ async with lock:
+ return self._register_page_locked(page)
+
+ async def _unregister_page_async(self, browser_id: int) -> None:
+ try:
+ lock = self._ensure_registry_lock()
+ async with lock:
+ self.pages.pop(browser_id, None)
+ if self.last_interacted_browser_id == browser_id:
+ self.last_interacted_browser_id = next(iter(sorted(self.pages)), None)
+ self._background_popup_pages.discard(browser_id)
+ except Exception as exc:
+ PrintStyle.warning(f"Page unregister failed: {exc}")
+
+ def _on_new_page_sync(self, page: Any) -> None:
+ if self._closing or self.context is None:
+ return
+ try:
+ asyncio.create_task(self._on_new_page_async(page))
+ except RuntimeError:
+ return
+
+ async def _on_new_page_async(self, page: Any) -> None:
+ try:
+ with contextlib.suppress(Exception):
+ await page.wait_for_load_state("domcontentloaded", timeout=2000)
+ if self._closing or page.is_closed():
+ return
+ lock = self._ensure_registry_lock()
+ async with lock:
+ if self._closing:
+ return
+ if self._browser_id_for_page(page) is not None:
+ return
+ browser_page = self._register_page_locked(page)
+ new_id = browser_page.id
+ while self._pending_popups:
+ waiter = self._pending_popups.pop(0)
+ if not waiter.done():
+ waiter.set_result(new_id)
+ break
+ if new_id not in self._background_popup_pages:
+ self.last_interacted_browser_id = new_id
+ else:
+ self._background_popup_pages.discard(new_id)
+ except Exception as exc:
+ PrintStyle.warning(f"Popup registration failed: {exc}")
+
def _browser_id_for_page(self, page: Any) -> int | None:
for browser_id, browser_page in self.pages.items():
if browser_page.page == page:
diff --git a/plugins/_browser/prompts/agent.system.tool.browser.md b/plugins/_browser/prompts/agent.system.tool.browser.md
index b0112ae25..e8c271792 100644
--- a/plugins/_browser/prompts/agent.system.tool.browser.md
+++ b/plugins/_browser/prompts/agent.system.tool.browser.md
@@ -4,8 +4,8 @@ use for web browsing, page inspection, forms, downloads, and browser-only tasks
state stays open per chat context
refs come from content as typed markers: [link 3], [button 6], [image 1], [input text 8]
-actions: open list state navigate back forward reload content detail click type submit type_submit scroll evaluate close close_all
-common args: action browser_id url ref text selector selectors script
+actions: open list state set_active navigate back forward reload content detail click type submit type_submit scroll evaluate key_chord mouse multi close close_all
+common args: action browser_id url ref text selector selectors script modifiers keys include_content focus_popup event_type x y button calls
workflow:
- open creates a new browser and returns id/state
@@ -13,7 +13,38 @@ workflow:
- detail inspects one ref, including link/image/input/button metadata
- click/type/type_submit/submit/scroll use refs from latest content capture and return {action,state}
- navigate/back/forward/reload return fresh state
-- list shows open browsers
+- list shows open browsers; pass include_content: true for one-call bulk read
+
+modifier clicks:
+- click accepts modifiers like ["Control"], ["Shift"], ["Alt"], ["Meta"]
+- ctrl/meta-click opens link in new tab in background (Chrome rule)
+- override with focus_popup: true (focus follows new tab) or false (always background)
+- the new tab id is reported in action.opened_browser_ids; list shows all tabs
+
+popup awareness:
+- tabs opened by site (window.open, target=_blank, ctrl-click) auto-register
+- list returns every tab; last_interacted_browser_id tracks current focus
+
+background work (do not steal focus):
+- operations on a non-active tab (read, click, type, evaluate, etc.) target that tab WITHOUT moving focus
+- last_interacted_browser_id (and the WebUI viewer that follows it) only changes on:
+ - open (new tab created)
+ - explicit set_active action
+ - action on the already-active tab
+ - chrome popup-focus rule (plain click on target=_blank → follow; ctrl-click → stay)
+- to switch focus deliberately: {"action":"set_active","browser_id":N}
+
+key_chord:
+- presses keys in order, releases in reverse; safe across exceptions
+- example: {"action":"key_chord","keys":["Control","a"]} selects all
+
+multi (parallel batch):
+- run many actions concurrently across tabs in one tool call
+- pass calls: array of action objects (each has its own action+args)
+- different browser_ids run in parallel; same browser_id runs in submit order
+- returns array of {"ok":true,"result":...} or {"ok":false,"error":"..."} matching input order
+- ideal for: scrape N tabs at once, fan-out reads, parallel evaluate
+- avoid mutating same tab twice in one batch unless serial order is intended
examples:
~~~json
@@ -46,3 +77,50 @@ examples:
}
}
~~~
+
+~~~json
+{
+ "tool_name": "browser",
+ "tool_args": {
+ "action": "click",
+ "browser_id": 1,
+ "ref": 3,
+ "modifiers": ["Control"]
+ }
+}
+~~~
+
+~~~json
+{
+ "tool_name": "browser",
+ "tool_args": {
+ "action": "key_chord",
+ "browser_id": 1,
+ "keys": ["Control", "a"]
+ }
+}
+~~~
+
+~~~json
+{
+ "tool_name": "browser",
+ "tool_args": {
+ "action": "list",
+ "include_content": true
+ }
+}
+~~~
+
+~~~json
+{
+ "tool_name": "browser",
+ "tool_args": {
+ "action": "multi",
+ "calls": [
+ {"action": "content", "browser_id": 1},
+ {"action": "content", "browser_id": 2},
+ {"action": "evaluate", "browser_id": 3, "script": "document.title"}
+ ]
+ }
+}
+~~~
diff --git a/plugins/_browser/tools/browser.py b/plugins/_browser/tools/browser.py
index 8ea6001e2..309428935 100644
--- a/plugins/_browser/tools/browser.py
+++ b/plugins/_browser/tools/browser.py
@@ -18,18 +18,34 @@ class Browser(Tool):
selector: str = "",
selectors: list[str] | None = None,
script: str = "",
+ modifiers: list[str] | str | None = None,
+ keys: list[str] | None = None,
+ include_content: bool = False,
+ focus_popup: bool | None = None,
+ event_type: str = "",
+ x: float = 0.0,
+ y: float = 0.0,
+ button: str = "left",
+ calls: list[dict[str, Any]] | None = None,
**kwargs: Any,
) -> Response:
action = str(action or self.method or "state").strip().lower().replace("-", "_")
runtime = await get_runtime(self.agent.context.id)
+ if isinstance(modifiers, str):
+ modifiers = [modifiers] if modifiers else None
+ elif isinstance(modifiers, list) and not modifiers:
+ modifiers = None
+
try:
if action == "open":
result = await runtime.call("open", url or "")
elif action == "list":
- result = await runtime.call("list")
+ result = await runtime.call("list", include_content=bool(include_content))
elif action == "state":
result = await runtime.call("state", browser_id)
+ elif action in {"set_active", "setactive", "activate", "focus"}:
+ result = await runtime.call("set_active", browser_id)
elif action == "navigate":
result = await runtime.call("navigate", browser_id, url)
elif action == "back":
@@ -44,7 +60,13 @@ class Browser(Tool):
elif action == "detail":
result = await runtime.call("detail", browser_id, self._require_ref(ref))
elif action == "click":
- result = await runtime.call("click", browser_id, self._require_ref(ref))
+ if modifiers:
+ result = await runtime.call(
+ "click", browser_id, self._require_ref(ref),
+ modifiers=modifiers, focus_popup=focus_popup,
+ )
+ else:
+ result = await runtime.call("click", browser_id, self._require_ref(ref))
elif action == "type":
result = await runtime.call("type", browser_id, self._require_ref(ref), text)
elif action == "submit":
@@ -60,6 +82,19 @@ class Browser(Tool):
result = await runtime.call("scroll", browser_id, self._require_ref(ref))
elif action == "evaluate":
result = await runtime.call("evaluate", browser_id, script)
+ elif action in {"key_chord", "keychord"}:
+ if not keys:
+ raise ValueError("key_chord requires non-empty 'keys' list")
+ result = await runtime.call("key_chord", browser_id, list(keys))
+ elif action == "mouse":
+ result = await runtime.call(
+ "mouse", browser_id, event_type or "click", x, y,
+ button=button or "left", modifiers=modifiers,
+ )
+ elif action == "multi":
+ if not calls:
+ raise ValueError("multi requires non-empty 'calls' list")
+ result = await runtime.call("multi", list(calls))
elif action == "close":
result = await runtime.call("close_browser", browser_id)
elif action == "close_all":