Agent Zero AI framework
Improve browser tool ergonomics for agent UI control
Alessandro7b1c84aeca74
diff --git a/plugins/_browser/assets/browser-page-content.js b/plugins/_browser/assets/browser-page-content.jsindex c1330d154..65a798ac6 100644--- a/plugins/_browser/assets/browser-page-content.js+++ b/plugins/_browser/assets/browser-page-content.js@@ -1,7 +1,7 @@(() => {const GLOBAL_KEY = "__spaceBrowserPageContent__";const DOM_HELPER_KEY = "__spaceBrowserDomHelper__";- const VERSION = "11";+ const VERSION = "12";const REQUIRED_API_NAMES = Object.freeze(["annotate","boundingBoxFor",@@ -646,6 +646,19 @@.split(/[.:]/u, 1)[0];}+ function isGlobalOrDelegatedEventBinding(value) {+ const parts = String(value || "")+ .trim()+ .toLowerCase()+ .split(/[.:]/u)+ .map((part) => part.trim())+ .filter(Boolean);+ return parts.includes("window")+ || parts.includes("document")+ || parts.includes("outside")+ || parts.includes("away");+ }+function isInteractiveEventName(value) {return INTERACTIVE_EVENT_NAMES.has(normalizeInteractiveEventName(value));}@@ -657,10 +670,16 @@}if (normalizedName.startsWith("@")) {+ if (isGlobalOrDelegatedEventBinding(normalizedName.slice(1))) {+ return false;+ }return isInteractiveEventName(normalizedName.slice(1));}if (normalizedName.startsWith("x-on:") || normalizedName.startsWith("v-on:")) {+ if (isGlobalOrDelegatedEventBinding(normalizedName.slice(5))) {+ return false;+ }return isInteractiveEventName(normalizedName.slice(5));}diff --git a/plugins/_browser/helpers/connector_runtime.py b/plugins/_browser/helpers/connector_runtime.pyindex 1d9fb86a2..c2daac636 100644--- a/plugins/_browser/helpers/connector_runtime.py+++ b/plugins/_browser/helpers/connector_runtime.py@@ -63,6 +63,18 @@ get_browser_config = browser_config.get_browser_config_LOCAL_PROVIDERS = {"ollama", "lm_studio"}_LOCAL_HOSTS = {"localhost", "127.0.0.1", "::1", "host.docker.internal"}_SENSITIVE_ACTIONS = {"content", "detail", "evaluate", "screenshot", "screenshot_file"}+_KEY_ALIASES = {+ "cmd": "Meta",+ "command": "Meta",+ "control": "Control",+ "ctrl": "Control",+ "escape": "Escape",+ "esc": "Escape",+ "meta": "Meta",+ "option": "Alt",+ "return": "Enter",+ "space": "Space",+}_REQUIRED_API_NAMES_RE = re.compile(r"const\s+REQUIRED_API_NAMES\s*=\s*Object\.freeze\(\[(?P<body>.*?)\]\);",re.S,@@ -151,7 +163,7 @@ class ConnectorBrowserRuntime:payload["text"] = kwargs.get("text", "")elif action == "key_chord":payload["browser_id"] = args[0] if args else None- payload["keys"] = args[1] if len(args) > 1 else []+ payload["keys"] = self._normalize_keys(args[1] if len(args) > 1 else [])elif action == "clipboard":payload["browser_id"] = args[0] if args else Nonepayload["clipboard_action"] = kwargs.get("action", "")@@ -196,11 +208,45 @@ class ConnectorBrowserRuntime:normalized["url"] = cls._normalize_open_url(normalized.get("url"))elif action == "navigate":normalized["url"] = normalize_url(normalized.get("url", ""))+ elif action == "click" and not normalized.get("ref") and (+ normalized.get("x") or normalized.get("y")+ ):+ normalized["action"] = "mouse"+ normalized.setdefault("event_type", "click")+ normalized.setdefault("button", "left")+ elif action == "type" and not normalized.get("ref"):+ normalized["action"] = "keyboard"+ normalized.setdefault("key", "")+ elif action in {"key_chord", "keychord"}:+ normalized["keys"] = cls._normalize_keys(normalized.get("keys"))elif action == "multi" or isinstance(normalized.get("calls"), list):normalized["calls"] = cls._normalize_multi_calls(normalized.get("calls", []))normalized_calls.append(normalized)return normalized_calls+ @staticmethod+ def _normalize_keys(keys: Any) -> list[str]:+ if keys is None:+ return []+ if isinstance(keys, str):+ raw = re.split(r"\s*\+\s*|\s*,\s*", keys.strip())+ elif isinstance(keys, list):+ raw = keys+ else:+ raw = [str(keys)]+ normalized: list[str] = []+ for key in raw:+ value = str(key or "").strip()+ if not value:+ continue+ normalized.append(+ _KEY_ALIASES.get(+ value.lower(),+ value.upper() if len(value) == 1 and value.isalpha() else value,+ )+ )+ return normalized+async def _dispatch(self, payload: dict[str, Any]) -> Any:payload.setdefault("profile_mode", self._host_browser_profile_mode())self._enforce_privacy(payload)diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.pyindex 4dac2e290..664c4e8c5 100644--- a/plugins/_browser/helpers/runtime.py+++ b/plugins/_browser/helpers/runtime.py@@ -535,6 +535,18 @@ class BrowserRuntime:class _BrowserRuntimeCore:_VALID_MODIFIERS = {"Control", "Shift", "Alt", "Meta"}+ _KEY_ALIASES = {+ "cmd": "Meta",+ "command": "Meta",+ "control": "Control",+ "ctrl": "Control",+ "escape": "Escape",+ "esc": "Escape",+ "meta": "Meta",+ "option": "Alt",+ "return": "Enter",+ "space": "Space",+ }_POPUP_WAIT_SECONDS = 2.0def __init__(self, context_id: str):@@ -596,6 +608,29 @@ class _BrowserRuntimeCore:)return normalized+ @classmethod+ def _normalize_keys(cls, keys: list[str] | str | None) -> list[str]:+ if keys is None:+ return []+ if isinstance(keys, str):+ raw = re.split(r"\s*\+\s*|\s*,\s*", keys.strip())+ elif isinstance(keys, list):+ raw = keys+ else:+ raw = [str(keys)]+ normalized: list[str] = []+ for key in raw:+ value = str(key or "").strip()+ if not value:+ continue+ normalized.append(+ cls._KEY_ALIASES.get(+ value.lower(),+ value.upper() if len(value) == 1 and value.isalpha() else value,+ )+ )+ return normalized+@staticmethoddef _has_reference(reference_id: int | str | None) -> bool:return reference_id is not None and str(reference_id).strip() != ""@@ -980,6 +1015,15 @@ class _BrowserRuntimeCore:return await self.detail(bid, ref)if action == "click":ref = call.get("ref")+ if ref is None and (call.get("x") or call.get("y")):+ return await self.mouse(+ bid,+ "click",+ float(call.get("x") or 0),+ float(call.get("y") or 0),+ button=call.get("button") or "left",+ modifiers=self._normalize_modifiers(call.get("modifiers")),+ )if ref is None:raise ValueError("click requires ref")return await self.click(@@ -990,7 +1034,11 @@ class _BrowserRuntimeCore:if action == "type":ref = call.get("ref")if ref is None:- raise ValueError("type requires ref")+ return await self.keyboard(+ bid,+ key="",+ text=str(call.get("text") or ""),+ )return await self.type(bid, ref, call.get("text") or "")if action == "submit":ref = call.get("ref")@@ -1010,10 +1058,10 @@ class _BrowserRuntimeCore:if action == "evaluate":return await self.evaluate(bid, call.get("script") or "")if action in {"key_chord", "keychord"}:- keys = call.get("keys") or []+ keys = self._normalize_keys(call.get("keys"))if not keys:raise ValueError("key_chord requires non-empty keys")- return await self.key_chord(bid, list(keys))+ return await self.key_chord(bid, keys)if action == "mouse":return await self.mouse(bid, call.get("event_type") or "click",diff --git a/plugins/_browser/prompts/agent.system.tool.browser.md b/plugins/_browser/prompts/agent.system.tool.browser.mdindex c5ca25d0b..992a66374 100644--- a/plugins/_browser/prompts/agent.system.tool.browser.md+++ b/plugins/_browser/prompts/agent.system.tool.browser.md@@ -15,6 +15,8 @@ Workflow:- `open` creates a tab and returns id/state.- `content` returns markdown with refs like `[link 3]`, `[button 6]`, `[input text 8]`.- Interactions use refs from the latest `content` capture.+- For same-page controls that are easier to identify structurally, `click`, `type`, `submit`, `type_submit`, `scroll`, `select_option`, `set_checked`, and `upload_file` may use `selector` instead of `ref`; the tool resolves the selector through `content` first.+- `click` with `x`/`y` and no `ref` is treated as a coordinate mouse click. `type` with text and no `ref` types into the currently focused element. `key_chord` accepts either `["Control", "A"]` or `"CTRL+A"`.- `navigate` reuses an existing `browser_id` and is preferred for serial browsing.- Screenshots are explicit only; the browser does not automatically load screenshots. Call `vision_load` with the returned path before reasoning visually.- Keep the tab set small; close pages after extracting what you need.diff --git a/plugins/_browser/tools/browser.py b/plugins/_browser/tools/browser.pyindex 1310da7e2..b0bad5476 100644--- a/plugins/_browser/tools/browser.py+++ b/plugins/_browser/tools/browser.py@@ -1,6 +1,7 @@from __future__ import annotationsimport json+import reimport timeimport uuidfrom pathlib import Path@@ -82,6 +83,7 @@ class Browser(Tool):modifiers = [modifiers] if modifiers else Noneelif isinstance(modifiers, list) and not modifiers:modifiers = None+ keys = self._normalize_keys(keys)try:if action == "open":@@ -114,34 +116,70 @@ class Browser(Tool):payload = self._selector_payload(selector, selectors)result = await runtime.call("content", browser_id, payload)elif action == "detail":- result = await runtime.call("detail", browser_id, self._require_ref(ref))+ result = await runtime.call(+ "detail",+ browser_id,+ await self._resolve_ref(runtime, browser_id, ref, selector, action),+ )elif action == "click":- if modifiers:+ resolved_ref = await self._resolve_ref(+ runtime,+ browser_id,+ ref,+ selector,+ action,+ required=not self._has_coordinates(x, y),+ )+ if resolved_ref is None and self._has_coordinates(x, y):result = await runtime.call(- "click", browser_id, self._require_ref(ref),+ "mouse", browser_id, "click", x, y,+ button=button or "left", modifiers=modifiers,+ )+ elif modifiers:+ result = await runtime.call(+ "click", browser_id, resolved_ref,modifiers=modifiers, focus_popup=focus_popup,)else:- result = await runtime.call("click", browser_id, self._require_ref(ref))+ result = await runtime.call("click", browser_id, resolved_ref)elif action == "type":- result = await runtime.call("type", browser_id, self._require_ref(ref), text)+ resolved_ref = await self._resolve_ref(+ runtime,+ browser_id,+ ref,+ selector,+ action,+ required=False,+ )+ if resolved_ref is None:+ result = await runtime.call("keyboard", browser_id, key="", text=text)+ else:+ result = await runtime.call("type", browser_id, resolved_ref, text)elif action == "submit":- result = await runtime.call("submit", browser_id, self._require_ref(ref))+ result = await runtime.call(+ "submit",+ browser_id,+ await self._resolve_ref(runtime, browser_id, ref, selector, action),+ )elif action in {"type_submit", "typesubmit"}:result = await runtime.call("type_submit",browser_id,- self._require_ref(ref),+ await self._resolve_ref(runtime, browser_id, ref, selector, action),text,)elif action == "scroll":- result = await runtime.call("scroll", browser_id, self._require_ref(ref))+ result = await runtime.call(+ "scroll",+ browser_id,+ await self._resolve_ref(runtime, browser_id, ref, selector, action),+ )elif action == "evaluate":result = await runtime.call("evaluate", browser_id, script)elif action in {"key_chord", "keychord"}:if not keys:raise ValueError("key_chord requires non-empty 'keys' list")- result = await runtime.call("key_chord", browser_id, list(keys))+ result = await runtime.call("key_chord", browser_id, keys)elif action == "hover":result = await runtime.call("hover",@@ -232,7 +270,7 @@ class Browser(Tool):result = await runtime.call("select_option",browser_id,- self._require_ref(ref),+ await self._resolve_ref(runtime, browser_id, ref, selector, action),value=value,values=values,)@@ -240,14 +278,14 @@ class Browser(Tool):result = await runtime.call("set_checked",browser_id,- self._require_ref(ref),+ await self._resolve_ref(runtime, browser_id, ref, selector, action),checked=True if checked is None else bool(checked),)elif action == "upload_file":result = await runtime.call("upload_file",browser_id,- self._require_ref(ref),+ await self._resolve_ref(runtime, browser_id, ref, selector, action),path=path,paths=paths,)@@ -290,6 +328,85 @@ class Browser(Tool):raise ValueError("ref is required for this browser action")return ref+ @staticmethod+ def _has_ref(ref: int | str | None) -> bool:+ return ref is not None and str(ref).strip() != ""++ @staticmethod+ def _has_coordinates(x: float, y: float) -> bool:+ return bool(float(x or 0) or float(y or 0))++ @classmethod+ async def _resolve_ref(+ cls,+ runtime: Any,+ browser_id: int | str | None,+ ref: int | str | None,+ selector: str = "",+ action: str = "action",+ *,+ required: bool = True,+ ) -> int | str | None:+ if cls._has_ref(ref):+ return ref++ selector = str(selector or "").strip()+ if selector:+ content = await runtime.call("content", browser_id, {"selector": selector})+ resolved = cls._first_ref_from_content(content, selector)+ if resolved is not None:+ return resolved+ raise ValueError(+ f"{action} could not resolve selector {selector!r} to a browser ref"+ )++ if required:+ return cls._require_ref(ref)+ return None++ @staticmethod+ def _first_ref_from_content(content: Any, selector: str = "") -> str | None:+ if isinstance(content, dict):+ values: list[Any] = []+ if selector and selector in content:+ values.append(content.get(selector))+ values.extend(value for key, value in content.items() if key != selector)+ text = "\n".join(str(value or "") for value in values)+ else:+ text = str(content or "")+ match = re.search(r"\[[^\]\n]*?\b(\d+)\]", text)+ return match.group(1) if match else None++ @staticmethod+ def _normalize_keys(keys: list[str] | str | None) -> list[str]:+ if keys is None:+ return []+ if isinstance(keys, str):+ raw = re.split(r"\s*\+\s*|\s*,\s*", keys.strip())+ elif isinstance(keys, list):+ raw = keys+ else:+ raw = [str(keys)]+ aliases = {+ "cmd": "Meta",+ "command": "Meta",+ "control": "Control",+ "ctrl": "Control",+ "escape": "Escape",+ "esc": "Escape",+ "meta": "Meta",+ "option": "Alt",+ "return": "Enter",+ "space": "Space",+ }+ normalized: list[str] = []+ for key in raw:+ value = str(key or "").strip()+ if not value:+ continue+ normalized.append(aliases.get(value.lower(), value.upper() if len(value) == 1 and value.isalpha() else value))+ return normalized+@staticmethoddef _selector_payload(selector: str = "", selectors: list[str] | None = None) -> dict | None:if selectors:diff --git a/plugins/_browser/webui/browser-panel.html b/plugins/_browser/webui/browser-panel.htmlindex 5bea3661d..ec579836a 100644--- a/plugins/_browser/webui/browser-panel.html+++ b/plugins/_browser/webui/browser-panel.html@@ -173,9 +173,9 @@</button></div>- <form class="browser-address-form" @submit.prevent="$store.browserPage.go()">+ <form class="browser-address-form" aria-label="Browser navigation" @submit.prevent="$store.browserPage.go()"><span class="material-symbols-outlined browser-address-icon">language</span>- <input class="browser-address" x-model="$store.browserPage.address"+ <input class="browser-address" aria-label="Browser address" name="browser_address" x-model="$store.browserPage.address"@focus="$store.browserPage.onAddressFocus()" @blur="$store.browserPage.onAddressBlur()":disabled="$store.browserPage.isBusy()"placeholder="https://example.com" autocomplete="off" />diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.pyindex bc637b97b..8d2ec8bd5 100644--- a/tests/test_browser_agent_regressions.py+++ b/tests/test_browser_agent_regressions.py@@ -1237,7 +1237,7 @@ def test_browser_content_helper_keeps_label_wrapped_controls_referenceable():PROJECT_ROOT / "plugins" / "_browser" / "assets" / "browser-page-content.js").read_text(encoding="utf-8")- assert 'const VERSION = "11"' in helper+ assert 'const VERSION = "12"' in helperassert "function patchOpenShadowDom" in helperassert "Element.prototype.attachShadow = patched" in helperassert "const REQUIRED_API_NAMES = Object.freeze([" in helper@@ -1249,6 +1249,18 @@ def test_browser_content_helper_keeps_label_wrapped_controls_referenceable():assert "getLabelElementText(labelElement, element)" in helperassert "return renderControlLabelReferences(node, context);" in helperassert "return renderControlLabelReferences(element, context);" in helper+ assert "function isGlobalOrDelegatedEventBinding" in helper+ assert 'parts.includes("window")' in helper+ assert 'parts.includes("outside")' in helper+++def test_browser_panel_exposes_agent_friendly_address_input():+ panel = (+ PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html"+ ).read_text(encoding="utf-8")++ assert 'class="browser-address-form" aria-label="Browser navigation"' in panel+ assert 'class="browser-address" aria-label="Browser address" name="browser_address"' in paneldef test_browser_runtime_requires_current_content_helper_for_modifier_clicks():@@ -1637,6 +1649,9 @@ async def test_browser_tool_dispatches_v1_agent_actions(monkeypatch):await execute(action="select_option", browser_id=1, ref=6, value="CA")await execute(action="set_checked", browser_id=1, ref=7, checked=False)await execute(action="upload_file", browser_id=1, ref=8, paths=["/tmp/a.txt"])+ await execute(action="key_chord", browser_id=1, keys="CTRL+A")+ await execute(action="click", browser_id=1, x=10, y=20)+ await execute(action="type", browser_id=1, text="agent-zero.ai")assert calls == [("screenshot_file", (1,), {"quality": 91, "full_page": True, "path": "/tmp/a.jpg"}),@@ -1690,6 +1705,81 @@ async def test_browser_tool_dispatches_v1_agent_actions(monkeypatch):("select_option", (1, 6), {"value": "CA", "values": None}),("set_checked", (1, 7), {"checked": False}),("upload_file", (1, 8), {"path": "", "paths": ["/tmp/a.txt"]}),+ ("key_chord", (1, ["Control", "A"]), {}),+ ("mouse", (1, "click", 10, 20), {"button": "left", "modifiers": None}),+ ("keyboard", (1,), {"key": "", "text": "agent-zero.ai"}),+ ]+++async def test_browser_tool_resolves_selector_for_reference_actions(monkeypatch):+ calls = []++ class FakeRuntime:+ async def call(self, method, *args, **kwargs):+ calls.append((method, args, kwargs))+ if method == "content":+ return {"input.browser-address": "[input text 31] Browser address"}+ return {"ok": True, "method": method, "args": args, "kwargs": kwargs}++ async def fake_get_runtime(context_id, create=True, agent=None):+ del create, agent+ assert context_id == "ctx"+ return FakeRuntime()++ monkeypatch.setattr(browser_tool_module, "get_runtime", fake_get_runtime)+ tool = browser_tool_module.Browser(+ agent=SimpleNamespace(context=SimpleNamespace(id="ctx")),+ name="browser",+ method=None,+ args={},+ message="",+ loop_data=None,+ )++ response = await tool.execute(+ action="type_submit",+ browser_id=1,+ selector="input.browser-address",+ text="agent-zero.ai",+ )++ assert response.break_loop is False+ assert calls == [+ ("content", (1, {"selector": "input.browser-address"}), {}),+ ("type_submit", (1, "31", "agent-zero.ai"), {}),+ ]+++async def test_browser_runtime_multi_accepts_human_shaped_input_calls():+ calls = []+ core = _BrowserRuntimeCore("ctx")++ async def fake_mouse(browser_id, event_type, x, y, *, button="left", modifiers=None):+ calls.append(("mouse", browser_id, event_type, x, y, button, modifiers))+ return {"ok": True}++ async def fake_keyboard(browser_id, *, key="", text=""):+ calls.append(("keyboard", browser_id, key, text))+ return {"ok": True}++ async def fake_key_chord(browser_id, keys):+ calls.append(("key_chord", browser_id, keys))+ return {"ok": True}++ core.mouse = fake_mouse+ core.keyboard = fake_keyboard+ core.key_chord = fake_key_chord++ assert await core._dispatch_call({"action": "click", "browser_id": 1, "x": 10, "y": 20}) == {"ok": True}+ assert await core._dispatch_call({"action": "type", "browser_id": 1, "text": "agent-zero.ai"}) == {"ok": True}+ assert await core._dispatch_call({"action": "key_chord", "browser_id": 1, "keys": "CTRL+A"}) == {"ok": True}++ assert calls == [+ ("mouse", 1, "click", 10.0, 20.0, "left", None),+ ("keyboard", 1, "", "agent-zero.ai"),+ ("key_chord", 1, ["Control", "A"]),]diff --git a/tests/test_host_browser_connector.py b/tests/test_host_browser_connector.pyindex 61874ec8a..e1be412c9 100644--- a/tests/test_host_browser_connector.py+++ b/tests/test_host_browser_connector.py@@ -245,6 +245,9 @@ def test_connector_runtime_normalizes_host_navigation_payloads(monkeypatch):[{"action": "open", "url": "example.com"},{"action": "navigate", "browser_id": 1, "url": "127.0.0.1:8000/path"},+ {"action": "click", "browser_id": 1, "x": 12, "y": 34},+ {"action": "type", "browser_id": 1, "text": "agent-zero.ai"},+ {"action": "key_chord", "browser_id": 1, "keys": "CTRL+A"},{"action": "multi","calls": [{"action": "open", "url": "nested.example"}],@@ -258,9 +261,25 @@ def test_connector_runtime_normalizes_host_navigation_payloads(monkeypatch):assert navigate_payload["url"] == "https://novinky.cz/"assert multi_payload["calls"][0]["url"] == "https://example.com/"assert multi_payload["calls"][1]["url"] == "http://127.0.0.1:8000/path"- assert multi_payload["calls"][2]["calls"][0]["url"] == "https://nested.example/"- assert multi_payload["calls"][3] == {"action": "content", "browser_id": 1}+ assert multi_payload["calls"][2] == {+ "action": "mouse",+ "browser_id": 1,+ "x": 12,+ "y": 34,+ "event_type": "click",+ "button": "left",+ }+ assert multi_payload["calls"][3] == {+ "action": "keyboard",+ "browser_id": 1,+ "text": "agent-zero.ai",+ "key": "",+ }+ assert multi_payload["calls"][4]["keys"] == ["Control", "A"]+ assert multi_payload["calls"][5]["calls"][0]["url"] == "https://nested.example/"+ assert multi_payload["calls"][6] == {"action": "content", "browser_id": 1}assert open_payload["profile_mode"] == "existing"+ assert runtime._payload_for_call("key_chord", 1, "CTRL+A")["keys"] == ["Control", "A"]def test_connector_runtime_forwards_host_profile_mode(monkeypatch):