Personal AI, On Personal Devices
Python
browser_click: accept param aliases and resolve natural-language descriptions (#631)
esslamali15faa06effebd2
diff --git a/src/openjarvis/tools/browser.py b/src/openjarvis/tools/browser.pyindex 59ce70c9..e62cf929 100644--- a/src/openjarvis/tools/browser.py+++ b/src/openjarvis/tools/browser.py@@ -162,15 +162,19 @@ class BrowserClickTool(BaseTool):return ToolSpec(name="browser_click",description=(- "Click an element on the current page."- " Use a CSS selector or text content to identify the element."+ "Click an element on the current page. Pass the text you see"+ " on the element (button label, link title), a CSS selector,"+ " or a short description like 'first video result'."),parameters={"type": "object","properties": {"selector": {"type": "string",- "description": "CSS selector or text content of the element.",+ "description": (+ "Visible text of the element, a CSS selector, or"+ " a short description (e.g. 'first video result')."+ ),},"by_text": {"type": "boolean",@@ -179,18 +183,66 @@ class BrowserClickTool(BaseTool):" instead of CSS selector. Default: false."),},+ "target": {+ "type": "string",+ "description": "Alias for selector.",+ },+ "text": {+ "type": "string",+ "description": "Alias for selector, for visible element text.",+ },+ "element": {+ "type": "string",+ "description": "Alias for selector.",+ },+ "query": {+ "type": "string",+ "description": "Alias for selector.",+ },},- "required": ["selector"],+ # Require one accepted spelling without forcing callers to+ # use ``selector`` when an advertised alias is supplied.+ "anyOf": [+ {"required": ["selector"]},+ {"required": ["target"]},+ {"required": ["text"]},+ {"required": ["element"]},+ {"required": ["query"]},+ ],},category="browser",)+ _ORDINALS = {"first": 0, "second": 1, "third": 2, "fourth": 3, "last": -1}++ # Filler words stripped from natural-language descriptions before+ # keyword matching ("the first video result" -> "video"). Derive the+ # ordinal entries from the supported mapping so adding one cannot leave it+ # behind as a literal role-name keyword.+ _DESC_STOPWORDS = frozenset(+ "the a an top main big red blue green"+ " result results item element link button option entry row".split()+ ) | frozenset(_ORDINALS)+def execute(self, **params: Any) -> ToolResult:- selector = params.get("selector", "")+ # Models routinely name this param `target`, `text`, `element` or+ # `query` instead of `selector`; rejecting those costs the whole+ # workflow, so accept the common aliases.+ selector = str(+ params.get("selector")+ or params.get("target")+ or params.get("text")+ or params.get("element")+ or params.get("query")+ or ""+ ).strip()if not selector:return ToolResult(tool_name="browser_click",- content="No selector provided.",+ content=(+ "No selector provided. Pass the visible text of the"+ " element or a CSS selector."+ ),success=False,)@@ -198,16 +250,16 @@ class BrowserClickTool(BaseTool):try:page = _session.page- if by_text:- page.get_by_text(selector).click()- else:- page.click(selector)-+ strategy = self._resolve_and_click(page, selector, by_text)return ToolResult(tool_name="browser_click",content=f"Clicked element: {selector}",success=True,- metadata={"selector": selector, "by_text": by_text},+ metadata={+ "selector": selector,+ "by_text": by_text,+ "strategy": strategy,+ },)except ImportError:return ToolResult(@@ -224,6 +276,111 @@ class BrowserClickTool(BaseTool):success=False,)+ def _resolve_and_click(self, page: Any, selector: str, by_text: bool) -> str:+ """Resolve *selector* through a strategy ladder and click it.++ LLMs pass anything from precise CSS to plain descriptions like+ "first video result". Strategies from most to least precise: CSS,+ exact text, fuzzy text, YouTube results video title, role-based+ keyword match. Existence is checked with ``count()`` (instant, no+ actionability wait) so bad strategies cost milliseconds; only a+ strategy that actually matched gets a real click timeout. Searches+ the main frame first, then child iframes (consent dialogs and ad+ overlays live there). Returns the name of the winning strategy;+ raises with a list of visible clickable elements when nothing+ matched, so the model can retry sensibly.+ """+ import re++ words = [w for w in re.split(r"\W+", selector.lower()) if w]+ ordinal = next((self._ORDINALS[w] for w in words if w in self._ORDINALS), None)++ def candidates(frame: Any) -> list:+ cands = []+ if not by_text:+ cands.append(("css", frame.locator(selector)))+ cands.append(("text-exact", frame.get_by_text(selector, exact=True)))+ cands.append(+ ("text-fuzzy", frame.get_by_text(re.compile(re.escape(selector), re.I)))+ )+ # YouTube search results: "first video result" must click a+ # video title, not the "Videos" filter chip — so this outranks+ # the generic role-based keyword match below.+ try:+ url = frame.url or ""+ except Exception:+ url = ""+ if "youtube." in url and ("video" in words or "result" in words):+ cands.append(("youtube-result", frame.locator("a#video-title")))+ keywords = [w for w in words if w not in self._DESC_STOPWORDS]+ if keywords:+ pat = re.compile(".*".join(re.escape(k) for k in keywords), re.I)+ for role in ("link", "button"):+ cands.append((f"role-{role}", frame.get_by_role(role, name=pat)))+ return cands++ def actionable_matches(locator: Any) -> list[Any]:+ """Return click-ordered matches, excluding hidden or disabled ones."""+ count = locator.count()+ matches = []+ # A locator can match hidden mobile/desktop duplicates before the+ # visible control. Build the actionable sequence first so an+ # ordinal such as "first" means the first element a user can+ # actually click, not the first raw DOM match.+ for index in range(count):+ match = locator.nth(index)+ try:+ if not match.is_visible() or not match.is_enabled():+ continue+ except Exception:+ # The click remains the final Playwright actionability+ # check, including stability and event-receivability.+ pass+ matches.append(match)++ if ordinal is not None:+ index = len(matches) - 1 if ordinal == -1 else ordinal+ return [matches[index]] if 0 <= index < len(matches) else []+ return matches++ last_error: Exception | None = None+ for index, frame in enumerate(page.frames):+ for strategy, locator in candidates(frame):+ try:+ matches = actionable_matches(locator)+ if not matches:+ continue+ except Exception:+ continue # e.g. selector isn't valid CSS — next rung+ for match in matches:+ try:+ match.click(timeout=8000 if index == 0 else 4000)+ return strategy+ except Exception as exc:+ last_error = exc+ continue++ hint = ""+ try:+ texts = page.evaluate(+ """(limit) => Array.from(+ document.querySelectorAll('a, button, [role=button], [role=link]')+ ).map(e => (e.innerText || e.getAttribute('aria-label') || '').trim())+ .filter(t => t && t.length < 80)+ .filter((t, i, arr) => arr.indexOf(t) === i)+ .slice(0, limit)""",+ 15,+ )+ if texts:+ hint = " Visible clickable elements: " + "; ".join(texts)+ except Exception:+ pass+ detail = f" Last error: {last_error}" if last_error else ""+ raise RuntimeError(+ f"No element matched '{selector}' in the main page or its"+ f" {len(page.frames) - 1} iframe(s).{hint}{detail}"+ )+# ---------------------------------------------------------------------------# Tool 3: BrowserTypeTooldiff --git a/tests/tools/test_browser.py b/tests/tools/test_browser.pyindex 1e070f5a..62623a37 100644--- a/tests/tools/test_browser.py+++ b/tests/tools/test_browser.py@@ -282,12 +282,20 @@ class TestBrowserClickTool:assert tool.spec.name == "browser_click"assert tool.spec.category == "browser"- def test_spec_requires_selector(self):+ def test_spec_advertises_selector_and_aliases(self):from openjarvis.tools.browser import BrowserClickTooltool = BrowserClickTool()- assert "selector" in tool.spec.parameters["properties"]- assert "selector" in tool.spec.parameters["required"]+ properties = tool.spec.parameters["properties"]+ assert {"selector", "target", "text", "element", "query"} <= set(properties)+ assert "required" not in tool.spec.parameters+ assert tool.spec.parameters["anyOf"] == [+ {"required": ["selector"]},+ {"required": ["target"]},+ {"required": ["text"]},+ {"required": ["element"]},+ {"required": ["query"]},+ ]def test_spec_has_by_text_parameter(self):from openjarvis.tools.browser import BrowserClickTool@@ -317,6 +325,141 @@ class TestBrowserClickTool:assert result.success is Falseassert "No selector" in result.content+ def test_resolver_skips_hidden_duplicate_match(self):+ from openjarvis.tools.browser import BrowserClickTool++ class Match:+ def __init__(self, *, visible=True, enabled=True):+ self.visible = visible+ self.enabled = enabled+ self.clicked = False++ def is_visible(self):+ return self.visible++ def is_enabled(self):+ return self.enabled++ def click(self, **_kwargs):+ self.clicked = True++ class Locator:+ def __init__(self, matches):+ self.matches = matches++ def count(self):+ return len(self.matches)++ def nth(self, index):+ return self.matches[index]++ hidden = Match(visible=False)+ visible = Match()+ empty = Locator([])+ duplicate = Locator([hidden, visible])+ frame = MagicMock(url="https://example.com")+ frame.locator.return_value = duplicate+ frame.get_by_text.return_value = empty+ frame.get_by_role.return_value = empty+ page = MagicMock(frames=[frame])++ strategy = BrowserClickTool()._resolve_and_click(page, ".save", False)++ assert strategy == "css"+ assert hidden.clicked is False+ assert visible.clicked is True++ def test_resolver_applies_ordinal_after_actionability_filter(self):+ from openjarvis.tools.browser import BrowserClickTool++ class Match:+ def __init__(self, *, visible=True):+ self.visible = visible+ self.clicked = False++ def is_visible(self):+ return self.visible++ def is_enabled(self):+ return True++ def click(self, **_kwargs):+ self.clicked = True++ class Locator:+ def __init__(self, matches):+ self.matches = matches++ def count(self):+ return len(self.matches)++ def nth(self, index):+ return self.matches[index]++ hidden = Match(visible=False)+ first_visible = Match()+ second_visible = Match()+ duplicate = Locator([hidden, first_visible, second_visible])+ empty = Locator([])+ frame = MagicMock(url="https://example.com")+ frame.get_by_text.return_value = duplicate+ frame.get_by_role.return_value = empty+ page = MagicMock(frames=[frame])++ strategy = BrowserClickTool()._resolve_and_click(page, "first save", True)++ assert strategy == "text-exact"+ assert hidden.clicked is False+ assert first_visible.clicked is True+ assert second_visible.clicked is False++ def test_resolver_clicks_fourth_actionable_role_match(self):+ from openjarvis.tools.browser import BrowserClickTool++ class Match:+ def __init__(self):+ self.clicked = False++ def is_visible(self):+ return True++ def is_enabled(self):+ return True++ def click(self, **_kwargs):+ self.clicked = True++ class Locator:+ def __init__(self, matches=()):+ self.matches = list(matches)++ def count(self):+ return len(self.matches)++ def nth(self, index):+ return self.matches[index]++ buttons = [Match() for _ in range(4)]+ empty = Locator()+ frame = MagicMock(url="https://example.com")+ frame.get_by_text.return_value = empty+ frame.get_by_role.side_effect = lambda role, name: (+ Locator(buttons) if role == "button" and name.search("Save") else empty+ )+ page = MagicMock(frames=[frame])++ strategy = BrowserClickTool()._resolve_and_click(+ page, "fourth save button", True+ )++ assert strategy == "role-button"+ assert [match.clicked for match in buttons] == [False, False, False, True]++ def test_every_supported_ordinal_is_a_description_stopword(self):+ from openjarvis.tools.browser import BrowserClickTool++ assert set(BrowserClickTool._ORDINALS) <= BrowserClickTool._DESC_STOPWORDS+def test_execute_playwright_not_installed(self):from openjarvis.tools.browser import BrowserClickTool@@ -331,6 +474,14 @@ class TestBrowserClickTool:from openjarvis.tools.browser import BrowserClickToolpage = _make_mock_page()+ page.frames = [page]+ match = MagicMock()+ match.is_visible.return_value = True+ match.is_enabled.return_value = True+ page.locator.return_value.count.return_value = 1+ page.locator.return_value.nth.return_value = match+ page.get_by_text.return_value.count.return_value = 0+ page.get_by_role.return_value.count.return_value = 0session = _make_mock_session(page)with patch("openjarvis.tools.browser._session", session):@@ -339,7 +490,7 @@ class TestBrowserClickTool:assert result.success is Trueassert "Clicked element" in result.content- page.click.assert_called_once_with("#submit-btn")+ match.click.assert_called_once_with(timeout=8000)assert result.metadata["selector"] == "#submit-btn"assert result.metadata["by_text"] is False@@ -347,6 +498,12 @@ class TestBrowserClickTool:from openjarvis.tools.browser import BrowserClickToolpage = _make_mock_page()+ page.frames = [page]+ match = MagicMock()+ match.is_visible.return_value = True+ match.is_enabled.return_value = True+ page.get_by_text.return_value.count.return_value = 1+ page.get_by_text.return_value.nth.return_value = matchsession = _make_mock_session(page)with patch("openjarvis.tools.browser._session", session):@@ -354,15 +511,23 @@ class TestBrowserClickTool:result = tool.execute(selector="Sign In", by_text=True)assert result.success is True- page.get_by_text.assert_called_once_with("Sign In")- page.get_by_text.return_value.click.assert_called_once()+ assert page.get_by_text.call_count >= 1+ match.click.assert_called_once_with(timeout=8000)assert result.metadata["by_text"] is Truedef test_execute_click_error(self):from openjarvis.tools.browser import BrowserClickToolpage = _make_mock_page()- page.click.side_effect = Exception("Element not found")+ page.frames = [page]+ match = MagicMock()+ match.is_visible.return_value = True+ match.is_enabled.return_value = True+ match.click.side_effect = Exception("Element not found")+ page.locator.return_value.count.return_value = 1+ page.locator.return_value.nth.return_value = match+ page.get_by_text.return_value.count.return_value = 0+ page.get_by_role.return_value.count.return_value = 0session = _make_mock_session(page)with patch("openjarvis.tools.browser._session", session):