Skip to content

Personal AI, On Personal Devices

Python

browser_click: accept param aliases and resolve natural-language descriptions (#631)

esslamali15faa06effebd2
diff --git a/src/openjarvis/tools/browser.py b/src/openjarvis/tools/browser.py
index 59ce70c9..e62cf929 100644
--- a/src/openjarvis/tools/browser.py
+++ b/src/openjarvis/tools/browser.py
@@ -162,15 +162,19 @@ class BrowserClickTool(BaseTool):
return ToolSpec(
name="browser_click",
description=(
- "Click an element on the current page."
- " Use a CSS selector or text content to identify the element."
+ "Click an element on the current page. Pass the text you see"
+ " on the element (button label, link title), a CSS selector,"
+ " or a short description like 'first video result'."
),
parameters={
"type": "object",
"properties": {
"selector": {
"type": "string",
- "description": "CSS selector or text content of the element.",
+ "description": (
+ "Visible text of the element, a CSS selector, or"
+ " a short description (e.g. 'first video result')."
+ ),
},
"by_text": {
"type": "boolean",
@@ -179,18 +183,66 @@ class BrowserClickTool(BaseTool):
" instead of CSS selector. Default: false."
),
},
+ "target": {
+ "type": "string",
+ "description": "Alias for selector.",
+ },
+ "text": {
+ "type": "string",
+ "description": "Alias for selector, for visible element text.",
+ },
+ "element": {
+ "type": "string",
+ "description": "Alias for selector.",
+ },
+ "query": {
+ "type": "string",
+ "description": "Alias for selector.",
+ },
},
- "required": ["selector"],
+ # Require one accepted spelling without forcing callers to
+ # use ``selector`` when an advertised alias is supplied.
+ "anyOf": [
+ {"required": ["selector"]},
+ {"required": ["target"]},
+ {"required": ["text"]},
+ {"required": ["element"]},
+ {"required": ["query"]},
+ ],
},
category="browser",
)
+ _ORDINALS = {"first": 0, "second": 1, "third": 2, "fourth": 3, "last": -1}
+
+ # Filler words stripped from natural-language descriptions before
+ # keyword matching ("the first video result" -> "video"). Derive the
+ # ordinal entries from the supported mapping so adding one cannot leave it
+ # behind as a literal role-name keyword.
+ _DESC_STOPWORDS = frozenset(
+ "the a an top main big red blue green"
+ " result results item element link button option entry row".split()
+ ) | frozenset(_ORDINALS)
+
def execute(self, **params: Any) -> ToolResult:
- selector = params.get("selector", "")
+ # Models routinely name this param `target`, `text`, `element` or
+ # `query` instead of `selector`; rejecting those costs the whole
+ # workflow, so accept the common aliases.
+ selector = str(
+ params.get("selector")
+ or params.get("target")
+ or params.get("text")
+ or params.get("element")
+ or params.get("query")
+ or ""
+ ).strip()
if not selector:
return ToolResult(
tool_name="browser_click",
- content="No selector provided.",
+ content=(
+ "No selector provided. Pass the visible text of the"
+ " element or a CSS selector."
+ ),
success=False,
)
@@ -198,16 +250,16 @@ class BrowserClickTool(BaseTool):
try:
page = _session.page
- if by_text:
- page.get_by_text(selector).click()
- else:
- page.click(selector)
-
+ strategy = self._resolve_and_click(page, selector, by_text)
return ToolResult(
tool_name="browser_click",
content=f"Clicked element: {selector}",
success=True,
- metadata={"selector": selector, "by_text": by_text},
+ metadata={
+ "selector": selector,
+ "by_text": by_text,
+ "strategy": strategy,
+ },
)
except ImportError:
return ToolResult(
@@ -224,6 +276,111 @@ class BrowserClickTool(BaseTool):
success=False,
)
+ def _resolve_and_click(self, page: Any, selector: str, by_text: bool) -> str:
+ """Resolve *selector* through a strategy ladder and click it.
+
+ LLMs pass anything from precise CSS to plain descriptions like
+ "first video result". Strategies from most to least precise: CSS,
+ exact text, fuzzy text, YouTube results video title, role-based
+ keyword match. Existence is checked with ``count()`` (instant, no
+ actionability wait) so bad strategies cost milliseconds; only a
+ strategy that actually matched gets a real click timeout. Searches
+ the main frame first, then child iframes (consent dialogs and ad
+ overlays live there). Returns the name of the winning strategy;
+ raises with a list of visible clickable elements when nothing
+ matched, so the model can retry sensibly.
+ """
+ import re
+
+ words = [w for w in re.split(r"\W+", selector.lower()) if w]
+ ordinal = next((self._ORDINALS[w] for w in words if w in self._ORDINALS), None)
+
+ def candidates(frame: Any) -> list:
+ cands = []
+ if not by_text:
+ cands.append(("css", frame.locator(selector)))
+ cands.append(("text-exact", frame.get_by_text(selector, exact=True)))
+ cands.append(
+ ("text-fuzzy", frame.get_by_text(re.compile(re.escape(selector), re.I)))
+ )
+ # YouTube search results: "first video result" must click a
+ # video title, not the "Videos" filter chip — so this outranks
+ # the generic role-based keyword match below.
+ try:
+ url = frame.url or ""
+ except Exception:
+ url = ""
+ if "youtube." in url and ("video" in words or "result" in words):
+ cands.append(("youtube-result", frame.locator("a#video-title")))
+ keywords = [w for w in words if w not in self._DESC_STOPWORDS]
+ if keywords:
+ pat = re.compile(".*".join(re.escape(k) for k in keywords), re.I)
+ for role in ("link", "button"):
+ cands.append((f"role-{role}", frame.get_by_role(role, name=pat)))
+ return cands
+
+ def actionable_matches(locator: Any) -> list[Any]:
+ """Return click-ordered matches, excluding hidden or disabled ones."""
+ count = locator.count()
+ matches = []
+ # A locator can match hidden mobile/desktop duplicates before the
+ # visible control. Build the actionable sequence first so an
+ # ordinal such as "first" means the first element a user can
+ # actually click, not the first raw DOM match.
+ for index in range(count):
+ match = locator.nth(index)
+ try:
+ if not match.is_visible() or not match.is_enabled():
+ continue
+ except Exception:
+ # The click remains the final Playwright actionability
+ # check, including stability and event-receivability.
+ pass
+ matches.append(match)
+
+ if ordinal is not None:
+ index = len(matches) - 1 if ordinal == -1 else ordinal
+ return [matches[index]] if 0 <= index < len(matches) else []
+ return matches
+
+ last_error: Exception | None = None
+ for index, frame in enumerate(page.frames):
+ for strategy, locator in candidates(frame):
+ try:
+ matches = actionable_matches(locator)
+ if not matches:
+ continue
+ except Exception:
+ continue # e.g. selector isn't valid CSS — next rung
+ for match in matches:
+ try:
+ match.click(timeout=8000 if index == 0 else 4000)
+ return strategy
+ except Exception as exc:
+ last_error = exc
+ continue
+
+ hint = ""
+ try:
+ texts = page.evaluate(
+ """(limit) => Array.from(
+ document.querySelectorAll('a, button, [role=button], [role=link]')
+ ).map(e => (e.innerText || e.getAttribute('aria-label') || '').trim())
+ .filter(t => t && t.length < 80)
+ .filter((t, i, arr) => arr.indexOf(t) === i)
+ .slice(0, limit)""",
+ 15,
+ )
+ if texts:
+ hint = " Visible clickable elements: " + "; ".join(texts)
+ except Exception:
+ pass
+ detail = f" Last error: {last_error}" if last_error else ""
+ raise RuntimeError(
+ f"No element matched '{selector}' in the main page or its"
+ f" {len(page.frames) - 1} iframe(s).{hint}{detail}"
+ )
+
# ---------------------------------------------------------------------------
# Tool 3: BrowserTypeTool
diff --git a/tests/tools/test_browser.py b/tests/tools/test_browser.py
index 1e070f5a..62623a37 100644
--- a/tests/tools/test_browser.py
+++ b/tests/tools/test_browser.py
@@ -282,12 +282,20 @@ class TestBrowserClickTool:
assert tool.spec.name == "browser_click"
assert tool.spec.category == "browser"
- def test_spec_requires_selector(self):
+ def test_spec_advertises_selector_and_aliases(self):
from openjarvis.tools.browser import BrowserClickTool
tool = BrowserClickTool()
- assert "selector" in tool.spec.parameters["properties"]
- assert "selector" in tool.spec.parameters["required"]
+ properties = tool.spec.parameters["properties"]
+ assert {"selector", "target", "text", "element", "query"} <= set(properties)
+ assert "required" not in tool.spec.parameters
+ assert tool.spec.parameters["anyOf"] == [
+ {"required": ["selector"]},
+ {"required": ["target"]},
+ {"required": ["text"]},
+ {"required": ["element"]},
+ {"required": ["query"]},
+ ]
def test_spec_has_by_text_parameter(self):
from openjarvis.tools.browser import BrowserClickTool
@@ -317,6 +325,141 @@ class TestBrowserClickTool:
assert result.success is False
assert "No selector" in result.content
+ def test_resolver_skips_hidden_duplicate_match(self):
+ from openjarvis.tools.browser import BrowserClickTool
+
+ class Match:
+ def __init__(self, *, visible=True, enabled=True):
+ self.visible = visible
+ self.enabled = enabled
+ self.clicked = False
+
+ def is_visible(self):
+ return self.visible
+
+ def is_enabled(self):
+ return self.enabled
+
+ def click(self, **_kwargs):
+ self.clicked = True
+
+ class Locator:
+ def __init__(self, matches):
+ self.matches = matches
+
+ def count(self):
+ return len(self.matches)
+
+ def nth(self, index):
+ return self.matches[index]
+
+ hidden = Match(visible=False)
+ visible = Match()
+ empty = Locator([])
+ duplicate = Locator([hidden, visible])
+ frame = MagicMock(url="https://example.com")
+ frame.locator.return_value = duplicate
+ frame.get_by_text.return_value = empty
+ frame.get_by_role.return_value = empty
+ page = MagicMock(frames=[frame])
+
+ strategy = BrowserClickTool()._resolve_and_click(page, ".save", False)
+
+ assert strategy == "css"
+ assert hidden.clicked is False
+ assert visible.clicked is True
+
+ def test_resolver_applies_ordinal_after_actionability_filter(self):
+ from openjarvis.tools.browser import BrowserClickTool
+
+ class Match:
+ def __init__(self, *, visible=True):
+ self.visible = visible
+ self.clicked = False
+
+ def is_visible(self):
+ return self.visible
+
+ def is_enabled(self):
+ return True
+
+ def click(self, **_kwargs):
+ self.clicked = True
+
+ class Locator:
+ def __init__(self, matches):
+ self.matches = matches
+
+ def count(self):
+ return len(self.matches)
+
+ def nth(self, index):
+ return self.matches[index]
+
+ hidden = Match(visible=False)
+ first_visible = Match()
+ second_visible = Match()
+ duplicate = Locator([hidden, first_visible, second_visible])
+ empty = Locator([])
+ frame = MagicMock(url="https://example.com")
+ frame.get_by_text.return_value = duplicate
+ frame.get_by_role.return_value = empty
+ page = MagicMock(frames=[frame])
+
+ strategy = BrowserClickTool()._resolve_and_click(page, "first save", True)
+
+ assert strategy == "text-exact"
+ assert hidden.clicked is False
+ assert first_visible.clicked is True
+ assert second_visible.clicked is False
+
+ def test_resolver_clicks_fourth_actionable_role_match(self):
+ from openjarvis.tools.browser import BrowserClickTool
+
+ class Match:
+ def __init__(self):
+ self.clicked = False
+
+ def is_visible(self):
+ return True
+
+ def is_enabled(self):
+ return True
+
+ def click(self, **_kwargs):
+ self.clicked = True
+
+ class Locator:
+ def __init__(self, matches=()):
+ self.matches = list(matches)
+
+ def count(self):
+ return len(self.matches)
+
+ def nth(self, index):
+ return self.matches[index]
+
+ buttons = [Match() for _ in range(4)]
+ empty = Locator()
+ frame = MagicMock(url="https://example.com")
+ frame.get_by_text.return_value = empty
+ frame.get_by_role.side_effect = lambda role, name: (
+ Locator(buttons) if role == "button" and name.search("Save") else empty
+ )
+ page = MagicMock(frames=[frame])
+
+ strategy = BrowserClickTool()._resolve_and_click(
+ page, "fourth save button", True
+ )
+
+ assert strategy == "role-button"
+ assert [match.clicked for match in buttons] == [False, False, False, True]
+
+ def test_every_supported_ordinal_is_a_description_stopword(self):
+ from openjarvis.tools.browser import BrowserClickTool
+
+ assert set(BrowserClickTool._ORDINALS) <= BrowserClickTool._DESC_STOPWORDS
+
def test_execute_playwright_not_installed(self):
from openjarvis.tools.browser import BrowserClickTool
@@ -331,6 +474,14 @@ class TestBrowserClickTool:
from openjarvis.tools.browser import BrowserClickTool
page = _make_mock_page()
+ page.frames = [page]
+ match = MagicMock()
+ match.is_visible.return_value = True
+ match.is_enabled.return_value = True
+ page.locator.return_value.count.return_value = 1
+ page.locator.return_value.nth.return_value = match
+ page.get_by_text.return_value.count.return_value = 0
+ page.get_by_role.return_value.count.return_value = 0
session = _make_mock_session(page)
with patch("openjarvis.tools.browser._session", session):
@@ -339,7 +490,7 @@ class TestBrowserClickTool:
assert result.success is True
assert "Clicked element" in result.content
- page.click.assert_called_once_with("#submit-btn")
+ match.click.assert_called_once_with(timeout=8000)
assert result.metadata["selector"] == "#submit-btn"
assert result.metadata["by_text"] is False
@@ -347,6 +498,12 @@ class TestBrowserClickTool:
from openjarvis.tools.browser import BrowserClickTool
page = _make_mock_page()
+ page.frames = [page]
+ match = MagicMock()
+ match.is_visible.return_value = True
+ match.is_enabled.return_value = True
+ page.get_by_text.return_value.count.return_value = 1
+ page.get_by_text.return_value.nth.return_value = match
session = _make_mock_session(page)
with patch("openjarvis.tools.browser._session", session):
@@ -354,15 +511,23 @@ class TestBrowserClickTool:
result = tool.execute(selector="Sign In", by_text=True)
assert result.success is True
- page.get_by_text.assert_called_once_with("Sign In")
- page.get_by_text.return_value.click.assert_called_once()
+ assert page.get_by_text.call_count >= 1
+ match.click.assert_called_once_with(timeout=8000)
assert result.metadata["by_text"] is True
def test_execute_click_error(self):
from openjarvis.tools.browser import BrowserClickTool
page = _make_mock_page()
- page.click.side_effect = Exception("Element not found")
+ page.frames = [page]
+ match = MagicMock()
+ match.is_visible.return_value = True
+ match.is_enabled.return_value = True
+ match.click.side_effect = Exception("Element not found")
+ page.locator.return_value.count.return_value = 1
+ page.locator.return_value.nth.return_value = match
+ page.get_by_text.return_value.count.return_value = 0
+ page.get_by_role.return_value.count.return_value = 0
session = _make_mock_session(page)
with patch("openjarvis.tools.browser._session", session):