Personal AI, On Personal Devices
Python
"""Browser automation tools — Playwright-based web interaction."""
from __future__ import annotations
import base64
from typing import Any
from openjarvis.core.registry import ToolRegistry
from openjarvis.core.types import ToolResult
from openjarvis.tools._stubs import BaseTool, ToolSpec
class _BrowserSession:
"""Manages a shared Playwright browser session (lazy init)."""
def __init__(self) -> None:
self._playwright = None
self._browser = None
self._page = None
def _ensure_browser(self) -> None:
if self._page is not None:
return
try:
from playwright.sync_api import sync_playwright
except ImportError:
raise ImportError(
"playwright not installed. Install with: uv sync --extra browser"
)
self._playwright = sync_playwright().start()
self._browser = self._playwright.chromium.launch(headless=True)
self._page = self._browser.new_page()
@property
def page(self):
self._ensure_browser()
return self._page
def close(self) -> None:
if self._browser:
self._browser.close()
if self._playwright:
self._playwright.stop()
self._playwright = self._browser = self._page = None
_session = _BrowserSession()
# ---------------------------------------------------------------------------
# Tool 1: BrowserNavigateTool
# ---------------------------------------------------------------------------
@ToolRegistry.register("browser_navigate")
class BrowserNavigateTool(BaseTool):
"""Navigate to a URL in the browser."""
tool_id = "browser_navigate"
is_local = False
@property
def spec(self) -> ToolSpec:
return ToolSpec(
name="browser_navigate",
description=(
"Navigate to a URL in the browser."
" Returns the page title and text content."
),
parameters={
"type": "object",
"properties": {
"url": {
"type": "string",
"description": "URL to navigate to.",
},
"wait_for": {
"type": "string",
"description": (
"Wait condition: 'load', 'domcontentloaded',"
" or 'networkidle'. Default: 'load'."
),
},
},
"required": ["url"],
},
category="browser",
required_capabilities=["network:fetch"],
)
def execute(self, **params: Any) -> ToolResult:
url = params.get("url", "")
if not url:
return ToolResult(
tool_name="browser_navigate",
content="No URL provided.",
success=False,
)
wait_for = params.get("wait_for", "load")
if wait_for not in ("load", "domcontentloaded", "networkidle"):
wait_for = "load"
# SSRF check — never skipped. check_ssrf falls back to a pure-Python
# implementation when the Rust backend is unavailable, so an
# uncompiled extension must not silently disable SSRF protection.
from openjarvis.security.ssrf import check_ssrf
ssrf_error = check_ssrf(url)
if ssrf_error:
return ToolResult(
tool_name="browser_navigate",
content=f"SSRF blocked: {ssrf_error}",
success=False,
)
try:
page = _session.page
response = page.goto(url, wait_until=wait_for)
title = page.title()
text_content = page.inner_text("body")
if len(text_content) > 5000:
text_content = text_content[:5000] + "\n\n[Content truncated]"
status = response.status if response else None
return ToolResult(
tool_name="browser_navigate",
content=f"Title: {title}\n\n{text_content}",
success=True,
metadata={"url": url, "title": title, "status": status},
)
except ImportError:
return ToolResult(
tool_name="browser_navigate",
content=(
"playwright not installed. Install with: uv sync --extra browser"
),
success=False,
)
except Exception as exc:
return ToolResult(
tool_name="browser_navigate",
content=f"Navigation error: {exc}",
success=False,
)
# ---------------------------------------------------------------------------
# Tool 2: BrowserClickTool
# ---------------------------------------------------------------------------
@ToolRegistry.register("browser_click")
class BrowserClickTool(BaseTool):
"""Click an element on the page."""
tool_id = "browser_click"
is_local = False
@property
def spec(self) -> ToolSpec:
return ToolSpec(
name="browser_click",
description=(
"Click an element on the current page. Pass the text you see"
" on the element (button label, link title), a CSS selector,"
" or a short description like 'first video result'."
),
parameters={
"type": "object",
"properties": {
"selector": {
"type": "string",
"description": (
"Visible text of the element, a CSS selector, or"
" a short description (e.g. 'first video result')."
),
},
"by_text": {
"type": "boolean",
"description": (
"If true, click by text content"
" instead of CSS selector. Default: false."
),
},
"target": {
"type": "string",
"description": "Alias for selector.",
},
"text": {
"type": "string",
"description": "Alias for selector, for visible element text.",
},
"element": {
"type": "string",
"description": "Alias for selector.",
},
"query": {
"type": "string",
"description": "Alias for selector.",
},
},
# Require one accepted spelling without forcing callers to
# use ``selector`` when an advertised alias is supplied.
"anyOf": [
{"required": ["selector"]},
{"required": ["target"]},
{"required": ["text"]},
{"required": ["element"]},
{"required": ["query"]},
],
},
category="browser",
)
_ORDINALS = {"first": 0, "second": 1, "third": 2, "fourth": 3, "last": -1}
# Filler words stripped from natural-language descriptions before
# keyword matching ("the first video result" -> "video"). Derive the
# ordinal entries from the supported mapping so adding one cannot leave it
# behind as a literal role-name keyword.
_DESC_STOPWORDS = frozenset(
"the a an top main big red blue green"
" result results item element link button option entry row".split()
) | frozenset(_ORDINALS)
def execute(self, **params: Any) -> ToolResult:
# Models routinely name this param `target`, `text`, `element` or
# `query` instead of `selector`; rejecting those costs the whole
# workflow, so accept the common aliases.
selector = str(
params.get("selector")
or params.get("target")
or params.get("text")
or params.get("element")
or params.get("query")
or ""
).strip()
if not selector:
return ToolResult(
tool_name="browser_click",
content=(
"No selector provided. Pass the visible text of the"
" element or a CSS selector."
),
success=False,
)
by_text = params.get("by_text", False)
try:
page = _session.page
strategy = self._resolve_and_click(page, selector, by_text)
return ToolResult(
tool_name="browser_click",
content=f"Clicked element: {selector}",
success=True,
metadata={
"selector": selector,
"by_text": by_text,
"strategy": strategy,
},
)
except ImportError:
return ToolResult(
tool_name="browser_click",
content=(
"playwright not installed. Install with: uv sync --extra browser"
),
success=False,
)
except Exception as exc:
return ToolResult(
tool_name="browser_click",
content=f"Click error: {exc}",
success=False,
)
def _resolve_and_click(self, page: Any, selector: str, by_text: bool) -> str:
"""Resolve *selector* through a strategy ladder and click it.
LLMs pass anything from precise CSS to plain descriptions like
"first video result". Strategies from most to least precise: CSS,
exact text, fuzzy text, YouTube results video title, role-based
keyword match. Existence is checked with ``count()`` (instant, no
actionability wait) so bad strategies cost milliseconds; only a
strategy that actually matched gets a real click timeout. Searches
the main frame first, then child iframes (consent dialogs and ad
overlays live there). Returns the name of the winning strategy;
raises with a list of visible clickable elements when nothing
matched, so the model can retry sensibly.
"""
import re
words = [w for w in re.split(r"\W+", selector.lower()) if w]
ordinal = next((self._ORDINALS[w] for w in words if w in self._ORDINALS), None)
def candidates(frame: Any) -> list:
cands = []
if not by_text:
cands.append(("css", frame.locator(selector)))
cands.append(("text-exact", frame.get_by_text(selector, exact=True)))
cands.append(
("text-fuzzy", frame.get_by_text(re.compile(re.escape(selector), re.I)))
)
# YouTube search results: "first video result" must click a
# video title, not the "Videos" filter chip — so this outranks
# the generic role-based keyword match below.
try:
url = frame.url or ""
except Exception:
url = ""
if "youtube." in url and ("video" in words or "result" in words):
cands.append(("youtube-result", frame.locator("a#video-title")))
keywords = [w for w in words if w not in self._DESC_STOPWORDS]
if keywords:
pat = re.compile(".*".join(re.escape(k) for k in keywords), re.I)
for role in ("link", "button"):
cands.append((f"role-{role}", frame.get_by_role(role, name=pat)))
return cands
def actionable_matches(locator: Any) -> list[Any]:
"""Return click-ordered matches, excluding hidden or disabled ones."""
count = locator.count()
matches = []
# A locator can match hidden mobile/desktop duplicates before the
# visible control. Build the actionable sequence first so an
# ordinal such as "first" means the first element a user can
# actually click, not the first raw DOM match.
for index in range(count):
match = locator.nth(index)
try:
if not match.is_visible() or not match.is_enabled():
continue
except Exception:
# The click remains the final Playwright actionability
# check, including stability and event-receivability.
pass
matches.append(match)
if ordinal is not None:
index = len(matches) - 1 if ordinal == -1 else ordinal
return [matches[index]] if 0 <= index < len(matches) else []
return matches
last_error: Exception | None = None
for index, frame in enumerate(page.frames):
for strategy, locator in candidates(frame):
try:
matches = actionable_matches(locator)
if not matches:
continue
except Exception:
continue # e.g. selector isn't valid CSS — next rung
for match in matches:
try:
match.click(timeout=8000 if index == 0 else 4000)
return strategy
except Exception as exc:
last_error = exc
continue
hint = ""
try:
texts = page.evaluate(
"""(limit) => Array.from(
document.querySelectorAll('a, button, [role=button], [role=link]')
).map(e => (e.innerText || e.getAttribute('aria-label') || '').trim())
.filter(t => t && t.length < 80)
.filter((t, i, arr) => arr.indexOf(t) === i)
.slice(0, limit)""",
15,
)
if texts:
hint = " Visible clickable elements: " + "; ".join(texts)
except Exception:
pass
detail = f" Last error: {last_error}" if last_error else ""
raise RuntimeError(
f"No element matched '{selector}' in the main page or its"
f" {len(page.frames) - 1} iframe(s).{hint}{detail}"
)
# ---------------------------------------------------------------------------
# Tool 3: BrowserTypeTool
# ---------------------------------------------------------------------------
@ToolRegistry.register("browser_type")
class BrowserTypeTool(BaseTool):
"""Type text into a form field."""
tool_id = "browser_type"
is_local = False
@property
def spec(self) -> ToolSpec:
return ToolSpec(
name="browser_type",
description=(
"Type text into a form field on the current page."
" Can clear the field first or append to existing content."
),
parameters={
"type": "object",
"properties": {
"selector": {
"type": "string",
"description": "CSS selector of the input field.",
},
"text": {
"type": "string",
"description": "Text to type into the field.",
},
"clear": {
"type": "boolean",
"description": (
"If true, clear the field before typing. Default: true."
),
},
},
"required": ["selector", "text"],
},
category="browser",
)
def execute(self, **params: Any) -> ToolResult:
selector = params.get("selector", "")
text = params.get("text", "")
if not selector:
return ToolResult(
tool_name="browser_type",
content="No selector provided.",
success=False,
)
if not text:
return ToolResult(
tool_name="browser_type",
content="No text provided.",
success=False,
)
clear = params.get("clear", True)
try:
page = _session.page
if clear:
page.fill(selector, text)
else:
page.type(selector, text)
return ToolResult(
tool_name="browser_type",
content=f"Typed text into: {selector}",
success=True,
metadata={"selector": selector},
)
except ImportError:
return ToolResult(
tool_name="browser_type",
content=(
"playwright not installed. Install with: uv sync --extra browser"
),
success=False,
)
except Exception as exc:
return ToolResult(
tool_name="browser_type",
content=f"Type error: {exc}",
success=False,
)
# ---------------------------------------------------------------------------
# Tool 4: BrowserScreenshotTool
# ---------------------------------------------------------------------------
@ToolRegistry.register("browser_screenshot")
class BrowserScreenshotTool(BaseTool):
"""Take a screenshot of the current page."""
tool_id = "browser_screenshot"
is_local = False
@property
def spec(self) -> ToolSpec:
return ToolSpec(
name="browser_screenshot",
description=(
"Take a screenshot of the current browser page."
" Returns the screenshot as base64-encoded data."
),
parameters={
"type": "object",
"properties": {
"path": {
"type": "string",
"description": "Optional file path to save the screenshot.",
},
"full_page": {
"type": "boolean",
"description": (
"If true, capture the full scrollable page. Default: false."
),
},
},
},
category="browser",
)
def execute(self, **params: Any) -> ToolResult:
path = params.get("path")
full_page = params.get("full_page", False)
try:
page = _session.page
screenshot_bytes = page.screenshot(full_page=full_page)
if path:
with open(path, "wb") as f:
f.write(screenshot_bytes)
b64_data = base64.b64encode(screenshot_bytes).decode("utf-8")
description = "Screenshot taken"
if full_page:
description += " (full page)"
if path:
description += f", saved to {path}"
return ToolResult(
tool_name="browser_screenshot",
content=description,
success=True,
metadata={"screenshot_base64": b64_data},
)
except ImportError:
return ToolResult(
tool_name="browser_screenshot",
content=(
"playwright not installed. Install with: uv sync --extra browser"
),
success=False,
)
except Exception as exc:
return ToolResult(
tool_name="browser_screenshot",
content=f"Screenshot error: {exc}",
success=False,
)
# ---------------------------------------------------------------------------
# Tool 5: BrowserExtractTool
# ---------------------------------------------------------------------------
@ToolRegistry.register("browser_extract")
class BrowserExtractTool(BaseTool):
"""Extract content from the current page."""
tool_id = "browser_extract"
is_local = False
@property
def spec(self) -> ToolSpec:
return ToolSpec(
name="browser_extract",
description=(
"Extract content from the current browser page."
" Supports extracting text, links, or tables."
),
parameters={
"type": "object",
"properties": {
"selector": {
"type": "string",
"description": (
"CSS selector to extract from. Default: 'body'."
),
},
"extract_type": {
"type": "string",
"description": (
"Type of extraction: 'text', 'links',"
" or 'tables'. Default: 'text'."
),
},
},
},
category="browser",
)
def execute(self, **params: Any) -> ToolResult:
selector = params.get("selector", "body")
extract_type = params.get("extract_type", "text")
if extract_type not in ("text", "links", "tables"):
return ToolResult(
tool_name="browser_extract",
content=(
f"Invalid extract_type: '{extract_type}'."
" Must be 'text', 'links', or 'tables'."
),
success=False,
)
try:
page = _session.page
if extract_type == "text":
content = page.inner_text(selector)
if len(content) > 10000:
content = content[:10000] + "\n\n[Content truncated]"
return ToolResult(
tool_name="browser_extract",
content=content,
success=True,
metadata={"selector": selector, "extract_type": extract_type},
)
elif extract_type == "links":
links = page.eval_on_selector_all(
f"{selector} a[href]",
"""elements => elements.map(el => ({
href: el.href,
text: el.innerText.trim()
}))""",
)
lines = []
for link in links:
text = link.get("text", "")
href = link.get("href", "")
lines.append(f"- [{text}]({href})")
content = "\n".join(lines) if lines else "No links found."
if len(content) > 10000:
content = content[:10000] + "\n\n[Content truncated]"
return ToolResult(
tool_name="browser_extract",
content=content,
success=True,
metadata={
"selector": selector,
"extract_type": extract_type,
"num_links": len(links),
},
)
else: # tables
tables_text = page.eval_on_selector_all(
f"{selector} table",
"""elements => elements.map(el => el.innerText)""",
)
if tables_text:
content = "\n\n---\n\n".join(tables_text)
else:
content = "No tables found."
if len(content) > 10000:
content = content[:10000] + "\n\n[Content truncated]"
return ToolResult(
tool_name="browser_extract",
content=content,
success=True,
metadata={
"selector": selector,
"extract_type": extract_type,
"num_tables": len(tables_text),
},
)
except ImportError:
return ToolResult(
tool_name="browser_extract",
content=(
"playwright not installed. Install with: uv sync --extra browser"
),
success=False,
)
except Exception as exc:
return ToolResult(
tool_name="browser_extract",
content=f"Extract error: {exc}",
success=False,
)
__all__ = [
"BrowserNavigateTool",
"BrowserClickTool",
"BrowserTypeTool",
"BrowserScreenshotTool",
"BrowserExtractTool",
]