AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(copilot): Add agent-browser multi-step browser automation tools (#12230)
Zamil Majdyb504cf9854b8
diff --git a/autogpt_platform/backend/backend/copilot/model.py b/autogpt_platform/backend/backend/copilot/model.pyindex fb0ac60e5b..c6ee34f62a 100644--- a/autogpt_platform/backend/backend/copilot/model.py+++ b/autogpt_platform/backend/backend/copilot/model.py@@ -672,6 +672,16 @@ async def delete_chat_session(session_id: str, user_id: str | None = None) -> boasync with _session_locks_mutex:_session_locks.pop(session_id, None)+ # Shut down any local browser daemon for this session (best-effort).+ # Inline import required: all tool modules import ChatSession from this+ # module, so any top-level import from tools.* would create a cycle.+ try:+ from .tools.agent_browser import close_browser_session++ await close_browser_session(session_id, user_id=user_id)+ except Exception as e:+ logger.debug(f"Browser cleanup for session {session_id}: {e}")+return Truediff --git a/autogpt_platform/backend/backend/copilot/service.py b/autogpt_platform/backend/backend/copilot/service.pyindex c4d20c5f67..52033ed130 100644--- a/autogpt_platform/backend/backend/copilot/service.py+++ b/autogpt_platform/backend/backend/copilot/service.py@@ -59,7 +59,7 @@ from .response_model import (StreamToolOutputAvailable,StreamUsage,)-from .tools import execute_tool, tools+from .tools import execute_tool, get_available_toolsfrom .tools.models import ErrorResponsefrom .tracking import track_user_message@@ -514,7 +514,7 @@ async def stream_chat_completion()async for chunk in _stream_chat_chunks(session=session,- tools=tools,+ tools=get_available_tools(),system_prompt=system_prompt,text_block_id=text_block_id,):diff --git a/autogpt_platform/backend/backend/copilot/tools/__init__.py b/autogpt_platform/backend/backend/copilot/tools/__init__.pyindex 8407af734c..5b8c9450fb 100644--- a/autogpt_platform/backend/backend/copilot/tools/__init__.py+++ b/autogpt_platform/backend/backend/copilot/tools/__init__.py@@ -1,12 +1,14 @@+from __future__ import annotations+import loggingfrom typing import TYPE_CHECKING, Anyfrom openai.types.chat import ChatCompletionToolParam-from backend.copilot.model import ChatSessionfrom backend.copilot.tracking import track_tool_calledfrom .add_understanding import AddUnderstandingTool+from .agent_browser import BrowserActTool, BrowserNavigateTool, BrowserScreenshotToolfrom .agent_output import AgentOutputToolfrom .base import BaseToolfrom .bash_exec import BashExecTool@@ -30,6 +32,7 @@ from .workspace_files import ()if TYPE_CHECKING:+ from backend.copilot.model import ChatSessionfrom backend.copilot.response_model import StreamToolOutputAvailablelogger = logging.getLogger(__name__)@@ -50,6 +53,10 @@ TOOL_REGISTRY: dict[str, BaseTool] = {"get_doc_page": GetDocPageTool(),# Web fetch for safe URL retrieval"web_fetch": WebFetchTool(),+ # Agent-browser multi-step automation (navigate, act, screenshot)+ "browser_navigate": BrowserNavigateTool(),+ "browser_act": BrowserActTool(),+ "browser_screenshot": BrowserScreenshotTool(),# Sandboxed code execution (bubblewrap)"bash_exec": BashExecTool(),# Persistent workspace tools (cloud storage, survives across sessions)@@ -67,10 +74,17 @@ TOOL_REGISTRY: dict[str, BaseTool] = {find_agent_tool = TOOL_REGISTRY["find_agent"]run_agent_tool = TOOL_REGISTRY["run_agent"]-# Generated from registry for OpenAI API-tools: list[ChatCompletionToolParam] = [- tool.as_openai_tool() for tool in TOOL_REGISTRY.values()-]++def get_available_tools() -> list[ChatCompletionToolParam]:+ """Return OpenAI tool schemas for tools available in the current environment.++ Called per-request so that env-var or binary availability is evaluated+ fresh each time (e.g. browser_* tools are excluded when agent-browser+ CLI is not installed).+ """+ return [+ tool.as_openai_tool() for tool in TOOL_REGISTRY.values() if tool.is_available+ ]def get_tool(tool_name: str) -> BaseTool | None:diff --git a/autogpt_platform/backend/backend/copilot/tools/agent_browser.py b/autogpt_platform/backend/backend/copilot/tools/agent_browser.pynew file mode 100644index 0000000000..6e76e1c62a--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/tools/agent_browser.py@@ -0,0 +1,876 @@+"""Agent-browser tools — multi-step browser automation for the Copilot.++Uses the agent-browser CLI (https://github.com/vercel-labs/agent-browser)+which runs a local Chromium instance managed by a persistent daemon.++- Runs locally — no cloud account required+- Full interaction support: click, fill, scroll, login flows, multi-step+- Session persistence via --session-name: cookies/auth carry across tool calls+ within the same Copilot session, enabling login → navigate → extract workflows+- Screenshot with --annotate overlays @ref labels, saved to workspace for user+- The Claude Agent SDK's multi-turn loop handles orchestration — each tool call+ is one browser action; the LLM chains them naturally++SSRF protection:+ Uses the shared validate_url() from backend.util.request, which is the same+ guard used by HTTP blocks and web_fetch. It resolves ALL DNS answers (not just+ the first), blocks RFC 1918, loopback, link-local, 0.0.0.0/8, multicast,+ and all relevant IPv6 ranges, and applies IDNA encoding to prevent Unicode+ domain attacks.++Requires:+ npm install -g agent-browser+ agent-browser install (downloads Chromium, one-time per machine)+"""++import asyncio+import base64+import json+import logging+import os+import shutil+import tempfile+from typing import Any++from backend.copilot.model import ChatSession+from backend.util.request import validate_url++from .base import BaseTool+from .models import (+ BrowserActResponse,+ BrowserNavigateResponse,+ BrowserScreenshotResponse,+ ErrorResponse,+ ToolResponseBase,+)+from .workspace_files import get_manager++logger = logging.getLogger(__name__)++# Per-command timeout (seconds). Navigation + networkidle wait can be slow.+_CMD_TIMEOUT = 45+# Accessibility tree can be very large; cap it to keep LLM context manageable.+_MAX_SNAPSHOT_CHARS = 20_000+++# ---------------------------------------------------------------------------+# Subprocess helper+# ---------------------------------------------------------------------------+++async def _run(+ session_name: str,+ *args: str,+ timeout: int = _CMD_TIMEOUT,+) -> tuple[int, str, str]:+ """Run agent-browser for the given session and return (rc, stdout, stderr).++ Uses both:+ --session <name> → isolated Chromium context (no shared history/cookies+ with other Copilot sessions — prevents cross-session+ browser state leakage)+ --session-name <name> → persist cookies/localStorage across tool calls within+ the same session (enables login → navigate flows)+ """+ cmd = [+ "agent-browser",+ "--session",+ session_name,+ "--session-name",+ session_name,+ *args,+ ]+ proc = None+ try:+ proc = await asyncio.create_subprocess_exec(+ *cmd,+ stdout=asyncio.subprocess.PIPE,+ stderr=asyncio.subprocess.PIPE,+ )+ stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)+ return proc.returncode or 0, stdout.decode(), stderr.decode()+ except asyncio.TimeoutError:+ # Kill the orphaned subprocess so it does not linger in the process table.+ if proc is not None and proc.returncode is None:+ proc.kill()+ try:+ await proc.communicate()+ except Exception:+ pass # Best-effort reap; ignore errors during cleanup.+ return 1, "", f"Command timed out after {timeout}s."+ except FileNotFoundError:+ return (+ 1,+ "",+ "agent-browser is not installed (run: npm install -g agent-browser && agent-browser install).",+ )+++async def _snapshot(session_name: str) -> str:+ """Return the current page's interactive accessibility tree, truncated."""+ rc, stdout, stderr = await _run(session_name, "snapshot", "-i", "-c")+ if rc != 0:+ return f"[snapshot failed: {stderr[:300]}]"+ text = stdout.strip()+ if len(text) > _MAX_SNAPSHOT_CHARS:+ suffix = "\n\n[Snapshot truncated — use browser_act to navigate further]"+ keep = max(0, _MAX_SNAPSHOT_CHARS - len(suffix))+ text = text[:keep] + suffix+ return text+++# ---------------------------------------------------------------------------+# Stateless session helpers — persist / restore browser state across pods+# ---------------------------------------------------------------------------++# Module-level cache of sessions known to be alive on this pod.+# Avoids the subprocess probe on every tool call within the same pod.+_alive_sessions: set[str] = set()++# Per-session locks to prevent concurrent _ensure_session calls from+# triggering duplicate _restore_browser_state for the same session.+# Protected by _session_locks_mutex to ensure setdefault/pop are not+# interleaved across await boundaries.+_session_locks: dict[str, asyncio.Lock] = {}+_session_locks_mutex = asyncio.Lock()++# Workspace filename for persisted browser state (auto-scoped to session).+# Dot-prefixed so it is hidden from user workspace listings.+_STATE_FILENAME = "._browser_state.json"++# Maximum concurrent subprocesses during cookie/storage restore.+_RESTORE_CONCURRENCY = 10++# Maximum cookies to restore per session. Pathological sites can accumulate+# thousands of cookies; restoring them all would be slow and is rarely useful.+_MAX_RESTORE_COOKIES = 100++# Background tasks for fire-and-forget state persistence.+# Prevents GC from collecting tasks before they complete.+_background_tasks: set[asyncio.Task] = set()+++def _fire_and_forget_save(+ session_name: str, user_id: str, session: ChatSession+) -> None:+ """Schedule state persistence as a background task (non-blocking).++ State save is already best-effort (errors are swallowed), so running it+ in the background avoids adding latency to tool responses.+ """+ task = asyncio.create_task(_save_browser_state(session_name, user_id, session))+ _background_tasks.add(task)+ task.add_done_callback(_background_tasks.discard)+++async def _has_local_session(session_name: str) -> bool:+ """Check if the local agent-browser daemon for this session is running."""+ rc, _, _ = await _run(session_name, "get", "url", timeout=5)+ return rc == 0+++async def _save_browser_state(+ session_name: str, user_id: str, session: ChatSession+) -> None:+ """Persist browser state (cookies, localStorage, URL) to workspace.++ Best-effort: errors are logged but never propagate to the tool response.+ """+ try:+ # Gather state in parallel+ (rc_url, url_out, _), (rc_ck, ck_out, _), (rc_ls, ls_out, _) = (+ await asyncio.gather(+ _run(session_name, "get", "url", timeout=10),+ _run(session_name, "cookies", "get", "--json", timeout=10),+ _run(session_name, "storage", "local", "--json", timeout=10),+ )+ )++ state = {+ "url": url_out.strip() if rc_url == 0 else "",+ "cookies": (json.loads(ck_out) if rc_ck == 0 and ck_out.strip() else []),+ "local_storage": (+ json.loads(ls_out) if rc_ls == 0 and ls_out.strip() else {}+ ),+ }++ manager = await get_manager(user_id, session.session_id)+ await manager.write_file(+ content=json.dumps(state).encode("utf-8"),+ filename=_STATE_FILENAME,+ mime_type="application/json",+ overwrite=True,+ )+ except Exception:+ logger.warning(+ "[browser] Failed to save browser state for session %s",+ session_name,+ exc_info=True,+ )+++async def _restore_browser_state(+ session_name: str, user_id: str, session: ChatSession+) -> bool:+ """Restore browser state from workspace storage into a fresh daemon.++ Best-effort: errors are logged but never propagate to the tool response.+ Returns True on success (or no state to restore), False on failure.+ """+ try:+ manager = await get_manager(user_id, session.session_id)++ file_info = await manager.get_file_info_by_path(_STATE_FILENAME)+ if file_info is None:+ return True # No saved state — first call or never saved++ state_bytes = await manager.read_file(_STATE_FILENAME)+ state = json.loads(state_bytes.decode("utf-8"))++ url = state.get("url", "")+ cookies = state.get("cookies", [])+ local_storage = state.get("local_storage", {})++ # Navigate first — starts daemon + sets the correct origin for cookies+ if url:+ # Validate the saved URL to prevent SSRF via stored redirect targets.+ try:+ await validate_url(url, trusted_origins=[])+ except ValueError:+ logger.warning(+ "[browser] State restore: blocked SSRF URL %s", url[:200]+ )+ return False++ rc, _, stderr = await _run(session_name, "open", url)+ if rc != 0:+ logger.warning(+ "[browser] State restore: failed to open %s: %s",+ url,+ stderr[:200],+ )+ return False+ await _run(session_name, "wait", "--load", "load", timeout=15)++ # Restore cookies and localStorage in parallel via asyncio.gather.+ # Semaphore caps concurrent subprocess spawns so we don't overwhelm the+ # system when a session has hundreds of cookies.+ sem = asyncio.Semaphore(_RESTORE_CONCURRENCY)++ # Guard against pathological sites with thousands of cookies.+ if len(cookies) > _MAX_RESTORE_COOKIES:+ logger.debug(+ "[browser] State restore: capping cookies from %d to %d",+ len(cookies),+ _MAX_RESTORE_COOKIES,+ )+ cookies = cookies[:_MAX_RESTORE_COOKIES]++ async def _set_cookie(c: dict[str, Any]) -> None:+ name = c.get("name", "")+ value = c.get("value", "")+ domain = c.get("domain", "")+ path = c.get("path", "/")+ if not (name and domain):+ return+ async with sem:+ rc, _, stderr = await _run(+ session_name,+ "cookies",+ "set",+ name,+ value,+ "--domain",+ domain,+ "--path",+ path,+ timeout=5,+ )+ if rc != 0:+ logger.debug(+ "[browser] State restore: cookie set failed for %s: %s",+ name,+ stderr[:100],+ )++ async def _set_storage(key: str, val: object) -> None:+ async with sem:+ rc, _, stderr = await _run(+ session_name,+ "storage",+ "local",+ "set",+ key,+ str(val),+ timeout=5,+ )+ if rc != 0:+ logger.debug(+ "[browser] State restore: localStorage set failed for %s: %s",+ key,+ stderr[:100],+ )++ await asyncio.gather(+ *[_set_cookie(c) for c in cookies],+ *[_set_storage(k, v) for k, v in local_storage.items()],+ )++ return True+ except Exception:+ logger.warning(+ "[browser] Failed to restore browser state for session %s",+ session_name,+ exc_info=True,+ )+ return False+++async def _ensure_session(+ session_name: str, user_id: str, session: ChatSession+) -> None:+ """Ensure the local browser daemon has state. Restore from cloud if needed."""+ if session_name in _alive_sessions:+ return+ async with _session_locks_mutex:+ lock = _session_locks.setdefault(session_name, asyncio.Lock())+ async with lock:+ # Double-check after acquiring lock — another coroutine may have restored.+ if session_name in _alive_sessions:+ return+ if await _has_local_session(session_name):+ _alive_sessions.add(session_name)+ return+ if await _restore_browser_state(session_name, user_id, session):+ _alive_sessions.add(session_name)+++async def close_browser_session(session_name: str, user_id: str | None = None) -> None:+ """Shut down the local agent-browser daemon and clean up stored state.++ Deletes ``._browser_state.json`` from workspace storage so cookies and+ other credentials do not linger after the session is deleted.++ Best-effort: errors are logged but never raised.+ """+ _alive_sessions.discard(session_name)+ async with _session_locks_mutex:+ _session_locks.pop(session_name, None)++ # Delete persisted browser state (cookies, localStorage) from workspace.+ if user_id:+ try:+ manager = await get_manager(user_id, session_name)+ file_info = await manager.get_file_info_by_path(_STATE_FILENAME)+ if file_info is not None:+ await manager.delete_file(file_info.id)+ except Exception:+ logger.debug(+ "[browser] Failed to delete state file for session %s",+ session_name,+ exc_info=True,+ )++ try:+ rc, _, stderr = await _run(session_name, "close", timeout=10)+ if rc != 0:+ logger.debug(+ "[browser] close failed for session %s: %s",+ session_name,+ stderr[:200],+ )+ except Exception:+ logger.debug(+ "[browser] Exception closing browser session %s",+ session_name,+ exc_info=True,+ )+++# ---------------------------------------------------------------------------+# Tool: browser_navigate+# ---------------------------------------------------------------------------+++class BrowserNavigateTool(BaseTool):+ """Navigate to a URL and return the page's interactive elements.++ The browser session persists across tool calls within this Copilot session+ (keyed to session_id), so cookies and auth state carry over. This enables+ full login flows: navigate to login page → browser_act to fill credentials+ → browser_act to submit → browser_navigate to the target page.+ """++ @property+ def name(self) -> str:+ return "browser_navigate"++ @property+ def description(self) -> str:+ return (+ "Navigate to a URL using a real browser. Returns an accessibility "+ "tree snapshot listing the page's interactive elements with @ref IDs "+ "(e.g. @e3) that can be used with browser_act. "+ "Session persists — cookies and login state carry over between calls. "+ "Use this (with browser_act) for multi-step interaction: login flows, "+ "form filling, button clicks, or anything requiring page interaction. "+ "For plain static pages, prefer web_fetch — no browser overhead. "+ "For authenticated pages: navigate to the login page first, use browser_act "+ "to fill credentials and submit, then navigate to the target page. "+ "Note: for slow SPAs, the returned snapshot may reflect a partially-loaded "+ "state. If elements seem missing, use browser_act with action='wait' and a "+ "CSS selector or millisecond delay, then take a browser_screenshot to verify."+ )++ @property+ def parameters(self) -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "url": {+ "type": "string",+ "description": "The HTTP/HTTPS URL to navigate to.",+ },+ "wait_for": {+ "type": "string",+ "enum": ["networkidle", "load", "domcontentloaded"],+ "default": "networkidle",+ "description": "When to consider navigation complete. Use 'networkidle' for SPAs (default).",+ },+ },+ "required": ["url"],+ }++ @property+ def requires_auth(self) -> bool:+ return True++ @property+ def is_available(self) -> bool:+ return shutil.which("agent-browser") is not None++ async def _execute(+ self,+ user_id: str | None,+ session: ChatSession,+ **kwargs: Any,+ ) -> ToolResponseBase:+ """Navigate to *url*, wait for the page to settle, and return a snapshot.++ The snapshot is an accessibility-tree listing of interactive elements.+ Note: for slow SPAs that never fully idle, the snapshot may reflect a+ partially-loaded state (the wait is best-effort).+ """+ url: str = (kwargs.get("url") or "").strip()+ wait_for: str = kwargs.get("wait_for") or "networkidle"+ session_name = session.session_id++ if not url:+ return ErrorResponse(+ message="Please provide a URL to navigate to.",+ error="missing_url",+ session_id=session_name,+ )++ try:+ await validate_url(url, trusted_origins=[])+ except ValueError as e:+ return ErrorResponse(+ message=str(e),+ error="blocked_url",+ session_id=session_name,+ )++ # Restore browser state from cloud if this is a different pod+ if user_id:+ await _ensure_session(session_name, user_id, session)++ # Navigate+ rc, _, stderr = await _run(session_name, "open", url)+ if rc != 0:+ logger.warning(+ "[browser_navigate] open failed for %s: %s", url, stderr[:300]+ )+ return ErrorResponse(+ message="Failed to navigate to URL.",+ error="navigation_failed",+ session_id=session_name,+ )++ # Wait for page to settle (best-effort: some SPAs never reach networkidle)+ wait_rc, _, wait_err = await _run(session_name, "wait", "--load", wait_for)+ if wait_rc != 0:+ logger.warning(+ "[browser_navigate] wait(%s) failed: %s", wait_for, wait_err[:300]+ )++ # Get current title and URL in parallel+ (_, title_out, _), (_, url_out, _) = await asyncio.gather(+ _run(session_name, "get", "title"),+ _run(session_name, "get", "url"),+ )++ snapshot = await _snapshot(session_name)++ result = BrowserNavigateResponse(+ message=f"Navigated to {url}",+ url=url_out.strip() or url,+ title=title_out.strip(),+ snapshot=snapshot,+ session_id=session_name,+ )++ # Persist browser state to cloud for cross-pod continuity+ if user_id:+ _fire_and_forget_save(session_name, user_id, session)++ return result+++# ---------------------------------------------------------------------------+# Tool: browser_act+# ---------------------------------------------------------------------------++_NO_TARGET_ACTIONS = frozenset({"back", "forward", "reload"})+_SCROLL_ACTIONS = frozenset({"scroll"})+_TARGET_ONLY_ACTIONS = frozenset({"click", "dblclick", "hover", "check", "uncheck"})+_TARGET_VALUE_ACTIONS = frozenset({"fill", "type", "select"})+# wait <selector|ms>: waits for a DOM element or a fixed delay (e.g. "1000" for 1 s)+_WAIT_ACTIONS = frozenset({"wait"})+++class BrowserActTool(BaseTool):+ """Perform an action on the current browser page and return the updated snapshot.++ Use @ref IDs from the snapshot returned by browser_navigate (e.g. '@e3').+ The LLM orchestrates multi-step flows by chaining browser_navigate and+ browser_act calls across turns of the Claude Agent SDK conversation.+ """++ @property+ def name(self) -> str:+ return "browser_act"++ @property+ def description(self) -> str:+ return (+ "Interact with the current browser page. Use @ref IDs from the "+ "snapshot (e.g. '@e3') to target elements. Returns an updated snapshot. "+ "Supported actions: click, dblclick, fill, type, scroll, hover, press, "+ "check, uncheck, select, wait, back, forward, reload. "+ "fill clears the field before typing; type appends without clearing. "+ "wait accepts a CSS selector (waits for element) or milliseconds string (e.g. '1000'). "+ "Example login flow: fill @e1 with email → fill @e2 with password → "+ "click @e3 (submit) → browser_navigate to the target page."+ )++ @property+ def parameters(self) -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "action": {+ "type": "string",+ "enum": [+ "click",+ "dblclick",+ "fill",+ "type",+ "scroll",+ "hover",+ "press",+ "check",+ "uncheck",+ "select",+ "wait",+ "back",+ "forward",+ "reload",+ ],+ "description": "The action to perform.",+ },+ "target": {+ "type": "string",+ "description": (+ "Element to target. Use @ref from snapshot (e.g. '@e3'), "+ "a CSS selector, or a text description. "+ "Required for: click, dblclick, fill, type, hover, check, uncheck, select. "+ "For wait: a CSS selector to wait for, or milliseconds as a string (e.g. '1000')."+ ),+ },+ "value": {+ "type": "string",+ "description": (+ "For fill/type: the text to enter. "+ "For press: key name (e.g. 'Enter', 'Tab', 'Control+a'). "+ "For select: the option value to select."+ ),+ },+ "direction": {+ "type": "string",+ "enum": ["up", "down", "left", "right"],+ "default": "down",+ "description": "For scroll: direction to scroll.",+ },+ },+ "required": ["action"],+ }++ @property+ def requires_auth(self) -> bool:+ return True++ @property+ def is_available(self) -> bool:+ return shutil.which("agent-browser") is not None++ async def _execute(+ self,+ user_id: str | None,+ session: ChatSession,+ **kwargs: Any,+ ) -> ToolResponseBase:+ """Perform a browser action and return an updated page snapshot.++ Validates the *action*/*target*/*value* combination, delegates to+ ``agent-browser``, waits for the page to settle, and returns the+ accessibility-tree snapshot so the LLM can plan the next step.+ """+ action: str = (kwargs.get("action") or "").strip()+ target: str = (kwargs.get("target") or "").strip()+ value: str = (kwargs.get("value") or "").strip()+ direction: str = (kwargs.get("direction") or "down").strip()+ session_name = session.session_id++ if not action:+ return ErrorResponse(+ message="Please specify an action.",+ error="missing_action",+ session_id=session_name,+ )++ # Build the agent-browser command args+ if action in _NO_TARGET_ACTIONS:+ cmd_args = [action]++ elif action in _SCROLL_ACTIONS:+ cmd_args = ["scroll", direction]++ elif action == "press":+ if not value:+ return ErrorResponse(+ message="'press' requires a 'value' (key name, e.g. 'Enter').",+ error="missing_value",+ session_id=session_name,+ )+ cmd_args = ["press", value]++ elif action in _TARGET_ONLY_ACTIONS:+ if not target:+ return ErrorResponse(+ message=f"'{action}' requires a 'target' element.",+ error="missing_target",+ session_id=session_name,+ )+ cmd_args = [action, target]++ elif action in _TARGET_VALUE_ACTIONS:+ if not target or not value:+ return ErrorResponse(+ message=f"'{action}' requires both 'target' and 'value'.",+ error="missing_params",+ session_id=session_name,+ )+ cmd_args = [action, target, value]++ elif action in _WAIT_ACTIONS:+ if not target:+ return ErrorResponse(+ message=(+ "'wait' requires a 'target': a CSS selector to wait for, "+ "or milliseconds as a string (e.g. '1000')."+ ),+ error="missing_target",+ session_id=session_name,+ )+ cmd_args = ["wait", target]++ else:+ return ErrorResponse(+ message=f"Unsupported action: {action}",+ error="invalid_action",+ session_id=session_name,+ )++ # Restore browser state from cloud if this is a different pod+ if user_id:+ await _ensure_session(session_name, user_id, session)++ rc, _, stderr = await _run(session_name, *cmd_args)+ if rc != 0:+ logger.warning("[browser_act] %s failed: %s", action, stderr[:300])+ return ErrorResponse(+ message=f"Action '{action}' failed.",+ error="action_failed",+ session_id=session_name,+ )++ # Allow the page to settle after interaction (best-effort: SPAs may not idle)+ settle_rc, _, settle_err = await _run(+ session_name, "wait", "--load", "networkidle"+ )+ if settle_rc != 0:+ logger.warning(+ "[browser_act] post-action wait failed: %s", settle_err[:300]+ )++ snapshot = await _snapshot(session_name)+ _, url_out, _ = await _run(session_name, "get", "url")++ result = BrowserActResponse(+ message=f"Performed '{action}'" + (f" on '{target}'" if target else ""),+ action=action,+ current_url=url_out.strip(),+ snapshot=snapshot,+ session_id=session_name,+ )++ # Persist browser state to cloud for cross-pod continuity+ if user_id:+ _fire_and_forget_save(session_name, user_id, session)++ return result+++# ---------------------------------------------------------------------------+# Tool: browser_screenshot+# ---------------------------------------------------------------------------+++class BrowserScreenshotTool(BaseTool):+ """Capture a screenshot of the current browser page and save it to the workspace."""++ @property+ def name(self) -> str:+ return "browser_screenshot"++ @property+ def description(self) -> str:+ return (+ "Take a screenshot of the current browser page and save it to the workspace. "+ "IMPORTANT: After calling this tool, immediately call read_workspace_file "+ "with the returned file_id to display the image inline to the user — "+ "the screenshot is not visible until you do this. "+ "With annotate=true (default), @ref labels are overlaid on interactive "+ "elements, making it easy to see which @ref ID maps to which element on screen."+ )++ @property+ def parameters(self) -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "annotate": {+ "type": "boolean",+ "default": True,+ "description": "Overlay @ref labels on interactive elements (default: true).",+ },+ "filename": {+ "type": "string",+ "default": "screenshot.png",+ "description": "Filename to save in the workspace.",+ },+ },+ }++ @property+ def requires_auth(self) -> bool:+ return True++ @property+ def is_available(self) -> bool:+ return shutil.which("agent-browser") is not None++ async def _execute(+ self,+ user_id: str | None,+ session: ChatSession,+ **kwargs: Any,+ ) -> ToolResponseBase:+ """Capture a PNG screenshot and upload it to the workspace.++ Handles string-to-bool coercion for *annotate* (OpenAI function-call+ payloads sometimes deliver ``"true"``/``"false"`` as strings).+ Returns a :class:`BrowserScreenshotResponse` with the workspace+ ``file_id`` the LLM should pass to ``read_workspace_file``.+ """+ raw_annotate = kwargs.get("annotate", True)+ if isinstance(raw_annotate, str):+ annotate = raw_annotate.strip().lower() in {"1", "true", "yes", "on"}+ else:+ annotate = bool(raw_annotate)+ filename: str = (kwargs.get("filename") or "screenshot.png").strip()+ session_name = session.session_id++ # Restore browser state from cloud if this is a different pod+ if user_id:+ await _ensure_session(session_name, user_id, session)++ tmp_fd, tmp_path = tempfile.mkstemp(suffix=".png")+ os.close(tmp_fd)+ try:+ cmd_args = ["screenshot"]+ if annotate:+ cmd_args.append("--annotate")+ cmd_args.append(tmp_path)++ rc, _, stderr = await _run(session_name, *cmd_args)+ if rc != 0:+ logger.warning("[browser_screenshot] failed: %s", stderr[:300])+ return ErrorResponse(+ message="Failed to take screenshot.",+ error="screenshot_failed",+ session_id=session_name,+ )++ with open(tmp_path, "rb") as f:+ png_bytes = f.read()++ finally:+ try:+ os.unlink(tmp_path)+ except OSError:+ pass # Best-effort temp file cleanup; not critical if it fails.++ # Upload to workspace so the user can view it+ png_b64 = base64.b64encode(png_bytes).decode()++ # Import here to avoid circular deps — workspace_files imports from .models+ from .workspace_files import WorkspaceWriteResponse, WriteWorkspaceFileTool++ write_resp = await WriteWorkspaceFileTool()._execute(+ user_id=user_id,+ session=session,+ filename=filename,+ content_base64=png_b64,+ )++ if not isinstance(write_resp, WorkspaceWriteResponse):+ return ErrorResponse(+ message="Screenshot taken but failed to save to workspace.",+ error="workspace_write_failed",+ session_id=session_name,+ )++ result = BrowserScreenshotResponse(+ message=f"Screenshot saved to workspace as '{filename}'. Use read_workspace_file with file_id='{write_resp.file_id}' to retrieve it.",+ file_id=write_resp.file_id,+ filename=filename,+ session_id=session_name,+ )++ # Persist browser state to cloud for cross-pod continuity+ if user_id:+ _fire_and_forget_save(session_name, user_id, session)++ return resultdiff --git a/autogpt_platform/backend/backend/copilot/tools/agent_browser_test.py b/autogpt_platform/backend/backend/copilot/tools/agent_browser_test.pynew file mode 100644index 0000000000..2dc3fd0cf2--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/tools/agent_browser_test.py@@ -0,0 +1,1663 @@+"""Unit tests for agent_browser tools.++All subprocess calls are mocked — no agent-browser binary or real browser needed.+"""++import asyncio+import json+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from backend.copilot.model import ChatSession++from .agent_browser import (+ BrowserActTool,+ BrowserNavigateTool,+ BrowserScreenshotTool,+ _ensure_session,+ _has_local_session,+ _restore_browser_state,+ _save_browser_state,+ close_browser_session,+)+from .models import (+ BrowserActResponse,+ BrowserNavigateResponse,+ BrowserScreenshotResponse,+ ErrorResponse,+ ResponseType,+)++# ---------------------------------------------------------------------------+# Autouse fixture — mock state persistence helpers for all existing tests.+# Dedicated tests for these helpers call the *original* functions directly+# (imported above), which are not affected by monkeypatch on the module attr.+# ---------------------------------------------------------------------------++[email protected](autouse=True)+def _reset_and_mock_state(monkeypatch):+ """Reset session cache and mock state persistence for all tests."""+ from . import agent_browser as _mod++ _mod._alive_sessions.clear()+ _mod._session_locks.clear()+ monkeypatch.setattr(+ "backend.copilot.tools.agent_browser._ensure_session", AsyncMock()+ )+ monkeypatch.setattr(+ "backend.copilot.tools.agent_browser._fire_and_forget_save", MagicMock()+ )+ yield+ _mod._alive_sessions.clear()+ _mod._session_locks.clear()+++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def make_session(session_id: str = "test-session-123") -> ChatSession:+ return MagicMock(spec=ChatSession, session_id=session_id)+++def _run_result(rc: int = 0, stdout: str = "", stderr: str = "") -> tuple:+ return (rc, stdout, stderr)+++# ---------------------------------------------------------------------------+# SSRF protection via shared validate_url (backend.util.request)+# ---------------------------------------------------------------------------++# Patch target: validate_url is imported directly into agent_browser's module scope.+_VALIDATE_URL = "backend.copilot.tools.agent_browser.validate_url"+++class TestSsrfViaValidateUrl:+ """Verify that browser_navigate uses validate_url for SSRF protection.++ We mock validate_url itself (not the low-level socket) so these tests+ exercise the integration point, not the internals of request.py+ (which has its own thorough test suite in request_test.py).+ """++ def setup_method(self):+ self.tool = BrowserNavigateTool()+ self.session = make_session()++ @pytest.mark.asyncio+ async def test_blocked_ip_returns_blocked_url_error(self):+ """validate_url raises ValueError → tool returns blocked_url ErrorResponse."""+ with patch(_VALIDATE_URL, new_callable=AsyncMock) as mock_validate:+ mock_validate.side_effect = ValueError(+ "Access to blocked IP 10.0.0.1 is not allowed."+ )+ result = await self.tool._execute(+ user_id="user1", session=self.session, url="http://internal.corp"+ )+ assert isinstance(result, ErrorResponse)+ assert result.error == "blocked_url"+ assert "10.0.0.1" in result.message++ @pytest.mark.asyncio+ async def test_invalid_scheme_returns_blocked_url_error(self):+ with patch(_VALIDATE_URL, new_callable=AsyncMock) as mock_validate:+ mock_validate.side_effect = ValueError("Scheme 'ftp' is not allowed.")+ result = await self.tool._execute(+ user_id="user1", session=self.session, url="ftp://example.com"+ )+ assert isinstance(result, ErrorResponse)+ assert result.error == "blocked_url"++ @pytest.mark.asyncio+ async def test_unresolvable_host_returns_blocked_url_error(self):+ with patch(_VALIDATE_URL, new_callable=AsyncMock) as mock_validate:+ mock_validate.side_effect = ValueError(+ "Unable to resolve IP address for hostname bad.invalid"+ )+ result = await self.tool._execute(+ user_id="user1", session=self.session, url="https://bad.invalid"+ )+ assert isinstance(result, ErrorResponse)+ assert result.error == "blocked_url"++ @pytest.mark.asyncio+ async def test_validate_url_called_with_empty_trusted_origins(self):+ """Confirms no trusted-origins bypass is granted — all URLs are validated."""+ with patch(_VALIDATE_URL, new_callable=AsyncMock) as mock_validate:+ mock_validate.return_value = (object(), False, ["1.2.3.4"])+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0),+ ):+ with patch(+ "backend.copilot.tools.agent_browser._snapshot",+ new_callable=AsyncMock,+ return_value="",+ ):+ await self.tool._execute(+ user_id="user1",+ session=self.session,+ url="https://example.com",+ )+ mock_validate.assert_called_once_with("https://example.com", trusted_origins=[])+++# ---------------------------------------------------------------------------+# BrowserNavigateTool metadata+# ---------------------------------------------------------------------------+++class TestAgentBrowserAvailability:+ """is_available reflects whether the agent-browser CLI binary is installed."""++ def test_available_when_binary_found(self):+ with patch("shutil.which", return_value="/usr/local/bin/agent-browser"):+ assert BrowserNavigateTool().is_available is True+ assert BrowserActTool().is_available is True+ assert BrowserScreenshotTool().is_available is True++ def test_unavailable_when_binary_missing(self):+ with patch("shutil.which", return_value=None):+ assert BrowserNavigateTool().is_available is False+ assert BrowserActTool().is_available is False+ assert BrowserScreenshotTool().is_available is False+++class TestBrowserNavigateMetadata:+ def setup_method(self):+ self.tool = BrowserNavigateTool()++ def test_name(self):+ assert self.tool.name == "browser_navigate"++ def test_requires_auth(self):+ assert self.tool.requires_auth is True++ def test_parameters_has_url_required(self):+ params = self.tool.parameters+ assert "url" in params["properties"]+ assert "url" in params["required"]++ def test_parameters_has_wait_for(self):+ params = self.tool.parameters+ assert "wait_for" in params["properties"]++ def test_as_openai_tool(self):+ ot = self.tool.as_openai_tool()+ assert ot["type"] == "function"+ assert ot["function"]["name"] == "browser_navigate"+++# ---------------------------------------------------------------------------+# BrowserNavigateTool execution+# ---------------------------------------------------------------------------+++class TestBrowserNavigateExecute:+ def setup_method(self):+ self.tool = BrowserNavigateTool()+ self.session = make_session()++ @pytest.mark.asyncio+ async def test_missing_url_returns_error(self):+ result = await self.tool._execute(user_id="user1", session=self.session, url="")+ assert isinstance(result, ErrorResponse)+ assert result.error == "missing_url"++ @pytest.mark.asyncio+ async def test_ssrf_blocked_url_returns_error(self):+ with patch(_VALIDATE_URL, new_callable=AsyncMock) as mock_validate:+ mock_validate.side_effect = ValueError(+ "Access to blocked IP 10.0.0.1 is not allowed."+ )+ result = await self.tool._execute(+ user_id="user1", session=self.session, url="http://internal.corp"+ )+ assert isinstance(result, ErrorResponse)+ assert result.error == "blocked_url"++ @pytest.mark.asyncio+ async def test_navigation_failure_returns_error(self):+ with patch(+ _VALIDATE_URL,+ new_callable=AsyncMock,+ return_value=(object(), False, ["1.2.3.4"]),+ ):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ ) as mock_run:+ mock_run.return_value = _run_result(rc=1, stderr="timeout")+ result = await self.tool._execute(+ user_id="user1",+ session=self.session,+ url="https://example.com",+ )+ assert isinstance(result, ErrorResponse)+ assert result.error == "navigation_failed"++ @pytest.mark.asyncio+ async def test_successful_navigation(self):+ with patch(+ _VALIDATE_URL,+ new_callable=AsyncMock,+ return_value=(object(), False, ["1.2.3.4"]),+ ):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ ) as mock_run:+ # open → success; wait → success; get title; get url+ mock_run.side_effect = [+ _run_result(rc=0), # open+ _run_result(rc=0), # wait+ _run_result(rc=0, stdout="Example Domain"), # get title+ _run_result(rc=0, stdout="https://example.com"), # get url+ ]+ with patch(+ "backend.copilot.tools.agent_browser._snapshot",+ new_callable=AsyncMock,+ return_value="[snapshot]",+ ):+ result = await self.tool._execute(+ user_id="user1",+ session=self.session,+ url="https://example.com",+ )++ assert isinstance(result, BrowserNavigateResponse)+ assert result.type == ResponseType.BROWSER_NAVIGATE+ assert result.url == "https://example.com"+ assert result.title == "Example Domain"+ assert result.snapshot == "[snapshot]"++ @pytest.mark.asyncio+ async def test_session_id_used_as_session_name(self):+ """Ensure session.session_id is passed as --session-name."""+ session = make_session("my-unique-session")+ captured_calls = []++ async def fake_run(session_name, *args, **kwargs):+ captured_calls.append((session_name, args))+ return _run_result(rc=0)++ with patch(+ _VALIDATE_URL,+ new_callable=AsyncMock,+ return_value=(object(), False, ["1.2.3.4"]),+ ):+ with patch(+ "backend.copilot.tools.agent_browser._run", side_effect=fake_run+ ):+ with patch(+ "backend.copilot.tools.agent_browser._snapshot",+ new_callable=AsyncMock,+ return_value="",+ ):+ await self.tool._execute(+ user_id="user1",+ session=session,+ url="https://example.com",+ )++ for session_name, _ in captured_calls:+ assert session_name == "my-unique-session"+++# ---------------------------------------------------------------------------+# BrowserActTool metadata+# ---------------------------------------------------------------------------+++class TestBrowserActMetadata:+ def setup_method(self):+ self.tool = BrowserActTool()++ def test_name(self):+ assert self.tool.name == "browser_act"++ def test_requires_auth(self):+ assert self.tool.requires_auth is True++ def test_parameters_has_action_required(self):+ params = self.tool.parameters+ assert "action" in params["properties"]+ assert "action" in params["required"]++ def test_action_enum_values(self):+ actions = self.tool.parameters["properties"]["action"]["enum"]+ expected = {+ "click",+ "dblclick",+ "fill",+ "type",+ "scroll",+ "hover",+ "press",+ "check",+ "uncheck",+ "select",+ "wait",+ "back",+ "forward",+ "reload",+ }+ assert set(actions) == expected+++# ---------------------------------------------------------------------------+# BrowserActTool execution — input validation+# ---------------------------------------------------------------------------+++class TestBrowserActValidation:+ def setup_method(self):+ self.tool = BrowserActTool()+ self.session = make_session()++ @pytest.mark.asyncio+ async def test_missing_action_returns_error(self):+ result = await self.tool._execute(+ user_id="user1", session=self.session, action=""+ )+ assert isinstance(result, ErrorResponse)+ assert result.error == "missing_action"++ @pytest.mark.asyncio+ async def test_click_without_target_returns_error(self):+ result = await self.tool._execute(+ user_id="user1", session=self.session, action="click", target=""+ )+ assert isinstance(result, ErrorResponse)+ assert result.error == "missing_target"++ @pytest.mark.asyncio+ async def test_fill_without_value_returns_error(self):+ result = await self.tool._execute(+ user_id="user1",+ session=self.session,+ action="fill",+ target="@e1",+ value="",+ )+ assert isinstance(result, ErrorResponse)+ assert result.error == "missing_params"++ @pytest.mark.asyncio+ async def test_press_without_value_returns_error(self):+ result = await self.tool._execute(+ user_id="user1", session=self.session, action="press", value=""+ )+ assert isinstance(result, ErrorResponse)+ assert result.error == "missing_value"++ @pytest.mark.asyncio+ async def test_unsupported_action_returns_error(self):+ result = await self.tool._execute(+ user_id="user1", session=self.session, action="teleport"+ )+ assert isinstance(result, ErrorResponse)+ assert result.error == "invalid_action"+++# ---------------------------------------------------------------------------+# BrowserActTool execution — successful actions+# ---------------------------------------------------------------------------+++class TestBrowserActExecute:+ def setup_method(self):+ self.tool = BrowserActTool()+ self.session = make_session()++ async def _act(self, **kwargs):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ ) as mock_run:+ mock_run.return_value = _run_result(rc=0, stdout="https://example.com")+ with patch(+ "backend.copilot.tools.agent_browser._snapshot",+ new_callable=AsyncMock,+ return_value="[updated-snapshot]",+ ):+ result = await self.tool._execute(+ user_id="user1", session=self.session, **kwargs+ )+ return result++ @pytest.mark.asyncio+ async def test_click_success(self):+ result = await self._act(action="click", target="@e3")+ assert isinstance(result, BrowserActResponse)+ assert result.type == ResponseType.BROWSER_ACT+ assert result.action == "click"+ assert result.snapshot == "[updated-snapshot]"++ @pytest.mark.asyncio+ async def test_fill_success(self):+ result = await self._act(action="fill", target="@e1", value="[email protected]")+ assert isinstance(result, BrowserActResponse)+ assert "fill" in result.message+ # value must NOT appear in the message — it may contain credentials+ assert "[email protected]" not in result.message++ @pytest.mark.asyncio+ async def test_back_success(self):+ result = await self._act(action="back")+ assert isinstance(result, BrowserActResponse)++ @pytest.mark.asyncio+ async def test_scroll_default_direction(self):+ result = await self._act(action="scroll")+ assert isinstance(result, BrowserActResponse)++ @pytest.mark.asyncio+ async def test_scroll_up(self):+ result = await self._act(action="scroll", direction="up")+ assert isinstance(result, BrowserActResponse)++ @pytest.mark.asyncio+ async def test_press_enter(self):+ result = await self._act(action="press", value="Enter")+ assert isinstance(result, BrowserActResponse)++ @pytest.mark.asyncio+ async def test_action_failure_returns_error(self):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=1, stderr="element not found"),+ ):+ result = await self.tool._execute(+ user_id="user1", session=self.session, action="click", target="@e99"+ )+ assert isinstance(result, ErrorResponse)+ assert result.error == "action_failed"++ @pytest.mark.asyncio+ async def test_select_success(self):+ result = await self._act(action="select", target="@e5", value="option1")+ assert isinstance(result, BrowserActResponse)++ @pytest.mark.asyncio+ async def test_check_success(self):+ result = await self._act(action="check", target="@e2")+ assert isinstance(result, BrowserActResponse)++ @pytest.mark.asyncio+ async def test_uncheck_success(self):+ result = await self._act(action="uncheck", target="@e2")+ assert isinstance(result, BrowserActResponse)++ @pytest.mark.asyncio+ async def test_dblclick_success(self):+ result = await self._act(action="dblclick", target="@e4")+ assert isinstance(result, BrowserActResponse)+ assert result.action == "dblclick"++ @pytest.mark.asyncio+ async def test_type_success(self):+ """type appends text without clearing the field first (unlike fill)."""+ result = await self._act(action="type", target="@e1", value="hello")+ assert isinstance(result, BrowserActResponse)+ assert result.action == "type"++ @pytest.mark.asyncio+ async def test_wait_selector_success(self):+ """wait with a CSS selector waits for the element to appear."""+ result = await self._act(action="wait", target=".my-element")+ assert isinstance(result, BrowserActResponse)+ assert result.action == "wait"++ @pytest.mark.asyncio+ async def test_wait_ms_success(self):+ """wait with a numeric string pauses for the given milliseconds."""+ result = await self._act(action="wait", target="1000")+ assert isinstance(result, BrowserActResponse)++ @pytest.mark.asyncio+ async def test_dblclick_without_target_returns_error(self):+ result = await self.tool._execute(+ user_id="user1", session=self.session, action="dblclick", target=""+ )+ assert isinstance(result, ErrorResponse)+ assert result.error == "missing_target"++ @pytest.mark.asyncio+ async def test_type_without_value_returns_error(self):+ result = await self.tool._execute(+ user_id="user1",+ session=self.session,+ action="type",+ target="@e1",+ value="",+ )+ assert isinstance(result, ErrorResponse)+ assert result.error == "missing_params"++ @pytest.mark.asyncio+ async def test_wait_without_target_returns_error(self):+ result = await self.tool._execute(+ user_id="user1", session=self.session, action="wait", target=""+ )+ assert isinstance(result, ErrorResponse)+ assert result.error == "missing_target"+++# ---------------------------------------------------------------------------+# BrowserScreenshotTool metadata+# ---------------------------------------------------------------------------+++class TestBrowserScreenshotMetadata:+ def setup_method(self):+ self.tool = BrowserScreenshotTool()++ def test_name(self):+ assert self.tool.name == "browser_screenshot"++ def test_requires_auth(self):+ assert self.tool.requires_auth is True++ def test_parameters_are_optional(self):+ params = self.tool.parameters+ # No required fields — both annotate and filename are optional+ assert "required" not in params or len(params.get("required", [])) == 0+++# ---------------------------------------------------------------------------+# BrowserScreenshotTool execution+# ---------------------------------------------------------------------------+++class TestBrowserScreenshotExecute:+ def setup_method(self):+ self.tool = BrowserScreenshotTool()+ self.session = make_session()++ @pytest.mark.asyncio+ async def test_screenshot_failure_returns_error(self):++ fake_fd, fake_path = 5, "/tmp/fake.png"+ with patch("tempfile.mkstemp", return_value=(fake_fd, fake_path)):+ with patch("os.close"):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=1, stderr="no browser"),+ ):+ with patch("os.unlink"):+ result = await self.tool._execute(+ user_id="user1",+ session=self.session,+ )+ assert isinstance(result, ErrorResponse)+ assert result.error == "screenshot_failed"++ @pytest.mark.asyncio+ async def test_screenshot_success(self):+ import os+ import tempfile++ from .workspace_files import WorkspaceWriteResponse++ png_bytes = b"\x89PNG\r\n\x1a\n" + b"\x00" * 100++ # Build a real WorkspaceWriteResponse to satisfy the isinstance check+ write_resp = WorkspaceWriteResponse(+ message="ok",+ file_id="file-abc-123",+ name="test.png",+ path="/workspace/test.png",+ mime_type="image/png",+ size_bytes=len(png_bytes),+ download_url="workspace://file-abc-123#image/png",+ session_id="test-session-123",+ )++ fd, path = tempfile.mkstemp(suffix=".png")+ os.close(fd)+ with open(path, "wb") as f:+ f.write(png_bytes)++ try:+ with patch("tempfile.mkstemp", return_value=(fd, path)):+ with patch("os.close"):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0),+ ):+ # Patch WriteWorkspaceFileTool in the workspace_files module+ # (the lazy import resolves to this target)+ with patch(+ "backend.copilot.tools.workspace_files.WriteWorkspaceFileTool._execute",+ new_callable=AsyncMock,+ return_value=write_resp,+ ):+ with patch("os.unlink"):+ result = await self.tool._execute(+ user_id="user1",+ session=self.session,+ filename="test.png",+ annotate=True,+ )+ finally:+ try:+ os.unlink(path)+ except OSError:+ pass # Best-effort test cleanup; not critical if temp file lingers.++ assert isinstance(result, BrowserScreenshotResponse)+ assert result.type == ResponseType.BROWSER_SCREENSHOT+ assert result.file_id == "file-abc-123"+ assert result.filename == "test.png"++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "input_val,expected",+ [+ ("true", True),+ ("false", False),+ ("0", False),+ (True, True),+ (False, False),+ ],+ )+ async def test_annotate_string_coercion(self, input_val, expected):+ """OpenAI function-call payloads may send annotate as a string."""+ import os+ import tempfile++ from .workspace_files import WorkspaceWriteResponse++ png_bytes = b"\x89PNG\r\n\x1a\n" + b"\x00" * 100+ write_resp = WorkspaceWriteResponse(+ message="ok",+ file_id="file-abc-123",+ name="shot.png",+ path="/workspace/shot.png",+ mime_type="image/png",+ size_bytes=len(png_bytes),+ download_url="workspace://file-abc-123#image/png",+ session_id="test-session-123",+ )++ fd, path = tempfile.mkstemp(suffix=".png")+ os.close(fd)+ with open(path, "wb") as f:+ f.write(png_bytes)++ try:+ with patch("tempfile.mkstemp", return_value=(fd, path)):+ with patch("os.close"):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0),+ ) as mock_run:+ with patch(+ "backend.copilot.tools.workspace_files.WriteWorkspaceFileTool._execute",+ new_callable=AsyncMock,+ return_value=write_resp,+ ):+ with patch("os.unlink"):+ result = await self.tool._execute(+ user_id="user1",+ session=self.session,+ annotate=input_val,+ )+ finally:+ try:+ os.unlink(path)+ except OSError:+ pass++ assert isinstance(result, BrowserScreenshotResponse)+ # Verify the --annotate flag was passed (or not) to agent-browser+ run_calls = mock_run.call_args_list+ screenshot_call = [c for c in run_calls if "screenshot" in c.args]+ assert len(screenshot_call) == 1+ if expected:+ assert "--annotate" in screenshot_call[0].args+ else:+ assert "--annotate" not in screenshot_call[0].args+++# ---------------------------------------------------------------------------+# BaseTool auth check (inherits to all browser tools)+# ---------------------------------------------------------------------------+++class TestAuthCheck:+ @pytest.mark.asyncio+ async def test_navigate_requires_login_if_no_user(self):+ tool = BrowserNavigateTool()+ session = make_session()+ # BaseTool.execute checks requires_auth before calling _execute+ result = await tool.execute(+ user_id=None,+ session=session,+ tool_call_id="call-1",+ url="https://example.com",+ )+ assert isinstance(result.output, str)+ data = json.loads(result.output)+ assert data["type"] == ResponseType.NEED_LOGIN++ @pytest.mark.asyncio+ async def test_act_requires_login_if_no_user(self):+ tool = BrowserActTool()+ session = make_session()+ result = await tool.execute(+ user_id=None,+ session=session,+ tool_call_id="call-2",+ action="click",+ target="@e1",+ )+ assert isinstance(result.output, str)+ data = json.loads(result.output)+ assert data["type"] == ResponseType.NEED_LOGIN++ @pytest.mark.asyncio+ async def test_screenshot_requires_login_if_no_user(self):+ tool = BrowserScreenshotTool()+ session = make_session()+ result = await tool.execute(+ user_id=None,+ session=session,+ tool_call_id="call-3",+ )+ assert isinstance(result.output, str)+ data = json.loads(result.output)+ assert data["type"] == ResponseType.NEED_LOGIN+++# ---------------------------------------------------------------------------+# _snapshot helper+# ---------------------------------------------------------------------------+++class TestSnapshot:+ @pytest.mark.asyncio+ async def test_snapshot_truncation(self):+ from .agent_browser import _MAX_SNAPSHOT_CHARS, _snapshot++ big_text = "x" * (_MAX_SNAPSHOT_CHARS + 1000)+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0, stdout=big_text),+ ):+ result = await _snapshot("test-session")+ assert len(result) <= _MAX_SNAPSHOT_CHARS+ assert "truncated" in result.lower()++ @pytest.mark.asyncio+ async def test_snapshot_failure(self):+ from .agent_browser import _snapshot++ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=1, stderr="daemon died"),+ ):+ result = await _snapshot("test-session")+ assert "snapshot failed" in result++ @pytest.mark.asyncio+ async def test_snapshot_short_content_not_truncated(self):+ from .agent_browser import _snapshot++ short = "button @e1 Submit"+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0, stdout=short),+ ):+ result = await _snapshot("test-session")+ assert result == short+ assert "truncated" not in result+++# ---------------------------------------------------------------------------+# _run helper (timeout and FileNotFoundError)+# ---------------------------------------------------------------------------+++class TestRunHelper:+ @pytest.mark.asyncio+ async def test_run_timeout(self):+ from .agent_browser import _run++ mock_proc = MagicMock()+ mock_proc.returncode = None # Simulate a process that is still running+ mock_proc.kill = MagicMock()+ mock_proc.communicate = AsyncMock(return_value=(b"", b""))++ with patch("asyncio.create_subprocess_exec", return_value=mock_proc):+ with patch("asyncio.wait_for", side_effect=asyncio.TimeoutError):+ rc, stdout, stderr = await _run(+ "sess", "open", "https://x.com", timeout=1+ )++ assert rc == 1+ assert "timed out" in stderr+ mock_proc.kill.assert_called_once()+ mock_proc.communicate.assert_called()++ @pytest.mark.asyncio+ async def test_run_agent_browser_not_installed(self):+ from .agent_browser import _run++ with patch(+ "asyncio.create_subprocess_exec",+ side_effect=FileNotFoundError("No such file"),+ ):+ rc, stdout, stderr = await _run("sess", "snapshot", "-i")++ assert rc == 1+ assert "agent-browser" in stderr+ assert "not installed" in stderr+++# ---------------------------------------------------------------------------+# _has_local_session+# ---------------------------------------------------------------------------+++class TestHasLocalSession:+ @pytest.mark.asyncio+ async def test_returns_true_when_daemon_alive(self):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0, stdout="https://example.com"),+ ):+ assert await _has_local_session("sess-1") is True++ @pytest.mark.asyncio+ async def test_returns_false_when_daemon_dead(self):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=1, stderr="no session"),+ ):+ assert await _has_local_session("sess-1") is False+++# ---------------------------------------------------------------------------+# _save_browser_state+# ---------------------------------------------------------------------------++_GET_MANAGER = "backend.copilot.tools.agent_browser.get_manager"+++def _make_mock_manager():+ mgr = AsyncMock()+ mgr.write_file = AsyncMock()+ mgr.read_file = AsyncMock()+ mgr.get_file_info_by_path = AsyncMock(return_value=None)+ return mgr+++class TestSaveBrowserState:+ @pytest.mark.asyncio+ async def test_writes_json_to_workspace(self):+ session = make_session("save-sess")+ cookies_json = json.dumps(+ [{"name": "sid", "value": "abc", "domain": ".example.com", "path": "/"}]+ )+ storage_json = json.dumps({"theme": "dark"})++ mock_mgr = _make_mock_manager()++ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ ) as mock_run:+ mock_run.side_effect = [+ _run_result(rc=0, stdout="https://example.com"), # get url+ _run_result(rc=0, stdout=cookies_json), # cookies get --json+ _run_result(rc=0, stdout=storage_json), # storage local --json+ ]+ with patch(_GET_MANAGER, new_callable=AsyncMock, return_value=mock_mgr):+ await _save_browser_state("save-sess", "user1", session)++ mock_mgr.write_file.assert_called_once()+ call_kwargs = mock_mgr.write_file.call_args+ written = json.loads(call_kwargs.kwargs["content"].decode("utf-8"))+ assert written["url"] == "https://example.com"+ assert written["cookies"] == [+ {"name": "sid", "value": "abc", "domain": ".example.com", "path": "/"}+ ]+ assert written["local_storage"] == {"theme": "dark"}+ assert call_kwargs.kwargs["overwrite"] is True+ assert call_kwargs.kwargs["filename"] == "._browser_state.json"++ @pytest.mark.asyncio+ async def test_error_is_swallowed(self):+ session = make_session("err-sess")+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ side_effect=RuntimeError("boom"),+ ):+ # Should not raise+ await _save_browser_state("err-sess", "user1", session)+++# ---------------------------------------------------------------------------+# _restore_browser_state+# ---------------------------------------------------------------------------+++class TestRestoreBrowserState:+ @pytest.mark.asyncio+ async def test_restores_url_cookies_and_storage(self):+ session = make_session("restore-sess")+ state = {+ "url": "https://example.com/page",+ "cookies": [+ {"name": "sid", "value": "abc", "domain": ".example.com", "path": "/"},+ {+ "name": "token",+ "value": "xyz",+ "domain": ".example.com",+ "path": "/api",+ },+ ],+ "local_storage": {"theme": "dark", "lang": "en"},+ }++ mock_mgr = _make_mock_manager()+ mock_mgr.get_file_info_by_path.return_value = MagicMock(id="file-1")+ mock_mgr.read_file.return_value = json.dumps(state).encode("utf-8")++ captured_cmds: list[tuple] = []++ async def fake_run(session_name, *args, **kwargs):+ captured_cmds.append((session_name, args))+ return _run_result(rc=0)++ with patch("backend.copilot.tools.agent_browser._run", side_effect=fake_run):+ with patch(_GET_MANAGER, new_callable=AsyncMock, return_value=mock_mgr):+ result = await _restore_browser_state("restore-sess", "user1", session)++ assert result is True+ # First call: open URL+ assert captured_cmds[0] == (+ "restore-sess",+ ("open", "https://example.com/page"),+ )+ # Second: wait for load+ assert captured_cmds[1][1] == ("wait", "--load", "load")+ # Then cookies (2 cookies)+ cookie_cmds = [+ c for c in captured_cmds if len(c[1]) > 1 and c[1][0] == "cookies"+ ]+ assert len(cookie_cmds) == 2+ assert cookie_cmds[0][1] == (+ "cookies",+ "set",+ "sid",+ "abc",+ "--domain",+ ".example.com",+ "--path",+ "/",+ )+ assert cookie_cmds[1][1] == (+ "cookies",+ "set",+ "token",+ "xyz",+ "--domain",+ ".example.com",+ "--path",+ "/api",+ )+ # Then localStorage (2 entries)+ storage_cmds = [+ c+ for c in captured_cmds+ if len(c[1]) > 2 and c[1][:2] == ("storage", "local")+ ]+ assert len(storage_cmds) == 2++ @pytest.mark.asyncio+ async def test_no_saved_state_returns_true(self):+ session = make_session("fresh-sess")+ mock_mgr = _make_mock_manager()+ mock_mgr.get_file_info_by_path.return_value = None++ with patch(_GET_MANAGER, new_callable=AsyncMock, return_value=mock_mgr):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ ) as mock_run:+ result = await _restore_browser_state("fresh-sess", "user1", session)++ assert result is True+ mock_run.assert_not_called()++ @pytest.mark.asyncio+ async def test_empty_url_skips_navigation(self):+ """State with url='' should skip navigation and still restore cookies/storage."""+ session = make_session("empty-url-sess")+ state = {+ "url": "",+ "cookies": [+ {"name": "c1", "value": "v1", "domain": ".example.com", "path": "/"},+ ],+ "local_storage": {"k": "v"},+ }++ mock_mgr = _make_mock_manager()+ mock_mgr.get_file_info_by_path.return_value = MagicMock(id="f-1")+ mock_mgr.read_file.return_value = json.dumps(state).encode("utf-8")++ captured_cmds: list[tuple] = []++ async def track_run(session_name, *args, **kwargs):+ captured_cmds.append((session_name, args))+ return _run_result(rc=0)++ with patch("backend.copilot.tools.agent_browser._run", side_effect=track_run):+ with patch(_GET_MANAGER, new_callable=AsyncMock, return_value=mock_mgr):+ result = await _restore_browser_state(+ "empty-url-sess", "user1", session+ )++ assert result is True+ # Should NOT have called "open" — no URL to navigate to+ open_cmds = [c for c in captured_cmds if c[1][:1] == ("open",)]+ assert open_cmds == []+ # But cookies and storage should still be restored+ cookie_cmds = [c for c in captured_cmds if c[1][:2] == ("cookies", "set")]+ storage_cmds = [+ c+ for c in captured_cmds+ if len(c[1]) > 2 and c[1][:2] == ("storage", "local")+ ]+ assert len(cookie_cmds) == 1+ assert len(storage_cmds) == 1++ @pytest.mark.asyncio+ async def test_error_returns_false(self):+ session = make_session("err-sess")+ with patch(+ _GET_MANAGER,+ new_callable=AsyncMock,+ side_effect=RuntimeError("db down"),+ ):+ result = await _restore_browser_state("err-sess", "user1", session)++ assert result is False+++# ---------------------------------------------------------------------------+# _ensure_session+# ---------------------------------------------------------------------------+++class TestEnsureSession:+ @pytest.mark.asyncio+ async def test_skips_probe_when_cached(self):+ from . import agent_browser as _mod++ session = make_session("cached-sess")+ _mod._alive_sessions.add("cached-sess")++ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ ) as mock_run:+ await _ensure_session("cached-sess", "user1", session)++ mock_run.assert_not_called()++ @pytest.mark.asyncio+ async def test_adds_to_cache_when_daemon_alive(self):+ from . import agent_browser as _mod++ session = make_session("alive-sess")+ with patch(+ "backend.copilot.tools.agent_browser._has_local_session",+ new_callable=AsyncMock,+ return_value=True,+ ):+ await _ensure_session("alive-sess", "user1", session)++ assert "alive-sess" in _mod._alive_sessions++ @pytest.mark.asyncio+ async def test_restores_and_caches_when_daemon_dead(self):+ from . import agent_browser as _mod++ session = make_session("dead-sess")+ with patch(+ "backend.copilot.tools.agent_browser._has_local_session",+ new_callable=AsyncMock,+ return_value=False,+ ):+ with patch(+ "backend.copilot.tools.agent_browser._restore_browser_state",+ new_callable=AsyncMock,+ return_value=True,+ ) as mock_restore:+ await _ensure_session("dead-sess", "user1", session)++ mock_restore.assert_called_once_with("dead-sess", "user1", session)+ assert "dead-sess" in _mod._alive_sessions++ @pytest.mark.asyncio+ async def test_no_cache_when_restore_fails(self):+ """Failed restore should not cache the session as alive."""+ from . import agent_browser as _mod++ session = make_session("fail-sess")+ with patch(+ "backend.copilot.tools.agent_browser._has_local_session",+ new_callable=AsyncMock,+ return_value=False,+ ):+ with patch(+ "backend.copilot.tools.agent_browser._restore_browser_state",+ new_callable=AsyncMock,+ return_value=False,+ ):+ await _ensure_session("fail-sess", "user1", session)++ assert "fail-sess" not in _mod._alive_sessions++ @pytest.mark.asyncio+ async def test_concurrent_calls_restore_once(self):+ """Two concurrent _ensure_session calls should only restore once."""+ session = make_session("race-sess")+ with patch(+ "backend.copilot.tools.agent_browser._has_local_session",+ new_callable=AsyncMock,+ return_value=False,+ ):+ with patch(+ "backend.copilot.tools.agent_browser._restore_browser_state",+ new_callable=AsyncMock,+ return_value=True,+ ) as mock_restore:+ await asyncio.gather(+ _ensure_session("race-sess", "user1", session),+ _ensure_session("race-sess", "user1", session),+ )++ mock_restore.assert_called_once()+++# ---------------------------------------------------------------------------+# close_browser_session+# ---------------------------------------------------------------------------+++class TestCloseBrowserSession:+ @pytest.mark.asyncio+ async def test_calls_close_command(self):+ from . import agent_browser as _mod++ _mod._alive_sessions.add("close-sess")+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0),+ ) as mock_run:+ await close_browser_session("close-sess")++ mock_run.assert_called_once_with("close-sess", "close", timeout=10)+ assert "close-sess" not in _mod._alive_sessions+ assert "close-sess" not in _mod._session_locks++ @pytest.mark.asyncio+ async def test_deletes_state_file_when_user_id_provided(self):+ mock_mgr = _make_mock_manager()+ mock_mgr.get_file_info_by_path.return_value = MagicMock(id="state-file-1")+ mock_mgr.delete_file = AsyncMock(return_value=True)++ with patch(_GET_MANAGER, new_callable=AsyncMock, return_value=mock_mgr):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0),+ ):+ await close_browser_session("close-sess", user_id="user1")++ mock_mgr.get_file_info_by_path.assert_called_once_with("._browser_state.json")+ mock_mgr.delete_file.assert_called_once_with("state-file-1")++ @pytest.mark.asyncio+ async def test_skips_state_cleanup_without_user_id(self):+ with patch(_GET_MANAGER, new_callable=AsyncMock) as mock_get_mgr:+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0),+ ):+ await close_browser_session("close-sess")++ mock_get_mgr.assert_not_called()++ @pytest.mark.asyncio+ async def test_state_file_not_found_skips_delete(self):+ """If get_file_info_by_path returns None, delete_file should not be called."""+ mock_mgr = _make_mock_manager()+ mock_mgr.get_file_info_by_path.return_value = None+ mock_mgr.delete_file = AsyncMock()++ with patch(_GET_MANAGER, new_callable=AsyncMock, return_value=mock_mgr):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0),+ ):+ await close_browser_session("close-sess", user_id="user1")++ mock_mgr.get_file_info_by_path.assert_called_once_with("._browser_state.json")+ mock_mgr.delete_file.assert_not_called()++ @pytest.mark.asyncio+ async def test_error_is_swallowed(self):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ side_effect=RuntimeError("gone"),+ ):+ # Should not raise+ await close_browser_session("bad-sess")+++# ---------------------------------------------------------------------------+# Integration: save/restore called from _execute methods+# ---------------------------------------------------------------------------+++class TestStatePersistenceIntegration:+ """Verify _execute methods call _ensure_session and _fire_and_forget_save."""++ def setup_method(self):+ self.session = make_session("int-sess")++ @pytest.mark.asyncio+ async def test_navigate_calls_ensure_and_save(self, monkeypatch):+ mock_ensure = AsyncMock()+ mock_save = MagicMock()+ monkeypatch.setattr(+ "backend.copilot.tools.agent_browser._ensure_session", mock_ensure+ )+ monkeypatch.setattr(+ "backend.copilot.tools.agent_browser._fire_and_forget_save", mock_save+ )++ tool = BrowserNavigateTool()+ with patch(+ _VALIDATE_URL,+ new_callable=AsyncMock,+ return_value=(object(), False, ["1.2.3.4"]),+ ):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0),+ ):+ with patch(+ "backend.copilot.tools.agent_browser._snapshot",+ new_callable=AsyncMock,+ return_value="",+ ):+ result = await tool._execute(+ user_id="user1",+ session=self.session,+ url="https://example.com",+ )++ assert isinstance(result, BrowserNavigateResponse)+ mock_ensure.assert_called_once_with("int-sess", "user1", self.session)+ mock_save.assert_called_once_with("int-sess", "user1", self.session)++ @pytest.mark.asyncio+ async def test_navigate_no_save_on_error(self, monkeypatch):+ mock_save = MagicMock()+ monkeypatch.setattr(+ "backend.copilot.tools.agent_browser._fire_and_forget_save", mock_save+ )++ tool = BrowserNavigateTool()+ with patch(+ _VALIDATE_URL,+ new_callable=AsyncMock,+ return_value=(object(), False, ["1.2.3.4"]),+ ):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=1, stderr="fail"),+ ):+ result = await tool._execute(+ user_id="user1",+ session=self.session,+ url="https://example.com",+ )++ assert isinstance(result, ErrorResponse)+ mock_save.assert_not_called()++ @pytest.mark.asyncio+ async def test_navigate_skips_persistence_without_user_id(self, monkeypatch):+ mock_ensure = AsyncMock()+ mock_save = MagicMock()+ monkeypatch.setattr(+ "backend.copilot.tools.agent_browser._ensure_session", mock_ensure+ )+ monkeypatch.setattr(+ "backend.copilot.tools.agent_browser._fire_and_forget_save", mock_save+ )++ tool = BrowserNavigateTool()+ with patch(+ _VALIDATE_URL,+ new_callable=AsyncMock,+ return_value=(object(), False, ["1.2.3.4"]),+ ):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0),+ ):+ with patch(+ "backend.copilot.tools.agent_browser._snapshot",+ new_callable=AsyncMock,+ return_value="",+ ):+ result = await tool._execute(+ user_id=None,+ session=self.session,+ url="https://example.com",+ )++ assert isinstance(result, BrowserNavigateResponse)+ mock_ensure.assert_not_called()+ mock_save.assert_not_called()++ @pytest.mark.asyncio+ async def test_act_calls_ensure_and_save(self, monkeypatch):+ mock_ensure = AsyncMock()+ mock_save = MagicMock()+ monkeypatch.setattr(+ "backend.copilot.tools.agent_browser._ensure_session", mock_ensure+ )+ monkeypatch.setattr(+ "backend.copilot.tools.agent_browser._fire_and_forget_save", mock_save+ )++ tool = BrowserActTool()+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0, stdout="https://example.com"),+ ):+ with patch(+ "backend.copilot.tools.agent_browser._snapshot",+ new_callable=AsyncMock,+ return_value="",+ ):+ result = await tool._execute(+ user_id="user1",+ session=self.session,+ action="click",+ target="@e1",+ )++ assert isinstance(result, BrowserActResponse)+ mock_ensure.assert_called_once()+ mock_save.assert_called_once()++ @pytest.mark.asyncio+ async def test_screenshot_calls_ensure_and_save(self, monkeypatch):+ import os+ import tempfile++ from .workspace_files import WorkspaceWriteResponse++ mock_ensure = AsyncMock()+ mock_save = MagicMock()+ monkeypatch.setattr(+ "backend.copilot.tools.agent_browser._ensure_session", mock_ensure+ )+ monkeypatch.setattr(+ "backend.copilot.tools.agent_browser._fire_and_forget_save", mock_save+ )++ png_bytes = b"\x89PNG\r\n\x1a\n" + b"\x00" * 100+ write_resp = WorkspaceWriteResponse(+ message="ok",+ file_id="f-1",+ name="s.png",+ path="/workspace/s.png",+ mime_type="image/png",+ size_bytes=len(png_bytes),+ download_url="workspace://f-1#image/png",+ session_id="int-sess",+ )++ fd, path = tempfile.mkstemp(suffix=".png")+ os.close(fd)+ with open(path, "wb") as f:+ f.write(png_bytes)++ try:+ tool = BrowserScreenshotTool()+ with patch("tempfile.mkstemp", return_value=(fd, path)):+ with patch("os.close"):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=0),+ ):+ with patch(+ "backend.copilot.tools.workspace_files.WriteWorkspaceFileTool._execute",+ new_callable=AsyncMock,+ return_value=write_resp,+ ):+ with patch("os.unlink"):+ result = await tool._execute(+ user_id="user1",+ session=self.session,+ )+ finally:+ try:+ os.unlink(path)+ except OSError:+ pass++ assert isinstance(result, BrowserScreenshotResponse)+ mock_ensure.assert_called_once()+ mock_save.assert_called_once()+++# ---------------------------------------------------------------------------+# SSRF protection in _restore_browser_state+# ---------------------------------------------------------------------------+++class TestRestoreSsrf:+ @pytest.mark.asyncio+ async def test_blocked_url_returns_false(self):+ """Saved URL pointing at a private IP should be blocked."""+ session = make_session("ssrf-sess")+ state = {+ "url": "http://169.254.169.254/metadata",+ "cookies": [],+ "local_storage": {},+ }++ mock_mgr = _make_mock_manager()+ mock_mgr.get_file_info_by_path.return_value = MagicMock(id="f-1")+ mock_mgr.read_file.return_value = json.dumps(state).encode("utf-8")++ with patch(_GET_MANAGER, new_callable=AsyncMock, return_value=mock_mgr):+ with patch(+ _VALIDATE_URL,+ new_callable=AsyncMock,+ side_effect=ValueError("blocked"),+ ):+ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ ) as mock_run:+ result = await _restore_browser_state("ssrf-sess", "user1", session)++ assert result is False+ # Should NOT have called _run at all — URL was blocked before open+ mock_run.assert_not_called()+++# ---------------------------------------------------------------------------+# Partial-failure tests for save/restore+# ---------------------------------------------------------------------------+++class TestPartialFailures:+ """Test mixed success/failure scenarios in save and restore."""++ @pytest.mark.asyncio+ async def test_save_partial_export_failure(self):+ """If cookies export fails but URL and storage succeed, state is still saved."""+ session = make_session("partial-save")+ mock_mgr = _make_mock_manager()++ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ ) as mock_run:+ mock_run.side_effect = [+ _run_result(rc=0, stdout="https://example.com"), # get url+ _run_result(rc=1, stderr="cookie export error"), # cookies get fails+ _run_result(rc=0, stdout='{"theme":"dark"}'), # storage local+ ]+ with patch(_GET_MANAGER, new_callable=AsyncMock, return_value=mock_mgr):+ await _save_browser_state("partial-save", "user1", session)++ # Should still write state — with empty cookies list+ mock_mgr.write_file.assert_called_once()+ written = json.loads(+ mock_mgr.write_file.call_args.kwargs["content"].decode("utf-8")+ )+ assert written["url"] == "https://example.com"+ assert written["cookies"] == [] # Failed export → empty+ assert written["local_storage"] == {"theme": "dark"}++ @pytest.mark.asyncio+ async def test_restore_cookies_fail_url_succeeds(self):+ """If some cookies fail to restore, the overall restore still succeeds."""+ session = make_session("partial-restore")+ state = {+ "url": "https://example.com",+ "cookies": [+ {"name": "good", "value": "1", "domain": ".example.com", "path": "/"},+ {"name": "bad", "value": "2", "domain": ".example.com", "path": "/"},+ ],+ "local_storage": {"key": "val"},+ }++ mock_mgr = _make_mock_manager()+ mock_mgr.get_file_info_by_path.return_value = MagicMock(id="f-1")+ mock_mgr.read_file.return_value = json.dumps(state).encode("utf-8")++ call_count = 0++ async def selective_fail(session_name, *args, **kwargs):+ nonlocal call_count+ call_count += 1+ # First two calls: open URL + wait for load → succeed+ if args[:1] == ("open",) or args[:1] == ("wait",):+ return _run_result(rc=0)+ # First cookie set succeeds, second fails+ if args[:2] == ("cookies", "set"):+ if args[2] == "bad":+ return _run_result(rc=1, stderr="cookie set error")+ return _run_result(rc=0)+ # Storage always succeeds+ return _run_result(rc=0)++ with patch(+ "backend.copilot.tools.agent_browser._run", side_effect=selective_fail+ ):+ with patch(_GET_MANAGER, new_callable=AsyncMock, return_value=mock_mgr):+ with patch(+ _VALIDATE_URL,+ new_callable=AsyncMock,+ return_value=(object(), False, ["1.2.3.4"]),+ ):+ result = await _restore_browser_state(+ "partial-restore", "user1", session+ )++ # Restore still returns True — individual cookie failures are non-fatal+ assert result is True++ @pytest.mark.asyncio+ async def test_restore_url_fails_returns_false(self):+ """If the URL navigation itself fails, restore returns False."""+ session = make_session("url-fail-sess")+ state = {+ "url": "https://example.com",+ "cookies": [],+ "local_storage": {},+ }++ mock_mgr = _make_mock_manager()+ mock_mgr.get_file_info_by_path.return_value = MagicMock(id="f-1")+ mock_mgr.read_file.return_value = json.dumps(state).encode("utf-8")++ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ return_value=_run_result(rc=1, stderr="navigation failed"),+ ):+ with patch(_GET_MANAGER, new_callable=AsyncMock, return_value=mock_mgr):+ with patch(+ _VALIDATE_URL,+ new_callable=AsyncMock,+ return_value=(object(), False, ["1.2.3.4"]),+ ):+ result = await _restore_browser_state(+ "url-fail-sess", "user1", session+ )++ assert result is False++ @pytest.mark.asyncio+ async def test_save_workspace_write_fails(self):+ """If workspace write raises, save swallows the error."""+ session = make_session("write-fail")+ mock_mgr = _make_mock_manager()+ mock_mgr.write_file.side_effect = RuntimeError("storage down")++ with patch(+ "backend.copilot.tools.agent_browser._run",+ new_callable=AsyncMock,+ ) as mock_run:+ mock_run.side_effect = [+ _run_result(rc=0, stdout="https://example.com"), # get url+ _run_result(rc=0, stdout="[]"), # cookies get --json+ _run_result(rc=0, stdout="{}"), # storage local --json+ ]+ with patch(_GET_MANAGER, new_callable=AsyncMock, return_value=mock_mgr):+ # Should not raise+ await _save_browser_state("write-fail", "user1", session)++ # write_file was actually called and its RuntimeError was swallowed+ mock_mgr.write_file.assert_called_once()diff --git a/autogpt_platform/backend/backend/copilot/tools/base.py b/autogpt_platform/backend/backend/copilot/tools/base.pyindex fa2858153e..da2a644edf 100644--- a/autogpt_platform/backend/backend/copilot/tools/base.py+++ b/autogpt_platform/backend/backend/copilot/tools/base.py@@ -36,6 +36,16 @@ class BaseTool:"""Whether this tool requires authentication."""return False+ @property+ def is_available(self) -> bool:+ """Whether this tool is available in the current environment.++ Override to check required env vars, binaries, or other dependencies.+ Unavailable tools are excluded from the LLM tool list so the model is+ never offered an option that will immediately fail.+ """+ return True+def as_openai_tool(self) -> ChatCompletionToolParam:"""Convert to OpenAI tool format."""return ChatCompletionToolParam(diff --git a/autogpt_platform/backend/backend/copilot/tools/models.py b/autogpt_platform/backend/backend/copilot/tools/models.pyindex 8c6740ce2e..125ff1c15f 100644--- a/autogpt_platform/backend/backend/copilot/tools/models.py+++ b/autogpt_platform/backend/backend/copilot/tools/models.py@@ -41,6 +41,10 @@ class ResponseType(str, Enum):INPUT_VALIDATION_ERROR = "input_validation_error"# Web fetchWEB_FETCH = "web_fetch"+ # Agent-browser multi-step automation (navigate, act, screenshot)+ BROWSER_NAVIGATE = "browser_navigate"+ BROWSER_ACT = "browser_act"+ BROWSER_SCREENSHOT = "browser_screenshot"# Code executionBASH_EXEC = "bash_exec"# Feature request types@@ -476,3 +480,32 @@ class FeatureRequestCreatedResponse(ToolResponseBase):issue_url: stris_new_issue: bool # False if added to existingcustomer_name: str+++# Agent-browser multi-step automation models+++class BrowserNavigateResponse(ToolResponseBase):+ """Response for browser_navigate tool."""++ type: ResponseType = ResponseType.BROWSER_NAVIGATE+ url: str+ title: str+ snapshot: str # Interactive accessibility tree with @ref IDs+++class BrowserActResponse(ToolResponseBase):+ """Response for browser_act tool."""++ type: ResponseType = ResponseType.BROWSER_ACT+ action: str+ current_url: str = ""+ snapshot: str # Updated accessibility tree after the action+++class BrowserScreenshotResponse(ToolResponseBase):+ """Response for browser_screenshot tool."""++ type: ResponseType = ResponseType.BROWSER_SCREENSHOT+ file_id: str # Workspace file ID — use read_workspace_file to retrieve+ filename: strdiff --git a/autogpt_platform/backend/backend/copilot/tools/workspace_files.py b/autogpt_platform/backend/backend/copilot/tools/workspace_files.pyindex 0032db63b1..1bbc36227f 100644--- a/autogpt_platform/backend/backend/copilot/tools/workspace_files.py+++ b/autogpt_platform/backend/backend/copilot/tools/workspace_files.py@@ -218,7 +218,7 @@ def _is_text_mime(mime_type: str) -> bool:return any(mime_type.startswith(t) for t in _TEXT_MIME_PREFIXES)-async def _get_manager(user_id: str, session_id: str) -> WorkspaceManager:+async def get_manager(user_id: str, session_id: str) -> WorkspaceManager:"""Create a session-scoped WorkspaceManager."""workspace = await workspace_db().get_or_create_workspace(user_id)return WorkspaceManager(user_id, workspace.id, session_id)@@ -386,7 +386,7 @@ class ListWorkspaceFilesTool(BaseTool):include_all_sessions: bool = kwargs.get("include_all_sessions", False)try:- manager = await _get_manager(user_id, session_id)+ manager = await get_manager(user_id, session_id)files = await manager.list_files(path=path_prefix, limit=limit, include_all_sessions=include_all_sessions)@@ -517,7 +517,7 @@ class ReadWorkspaceFileTool(BaseTool):)try:- manager = await _get_manager(user_id, session_id)+ manager = await get_manager(user_id, session_id)resolved = await _resolve_file(manager, file_id, path, session_id)if isinstance(resolved, ErrorResponse):return resolved@@ -725,7 +725,7 @@ class WriteWorkspaceFileTool(BaseTool):try:await scan_content_safe(content, filename=filename)- manager = await _get_manager(user_id, session_id)+ manager = await get_manager(user_id, session_id)rec = await manager.write_file(content=content,filename=filename,@@ -852,7 +852,7 @@ class DeleteWorkspaceFileTool(BaseTool):)try:- manager = await _get_manager(user_id, session_id)+ manager = await get_manager(user_id, session_id)resolved = await _resolve_file(manager, file_id, path, session_id)if isinstance(resolved, ErrorResponse):return resolveddiff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/tools/GenericTool/GenericTool.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/tools/GenericTool/GenericTool.tsxindex 8c0cccce83..ad6753e6ac 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/tools/GenericTool/GenericTool.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/tools/GenericTool/GenericTool.tsx@@ -12,6 +12,7 @@ import {GlobeIcon,ListChecksIcon,MagnifyingGlassIcon,+ MonitorIcon,PencilSimpleIcon,TerminalIcon,TrashIcon,@@ -48,6 +49,7 @@ function formatToolName(name: string): string {type ToolCategory =| "bash"| "web"+ | "browser"| "file-read"| "file-write"| "file-delete"@@ -65,6 +67,10 @@ function getToolCategory(toolName: string): ToolCategory {case "WebSearch":case "WebFetch":return "web";+ case "browser_navigate":+ case "browser_act":+ case "browser_screenshot":+ return "browser";case "read_workspace_file":case "read_file":case "Read":@@ -120,6 +126,8 @@ function ToolIcon({return <TerminalIcon size={14} weight="regular" className={iconClass} />;case "web":return <GlobeIcon size={14} weight="regular" className={iconClass} />;+ case "browser":+ return <MonitorIcon size={14} weight="regular" className={iconClass} />;case "file-read":return <FileIcon size={14} weight="regular" className={iconClass} />;case "file-write":@@ -155,6 +163,8 @@ function AccordionIcon({ category }: { category: ToolCategory }) {return <TerminalIcon size={32} weight="light" />;case "web":return <GlobeIcon size={32} weight="light" />;+ case "browser":+ return <MonitorIcon size={32} weight="light" />;case "file-read":case "file-write":return <FileIcon size={32} weight="light" />;@@ -189,6 +199,16 @@ function getInputSummary(toolName: string, input: unknown): string | null {return typeof inp.url === "string" ? inp.url : null;case "WebSearch":return typeof inp.query === "string" ? inp.query : null;+ case "browser_navigate":+ return typeof inp.url === "string" ? inp.url : null;+ case "browser_act":+ return typeof inp.action === "string"+ ? inp.target+ ? `${inp.action} ${inp.target}`+ : (inp.action as string)+ : null;+ case "browser_screenshot":+ return null;case "read_workspace_file":case "read_file":case "Read":@@ -259,6 +279,11 @@ function getAnimationText(part: ToolUIPart, category: ToolCategory): string {return shortSummary? `Fetching ${shortSummary}`: "Fetching web content…";+ case "browser":+ if (toolName === "browser_screenshot") return "Taking screenshot…";+ return shortSummary+ ? `Browsing ${shortSummary}`+ : "Interacting with browser…";case "file-read":return shortSummary ? `Reading ${shortSummary}` : "Reading file…";case "file-write":@@ -297,6 +322,11 @@ function getAnimationText(part: ToolUIPart, category: ToolCategory): string {return shortSummary? `Fetched ${shortSummary}`: "Fetched web content";+ case "browser":+ if (toolName === "browser_screenshot") return "Screenshot captured";+ return shortSummary+ ? `Browsed ${shortSummary}`+ : "Browser action completed";case "file-read":return shortSummary ? `Read ${shortSummary}` : "File read completed";case "file-write":@@ -323,6 +353,8 @@ function getAnimationText(part: ToolUIPart, category: ToolCategory): string {return "Command failed";case "web":return toolName === "WebSearch" ? "Search failed" : "Fetch failed";+ case "browser":+ return "Browser action failed";default:return `${formatToolName(toolName)} failed`;}@@ -500,6 +532,44 @@ function getWebAccordionData(};}+function getBrowserAccordionData(+ toolName: string,+ input: unknown,+ output: Record<string, unknown>,+): AccordionData {+ const message = getStringField(output, "message");+ const snapshot = getStringField(output, "snapshot");++ // Screenshot tool: show the file_id so the user knows it was saved+ if (toolName === "browser_screenshot") {+ const fileId = getStringField(output, "file_id");+ const filename = getStringField(output, "filename");+ return {+ title: filename ? `Screenshot: ${filename}` : "Screenshot captured",+ description: fileId ? `file_id: ${fileId}` : undefined,+ content: message ? <ContentMessage>{message}</ContentMessage> : null,+ };+ }++ // Navigate / act tools: show snapshot if available+ const title =+ toolName === "browser_navigate"+ ? (getStringField(output, "title") ?? "Page loaded")+ : (message ?? "Action completed");++ const url = getStringField(output, "url", "current_url");++ return {+ title,+ description: url ? truncate(url, 80) : undefined,+ content: snapshot ? (+ <ContentCodeBlock>{truncate(snapshot, 3000)}</ContentCodeBlock>+ ) : message ? (+ <ContentMessage>{message}</ContentMessage>+ ) : null,+ };+}+function getFileAccordionData(category: ToolCategory,input: unknown,@@ -725,6 +795,7 @@ function getDefaultAccordionData(function getAccordionData(category: ToolCategory,+ toolName: string,input: unknown,output: Record<string, unknown>,): AccordionData {@@ -733,6 +804,8 @@ function getAccordionData(return getBashAccordionData(input, output);case "web":return getWebAccordionData(input, output);+ case "browser":+ return getBrowserAccordionData(toolName, input, output);case "file-read":case "file-write":case "file-delete":@@ -775,7 +848,7 @@ export function GenericTool({ part }: Props) {const showAccordion = hasOutput || hasError || hasTodoInput;const accordionData = showAccordion- ? getAccordionData(category, part.input, output ?? {})+ ? getAccordionData(category, toolName, part.input, output ?? {}): null;return (diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.jsonindex 44ddcac4b1..d2f1b8413d 100644--- a/autogpt_platform/frontend/src/app/api/openapi.json+++ b/autogpt_platform/frontend/src/app/api/openapi.json@@ -11254,6 +11254,9 @@"operation_in_progress","input_validation_error","web_fetch",+ "browser_navigate",+ "browser_act",+ "browser_screenshot","bash_exec","feature_request_search","feature_request_created",