Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: add som screenshot in non vision browser (#2759)

Puzhen Zhang7480c48fec38
diff --git a/camel/agents/_types.py b/camel/agents/_types.py
index ad6881726..07a86f9cd 100644
--- a/camel/agents/_types.py
+++ b/camel/agents/_types.py
@@ -14,6 +14,7 @@
from typing import Any, Dict, List, Optional, Union
from openai import AsyncStream, Stream
+from openai.types.chat import ChatCompletionChunk
from pydantic import BaseModel, ConfigDict
from camel.messages import BaseMessage
@@ -32,8 +33,11 @@ class ModelResponse(BaseModel):
r"""The response from the model."""
model_config = ConfigDict(arbitrary_types_allowed=True)
-
- response: Union[ChatCompletion, Stream, AsyncStream]
+ response: Union[
+ ChatCompletion,
+ Stream[ChatCompletionChunk],
+ AsyncStream[ChatCompletionChunk],
+ ]
tool_call_requests: Optional[List[ToolCallRequest]]
output_messages: List[BaseMessage]
finish_reasons: List[str]
diff --git a/camel/agents/chat_agent.py b/camel/agents/chat_agent.py
index 5bc13d4f9..4ff43894b 100644
--- a/camel/agents/chat_agent.py
+++ b/camel/agents/chat_agent.py
@@ -85,6 +85,7 @@ from camel.utils import (
model_from_json_schema,
)
from camel.utils.commons import dependencies_required
+from camel.utils.tool_result import ToolResult
if TYPE_CHECKING:
from camel.terminators import ResponseTerminator
@@ -291,12 +292,17 @@ class ChatAgent(BaseAgent):
self.stop_event = stop_event
self.mask_tool_output = mask_tool_output
self._secure_result_store: Dict[str, Any] = {}
+ self._pending_images: List[str] = []
+ self._image_retry_count: Dict[str, int] = {}
+ # Store images to attach to next user message
self.pause_event = pause_event
def reset(self):
r"""Resets the :obj:`ChatAgent` to its initial state."""
self.terminated = False
self.init_messages()
+ self._pending_images = []
+ self._image_retry_count = {}
for terminator in self.response_terminators:
terminator.reset()
@@ -1140,6 +1146,16 @@ class ChatAgent(BaseAgent):
role_name="User", content=input_message
)
+ # Attach any pending images from previous tool calls
+ image_list = self._process_pending_images()
+ if image_list:
+ # Create new message with images attached
+ input_message = BaseMessage.make_user_message(
+ role_name="User",
+ content=input_message.content,
+ image_list=image_list,
+ )
+
# Add user input to memory
self.update_memory(input_message, OpenAIBackendRole.USER)
@@ -1297,6 +1313,16 @@ class ChatAgent(BaseAgent):
role_name="User", content=input_message
)
+ # Attach any pending images from previous tool calls
+ image_list = self._process_pending_images()
+ if image_list:
+ # Create new message with images attached
+ input_message = BaseMessage.make_user_message(
+ role_name="User",
+ content=input_message.content,
+ image_list=image_list,
+ )
+
self.update_memory(input_message, OpenAIBackendRole.USER)
tool_call_records: List[ToolCallingRecord] = []
@@ -1343,6 +1369,7 @@ class ChatAgent(BaseAgent):
if tool_call_requests := response.tool_call_requests:
# Process all tool calls
+ new_images_from_tools = []
for tool_call_request in tool_call_requests:
if (
tool_call_request.tool_name
@@ -1362,10 +1389,72 @@ class ChatAgent(BaseAgent):
)
tool_call_records.append(tool_call_record)
+ # Check if this tool call produced images
+ if (
+ hasattr(tool_call_record, 'images')
+ and tool_call_record.images
+ ):
+ new_images_from_tools.extend(
+ tool_call_record.images
+ )
+
# If we found an external tool call, break the loop
if external_tool_call_requests:
break
+ # If tools produced images
+ # send them to the model as a user message
+ if new_images_from_tools:
+ # Convert base64 images to PIL Images
+ image_list = []
+ for img_data in new_images_from_tools:
+ try:
+ import base64
+ import io
+
+ from PIL import Image
+
+ # Extract base64 data from data URL format
+ if img_data.startswith("data:image"):
+ # Format:
+ # "data:image/png;base64,iVBORw0KGgo..."
+ base64_data = img_data.split(',', 1)[1]
+ else:
+ # Raw base64 data
+ base64_data = img_data
+
+ # Decode and create PIL Image
+ image_bytes = base64.b64decode(base64_data)
+ pil_image = Image.open(io.BytesIO(image_bytes))
+ # Convert to ensure proper
+ # Image.Image type for compatibility
+ pil_image_tool_result: Image.Image = (
+ pil_image.convert('RGB')
+ )
+ image_list.append(pil_image_tool_result)
+
+ except Exception as e:
+ logger.warning(
+ f"Failed to convert "
+ f"base64 image to PIL for immediate use: {e}"
+ )
+ continue
+
+ # If we have valid images
+ # create a user message with images
+ if image_list:
+ # Create a user message with images
+ # to provide visual context immediately
+ image_message = BaseMessage.make_user_message(
+ role_name="User",
+ content="[Visual content from tool execution - please analyze and continue]", # noqa: E501
+ image_list=image_list,
+ )
+
+ self.update_memory(
+ image_message, OpenAIBackendRole.USER
+ )
+
if (
self.max_iteration is not None
and iteration_count >= self.max_iteration
@@ -1452,6 +1541,69 @@ class ChatAgent(BaseAgent):
info=info,
)
+ def _process_pending_images(self) -> List:
+ r"""Process pending images with retry logic and return PIL Image list.
+
+ Returns:
+ List: List of successfully converted PIL Images.
+ """
+ if not self._pending_images:
+ return []
+
+ image_list = []
+ successfully_processed = []
+ failed_images = []
+
+ for img_data in self._pending_images:
+ # Track retry count
+ retry_count = self._image_retry_count.get(img_data, 0)
+
+ # Remove images that have failed too many times (max 3 attempts)
+ if retry_count >= 3:
+ failed_images.append(img_data)
+ logger.warning(
+ f"Removing image after {retry_count} failed attempts"
+ )
+ continue
+
+ try:
+ import base64
+ import io
+
+ from PIL import Image
+
+ # Extract base64 data from data URL format
+ if img_data.startswith("data:image"):
+ # Format: "data:image/png;base64,iVBORw0KGgo..."
+ base64_data = img_data.split(',', 1)[1]
+ else:
+ # Raw base64 data
+ base64_data = img_data
+
+ # Decode and create PIL Image
+ image_bytes = base64.b64decode(base64_data)
+ pil_image = Image.open(io.BytesIO(image_bytes))
+ pil_image_converted: Image.Image = pil_image.convert('RGB')
+ image_list.append(pil_image_converted)
+ successfully_processed.append(img_data)
+
+ except Exception as e:
+ # Increment retry count for failed conversion
+ self._image_retry_count[img_data] = retry_count + 1
+ logger.warning(
+ f"Failed to convert base64 image to PIL "
+ f"(attempt {retry_count + 1}/3): {e}"
+ )
+ continue
+
+ # Clean up processed and failed images
+ for img in successfully_processed + failed_images:
+ self._pending_images.remove(img)
+ # Clean up retry count for processed/removed images
+ self._image_retry_count.pop(img, None)
+
+ return image_list
+
def _record_final_output(self, output_messages: List[BaseMessage]) -> None:
r"""Log final messages or warnings about multiple responses."""
if len(output_messages) == 1:
@@ -1462,6 +1614,61 @@ class ChatAgent(BaseAgent):
"selected message manually using `record_message()`."
)
+ def _is_vision_error(self, exc: Exception) -> bool:
+ r"""Check if the exception is likely related to vision/image is not
+ supported by the model."""
+ # TODO: more robust vision error detection
+ error_msg = str(exc).lower()
+ vision_keywords = [
+ 'vision',
+ 'image',
+ 'multimodal',
+ 'unsupported',
+ 'invalid content type',
+ 'image_url',
+ 'visual',
+ ]
+ return any(keyword in error_msg for keyword in vision_keywords)
+
+ def _has_images(self, messages: List[OpenAIMessage]) -> bool:
+ r"""Check if any message contains images."""
+ for msg in messages:
+ content = msg.get('content')
+ if isinstance(content, list):
+ for item in content:
+ if (
+ isinstance(item, dict)
+ and item.get('type') == 'image_url'
+ ):
+ return True
+ return False
+
+ def _strip_images_from_messages(
+ self, messages: List[OpenAIMessage]
+ ) -> List[OpenAIMessage]:
+ r"""Remove images from messages, keeping only text content."""
+ stripped_messages = []
+ for msg in messages:
+ content = msg.get('content')
+ if isinstance(content, list):
+ # Extract only text content from multimodal messages
+ text_content = ""
+ for item in content:
+ if isinstance(item, dict) and item.get('type') == 'text':
+ text_content += item.get('text', '')
+
+ # Create new message with only text content
+ new_msg = msg.copy()
+ new_msg['content'] = (
+ text_content
+ or "[Image content removed - model doesn't support vision]"
+ )
+ stripped_messages.append(new_msg)
+ else:
+ # Regular text message, keep as is
+ stripped_messages.append(msg)
+ return stripped_messages
+
def _get_model_response(
self,
openai_messages: List[OpenAIMessage],
@@ -1477,13 +1684,33 @@ class ChatAgent(BaseAgent):
openai_messages, response_format, tool_schemas or None
)
except Exception as exc:
- logger.error(
- f"An error occurred while running model "
- f"{self.model_backend.model_type}, "
- f"index: {self.model_backend.current_model_index}",
- exc_info=exc,
- )
- error_info = str(exc)
+ # Try again without images if the error might be vision-related
+ if self._is_vision_error(exc) and self._has_images(
+ openai_messages
+ ):
+ logger.warning(
+ "Model appears to not support vision. Retrying without images." # noqa: E501
+ )
+ try:
+ stripped_messages = self._strip_images_from_messages(
+ openai_messages
+ )
+ response = self.model_backend.run(
+ stripped_messages,
+ response_format,
+ tool_schemas or None,
+ )
+ except Exception:
+ pass # Fall through to original error handling
+
+ if not response:
+ logger.error(
+ f"An error occurred while running model "
+ f"{self.model_backend.model_type}, "
+ f"index: {self.model_backend.current_model_index}",
+ exc_info=exc,
+ )
+ error_info = str(exc)
if not response and self.model_backend.num_models > 1:
raise ModelProcessingError(
@@ -1525,13 +1752,33 @@ class ChatAgent(BaseAgent):
openai_messages, response_format, tool_schemas or None
)
except Exception as exc:
- logger.error(
- f"An error occurred while running model "
- f"{self.model_backend.model_type}, "
- f"index: {self.model_backend.current_model_index}",
- exc_info=exc,
- )
- error_info = str(exc)
+ # Try again without images if the error might be vision-related
+ if self._is_vision_error(exc) and self._has_images(
+ openai_messages
+ ):
+ logger.warning(
+ "Model appears to not support vision. Retrying without images." # noqa: E501
+ )
+ try:
+ stripped_messages = self._strip_images_from_messages(
+ openai_messages
+ )
+ response = await self.model_backend.arun(
+ stripped_messages,
+ response_format,
+ tool_schemas or None,
+ )
+ except Exception:
+ pass # Fall through to original error handling
+
+ if not response:
+ logger.error(
+ f"An error occurred while running model "
+ f"{self.model_backend.model_type}, "
+ f"index: {self.model_backend.current_model_index}",
+ exc_info=exc,
+ )
+ error_info = str(exc)
if not response and self.model_backend.num_models > 1:
raise ModelProcessingError(
@@ -2005,10 +2252,26 @@ class ChatAgent(BaseAgent):
mask_flag = False
logging.warning(error_msg)
- return self._record_tool_calling(
+ # Check if result is a ToolResult with images
+ images_to_attach = None
+ if isinstance(result, ToolResult):
+ images_to_attach = result.images
+ result = str(result) # Use string representation for storage
+
+ tool_record = self._record_tool_calling(
func_name, args, result, tool_call_id, mask_output=mask_flag
)
+ # Store images for later attachment to next user message
+ if images_to_attach:
+ tool_record.images = images_to_attach
+ # Add images with duplicate prevention
+ for img in images_to_attach:
+ if img not in self._pending_images:
+ self._pending_images.append(img)
+
+ return tool_record
+
async def _aexecute_tool(
self,
tool_call_request: ToolCallRequest,
@@ -2049,7 +2312,25 @@ class ChatAgent(BaseAgent):
result = {"error": error_msg}
logging.warning(error_msg)
- return self._record_tool_calling(func_name, args, result, tool_call_id)
+ # Check if result is a ToolResult with images
+ images_to_attach = None
+ if isinstance(result, ToolResult):
+ images_to_attach = result.images
+ result = str(result) # Use string representation for storage
+
+ tool_record = self._record_tool_calling(
+ func_name, args, result, tool_call_id
+ )
+
+ # Store images for later attachment to next user message
+ if images_to_attach:
+ tool_record.images = images_to_attach
+ # Add images with duplicate prevention
+ for img in images_to_attach:
+ if img not in self._pending_images:
+ self._pending_images.append(img)
+
+ return tool_record
def _record_tool_calling(
self,
diff --git a/camel/messages/base.py b/camel/messages/base.py
index 9dd379831..3c78efde8 100644
--- a/camel/messages/base.py
+++ b/camel/messages/base.py
@@ -437,12 +437,8 @@ class BaseMessage:
if self.image_list and len(self.image_list) > 0:
for image in self.image_list:
if image.format is None:
- raise ValueError(
- f"Image's `format` is `None`, please "
- f"transform the `PIL.Image.Image` to one of "
- f"following supported formats, such as "
- f"{list(OpenAIImageType)}"
- )
+ # Set default format to PNG as fallback
+ image.format = 'PNG'
image_type: str = image.format.lower()
if image_type not in OpenAIImageType:
diff --git a/camel/toolkits/__init__.py b/camel/toolkits/__init__.py
index 3fdb08cc6..f4f797123 100644
--- a/camel/toolkits/__init__.py
+++ b/camel/toolkits/__init__.py
@@ -77,7 +77,7 @@ from .aci_toolkit import ACIToolkit
from .playwright_mcp_toolkit import PlaywrightMCPToolkit
from .wolfram_alpha_toolkit import WolframAlphaToolkit
from .task_planning_toolkit import TaskPlanningToolkit
-from .non_visual_browser_toolkit import BrowserNonVisualToolkit
+from .hybrid_browser_toolkit import HybridBrowserToolkit
from .edgeone_pages_mcp_toolkit import EdgeOnePagesMCPToolkit
from .google_drive_mcp_toolkit import GoogleDriveMCPToolkit
from .craw4ai_toolkit import Crawl4AIToolkit
@@ -146,7 +146,7 @@ __all__ = [
'WolframAlphaToolkit',
'BohriumToolkit',
'TaskPlanningToolkit',
- 'BrowserNonVisualToolkit',
+ 'HybridBrowserToolkit',
'EdgeOnePagesMCPToolkit',
'GoogleDriveMCPToolkit',
'Crawl4AIToolkit',
diff --git a/camel/toolkits/non_visual_browser_toolkit/__init__.py b/camel/toolkits/hybrid_browser_toolkit/__init__.py
similarity index 88%
rename from camel/toolkits/non_visual_browser_toolkit/__init__.py
rename to camel/toolkits/hybrid_browser_toolkit/__init__.py
index 0fff38941..b3e336c9e 100644
--- a/camel/toolkits/non_visual_browser_toolkit/__init__.py
+++ b/camel/toolkits/hybrid_browser_toolkit/__init__.py
@@ -11,8 +11,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-from .browser_non_visual_toolkit import BrowserNonVisualToolkit
+from .hybrid_browser_toolkit import HybridBrowserToolkit
__all__ = [
- "BrowserNonVisualToolkit",
+ "HybridBrowserToolkit",
]
diff --git a/camel/toolkits/non_visual_browser_toolkit/actions.py b/camel/toolkits/hybrid_browser_toolkit/actions.py
similarity index 76%
rename from camel/toolkits/non_visual_browser_toolkit/actions.py
rename to camel/toolkits/hybrid_browser_toolkit/actions.py
index 4c5cb3799..1054ccfa7 100644
--- a/camel/toolkits/non_visual_browser_toolkit/actions.py
+++ b/camel/toolkits/hybrid_browser_toolkit/actions.py
@@ -24,6 +24,7 @@ class ActionExecutor:
# Configuration constants
DEFAULT_TIMEOUT = 5000 # 5 seconds
SHORT_TIMEOUT = 2000 # 2 seconds
+ MAX_SCROLL_AMOUNT = 5000 # Maximum scroll distance in pixels
def __init__(self, page: "Page"):
self.page = page
@@ -32,6 +33,7 @@ class ActionExecutor:
# Public helpers
# ------------------------------------------------------------------
async def execute(self, action: Dict[str, Any]) -> str:
+ r"""Execute an action and return the result description."""
if not action:
return "No action to execute"
@@ -64,32 +66,46 @@ class ActionExecutor:
# Internal handlers
# ------------------------------------------------------------------
async def _click(self, action: Dict[str, Any]) -> str:
+ r"""Handle click actions with multiple fallback strategies."""
ref = action.get("ref")
text = action.get("text")
selector = action.get("selector")
if not (ref or text or selector):
return "Error: click requires ref/text/selector"
+ # Build strategies in priority order: ref > selector > text
strategies = []
+ if ref:
+ strategies.append(f"[aria-ref='{ref}']")
if selector:
strategies.append(selector)
if text:
strategies.append(f'text="{text}"')
- if ref:
- strategies.append(f"[aria-ref='{ref}']")
+ # Strategy 1: Try Playwright force click for each selector
for sel in strategies:
try:
if await self.page.locator(sel).count() > 0:
await self.page.click(
- sel, timeout=self.SHORT_TIMEOUT, force=True
+ sel, timeout=self.DEFAULT_TIMEOUT, force=True
)
- return f"Clicked element via {sel}"
+ return f"Clicked element via force: {sel}"
except Exception:
- pass
- return "Error: Could not click element"
+ continue
+
+ # Strategy 2: Try JavaScript click as fallback
+ for sel in strategies:
+ try:
+ await self.page.locator(sel).first.evaluate("el => el.click()")
+ await asyncio.sleep(0.1) # Brief wait for effects
+ return f"Clicked element via JS: {sel}"
+ except Exception:
+ continue
+
+ return "Error: All click strategies failed"
async def _type(self, action: Dict[str, Any]) -> str:
+ r"""Handle typing text into input fields."""
ref = action.get("ref")
selector = action.get("selector")
text = action.get("text", "")
@@ -103,6 +119,7 @@ class ActionExecutor:
return f"Type failed: {exc}"
async def _select(self, action: Dict[str, Any]) -> str:
+ r"""Handle selecting options from dropdowns."""
ref = action.get("ref")
selector = action.get("selector")
value = action.get("value", "")
@@ -118,8 +135,9 @@ class ActionExecutor:
return f"Select failed: {exc}"
async def _wait(self, action: Dict[str, Any]) -> str:
+ r"""Handle wait actions."""
if "timeout" in action:
- ms = action["timeout"]
+ ms = int(action["timeout"])
await asyncio.sleep(ms / 1000)
return f"Waited {ms}ms"
if "selector" in action:
@@ -131,6 +149,7 @@ class ActionExecutor:
return "Error: wait requires timeout/selector"
async def _extract(self, action: Dict[str, Any]) -> str:
+ r"""Handle text extraction from elements."""
ref = action.get("ref")
if not ref:
return "Error: extract requires ref"
@@ -140,6 +159,7 @@ class ActionExecutor:
return f"Extracted: {txt[:100] if txt else 'None'}"
async def _scroll(self, action: Dict[str, Any]) -> str:
+ r"""Handle page scrolling with safe parameter validation."""
direction = action.get("direction", "down")
amount = action.get("amount", 300)
@@ -151,18 +171,22 @@ class ActionExecutor:
# Safely convert amount to integer and clamp to reasonable range
amount_int = int(amount)
amount_int = max(
- -5000, min(5000, amount_int)
- ) # Clamp between -5000 and 5000
+ -self.MAX_SCROLL_AMOUNT,
+ min(self.MAX_SCROLL_AMOUNT, amount_int),
+ ) # Clamp to MAX_SCROLL_AMOUNT range
except (ValueError, TypeError):
return "Error: amount must be a valid number"
# Use safe evaluation with bound parameters
scroll_offset = amount_int if direction == "down" else -amount_int
- await self.page.evaluate(f"window.scrollBy(0, {scroll_offset})")
+ await self.page.evaluate(
+ "offset => window.scrollBy(0, offset)", scroll_offset
+ )
await asyncio.sleep(0.5)
return f"Scrolled {direction} by {abs(amount_int)}px"
async def _enter(self, action: Dict[str, Any]) -> str:
+ r"""Handle Enter key press actions."""
ref = action.get("ref")
selector = action.get("selector")
if ref:
@@ -175,16 +199,28 @@ class ActionExecutor:
# utilities
async def _wait_dom_stable(self) -> None:
+ r"""Wait for DOM to become stable before executing actions."""
try:
+ # Wait for basic DOM content loading
await self.page.wait_for_load_state(
'domcontentloaded', timeout=self.SHORT_TIMEOUT
)
+
+ # Try to wait for network idle briefly
+ try:
+ await self.page.wait_for_load_state(
+ 'networkidle', timeout=self.SHORT_TIMEOUT
+ )
+ except Exception:
+ pass # Network idle is optional
+
except Exception:
- pass
+ pass # Don't fail if wait times out
# static helpers
@staticmethod
def should_update_snapshot(action: Dict[str, Any]) -> bool:
+ r"""Determine if an action requires a snapshot update."""
change_types = {
"click",
"type",
diff --git a/camel/toolkits/non_visual_browser_toolkit/agent.py b/camel/toolkits/hybrid_browser_toolkit/agent.py
similarity index 89%
rename from camel/toolkits/non_visual_browser_toolkit/agent.py
rename to camel/toolkits/hybrid_browser_toolkit/agent.py
index 1e166b0cf..c7b5bb68a 100644
--- a/camel/toolkits/non_visual_browser_toolkit/agent.py
+++ b/camel/toolkits/hybrid_browser_toolkit/agent.py
@@ -12,24 +12,24 @@
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
import json
-import logging
import re
from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from camel.logger import get_logger
from camel.models import BaseModelBackend, ModelFactory
from camel.types import ModelPlatformType, ModelType
from .actions import ActionExecutor
-from .nv_browser_session import NVBrowserSession
+from .browser_session import NVBrowserSession
if TYPE_CHECKING:
from camel.agents import ChatAgent
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
class PlaywrightLLMAgent:
- """High-level orchestration: snapshot ↔ LLM ↔ action executor."""
+ r"""High-level orchestration: snapshot ↔ LLM ↔ action executor."""
# System prompt as class constant to avoid recreation
SYSTEM_PROMPT = """
@@ -90,8 +90,8 @@ what was accomplished
self.action_history: List[Dict[str, Any]] = []
if model_backend is None:
model_backend = ModelFactory.create(
- model_platform=ModelPlatformType.OPENAI,
- model_type=ModelType.GPT_4O_MINI,
+ model_platform=ModelPlatformType.DEFAULT,
+ model_type=ModelType.DEFAULT,
model_config_dict={"temperature": 0, "top_p": 1},
)
self.model_backend = model_backend
@@ -99,16 +99,19 @@ what was accomplished
self._chat_agent: Optional[ChatAgent] = None
async def navigate(self, url: str) -> str:
+ r"""Navigate to a URL and return the snapshot."""
try:
# NVBrowserSession handles waits internally
logger.debug("Navigated to URL: %s", url)
await self._session.visit(url)
return await self._session.get_snapshot(force_refresh=True)
except Exception as exc:
- return f"Error: could not navigate - {exc}"
+ error_msg = f"Error: could not navigate to {url} - {exc}"
+ logger.error(error_msg)
+ return error_msg
def _get_chat_agent(self) -> "ChatAgent":
- """Get or create the ChatAgent instance."""
+ r"""Get or create the ChatAgent instance."""
from camel.agents import ChatAgent
if self._chat_agent is None:
@@ -165,12 +168,16 @@ what was accomplished
logger.warning(
"Could not parse JSON from LLM response: %s", content[:200]
)
+ return self._get_fallback_response("Parsing error")
+
+ def _get_fallback_response(self, error_msg: str) -> Dict[str, Any]:
+ r"""Generate a fallback response structure."""
return {
- "plan": ["Could not parse response"],
+ "plan": [f"Could not parse response: {error_msg}"],
"action": {
"type": "finish",
"ref": None,
- "summary": "Parsing error",
+ "summary": f"Parsing error: {error_msg}",
},
}
@@ -181,7 +188,7 @@ what was accomplished
is_initial: bool,
history: Optional[List[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
- """Call the LLM (via CAMEL ChatAgent) to get plan & next action."""
+ r"""Call the LLM (via CAMEL ChatAgent) to get plan & next action."""
# Build user message
if is_initial:
user_content = f"Snapshot:\n{snapshot}\n\nTask: {prompt}"
@@ -208,6 +215,7 @@ what was accomplished
return self._safe_parse_json(content)
async def process_command(self, prompt: str, max_steps: int = 15):
+ r"""Process a command using LLM-guided browser automation."""
# initial full snapshot
full_snapshot = await self._session.get_snapshot()
assert self._session.snapshot is not None
@@ -270,9 +278,11 @@ what was accomplished
logger.info("Process completed with %d steps", steps)
async def _run_action(self, action: Dict[str, Any]) -> str:
+ r"""Execute a single action and return the result."""
if action.get("type") == "navigate":
return await self.navigate(action.get("url", ""))
return await self._session.exec_action(action)
async def close(self):
+ r"""Clean up browser session and resources."""
await self._session.close()
diff --git a/camel/toolkits/non_visual_browser_toolkit/nv_browser_session.py b/camel/toolkits/hybrid_browser_toolkit/browser_session.py
similarity index 74%
rename from camel/toolkits/non_visual_browser_toolkit/nv_browser_session.py
rename to camel/toolkits/hybrid_browser_toolkit/browser_session.py
index 2b4e6c14c..cafcfaf3b 100644
--- a/camel/toolkits/non_visual_browser_toolkit/nv_browser_session.py
+++ b/camel/toolkits/hybrid_browser_toolkit/browser_session.py
@@ -57,13 +57,12 @@ class NVBrowserSession:
def __new__(
cls, *, headless: bool = True, user_data_dir: Optional[str] = None
- ):
- loop = asyncio.get_running_loop()
- if loop not in cls._sessions:
- instance = super().__new__(cls)
- instance._initialized = False
- cls._sessions[loop] = instance
- return cls._sessions[loop]
+ ) -> "NVBrowserSession":
+ # Defer event loop lookup until we actually need it
+ # This allows creation outside of async context
+ instance = super().__new__(cls)
+ instance._initialized = False
+ return instance
def __init__(
self, *, headless: bool = True, user_data_dir: Optional[str] = None
@@ -90,6 +89,47 @@ class NVBrowserSession:
# Browser lifecycle helpers
# ------------------------------------------------------------------
async def ensure_browser(self) -> None:
+ r"""Ensure browser is ready, implementing singleton pattern per event
+ loop.
+ """
+ # Check if we need to reuse or create a session for this event loop
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError as e:
+ raise RuntimeError(
+ "ensure_browser() must be called from within an async context"
+ ) from e
+
+ # Check if there's already a session for this loop
+ if loop in self._sessions and self._sessions[loop] is not self:
+ # Copy the existing session's browser resources
+ existing = self._sessions[loop]
+ # Wait for existing session to be fully initialized
+ async with existing._ensure_lock:
+ if (
+ existing._initialized
+ and existing._page is not None
+ and existing._playwright is not None
+ ):
+ try:
+ # Verify the page is still responsive
+ await existing._page.title()
+ self._playwright = existing._playwright
+ self._browser = existing._browser
+ self._context = existing._context
+ self._page = existing._page
+ self.snapshot = existing.snapshot
+ self.executor = existing.executor
+ self._initialized = True
+ return
+ except Exception:
+ # Existing session is broken, continue with new
+ # initialization
+ pass
+
+ # Register this instance for the current loop
+ self._sessions[loop] = self
+
# Serialise initialisation to avoid race conditions where multiple
# concurrent coroutine calls create multiple browser instances for
# the same NVBrowserSession.
@@ -98,6 +138,7 @@ class NVBrowserSession:
# Moved original logic to helper
async def _ensure_browser_inner(self) -> None:
+ r"""Internal browser initialization logic."""
from playwright.async_api import async_playwright
if self._page is not None:
@@ -144,11 +185,23 @@ class NVBrowserSession:
r"""Close all browser resources, ensuring cleanup even if some
operations fail.
"""
- # The close method will now only close the *current* event-loop's
- # browser instance. Use `close_all_sessions` for a full cleanup.
+ # Remove this session from the sessions dict and close resources
+ try:
+ loop = asyncio.get_running_loop()
+ if loop in self._sessions and self._sessions[loop] is self:
+ del self._sessions[loop]
+ except RuntimeError:
+ pass # No running loop, that's okay
+
+ # Clean up any stale loop references
+ stale_loops = [loop for loop in self._sessions if loop.is_closed()]
+ for loop in stale_loops:
+ del self._sessions[loop]
+
await self._close_session()
async def _close_session(self) -> None:
+ r"""Internal session cleanup with comprehensive error handling."""
errors: list[str] = []
# Close context first (which closes pages)
@@ -204,6 +257,7 @@ class NVBrowserSession:
# Convenience wrappers around common actions
# ------------------------------------------------------------------
async def visit(self, url: str) -> str:
+ r"""Navigate to a URL with proper error handling."""
await self.ensure_browser()
assert self._page is not None
@@ -233,7 +287,7 @@ class NVBrowserSession:
force_refresh=force_refresh, diff_only=diff_only
)
- async def exec_action(self, action: dict[str, Any]) -> str:
+ async def exec_action(self, action: Dict[str, Any]) -> str:
await self.ensure_browser()
assert self.executor is not None
return await self.executor.execute(action)
diff --git a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py
new file mode 100644
index 000000000..c27860bdf
--- /dev/null
+++ b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py
@@ -0,0 +1,1002 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import base64
+import datetime
+import io
+import os
+import time
+import urllib.parse
+from typing import Any, Callable, ClassVar, Dict, List, Optional, cast
+
+from camel.logger import get_logger
+from camel.models import BaseModelBackend
+from camel.toolkits.base import BaseToolkit
+from camel.toolkits.function_tool import FunctionTool
+from camel.utils import sanitize_filename
+from camel.utils.commons import dependencies_required
+
+from .agent import PlaywrightLLMAgent
+from .browser_session import NVBrowserSession
+
+logger = get_logger(__name__)
+
+
+class HybridBrowserToolkit(BaseToolkit):
+ r"""A hybrid browser toolkit that combines non-visual, DOM-based browser
+ automation with visual, screenshot-based capabilities.
+
+ This toolkit exposes a set of actions as CAMEL FunctionTools for agents
+ to interact with web pages. It can operate in headless mode and supports
+ both programmatic control of browser actions (like clicking and typing)
+ and visual analysis of the page layout through screenshots with marked
+ interactive elements.
+ """
+
+ # Configuration constants
+ DEFAULT_SCREENSHOT_TIMEOUT = 60000 # 60 seconds for screenshots
+ PAGE_STABILITY_TIMEOUT = 3000 # 3 seconds for DOM stability
+ NETWORK_IDLE_TIMEOUT = 2000 # 2 seconds for network idle
+
+ # Default tool list - core browser functionality
+ DEFAULT_TOOLS: ClassVar[List[str]] = [
+ "open_browser",
+ "close_browser",
+ "visit_page",
+ "click",
+ "type",
+ ]
+
+ # All available tools
+ ALL_TOOLS: ClassVar[List[str]] = [
+ "open_browser",
+ "close_browser",
+ "visit_page",
+ "get_page_snapshot",
+ "get_som_screenshot",
+ "get_page_links",
+ "click",
+ "type",
+ "select",
+ "scroll",
+ "enter",
+ "wait_user",
+ "solve_task",
+ ]
+
+ def __init__(
+ self,
+ *,
+ headless: bool = True,
+ user_data_dir: Optional[str] = None,
+ web_agent_model: Optional[BaseModelBackend] = None,
+ cache_dir: str = "tmp/",
+ enabled_tools: Optional[List[str]] = None,
+ ) -> None:
+ r"""Initialize the HybridBrowserToolkit.
+
+ Args:
+ headless (bool): Whether to run the browser in headless mode.
+ Defaults to `True`.
+ user_data_dir (Optional[str]): Path to a directory for storing
+ browser data like cookies and local storage. Useful for
+ maintaining sessions across runs. Defaults to `None` (a
+ temporary directory is used).
+ web_agent_model (Optional[BaseModelBackend]): The language model
+ backend to use for the high-level `solve_task` agent. This is
+ required only if you plan to use `solve_task`.
+ Defaults to `None`.
+ cache_dir (str): The directory to store cached files, such as
+ screenshots. Defaults to `"tmp/"`.
+ enabled_tools (Optional[List[str]]): List of tool names to enable.
+ If None, uses DEFAULT_TOOLS. Available tools: open_browser,
+ close_browser, visit_page, get_page_snapshot,
+ get_som_screenshot, get_page_links, click, type, select,
+ scroll, enter, wait_user, solve_task.
+ Defaults to `None`.
+ """
+ super().__init__()
+ self._headless = headless
+ self._user_data_dir = user_data_dir
+ self.web_agent_model = web_agent_model
+ self.cache_dir = cache_dir
+ os.makedirs(self.cache_dir, exist_ok=True)
+
+ # Configure enabled tools
+ if enabled_tools is None:
+ self.enabled_tools = self.DEFAULT_TOOLS.copy()
+ else:
+ # Validate enabled tools
+ invalid_tools = [
+ tool for tool in enabled_tools if tool not in self.ALL_TOOLS
+ ]
+ if invalid_tools:
+ raise ValueError(
+ f"Invalid tools specified: {invalid_tools}. "
+ f"Available tools: {self.ALL_TOOLS}"
+ )
+ self.enabled_tools = enabled_tools.copy()
+
+ logger.info(f"Enabled tools: {self.enabled_tools}")
+
+ # Core components
+ self._session = NVBrowserSession(
+ headless=headless, user_data_dir=user_data_dir
+ )
+ self._agent: Optional[PlaywrightLLMAgent] = None
+ self._unified_script = self._load_unified_analyzer()
+
+ def __del__(self):
+ r"""Cleanup browser resources on garbage collection."""
+ try:
+ import sys
+
+ if getattr(sys, "is_finalizing", lambda: False)():
+ return
+
+ import asyncio
+
+ try:
+ loop = asyncio.get_event_loop()
+ if not loop.is_closed() and not loop.is_running():
+ loop.run_until_complete(self.close_browser())
+ except (RuntimeError, ImportError):
+ pass # Event loop unavailable, skip cleanup
+ except Exception:
+ pass # Suppress all errors during garbage collection
+
+ def _load_unified_analyzer(self) -> str:
+ r"""Load the unified analyzer JavaScript script."""
+ script_path = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), "unified_analyzer.js"
+ )
+
+ try:
+ with open(
+ script_path, "r", encoding='utf-8', errors='replace'
+ ) as f:
+ script_content = f.read()
+
+ if not script_content.strip():
+ raise ValueError(f"Script is empty: {script_path}")
+
+ logger.debug(
+ f"Loaded unified analyzer ({len(script_content)} chars)"
+ )
+ return script_content
+ except FileNotFoundError:
+ raise FileNotFoundError(f"Script not found: {script_path}")
+
+ def _validate_ref(self, ref: str, method_name: str) -> None:
+ r"""Validate ref parameter."""
+ if not ref or not isinstance(ref, str):
+ raise ValueError(
+ f"{method_name}: 'ref' must be a non-empty string"
+ )
+
+ async def _ensure_browser(self):
+ await self._session.ensure_browser()
+
+ async def _require_page(self):
+ await self._session.ensure_browser()
+ return await self._session.get_page()
+
+ async def _wait_for_page_stability(self):
+ r"""Wait for page to become stable after actions that might trigger
+ updates.
+ """
+ page = await self._require_page()
+ import asyncio
+
+ try:
+ # Wait for DOM content to be loaded
+ await page.wait_for_load_state(
+ 'domcontentloaded', timeout=self.PAGE_STABILITY_TIMEOUT
+ )
+ logger.debug("DOM content loaded")
+
+ # Try to wait for network idle (important for AJAX/SPA)
+ try:
+ await page.wait_for_load_state(
+ 'networkidle', timeout=self.NETWORK_IDLE_TIMEOUT
+ )
+ logger.debug("Network idle achieved")
+ except Exception:
+ logger.debug("Network idle timeout - continuing anyway")
+
+ # Additional small delay for JavaScript execution
+ await asyncio.sleep(0.5)
+ logger.debug("Page stability wait completed")
+
+ except Exception as e:
+ logger.debug(
+ f"Page stability wait failed: {e} - continuing anyway"
+ )
+
+ async def _get_unified_analysis(self) -> Dict[str, Any]:
+ r"""Get unified analysis data from the page."""
+ page = await self._require_page()
+ try:
+ if not self._unified_script:
+ logger.error("Unified analyzer script not loaded")
+ return {"elements": {}, "metadata": {"elementCount": 0}}
+
+ result = await page.evaluate(self._unified_script)
+
+ if not isinstance(result, dict):
+ logger.warning(f"Invalid result type: {type(result)}")
+ return {"elements": {}, "metadata": {"elementCount": 0}}
+
+ return result
+ except Exception as e:
+ logger.warning(f"Error in unified analysis: {e}")
+ return {"elements": {}, "metadata": {"elementCount": 0}}
+
+ def _convert_analysis_to_rects(
+ self, analysis_data: Dict[str, Any]
+ ) -> Dict[str, Any]:
+ r"""Convert analysis data to rect format for visual marking."""
+ rects = {}
+ elements = analysis_data.get("elements", {})
+
+ for ref, element_data in elements.items():
+ coordinates = element_data.get("coordinates", [])
+ if coordinates:
+ rects[ref] = {
+ "role": element_data.get("role", "generic"),
+ "aria-name": element_data.get("name", ""),
+ "rects": [coordinates[0]],
+ }
+ return rects
+
+ def _add_set_of_mark(self, image, rects):
+ r"""Add visual marks to the image."""
+ try:
+ from PIL import ImageDraw, ImageFont
+ except ImportError:
+ logger.warning("PIL not available, returning original image")
+ return image
+
+ marked_image = image.copy()
+ draw = ImageDraw.Draw(marked_image)
+
+ # Try to get font
+ try:
+ font = ImageFont.truetype("arial.ttf", 16)
+ except (OSError, IOError):
+ try:
+ font = ImageFont.load_default()
+ except (OSError, IOError):
+ font = None
+
+ # Color scheme
+ colors = {
+ "button": "#FF6B6B",
+ "link": "#4ECDC4",
+ "textbox": "#45B7D1",
+ "select": "#96CEB4",
+ "checkbox": "#FECA57",
+ "radio": "#FF9FF3",
+ "default": "#DDA0DD",
+ }
+
+ for ref, rect_data in rects.items():
+ rects_list = rect_data.get("rects", [])
+ role = rect_data.get("role", "generic")
+ color = colors.get(role, colors["default"])
+
+ for rect in rects_list:
+ x, y = rect.get("x", 0), rect.get("y", 0)
+ width, height = rect.get("width", 0), rect.get("height", 0)
+
+ # Draw rectangle outline
+ draw.rectangle(
+ [x, y, x + width, y + height], outline=color, width=2
+ )
+
+ # Draw reference label
+ label_text = ref
+ if font:
+ bbox = draw.textbbox((0, 0), label_text, font=font)
+ text_width, text_height = (
+ bbox[2] - bbox[0],
+ bbox[3] - bbox[1],
+ )
+ else:
+ text_width, text_height = len(label_text) * 8, 16
+
+ label_x, label_y = max(0, x - 2), max(0, y - text_height - 2)
+
+ # Background and text
+ draw.rectangle(
+ [
+ label_x,
+ label_y,
+ label_x + text_width + 4,
+ label_y + text_height + 2,
+ ],
+ fill=color,
+ )
+ draw.text(
+ (label_x + 2, label_y + 1),
+ label_text,
+ fill="white",
+ font=font,
+ )
+
+ return marked_image
+
+ def _format_snapshot_from_analysis(
+ self, analysis_data: Dict[str, Any]
+ ) -> str:
+ r"""Format analysis data into snapshot string."""
+ lines = []
+ elements = analysis_data.get("elements", {})
+
+ for ref, element_data in elements.items():
+ role = element_data.get("role", "generic")
+ name = element_data.get("name", "")
+
+ line = f"- {role}"
+ if name:
+ line += f' "{name}"'
+
+ # Add properties
+ props = []
+ for prop in ["disabled", "checked", "expanded"]:
+ value = element_data.get(prop)
+ if value is True:
+ props.append(prop)
+ elif value is not None and prop in ["checked", "expanded"]:
+ props.append(f"{prop}={value}")
+
+ if props:
+ line += f" {' '.join(props)}"
+
+ line += f" [ref={ref}]"
+ lines.append(line)
+
+ return "\n".join(lines)
+
+ async def _exec_with_snapshot(
+ self, action: Dict[str, Any]
+ ) -> Dict[str, str]:
+ r"""Execute action and return result with snapshot comparison."""
+
+ # Log action execution start
+ action_type = action.get("type", "unknown")
+ logger.info(f"Executing action: {action_type}")
+
+ # Get before snapshot
+ logger.info("Capturing pre-action snapshot...")
+ snapshot_start = time.time()
+ before_snapshot = await self._session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ snapshot_time = time.time() - snapshot_start
+ logger.info(f"Pre-action snapshot captured in {snapshot_time:.2f}s")
+
+ # Execute action
+ logger.info(f"Executing {action_type} action...")
+ action_start = time.time()
+ result = await self._session.exec_action(action)
+ action_time = time.time() - action_start
+ logger.info(f"Action {action_type} completed in {action_time:.2f}s")
+
+ # Wait for page stability after action (especially important for click)
+ if action_type in ["click", "type", "select", "enter"]:
+ logger.info(
+ f"Waiting for page stability " f"after {action_type}..."
+ )
+ stability_start = time.time()
+ await self._wait_for_page_stability()
+ stability_time = time.time() - stability_start
+ logger.info(
+ f"Page stability wait " f"completed in {stability_time:.2f}s"
+ )
+
+ # Get after snapshot
+ logger.info("Capturing post-action snapshot...")
+ snapshot_start = time.time()
+ after_snapshot = await self._session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ snapshot_time = time.time() - snapshot_start
+ logger.info(
+ f"Post-action snapshot " f"captured in {snapshot_time:.2f}s"
+ )
+
+ # Check for snapshot quality and log warnings
+ if before_snapshot == after_snapshot:
+ snapshot = "snapshot not changed"
+ logger.debug("Page snapshot unchanged after action")
+ else:
+ snapshot = after_snapshot
+ # Check if snapshot is empty or problematic
+ if "<empty>" in after_snapshot:
+ logger.warning(
+ f"Action {action_type} resulted "
+ f"in empty snapshot - "
+ f"page may still be loading"
+ )
+ elif len(after_snapshot.strip()) < 50:
+ logger.warning(
+ f"Action {action_type} resulted "
+ f"in very short snapshot:"
+ f" {len(after_snapshot)} chars"
+ )
+ else:
+ logger.debug(
+ f"Action {action_type} resulted "
+ f"in updated snapshot: "
+ f"{len(after_snapshot)} chars"
+ )
+
+ return {"result": result, "snapshot": snapshot}
+
+ async def _extract_links_by_refs(
+ self, snapshot: str, page, refs: List[str]
+ ) -> List[Dict[str, str]]:
+ r"""Extract multiple links by their reference IDs."""
+ import re
+
+ found_links = []
+ ref_set = set(refs)
+ lines = snapshot.split('\n')
+
+ for line in lines:
+ link_match = re.search(
+ r'- link\s+"([^"]+)"\s+\[ref=([^\]]+)\]', line
+ )
+ if link_match and link_match.group(2) in ref_set:
+ text, found_ref = link_match.groups()
+ try:
+ url = await self._get_link_url_by_ref(page, found_ref)
+ found_links.append(
+ {"text": text, "ref": found_ref, "url": url or ""}
+ )
+ except Exception as e:
+ logger.warning(
+ f"Failed to get URL for ref {found_ref}: {e}"
+ )
+ found_links.append(
+ {"text": text, "ref": found_ref, "url": ""}
+ )
+
+ return found_links
+
+ async def _get_link_url_by_ref(self, page, ref: str) -> str:
+ r"""Get URL of a link element by reference ID."""
+ try:
+ element = await page.query_selector(f'[aria-ref="{ref}"]')
+ if element:
+ href = await element.get_attribute('href')
+ if href:
+ from urllib.parse import urljoin
+
+ return urljoin(page.url, href)
+ return ""
+ except Exception as e:
+ logger.warning(f"Failed to get URL for ref {ref}: {e}")
+ return ""
+
+ def _ensure_agent(self) -> PlaywrightLLMAgent:
+ r"""Create PlaywrightLLMAgent on first use."""
+ if self.web_agent_model is None:
+ raise RuntimeError(
+ "web_agent_model required for high-level task planning"
+ )
+
+ if self._agent is None:
+ self._agent = PlaywrightLLMAgent(
+ headless=self._headless,
+ user_data_dir=self._user_data_dir,
+ model_backend=self.web_agent_model,
+ )
+ return self._agent
+
+ # Public API Methods
+
+ async def open_browser(
+ self, start_url: Optional[str] = None
+ ) -> Dict[str, str]:
+ r"""Launches a new browser session, making it ready for web automation.
+
+ This method initializes the underlying browser instance. If a
+ `start_url` is provided, it will also navigate to that URL.
+
+ Args:
+ start_url (Optional[str]): The initial URL to navigate to after the
+ browser is launched. If not provided, the browser will start
+ with a blank page.
+
+ Returns:
+ Dict[str, str]: A dictionary containing:
+ - "result": A string confirming that the browser session has
+ started.
+ - "snapshot": A textual representation of the current page's
+ interactive elements. This snapshot is crucial for
+ identifying elements for subsequent actions.
+ """
+ logger.info("Starting browser session...")
+
+ browser_start = time.time()
+ await self._session.ensure_browser()
+ browser_time = time.time() - browser_start
+ logger.info(f"Browser session started in {browser_time:.2f}s")
+
+ if start_url:
+ logger.info(f"Auto-navigating to start URL: {start_url}")
+ return await self.visit_page(start_url)
+
+ logger.info("Capturing initial browser snapshot...")
+ snapshot_start = time.time()
+ snapshot = await self._session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ snapshot_time = time.time() - snapshot_start
+ logger.info(f"Initial snapshot captured in {snapshot_time:.2f}s")
+
+ return {"result": "Browser session started.", "snapshot": snapshot}
+
+ async def close_browser(self) -> str:
+ r"""Closes the current browser session and releases all associated
+ resources.
+
+ This should be called at the end of a web automation task to ensure a
+ clean shutdown of the browser instance.
+
+ Returns:
+ str: A confirmation message indicating the session is closed.
+ """
+ if self._agent is not None:
+ try:
+ await self._agent.close()
+ except Exception:
+ pass
+ self._agent = None
+
+ await self._session.close()
+ return "Browser session closed."
+
+ async def visit_page(self, url: str) -> Dict[str, str]:
+ r"""Navigates the current browser page to a specified URL.
+
+ Args:
+ url (str): The web address to load in the browser. Must be a
+ valid URL.
+
+ Returns:
+ Dict[str, str]: A dictionary containing:
+ - "result": A message indicating the outcome of the navigation,
+ e.g., "Navigation successful.".
+ - "snapshot": A new textual snapshot of the page's interactive
+ elements after the new page has loaded.
+ """
+ if not url or not isinstance(url, str):
+ return {
+ "result": "Error: 'url' must be a non-empty string",
+ "snapshot": "",
+ }
+
+ logger.info(f"Navigating to URL: {url}")
+
+ # Navigate to page
+ nav_start = time.time()
+ nav_result = await self._session.visit(url)
+ nav_time = time.time() - nav_start
+ logger.info(f"Page navigation completed in {nav_time:.2f}s")
+
+ # Get snapshot
+ logger.info("Capturing page snapshot after navigation...")
+ snapshot_start = time.time()
+ snapshot = await self._session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ snapshot_time = time.time() - snapshot_start
+ logger.info(f"Navigation snapshot captured in {snapshot_time:.2f}s")
+
+ return {"result": nav_result, "snapshot": snapshot}
+
+ async def get_page_snapshot(self) -> str:
+ r"""Captures a textual representation of the current page's content.
+
+ This "snapshot" provides a simplified view of the DOM, focusing on
+ interactive elements like links, buttons, and input fields. Each
+ element is assigned a unique reference ID (`ref`) that can be used in
+ other actions like `click` or `type`.
+
+ The snapshot is useful for understanding the page structure and
+ identifying elements to interact with without needing to parse raw
+ HTML. A new snapshot is generated on each call.
+
+ Returns:
+ str: A formatted string representing the interactive elements on
+ the page. For example:
+ '- link "Sign In" [ref=1]'
+ '- textbox "Username" [ref=2]'
+ """
+ logger.info("Capturing page snapshot")
+
+ analysis_start = time.time()
+ analysis_data = await self._get_unified_analysis()
+ analysis_time = time.time() - analysis_start
+ logger.info(
+ f"Page snapshot analysis " f"completed in {analysis_time:.2f}s"
+ )
+
+ snapshot_text = analysis_data.get("snapshotText", "")
+ return (
+ snapshot_text
+ if snapshot_text
+ else self._format_snapshot_from_analysis(analysis_data)
+ )
+
+ @dependencies_required('PIL')
+ async def get_som_screenshot(self):
+ r"""Captures a screenshot of the current webpage and visually marks all
+ interactive elements. "SoM" stands for "Set of Marks".
+
+ This method is essential for tasks requiring visual understanding of
+ the page layout. It works by:
+ 1. Taking a full-page screenshot.
+ 2. Identifying all interactive elements (buttons, links, inputs, etc.).
+ 3. Drawing colored boxes and reference IDs (`ref`) over these elements
+ on the screenshot.
+ 4. Saving the annotated image to a cache directory.
+ 5. Returning the image as a base64-encoded string along with a summary.
+
+ Use this when the textual snapshot from `get_page_snapshot` is
+ insufficient and visual context is needed to decide the next action.
+
+ Returns:
+ ToolResult: An object containing:
+ - `text`: A summary string, e.g., "Visual webpage screenshot
+ captured with 42 interactive elements".
+ - `images`: A list containing a single base64-encoded PNG image
+ as a data URL.
+ """
+ from PIL import Image
+
+ from camel.utils.tool_result import ToolResult
+
+ # Get screenshot and analysis
+ page = await self._require_page()
+
+ # Log screenshot timeout start
+ logger.info(
+ f"Starting screenshot capture"
+ f"with timeout: {self.DEFAULT_SCREENSHOT_TIMEOUT}ms"
+ )
+
+ start_time = time.time()
+ image_data = await page.screenshot(
+ timeout=self.DEFAULT_SCREENSHOT_TIMEOUT
+ )
+ screenshot_time = time.time() - start_time
+
+ logger.info(f"Screenshot capture completed in {screenshot_time:.2f}s")
+ image = Image.open(io.BytesIO(image_data))
+
+ # Log unified analysis start
+ logger.info("Starting unified page analysis...")
+ analysis_start_time = time.time()
+ analysis_data = await self._get_unified_analysis()
+ analysis_time = time.time() - analysis_start_time
+ logger.info(f"Unified page analysis completed in {analysis_time:.2f}s")
+
+ # Log image processing
+ logger.info("Processing visual marks on screenshot...")
+ mark_start_time = time.time()
+ rects = self._convert_analysis_to_rects(analysis_data)
+ marked_image = self._add_set_of_mark(image, rects)
+ mark_time = time.time() - mark_start_time
+ logger.info(f"Visual marks processing completed in {mark_time:.2f}s")
+
+ # Save screenshot to cache directory
+ parsed_url = urllib.parse.urlparse(page.url)
+ url_name = sanitize_filename(str(parsed_url.path), max_length=241)
+ timestamp = datetime.datetime.now().strftime("%m%d%H%M%S")
+ file_path = os.path.join(
+ self.cache_dir, f"{url_name}_{timestamp}_som.png"
+ )
+ marked_image.save(file_path, "PNG")
+
+ # Convert to base64
+ img_buffer = io.BytesIO()
+ marked_image.save(img_buffer, format="PNG")
+ img_buffer.seek(0)
+ img_base64 = base64.b64encode(img_buffer.getvalue()).decode('utf-8')
+ img_data_url = f"data:image/png;base64,{img_base64}"
+
+ text_result = (
+ f"Visual webpage screenshot "
+ f"captured with {len(rects)} interactive elements"
+ )
+
+ return ToolResult(text=text_result, images=[img_data_url])
+
+ async def click(self, *, ref: str) -> Dict[str, str]:
+ r"""Clicks on an interactive element on the page.
+
+ Args:
+ ref (str): The reference ID of the element to click. This ID is
+ obtained from the page snapshot (see `get_page_snapshot` or
+ `get_som_screenshot`).
+
+ Returns:
+ Dict[str, str]: A dictionary containing:
+ - "result": A message confirming the click action.
+ - "snapshot": A new textual snapshot of the page after the
+ click, which may have changed as a result of the action. If
+ the snapshot is unchanged, it will be the string "snapshot
+ not changed".
+ """
+ self._validate_ref(ref, "click")
+
+ analysis = await self._get_unified_analysis()
+ elements = analysis.get("elements", {})
+ if ref not in elements:
+ available_refs = list(elements.keys())
+ logger.error(
+ f"Error: Element reference '{ref}' not found. "
+ f"Available refs: {available_refs}"
+ )
+ return {
+ "result": f"Error: Element reference '{ref}' not found. "
+ f"Available refs: {available_refs}"
+ }
+
+ action = {"type": "click", "ref": ref}
+ return await self._exec_with_snapshot(action)
+
+ async def type(self, *, ref: str, text: str) -> Dict[str, str]:
+ r"""Types text into an input field, such as a textbox or search bar.
+
+ Args:
+ ref (str): The reference ID of the input element.
+ text (str): The text to be typed into the element.
+
+ Returns:
+ Dict[str, str]: A dictionary containing:
+ - "result": A message confirming the type action.
+ - "snapshot": A new textual snapshot of the page after the
+ text has been entered.
+ """
+ self._validate_ref(ref, "type")
+ await self._get_unified_analysis() # Ensure aria-ref attributes
+
+ action = {"type": "type", "ref": ref, "text": text}
+ return await self._exec_with_snapshot(action)
+
+ async def select(self, *, ref: str, value: str) -> Dict[str, str]:
+ r"""Selects an option from a dropdown (`<select>`) element.
+
+ Args:
+ ref (str): The reference ID of the `<select>` element.
+ value (str): The value of the `<option>` to be selected. This
+ should match the `value` attribute of the option, not the
+ visible text.
+
+ Returns:
+ Dict[str, str]: A dictionary containing:
+ - "result": A message confirming the select action.
+ - "snapshot": A new snapshot of the page after the selection.
+ """
+ self._validate_ref(ref, "select")
+ await self._get_unified_analysis()
+
+ action = {"type": "select", "ref": ref, "value": value}
+ return await self._exec_with_snapshot(action)
+
+ async def scroll(self, *, direction: str, amount: int) -> Dict[str, str]:
+ r"""Scrolls the page window up or down by a specified amount.
+
+ Args:
+ direction (str): The direction to scroll. Must be either 'up' or
+ 'down'.
+ amount (int): The number of pixels to scroll.
+
+ Returns:
+ Dict[str, str]: A dictionary containing:
+ - "result": A confirmation of the scroll action.
+ - "snapshot": A new snapshot of the page after scrolling.
+ """
+ if direction not in ("up", "down"):
+ return {
+ "result": "Error: direction must be 'up' or 'down'",
+ "snapshot": "",
+ }
+
+ action = {"type": "scroll", "direction": direction, "amount": amount}
+ return await self._exec_with_snapshot(action)
+
+ async def enter(self, *, ref: str) -> Dict[str, str]:
+ r"""Simulates pressing the Enter key on a specific element.
+
+ This is often used to submit forms after filling them out.
+
+ Args:
+ ref (str): The reference ID of the element to press Enter on.
+
+ Returns:
+ Dict[str, str]: A dictionary containing:
+ - "result": A confirmation of the action.
+ - "snapshot": A new page snapshot, as this action often
+ triggers navigation or page updates.
+ """
+ self._validate_ref(ref, "enter")
+ action = {"type": "enter", "ref": ref}
+ return await self._exec_with_snapshot(action)
+
+ async def wait_user(
+ self, timeout_sec: Optional[float] = None
+ ) -> Dict[str, str]:
+ r"""Pauses the agent's execution and waits for human intervention.
+
+ This is useful for tasks that require manual steps, like solving a
+ CAPTCHA. The agent will print a message to the console and wait
+ until the user presses the Enter key.
+
+ Args:
+ timeout_sec (Optional[float]): The maximum time to wait in
+ seconds. If the timeout is reached, the agent will resume
+ automatically. If `None`, it will wait indefinitely.
+
+ Returns:
+ Dict[str, str]: A dictionary containing:
+ - "result": A message indicating how the wait ended (e.g.,
+ "User resumed." or "Timeout... reached, auto-resumed.").
+ - "snapshot": The current page snapshot after the wait.
+ """
+ import asyncio
+
+ prompt = (
+ "🕑 Agent waiting for human input. "
+ "Complete action in browser, then press Enter..."
+ )
+ logger.info(f"\n{prompt}\n")
+
+ async def _await_enter():
+ await asyncio.to_thread(input, ">>> Press Enter to resume <<<\n")
+
+ try:
+ if timeout_sec is not None:
+ logger.info(
+ f"Waiting for user input with timeout: {timeout_sec}s"
+ )
+ start_time = time.time()
+ await asyncio.wait_for(_await_enter(), timeout=timeout_sec)
+ wait_time = time.time() - start_time
+ logger.info(f"User input received after {wait_time:.2f}s")
+ result_msg = "User resumed."
+ else:
+ logger.info("Waiting for user " "input (no timeout)")
+ start_time = time.time()
+ await _await_enter()
+ wait_time = time.time() - start_time
+ logger.info(f"User input received " f"after {wait_time:.2f}s")
+ result_msg = "User resumed."
+ except asyncio.TimeoutError:
+ wait_time = timeout_sec or 0.0
+ logger.info(
+ f"User input timeout reached "
+ f"after {wait_time}s, auto-resuming"
+ )
+ result_msg = f"Timeout {timeout_sec}s reached, auto-resumed."
+
+ snapshot = await self._session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ return {"result": result_msg, "snapshot": snapshot}
+
+ async def get_page_links(self, *, ref: List[str]) -> Dict[str, Any]:
+ r"""Retrieves the full URLs for a given list of link reference IDs.
+
+ This is useful when you need to know the destination of a link before
+ clicking it.
+
+ Args:
+ ref (List[str]): A list of reference IDs for link elements,
+ obtained from a page snapshot.
+
+ Returns:
+ Dict[str, Any]: A dictionary containing:
+ - "links": A list of dictionaries, where each dictionary
+ represents a found link and has "text", "ref", and "url"
+ keys.
+ """
+ if not ref or not isinstance(ref, list):
+ return {"links": []}
+
+ for r in ref:
+ if not r or not isinstance(r, str):
+ return {"links": []}
+
+ page = await self._require_page()
+ snapshot = await self._session.get_snapshot(
+ force_refresh=True, diff_only=False
+ )
+ links = await self._extract_links_by_refs(snapshot, page, ref)
+
+ return {"links": links}
+
+ async def solve_task(
+ self, task_prompt: str, start_url: str, max_steps: int = 15
+ ) -> str:
+ r"""Uses a high-level LLM agent to autonomously complete a task.
+
+ This function delegates control to another agent that can reason about
+ a task, break it down into steps, and execute browser actions to
+ achieve the goal. It is suitable for complex, multi-step tasks.
+
+ Note: `web_agent_model` must be provided during the toolkit's
+ initialization to use this function.
+
+ Args:
+ task_prompt (str): A natural language description of the task to
+ be completed (e.g., "log into my account on example.com").
+ start_url (str): The URL to start the task from.
+ max_steps (int): The maximum number of steps the agent is allowed
+ to take before stopping.
+
+ Returns:
+ str: A summary message indicating that the task processing has
+ finished. The detailed trace of the agent's actions will be
+ printed to the standard output.
+ """
+ agent = self._ensure_agent()
+ await agent.navigate(start_url)
+ await agent.process_command(task_prompt, max_steps=max_steps)
+ return "Task processing finished - see stdout for detailed trace."
+
+ def get_tools(self) -> List[FunctionTool]:
+ r"""Get available function tools
+ based on enabled_tools configuration."""
+ # Map tool names to their corresponding methods
+ tool_map = {
+ "open_browser": self.open_browser,
+ "close_browser": self.close_browser,
+ "visit_page": self.visit_page,
+ "get_page_snapshot": self.get_page_snapshot,
+ "get_som_screenshot": self.get_som_screenshot,
+ "get_page_links": self.get_page_links,
+ "click": self.click,
+ "type": self.type,
+ "select": self.select,
+ "scroll": self.scroll,
+ "enter": self.enter,
+ "wait_user": self.wait_user,
+ "solve_task": self.solve_task,
+ }
+
+ enabled_tools = []
+
+ for tool_name in self.enabled_tools:
+ if tool_name == "solve_task" and self.web_agent_model is None:
+ logger.warning(
+ f"Tool '{tool_name}' is enabled but web_agent_model "
+ f"is not provided. Skipping this tool."
+ )
+ continue
+
+ if tool_name in tool_map:
+ enabled_tools.append(
+ FunctionTool(cast(Callable, tool_map[tool_name]))
+ )
+ else:
+ logger.warning(f"Unknown tool name: {tool_name}")
+
+ logger.info(f"Returning {len(enabled_tools)} enabled tools")
+ return enabled_tools
diff --git a/camel/toolkits/non_visual_browser_toolkit/snapshot.py b/camel/toolkits/hybrid_browser_toolkit/snapshot.py
similarity index 92%
rename from camel/toolkits/non_visual_browser_toolkit/snapshot.py
rename to camel/toolkits/hybrid_browser_toolkit/snapshot.py
index 46c756da1..bdda07896 100644
--- a/camel/toolkits/non_visual_browser_toolkit/snapshot.py
+++ b/camel/toolkits/hybrid_browser_toolkit/snapshot.py
@@ -12,7 +12,7 @@
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
from pathlib import Path
-from typing import TYPE_CHECKING, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
if TYPE_CHECKING:
from playwright.async_api import Page
@@ -64,7 +64,17 @@ class PageSnapshot:
)
logger.debug("Capturing page snapshot …")
- snapshot_text = await self._get_snapshot_direct()
+ snapshot_result = await self._get_snapshot_direct()
+
+ # Extract snapshot text from the unified analyzer result
+ if (
+ isinstance(snapshot_result, dict)
+ and 'snapshotText' in snapshot_result
+ ):
+ snapshot_text = snapshot_result['snapshotText']
+ else:
+ snapshot_text = snapshot_result
+
formatted = self._format_snapshot(snapshot_text or "<empty>")
output = formatted
@@ -99,7 +109,9 @@ class PageSnapshot:
# ------------------------------------------------------------------
_snapshot_js_cache: Optional[str] = None # class-level cache
- async def _get_snapshot_direct(self) -> Optional[str]:
+ async def _get_snapshot_direct(
+ self,
+ ) -> Optional[Union[str, Dict[str, Any]]]:
r"""Evaluate the snapshot-extraction JS with simple retry logic.
Playwright throws *Execution context was destroyed* when a new page
@@ -110,7 +122,7 @@ class PageSnapshot:
# Load JS once and cache it at class level
if PageSnapshot._snapshot_js_cache is None:
- js_path = Path(__file__).parent / "snapshot.js"
+ js_path = Path(__file__).parent / "unified_analyzer.js"
PageSnapshot._snapshot_js_cache = js_path.read_text(
encoding="utf-8"
)
diff --git a/camel/toolkits/non_visual_browser_toolkit/snapshot.js b/camel/toolkits/hybrid_browser_toolkit/unified_analyzer.js
similarity index 60%
rename from camel/toolkits/non_visual_browser_toolkit/snapshot.js
rename to camel/toolkits/hybrid_browser_toolkit/unified_analyzer.js
index ae42bdc7f..99c9795e2 100644
--- a/camel/toolkits/non_visual_browser_toolkit/snapshot.js
+++ b/camel/toolkits/hybrid_browser_toolkit/unified_analyzer.js
@@ -1,6 +1,13 @@
(() => {
- // Playwright's snapshot logic focuses on semantics and visibility, not arbitrary limits.
- // We will first build a semantic tree in memory, then render it.
+ // Unified analyzer that combines visual and structural analysis
+ // Preserves complete snapshot.js logic while adding visual coordinate information
+
+ let refCounter = 1;
+ function generateRef() {
+ return `e${refCounter++}`;
+ }
+
+ // === Complete snapshot.js logic preservation ===
function isVisible(node) {
if (node.nodeType !== Node.ELEMENT_NODE) return true;
@@ -70,13 +77,9 @@
return result;
}
- let refCounter = 1;
- function generateRef() {
- return `e${refCounter++}`;
- }
-
/**
* Phase 1: Build an in-memory representation of the accessibility tree.
+ * Complete preservation of snapshot.js buildAriaTree logic
*/
function buildAriaTree(rootElement) {
const visited = new Set();
@@ -153,9 +156,34 @@
}
}
- // FIX: If an element's name is the same as its only text child, remove the redundant child.
- if (ariaNode && ariaNode.children.length === 1 && typeof ariaNode.children[0] === 'string' && ariaNode.name === ariaNode.children[0]) {
- ariaNode.children = [];
+ // FIX: Remove redundant text children that match the element's name
+ if (ariaNode && ariaNode.children.length > 0) {
+ // Remove text children that are the same as the parent's name or are contained in it
+ ariaNode.children = ariaNode.children.filter(child => {
+ if (typeof child === 'string') {
+ const childText = child.trim();
+ const parentName = ariaNode.name.trim();
+
+ // Remove if text child exactly matches parent name
+ if (childText === parentName) {
+ return false;
+ }
+
+ // Also remove if the child text is completely contained in parent name
+ // and represents a significant portion (to avoid removing important partial text)
+ if (childText.length > 3 && parentName.includes(childText)) {
+ return false;
+ }
+
+ return true;
+ }
+ return true;
+ });
+
+ // If after filtering, we have only one text child that equals the name, remove it
+ if (ariaNode.children.length === 1 && typeof ariaNode.children[0] === 'string' && ariaNode.name === ariaNode.children[0]) {
+ ariaNode.children = [];
+ }
}
}
@@ -166,7 +194,7 @@
/**
* Phase 2: Normalize the tree by removing redundant generic wrappers.
- * This is a key optimization in Playwright to simplify the structure.
+ * Complete preservation of snapshot.js normalizeTree logic
*/
function normalizeTree(node) {
if (typeof node === 'string') return [node];
@@ -178,6 +206,24 @@
node.children = newChildren;
// Remove child elements that have the same name as their parent
+ const filteredChildren = [];
+ for (const child of node.children) {
+ if (typeof child !== 'string' && child.name && node.name) {
+ const childName = child.name.trim();
+ const parentName = node.name.trim();
+ if (childName === parentName) {
+ // If child has same name as parent, merge its children into parent
+ filteredChildren.push(...(child.children || []));
+ } else {
+ filteredChildren.push(child);
+ }
+ } else {
+ filteredChildren.push(child);
+ }
+ }
+ node.children = filteredChildren;
+
+ // Also handle the case where we have only one child with same name
if (node.children.length === 1 && typeof node.children[0] !== 'string') {
const child = node.children[0];
if (child.name && node.name && child.name.trim() === node.name.trim()) {
@@ -195,9 +241,9 @@
return [node];
}
-
/**
* Phase 3: Render the normalized tree into the final string format.
+ * Complete preservation of snapshot.js renderTree logic
*/
function renderTree(node, indent = '') {
const lines = [];
@@ -263,6 +309,116 @@
return lines;
}
- const outputLines = processDocument(document);
- return outputLines.join('\n');
-})();
+ // === Visual analysis functions from page_script.js ===
+
+ // From page_script.js - check if element is topmost at coordinates
+ function isTopmost(element, x, y) {
+ let hit = document.elementFromPoint(x, y);
+ if (hit === null) return true;
+
+ while (hit) {
+ if (hit == element) return true;
+ hit = hit.parentNode;
+ }
+ return false;
+ }
+
+ // From page_script.js - get visual coordinates
+ function getElementCoordinates(element) {
+ let rects = element.getClientRects();
+ let scale = window.devicePixelRatio || 1;
+ let validRects = [];
+
+ for (const rect of rects) {
+ let x = rect.left + rect.width / 2;
+ let y = rect.top + rect.height / 2;
+ if (isTopmost(element, x, y)) {
+ validRects.push({
+ x: rect.x * scale,
+ y: rect.y * scale,
+ width: rect.width * scale,
+ height: rect.height * scale,
+ top: rect.top * scale,
+ left: rect.left * scale,
+ right: rect.right * scale,
+ bottom: rect.bottom * scale
+ });
+ }
+ }
+
+ return validRects;
+ }
+
+ // === Unified analysis function ===
+
+ function collectElementsFromTree(node, elementsMap) {
+ if (typeof node === 'string') return;
+
+ if (node.element && node.ref) {
+ // Get visual coordinates for this element
+ const coordinates = getElementCoordinates(node.element);
+
+ // Store comprehensive element information
+ elementsMap[node.ref] = {
+ // Structural information (preserved from snapshot.js)
+ role: node.role,
+ name: node.name,
+ tagName: node.element.tagName.toLowerCase(),
+ disabled: node.disabled,
+ checked: node.checked,
+ expanded: node.expanded,
+
+ // Visual information (from page_script.js)
+ coordinates: coordinates,
+
+ // Additional metadata
+ href: node.element.href || null,
+ value: node.element.value || null,
+ placeholder: node.element.placeholder || null,
+ scrollable: node.element.scrollHeight > node.element.clientHeight
+ };
+ }
+
+ // Recursively process children
+ if (node.children) {
+ for (const child of node.children) {
+ collectElementsFromTree(child, elementsMap);
+ }
+ }
+ }
+
+ function analyzePageElements() {
+ // Generate the complete structured snapshot using original snapshot.js logic
+ const outputLines = processDocument(document);
+ const snapshotText = outputLines.join('\n');
+
+ // Build the tree again to collect element information with visual data
+ textCache.clear();
+ refCounter = 1; // Reset counter to match snapshot generation
+ let tree = buildAriaTree(document.body);
+ [tree] = normalizeTree(tree);
+
+ const elementsMap = {};
+ collectElementsFromTree(tree, elementsMap);
+
+ const result = {
+ url: window.location.href,
+ elements: elementsMap,
+ snapshotText: snapshotText,
+ metadata: {
+ timestamp: new Date().toISOString(),
+ elementCount: Object.keys(elementsMap).length,
+ screenInfo: {
+ width: window.innerWidth,
+ height: window.innerHeight,
+ devicePixelRatio: window.devicePixelRatio || 1
+ }
+ }
+ };
+
+ return result;
+ }
+
+ // Execute analysis and return result
+ return analyzePageElements();
+})();
\ No newline at end of file
diff --git a/camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.py b/camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.py
deleted file mode 100644
index 167ed818b..000000000
--- a/camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.py
+++ /dev/null
@@ -1,446 +0,0 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-from __future__ import annotations
-
-from typing import Any, Dict, List, Optional
-
-from camel.logger import get_logger
-from camel.models import BaseModelBackend
-from camel.toolkits.base import BaseToolkit
-from camel.toolkits.function_tool import FunctionTool
-
-from .agent import PlaywrightLLMAgent
-from .nv_browser_session import NVBrowserSession
-
-logger = get_logger(__name__)
-
-
-class BrowserNonVisualToolkit(BaseToolkit):
- r"""A lightweight, *non-visual* browser toolkit exposing primitive
- Playwright actions as CAMEL `FunctionTool`s.
- """
-
- def __init__(
- self,
- *,
- headless: bool = True,
- user_data_dir: Optional[str] = None,
- web_agent_model: Optional[BaseModelBackend] = None,
- ) -> None:
- super().__init__()
- self._headless = headless
- self._user_data_dir = user_data_dir
- self.web_agent_model = web_agent_model # Currently unused but kept
- # for compatibility
-
- # Encapsulated browser session
- self._session = NVBrowserSession(
- headless=headless, user_data_dir=user_data_dir
- )
-
- # Optional higher-level agent (only if user supplies model)
- self._agent: Optional[PlaywrightLLMAgent] = None
-
- def __del__(self):
- r"""Best-effort cleanup when toolkit is garbage collected.
-
- 1. We *avoid* running during the Python interpreter shutdown phase
- (`sys.is_finalizing()`), because the import machinery and/or event
- loop may already be torn down which leads to noisy exceptions such
- as `ImportError: sys.meta_path is None` or
- `RuntimeError: Event loop is closed`.
- 2. We protect all imports and event-loop operations with defensive
- `try/except` blocks. This ensures that, even if cleanup cannot be
- carried out, we silently ignore the failure instead of polluting
- stderr on program exit.
- """
- try:
- import sys
-
- if getattr(sys, "is_finalizing", lambda: False)():
- return # Skip cleanup during interpreter shutdown
-
- import asyncio
-
- try:
- loop = asyncio.get_event_loop()
- except RuntimeError:
- # No event loop in current thread → nothing to clean
- return
-
- if loop.is_closed():
- # The default loop is closed, create a *temporary* loop just
- # for cleanup so that Playwright / asyncio transports are
- # gracefully shut down. This avoids noisy warnings such as
- # "RuntimeError: Event loop is closed" when the program
- # exits.
- try:
- tmp_loop = asyncio.new_event_loop()
- try:
- asyncio.set_event_loop(tmp_loop)
- tmp_loop.run_until_complete(self.close_browser())
- finally:
- # Best-effort shutdown of async generators and loop
- # itself (Python ≥3.6).
- if hasattr(tmp_loop, "shutdown_asyncgens"):
- tmp_loop.run_until_complete(
- tmp_loop.shutdown_asyncgens()
- )
- tmp_loop.close()
- finally:
- # Ensure no subsequent get_event_loop() picks up a now
- # closed temporary loop.
- asyncio.set_event_loop(None)
- return
-
- if loop.is_running():
- try:
- task = loop.create_task(self.close_browser())
- del task # Fire-and-forget
- except RuntimeError:
- # Loop is running but not in this thread → ignore
- pass
- else:
- # Own the loop → safe to run
- asyncio.run(self.close_browser())
- except Exception:
- # Suppress *all* errors during garbage collection
- pass
-
- async def _ensure_browser(self):
- await self._session.ensure_browser()
-
- async def _require_page(self):
- await self._session.ensure_browser()
- return await self._session.get_page()
-
- def _validate_ref(self, ref: str, method_name: str) -> None:
- r"""Validate that ref parameter is a non-empty string."""
- if not ref or not isinstance(ref, str):
- logger.error(
- f"{method_name}(): 'ref' must be a non-empty string, "
- f"got: {ref}"
- )
-
- async def open_browser(
- self, start_url: Optional[str] = None
- ) -> Dict[str, str]:
- r"""Launches a new browser session. This should be the first step.
-
- Args:
- start_url (Optional[str]): The URL to navigate to after the browser
- launches. If not provided, the browser will open with a blank
- page. (default: :obj:`None`)
-
- Returns:
- Dict[str, str]: A dictionary containing the result of the action
- and a snapshot of the page. The keys are "result" and
- "snapshot". The "snapshot" is a YAML-like representation of
- the page's DOM structure, including element references
- (e.g., "e3") that can be used in other tool calls.
- """
- await self._session.ensure_browser()
- if start_url:
- return await self.visit_page(start_url)
- # If no start_url provided, still capture initial snapshot
- snapshot = await self._session.get_snapshot(
- force_refresh=True, diff_only=False
- )
- return {"result": "Browser session started.", "snapshot": snapshot}
-
- async def close_browser(self) -> str:
- r"""Closes the current browser session, freeing up all associated
- resources. This should be called when the browsing task is complete.
-
- Returns:
- str: A confirmation message indicating the session has been
- closed.
- """
- # Close agent if it exists
- if self._agent is not None:
- try:
- await self._agent.close()
- except Exception:
- pass # Don't fail if agent cleanup fails
- self._agent = None
-
- # Close session
- await NVBrowserSession.close_all_sessions()
- return "Browser session closed."
-
- async def visit_page(self, url: str) -> Dict[str, str]:
- r"""Navigates the current browser page to a new URL.
-
- Args:
- url (str): The URL to navigate to. Must be a fully qualified URL
- (e.g., "https://www.google.com").
-
- Returns:
- Dict[str, str]: A dictionary containing the navigation result and a
- snapshot of the new page. The keys are "result" and
- "snapshot". The "snapshot" provides a fresh view of the
- page's DOM structure.
- """
- if not url or not isinstance(url, str):
- raise ValueError("visit_page(): 'url' must be a non-empty string")
-
- nav_result = await self._session.visit(url)
- snapshot = await self._session.get_snapshot(
- force_refresh=True, diff_only=False
- )
- return {"result": nav_result, "snapshot": snapshot}
-
- async def get_page_snapshot(
- self, *, force_refresh: bool = False, diff_only: bool = False
- ) -> str:
- r"""Capture a YAML-like structural snapshot of the DOM.
-
- Args:
- force_refresh (bool): When ``True`` always re-generate the
- snapshot even if the URL has not changed. (default:
- :obj:`False`)
- diff_only (bool): When ``True`` return only the diff relative to
- the previous snapshot. (default: :obj:`False`)
-
- Returns:
- str: Formatted snapshot string.
- """
- return await self._session.get_snapshot(
- force_refresh=force_refresh, diff_only=diff_only
- )
-
- async def click(self, *, ref: str) -> Dict[str, str]:
- r"""Performs a click action on a specified element on the current page.
-
- Args:
- ref (str): The reference ID of the element to click. This ID is
- obtained from the page snapshot (e.g., "e12").
-
- Returns:
- Dict[str, str]: A dictionary containing the result of the action.
- If the click causes a change in the page's structure, a
- "snapshot" key will be included with a diff of the changes.
- """
- self._validate_ref(ref, "click")
-
- action: Dict[str, Any] = {"type": "click", "ref": ref}
- return await self._exec_with_snapshot(action)
-
- async def type(self, *, ref: str, text: str) -> Dict[str, str]:
- r"""Types text into an input field or textarea on the current page.
-
- Args:
- ref (str): The reference ID of the input element. This ID is
- obtained from the page snapshot (e.g., "e25").
- text (str): The text to be typed into the element.
-
- Returns:
- Dict[str, str]: A dictionary containing the result of the action.
- The key is "result".
- """
- self._validate_ref(ref, "type")
-
- action: Dict[str, Any] = {"type": "type", "ref": ref, "text": text}
- return await self._exec_with_snapshot(action)
-
- async def select(self, *, ref: str, value: str) -> Dict[str, str]:
- r"""Selects an option from a dropdown (<select>) element on the page.
-
- Args:
- ref (str): The reference ID of the <select> element. This ID is
- obtained from the page snapshot.
- value (str): The value of the option to be selected. This should
- match the 'value' attribute of an <option> tag.
-
- Returns:
- Dict[str, str]: A dictionary containing the result of the action.
- The key is "result".
- """
- self._validate_ref(ref, "select")
-
- action: Dict[str, Any] = {"type": "select", "ref": ref, "value": value}
- return await self._exec_with_snapshot(action)
-
- async def scroll(self, *, direction: str, amount: int) -> Dict[str, str]:
- r"""Scrolls the current page up or down by a specified amount.
-
- Args:
- direction (str): The direction to scroll. Must be either "up" or
- "down".
- amount (int): The number of pixels to scroll.
-
- Returns:
- Dict[str, str]: A dictionary containing the result of the action.
- The key is "result".
- """
- if direction not in ("up", "down"):
- logger.error("scroll(): 'direction' must be 'up' or 'down'")
- return {
- "result": "scroll() Error: 'direction' must be 'up' or 'down'"
- }
-
- action = {"type": "scroll", "direction": direction, "amount": amount}
- return await self._exec_with_snapshot(action)
-
- async def enter(self, *, ref: str) -> Dict[str, str]:
- r"""Simulates pressing the Enter key on a specific element.
- This is often used to submit forms.
-
- Args:
- ref (str): The reference ID of the element to focus before
- pressing Enter. This ID is obtained from the page snapshot.
-
- Returns:
- Dict[str, str]: A dictionary containing the result of the action.
- If pressing Enter causes a page navigation or DOM change, a
- "snapshot" key will be included with a diff of the changes.
- """
- self._validate_ref(ref, "enter")
-
- action: Dict[str, Any] = {"type": "enter", "ref": ref}
- return await self._exec_with_snapshot(action)
-
- async def wait_user(
- self,
- timeout_sec: Optional[float] = None,
- ) -> Dict[str, str]:
- r"""Pauses the agent's execution and waits for human intervention.
- This is useful for tasks that require manual steps, like solving a
- CAPTCHA. The agent will print a message and wait for the user to
- press the Enter key in the console.
-
- Args:
- timeout_sec (Optional[float]): The maximum time in seconds to wait
- for the user. If `None`, it will wait indefinitely. Defaults
- to `None`. (default: :obj:`None`)
-
- Returns:
- Dict[str, str]: A dictionary containing a result message and a
- full snapshot of the current page after the user has acted.
- The keys are "result" and "snapshot".
- """
-
- import asyncio
-
- prompt = (
- "🕑 Agent is waiting for human input. "
- "Complete the required action in the browser, then press Enter "
- "to continue..."
- )
-
- logger.info(f"\n{prompt}\n")
-
- async def _await_enter():
- await asyncio.to_thread(input, ">>> Press Enter to resume <<<\n")
-
- try:
- if timeout_sec is not None:
- await asyncio.wait_for(_await_enter(), timeout=timeout_sec)
- result_msg = "User resumed."
- else:
- await _await_enter()
- result_msg = "User resumed."
- except asyncio.TimeoutError:
- result_msg = f"Timeout {timeout_sec}s reached, auto-resumed."
-
- snapshot = await self._session.get_snapshot(
- force_refresh=True,
- diff_only=False,
- )
-
- return {"result": result_msg, "snapshot": snapshot}
-
- # Helper to run through ActionExecutor
- async def _exec(self, action: Dict[str, Any]) -> str:
- return await self._session.exec_action(action)
-
- async def _exec_with_snapshot(
- self, action: Dict[str, Any]
- ) -> Dict[str, str]:
- r"""Execute action and, if DOM structure changed, include snapshot
- diff.
- """
- result = await self._session.exec_action(action)
-
- # Only capture diff if action type typically changes DOM
- from .actions import ActionExecutor
-
- if not ActionExecutor.should_update_snapshot(action):
- return {"result": result}
-
- # Capture structural diff to previous snapshot
- diff = await self._session.get_snapshot(
- force_refresh=True, diff_only=True
- )
-
- if diff.startswith("- Page Snapshot (no structural changes)"):
- return {"result": result}
-
- return {"result": result, "snapshot": diff}
-
- def _ensure_agent(self) -> PlaywrightLLMAgent:
- r"""Create PlaywrightLLMAgent on first use if `web_agent_model`
- provided."""
- if self.web_agent_model is None:
- raise RuntimeError(
- "web_agent_model not supplied - high-level task planning is "
- "unavailable."
- )
-
- if self._agent is None:
- self._agent = PlaywrightLLMAgent(
- headless=self._headless,
- user_data_dir=self._user_data_dir,
- model_backend=self.web_agent_model,
- )
- return self._agent
-
- async def solve_task(
- self, task_prompt: str, start_url: str, max_steps: int = 15
- ) -> str:
- r"""Use LLM agent to autonomously complete the task (requires
- `web_agent_model`).
-
- Args:
- task_prompt (str): The task prompt to complete.
- start_url (str): The URL to navigate to.
- max_steps (int): The maximum number of steps to take.
- (default: :obj:`15`)
-
- Returns:
- str: The result of the task.
- """
-
- agent = self._ensure_agent()
- await agent.navigate(start_url)
- await agent.process_command(task_prompt, max_steps=max_steps)
- return "Task processing finished - see stdout for detailed trace."
-
- def get_tools(self) -> List[FunctionTool]:
- base_tools = [
- FunctionTool(self.open_browser),
- FunctionTool(self.close_browser),
- FunctionTool(self.visit_page),
- FunctionTool(self.click),
- FunctionTool(self.type),
- FunctionTool(self.select),
- FunctionTool(self.scroll),
- FunctionTool(self.enter),
- FunctionTool(self.wait_user),
- ]
-
- if self.web_agent_model is not None:
- base_tools.append(FunctionTool(self.solve_task))
-
- return base_tools
diff --git a/camel/types/agents/tool_calling_record.py b/camel/types/agents/tool_calling_record.py
index d3359b4de..3e0b4c91e 100644
--- a/camel/types/agents/tool_calling_record.py
+++ b/camel/types/agents/tool_calling_record.py
@@ -11,7 +11,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-from typing import Any, Dict
+from typing import Any, Dict, List, Optional
from pydantic import BaseModel
@@ -24,12 +24,15 @@ class ToolCallingRecord(BaseModel):
args (Dict[str, Any]): The dictionary of arguments passed to the tool.
result (Any): The execution result of calling this tool.
tool_call_id (str): The ID of the tool call, if available.
+ images (Optional[List[str]]): List of base64-encoded images returned
+ by the tool, if any.
"""
tool_name: str
args: Dict[str, Any]
result: Any
tool_call_id: str
+ images: Optional[List[str]] = None
def __str__(self) -> str:
r"""Overridden version of the string function.
diff --git a/camel/utils/tool_result.py b/camel/utils/tool_result.py
new file mode 100644
index 000000000..48e87269d
--- /dev/null
+++ b/camel/utils/tool_result.py
@@ -0,0 +1,44 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from typing import List, Optional
+
+
+class ToolResult:
+ r"""Special result type for tools that can return images along with text.
+
+ This class is used by ChatAgent to detect when a tool returns visual
+ content that should be included in the conversation context.
+ """
+
+ def __init__(self, text: str, images: Optional[List[str]] = None):
+ r"""Initialize a tool result.
+
+ Args:
+ text (str): The text description or result of the tool operation.
+ images (Optional[List[str]]): List of base64-encoded images to
+ include in the conversation context. Images should be encoded
+ as "data:image/{format};base64,{data}" format.
+ """
+ self.text = text
+ self.images = images or []
+
+ def __str__(self) -> str:
+ r"""Return the text representation of the result."""
+ return self.text
+
+ def __repr__(self) -> str:
+ r"""Return a detailed representation of the result."""
+ img_count = len(self.images) if self.images else 0
+ return f"ToolResult(text='{self.text[:50]}...', images={img_count})"
diff --git a/docs/mintlify/docs.json b/docs/mintlify/docs.json
index 09687bed4..cd523156a 100644
--- a/docs/mintlify/docs.json
+++ b/docs/mintlify/docs.json
@@ -515,11 +515,11 @@
"reference/camel.toolkits.whatsapp_toolkit",
"reference/camel.toolkits.wolfram_alpha_toolkit",
"reference/camel.toolkits.zapier_toolkit",
- "reference/camel.toolkits.non_visual_browser_toolkit.actions",
- "reference/camel.toolkits.non_visual_browser_toolkit.agent",
- "reference/camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit",
- "reference/camel.toolkits.non_visual_browser_toolkit.nv_browser_session",
- "reference/camel.toolkits.non_visual_browser_toolkit.snapshot"
+ "reference/camel.toolkits.hybrid_browser_toolkit.actions",
+ "reference/camel.toolkits.hybrid_browser_toolkit.agent",
+ "reference/camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit",
+ "reference/camel.toolkits.hybrid_browser_toolkit.browser_session",
+ "reference/camel.toolkits.hybrid_browser_toolkit.snapshot"
]
},
{
diff --git a/docs/reference/camel.toolkits.hybrid_browser_toolkit.actions.md b/docs/reference/camel.toolkits.hybrid_browser_toolkit.actions.md
new file mode 100644
index 000000000..2f3759a2f
--- /dev/null
+++ b/docs/reference/camel.toolkits.hybrid_browser_toolkit.actions.md
@@ -0,0 +1,27 @@
+<a id="camel.toolkits.hybrid_browser_toolkit.actions"></a>
+
+<a id="camel.toolkits.hybrid_browser_toolkit.actions.ActionExecutor"></a>
+
+## ActionExecutor
+
+```python
+class ActionExecutor:
+```
+
+Executes high-level actions (click, type …) on a Playwright Page.
+
+<a id="camel.toolkits.hybrid_browser_toolkit.actions.ActionExecutor.__init__"></a>
+
+### __init__
+
+```python
+def __init__(self, page: 'Page'):
+```
+
+<a id="camel.toolkits.hybrid_browser_toolkit.actions.ActionExecutor.should_update_snapshot"></a>
+
+### should_update_snapshot
+
+```python
+def should_update_snapshot(action: Dict[str, Any]):
+```
diff --git a/docs/reference/camel.toolkits.non_visual_browser_toolkit.agent.md b/docs/reference/camel.toolkits.hybrid_browser_toolkit.agent.md
similarity index 56%
rename from docs/reference/camel.toolkits.non_visual_browser_toolkit.agent.md
rename to docs/reference/camel.toolkits.hybrid_browser_toolkit.agent.md
index 84ad5790f..bd9963a89 100644
--- a/docs/reference/camel.toolkits.non_visual_browser_toolkit.agent.md
+++ b/docs/reference/camel.toolkits.hybrid_browser_toolkit.agent.md
@@ -1,6 +1,6 @@
-<a id="camel.toolkits.non_visual_browser_toolkit.agent"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.agent"></a>
-<a id="camel.toolkits.non_visual_browser_toolkit.agent.PlaywrightLLMAgent"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.agent.PlaywrightLLMAgent"></a>
## PlaywrightLLMAgent
@@ -10,7 +10,7 @@ class PlaywrightLLMAgent:
High-level orchestration: snapshot ↔ LLM ↔ action executor.
-<a id="camel.toolkits.non_visual_browser_toolkit.agent.PlaywrightLLMAgent.__init__"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.agent.PlaywrightLLMAgent.__init__"></a>
### __init__
@@ -18,7 +18,7 @@ High-level orchestration: snapshot ↔ LLM ↔ action executor.
def __init__(self):
```
-<a id="camel.toolkits.non_visual_browser_toolkit.agent.PlaywrightLLMAgent._get_chat_agent"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.agent.PlaywrightLLMAgent._get_chat_agent"></a>
### _get_chat_agent
@@ -28,7 +28,7 @@ def _get_chat_agent(self):
Get or create the ChatAgent instance.
-<a id="camel.toolkits.non_visual_browser_toolkit.agent.PlaywrightLLMAgent._safe_parse_json"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.agent.PlaywrightLLMAgent._safe_parse_json"></a>
### _safe_parse_json
@@ -39,7 +39,7 @@ def _safe_parse_json(self, content: str):
Safely parse JSON from LLM response with multiple fallback
strategies.
-<a id="camel.toolkits.non_visual_browser_toolkit.agent.PlaywrightLLMAgent._llm_call"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.agent.PlaywrightLLMAgent._llm_call"></a>
### _llm_call
diff --git a/docs/reference/camel.toolkits.non_visual_browser_toolkit.nv_browser_session.md b/docs/reference/camel.toolkits.hybrid_browser_toolkit.browser_session.md
similarity index 55%
rename from docs/reference/camel.toolkits.non_visual_browser_toolkit.nv_browser_session.md
rename to docs/reference/camel.toolkits.hybrid_browser_toolkit.browser_session.md
index bd74fa5f9..8b806536c 100644
--- a/docs/reference/camel.toolkits.non_visual_browser_toolkit.nv_browser_session.md
+++ b/docs/reference/camel.toolkits.hybrid_browser_toolkit.browser_session.md
@@ -1,6 +1,6 @@
-<a id="camel.toolkits.non_visual_browser_toolkit.nv_browser_session"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.browser_session"></a>
-<a id="camel.toolkits.non_visual_browser_toolkit.nv_browser_session.NVBrowserSession"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession"></a>
## NVBrowserSession
@@ -17,7 +17,7 @@ duplicating Playwright setup code.
This class is a singleton per event-loop.
-<a id="camel.toolkits.non_visual_browser_toolkit.nv_browser_session.NVBrowserSession.__new__"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession.__new__"></a>
### __new__
@@ -25,7 +25,7 @@ This class is a singleton per event-loop.
def __new__(cls):
```
-<a id="camel.toolkits.non_visual_browser_toolkit.nv_browser_session.NVBrowserSession.__init__"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession.__init__"></a>
### __init__
diff --git a/docs/reference/camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.md b/docs/reference/camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.md
similarity index 56%
rename from docs/reference/camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.md
rename to docs/reference/camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.md
index fbb584892..a3c5e414c 100644
--- a/docs/reference/camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.md
+++ b/docs/reference/camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.md
@@ -1,17 +1,17 @@
-<a id="camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit"></a>
-<a id="camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.BrowserNonVisualToolkit"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.HybridBrowserToolkit"></a>
-## BrowserNonVisualToolkit
+## HybridBrowserToolkit
```python
-class BrowserNonVisualToolkit(BaseToolkit):
+class HybridBrowserToolkit(BaseToolkit):
```
A lightweight, *non-visual* browser toolkit exposing primitive
Playwright actions as CAMEL `FunctionTool`s.
-<a id="camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.BrowserNonVisualToolkit.__init__"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.HybridBrowserToolkit.__init__"></a>
### __init__
@@ -19,7 +19,7 @@ Playwright actions as CAMEL `FunctionTool`s.
def __init__(self):
```
-<a id="camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.BrowserNonVisualToolkit.__del__"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.HybridBrowserToolkit.__del__"></a>
### __del__
@@ -39,7 +39,7 @@ as `ImportError: sys.meta_path is None` or
carried out, we silently ignore the failure instead of polluting
stderr on program exit.
-<a id="camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.BrowserNonVisualToolkit._validate_ref"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.HybridBrowserToolkit._validate_ref"></a>
### _validate_ref
@@ -49,7 +49,7 @@ def _validate_ref(self, ref: str, method_name: str):
Validate that ref parameter is a non-empty string.
-<a id="camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.BrowserNonVisualToolkit._ensure_agent"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.HybridBrowserToolkit._ensure_agent"></a>
### _ensure_agent
@@ -60,7 +60,7 @@ def _ensure_agent(self):
Create PlaywrightLLMAgent on first use if `web_agent_model`
provided.
-<a id="camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.BrowserNonVisualToolkit.get_tools"></a>
+<a id="camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.HybridBrowserToolkit.get_tools"></a>
### get_tools
diff --git a/docs/reference/camel.toolkits.hybrid_browser_toolkit.snapshot.md b/docs/reference/camel.toolkits.hybrid_browser_toolkit.snapshot.md
new file mode 100644
index 000000000..cccf26330
--- /dev/null
+++ b/docs/reference/camel.toolkits.hybrid_browser_toolkit.snapshot.md
@@ -0,0 +1,46 @@
+<a id="camel.toolkits.hybrid_browser_toolkit.snapshot"></a>
+
+<a id="camel.toolkits.hybrid_browser_toolkit.snapshot.PageSnapshot"></a>
+
+## PageSnapshot
+
+```python
+class PageSnapshot:
+```
+
+Utility for capturing YAML-like page snapshots and diff-only
+variants.
+
+<a id="camel.toolkits.hybrid_browser_toolkit.snapshot.PageSnapshot.__init__"></a>
+
+### __init__
+
+```python
+def __init__(self, page: 'Page'):
+```
+
+<a id="camel.toolkits.hybrid_browser_toolkit.snapshot.PageSnapshot._format_snapshot"></a>
+
+### _format_snapshot
+
+```python
+def _format_snapshot(text: str):
+```
+
+<a id="camel.toolkits.hybrid_browser_toolkit.snapshot.PageSnapshot._compute_diff"></a>
+
+### _compute_diff
+
+```python
+def _compute_diff(old: str, new: str):
+```
+
+<a id="camel.toolkits.hybrid_browser_toolkit.snapshot.PageSnapshot._detect_priorities"></a>
+
+### _detect_priorities
+
+```python
+def _detect_priorities(self, snapshot_yaml: str):
+```
+
+Return sorted list of priorities present (1,2,3).
diff --git a/docs/reference/camel.toolkits.non_visual_browser_toolkit.actions.md b/docs/reference/camel.toolkits.non_visual_browser_toolkit.actions.md
deleted file mode 100644
index 024feec52..000000000
--- a/docs/reference/camel.toolkits.non_visual_browser_toolkit.actions.md
+++ /dev/null
@@ -1,27 +0,0 @@
-<a id="camel.toolkits.non_visual_browser_toolkit.actions"></a>
-
-<a id="camel.toolkits.non_visual_browser_toolkit.actions.ActionExecutor"></a>
-
-## ActionExecutor
-
-```python
-class ActionExecutor:
-```
-
-Executes high-level actions (click, type …) on a Playwright Page.
-
-<a id="camel.toolkits.non_visual_browser_toolkit.actions.ActionExecutor.__init__"></a>
-
-### __init__
-
-```python
-def __init__(self, page: 'Page'):
-```
-
-<a id="camel.toolkits.non_visual_browser_toolkit.actions.ActionExecutor.should_update_snapshot"></a>
-
-### should_update_snapshot
-
-```python
-def should_update_snapshot(action: Dict[str, Any]):
-```
diff --git a/docs/reference/camel.toolkits.non_visual_browser_toolkit.snapshot.md b/docs/reference/camel.toolkits.non_visual_browser_toolkit.snapshot.md
deleted file mode 100644
index 81522a636..000000000
--- a/docs/reference/camel.toolkits.non_visual_browser_toolkit.snapshot.md
+++ /dev/null
@@ -1,46 +0,0 @@
-<a id="camel.toolkits.non_visual_browser_toolkit.snapshot"></a>
-
-<a id="camel.toolkits.non_visual_browser_toolkit.snapshot.PageSnapshot"></a>
-
-## PageSnapshot
-
-```python
-class PageSnapshot:
-```
-
-Utility for capturing YAML-like page snapshots and diff-only
-variants.
-
-<a id="camel.toolkits.non_visual_browser_toolkit.snapshot.PageSnapshot.__init__"></a>
-
-### __init__
-
-```python
-def __init__(self, page: 'Page'):
-```
-
-<a id="camel.toolkits.non_visual_browser_toolkit.snapshot.PageSnapshot._format_snapshot"></a>
-
-### _format_snapshot
-
-```python
-def _format_snapshot(text: str):
-```
-
-<a id="camel.toolkits.non_visual_browser_toolkit.snapshot.PageSnapshot._compute_diff"></a>
-
-### _compute_diff
-
-```python
-def _compute_diff(old: str, new: str):
-```
-
-<a id="camel.toolkits.non_visual_browser_toolkit.snapshot.PageSnapshot._detect_priorities"></a>
-
-### _detect_priorities
-
-```python
-def _detect_priorities(self, snapshot_yaml: str):
-```
-
-Return sorted list of priorities present (1,2,3).
diff --git a/examples/toolkits/browser_non_visual_toolkit_example.py b/examples/toolkits/hybrid_browser_toolkit_example.py
similarity index 52%
rename from examples/toolkits/browser_non_visual_toolkit_example.py
rename to examples/toolkits/hybrid_browser_toolkit_example.py
index ecee5b084..22b635b62 100644
--- a/examples/toolkits/browser_non_visual_toolkit_example.py
+++ b/examples/toolkits/hybrid_browser_toolkit_example.py
@@ -16,7 +16,7 @@ import logging
from camel.agents import ChatAgent
from camel.models import ModelFactory
-from camel.toolkits import BrowserNonVisualToolkit
+from camel.toolkits import HybridBrowserToolkit
from camel.types import ModelPlatformType, ModelType
logging.basicConfig(
@@ -30,49 +30,77 @@ logging.basicConfig(
logging.getLogger('camel.agents').setLevel(logging.INFO)
logging.getLogger('camel.models').setLevel(logging.INFO)
logging.getLogger('camel.models').setLevel(logging.INFO)
-logging.getLogger('camel.toolkits.non_visual_browser_toolkit').setLevel(
+logging.getLogger('camel.toolkits.hybrid_browser_toolkit').setLevel(
logging.DEBUG
)
USER_DATA_DIR = "User_Data"
model_backend = ModelFactory.create(
model_platform=ModelPlatformType.OPENAI,
- model_type=ModelType.GPT_4_1,
+ model_type=ModelType.GPT_4O,
model_config_dict={"temperature": 0.0, "top_p": 1},
)
-web_toolkit = BrowserNonVisualToolkit(
- headless=False, user_data_dir=USER_DATA_DIR
+# Example 1: Use default tools (basic functionality)
+# web_toolkit_default = HybridBrowserToolkit(
+# headless=False,
+# user_data_dir=USER_DATA_DIR
+# )
+# print(f"Default tools: {web_toolkit_default.enabled_tools}")
+
+# Example 2: Use all available tools
+# web_toolkit_all = HybridBrowserToolkit(
+# headless=False,
+# user_data_dir=USER_DATA_DIR,
+# enabled_tools=HybridBrowserToolkit.ALL_TOOLS
+# )
+# print(f"All tools: {web_toolkit_all.enabled_tools}")
+
+# Example 3: Use custom tools selection
+custom_tools = [
+ "open_browser",
+ "close_browser",
+ "visit_page",
+ "get_som_screenshot", # Add screenshot capability
+ "click",
+ "type",
+]
+
+web_toolkit_custom = HybridBrowserToolkit(
+ headless=False, user_data_dir=USER_DATA_DIR, enabled_tools=custom_tools
)
+print(f"Custom tools: {web_toolkit_custom.enabled_tools}")
+
+# Use the custom toolkit for the actual task
agent = ChatAgent(
model=model_backend,
- tools=[*web_toolkit.get_tools()],
+ tools=[*web_toolkit_custom.get_tools()],
max_iteration=10,
)
-TASK_PROMPT = """Play this game:
-https://www.nytimes.com/games/wordle/index.html
-You need to enter meaningful five-letter words, such as "Apple".
-Before completing the task, do not do anything outside of using the call tool.
-If the game tells you that something is wrong, you must resolve it yourself.
-Also, you must attempt all six tries.
-
-If the registration window can be closed, please close it.
+TASK_PROMPT = r"""
+Open Amazon.com, search for a beautiful keyboard,
+click it and add it to the cart
+and use get_som_screenshot to see the final
+picture of keyboard and describe its appearance
"""
async def main() -> None:
response = await agent.astep(TASK_PROMPT)
print("Task:", TASK_PROMPT)
- print(f"Using user data directory: {USER_DATA_DIR}\n")
- print("Response from agent:")
+ print(f"Using user data directory: {USER_DATA_DIR}")
+ print(f"Enabled tools: {web_toolkit_custom.enabled_tools}")
+ print("\nResponse from agent:")
print(response.msgs[0].content if response.msgs else "<no response>")
try:
- await web_toolkit.close_browser()
+ await web_toolkit_custom.close_browser()
except Exception as err:
- logging.warning("Failed to close browser session explicitly: %s", err)
+ logging.warning(
+ "Failed to close " "browser session explicitly: %s", err
+ )
if __name__ == "__main__":
diff --git a/examples/workforce/eigent.py b/examples/workforce/eigent.py
index b57bbeb11..38d1f0dbd 100644
--- a/examples/workforce/eigent.py
+++ b/examples/workforce/eigent.py
@@ -22,7 +22,6 @@ from camel.societies.workforce import Workforce
from camel.tasks.task import Task
from camel.toolkits import (
AudioAnalysisToolkit,
- BrowserNonVisualToolkit,
CodeExecutionToolkit,
Crawl4AIToolkit,
DalleToolkit,
@@ -32,6 +31,7 @@ from camel.toolkits import (
FunctionTool,
# GoogleDriveMCPToolkit,
HumanToolkit,
+ HybridBrowserToolkit,
ImageAnalysisToolkit,
LinkedInToolkit,
MarkItDownToolkit,
@@ -140,7 +140,7 @@ def search_agent_factory(
FunctionTool(SearchToolkit().search_exa),
# FunctionTool(SearchToolkit().search_bing),
# FunctionTool(SearchToolkit().search_baidu),
- *BrowserNonVisualToolkit(headless=False).get_tools(),
+ *HybridBrowserToolkit(headless=False).get_tools(),
*TerminalToolkit().get_tools(),
send_message_to_user,
HumanToolkit().ask_human_via_console,
diff --git a/pyproject.toml b/pyproject.toml
index 00d04629f..8e4e7ccc9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -601,4 +601,4 @@ skip = '.git*,*.pdf,*.svg,*.lock,*.css,examples/toolkits/semantic_scholar_toolki
check-hidden = true
# Ignore long lines as usually some kind of base64 encodings e.g. in ipynb
ignore-regex = '^\s*"image/\S+": ".*|^.{300,}|\b(NotIn|Unsupport)\b'
-ignore-words-list = 'ans,datas,rouge,doubleClick,OCE,ACI,aci,Pinter,'
+ignore-words-list = 'ans,datas,rouge,doubleClick,OCE,ACI,aci,Pinter,SoM,'
diff --git a/test/toolkits/test_file_write_toolkit.py b/test/toolkits/test_file_write_toolkit.py
index 5163cf424..03a9b630f 100644
--- a/test/toolkits/test_file_write_toolkit.py
+++ b/test/toolkits/test_file_write_toolkit.py
@@ -69,7 +69,9 @@ def test_write_to_file_text(file_write_toolkit):
content = "Hello, world!"
filename = "test.txt"
- result = file_write_toolkit.write_to_file(content, filename)
+ result = file_write_toolkit.write_to_file(
+ "Test Document", content, filename
+ )
# Check the result message
assert "successfully written" in result
@@ -89,7 +91,9 @@ def test_write_to_file_markdown(file_write_toolkit):
content = "# Heading\n\nThis is a paragraph with **bold** text."
filename = "test.md"
- result = file_write_toolkit.write_to_file(content, filename)
+ result = file_write_toolkit.write_to_file(
+ "Markdown Document", content, filename
+ )
# Check the result message
assert "successfully written" in result
@@ -109,7 +113,7 @@ def test_write_to_file_json(file_write_toolkit):
data = {"name": "John", "age": 30, "city": "New York"}
filename = "test.json"
- result = file_write_toolkit.write_to_file(data, filename)
+ result = file_write_toolkit.write_to_file("JSON Document", data, filename)
# Check the result message
assert "successfully written" in result
@@ -129,7 +133,7 @@ def test_write_to_file_yaml(file_write_toolkit):
data = {"name": "John", "age": 30, "city": "New York"}
filename = "test.yaml"
- result = file_write_toolkit.write_to_file(data, filename)
+ result = file_write_toolkit.write_to_file("YAML Document", data, filename)
# Check the result message
assert "successfully written" in result
@@ -149,7 +153,9 @@ def test_write_to_file_csv_string(file_write_toolkit):
content = "name,age,city\nJohn,30,New York\nJane,25,San Francisco"
filename = "test.csv"
- result = file_write_toolkit.write_to_file(content, filename)
+ result = file_write_toolkit.write_to_file(
+ "CSV Document", content, filename
+ )
# Check the result message
assert "successfully written" in result
@@ -173,7 +179,9 @@ def test_write_to_file_csv_list(file_write_toolkit):
]
filename = "test_list.csv"
- result = file_write_toolkit.write_to_file(content, filename)
+ result = file_write_toolkit.write_to_file(
+ "CSV List Document", content, filename
+ )
# Check the result message
assert "successfully written" in result
@@ -196,7 +204,9 @@ def test_write_to_file_html(file_write_toolkit):
content = "<html><body><h1>Hello, world!</h1></body></html>"
filename = "test.html"
- result = file_write_toolkit.write_to_file(content, filename)
+ result = file_write_toolkit.write_to_file(
+ "HTML Document", content, filename
+ )
# Check the result message
assert "successfully written" in result
@@ -218,7 +228,9 @@ def test_write_to_file_no_extension(file_write_toolkit):
content = "# Default format test"
filename = "test_no_extension"
- result = file_write_toolkit.write_to_file(content, filename)
+ result = file_write_toolkit.write_to_file(
+ "Default Format Document", content, filename
+ )
# Check the result message
assert "successfully written" in result
@@ -241,7 +253,7 @@ def test_write_to_file_custom_encoding(file_write_toolkit):
encoding = "latin-1"
result = file_write_toolkit.write_to_file(
- content, filename, encoding=encoding
+ "Custom Encoding Document", content, filename, encoding=encoding
)
# Check the result message
@@ -263,7 +275,9 @@ def test_write_to_file_nested_directory(file_write_toolkit):
content = "Content in nested directory"
filename = "nested/dir/test.txt"
- result = file_write_toolkit.write_to_file(content, filename)
+ result = file_write_toolkit.write_to_file(
+ "Nested Directory Document", content, filename
+ )
# Check the result message
assert "successfully written" in result
@@ -284,7 +298,7 @@ def test_write_to_file_absolute_path(temp_dir):
content = "Content with absolute path"
filename = os.path.join(temp_dir, "absolute_path.txt")
- result = toolkit.write_to_file(content, filename)
+ result = toolkit.write_to_file("Absolute Path Document", content, filename)
# Check the result message
assert "successfully written" in result
diff --git a/test/toolkits/test_hybrid_browser_toolkit.py b/test/toolkits/test_hybrid_browser_toolkit.py
new file mode 100644
index 000000000..4d36bb91f
--- /dev/null
+++ b/test/toolkits/test_hybrid_browser_toolkit.py
@@ -0,0 +1,1348 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import io
+import os
+import shutil
+from unittest.mock import AsyncMock, MagicMock, mock_open, patch
+
+import pytest
+from PIL import Image
+
+# Import the modules directly without global sys.modules mocking
+from camel.toolkits.hybrid_browser_toolkit import HybridBrowserToolkit
+from camel.utils.tool_result import ToolResult
+
+TEST_URL = "https://example.com"
+TEST_FILE_URL = "file:///test.html"
+
+
+def create_mock_playwright_objects():
+ """Create mock playwright objects for testing."""
+ # Create comprehensive mock objects
+ mock_page = AsyncMock()
+ mock_page.url = "https://example.com"
+ mock_page.goto = AsyncMock()
+ mock_page.wait_for_load_state = AsyncMock()
+ mock_page.screenshot = AsyncMock(return_value=b"fake_screenshot_data")
+ mock_page.evaluate = AsyncMock(return_value={"elements": {}})
+ mock_page.query_selector = AsyncMock(return_value=None)
+ mock_page.query_selector_all = AsyncMock(return_value=[])
+ mock_page.click = AsyncMock()
+ mock_page.fill = AsyncMock()
+ mock_page.select_option = AsyncMock()
+ mock_page.mouse = MagicMock()
+ mock_page.mouse.wheel = AsyncMock()
+
+ mock_context = AsyncMock()
+ mock_context.pages = [mock_page]
+ mock_context.new_page = AsyncMock(return_value=mock_page)
+ mock_context.close = AsyncMock()
+ mock_context.browser = None
+
+ mock_browser = AsyncMock()
+ mock_browser.new_context = AsyncMock(return_value=mock_context)
+ mock_browser.close = AsyncMock()
+ mock_browser.contexts = [mock_context]
+
+ mock_chromium = MagicMock()
+ mock_chromium.launch = AsyncMock(return_value=mock_browser)
+ mock_chromium.launch_persistent_context = AsyncMock(
+ return_value=mock_context
+ )
+
+ mock_playwright_instance = AsyncMock()
+ mock_playwright_instance.chromium = mock_chromium
+ mock_playwright_instance.firefox = mock_chromium
+ mock_playwright_instance.webkit = mock_chromium
+ mock_playwright_instance.stop = AsyncMock()
+
+ # Mock the async_playwright context manager
+ mock_async_playwright_cm = AsyncMock()
+ mock_async_playwright_cm.__aenter__ = AsyncMock(
+ return_value=mock_playwright_instance
+ )
+ mock_async_playwright_cm.__aexit__ = AsyncMock(return_value=None)
+
+ # Mock the async_playwright function
+ mock_async_playwright = MagicMock(return_value=mock_async_playwright_cm)
+
+ return (
+ mock_async_playwright,
+ mock_playwright_instance,
+ mock_page,
+ mock_context,
+ mock_browser,
+ )
+
+
+# Create global mock objects for backward compatibility
+(
+ mock_async_playwright,
+ mock_playwright_instance,
+ mock_page,
+ mock_context,
+ mock_browser,
+) = create_mock_playwright_objects()
+
+
[email protected](scope="function")
+def mock_playwright_setup():
+ """Set up playwright mocks for individual tests."""
+ return create_mock_playwright_objects()
+
+
[email protected](autouse=True)
+def mock_browser_dependencies():
+ """Mock all browser-related dependencies for each test."""
+ with (
+ patch('os.makedirs'),
+ patch('os.path.exists', return_value=True),
+ patch(
+ 'playwright.async_api.async_playwright',
+ return_value=mock_async_playwright(),
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.async_playwright',
+ return_value=mock_async_playwright(),
+ create=True,
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock unified analyzer');"),
+ ),
+ ):
+ yield
+
+
[email protected](scope="function")
+async def browser_toolkit_fixture():
+ """Create a HybridBrowserToolkit instance for testing."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open', mock_open(read_data="console.log('mock script');")
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session # Ensure the mock session is used
+ yield toolkit
+
+ # Cleanup
+ try:
+ await toolkit.close_browser()
+ except Exception:
+ pass
+
+ if os.path.exists("test_cache"):
+ shutil.rmtree("test_cache", ignore_errors=True)
+
+
[email protected](scope="function")
+def mock_session():
+ """Create a mock browser session."""
+ session = AsyncMock()
+ session.ensure_browser = AsyncMock()
+ session.get_page = AsyncMock(return_value=mock_page)
+ session.visit = AsyncMock(return_value="Visited test URL")
+ session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ session.close = AsyncMock()
+ return session
+
+
[email protected](scope="function")
+def mock_page_fixture():
+ """Create a mock page object."""
+ page = AsyncMock()
+ page.url = TEST_URL
+ page.screenshot = AsyncMock(return_value=b"fake_screenshot_data")
+ page.evaluate = AsyncMock(return_value={"elements": {}})
+ page.query_selector = AsyncMock(return_value=None)
+ page.query_selector_all = AsyncMock(return_value=[])
+ page.click = AsyncMock()
+ page.fill = AsyncMock()
+ page.select_option = AsyncMock()
+ page.mouse = MagicMock()
+ page.mouse.wheel = AsyncMock()
+ return page
+
+
[email protected](scope="function")
+def sync_browser_toolkit():
+ """Create a HybridBrowserToolkit instance for synchronous testing."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open', mock_open(read_data="console.log('mock script');")
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session # Ensure the mock session is used
+ return toolkit
+
+
+class TestHybridBrowserToolkit:
+ """Test cases for HybridBrowserToolkit."""
+
+ def test_init_default_params(self):
+ """Test initialization with default parameters."""
+ mock_session = AsyncMock()
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session'
+ '.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock unified analyzer');"),
+ ),
+ patch('os.path.exists', return_value=True),
+ patch('os.makedirs'),
+ ):
+ toolkit = HybridBrowserToolkit()
+
+ assert toolkit._headless is True
+ assert toolkit._user_data_dir is None
+ assert toolkit.web_agent_model is None
+ assert toolkit.cache_dir == "tmp/"
+ assert toolkit._agent is None
+ assert toolkit._unified_script is not None
+
+ def test_init_custom_params(self):
+ """Test initialization with custom parameters."""
+ test_cache_dir = "custom_cache"
+ test_user_data_dir = "custom_user_data"
+
+ mock_session = AsyncMock()
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session'
+ '.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock unified analyzer');"),
+ ),
+ patch('os.path.exists', return_value=True),
+ patch('os.makedirs'),
+ ):
+ toolkit = HybridBrowserToolkit(
+ headless=False,
+ user_data_dir=test_user_data_dir,
+ )
+
+ assert toolkit._headless is False
+ assert toolkit._user_data_dir == test_user_data_dir
+ assert toolkit.cache_dir == "tmp/" # Default cache dir
+
+ # Cleanup
+ if os.path.exists(test_cache_dir):
+ shutil.rmtree(test_cache_dir, ignore_errors=True)
+
+ def test_load_unified_analyzer_success(self):
+ """Test successful loading of unified analyzer script."""
+ mock_session = AsyncMock()
+ with patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session'
+ '.NVBrowserSession',
+ return_value=mock_session,
+ ):
+ with patch('builtins.open', create=True) as mock_file:
+ # ruff:noqa:E501
+ mock_file.return_value.__enter__.return_value.read.return_value = "console.log('test');"
+
+ with (
+ patch('os.makedirs'),
+ patch('os.path.exists', return_value=True),
+ ):
+ toolkit = HybridBrowserToolkit()
+ assert "console.log('test');" in toolkit._unified_script
+
+ def test_load_unified_analyzer_file_not_found(self):
+ """Test handling of missing unified analyzer script."""
+ mock_session = AsyncMock()
+ with patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session'
+ '.NVBrowserSession',
+ return_value=mock_session,
+ ):
+ with (
+ patch('builtins.open', side_effect=FileNotFoundError),
+ patch('os.makedirs'),
+ patch('os.path.exists', return_value=True),
+ ):
+ with pytest.raises(FileNotFoundError):
+ HybridBrowserToolkit()
+
+ def test_validate_ref_valid(self):
+ """Test ref validation with valid input."""
+ mock_session = AsyncMock()
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session'
+ '.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ patch('os.makedirs'),
+ patch('os.path.exists', return_value=True),
+ ):
+ toolkit = HybridBrowserToolkit()
+ # Should not raise exception
+ toolkit._validate_ref("e1", "test_method")
+
+ def test_validate_ref_invalid(self):
+ """Test ref validation with invalid input."""
+ mock_session = AsyncMock()
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session'
+ '.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ patch('os.makedirs'),
+ patch('os.path.exists', return_value=True),
+ ):
+ toolkit = HybridBrowserToolkit()
+
+ # Test with empty string
+ with pytest.raises(ValueError):
+ toolkit._validate_ref("", "test_method")
+
+ @pytest.mark.asyncio
+ async def test_open_browser_no_start_url(self):
+ """Test opening browser without start URL."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ result = await toolkit.open_browser()
+
+ assert result["result"] == "Browser session started."
+ assert "snapshot" in result
+
+ @pytest.mark.asyncio
+ async def test_open_browser_with_start_url(self):
+ """Test opening browser with start URL."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ # Mock visit_page method
+ with patch.object(
+ toolkit, 'visit_page', new_callable=AsyncMock
+ ) as mock_visit:
+ mock_visit.return_value = {
+ "result": "Visited",
+ "snapshot": "test",
+ }
+
+ await toolkit.open_browser(start_url=TEST_URL)
+
+ mock_visit.assert_called_once_with(TEST_URL)
+
+ @pytest.mark.asyncio
+ async def test_close_browser(self):
+ """Test closing browser."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ result = await toolkit.close_browser()
+
+ assert result == "Browser session closed."
+
+ @pytest.mark.asyncio
+ async def test_visit_page_valid_url(self):
+ """Test visiting a valid URL."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ result = await toolkit.visit_page(TEST_URL)
+
+ assert "Visited" in result["result"]
+ assert "snapshot" in result
+
+ @pytest.mark.asyncio
+ async def test_visit_page_invalid_url(self):
+ """Test visiting with invalid URL."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ result = await toolkit.visit_page("")
+ assert "Error" in result.get("result", "")
+ assert "'url' must be a non-empty string" in result.get(
+ "result", ""
+ )
+
+ @pytest.mark.asyncio
+ async def test_get_page_snapshot(self):
+ """Test getting page snapshot."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ # Mock unified analysis
+ mock_analysis = {
+ "snapshotText": "Test snapshot content",
+ "elements": {"e1": {"role": "button", "name": "Test"}},
+ }
+
+ # Ensure the mock page has the analysis
+ mock_page.evaluate = AsyncMock(return_value=mock_analysis)
+
+ result = await toolkit.get_page_snapshot()
+
+ assert result == "Test snapshot content"
+
+ @pytest.mark.asyncio
+ async def test_get_som_screenshot_success(self):
+ """Test successful Set of Marks screenshot generation."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ # Create a mock image
+ mock_image = Image.new('RGB', (100, 100), color='red')
+ img_byte_arr = io.BytesIO()
+ mock_image.save(img_byte_arr, format='PNG')
+ mock_image_data = img_byte_arr.getvalue()
+
+ # Set up the mock page with screenshot and analysis data
+ mock_page.screenshot = AsyncMock(return_value=mock_image_data)
+ mock_page.url = TEST_URL
+
+ # Mock unified analysis
+ mock_analysis = {
+ "elements": {
+ "e1": {
+ "role": "button",
+ "name": "Test Button",
+ "coordinates": [
+ {"x": 10, "y": 10, "width": 50, "height": 20}
+ ],
+ }
+ }
+ }
+ mock_page.evaluate = AsyncMock(return_value=mock_analysis)
+
+ result = await toolkit.get_som_screenshot()
+
+ assert isinstance(result, ToolResult)
+ assert "Visual webpage screenshot captured" in result.text
+ assert len(result.images) == 1
+ assert result.images[0].startswith("data:image/png;base64,")
+
+ @pytest.mark.asyncio
+ async def test_get_som_screenshot_pil_not_available(self):
+ """Test Set of Marks screenshot when PIL is not available."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ # Create a mock image for the screenshot
+ mock_image = Image.new('RGB', (100, 100), color='red')
+ img_byte_arr = io.BytesIO()
+ mock_image.save(img_byte_arr, format='PNG')
+ mock_image_data = img_byte_arr.getvalue()
+
+ # Set up the mock page with screenshot data
+ mock_page.screenshot = AsyncMock(return_value=mock_image_data)
+ mock_page.url = TEST_URL
+
+ # Mock unified analysis - provide empty elements to avoid coordinate processing
+ mock_analysis = {"elements": {}}
+ mock_page.evaluate = AsyncMock(return_value=mock_analysis)
+
+ with patch('PIL.ImageDraw', side_effect=ImportError):
+ result = await toolkit.get_som_screenshot()
+
+ assert isinstance(result, ToolResult)
+ assert "Visual webpage screenshot captured" in result.text
+
+ @pytest.mark.asyncio
+ async def test_click_valid_ref(self):
+ """Test clicking with valid reference."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ # Mock the _get_unified_analysis method to include e1 element
+ mock_analysis = {"elements": {"e1": {"role": "button"}}}
+ toolkit._get_unified_analysis = AsyncMock(
+ return_value=mock_analysis
+ )
+
+ result = await toolkit.click(ref="e1")
+
+ assert result["result"] == "Action executed successfully"
+ assert "snapshot" in result
+
+ @pytest.mark.asyncio
+ async def test_click_invalid_ref(self):
+ """Test clicking with invalid reference."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ with pytest.raises(ValueError):
+ await toolkit.click(ref="")
+
+ @pytest.mark.asyncio
+ async def test_type_text(self):
+ """Test typing text in element."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ result = await toolkit.type(
+ ref="input_element", text="Hello World"
+ )
+
+ assert result["result"] == "Action executed successfully"
+ assert "snapshot" in result
+
+ @pytest.mark.asyncio
+ async def test_select_option(self):
+ """Test selecting option in dropdown."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ result = await toolkit.select(
+ ref="select_element", value="option1"
+ )
+
+ assert result["result"] == "Action executed successfully"
+ assert "snapshot" in result
+
+ @pytest.mark.asyncio
+ async def test_scroll_valid_direction(self):
+ """Test scrolling with valid direction."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ result = await toolkit.scroll(direction="down", amount=100)
+
+ assert result["result"] == "Action executed successfully"
+ assert "snapshot" in result
+
+ @pytest.mark.asyncio
+ async def test_scroll_invalid_direction(self):
+ """Test scrolling with invalid direction."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ result = await toolkit.scroll(
+ direction="invalid_direction", amount=100
+ )
+ assert (
+ "Error: direction must be 'up' or 'down'" in result["result"]
+ )
+
+ @pytest.mark.asyncio
+ async def test_get_page_links_valid_refs(self):
+ """Test getting page links with valid references."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ # Mock page evaluate to return links
+ mock_links = [
+ {"href": "https://example.com/page1", "text": "Page 1"},
+ {"href": "https://example.com/page2", "text": "Page 2"},
+ ]
+ mock_page.evaluate = AsyncMock(return_value=mock_links)
+
+ result = await toolkit.get_page_links(ref=["e1", "e2"])
+
+ assert "links" in result
+ assert isinstance(result["links"], list)
+
+ @pytest.mark.asyncio
+ async def test_get_page_links_invalid_refs(self):
+ """Test getting page links with invalid references."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ result = await toolkit.get_page_links(ref=[])
+ assert result["links"] == []
+
+ @pytest.mark.asyncio
+ async def test_wait_user_no_timeout(self):
+ """Test waiting for user input without timeout."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ # Mock user input
+ with patch('builtins.input', return_value='continue'):
+ result = await toolkit.wait_user()
+
+ assert result["result"] == "User resumed."
+ assert "snapshot" in result
+
+ @pytest.mark.asyncio
+ async def test_wait_user_with_timeout(self):
+ """Test waiting for user input with timeout."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ # Mock asyncio.wait_for to simulate timeout
+ import asyncio
+
+ with patch('asyncio.wait_for', side_effect=asyncio.TimeoutError()):
+ result = await toolkit.wait_user(timeout_sec=1.0)
+
+ assert (
+ "Timeout 1.0s reached, auto-resumed." in result["result"]
+ )
+ assert "snapshot" in result
+
+ def test_get_tools(self, sync_browser_toolkit):
+ """Test getting available tools with default configuration."""
+ toolkit = sync_browser_toolkit
+
+ tools = toolkit.get_tools()
+
+ # Default tools should be 5 tools (updated default configuration)
+ assert len(tools) == 5
+ tool_names = [tool.func.__name__ for tool in tools]
+
+ expected_default_tools = [
+ 'open_browser',
+ 'close_browser',
+ 'visit_page',
+ 'click',
+ 'type',
+ ]
+
+ for expected_tool in expected_default_tools:
+ assert expected_tool in tool_names
+
+ def test_get_tools_with_web_agent_model(self, sync_browser_toolkit):
+ """Test getting tools when web_agent_model is provided."""
+ mock_model = MagicMock()
+ toolkit = sync_browser_toolkit
+
+ # Simulate having a web agent model
+ toolkit.web_agent_model = mock_model
+
+ tools = toolkit.get_tools()
+
+ # Default tools should be 5, even with web_agent_model set
+ # unless solve_task is explicitly enabled
+ assert len(tools) == 5
+ tool_names = [tool.func.__name__ for tool in tools]
+
+ # Should not automatically include solve_task unless explicitly enabled
+ assert 'solve_task' not in tool_names
+
+ def test_get_tools_with_solve_task_enabled(self):
+ """Test getting tools when solve_task is explicitly enabled."""
+ mock_model = MagicMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=AsyncMock(),
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(
+ headless=True,
+ web_agent_model=mock_model,
+ enabled_tools=['open_browser', 'close_browser', 'solve_task'],
+ )
+
+ tools = toolkit.get_tools()
+ tool_names = [tool.func.__name__ for tool in tools]
+
+ assert len(tools) == 3
+ assert 'solve_task' in tool_names
+ assert 'open_browser' in tool_names
+ assert 'close_browser' in tool_names
+
+ def test_get_tools_custom_selection(self):
+ """Test getting tools with custom tool selection."""
+ custom_tools = [
+ 'open_browser',
+ 'visit_page',
+ 'get_som_screenshot',
+ 'click',
+ 'scroll',
+ ]
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=AsyncMock(),
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(
+ headless=True, enabled_tools=custom_tools
+ )
+
+ tools = toolkit.get_tools()
+ tool_names = [tool.func.__name__ for tool in tools]
+
+ assert len(tools) == 5
+ for expected_tool in custom_tools:
+ assert expected_tool in tool_names
+
+ def test_convert_analysis_to_rects(self, sync_browser_toolkit):
+ """Test converting analysis data to rect format."""
+ toolkit = sync_browser_toolkit
+
+ analysis_data = {
+ "elements": {
+ "e1": {
+ "role": "button",
+ "name": "Submit",
+ "coordinates": [
+ {"x": 10, "y": 20, "width": 50, "height": 30}
+ ],
+ }
+ }
+ }
+
+ rects = toolkit._convert_analysis_to_rects(analysis_data)
+
+ assert "e1" in rects
+ assert rects["e1"]["role"] == "button"
+ assert rects["e1"]["aria-name"] == "Submit"
+ assert len(rects["e1"]["rects"]) == 1
+ assert rects["e1"]["rects"][0]["x"] == 10
+
+ def test_format_snapshot_from_analysis(self, sync_browser_toolkit):
+ """Test formatting snapshot from analysis data."""
+ toolkit = sync_browser_toolkit
+
+ analysis_data = {
+ "elements": {
+ "e1": {
+ "role": "button",
+ "name": "Submit",
+ "disabled": True,
+ },
+ "e2": {
+ "role": "checkbox",
+ "name": "Agree",
+ "checked": True,
+ },
+ }
+ }
+
+ snapshot = toolkit._format_snapshot_from_analysis(analysis_data)
+
+ assert "[ref=e1]" in snapshot
+ assert "button" in snapshot
+ assert "Submit" in snapshot
+ assert "disabled" in snapshot
+ assert "[ref=e2]" in snapshot
+ assert "checkbox" in snapshot
+ assert "checked" in snapshot
+
+ def test_add_set_of_mark_with_pil(self, sync_browser_toolkit):
+ """Test adding visual marks to image."""
+ toolkit = sync_browser_toolkit
+
+ # Create test image
+ test_image = Image.new('RGB', (200, 200), color='white')
+
+ # Test rects data
+ rects = {
+ "e1": {
+ "role": "button",
+ "rects": [{"x": 10, "y": 10, "width": 50, "height": 20}],
+ }
+ }
+
+ marked_image = toolkit._add_set_of_mark(test_image, rects)
+
+ assert isinstance(marked_image, Image.Image)
+ assert marked_image.size == test_image.size
+
+ def test_add_set_of_mark_without_pil(self, sync_browser_toolkit):
+ """Test adding visual marks when PIL is not available."""
+ toolkit = sync_browser_toolkit
+
+ test_image = Image.new('RGB', (200, 200), color='white')
+
+ with patch('PIL.ImageDraw', side_effect=ImportError):
+ result = toolkit._add_set_of_mark(test_image, {})
+
+ # Should return original image when PIL not available
+ assert result == test_image
+
+ @pytest.mark.asyncio
+ async def test_simple_async_creation(self):
+ """Simple test to verify async toolkit creation works with mocks."""
+ mock_session = AsyncMock()
+ mock_session.ensure_browser = AsyncMock()
+ mock_session.get_page = AsyncMock(return_value=mock_page)
+ mock_session.visit = AsyncMock(return_value="Visited test URL")
+ mock_session.get_snapshot = AsyncMock(
+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"
+ )
+ mock_session.exec_action = AsyncMock(
+ return_value="Action executed successfully"
+ )
+ mock_session.close = AsyncMock()
+
+ with (
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',
+ return_value=mock_session,
+ ),
+ patch(
+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',
+ return_value='mock_script_path',
+ ),
+ patch(
+ 'builtins.open',
+ mock_open(read_data="console.log('mock script');"),
+ ),
+ ):
+ toolkit = HybridBrowserToolkit(headless=True)
+ toolkit._session = mock_session
+
+ # Test a simple async operation
+ result = await toolkit.open_browser()
+
+ assert result["result"] == "Browser session started."
+ assert "snapshot" in result