🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: add som screenshot in non vision browser (#2759)
Puzhen Zhang7480c48fec38
diff --git a/camel/agents/_types.py b/camel/agents/_types.pyindex ad6881726..07a86f9cd 100644--- a/camel/agents/_types.py+++ b/camel/agents/_types.py@@ -14,6 +14,7 @@from typing import Any, Dict, List, Optional, Unionfrom openai import AsyncStream, Stream+from openai.types.chat import ChatCompletionChunkfrom pydantic import BaseModel, ConfigDictfrom camel.messages import BaseMessage@@ -32,8 +33,11 @@ class ModelResponse(BaseModel):r"""The response from the model."""model_config = ConfigDict(arbitrary_types_allowed=True)-- response: Union[ChatCompletion, Stream, AsyncStream]+ response: Union[+ ChatCompletion,+ Stream[ChatCompletionChunk],+ AsyncStream[ChatCompletionChunk],+ ]tool_call_requests: Optional[List[ToolCallRequest]]output_messages: List[BaseMessage]finish_reasons: List[str]diff --git a/camel/agents/chat_agent.py b/camel/agents/chat_agent.pyindex 5bc13d4f9..4ff43894b 100644--- a/camel/agents/chat_agent.py+++ b/camel/agents/chat_agent.py@@ -85,6 +85,7 @@ from camel.utils import (model_from_json_schema,)from camel.utils.commons import dependencies_required+from camel.utils.tool_result import ToolResultif TYPE_CHECKING:from camel.terminators import ResponseTerminator@@ -291,12 +292,17 @@ class ChatAgent(BaseAgent):self.stop_event = stop_eventself.mask_tool_output = mask_tool_outputself._secure_result_store: Dict[str, Any] = {}+ self._pending_images: List[str] = []+ self._image_retry_count: Dict[str, int] = {}+ # Store images to attach to next user messageself.pause_event = pause_eventdef reset(self):r"""Resets the :obj:`ChatAgent` to its initial state."""self.terminated = Falseself.init_messages()+ self._pending_images = []+ self._image_retry_count = {}for terminator in self.response_terminators:terminator.reset()@@ -1140,6 +1146,16 @@ class ChatAgent(BaseAgent):role_name="User", content=input_message)+ # Attach any pending images from previous tool calls+ image_list = self._process_pending_images()+ if image_list:+ # Create new message with images attached+ input_message = BaseMessage.make_user_message(+ role_name="User",+ content=input_message.content,+ image_list=image_list,+ )+# Add user input to memoryself.update_memory(input_message, OpenAIBackendRole.USER)@@ -1297,6 +1313,16 @@ class ChatAgent(BaseAgent):role_name="User", content=input_message)+ # Attach any pending images from previous tool calls+ image_list = self._process_pending_images()+ if image_list:+ # Create new message with images attached+ input_message = BaseMessage.make_user_message(+ role_name="User",+ content=input_message.content,+ image_list=image_list,+ )+self.update_memory(input_message, OpenAIBackendRole.USER)tool_call_records: List[ToolCallingRecord] = []@@ -1343,6 +1369,7 @@ class ChatAgent(BaseAgent):if tool_call_requests := response.tool_call_requests:# Process all tool calls+ new_images_from_tools = []for tool_call_request in tool_call_requests:if (tool_call_request.tool_name@@ -1362,10 +1389,72 @@ class ChatAgent(BaseAgent):)tool_call_records.append(tool_call_record)+ # Check if this tool call produced images+ if (+ hasattr(tool_call_record, 'images')+ and tool_call_record.images+ ):+ new_images_from_tools.extend(+ tool_call_record.images+ )+# If we found an external tool call, break the loopif external_tool_call_requests:break+ # If tools produced images+ # send them to the model as a user message+ if new_images_from_tools:+ # Convert base64 images to PIL Images+ image_list = []+ for img_data in new_images_from_tools:+ try:+ import base64+ import io++ from PIL import Image++ # Extract base64 data from data URL format+ if img_data.startswith("data:image"):+ # Format:+ # "data:image/png;base64,iVBORw0KGgo..."+ base64_data = img_data.split(',', 1)[1]+ else:+ # Raw base64 data+ base64_data = img_data++ # Decode and create PIL Image+ image_bytes = base64.b64decode(base64_data)+ pil_image = Image.open(io.BytesIO(image_bytes))+ # Convert to ensure proper+ # Image.Image type for compatibility+ pil_image_tool_result: Image.Image = (+ pil_image.convert('RGB')+ )+ image_list.append(pil_image_tool_result)++ except Exception as e:+ logger.warning(+ f"Failed to convert "+ f"base64 image to PIL for immediate use: {e}"+ )+ continue++ # If we have valid images+ # create a user message with images+ if image_list:+ # Create a user message with images+ # to provide visual context immediately+ image_message = BaseMessage.make_user_message(+ role_name="User",+ content="[Visual content from tool execution - please analyze and continue]", # noqa: E501+ image_list=image_list,+ )++ self.update_memory(+ image_message, OpenAIBackendRole.USER+ )+if (self.max_iteration is not Noneand iteration_count >= self.max_iteration@@ -1452,6 +1541,69 @@ class ChatAgent(BaseAgent):info=info,)+ def _process_pending_images(self) -> List:+ r"""Process pending images with retry logic and return PIL Image list.++ Returns:+ List: List of successfully converted PIL Images.+ """+ if not self._pending_images:+ return []++ image_list = []+ successfully_processed = []+ failed_images = []++ for img_data in self._pending_images:+ # Track retry count+ retry_count = self._image_retry_count.get(img_data, 0)++ # Remove images that have failed too many times (max 3 attempts)+ if retry_count >= 3:+ failed_images.append(img_data)+ logger.warning(+ f"Removing image after {retry_count} failed attempts"+ )+ continue++ try:+ import base64+ import io++ from PIL import Image++ # Extract base64 data from data URL format+ if img_data.startswith("data:image"):+ # Format: "data:image/png;base64,iVBORw0KGgo..."+ base64_data = img_data.split(',', 1)[1]+ else:+ # Raw base64 data+ base64_data = img_data++ # Decode and create PIL Image+ image_bytes = base64.b64decode(base64_data)+ pil_image = Image.open(io.BytesIO(image_bytes))+ pil_image_converted: Image.Image = pil_image.convert('RGB')+ image_list.append(pil_image_converted)+ successfully_processed.append(img_data)++ except Exception as e:+ # Increment retry count for failed conversion+ self._image_retry_count[img_data] = retry_count + 1+ logger.warning(+ f"Failed to convert base64 image to PIL "+ f"(attempt {retry_count + 1}/3): {e}"+ )+ continue++ # Clean up processed and failed images+ for img in successfully_processed + failed_images:+ self._pending_images.remove(img)+ # Clean up retry count for processed/removed images+ self._image_retry_count.pop(img, None)++ return image_list+def _record_final_output(self, output_messages: List[BaseMessage]) -> None:r"""Log final messages or warnings about multiple responses."""if len(output_messages) == 1:@@ -1462,6 +1614,61 @@ class ChatAgent(BaseAgent):"selected message manually using `record_message()`.")+ def _is_vision_error(self, exc: Exception) -> bool:+ r"""Check if the exception is likely related to vision/image is not+ supported by the model."""+ # TODO: more robust vision error detection+ error_msg = str(exc).lower()+ vision_keywords = [+ 'vision',+ 'image',+ 'multimodal',+ 'unsupported',+ 'invalid content type',+ 'image_url',+ 'visual',+ ]+ return any(keyword in error_msg for keyword in vision_keywords)++ def _has_images(self, messages: List[OpenAIMessage]) -> bool:+ r"""Check if any message contains images."""+ for msg in messages:+ content = msg.get('content')+ if isinstance(content, list):+ for item in content:+ if (+ isinstance(item, dict)+ and item.get('type') == 'image_url'+ ):+ return True+ return False++ def _strip_images_from_messages(+ self, messages: List[OpenAIMessage]+ ) -> List[OpenAIMessage]:+ r"""Remove images from messages, keeping only text content."""+ stripped_messages = []+ for msg in messages:+ content = msg.get('content')+ if isinstance(content, list):+ # Extract only text content from multimodal messages+ text_content = ""+ for item in content:+ if isinstance(item, dict) and item.get('type') == 'text':+ text_content += item.get('text', '')++ # Create new message with only text content+ new_msg = msg.copy()+ new_msg['content'] = (+ text_content+ or "[Image content removed - model doesn't support vision]"+ )+ stripped_messages.append(new_msg)+ else:+ # Regular text message, keep as is+ stripped_messages.append(msg)+ return stripped_messages+def _get_model_response(self,openai_messages: List[OpenAIMessage],@@ -1477,13 +1684,33 @@ class ChatAgent(BaseAgent):openai_messages, response_format, tool_schemas or None)except Exception as exc:- logger.error(- f"An error occurred while running model "- f"{self.model_backend.model_type}, "- f"index: {self.model_backend.current_model_index}",- exc_info=exc,- )- error_info = str(exc)+ # Try again without images if the error might be vision-related+ if self._is_vision_error(exc) and self._has_images(+ openai_messages+ ):+ logger.warning(+ "Model appears to not support vision. Retrying without images." # noqa: E501+ )+ try:+ stripped_messages = self._strip_images_from_messages(+ openai_messages+ )+ response = self.model_backend.run(+ stripped_messages,+ response_format,+ tool_schemas or None,+ )+ except Exception:+ pass # Fall through to original error handling++ if not response:+ logger.error(+ f"An error occurred while running model "+ f"{self.model_backend.model_type}, "+ f"index: {self.model_backend.current_model_index}",+ exc_info=exc,+ )+ error_info = str(exc)if not response and self.model_backend.num_models > 1:raise ModelProcessingError(@@ -1525,13 +1752,33 @@ class ChatAgent(BaseAgent):openai_messages, response_format, tool_schemas or None)except Exception as exc:- logger.error(- f"An error occurred while running model "- f"{self.model_backend.model_type}, "- f"index: {self.model_backend.current_model_index}",- exc_info=exc,- )- error_info = str(exc)+ # Try again without images if the error might be vision-related+ if self._is_vision_error(exc) and self._has_images(+ openai_messages+ ):+ logger.warning(+ "Model appears to not support vision. Retrying without images." # noqa: E501+ )+ try:+ stripped_messages = self._strip_images_from_messages(+ openai_messages+ )+ response = await self.model_backend.arun(+ stripped_messages,+ response_format,+ tool_schemas or None,+ )+ except Exception:+ pass # Fall through to original error handling++ if not response:+ logger.error(+ f"An error occurred while running model "+ f"{self.model_backend.model_type}, "+ f"index: {self.model_backend.current_model_index}",+ exc_info=exc,+ )+ error_info = str(exc)if not response and self.model_backend.num_models > 1:raise ModelProcessingError(@@ -2005,10 +2252,26 @@ class ChatAgent(BaseAgent):mask_flag = Falselogging.warning(error_msg)- return self._record_tool_calling(+ # Check if result is a ToolResult with images+ images_to_attach = None+ if isinstance(result, ToolResult):+ images_to_attach = result.images+ result = str(result) # Use string representation for storage++ tool_record = self._record_tool_calling(func_name, args, result, tool_call_id, mask_output=mask_flag)+ # Store images for later attachment to next user message+ if images_to_attach:+ tool_record.images = images_to_attach+ # Add images with duplicate prevention+ for img in images_to_attach:+ if img not in self._pending_images:+ self._pending_images.append(img)++ return tool_record+async def _aexecute_tool(self,tool_call_request: ToolCallRequest,@@ -2049,7 +2312,25 @@ class ChatAgent(BaseAgent):result = {"error": error_msg}logging.warning(error_msg)- return self._record_tool_calling(func_name, args, result, tool_call_id)+ # Check if result is a ToolResult with images+ images_to_attach = None+ if isinstance(result, ToolResult):+ images_to_attach = result.images+ result = str(result) # Use string representation for storage++ tool_record = self._record_tool_calling(+ func_name, args, result, tool_call_id+ )++ # Store images for later attachment to next user message+ if images_to_attach:+ tool_record.images = images_to_attach+ # Add images with duplicate prevention+ for img in images_to_attach:+ if img not in self._pending_images:+ self._pending_images.append(img)++ return tool_recorddef _record_tool_calling(self,diff --git a/camel/messages/base.py b/camel/messages/base.pyindex 9dd379831..3c78efde8 100644--- a/camel/messages/base.py+++ b/camel/messages/base.py@@ -437,12 +437,8 @@ class BaseMessage:if self.image_list and len(self.image_list) > 0:for image in self.image_list:if image.format is None:- raise ValueError(- f"Image's `format` is `None`, please "- f"transform the `PIL.Image.Image` to one of "- f"following supported formats, such as "- f"{list(OpenAIImageType)}"- )+ # Set default format to PNG as fallback+ image.format = 'PNG'image_type: str = image.format.lower()if image_type not in OpenAIImageType:diff --git a/camel/toolkits/__init__.py b/camel/toolkits/__init__.pyindex 3fdb08cc6..f4f797123 100644--- a/camel/toolkits/__init__.py+++ b/camel/toolkits/__init__.py@@ -77,7 +77,7 @@ from .aci_toolkit import ACIToolkitfrom .playwright_mcp_toolkit import PlaywrightMCPToolkitfrom .wolfram_alpha_toolkit import WolframAlphaToolkitfrom .task_planning_toolkit import TaskPlanningToolkit-from .non_visual_browser_toolkit import BrowserNonVisualToolkit+from .hybrid_browser_toolkit import HybridBrowserToolkitfrom .edgeone_pages_mcp_toolkit import EdgeOnePagesMCPToolkitfrom .google_drive_mcp_toolkit import GoogleDriveMCPToolkitfrom .craw4ai_toolkit import Crawl4AIToolkit@@ -146,7 +146,7 @@ __all__ = ['WolframAlphaToolkit','BohriumToolkit','TaskPlanningToolkit',- 'BrowserNonVisualToolkit',+ 'HybridBrowserToolkit','EdgeOnePagesMCPToolkit','GoogleDriveMCPToolkit','Crawl4AIToolkit',diff --git a/camel/toolkits/non_visual_browser_toolkit/__init__.py b/camel/toolkits/hybrid_browser_toolkit/__init__.pysimilarity index 88%rename from camel/toolkits/non_visual_browser_toolkit/__init__.pyrename to camel/toolkits/hybrid_browser_toolkit/__init__.pyindex 0fff38941..b3e336c9e 100644--- a/camel/toolkits/non_visual_browser_toolkit/__init__.py+++ b/camel/toolkits/hybrid_browser_toolkit/__init__.py@@ -11,8 +11,8 @@# See the License for the specific language governing permissions and# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========-from .browser_non_visual_toolkit import BrowserNonVisualToolkit+from .hybrid_browser_toolkit import HybridBrowserToolkit__all__ = [- "BrowserNonVisualToolkit",+ "HybridBrowserToolkit",]diff --git a/camel/toolkits/non_visual_browser_toolkit/actions.py b/camel/toolkits/hybrid_browser_toolkit/actions.pysimilarity index 76%rename from camel/toolkits/non_visual_browser_toolkit/actions.pyrename to camel/toolkits/hybrid_browser_toolkit/actions.pyindex 4c5cb3799..1054ccfa7 100644--- a/camel/toolkits/non_visual_browser_toolkit/actions.py+++ b/camel/toolkits/hybrid_browser_toolkit/actions.py@@ -24,6 +24,7 @@ class ActionExecutor:# Configuration constantsDEFAULT_TIMEOUT = 5000 # 5 secondsSHORT_TIMEOUT = 2000 # 2 seconds+ MAX_SCROLL_AMOUNT = 5000 # Maximum scroll distance in pixelsdef __init__(self, page: "Page"):self.page = page@@ -32,6 +33,7 @@ class ActionExecutor:# Public helpers# ------------------------------------------------------------------async def execute(self, action: Dict[str, Any]) -> str:+ r"""Execute an action and return the result description."""if not action:return "No action to execute"@@ -64,32 +66,46 @@ class ActionExecutor:# Internal handlers# ------------------------------------------------------------------async def _click(self, action: Dict[str, Any]) -> str:+ r"""Handle click actions with multiple fallback strategies."""ref = action.get("ref")text = action.get("text")selector = action.get("selector")if not (ref or text or selector):return "Error: click requires ref/text/selector"+ # Build strategies in priority order: ref > selector > textstrategies = []+ if ref:+ strategies.append(f"[aria-ref='{ref}']")if selector:strategies.append(selector)if text:strategies.append(f'text="{text}"')- if ref:- strategies.append(f"[aria-ref='{ref}']")+ # Strategy 1: Try Playwright force click for each selectorfor sel in strategies:try:if await self.page.locator(sel).count() > 0:await self.page.click(- sel, timeout=self.SHORT_TIMEOUT, force=True+ sel, timeout=self.DEFAULT_TIMEOUT, force=True)- return f"Clicked element via {sel}"+ return f"Clicked element via force: {sel}"except Exception:- pass- return "Error: Could not click element"+ continue++ # Strategy 2: Try JavaScript click as fallback+ for sel in strategies:+ try:+ await self.page.locator(sel).first.evaluate("el => el.click()")+ await asyncio.sleep(0.1) # Brief wait for effects+ return f"Clicked element via JS: {sel}"+ except Exception:+ continue++ return "Error: All click strategies failed"async def _type(self, action: Dict[str, Any]) -> str:+ r"""Handle typing text into input fields."""ref = action.get("ref")selector = action.get("selector")text = action.get("text", "")@@ -103,6 +119,7 @@ class ActionExecutor:return f"Type failed: {exc}"async def _select(self, action: Dict[str, Any]) -> str:+ r"""Handle selecting options from dropdowns."""ref = action.get("ref")selector = action.get("selector")value = action.get("value", "")@@ -118,8 +135,9 @@ class ActionExecutor:return f"Select failed: {exc}"async def _wait(self, action: Dict[str, Any]) -> str:+ r"""Handle wait actions."""if "timeout" in action:- ms = action["timeout"]+ ms = int(action["timeout"])await asyncio.sleep(ms / 1000)return f"Waited {ms}ms"if "selector" in action:@@ -131,6 +149,7 @@ class ActionExecutor:return "Error: wait requires timeout/selector"async def _extract(self, action: Dict[str, Any]) -> str:+ r"""Handle text extraction from elements."""ref = action.get("ref")if not ref:return "Error: extract requires ref"@@ -140,6 +159,7 @@ class ActionExecutor:return f"Extracted: {txt[:100] if txt else 'None'}"async def _scroll(self, action: Dict[str, Any]) -> str:+ r"""Handle page scrolling with safe parameter validation."""direction = action.get("direction", "down")amount = action.get("amount", 300)@@ -151,18 +171,22 @@ class ActionExecutor:# Safely convert amount to integer and clamp to reasonable rangeamount_int = int(amount)amount_int = max(- -5000, min(5000, amount_int)- ) # Clamp between -5000 and 5000+ -self.MAX_SCROLL_AMOUNT,+ min(self.MAX_SCROLL_AMOUNT, amount_int),+ ) # Clamp to MAX_SCROLL_AMOUNT rangeexcept (ValueError, TypeError):return "Error: amount must be a valid number"# Use safe evaluation with bound parametersscroll_offset = amount_int if direction == "down" else -amount_int- await self.page.evaluate(f"window.scrollBy(0, {scroll_offset})")+ await self.page.evaluate(+ "offset => window.scrollBy(0, offset)", scroll_offset+ )await asyncio.sleep(0.5)return f"Scrolled {direction} by {abs(amount_int)}px"async def _enter(self, action: Dict[str, Any]) -> str:+ r"""Handle Enter key press actions."""ref = action.get("ref")selector = action.get("selector")if ref:@@ -175,16 +199,28 @@ class ActionExecutor:# utilitiesasync def _wait_dom_stable(self) -> None:+ r"""Wait for DOM to become stable before executing actions."""try:+ # Wait for basic DOM content loadingawait self.page.wait_for_load_state('domcontentloaded', timeout=self.SHORT_TIMEOUT)++ # Try to wait for network idle briefly+ try:+ await self.page.wait_for_load_state(+ 'networkidle', timeout=self.SHORT_TIMEOUT+ )+ except Exception:+ pass # Network idle is optional+except Exception:- pass+ pass # Don't fail if wait times out# static helpers@staticmethoddef should_update_snapshot(action: Dict[str, Any]) -> bool:+ r"""Determine if an action requires a snapshot update."""change_types = {"click","type",diff --git a/camel/toolkits/non_visual_browser_toolkit/agent.py b/camel/toolkits/hybrid_browser_toolkit/agent.pysimilarity index 89%rename from camel/toolkits/non_visual_browser_toolkit/agent.pyrename to camel/toolkits/hybrid_browser_toolkit/agent.pyindex 1e166b0cf..c7b5bb68a 100644--- a/camel/toolkits/non_visual_browser_toolkit/agent.py+++ b/camel/toolkits/hybrid_browser_toolkit/agent.py@@ -12,24 +12,24 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========import json-import loggingimport refrom typing import TYPE_CHECKING, Any, Dict, List, Optional+from camel.logger import get_loggerfrom camel.models import BaseModelBackend, ModelFactoryfrom camel.types import ModelPlatformType, ModelTypefrom .actions import ActionExecutor-from .nv_browser_session import NVBrowserSession+from .browser_session import NVBrowserSessionif TYPE_CHECKING:from camel.agents import ChatAgent-logger = logging.getLogger(__name__)+logger = get_logger(__name__)class PlaywrightLLMAgent:- """High-level orchestration: snapshot ↔ LLM ↔ action executor."""+ r"""High-level orchestration: snapshot ↔ LLM ↔ action executor."""# System prompt as class constant to avoid recreationSYSTEM_PROMPT = """@@ -90,8 +90,8 @@ what was accomplishedself.action_history: List[Dict[str, Any]] = []if model_backend is None:model_backend = ModelFactory.create(- model_platform=ModelPlatformType.OPENAI,- model_type=ModelType.GPT_4O_MINI,+ model_platform=ModelPlatformType.DEFAULT,+ model_type=ModelType.DEFAULT,model_config_dict={"temperature": 0, "top_p": 1},)self.model_backend = model_backend@@ -99,16 +99,19 @@ what was accomplishedself._chat_agent: Optional[ChatAgent] = Noneasync def navigate(self, url: str) -> str:+ r"""Navigate to a URL and return the snapshot."""try:# NVBrowserSession handles waits internallylogger.debug("Navigated to URL: %s", url)await self._session.visit(url)return await self._session.get_snapshot(force_refresh=True)except Exception as exc:- return f"Error: could not navigate - {exc}"+ error_msg = f"Error: could not navigate to {url} - {exc}"+ logger.error(error_msg)+ return error_msgdef _get_chat_agent(self) -> "ChatAgent":- """Get or create the ChatAgent instance."""+ r"""Get or create the ChatAgent instance."""from camel.agents import ChatAgentif self._chat_agent is None:@@ -165,12 +168,16 @@ what was accomplishedlogger.warning("Could not parse JSON from LLM response: %s", content[:200])+ return self._get_fallback_response("Parsing error")++ def _get_fallback_response(self, error_msg: str) -> Dict[str, Any]:+ r"""Generate a fallback response structure."""return {- "plan": ["Could not parse response"],+ "plan": [f"Could not parse response: {error_msg}"],"action": {"type": "finish","ref": None,- "summary": "Parsing error",+ "summary": f"Parsing error: {error_msg}",},}@@ -181,7 +188,7 @@ what was accomplishedis_initial: bool,history: Optional[List[Dict[str, Any]]] = None,) -> Dict[str, Any]:- """Call the LLM (via CAMEL ChatAgent) to get plan & next action."""+ r"""Call the LLM (via CAMEL ChatAgent) to get plan & next action."""# Build user messageif is_initial:user_content = f"Snapshot:\n{snapshot}\n\nTask: {prompt}"@@ -208,6 +215,7 @@ what was accomplishedreturn self._safe_parse_json(content)async def process_command(self, prompt: str, max_steps: int = 15):+ r"""Process a command using LLM-guided browser automation."""# initial full snapshotfull_snapshot = await self._session.get_snapshot()assert self._session.snapshot is not None@@ -270,9 +278,11 @@ what was accomplishedlogger.info("Process completed with %d steps", steps)async def _run_action(self, action: Dict[str, Any]) -> str:+ r"""Execute a single action and return the result."""if action.get("type") == "navigate":return await self.navigate(action.get("url", ""))return await self._session.exec_action(action)async def close(self):+ r"""Clean up browser session and resources."""await self._session.close()diff --git a/camel/toolkits/non_visual_browser_toolkit/nv_browser_session.py b/camel/toolkits/hybrid_browser_toolkit/browser_session.pysimilarity index 74%rename from camel/toolkits/non_visual_browser_toolkit/nv_browser_session.pyrename to camel/toolkits/hybrid_browser_toolkit/browser_session.pyindex 2b4e6c14c..cafcfaf3b 100644--- a/camel/toolkits/non_visual_browser_toolkit/nv_browser_session.py+++ b/camel/toolkits/hybrid_browser_toolkit/browser_session.py@@ -57,13 +57,12 @@ class NVBrowserSession:def __new__(cls, *, headless: bool = True, user_data_dir: Optional[str] = None- ):- loop = asyncio.get_running_loop()- if loop not in cls._sessions:- instance = super().__new__(cls)- instance._initialized = False- cls._sessions[loop] = instance- return cls._sessions[loop]+ ) -> "NVBrowserSession":+ # Defer event loop lookup until we actually need it+ # This allows creation outside of async context+ instance = super().__new__(cls)+ instance._initialized = False+ return instancedef __init__(self, *, headless: bool = True, user_data_dir: Optional[str] = None@@ -90,6 +89,47 @@ class NVBrowserSession:# Browser lifecycle helpers# ------------------------------------------------------------------async def ensure_browser(self) -> None:+ r"""Ensure browser is ready, implementing singleton pattern per event+ loop.+ """+ # Check if we need to reuse or create a session for this event loop+ try:+ loop = asyncio.get_running_loop()+ except RuntimeError as e:+ raise RuntimeError(+ "ensure_browser() must be called from within an async context"+ ) from e++ # Check if there's already a session for this loop+ if loop in self._sessions and self._sessions[loop] is not self:+ # Copy the existing session's browser resources+ existing = self._sessions[loop]+ # Wait for existing session to be fully initialized+ async with existing._ensure_lock:+ if (+ existing._initialized+ and existing._page is not None+ and existing._playwright is not None+ ):+ try:+ # Verify the page is still responsive+ await existing._page.title()+ self._playwright = existing._playwright+ self._browser = existing._browser+ self._context = existing._context+ self._page = existing._page+ self.snapshot = existing.snapshot+ self.executor = existing.executor+ self._initialized = True+ return+ except Exception:+ # Existing session is broken, continue with new+ # initialization+ pass++ # Register this instance for the current loop+ self._sessions[loop] = self+# Serialise initialisation to avoid race conditions where multiple# concurrent coroutine calls create multiple browser instances for# the same NVBrowserSession.@@ -98,6 +138,7 @@ class NVBrowserSession:# Moved original logic to helperasync def _ensure_browser_inner(self) -> None:+ r"""Internal browser initialization logic."""from playwright.async_api import async_playwrightif self._page is not None:@@ -144,11 +185,23 @@ class NVBrowserSession:r"""Close all browser resources, ensuring cleanup even if someoperations fail."""- # The close method will now only close the *current* event-loop's- # browser instance. Use `close_all_sessions` for a full cleanup.+ # Remove this session from the sessions dict and close resources+ try:+ loop = asyncio.get_running_loop()+ if loop in self._sessions and self._sessions[loop] is self:+ del self._sessions[loop]+ except RuntimeError:+ pass # No running loop, that's okay++ # Clean up any stale loop references+ stale_loops = [loop for loop in self._sessions if loop.is_closed()]+ for loop in stale_loops:+ del self._sessions[loop]+await self._close_session()async def _close_session(self) -> None:+ r"""Internal session cleanup with comprehensive error handling."""errors: list[str] = []# Close context first (which closes pages)@@ -204,6 +257,7 @@ class NVBrowserSession:# Convenience wrappers around common actions# ------------------------------------------------------------------async def visit(self, url: str) -> str:+ r"""Navigate to a URL with proper error handling."""await self.ensure_browser()assert self._page is not None@@ -233,7 +287,7 @@ class NVBrowserSession:force_refresh=force_refresh, diff_only=diff_only)- async def exec_action(self, action: dict[str, Any]) -> str:+ async def exec_action(self, action: Dict[str, Any]) -> str:await self.ensure_browser()assert self.executor is not Nonereturn await self.executor.execute(action)diff --git a/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.pynew file mode 100644index 000000000..c27860bdf--- /dev/null+++ b/camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py@@ -0,0 +1,1002 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import base64+import datetime+import io+import os+import time+import urllib.parse+from typing import Any, Callable, ClassVar, Dict, List, Optional, cast++from camel.logger import get_logger+from camel.models import BaseModelBackend+from camel.toolkits.base import BaseToolkit+from camel.toolkits.function_tool import FunctionTool+from camel.utils import sanitize_filename+from camel.utils.commons import dependencies_required++from .agent import PlaywrightLLMAgent+from .browser_session import NVBrowserSession++logger = get_logger(__name__)+++class HybridBrowserToolkit(BaseToolkit):+ r"""A hybrid browser toolkit that combines non-visual, DOM-based browser+ automation with visual, screenshot-based capabilities.++ This toolkit exposes a set of actions as CAMEL FunctionTools for agents+ to interact with web pages. It can operate in headless mode and supports+ both programmatic control of browser actions (like clicking and typing)+ and visual analysis of the page layout through screenshots with marked+ interactive elements.+ """++ # Configuration constants+ DEFAULT_SCREENSHOT_TIMEOUT = 60000 # 60 seconds for screenshots+ PAGE_STABILITY_TIMEOUT = 3000 # 3 seconds for DOM stability+ NETWORK_IDLE_TIMEOUT = 2000 # 2 seconds for network idle++ # Default tool list - core browser functionality+ DEFAULT_TOOLS: ClassVar[List[str]] = [+ "open_browser",+ "close_browser",+ "visit_page",+ "click",+ "type",+ ]++ # All available tools+ ALL_TOOLS: ClassVar[List[str]] = [+ "open_browser",+ "close_browser",+ "visit_page",+ "get_page_snapshot",+ "get_som_screenshot",+ "get_page_links",+ "click",+ "type",+ "select",+ "scroll",+ "enter",+ "wait_user",+ "solve_task",+ ]++ def __init__(+ self,+ *,+ headless: bool = True,+ user_data_dir: Optional[str] = None,+ web_agent_model: Optional[BaseModelBackend] = None,+ cache_dir: str = "tmp/",+ enabled_tools: Optional[List[str]] = None,+ ) -> None:+ r"""Initialize the HybridBrowserToolkit.++ Args:+ headless (bool): Whether to run the browser in headless mode.+ Defaults to `True`.+ user_data_dir (Optional[str]): Path to a directory for storing+ browser data like cookies and local storage. Useful for+ maintaining sessions across runs. Defaults to `None` (a+ temporary directory is used).+ web_agent_model (Optional[BaseModelBackend]): The language model+ backend to use for the high-level `solve_task` agent. This is+ required only if you plan to use `solve_task`.+ Defaults to `None`.+ cache_dir (str): The directory to store cached files, such as+ screenshots. Defaults to `"tmp/"`.+ enabled_tools (Optional[List[str]]): List of tool names to enable.+ If None, uses DEFAULT_TOOLS. Available tools: open_browser,+ close_browser, visit_page, get_page_snapshot,+ get_som_screenshot, get_page_links, click, type, select,+ scroll, enter, wait_user, solve_task.+ Defaults to `None`.+ """+ super().__init__()+ self._headless = headless+ self._user_data_dir = user_data_dir+ self.web_agent_model = web_agent_model+ self.cache_dir = cache_dir+ os.makedirs(self.cache_dir, exist_ok=True)++ # Configure enabled tools+ if enabled_tools is None:+ self.enabled_tools = self.DEFAULT_TOOLS.copy()+ else:+ # Validate enabled tools+ invalid_tools = [+ tool for tool in enabled_tools if tool not in self.ALL_TOOLS+ ]+ if invalid_tools:+ raise ValueError(+ f"Invalid tools specified: {invalid_tools}. "+ f"Available tools: {self.ALL_TOOLS}"+ )+ self.enabled_tools = enabled_tools.copy()++ logger.info(f"Enabled tools: {self.enabled_tools}")++ # Core components+ self._session = NVBrowserSession(+ headless=headless, user_data_dir=user_data_dir+ )+ self._agent: Optional[PlaywrightLLMAgent] = None+ self._unified_script = self._load_unified_analyzer()++ def __del__(self):+ r"""Cleanup browser resources on garbage collection."""+ try:+ import sys++ if getattr(sys, "is_finalizing", lambda: False)():+ return++ import asyncio++ try:+ loop = asyncio.get_event_loop()+ if not loop.is_closed() and not loop.is_running():+ loop.run_until_complete(self.close_browser())+ except (RuntimeError, ImportError):+ pass # Event loop unavailable, skip cleanup+ except Exception:+ pass # Suppress all errors during garbage collection++ def _load_unified_analyzer(self) -> str:+ r"""Load the unified analyzer JavaScript script."""+ script_path = os.path.join(+ os.path.dirname(os.path.abspath(__file__)), "unified_analyzer.js"+ )++ try:+ with open(+ script_path, "r", encoding='utf-8', errors='replace'+ ) as f:+ script_content = f.read()++ if not script_content.strip():+ raise ValueError(f"Script is empty: {script_path}")++ logger.debug(+ f"Loaded unified analyzer ({len(script_content)} chars)"+ )+ return script_content+ except FileNotFoundError:+ raise FileNotFoundError(f"Script not found: {script_path}")++ def _validate_ref(self, ref: str, method_name: str) -> None:+ r"""Validate ref parameter."""+ if not ref or not isinstance(ref, str):+ raise ValueError(+ f"{method_name}: 'ref' must be a non-empty string"+ )++ async def _ensure_browser(self):+ await self._session.ensure_browser()++ async def _require_page(self):+ await self._session.ensure_browser()+ return await self._session.get_page()++ async def _wait_for_page_stability(self):+ r"""Wait for page to become stable after actions that might trigger+ updates.+ """+ page = await self._require_page()+ import asyncio++ try:+ # Wait for DOM content to be loaded+ await page.wait_for_load_state(+ 'domcontentloaded', timeout=self.PAGE_STABILITY_TIMEOUT+ )+ logger.debug("DOM content loaded")++ # Try to wait for network idle (important for AJAX/SPA)+ try:+ await page.wait_for_load_state(+ 'networkidle', timeout=self.NETWORK_IDLE_TIMEOUT+ )+ logger.debug("Network idle achieved")+ except Exception:+ logger.debug("Network idle timeout - continuing anyway")++ # Additional small delay for JavaScript execution+ await asyncio.sleep(0.5)+ logger.debug("Page stability wait completed")++ except Exception as e:+ logger.debug(+ f"Page stability wait failed: {e} - continuing anyway"+ )++ async def _get_unified_analysis(self) -> Dict[str, Any]:+ r"""Get unified analysis data from the page."""+ page = await self._require_page()+ try:+ if not self._unified_script:+ logger.error("Unified analyzer script not loaded")+ return {"elements": {}, "metadata": {"elementCount": 0}}++ result = await page.evaluate(self._unified_script)++ if not isinstance(result, dict):+ logger.warning(f"Invalid result type: {type(result)}")+ return {"elements": {}, "metadata": {"elementCount": 0}}++ return result+ except Exception as e:+ logger.warning(f"Error in unified analysis: {e}")+ return {"elements": {}, "metadata": {"elementCount": 0}}++ def _convert_analysis_to_rects(+ self, analysis_data: Dict[str, Any]+ ) -> Dict[str, Any]:+ r"""Convert analysis data to rect format for visual marking."""+ rects = {}+ elements = analysis_data.get("elements", {})++ for ref, element_data in elements.items():+ coordinates = element_data.get("coordinates", [])+ if coordinates:+ rects[ref] = {+ "role": element_data.get("role", "generic"),+ "aria-name": element_data.get("name", ""),+ "rects": [coordinates[0]],+ }+ return rects++ def _add_set_of_mark(self, image, rects):+ r"""Add visual marks to the image."""+ try:+ from PIL import ImageDraw, ImageFont+ except ImportError:+ logger.warning("PIL not available, returning original image")+ return image++ marked_image = image.copy()+ draw = ImageDraw.Draw(marked_image)++ # Try to get font+ try:+ font = ImageFont.truetype("arial.ttf", 16)+ except (OSError, IOError):+ try:+ font = ImageFont.load_default()+ except (OSError, IOError):+ font = None++ # Color scheme+ colors = {+ "button": "#FF6B6B",+ "link": "#4ECDC4",+ "textbox": "#45B7D1",+ "select": "#96CEB4",+ "checkbox": "#FECA57",+ "radio": "#FF9FF3",+ "default": "#DDA0DD",+ }++ for ref, rect_data in rects.items():+ rects_list = rect_data.get("rects", [])+ role = rect_data.get("role", "generic")+ color = colors.get(role, colors["default"])++ for rect in rects_list:+ x, y = rect.get("x", 0), rect.get("y", 0)+ width, height = rect.get("width", 0), rect.get("height", 0)++ # Draw rectangle outline+ draw.rectangle(+ [x, y, x + width, y + height], outline=color, width=2+ )++ # Draw reference label+ label_text = ref+ if font:+ bbox = draw.textbbox((0, 0), label_text, font=font)+ text_width, text_height = (+ bbox[2] - bbox[0],+ bbox[3] - bbox[1],+ )+ else:+ text_width, text_height = len(label_text) * 8, 16++ label_x, label_y = max(0, x - 2), max(0, y - text_height - 2)++ # Background and text+ draw.rectangle(+ [+ label_x,+ label_y,+ label_x + text_width + 4,+ label_y + text_height + 2,+ ],+ fill=color,+ )+ draw.text(+ (label_x + 2, label_y + 1),+ label_text,+ fill="white",+ font=font,+ )++ return marked_image++ def _format_snapshot_from_analysis(+ self, analysis_data: Dict[str, Any]+ ) -> str:+ r"""Format analysis data into snapshot string."""+ lines = []+ elements = analysis_data.get("elements", {})++ for ref, element_data in elements.items():+ role = element_data.get("role", "generic")+ name = element_data.get("name", "")++ line = f"- {role}"+ if name:+ line += f' "{name}"'++ # Add properties+ props = []+ for prop in ["disabled", "checked", "expanded"]:+ value = element_data.get(prop)+ if value is True:+ props.append(prop)+ elif value is not None and prop in ["checked", "expanded"]:+ props.append(f"{prop}={value}")++ if props:+ line += f" {' '.join(props)}"++ line += f" [ref={ref}]"+ lines.append(line)++ return "\n".join(lines)++ async def _exec_with_snapshot(+ self, action: Dict[str, Any]+ ) -> Dict[str, str]:+ r"""Execute action and return result with snapshot comparison."""++ # Log action execution start+ action_type = action.get("type", "unknown")+ logger.info(f"Executing action: {action_type}")++ # Get before snapshot+ logger.info("Capturing pre-action snapshot...")+ snapshot_start = time.time()+ before_snapshot = await self._session.get_snapshot(+ force_refresh=True, diff_only=False+ )+ snapshot_time = time.time() - snapshot_start+ logger.info(f"Pre-action snapshot captured in {snapshot_time:.2f}s")++ # Execute action+ logger.info(f"Executing {action_type} action...")+ action_start = time.time()+ result = await self._session.exec_action(action)+ action_time = time.time() - action_start+ logger.info(f"Action {action_type} completed in {action_time:.2f}s")++ # Wait for page stability after action (especially important for click)+ if action_type in ["click", "type", "select", "enter"]:+ logger.info(+ f"Waiting for page stability " f"after {action_type}..."+ )+ stability_start = time.time()+ await self._wait_for_page_stability()+ stability_time = time.time() - stability_start+ logger.info(+ f"Page stability wait " f"completed in {stability_time:.2f}s"+ )++ # Get after snapshot+ logger.info("Capturing post-action snapshot...")+ snapshot_start = time.time()+ after_snapshot = await self._session.get_snapshot(+ force_refresh=True, diff_only=False+ )+ snapshot_time = time.time() - snapshot_start+ logger.info(+ f"Post-action snapshot " f"captured in {snapshot_time:.2f}s"+ )++ # Check for snapshot quality and log warnings+ if before_snapshot == after_snapshot:+ snapshot = "snapshot not changed"+ logger.debug("Page snapshot unchanged after action")+ else:+ snapshot = after_snapshot+ # Check if snapshot is empty or problematic+ if "<empty>" in after_snapshot:+ logger.warning(+ f"Action {action_type} resulted "+ f"in empty snapshot - "+ f"page may still be loading"+ )+ elif len(after_snapshot.strip()) < 50:+ logger.warning(+ f"Action {action_type} resulted "+ f"in very short snapshot:"+ f" {len(after_snapshot)} chars"+ )+ else:+ logger.debug(+ f"Action {action_type} resulted "+ f"in updated snapshot: "+ f"{len(after_snapshot)} chars"+ )++ return {"result": result, "snapshot": snapshot}++ async def _extract_links_by_refs(+ self, snapshot: str, page, refs: List[str]+ ) -> List[Dict[str, str]]:+ r"""Extract multiple links by their reference IDs."""+ import re++ found_links = []+ ref_set = set(refs)+ lines = snapshot.split('\n')++ for line in lines:+ link_match = re.search(+ r'- link\s+"([^"]+)"\s+\[ref=([^\]]+)\]', line+ )+ if link_match and link_match.group(2) in ref_set:+ text, found_ref = link_match.groups()+ try:+ url = await self._get_link_url_by_ref(page, found_ref)+ found_links.append(+ {"text": text, "ref": found_ref, "url": url or ""}+ )+ except Exception as e:+ logger.warning(+ f"Failed to get URL for ref {found_ref}: {e}"+ )+ found_links.append(+ {"text": text, "ref": found_ref, "url": ""}+ )++ return found_links++ async def _get_link_url_by_ref(self, page, ref: str) -> str:+ r"""Get URL of a link element by reference ID."""+ try:+ element = await page.query_selector(f'[aria-ref="{ref}"]')+ if element:+ href = await element.get_attribute('href')+ if href:+ from urllib.parse import urljoin++ return urljoin(page.url, href)+ return ""+ except Exception as e:+ logger.warning(f"Failed to get URL for ref {ref}: {e}")+ return ""++ def _ensure_agent(self) -> PlaywrightLLMAgent:+ r"""Create PlaywrightLLMAgent on first use."""+ if self.web_agent_model is None:+ raise RuntimeError(+ "web_agent_model required for high-level task planning"+ )++ if self._agent is None:+ self._agent = PlaywrightLLMAgent(+ headless=self._headless,+ user_data_dir=self._user_data_dir,+ model_backend=self.web_agent_model,+ )+ return self._agent++ # Public API Methods++ async def open_browser(+ self, start_url: Optional[str] = None+ ) -> Dict[str, str]:+ r"""Launches a new browser session, making it ready for web automation.++ This method initializes the underlying browser instance. If a+ `start_url` is provided, it will also navigate to that URL.++ Args:+ start_url (Optional[str]): The initial URL to navigate to after the+ browser is launched. If not provided, the browser will start+ with a blank page.++ Returns:+ Dict[str, str]: A dictionary containing:+ - "result": A string confirming that the browser session has+ started.+ - "snapshot": A textual representation of the current page's+ interactive elements. This snapshot is crucial for+ identifying elements for subsequent actions.+ """+ logger.info("Starting browser session...")++ browser_start = time.time()+ await self._session.ensure_browser()+ browser_time = time.time() - browser_start+ logger.info(f"Browser session started in {browser_time:.2f}s")++ if start_url:+ logger.info(f"Auto-navigating to start URL: {start_url}")+ return await self.visit_page(start_url)++ logger.info("Capturing initial browser snapshot...")+ snapshot_start = time.time()+ snapshot = await self._session.get_snapshot(+ force_refresh=True, diff_only=False+ )+ snapshot_time = time.time() - snapshot_start+ logger.info(f"Initial snapshot captured in {snapshot_time:.2f}s")++ return {"result": "Browser session started.", "snapshot": snapshot}++ async def close_browser(self) -> str:+ r"""Closes the current browser session and releases all associated+ resources.++ This should be called at the end of a web automation task to ensure a+ clean shutdown of the browser instance.++ Returns:+ str: A confirmation message indicating the session is closed.+ """+ if self._agent is not None:+ try:+ await self._agent.close()+ except Exception:+ pass+ self._agent = None++ await self._session.close()+ return "Browser session closed."++ async def visit_page(self, url: str) -> Dict[str, str]:+ r"""Navigates the current browser page to a specified URL.++ Args:+ url (str): The web address to load in the browser. Must be a+ valid URL.++ Returns:+ Dict[str, str]: A dictionary containing:+ - "result": A message indicating the outcome of the navigation,+ e.g., "Navigation successful.".+ - "snapshot": A new textual snapshot of the page's interactive+ elements after the new page has loaded.+ """+ if not url or not isinstance(url, str):+ return {+ "result": "Error: 'url' must be a non-empty string",+ "snapshot": "",+ }++ logger.info(f"Navigating to URL: {url}")++ # Navigate to page+ nav_start = time.time()+ nav_result = await self._session.visit(url)+ nav_time = time.time() - nav_start+ logger.info(f"Page navigation completed in {nav_time:.2f}s")++ # Get snapshot+ logger.info("Capturing page snapshot after navigation...")+ snapshot_start = time.time()+ snapshot = await self._session.get_snapshot(+ force_refresh=True, diff_only=False+ )+ snapshot_time = time.time() - snapshot_start+ logger.info(f"Navigation snapshot captured in {snapshot_time:.2f}s")++ return {"result": nav_result, "snapshot": snapshot}++ async def get_page_snapshot(self) -> str:+ r"""Captures a textual representation of the current page's content.++ This "snapshot" provides a simplified view of the DOM, focusing on+ interactive elements like links, buttons, and input fields. Each+ element is assigned a unique reference ID (`ref`) that can be used in+ other actions like `click` or `type`.++ The snapshot is useful for understanding the page structure and+ identifying elements to interact with without needing to parse raw+ HTML. A new snapshot is generated on each call.++ Returns:+ str: A formatted string representing the interactive elements on+ the page. For example:+ '- link "Sign In" [ref=1]'+ '- textbox "Username" [ref=2]'+ """+ logger.info("Capturing page snapshot")++ analysis_start = time.time()+ analysis_data = await self._get_unified_analysis()+ analysis_time = time.time() - analysis_start+ logger.info(+ f"Page snapshot analysis " f"completed in {analysis_time:.2f}s"+ )++ snapshot_text = analysis_data.get("snapshotText", "")+ return (+ snapshot_text+ if snapshot_text+ else self._format_snapshot_from_analysis(analysis_data)+ )++ @dependencies_required('PIL')+ async def get_som_screenshot(self):+ r"""Captures a screenshot of the current webpage and visually marks all+ interactive elements. "SoM" stands for "Set of Marks".++ This method is essential for tasks requiring visual understanding of+ the page layout. It works by:+ 1. Taking a full-page screenshot.+ 2. Identifying all interactive elements (buttons, links, inputs, etc.).+ 3. Drawing colored boxes and reference IDs (`ref`) over these elements+ on the screenshot.+ 4. Saving the annotated image to a cache directory.+ 5. Returning the image as a base64-encoded string along with a summary.++ Use this when the textual snapshot from `get_page_snapshot` is+ insufficient and visual context is needed to decide the next action.++ Returns:+ ToolResult: An object containing:+ - `text`: A summary string, e.g., "Visual webpage screenshot+ captured with 42 interactive elements".+ - `images`: A list containing a single base64-encoded PNG image+ as a data URL.+ """+ from PIL import Image++ from camel.utils.tool_result import ToolResult++ # Get screenshot and analysis+ page = await self._require_page()++ # Log screenshot timeout start+ logger.info(+ f"Starting screenshot capture"+ f"with timeout: {self.DEFAULT_SCREENSHOT_TIMEOUT}ms"+ )++ start_time = time.time()+ image_data = await page.screenshot(+ timeout=self.DEFAULT_SCREENSHOT_TIMEOUT+ )+ screenshot_time = time.time() - start_time++ logger.info(f"Screenshot capture completed in {screenshot_time:.2f}s")+ image = Image.open(io.BytesIO(image_data))++ # Log unified analysis start+ logger.info("Starting unified page analysis...")+ analysis_start_time = time.time()+ analysis_data = await self._get_unified_analysis()+ analysis_time = time.time() - analysis_start_time+ logger.info(f"Unified page analysis completed in {analysis_time:.2f}s")++ # Log image processing+ logger.info("Processing visual marks on screenshot...")+ mark_start_time = time.time()+ rects = self._convert_analysis_to_rects(analysis_data)+ marked_image = self._add_set_of_mark(image, rects)+ mark_time = time.time() - mark_start_time+ logger.info(f"Visual marks processing completed in {mark_time:.2f}s")++ # Save screenshot to cache directory+ parsed_url = urllib.parse.urlparse(page.url)+ url_name = sanitize_filename(str(parsed_url.path), max_length=241)+ timestamp = datetime.datetime.now().strftime("%m%d%H%M%S")+ file_path = os.path.join(+ self.cache_dir, f"{url_name}_{timestamp}_som.png"+ )+ marked_image.save(file_path, "PNG")++ # Convert to base64+ img_buffer = io.BytesIO()+ marked_image.save(img_buffer, format="PNG")+ img_buffer.seek(0)+ img_base64 = base64.b64encode(img_buffer.getvalue()).decode('utf-8')+ img_data_url = f"data:image/png;base64,{img_base64}"++ text_result = (+ f"Visual webpage screenshot "+ f"captured with {len(rects)} interactive elements"+ )++ return ToolResult(text=text_result, images=[img_data_url])++ async def click(self, *, ref: str) -> Dict[str, str]:+ r"""Clicks on an interactive element on the page.++ Args:+ ref (str): The reference ID of the element to click. This ID is+ obtained from the page snapshot (see `get_page_snapshot` or+ `get_som_screenshot`).++ Returns:+ Dict[str, str]: A dictionary containing:+ - "result": A message confirming the click action.+ - "snapshot": A new textual snapshot of the page after the+ click, which may have changed as a result of the action. If+ the snapshot is unchanged, it will be the string "snapshot+ not changed".+ """+ self._validate_ref(ref, "click")++ analysis = await self._get_unified_analysis()+ elements = analysis.get("elements", {})+ if ref not in elements:+ available_refs = list(elements.keys())+ logger.error(+ f"Error: Element reference '{ref}' not found. "+ f"Available refs: {available_refs}"+ )+ return {+ "result": f"Error: Element reference '{ref}' not found. "+ f"Available refs: {available_refs}"+ }++ action = {"type": "click", "ref": ref}+ return await self._exec_with_snapshot(action)++ async def type(self, *, ref: str, text: str) -> Dict[str, str]:+ r"""Types text into an input field, such as a textbox or search bar.++ Args:+ ref (str): The reference ID of the input element.+ text (str): The text to be typed into the element.++ Returns:+ Dict[str, str]: A dictionary containing:+ - "result": A message confirming the type action.+ - "snapshot": A new textual snapshot of the page after the+ text has been entered.+ """+ self._validate_ref(ref, "type")+ await self._get_unified_analysis() # Ensure aria-ref attributes++ action = {"type": "type", "ref": ref, "text": text}+ return await self._exec_with_snapshot(action)++ async def select(self, *, ref: str, value: str) -> Dict[str, str]:+ r"""Selects an option from a dropdown (`<select>`) element.++ Args:+ ref (str): The reference ID of the `<select>` element.+ value (str): The value of the `<option>` to be selected. This+ should match the `value` attribute of the option, not the+ visible text.++ Returns:+ Dict[str, str]: A dictionary containing:+ - "result": A message confirming the select action.+ - "snapshot": A new snapshot of the page after the selection.+ """+ self._validate_ref(ref, "select")+ await self._get_unified_analysis()++ action = {"type": "select", "ref": ref, "value": value}+ return await self._exec_with_snapshot(action)++ async def scroll(self, *, direction: str, amount: int) -> Dict[str, str]:+ r"""Scrolls the page window up or down by a specified amount.++ Args:+ direction (str): The direction to scroll. Must be either 'up' or+ 'down'.+ amount (int): The number of pixels to scroll.++ Returns:+ Dict[str, str]: A dictionary containing:+ - "result": A confirmation of the scroll action.+ - "snapshot": A new snapshot of the page after scrolling.+ """+ if direction not in ("up", "down"):+ return {+ "result": "Error: direction must be 'up' or 'down'",+ "snapshot": "",+ }++ action = {"type": "scroll", "direction": direction, "amount": amount}+ return await self._exec_with_snapshot(action)++ async def enter(self, *, ref: str) -> Dict[str, str]:+ r"""Simulates pressing the Enter key on a specific element.++ This is often used to submit forms after filling them out.++ Args:+ ref (str): The reference ID of the element to press Enter on.++ Returns:+ Dict[str, str]: A dictionary containing:+ - "result": A confirmation of the action.+ - "snapshot": A new page snapshot, as this action often+ triggers navigation or page updates.+ """+ self._validate_ref(ref, "enter")+ action = {"type": "enter", "ref": ref}+ return await self._exec_with_snapshot(action)++ async def wait_user(+ self, timeout_sec: Optional[float] = None+ ) -> Dict[str, str]:+ r"""Pauses the agent's execution and waits for human intervention.++ This is useful for tasks that require manual steps, like solving a+ CAPTCHA. The agent will print a message to the console and wait+ until the user presses the Enter key.++ Args:+ timeout_sec (Optional[float]): The maximum time to wait in+ seconds. If the timeout is reached, the agent will resume+ automatically. If `None`, it will wait indefinitely.++ Returns:+ Dict[str, str]: A dictionary containing:+ - "result": A message indicating how the wait ended (e.g.,+ "User resumed." or "Timeout... reached, auto-resumed.").+ - "snapshot": The current page snapshot after the wait.+ """+ import asyncio++ prompt = (+ "🕑 Agent waiting for human input. "+ "Complete action in browser, then press Enter..."+ )+ logger.info(f"\n{prompt}\n")++ async def _await_enter():+ await asyncio.to_thread(input, ">>> Press Enter to resume <<<\n")++ try:+ if timeout_sec is not None:+ logger.info(+ f"Waiting for user input with timeout: {timeout_sec}s"+ )+ start_time = time.time()+ await asyncio.wait_for(_await_enter(), timeout=timeout_sec)+ wait_time = time.time() - start_time+ logger.info(f"User input received after {wait_time:.2f}s")+ result_msg = "User resumed."+ else:+ logger.info("Waiting for user " "input (no timeout)")+ start_time = time.time()+ await _await_enter()+ wait_time = time.time() - start_time+ logger.info(f"User input received " f"after {wait_time:.2f}s")+ result_msg = "User resumed."+ except asyncio.TimeoutError:+ wait_time = timeout_sec or 0.0+ logger.info(+ f"User input timeout reached "+ f"after {wait_time}s, auto-resuming"+ )+ result_msg = f"Timeout {timeout_sec}s reached, auto-resumed."++ snapshot = await self._session.get_snapshot(+ force_refresh=True, diff_only=False+ )+ return {"result": result_msg, "snapshot": snapshot}++ async def get_page_links(self, *, ref: List[str]) -> Dict[str, Any]:+ r"""Retrieves the full URLs for a given list of link reference IDs.++ This is useful when you need to know the destination of a link before+ clicking it.++ Args:+ ref (List[str]): A list of reference IDs for link elements,+ obtained from a page snapshot.++ Returns:+ Dict[str, Any]: A dictionary containing:+ - "links": A list of dictionaries, where each dictionary+ represents a found link and has "text", "ref", and "url"+ keys.+ """+ if not ref or not isinstance(ref, list):+ return {"links": []}++ for r in ref:+ if not r or not isinstance(r, str):+ return {"links": []}++ page = await self._require_page()+ snapshot = await self._session.get_snapshot(+ force_refresh=True, diff_only=False+ )+ links = await self._extract_links_by_refs(snapshot, page, ref)++ return {"links": links}++ async def solve_task(+ self, task_prompt: str, start_url: str, max_steps: int = 15+ ) -> str:+ r"""Uses a high-level LLM agent to autonomously complete a task.++ This function delegates control to another agent that can reason about+ a task, break it down into steps, and execute browser actions to+ achieve the goal. It is suitable for complex, multi-step tasks.++ Note: `web_agent_model` must be provided during the toolkit's+ initialization to use this function.++ Args:+ task_prompt (str): A natural language description of the task to+ be completed (e.g., "log into my account on example.com").+ start_url (str): The URL to start the task from.+ max_steps (int): The maximum number of steps the agent is allowed+ to take before stopping.++ Returns:+ str: A summary message indicating that the task processing has+ finished. The detailed trace of the agent's actions will be+ printed to the standard output.+ """+ agent = self._ensure_agent()+ await agent.navigate(start_url)+ await agent.process_command(task_prompt, max_steps=max_steps)+ return "Task processing finished - see stdout for detailed trace."++ def get_tools(self) -> List[FunctionTool]:+ r"""Get available function tools+ based on enabled_tools configuration."""+ # Map tool names to their corresponding methods+ tool_map = {+ "open_browser": self.open_browser,+ "close_browser": self.close_browser,+ "visit_page": self.visit_page,+ "get_page_snapshot": self.get_page_snapshot,+ "get_som_screenshot": self.get_som_screenshot,+ "get_page_links": self.get_page_links,+ "click": self.click,+ "type": self.type,+ "select": self.select,+ "scroll": self.scroll,+ "enter": self.enter,+ "wait_user": self.wait_user,+ "solve_task": self.solve_task,+ }++ enabled_tools = []++ for tool_name in self.enabled_tools:+ if tool_name == "solve_task" and self.web_agent_model is None:+ logger.warning(+ f"Tool '{tool_name}' is enabled but web_agent_model "+ f"is not provided. Skipping this tool."+ )+ continue++ if tool_name in tool_map:+ enabled_tools.append(+ FunctionTool(cast(Callable, tool_map[tool_name]))+ )+ else:+ logger.warning(f"Unknown tool name: {tool_name}")++ logger.info(f"Returning {len(enabled_tools)} enabled tools")+ return enabled_toolsdiff --git a/camel/toolkits/non_visual_browser_toolkit/snapshot.py b/camel/toolkits/hybrid_browser_toolkit/snapshot.pysimilarity index 92%rename from camel/toolkits/non_visual_browser_toolkit/snapshot.pyrename to camel/toolkits/hybrid_browser_toolkit/snapshot.pyindex 46c756da1..bdda07896 100644--- a/camel/toolkits/non_visual_browser_toolkit/snapshot.py+++ b/camel/toolkits/hybrid_browser_toolkit/snapshot.py@@ -12,7 +12,7 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========from pathlib import Path-from typing import TYPE_CHECKING, Dict, List, Optional+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Unionif TYPE_CHECKING:from playwright.async_api import Page@@ -64,7 +64,17 @@ class PageSnapshot:)logger.debug("Capturing page snapshot …")- snapshot_text = await self._get_snapshot_direct()+ snapshot_result = await self._get_snapshot_direct()++ # Extract snapshot text from the unified analyzer result+ if (+ isinstance(snapshot_result, dict)+ and 'snapshotText' in snapshot_result+ ):+ snapshot_text = snapshot_result['snapshotText']+ else:+ snapshot_text = snapshot_result+formatted = self._format_snapshot(snapshot_text or "<empty>")output = formatted@@ -99,7 +109,9 @@ class PageSnapshot:# ------------------------------------------------------------------_snapshot_js_cache: Optional[str] = None # class-level cache- async def _get_snapshot_direct(self) -> Optional[str]:+ async def _get_snapshot_direct(+ self,+ ) -> Optional[Union[str, Dict[str, Any]]]:r"""Evaluate the snapshot-extraction JS with simple retry logic.Playwright throws *Execution context was destroyed* when a new page@@ -110,7 +122,7 @@ class PageSnapshot:# Load JS once and cache it at class levelif PageSnapshot._snapshot_js_cache is None:- js_path = Path(__file__).parent / "snapshot.js"+ js_path = Path(__file__).parent / "unified_analyzer.js"PageSnapshot._snapshot_js_cache = js_path.read_text(encoding="utf-8")diff --git a/camel/toolkits/non_visual_browser_toolkit/snapshot.js b/camel/toolkits/hybrid_browser_toolkit/unified_analyzer.jssimilarity index 60%rename from camel/toolkits/non_visual_browser_toolkit/snapshot.jsrename to camel/toolkits/hybrid_browser_toolkit/unified_analyzer.jsindex ae42bdc7f..99c9795e2 100644--- a/camel/toolkits/non_visual_browser_toolkit/snapshot.js+++ b/camel/toolkits/hybrid_browser_toolkit/unified_analyzer.js@@ -1,6 +1,13 @@(() => {- // Playwright's snapshot logic focuses on semantics and visibility, not arbitrary limits.- // We will first build a semantic tree in memory, then render it.+ // Unified analyzer that combines visual and structural analysis+ // Preserves complete snapshot.js logic while adding visual coordinate information++ let refCounter = 1;+ function generateRef() {+ return `e${refCounter++}`;+ }++ // === Complete snapshot.js logic preservation ===function isVisible(node) {if (node.nodeType !== Node.ELEMENT_NODE) return true;@@ -70,13 +77,9 @@return result;}- let refCounter = 1;- function generateRef() {- return `e${refCounter++}`;- }-/*** Phase 1: Build an in-memory representation of the accessibility tree.+ * Complete preservation of snapshot.js buildAriaTree logic*/function buildAriaTree(rootElement) {const visited = new Set();@@ -153,9 +156,34 @@}}- // FIX: If an element's name is the same as its only text child, remove the redundant child.- if (ariaNode && ariaNode.children.length === 1 && typeof ariaNode.children[0] === 'string' && ariaNode.name === ariaNode.children[0]) {- ariaNode.children = [];+ // FIX: Remove redundant text children that match the element's name+ if (ariaNode && ariaNode.children.length > 0) {+ // Remove text children that are the same as the parent's name or are contained in it+ ariaNode.children = ariaNode.children.filter(child => {+ if (typeof child === 'string') {+ const childText = child.trim();+ const parentName = ariaNode.name.trim();++ // Remove if text child exactly matches parent name+ if (childText === parentName) {+ return false;+ }++ // Also remove if the child text is completely contained in parent name+ // and represents a significant portion (to avoid removing important partial text)+ if (childText.length > 3 && parentName.includes(childText)) {+ return false;+ }++ return true;+ }+ return true;+ });++ // If after filtering, we have only one text child that equals the name, remove it+ if (ariaNode.children.length === 1 && typeof ariaNode.children[0] === 'string' && ariaNode.name === ariaNode.children[0]) {+ ariaNode.children = [];+ }}}@@ -166,7 +194,7 @@/*** Phase 2: Normalize the tree by removing redundant generic wrappers.- * This is a key optimization in Playwright to simplify the structure.+ * Complete preservation of snapshot.js normalizeTree logic*/function normalizeTree(node) {if (typeof node === 'string') return [node];@@ -178,6 +206,24 @@node.children = newChildren;// Remove child elements that have the same name as their parent+ const filteredChildren = [];+ for (const child of node.children) {+ if (typeof child !== 'string' && child.name && node.name) {+ const childName = child.name.trim();+ const parentName = node.name.trim();+ if (childName === parentName) {+ // If child has same name as parent, merge its children into parent+ filteredChildren.push(...(child.children || []));+ } else {+ filteredChildren.push(child);+ }+ } else {+ filteredChildren.push(child);+ }+ }+ node.children = filteredChildren;++ // Also handle the case where we have only one child with same nameif (node.children.length === 1 && typeof node.children[0] !== 'string') {const child = node.children[0];if (child.name && node.name && child.name.trim() === node.name.trim()) {@@ -195,9 +241,9 @@return [node];}-/*** Phase 3: Render the normalized tree into the final string format.+ * Complete preservation of snapshot.js renderTree logic*/function renderTree(node, indent = '') {const lines = [];@@ -263,6 +309,116 @@return lines;}- const outputLines = processDocument(document);- return outputLines.join('\n');-})();+ // === Visual analysis functions from page_script.js ===++ // From page_script.js - check if element is topmost at coordinates+ function isTopmost(element, x, y) {+ let hit = document.elementFromPoint(x, y);+ if (hit === null) return true;++ while (hit) {+ if (hit == element) return true;+ hit = hit.parentNode;+ }+ return false;+ }++ // From page_script.js - get visual coordinates+ function getElementCoordinates(element) {+ let rects = element.getClientRects();+ let scale = window.devicePixelRatio || 1;+ let validRects = [];++ for (const rect of rects) {+ let x = rect.left + rect.width / 2;+ let y = rect.top + rect.height / 2;+ if (isTopmost(element, x, y)) {+ validRects.push({+ x: rect.x * scale,+ y: rect.y * scale,+ width: rect.width * scale,+ height: rect.height * scale,+ top: rect.top * scale,+ left: rect.left * scale,+ right: rect.right * scale,+ bottom: rect.bottom * scale+ });+ }+ }++ return validRects;+ }++ // === Unified analysis function ===++ function collectElementsFromTree(node, elementsMap) {+ if (typeof node === 'string') return;++ if (node.element && node.ref) {+ // Get visual coordinates for this element+ const coordinates = getElementCoordinates(node.element);++ // Store comprehensive element information+ elementsMap[node.ref] = {+ // Structural information (preserved from snapshot.js)+ role: node.role,+ name: node.name,+ tagName: node.element.tagName.toLowerCase(),+ disabled: node.disabled,+ checked: node.checked,+ expanded: node.expanded,++ // Visual information (from page_script.js)+ coordinates: coordinates,++ // Additional metadata+ href: node.element.href || null,+ value: node.element.value || null,+ placeholder: node.element.placeholder || null,+ scrollable: node.element.scrollHeight > node.element.clientHeight+ };+ }++ // Recursively process children+ if (node.children) {+ for (const child of node.children) {+ collectElementsFromTree(child, elementsMap);+ }+ }+ }++ function analyzePageElements() {+ // Generate the complete structured snapshot using original snapshot.js logic+ const outputLines = processDocument(document);+ const snapshotText = outputLines.join('\n');++ // Build the tree again to collect element information with visual data+ textCache.clear();+ refCounter = 1; // Reset counter to match snapshot generation+ let tree = buildAriaTree(document.body);+ [tree] = normalizeTree(tree);++ const elementsMap = {};+ collectElementsFromTree(tree, elementsMap);++ const result = {+ url: window.location.href,+ elements: elementsMap,+ snapshotText: snapshotText,+ metadata: {+ timestamp: new Date().toISOString(),+ elementCount: Object.keys(elementsMap).length,+ screenInfo: {+ width: window.innerWidth,+ height: window.innerHeight,+ devicePixelRatio: window.devicePixelRatio || 1+ }+ }+ };++ return result;+ }++ // Execute analysis and return result+ return analyzePageElements();+})();\ No newline at end of filediff --git a/camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.py b/camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.pydeleted file mode 100644index 167ed818b..000000000--- a/camel/toolkits/non_visual_browser_toolkit/browser_non_visual_toolkit.py+++ /dev/null@@ -1,446 +0,0 @@-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========-from __future__ import annotations--from typing import Any, Dict, List, Optional--from camel.logger import get_logger-from camel.models import BaseModelBackend-from camel.toolkits.base import BaseToolkit-from camel.toolkits.function_tool import FunctionTool--from .agent import PlaywrightLLMAgent-from .nv_browser_session import NVBrowserSession--logger = get_logger(__name__)---class BrowserNonVisualToolkit(BaseToolkit):- r"""A lightweight, *non-visual* browser toolkit exposing primitive- Playwright actions as CAMEL `FunctionTool`s.- """-- def __init__(- self,- *,- headless: bool = True,- user_data_dir: Optional[str] = None,- web_agent_model: Optional[BaseModelBackend] = None,- ) -> None:- super().__init__()- self._headless = headless- self._user_data_dir = user_data_dir- self.web_agent_model = web_agent_model # Currently unused but kept- # for compatibility-- # Encapsulated browser session- self._session = NVBrowserSession(- headless=headless, user_data_dir=user_data_dir- )-- # Optional higher-level agent (only if user supplies model)- self._agent: Optional[PlaywrightLLMAgent] = None-- def __del__(self):- r"""Best-effort cleanup when toolkit is garbage collected.-- 1. We *avoid* running during the Python interpreter shutdown phase- (`sys.is_finalizing()`), because the import machinery and/or event- loop may already be torn down which leads to noisy exceptions such- as `ImportError: sys.meta_path is None` or- `RuntimeError: Event loop is closed`.- 2. We protect all imports and event-loop operations with defensive- `try/except` blocks. This ensures that, even if cleanup cannot be- carried out, we silently ignore the failure instead of polluting- stderr on program exit.- """- try:- import sys-- if getattr(sys, "is_finalizing", lambda: False)():- return # Skip cleanup during interpreter shutdown-- import asyncio-- try:- loop = asyncio.get_event_loop()- except RuntimeError:- # No event loop in current thread → nothing to clean- return-- if loop.is_closed():- # The default loop is closed, create a *temporary* loop just- # for cleanup so that Playwright / asyncio transports are- # gracefully shut down. This avoids noisy warnings such as- # "RuntimeError: Event loop is closed" when the program- # exits.- try:- tmp_loop = asyncio.new_event_loop()- try:- asyncio.set_event_loop(tmp_loop)- tmp_loop.run_until_complete(self.close_browser())- finally:- # Best-effort shutdown of async generators and loop- # itself (Python ≥3.6).- if hasattr(tmp_loop, "shutdown_asyncgens"):- tmp_loop.run_until_complete(- tmp_loop.shutdown_asyncgens()- )- tmp_loop.close()- finally:- # Ensure no subsequent get_event_loop() picks up a now- # closed temporary loop.- asyncio.set_event_loop(None)- return-- if loop.is_running():- try:- task = loop.create_task(self.close_browser())- del task # Fire-and-forget- except RuntimeError:- # Loop is running but not in this thread → ignore- pass- else:- # Own the loop → safe to run- asyncio.run(self.close_browser())- except Exception:- # Suppress *all* errors during garbage collection- pass-- async def _ensure_browser(self):- await self._session.ensure_browser()-- async def _require_page(self):- await self._session.ensure_browser()- return await self._session.get_page()-- def _validate_ref(self, ref: str, method_name: str) -> None:- r"""Validate that ref parameter is a non-empty string."""- if not ref or not isinstance(ref, str):- logger.error(- f"{method_name}(): 'ref' must be a non-empty string, "- f"got: {ref}"- )-- async def open_browser(- self, start_url: Optional[str] = None- ) -> Dict[str, str]:- r"""Launches a new browser session. This should be the first step.-- Args:- start_url (Optional[str]): The URL to navigate to after the browser- launches. If not provided, the browser will open with a blank- page. (default: :obj:`None`)-- Returns:- Dict[str, str]: A dictionary containing the result of the action- and a snapshot of the page. The keys are "result" and- "snapshot". The "snapshot" is a YAML-like representation of- the page's DOM structure, including element references- (e.g., "e3") that can be used in other tool calls.- """- await self._session.ensure_browser()- if start_url:- return await self.visit_page(start_url)- # If no start_url provided, still capture initial snapshot- snapshot = await self._session.get_snapshot(- force_refresh=True, diff_only=False- )- return {"result": "Browser session started.", "snapshot": snapshot}-- async def close_browser(self) -> str:- r"""Closes the current browser session, freeing up all associated- resources. This should be called when the browsing task is complete.-- Returns:- str: A confirmation message indicating the session has been- closed.- """- # Close agent if it exists- if self._agent is not None:- try:- await self._agent.close()- except Exception:- pass # Don't fail if agent cleanup fails- self._agent = None-- # Close session- await NVBrowserSession.close_all_sessions()- return "Browser session closed."-- async def visit_page(self, url: str) -> Dict[str, str]:- r"""Navigates the current browser page to a new URL.-- Args:- url (str): The URL to navigate to. Must be a fully qualified URL- (e.g., "https://www.google.com").-- Returns:- Dict[str, str]: A dictionary containing the navigation result and a- snapshot of the new page. The keys are "result" and- "snapshot". The "snapshot" provides a fresh view of the- page's DOM structure.- """- if not url or not isinstance(url, str):- raise ValueError("visit_page(): 'url' must be a non-empty string")-- nav_result = await self._session.visit(url)- snapshot = await self._session.get_snapshot(- force_refresh=True, diff_only=False- )- return {"result": nav_result, "snapshot": snapshot}-- async def get_page_snapshot(- self, *, force_refresh: bool = False, diff_only: bool = False- ) -> str:- r"""Capture a YAML-like structural snapshot of the DOM.-- Args:- force_refresh (bool): When ``True`` always re-generate the- snapshot even if the URL has not changed. (default:- :obj:`False`)- diff_only (bool): When ``True`` return only the diff relative to- the previous snapshot. (default: :obj:`False`)-- Returns:- str: Formatted snapshot string.- """- return await self._session.get_snapshot(- force_refresh=force_refresh, diff_only=diff_only- )-- async def click(self, *, ref: str) -> Dict[str, str]:- r"""Performs a click action on a specified element on the current page.-- Args:- ref (str): The reference ID of the element to click. This ID is- obtained from the page snapshot (e.g., "e12").-- Returns:- Dict[str, str]: A dictionary containing the result of the action.- If the click causes a change in the page's structure, a- "snapshot" key will be included with a diff of the changes.- """- self._validate_ref(ref, "click")-- action: Dict[str, Any] = {"type": "click", "ref": ref}- return await self._exec_with_snapshot(action)-- async def type(self, *, ref: str, text: str) -> Dict[str, str]:- r"""Types text into an input field or textarea on the current page.-- Args:- ref (str): The reference ID of the input element. This ID is- obtained from the page snapshot (e.g., "e25").- text (str): The text to be typed into the element.-- Returns:- Dict[str, str]: A dictionary containing the result of the action.- The key is "result".- """- self._validate_ref(ref, "type")-- action: Dict[str, Any] = {"type": "type", "ref": ref, "text": text}- return await self._exec_with_snapshot(action)-- async def select(self, *, ref: str, value: str) -> Dict[str, str]:- r"""Selects an option from a dropdown (<select>) element on the page.-- Args:- ref (str): The reference ID of the <select> element. This ID is- obtained from the page snapshot.- value (str): The value of the option to be selected. This should- match the 'value' attribute of an <option> tag.-- Returns:- Dict[str, str]: A dictionary containing the result of the action.- The key is "result".- """- self._validate_ref(ref, "select")-- action: Dict[str, Any] = {"type": "select", "ref": ref, "value": value}- return await self._exec_with_snapshot(action)-- async def scroll(self, *, direction: str, amount: int) -> Dict[str, str]:- r"""Scrolls the current page up or down by a specified amount.-- Args:- direction (str): The direction to scroll. Must be either "up" or- "down".- amount (int): The number of pixels to scroll.-- Returns:- Dict[str, str]: A dictionary containing the result of the action.- The key is "result".- """- if direction not in ("up", "down"):- logger.error("scroll(): 'direction' must be 'up' or 'down'")- return {- "result": "scroll() Error: 'direction' must be 'up' or 'down'"- }-- action = {"type": "scroll", "direction": direction, "amount": amount}- return await self._exec_with_snapshot(action)-- async def enter(self, *, ref: str) -> Dict[str, str]:- r"""Simulates pressing the Enter key on a specific element.- This is often used to submit forms.-- Args:- ref (str): The reference ID of the element to focus before- pressing Enter. This ID is obtained from the page snapshot.-- Returns:- Dict[str, str]: A dictionary containing the result of the action.- If pressing Enter causes a page navigation or DOM change, a- "snapshot" key will be included with a diff of the changes.- """- self._validate_ref(ref, "enter")-- action: Dict[str, Any] = {"type": "enter", "ref": ref}- return await self._exec_with_snapshot(action)-- async def wait_user(- self,- timeout_sec: Optional[float] = None,- ) -> Dict[str, str]:- r"""Pauses the agent's execution and waits for human intervention.- This is useful for tasks that require manual steps, like solving a- CAPTCHA. The agent will print a message and wait for the user to- press the Enter key in the console.-- Args:- timeout_sec (Optional[float]): The maximum time in seconds to wait- for the user. If `None`, it will wait indefinitely. Defaults- to `None`. (default: :obj:`None`)-- Returns:- Dict[str, str]: A dictionary containing a result message and a- full snapshot of the current page after the user has acted.- The keys are "result" and "snapshot".- """-- import asyncio-- prompt = (- "🕑 Agent is waiting for human input. "- "Complete the required action in the browser, then press Enter "- "to continue..."- )-- logger.info(f"\n{prompt}\n")-- async def _await_enter():- await asyncio.to_thread(input, ">>> Press Enter to resume <<<\n")-- try:- if timeout_sec is not None:- await asyncio.wait_for(_await_enter(), timeout=timeout_sec)- result_msg = "User resumed."- else:- await _await_enter()- result_msg = "User resumed."- except asyncio.TimeoutError:- result_msg = f"Timeout {timeout_sec}s reached, auto-resumed."-- snapshot = await self._session.get_snapshot(- force_refresh=True,- diff_only=False,- )-- return {"result": result_msg, "snapshot": snapshot}-- # Helper to run through ActionExecutor- async def _exec(self, action: Dict[str, Any]) -> str:- return await self._session.exec_action(action)-- async def _exec_with_snapshot(- self, action: Dict[str, Any]- ) -> Dict[str, str]:- r"""Execute action and, if DOM structure changed, include snapshot- diff.- """- result = await self._session.exec_action(action)-- # Only capture diff if action type typically changes DOM- from .actions import ActionExecutor-- if not ActionExecutor.should_update_snapshot(action):- return {"result": result}-- # Capture structural diff to previous snapshot- diff = await self._session.get_snapshot(- force_refresh=True, diff_only=True- )-- if diff.startswith("- Page Snapshot (no structural changes)"):- return {"result": result}-- return {"result": result, "snapshot": diff}-- def _ensure_agent(self) -> PlaywrightLLMAgent:- r"""Create PlaywrightLLMAgent on first use if `web_agent_model`- provided."""- if self.web_agent_model is None:- raise RuntimeError(- "web_agent_model not supplied - high-level task planning is "- "unavailable."- )-- if self._agent is None:- self._agent = PlaywrightLLMAgent(- headless=self._headless,- user_data_dir=self._user_data_dir,- model_backend=self.web_agent_model,- )- return self._agent-- async def solve_task(- self, task_prompt: str, start_url: str, max_steps: int = 15- ) -> str:- r"""Use LLM agent to autonomously complete the task (requires- `web_agent_model`).-- Args:- task_prompt (str): The task prompt to complete.- start_url (str): The URL to navigate to.- max_steps (int): The maximum number of steps to take.- (default: :obj:`15`)-- Returns:- str: The result of the task.- """-- agent = self._ensure_agent()- await agent.navigate(start_url)- await agent.process_command(task_prompt, max_steps=max_steps)- return "Task processing finished - see stdout for detailed trace."-- def get_tools(self) -> List[FunctionTool]:- base_tools = [- FunctionTool(self.open_browser),- FunctionTool(self.close_browser),- FunctionTool(self.visit_page),- FunctionTool(self.click),- FunctionTool(self.type),- FunctionTool(self.select),- FunctionTool(self.scroll),- FunctionTool(self.enter),- FunctionTool(self.wait_user),- ]-- if self.web_agent_model is not None:- base_tools.append(FunctionTool(self.solve_task))-- return base_toolsdiff --git a/camel/types/agents/tool_calling_record.py b/camel/types/agents/tool_calling_record.pyindex d3359b4de..3e0b4c91e 100644--- a/camel/types/agents/tool_calling_record.py+++ b/camel/types/agents/tool_calling_record.py@@ -11,7 +11,7 @@# See the License for the specific language governing permissions and# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========-from typing import Any, Dict+from typing import Any, Dict, List, Optionalfrom pydantic import BaseModel@@ -24,12 +24,15 @@ class ToolCallingRecord(BaseModel):args (Dict[str, Any]): The dictionary of arguments passed to the tool.result (Any): The execution result of calling this tool.tool_call_id (str): The ID of the tool call, if available.+ images (Optional[List[str]]): List of base64-encoded images returned+ by the tool, if any."""tool_name: strargs: Dict[str, Any]result: Anytool_call_id: str+ images: Optional[List[str]] = Nonedef __str__(self) -> str:r"""Overridden version of the string function.diff --git a/camel/utils/tool_result.py b/camel/utils/tool_result.pynew file mode 100644index 000000000..48e87269d--- /dev/null+++ b/camel/utils/tool_result.py@@ -0,0 +1,44 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from typing import List, Optional+++class ToolResult:+ r"""Special result type for tools that can return images along with text.++ This class is used by ChatAgent to detect when a tool returns visual+ content that should be included in the conversation context.+ """++ def __init__(self, text: str, images: Optional[List[str]] = None):+ r"""Initialize a tool result.++ Args:+ text (str): The text description or result of the tool operation.+ images (Optional[List[str]]): List of base64-encoded images to+ include in the conversation context. Images should be encoded+ as "data:image/{format};base64,{data}" format.+ """+ self.text = text+ self.images = images or []++ def __str__(self) -> str:+ r"""Return the text representation of the result."""+ return self.text++ def __repr__(self) -> str:+ r"""Return a detailed representation of the result."""+ img_count = len(self.images) if self.images else 0+ return f"ToolResult(text='{self.text[:50]}...', images={img_count})"diff --git a/docs/mintlify/docs.json b/docs/mintlify/docs.jsonindex 09687bed4..cd523156a 100644--- a/docs/mintlify/docs.json+++ b/docs/mintlify/docs.json@@ -515,11 +515,11 @@"reference/camel.toolkits.whatsapp_toolkit","reference/camel.toolkits.wolfram_alpha_toolkit","reference/camel.toolkits.zapier_toolkit",- "reference/camel.toolkits.non_visual_browser_toolkit.actions",- "reference/camel.toolkits.non_visual_browser_toolkit.agent",- "reference/camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit",- "reference/camel.toolkits.non_visual_browser_toolkit.nv_browser_session",- "reference/camel.toolkits.non_visual_browser_toolkit.snapshot"+ "reference/camel.toolkits.hybrid_browser_toolkit.actions",+ "reference/camel.toolkits.hybrid_browser_toolkit.agent",+ "reference/camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit",+ "reference/camel.toolkits.hybrid_browser_toolkit.browser_session",+ "reference/camel.toolkits.hybrid_browser_toolkit.snapshot"]},{diff --git a/docs/reference/camel.toolkits.hybrid_browser_toolkit.actions.md b/docs/reference/camel.toolkits.hybrid_browser_toolkit.actions.mdnew file mode 100644index 000000000..2f3759a2f--- /dev/null+++ b/docs/reference/camel.toolkits.hybrid_browser_toolkit.actions.md@@ -0,0 +1,27 @@+<a id="camel.toolkits.hybrid_browser_toolkit.actions"></a>++<a id="camel.toolkits.hybrid_browser_toolkit.actions.ActionExecutor"></a>++## ActionExecutor++```python+class ActionExecutor:+```++Executes high-level actions (click, type …) on a Playwright Page.++<a id="camel.toolkits.hybrid_browser_toolkit.actions.ActionExecutor.__init__"></a>++### __init__++```python+def __init__(self, page: 'Page'):+```++<a id="camel.toolkits.hybrid_browser_toolkit.actions.ActionExecutor.should_update_snapshot"></a>++### should_update_snapshot++```python+def should_update_snapshot(action: Dict[str, Any]):+```diff --git a/docs/reference/camel.toolkits.non_visual_browser_toolkit.agent.md b/docs/reference/camel.toolkits.hybrid_browser_toolkit.agent.mdsimilarity index 56%rename from docs/reference/camel.toolkits.non_visual_browser_toolkit.agent.mdrename to docs/reference/camel.toolkits.hybrid_browser_toolkit.agent.mdindex 84ad5790f..bd9963a89 100644--- a/docs/reference/camel.toolkits.non_visual_browser_toolkit.agent.md+++ b/docs/reference/camel.toolkits.hybrid_browser_toolkit.agent.md@@ -1,6 +1,6 @@-<a id="camel.toolkits.non_visual_browser_toolkit.agent"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.agent"></a>-<a id="camel.toolkits.non_visual_browser_toolkit.agent.PlaywrightLLMAgent"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.agent.PlaywrightLLMAgent"></a>## PlaywrightLLMAgent@@ -10,7 +10,7 @@ class PlaywrightLLMAgent:High-level orchestration: snapshot ↔ LLM ↔ action executor.-<a id="camel.toolkits.non_visual_browser_toolkit.agent.PlaywrightLLMAgent.__init__"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.agent.PlaywrightLLMAgent.__init__"></a>### __init__@@ -18,7 +18,7 @@ High-level orchestration: snapshot ↔ LLM ↔ action executor.def __init__(self):```-<a id="camel.toolkits.non_visual_browser_toolkit.agent.PlaywrightLLMAgent._get_chat_agent"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.agent.PlaywrightLLMAgent._get_chat_agent"></a>### _get_chat_agent@@ -28,7 +28,7 @@ def _get_chat_agent(self):Get or create the ChatAgent instance.-<a id="camel.toolkits.non_visual_browser_toolkit.agent.PlaywrightLLMAgent._safe_parse_json"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.agent.PlaywrightLLMAgent._safe_parse_json"></a>### _safe_parse_json@@ -39,7 +39,7 @@ def _safe_parse_json(self, content: str):Safely parse JSON from LLM response with multiple fallbackstrategies.-<a id="camel.toolkits.non_visual_browser_toolkit.agent.PlaywrightLLMAgent._llm_call"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.agent.PlaywrightLLMAgent._llm_call"></a>### _llm_calldiff --git a/docs/reference/camel.toolkits.non_visual_browser_toolkit.nv_browser_session.md b/docs/reference/camel.toolkits.hybrid_browser_toolkit.browser_session.mdsimilarity index 55%rename from docs/reference/camel.toolkits.non_visual_browser_toolkit.nv_browser_session.mdrename to docs/reference/camel.toolkits.hybrid_browser_toolkit.browser_session.mdindex bd74fa5f9..8b806536c 100644--- a/docs/reference/camel.toolkits.non_visual_browser_toolkit.nv_browser_session.md+++ b/docs/reference/camel.toolkits.hybrid_browser_toolkit.browser_session.md@@ -1,6 +1,6 @@-<a id="camel.toolkits.non_visual_browser_toolkit.nv_browser_session"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.browser_session"></a>-<a id="camel.toolkits.non_visual_browser_toolkit.nv_browser_session.NVBrowserSession"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession"></a>## NVBrowserSession@@ -17,7 +17,7 @@ duplicating Playwright setup code.This class is a singleton per event-loop.-<a id="camel.toolkits.non_visual_browser_toolkit.nv_browser_session.NVBrowserSession.__new__"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession.__new__"></a>### __new__@@ -25,7 +25,7 @@ This class is a singleton per event-loop.def __new__(cls):```-<a id="camel.toolkits.non_visual_browser_toolkit.nv_browser_session.NVBrowserSession.__init__"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession.__init__"></a>### __init__diff --git a/docs/reference/camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.md b/docs/reference/camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.mdsimilarity index 56%rename from docs/reference/camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.mdrename to docs/reference/camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.mdindex fbb584892..a3c5e414c 100644--- a/docs/reference/camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.md+++ b/docs/reference/camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.md@@ -1,17 +1,17 @@-<a id="camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit"></a>-<a id="camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.BrowserNonVisualToolkit"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.HybridBrowserToolkit"></a>-## BrowserNonVisualToolkit+## HybridBrowserToolkit```python-class BrowserNonVisualToolkit(BaseToolkit):+class HybridBrowserToolkit(BaseToolkit):```A lightweight, *non-visual* browser toolkit exposing primitivePlaywright actions as CAMEL `FunctionTool`s.-<a id="camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.BrowserNonVisualToolkit.__init__"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.HybridBrowserToolkit.__init__"></a>### __init__@@ -19,7 +19,7 @@ Playwright actions as CAMEL `FunctionTool`s.def __init__(self):```-<a id="camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.BrowserNonVisualToolkit.__del__"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.HybridBrowserToolkit.__del__"></a>### __del__@@ -39,7 +39,7 @@ as `ImportError: sys.meta_path is None` orcarried out, we silently ignore the failure instead of pollutingstderr on program exit.-<a id="camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.BrowserNonVisualToolkit._validate_ref"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.HybridBrowserToolkit._validate_ref"></a>### _validate_ref@@ -49,7 +49,7 @@ def _validate_ref(self, ref: str, method_name: str):Validate that ref parameter is a non-empty string.-<a id="camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.BrowserNonVisualToolkit._ensure_agent"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.HybridBrowserToolkit._ensure_agent"></a>### _ensure_agent@@ -60,7 +60,7 @@ def _ensure_agent(self):Create PlaywrightLLMAgent on first use if `web_agent_model`provided.-<a id="camel.toolkits.non_visual_browser_toolkit.browser_non_visual_toolkit.BrowserNonVisualToolkit.get_tools"></a>+<a id="camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.HybridBrowserToolkit.get_tools"></a>### get_toolsdiff --git a/docs/reference/camel.toolkits.hybrid_browser_toolkit.snapshot.md b/docs/reference/camel.toolkits.hybrid_browser_toolkit.snapshot.mdnew file mode 100644index 000000000..cccf26330--- /dev/null+++ b/docs/reference/camel.toolkits.hybrid_browser_toolkit.snapshot.md@@ -0,0 +1,46 @@+<a id="camel.toolkits.hybrid_browser_toolkit.snapshot"></a>++<a id="camel.toolkits.hybrid_browser_toolkit.snapshot.PageSnapshot"></a>++## PageSnapshot++```python+class PageSnapshot:+```++Utility for capturing YAML-like page snapshots and diff-only+variants.++<a id="camel.toolkits.hybrid_browser_toolkit.snapshot.PageSnapshot.__init__"></a>++### __init__++```python+def __init__(self, page: 'Page'):+```++<a id="camel.toolkits.hybrid_browser_toolkit.snapshot.PageSnapshot._format_snapshot"></a>++### _format_snapshot++```python+def _format_snapshot(text: str):+```++<a id="camel.toolkits.hybrid_browser_toolkit.snapshot.PageSnapshot._compute_diff"></a>++### _compute_diff++```python+def _compute_diff(old: str, new: str):+```++<a id="camel.toolkits.hybrid_browser_toolkit.snapshot.PageSnapshot._detect_priorities"></a>++### _detect_priorities++```python+def _detect_priorities(self, snapshot_yaml: str):+```++Return sorted list of priorities present (1,2,3).diff --git a/docs/reference/camel.toolkits.non_visual_browser_toolkit.actions.md b/docs/reference/camel.toolkits.non_visual_browser_toolkit.actions.mddeleted file mode 100644index 024feec52..000000000--- a/docs/reference/camel.toolkits.non_visual_browser_toolkit.actions.md+++ /dev/null@@ -1,27 +0,0 @@-<a id="camel.toolkits.non_visual_browser_toolkit.actions"></a>--<a id="camel.toolkits.non_visual_browser_toolkit.actions.ActionExecutor"></a>--## ActionExecutor--```python-class ActionExecutor:-```--Executes high-level actions (click, type …) on a Playwright Page.--<a id="camel.toolkits.non_visual_browser_toolkit.actions.ActionExecutor.__init__"></a>--### __init__--```python-def __init__(self, page: 'Page'):-```--<a id="camel.toolkits.non_visual_browser_toolkit.actions.ActionExecutor.should_update_snapshot"></a>--### should_update_snapshot--```python-def should_update_snapshot(action: Dict[str, Any]):-```diff --git a/docs/reference/camel.toolkits.non_visual_browser_toolkit.snapshot.md b/docs/reference/camel.toolkits.non_visual_browser_toolkit.snapshot.mddeleted file mode 100644index 81522a636..000000000--- a/docs/reference/camel.toolkits.non_visual_browser_toolkit.snapshot.md+++ /dev/null@@ -1,46 +0,0 @@-<a id="camel.toolkits.non_visual_browser_toolkit.snapshot"></a>--<a id="camel.toolkits.non_visual_browser_toolkit.snapshot.PageSnapshot"></a>--## PageSnapshot--```python-class PageSnapshot:-```--Utility for capturing YAML-like page snapshots and diff-only-variants.--<a id="camel.toolkits.non_visual_browser_toolkit.snapshot.PageSnapshot.__init__"></a>--### __init__--```python-def __init__(self, page: 'Page'):-```--<a id="camel.toolkits.non_visual_browser_toolkit.snapshot.PageSnapshot._format_snapshot"></a>--### _format_snapshot--```python-def _format_snapshot(text: str):-```--<a id="camel.toolkits.non_visual_browser_toolkit.snapshot.PageSnapshot._compute_diff"></a>--### _compute_diff--```python-def _compute_diff(old: str, new: str):-```--<a id="camel.toolkits.non_visual_browser_toolkit.snapshot.PageSnapshot._detect_priorities"></a>--### _detect_priorities--```python-def _detect_priorities(self, snapshot_yaml: str):-```--Return sorted list of priorities present (1,2,3).diff --git a/examples/toolkits/browser_non_visual_toolkit_example.py b/examples/toolkits/hybrid_browser_toolkit_example.pysimilarity index 52%rename from examples/toolkits/browser_non_visual_toolkit_example.pyrename to examples/toolkits/hybrid_browser_toolkit_example.pyindex ecee5b084..22b635b62 100644--- a/examples/toolkits/browser_non_visual_toolkit_example.py+++ b/examples/toolkits/hybrid_browser_toolkit_example.py@@ -16,7 +16,7 @@ import loggingfrom camel.agents import ChatAgentfrom camel.models import ModelFactory-from camel.toolkits import BrowserNonVisualToolkit+from camel.toolkits import HybridBrowserToolkitfrom camel.types import ModelPlatformType, ModelTypelogging.basicConfig(@@ -30,49 +30,77 @@ logging.basicConfig(logging.getLogger('camel.agents').setLevel(logging.INFO)logging.getLogger('camel.models').setLevel(logging.INFO)logging.getLogger('camel.models').setLevel(logging.INFO)-logging.getLogger('camel.toolkits.non_visual_browser_toolkit').setLevel(+logging.getLogger('camel.toolkits.hybrid_browser_toolkit').setLevel(logging.DEBUG)USER_DATA_DIR = "User_Data"model_backend = ModelFactory.create(model_platform=ModelPlatformType.OPENAI,- model_type=ModelType.GPT_4_1,+ model_type=ModelType.GPT_4O,model_config_dict={"temperature": 0.0, "top_p": 1},)-web_toolkit = BrowserNonVisualToolkit(- headless=False, user_data_dir=USER_DATA_DIR+# Example 1: Use default tools (basic functionality)+# web_toolkit_default = HybridBrowserToolkit(+# headless=False,+# user_data_dir=USER_DATA_DIR+# )+# print(f"Default tools: {web_toolkit_default.enabled_tools}")++# Example 2: Use all available tools+# web_toolkit_all = HybridBrowserToolkit(+# headless=False,+# user_data_dir=USER_DATA_DIR,+# enabled_tools=HybridBrowserToolkit.ALL_TOOLS+# )+# print(f"All tools: {web_toolkit_all.enabled_tools}")++# Example 3: Use custom tools selection+custom_tools = [+ "open_browser",+ "close_browser",+ "visit_page",+ "get_som_screenshot", # Add screenshot capability+ "click",+ "type",+]++web_toolkit_custom = HybridBrowserToolkit(+ headless=False, user_data_dir=USER_DATA_DIR, enabled_tools=custom_tools)+print(f"Custom tools: {web_toolkit_custom.enabled_tools}")++# Use the custom toolkit for the actual taskagent = ChatAgent(model=model_backend,- tools=[*web_toolkit.get_tools()],+ tools=[*web_toolkit_custom.get_tools()],max_iteration=10,)-TASK_PROMPT = """Play this game:-https://www.nytimes.com/games/wordle/index.html-You need to enter meaningful five-letter words, such as "Apple".-Before completing the task, do not do anything outside of using the call tool.-If the game tells you that something is wrong, you must resolve it yourself.-Also, you must attempt all six tries.--If the registration window can be closed, please close it.+TASK_PROMPT = r"""+Open Amazon.com, search for a beautiful keyboard,+click it and add it to the cart+and use get_som_screenshot to see the final+picture of keyboard and describe its appearance"""async def main() -> None:response = await agent.astep(TASK_PROMPT)print("Task:", TASK_PROMPT)- print(f"Using user data directory: {USER_DATA_DIR}\n")- print("Response from agent:")+ print(f"Using user data directory: {USER_DATA_DIR}")+ print(f"Enabled tools: {web_toolkit_custom.enabled_tools}")+ print("\nResponse from agent:")print(response.msgs[0].content if response.msgs else "<no response>")try:- await web_toolkit.close_browser()+ await web_toolkit_custom.close_browser()except Exception as err:- logging.warning("Failed to close browser session explicitly: %s", err)+ logging.warning(+ "Failed to close " "browser session explicitly: %s", err+ )if __name__ == "__main__":diff --git a/examples/workforce/eigent.py b/examples/workforce/eigent.pyindex b57bbeb11..38d1f0dbd 100644--- a/examples/workforce/eigent.py+++ b/examples/workforce/eigent.py@@ -22,7 +22,6 @@ from camel.societies.workforce import Workforcefrom camel.tasks.task import Taskfrom camel.toolkits import (AudioAnalysisToolkit,- BrowserNonVisualToolkit,CodeExecutionToolkit,Crawl4AIToolkit,DalleToolkit,@@ -32,6 +31,7 @@ from camel.toolkits import (FunctionTool,# GoogleDriveMCPToolkit,HumanToolkit,+ HybridBrowserToolkit,ImageAnalysisToolkit,LinkedInToolkit,MarkItDownToolkit,@@ -140,7 +140,7 @@ def search_agent_factory(FunctionTool(SearchToolkit().search_exa),# FunctionTool(SearchToolkit().search_bing),# FunctionTool(SearchToolkit().search_baidu),- *BrowserNonVisualToolkit(headless=False).get_tools(),+ *HybridBrowserToolkit(headless=False).get_tools(),*TerminalToolkit().get_tools(),send_message_to_user,HumanToolkit().ask_human_via_console,diff --git a/pyproject.toml b/pyproject.tomlindex 00d04629f..8e4e7ccc9 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -601,4 +601,4 @@ skip = '.git*,*.pdf,*.svg,*.lock,*.css,examples/toolkits/semantic_scholar_toolkicheck-hidden = true# Ignore long lines as usually some kind of base64 encodings e.g. in ipynbignore-regex = '^\s*"image/\S+": ".*|^.{300,}|\b(NotIn|Unsupport)\b'-ignore-words-list = 'ans,datas,rouge,doubleClick,OCE,ACI,aci,Pinter,'+ignore-words-list = 'ans,datas,rouge,doubleClick,OCE,ACI,aci,Pinter,SoM,'diff --git a/test/toolkits/test_file_write_toolkit.py b/test/toolkits/test_file_write_toolkit.pyindex 5163cf424..03a9b630f 100644--- a/test/toolkits/test_file_write_toolkit.py+++ b/test/toolkits/test_file_write_toolkit.py@@ -69,7 +69,9 @@ def test_write_to_file_text(file_write_toolkit):content = "Hello, world!"filename = "test.txt"- result = file_write_toolkit.write_to_file(content, filename)+ result = file_write_toolkit.write_to_file(+ "Test Document", content, filename+ )# Check the result messageassert "successfully written" in result@@ -89,7 +91,9 @@ def test_write_to_file_markdown(file_write_toolkit):content = "# Heading\n\nThis is a paragraph with **bold** text."filename = "test.md"- result = file_write_toolkit.write_to_file(content, filename)+ result = file_write_toolkit.write_to_file(+ "Markdown Document", content, filename+ )# Check the result messageassert "successfully written" in result@@ -109,7 +113,7 @@ def test_write_to_file_json(file_write_toolkit):data = {"name": "John", "age": 30, "city": "New York"}filename = "test.json"- result = file_write_toolkit.write_to_file(data, filename)+ result = file_write_toolkit.write_to_file("JSON Document", data, filename)# Check the result messageassert "successfully written" in result@@ -129,7 +133,7 @@ def test_write_to_file_yaml(file_write_toolkit):data = {"name": "John", "age": 30, "city": "New York"}filename = "test.yaml"- result = file_write_toolkit.write_to_file(data, filename)+ result = file_write_toolkit.write_to_file("YAML Document", data, filename)# Check the result messageassert "successfully written" in result@@ -149,7 +153,9 @@ def test_write_to_file_csv_string(file_write_toolkit):content = "name,age,city\nJohn,30,New York\nJane,25,San Francisco"filename = "test.csv"- result = file_write_toolkit.write_to_file(content, filename)+ result = file_write_toolkit.write_to_file(+ "CSV Document", content, filename+ )# Check the result messageassert "successfully written" in result@@ -173,7 +179,9 @@ def test_write_to_file_csv_list(file_write_toolkit):]filename = "test_list.csv"- result = file_write_toolkit.write_to_file(content, filename)+ result = file_write_toolkit.write_to_file(+ "CSV List Document", content, filename+ )# Check the result messageassert "successfully written" in result@@ -196,7 +204,9 @@ def test_write_to_file_html(file_write_toolkit):content = "<html><body><h1>Hello, world!</h1></body></html>"filename = "test.html"- result = file_write_toolkit.write_to_file(content, filename)+ result = file_write_toolkit.write_to_file(+ "HTML Document", content, filename+ )# Check the result messageassert "successfully written" in result@@ -218,7 +228,9 @@ def test_write_to_file_no_extension(file_write_toolkit):content = "# Default format test"filename = "test_no_extension"- result = file_write_toolkit.write_to_file(content, filename)+ result = file_write_toolkit.write_to_file(+ "Default Format Document", content, filename+ )# Check the result messageassert "successfully written" in result@@ -241,7 +253,7 @@ def test_write_to_file_custom_encoding(file_write_toolkit):encoding = "latin-1"result = file_write_toolkit.write_to_file(- content, filename, encoding=encoding+ "Custom Encoding Document", content, filename, encoding=encoding)# Check the result message@@ -263,7 +275,9 @@ def test_write_to_file_nested_directory(file_write_toolkit):content = "Content in nested directory"filename = "nested/dir/test.txt"- result = file_write_toolkit.write_to_file(content, filename)+ result = file_write_toolkit.write_to_file(+ "Nested Directory Document", content, filename+ )# Check the result messageassert "successfully written" in result@@ -284,7 +298,7 @@ def test_write_to_file_absolute_path(temp_dir):content = "Content with absolute path"filename = os.path.join(temp_dir, "absolute_path.txt")- result = toolkit.write_to_file(content, filename)+ result = toolkit.write_to_file("Absolute Path Document", content, filename)# Check the result messageassert "successfully written" in resultdiff --git a/test/toolkits/test_hybrid_browser_toolkit.py b/test/toolkits/test_hybrid_browser_toolkit.pynew file mode 100644index 000000000..4d36bb91f--- /dev/null+++ b/test/toolkits/test_hybrid_browser_toolkit.py@@ -0,0 +1,1348 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import io+import os+import shutil+from unittest.mock import AsyncMock, MagicMock, mock_open, patch++import pytest+from PIL import Image++# Import the modules directly without global sys.modules mocking+from camel.toolkits.hybrid_browser_toolkit import HybridBrowserToolkit+from camel.utils.tool_result import ToolResult++TEST_URL = "https://example.com"+TEST_FILE_URL = "file:///test.html"+++def create_mock_playwright_objects():+ """Create mock playwright objects for testing."""+ # Create comprehensive mock objects+ mock_page = AsyncMock()+ mock_page.url = "https://example.com"+ mock_page.goto = AsyncMock()+ mock_page.wait_for_load_state = AsyncMock()+ mock_page.screenshot = AsyncMock(return_value=b"fake_screenshot_data")+ mock_page.evaluate = AsyncMock(return_value={"elements": {}})+ mock_page.query_selector = AsyncMock(return_value=None)+ mock_page.query_selector_all = AsyncMock(return_value=[])+ mock_page.click = AsyncMock()+ mock_page.fill = AsyncMock()+ mock_page.select_option = AsyncMock()+ mock_page.mouse = MagicMock()+ mock_page.mouse.wheel = AsyncMock()++ mock_context = AsyncMock()+ mock_context.pages = [mock_page]+ mock_context.new_page = AsyncMock(return_value=mock_page)+ mock_context.close = AsyncMock()+ mock_context.browser = None++ mock_browser = AsyncMock()+ mock_browser.new_context = AsyncMock(return_value=mock_context)+ mock_browser.close = AsyncMock()+ mock_browser.contexts = [mock_context]++ mock_chromium = MagicMock()+ mock_chromium.launch = AsyncMock(return_value=mock_browser)+ mock_chromium.launch_persistent_context = AsyncMock(+ return_value=mock_context+ )++ mock_playwright_instance = AsyncMock()+ mock_playwright_instance.chromium = mock_chromium+ mock_playwright_instance.firefox = mock_chromium+ mock_playwright_instance.webkit = mock_chromium+ mock_playwright_instance.stop = AsyncMock()++ # Mock the async_playwright context manager+ mock_async_playwright_cm = AsyncMock()+ mock_async_playwright_cm.__aenter__ = AsyncMock(+ return_value=mock_playwright_instance+ )+ mock_async_playwright_cm.__aexit__ = AsyncMock(return_value=None)++ # Mock the async_playwright function+ mock_async_playwright = MagicMock(return_value=mock_async_playwright_cm)++ return (+ mock_async_playwright,+ mock_playwright_instance,+ mock_page,+ mock_context,+ mock_browser,+ )+++# Create global mock objects for backward compatibility+(+ mock_async_playwright,+ mock_playwright_instance,+ mock_page,+ mock_context,+ mock_browser,+) = create_mock_playwright_objects()++[email protected](scope="function")+def mock_playwright_setup():+ """Set up playwright mocks for individual tests."""+ return create_mock_playwright_objects()++[email protected](autouse=True)+def mock_browser_dependencies():+ """Mock all browser-related dependencies for each test."""+ with (+ patch('os.makedirs'),+ patch('os.path.exists', return_value=True),+ patch(+ 'playwright.async_api.async_playwright',+ return_value=mock_async_playwright(),+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.async_playwright',+ return_value=mock_async_playwright(),+ create=True,+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock unified analyzer');"),+ ),+ ):+ yield++[email protected](scope="function")+async def browser_toolkit_fixture():+ """Create a HybridBrowserToolkit instance for testing."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open', mock_open(read_data="console.log('mock script');")+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session # Ensure the mock session is used+ yield toolkit++ # Cleanup+ try:+ await toolkit.close_browser()+ except Exception:+ pass++ if os.path.exists("test_cache"):+ shutil.rmtree("test_cache", ignore_errors=True)++[email protected](scope="function")+def mock_session():+ """Create a mock browser session."""+ session = AsyncMock()+ session.ensure_browser = AsyncMock()+ session.get_page = AsyncMock(return_value=mock_page)+ session.visit = AsyncMock(return_value="Visited test URL")+ session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ session.close = AsyncMock()+ return session++[email protected](scope="function")+def mock_page_fixture():+ """Create a mock page object."""+ page = AsyncMock()+ page.url = TEST_URL+ page.screenshot = AsyncMock(return_value=b"fake_screenshot_data")+ page.evaluate = AsyncMock(return_value={"elements": {}})+ page.query_selector = AsyncMock(return_value=None)+ page.query_selector_all = AsyncMock(return_value=[])+ page.click = AsyncMock()+ page.fill = AsyncMock()+ page.select_option = AsyncMock()+ page.mouse = MagicMock()+ page.mouse.wheel = AsyncMock()+ return page++[email protected](scope="function")+def sync_browser_toolkit():+ """Create a HybridBrowserToolkit instance for synchronous testing."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open', mock_open(read_data="console.log('mock script');")+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session # Ensure the mock session is used+ return toolkit+++class TestHybridBrowserToolkit:+ """Test cases for HybridBrowserToolkit."""++ def test_init_default_params(self):+ """Test initialization with default parameters."""+ mock_session = AsyncMock()+ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session'+ '.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock unified analyzer');"),+ ),+ patch('os.path.exists', return_value=True),+ patch('os.makedirs'),+ ):+ toolkit = HybridBrowserToolkit()++ assert toolkit._headless is True+ assert toolkit._user_data_dir is None+ assert toolkit.web_agent_model is None+ assert toolkit.cache_dir == "tmp/"+ assert toolkit._agent is None+ assert toolkit._unified_script is not None++ def test_init_custom_params(self):+ """Test initialization with custom parameters."""+ test_cache_dir = "custom_cache"+ test_user_data_dir = "custom_user_data"++ mock_session = AsyncMock()+ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session'+ '.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock unified analyzer');"),+ ),+ patch('os.path.exists', return_value=True),+ patch('os.makedirs'),+ ):+ toolkit = HybridBrowserToolkit(+ headless=False,+ user_data_dir=test_user_data_dir,+ )++ assert toolkit._headless is False+ assert toolkit._user_data_dir == test_user_data_dir+ assert toolkit.cache_dir == "tmp/" # Default cache dir++ # Cleanup+ if os.path.exists(test_cache_dir):+ shutil.rmtree(test_cache_dir, ignore_errors=True)++ def test_load_unified_analyzer_success(self):+ """Test successful loading of unified analyzer script."""+ mock_session = AsyncMock()+ with patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session'+ '.NVBrowserSession',+ return_value=mock_session,+ ):+ with patch('builtins.open', create=True) as mock_file:+ # ruff:noqa:E501+ mock_file.return_value.__enter__.return_value.read.return_value = "console.log('test');"++ with (+ patch('os.makedirs'),+ patch('os.path.exists', return_value=True),+ ):+ toolkit = HybridBrowserToolkit()+ assert "console.log('test');" in toolkit._unified_script++ def test_load_unified_analyzer_file_not_found(self):+ """Test handling of missing unified analyzer script."""+ mock_session = AsyncMock()+ with patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session'+ '.NVBrowserSession',+ return_value=mock_session,+ ):+ with (+ patch('builtins.open', side_effect=FileNotFoundError),+ patch('os.makedirs'),+ patch('os.path.exists', return_value=True),+ ):+ with pytest.raises(FileNotFoundError):+ HybridBrowserToolkit()++ def test_validate_ref_valid(self):+ """Test ref validation with valid input."""+ mock_session = AsyncMock()+ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session'+ '.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ patch('os.makedirs'),+ patch('os.path.exists', return_value=True),+ ):+ toolkit = HybridBrowserToolkit()+ # Should not raise exception+ toolkit._validate_ref("e1", "test_method")++ def test_validate_ref_invalid(self):+ """Test ref validation with invalid input."""+ mock_session = AsyncMock()+ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session'+ '.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ patch('os.makedirs'),+ patch('os.path.exists', return_value=True),+ ):+ toolkit = HybridBrowserToolkit()++ # Test with empty string+ with pytest.raises(ValueError):+ toolkit._validate_ref("", "test_method")++ @pytest.mark.asyncio+ async def test_open_browser_no_start_url(self):+ """Test opening browser without start URL."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ result = await toolkit.open_browser()++ assert result["result"] == "Browser session started."+ assert "snapshot" in result++ @pytest.mark.asyncio+ async def test_open_browser_with_start_url(self):+ """Test opening browser with start URL."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ # Mock visit_page method+ with patch.object(+ toolkit, 'visit_page', new_callable=AsyncMock+ ) as mock_visit:+ mock_visit.return_value = {+ "result": "Visited",+ "snapshot": "test",+ }++ await toolkit.open_browser(start_url=TEST_URL)++ mock_visit.assert_called_once_with(TEST_URL)++ @pytest.mark.asyncio+ async def test_close_browser(self):+ """Test closing browser."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ result = await toolkit.close_browser()++ assert result == "Browser session closed."++ @pytest.mark.asyncio+ async def test_visit_page_valid_url(self):+ """Test visiting a valid URL."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ result = await toolkit.visit_page(TEST_URL)++ assert "Visited" in result["result"]+ assert "snapshot" in result++ @pytest.mark.asyncio+ async def test_visit_page_invalid_url(self):+ """Test visiting with invalid URL."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ result = await toolkit.visit_page("")+ assert "Error" in result.get("result", "")+ assert "'url' must be a non-empty string" in result.get(+ "result", ""+ )++ @pytest.mark.asyncio+ async def test_get_page_snapshot(self):+ """Test getting page snapshot."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ # Mock unified analysis+ mock_analysis = {+ "snapshotText": "Test snapshot content",+ "elements": {"e1": {"role": "button", "name": "Test"}},+ }++ # Ensure the mock page has the analysis+ mock_page.evaluate = AsyncMock(return_value=mock_analysis)++ result = await toolkit.get_page_snapshot()++ assert result == "Test snapshot content"++ @pytest.mark.asyncio+ async def test_get_som_screenshot_success(self):+ """Test successful Set of Marks screenshot generation."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ # Create a mock image+ mock_image = Image.new('RGB', (100, 100), color='red')+ img_byte_arr = io.BytesIO()+ mock_image.save(img_byte_arr, format='PNG')+ mock_image_data = img_byte_arr.getvalue()++ # Set up the mock page with screenshot and analysis data+ mock_page.screenshot = AsyncMock(return_value=mock_image_data)+ mock_page.url = TEST_URL++ # Mock unified analysis+ mock_analysis = {+ "elements": {+ "e1": {+ "role": "button",+ "name": "Test Button",+ "coordinates": [+ {"x": 10, "y": 10, "width": 50, "height": 20}+ ],+ }+ }+ }+ mock_page.evaluate = AsyncMock(return_value=mock_analysis)++ result = await toolkit.get_som_screenshot()++ assert isinstance(result, ToolResult)+ assert "Visual webpage screenshot captured" in result.text+ assert len(result.images) == 1+ assert result.images[0].startswith("data:image/png;base64,")++ @pytest.mark.asyncio+ async def test_get_som_screenshot_pil_not_available(self):+ """Test Set of Marks screenshot when PIL is not available."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ # Create a mock image for the screenshot+ mock_image = Image.new('RGB', (100, 100), color='red')+ img_byte_arr = io.BytesIO()+ mock_image.save(img_byte_arr, format='PNG')+ mock_image_data = img_byte_arr.getvalue()++ # Set up the mock page with screenshot data+ mock_page.screenshot = AsyncMock(return_value=mock_image_data)+ mock_page.url = TEST_URL++ # Mock unified analysis - provide empty elements to avoid coordinate processing+ mock_analysis = {"elements": {}}+ mock_page.evaluate = AsyncMock(return_value=mock_analysis)++ with patch('PIL.ImageDraw', side_effect=ImportError):+ result = await toolkit.get_som_screenshot()++ assert isinstance(result, ToolResult)+ assert "Visual webpage screenshot captured" in result.text++ @pytest.mark.asyncio+ async def test_click_valid_ref(self):+ """Test clicking with valid reference."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ # Mock the _get_unified_analysis method to include e1 element+ mock_analysis = {"elements": {"e1": {"role": "button"}}}+ toolkit._get_unified_analysis = AsyncMock(+ return_value=mock_analysis+ )++ result = await toolkit.click(ref="e1")++ assert result["result"] == "Action executed successfully"+ assert "snapshot" in result++ @pytest.mark.asyncio+ async def test_click_invalid_ref(self):+ """Test clicking with invalid reference."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ with pytest.raises(ValueError):+ await toolkit.click(ref="")++ @pytest.mark.asyncio+ async def test_type_text(self):+ """Test typing text in element."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ result = await toolkit.type(+ ref="input_element", text="Hello World"+ )++ assert result["result"] == "Action executed successfully"+ assert "snapshot" in result++ @pytest.mark.asyncio+ async def test_select_option(self):+ """Test selecting option in dropdown."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ result = await toolkit.select(+ ref="select_element", value="option1"+ )++ assert result["result"] == "Action executed successfully"+ assert "snapshot" in result++ @pytest.mark.asyncio+ async def test_scroll_valid_direction(self):+ """Test scrolling with valid direction."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ result = await toolkit.scroll(direction="down", amount=100)++ assert result["result"] == "Action executed successfully"+ assert "snapshot" in result++ @pytest.mark.asyncio+ async def test_scroll_invalid_direction(self):+ """Test scrolling with invalid direction."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ result = await toolkit.scroll(+ direction="invalid_direction", amount=100+ )+ assert (+ "Error: direction must be 'up' or 'down'" in result["result"]+ )++ @pytest.mark.asyncio+ async def test_get_page_links_valid_refs(self):+ """Test getting page links with valid references."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ # Mock page evaluate to return links+ mock_links = [+ {"href": "https://example.com/page1", "text": "Page 1"},+ {"href": "https://example.com/page2", "text": "Page 2"},+ ]+ mock_page.evaluate = AsyncMock(return_value=mock_links)++ result = await toolkit.get_page_links(ref=["e1", "e2"])++ assert "links" in result+ assert isinstance(result["links"], list)++ @pytest.mark.asyncio+ async def test_get_page_links_invalid_refs(self):+ """Test getting page links with invalid references."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ result = await toolkit.get_page_links(ref=[])+ assert result["links"] == []++ @pytest.mark.asyncio+ async def test_wait_user_no_timeout(self):+ """Test waiting for user input without timeout."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ # Mock user input+ with patch('builtins.input', return_value='continue'):+ result = await toolkit.wait_user()++ assert result["result"] == "User resumed."+ assert "snapshot" in result++ @pytest.mark.asyncio+ async def test_wait_user_with_timeout(self):+ """Test waiting for user input with timeout."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ # Mock asyncio.wait_for to simulate timeout+ import asyncio++ with patch('asyncio.wait_for', side_effect=asyncio.TimeoutError()):+ result = await toolkit.wait_user(timeout_sec=1.0)++ assert (+ "Timeout 1.0s reached, auto-resumed." in result["result"]+ )+ assert "snapshot" in result++ def test_get_tools(self, sync_browser_toolkit):+ """Test getting available tools with default configuration."""+ toolkit = sync_browser_toolkit++ tools = toolkit.get_tools()++ # Default tools should be 5 tools (updated default configuration)+ assert len(tools) == 5+ tool_names = [tool.func.__name__ for tool in tools]++ expected_default_tools = [+ 'open_browser',+ 'close_browser',+ 'visit_page',+ 'click',+ 'type',+ ]++ for expected_tool in expected_default_tools:+ assert expected_tool in tool_names++ def test_get_tools_with_web_agent_model(self, sync_browser_toolkit):+ """Test getting tools when web_agent_model is provided."""+ mock_model = MagicMock()+ toolkit = sync_browser_toolkit++ # Simulate having a web agent model+ toolkit.web_agent_model = mock_model++ tools = toolkit.get_tools()++ # Default tools should be 5, even with web_agent_model set+ # unless solve_task is explicitly enabled+ assert len(tools) == 5+ tool_names = [tool.func.__name__ for tool in tools]++ # Should not automatically include solve_task unless explicitly enabled+ assert 'solve_task' not in tool_names++ def test_get_tools_with_solve_task_enabled(self):+ """Test getting tools when solve_task is explicitly enabled."""+ mock_model = MagicMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=AsyncMock(),+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(+ headless=True,+ web_agent_model=mock_model,+ enabled_tools=['open_browser', 'close_browser', 'solve_task'],+ )++ tools = toolkit.get_tools()+ tool_names = [tool.func.__name__ for tool in tools]++ assert len(tools) == 3+ assert 'solve_task' in tool_names+ assert 'open_browser' in tool_names+ assert 'close_browser' in tool_names++ def test_get_tools_custom_selection(self):+ """Test getting tools with custom tool selection."""+ custom_tools = [+ 'open_browser',+ 'visit_page',+ 'get_som_screenshot',+ 'click',+ 'scroll',+ ]++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=AsyncMock(),+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(+ headless=True, enabled_tools=custom_tools+ )++ tools = toolkit.get_tools()+ tool_names = [tool.func.__name__ for tool in tools]++ assert len(tools) == 5+ for expected_tool in custom_tools:+ assert expected_tool in tool_names++ def test_convert_analysis_to_rects(self, sync_browser_toolkit):+ """Test converting analysis data to rect format."""+ toolkit = sync_browser_toolkit++ analysis_data = {+ "elements": {+ "e1": {+ "role": "button",+ "name": "Submit",+ "coordinates": [+ {"x": 10, "y": 20, "width": 50, "height": 30}+ ],+ }+ }+ }++ rects = toolkit._convert_analysis_to_rects(analysis_data)++ assert "e1" in rects+ assert rects["e1"]["role"] == "button"+ assert rects["e1"]["aria-name"] == "Submit"+ assert len(rects["e1"]["rects"]) == 1+ assert rects["e1"]["rects"][0]["x"] == 10++ def test_format_snapshot_from_analysis(self, sync_browser_toolkit):+ """Test formatting snapshot from analysis data."""+ toolkit = sync_browser_toolkit++ analysis_data = {+ "elements": {+ "e1": {+ "role": "button",+ "name": "Submit",+ "disabled": True,+ },+ "e2": {+ "role": "checkbox",+ "name": "Agree",+ "checked": True,+ },+ }+ }++ snapshot = toolkit._format_snapshot_from_analysis(analysis_data)++ assert "[ref=e1]" in snapshot+ assert "button" in snapshot+ assert "Submit" in snapshot+ assert "disabled" in snapshot+ assert "[ref=e2]" in snapshot+ assert "checkbox" in snapshot+ assert "checked" in snapshot++ def test_add_set_of_mark_with_pil(self, sync_browser_toolkit):+ """Test adding visual marks to image."""+ toolkit = sync_browser_toolkit++ # Create test image+ test_image = Image.new('RGB', (200, 200), color='white')++ # Test rects data+ rects = {+ "e1": {+ "role": "button",+ "rects": [{"x": 10, "y": 10, "width": 50, "height": 20}],+ }+ }++ marked_image = toolkit._add_set_of_mark(test_image, rects)++ assert isinstance(marked_image, Image.Image)+ assert marked_image.size == test_image.size++ def test_add_set_of_mark_without_pil(self, sync_browser_toolkit):+ """Test adding visual marks when PIL is not available."""+ toolkit = sync_browser_toolkit++ test_image = Image.new('RGB', (200, 200), color='white')++ with patch('PIL.ImageDraw', side_effect=ImportError):+ result = toolkit._add_set_of_mark(test_image, {})++ # Should return original image when PIL not available+ assert result == test_image++ @pytest.mark.asyncio+ async def test_simple_async_creation(self):+ """Simple test to verify async toolkit creation works with mocks."""+ mock_session = AsyncMock()+ mock_session.ensure_browser = AsyncMock()+ mock_session.get_page = AsyncMock(return_value=mock_page)+ mock_session.visit = AsyncMock(return_value="Visited test URL")+ mock_session.get_snapshot = AsyncMock(+ return_value="- Page Snapshot\\n```yaml\\n<test content>\\n```"+ )+ mock_session.exec_action = AsyncMock(+ return_value="Action executed successfully"+ )+ mock_session.close = AsyncMock()++ with (+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.browser_session.NVBrowserSession',+ return_value=mock_session,+ ),+ patch(+ 'camel.toolkits.hybrid_browser_toolkit.hybrid_browser_toolkit.os.path.join',+ return_value='mock_script_path',+ ),+ patch(+ 'builtins.open',+ mock_open(read_data="console.log('mock script');"),+ ),+ ):+ toolkit = HybridBrowserToolkit(headless=True)+ toolkit._session = mock_session++ # Test a simple async operation+ result = await toolkit.open_browser()++ assert result["result"] == "Browser session started."+ assert "snapshot" in result