🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
[refactor 1/n] Web toolkit (#1820)
Zecheng Zhangaed5006c8f0a
diff --git a/camel/toolkits/web_toolkit.py b/camel/toolkits/web_toolkit.pyindex 7afeaee3e..76113f6c7 100644--- a/camel/toolkits/web_toolkit.py+++ b/camel/toolkits/web_toolkit.py@@ -92,7 +92,7 @@ ACTION_WITH_FEEDBACK_LIST = []-# codes from magentic-one+# Code from magentic-oneclass DOMRectangle(TypedDict):x: Union[int, float]y: Union[int, float]@@ -127,21 +127,36 @@ class InteractiveRegion(TypedDict):def _get_str(d: Any, k: str) -> str:+ r"""Safely retrieve a string value from a dictionary."""+ if k not in d:+ raise KeyError(f"Missing required key: '{k}'")val = d[k]- assert isinstance(val, str)- return val+ if isinstance(val, str):+ return val+ raise TypeError(+ f"Expected a string for key '{k}', " f"but got {type(val).__name__}"+ )def _get_number(d: Any, k: str) -> Union[int, float]:+ r"""Safely retrieve a number (int or float) from a dictionary"""val = d[k]- assert isinstance(val, int) or isinstance(val, float)- return val+ if isinstance(val, (int, float)):+ return val+ raise TypeError(+ f"Expected a number (int/float) for key "+ f"'{k}', but got {type(val).__name__}"+ )def _get_bool(d: Any, k: str) -> bool:+ r"""Safely retrieve a boolean value from a dictionary."""val = d[k]- assert isinstance(val, bool)- return val+ if isinstance(val, bool):+ return val+ raise TypeError(+ f"Expected a boolean for key '{k}', " f"but got {type(val).__name__}"+ )def _parse_json_output(text: str) -> Dict[str, Any]:@@ -208,7 +223,8 @@ def _reload_image(image: Image.Image):return Image.open(buffer)-def domrectangle_from_dict(rect: Dict[str, Any]) -> DOMRectangle:+def dom_rectangle_from_dict(rect: Dict[str, Any]) -> DOMRectangle:+ r"""Create a DOMRectangle object from a dictionary."""return DOMRectangle(x=_get_number(rect, "x"),y=_get_number(rect, "y"),@@ -221,10 +237,11 @@ def domrectangle_from_dict(rect: Dict[str, Any]) -> DOMRectangle:)-def interactiveregion_from_dict(region: Dict[str, Any]) -> InteractiveRegion:+def interactive_region_from_dict(region: Dict[str, Any]) -> InteractiveRegion:+ r"""Create an :class:`InteractiveRegion` object from a dictionary."""typed_rects: List[DOMRectangle] = []for rect in region["rects"]:- typed_rects.append(domrectangle_from_dict(rect))+ typed_rects.append(dom_rectangle_from_dict(rect))return InteractiveRegion(tag_name=_get_str(region, "tag_name"),@@ -235,7 +252,8 @@ def interactiveregion_from_dict(region: Dict[str, Any]) -> InteractiveRegion:)-def visualviewport_from_dict(viewport: Dict[str, Any]) -> VisualViewport:+def visual_viewport_from_dict(viewport: Dict[str, Any]) -> VisualViewport:+ r"""Create a :class:`VisualViewport` object from a dictionary."""return VisualViewport(height=_get_number(viewport, "height"),width=_get_number(viewport, "width"),@@ -252,7 +270,7 @@ def visualviewport_from_dict(viewport: Dict[str, Any]) -> VisualViewport:def add_set_of_mark(- screenshot: bytes | Image.Image | io.BufferedIOBase,+ screenshot: Union[bytes, Image.Image, io.BufferedIOBase],ROIs: Dict[str, InteractiveRegion],) -> Tuple[Image.Image, List[str], List[str], List[str]]:if isinstance(screenshot, Image.Image):@@ -272,6 +290,18 @@ def add_set_of_mark(def _add_set_of_mark(screenshot: Image.Image, ROIs: Dict[str, InteractiveRegion]) -> Tuple[Image.Image, List[str], List[str], List[str]]:+ r"""Add a set of marks to the screenshot.++ Args:+ screenshot (Image.Image): The screenshot to add marks to.+ ROIs (Dict[str, InteractiveRegion]): The regions to add marks to.++ Returns:+ Tuple[Image.Image, List[str], List[str], List[str]]: A tuple+ containing the screenshot with marked ROIs, ROIs fully within the+ images, ROIs located above the visible area, and ROIs located below+ the visible area.+ """visible_rects: List[str] = list()rects_above: List[str] = list() # Scroll up to seerects_below: List[str] = list() # Scroll down to see@@ -284,22 +314,22 @@ def _add_set_of_mark(for r in ROIs:for rect in ROIs[r]["rects"]:# Empty rectangles- if not rect:- continue- if rect["width"] * rect["height"] == 0:+ if not rect or rect["width"] == 0 or rect["height"] == 0:continue- mid = (- (rect["right"] + rect["left"]) / 2.0,- (rect["top"] + rect["bottom"]) / 2.0,- )+ # TODO: add scroll left and right?+ horizontal_center = (rect["right"] + rect["left"]) / 2.0+ vertical_center = (rect["top"] + rect["bottom"]) / 2.0+ is_within_horizon = 0 <= horizontal_center < base.size[0]+ is_above_viewport = vertical_center < 0+ is_below_viewport = vertical_center >= base.size[1]- if 0 <= mid[0] and mid[0] < base.size[0]:- if mid[1] < 0:+ if is_within_horizon:+ if is_above_viewport:rects_above.append(r)- elif mid[1] >= base.size[1]:+ elif is_below_viewport:rects_below.append(r)- else:+ else: # Fully visiblevisible_rects.append(r)_draw_roi(draw, int(r), fnt, rect)@@ -314,9 +344,16 @@ def _draw_roi(font: ImageFont.FreeTypeFont | ImageFont.ImageFont,rect: DOMRectangle,) -> None:- color = _color(idx)- luminance = color[0] * 0.3 + color[1] * 0.59 + color[2] * 0.11- text_color = (0, 0, 0, 255) if luminance > 90 else (255, 255, 255, 255)+ r"""Draw a ROI on the image.++ Args:+ draw (ImageDraw.ImageDraw): The draw object.+ idx (int): The index of the ROI.+ font (ImageFont.FreeTypeFont | ImageFont.ImageFont): The font.+ rect (DOMRectangle): The DOM rectangle.+ """+ color = _get_random_color(idx)+ text_color = _get_text_color(color)roi = ((rect["left"], rect["top"]), (rect["right"], rect["bottom"]))@@ -351,9 +388,36 @@ def _draw_roi()-def _color(identifier: int) -> Tuple[int, int, int, int]:+def _get_text_color(+ bg_color: Tuple[int, int, int, int],+) -> Tuple[int, int, int, int]:+ r"""Determine the ideal text color (black or white) for contrast.++ Args:+ bg_color: The background color (R, G, B, A).++ Returns:+ A tuple representing black or white color for text.+ """+ luminance = bg_color[0] * 0.3 + bg_color[1] * 0.59 + bg_color[2] * 0.11+ return (0, 0, 0, 255) if luminance > 120 else (255, 255, 255, 255)+++def _get_random_color(identifier: int) -> Tuple[int, int, int, int]:+ r"""Generate a consistent random RGBA color based on the identifier.++ Args:+ identifier: The ID used as a seed to ensure color consistency.++ Returns:+ A tuple representing (R, G, B, A) values.+ """rnd = random.Random(int(identifier))- color = [rnd.randint(0, 255), rnd.randint(125, 255), rnd.randint(0, 50)]+ r = rnd.randint(0, 255)+ g = rnd.randint(125, 255)+ b = rnd.randint(0, 50)+ color = [r, g, b]+ # TODO: check why shuffle is needed?rnd.shuffle(color)color.append(255)return cast(Tuple[int, int, int, int], tuple(color))@@ -379,13 +443,11 @@ class BaseBrowser:self.playwright = sync_playwright().start()self.page_history: list = [] # stores the history of visited pages- # set the cache directory- self.cache_dir = "tmp/"+ # Set the cache directory+ self.cache_dir = "tmp/" if cache_dir is None else cache_diros.makedirs(self.cache_dir, exist_ok=True)- if cache_dir is not None:- self.cache_dir = cache_dir- # load the page script+ # Load the page scriptabs_dir_path = os.path.dirname(os.path.abspath(__file__))page_script_path = os.path.join(abs_dir_path, "page_script.js")@@ -398,34 +460,35 @@ class BaseBrowser:f"Page script file not found at path: {page_script_path}")- def init(self):+ def init(self) -> None:r"""Initialize the browser."""- self.browser = self.playwright.chromium.launch(- headless=self.headless- ) # Launch the browser, if headless is False, the browser will display- self.context = self.browser.new_context(- accept_downloads=True- ) # create a new context- self.page = self.context.new_page() # create a new page-- def clean_cache(self):- r"""delete the cache directory and its contents."""+ # Launch the browser, if headless is False, the browser will display+ self.browser = self.playwright.chromium.launch(headless=self.headless)+ # Create a new context+ self.context = self.browser.new_context(accept_downloads=True)+ # Create a new page+ self.page = self.context.new_page()++ def clean_cache(self) -> None:+ r"""Delete the cache directory and its contents."""if os.path.exists(self.cache_dir):shutil.rmtree(self.cache_dir)- def _wait_for_load(self, timeout: int = 20):+ def _wait_for_load(self, timeout: int = 20) -> None:r"""Wait for a certain amount of time for the page to load."""timeout_ms = timeout * 1000self.page.wait_for_load_state("load", timeout=timeout_ms)++ # TODO: check if this is neededtime.sleep(2)- def click_blank_area(self):+ def click_blank_area(self) -> None:r"""Click a blank area of the page to unfocus the current element."""self.page.mouse.click(0, 0)self._wait_for_load()- def visit_page(self, url: str):+ def visit_page(self, url: str) -> None:r"""Visit a page with the given URL."""self.page.goto(url)@@ -433,8 +496,8 @@ class BaseBrowser:self.page_url = urldef ask_question_about_video(self, question: str) -> str:- r"""Ask a question about the video on the current page. It is suitable- to process youtube video.+ r"""Ask a question about the video on the current page,+ such as YouTube video.Args:question (str): The question to ask.@@ -459,8 +522,9 @@ class BaseBrowser:directory.Returns:- Tuple[Image.Image, str]: A tuple containing the screenshot image- and the path to the image file.+ Tuple[Image.Image, str]: A tuple containing the screenshot+ image and the path to the image file if saved, otherwise+ :obj:`None`."""image_data = self.page.screenshot(timeout=60000)@@ -468,12 +532,13 @@ class BaseBrowser:file_path = Noneif save_image:- # get url name to form a file name+ # Get url name to form a file name+ # TODO: Use a safer way for the url nameurl_name = self.page_url.split("/")[-1]for char in ['\\', '/', ':', '*', '?', '"', '<', '>', '|', '.']:url_name = url_name.replace(char, "_")- # get formatted time: mmddhhmmss+ # Get formatted time: mmddhhmmsstimestamp = datetime.datetime.now().strftime("%m%d%H%M%S")file_path = os.path.join(self.cache_dir, f"{url_name}_{timestamp}.png"@@ -492,23 +557,18 @@ class BaseBrowser:Args:scroll_ratio (float): The ratio of viewport height to scroll each- step (default: 0.7).+ step (default: 0.8).Returns:List[str]: A list of paths to the screenshot files."""screenshots = []scroll_height = self.page.evaluate("document.body.scrollHeight")+ assert self.page.viewport_size is not Noneviewport_height = self.page.viewport_size["height"]current_scroll = 0screenshot_index = 1- url_name = self.page.url.split("/")[-1].replace(".", "_")- timestamp = datetime.datetime.now().strftime("%m%d%H%M%S")- base_file_path = os.path.join(- self.cache_dir, f"{url_name}_{timestamp}"- )-max_height = scroll_height - viewport_heightscroll_step = int(viewport_height * scroll_ratio)@@ -520,14 +580,15 @@ class BaseBrowser:f"{max_height}, step: {scroll_step}")- file_path = f"{base_file_path}_{screenshot_index}.png"_, file_path = self.get_screenshot(save_image=True)screenshots.append(file_path)self.page.evaluate(f"window.scrollBy(0, {scroll_step})")+ # Allow time for content to loadtime.sleep(0.5)current_scroll = self.page.evaluate("window.scrollY")+ # Break if there is no significant scrollif abs(current_scroll - last_height) < viewport_height * 0.1:break@@ -547,12 +608,16 @@ class BaseBrowser:except Exception as e:logger.warning(f"Error evaluating page script: {e}")- return visualviewport_from_dict(+ return visual_viewport_from_dict(self.page.evaluate("MultimodalWebSurfer.getVisualViewport();"))- def get_interactive_elements(self) -> List[Dict[str, Any]]:- # codes from magentic-one+ def get_interactive_elements(self) -> Dict[str, InteractiveRegion]:+ r"""Get the interactive elements of the current page.++ Returns:+ Dict[str, InteractiveRegion]: A dictionary of interactive elements.+ """try:self.page.evaluate(self.page_script)except Exception as e:@@ -565,12 +630,13 @@ class BaseBrowser:typed_results: Dict[str, InteractiveRegion] = {}for k in result:- typed_results[k] = interactiveregion_from_dict(result[k])+ typed_results[k] = interactive_region_from_dict(result[k])return typed_results # type: ignore[return-value]def get_som_screenshot(- self, save_image: bool = False+ self,+ save_image: bool = False,) -> Tuple[Image.Image, Union[str, None]]:r"""Get a screenshot of the current viewport with interactive elementsmarked.@@ -608,15 +674,19 @@ class BaseBrowser:return comp, file_pathdef scroll_up(self) -> None:+ r"""Scroll up the page."""self.page.keyboard.press("PageUp")def scroll_down(self) -> None:+ r"""Scroll down the page."""self.page.keyboard.press("PageDown")def get_url(self) -> str:+ r"""Get the URL of the current page."""return self.page.url- def click_id(self, identifier: Union[str, int]):+ def click_id(self, identifier: Union[str, int]) -> None:+ r"""Click an element with the given identifier."""if isinstance(identifier, int):identifier = str(identifier)target = self.page.locator(f"[__elementId='{identifier}']")@@ -649,7 +719,7 @@ class BaseBrowser:self._wait_for_load()- def extract_url_content(self):+ def extract_url_content(self) -> str:r"""Extract the content of the current page."""content = self.page.content()return content@@ -1026,9 +1096,7 @@ out the information you need. Sometimes they are extremely useful."""# get current state- som_screenshot, som_screenshot_path = self.browser.get_som_screenshot(- save_image=True- )+ som_screenshot, _ = self.browser.get_som_screenshot(save_image=True)img = _reload_image(som_screenshot)message = BaseMessage.make_user_message(role_name='user', content=observe_prompt, image_list=[img]