Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

[Browser Toolkit] Better filename from url and browser toolkit cleaning (#2232)

Zecheng Zhang6f25add5410b
diff --git a/camel/toolkits/browser_toolkit.py b/camel/toolkits/browser_toolkit.py
index 89f95e307..1aa7d034c 100644
--- a/camel/toolkits/browser_toolkit.py
+++ b/camel/toolkits/browser_toolkit.py
@@ -47,7 +47,11 @@ from camel.toolkits.base import BaseToolkit
from camel.toolkits.function_tool import FunctionTool
from camel.toolkits.video_analysis_toolkit import VideoAnalysisToolkit
from camel.types import ModelPlatformType, ModelType
-from camel.utils import dependencies_required, retry_on_error
+from camel.utils import (
+ dependencies_required,
+ retry_on_error,
+ sanitize_filename,
+)
logger = get_logger(__name__)
@@ -219,7 +223,7 @@ def _parse_json_output(text: str) -> Dict[str, Any]:
return {}
-def _reload_image(image: Image.Image):
+def _reload_image(image: Image.Image) -> Image.Image:
buffer = io.BytesIO()
image.save(buffer, format="PNG")
buffer.seek(0)
@@ -550,12 +554,9 @@ class BaseBrowser:
# Get url name to form a file name
# Use urlparser for a safer extraction the url name
parsed_url = urllib.parse.urlparse(self.page_url)
- url_name = os.path.basename(str(parsed_url.path)) or "index"
-
- for char in ['\\', '/', ':', '*', '?', '"', '<', '>', '|', '.']:
- url_name = url_name.replace(char, "_")
-
- # Get formatted time: mmddhhmmss
+ # Max length is set to 241 as there are 10 characters for the
+ # timestamp and 4 characters for the file extension:
+ url_name = sanitize_filename(str(parsed_url.path), max_length=241)
timestamp = datetime.datetime.now().strftime("%m%d%H%M%S")
file_path = os.path.join(
self.cache_dir, f"{url_name}_{timestamp}.png"
@@ -663,24 +664,25 @@ class BaseBrowser:
directory.
Returns:
- Tuple[Image.Image, str]: A tuple containing the screenshot image
- and the path to the image file.
+ Tuple[Image.Image, Union[str, None]]: A tuple containing the screenshot image
+ and an optional path to the image file if saved, otherwise
+ :obj:`None`.
"""
self._wait_for_load()
screenshot, _ = self.get_screenshot(save_image=False)
rects = self.get_interactive_elements()
- file_path = None
- comp, visible_rects, rects_above, rects_below = add_set_of_mark(
+ file_path: str | None = None
+ comp, _, _, _ = add_set_of_mark(
screenshot,
rects, # type: ignore[arg-type]
)
if save_image:
parsed_url = urllib.parse.urlparse(self.page_url)
- url_name = os.path.basename(str(parsed_url.path)) or "index"
- for char in ['\\', '/', ':', '*', '?', '"', '<', '>', '|', '.']:
- url_name = url_name.replace(char, "_")
+ # Max length is set to 241 as there are 10 characters for the
+ # timestamp and 4 characters for the file extension:
+ url_name = sanitize_filename(str(parsed_url.path), max_length=241)
timestamp = datetime.datetime.now().strftime("%m%d%H%M%S")
file_path = os.path.join(
self.cache_dir, f"{url_name}_{timestamp}.png"
@@ -1322,36 +1324,6 @@ Please find the final answer, or give valuable insights and founds (e.g. if prev
resp = self.web_agent.step(message)
return resp.msgs[0].content
- def _make_reflection(self, task_prompt: str) -> str:
- r"""Make a reflection about the current state and the task prompt."""
-
- reflection_prompt = f"""
-Now we are working on a complex task that requires multi-step browser interaction. The task is: <task>{task_prompt}</task>
-To achieve this goal, we have made a series of observations, reasonings, and actions. We have also made a reflection on previous states.
-
-Here are the global available browser functions we can use:
-{AVAILABLE_ACTIONS_PROMPT}
-
-Here are the latest {self.history_window} trajectory (at most) we have taken:
-<history>{self.history[-self.history_window:]}</history>
-
-The image provided is the current state of the browser, where we have marked interactive elements.
-Please carefully examine the requirements of the task, and the current state of the browser, and then make reflections on the previous steps, thinking about whether they are helpful or not, and why, offering detailed feedback and suggestions for the next steps.
-Your output should be in json format, including the following fields:
-- `reflection`: The reflection about the previous steps, thinking about whether they are helpful or not, and why, offering detailed feedback.
-- `suggestion`: The suggestion for the next steps, offering detailed suggestions, including the common solutions to the overall task based on the current state of the browser.
- """
- som_image, _ = self.browser.get_som_screenshot()
- img = _reload_image(som_image)
-
- message = BaseMessage.make_user_message(
- role_name='user', content=reflection_prompt, image_list=[img]
- )
-
- resp = self.web_agent.step(message)
-
- return resp.msgs[0].content
-
def _task_planning(self, task_prompt: str, start_url: str) -> str:
r"""Plan the task based on the given task prompt."""
@@ -1422,7 +1394,8 @@ Your output should be in json format, including the following fields:
def browse_url(
self, task_prompt: str, start_url: str, round_limit: int = 12
) -> str:
- r"""A powerful toolkit which can simulate the browser interaction to solve the task which needs multi-step actions.
+ r"""A powerful toolkit which can simulate the browser interaction to
+ solve the task which needs multi-step actions.
Args:
task_prompt (str): The task prompt to solve.
@@ -1480,7 +1453,7 @@ Your output should be in json format, including the following fields:
}
self.history.append(trajectory_info)
- # replan the task if necessary
+ # Replan the task if necessary
if_need_replan, replanned_schema = self._task_replanning(
task_prompt, detailed_plan
)
diff --git a/camel/utils/__init__.py b/camel/utils/__init__.py
index 64eff9fcd..3800a6854 100644
--- a/camel/utils/__init__.py
+++ b/camel/utils/__init__.py
@@ -41,6 +41,7 @@ from .commons import (
)
from .constants import Constants
from .deduplication import DeduplicationResult, deduplicate_internally
+from .filename import sanitize_filename
from .mcp import MCPServer
from .response_format import get_pydantic_model
from .token_counting import (
@@ -90,4 +91,5 @@ __all__ = [
"BatchProcessor",
"with_timeout",
"MCPServer",
+ "sanitize_filename",
]
diff --git a/camel/utils/filename.py b/camel/utils/filename.py
new file mode 100644
index 000000000..3fc671edd
--- /dev/null
+++ b/camel/utils/filename.py
@@ -0,0 +1,80 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import platform
+import re
+import unicodedata
+
+MAX_FILENAME_LENGTH = 255
+WINDOWS_RESERVED = {
+ 'CON',
+ 'PRN',
+ 'AUX',
+ 'NUL',
+ 'COM1',
+ 'COM2',
+ 'COM3',
+ 'COM4',
+ 'LPT1',
+ 'LPT2',
+ 'LPT3',
+}
+
+
+def sanitize_filename(
+ url_name: str,
+ default: str = "index",
+ max_length: int = MAX_FILENAME_LENGTH,
+) -> str:
+ r"""Sanitize a URL path into a safe filename that is safe for
+ most platforms.
+
+ Args:
+ url_name (str): The URL path to sanitize.
+ default (str): Default name if sanitization results in empty string.
+ (default: :obj:`"index"`)
+ max_length (int): Maximum length of the filename.
+ (default: :obj:`MAX_FILENAME_LENGTH`)
+
+ Returns:
+ str: A sanitized filename safe for most platforms.
+ """
+ if max_length < 1:
+ raise ValueError(
+ f"`max_length` must be greater than " f"0, got {max_length}"
+ )
+
+ if not url_name:
+ return default
+
+ # Normalize Unicode characters by removing characters
+ # such as accents and special characters:
+ # café☕.txt -> cafe.txt
+ url_name = unicodedata.normalize('NFKD', url_name)
+ url_name = url_name.encode('ASCII', 'ignore').decode('ASCII')
+
+ # Replace special characters such as:
+ # Separators: my/file:name*.txt -> my_file_name.txt etc.
+ url_name = re.sub(r'[\\/:*?"<>|.]', '_', url_name)
+ url_name = re.sub(r'_+', '_', url_name) # Collapse multiple underscores
+ url_name = url_name.strip('_') # Remove leading/trailing underscores
+
+ # Handle empty result if all characters are invalid:
+ if not url_name:
+ return default
+
+ # Handle Windows reserved names
+ if platform.system() == "Windows" and url_name.upper() in WINDOWS_RESERVED:
+ url_name = f"_{url_name}"
+
+ return url_name[:max_length]
diff --git a/camel/verifiers/physics_verifier.py b/camel/verifiers/physics_verifier.py
index b967079a4..70f5b440b 100644
--- a/camel/verifiers/physics_verifier.py
+++ b/camel/verifiers/physics_verifier.py
@@ -20,9 +20,9 @@ from typing import Any, Dict, List, Optional, Tuple, Union
from camel.extractors.base import BaseExtractor
from camel.logger import get_logger
-from camel.verifiers import PythonVerifier
from .models import VerificationOutcome, VerificationResult
+from .python_verifier import PythonVerifier
logger = get_logger(__name__)
diff --git a/test/utils/test_filename.py b/test/utils/test_filename.py
new file mode 100644
index 000000000..366fd6c74
--- /dev/null
+++ b/test/utils/test_filename.py
@@ -0,0 +1,118 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from urllib.parse import urlparse
+
+import pytest
+
+from camel.utils.filename import MAX_FILENAME_LENGTH, sanitize_filename
+
+
+def test_sanitize_filename_basic():
+ """Test basic filename sanitization."""
+ assert sanitize_filename("test.txt") == "test_txt"
+ path = urlparse("https://example.com/my_file_name").path
+ assert sanitize_filename(path) == "my_file_name"
+ path = urlparse("https://example.com/file:name").path
+ assert sanitize_filename(path) == "file_name"
+ path = urlparse("https://example.com/file<name").path
+ assert sanitize_filename(path) == "file_name"
+ path = urlparse("https://example.com/file>name").path
+ assert sanitize_filename(path) == "file_name"
+ path = urlparse("https://example.com/file|name").path
+ assert sanitize_filename(path) == "file_name"
+ path = urlparse("https://example.com/file\"name").path
+
+
+def test_sanitize_filename_unicode_normalization():
+ """Test Unicode normalization (NFKD) and ASCII conversion."""
+ # Test with accented characters
+ path = urlparse("https://example.com/café.txt").path
+ assert sanitize_filename(path) == "cafe_txt"
+ path = urlparse("https://example.com/naïve.txt").path
+ assert sanitize_filename(path) == "naive_txt"
+ path = urlparse("https://example.com/résumé.pdf").path
+ assert sanitize_filename(path) == "resume_pdf"
+
+ # Test with special Unicode characters
+ path = urlparse("https://example.com/file☕.txt").path
+ assert sanitize_filename(path) == "file_txt"
+ path = urlparse("https://example.com/file🎉.txt").path
+ assert sanitize_filename(path) == "file_txt"
+
+
+def test_sanitize_filename_special_characters():
+ """Test handling of special characters."""
+ path = urlparse("https://example.com/file/name.txt").path
+ assert sanitize_filename(path) == "file_name_txt"
+ path = urlparse("https://example.com/file:name.txt").path
+ assert sanitize_filename(path) == "file_name_txt"
+ path = urlparse("https://example.com/file\\name.txt").path
+ assert sanitize_filename(path) == "file_name_txt"
+ path = urlparse("https://example.com/file*name.txt").path
+ assert sanitize_filename(path) == "file_name_txt"
+ path = urlparse("https://example.com/file<name.txt").path
+ assert sanitize_filename(path) == "file_name_txt"
+ path = urlparse("https://example.com/file>name.txt").path
+ assert sanitize_filename(path) == "file_name_txt"
+ path = urlparse("https://example.com/file|name.txt").path
+ assert sanitize_filename(path) == "file_name_txt"
+ path = urlparse("https://example.com/file\"name.txt").path
+ assert sanitize_filename(path) == "file_name_txt"
+
+
+def test_sanitize_filename_multiple_special_chars():
+ """Test handling of multiple consecutive special characters."""
+ path = urlparse("https://example.com/file//name.txt").path
+ assert sanitize_filename(path) == "file_name_txt"
+ path = urlparse("https://example.com/file\\\\name.txt").path
+ assert sanitize_filename(path) == "file_name_txt"
+ path = urlparse("https://example.com/file::name.txt").path
+ assert sanitize_filename(path) == "file_name_txt"
+ path = urlparse("https://example.com/file**name.txt").path
+ assert sanitize_filename(path) == "file_name_txt"
+
+
+def test_sanitize_filename_leading_trailing():
+ """Test handling of leading and trailing special characters."""
+ path = urlparse("https://example.com/_file.txt").path
+ assert sanitize_filename(path) == "file_txt"
+ path = urlparse("https://example.com/file.txt_").path
+ assert sanitize_filename(path) == "file_txt"
+ path = urlparse("https://example.com/_file.txt_").path
+ assert sanitize_filename(path) == "file_txt"
+ path = urlparse("https://example.com/__file.txt__").path
+ assert sanitize_filename(path) == "file_txt"
+
+
+def test_sanitize_filename_empty():
+ """Test handling of empty input."""
+ assert sanitize_filename("") == "index"
+ assert sanitize_filename("", default="default") == "default"
+
+
+def test_sanitize_filename_max_length():
+ """Test maximum length handling."""
+ long_name = "a" * (MAX_FILENAME_LENGTH + 10)
+ assert len(sanitize_filename(long_name)) == MAX_FILENAME_LENGTH
+
+ # Test with custom max_length
+ assert len(sanitize_filename("test.txt", max_length=5)) == 5
+
+
+def test_sanitize_filename_invalid_max_length():
+ """Test invalid max_length parameter."""
+ with pytest.raises(ValueError):
+ sanitize_filename("test.txt", max_length=0)
+ with pytest.raises(ValueError):
+ sanitize_filename("test.txt", max_length=-1)