Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

sanitize everything now

mertunsalla64c82ef2c63
diff --git a/browser_use/agent/prompts.py b/browser_use/agent/prompts.py
index 3532d2e48..b781a7766 100644
--- a/browser_use/agent/prompts.py
+++ b/browser_use/agent/prompts.py
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Literal, Optional
from browser_use.dom.views import NodeType, SimplifiedNode
from browser_use.llm.messages import ContentPartImageParam, ContentPartTextParam, ImageURL, SystemMessage, UserMessage
from browser_use.observability import observe_debug
-from browser_use.utils import is_new_tab_page
+from browser_use.utils import is_new_tab_page, sanitize_surrogates
if TYPE_CHECKING:
from browser_use.agent.views import AgentStepInfo
@@ -352,15 +352,6 @@ Available tabs:
logging.getLogger(__name__).warning(f'Failed to resize screenshot: {e}, using original')
return screenshot_b64
- @staticmethod
- def _sanitize_surrogates(text: str) -> str:
- """Remove surrogate characters that can't be encoded in UTF-8.
-
- Surrogate pairs (U+D800 to U+DFFF) are invalid in UTF-8 when unpaired.
- These often appear in DOM content from mathematical symbols or emojis.
- """
- return text.encode('utf-8', errors='ignore').decode('utf-8')
-
@observe_debug(ignore_input=True, ignore_output=True, name='get_user_message')
def get_user_message(self, use_vision: bool = True) -> UserMessage:
"""Get complete state as a single cached message"""
@@ -392,7 +383,7 @@ Available tabs:
state_description += '</page_specific_actions>\n'
# Sanitize surrogates from all text content
- state_description = self._sanitize_surrogates(state_description)
+ state_description = sanitize_surrogates(state_description)
# Check if we have images to include (from read_file action)
has_images = bool(self.read_state_images)
diff --git a/browser_use/tools/service.py b/browser_use/tools/service.py
index 6609141f8..5712b2bd3 100644
--- a/browser_use/tools/service.py
+++ b/browser_use/tools/service.py
@@ -51,7 +51,7 @@ from browser_use.tools.views import (
SwitchTabAction,
UploadFileAction,
)
-from browser_use.utils import create_task_with_error_handling, time_execution_sync
+from browser_use.utils import create_task_with_error_handling, sanitize_surrogates, time_execution_sync
logger = logging.getLogger(__name__)
@@ -742,6 +742,10 @@ You will be given a query and the markdown of a webpage that has been filtered t
</output>
""".strip()
+ # Sanitize surrogates from content to prevent UTF-8 encoding errors
+ content = sanitize_surrogates(content)
+ query = sanitize_surrogates(query)
+
prompt = f'<query>\n{query}\n</query>\n\n<content_stats>\n{stats_summary}\n</content_stats>\n\n<webpage_content>\n{content}\n</webpage_content>'
try:
diff --git a/browser_use/utils.py b/browser_use/utils.py
index 6fc2a5557..6d836eb59 100644
--- a/browser_use/utils.py
+++ b/browser_use/utils.py
@@ -733,3 +733,18 @@ def create_task_with_error_handling(
task.add_done_callback(_handle_task_exception)
return task
+
+
+def sanitize_surrogates(text: str) -> str:
+ """Remove surrogate characters that can't be encoded in UTF-8.
+
+ Surrogate pairs (U+D800 to U+DFFF) are invalid in UTF-8 when unpaired.
+ These often appear in DOM content from mathematical symbols or emojis.
+
+ Args:
+ text: The text to sanitize
+
+ Returns:
+ Text with surrogate characters removed
+ """
+ return text.encode('utf-8', errors='ignore').decode('utf-8')