Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
sanitize everything now
mertunsalla64c82ef2c63
diff --git a/browser_use/agent/prompts.py b/browser_use/agent/prompts.pyindex 3532d2e48..b781a7766 100644--- a/browser_use/agent/prompts.py+++ b/browser_use/agent/prompts.py@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Literal, Optionalfrom browser_use.dom.views import NodeType, SimplifiedNodefrom browser_use.llm.messages import ContentPartImageParam, ContentPartTextParam, ImageURL, SystemMessage, UserMessagefrom browser_use.observability import observe_debug-from browser_use.utils import is_new_tab_page+from browser_use.utils import is_new_tab_page, sanitize_surrogatesif TYPE_CHECKING:from browser_use.agent.views import AgentStepInfo@@ -352,15 +352,6 @@ Available tabs:logging.getLogger(__name__).warning(f'Failed to resize screenshot: {e}, using original')return screenshot_b64- @staticmethod- def _sanitize_surrogates(text: str) -> str:- """Remove surrogate characters that can't be encoded in UTF-8.-- Surrogate pairs (U+D800 to U+DFFF) are invalid in UTF-8 when unpaired.- These often appear in DOM content from mathematical symbols or emojis.- """- return text.encode('utf-8', errors='ignore').decode('utf-8')-@observe_debug(ignore_input=True, ignore_output=True, name='get_user_message')def get_user_message(self, use_vision: bool = True) -> UserMessage:"""Get complete state as a single cached message"""@@ -392,7 +383,7 @@ Available tabs:state_description += '</page_specific_actions>\n'# Sanitize surrogates from all text content- state_description = self._sanitize_surrogates(state_description)+ state_description = sanitize_surrogates(state_description)# Check if we have images to include (from read_file action)has_images = bool(self.read_state_images)diff --git a/browser_use/tools/service.py b/browser_use/tools/service.pyindex 6609141f8..5712b2bd3 100644--- a/browser_use/tools/service.py+++ b/browser_use/tools/service.py@@ -51,7 +51,7 @@ from browser_use.tools.views import (SwitchTabAction,UploadFileAction,)-from browser_use.utils import create_task_with_error_handling, time_execution_sync+from browser_use.utils import create_task_with_error_handling, sanitize_surrogates, time_execution_synclogger = logging.getLogger(__name__)@@ -742,6 +742,10 @@ You will be given a query and the markdown of a webpage that has been filtered t</output>""".strip()+ # Sanitize surrogates from content to prevent UTF-8 encoding errors+ content = sanitize_surrogates(content)+ query = sanitize_surrogates(query)+prompt = f'<query>\n{query}\n</query>\n\n<content_stats>\n{stats_summary}\n</content_stats>\n\n<webpage_content>\n{content}\n</webpage_content>'try:diff --git a/browser_use/utils.py b/browser_use/utils.pyindex 6fc2a5557..6d836eb59 100644--- a/browser_use/utils.py+++ b/browser_use/utils.py@@ -733,3 +733,18 @@ def create_task_with_error_handling(task.add_done_callback(_handle_task_exception)return task+++def sanitize_surrogates(text: str) -> str:+ """Remove surrogate characters that can't be encoded in UTF-8.++ Surrogate pairs (U+D800 to U+DFFF) are invalid in UTF-8 when unpaired.+ These often appear in DOM content from mathematical symbols or emojis.++ Args:+ text: The text to sanitize++ Returns:+ Text with surrogate characters removed+ """+ return text.encode('utf-8', errors='ignore').decode('utf-8')