Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
shorten super long URLs
Gregor Žuničb5aebf0b034b
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.pyindex 22d8873e4..03662c753 100644--- a/browser_use/agent/service.py+++ b/browser_use/agent/service.py@@ -30,7 +30,7 @@ from browser_use.tokens.service import TokenCostload_dotenv()from bubus import EventBus-from pydantic import ValidationError+from pydantic import BaseModel, ValidationErrorfrom uuid_extensions import uuid7strfrom browser_use import Browser, BrowserProfile, BrowserSession@@ -66,6 +66,7 @@ from browser_use.telemetry.views import AgentTelemetryEventfrom browser_use.tools.registry.views import ActionModelfrom browser_use.tools.service import Toolsfrom browser_use.utils import (+ URL_PATTERN,_log_pretty_path,get_browser_use_version,get_git_info,@@ -179,6 +180,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):directly_open_url: bool = True,include_recent_events: bool = False,sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,+ _url_shortening_limit: int = 25,**kwargs,):if llm is None:@@ -227,6 +229,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):self.llm = llmself.directly_open_url = directly_open_urlself.include_recent_events = include_recent_events+ self._url_shortening_limit = _url_shortening_limitif tools is not None:self.tools = toolselif controller is not None:@@ -970,15 +973,151 @@ class Agent(Generic[Context, AgentStructuredOutput]):text = re.sub(STRAY_CLOSE_TAG, '', text)return text.strip()+ # region - URL replacement+ def _replace_urls_in_text(self, text: str) -> tuple[str, dict[str, str]]:+ """Replace URLs in a text string"""++ replaced_urls: dict[str, str] = {}++ def replace_url(match: re.Match) -> str:+ """Url can only have 1 query and 1 fragment"""+ import hashlib++ original_url = match.group(0)++ # Find where the query/fragment starts+ query_start = original_url.find('?')+ fragment_start = original_url.find('#')++ # Find the earliest position of query or fragment+ after_path_start = len(original_url) # Default: no query/fragment+ if query_start != -1:+ after_path_start = min(after_path_start, query_start)+ if fragment_start != -1:+ after_path_start = min(after_path_start, fragment_start)++ # Split URL into base (up to path) and after_path (query + fragment)+ base_url = original_url[:after_path_start]+ after_path = original_url[after_path_start:]++ # If after_path is within the limit, don't shorten+ if len(after_path) <= self._url_shortening_limit:+ return original_url++ # If after_path is too long, truncate and add hash+ if after_path:+ truncated_after_path = after_path[: self._url_shortening_limit]+ # Create a short hash of the full after_path content+ hash_obj = hashlib.md5(after_path.encode('utf-8'))+ short_hash = hash_obj.hexdigest()[:7]+ # Create shortened URL+ shortened = f'{base_url}{truncated_after_path}...{short_hash}'+ # Only use shortened URL if it's actually shorter than the original+ if len(shortened) < len(original_url):+ replaced_urls[shortened] = original_url+ return shortened++ return original_url++ return URL_PATTERN.sub(replace_url, text), replaced_urls++ def _process_messsages_and_replace_long_urls_shorter_ones(self, input_messages: list[BaseMessage]) -> dict[str, str]:+ """Replace long URLs with shorter ones+ ? @dev edits input_messages in place++ returns:+ tuple[filtered_input_messages, urls we replaced {shorter_url: original_url}]+ """+ from browser_use.llm.messages import AssistantMessage, UserMessage++ urls_replaced: dict[str, str] = {}++ # Process each message, in place+ for message in input_messages:+ # no need to process SystemMessage, we have control over that anyway+ if isinstance(message, (UserMessage, AssistantMessage)):+ if isinstance(message.content, str):+ # Simple string content+ message.content, replaced_urls = self._replace_urls_in_text(message.content)+ urls_replaced.update(replaced_urls)++ elif isinstance(message.content, list):+ # List of content parts+ for part in message.content:+ if isinstance(part, ContentPartTextParam):+ part.text, replaced_urls = self._replace_urls_in_text(part.text)+ urls_replaced.update(replaced_urls)++ return urls_replaced++ @staticmethod+ def _recursive_process_all_strings_inside_pydantic_model(model: BaseModel, url_replacements: dict[str, str]) -> None:+ """Recursively process all strings inside a Pydantic model, replacing shortened URLs with originals in place."""+ for field_name, field_value in model.__dict__.items():+ if isinstance(field_value, str):+ # Replace shortened URLs with original URLs in string+ processed_string = Agent._replace_shortened_urls_in_string(field_value, url_replacements)+ setattr(model, field_name, processed_string)+ elif isinstance(field_value, BaseModel):+ # Recursively process nested Pydantic models+ Agent._recursive_process_all_strings_inside_pydantic_model(field_value, url_replacements)+ elif isinstance(field_value, dict):+ # Process dictionary values in place+ Agent._recursive_process_dict(field_value, url_replacements)+ elif isinstance(field_value, (list, tuple)):+ Agent._recursive_process_list_or_tuple(field_value, url_replacements)++ @staticmethod+ def _recursive_process_dict(dictionary: dict, url_replacements: dict[str, str]) -> None:+ """Helper method to process dictionaries."""+ for k, v in dictionary.items():+ if isinstance(v, str):+ dictionary[k] = Agent._replace_shortened_urls_in_string(v, url_replacements)+ elif isinstance(v, BaseModel):+ Agent._recursive_process_all_strings_inside_pydantic_model(v, url_replacements)+ elif isinstance(v, dict):+ Agent._recursive_process_dict(v, url_replacements)+ elif isinstance(v, (list, tuple)):+ Agent._recursive_process_list_or_tuple(v, url_replacements)++ @staticmethod+ def _recursive_process_list_or_tuple(container, url_replacements: dict[str, str]) -> None:+ """Helper method to process lists and tuples."""+ for i, item in enumerate(container):+ if isinstance(item, str):+ container[i] = Agent._replace_shortened_urls_in_string(item, url_replacements)+ elif isinstance(item, BaseModel):+ Agent._recursive_process_all_strings_inside_pydantic_model(item, url_replacements)+ elif isinstance(item, dict):+ Agent._recursive_process_dict(item, url_replacements)+ elif isinstance(item, (list, tuple)):+ Agent._recursive_process_list_or_tuple(item, url_replacements)++ @staticmethod+ def _replace_shortened_urls_in_string(text: str, url_replacements: dict[str, str]) -> str:+ """Replace all shortened URLs in a string with their original URLs."""+ result = text+ for shortened_url, original_url in url_replacements.items():+ result = result.replace(shortened_url, original_url)+ return result++ # endregion - URL replacement+@time_execution_async('--get_next_action')@observe_debug(ignore_input=True, ignore_output=True, name='get_model_output')async def get_model_output(self, input_messages: list[BaseMessage]) -> AgentOutput:"""Get next action from LLM based on current state"""+ urls_replaced = self._process_messsages_and_replace_long_urls_shorter_ones(input_messages)+try:response = await self.llm.ainvoke(input_messages, output_format=self.AgentOutput)parsed = response.completion+ # Replace any shortened URLs in the LLM response back to original URLs+ if urls_replaced:+ self._recursive_process_all_strings_inside_pydantic_model(parsed, urls_replaced)+# cut the number of actions to max_actions_per_step if neededif len(parsed.action) > self.settings.max_actions_per_step:parsed.action = parsed.action[: self.settings.max_actions_per_step]diff --git a/browser_use/utils.py b/browser_use/utils.pyindex 3fd4c4ba7..35f9a9db0 100644--- a/browser_use/utils.py+++ b/browser_use/utils.py@@ -2,6 +2,7 @@ import asyncioimport loggingimport osimport platform+import reimport signalimport timefrom collections.abc import Callable, Coroutine@@ -16,6 +17,9 @@ from dotenv import load_dotenvload_dotenv()+# Pre-compiled regex for URL detection - used in URL shortening+URL_PATTERN = re.compile(r'https?://[^\s<>"\']+|www\.[^\s<>"\']+|[^\s<>"\']+\.[a-z]{2,}(?:/[^\s<>"\']*)?', re.IGNORECASE)+logger = logging.getLogger(__name__)diff --git a/tests/ci/test_url_shortening.py b/tests/ci/test_url_shortening.pynew file mode 100644index 000000000..6ea45b15d--- /dev/null+++ b/tests/ci/test_url_shortening.py@@ -0,0 +1,161 @@+"""+Simplified tests for URL shortening functionality in Agent service.++Three focused tests:+1. Input message processing with URL shortening+2. Output processing with custom actions and URL restoration+3. End-to-end pipeline test+"""++import json++import pytest++from browser_use.agent.service import Agent+from browser_use.agent.views import AgentOutput+from browser_use.llm.messages import AssistantMessage, BaseMessage, UserMessage++# Super long URL to reuse across tests - much longer than the 25 character limit+# Includes both query params (?...) and fragment params (#...)+SUPER_LONG_URL = 'https://documentation.example-company.com/api/v3/enterprise/user-management/endpoints/administration/create-new-user-account-with-permissions/advanced-settings?format=detailed-json&version=3.2.1×tamp=1699123456789&session_id=abc123def456ghi789&authentication_token=very_long_authentication_token_string_here&include_metadata=true&expand_relationships=user_groups,permissions,roles&sort_by=created_at&order=desc&page_size=100&include_deprecated_fields=false&api_key=super_long_api_key_that_exceeds_normal_limits#section=user_management&tab=advanced&view=detailed&scroll_to=permissions_table&highlight=admin_settings&filter=active_users&expand_all=true&debug_mode=enabled'+++def agent():+ """Create an agent instance for testing URL shortening functionality."""+ from tests.ci.conftest import create_mock_llm++ return Agent(task='Test URL shortening', llm=create_mock_llm(), url_shortening_limit=25)+++class TestUrlShorteningInputProcessing:+ """Test URL shortening for input messages."""++ def test_process_input_messages_with_url_shortening(self, agent: Agent):+ """Test that long URLs in input messages are shortened and mappings stored."""+ original_content = f'Please visit {SUPER_LONG_URL} and extract information'++ messages: list[BaseMessage] = [UserMessage(content=original_content)]++ # Process messages (modifies messages in-place and returns URL mappings)+ url_mappings = agent._process_messsages_and_replace_long_urls_shorter_ones(messages)++ # Verify URL was shortened in the message (modified in-place)+ processed_content = messages[0].content or ''+ assert processed_content != original_content+ assert 'https://documentation.example-company.com' in processed_content+ assert len(processed_content) < len(original_content)++ # Verify URL mapping was returned+ assert len(url_mappings) == 1+ shortened_url = next(iter(url_mappings.keys()))+ assert url_mappings[shortened_url] == SUPER_LONG_URL++ def test_process_user_and_assistant_messages_with_url_shortening(self, agent: Agent):+ """Test URL shortening in both UserMessage and AssistantMessage."""+ user_content = f'I need to access {SUPER_LONG_URL} for the API documentation'+ assistant_content = f'I will help you navigate to {SUPER_LONG_URL} to retrieve the documentation'++ messages: list[BaseMessage] = [UserMessage(content=user_content), AssistantMessage(content=assistant_content)]++ # Process messages (modifies messages in-place and returns URL mappings)+ url_mappings = agent._process_messsages_and_replace_long_urls_shorter_ones(messages)++ # Verify URL was shortened in both messages+ user_processed_content = messages[0].content or ''+ assistant_processed_content = messages[1].content or ''++ assert user_processed_content != user_content+ assert assistant_processed_content != assistant_content+ assert 'https://documentation.example-company.com' in user_processed_content+ assert 'https://documentation.example-company.com' in assistant_processed_content+ assert len(user_processed_content) < len(user_content)+ assert len(assistant_processed_content) < len(assistant_content)++ # Verify URL mapping was returned (should be same shortened URL for both occurrences)+ assert len(url_mappings) == 1+ shortened_url = next(iter(url_mappings.keys()))+ assert url_mappings[shortened_url] == SUPER_LONG_URL+++class TestUrlShorteningOutputProcessing:+ """Test URL restoration for output processing with custom actions."""++ def test_process_output_with_custom_actions_and_url_restoration(self, agent: Agent):+ """Test that shortened URLs in AgentOutput with custom actions are restored."""+ # Set up URL mapping (simulating previous shortening)+ shortened_url: str = agent._replace_urls_in_text(SUPER_LONG_URL)[0]+ url_mappings = {shortened_url: SUPER_LONG_URL}++ # Create AgentOutput with shortened URLs using JSON parsing+ output_json = {+ 'thinking': f'I need to navigate to {shortened_url} for documentation',+ 'evaluation_previous_goal': 'Successfully processed the request',+ 'memory': f'Found useful info at {shortened_url}',+ 'next_goal': 'Complete the documentation review',+ 'action': [{'go_to_url': {'url': shortened_url, 'new_tab': False}}],+ }++ # Create properly typed AgentOutput with custom actions+ tools = agent.tools+ ActionModel = tools.registry.create_action_model()+ AgentOutputWithActions = AgentOutput.type_with_custom_actions(ActionModel)+ agent_output = AgentOutputWithActions.model_validate_json(json.dumps(output_json))++ # Process the output to restore URLs (modifies agent_output in-place)+ agent._recursive_process_all_strings_inside_pydantic_model(agent_output, url_mappings)++ # Verify URLs were restored in all locations+ assert SUPER_LONG_URL in (agent_output.thinking or '')+ assert SUPER_LONG_URL in (agent_output.memory or '')+ action_data = agent_output.action[0].model_dump()+ assert action_data['go_to_url']['url'] == SUPER_LONG_URL+++class TestUrlShorteningEndToEnd:+ """Test complete URL shortening pipeline end-to-end."""++ def test_complete_url_shortening_pipeline(self, agent: Agent):+ """Test the complete pipeline: input shortening -> processing -> output restoration."""++ # Step 1: Input processing with URL shortening+ original_content = f'Navigate to {SUPER_LONG_URL} and extract the API documentation'++ messages: list[BaseMessage] = [UserMessage(content=original_content)]++ url_mappings = agent._process_messsages_and_replace_long_urls_shorter_ones(messages)++ # Verify URL was shortened in input+ assert len(url_mappings) == 1+ shortened_url = next(iter(url_mappings.keys()))+ assert url_mappings[shortened_url] == SUPER_LONG_URL+ assert shortened_url in (messages[0].content or '')++ # Step 2: Simulate agent output with shortened URL+ output_json = {+ 'thinking': f'I will navigate to {shortened_url} to get the documentation',+ 'evaluation_previous_goal': 'Starting documentation extraction',+ 'memory': f'Target URL: {shortened_url}',+ 'next_goal': 'Extract API documentation',+ 'action': [{'go_to_url': {'url': shortened_url, 'new_tab': True}}],+ }++ # Create AgentOutput with custom actions+ tools = agent.tools+ ActionModel = tools.registry.create_action_model()+ AgentOutputWithActions = AgentOutput.type_with_custom_actions(ActionModel)+ agent_output = AgentOutputWithActions.model_validate_json(json.dumps(output_json))++ # Step 3: Output processing with URL restoration (modifies agent_output in-place)+ agent._recursive_process_all_strings_inside_pydantic_model(agent_output, url_mappings)++ # Verify complete pipeline worked correctly+ assert SUPER_LONG_URL in (agent_output.thinking or '')+ assert SUPER_LONG_URL in (agent_output.memory or '')+ action_data = agent_output.action[0].model_dump()+ assert action_data['go_to_url']['url'] == SUPER_LONG_URL+ assert action_data['go_to_url']['new_tab'] is True++ # Verify original shortened content is no longer present+ assert shortened_url not in (agent_output.thinking or '')+ assert shortened_url not in (agent_output.memory or '')