Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

shorten super long URLs

Gregor Žuničb5aebf0b034b
diff --git a/browser_use/agent/service.py b/browser_use/agent/service.py
index 22d8873e4..03662c753 100644
--- a/browser_use/agent/service.py
+++ b/browser_use/agent/service.py
@@ -30,7 +30,7 @@ from browser_use.tokens.service import TokenCost
load_dotenv()
from bubus import EventBus
-from pydantic import ValidationError
+from pydantic import BaseModel, ValidationError
from uuid_extensions import uuid7str
from browser_use import Browser, BrowserProfile, BrowserSession
@@ -66,6 +66,7 @@ from browser_use.telemetry.views import AgentTelemetryEvent
from browser_use.tools.registry.views import ActionModel
from browser_use.tools.service import Tools
from browser_use.utils import (
+ URL_PATTERN,
_log_pretty_path,
get_browser_use_version,
get_git_info,
@@ -179,6 +180,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
directly_open_url: bool = True,
include_recent_events: bool = False,
sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
+ _url_shortening_limit: int = 25,
**kwargs,
):
if llm is None:
@@ -227,6 +229,7 @@ class Agent(Generic[Context, AgentStructuredOutput]):
self.llm = llm
self.directly_open_url = directly_open_url
self.include_recent_events = include_recent_events
+ self._url_shortening_limit = _url_shortening_limit
if tools is not None:
self.tools = tools
elif controller is not None:
@@ -970,15 +973,151 @@ class Agent(Generic[Context, AgentStructuredOutput]):
text = re.sub(STRAY_CLOSE_TAG, '', text)
return text.strip()
+ # region - URL replacement
+ def _replace_urls_in_text(self, text: str) -> tuple[str, dict[str, str]]:
+ """Replace URLs in a text string"""
+
+ replaced_urls: dict[str, str] = {}
+
+ def replace_url(match: re.Match) -> str:
+ """Url can only have 1 query and 1 fragment"""
+ import hashlib
+
+ original_url = match.group(0)
+
+ # Find where the query/fragment starts
+ query_start = original_url.find('?')
+ fragment_start = original_url.find('#')
+
+ # Find the earliest position of query or fragment
+ after_path_start = len(original_url) # Default: no query/fragment
+ if query_start != -1:
+ after_path_start = min(after_path_start, query_start)
+ if fragment_start != -1:
+ after_path_start = min(after_path_start, fragment_start)
+
+ # Split URL into base (up to path) and after_path (query + fragment)
+ base_url = original_url[:after_path_start]
+ after_path = original_url[after_path_start:]
+
+ # If after_path is within the limit, don't shorten
+ if len(after_path) <= self._url_shortening_limit:
+ return original_url
+
+ # If after_path is too long, truncate and add hash
+ if after_path:
+ truncated_after_path = after_path[: self._url_shortening_limit]
+ # Create a short hash of the full after_path content
+ hash_obj = hashlib.md5(after_path.encode('utf-8'))
+ short_hash = hash_obj.hexdigest()[:7]
+ # Create shortened URL
+ shortened = f'{base_url}{truncated_after_path}...{short_hash}'
+ # Only use shortened URL if it's actually shorter than the original
+ if len(shortened) < len(original_url):
+ replaced_urls[shortened] = original_url
+ return shortened
+
+ return original_url
+
+ return URL_PATTERN.sub(replace_url, text), replaced_urls
+
+ def _process_messsages_and_replace_long_urls_shorter_ones(self, input_messages: list[BaseMessage]) -> dict[str, str]:
+ """Replace long URLs with shorter ones
+ ? @dev edits input_messages in place
+
+ returns:
+ tuple[filtered_input_messages, urls we replaced {shorter_url: original_url}]
+ """
+ from browser_use.llm.messages import AssistantMessage, UserMessage
+
+ urls_replaced: dict[str, str] = {}
+
+ # Process each message, in place
+ for message in input_messages:
+ # no need to process SystemMessage, we have control over that anyway
+ if isinstance(message, (UserMessage, AssistantMessage)):
+ if isinstance(message.content, str):
+ # Simple string content
+ message.content, replaced_urls = self._replace_urls_in_text(message.content)
+ urls_replaced.update(replaced_urls)
+
+ elif isinstance(message.content, list):
+ # List of content parts
+ for part in message.content:
+ if isinstance(part, ContentPartTextParam):
+ part.text, replaced_urls = self._replace_urls_in_text(part.text)
+ urls_replaced.update(replaced_urls)
+
+ return urls_replaced
+
+ @staticmethod
+ def _recursive_process_all_strings_inside_pydantic_model(model: BaseModel, url_replacements: dict[str, str]) -> None:
+ """Recursively process all strings inside a Pydantic model, replacing shortened URLs with originals in place."""
+ for field_name, field_value in model.__dict__.items():
+ if isinstance(field_value, str):
+ # Replace shortened URLs with original URLs in string
+ processed_string = Agent._replace_shortened_urls_in_string(field_value, url_replacements)
+ setattr(model, field_name, processed_string)
+ elif isinstance(field_value, BaseModel):
+ # Recursively process nested Pydantic models
+ Agent._recursive_process_all_strings_inside_pydantic_model(field_value, url_replacements)
+ elif isinstance(field_value, dict):
+ # Process dictionary values in place
+ Agent._recursive_process_dict(field_value, url_replacements)
+ elif isinstance(field_value, (list, tuple)):
+ Agent._recursive_process_list_or_tuple(field_value, url_replacements)
+
+ @staticmethod
+ def _recursive_process_dict(dictionary: dict, url_replacements: dict[str, str]) -> None:
+ """Helper method to process dictionaries."""
+ for k, v in dictionary.items():
+ if isinstance(v, str):
+ dictionary[k] = Agent._replace_shortened_urls_in_string(v, url_replacements)
+ elif isinstance(v, BaseModel):
+ Agent._recursive_process_all_strings_inside_pydantic_model(v, url_replacements)
+ elif isinstance(v, dict):
+ Agent._recursive_process_dict(v, url_replacements)
+ elif isinstance(v, (list, tuple)):
+ Agent._recursive_process_list_or_tuple(v, url_replacements)
+
+ @staticmethod
+ def _recursive_process_list_or_tuple(container, url_replacements: dict[str, str]) -> None:
+ """Helper method to process lists and tuples."""
+ for i, item in enumerate(container):
+ if isinstance(item, str):
+ container[i] = Agent._replace_shortened_urls_in_string(item, url_replacements)
+ elif isinstance(item, BaseModel):
+ Agent._recursive_process_all_strings_inside_pydantic_model(item, url_replacements)
+ elif isinstance(item, dict):
+ Agent._recursive_process_dict(item, url_replacements)
+ elif isinstance(item, (list, tuple)):
+ Agent._recursive_process_list_or_tuple(item, url_replacements)
+
+ @staticmethod
+ def _replace_shortened_urls_in_string(text: str, url_replacements: dict[str, str]) -> str:
+ """Replace all shortened URLs in a string with their original URLs."""
+ result = text
+ for shortened_url, original_url in url_replacements.items():
+ result = result.replace(shortened_url, original_url)
+ return result
+
+ # endregion - URL replacement
+
@time_execution_async('--get_next_action')
@observe_debug(ignore_input=True, ignore_output=True, name='get_model_output')
async def get_model_output(self, input_messages: list[BaseMessage]) -> AgentOutput:
"""Get next action from LLM based on current state"""
+ urls_replaced = self._process_messsages_and_replace_long_urls_shorter_ones(input_messages)
+
try:
response = await self.llm.ainvoke(input_messages, output_format=self.AgentOutput)
parsed = response.completion
+ # Replace any shortened URLs in the LLM response back to original URLs
+ if urls_replaced:
+ self._recursive_process_all_strings_inside_pydantic_model(parsed, urls_replaced)
+
# cut the number of actions to max_actions_per_step if needed
if len(parsed.action) > self.settings.max_actions_per_step:
parsed.action = parsed.action[: self.settings.max_actions_per_step]
diff --git a/browser_use/utils.py b/browser_use/utils.py
index 3fd4c4ba7..35f9a9db0 100644
--- a/browser_use/utils.py
+++ b/browser_use/utils.py
@@ -2,6 +2,7 @@ import asyncio
import logging
import os
import platform
+import re
import signal
import time
from collections.abc import Callable, Coroutine
@@ -16,6 +17,9 @@ from dotenv import load_dotenv
load_dotenv()
+# Pre-compiled regex for URL detection - used in URL shortening
+URL_PATTERN = re.compile(r'https?://[^\s<>"\']+|www\.[^\s<>"\']+|[^\s<>"\']+\.[a-z]{2,}(?:/[^\s<>"\']*)?', re.IGNORECASE)
+
logger = logging.getLogger(__name__)
diff --git a/tests/ci/test_url_shortening.py b/tests/ci/test_url_shortening.py
new file mode 100644
index 000000000..6ea45b15d
--- /dev/null
+++ b/tests/ci/test_url_shortening.py
@@ -0,0 +1,161 @@
+"""
+Simplified tests for URL shortening functionality in Agent service.
+
+Three focused tests:
+1. Input message processing with URL shortening
+2. Output processing with custom actions and URL restoration
+3. End-to-end pipeline test
+"""
+
+import json
+
+import pytest
+
+from browser_use.agent.service import Agent
+from browser_use.agent.views import AgentOutput
+from browser_use.llm.messages import AssistantMessage, BaseMessage, UserMessage
+
+# Super long URL to reuse across tests - much longer than the 25 character limit
+# Includes both query params (?...) and fragment params (#...)
+SUPER_LONG_URL = 'https://documentation.example-company.com/api/v3/enterprise/user-management/endpoints/administration/create-new-user-account-with-permissions/advanced-settings?format=detailed-json&version=3.2.1&timestamp=1699123456789&session_id=abc123def456ghi789&authentication_token=very_long_authentication_token_string_here&include_metadata=true&expand_relationships=user_groups,permissions,roles&sort_by=created_at&order=desc&page_size=100&include_deprecated_fields=false&api_key=super_long_api_key_that_exceeds_normal_limits#section=user_management&tab=advanced&view=detailed&scroll_to=permissions_table&highlight=admin_settings&filter=active_users&expand_all=true&debug_mode=enabled'
+
+
+def agent():
+ """Create an agent instance for testing URL shortening functionality."""
+ from tests.ci.conftest import create_mock_llm
+
+ return Agent(task='Test URL shortening', llm=create_mock_llm(), url_shortening_limit=25)
+
+
+class TestUrlShorteningInputProcessing:
+ """Test URL shortening for input messages."""
+
+ def test_process_input_messages_with_url_shortening(self, agent: Agent):
+ """Test that long URLs in input messages are shortened and mappings stored."""
+ original_content = f'Please visit {SUPER_LONG_URL} and extract information'
+
+ messages: list[BaseMessage] = [UserMessage(content=original_content)]
+
+ # Process messages (modifies messages in-place and returns URL mappings)
+ url_mappings = agent._process_messsages_and_replace_long_urls_shorter_ones(messages)
+
+ # Verify URL was shortened in the message (modified in-place)
+ processed_content = messages[0].content or ''
+ assert processed_content != original_content
+ assert 'https://documentation.example-company.com' in processed_content
+ assert len(processed_content) < len(original_content)
+
+ # Verify URL mapping was returned
+ assert len(url_mappings) == 1
+ shortened_url = next(iter(url_mappings.keys()))
+ assert url_mappings[shortened_url] == SUPER_LONG_URL
+
+ def test_process_user_and_assistant_messages_with_url_shortening(self, agent: Agent):
+ """Test URL shortening in both UserMessage and AssistantMessage."""
+ user_content = f'I need to access {SUPER_LONG_URL} for the API documentation'
+ assistant_content = f'I will help you navigate to {SUPER_LONG_URL} to retrieve the documentation'
+
+ messages: list[BaseMessage] = [UserMessage(content=user_content), AssistantMessage(content=assistant_content)]
+
+ # Process messages (modifies messages in-place and returns URL mappings)
+ url_mappings = agent._process_messsages_and_replace_long_urls_shorter_ones(messages)
+
+ # Verify URL was shortened in both messages
+ user_processed_content = messages[0].content or ''
+ assistant_processed_content = messages[1].content or ''
+
+ assert user_processed_content != user_content
+ assert assistant_processed_content != assistant_content
+ assert 'https://documentation.example-company.com' in user_processed_content
+ assert 'https://documentation.example-company.com' in assistant_processed_content
+ assert len(user_processed_content) < len(user_content)
+ assert len(assistant_processed_content) < len(assistant_content)
+
+ # Verify URL mapping was returned (should be same shortened URL for both occurrences)
+ assert len(url_mappings) == 1
+ shortened_url = next(iter(url_mappings.keys()))
+ assert url_mappings[shortened_url] == SUPER_LONG_URL
+
+
+class TestUrlShorteningOutputProcessing:
+ """Test URL restoration for output processing with custom actions."""
+
+ def test_process_output_with_custom_actions_and_url_restoration(self, agent: Agent):
+ """Test that shortened URLs in AgentOutput with custom actions are restored."""
+ # Set up URL mapping (simulating previous shortening)
+ shortened_url: str = agent._replace_urls_in_text(SUPER_LONG_URL)[0]
+ url_mappings = {shortened_url: SUPER_LONG_URL}
+
+ # Create AgentOutput with shortened URLs using JSON parsing
+ output_json = {
+ 'thinking': f'I need to navigate to {shortened_url} for documentation',
+ 'evaluation_previous_goal': 'Successfully processed the request',
+ 'memory': f'Found useful info at {shortened_url}',
+ 'next_goal': 'Complete the documentation review',
+ 'action': [{'go_to_url': {'url': shortened_url, 'new_tab': False}}],
+ }
+
+ # Create properly typed AgentOutput with custom actions
+ tools = agent.tools
+ ActionModel = tools.registry.create_action_model()
+ AgentOutputWithActions = AgentOutput.type_with_custom_actions(ActionModel)
+ agent_output = AgentOutputWithActions.model_validate_json(json.dumps(output_json))
+
+ # Process the output to restore URLs (modifies agent_output in-place)
+ agent._recursive_process_all_strings_inside_pydantic_model(agent_output, url_mappings)
+
+ # Verify URLs were restored in all locations
+ assert SUPER_LONG_URL in (agent_output.thinking or '')
+ assert SUPER_LONG_URL in (agent_output.memory or '')
+ action_data = agent_output.action[0].model_dump()
+ assert action_data['go_to_url']['url'] == SUPER_LONG_URL
+
+
+class TestUrlShorteningEndToEnd:
+ """Test complete URL shortening pipeline end-to-end."""
+
+ def test_complete_url_shortening_pipeline(self, agent: Agent):
+ """Test the complete pipeline: input shortening -> processing -> output restoration."""
+
+ # Step 1: Input processing with URL shortening
+ original_content = f'Navigate to {SUPER_LONG_URL} and extract the API documentation'
+
+ messages: list[BaseMessage] = [UserMessage(content=original_content)]
+
+ url_mappings = agent._process_messsages_and_replace_long_urls_shorter_ones(messages)
+
+ # Verify URL was shortened in input
+ assert len(url_mappings) == 1
+ shortened_url = next(iter(url_mappings.keys()))
+ assert url_mappings[shortened_url] == SUPER_LONG_URL
+ assert shortened_url in (messages[0].content or '')
+
+ # Step 2: Simulate agent output with shortened URL
+ output_json = {
+ 'thinking': f'I will navigate to {shortened_url} to get the documentation',
+ 'evaluation_previous_goal': 'Starting documentation extraction',
+ 'memory': f'Target URL: {shortened_url}',
+ 'next_goal': 'Extract API documentation',
+ 'action': [{'go_to_url': {'url': shortened_url, 'new_tab': True}}],
+ }
+
+ # Create AgentOutput with custom actions
+ tools = agent.tools
+ ActionModel = tools.registry.create_action_model()
+ AgentOutputWithActions = AgentOutput.type_with_custom_actions(ActionModel)
+ agent_output = AgentOutputWithActions.model_validate_json(json.dumps(output_json))
+
+ # Step 3: Output processing with URL restoration (modifies agent_output in-place)
+ agent._recursive_process_all_strings_inside_pydantic_model(agent_output, url_mappings)
+
+ # Verify complete pipeline worked correctly
+ assert SUPER_LONG_URL in (agent_output.thinking or '')
+ assert SUPER_LONG_URL in (agent_output.memory or '')
+ action_data = agent_output.action[0].model_dump()
+ assert action_data['go_to_url']['url'] == SUPER_LONG_URL
+ assert action_data['go_to_url']['new_tab'] is True
+
+ # Verify original shortened content is no longer present
+ assert shortened_url not in (agent_output.thinking or '')
+ assert shortened_url not in (agent_output.memory or '')