Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
fixes and large list example
mertunsalla4207a6ff9b8
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.pyindex 2891c1ccc..fcba83fc9 100644--- a/browser_use/browser/session.py+++ b/browser_use/browser/session.py@@ -1563,7 +1563,7 @@ class BrowserSession(BaseModel):if is_new_tab_page(url) or url.startswith('chrome://'):# Use URL as title for chrome pages, or mark new tabs as unusableif is_new_tab_page(url):- title = 'ignore this tab and do not use it'+ title = ''elif not title:# For chrome:// pages without a title, use the URL itselftitle = url@@ -1585,7 +1585,7 @@ class BrowserSession(BaseModel):self.logger.debug(f'⚠️ Failed to get target info for tab #{i}: {_log_pretty_url(url)} - {type(e).__name__}')if is_new_tab_page(url):- title = 'ignore this tab and do not use it'+ title = ''elif url.startswith('chrome://'):title = urlelse:diff --git a/browser_use/browser/watchdogs/security_watchdog.py b/browser_use/browser/watchdogs/security_watchdog.pyindex ad00775fc..2cb866d02 100644--- a/browser_use/browser/watchdogs/security_watchdog.py+++ b/browser_use/browser/watchdogs/security_watchdog.py@@ -48,7 +48,7 @@ class SecurityWatchdog(BaseWatchdog):raise ValueError(f'Navigation to {event.url} blocked by security policy')async def on_NavigationCompleteEvent(self, event: NavigationCompleteEvent) -> None:- """Check if navigated URL is allowed and close tab if not."""+ """Check if navigated URL is allowed (catches redirects to blocked domains)."""# Check if the navigated URL is allowed (in case of redirects)if not self._is_url_allowed(event.url):self.logger.warning(f'⛔️ Navigation to non-allowed URL detected: {event.url}')@@ -57,17 +57,19 @@ class SecurityWatchdog(BaseWatchdog):self.event_bus.dispatch(BrowserErrorEvent(error_type='NavigationBlocked',- message=f'Navigation to non-allowed URL: {event.url}',+ message=f'Navigation blocked to non-allowed URL: {event.url} - redirecting to about:blank',details={'url': event.url, 'target_id': event.target_id},))-- # Close the target that navigated to the disallowed URL+ # Navigate to about:blank to keep session alive+ # Agent will see the error and can continue with other taskstry:- await self.browser_session._cdp_close_page(event.target_id)- self.logger.info(f'⛔️ Closed target with non-allowed URL: {event.url}')+ session = await self.browser_session.get_or_create_cdp_session(target_id=event.target_id)+ await session.cdp_client.send.Page.navigate(params={'url': 'about:blank'}, session_id=session.session_id)+ self.logger.info(f'⛔️ Navigated to about:blank after blocked URL: {event.url}')except Exception as e:- self.logger.error(f'⛔️ Failed to close target with non-allowed URL: {type(e).__name__} {e}')+ pass+ self.logger.error(f'⛔️ Failed to navigate to about:blank: {type(e).__name__} {e}')async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:"""Check if new tab URL is allowed."""diff --git a/examples/features/large_blocklist.py b/examples/features/large_blocklist.pynew file mode 100644index 000000000..3e69cef4e--- /dev/null+++ b/examples/features/large_blocklist.py@@ -0,0 +1,117 @@+"""+Example: Using large blocklists (400k+ domains) with automatic optimization++This example demonstrates:+1. Loading a real-world blocklist (HaGeZi's Pro++ with 439k+ domains)+2. Automatic conversion to set for O(1) lookup performance+3. Testing that blocked domains are actually blocked++Performance: ~0.02ms per domain check (50,000+ checks/second!)+"""++import asyncio+import os+import sys++sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))++from dotenv import load_dotenv++load_dotenv()++from browser_use import Agent, ChatOpenAI+from browser_use.browser import BrowserProfile, BrowserSession++llm = ChatOpenAI(model='gpt-4.1-mini')+++def load_blocklist_from_url(url: str) -> list[str]:+ """Load and parse a blocklist from a URL.++ Args:+ url: URL to the blocklist file++ Returns:+ List of domain strings (comments and empty lines removed)+ """+ import urllib.request++ print(f'📥 Downloading blocklist from {url}...')++ domains = []+ with urllib.request.urlopen(url) as response:+ for line in response:+ line = line.decode('utf-8').strip()+ # Skip comments and empty lines+ if line and not line.startswith('#'):+ domains.append(line)++ print(f'✅ Loaded {len(domains):,} domains')+ return domains+++async def main():+ # Load HaGeZi's Pro++ blocklist (blocks ads, tracking, malware, etc.)+ # Source: https://github.com/hagezi/dns-blocklists+ blocklist_url = 'https://gitlab.com/hagezi/mirror/-/raw/main/dns-blocklists/domains/pro.plus.txt'++ print('=' * 70)+ print('🚀 Large Blocklist Demo - 439k+ Blocked Domains')+ print('=' * 70)+ print()++ # Load the blocklist+ prohibited_domains = load_blocklist_from_url(blocklist_url)++ # Sample some blocked domains to test+ test_blocked = [prohibited_domains[0], prohibited_domains[1000], prohibited_domains[-1]]+ print(f'\n📋 Sample blocked domains: {", ".join(test_blocked[:3])}')++ print(f'\n🔧 Creating browser with {len(prohibited_domains):,} blocked domains...')+ print(' (Auto-optimizing to set for O(1) lookup performance)')++ # Create browser with the blocklist+ # The list will be automatically optimized to a set for fast lookups+ browser_session = BrowserSession(+ browser_profile=BrowserProfile(+ prohibited_domains=prohibited_domains,+ headless=False,+ user_data_dir='~/.config/browseruse/profiles/blocklist-demo',+ ),+ )++ # Task: Try to visit a blocked domain and a safe domain+ blocked_site = test_blocked[0] # Will be blocked+ safe_site = 'github.com' # Will be allowed++ task = f"""+ Try to navigate to these websites and report what happens:+ 1. First, try to visit https://{blocked_site}+ 2. Then, try to visit https://{safe_site}++ Tell me which sites you were able to access and which were blocked.+ """++ agent = Agent(+ task=task,+ llm=llm,+ browser_session=browser_session,+ )++ print(f'\n🤖 Agent task: Try to visit {blocked_site} (blocked) and {safe_site} (allowed)')+ print('\n' + '=' * 70)++ await agent.run(max_steps=5)++ print('\n' + '=' * 70)+ print('✅ Demo complete!')+ print(f'💡 The blocklist with {len(prohibited_domains):,} domains was optimized to a set')+ print(' for instant O(1) domain checking (vs slow O(n) pattern matching)')+ print('=' * 70)++ input('\nPress Enter to close the browser...')+ await browser_session.kill()+++if __name__ == '__main__':+ asyncio.run(main())