Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

fixes and large list example

mertunsalla4207a6ff9b8
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index 2891c1ccc..fcba83fc9 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -1563,7 +1563,7 @@ class BrowserSession(BaseModel):
if is_new_tab_page(url) or url.startswith('chrome://'):
# Use URL as title for chrome pages, or mark new tabs as unusable
if is_new_tab_page(url):
- title = 'ignore this tab and do not use it'
+ title = ''
elif not title:
# For chrome:// pages without a title, use the URL itself
title = url
@@ -1585,7 +1585,7 @@ class BrowserSession(BaseModel):
self.logger.debug(f'⚠️ Failed to get target info for tab #{i}: {_log_pretty_url(url)} - {type(e).__name__}')
if is_new_tab_page(url):
- title = 'ignore this tab and do not use it'
+ title = ''
elif url.startswith('chrome://'):
title = url
else:
diff --git a/browser_use/browser/watchdogs/security_watchdog.py b/browser_use/browser/watchdogs/security_watchdog.py
index ad00775fc..2cb866d02 100644
--- a/browser_use/browser/watchdogs/security_watchdog.py
+++ b/browser_use/browser/watchdogs/security_watchdog.py
@@ -48,7 +48,7 @@ class SecurityWatchdog(BaseWatchdog):
raise ValueError(f'Navigation to {event.url} blocked by security policy')
async def on_NavigationCompleteEvent(self, event: NavigationCompleteEvent) -> None:
- """Check if navigated URL is allowed and close tab if not."""
+ """Check if navigated URL is allowed (catches redirects to blocked domains)."""
# Check if the navigated URL is allowed (in case of redirects)
if not self._is_url_allowed(event.url):
self.logger.warning(f'⛔️ Navigation to non-allowed URL detected: {event.url}')
@@ -57,17 +57,19 @@ class SecurityWatchdog(BaseWatchdog):
self.event_bus.dispatch(
BrowserErrorEvent(
error_type='NavigationBlocked',
- message=f'Navigation to non-allowed URL: {event.url}',
+ message=f'Navigation blocked to non-allowed URL: {event.url} - redirecting to about:blank',
details={'url': event.url, 'target_id': event.target_id},
)
)
-
- # Close the target that navigated to the disallowed URL
+ # Navigate to about:blank to keep session alive
+ # Agent will see the error and can continue with other tasks
try:
- await self.browser_session._cdp_close_page(event.target_id)
- self.logger.info(f'⛔️ Closed target with non-allowed URL: {event.url}')
+ session = await self.browser_session.get_or_create_cdp_session(target_id=event.target_id)
+ await session.cdp_client.send.Page.navigate(params={'url': 'about:blank'}, session_id=session.session_id)
+ self.logger.info(f'⛔️ Navigated to about:blank after blocked URL: {event.url}')
except Exception as e:
- self.logger.error(f'⛔️ Failed to close target with non-allowed URL: {type(e).__name__} {e}')
+ pass
+ self.logger.error(f'⛔️ Failed to navigate to about:blank: {type(e).__name__} {e}')
async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
"""Check if new tab URL is allowed."""
diff --git a/examples/features/large_blocklist.py b/examples/features/large_blocklist.py
new file mode 100644
index 000000000..3e69cef4e
--- /dev/null
+++ b/examples/features/large_blocklist.py
@@ -0,0 +1,117 @@
+"""
+Example: Using large blocklists (400k+ domains) with automatic optimization
+
+This example demonstrates:
+1. Loading a real-world blocklist (HaGeZi's Pro++ with 439k+ domains)
+2. Automatic conversion to set for O(1) lookup performance
+3. Testing that blocked domains are actually blocked
+
+Performance: ~0.02ms per domain check (50,000+ checks/second!)
+"""
+
+import asyncio
+import os
+import sys
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from browser_use import Agent, ChatOpenAI
+from browser_use.browser import BrowserProfile, BrowserSession
+
+llm = ChatOpenAI(model='gpt-4.1-mini')
+
+
+def load_blocklist_from_url(url: str) -> list[str]:
+ """Load and parse a blocklist from a URL.
+
+ Args:
+ url: URL to the blocklist file
+
+ Returns:
+ List of domain strings (comments and empty lines removed)
+ """
+ import urllib.request
+
+ print(f'📥 Downloading blocklist from {url}...')
+
+ domains = []
+ with urllib.request.urlopen(url) as response:
+ for line in response:
+ line = line.decode('utf-8').strip()
+ # Skip comments and empty lines
+ if line and not line.startswith('#'):
+ domains.append(line)
+
+ print(f'✅ Loaded {len(domains):,} domains')
+ return domains
+
+
+async def main():
+ # Load HaGeZi's Pro++ blocklist (blocks ads, tracking, malware, etc.)
+ # Source: https://github.com/hagezi/dns-blocklists
+ blocklist_url = 'https://gitlab.com/hagezi/mirror/-/raw/main/dns-blocklists/domains/pro.plus.txt'
+
+ print('=' * 70)
+ print('🚀 Large Blocklist Demo - 439k+ Blocked Domains')
+ print('=' * 70)
+ print()
+
+ # Load the blocklist
+ prohibited_domains = load_blocklist_from_url(blocklist_url)
+
+ # Sample some blocked domains to test
+ test_blocked = [prohibited_domains[0], prohibited_domains[1000], prohibited_domains[-1]]
+ print(f'\n📋 Sample blocked domains: {", ".join(test_blocked[:3])}')
+
+ print(f'\n🔧 Creating browser with {len(prohibited_domains):,} blocked domains...')
+ print(' (Auto-optimizing to set for O(1) lookup performance)')
+
+ # Create browser with the blocklist
+ # The list will be automatically optimized to a set for fast lookups
+ browser_session = BrowserSession(
+ browser_profile=BrowserProfile(
+ prohibited_domains=prohibited_domains,
+ headless=False,
+ user_data_dir='~/.config/browseruse/profiles/blocklist-demo',
+ ),
+ )
+
+ # Task: Try to visit a blocked domain and a safe domain
+ blocked_site = test_blocked[0] # Will be blocked
+ safe_site = 'github.com' # Will be allowed
+
+ task = f"""
+ Try to navigate to these websites and report what happens:
+ 1. First, try to visit https://{blocked_site}
+ 2. Then, try to visit https://{safe_site}
+
+ Tell me which sites you were able to access and which were blocked.
+ """
+
+ agent = Agent(
+ task=task,
+ llm=llm,
+ browser_session=browser_session,
+ )
+
+ print(f'\n🤖 Agent task: Try to visit {blocked_site} (blocked) and {safe_site} (allowed)')
+ print('\n' + '=' * 70)
+
+ await agent.run(max_steps=5)
+
+ print('\n' + '=' * 70)
+ print('✅ Demo complete!')
+ print(f'💡 The blocklist with {len(prohibited_domains):,} domains was optimized to a set')
+ print(' for instant O(1) domain checking (vs slow O(n) pattern matching)')
+ print('=' * 70)
+
+ input('\nPress Enter to close the browser...')
+ await browser_session.kill()
+
+
+if __name__ == '__main__':
+ asyncio.run(main())