Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Blocked domains are optimized!!!
mertunsall16457c2a5e9f
diff --git a/browser_use/browser/profile.py b/browser_use/browser/profile.pyindex 513770424..044a021b4 100644--- a/browser_use/browser/profile.py+++ b/browser_use/browser/profile.py@@ -13,6 +13,7 @@ from browser_use.config import CONFIGfrom browser_use.utils import _log_pretty_path, loggerCHROME_DEBUG_PORT = 9242 # use a non-default port to avoid conflicts with other tools / devs using 9222+DOMAIN_OPTIMIZATION_THRESHOLD = 100 # Convert domain lists to sets for O(1) lookup when >= this sizeCHROME_DISABLED_COMPONENTS = [# Playwright defaults: https://github.com/microsoft/playwright/blob/41008eeddd020e2dee1c540f7c0cdfa337e99637/packages/playwright-core/src/server/chromium/chromiumSwitches.ts#L76# AcceptCHFrame,AutoExpandDetailsElement,AvoidUnnecessaryBeforeUnloadCheckSync,CertificateTransparencyComponentUpdater,DeferRendererTasksAfterInput,DestroyProfileOnBrowserClose,DialMediaRouteProvider,ExtensionManifestV2Disabled,GlobalMediaControls,HttpsUpgrades,ImprovedCookieControls,LazyFrameLoading,LensOverlay,MediaRouter,PaintHolding,ThirdPartyStoragePartitioning,Translate@@ -563,13 +564,13 @@ class BrowserProfile(BrowserConnectArgs, BrowserLaunchPersistentContextArgs, Bro# custom options we provide that aren't native playwright kwargsdisable_security: bool = Field(default=False, description='Disable browser security features.')deterministic_rendering: bool = Field(default=False, description='Enable deterministic rendering flags.')- allowed_domains: list[str] | None = Field(+ allowed_domains: list[str] | set[str] | None = Field(default=None,- description='List of allowed domains for navigation e.g. ["*.google.com", "https://example.com", "chrome-extension://*"]',+ description='List of allowed domains for navigation e.g. ["*.google.com", "https://example.com", "chrome-extension://*"]. Lists with 100+ items are auto-optimized to sets (no pattern matching).',)- prohibited_domains: list[str] | None = Field(+ prohibited_domains: list[str] | set[str] | None = Field(default=None,- description='List of prohibited domains for navigation e.g. ["*.google.com", "https://example.com", "chrome-extension://*"]. Allowed domains take precedence over prohibited domains.',+ description='List of prohibited domains for navigation e.g. ["*.google.com", "https://example.com", "chrome-extension://*"]. Allowed domains take precedence over prohibited domains. Lists with 100+ items are auto-optimized to sets (no pattern matching).',)keep_alive: bool | None = Field(default=None, description='Keep browser alive after agent run.')@@ -665,6 +666,23 @@ class BrowserProfile(BrowserConnectArgs, BrowserLaunchPersistentContextArgs, Brodef __str__(self) -> str:return 'BrowserProfile'+ @field_validator('allowed_domains', 'prohibited_domains', mode='after')+ @classmethod+ def optimize_large_domain_lists(cls, v: list[str] | set[str] | None) -> list[str] | set[str] | None:+ """Convert large domain lists (>=100 items) to sets for O(1) lookup performance."""+ if v is None or isinstance(v, set):+ return v++ if len(v) >= DOMAIN_OPTIMIZATION_THRESHOLD:+ logger.warning(+ f'🔧 Optimizing domain list with {len(v)} items to set for O(1) lookup. '+ f'Note: Pattern matching (*.domain.com, etc.) is not supported for lists >= {DOMAIN_OPTIMIZATION_THRESHOLD} items. '+ f'Use exact domains only or keep list size < {DOMAIN_OPTIMIZATION_THRESHOLD} for pattern support.'+ )+ return set(v)++ return v+@model_validator(mode='after')def copy_old_config_names_to_new(self) -> Self:"""Copy old config window_width & window_height to window_size."""diff --git a/browser_use/browser/watchdogs/security_watchdog.py b/browser_use/browser/watchdogs/security_watchdog.pyindex c9f7caad7..ad00775fc 100644--- a/browser_use/browser/watchdogs/security_watchdog.py+++ b/browser_use/browser/watchdogs/security_watchdog.py@@ -118,6 +118,22 @@ class SecurityWatchdog(BaseWatchdog):'Note: Patterns like "*.example.com" will match both subdomains AND the main domain.')+ def _get_domain_variants(self, host: str) -> tuple[str, str]:+ """Get both variants of a domain (with and without www prefix).++ Args:+ host: The hostname to process++ Returns:+ Tuple of (original_host, variant_host)+ - If host starts with www., variant is without www.+ - Otherwise, variant is with www. prefix+ """+ if host.startswith('www.'):+ return (host, host[4:]) # ('www.example.com', 'example.com')+ else:+ return (host, f'www.{host}') # ('example.com', 'www.example.com')+def _is_url_allowed(self, url: str) -> bool:"""Check if a URL is allowed based on the allowed_domains configuration.@@ -153,21 +169,35 @@ class SecurityWatchdog(BaseWatchdog):if not host:return False- # Check each allowed domain pattern+ # Check allowed domains (fast path for sets, slow path for lists with patterns)if self.browser_session.browser_profile.allowed_domains:- for pattern in self.browser_session.browser_profile.allowed_domains:- if self._is_url_match(url, host, parsed.scheme, pattern):- return True+ allowed_domains = self.browser_session.browser_profile.allowed_domains- return False+ if isinstance(allowed_domains, set):+ # Fast path: O(1) exact hostname match - check both www and non-www variants+ host_variant, host_alt = self._get_domain_variants(host)+ return host_variant in allowed_domains or host_alt in allowed_domains+ else:+ # Slow path: O(n) pattern matching for lists+ for pattern in allowed_domains:+ if self._is_url_match(url, host, parsed.scheme, pattern):+ return True+ return False- # Check each prohibited domain pattern+ # Check prohibited domains (fast path for sets, slow path for lists with patterns)if self.browser_session.browser_profile.prohibited_domains:- for pattern in self.browser_session.browser_profile.prohibited_domains:- if self._is_url_match(url, host, parsed.scheme, pattern):- return False+ prohibited_domains = self.browser_session.browser_profile.prohibited_domains- return True+ if isinstance(prohibited_domains, set):+ # Fast path: O(1) exact hostname match - check both www and non-www variants+ host_variant, host_alt = self._get_domain_variants(host)+ return host_variant not in prohibited_domains and host_alt not in prohibited_domains+ else:+ # Slow path: O(n) pattern matching for lists+ for pattern in prohibited_domains:+ if self._is_url_match(url, host, parsed.scheme, pattern):+ return False+ return Truereturn Truediff --git a/docs/customize/browser/all-parameters.mdx b/docs/customize/browser/all-parameters.mdxindex f1401df71..deb253ace 100644--- a/docs/customize/browser/all-parameters.mdx+++ b/docs/customize/browser/all-parameters.mdx@@ -32,9 +32,11 @@ The `Browser` instance also provides all [Actor](/customize/actor/all-parameters- `'chrome-extension://*'` - Matches any Chrome extension URL- **Security**: Wildcards in TLD (e.g., `example.*`) are **not allowed** for security- Use list like `['*.google.com', 'https://example.com', 'chrome-extension://*']`+ - **Performance**: Lists with 100+ domains are automatically optimized to sets for O(1) lookup. Pattern matching is disabled for optimized lists. Both `www.example.com` and `example.com` variants are checked automatically.- `prohibited_domains`: Block navigation to specific domains. Uses same pattern formats as `allowed_domains`. When both `allowed_domains` and `prohibited_domains` are set, `allowed_domains` takes precedence. Examples:- `['pornhub.com', '*.gambling-site.net']` - Block specific sites and all subdomains- `['https://explicit-content.org']` - Block specific protocol/domain combination+ - **Performance**: Lists with 100+ domains are automatically optimized to sets for O(1) lookup (same as `allowed_domains`)- `enable_default_extensions` (default: `True`): Load automation extensions (uBlock Origin, cookie handlers, ClearURLs)- `cross_origin_iframes` (default: `False`): Enable cross-origin iframe support (may cause complexity)- `is_local` (default: `True`): Whether this is a local browser instance. Set to `False` for remote browsers. If we have a `executable_path` set, it will be automatically set to `True`. This can effect your download behavior.diff --git a/examples/features/blocked_domains.py b/examples/features/blocked_domains.pyindex a6f9e0738..a0cfb0640 100644--- a/examples/features/blocked_domains.py+++ b/examples/features/blocked_domains.py@@ -20,12 +20,18 @@ prohibited_domains = ['x.com', # Block X (formerly Twitter) - "locked the f in"'twitter.com', # Block Twitter (redirects to x.com anyway)'facebook.com', # Lock the F in Facebook too- '*.meta.com', # Block all Meta properties+ '*.meta.com', # Block all Meta properties (wildcard pattern)'*.adult-site.com', # Block all subdomains of adult sites'https://explicit-content.org', # Block specific protocol/domain'gambling-site.net', # Block gambling sites]+# Note: For lists with 100+ domains, automatic optimization kicks in:+# - Converts list to set for O(1) lookup (blazingly fast!)+# - Pattern matching (*.domain) is disabled for large lists+# - Both www.example.com and example.com variants are checked automatically+# Perfect for ad blockers or large malware domain lists (e.g., 400k+ domains)+browser_session = BrowserSession(browser_profile=BrowserProfile(prohibited_domains=prohibited_domains,diff --git a/tests/ci/test_browser_watchdog_security2.py b/tests/ci/test_browser_watchdog_security2.pyindex b283d1459..2a5bc29f1 100644--- a/tests/ci/test_browser_watchdog_security2.py+++ b/tests/ci/test_browser_watchdog_security2.py@@ -433,3 +433,137 @@ class TestUrlProhibitlistSecurity:assert watchdog._is_url_allowed('https://example.com') is Falseassert watchdog._is_url_allowed('https://WWW.EXAMPLE.COM') is Falseassert watchdog._is_url_allowed('https://mail.example.com') is True+++class TestDomainListOptimization:+ """Tests for domain list optimization (set conversion for large lists)."""++ def test_small_list_keeps_pattern_support(self):+ """Test that lists < 100 items keep pattern matching support."""+ from bubus import EventBus++ from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog++ browser_profile = BrowserProfile(+ prohibited_domains=['*.google.com', 'x.com', 'facebook.com'], headless=True, user_data_dir=None+ )+ browser_session = BrowserSession(browser_profile=browser_profile)+ event_bus = EventBus()+ watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)++ # Should still be a list+ assert isinstance(browser_session.browser_profile.prohibited_domains, list)++ # Pattern matching should work+ assert watchdog._is_url_allowed('https://www.google.com') is False+ assert watchdog._is_url_allowed('https://mail.google.com') is False+ assert watchdog._is_url_allowed('https://google.com') is False++ # Exact matches should work+ assert watchdog._is_url_allowed('https://x.com') is False+ assert watchdog._is_url_allowed('https://facebook.com') is False++ # Other domains should be allowed+ assert watchdog._is_url_allowed('https://example.com') is True++ def test_large_list_converts_to_set(self):+ """Test that lists >= 100 items are converted to sets."""+ from bubus import EventBus++ from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog++ # Create a list of 100 domains+ large_list = [f'blocked{i}.com' for i in range(100)]++ browser_profile = BrowserProfile(prohibited_domains=large_list, headless=True, user_data_dir=None)+ browser_session = BrowserSession(browser_profile=browser_profile)+ event_bus = EventBus()+ watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)++ # Should be converted to set+ assert isinstance(browser_session.browser_profile.prohibited_domains, set)+ assert len(browser_session.browser_profile.prohibited_domains) == 100++ # Exact matches should work+ assert watchdog._is_url_allowed('https://blocked0.com') is False+ assert watchdog._is_url_allowed('https://blocked50.com') is False+ assert watchdog._is_url_allowed('https://blocked99.com') is False++ # Other domains should be allowed+ assert watchdog._is_url_allowed('https://example.com') is True+ assert watchdog._is_url_allowed('https://blocked100.com') is True # Not in list++ def test_www_variant_matching_with_sets(self):+ """Test that www variants are checked in set-based lookups."""+ from bubus import EventBus++ from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog++ # Create a list with 100 domains (some with www, some without)+ large_list = [f'site{i}.com' for i in range(50)] + [f'www.domain{i}.org' for i in range(50)]++ browser_profile = BrowserProfile(prohibited_domains=large_list, headless=True, user_data_dir=None)+ browser_session = BrowserSession(browser_profile=browser_profile)+ event_bus = EventBus()+ watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)++ # Should be converted to set+ assert isinstance(browser_session.browser_profile.prohibited_domains, set)++ # Test www variant matching for domains without www prefix+ assert watchdog._is_url_allowed('https://site0.com') is False+ assert watchdog._is_url_allowed('https://www.site0.com') is False # Should also be blocked++ # Test www variant matching for domains with www prefix+ assert watchdog._is_url_allowed('https://www.domain0.org') is False+ assert watchdog._is_url_allowed('https://domain0.org') is False # Should also be blocked++ # Test that unrelated domains are allowed+ assert watchdog._is_url_allowed('https://example.com') is True+ assert watchdog._is_url_allowed('https://www.example.com') is True++ def test_allowed_domains_with_sets(self):+ """Test that allowed_domains also works with set optimization."""+ from bubus import EventBus++ from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog++ # Create a large allowlist+ large_list = [f'allowed{i}.com' for i in range(100)]++ browser_profile = BrowserProfile(allowed_domains=large_list, headless=True, user_data_dir=None)+ browser_session = BrowserSession(browser_profile=browser_profile)+ event_bus = EventBus()+ watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)++ # Should be converted to set+ assert isinstance(browser_session.browser_profile.allowed_domains, set)++ # Allowed domains should work+ assert watchdog._is_url_allowed('https://allowed0.com') is True+ assert watchdog._is_url_allowed('https://www.allowed0.com') is True+ assert watchdog._is_url_allowed('https://allowed99.com') is True++ # Other domains should be blocked+ assert watchdog._is_url_allowed('https://example.com') is False+ assert watchdog._is_url_allowed('https://notallowed.com') is False++ def test_manual_set_input(self):+ """Test that users can directly provide a set."""+ from bubus import EventBus++ from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog++ blocked_set = {f'blocked{i}.com' for i in range(50)}++ browser_profile = BrowserProfile(prohibited_domains=blocked_set, headless=True, user_data_dir=None)+ browser_session = BrowserSession(browser_profile=browser_profile)+ event_bus = EventBus()+ watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)++ # Should remain a set+ assert isinstance(browser_session.browser_profile.prohibited_domains, set)++ # Should work correctly+ assert watchdog._is_url_allowed('https://blocked0.com') is False+ assert watchdog._is_url_allowed('https://example.com') is True