Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Blocked domains are optimized!!!

mertunsall16457c2a5e9f
diff --git a/browser_use/browser/profile.py b/browser_use/browser/profile.py
index 513770424..044a021b4 100644
--- a/browser_use/browser/profile.py
+++ b/browser_use/browser/profile.py
@@ -13,6 +13,7 @@ from browser_use.config import CONFIG
from browser_use.utils import _log_pretty_path, logger
CHROME_DEBUG_PORT = 9242 # use a non-default port to avoid conflicts with other tools / devs using 9222
+DOMAIN_OPTIMIZATION_THRESHOLD = 100 # Convert domain lists to sets for O(1) lookup when >= this size
CHROME_DISABLED_COMPONENTS = [
# Playwright defaults: https://github.com/microsoft/playwright/blob/41008eeddd020e2dee1c540f7c0cdfa337e99637/packages/playwright-core/src/server/chromium/chromiumSwitches.ts#L76
# AcceptCHFrame,AutoExpandDetailsElement,AvoidUnnecessaryBeforeUnloadCheckSync,CertificateTransparencyComponentUpdater,DeferRendererTasksAfterInput,DestroyProfileOnBrowserClose,DialMediaRouteProvider,ExtensionManifestV2Disabled,GlobalMediaControls,HttpsUpgrades,ImprovedCookieControls,LazyFrameLoading,LensOverlay,MediaRouter,PaintHolding,ThirdPartyStoragePartitioning,Translate
@@ -563,13 +564,13 @@ class BrowserProfile(BrowserConnectArgs, BrowserLaunchPersistentContextArgs, Bro
# custom options we provide that aren't native playwright kwargs
disable_security: bool = Field(default=False, description='Disable browser security features.')
deterministic_rendering: bool = Field(default=False, description='Enable deterministic rendering flags.')
- allowed_domains: list[str] | None = Field(
+ allowed_domains: list[str] | set[str] | None = Field(
default=None,
- description='List of allowed domains for navigation e.g. ["*.google.com", "https://example.com", "chrome-extension://*"]',
+ description='List of allowed domains for navigation e.g. ["*.google.com", "https://example.com", "chrome-extension://*"]. Lists with 100+ items are auto-optimized to sets (no pattern matching).',
)
- prohibited_domains: list[str] | None = Field(
+ prohibited_domains: list[str] | set[str] | None = Field(
default=None,
- description='List of prohibited domains for navigation e.g. ["*.google.com", "https://example.com", "chrome-extension://*"]. Allowed domains take precedence over prohibited domains.',
+ description='List of prohibited domains for navigation e.g. ["*.google.com", "https://example.com", "chrome-extension://*"]. Allowed domains take precedence over prohibited domains. Lists with 100+ items are auto-optimized to sets (no pattern matching).',
)
keep_alive: bool | None = Field(default=None, description='Keep browser alive after agent run.')
@@ -665,6 +666,23 @@ class BrowserProfile(BrowserConnectArgs, BrowserLaunchPersistentContextArgs, Bro
def __str__(self) -> str:
return 'BrowserProfile'
+ @field_validator('allowed_domains', 'prohibited_domains', mode='after')
+ @classmethod
+ def optimize_large_domain_lists(cls, v: list[str] | set[str] | None) -> list[str] | set[str] | None:
+ """Convert large domain lists (>=100 items) to sets for O(1) lookup performance."""
+ if v is None or isinstance(v, set):
+ return v
+
+ if len(v) >= DOMAIN_OPTIMIZATION_THRESHOLD:
+ logger.warning(
+ f'🔧 Optimizing domain list with {len(v)} items to set for O(1) lookup. '
+ f'Note: Pattern matching (*.domain.com, etc.) is not supported for lists >= {DOMAIN_OPTIMIZATION_THRESHOLD} items. '
+ f'Use exact domains only or keep list size < {DOMAIN_OPTIMIZATION_THRESHOLD} for pattern support.'
+ )
+ return set(v)
+
+ return v
+
@model_validator(mode='after')
def copy_old_config_names_to_new(self) -> Self:
"""Copy old config window_width & window_height to window_size."""
diff --git a/browser_use/browser/watchdogs/security_watchdog.py b/browser_use/browser/watchdogs/security_watchdog.py
index c9f7caad7..ad00775fc 100644
--- a/browser_use/browser/watchdogs/security_watchdog.py
+++ b/browser_use/browser/watchdogs/security_watchdog.py
@@ -118,6 +118,22 @@ class SecurityWatchdog(BaseWatchdog):
'Note: Patterns like "*.example.com" will match both subdomains AND the main domain.'
)
+ def _get_domain_variants(self, host: str) -> tuple[str, str]:
+ """Get both variants of a domain (with and without www prefix).
+
+ Args:
+ host: The hostname to process
+
+ Returns:
+ Tuple of (original_host, variant_host)
+ - If host starts with www., variant is without www.
+ - Otherwise, variant is with www. prefix
+ """
+ if host.startswith('www.'):
+ return (host, host[4:]) # ('www.example.com', 'example.com')
+ else:
+ return (host, f'www.{host}') # ('example.com', 'www.example.com')
+
def _is_url_allowed(self, url: str) -> bool:
"""Check if a URL is allowed based on the allowed_domains configuration.
@@ -153,21 +169,35 @@ class SecurityWatchdog(BaseWatchdog):
if not host:
return False
- # Check each allowed domain pattern
+ # Check allowed domains (fast path for sets, slow path for lists with patterns)
if self.browser_session.browser_profile.allowed_domains:
- for pattern in self.browser_session.browser_profile.allowed_domains:
- if self._is_url_match(url, host, parsed.scheme, pattern):
- return True
+ allowed_domains = self.browser_session.browser_profile.allowed_domains
- return False
+ if isinstance(allowed_domains, set):
+ # Fast path: O(1) exact hostname match - check both www and non-www variants
+ host_variant, host_alt = self._get_domain_variants(host)
+ return host_variant in allowed_domains or host_alt in allowed_domains
+ else:
+ # Slow path: O(n) pattern matching for lists
+ for pattern in allowed_domains:
+ if self._is_url_match(url, host, parsed.scheme, pattern):
+ return True
+ return False
- # Check each prohibited domain pattern
+ # Check prohibited domains (fast path for sets, slow path for lists with patterns)
if self.browser_session.browser_profile.prohibited_domains:
- for pattern in self.browser_session.browser_profile.prohibited_domains:
- if self._is_url_match(url, host, parsed.scheme, pattern):
- return False
+ prohibited_domains = self.browser_session.browser_profile.prohibited_domains
- return True
+ if isinstance(prohibited_domains, set):
+ # Fast path: O(1) exact hostname match - check both www and non-www variants
+ host_variant, host_alt = self._get_domain_variants(host)
+ return host_variant not in prohibited_domains and host_alt not in prohibited_domains
+ else:
+ # Slow path: O(n) pattern matching for lists
+ for pattern in prohibited_domains:
+ if self._is_url_match(url, host, parsed.scheme, pattern):
+ return False
+ return True
return True
diff --git a/docs/customize/browser/all-parameters.mdx b/docs/customize/browser/all-parameters.mdx
index f1401df71..deb253ace 100644
--- a/docs/customize/browser/all-parameters.mdx
+++ b/docs/customize/browser/all-parameters.mdx
@@ -32,9 +32,11 @@ The `Browser` instance also provides all [Actor](/customize/actor/all-parameters
- `'chrome-extension://*'` - Matches any Chrome extension URL
- **Security**: Wildcards in TLD (e.g., `example.*`) are **not allowed** for security
- Use list like `['*.google.com', 'https://example.com', 'chrome-extension://*']`
+ - **Performance**: Lists with 100+ domains are automatically optimized to sets for O(1) lookup. Pattern matching is disabled for optimized lists. Both `www.example.com` and `example.com` variants are checked automatically.
- `prohibited_domains`: Block navigation to specific domains. Uses same pattern formats as `allowed_domains`. When both `allowed_domains` and `prohibited_domains` are set, `allowed_domains` takes precedence. Examples:
- `['pornhub.com', '*.gambling-site.net']` - Block specific sites and all subdomains
- `['https://explicit-content.org']` - Block specific protocol/domain combination
+ - **Performance**: Lists with 100+ domains are automatically optimized to sets for O(1) lookup (same as `allowed_domains`)
- `enable_default_extensions` (default: `True`): Load automation extensions (uBlock Origin, cookie handlers, ClearURLs)
- `cross_origin_iframes` (default: `False`): Enable cross-origin iframe support (may cause complexity)
- `is_local` (default: `True`): Whether this is a local browser instance. Set to `False` for remote browsers. If we have a `executable_path` set, it will be automatically set to `True`. This can effect your download behavior.
diff --git a/examples/features/blocked_domains.py b/examples/features/blocked_domains.py
index a6f9e0738..a0cfb0640 100644
--- a/examples/features/blocked_domains.py
+++ b/examples/features/blocked_domains.py
@@ -20,12 +20,18 @@ prohibited_domains = [
'x.com', # Block X (formerly Twitter) - "locked the f in"
'twitter.com', # Block Twitter (redirects to x.com anyway)
'facebook.com', # Lock the F in Facebook too
- '*.meta.com', # Block all Meta properties
+ '*.meta.com', # Block all Meta properties (wildcard pattern)
'*.adult-site.com', # Block all subdomains of adult sites
'https://explicit-content.org', # Block specific protocol/domain
'gambling-site.net', # Block gambling sites
]
+# Note: For lists with 100+ domains, automatic optimization kicks in:
+# - Converts list to set for O(1) lookup (blazingly fast!)
+# - Pattern matching (*.domain) is disabled for large lists
+# - Both www.example.com and example.com variants are checked automatically
+# Perfect for ad blockers or large malware domain lists (e.g., 400k+ domains)
+
browser_session = BrowserSession(
browser_profile=BrowserProfile(
prohibited_domains=prohibited_domains,
diff --git a/tests/ci/test_browser_watchdog_security2.py b/tests/ci/test_browser_watchdog_security2.py
index b283d1459..2a5bc29f1 100644
--- a/tests/ci/test_browser_watchdog_security2.py
+++ b/tests/ci/test_browser_watchdog_security2.py
@@ -433,3 +433,137 @@ class TestUrlProhibitlistSecurity:
assert watchdog._is_url_allowed('https://example.com') is False
assert watchdog._is_url_allowed('https://WWW.EXAMPLE.COM') is False
assert watchdog._is_url_allowed('https://mail.example.com') is True
+
+
+class TestDomainListOptimization:
+ """Tests for domain list optimization (set conversion for large lists)."""
+
+ def test_small_list_keeps_pattern_support(self):
+ """Test that lists < 100 items keep pattern matching support."""
+ from bubus import EventBus
+
+ from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+ browser_profile = BrowserProfile(
+ prohibited_domains=['*.google.com', 'x.com', 'facebook.com'], headless=True, user_data_dir=None
+ )
+ browser_session = BrowserSession(browser_profile=browser_profile)
+ event_bus = EventBus()
+ watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+ # Should still be a list
+ assert isinstance(browser_session.browser_profile.prohibited_domains, list)
+
+ # Pattern matching should work
+ assert watchdog._is_url_allowed('https://www.google.com') is False
+ assert watchdog._is_url_allowed('https://mail.google.com') is False
+ assert watchdog._is_url_allowed('https://google.com') is False
+
+ # Exact matches should work
+ assert watchdog._is_url_allowed('https://x.com') is False
+ assert watchdog._is_url_allowed('https://facebook.com') is False
+
+ # Other domains should be allowed
+ assert watchdog._is_url_allowed('https://example.com') is True
+
+ def test_large_list_converts_to_set(self):
+ """Test that lists >= 100 items are converted to sets."""
+ from bubus import EventBus
+
+ from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+ # Create a list of 100 domains
+ large_list = [f'blocked{i}.com' for i in range(100)]
+
+ browser_profile = BrowserProfile(prohibited_domains=large_list, headless=True, user_data_dir=None)
+ browser_session = BrowserSession(browser_profile=browser_profile)
+ event_bus = EventBus()
+ watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+ # Should be converted to set
+ assert isinstance(browser_session.browser_profile.prohibited_domains, set)
+ assert len(browser_session.browser_profile.prohibited_domains) == 100
+
+ # Exact matches should work
+ assert watchdog._is_url_allowed('https://blocked0.com') is False
+ assert watchdog._is_url_allowed('https://blocked50.com') is False
+ assert watchdog._is_url_allowed('https://blocked99.com') is False
+
+ # Other domains should be allowed
+ assert watchdog._is_url_allowed('https://example.com') is True
+ assert watchdog._is_url_allowed('https://blocked100.com') is True # Not in list
+
+ def test_www_variant_matching_with_sets(self):
+ """Test that www variants are checked in set-based lookups."""
+ from bubus import EventBus
+
+ from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+ # Create a list with 100 domains (some with www, some without)
+ large_list = [f'site{i}.com' for i in range(50)] + [f'www.domain{i}.org' for i in range(50)]
+
+ browser_profile = BrowserProfile(prohibited_domains=large_list, headless=True, user_data_dir=None)
+ browser_session = BrowserSession(browser_profile=browser_profile)
+ event_bus = EventBus()
+ watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+ # Should be converted to set
+ assert isinstance(browser_session.browser_profile.prohibited_domains, set)
+
+ # Test www variant matching for domains without www prefix
+ assert watchdog._is_url_allowed('https://site0.com') is False
+ assert watchdog._is_url_allowed('https://www.site0.com') is False # Should also be blocked
+
+ # Test www variant matching for domains with www prefix
+ assert watchdog._is_url_allowed('https://www.domain0.org') is False
+ assert watchdog._is_url_allowed('https://domain0.org') is False # Should also be blocked
+
+ # Test that unrelated domains are allowed
+ assert watchdog._is_url_allowed('https://example.com') is True
+ assert watchdog._is_url_allowed('https://www.example.com') is True
+
+ def test_allowed_domains_with_sets(self):
+ """Test that allowed_domains also works with set optimization."""
+ from bubus import EventBus
+
+ from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+ # Create a large allowlist
+ large_list = [f'allowed{i}.com' for i in range(100)]
+
+ browser_profile = BrowserProfile(allowed_domains=large_list, headless=True, user_data_dir=None)
+ browser_session = BrowserSession(browser_profile=browser_profile)
+ event_bus = EventBus()
+ watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+ # Should be converted to set
+ assert isinstance(browser_session.browser_profile.allowed_domains, set)
+
+ # Allowed domains should work
+ assert watchdog._is_url_allowed('https://allowed0.com') is True
+ assert watchdog._is_url_allowed('https://www.allowed0.com') is True
+ assert watchdog._is_url_allowed('https://allowed99.com') is True
+
+ # Other domains should be blocked
+ assert watchdog._is_url_allowed('https://example.com') is False
+ assert watchdog._is_url_allowed('https://notallowed.com') is False
+
+ def test_manual_set_input(self):
+ """Test that users can directly provide a set."""
+ from bubus import EventBus
+
+ from browser_use.browser.watchdogs.security_watchdog import SecurityWatchdog
+
+ blocked_set = {f'blocked{i}.com' for i in range(50)}
+
+ browser_profile = BrowserProfile(prohibited_domains=blocked_set, headless=True, user_data_dir=None)
+ browser_session = BrowserSession(browser_profile=browser_profile)
+ event_bus = EventBus()
+ watchdog = SecurityWatchdog(browser_session=browser_session, event_bus=event_bus)
+
+ # Should remain a set
+ assert isinstance(browser_session.browser_profile.prohibited_domains, set)
+
+ # Should work correctly
+ assert watchdog._is_url_allowed('https://blocked0.com') is False
+ assert watchdog._is_url_allowed('https://example.com') is True