Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

fix cdp logging

Nick Sweeting645742fb17c3
diff --git a/browser_use/browser/session.py b/browser_use/browser/session.py
index 631298a20..3c4b32db2 100644
--- a/browser_use/browser/session.py
+++ b/browser_use/browser/session.py
@@ -14,17 +14,8 @@ from cdp_use.cdp.network import Cookie
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
from uuid_extensions import uuid7str
-# Note: CDP logging level is controlled by the CLI when needed
-# By default we silence it unless explicitly enabled
-if os.environ.get('BROWSER_USE_CDP_DEBUG') != 'true':
- # Silence cdp_use logging - it's too verbose at DEBUG level
- cdp_use.client.logger.setLevel(logging.ERROR)
-
- # Also configure all cdp_use loggers
- for logger_name in ['cdp_use', 'cdp_use.client', 'cdp_use.cdp', 'cdp_use.cdp.registry']:
- cdp_logger = logging.getLogger(logger_name)
- cdp_logger.setLevel(logging.ERROR)
- cdp_logger.propagate = False # Don't propagate to root logger
+# CDP logging is now handled by setup_logging() in logging_config.py
+# It automatically sets CDP logs to the same level as browser_use logs
from browser_use.browser.events import (
AgentFocusChangedEvent,
@@ -1280,6 +1271,12 @@ class BrowserSession(BaseModel):
url_allowed, type_allowed = False, False
+ # Always allow new tab pages (chrome://new-tab-page/, chrome://newtab/, about:blank)
+ # so they can be redirected to about:blank in connect()
+ from browser_use.utils import is_new_tab_page
+ if is_new_tab_page(url):
+ url_allowed = True
+
if url.startswith('chrome-error://') and include_chrome_error:
url_allowed = True
diff --git a/browser_use/browser/storage_state_watchdog.py b/browser_use/browser/storage_state_watchdog.py
index fcbe697a5..37b3aa199 100644
--- a/browser_use/browser/storage_state_watchdog.py
+++ b/browser_use/browser/storage_state_watchdog.py
@@ -175,8 +175,9 @@ class StorageStateWatchdog(BaseWatchdog):
async def _save_storage_state(self, path: str | None = None) -> None:
"""Save browser storage state to file."""
async with self._save_lock:
- if not self.browser_session.cdp_client:
- self.logger.warning('[StorageStateWatchdog] No CDP client available for saving')
+ # Check if CDP client is available without raising an assertion error
+ if self.browser_session._cdp_client_root is None:
+ self.logger.debug('[StorageStateWatchdog] CDP client already disconnected, skipping save')
return
save_path = path or self.browser_session.browser_profile.storage_state
diff --git a/browser_use/cli.py b/browser_use/cli.py
index baebd742f..9463c3715 100644
--- a/browser_use/cli.py
+++ b/browser_use/cli.py
@@ -8,9 +8,6 @@ import time
from pathlib import Path
from typing import Any
-# Enable CDP debug logging BEFORE importing anything that uses cdp_use
-os.environ['BROWSER_USE_CDP_DEBUG'] = 'true'
-
from dotenv import load_dotenv
from browser_use.llm.anthropic.chat import ChatAnthropic
@@ -406,6 +403,7 @@ class BrowserUseApp(App):
self._telemetry = ProductTelemetry()
# Store for event bus handler
self._event_bus_handler_id = None
+ self._event_bus_handler_func = None
def setup_richlog_logging(self) -> None:
"""Set up logging to redirect to RichLog widget instead of stdout."""
@@ -659,15 +657,19 @@ class BrowserUseApp(App):
return
# Clean up any existing handler before registering a new one
- if self._event_bus_handler_id is not None:
+ if self._event_bus_handler_func is not None:
try:
# Remove handler from the event bus's internal handlers dict
if hasattr(self.browser_session.event_bus, 'handlers'):
- for event_type, handlers in list(self.browser_session.event_bus.handlers.items()):
- if self._event_bus_handler_id in handlers:
- del handlers[self._event_bus_handler_id]
- except Exception:
- pass # Handler might not exist anymore
+ # Find and remove our handler function from all event patterns
+ for event_type, handler_list in list(self.browser_session.event_bus.handlers.items()):
+ # Remove our specific handler function object
+ if self._event_bus_handler_func in handler_list:
+ handler_list.remove(self._event_bus_handler_func)
+ logging.debug(f'Removed old handler from event type: {event_type}')
+ except Exception as e:
+ logging.debug(f'Error cleaning up event bus handler: {e}')
+ self._event_bus_handler_func = None
self._event_bus_handler_id = None
try:
@@ -701,15 +703,18 @@ class BrowserUseApp(App):
except Exception as e:
events_log.write(f'[red]→ {event_name}[/] (error formatting: {e})')
+ # Store the handler function before registering it
+ self._event_bus_handler_func = log_event
+ self._event_bus_handler_id = id(log_event)
+
# Register wildcard handler for all events
- self._event_bus_handler_id = self.browser_session.event_bus.on('*', log_event)
+ self.browser_session.event_bus.on('*', log_event)
+ logging.debug(f'Registered new event bus handler with id: {self._event_bus_handler_id}')
def setup_cdp_logger(self) -> None:
- """Setup CDP message logger."""
- # CDP debug logging is enabled at module import time via environment variable
- # Just configure the loggers to use our handler
- import cdp_use.client
- cdp_use.client.logger.setLevel(logging.DEBUG)
+ """Setup CDP message logger to capture already-transformed CDP logs."""
+ # No need to configure levels - setup_logging() already handles that
+ # We just need to capture the transformed logs and route them to the CDP pane
# Get the CDP log widget
cdp_log = self.query_one('#cdp-log', RichLog)
@@ -738,15 +743,16 @@ class BrowserUseApp(App):
# Setup handler for cdp_use loggers
cdp_handler = CDPLogHandler(cdp_log)
- cdp_handler.setFormatter(logging.Formatter('%(name)s: %(message)s'))
+ cdp_handler.setFormatter(logging.Formatter('%(message)s'))
cdp_handler.setLevel(logging.DEBUG)
- # Configure cdp_use loggers to use our handler
- for logger_name in ['cdp_use', 'cdp_use.client', 'cdp_use.cdp', 'cdp_use.cdp.registry']:
- cdp_logger = logging.getLogger(logger_name)
- cdp_logger.setLevel(logging.DEBUG)
- cdp_logger.handlers = [cdp_handler] # Replace existing handlers
- cdp_logger.propagate = False
+ # Route CDP logs to the CDP pane
+ # These are already transformed by cdp_use and at the right level from setup_logging
+ for logger_name in ['websockets.client', 'cdp_use', 'cdp_use.client', 'cdp_use.cdp', 'cdp_use.cdp.registry']:
+ logger = logging.getLogger(logger_name)
+ # Add our handler (don't replace - keep existing console handler too)
+ if cdp_handler not in logger.handlers:
+ logger.addHandler(cdp_handler)
def scroll_to_input(self) -> None:
"""Scroll to the input field to ensure it's visible."""
@@ -1031,9 +1037,16 @@ async def run_prompt_mode(prompt: str, ctx: click.Context, debug: bool = False):
await agent.run()
- # Note: We don't close the browser session here because:
- # 1. The agent already called browser_session.stop() in its run() method
- # 2. This prevents duplicate "stop() called" messages in the logs
+ # Ensure the browser session is fully stopped
+ # The agent's close() method only kills the browser if keep_alive=False,
+ # but we need to ensure all background tasks are stopped regardless
+ if browser_session:
+ try:
+ # Kill the browser session to stop all background tasks
+ await browser_session.kill()
+ except Exception:
+ # Ignore errors during cleanup
+ pass
# Capture telemetry for successful completion
telemetry.capture(
@@ -1071,6 +1084,18 @@ async def run_prompt_mode(prompt: str, ctx: click.Context, debug: bool = False):
finally:
# Ensure telemetry is flushed
telemetry.flush()
+
+ # Give a brief moment for cleanup to complete
+ await asyncio.sleep(0.1)
+
+ # Cancel any remaining tasks to ensure clean exit
+ tasks = [t for t in asyncio.all_tasks() if t != asyncio.current_task()]
+ for task in tasks:
+ task.cancel()
+
+ # Wait for all tasks to be cancelled
+ if tasks:
+ await asyncio.gather(*tasks, return_exceptions=True)
async def textual_interface(config: dict[str, Any]):
diff --git a/browser_use/dom/debug/highlights.py b/browser_use/dom/debug/highlights.py
index 78a9bd26c..ef4f43055 100644
--- a/browser_use/dom/debug/highlights.py
+++ b/browser_use/dom/debug/highlights.py
@@ -60,10 +60,21 @@ async def remove_highlighting_script(dom_service: DomService) -> None:
# Create script to remove all highlights
script = """
(function() {
- // Remove any existing highlights
+ // Remove any existing highlights - be thorough
const existingHighlights = document.querySelectorAll('[data-browser-use-highlight]');
console.log('Removing', existingHighlights.length, 'browser-use highlight elements');
existingHighlights.forEach(el => el.remove());
+
+ // Also remove by ID in case selector missed anything
+ const highlightContainer = document.getElementById('browser-use-debug-highlights');
+ if (highlightContainer) {
+ console.log('Removing highlight container by ID');
+ highlightContainer.remove();
+ }
+
+ // Final cleanup - remove any orphaned tooltips
+ const orphanedTooltips = document.querySelectorAll('[data-browser-use-highlight="tooltip"]');
+ orphanedTooltips.forEach(el => el.remove());
})();
"""
@@ -90,8 +101,12 @@ async def inject_highlighting_script(dom_service: DomService, interactive_elemen
print(f'📍 Creating CSP-safe highlighting for {len(converted_elements)} elements')
- # Remove any existing highlights first
+ # ALWAYS remove any existing highlights first to prevent double-highlighting
await remove_highlighting_script(dom_service)
+
+ # Add a small delay to ensure removal completes
+ import asyncio
+ await asyncio.sleep(0.05)
# Create CSP-safe highlighting script using DOM methods instead of innerHTML
# Uses outline-only highlights with reasonable z-index to avoid blocking page content
@@ -103,6 +118,20 @@ async def inject_highlighting_script(dom_service: DomService, interactive_elemen
console.log('=== BROWSER-USE HIGHLIGHTING ===');
console.log('Highlighting', interactiveElements.length, 'interactive elements');
+ // Double-check: Remove any existing highlight container first to prevent duplicates
+ const existingContainer = document.getElementById('browser-use-debug-highlights');
+ if (existingContainer) {{
+ console.log('⚠️ Found existing highlight container, removing it first');
+ existingContainer.remove();
+ }}
+
+ // Also remove any stray highlight elements
+ const strayHighlights = document.querySelectorAll('[data-browser-use-highlight]');
+ if (strayHighlights.length > 0) {{
+ console.log('⚠️ Found', strayHighlights.length, 'stray highlight elements, removing them');
+ strayHighlights.forEach(el => el.remove());
+ }}
+
// Use a high but reasonable z-index to be visible without covering important content
// High enough for most content but not maximum to avoid blocking critical popups/modals
const HIGHLIGHT_Z_INDEX = 999999; // High but reasonable z-index
diff --git a/browser_use/logging_config.py b/browser_use/logging_config.py
index 2245565d8..d8fdd74fb 100644
--- a/browser_use/logging_config.py
+++ b/browser_use/logging_config.py
@@ -102,23 +102,42 @@ def setup_logging(stream=None, log_level=None, force_setup=False):
# Configure root logger only
root.addHandler(console)
- # switch cases for log_type
+ # Determine the log level to use
if log_type == 'result':
- root.setLevel('RESULT') # string usage to avoid syntax error
+ log_level = 'RESULT' # string usage to avoid syntax error
elif log_type == 'debug':
- root.setLevel(logging.DEBUG)
+ log_level = logging.DEBUG
else:
- root.setLevel(logging.INFO)
+ log_level = logging.INFO
+
+ # Configure root logger
+ root.setLevel(log_level)
# Configure browser_use logger
browser_use_logger = logging.getLogger('browser_use')
browser_use_logger.propagate = False # Don't propagate to root logger
browser_use_logger.addHandler(console)
- browser_use_logger.setLevel(root.level) # Set same level as root logger
+ browser_use_logger.setLevel(log_level)
+
+ # Configure CDP logging to match browser_use level
+ # websockets.client emits the logs that cdp_use transforms to appear as cdp_use.client
+ cdp_loggers = [
+ 'websockets.client', # Controls emission of transformed cdp_use.client logs
+ 'cdp_use',
+ 'cdp_use.client',
+ 'cdp_use.cdp',
+ 'cdp_use.cdp.registry',
+ ]
+ for logger_name in cdp_loggers:
+ cdp_logger = logging.getLogger(logger_name)
+ cdp_logger.setLevel(log_level) # Same level as browser_use
+ cdp_logger.addHandler(console) # Same handler as browser_use
+ cdp_logger.propagate = False
logger = logging.getLogger('browser_use')
# logger.info('BrowserUse logging setup complete with level %s', log_type)
- # Silence or adjust third-party loggers
+
+ # Silence third-party loggers (but not CDP ones which we configured above)
third_party_loggers = [
'WDM',
'httpx',
@@ -138,12 +157,7 @@ def setup_logging(stream=None, log_level=None, force_setup=False):
'groq',
'portalocker',
'portalocker.utils',
- 'cdp_use',
- 'cdp_use.client',
- 'cdp_use.cdp',
- 'cdp_use.cdp.registry',
- 'websockets',
- 'websockets.client',
+ 'websockets', # General websockets (but not websockets.client which we need)
]
for logger_name in third_party_loggers:
third_party = logging.getLogger(logger_name)