Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.
Python ai-agentsai-toolsbrowser-agentbrowser-automationbrowser-usecdpcloud-browsercomputer-useheadless-browserllmmcpplaywrightpythonremote-browserstealth-browserweb-automationweb-scraping
Fix download monitoring issues with fetch requests and blob downloads
Sandeep Salwan960ab9b58faf
diff --git a/browser_use/browser/watchdogs/downloads_watchdog.py b/browser_use/browser/watchdogs/downloads_watchdog.pyindex d44f51ff0..51bb0a49f 100644--- a/browser_use/browser/watchdogs/downloads_watchdog.py+++ b/browser_use/browser/watchdogs/downloads_watchdog.py@@ -56,8 +56,8 @@ class DownloadsWatchdog(BaseWatchdog):_download_cdp_session: Any = PrivateAttr(default=None) # Store CDP session reference_cdp_event_tasks: set[asyncio.Task] = PrivateAttr(default_factory=set) # Track CDP event handler tasks_cdp_downloads_info: dict[str, dict[str, Any]] = PrivateAttr(default_factory=dict) # Map guid -> info- _use_js_fetch_for_local: bool = PrivateAttr(default=False) # Guard JS fetch path for local regular downloads_session_pdf_urls: dict[str, str] = PrivateAttr(default_factory=dict) # URL -> path for PDFs downloaded this session+ _initial_downloads_snapshot: set[str] = PrivateAttr(default_factory=set) # Files present when watchdog started_network_monitored_targets: set[str] = PrivateAttr(default_factory=set) # Track targets with network monitoring enabled_detected_downloads: set[str] = PrivateAttr(default_factory=set) # Track detected download URLs to avoid duplicates_network_callback_registered: bool = PrivateAttr(default=False) # Track if global network callback is registered@@ -71,6 +71,15 @@ class DownloadsWatchdog(BaseWatchdog):expanded_path.mkdir(parents=True, exist_ok=True)self.logger.debug(f'[DownloadsWatchdog] Ensured downloads directory exists: {expanded_path}')+ # Capture initial files to detect new downloads reliably+ if expanded_path.exists():+ for f in expanded_path.iterdir():+ if f.is_file() and not f.name.startswith('.'):+ self._initial_downloads_snapshot.add(f.name)+ self.logger.debug(+ f'[DownloadsWatchdog] Captured initial downloads: {len(self._initial_downloads_snapshot)} files'+ )+async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:"""Monitor new tabs for downloads."""# logger.info(f'[DownloadsWatchdog] TabCreatedEvent received for tab {event.target_id[-4:]}: {event.url}')@@ -131,6 +140,7 @@ class DownloadsWatchdog(BaseWatchdog):self._session_pdf_urls.clear()self._network_monitored_targets.clear()self._detected_downloads.clear()+ self._initial_downloads_snapshot.clear()self._network_callback_registered = Falseasync def on_NavigationCompleteEvent(self, event: NavigationCompleteEvent) -> None:@@ -203,10 +213,31 @@ class DownloadsWatchdog(BaseWatchdog):except (KeyError, AttributeError):passelse:- # No local file path provided, local polling in _handle_cdp_download will handle it- self.logger.debug(- '[DownloadsWatchdog] No filePath in progress event (local); polling will handle detection'- )+ # No filePath provided - detect by comparing with initial snapshot+ self.logger.debug('[DownloadsWatchdog] No filePath in progress event; detecting via filesystem')+ downloads_path = self.browser_session.browser_profile.downloads_path+ if downloads_path:+ downloads_dir = Path(downloads_path).expanduser().resolve()+ if downloads_dir.exists():+ for f in downloads_dir.iterdir():+ if (+ f.is_file()+ and not f.name.startswith('.')+ and f.name not in self._initial_downloads_snapshot+ ):+ # Check file has content before processing+ if f.stat().st_size > 4:+ # Found a new file! Add to snapshot immediately to prevent duplicate detection+ self._initial_downloads_snapshot.add(f.name)+ self.logger.debug(f'[DownloadsWatchdog] Detected new download: {f.name}')+ self._track_download(str(f))+ # Mark as handled+ try:+ if guid in self._cdp_downloads_info:+ self._cdp_downloads_info[guid]['handled'] = True+ except (KeyError, AttributeError):+ pass+ breakelse:# Remote browser: do not touch local filesystem. Fallback to downloadPath+suggestedFilenameinfo = self._cdp_downloads_info.get(guid, {})@@ -327,17 +358,24 @@ class DownloadsWatchdog(BaseWatchdog):response = event.get('response', {})url = response.get('url', '')content_type = response.get('mimeType', '').lower()- headers = response.get('headers', {})+ headers = {+ k.lower(): v for k, v in response.get('headers', {}).items()+ } # Normalize for case-insensitive lookup+ request_type = event.get('type', '')# Skip non-HTTP URLs (data:, about:, chrome-extension:, etc.)if not url.startswith('http'):return+ # Skip fetch/XHR - real browsers don't download PDFs from programmatic requests+ if request_type in ('Fetch', 'XHR'):+ return+# Check if it's a PDFis_pdf = 'application/pdf' in content_type# Check if it's marked as download via Content-Disposition header- content_disposition = headers.get('content-disposition', '').lower()+ content_disposition = str(headers.get('content-disposition', '')).lower()is_download_attachment = 'attachment' in content_disposition# Filter out image/video/audio files even if marked as attachment@@ -424,6 +462,8 @@ class DownloadsWatchdog(BaseWatchdog):if download_path:self.logger.info(f'[DownloadsWatchdog] ✅ Successfully downloaded: {download_path}')+ # Clean up from detected downloads set after success+ self._detected_downloads.discard(url)else:self.logger.warning(f'[DownloadsWatchdog] ⚠️ Failed to download: {url[:80]}...')except Exception as e:@@ -659,106 +699,6 @@ class DownloadsWatchdog(BaseWatchdog):# We just need to wait for it to appear in the downloads directoryexpected_path = downloads_dir / suggested_filename- # Debug: List current directory contents- self.logger.debug(f'[DownloadsWatchdog] Downloads directory: {downloads_dir}')- if downloads_dir.exists():- files_before = list(downloads_dir.iterdir())- self.logger.debug(f'[DownloadsWatchdog] Files before download: {[f.name for f in files_before]}')-- # Try manual JavaScript fetch as a fallback for local browsers (disabled for regular local downloads)- if self.browser_session.is_local and self._use_js_fetch_for_local:- self.logger.debug(f'[DownloadsWatchdog] Attempting JS fetch fallback for {download_url}')-- unique_filename = None- file_size = None- download_result = None- try:- # Escape the URL for JavaScript- import json-- escaped_url = json.dumps(download_url)-- # Get the proper session for the frame that initiated the download- cdp_session = await self.browser_session.cdp_client_for_frame(event.get('frameId'))- assert cdp_session-- result = await cdp_session.cdp_client.send.Runtime.evaluate(- params={- 'expression': f"""- (async () => {{- try {{- const response = await fetch({escaped_url});- if (!response.ok) {{- throw new Error(`HTTP error! status: ${{response.status}}`);- }}- const blob = await response.blob();- const arrayBuffer = await blob.arrayBuffer();- const uint8Array = new Uint8Array(arrayBuffer);- return {{- data: Array.from(uint8Array),- size: uint8Array.length,- contentType: response.headers.get('content-type') || 'application/octet-stream'- }};- }} catch (error) {{- throw new Error(`Fetch failed: ${{error.message}}`);- }}- }})()- """,- 'awaitPromise': True,- 'returnByValue': True,- },- session_id=cdp_session.session_id,- )- download_result = result.get('result', {}).get('value')-- if download_result and download_result.get('data'):- # Save the file- file_data = bytes(download_result['data'])- file_size = len(file_data)-- # Ensure unique filename- unique_filename = await self._get_unique_filename(str(downloads_dir), suggested_filename)- final_path = downloads_dir / unique_filename-- # Write the file- import anyio-- async with await anyio.open_file(final_path, 'wb') as f:- await f.write(file_data)-- self.logger.debug(f'[DownloadsWatchdog] ✅ Downloaded and saved file: {final_path} ({file_size} bytes)')- expected_path = final_path- # Emit download event immediately- file_ext = expected_path.suffix.lower().lstrip('.')- file_type = file_ext if file_ext else None- self.event_bus.dispatch(- FileDownloadedEvent(- url=download_url,- path=str(expected_path),- file_name=unique_filename or expected_path.name,- file_size=file_size or 0,- file_type=file_type,- mime_type=(download_result.get('contentType') if download_result else None),- from_cache=False,- auto_download=False,- )- )- # Mark as handled to prevent duplicate dispatch from progress/polling paths- try:- if guid in self._cdp_downloads_info:- self._cdp_downloads_info[guid]['handled'] = True- except (KeyError, AttributeError):- pass- self.logger.debug(- f'[DownloadsWatchdog] ✅ File download completed via CDP: {suggested_filename} ({file_size} bytes) saved to {expected_path}'- )- return- else:- self.logger.error('[DownloadsWatchdog] ❌ No data received from fetch')-- except Exception as fetch_error:- self.logger.error(f'[DownloadsWatchdog] ❌ Failed to download file via fetch: {fetch_error}')-# For remote browsers, don't poll local filesystem; downloadProgress handler will emit the eventif not self.browser_session.is_local:return@@ -769,24 +709,23 @@ class DownloadsWatchdog(BaseWatchdog):# Poll the downloads directory for new filesself.logger.debug(f'[DownloadsWatchdog] Checking if browser auto-download saved the file for us: {suggested_filename}')- # Get initial list of files in downloads directory- initial_files = set()- if Path(downloads_dir).exists():- for f in Path(downloads_dir).iterdir():- if f.is_file() and not f.name.startswith('.'):- initial_files.add(f.name)-# Poll for new filesmax_wait = 20 # secondsstart_time = asyncio.get_event_loop().time()- while asyncio.get_event_loop().time() - start_time < max_wait:+ while asyncio.get_event_loop().time() - start_time < max_wait: # noqa: ASYNC110await asyncio.sleep(5.0) # Check every 5 secondsif Path(downloads_dir).exists():for file_path in Path(downloads_dir).iterdir():# Skip hidden files and files that were already there- if file_path.is_file() and not file_path.name.startswith('.') and file_path.name not in initial_files:+ if (+ file_path.is_file()+ and not file_path.name.startswith('.')+ and file_path.name not in self._initial_downloads_snapshot+ ):+ # Add to snapshot immediately to prevent duplicate detection+ self._initial_downloads_snapshot.add(file_path.name)# Check if file has content (> 4 bytes)try:file_size = file_path.stat().st_size@@ -814,13 +753,13 @@ class DownloadsWatchdog(BaseWatchdog):file_type=file_type,))- # Mark as handled after dispatch- try:- if guid in self._cdp_downloads_info:- self._cdp_downloads_info[guid]['handled'] = True- except (KeyError, AttributeError):- pass- return+ # Mark as handled after dispatch+ try:+ if guid in self._cdp_downloads_info:+ self._cdp_downloads_info[guid]['handled'] = True+ except (KeyError, AttributeError):+ pass+ returnexcept Exception as e:self.logger.debug(f'[DownloadsWatchdog] Error checking file {file_path}: {e}')