Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Fix download monitoring issues with fetch requests and blob downloads

Sandeep Salwan960ab9b58faf
diff --git a/browser_use/browser/watchdogs/downloads_watchdog.py b/browser_use/browser/watchdogs/downloads_watchdog.py
index d44f51ff0..51bb0a49f 100644
--- a/browser_use/browser/watchdogs/downloads_watchdog.py
+++ b/browser_use/browser/watchdogs/downloads_watchdog.py
@@ -56,8 +56,8 @@ class DownloadsWatchdog(BaseWatchdog):
_download_cdp_session: Any = PrivateAttr(default=None) # Store CDP session reference
_cdp_event_tasks: set[asyncio.Task] = PrivateAttr(default_factory=set) # Track CDP event handler tasks
_cdp_downloads_info: dict[str, dict[str, Any]] = PrivateAttr(default_factory=dict) # Map guid -> info
- _use_js_fetch_for_local: bool = PrivateAttr(default=False) # Guard JS fetch path for local regular downloads
_session_pdf_urls: dict[str, str] = PrivateAttr(default_factory=dict) # URL -> path for PDFs downloaded this session
+ _initial_downloads_snapshot: set[str] = PrivateAttr(default_factory=set) # Files present when watchdog started
_network_monitored_targets: set[str] = PrivateAttr(default_factory=set) # Track targets with network monitoring enabled
_detected_downloads: set[str] = PrivateAttr(default_factory=set) # Track detected download URLs to avoid duplicates
_network_callback_registered: bool = PrivateAttr(default=False) # Track if global network callback is registered
@@ -71,6 +71,15 @@ class DownloadsWatchdog(BaseWatchdog):
expanded_path.mkdir(parents=True, exist_ok=True)
self.logger.debug(f'[DownloadsWatchdog] Ensured downloads directory exists: {expanded_path}')
+ # Capture initial files to detect new downloads reliably
+ if expanded_path.exists():
+ for f in expanded_path.iterdir():
+ if f.is_file() and not f.name.startswith('.'):
+ self._initial_downloads_snapshot.add(f.name)
+ self.logger.debug(
+ f'[DownloadsWatchdog] Captured initial downloads: {len(self._initial_downloads_snapshot)} files'
+ )
+
async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
"""Monitor new tabs for downloads."""
# logger.info(f'[DownloadsWatchdog] TabCreatedEvent received for tab {event.target_id[-4:]}: {event.url}')
@@ -131,6 +140,7 @@ class DownloadsWatchdog(BaseWatchdog):
self._session_pdf_urls.clear()
self._network_monitored_targets.clear()
self._detected_downloads.clear()
+ self._initial_downloads_snapshot.clear()
self._network_callback_registered = False
async def on_NavigationCompleteEvent(self, event: NavigationCompleteEvent) -> None:
@@ -203,10 +213,31 @@ class DownloadsWatchdog(BaseWatchdog):
except (KeyError, AttributeError):
pass
else:
- # No local file path provided, local polling in _handle_cdp_download will handle it
- self.logger.debug(
- '[DownloadsWatchdog] No filePath in progress event (local); polling will handle detection'
- )
+ # No filePath provided - detect by comparing with initial snapshot
+ self.logger.debug('[DownloadsWatchdog] No filePath in progress event; detecting via filesystem')
+ downloads_path = self.browser_session.browser_profile.downloads_path
+ if downloads_path:
+ downloads_dir = Path(downloads_path).expanduser().resolve()
+ if downloads_dir.exists():
+ for f in downloads_dir.iterdir():
+ if (
+ f.is_file()
+ and not f.name.startswith('.')
+ and f.name not in self._initial_downloads_snapshot
+ ):
+ # Check file has content before processing
+ if f.stat().st_size > 4:
+ # Found a new file! Add to snapshot immediately to prevent duplicate detection
+ self._initial_downloads_snapshot.add(f.name)
+ self.logger.debug(f'[DownloadsWatchdog] Detected new download: {f.name}')
+ self._track_download(str(f))
+ # Mark as handled
+ try:
+ if guid in self._cdp_downloads_info:
+ self._cdp_downloads_info[guid]['handled'] = True
+ except (KeyError, AttributeError):
+ pass
+ break
else:
# Remote browser: do not touch local filesystem. Fallback to downloadPath+suggestedFilename
info = self._cdp_downloads_info.get(guid, {})
@@ -327,17 +358,24 @@ class DownloadsWatchdog(BaseWatchdog):
response = event.get('response', {})
url = response.get('url', '')
content_type = response.get('mimeType', '').lower()
- headers = response.get('headers', {})
+ headers = {
+ k.lower(): v for k, v in response.get('headers', {}).items()
+ } # Normalize for case-insensitive lookup
+ request_type = event.get('type', '')
# Skip non-HTTP URLs (data:, about:, chrome-extension:, etc.)
if not url.startswith('http'):
return
+ # Skip fetch/XHR - real browsers don't download PDFs from programmatic requests
+ if request_type in ('Fetch', 'XHR'):
+ return
+
# Check if it's a PDF
is_pdf = 'application/pdf' in content_type
# Check if it's marked as download via Content-Disposition header
- content_disposition = headers.get('content-disposition', '').lower()
+ content_disposition = str(headers.get('content-disposition', '')).lower()
is_download_attachment = 'attachment' in content_disposition
# Filter out image/video/audio files even if marked as attachment
@@ -424,6 +462,8 @@ class DownloadsWatchdog(BaseWatchdog):
if download_path:
self.logger.info(f'[DownloadsWatchdog] ✅ Successfully downloaded: {download_path}')
+ # Clean up from detected downloads set after success
+ self._detected_downloads.discard(url)
else:
self.logger.warning(f'[DownloadsWatchdog] ⚠️ Failed to download: {url[:80]}...')
except Exception as e:
@@ -659,106 +699,6 @@ class DownloadsWatchdog(BaseWatchdog):
# We just need to wait for it to appear in the downloads directory
expected_path = downloads_dir / suggested_filename
- # Debug: List current directory contents
- self.logger.debug(f'[DownloadsWatchdog] Downloads directory: {downloads_dir}')
- if downloads_dir.exists():
- files_before = list(downloads_dir.iterdir())
- self.logger.debug(f'[DownloadsWatchdog] Files before download: {[f.name for f in files_before]}')
-
- # Try manual JavaScript fetch as a fallback for local browsers (disabled for regular local downloads)
- if self.browser_session.is_local and self._use_js_fetch_for_local:
- self.logger.debug(f'[DownloadsWatchdog] Attempting JS fetch fallback for {download_url}')
-
- unique_filename = None
- file_size = None
- download_result = None
- try:
- # Escape the URL for JavaScript
- import json
-
- escaped_url = json.dumps(download_url)
-
- # Get the proper session for the frame that initiated the download
- cdp_session = await self.browser_session.cdp_client_for_frame(event.get('frameId'))
- assert cdp_session
-
- result = await cdp_session.cdp_client.send.Runtime.evaluate(
- params={
- 'expression': f"""
- (async () => {{
- try {{
- const response = await fetch({escaped_url});
- if (!response.ok) {{
- throw new Error(`HTTP error! status: ${{response.status}}`);
- }}
- const blob = await response.blob();
- const arrayBuffer = await blob.arrayBuffer();
- const uint8Array = new Uint8Array(arrayBuffer);
- return {{
- data: Array.from(uint8Array),
- size: uint8Array.length,
- contentType: response.headers.get('content-type') || 'application/octet-stream'
- }};
- }} catch (error) {{
- throw new Error(`Fetch failed: ${{error.message}}`);
- }}
- }})()
- """,
- 'awaitPromise': True,
- 'returnByValue': True,
- },
- session_id=cdp_session.session_id,
- )
- download_result = result.get('result', {}).get('value')
-
- if download_result and download_result.get('data'):
- # Save the file
- file_data = bytes(download_result['data'])
- file_size = len(file_data)
-
- # Ensure unique filename
- unique_filename = await self._get_unique_filename(str(downloads_dir), suggested_filename)
- final_path = downloads_dir / unique_filename
-
- # Write the file
- import anyio
-
- async with await anyio.open_file(final_path, 'wb') as f:
- await f.write(file_data)
-
- self.logger.debug(f'[DownloadsWatchdog] ✅ Downloaded and saved file: {final_path} ({file_size} bytes)')
- expected_path = final_path
- # Emit download event immediately
- file_ext = expected_path.suffix.lower().lstrip('.')
- file_type = file_ext if file_ext else None
- self.event_bus.dispatch(
- FileDownloadedEvent(
- url=download_url,
- path=str(expected_path),
- file_name=unique_filename or expected_path.name,
- file_size=file_size or 0,
- file_type=file_type,
- mime_type=(download_result.get('contentType') if download_result else None),
- from_cache=False,
- auto_download=False,
- )
- )
- # Mark as handled to prevent duplicate dispatch from progress/polling paths
- try:
- if guid in self._cdp_downloads_info:
- self._cdp_downloads_info[guid]['handled'] = True
- except (KeyError, AttributeError):
- pass
- self.logger.debug(
- f'[DownloadsWatchdog] ✅ File download completed via CDP: {suggested_filename} ({file_size} bytes) saved to {expected_path}'
- )
- return
- else:
- self.logger.error('[DownloadsWatchdog] ❌ No data received from fetch')
-
- except Exception as fetch_error:
- self.logger.error(f'[DownloadsWatchdog] ❌ Failed to download file via fetch: {fetch_error}')
-
# For remote browsers, don't poll local filesystem; downloadProgress handler will emit the event
if not self.browser_session.is_local:
return
@@ -769,24 +709,23 @@ class DownloadsWatchdog(BaseWatchdog):
# Poll the downloads directory for new files
self.logger.debug(f'[DownloadsWatchdog] Checking if browser auto-download saved the file for us: {suggested_filename}')
- # Get initial list of files in downloads directory
- initial_files = set()
- if Path(downloads_dir).exists():
- for f in Path(downloads_dir).iterdir():
- if f.is_file() and not f.name.startswith('.'):
- initial_files.add(f.name)
-
# Poll for new files
max_wait = 20 # seconds
start_time = asyncio.get_event_loop().time()
- while asyncio.get_event_loop().time() - start_time < max_wait:
+ while asyncio.get_event_loop().time() - start_time < max_wait: # noqa: ASYNC110
await asyncio.sleep(5.0) # Check every 5 seconds
if Path(downloads_dir).exists():
for file_path in Path(downloads_dir).iterdir():
# Skip hidden files and files that were already there
- if file_path.is_file() and not file_path.name.startswith('.') and file_path.name not in initial_files:
+ if (
+ file_path.is_file()
+ and not file_path.name.startswith('.')
+ and file_path.name not in self._initial_downloads_snapshot
+ ):
+ # Add to snapshot immediately to prevent duplicate detection
+ self._initial_downloads_snapshot.add(file_path.name)
# Check if file has content (> 4 bytes)
try:
file_size = file_path.stat().st_size
@@ -814,13 +753,13 @@ class DownloadsWatchdog(BaseWatchdog):
file_type=file_type,
)
)
- # Mark as handled after dispatch
- try:
- if guid in self._cdp_downloads_info:
- self._cdp_downloads_info[guid]['handled'] = True
- except (KeyError, AttributeError):
- pass
- return
+ # Mark as handled after dispatch
+ try:
+ if guid in self._cdp_downloads_info:
+ self._cdp_downloads_info[guid]['handled'] = True
+ except (KeyError, AttributeError):
+ pass
+ return
except Exception as e:
self.logger.debug(f'[DownloadsWatchdog] Error checking file {file_path}: {e}')