Skip to content

Open-source Python library for AI browser agents, plus Browser Use Cloud: stealth browsers and a hosted agent API.

Improve auto download for inside iframe

Magnus Müller2e6db8a13577
diff --git a/browser_use/browser/watchdogs/downloads_watchdog.py b/browser_use/browser/watchdogs/downloads_watchdog.py
index bc6f0d4c6..b85ed7968 100644
--- a/browser_use/browser/watchdogs/downloads_watchdog.py
+++ b/browser_use/browser/watchdogs/downloads_watchdog.py
@@ -11,6 +11,7 @@ from urllib.parse import urlparse
import anyio
from bubus import BaseEvent
from cdp_use.cdp.browser import DownloadProgressEvent, DownloadWillBeginEvent
+from cdp_use.cdp.network import ResponseReceivedEvent
from cdp_use.cdp.target import SessionID, TargetID
from pydantic import PrivateAttr
@@ -57,6 +58,8 @@ class DownloadsWatchdog(BaseWatchdog):
_cdp_downloads_info: dict[str, dict[str, Any]] = PrivateAttr(default_factory=dict) # Map guid -> info
_use_js_fetch_for_local: bool = PrivateAttr(default=False) # Guard JS fetch path for local regular downloads
_session_pdf_urls: dict[str, str] = PrivateAttr(default_factory=dict) # URL -> path for PDFs downloaded this session
+ _network_monitored_targets: set[str] = PrivateAttr(default_factory=set) # Track targets with network monitoring enabled
+ _detected_downloads: set[str] = PrivateAttr(default_factory=set) # Track detected download URLs to avoid duplicates
async def on_BrowserLaunchEvent(self, event: BrowserLaunchEvent) -> None:
self.logger.debug(f'[DownloadsWatchdog] Received BrowserLaunchEvent, EventBus ID: {id(self.event_bus)}')
@@ -125,6 +128,8 @@ class DownloadsWatchdog(BaseWatchdog):
self._active_downloads.clear()
self._pdf_viewer_cache.clear()
self._session_pdf_urls.clear()
+ self._network_monitored_targets.clear()
+ self._detected_downloads.clear()
async def on_NavigationCompleteEvent(self, event: NavigationCompleteEvent) -> None:
"""Check for PDFs after navigation completes."""
@@ -272,6 +277,262 @@ class DownloadsWatchdog(BaseWatchdog):
except Exception as e:
self.logger.warning(f'[DownloadsWatchdog] Failed to set up CDP download listener for target {target_id}: {e}')
+ # Set up network monitoring for this target (catches ALL download variants)
+ await self._setup_network_monitoring(target_id)
+
+ async def _setup_network_monitoring(self, target_id: TargetID) -> None:
+ """Set up network monitoring to detect PDFs and downloads from ALL sources.
+
+ This catches:
+ - Direct PDF navigation
+ - PDFs in iframes
+ - PDFs with embed/object tags
+ - JavaScript-triggered downloads
+ - Any Content-Disposition: attachment headers
+ """
+ # Skip if already monitoring this target
+ if target_id in self._network_monitored_targets:
+ self.logger.debug(f'[DownloadsWatchdog] Network monitoring already enabled for target {target_id[-4:]}')
+ return
+
+ # Check if auto-download is enabled
+ if not self._is_auto_download_enabled():
+ self.logger.debug('[DownloadsWatchdog] Auto-download disabled, skipping network monitoring')
+ return
+
+ try:
+ cdp_client = self.browser_session.cdp_client
+
+ # Get or create CDP session for this target
+ cdp_session = await self.browser_session.get_or_create_cdp_session(target_id, focus=False)
+
+ # Enable Network domain to monitor HTTP responses
+ await cdp_client.send.Network.enable(session_id=cdp_session.session_id)
+ self.logger.debug(f'[DownloadsWatchdog] Enabled Network domain for target {target_id[-4:]}')
+
+ # Register callback for network responses
+ async def on_response_received(event: ResponseReceivedEvent, session_id: str | None) -> None:
+ """Handle Network.responseReceived event to detect downloadable content."""
+ try:
+ response = event.get('response', {})
+ url = response.get('url', '')
+ content_type = response.get('mimeType', '').lower()
+ headers = response.get('headers', {})
+
+ # Skip non-HTTP URLs (data:, about:, chrome-extension:, etc.)
+ if not url.startswith('http'):
+ return
+
+ # Check if it's a PDF
+ is_pdf = 'application/pdf' in content_type
+
+ # Check if it's marked as download via Content-Disposition header
+ content_disposition = headers.get('content-disposition', '').lower()
+ is_download_attachment = 'attachment' in content_disposition
+
+ # Only process if it's a PDF or download
+ if not (is_pdf or is_download_attachment):
+ return
+
+ # Check if we've already processed this URL in this session
+ if url in self._detected_downloads:
+ self.logger.debug(f'[DownloadsWatchdog] Already detected download: {url[:80]}...')
+ return
+
+ # Mark as detected to avoid duplicates
+ self._detected_downloads.add(url)
+
+ # Extract filename from Content-Disposition if available
+ suggested_filename = None
+ if 'filename=' in content_disposition:
+ # Parse filename from Content-Disposition header
+ import re
+
+ filename_match = re.search(r'filename[^;=\n]*=(([\'"]).*?\2|[^;\n]*)', content_disposition)
+ if filename_match:
+ suggested_filename = filename_match.group(1).strip('\'"')
+
+ self.logger.info(f'[DownloadsWatchdog] 🔍 Detected downloadable content via network: {url[:80]}...')
+ self.logger.debug(
+ f'[DownloadsWatchdog] Content-Type: {content_type}, Is PDF: {is_pdf}, Is Attachment: {is_download_attachment}'
+ )
+
+ # Trigger download asynchronously in background (don't block event handler)
+ async def download_in_background():
+ try:
+ download_path = await self.download_file_from_url(
+ url=url, target_id=target_id, content_type=content_type, suggested_filename=suggested_filename
+ )
+
+ if download_path:
+ self.logger.info(f'[DownloadsWatchdog] ✅ Successfully downloaded: {download_path}')
+ else:
+ self.logger.warning(f'[DownloadsWatchdog] ⚠️ Failed to download: {url[:80]}...')
+ except Exception as e:
+ self.logger.error(f'[DownloadsWatchdog] Error downloading in background: {type(e).__name__}: {e}')
+
+ # Create background task
+ task = asyncio.create_task(download_in_background())
+ self._cdp_event_tasks.add(task)
+ task.add_done_callback(lambda t: self._cdp_event_tasks.discard(t))
+
+ except Exception as e:
+ self.logger.error(f'[DownloadsWatchdog] Error in network response handler: {type(e).__name__}: {e}')
+
+ # Register the callback
+ cdp_client.register.Network.responseReceived(on_response_received)
+
+ # Mark this target as monitored
+ self._network_monitored_targets.add(target_id)
+ self.logger.debug(f'[DownloadsWatchdog] ✅ Network monitoring enabled for target {target_id[-4:]}')
+
+ except Exception as e:
+ self.logger.warning(f'[DownloadsWatchdog] Failed to set up network monitoring for target {target_id}: {e}')
+
+ async def download_file_from_url(
+ self, url: str, target_id: TargetID, content_type: str | None = None, suggested_filename: str | None = None
+ ) -> str | None:
+ """Generic method to download any file from a URL.
+
+ Args:
+ url: The URL to download
+ target_id: The target ID for CDP session
+ content_type: Optional content type (e.g., 'application/pdf')
+ suggested_filename: Optional filename from Content-Disposition header
+
+ Returns:
+ Path to downloaded file, or None if download failed
+ """
+ if not self.browser_session.browser_profile.downloads_path:
+ self.logger.warning('[DownloadsWatchdog] No downloads path configured')
+ return None
+
+ # Check if already downloaded in this session
+ if url in self._session_pdf_urls:
+ existing_path = self._session_pdf_urls[url]
+ self.logger.debug(f'[DownloadsWatchdog] File already downloaded in session: {existing_path}')
+ return existing_path
+
+ try:
+ # Get or create CDP session for this target
+ temp_session = await self.browser_session.get_or_create_cdp_session(target_id, focus=False)
+
+ # Determine filename
+ if suggested_filename:
+ filename = suggested_filename
+ else:
+ # Extract from URL
+ filename = os.path.basename(url.split('?')[0]) # Remove query params
+ if not filename or '.' not in filename:
+ # Fallback: use content type to determine extension
+ if content_type and 'pdf' in content_type:
+ filename = 'document.pdf'
+ else:
+ filename = 'download'
+
+ # Ensure downloads directory exists
+ downloads_dir = str(self.browser_session.browser_profile.downloads_path)
+ os.makedirs(downloads_dir, exist_ok=True)
+
+ # Generate unique filename if file exists
+ final_filename = filename
+ existing_files = os.listdir(downloads_dir)
+ if filename in existing_files:
+ base, ext = os.path.splitext(filename)
+ counter = 1
+ while f'{base} ({counter}){ext}' in existing_files:
+ counter += 1
+ final_filename = f'{base} ({counter}){ext}'
+ self.logger.debug(f'[DownloadsWatchdog] File exists, using: {final_filename}')
+
+ self.logger.debug(f'[DownloadsWatchdog] Downloading from: {url[:100]}...')
+
+ # Download using JavaScript fetch to leverage browser cache
+ escaped_url = json.dumps(url)
+
+ result = await asyncio.wait_for(
+ temp_session.cdp_client.send.Runtime.evaluate(
+ params={
+ 'expression': f"""
+ (async () => {{
+ try {{
+ const response = await fetch({escaped_url}, {{
+ cache: 'force-cache'
+ }});
+ if (!response.ok) {{
+ throw new Error(`HTTP error! status: ${{response.status}}`);
+ }}
+ const blob = await response.blob();
+ const arrayBuffer = await blob.arrayBuffer();
+ const uint8Array = new Uint8Array(arrayBuffer);
+
+ return {{
+ data: Array.from(uint8Array),
+ responseSize: uint8Array.length
+ }};
+ }} catch (error) {{
+ throw new Error(`Fetch failed: ${{error.message}}`);
+ }}
+ }})()
+ """,
+ 'awaitPromise': True,
+ 'returnByValue': True,
+ },
+ session_id=temp_session.session_id,
+ ),
+ timeout=15.0, # 15 second timeout
+ )
+
+ download_result = result.get('result', {}).get('value', {})
+
+ if download_result and download_result.get('data') and len(download_result['data']) > 0:
+ download_path = os.path.join(downloads_dir, final_filename)
+
+ # Save the file asynchronously
+ async with await anyio.open_file(download_path, 'wb') as f:
+ await f.write(bytes(download_result['data']))
+
+ # Verify file was written successfully
+ if os.path.exists(download_path):
+ actual_size = os.path.getsize(download_path)
+ self.logger.debug(f'[DownloadsWatchdog] File written: {download_path} ({actual_size} bytes)')
+
+ # Determine file type
+ file_ext = Path(final_filename).suffix.lower().lstrip('.')
+ mime_type = content_type or f'application/{file_ext}'
+
+ # Store URL->path mapping for this session
+ self._session_pdf_urls[url] = download_path
+
+ # Emit file downloaded event
+ self.logger.debug(f'[DownloadsWatchdog] Dispatching FileDownloadedEvent for {final_filename}')
+ self.event_bus.dispatch(
+ FileDownloadedEvent(
+ url=url,
+ path=download_path,
+ file_name=final_filename,
+ file_size=actual_size,
+ file_type=file_ext if file_ext else None,
+ mime_type=mime_type,
+ auto_download=True,
+ )
+ )
+
+ return download_path
+ else:
+ self.logger.error(f'[DownloadsWatchdog] Failed to write file: {download_path}')
+ return None
+ else:
+ self.logger.warning(f'[DownloadsWatchdog] No data received when downloading from {url}')
+ return None
+
+ except TimeoutError:
+ self.logger.warning(f'[DownloadsWatchdog] Download timed out: {url[:80]}...')
+ return None
+ except Exception as e:
+ self.logger.warning(f'[DownloadsWatchdog] Download failed: {type(e).__name__}: {e}')
+ return None
+
def _track_download(self, file_path: str) -> None:
"""Track a completed download and dispatch the appropriate event.