Skip to content

Automate browser based workflows with AI

Add CDPFetchDownloadBrowser for remote Windows Chrome download interception (#4906)

LawyZheng665c7b6bc554
diff --git a/skyvern/webeye/cdp_download_interceptor.py b/skyvern/webeye/cdp_download_interceptor.py
new file mode 100644
index 000000000..28f1ce2cc
--- /dev/null
+++ b/skyvern/webeye/cdp_download_interceptor.py
@@ -0,0 +1,453 @@
+"""
+CDP Fetch Download Interceptor
+
+Intercepts download responses via the CDP Fetch domain and saves files locally.
+Used for remote CDP browsers where Browser.setDownloadBehavior with a local
+downloadPath does not work (e.g., Playwright bug #38805 — remote Windows Chrome
+ignoring Linux paths).
+
+Flow:
+1. Enable Fetch interception at Response stage for each page
+2. On each paused request:
+ - Non-download → Fetch.continueResponse (pass through)
+ - Download → extract body via stream → save to disk → Fetch.fulfillRequest (empty body blocks browser save)
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import re
+import time
+from pathlib import Path
+from typing import Any
+from urllib.parse import unquote, urlparse
+
+import structlog
+from playwright.async_api import CDPSession, Page
+
+LOG = structlog.get_logger()
+
+# Chunk size for IO.read streaming
+IO_READ_CHUNK_SIZE = 64 * 1024 # 64 KB
+
+# Maximum file size we'll attempt to download
+MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024 # 100 MB
+
+# Resource types that should NEVER be treated as downloads.
+# Sub-resources (Font, Stylesheet, etc.) are loaded by the page, not user-initiated.
+# XHR/Fetch are programmatic JS API calls (e.g. Google APIs return Content-Disposition:
+# attachment on JSON responses, which would cause false positives).
+# Real user downloads come through as "Document" (link click / navigation).
+NON_DOWNLOAD_RESOURCE_TYPES = frozenset(
+ {
+ "XHR",
+ "Fetch",
+ "Font",
+ "Stylesheet",
+ "Script",
+ "Image",
+ "Media",
+ "Manifest",
+ "SignedExchange",
+ "Ping",
+ "Preflight",
+ "CSPViolationReport",
+ "Prefetch",
+ }
+)
+
+# Content types that are clearly API / data responses, never user-facing downloads,
+# even if the server includes Content-Disposition: attachment.
+NON_DOWNLOAD_CONTENT_TYPES = frozenset(
+ {
+ "application/json",
+ "application/xml",
+ "text/xml",
+ "application/grpc",
+ "application/grpc-web",
+ "application/grpc-web+proto",
+ }
+)
+
+# MIME types that are almost always downloads (even without Content-Disposition)
+DOWNLOAD_MIME_TYPES = frozenset(
+ {
+ "application/octet-stream",
+ "application/zip",
+ "application/x-zip-compressed",
+ "application/gzip",
+ "application/x-gzip",
+ "application/x-tar",
+ "application/x-7z-compressed",
+ "application/x-rar-compressed",
+ "application/x-msdownload",
+ "application/x-download",
+ "application/force-download",
+ "application/pdf",
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ "application/vnd.ms-excel",
+ "application/msword",
+ }
+)
+
+
+def _parse_headers(raw_headers: list[dict[str, str]]) -> dict[str, str]:
+ """Convert CDP header list [{name, value}] to a lowercase-keyed dict (last value wins)."""
+ result: dict[str, str] = {}
+ for h in raw_headers:
+ result[h["name"].lower()] = h["value"]
+ return result
+
+
+def _parse_content_length(headers: dict[str, str]) -> int | None:
+ """Extract Content-Length as int, or None if absent/invalid."""
+ val = headers.get("content-length")
+ if val is None:
+ return None
+ try:
+ return int(val)
+ except ValueError:
+ return None
+
+
+def is_download_response(headers: dict[str, str], status_code: int, resource_type: str = "") -> bool:
+ """
+ Determine if a response is a file download.
+
+ Checks:
+ 0. Skip sub-resource types (Font, Stylesheet, Script, Image, etc.)
+ 1. Skip API content types (application/json, etc.)
+ 2. Content-Disposition contains "attachment"
+ 3. Content-Type is a known download MIME type
+ """
+ if status_code >= 400:
+ return False
+
+ if resource_type in NON_DOWNLOAD_RESOURCE_TYPES:
+ return False
+
+ content_disposition = headers.get("content-disposition", "")
+ content_type = headers.get("content-type", "").split(";")[0].strip().lower()
+
+ if content_type in NON_DOWNLOAD_CONTENT_TYPES:
+ return False
+
+ if "attachment" in content_disposition.lower():
+ return True
+
+ if content_type in DOWNLOAD_MIME_TYPES:
+ return True
+
+ return False
+
+
+def extract_filename(headers: dict[str, str], url: str, index: int) -> str:
+ """
+ Extract filename from response headers or URL.
+
+ Priority:
+ 1. Content-Disposition filename*= (RFC 5987, UTF-8)
+ 2. Content-Disposition filename=
+ 3. URL path last segment (if it has an extension)
+ 4. Fallback: download_{timestamp}_{index}
+ """
+ content_disposition = headers.get("content-disposition", "")
+
+ if content_disposition:
+ # Try RFC 5987 filename*= first
+ match = re.search(r"filename\*\s*=\s*(?:UTF-8|utf-8)''(.+?)(?:;|$)", content_disposition)
+ if match:
+ return unquote(match.group(1).strip())
+
+ # Try regular filename=
+ match = re.search(r'filename\s*=\s*"?([^";]+)"?', content_disposition)
+ if match:
+ return match.group(1).strip()
+
+ # Try URL path
+ parsed = urlparse(url)
+ path_segments = [s for s in parsed.path.split("/") if s]
+ if path_segments:
+ last_segment = unquote(path_segments[-1])
+ if "." in last_segment:
+ return last_segment
+
+ return f"download_{int(time.time())}_{index}"
+
+
+class CDPDownloadInterceptor:
+ """
+ Intercepts download responses via the CDP Fetch domain.
+
+ Flow:
+ 1. Enable Fetch interception at Response stage
+ 2. On each paused request:
+ - Non-download → Fetch.continueResponse (pass through)
+ - Download → extract body → save to disk → Fetch.fulfillRequest (block browser save)
+ """
+
+ def __init__(self, output_dir: str | None = None) -> None:
+ self._output_dir: Path | None = Path(output_dir) if output_dir else None
+ self._cdp_sessions: list[CDPSession] = []
+ self._enabled = False
+ self._download_index = 0
+
+ def set_download_dir(self, download_dir: str) -> None:
+ """Set or update the download directory. Can be called after init when run_id becomes available."""
+ self._output_dir = Path(download_dir)
+ self._output_dir.mkdir(parents=True, exist_ok=True)
+ LOG.info("CDP download interceptor download dir set", download_dir=download_dir)
+
+ async def enable_for_page(self, page: Page) -> None:
+ """Create a CDP session for the given page and enable Fetch interception."""
+ cdp_session = await page.context.new_cdp_session(page)
+ # Capture cdp_session in the closure so the handler uses the correct session
+ # (requestId is scoped to the session that fired Fetch.requestPaused).
+ cdp_session.on("Fetch.requestPaused", lambda event: self._on_request_paused(event, cdp_session))
+ await cdp_session.send(
+ "Fetch.enable",
+ {"patterns": [{"requestStage": "Response"}]},
+ )
+ self._cdp_sessions.append(cdp_session)
+ self._enabled = True
+ LOG.info(
+ "CDP Fetch interception enabled for page",
+ page_url=page.url,
+ session_count=len(self._cdp_sessions),
+ output_dir=str(self._output_dir),
+ )
+
+ async def disable(self) -> None:
+ """Disable Fetch interception on all CDP sessions."""
+ session_count = len(self._cdp_sessions)
+ for cdp_session in self._cdp_sessions:
+ try:
+ await cdp_session.send("Fetch.disable")
+ except Exception:
+ pass
+ self._cdp_sessions.clear()
+ self._enabled = False
+ LOG.info(
+ "CDP Fetch interception disabled",
+ session_count=session_count,
+ downloads_intercepted=self._download_index,
+ )
+
+ def _on_request_paused(self, event: dict[str, Any], cdp_session: CDPSession) -> None:
+ """Handle Fetch.requestPaused — schedule async handler with the originating session."""
+ asyncio.ensure_future(self._handle_request_paused(event, cdp_session))
+
+ async def _handle_request_paused(self, event: dict[str, Any], cdp_session: CDPSession) -> None:
+ """Async handler for paused requests."""
+ request_id = event["requestId"]
+ response_status = event.get("responseStatusCode", 0)
+ response_headers = _parse_headers(event.get("responseHeaders", []))
+ url = event.get("request", {}).get("url", "<unknown>")
+ resource_type = event.get("resourceType", "")
+
+ try:
+ if is_download_response(response_headers, response_status, resource_type):
+ LOG.info(
+ "CDP download response detected",
+ url=url,
+ resource_type=resource_type,
+ status_code=response_status,
+ content_type=response_headers.get("content-type", ""),
+ content_disposition=response_headers.get("content-disposition", ""),
+ )
+ await self._handle_download(cdp_session, request_id, url, response_headers)
+ else:
+ await self._continue_response(cdp_session, request_id)
+ except Exception as e:
+ LOG.error(
+ "Error handling CDP request",
+ request_id=request_id,
+ url=url,
+ exc_info=True,
+ error=str(e),
+ )
+ try:
+ await self._continue_response(cdp_session, request_id)
+ except Exception:
+ pass
+
+ async def _continue_response(self, cdp_session: CDPSession, request_id: str) -> None:
+ """Let a non-download response pass through to the browser."""
+ await cdp_session.send("Fetch.continueResponse", {"requestId": request_id})
+
+ async def _handle_download(
+ self,
+ cdp_session: CDPSession,
+ request_id: str,
+ url: str,
+ headers: dict[str, str],
+ ) -> None:
+ """Extract a download file and save it to disk."""
+ if not self._output_dir:
+ LOG.warning("CDP download intercepted but no output_dir set, passing through", url=url)
+ await self._continue_response(cdp_session, request_id)
+ return
+
+ self._output_dir.mkdir(parents=True, exist_ok=True)
+
+ self._download_index += 1
+ content_length = _parse_content_length(headers)
+ content_type = headers.get("content-type", "").split(";")[0].strip()
+ filename = extract_filename(headers, url, self._download_index)
+ # Sanitize filename to prevent path traversal (e.g. "../../etc/evil")
+ filename = Path(filename).name
+ if not filename:
+ filename = f"download_{int(time.time())}_{self._download_index}"
+ save_path = self._output_dir / filename
+
+ # Deduplicate filename if a file with the same name already exists
+ if save_path.exists():
+ stem = Path(filename).stem
+ suffix = Path(filename).suffix
+ filename = f"{stem}_{self._download_index}{suffix}"
+ save_path = self._output_dir / filename
+
+ LOG.info(
+ "CDP download detected",
+ filename=filename,
+ url=url,
+ content_type=content_type,
+ content_length=content_length,
+ )
+
+ if content_length and content_length > MAX_FILE_SIZE_BYTES:
+ LOG.warning(
+ "CDP download file exceeds size limit, passing through",
+ filename=filename,
+ content_length=content_length,
+ max_size=MAX_FILE_SIZE_BYTES,
+ )
+ await self._continue_response(cdp_session, request_id)
+ return
+
+ t0 = time.monotonic()
+
+ try:
+ # Stream-first strategy: try takeResponseBodyAsStream, fallback to getResponseBody.
+ # Note: if stream partially consumes the body before failing, the direct fallback
+ # will also fail since the body is already consumed. The outer handler catches this.
+ extraction_method = "stream"
+ try:
+ data = await self._extract_body_stream(cdp_session, request_id)
+ except Exception as e:
+ extraction_method = "direct"
+ LOG.warning(
+ "takeResponseBodyAsStream failed, trying getResponseBody",
+ filename=filename,
+ url=url,
+ error=str(e),
+ )
+ data = await self._extract_body_direct(cdp_session, request_id)
+
+ with open(save_path, "wb") as f:
+ f.write(data)
+
+ elapsed_ms = (time.monotonic() - t0) * 1000
+ LOG.info(
+ "CDP download saved",
+ filename=filename,
+ size=len(data),
+ duration_ms=round(elapsed_ms, 1),
+ save_path=str(save_path),
+ extraction_method=extraction_method,
+ download_index=self._download_index,
+ )
+
+ except Exception as e:
+ LOG.error(
+ "Failed to extract CDP download",
+ filename=filename,
+ url=url,
+ content_type=content_type,
+ content_length=content_length,
+ error=str(e),
+ exc_info=True,
+ )
+ try:
+ await self._continue_response(cdp_session, request_id)
+ except Exception:
+ pass
+ return
+
+ # Block the browser from saving the file locally by fulfilling with an empty response.
+ # After body extraction (especially via takeResponseBodyAsStream), failRequest may
+ # return "Invalid parameters". fulfillRequest with empty body works reliably.
+ try:
+ await self._fulfill_empty(cdp_session, request_id)
+ except Exception as e:
+ LOG.warning("fulfillRequest failed after download", filename=filename, url=url, error=str(e))
+ try:
+ await self._continue_response(cdp_session, request_id)
+ except Exception:
+ pass
+
+ async def _fulfill_empty(self, cdp_session: CDPSession, request_id: str) -> None:
+ """Fulfill a request with an empty response to block browser-side download."""
+ await cdp_session.send(
+ "Fetch.fulfillRequest",
+ {
+ "requestId": request_id,
+ "responseCode": 200,
+ "responseHeaders": [{"name": "Content-Type", "value": "text/plain"}],
+ "body": base64.b64encode(b"").decode(),
+ },
+ )
+
+ async def _extract_body_direct(self, cdp_session: CDPSession, request_id: str) -> bytes:
+ """Extract response body using Fetch.getResponseBody (single call, base64)."""
+ result = await cdp_session.send(
+ "Fetch.getResponseBody",
+ {"requestId": request_id},
+ )
+ body = result.get("body", "")
+ is_base64 = result.get("base64Encoded", False)
+ if is_base64:
+ return base64.b64decode(body)
+ return body.encode("utf-8")
+
+ async def _extract_body_stream(self, cdp_session: CDPSession, request_id: str) -> bytes:
+ """Extract response body using Fetch.takeResponseBodyAsStream + IO.read."""
+ result = await cdp_session.send(
+ "Fetch.takeResponseBodyAsStream",
+ {"requestId": request_id},
+ )
+ stream_handle = result["stream"]
+
+ chunks: list[bytes] = []
+ total_read = 0
+
+ try:
+ while True:
+ read_result = await cdp_session.send(
+ "IO.read",
+ {"handle": stream_handle, "size": IO_READ_CHUNK_SIZE},
+ )
+ data = read_result.get("data", "")
+ is_base64 = read_result.get("base64Encoded", False)
+ eof = read_result.get("eof", False)
+
+ if data:
+ chunk = base64.b64decode(data) if is_base64 else data.encode("utf-8")
+ chunks.append(chunk)
+ total_read += len(chunk)
+
+ if total_read > MAX_FILE_SIZE_BYTES:
+ LOG.warning("Stream exceeded max file size during read", total_read=total_read)
+ break
+
+ if eof:
+ break
+ finally:
+ try:
+ await cdp_session.send("IO.close", {"handle": stream_handle})
+ except Exception:
+ pass
+
+ return b"".join(chunks)
diff --git a/tests/unit/test_cdp_download_interceptor.py b/tests/unit/test_cdp_download_interceptor.py
new file mode 100644
index 000000000..653573c3f
--- /dev/null
+++ b/tests/unit/test_cdp_download_interceptor.py
@@ -0,0 +1,179 @@
+"""Unit tests for CDPDownloadInterceptor pure functions (no browser needed)."""
+
+import time
+
+from skyvern.webeye.cdp_download_interceptor import extract_filename, is_download_response
+
+
+class TestIsDownloadResponse:
+ """Tests for is_download_response()."""
+
+ def test_attachment_header(self) -> None:
+ headers = {"content-disposition": 'attachment; filename="report.csv"', "content-type": "text/csv"}
+ assert is_download_response(headers, 200) is True
+
+ def test_attachment_header_case_insensitive(self) -> None:
+ headers = {"content-disposition": 'Attachment; filename="report.csv"', "content-type": "text/csv"}
+ assert is_download_response(headers, 200) is True
+
+ def test_download_mime_type_pdf(self) -> None:
+ headers = {"content-type": "application/pdf"}
+ assert is_download_response(headers, 200) is True
+
+ def test_download_mime_type_zip(self) -> None:
+ headers = {"content-type": "application/zip"}
+ assert is_download_response(headers, 200) is True
+
+ def test_download_mime_type_xlsx(self) -> None:
+ headers = {
+ "content-type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ }
+ assert is_download_response(headers, 200) is True
+
+ def test_download_mime_type_octet_stream(self) -> None:
+ headers = {"content-type": "application/octet-stream"}
+ assert is_download_response(headers, 200) is True
+
+ def test_download_mime_type_with_charset(self) -> None:
+ headers = {"content-type": "application/pdf; charset=utf-8"}
+ assert is_download_response(headers, 200) is True
+
+ def test_html_not_download(self) -> None:
+ headers = {"content-type": "text/html"}
+ assert is_download_response(headers, 200) is False
+
+ def test_json_not_download(self) -> None:
+ headers = {"content-type": "application/json"}
+ assert is_download_response(headers, 200) is False
+
+ def test_json_with_attachment_not_download(self) -> None:
+ """JSON responses with Content-Disposition: attachment should NOT be treated as downloads."""
+ headers = {
+ "content-disposition": "attachment",
+ "content-type": "application/json",
+ }
+ assert is_download_response(headers, 200) is False
+
+ def test_xml_not_download(self) -> None:
+ headers = {"content-type": "application/xml"}
+ assert is_download_response(headers, 200) is False
+
+ def test_grpc_not_download(self) -> None:
+ headers = {"content-type": "application/grpc"}
+ assert is_download_response(headers, 200) is False
+
+ def test_empty_headers_not_download(self) -> None:
+ assert is_download_response({}, 200) is False
+
+ # Resource type filtering
+ def test_xhr_resource_type_not_download(self) -> None:
+ headers = {"content-disposition": "attachment", "content-type": "application/octet-stream"}
+ assert is_download_response(headers, 200, resource_type="XHR") is False
+
+ def test_fetch_resource_type_not_download(self) -> None:
+ headers = {"content-disposition": "attachment"}
+ assert is_download_response(headers, 200, resource_type="Fetch") is False
+
+ def test_font_resource_type_not_download(self) -> None:
+ headers = {"content-type": "application/octet-stream"}
+ assert is_download_response(headers, 200, resource_type="Font") is False
+
+ def test_stylesheet_resource_type_not_download(self) -> None:
+ headers = {"content-type": "application/octet-stream"}
+ assert is_download_response(headers, 200, resource_type="Stylesheet") is False
+
+ def test_script_resource_type_not_download(self) -> None:
+ headers = {"content-type": "application/octet-stream"}
+ assert is_download_response(headers, 200, resource_type="Script") is False
+
+ def test_image_resource_type_not_download(self) -> None:
+ headers = {"content-type": "application/octet-stream"}
+ assert is_download_response(headers, 200, resource_type="Image") is False
+
+ def test_document_resource_type_is_download(self) -> None:
+ """Document resource type (link click) should allow download detection."""
+ headers = {"content-disposition": "attachment", "content-type": "application/pdf"}
+ assert is_download_response(headers, 200, resource_type="Document") is True
+
+ def test_empty_resource_type_is_download(self) -> None:
+ headers = {"content-type": "application/pdf"}
+ assert is_download_response(headers, 200, resource_type="") is True
+
+ def test_error_status_code_not_download(self) -> None:
+ headers = {"content-disposition": "attachment", "content-type": "application/pdf"}
+ assert is_download_response(headers, 404) is False
+
+ def test_server_error_not_download(self) -> None:
+ headers = {"content-type": "application/octet-stream"}
+ assert is_download_response(headers, 500) is False
+
+
+class TestExtractFilename:
+ """Tests for extract_filename()."""
+
+ def test_rfc5987_filename_star(self) -> None:
+ headers = {"content-disposition": "attachment; filename*=UTF-8''my%20report%282024%29.pdf"}
+ result = extract_filename(headers, "https://example.com/download", 1)
+ assert result == "my report(2024).pdf"
+
+ def test_regular_filename(self) -> None:
+ headers = {"content-disposition": 'attachment; filename="report.csv"'}
+ result = extract_filename(headers, "https://example.com/download", 1)
+ assert result == "report.csv"
+
+ def test_unquoted_filename(self) -> None:
+ headers = {"content-disposition": "attachment; filename=report.csv"}
+ result = extract_filename(headers, "https://example.com/download", 1)
+ assert result == "report.csv"
+
+ def test_filename_star_takes_priority(self) -> None:
+ headers = {
+ "content-disposition": "attachment; filename=\"fallback.csv\"; filename*=UTF-8''preferred.csv",
+ }
+ result = extract_filename(headers, "https://example.com/download", 1)
+ assert result == "preferred.csv"
+
+ def test_url_path_fallback(self) -> None:
+ headers: dict[str, str] = {}
+ result = extract_filename(headers, "https://example.com/files/document.pdf", 1)
+ assert result == "document.pdf"
+
+ def test_url_path_with_encoded_chars(self) -> None:
+ headers: dict[str, str] = {}
+ result = extract_filename(headers, "https://example.com/files/my%20report.xlsx", 1)
+ assert result == "my report.xlsx"
+
+ def test_url_path_no_extension_uses_fallback(self) -> None:
+ headers: dict[str, str] = {}
+ result = extract_filename(headers, "https://example.com/download", 1)
+ assert result.startswith("download_")
+
+ def test_fallback_format(self) -> None:
+ headers: dict[str, str] = {}
+ before = int(time.time())
+ result = extract_filename(headers, "https://example.com/api/export", 42)
+ after = int(time.time())
+ # Should be download_{timestamp}_{index}
+ parts = result.split("_")
+ assert parts[0] == "download"
+ assert before <= int(parts[1]) <= after
+ assert parts[2] == "42"
+
+ def test_empty_content_disposition(self) -> None:
+ headers = {"content-disposition": ""}
+ result = extract_filename(headers, "https://example.com/files/data.csv", 1)
+ assert result == "data.csv"
+
+ def test_content_disposition_inline(self) -> None:
+ """inline disposition without filename should fall back to URL."""
+ headers = {"content-disposition": "inline"}
+ result = extract_filename(headers, "https://example.com/files/report.pdf", 1)
+ assert result == "report.pdf"
+
+ def test_path_traversal_stripped(self) -> None:
+ """Path traversal in filename should be sanitized to just the filename part."""
+ headers = {"content-disposition": 'attachment; filename="../../etc/cron.d/evil"'}
+ result = extract_filename(headers, "https://example.com/download", 1)
+ # extract_filename returns the raw name; sanitization is done in _handle_download.
+ # But verify the raw output so tests document the behavior.
+ assert result == "../../etc/cron.d/evil"