Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Fix CDP download interceptor missing XHR-triggered downloads (#4934)
LawyZheng44d3635cef01
diff --git a/skyvern/webeye/cdp_download_interceptor.py b/skyvern/webeye/cdp_download_interceptor.pyindex 0a1e9b9fe..40f06395a 100644--- a/skyvern/webeye/cdp_download_interceptor.py+++ b/skyvern/webeye/cdp_download_interceptor.py@@ -36,13 +36,9 @@ MAX_FILE_SIZE_BYTES = 100 * 1024 * 1024 # 100 MB# Resource types that should NEVER be treated as downloads.# Sub-resources (Font, Stylesheet, etc.) are loaded by the page, not user-initiated.-# XHR/Fetch are programmatic JS API calls (e.g. Google APIs return Content-Disposition:-# attachment on JSON responses, which would cause false positives).# Real user downloads come through as "Document" (link click / navigation).NON_DOWNLOAD_RESOURCE_TYPES = frozenset({- "XHR",- "Fetch","Font","Stylesheet","Script",@@ -57,6 +53,14 @@ NON_DOWNLOAD_RESOURCE_TYPES = frozenset(})+# XHR/Fetch are programmatic JS API calls that sometimes carry Content-Disposition:+# attachment (e.g. Google APIs on JSON responses). We don't fully block them —+# instead, we only allow them through if there's an explicit attachment header,+# and rely on NON_DOWNLOAD_CONTENT_TYPES to filter out API false-positives.+# Without an explicit attachment header, we skip XHR/Fetch to avoid MIME-only+# false positives.+XHR_FETCH_RESOURCE_TYPES = frozenset({"XHR", "Fetch"})+# Content types that are clearly API / data responses, never user-facing downloads,# even if the server includes Content-Disposition: attachment.NON_DOWNLOAD_CONTENT_TYPES = frozenset(@@ -117,10 +121,13 @@ def is_download_response(headers: dict[str, str], status_code: int, resource_typDetermine if a response is a file download.Checks:- 0. Skip sub-resource types (Font, Stylesheet, Script, Image, etc.)- 1. Skip API content types (application/json, etc.)- 2. Content-Disposition contains "attachment"- 3. Content-Type is a known download MIME type+ 0. Skip error responses (status >= 400)+ 1. Skip sub-resource types (Font, Stylesheet, Script, Image, etc.)+ 2. Skip API content types (application/json, etc.)+ 3. For XHR/Fetch: require BOTH attachment header AND download MIME type+ (prevents false positives like Google's text/plain + attachment XHR responses)+ 4. Content-Disposition contains "attachment"+ 5. Content-Type is a known download MIME type"""if status_code >= 400:return False@@ -134,10 +141,18 @@ def is_download_response(headers: dict[str, str], status_code: int, resource_typif content_type in NON_DOWNLOAD_CONTENT_TYPES:return False- if "attachment" in content_disposition.lower():+ is_attachment = "attachment" in content_disposition.lower()+ is_download_mime = content_type in DOWNLOAD_MIME_TYPES++ # XHR/Fetch require both signals to avoid false positives+ # (e.g. Google async requests: text/plain + attachment; filename="f.txt")+ if resource_type in XHR_FETCH_RESOURCE_TYPES:+ return is_attachment and is_download_mime++ if is_attachment:return True- if content_type in DOWNLOAD_MIME_TYPES:+ if is_download_mime:return Truereturn Falsediff --git a/tests/unit/test_cdp_download_interceptor.py b/tests/unit/test_cdp_download_interceptor.pyindex 653573c3f..acfa740b4 100644--- a/tests/unit/test_cdp_download_interceptor.py+++ b/tests/unit/test_cdp_download_interceptor.py@@ -65,15 +65,63 @@ class TestIsDownloadResponse:def test_empty_headers_not_download(self) -> None:assert is_download_response({}, 200) is False- # Resource type filtering- def test_xhr_resource_type_not_download(self) -> None:+ # Resource type filtering — XHR/Fetch require BOTH attachment AND download MIME+ def test_xhr_attachment_and_download_mime_is_download(self) -> None:+ """XHR with attachment + download MIME type should be treated as download."""headers = {"content-disposition": "attachment", "content-type": "application/octet-stream"}+ assert is_download_response(headers, 200, resource_type="XHR") is True++ def test_fetch_attachment_and_download_mime_is_download(self) -> None:+ """Fetch with attachment + download MIME type should be treated as download."""+ headers = {"content-disposition": "attachment", "content-type": "application/pdf"}+ assert is_download_response(headers, 200, resource_type="Fetch") is True++ def test_xhr_attachment_pdf_is_download(self) -> None:+ """Real-world case: XHR download with attachment header and PDF content-type."""+ headers = {+ "content-disposition": "attachment; filename=Invoice_12345.pdf; filename*=UTF-8''Invoice_12345.pdf",+ "content-type": "application/pdf",+ }+ assert is_download_response(headers, 200, resource_type="XHR") is True++ # XHR/Fetch with attachment but non-download MIME should NOT be download+ def test_xhr_attachment_text_plain_not_download(self) -> None:+ """Google-style XHR: text/plain + attachment should NOT be treated as download."""+ headers = {"content-disposition": 'attachment; filename="f.txt"', "content-type": "text/plain; charset=UTF-8"}+ assert is_download_response(headers, 200, resource_type="XHR") is False++ def test_xhr_attachment_text_html_not_download(self) -> None:+ """XHR with text/html + attachment should NOT be treated as download."""+ headers = {"content-disposition": "attachment", "content-type": "text/html"}assert is_download_response(headers, 200, resource_type="XHR") is False- def test_fetch_resource_type_not_download(self) -> None:+ def test_fetch_attachment_only_not_download(self) -> None:+ """Fetch with attachment but no download MIME type should NOT be download."""headers = {"content-disposition": "attachment"}assert is_download_response(headers, 200, resource_type="Fetch") is False+ def test_xhr_attachment_csv_not_download(self) -> None:+ """Known limitation: CSV via XHR is not detected because text/csv is not in DOWNLOAD_MIME_TYPES."""+ headers = {"content-disposition": "attachment", "content-type": "text/csv"}+ assert is_download_response(headers, 200, resource_type="XHR") is False++ # XHR/Fetch without attachment header should NOT be download (MIME-only false positive)+ def test_xhr_mime_only_not_download(self) -> None:+ """XHR with download MIME type but no attachment header should NOT be treated as download."""+ headers = {"content-type": "application/pdf"}+ assert is_download_response(headers, 200, resource_type="XHR") is False++ def test_fetch_mime_only_not_download(self) -> None:+ """Fetch with download MIME type but no attachment header should NOT be treated as download."""+ headers = {"content-type": "application/octet-stream"}+ assert is_download_response(headers, 200, resource_type="Fetch") is False++ # XHR/Fetch with attachment but API content-type should still be filtered+ def test_xhr_json_attachment_not_download(self) -> None:+ """XHR with JSON content-type and attachment header should NOT be download."""+ headers = {"content-disposition": "attachment", "content-type": "application/json"}+ assert is_download_response(headers, 200, resource_type="XHR") is False+def test_font_resource_type_not_download(self) -> None:headers = {"content-type": "application/octet-stream"}assert is_download_response(headers, 200, resource_type="Font") is False