Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-12018): classify external download HTTP errors (#7210)
Aaron Perez2452bf60e190
diff --git a/skyvern/forge/sdk/api/files.py b/skyvern/forge/sdk/api/files.pyindex 343862b7c..82f042e8c 100644--- a/skyvern/forge/sdk/api/files.py+++ b/skyvern/forge/sdk/api/files.py@@ -116,6 +116,19 @@ def _determine_download_filename(return sanitize_filename(file_name)+def _raise_download_response_for_status(response: aiohttp.ClientResponse) -> None:+ if response.status < HTTPStatus.BAD_REQUEST:+ return++ raise aiohttp.ClientResponseError(+ request_info=response.request_info,+ history=response.history,+ status=response.status,+ message=response.reason,+ headers=response.headers,+ )++def validate_download_url(url: str, organization_id: str | None = None) -> bool:"""Validate if a URL is supported for downloading.@@ -224,9 +237,7 @@ async def download_file(resolver = SSRFGuardedResolver()current_url = await validate_and_pin_fetch_url(url, resolver)request_headers = dict(headers or {})- async with aiohttp.ClientSession(- raise_for_status=True, connector=ssrf_guarded_tcp_connector(resolver)- ) as session:+ async with aiohttp.ClientSession(connector=ssrf_guarded_tcp_connector(resolver)) as session:LOG.info("Starting to download file", url=url)for _ in range(MAX_SAFE_REDIRECTS + 1):encoded_url = encode_url(current_url)@@ -243,6 +254,8 @@ async def download_file(current_url = next_urlcontinue+ _raise_download_response_for_status(response)+# Check the content length if availableif max_size_mb and response.content_length and response.content_length > max_size_mb * 1024 * 1024:# todo: move to root exception.py@@ -301,7 +314,7 @@ async def download_file()except aiohttp.ClientResponseError as e:# Re-raised and handled at the action/block boundary; server rejections are external.- LOG.warning(f"Failed to download file, status code: {e.status}", exc_info=True)+ LOG.warning("Failed to download file", status_code=e.status)raiseexcept DownloadFileMaxSizeExceeded as e:LOG.warning(f"Failed to download file, max size exceeded: {e.max_size}", exc_info=True)diff --git a/tests/unit/test_files_resolve_local_or_download.py b/tests/unit/test_files_resolve_local_or_download.pyindex 873e721ae..3b0db06ae 100644--- a/tests/unit/test_files_resolve_local_or_download.py+++ b/tests/unit/test_files_resolve_local_or_download.py@@ -2,8 +2,9 @@ from __future__ import annotationsfrom collections.abc import AsyncIteratorfrom pathlib import Path-from unittest.mock import AsyncMock+from unittest.mock import AsyncMock, MagicMock+import aiohttpimport pytestfrom multidict import CIMultiDict, CIMultiDictProxy@@ -13,33 +14,58 @@ from skyvern.forge.sdk.api import filesclass _FakeDownloadResponse:- def __init__(self, data: bytes, headers: dict[str, str] | None = None, advertise_length: bool = True) -> None:+ def __init__(+ self,+ data: bytes,+ headers: dict[str, str] | None = None,+ advertise_length: bool = True,+ status: int = 200,+ ) -> None:self._data = data# aiohttp exposes headers as a case-insensitive CIMultiDictProxy; mirror that.self.headers = CIMultiDictProxy(CIMultiDict(headers or {}))- self.status = 200+ self.status = status+ self.reason = "Unknown Error" if status >= 400 else "OK"+ self.history = ()+ self.request_info = MagicMock(real_url="https://example.com/files/rate-limited.png")self.content_length = len(data) if advertise_length else Noneself.content = selfself.body_read = False+ self.auto_raise_for_status = Falseasync def iter_chunked(self, chunk_size: int) -> AsyncIterator[bytes]:self.body_read = Trueyield self._dataasync def __aenter__(self) -> _FakeDownloadResponse:+ if self.auto_raise_for_status:+ self.raise_for_status()return selfasync def __aexit__(self, *exc: object) -> None:return None+ def raise_for_status(self) -> None:+ if self.status < 400:+ return+ raise aiohttp.ClientResponseError(+ request_info=self.request_info,+ history=self.history,+ status=self.status,+ message=self.reason,+ headers=self.headers,+ )+class _FakeDownloadSession:- def __init__(self, response: _FakeDownloadResponse) -> None:+ def __init__(self, response: _FakeDownloadResponse, *, raise_for_status: bool = False) -> None:self._response = response+ self._raise_for_status = raise_for_statusdef get(self, url: object, headers: dict[str, str] | None = None, allow_redirects: bool = True) -> _FakeDownloadResponse:+ self._response.auto_raise_for_status = self._raise_for_statusreturn self._responseasync def __aenter__(self) -> _FakeDownloadSession:@@ -54,9 +80,17 @@ def _patch_download_session(data: bytes,headers: dict[str, str] | None = None,advertise_length: bool = True,+ status: int = 200,+ captured_session_kwargs: dict[str, object] | None = None,) -> _FakeDownloadResponse:- response = _FakeDownloadResponse(data, headers, advertise_length=advertise_length)- monkeypatch.setattr(files.aiohttp, "ClientSession", lambda **kwargs: _FakeDownloadSession(response))+ response = _FakeDownloadResponse(data, headers, advertise_length=advertise_length, status=status)++ def make_session(**kwargs: object) -> _FakeDownloadSession:+ if captured_session_kwargs is not None:+ captured_session_kwargs.update(kwargs)+ return _FakeDownloadSession(response, raise_for_status=kwargs.get("raise_for_status") is True)++ monkeypatch.setattr(files.aiohttp, "ClientSession", make_session)return response@@ -196,3 +230,24 @@ async def test_download_file_cleans_up_temp_file_when_max_size_exceeded_mid_streawait files.download_file("https://example.com/files/big.bin", output_dir=str(tmp_path), max_size_mb=1)assert list(tmp_path.iterdir()) == []+++async def test_download_file_raises_http_error_without_aiohttp_auto_raise(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ captured_session_kwargs: dict[str, object] = {}+ response = _patch_download_session(+ monkeypatch,+ b"",+ status=429,+ captured_session_kwargs=captured_session_kwargs,+ )++ with pytest.raises(aiohttp.ClientResponseError) as exc_info:+ await files.download_file("https://example.com/files/rate-limited.png", output_dir=str(tmp_path))++ assert exc_info.value.status == 429+ assert captured_session_kwargs.get("raise_for_status") is not True+ assert not response.body_read+ assert list(tmp_path.iterdir()) == []