AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): return 404/502 instead of 500 for failed workspace file downloads (#14881)
Bently20fcab7c6292
diff --git a/autogpt_platform/backend/backend/api/features/workspace/routes.py b/autogpt_platform/backend/backend/api/features/workspace/routes.pyindex dbc8c00bf1..681b207bc3 100644--- a/autogpt_platform/backend/backend/api/features/workspace/routes.py+++ b/autogpt_platform/backend/backend/api/features/workspace/routes.py@@ -9,6 +9,7 @@ import refrom typing import Annotated, Any, Literalfrom urllib.parse import quote+import aiohttpimport fastapifrom autogpt_libs.auth.dependencies import get_user_id, requires_userfrom fastapi import Query, UploadFile@@ -36,7 +37,10 @@ from backend.data.workspace_scope import ()from backend.util.settings import Configfrom backend.util.workspace import WorkspaceManager, format_bytes-from backend.util.workspace_storage import get_workspace_storage+from backend.util.workspace_storage import (+ WorkspaceStorageBackend,+ get_workspace_storage,+)def _sanitize_filename_for_header(@@ -89,48 +93,71 @@ def _create_streaming_response()+_STORAGE_READ_ATTEMPTS = 2+++async def _read_file_content(+ storage: WorkspaceStorageBackend, file: WorkspaceFile+) -> bytes:+ """Read a file's bytes, retrying once on a transient storage error.++ Content missing from storage is a 404, not a server error. A storage read+ that keeps failing (a GCS body cut off mid-transfer, a timeout, a 5xx from+ the bucket) is a 502, logged with its traceback so it reaches Sentry.+ """+ attempt = 1+ while True:+ try:+ return await storage.retrieve(file.storage_path)+ except FileNotFoundError:+ logger.warning(+ f"File {file.id} has a record but no content in storage "+ f"(storagePath={file.storage_path})"+ )+ raise fastapi.HTTPException(status_code=404, detail="File not found")+ except (aiohttp.ClientError, OSError) as e:+ if attempt < _STORAGE_READ_ATTEMPTS:+ logger.warning(f"Retrying read of file {file.id} after {e!r}")+ attempt += 1+ continue+ logger.error(+ f"Failed to read file {file.id} from storage after {attempt} "+ f"attempts (storagePath={file.storage_path}): {e!r}",+ exc_info=True,+ )+ raise fastapi.HTTPException(+ status_code=502,+ detail="File storage is temporarily unavailable, please retry",+ ) from e++async def create_file_download_response(file: WorkspaceFile, *, inline: bool = False) -> Response:"""Create a download response for a workspace file.- Handles both local storage (direct streaming) and GCS (signed URL redirect- with fallback to streaming).+ GCS files redirect to a signed URL when the credentials can sign one;+ otherwise (local storage, or GCS without a signing key) the content is+ streamed through the API."""storage = await get_workspace_storage()- # For local storage, stream the file directly- if file.storage_path.startswith("local://"):- content = await storage.retrieve(file.storage_path)- return _create_streaming_response(content, file, inline=inline)-- # For GCS, try to redirect to signed URL, fall back to streaming- try:- url = await storage.get_download_url(file.storage_path, expires_in=300)- # If we got back an API path (fallback), stream directly instead- if url.startswith("/api/"):- content = await storage.retrieve(file.storage_path)- return _create_streaming_response(content, file, inline=inline)- return fastapi.responses.RedirectResponse(url=url, status_code=302)- except Exception as e:- # Log the signed URL failure with context- logger.error(- f"Failed to get signed URL for file {file.id} "- f"(storagePath={file.storage_path}): {e}",- exc_info=True,- )- # Fall back to streaming directly from GCS+ if not file.storage_path.startswith("local://"):try:- content = await storage.retrieve(file.storage_path)- return _create_streaming_response(content, file, inline=inline)- except Exception as fallback_error:- logger.error(- f"Fallback streaming also failed for file {file.id} "- f"(storagePath={file.storage_path}): {fallback_error}",- exc_info=True,+ url = await storage.get_download_url(file.storage_path, expires_in=300)+ except Exception as e:+ logger.warning(+ f"Could not sign a download URL for file {file.id}, "+ f"streaming it instead: {e!r}")- raise+ url = None+ # An /api/ path means the backend cannot sign URLs, so stream instead.+ if url and not url.startswith("/api/"):+ return fastapi.responses.RedirectResponse(url=url, status_code=302)++ content = await _read_file_content(storage, file)+ return _create_streaming_response(content, file, inline=inline)class WorkspaceFileUploadResponse(BaseModel):diff --git a/autogpt_platform/backend/backend/api/features/workspace/routes_test.py b/autogpt_platform/backend/backend/api/features/workspace/routes_test.pyindex 0065aeaeed..44993fd609 100644--- a/autogpt_platform/backend/backend/api/features/workspace/routes_test.py+++ b/autogpt_platform/backend/backend/api/features/workspace/routes_test.py@@ -2,6 +2,7 @@ import iofrom datetime import datetime, timezonefrom unittest.mock import AsyncMock, MagicMock, patch+import aiohttpimport fastapiimport fastapi.testclientimport pytest@@ -1171,6 +1172,105 @@ class TestCreateFileDownloadResponse:with pytest.raises(RuntimeError, match="Also failed"):await create_file_download_response(file)+ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "storage_path", ["gcs://bucket/file.txt", "local://ws/file-001/file.txt"]+ )+ async def test_missing_content_returns_404_without_retry(+ self, mocker, storage_path+ ):+ from backend.api.features.workspace.routes import create_file_download_response++ mock_storage = AsyncMock()+ mock_storage.get_download_url.return_value = "/api/fallback"+ mock_storage.retrieve.side_effect = FileNotFoundError("File not found")+ mocker.patch(+ "backend.api.features.workspace.routes.get_workspace_storage",+ return_value=mock_storage,+ )++ file = _make_file(storage_path=storage_path)+ with pytest.raises(fastapi.HTTPException) as exc_info:+ await create_file_download_response(file)+ assert exc_info.value.status_code == 404+ assert mock_storage.retrieve.await_count == 1++ @pytest.mark.asyncio+ async def test_transient_storage_error_is_retried(self, mocker, caplog):+ from backend.api.features.workspace.routes import create_file_download_response++ mock_storage = AsyncMock()+ mock_storage.get_download_url.return_value = "/api/fallback"+ mock_storage.retrieve.side_effect = [+ aiohttp.ClientPayloadError("Response payload is not completed"),+ b"second try",+ ]+ mocker.patch(+ "backend.api.features.workspace.routes.get_workspace_storage",+ return_value=mock_storage,+ )++ file = _make_file(storage_path="gcs://bucket/file.txt")+ with caplog.at_level("WARNING"):+ response = await create_file_download_response(file)+ assert response.status_code == 200+ assert response.body == b"second try"+ assert not [r for r in caplog.records if r.levelname == "ERROR"]++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "error",+ [+ aiohttp.ClientPayloadError("Response payload is not completed"),+ TimeoutError(),+ ],+ )+ async def test_persistent_storage_error_returns_502_and_logs(+ self, mocker, caplog, error+ ):+ from backend.api.features.workspace.routes import create_file_download_response++ mock_storage = AsyncMock()+ mock_storage.get_download_url.return_value = "/api/fallback"+ mock_storage.retrieve.side_effect = error+ mocker.patch(+ "backend.api.features.workspace.routes.get_workspace_storage",+ return_value=mock_storage,+ )++ file = _make_file(storage_path="gcs://bucket/file.txt")+ with caplog.at_level("WARNING"), pytest.raises(+ fastapi.HTTPException+ ) as exc_info:+ await create_file_download_response(file)+ assert exc_info.value.status_code == 502+ assert mock_storage.retrieve.await_count == 2+ errors = [r for r in caplog.records if r.levelname == "ERROR"]+ assert len(errors) == 1+ assert errors[0].exc_info is not None+ assert type(error).__name__ in errors[0].getMessage()+++def test_download_file_missing_content_returns_404(mocker):+ mocker.patch(+ "backend.api.features.workspace.routes.get_workspace",+ return_value=_make_workspace(),+ )+ mocker.patch(+ "backend.api.features.workspace.routes.get_workspace_file",+ return_value=_make_file(storage_path="gcs://bucket/ws/file-001/gone.png"),+ )+ mock_storage = AsyncMock()+ mock_storage.get_download_url.return_value = "/api/fallback"+ mock_storage.retrieve.side_effect = FileNotFoundError("File not found")+ mocker.patch(+ "backend.api.features.workspace.routes.get_workspace_storage",+ return_value=mock_storage,+ )++ response = client.get("/files/file-001/download")+ assert response.status_code == 404+# -- list_workspace_files: expert filter + attribution --diff --git a/autogpt_platform/backend/backend/util/cloud_storage.py b/autogpt_platform/backend/backend/util/cloud_storage.pyindex 3020d731fe..0597abad46 100644--- a/autogpt_platform/backend/backend/util/cloud_storage.py+++ b/autogpt_platform/backend/backend/util/cloud_storage.py@@ -13,7 +13,11 @@ import aiohttpfrom gcloud.aio import storage as async_gcs_storagefrom google.cloud import storage as gcs_storage-from backend.util.gcs_utils import download_with_fresh_session, generate_signed_url+from backend.util.gcs_utils import (+ download_with_fresh_session,+ generate_signed_url,+ is_not_found_error,+)from backend.util.settings import Configlogger = logging.getLogger(__name__)@@ -544,7 +548,7 @@ class CloudStorageHandler:except Exception as e:# If file doesn't exist or we can't read metadata- if "404" in str(e) or "Not Found" in str(e):+ if is_not_found_error(e):logger.warning(f"[CloudStorage] File not found during expiration check: {blob_name}")diff --git a/autogpt_platform/backend/backend/util/cloud_storage_test.py b/autogpt_platform/backend/backend/util/cloud_storage_test.pyindex d81187a879..92bd4d2b71 100644--- a/autogpt_platform/backend/backend/util/cloud_storage_test.py+++ b/autogpt_platform/backend/backend/util/cloud_storage_test.py@@ -4,9 +4,11 @@ Tests for cloud storage utilities.from unittest.mock import AsyncMock, MagicMock, patch+import aiohttpimport pytestfrom backend.util.cloud_storage import CloudStorageConfig, CloudStorageHandler+from backend.util.gcs_utils_test import _URL_CONTAINING_404, _gcs_http_errorclass TestCloudStorageHandler:@@ -101,8 +103,10 @@ class TestCloudStorageHandler:mock_async_client = AsyncMock()mock_storage_class.return_value = mock_async_client- # Mock the download method to raise a 404 exception- mock_async_client.download = AsyncMock(side_effect=Exception("404 Not Found"))+ # gcloud-aio raises aiohttp.ClientResponseError for a non-2xx response+ mock_async_client.download = AsyncMock(+ side_effect=_gcs_http_error(404, _URL_CONTAINING_404)+ )mock_async_client.close = AsyncMock()with pytest.raises(FileNotFoundError):@@ -110,6 +114,25 @@ class TestCloudStorageHandler:"gcs://test-bucket/uploads/system/uuid123/nonexistent.txt")+ @patch("backend.util.cloud_storage.async_gcs_storage.Storage")+ @pytest.mark.asyncio+ async def test_retrieve_file_keeps_non_404_errors(+ self, mock_storage_class, handler+ ):+ """A 503 whose object URL contains "404" is not a missing file."""+ mock_async_client = AsyncMock()+ mock_storage_class.return_value = mock_async_client+ mock_async_client.download = AsyncMock(+ side_effect=_gcs_http_error(503, _URL_CONTAINING_404)+ )+ mock_async_client.close = AsyncMock()++ with pytest.raises(aiohttp.ClientResponseError) as exc_info:+ await handler.retrieve_file(+ "gcs://test-bucket/uploads/system/uuid123/file.txt"+ )+ assert exc_info.value.status == 503+@patch.object(CloudStorageHandler, "_get_sync_gcs_client")@pytest.mark.asyncioasync def test_generate_signed_url_gcs(self, mock_get_sync_client, handler):@@ -211,6 +234,25 @@ class TestCloudStorageHandler:result = await handler.check_file_expired("gcs://test-bucket/valid-file.txt")assert result is False+ @patch.object(CloudStorageHandler, "_get_async_gcs_client")+ @pytest.mark.asyncio+ async def test_check_file_expired_gcs_by_status(+ self, mock_get_async_client, handler+ ):+ """Only a 404 counts as gone; other errors whose URL contains "404" don't."""+ mock_async_client = AsyncMock()+ mock_get_async_client.return_value = mock_async_client++ mock_async_client.download_metadata = AsyncMock(+ side_effect=_gcs_http_error(404, _URL_CONTAINING_404)+ )+ assert await handler.check_file_expired("gcs://test-bucket/gone.txt") is True++ mock_async_client.download_metadata = AsyncMock(+ side_effect=_gcs_http_error(401, _URL_CONTAINING_404)+ )+ assert await handler.check_file_expired("gcs://test-bucket/file.txt") is False+@patch("backend.util.cloud_storage.get_cloud_storage_handler")@pytest.mark.asyncioasync def test_cleanup_expired_files_async(self, mock_get_handler):diff --git a/autogpt_platform/backend/backend/util/gcs_utils.py b/autogpt_platform/backend/backend/util/gcs_utils.pyindex 10bb58d75c..180a17c318 100644--- a/autogpt_platform/backend/backend/util/gcs_utils.py+++ b/autogpt_platform/backend/backend/util/gcs_utils.py@@ -40,6 +40,16 @@ def parse_gcs_path(path: str) -> tuple[str, str]:return parts[0], parts[1]+def is_not_found_error(error: Exception) -> bool:+ """True only for a real GCS 404.++ Matching "404" in the message is not enough: the message carries the object+ URL, and file ids are UUIDs, so a 503 or 401 for a blob whose id happens to+ contain "404" would be misreported as a missing file.+ """+ return isinstance(error, aiohttp.ClientResponseError) and error.status == 404++async def download_with_fresh_session(bucket: str, blob: str) -> bytes:"""Download file content using a fresh session.@@ -66,8 +76,8 @@ async def download_with_fresh_session(bucket: str, blob: str) -> bytes:content = await client.download(bucket, blob)return contentexcept Exception as e:- if "404" in str(e) or "Not Found" in str(e):- raise FileNotFoundError(f"File not found: gcs://{bucket}/{blob}")+ if is_not_found_error(e):+ raise FileNotFoundError(f"File not found: gcs://{bucket}/{blob}") from eraisefinally:if client:@@ -112,8 +122,8 @@ async def download_range(bucket: str, blob: str, max_bytes: int) -> bytes:content = await client.download(bucket, blob)return content[:max_bytes]except Exception as e:- if "404" in str(e) or "Not Found" in str(e):- raise FileNotFoundError(f"File not found: gcs://{bucket}/{blob}")+ if is_not_found_error(e):+ raise FileNotFoundError(f"File not found: gcs://{bucket}/{blob}") from eraisefinally:if client:diff --git a/autogpt_platform/backend/backend/util/gcs_utils_test.py b/autogpt_platform/backend/backend/util/gcs_utils_test.pyindex 2be64a06fb..fa12b698e7 100644--- a/autogpt_platform/backend/backend/util/gcs_utils_test.py+++ b/autogpt_platform/backend/backend/util/gcs_utils_test.py@@ -1,8 +1,15 @@from unittest.mock import AsyncMock, MagicMock+import aiohttpimport pytest+from multidict import CIMultiDict, CIMultiDictProxy+from yarl import URL-from backend.util.gcs_utils import download_range, parse_gcs_path+from backend.util.gcs_utils import (+ download_range,+ download_with_fresh_session,+ parse_gcs_path,+)def _mock_client(mocker, *, download: AsyncMock) -> MagicMock:@@ -63,10 +70,50 @@ async def test_download_range_falls_back_when_headers_unsupported(mocker):assert download.await_count == 2+def _gcs_http_error(status: int, url: str) -> aiohttp.ClientResponseError:+ """The error gcloud-aio raises for a non-2xx GCS response."""+ request_info = aiohttp.RequestInfo(+ url=URL(url), method="GET", headers=CIMultiDictProxy(CIMultiDict())+ )+ return aiohttp.ClientResponseError(+ request_info, (), status=status, message="GCS error"+ )+++# File ids are UUIDs, so "404" turns up in object URLs by chance.+_URL_CONTAINING_404 = (+ "https://storage.googleapis.com/storage/v1/b/bucket/o/"+ "workspaces%2Fws%2F1aa50e64-f289-404d-889a-935d443c0ca0%2Fa.png?alt=media"+)++@pytest.mark.asyncioasync def test_download_range_maps_404_to_file_not_found(mocker):- download = AsyncMock(side_effect=Exception("404 Not Found"))+ download = AsyncMock(side_effect=_gcs_http_error(404, _URL_CONTAINING_404))_mock_client(mocker, download=download)with pytest.raises(FileNotFoundError):await download_range("bucket", "missing", 16)+++async def test_download_maps_404_to_file_not_found(mocker):+ download = AsyncMock(side_effect=_gcs_http_error(404, _URL_CONTAINING_404))+ _mock_client(mocker, download=download)++ with pytest.raises(FileNotFoundError):+ await download_with_fresh_session("bucket", "missing")++[email protected]("status", [401, 403, 500, 503])+async def test_download_keeps_other_errors_when_url_contains_404(mocker, status):+ download = AsyncMock(side_effect=_gcs_http_error(status, _URL_CONTAINING_404))+ _mock_client(mocker, download=download)++ with pytest.raises(aiohttp.ClientResponseError) as exc_info:+ await download_with_fresh_session("bucket", "blob")+ assert exc_info.value.status == status++ with pytest.raises(aiohttp.ClientResponseError):+ await download_range("bucket", "blob", 16)diff --git a/autogpt_platform/backend/backend/util/workspace_storage.py b/autogpt_platform/backend/backend/util/workspace_storage.pyindex b0462253eb..d821eb4866 100644--- a/autogpt_platform/backend/backend/util/workspace_storage.py+++ b/autogpt_platform/backend/backend/util/workspace_storage.py@@ -23,6 +23,7 @@ from backend.util.gcs_utils import (download_range,download_with_fresh_session,generate_signed_url,+ is_not_found_error,parse_gcs_path,)from backend.util.settings import Config@@ -204,7 +205,7 @@ class GCSWorkspaceStorage(WorkspaceStorageBackend):try:await client.delete(bucket_name, blob_name)except Exception as e:- if "404" not in str(e) and "Not Found" not in str(e):+ if not is_not_found_error(e):raise# File already deleted, that's finediff --git a/autogpt_platform/backend/backend/util/workspace_storage_test.py b/autogpt_platform/backend/backend/util/workspace_storage_test.pyindex b88e2332e7..d41a20e674 100644--- a/autogpt_platform/backend/backend/util/workspace_storage_test.py+++ b/autogpt_platform/backend/backend/util/workspace_storage_test.py@@ -1,7 +1,9 @@-from unittest.mock import AsyncMock+from unittest.mock import AsyncMock, MagicMock+import aiohttpimport pytest+from backend.util.gcs_utils_test import _URL_CONTAINING_404, _gcs_http_errorfrom backend.util.workspace_storage import GCSWorkspaceStorage, LocalWorkspaceStorage@@ -45,3 +47,31 @@ async def test_gcs_retrieve_partial_delegates_to_download_range(mocker):assert result == b"head"download_range.assert_awaited_once_with("my-bucket", "path/file.txt", 4)+++def _gcs_storage_whose_delete_raises(mocker, error: Exception) -> GCSWorkspaceStorage:+ storage = GCSWorkspaceStorage(bucket_name="my-bucket")+ client = MagicMock()+ client.delete = AsyncMock(side_effect=error)+ mocker.patch.object(storage, "_get_async_client", AsyncMock(return_value=client))+ return storage+++async def test_gcs_delete_ignores_already_deleted_file(mocker):+ storage = _gcs_storage_whose_delete_raises(+ mocker, _gcs_http_error(404, _URL_CONTAINING_404)+ )++ await storage.delete("gcs://my-bucket/path/file.txt")++[email protected]("status", [401, 403, 503])+async def test_gcs_delete_raises_other_errors_when_url_contains_404(mocker, status):+ storage = _gcs_storage_whose_delete_raises(+ mocker, _gcs_http_error(status, _URL_CONTAINING_404)+ )++ with pytest.raises(aiohttp.ClientResponseError):+ await storage.delete("gcs://my-bucket/path/file.txt")