Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
refactor(api): centralize remote file handling (#41627)
非法操作ba1d5ba172ad
diff --git a/api/controllers/common/errors.py b/api/controllers/common/errors.pyindex 31dd2fcd74f..c865e5eceb4 100644--- a/api/controllers/common/errors.py+++ b/api/controllers/common/errors.py@@ -1,18 +1,54 @@-from werkzeug.exceptions import HTTPException-from libs.exception import BaseHTTPException-class FilenameNotExistsError(HTTPException):+class FilenameNotExistsError(BaseHTTPException):+ error_code = "filename_not_exists_error"code = 400description = "The specified filename does not exist."-class RemoteFileUploadError(HTTPException):+class RemoteFileUploadError(BaseHTTPException):+ error_code = "remote_file_upload_error"code = 400description = "Error uploading remote file."+class RemoteFileInvalidUrlError(BaseHTTPException):+ error_code = "remote_file_invalid_url"+ description = "The remote file URL is invalid."+ code = 400+++class RemoteFileUrlBlockedError(BaseHTTPException):+ error_code = "remote_file_url_blocked"+ description = "The remote file URL is not allowed."+ code = 400+++class RemoteFileNotFoundError(BaseHTTPException):+ error_code = "remote_file_not_found"+ description = "The remote file could not be found."+ code = 404+++class RemoteFileAccessDeniedError(BaseHTTPException):+ error_code = "remote_file_access_denied"+ description = "The remote file cannot be accessed without authorization."+ code = 400+++class RemoteFileUnavailableError(BaseHTTPException):+ error_code = "remote_file_unavailable"+ description = "The remote file is temporarily unavailable."+ code = 502+++class RemoteFileInvalidResponseError(BaseHTTPException):+ error_code = "remote_file_invalid_response"+ description = "The remote file server returned an invalid response."+ code = 502++class FileTooLargeError(BaseHTTPException):error_code = "file_too_large"description = "File size exceeded. {message}"diff --git a/api/controllers/common/helpers.py b/api/controllers/common/helpers.pyindex 56a07a8b4a9..23d5e2c56b7 100644--- a/api/controllers/common/helpers.py+++ b/api/controllers/common/helpers.py@@ -1,39 +1,4 @@-import contextlib-import mimetypes-import os-import platform-import reimport urllib.parse-import warnings-from uuid import uuid4--import httpx--try:- import magic-except ImportError:- if platform.system() == "Windows":- warnings.warn(- "To use python-magic guess MIMETYPE, you need to run `pip install python-magic-bin`", stacklevel=2- )- elif platform.system() == "Darwin":- warnings.warn("To use python-magic guess MIMETYPE, you need to run `brew install libmagic`", stacklevel=2)- elif platform.system() == "Linux":- warnings.warn(- "To use python-magic guess MIMETYPE, you need to run `sudo apt-get install libmagic1`", stacklevel=2- )- else:- warnings.warn("To use python-magic guess MIMETYPE, you need to install `libmagic`", stacklevel=2)- magic = None # type: ignore[assignment]--from pydantic import BaseModel---class FileInfo(BaseModel):- filename: str- extension: str- mimetype: str- size: intdef decode_remote_url(url: str, query_string: bytes | str = b"") -> str:@@ -45,59 +10,15 @@ def decode_remote_url(url: str, query_string: bytes | str = b"") -> str:if not raw_query:return decoded_url+ try:+ has_query = bool(urllib.parse.urlsplit(decoded_url).query)+ except ValueError:+ has_query = False+if decoded_url.endswith(("?", "&")):separator = ""- elif urllib.parse.urlsplit(decoded_url).query:+ elif has_query:separator = "&"else:separator = "?"return f"{decoded_url}{separator}{raw_query}"---def guess_file_info_from_response(response: httpx.Response):- url = str(response.url)- # Try to extract filename from URL- parsed_url = urllib.parse.urlparse(url)- url_path = parsed_url.path- # Decode percent-encoded characters in the path segment- filename = urllib.parse.unquote(os.path.basename(url_path))-- # If filename couldn't be extracted, use Content-Disposition header- if not filename:- content_disposition = response.headers.get("Content-Disposition")- if content_disposition:- filename_match = re.search(r'filename="?(.+)"?', content_disposition)- if filename_match:- filename = filename_match.group(1)-- # If still no filename, generate a unique one- if not filename:- unique_name = str(uuid4())- filename = f"{unique_name}"-- # Guess MIME type from filename first, then URL- mimetype, _ = mimetypes.guess_type(filename)- if mimetype is None:- mimetype, _ = mimetypes.guess_type(url)- if mimetype is None:- # If guessing fails, use Content-Type from response headers- mimetype = response.headers.get("Content-Type", "application/octet-stream")-- # Use python-magic to guess MIME type if still unknown or generic- if mimetype == "application/octet-stream" and magic is not None:- with contextlib.suppress(magic.MagicException):- mimetype = magic.from_buffer(response.content[:1024], mime=True)-- extension = os.path.splitext(filename)[1]-- # Ensure filename has an extension- if not extension:- extension = mimetypes.guess_extension(mimetype) or ".bin"- filename = f"{filename}{extension}"-- return FileInfo(- filename=filename,- extension=extension,- mimetype=mimetype,- size=int(response.headers.get("Content-Length", -1)),- )diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.pyindex f14fe1eca95..87ff0317971 100644--- a/api/controllers/console/app/app.py+++ b/api/controllers/console/app/app.py@@ -13,7 +13,6 @@ from werkzeug.exceptions import BadRequest, Forbidden, NotFoundfrom configs import dify_configfrom controllers.common.app_access import resolve_app_access_filterfrom controllers.common.fields import RedirectUrlResponse, SimpleResultResponse-from controllers.common.helpers import FileInfofrom controllers.common.schema import (query_params_from_model,query_params_from_request,@@ -39,6 +38,7 @@ from controllers.console.wraps import (with_current_user,with_current_user_id,)+from core.file.remote_file_metadata import FileInfofrom core.ops.ops_trace_manager import OpsTraceManagerfrom core.rag.entities import PreProcessingRule, Rule, Segmentationfrom core.rag.retrieval.retrieval_methods import RetrievalMethoddiff --git a/api/controllers/console/explore/trial.py b/api/controllers/console/explore/trial.pyindex d5388c7b542..e481c80b756 100644--- a/api/controllers/console/explore/trial.py+++ b/api/controllers/console/explore/trial.py@@ -47,7 +47,7 @@ from controllers.console.explore.error import ()from controllers.console.explore.wraps import TrialAppResourcefrom controllers.console.files import FILE_UPLOAD_PARAMS, upload_file_from_request-from controllers.console.remote_files import RemoteFileUploadPayload, upload_remote_file_from_request+from controllers.console.remote_files import RemoteFileUploadPayload, upload_remote_filefrom controllers.console.wraps import cloud_edition_billing_resource_check, model_validate, with_current_userfrom controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpErrorfrom core.app.apps.base_app_queue_manager import AppQueueManager@@ -472,13 +472,15 @@ class TrialAppRemoteFileUploadApi(TrialAppResource):@console_ns.expect(console_ns.models[RemoteFileUploadPayload.__name__])@console_ns.response(201, "File uploaded successfully", console_ns.models[FileWithSignedUrl.__name__])@with_current_user- def post(self, current_user: Account, app_model: App):+ @model_validate(RemoteFileUploadPayload)+ def post(self, payload: RemoteFileUploadPayload, current_user: Account, app_model: App):"""Upload a remote file into the tenant that owns the trial app."""- remote_file = upload_remote_file_from_request(+ remote_file = upload_remote_file(+ url=payload.url,current_user=current_user,resource_tenant_id=app_model.tenant_id,)- return remote_file.model_dump(mode="json"), 201+ return dump_response(FileWithSignedUrl, remote_file), 201class TrialAppWorkflowRunApi(TrialAppResource):diff --git a/api/controllers/console/remote_files.py b/api/controllers/console/remote_files.pyindex e3aefb72595..e7e4eef2f53 100644--- a/api/controllers/console/remote_files.py+++ b/api/controllers/console/remote_files.py@@ -1,4 +1,3 @@-import httpxfrom flask import requestfrom flask_restx import Resourcefrom pydantic import BaseModel, Field@@ -6,24 +5,41 @@ from pydantic import BaseModel, Fieldimport servicesfrom controllers.common import helpersfrom controllers.common.errors import (+ BlockedFileExtensionError,FileTooLargeError,- RemoteFileUploadError,+ RemoteFileAccessDeniedError,+ RemoteFileInvalidResponseError,+ RemoteFileInvalidUrlError,+ RemoteFileNotFoundError,+ RemoteFileUnavailableError,+ RemoteFileUrlBlockedError,UnsupportedFileTypeError,)-from controllers.common.schema import register_response_schema_models, register_schema_models+from controllers.common.schema import JsonResponseWithStatus, register_response_schema_models, register_schema_modelsfrom controllers.console import console_ns-from controllers.console.wraps import with_current_user-from core.file import remote_fetcher-from extensions.ext_database import db+from controllers.console.flask_admission import console_account_admission+from controllers.console.wraps import model_validate, with_current_user+from extensions.ext_application_services import application_servicesfrom fields.file_fields import FileWithSignedUrl, RemoteFileInfo-from graphon.file import helpers as file_helpers+from libs.helper import dump_responsefrom libs.login import login_required+from machinery.context import RequestContextfrom models import Account-from services.file_service import FileService+from services.remote_file_service import (+ RemoteFileAccessDeniedError as RemoteFileAccessDeniedServiceError,+)+from services.remote_file_service import (+ RemoteFileInvalidResponseError as RemoteFileInvalidResponseServiceError,+)+from services.remote_file_service import RemoteFileInvalidUrlError as RemoteFileInvalidUrlServiceError+from services.remote_file_service import RemoteFileNotFoundError as RemoteFileNotFoundServiceError+from services.remote_file_service import RemoteFileUnavailableError as RemoteFileUnavailableServiceError+from services.remote_file_service import RemoteFileUploadResult+from services.remote_file_service import RemoteFileUrlBlockedError as RemoteFileUrlBlockedServiceErrorclass RemoteFileUploadPayload(BaseModel):- url: str = Field(..., description="URL to fetch")+ url: str = Field(description="URL to fetch", json_schema_extra={"format": "uri"})register_schema_models(console_ns, RemoteFileUploadPayload)@@ -32,84 +48,96 @@ register_response_schema_models(console_ns, FileWithSignedUrl, RemoteFileInfo)@console_ns.route("/remote-files/<path:url>")class GetRemoteFileInfo(Resource):+ @console_ns.doc(+ responses={+ 400: "Invalid, blocked, or inaccessible remote file URL",+ 404: "Remote file not found",+ 502: "Remote file unavailable or returned an invalid response",+ 500: "Internal server error",+ }+ )@console_ns.response(200, "Success", console_ns.models[RemoteFileInfo.__name__])- @login_required- def get(self, url: str):+ @console_account_admission()+ def get(self, _request_context: RequestContext, url: str):decoded_url = helpers.decode_remote_url(url, request.query_string)- resp = remote_fetcher.make_request("HEAD", decoded_url)- if resp.status_code != httpx.codes.OK:- resp = remote_fetcher.make_request("GET", decoded_url, timeout=3)- resp.raise_for_status()- return RemoteFileInfo(- file_type=resp.headers.get("Content-Type", "application/octet-stream"),- file_length=int(resp.headers.get("Content-Length", 0)),- ).model_dump(mode="json")+ try:+ file_info = application_services().remote_files.fetch_info(url=decoded_url)+ except RemoteFileInvalidUrlServiceError as error:+ raise RemoteFileInvalidUrlError from error+ except RemoteFileUrlBlockedServiceError as error:+ raise RemoteFileUrlBlockedError from error+ except RemoteFileNotFoundServiceError as error:+ raise RemoteFileNotFoundError from error+ except RemoteFileAccessDeniedServiceError as error:+ raise RemoteFileAccessDeniedError from error+ except RemoteFileUnavailableServiceError as error:+ raise RemoteFileUnavailableError from error+ except RemoteFileInvalidResponseServiceError as error:+ raise RemoteFileInvalidResponseError from error++ return dump_response(+ RemoteFileInfo,+ {+ "file_type": file_info.content_type,+ "file_length": file_info.content_length if file_info.content_length is not None else 0,+ },+ )-def upload_remote_file_from_request(+def upload_remote_file(*,+ url: str,current_user: Account,resource_tenant_id: str | None = None,-) -> FileWithSignedUrl:- payload = RemoteFileUploadPayload.model_validate(console_ns.payload)- """Validate the JSON request, fetch its remote file, and persist it under the requested tenant."""- url = payload.url-- # Try to fetch remote file metadata/content first- try:- resp = remote_fetcher.make_request("HEAD", url=url)- if resp.status_code != httpx.codes.OK:- resp = remote_fetcher.make_request("GET", url=url, timeout=3, follow_redirects=True)- if resp.status_code != httpx.codes.OK:- # Normalize into a user-friendly error message expected by tests- raise RemoteFileUploadError(f"Failed to fetch file from {url}: {resp.text}")- except httpx.RequestError as e:- raise RemoteFileUploadError(f"Failed to fetch file from {url}: {str(e)}")-- file_info = helpers.guess_file_info_from_response(resp)-- # Enforce file size limit with 400 (Bad Request) per tests' expectation- if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):- raise FileTooLargeError()-- # Load content if needed- content = resp.content if resp.request.method == "GET" else remote_fetcher.make_request("GET", url).content-+) -> RemoteFileUploadResult:+ """Fetch a remote file and persist it under the requested tenant."""try:- upload_file = FileService(db.engine).upload_file(- filename=file_info.filename,- content=content,- mimetype=file_info.mimetype,+ return application_services().remote_files.upload_from_url(+ url=url,user=current_user,tenant_id=resource_tenant_id,- source_url=url,)- except services.errors.file.FileTooLargeError as file_too_large_error:- raise FileTooLargeError(file_too_large_error.description)- except services.errors.file.UnsupportedFileTypeError:- raise UnsupportedFileTypeError()-- return FileWithSignedUrl(- id=upload_file.id,- name=upload_file.name,- size=upload_file.size,- extension=upload_file.extension,- url=file_helpers.get_signed_file_url(upload_file_id=upload_file.id),- mime_type=upload_file.mime_type,- created_by=upload_file.created_by,- created_at=int(upload_file.created_at.timestamp()),- )+ except RemoteFileInvalidUrlServiceError as error:+ raise RemoteFileInvalidUrlError from error+ except RemoteFileUrlBlockedServiceError as error:+ raise RemoteFileUrlBlockedError from error+ except RemoteFileNotFoundServiceError as error:+ raise RemoteFileNotFoundError from error+ except RemoteFileAccessDeniedServiceError as error:+ raise RemoteFileAccessDeniedError from error+ except RemoteFileUnavailableServiceError as error:+ raise RemoteFileUnavailableError from error+ except RemoteFileInvalidResponseServiceError as error:+ raise RemoteFileInvalidResponseError from error+ except services.errors.file.FileTooLargeError as error:+ raise FileTooLargeError(error.description or "File size exceeded.") from error+ except services.errors.file.UnsupportedFileTypeError as error:+ raise UnsupportedFileTypeError from error+ except services.errors.file.BlockedFileExtensionError as error:+ raise BlockedFileExtensionError(error.description) from error@console_ns.route("/remote-files/upload")class RemoteFileUpload(Resource):+ @console_ns.doc(+ responses={+ 400: "Invalid, blocked, or inaccessible remote file URL",+ 404: "Remote file not found",+ 413: "File too large",+ 415: "Unsupported file type",+ 422: "Request payload validation failed",+ 502: "Remote file unavailable or returned an invalid response",+ 500: "Internal server error",+ }+ )@console_ns.expect(console_ns.models[RemoteFileUploadPayload.__name__])@console_ns.response(201, "File uploaded successfully", console_ns.models[FileWithSignedUrl.__name__])@login_required@with_current_user- def post(self, current_user: Account):- remote_file = upload_remote_file_from_request(current_user=current_user)- return (- remote_file.model_dump(mode="json"),- 201,+ @model_validate(RemoteFileUploadPayload)+ def post(self, payload: RemoteFileUploadPayload, current_user: Account) -> JsonResponseWithStatus:+ remote_file = upload_remote_file(+ url=payload.url,+ current_user=current_user,)+ return dump_response(FileWithSignedUrl, remote_file), 201diff --git a/api/controllers/web/human_input_file_upload.py b/api/controllers/web/human_input_file_upload.pyindex 5203951a993..dcddec8f35b 100644--- a/api/controllers/web/human_input_file_upload.py+++ b/api/controllers/web/human_input_file_upload.py@@ -13,7 +13,6 @@ from pydantic import BaseModel, ConfigDict, Field, HttpUrlfrom sqlalchemy.orm import sessionmakerimport services-from controllers.common import helpersfrom controllers.common.errors import (BlockedFileExtensionError,FileTooLargeError,@@ -24,6 +23,7 @@ from controllers.common.errors import ()from controllers.common.schema import register_schema_modelsfrom controllers.web import web_ns+from core.file.remote_file_metadata import guess_file_info_from_responsefrom core.helper import ssrf_proxyfrom extensions.ext_database import dbfrom fields.file_fields import FileResponse, FileWithSignedUrl@@ -155,7 +155,7 @@ def _upload_remote_file(context, url: str):except httpx.RequestError as exc:raise RemoteFileUploadError(f"Failed to fetch file from {url}: {str(exc)}")- file_info = helpers.guess_file_info_from_response(resp)+ file_info = guess_file_info_from_response(resp)if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):raise FileTooLargeError()diff --git a/api/controllers/web/remote_files.py b/api/controllers/web/remote_files.pyindex ab099077b79..51004073c91 100644--- a/api/controllers/web/remote_files.py+++ b/api/controllers/web/remote_files.py@@ -1,21 +1,34 @@-import httpxfrom flask import request-from pydantic import BaseModel, Field, HttpUrl+from pydantic import BaseModel, Fieldimport servicesfrom controllers.common import helpersfrom controllers.common.errors import (+ BlockedFileExtensionError,FileTooLargeError,- RemoteFileUploadError,+ RemoteFileAccessDeniedError,+ RemoteFileInvalidResponseError,+ RemoteFileInvalidUrlError,+ RemoteFileNotFoundError,+ RemoteFileUnavailableError,+ RemoteFileUrlBlockedError,UnsupportedFileTypeError,)from controllers.console.wraps import model_validate-from core.file import remote_fetcher-from extensions.ext_database import db+from extensions.ext_application_services import application_servicesfrom fields.file_fields import FileWithSignedUrl, RemoteFileInfo-from graphon.file import helpers as file_helpers+from libs.helper import dump_responsefrom models.model import App, EndUser-from services.file_service import FileService+from services.remote_file_service import (+ RemoteFileAccessDeniedError as RemoteFileAccessDeniedServiceError,+)+from services.remote_file_service import (+ RemoteFileInvalidResponseError as RemoteFileInvalidResponseServiceError,+)+from services.remote_file_service import RemoteFileInvalidUrlError as RemoteFileInvalidUrlServiceError+from services.remote_file_service import RemoteFileNotFoundError as RemoteFileNotFoundServiceError+from services.remote_file_service import RemoteFileUnavailableError as RemoteFileUnavailableServiceError+from services.remote_file_service import RemoteFileUrlBlockedError as RemoteFileUrlBlockedServiceErrorfrom ..common.schema import register_response_schema_models, register_schema_modelsfrom . import web_ns@@ -23,7 +36,7 @@ from .wraps import WebApiResourceclass RemoteFileUploadPayload(BaseModel):- url: HttpUrl = Field(description="Remote file URL")+ url: str = Field(description="Remote file URL", json_schema_extra={"format": "uri"})register_schema_models(web_ns, RemoteFileUploadPayload)@@ -37,9 +50,10 @@ class RemoteFileInfoApi(WebApiResource):@web_ns.doc(responses={200: "Remote file information retrieved successfully",- 400: "Bad request - invalid URL",+ 400: "Invalid, blocked, or inaccessible remote file URL",404: "Remote file not found",- 500: "Failed to fetch remote file",+ 502: "Remote file unavailable or returned an invalid response",+ 500: "Internal server error",})@web_ns.response(200, "Remote file info", web_ns.models[RemoteFileInfo.__name__])@@ -61,15 +75,28 @@ class RemoteFileInfoApi(WebApiResource):HTTPException: If the remote file cannot be accessed"""decoded_url = helpers.decode_remote_url(url, request.query_string)- resp = remote_fetcher.make_request("HEAD", decoded_url)- if resp.status_code != httpx.codes.OK:- # failed back to get method- resp = remote_fetcher.make_request("GET", decoded_url, timeout=3)- resp.raise_for_status()- return RemoteFileInfo(- file_type=resp.headers.get("Content-Type", "application/octet-stream"),- file_length=int(resp.headers.get("Content-Length", -1)),- ).model_dump(mode="json")+ try:+ file_info = application_services().remote_files.fetch_info(url=decoded_url)+ except RemoteFileInvalidUrlServiceError as error:+ raise RemoteFileInvalidUrlError from error+ except RemoteFileUrlBlockedServiceError as error:+ raise RemoteFileUrlBlockedError from error+ except RemoteFileNotFoundServiceError as error:+ raise RemoteFileNotFoundError from error+ except RemoteFileAccessDeniedServiceError as error:+ raise RemoteFileAccessDeniedError from error+ except RemoteFileUnavailableServiceError as error:+ raise RemoteFileUnavailableError from error+ except RemoteFileInvalidResponseServiceError as error:+ raise RemoteFileInvalidResponseError from error++ return dump_response(+ RemoteFileInfo,+ {+ "file_type": file_info.content_type,+ "file_length": file_info.content_length if file_info.content_length is not None else -1,+ },+ )@web_ns.route("/remote-files/upload")@@ -79,10 +106,13 @@ class RemoteFileUploadApi(WebApiResource):@web_ns.doc(responses={201: "Remote file uploaded successfully",- 400: "Bad request - invalid URL or parameters",+ 400: "Invalid, blocked, or inaccessible remote file URL",+ 404: "Remote file not found",413: "File too large",415: "Unsupported file type",- 500: "Failed to fetch remote file",+ 422: "Request payload validation failed",+ 502: "Remote file unavailable or returned an invalid response",+ 500: "Internal server error",})@web_ns.response(201, "Remote file uploaded", web_ns.models[FileWithSignedUrl.__name__])@@ -106,48 +136,37 @@ class RemoteFileUploadApi(WebApiResource):int: HTTP status code 201 for successRaises:- RemoteFileUploadError: Failed to fetch file from remote URL+ RemoteFileInvalidUrlError: Remote file URL is invalid+ RemoteFileUrlBlockedError: Remote file URL is blocked+ RemoteFileNotFoundError: Remote file does not exist+ RemoteFileAccessDeniedError: Remote file requires authorization+ RemoteFileUnavailableError: Remote file is unavailable+ RemoteFileInvalidResponseError: Remote file response is invalidFileTooLargeError: File exceeds size limitUnsupportedFileTypeError: File type not supported"""- url = str(payload.url)-- try:- resp = remote_fetcher.make_request("HEAD", url=url)- if resp.status_code != httpx.codes.OK:- resp = remote_fetcher.make_request("GET", url=url, timeout=3, follow_redirects=True)- if resp.status_code != httpx.codes.OK:- raise RemoteFileUploadError(f"Failed to fetch file from {url}: {resp.text}")- except httpx.RequestError as e:- raise RemoteFileUploadError(f"Failed to fetch file from {url}: {str(e)}")-- file_info = helpers.guess_file_info_from_response(resp)-- if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):- raise FileTooLargeError-- content = resp.content if resp.request.method == "GET" else remote_fetcher.make_request("GET", url).content-try:- upload_file = FileService(db.engine).upload_file(- filename=file_info.filename,- content=content,- mimetype=file_info.mimetype,+ remote_file = application_services().remote_files.upload_from_url(+ url=payload.url,user=end_user,- source_url=url,)- except services.errors.file.FileTooLargeError as file_too_large_error:- raise FileTooLargeError(file_too_large_error.description)- except services.errors.file.UnsupportedFileTypeError:- raise UnsupportedFileTypeError-- return FileWithSignedUrl(- id=upload_file.id,- name=upload_file.name,- size=upload_file.size,- extension=upload_file.extension,- url=file_helpers.get_signed_file_url(upload_file_id=upload_file.id),- mime_type=upload_file.mime_type,- created_by=upload_file.created_by,- created_at=int(upload_file.created_at.timestamp()),- ).model_dump(mode="json"), 201+ except RemoteFileInvalidUrlServiceError as error:+ raise RemoteFileInvalidUrlError from error+ except RemoteFileUrlBlockedServiceError as error:+ raise RemoteFileUrlBlockedError from error+ except RemoteFileNotFoundServiceError as error:+ raise RemoteFileNotFoundError from error+ except RemoteFileAccessDeniedServiceError as error:+ raise RemoteFileAccessDeniedError from error+ except RemoteFileUnavailableServiceError as error:+ raise RemoteFileUnavailableError from error+ except RemoteFileInvalidResponseServiceError as error:+ raise RemoteFileInvalidResponseError from error+ except services.errors.file.FileTooLargeError as error:+ raise FileTooLargeError(error.description or "File size exceeded.") from error+ except services.errors.file.UnsupportedFileTypeError as error:+ raise UnsupportedFileTypeError from error+ except services.errors.file.BlockedFileExtensionError as error:+ raise BlockedFileExtensionError(error.description) from error++ return dump_response(FileWithSignedUrl, remote_file), 201diff --git a/api/core/file/remote_file_metadata.py b/api/core/file/remote_file_metadata.pynew file mode 100644index 00000000000..d4d14860fcc--- /dev/null+++ b/api/core/file/remote_file_metadata.py@@ -0,0 +1,94 @@+import contextlib+import mimetypes+import os+import platform+import re+import urllib.parse+import warnings+from uuid import uuid4++import httpx++try:+ import magic+except ImportError:+ if platform.system() == "Windows":+ warnings.warn(+ "To use python-magic guess MIMETYPE, you need to run `pip install python-magic-bin`", stacklevel=2+ )+ elif platform.system() == "Darwin":+ warnings.warn("To use python-magic guess MIMETYPE, you need to run `brew install libmagic`", stacklevel=2)+ elif platform.system() == "Linux":+ warnings.warn(+ "To use python-magic guess MIMETYPE, you need to run `sudo apt-get install libmagic1`", stacklevel=2+ )+ else:+ warnings.warn("To use python-magic guess MIMETYPE, you need to install `libmagic`", stacklevel=2)+ magic = None # type: ignore[assignment]++from pydantic import BaseModel+++class FileInfo(BaseModel):+ filename: str+ extension: str+ mimetype: str+ size: int+++class InvalidRemoteFileMetadataError(ValueError):+ pass+++def guess_file_info_from_response(response: httpx.Response) -> FileInfo:+ url = str(response.url)+ # Try to extract filename from URL+ parsed_url = urllib.parse.urlparse(url)+ url_path = parsed_url.path+ # Decode percent-encoded characters in the path segment+ filename = urllib.parse.unquote(os.path.basename(url_path))++ # If filename couldn't be extracted, use Content-Disposition header+ if not filename:+ content_disposition = response.headers.get("Content-Disposition")+ if content_disposition:+ filename_match = re.search(r'filename="?(.+)"?', content_disposition)+ if filename_match:+ filename = filename_match.group(1)++ # If still no filename, generate a unique one+ if not filename:+ unique_name = str(uuid4())+ filename = f"{unique_name}"++ # Guess MIME type from filename first, then URL+ mimetype, _ = mimetypes.guess_type(filename)+ if mimetype is None:+ mimetype, _ = mimetypes.guess_type(url)+ if mimetype is None:+ # If guessing fails, use Content-Type from response headers+ mimetype = response.headers.get("Content-Type", "application/octet-stream")++ # Use python-magic to guess MIME type if still unknown or generic+ if mimetype == "application/octet-stream" and magic is not None:+ with contextlib.suppress(magic.MagicException):+ mimetype = magic.from_buffer(response.content[:1024], mime=True)++ extension = os.path.splitext(filename)[1]++ # Ensure filename has an extension+ if not extension:+ extension = mimetypes.guess_extension(mimetype) or ".bin"+ filename = f"{filename}{extension}"++ try:+ size = int(response.headers.get("Content-Length", -1))+ except ValueError as error:+ raise InvalidRemoteFileMetadataError("The Content-Length header is invalid") from error++ return FileInfo(+ filename=filename,+ extension=extension,+ mimetype=mimetype,+ size=size,+ )diff --git a/api/extensions/ext_application_services.py b/api/extensions/ext_application_services.pyindex 1ceaf4a4fcc..986fc5a79c4 100644--- a/api/extensions/ext_application_services.py+++ b/api/extensions/ext_application_services.py@@ -129,6 +129,7 @@ from services.recommended_app_catalog_gateway import (RemoteRecommendedAppCatalogGateway,)from services.recommended_app_query_service import RecommendedAppQueryService+from services.remote_file_service import RemoteFileServicefrom services.retention.workflow_run.archive_download_adapters import (dispatch_workflow_run_archive_download_task,sign_workflow_run_archive_download_url,@@ -218,6 +219,7 @@ class ApplicationServices:step_by_step_tour: StepByStepTourServicepartner_tenant_bindings: PartnerTenantBindingServicerecommended_app_queries: RecommendedAppQueryService+ remote_files: RemoteFileServicetrial_app_usage: TrialAppUsageRecorderworkflow_run_archives: WorkflowRunArchiveServiceworkspace_queries: WorkspaceQueryService@@ -502,6 +504,9 @@ def build_application_services(trial_apps=TrialAppQueryRepository(session_factory=database_client),trial_enabled=trial_app_enabled,),+ remote_files=RemoteFileService(+ files=FileService(session_factory=database_client),+ ),trial_app_usage=TrialAppUsageRepository(session_factory=database_client),workflow_run_archives=WorkflowRunArchiveService(bundles=WorkflowRunArchiveBundleQueryRepository(session_factory=database_client),diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.mdindex 79313ba2ebb..4a63869ccd9 100644--- a/api/openapi/markdown/console-openapi.md+++ b/api/openapi/markdown/console-openapi.md@@ -8393,6 +8393,13 @@ Update account-level Step-by-step Tour state| Code | Description | Schema || ---- | ----------- | ------ || 201 | File uploaded successfully | **application/json**: [FileWithSignedUrl](#filewithsignedurl)<br> |+| 400 | Invalid, blocked, or inaccessible remote file URL | |+| 404 | Remote file not found | |+| 413 | File too large | |+| 415 | Unsupported file type | |+| 422 | Request payload validation failed | |+| 500 | Internal server error | |+| 502 | Remote file unavailable or returned an invalid response | |### [GET] /remote-files/{url}#### Parameters@@ -8406,6 +8413,10 @@ Update account-level Step-by-step Tour state| Code | Description | Schema || ---- | ----------- | ------ || 200 | Success | **application/json**: [RemoteFileInfo](#remotefileinfo)<br> |+| 400 | Invalid, blocked, or inaccessible remote file URL | |+| 404 | Remote file not found | |+| 500 | Internal server error | |+| 502 | Remote file unavailable or returned an invalid response | |### [POST] /reset-password#### Request Body@@ -21347,7 +21358,7 @@ Model class for provider quota configuration.| Name | Type | Description | Required || ---- | ---- | ----------- | -------- |-| url | string | URL to fetch | Yes |+| url | string (uri) | URL to fetch | Yes |#### ReplaceUserAccessPoliciesdiff --git a/api/openapi/markdown/web-openapi.md b/api/openapi/markdown/web-openapi.mdindex 1cb85bf66d8..1f6d7a8e723 100644--- a/api/openapi/markdown/web-openapi.md+++ b/api/openapi/markdown/web-openapi.md@@ -638,7 +638,12 @@ Returns:int: HTTP status code 201 for successRaises:- RemoteFileUploadError: Failed to fetch file from remote URL+ RemoteFileInvalidUrlError: Remote file URL is invalid+ RemoteFileUrlBlockedError: Remote file URL is blocked+ RemoteFileNotFoundError: Remote file does not exist+ RemoteFileAccessDeniedError: Remote file requires authorization+ RemoteFileUnavailableError: Remote file is unavailable+ RemoteFileInvalidResponseError: Remote file response is invalidFileTooLargeError: File exceeds size limitUnsupportedFileTypeError: File type not supported@@ -653,10 +658,13 @@ Raises:| Code | Description | Schema || ---- | ----------- | ------ || 201 | Remote file uploaded successfully | **application/json**: [FileWithSignedUrl](#filewithsignedurl)<br> |-| 400 | Bad request - invalid URL or parameters | |+| 400 | Invalid, blocked, or inaccessible remote file URL | |+| 404 | Remote file not found | || 413 | File too large | || 415 | Unsupported file type | |-| 500 | Failed to fetch remote file | |+| 422 | Request payload validation failed | |+| 500 | Internal server error | |+| 502 | Remote file unavailable or returned an invalid response | |### [GET] /remote-files/{url}**Get information about a remote file**@@ -687,9 +695,10 @@ Raises:| Code | Description | Schema || ---- | ----------- | ------ || 200 | Remote file information retrieved successfully | **application/json**: [RemoteFileInfo](#remotefileinfo)<br> |-| 400 | Bad request - invalid URL | |+| 400 | Invalid, blocked, or inaccessible remote file URL | || 404 | Remote file not found | |-| 500 | Failed to fetch remote file | |+| 500 | Internal server error | |+| 502 | Remote file unavailable or returned an invalid response | |### [GET] /saved-messagesRetrieve paginated list of saved messages for a completion application.diff --git a/api/services/remote_file_service.py b/api/services/remote_file_service.pynew file mode 100644index 00000000000..1ccb63ebb9a--- /dev/null+++ b/api/services/remote_file_service.py@@ -0,0 +1,185 @@+import urllib.parse+from dataclasses import dataclass+from datetime import datetime+from typing import Any, Literal++import httpx++from core.file import remote_fetcher+from core.file.remote_file_metadata import InvalidRemoteFileMetadataError, guess_file_info_from_response+from core.helper.ssrf_proxy import MaxRetriesExceededError+from core.tools.errors import ToolSSRFError+from graphon.file import helpers as file_helpers+from models import Account+from models.model import EndUser+from services.errors.file import FileTooLargeError+from services.file_service import FileService+++@dataclass(frozen=True, slots=True)+class RemoteFileInfoResult:+ content_type: str+ content_length: int | None+++@dataclass(frozen=True, slots=True)+class RemoteFileUploadResult:+ id: str+ name: str+ size: int+ extension: str+ url: str+ mime_type: str | None+ created_by: str+ created_at: datetime+++class RemoteFileError(Exception):+ pass+++class RemoteFileInvalidUrlError(RemoteFileError):+ pass+++class RemoteFileUrlBlockedError(RemoteFileError):+ pass+++class RemoteFileNotFoundError(RemoteFileError):+ pass+++class RemoteFileAccessDeniedError(RemoteFileError):+ pass+++class RemoteFileUnavailableError(RemoteFileError):+ pass+++class RemoteFileInvalidResponseError(RemoteFileError):+ pass+++class RemoteFileService:+ def __init__(self, *, files: FileService) -> None:+ self._files = files++ def fetch_info(self, *, url: str) -> RemoteFileInfoResult:+ response = self._request("HEAD", url=url)+ if response.status_code != httpx.codes.OK:+ response = self._request("GET", url=url, timeout=3)+ self._ensure_success(response)++ content_length = response.headers.get("Content-Length")+ try:+ parsed_content_length = int(content_length) if content_length is not None else None+ except ValueError as error:+ raise RemoteFileInvalidResponseError("The remote response has an invalid Content-Length header") from error++ return RemoteFileInfoResult(+ content_type=response.headers.get("Content-Type", "application/octet-stream"),+ content_length=parsed_content_length,+ )++ def upload_from_url(+ self,+ *,+ url: str,+ user: Account | EndUser,+ tenant_id: str | None = None,+ ) -> RemoteFileUploadResult:+ response = self._fetch_for_upload(url=url)+ try:+ file_info = guess_file_info_from_response(response)+ except InvalidRemoteFileMetadataError as error:+ raise RemoteFileInvalidResponseError("The remote response contains invalid file metadata") from error+ except ValueError as error:+ # Unclassified parser failures are server bugs, not invalid request parameters.+ raise RuntimeError("Unexpected remote file metadata parsing failure") from error++ if any(separator in file_info.filename for separator in ("/", "\\")):+ raise RemoteFileInvalidResponseError("The remote response contains an invalid filename")++ if not self._files.is_file_size_within_limit(+ extension=file_info.extension,+ file_size=file_info.size,+ ):+ raise FileTooLargeError()++ if response.request.method == "GET":+ content = response.content+ else:+ content = self._fetch_content(url=url)++ upload_file = self._files.upload_file(+ filename=file_info.filename,+ content=content,+ mimetype=file_info.mimetype,+ user=user,+ tenant_id=tenant_id,+ source_url=url,+ )+ return RemoteFileUploadResult(+ id=upload_file.id,+ name=upload_file.name,+ size=upload_file.size,+ extension=upload_file.extension,+ url=file_helpers.get_signed_file_url(upload_file_id=upload_file.id),+ mime_type=upload_file.mime_type,+ created_by=upload_file.created_by,+ created_at=upload_file.created_at,+ )++ @staticmethod+ def _request(+ method: Literal["GET", "HEAD"],+ *,+ url: str,+ **kwargs: Any,+ ) -> httpx.Response:+ try:+ parsed_url = urllib.parse.urlsplit(url)+ port = parsed_url.port+ except ValueError as error:+ raise RemoteFileInvalidUrlError("The remote file URL is invalid") from error++ if parsed_url.scheme not in {"http", "https"} or parsed_url.hostname is None or port == 0:+ raise RemoteFileInvalidUrlError("The remote file URL is invalid")++ try:+ return remote_fetcher.make_request(method, url=url, **kwargs)+ except httpx.InvalidURL as error:+ raise RemoteFileInvalidUrlError("The remote file URL is invalid") from error+ except ToolSSRFError as error:+ raise RemoteFileUrlBlockedError("The remote file URL was blocked by SSRF protection") from error+ except (MaxRetriesExceededError, httpx.RequestError) as error:+ raise RemoteFileUnavailableError("The remote file request failed") from error++ @classmethod+ def _fetch_for_upload(cls, *, url: str) -> httpx.Response:+ response = cls._request("HEAD", url=url)+ if response.status_code != httpx.codes.OK:+ response = cls._request("GET", url=url, timeout=3, follow_redirects=True)++ cls._ensure_success(response)+ return response++ @staticmethod+ def _ensure_success(response: httpx.Response) -> None:+ if response.status_code == httpx.codes.OK:+ return++ if response.status_code in {httpx.codes.NOT_FOUND, httpx.codes.GONE}:+ raise RemoteFileNotFoundError("The remote file does not exist")+ if response.status_code in {httpx.codes.UNAUTHORIZED, httpx.codes.FORBIDDEN}:+ raise RemoteFileAccessDeniedError("The remote file cannot be accessed anonymously")++ raise RemoteFileUnavailableError(f"The remote file request returned HTTP {response.status_code}")++ @classmethod+ def _fetch_content(cls, *, url: str) -> bytes:+ response = cls._request("GET", url=url)+ cls._ensure_success(response)+ return response.contentdiff --git a/api/tests/unit_tests/controllers/common/test_errors.py b/api/tests/unit_tests/controllers/common/test_errors.pyindex 25a9fe5b668..ada7e6d379e 100644--- a/api/tests/unit_tests/controllers/common/test_errors.py+++ b/api/tests/unit_tests/controllers/common/test_errors.py@@ -1,12 +1,21 @@+import pytest+from controllers.common.errors import (BlockedFileExtensionError,FilenameNotExistsError,FileTooLargeError,NoFileUploadedError,+ RemoteFileAccessDeniedError,+ RemoteFileInvalidResponseError,+ RemoteFileInvalidUrlError,+ RemoteFileNotFoundError,+ RemoteFileUnavailableError,RemoteFileUploadError,+ RemoteFileUrlBlockedError,TooManyFilesError,UnsupportedFileTypeError,)+from libs.exception import BaseHTTPExceptionclass TestFilenameNotExistsError:@@ -14,7 +23,13 @@ class TestFilenameNotExistsError:error = FilenameNotExistsError()assert error.code == 400+ assert error.error_code == "filename_not_exists_error"assert error.description == "The specified filename does not exist."+ assert error.data == {+ "code": "filename_not_exists_error",+ "message": "The specified filename does not exist.",+ "status": 400,+ }class TestRemoteFileUploadError:@@ -22,7 +37,53 @@ class TestRemoteFileUploadError:error = RemoteFileUploadError()assert error.code == 400+ assert error.error_code == "remote_file_upload_error"assert error.description == "Error uploading remote file."+ assert error.data == {+ "code": "remote_file_upload_error",+ "message": "Error uploading remote file.",+ "status": 400,+ }+++ ("error_type", "error_code", "description", "status"),+ [+ (RemoteFileInvalidUrlError, "remote_file_invalid_url", "The remote file URL is invalid.", 400),+ (RemoteFileUrlBlockedError, "remote_file_url_blocked", "The remote file URL is not allowed.", 400),+ (RemoteFileNotFoundError, "remote_file_not_found", "The remote file could not be found.", 404),+ (+ RemoteFileAccessDeniedError,+ "remote_file_access_denied",+ "The remote file cannot be accessed without authorization.",+ 400,+ ),+ (+ RemoteFileUnavailableError,+ "remote_file_unavailable",+ "The remote file is temporarily unavailable.",+ 502,+ ),+ (+ RemoteFileInvalidResponseError,+ "remote_file_invalid_response",+ "The remote file server returned an invalid response.",+ 502,+ ),+ ],+)+def test_remote_file_errors(+ error_type: type[BaseHTTPException],+ error_code: str,+ description: str,+ status: int,+) -> None:+ error = error_type()++ assert error.code == status+ assert error.error_code == error_code+ assert error.description == description+ assert error.data == {"code": error_code, "message": description, "status": status}class TestFileTooLargeError:diff --git a/api/tests/unit_tests/controllers/common/test_helpers.py b/api/tests/unit_tests/controllers/common/test_helpers.pyindex 376a7a90c55..b06bd1e3ea2 100644--- a/api/tests/unit_tests/controllers/common/test_helpers.py+++ b/api/tests/unit_tests/controllers/common/test_helpers.py@@ -1,188 +1,30 @@-from uuid import UUID+import urllib.parse-import httpximport pytest-from controllers.common import helpers-from controllers.common.helpers import FileInfo, guess_file_info_from_response---def make_response(- url="https://example.com/file.txt",- headers=None,- content=None,-):- return httpx.Response(- 200,- request=httpx.Request("GET", url),- headers=headers or {},- content=content or b"",- )---class TestGuessFileInfoFromResponse:- def test_filename_from_url(self):- response = make_response(- url="https://example.com/test.pdf",- content=b"Hello World",- )-- info = guess_file_info_from_response(response)-- assert info.filename == "test.pdf"- assert info.extension == ".pdf"- assert info.mimetype == "application/pdf"-- def test_filename_from_content_disposition(self):- headers = {- "Content-Disposition": "attachment; filename=myfile.csv",- "Content-Type": "text/csv",- }- response = make_response(- url="https://example.com/",- headers=headers,- content=b"Hello World",- )-- info = guess_file_info_from_response(response)-- assert info.filename == "myfile.csv"- assert info.extension == ".csv"- assert info.mimetype == "text/csv"-- @pytest.mark.parametrize(- ("magic_available", "expected_ext"),- [- (True, "txt"),- (False, "bin"),- ],- )- def test_generated_filename_when_missing(self, monkeypatch: pytest.MonkeyPatch, magic_available, expected_ext):- if magic_available:- if helpers.magic is None:- pytest.skip("python-magic is not installed, cannot run 'magic_available=True' test variant")- else:- monkeypatch.setattr(helpers, "magic", None)-- response = make_response(- url="https://example.com/",- content=b"Hello World",- )-- info = guess_file_info_from_response(response)-- name, ext = info.filename.split(".")- UUID(name)- assert ext == expected_ext-- def test_mimetype_from_header_when_unknown(self):- headers = {"Content-Type": "application/json"}- response = make_response(- url="https://example.com/file.unknown",- headers=headers,- content=b'{"a": 1}',- )-- info = guess_file_info_from_response(response)-- assert info.mimetype == "application/json"-- def test_extension_added_when_missing(self):- headers = {"Content-Type": "image/png"}- response = make_response(- url="https://example.com/image",- headers=headers,- content=b"fakepngdata",- )-- info = guess_file_info_from_response(response)-- assert info.extension == ".png"- assert info.filename.endswith(".png")-- def test_content_length_used_as_size(self):- headers = {- "Content-Length": "1234",- "Content-Type": "text/plain",- }- response = make_response(- url="https://example.com/a.txt",- headers=headers,- content=b"a" * 1234,- )-- info = guess_file_info_from_response(response)-- assert info.size == 1234-- def test_size_minus_one_when_header_missing(self):- response = make_response(url="https://example.com/a.txt")-- info = guess_file_info_from_response(response)-- assert info.size == -1-- def test_fallback_to_bin_extension(self):- headers = {"Content-Type": "application/octet-stream"}- response = make_response(- url="https://example.com/download",- headers=headers,- content=b"\x00\x01\x02\x03",- )-- info = guess_file_info_from_response(response)-- assert info.extension == ".bin"- assert info.filename.endswith(".bin")-- def test_return_type(self):- response = make_response()-- info = guess_file_info_from_response(response)-- assert isinstance(info, FileInfo)---class TestMagicImportWarnings:- @pytest.mark.parametrize(- ("platform_name", "expected_message"),- [- ("Windows", "pip install python-magic-bin"),- ("Darwin", "brew install libmagic"),- ("Linux", "sudo apt-get install libmagic1"),- ("Other", "install `libmagic`"),- ],- )- def test_magic_import_warning_per_platform(- self,- monkeypatch: pytest.MonkeyPatch,- platform_name,- expected_message,- ):- import builtins- import importlib-- # Force ImportError when "magic" is imported- real_import = builtins.__import__-- def fake_import(name, *args, **kwargs):- if name == "magic":- raise ImportError("No module named magic")- return real_import(name, *args, **kwargs)-- monkeypatch.setattr(builtins, "__import__", fake_import)- monkeypatch.setattr("platform.system", lambda: platform_name)-- # Remove helpers so it imports fresh- import sys-- original_helpers = sys.modules.get(helpers.__name__)- sys.modules.pop(helpers.__name__, None)-- try:- with pytest.warns(UserWarning, match="To use python-magic") as warning:- imported_helpers = importlib.import_module(helpers.__name__)- assert expected_message in str(warning[0].message)- finally:- if original_helpers is not None:- sys.modules[helpers.__name__] = original_helpers+from controllers.common.helpers import decode_remote_url+++ ("url", "query_string", "expected"),+ [+ (+ urllib.parse.quote("https://example.com/report.txt", safe=""),+ b"download=1",+ "https://example.com/report.txt?download=1",+ ),+ (+ urllib.parse.quote("https://example.com/report.txt?version=1", safe=""),+ "download=1",+ "https://example.com/report.txt?version=1&download=1",+ ),+ ("https://example.com/report.txt?", "download=1", "https://example.com/report.txt?download=1"),+ ("https://example.com/report.txt", b"", "https://example.com/report.txt"),+ ],+)+def test_decode_remote_url(url: str, query_string: bytes | str, expected: str) -> None:+ assert decode_remote_url(url, query_string) == expected+++def test_decode_remote_url_defers_invalid_url_validation() -> None:+ assert decode_remote_url("http://[invalid", "download=1") == "http://[invalid?download=1"diff --git a/api/tests/unit_tests/controllers/console/explore/test_trial.py b/api/tests/unit_tests/controllers/console/explore/test_trial.pyindex 2f2966b3844..03d93348382 100644--- a/api/tests/unit_tests/controllers/console/explore/test_trial.py+++ b/api/tests/unit_tests/controllers/console/explore/test_trial.py@@ -252,17 +252,23 @@ class TestTrialAppRemoteFileUploadApi:method = unwrap(api.post)app_model = _app(app_id="app-1", mode=AppMode.CHAT, tenant_id="app-tenant-id")remote_file = MagicMock()- remote_file.model_dump.return_value = {"id": "upload-file-id"}+ payload = module.RemoteFileUploadPayload(url="https://example.com/file.txt")with (app.test_request_context("/", method="POST", json={"url": "https://example.com/file.txt"}),- patch.object(module, "upload_remote_file_from_request", return_value=remote_file) as upload,+ patch.object(module, "upload_remote_file", return_value=remote_file) as upload,+ patch.object(module, "dump_response", return_value={"id": "upload-file-id"}) as dump,):- response, status = method(api, account, app_model)+ response, status = method(api, payload, account, app_model)assert status == 201assert response == {"id": "upload-file-id"}- upload.assert_called_once_with(current_user=account, resource_tenant_id="app-tenant-id")+ upload.assert_called_once_with(+ url="https://example.com/file.txt",+ current_user=account,+ resource_tenant_id="app-tenant-id",+ )+ dump.assert_called_once_with(module.FileWithSignedUrl, remote_file)class TestTrialAppWorkflowRunApi(_UsesSQLiteSession):diff --git a/api/tests/unit_tests/controllers/console/test_remote_files.py b/api/tests/unit_tests/controllers/console/test_remote_files.pyindex 6eeb68f79ac..c0bd04614ab 100644--- a/api/tests/unit_tests/controllers/console/test_remote_files.py+++ b/api/tests/unit_tests/controllers/console/test_remote_files.py@@ -1,349 +1,260 @@from __future__ import annotationsimport urllib.parse+from collections.abc import Generator+from contextlib import contextmanagerfrom datetime import UTC, datetimefrom inspect import unwrapfrom types import SimpleNamespace-from unittest.mock import MagicMock+from unittest.mock import MagicMock, patch-import httpximport pytestfrom flask import Flask-from controllers.common.errors import FileTooLargeError, RemoteFileUploadError, UnsupportedFileTypeError+from controllers.common.errors import (+ BlockedFileExtensionError,+ FileTooLargeError,+ RemoteFileAccessDeniedError,+ RemoteFileInvalidResponseError,+ RemoteFileInvalidUrlError,+ RemoteFileNotFoundError,+ RemoteFileUnavailableError,+ RemoteFileUrlBlockedError,+ UnsupportedFileTypeError,+)from controllers.console import remote_files as remote_files_module-from extensions.storage.storage_type import StorageType+from libs.exception import BaseHTTPException+from machinery.context import RequestContextfrom models import Accountfrom models.account import AccountStatus, TenantAccountRole-from models.enums import CreatorUserRole-from models.model import UploadFile+from services.errors.file import (+ BlockedFileExtensionError as ServiceBlockedFileExtensionError,+)from services.errors.file import FileTooLargeError as ServiceFileTooLargeErrorfrom services.errors.file import UnsupportedFileTypeError as ServiceUnsupportedFileTypeError---def _make_account(account_id: str = "u1") -> Account:+from services.remote_file_service import (+ RemoteFileAccessDeniedError as RemoteFileAccessDeniedServiceError,+)+from services.remote_file_service import RemoteFileError, RemoteFileInfoResult, RemoteFileUploadResult+from services.remote_file_service import (+ RemoteFileInvalidResponseError as RemoteFileInvalidResponseServiceError,+)+from services.remote_file_service import RemoteFileInvalidUrlError as RemoteFileInvalidUrlServiceError+from services.remote_file_service import RemoteFileNotFoundError as RemoteFileNotFoundServiceError+from services.remote_file_service import RemoteFileUnavailableError as RemoteFileUnavailableServiceError+from services.remote_file_service import RemoteFileUrlBlockedError as RemoteFileUrlBlockedServiceError++REMOTE_FILE_ERROR_CASES: tuple[tuple[type[RemoteFileError], type[BaseHTTPException], str, int], ...] = (+ (RemoteFileInvalidUrlServiceError, RemoteFileInvalidUrlError, "remote_file_invalid_url", 400),+ (RemoteFileUrlBlockedServiceError, RemoteFileUrlBlockedError, "remote_file_url_blocked", 400),+ (RemoteFileNotFoundServiceError, RemoteFileNotFoundError, "remote_file_not_found", 404),+ (RemoteFileAccessDeniedServiceError, RemoteFileAccessDeniedError, "remote_file_access_denied", 400),+ (RemoteFileUnavailableServiceError, RemoteFileUnavailableError, "remote_file_unavailable", 502),+ (+ RemoteFileInvalidResponseServiceError,+ RemoteFileInvalidResponseError,+ "remote_file_invalid_response",+ 502,+ ),+)+++def _account() -> Account:account = Account(name="Test User",- email=f"{account_id}@example.com",+ email="[email protected]",status=AccountStatus.ACTIVE,)- account.id = account_id+ account.id = "account-1"account.role = TenantAccountRole.OWNERreturn account-def _upload_file(- *,- file_id: str = "file-1",- name: str = "report.txt",- size: int = 16,- extension: str = ".txt",- mime_type: str = "text/plain",- created_at: datetime | None = None,-) -> UploadFile:- upload_file = UploadFile(- tenant_id="tenant-1",- storage_type=StorageType.LOCAL,- key=f"upload/{name}",- name=name,- size=size,- extension=extension,- mime_type=mime_type,- created_by_role=CreatorUserRole.ACCOUNT,- created_by="u1",- created_at=created_at or datetime(2024, 1, 1, tzinfo=UTC),- used=False,+def _upload_result() -> RemoteFileUploadResult:+ return RemoteFileUploadResult(+ id="file-1",+ name="report.txt",+ size=16,+ extension="txt",+ url="https://signed.example/file-1",+ mime_type="text/plain",+ created_by="account-1",+ created_at=datetime(2024, 1, 1, tzinfo=UTC),)- upload_file.id = file_id- return upload_file-class _FakeResponse:- def __init__(- self,- *,- status_code: int = 200,- headers: dict[str, str] | None = None,- method: str = "GET",- content: bytes = b"",- text: str = "",- error: Exception | None = None,- ) -> None:- self.status_code = status_code- self.headers = headers or {}- self.request = SimpleNamespace(method=method)- self.content = content- self.text = text- self._error = error-- def raise_for_status(self) -> None:- if self._error:- raise self._error---def _mock_upload_dependencies(- monkeypatch: pytest.MonkeyPatch,- *,- file_size_within_limit: bool = True,-):- file_info = SimpleNamespace(- filename="report.txt",- extension=".txt",- mimetype="text/plain",- size=3,- )- monkeypatch.setattr(- remote_files_module.helpers,- "guess_file_info_from_response",- MagicMock(return_value=file_info),- )-- file_service_cls = MagicMock()- file_service_cls.is_file_size_within_limit.return_value = file_size_within_limit- monkeypatch.setattr(remote_files_module, "FileService", file_service_cls)- current_user = _make_account()- monkeypatch.setattr(remote_files_module, "db", SimpleNamespace(engine=object()))- monkeypatch.setattr(- remote_files_module.file_helpers,- "get_signed_file_url",- lambda upload_file_id: f"https://signed.example/{upload_file_id}",+def _request_context() -> RequestContext:+ return RequestContext(+ request_id="request-1",+ trace_id="trace-1",+ account_id="account-1",+ active_workspace_id="tenant-1",)- return file_service_cls, current_user---def test_get_remote_file_info_uses_head_when_successful(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:- api = remote_files_module.GetRemoteFileInfo()- handler = unwrap(api.get)- decoded_url = "https://example.com/test.txt"- encoded_url = urllib.parse.quote(decoded_url, safe="")-- head_resp = _FakeResponse(- status_code=200,- headers={"Content-Type": "text/plain", "Content-Length": "128"},- method="HEAD",- )- make_request = MagicMock(return_value=head_resp)- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)-- with app.test_request_context(method="GET"):- payload = handler(api, url=encoded_url)-- assert payload == {"file_type": "text/plain", "file_length": 128}- make_request.assert_called_once_with("HEAD", decoded_url)---def test_get_remote_file_info_preserves_unencoded_target_query(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:- api = remote_files_module.GetRemoteFileInfo()- handler = unwrap(api.get)- target_url = "http://example.com/api/aiagent/httpview/txt"- query = "fileNameKey=cankao1_ce4305bc-be20-4c5d-8732-de1741d28e27"- head_resp = _FakeResponse(- status_code=200,- headers={"Content-Type": "text/plain", "Content-Length": "128"},- method="HEAD",- )- make_request = MagicMock(return_value=head_resp)- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)-- with app.test_request_context(f"/remote-files/{target_url}?{query}", method="GET"):- payload = handler(api, url=target_url)-- assert payload == {"file_type": "text/plain", "file_length": 128}- make_request.assert_called_once_with("HEAD", f"{target_url}?{query}")---def test_get_remote_file_info_falls_back_to_get_and_uses_default_headers(- app: Flask, monkeypatch: pytest.MonkeyPatch-) -> None:- api = remote_files_module.GetRemoteFileInfo()- handler = unwrap(api.get)- decoded_url = "https://example.com/test.txt"- encoded_url = urllib.parse.quote(decoded_url, safe="")-- make_request = MagicMock(- side_effect=[- _FakeResponse(status_code=503),- _FakeResponse(status_code=200, headers={}, method="GET"),- ]- )- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)-- with app.test_request_context(method="GET"):- payload = handler(api, url=encoded_url)-- assert payload == {"file_type": "application/octet-stream", "file_length": 0}- assert make_request.call_args_list[0].args == ("HEAD", decoded_url)- assert make_request.call_args_list[1].args == ("GET", decoded_url)- assert make_request.call_args_list[1].kwargs == {"timeout": 3}---def test_remote_file_upload_success_when_fetch_falls_back_to_get(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:- api = remote_files_module.RemoteFileUpload()- handler = unwrap(api.post)- url = "https://example.com/report.txt"-- get_resp = _FakeResponse(status_code=200, method="GET", content=b"fallback-content")- make_request = MagicMock(side_effect=[_FakeResponse(status_code=404), get_resp])- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)-- file_service_cls, current_user = _mock_upload_dependencies(monkeypatch)- upload_file = _upload_file()- file_service_cls.return_value.upload_file.return_value = upload_file-- with app.test_request_context(method="POST", json={"url": url}):- payload, status = handler(api, current_user)-- assert status == 201- assert payload["id"] == "file-1"- assert payload["url"] == "https://signed.example/file-1"- assert make_request.call_args_list[0].args == ("HEAD",)- assert make_request.call_args_list[0].kwargs == {"url": url}- assert make_request.call_args_list[1].args == ("GET",)- assert make_request.call_args_list[1].kwargs == {"url": url, "timeout": 3, "follow_redirects": True}- file_service_cls.return_value.upload_file.assert_called_once_with(- filename="report.txt",- content=b"fallback-content",- mimetype="text/plain",- user=current_user,- tenant_id=None,- source_url=url,- )---def test_remote_file_upload_assigns_resource_tenant(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:- url = "https://example.com/report.txt"- response = _FakeResponse(status_code=200, method="GET", content=b"content")- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", MagicMock(return_value=response))-- file_service_cls, current_user = _mock_upload_dependencies(monkeypatch)- file_service_cls.return_value.upload_file.return_value = _upload_file(size=7)-- with app.test_request_context(method="POST", json={"url": url}):- remote_files_module.upload_remote_file_from_request(- current_user=current_user,- resource_tenant_id="app-tenant-id",+@contextmanager+def _patch_application_services(remote_files: MagicMock) -> Generator[None]:+ with patch.object(+ remote_files_module,+ "application_services",+ return_value=SimpleNamespace(remote_files=remote_files),+ ):+ yield+++class TestGetRemoteFileInfo:+ def test_decodes_url_and_preserves_console_missing_length_default(self, app: Flask) -> None:+ api = remote_files_module.GetRemoteFileInfo()+ handler = unwrap(api.get)+ remote_files = MagicMock()+ remote_files.fetch_info.return_value = RemoteFileInfoResult(+ content_type="text/plain",+ content_length=None,)+ target_url = "https://example.com/file?name=report.txt"+ encoded_url = urllib.parse.quote(target_url, safe="")- file_service_cls.return_value.upload_file.assert_called_once_with(- filename="report.txt",- content=b"content",- mimetype="text/plain",- user=current_user,- tenant_id="app-tenant-id",- source_url=url,- )-+ with (+ app.test_request_context(f"/remote-files/{encoded_url}", method="GET"),+ _patch_application_services(remote_files),+ ):+ response = handler(api, _request_context(), encoded_url)-def test_remote_file_upload_fetches_content_with_second_get_when_head_succeeds(- app, monkeypatch: pytest.MonkeyPatch-) -> None:- api = remote_files_module.RemoteFileUpload()- handler = unwrap(api.post)- url = "https://example.com/photo.jpg"-- head_resp = _FakeResponse(status_code=200, method="HEAD", content=b"head-content")- extra_get_resp = _FakeResponse(status_code=200, method="GET", content=b"downloaded-content")- make_request = MagicMock(side_effect=[head_resp, extra_get_resp])- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)-- file_service_cls, current_user = _mock_upload_dependencies(monkeypatch)- upload_file = _upload_file(- file_id="file-2",- name="photo.jpg",- size=18,- extension=".jpg",- mime_type="image/jpeg",- created_at=datetime(2024, 1, 2, tzinfo=UTC),- )- file_service_cls.return_value.upload_file.return_value = upload_file-- with app.test_request_context(method="POST", json={"url": url}):- payload, status = handler(api, current_user)-- assert status == 201- assert payload["id"] == "file-2"- assert make_request.call_args_list[1].args == ("GET", url)- assert file_service_cls.return_value.upload_file.call_args.kwargs["content"] == b"downloaded-content"+ assert response == {"file_type": "text/plain", "file_length": 0}+ remote_files.fetch_info.assert_called_once_with(url=target_url)+ @pytest.mark.parametrize(("service_error_type", "http_error", "error_code", "status"), REMOTE_FILE_ERROR_CASES)+ def test_translates_remote_file_errors(+ self,+ app: Flask,+ service_error_type: type[RemoteFileError],+ http_error: type[BaseHTTPException],+ error_code: str,+ status: int,+ ) -> None:+ api = remote_files_module.GetRemoteFileInfo()+ handler = unwrap(api.get)+ service_error = service_error_type("sensitive upstream details")+ remote_files = MagicMock()+ remote_files.fetch_info.side_effect = service_error++ with (+ app.test_request_context("/remote-files/url", method="GET"),+ _patch_application_services(remote_files),+ pytest.raises(http_error) as error_info,+ ):+ handler(api, _request_context(), "url")++ assert error_info.value.__cause__ is service_error+ assert error_info.value.data is not None+ assert error_info.value.data["code"] == error_code+ assert error_info.value.data["status"] == status+ assert "sensitive upstream details" not in error_info.value.data["message"]+++class TestRemoteFileUpload:+ def test_controller_delegates_with_validated_payload(self, app: Flask) -> None:+ api = remote_files_module.RemoteFileUpload()+ handler = unwrap(api.post)+ account = _account()+ remote_file = _upload_result()++ with (+ app.test_request_context("/remote-files/upload", method="POST"),+ patch.object(remote_files_module, "upload_remote_file", return_value=remote_file) as upload,+ ):+ response, status = handler(+ api,+ remote_files_module.RemoteFileUploadPayload(url="https://example.com/report.txt"),+ account,+ )++ assert status == 201+ assert response["created_at"] == 1704067200+ upload.assert_called_once_with(+ url="https://example.com/report.txt",+ current_user=account,+ )-def test_remote_file_upload_raises_when_fallback_get_still_not_ok(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:- api = remote_files_module.RemoteFileUpload()- handler = unwrap(api.post)- url = "https://example.com/fail.txt"+ def test_helper_preserves_explicit_resource_tenant(self, app: Flask) -> None:+ account = _account()+ remote_files = MagicMock()+ remote_files.upload_from_url.return_value = _upload_result()++ with (+ app.test_request_context("/remote-files/upload", method="POST"),+ _patch_application_services(remote_files),+ ):+ result = remote_files_module.upload_remote_file(+ url="https://example.com/report.txt",+ current_user=account,+ resource_tenant_id="app-tenant-id",+ )++ assert result == _upload_result()+ remote_files.upload_from_url.assert_called_once_with(+ url="https://example.com/report.txt",+ user=account,+ tenant_id="app-tenant-id",+ )- make_request = MagicMock(- side_effect=[- _FakeResponse(status_code=500),- _FakeResponse(status_code=502, text="bad gateway"),- ]+ @pytest.mark.parametrize(("service_error_type", "http_error", "error_code", "status"), REMOTE_FILE_ERROR_CASES)+ def test_helper_translates_remote_file_errors(+ self,+ app: Flask,+ service_error_type: type[RemoteFileError],+ http_error: type[BaseHTTPException],+ error_code: str,+ status: int,+ ) -> None:+ service_error = service_error_type("sensitive upstream details")+ remote_files = MagicMock()+ remote_files.upload_from_url.side_effect = service_error++ with (+ app.test_request_context("/remote-files/upload", method="POST"),+ _patch_application_services(remote_files),+ pytest.raises(http_error) as error_info,+ ):+ remote_files_module.upload_remote_file(+ url="https://example.com/report.txt",+ current_user=_account(),+ )++ assert error_info.value.__cause__ is service_error+ assert error_info.value.data is not None+ assert error_info.value.data["code"] == error_code+ assert error_info.value.data["status"] == status+ assert "sensitive upstream details" not in error_info.value.data["message"]++ @pytest.mark.parametrize(+ ("service_error", "http_error"),+ [+ (ServiceFileTooLargeError("size exceeded"), FileTooLargeError),+ (ServiceUnsupportedFileTypeError(), UnsupportedFileTypeError),+ (+ ServiceBlockedFileExtensionError("File extension '.exe' is not allowed"),+ BlockedFileExtensionError,+ ),+ ],)- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)-- with app.test_request_context(method="POST", json={"url": url}):- with pytest.raises(RemoteFileUploadError, match=f"Failed to fetch file from {url}: bad gateway"):- handler(api, _make_account())---def test_remote_file_upload_raises_on_httpx_request_error(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:- api = remote_files_module.RemoteFileUpload()- handler = unwrap(api.post)- url = "https://example.com/fail.txt"-- request = httpx.Request("HEAD", url)- make_request = MagicMock(side_effect=httpx.RequestError("network down", request=request))- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)-- with app.test_request_context(method="POST", json={"url": url}):- with pytest.raises(RemoteFileUploadError, match=f"Failed to fetch file from {url}: network down"):- handler(api, _make_account())---def test_remote_file_upload_rejects_oversized_file(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:- api = remote_files_module.RemoteFileUpload()- handler = unwrap(api.post)- url = "https://example.com/large.bin"-- make_request = MagicMock(return_value=_FakeResponse(status_code=200, method="GET", content=b"payload"))- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)-- _, current_user = _mock_upload_dependencies(monkeypatch, file_size_within_limit=False)-- with app.test_request_context(method="POST", json={"url": url}):- with pytest.raises(FileTooLargeError):- handler(api, current_user)---def test_remote_file_upload_translates_service_file_too_large_error(- app: Flask, monkeypatch: pytest.MonkeyPatch-) -> None:- api = remote_files_module.RemoteFileUpload()- handler = unwrap(api.post)- url = "https://example.com/large.bin"-- make_request = MagicMock(return_value=_FakeResponse(status_code=200, method="GET", content=b"payload"))- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)- file_service_cls, current_user = _mock_upload_dependencies(monkeypatch)- file_service_cls.return_value.upload_file.side_effect = ServiceFileTooLargeError("size exceeded")-- with app.test_request_context(method="POST", json={"url": url}):- with pytest.raises(FileTooLargeError, match="size exceeded"):- handler(api, current_user)---def test_remote_file_upload_translates_service_unsupported_type_error(- app: Flask, monkeypatch: pytest.MonkeyPatch-) -> None:- api = remote_files_module.RemoteFileUpload()- handler = unwrap(api.post)- url = "https://example.com/file.exe"-- make_request = MagicMock(return_value=_FakeResponse(status_code=200, method="GET", content=b"payload"))- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)- file_service_cls, current_user = _mock_upload_dependencies(monkeypatch)- file_service_cls.return_value.upload_file.side_effect = ServiceUnsupportedFileTypeError()-- with app.test_request_context(method="POST", json={"url": url}):- with pytest.raises(UnsupportedFileTypeError):- handler(api, current_user)+ def test_helper_translates_file_errors(+ self,+ app: Flask,+ service_error: Exception,+ http_error: type[Exception],+ ) -> None:+ remote_files = MagicMock()+ remote_files.upload_from_url.side_effect = service_error++ with (+ app.test_request_context("/remote-files/upload", method="POST"),+ _patch_application_services(remote_files),+ pytest.raises(http_error) as error_info,+ ):+ remote_files_module.upload_remote_file(+ url="https://example.com/report.txt",+ current_user=_account(),+ )++ assert error_info.value.__cause__ is service_errordiff --git a/api/tests/unit_tests/controllers/web/test_human_input_file_upload.py b/api/tests/unit_tests/controllers/web/test_human_input_file_upload.pyindex 81e361d494c..3dfa2be777d 100644--- a/api/tests/unit_tests/controllers/web/test_human_input_file_upload.py+++ b/api/tests/unit_tests/controllers/web/test_human_input_file_upload.py@@ -210,7 +210,7 @@ def test_remote_upload_records_form_file_link(ssrf_proxy.head.return_value = responsemonkeypatch.setattr(upload_module, "ssrf_proxy", ssrf_proxy)monkeypatch.setattr(- upload_module.helpers,+ upload_module,"guess_file_info_from_response",lambda _response: SimpleNamespace(filename="sample.txt", extension="txt", mimetype="text/plain", size=6),)diff --git a/api/tests/unit_tests/controllers/web/test_pydantic_models.py b/api/tests/unit_tests/controllers/web/test_pydantic_models.pyindex bceb65b89f6..c2d5b24b3c6 100644--- a/api/tests/unit_tests/controllers/web/test_pydantic_models.py+++ b/api/tests/unit_tests/controllers/web/test_pydantic_models.py@@ -260,9 +260,10 @@ class TestRemoteFileUploadPayload:p = RemoteFileUploadPayload(url="https://example.com/file.pdf")assert str(p.url) == "https://example.com/file.pdf"- def test_invalid_url(self) -> None:- with pytest.raises(ValidationError):- RemoteFileUploadPayload(url="not-a-url")+ def test_url_syntax_is_validated_by_remote_file_service(self) -> None:+ payload = RemoteFileUploadPayload(url="not-a-url")++ assert payload.url == "not-a-url"def test_url_required(self) -> None:with pytest.raises(ValidationError):diff --git a/api/tests/unit_tests/controllers/web/test_remote_files.py b/api/tests/unit_tests/controllers/web/test_remote_files.pyindex 68cb0e6b6bb..791ec2770fb 100644--- a/api/tests/unit_tests/controllers/web/test_remote_files.py+++ b/api/tests/unit_tests/controllers/web/test_remote_files.py@@ -1,21 +1,60 @@-"""Unit tests for controllers.web.remote_files endpoints."""-from __future__ import annotationsimport urllib.parse-from datetime import datetime+from collections.abc import Generator+from contextlib import contextmanager+from datetime import UTC, datetimefrom types import SimpleNamespacefrom unittest.mock import MagicMock, patchimport pytestfrom flask import Flask-from sqlalchemy import Engine-from controllers.common.errors import FileTooLargeError, RemoteFileUploadError-from controllers.web.remote_files import RemoteFileInfoApi, RemoteFileUploadApi-from extensions.storage.storage_type import StorageType-from models.enums import CreatorUserRole, EndUserType-from models.model import App, AppMode, EndUser, UploadFile+from controllers.common.errors import (+ BlockedFileExtensionError,+ FileTooLargeError,+ RemoteFileAccessDeniedError,+ RemoteFileInvalidResponseError,+ RemoteFileInvalidUrlError,+ RemoteFileNotFoundError,+ RemoteFileUnavailableError,+ RemoteFileUrlBlockedError,+ UnsupportedFileTypeError,+)+from controllers.web import remote_files as remote_files_module+from libs.exception import BaseHTTPException+from models.enums import EndUserType+from models.model import App, AppMode, EndUser+from services.errors.file import (+ BlockedFileExtensionError as ServiceBlockedFileExtensionError,+)+from services.errors.file import FileTooLargeError as ServiceFileTooLargeError+from services.errors.file import UnsupportedFileTypeError as ServiceUnsupportedFileTypeError+from services.remote_file_service import (+ RemoteFileAccessDeniedError as RemoteFileAccessDeniedServiceError,+)+from services.remote_file_service import RemoteFileError, RemoteFileInfoResult, RemoteFileUploadResult+from services.remote_file_service import (+ RemoteFileInvalidResponseError as RemoteFileInvalidResponseServiceError,+)+from services.remote_file_service import RemoteFileInvalidUrlError as RemoteFileInvalidUrlServiceError+from services.remote_file_service import RemoteFileNotFoundError as RemoteFileNotFoundServiceError+from services.remote_file_service import RemoteFileUnavailableError as RemoteFileUnavailableServiceError+from services.remote_file_service import RemoteFileUrlBlockedError as RemoteFileUrlBlockedServiceError++REMOTE_FILE_ERROR_CASES: tuple[tuple[type[RemoteFileError], type[BaseHTTPException], str, int], ...] = (+ (RemoteFileInvalidUrlServiceError, RemoteFileInvalidUrlError, "remote_file_invalid_url", 400),+ (RemoteFileUrlBlockedServiceError, RemoteFileUrlBlockedError, "remote_file_url_blocked", 400),+ (RemoteFileNotFoundServiceError, RemoteFileNotFoundError, "remote_file_not_found", 404),+ (RemoteFileAccessDeniedServiceError, RemoteFileAccessDeniedError, "remote_file_access_denied", 400),+ (RemoteFileUnavailableServiceError, RemoteFileUnavailableError, "remote_file_unavailable", 502),+ (+ RemoteFileInvalidResponseServiceError,+ RemoteFileInvalidResponseError,+ "remote_file_invalid_response",+ 502,+ ),+)def _app_model() -> App:@@ -41,163 +80,172 @@ def _end_user() -> EndUser:)-def _upload_file() -> UploadFile:- upload_file = UploadFile(- tenant_id="tenant-1",- storage_type=StorageType.LOCAL,- key="upload/file.pdf",- name="file.pdf",- size=100,- extension="pdf",- mime_type="application/pdf",- created_by_role=CreatorUserRole.END_USER,+def _upload_result() -> RemoteFileUploadResult:+ return RemoteFileUploadResult(+ id="file-1",+ name="report.txt",+ size=16,+ extension="txt",+ url="https://signed.example/file-1",+ mime_type="text/plain",created_by="eu-1",- created_at=datetime(2024, 1, 1),- used=False,+ created_at=datetime(2024, 1, 1, tzinfo=UTC),)- upload_file.id = "f-1"- return upload_file---# ----------------------------------------------------------------------------# RemoteFileInfoApi-# ----------------------------------------------------------------------------class TestRemoteFileInfoApi:- @patch("controllers.web.remote_files.remote_fetcher")- def test_head_success(self, mock_proxy: MagicMock, app: Flask) -> None:- mock_resp = MagicMock()- mock_resp.status_code = 200- mock_resp.headers = {"Content-Type": "application/pdf", "Content-Length": "1024"}- mock_proxy.make_request.return_value = mock_resp-- with app.test_request_context("/remote-files/https%3A%2F%2Fexample.com%2Ffile.pdf"):- result = RemoteFileInfoApi().get(_app_model(), _end_user(), "https%3A%2F%2Fexample.com%2Ffile.pdf")-- assert result["file_type"] == "application/pdf"- assert result["file_length"] == 1024- mock_proxy.make_request.assert_called_once_with("HEAD", "https://example.com/file.pdf")-- @patch("controllers.web.remote_files.remote_fetcher")- def test_preserves_unencoded_target_query(self, mock_proxy: MagicMock, app: Flask) -> None:- mock_resp = MagicMock()- mock_resp.status_code = 200- mock_resp.headers = {"Content-Type": "text/plain", "Content-Length": "128"}- mock_proxy.make_request.return_value = mock_resp-- target_url = "http://example.com/api/aiagent/httpview/txt"- query = "fileNameKey=cankao1_ce4305bc-be20-4c5d-8732-de1741d28e27"-- with app.test_request_context(f"/remote-files/{target_url}?{query}"):- result = RemoteFileInfoApi().get(_app_model(), _end_user(), target_url)-- assert result["file_type"] == "text/plain"- mock_proxy.make_request.assert_called_once_with("HEAD", f"{target_url}?{query}")-- @patch("controllers.web.remote_files.remote_fetcher")- def test_preserves_encoded_target_query(self, mock_proxy: MagicMock, app: Flask) -> None:- mock_resp = MagicMock()- mock_resp.status_code = 200- mock_resp.headers = {"Content-Type": "text/plain", "Content-Length": "128"}- mock_proxy.make_request.return_value = mock_resp-- target_url = "http://example.com/api/aiagent/httpview/txt?fileNameKey=cankao1"- encoded_url = urllib.parse.quote(target_url, safe="")- with app.test_request_context(f"/remote-files/{encoded_url}"):- result = RemoteFileInfoApi().get(_app_model(), _end_user(), encoded_url)- assert result["file_type"] == "text/plain"- mock_proxy.make_request.assert_called_once_with("HEAD", target_url)+@contextmanager+def _patch_application_services(remote_files: MagicMock) -> Generator[None]:+ with patch.object(+ remote_files_module,+ "application_services",+ return_value=SimpleNamespace(remote_files=remote_files),+ ):+ yield- @patch("controllers.web.remote_files.remote_fetcher")- def test_fallback_to_get(self, mock_proxy: MagicMock, app: Flask) -> None:- head_resp = MagicMock()- head_resp.status_code = 405 # Method not allowed- get_resp = MagicMock()- get_resp.status_code = 200- get_resp.headers = {"Content-Type": "text/plain", "Content-Length": "42"}- get_resp.raise_for_status = MagicMock()- mock_proxy.make_request.side_effect = [head_resp, get_resp]- with app.test_request_context("/remote-files/https%3A%2F%2Fexample.com%2Ffile.txt"):- result = RemoteFileInfoApi().get(_app_model(), _end_user(), "https%3A%2F%2Fexample.com%2Ffile.txt")+class TestRemoteFileInfoApi:+ def test_decodes_url_and_preserves_web_missing_length_default(self, app: Flask) -> None:+ remote_files = MagicMock()+ remote_files.fetch_info.return_value = RemoteFileInfoResult(+ content_type="text/plain",+ content_length=None,+ )+ target_url = "http://example.com/api/file?name=report.txt"+ encoded_url = urllib.parse.quote(target_url, safe="")- assert result["file_type"] == "text/plain"- assert mock_proxy.make_request.call_args_list[1].args == ("GET", "https://example.com/file.txt")+ with (+ app.test_request_context(f"/remote-files/{encoded_url}"),+ _patch_application_services(remote_files),+ ):+ result = remote_files_module.RemoteFileInfoApi().get(_app_model(), _end_user(), encoded_url)+ assert result == {"file_type": "text/plain", "file_length": -1}+ remote_files.fetch_info.assert_called_once_with(url=target_url)-# ----------------------------------------------------------------------------# RemoteFileUploadApi-# ----------------------------------------------------------------------------class TestRemoteFileUploadApi:- @patch("controllers.web.remote_files.file_helpers.get_signed_file_url", return_value="https://signed-url")- @patch("controllers.web.remote_files.FileService")- @patch("controllers.web.remote_files.helpers.guess_file_info_from_response")- @patch("controllers.web.remote_files.remote_fetcher")- @patch("controllers.web.remote_files.db")- def test_upload_success(+ @pytest.mark.parametrize(("service_error_type", "http_error", "error_code", "status"), REMOTE_FILE_ERROR_CASES)+ def test_translates_remote_file_errors(self,- mock_db: MagicMock,- mock_proxy: MagicMock,- mock_guess: MagicMock,- mock_file_svc_cls: MagicMock,- mock_signed: MagicMock,app: Flask,- sqlite_engine: Engine,+ service_error_type: type[RemoteFileError],+ http_error: type[BaseHTTPException],+ error_code: str,+ status: int,) -> None:- mock_db.engine = sqlite_engine- head_resp = MagicMock()- head_resp.status_code = 200- head_resp.content = b"pdf-content"- head_resp.request.method = "HEAD"- get_resp = MagicMock()- get_resp.content = b"pdf-content"- mock_proxy.make_request.side_effect = [head_resp, get_resp]-- mock_guess.return_value = SimpleNamespace(- filename="file.pdf", extension="pdf", mimetype="application/pdf", size=100- )- mock_file_svc_cls.is_file_size_within_limit.return_value = True+ service_error = service_error_type("sensitive upstream details")+ remote_files = MagicMock()+ remote_files.fetch_info.side_effect = service_error++ with (+ app.test_request_context("/remote-files/url"),+ _patch_application_services(remote_files),+ pytest.raises(http_error) as error_info,+ ):+ remote_files_module.RemoteFileInfoApi().get(_app_model(), _end_user(), "url")++ assert error_info.value.__cause__ is service_error+ assert error_info.value.data is not None+ assert error_info.value.data["code"] == error_code+ assert error_info.value.data["status"] == status+ assert "sensitive upstream details" not in error_info.value.data["message"]- mock_file_svc_cls.return_value.upload_file.return_value = _upload_file()- with app.test_request_context(- "/remote-files/upload", method="POST", json={"url": "https://example.com/file.pdf"}+class TestRemoteFileUploadApi:+ def test_upload_delegates_without_overriding_web_tenant(self, app: Flask) -> None:+ remote_files = MagicMock()+ remote_files.upload_from_url.return_value = _upload_result()+ end_user = _end_user()+ url = "https://example.com/report.txt"++ with (+ app.test_request_context("/remote-files/upload", method="POST", json={"url": url}),+ _patch_application_services(remote_files),):- result, status = RemoteFileUploadApi().post(_app_model(), _end_user())+ response, status = remote_files_module.RemoteFileUploadApi().post(_app_model(), end_user)assert status == 201- assert result["id"] == "f-1"+ assert response == {+ "id": "file-1",+ "name": "report.txt",+ "size": 16,+ "extension": "txt",+ "url": "https://signed.example/file-1",+ "mime_type": "text/plain",+ "created_by": "eu-1",+ "created_at": 1704067200,+ }+ remote_files.upload_from_url.assert_called_once_with(url=url, user=end_user)++ def test_uses_complete_message_for_empty_file_size_error(self, app: Flask) -> None:+ remote_files = MagicMock()+ remote_files.upload_from_url.side_effect = ServiceFileTooLargeError()++ with (+ app.test_request_context(+ "/remote-files/upload",+ method="POST",+ json={"url": "https://example.com/report.txt"},+ ),+ _patch_application_services(remote_files),+ pytest.raises(FileTooLargeError) as error_info,+ ):+ remote_files_module.RemoteFileUploadApi().post(_app_model(), _end_user())++ assert error_info.value.description == "File size exceeded."- @patch("controllers.web.remote_files.FileService.is_file_size_within_limit", return_value=False)- @patch("controllers.web.remote_files.helpers.guess_file_info_from_response")- @patch("controllers.web.remote_files.remote_fetcher")- def test_file_too_large(+ @pytest.mark.parametrize(("service_error_type", "http_error", "error_code", "status"), REMOTE_FILE_ERROR_CASES)+ def test_translates_remote_file_errors(self,- mock_proxy: MagicMock,- mock_guess: MagicMock,- mock_size_check: MagicMock,app: Flask,+ service_error_type: type[RemoteFileError],+ http_error: type[BaseHTTPException],+ error_code: str,+ status: int,) -> None:- head_resp = MagicMock()- head_resp.status_code = 200- mock_proxy.make_request.return_value = head_resp- mock_guess.return_value = SimpleNamespace(- filename="big.zip", extension="zip", mimetype="application/zip", size=999999999- )-- with app.test_request_context(- "/remote-files/upload", method="POST", json={"url": "https://example.com/big.zip"}+ service_error = service_error_type("sensitive upstream details")+ remote_files = MagicMock()+ remote_files.upload_from_url.side_effect = service_error++ with (+ app.test_request_context(+ "/remote-files/upload",+ method="POST",+ json={"url": "https://example.com/report.txt"},+ ),+ _patch_application_services(remote_files),+ pytest.raises(http_error) as error_info,):- with pytest.raises(FileTooLargeError):- RemoteFileUploadApi().post(_app_model(), _end_user())-- @patch("controllers.web.remote_files.remote_fetcher")- def test_fetch_failure_raises(self, mock_proxy: MagicMock, app: Flask) -> None:- import httpx-- mock_proxy.make_request.side_effect = httpx.RequestError("connection failed")+ remote_files_module.RemoteFileUploadApi().post(_app_model(), _end_user())++ assert error_info.value.__cause__ is service_error+ assert error_info.value.data is not None+ assert error_info.value.data["code"] == error_code+ assert error_info.value.data["status"] == status+ assert "sensitive upstream details" not in error_info.value.data["message"]++ @pytest.mark.parametrize(+ ("service_error", "http_error"),+ [+ (ServiceFileTooLargeError("size exceeded"), FileTooLargeError),+ (ServiceUnsupportedFileTypeError(), UnsupportedFileTypeError),+ (+ ServiceBlockedFileExtensionError("File extension '.exe' is not allowed"),+ BlockedFileExtensionError,+ ),+ ],+ )+ def test_translates_file_errors(self, app: Flask, service_error: Exception, http_error: type[Exception]) -> None:+ remote_files = MagicMock()+ remote_files.upload_from_url.side_effect = service_error++ with (+ app.test_request_context(+ "/remote-files/upload",+ method="POST",+ json={"url": "https://example.com/report.txt"},+ ),+ _patch_application_services(remote_files),+ pytest.raises(http_error) as error_info,+ ):+ remote_files_module.RemoteFileUploadApi().post(_app_model(), _end_user())- with app.test_request_context("/remote-files/upload", method="POST", json={"url": "https://example.com/bad"}):- with pytest.raises(RemoteFileUploadError):- RemoteFileUploadApi().post(_app_model(), _end_user())+ assert error_info.value.__cause__ is service_errordiff --git a/api/tests/unit_tests/core/file/test_remote_file_metadata.py b/api/tests/unit_tests/core/file/test_remote_file_metadata.pynew file mode 100644index 00000000000..2e9425d65dc--- /dev/null+++ b/api/tests/unit_tests/core/file/test_remote_file_metadata.py@@ -0,0 +1,209 @@+from collections.abc import Mapping, Sequence+from types import ModuleType+from uuid import UUID++import httpx+import pytest++from core.file import remote_file_metadata+from core.file.remote_file_metadata import FileInfo, InvalidRemoteFileMetadataError, guess_file_info_from_response+++def make_response(+ url: str = "https://example.com/file.txt",+ headers: dict[str, str] | None = None,+ content: bytes | None = None,+) -> httpx.Response:+ return httpx.Response(+ 200,+ request=httpx.Request("GET", url),+ headers=headers or {},+ content=content or b"",+ )+++class TestGuessFileInfoFromResponse:+ def test_filename_from_url(self) -> None:+ response = make_response(+ url="https://example.com/test.pdf",+ content=b"Hello World",+ )++ info = guess_file_info_from_response(response)++ assert info.filename == "test.pdf"+ assert info.extension == ".pdf"+ assert info.mimetype == "application/pdf"++ def test_filename_from_content_disposition(self) -> None:+ headers = {+ "Content-Disposition": "attachment; filename=myfile.csv",+ "Content-Type": "text/csv",+ }+ response = make_response(+ url="https://example.com/",+ headers=headers,+ content=b"Hello World",+ )++ info = guess_file_info_from_response(response)++ assert info.filename == "myfile.csv"+ assert info.extension == ".csv"+ assert info.mimetype == "text/csv"++ @pytest.mark.parametrize(+ ("magic_available", "expected_ext"),+ [+ (True, "txt"),+ (False, "bin"),+ ],+ )+ def test_generated_filename_when_missing(+ self,+ monkeypatch: pytest.MonkeyPatch,+ magic_available: bool,+ expected_ext: str,+ ) -> None:+ if magic_available:+ if remote_file_metadata.magic is None:+ pytest.skip("python-magic is not installed, cannot run 'magic_available=True' test variant")+ else:+ monkeypatch.setattr(remote_file_metadata, "magic", None)++ response = make_response(+ url="https://example.com/",+ content=b"Hello World",+ )++ info = guess_file_info_from_response(response)++ name, ext = info.filename.split(".")+ UUID(name)+ assert ext == expected_ext++ def test_mimetype_from_header_when_unknown(self) -> None:+ headers = {"Content-Type": "application/json"}+ response = make_response(+ url="https://example.com/file.unknown",+ headers=headers,+ content=b'{"a": 1}',+ )++ info = guess_file_info_from_response(response)++ assert info.mimetype == "application/json"++ def test_extension_added_when_missing(self) -> None:+ headers = {"Content-Type": "image/png"}+ response = make_response(+ url="https://example.com/image",+ headers=headers,+ content=b"fakepngdata",+ )++ info = guess_file_info_from_response(response)++ assert info.extension == ".png"+ assert info.filename.endswith(".png")++ def test_content_length_used_as_size(self) -> None:+ headers = {+ "Content-Length": "1234",+ "Content-Type": "text/plain",+ }+ response = make_response(+ url="https://example.com/a.txt",+ headers=headers,+ content=b"a" * 1234,+ )++ info = guess_file_info_from_response(response)++ assert info.size == 1234++ def test_size_minus_one_when_header_missing(self) -> None:+ response = make_response(url="https://example.com/a.txt")++ info = guess_file_info_from_response(response)++ assert info.size == -1++ def test_invalid_content_length_raises_typed_metadata_error(self) -> None:+ response = make_response(headers={"Content-Length": "not-a-number"})++ with pytest.raises(InvalidRemoteFileMetadataError) as error_info:+ guess_file_info_from_response(response)++ assert isinstance(error_info.value.__cause__, ValueError)++ def test_fallback_to_bin_extension(self) -> None:+ headers = {"Content-Type": "application/octet-stream"}+ response = make_response(+ url="https://example.com/download",+ headers=headers,+ content=b"\x00\x01\x02\x03",+ )++ info = guess_file_info_from_response(response)++ assert info.extension == ".bin"+ assert info.filename.endswith(".bin")++ def test_return_type(self) -> None:+ response = make_response()++ info = guess_file_info_from_response(response)++ assert isinstance(info, FileInfo)+++class TestMagicImportWarnings:+ @pytest.mark.parametrize(+ ("platform_name", "expected_message"),+ [+ ("Windows", "pip install python-magic-bin"),+ ("Darwin", "brew install libmagic"),+ ("Linux", "sudo apt-get install libmagic1"),+ ("Other", "install `libmagic`"),+ ],+ )+ def test_magic_import_warning_per_platform(+ self,+ monkeypatch: pytest.MonkeyPatch,+ platform_name: str,+ expected_message: str,+ ) -> None:+ import builtins+ import importlib++ # Force ImportError when "magic" is imported+ real_import = builtins.__import__++ def fake_import(+ name: str,+ global_vars: Mapping[str, object] | None = None,+ local_vars: Mapping[str, object] | None = None,+ fromlist: Sequence[str] = (),+ level: int = 0,+ ) -> ModuleType:+ if name == "magic":+ raise ImportError("No module named magic")+ return real_import(name, global_vars, local_vars, fromlist, level)++ monkeypatch.setattr(builtins, "__import__", fake_import)+ monkeypatch.setattr("platform.system", lambda: platform_name)++ # Remove the metadata module so it imports fresh+ import sys++ original_metadata = sys.modules.get(remote_file_metadata.__name__)+ sys.modules.pop(remote_file_metadata.__name__, None)++ try:+ with pytest.warns(UserWarning, match="To use python-magic") as warning:+ importlib.import_module(remote_file_metadata.__name__)+ assert expected_message in str(warning[0].message)+ finally:+ if original_metadata is not None:+ sys.modules[remote_file_metadata.__name__] = original_metadatadiff --git a/api/tests/unit_tests/services/test_remote_file_service.py b/api/tests/unit_tests/services/test_remote_file_service.pynew file mode 100644index 00000000000..cea00fd8ade--- /dev/null+++ b/api/tests/unit_tests/services/test_remote_file_service.py@@ -0,0 +1,357 @@+from datetime import UTC, datetime+from types import SimpleNamespace+from unittest.mock import MagicMock, call, patch++import httpx+import pytest++from core.file.remote_file_metadata import FileInfo, InvalidRemoteFileMetadataError+from core.helper.ssrf_proxy import MaxRetriesExceededError+from core.tools.errors import ToolSSRFError+from models.model import Account+from services.errors.file import FileTooLargeError+from services.file_service import FileService+from services.remote_file_service import (+ RemoteFileAccessDeniedError,+ RemoteFileError,+ RemoteFileInvalidResponseError,+ RemoteFileInvalidUrlError,+ RemoteFileNotFoundError,+ RemoteFileService,+ RemoteFileUnavailableError,+ RemoteFileUploadResult,+ RemoteFileUrlBlockedError,+)++REMOTE_URL = "https://example.com/files/report.pdf"+++def _response(+ method: str,+ status_code: int,+ *,+ content: bytes = b"",+ headers: dict[str, str] | None = None,+) -> httpx.Response:+ return httpx.Response(+ status_code,+ content=content,+ headers=headers,+ request=httpx.Request(method, REMOTE_URL),+ )+++def _account() -> Account:+ account = Account(name="Test Account", email="[email protected]")+ account.id = "account-id"+ return account+++def file_service() -> MagicMock:+ service = MagicMock(spec=FileService)+ service.is_file_size_within_limit.return_value = True+ return service+++def remote_file_service(file_service: MagicMock) -> RemoteFileService:+ return RemoteFileService(files=file_service)+++def test_fetch_info_uses_successful_head_response(remote_file_service: RemoteFileService) -> None:+ response = _response(+ "HEAD",+ httpx.codes.OK,+ headers={"Content-Type": "application/pdf", "Content-Length": "42"},+ )++ with patch("services.remote_file_service.remote_fetcher.make_request", return_value=response) as make_request:+ result = remote_file_service.fetch_info(url=REMOTE_URL)++ assert result.content_type == "application/pdf"+ assert result.content_length == 42+ make_request.assert_called_once_with("HEAD", url=REMOTE_URL)+++def test_fetch_info_falls_back_to_get(remote_file_service: RemoteFileService) -> None:+ head_response = _response("HEAD", httpx.codes.METHOD_NOT_ALLOWED)+ get_response = _response("GET", httpx.codes.OK)++ with patch(+ "services.remote_file_service.remote_fetcher.make_request",+ side_effect=[head_response, get_response],+ ) as make_request:+ result = remote_file_service.fetch_info(url=REMOTE_URL)++ assert result.content_type == "application/octet-stream"+ assert result.content_length is None+ assert make_request.call_args_list == [call("HEAD", url=REMOTE_URL), call("GET", url=REMOTE_URL, timeout=3)]+++ ("source_error", "service_error"),+ [+ (httpx.InvalidURL("invalid URL"), RemoteFileInvalidUrlError),+ (ToolSSRFError("blocked URL and internal proxy details"), RemoteFileUrlBlockedError),+ (MaxRetriesExceededError("maximum retries reached"), RemoteFileUnavailableError),+ (+ httpx.ConnectError("network down", request=httpx.Request("HEAD", REMOTE_URL)),+ RemoteFileUnavailableError,+ ),+ ],+)+def test_fetch_info_translates_known_request_failures(+ remote_file_service: RemoteFileService,+ source_error: Exception,+ service_error: type[RemoteFileError],+) -> None:+ with patch("services.remote_file_service.remote_fetcher.make_request", side_effect=source_error):+ with pytest.raises(service_error) as error_info:+ remote_file_service.fetch_info(url=REMOTE_URL)++ assert error_info.value.__cause__ is source_error++[email protected]("url", ["not-a-url", "ftp://example.com/report.pdf", "http://", "http://[invalid"])+def test_fetch_info_rejects_invalid_url_without_requesting_it(+ remote_file_service: RemoteFileService,+ url: str,+) -> None:+ with patch("services.remote_file_service.remote_fetcher.make_request") as make_request:+ with pytest.raises(RemoteFileInvalidUrlError):+ remote_file_service.fetch_info(url=url)++ make_request.assert_not_called()+++def test_fetch_info_does_not_translate_unknown_failure(remote_file_service: RemoteFileService) -> None:+ source_error = RuntimeError("database failure while resolving a signed file URL")++ with patch("services.remote_file_service.remote_fetcher.make_request", side_effect=source_error):+ with pytest.raises(RuntimeError) as error_info:+ remote_file_service.fetch_info(url=REMOTE_URL)++ assert error_info.value is source_error+++ ("status_code", "service_error"),+ [+ (httpx.codes.NOT_FOUND, RemoteFileNotFoundError),+ (httpx.codes.GONE, RemoteFileNotFoundError),+ (httpx.codes.UNAUTHORIZED, RemoteFileAccessDeniedError),+ (httpx.codes.FORBIDDEN, RemoteFileAccessDeniedError),+ (httpx.codes.BAD_GATEWAY, RemoteFileUnavailableError),+ ],+)+def test_fetch_info_classifies_remote_status(+ remote_file_service: RemoteFileService,+ status_code: int,+ service_error: type[RemoteFileError],+) -> None:+ request = httpx.Request("HEAD", REMOTE_URL)+ response = httpx.Response(status_code, request=request, text="upstream response must not reach the client")++ with patch("services.remote_file_service.remote_fetcher.make_request", return_value=response):+ with pytest.raises(service_error):+ remote_file_service.fetch_info(url=REMOTE_URL)+++def test_fetch_info_rejects_invalid_content_length(remote_file_service: RemoteFileService) -> None:+ response = _response("HEAD", httpx.codes.OK, headers={"Content-Length": "not-a-number"})++ with patch("services.remote_file_service.remote_fetcher.make_request", return_value=response):+ with pytest.raises(RemoteFileInvalidResponseError) as error_info:+ remote_file_service.fetch_info(url=REMOTE_URL)++ assert isinstance(error_info.value.__cause__, ValueError)+++def test_upload_reuses_get_fallback_content_and_returns_signed_result(+ remote_file_service: RemoteFileService,+ file_service: MagicMock,+) -> None:+ account = _account()+ head_response = _response("HEAD", httpx.codes.METHOD_NOT_ALLOWED)+ get_response = _response("GET", httpx.codes.OK, content=b"remote content")+ file_info = FileInfo(filename="report.pdf", extension=".pdf", mimetype="application/pdf", size=14)+ created_at = datetime(2026, 9, 2, tzinfo=UTC)+ upload_file = SimpleNamespace(+ id="upload-id",+ name="report.pdf",+ size=14,+ extension="pdf",+ mime_type="application/pdf",+ created_by=account.id,+ created_at=created_at,+ )+ file_service.upload_file.return_value = upload_file++ with (+ patch(+ "services.remote_file_service.remote_fetcher.make_request",+ side_effect=[head_response, get_response],+ ) as make_request,+ patch("services.remote_file_service.guess_file_info_from_response", return_value=file_info),+ patch(+ "services.remote_file_service.file_helpers.get_signed_file_url",+ return_value="https://example.com/signed/upload-id",+ ) as get_signed_file_url,+ ):+ result = remote_file_service.upload_from_url(+ url=REMOTE_URL,+ user=account,+ tenant_id="tenant-id",+ )++ assert make_request.call_args_list == [+ call("HEAD", url=REMOTE_URL),+ call("GET", url=REMOTE_URL, timeout=3, follow_redirects=True),+ ]+ file_service.is_file_size_within_limit.assert_called_once_with(extension=".pdf", file_size=14)+ file_service.upload_file.assert_called_once_with(+ filename="report.pdf",+ content=b"remote content",+ mimetype="application/pdf",+ user=account,+ tenant_id="tenant-id",+ source_url=REMOTE_URL,+ )+ get_signed_file_url.assert_called_once_with(upload_file_id="upload-id")+ assert result == RemoteFileUploadResult(+ id="upload-id",+ name="report.pdf",+ size=14,+ extension="pdf",+ url="https://example.com/signed/upload-id",+ mime_type="application/pdf",+ created_by="account-id",+ created_at=created_at,+ )+++def test_upload_fetches_content_after_successful_head(+ remote_file_service: RemoteFileService,+ file_service: MagicMock,+) -> None:+ account = _account()+ head_response = _response("HEAD", httpx.codes.OK)+ content_response = _response("GET", httpx.codes.OK, content=b"downloaded content")+ file_info = FileInfo(filename="report.pdf", extension=".pdf", mimetype="application/pdf", size=18)+ file_service.upload_file.return_value = SimpleNamespace(+ id="upload-id",+ name="report.pdf",+ size=18,+ extension="pdf",+ mime_type="application/pdf",+ created_by=account.id,+ created_at=datetime(2026, 9, 2, tzinfo=UTC),+ )++ with (+ patch(+ "services.remote_file_service.remote_fetcher.make_request",+ side_effect=[head_response, content_response],+ ) as make_request,+ patch("services.remote_file_service.guess_file_info_from_response", return_value=file_info),+ patch("services.remote_file_service.file_helpers.get_signed_file_url", return_value="signed-url"),+ ):+ remote_file_service.upload_from_url(url=REMOTE_URL, user=account)++ assert make_request.call_args_list == [call("HEAD", url=REMOTE_URL), call("GET", url=REMOTE_URL)]+ assert file_service.upload_file.call_args.kwargs["content"] == b"downloaded content"+ assert file_service.upload_file.call_args.kwargs["tenant_id"] is None+++def test_upload_rejects_failed_content_download(+ remote_file_service: RemoteFileService,+ file_service: MagicMock,+) -> None:+ head_response = _response("HEAD", httpx.codes.OK)+ content_response = _response("GET", httpx.codes.BAD_GATEWAY, content=b"bad gateway")+ file_info = FileInfo(filename="report.pdf", extension=".pdf", mimetype="application/pdf", size=14)++ with (+ patch(+ "services.remote_file_service.remote_fetcher.make_request",+ side_effect=[head_response, content_response],+ ),+ patch("services.remote_file_service.guess_file_info_from_response", return_value=file_info),+ ):+ with pytest.raises(RemoteFileUnavailableError):+ remote_file_service.upload_from_url(url=REMOTE_URL, user=_account())++ file_service.upload_file.assert_not_called()+++def test_upload_rejects_invalid_remote_metadata(+ remote_file_service: RemoteFileService,+ file_service: MagicMock,+) -> None:+ response = _response("GET", httpx.codes.OK, content=b"remote content")+ metadata_error = InvalidRemoteFileMetadataError("invalid Content-Length")++ with (+ patch("services.remote_file_service.remote_fetcher.make_request", return_value=response),+ patch("services.remote_file_service.guess_file_info_from_response", side_effect=metadata_error),+ ):+ with pytest.raises(RemoteFileInvalidResponseError) as error_info:+ remote_file_service.upload_from_url(url=REMOTE_URL, user=_account())++ assert error_info.value.__cause__ is metadata_error+ file_service.upload_file.assert_not_called()+++def test_upload_does_not_translate_unknown_metadata_failure(+ remote_file_service: RemoteFileService,+ file_service: MagicMock,+) -> None:+ response = _response("GET", httpx.codes.OK, content=b"remote content")+ source_error = ValueError("unexpected parser bug")++ with (+ patch("services.remote_file_service.remote_fetcher.make_request", return_value=response),+ patch("services.remote_file_service.guess_file_info_from_response", side_effect=source_error),+ ):+ with pytest.raises(RuntimeError) as error_info:+ remote_file_service.upload_from_url(url=REMOTE_URL, user=_account())++ assert error_info.value.__cause__ is source_error+ file_service.upload_file.assert_not_called()+++def test_upload_rejects_remote_filename_with_path_separator(+ remote_file_service: RemoteFileService,+ file_service: MagicMock,+) -> None:+ response = _response("GET", httpx.codes.OK, content=b"remote content")+ file_info = FileInfo(filename="folder/report.pdf", extension=".pdf", mimetype="application/pdf", size=14)++ with (+ patch("services.remote_file_service.remote_fetcher.make_request", return_value=response),+ patch("services.remote_file_service.guess_file_info_from_response", return_value=file_info),+ ):+ with pytest.raises(RemoteFileInvalidResponseError):+ remote_file_service.upload_from_url(url=REMOTE_URL, user=_account())++ file_service.upload_file.assert_not_called()+++def test_upload_rejects_file_that_exceeds_size_limit(+ remote_file_service: RemoteFileService,+ file_service: MagicMock,+) -> None:+ response = _response("GET", httpx.codes.OK, content=b"remote content")+ file_info = FileInfo(filename="report.pdf", extension=".pdf", mimetype="application/pdf", size=1024)+ file_service.is_file_size_within_limit.return_value = False++ with (+ patch("services.remote_file_service.remote_fetcher.make_request", return_value=response),+ patch("services.remote_file_service.guess_file_info_from_response", return_value=file_info),+ ):+ with pytest.raises(FileTooLargeError):+ remote_file_service.upload_from_url(url=REMOTE_URL, user=_account())++ file_service.is_file_size_within_limit.assert_called_once_with(extension=".pdf", file_size=1024)+ file_service.upload_file.assert_not_called()diff --git a/packages/contracts/generated/api/console/remote-files/types.gen.ts b/packages/contracts/generated/api/console/remote-files/types.gen.tsindex 7a7e3ffa2ed..3edb44fe765 100644--- a/packages/contracts/generated/api/console/remote-files/types.gen.ts+++ b/packages/contracts/generated/api/console/remote-files/types.gen.ts@@ -31,6 +31,16 @@ export type PostRemoteFilesUploadData = {url: '/remote-files/upload'}+export type PostRemoteFilesUploadErrors = {+ 400: unknown+ 404: unknown+ 413: unknown+ 415: unknown+ 422: unknown+ 500: unknown+ 502: unknown+}+export type PostRemoteFilesUploadResponses = {201: FileWithSignedUrl}@@ -47,6 +57,13 @@ export type GetRemoteFilesByUrlData = {url: '/remote-files/{url}'}+export type GetRemoteFilesByUrlErrors = {+ 400: unknown+ 404: unknown+ 500: unknown+ 502: unknown+}+export type GetRemoteFilesByUrlResponses = {200: RemoteFileInfo}diff --git a/packages/contracts/generated/api/console/remote-files/zod.gen.ts b/packages/contracts/generated/api/console/remote-files/zod.gen.tsindex f3892990592..f132f9b564a 100644--- a/packages/contracts/generated/api/console/remote-files/zod.gen.ts+++ b/packages/contracts/generated/api/console/remote-files/zod.gen.ts@@ -6,7 +6,7 @@ import * as z from 'zod'* RemoteFileUploadPayload*/export const zRemoteFileUploadPayload = z.object({- url: z.string(),+ url: z.url(),})/**diff --git a/packages/contracts/generated/api/console/trial-apps/zod.gen.ts b/packages/contracts/generated/api/console/trial-apps/zod.gen.tsindex 395446a2002..9cdf7a94ca5 100644--- a/packages/contracts/generated/api/console/trial-apps/zod.gen.ts+++ b/packages/contracts/generated/api/console/trial-apps/zod.gen.ts@@ -64,7 +64,7 @@ export const zSuggestedQuestionsResponse = z.object({* RemoteFileUploadPayload*/export const zRemoteFileUploadPayload = z.object({- url: z.string(),+ url: z.url(),})/**diff --git a/packages/contracts/generated/api/web/orpc.gen.ts b/packages/contracts/generated/api/web/orpc.gen.tsindex 583c974803d..d235d56a60f 100644--- a/packages/contracts/generated/api/web/orpc.gen.ts+++ b/packages/contracts/generated/api/web/orpc.gen.ts@@ -798,14 +798,19 @@ export const passport = {* int: HTTP status code 201 for success** Raises:- * RemoteFileUploadError: Failed to fetch file from remote URL+ * RemoteFileInvalidUrlError: Remote file URL is invalid+ * RemoteFileUrlBlockedError: Remote file URL is blocked+ * RemoteFileNotFoundError: Remote file does not exist+ * RemoteFileAccessDeniedError: Remote file requires authorization+ * RemoteFileUnavailableError: Remote file is unavailable+ * RemoteFileInvalidResponseError: Remote file response is invalid* FileTooLargeError: File exceeds size limit* UnsupportedFileTypeError: File type not supported*/export const post19 = oc.route({description:- 'Upload a file from a remote URL\nDownloads a file from the provided remote URL and uploads it\nto the platform storage for use in web applications.\n\nArgs:\n app_model: The associated application model\n end_user: The end user making the request\n\nJSON Parameters:\n url: The remote URL to download the file from (required)\n\nReturns:\n dict: File information including ID, signed URL, and metadata\n int: HTTP status code 201 for success\n\nRaises:\n RemoteFileUploadError: Failed to fetch file from remote URL\n FileTooLargeError: File exceeds size limit\n UnsupportedFileTypeError: File type not supported',+ 'Upload a file from a remote URL\nDownloads a file from the provided remote URL and uploads it\nto the platform storage for use in web applications.\n\nArgs:\n app_model: The associated application model\n end_user: The end user making the request\n\nJSON Parameters:\n url: The remote URL to download the file from (required)\n\nReturns:\n dict: File information including ID, signed URL, and metadata\n int: HTTP status code 201 for success\n\nRaises:\n RemoteFileInvalidUrlError: Remote file URL is invalid\n RemoteFileUrlBlockedError: Remote file URL is blocked\n RemoteFileNotFoundError: Remote file does not exist\n RemoteFileAccessDeniedError: Remote file requires authorization\n RemoteFileUnavailableError: Remote file is unavailable\n RemoteFileInvalidResponseError: Remote file response is invalid\n FileTooLargeError: File exceeds size limit\n UnsupportedFileTypeError: File type not supported',inputStructure: 'detailed',method: 'POST',operationId: 'postRemoteFilesUpload',diff --git a/packages/contracts/generated/api/web/types.gen.ts b/packages/contracts/generated/api/web/types.gen.tsindex 8db116c5fef..09672af5c97 100644--- a/packages/contracts/generated/api/web/types.gen.ts+++ b/packages/contracts/generated/api/web/types.gen.ts@@ -1695,9 +1695,12 @@ export type PostRemoteFilesUploadData = {export type PostRemoteFilesUploadErrors = {400: unknown+ 404: unknown413: unknown415: unknown+ 422: unknown500: unknown+ 502: unknown}export type PostRemoteFilesUploadResponses = {@@ -1720,6 +1723,7 @@ export type GetRemoteFilesByUrlErrors = {400: unknown404: unknown500: unknown+ 502: unknown}export type GetRemoteFilesByUrlResponses = {diff --git a/packages/contracts/generated/api/web/zod.gen.ts b/packages/contracts/generated/api/web/zod.gen.tsindex 7b6aed9e8b3..d6c3a3a31d9 100644--- a/packages/contracts/generated/api/web/zod.gen.ts+++ b/packages/contracts/generated/api/web/zod.gen.ts@@ -478,7 +478,7 @@ export const zRemoteFileInfo = z.object({* RemoteFileUploadPayload*/export const zRemoteFileUploadPayload = z.object({- url: z.url().min(1).max(2083),+ url: z.url(),})/**