Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

refactor(api): centralize remote file handling (#41627)

非法操作ba1d5ba172ad
diff --git a/api/controllers/common/errors.py b/api/controllers/common/errors.py
index 31dd2fcd74f..c865e5eceb4 100644
--- a/api/controllers/common/errors.py
+++ b/api/controllers/common/errors.py
@@ -1,18 +1,54 @@
-from werkzeug.exceptions import HTTPException
-
from libs.exception import BaseHTTPException
-class FilenameNotExistsError(HTTPException):
+class FilenameNotExistsError(BaseHTTPException):
+ error_code = "filename_not_exists_error"
code = 400
description = "The specified filename does not exist."
-class RemoteFileUploadError(HTTPException):
+class RemoteFileUploadError(BaseHTTPException):
+ error_code = "remote_file_upload_error"
code = 400
description = "Error uploading remote file."
+class RemoteFileInvalidUrlError(BaseHTTPException):
+ error_code = "remote_file_invalid_url"
+ description = "The remote file URL is invalid."
+ code = 400
+
+
+class RemoteFileUrlBlockedError(BaseHTTPException):
+ error_code = "remote_file_url_blocked"
+ description = "The remote file URL is not allowed."
+ code = 400
+
+
+class RemoteFileNotFoundError(BaseHTTPException):
+ error_code = "remote_file_not_found"
+ description = "The remote file could not be found."
+ code = 404
+
+
+class RemoteFileAccessDeniedError(BaseHTTPException):
+ error_code = "remote_file_access_denied"
+ description = "The remote file cannot be accessed without authorization."
+ code = 400
+
+
+class RemoteFileUnavailableError(BaseHTTPException):
+ error_code = "remote_file_unavailable"
+ description = "The remote file is temporarily unavailable."
+ code = 502
+
+
+class RemoteFileInvalidResponseError(BaseHTTPException):
+ error_code = "remote_file_invalid_response"
+ description = "The remote file server returned an invalid response."
+ code = 502
+
+
class FileTooLargeError(BaseHTTPException):
error_code = "file_too_large"
description = "File size exceeded. {message}"
diff --git a/api/controllers/common/helpers.py b/api/controllers/common/helpers.py
index 56a07a8b4a9..23d5e2c56b7 100644
--- a/api/controllers/common/helpers.py
+++ b/api/controllers/common/helpers.py
@@ -1,39 +1,4 @@
-import contextlib
-import mimetypes
-import os
-import platform
-import re
import urllib.parse
-import warnings
-from uuid import uuid4
-
-import httpx
-
-try:
- import magic
-except ImportError:
- if platform.system() == "Windows":
- warnings.warn(
- "To use python-magic guess MIMETYPE, you need to run `pip install python-magic-bin`", stacklevel=2
- )
- elif platform.system() == "Darwin":
- warnings.warn("To use python-magic guess MIMETYPE, you need to run `brew install libmagic`", stacklevel=2)
- elif platform.system() == "Linux":
- warnings.warn(
- "To use python-magic guess MIMETYPE, you need to run `sudo apt-get install libmagic1`", stacklevel=2
- )
- else:
- warnings.warn("To use python-magic guess MIMETYPE, you need to install `libmagic`", stacklevel=2)
- magic = None # type: ignore[assignment]
-
-from pydantic import BaseModel
-
-
-class FileInfo(BaseModel):
- filename: str
- extension: str
- mimetype: str
- size: int
def decode_remote_url(url: str, query_string: bytes | str = b"") -> str:
@@ -45,59 +10,15 @@ def decode_remote_url(url: str, query_string: bytes | str = b"") -> str:
if not raw_query:
return decoded_url
+ try:
+ has_query = bool(urllib.parse.urlsplit(decoded_url).query)
+ except ValueError:
+ has_query = False
+
if decoded_url.endswith(("?", "&")):
separator = ""
- elif urllib.parse.urlsplit(decoded_url).query:
+ elif has_query:
separator = "&"
else:
separator = "?"
return f"{decoded_url}{separator}{raw_query}"
-
-
-def guess_file_info_from_response(response: httpx.Response):
- url = str(response.url)
- # Try to extract filename from URL
- parsed_url = urllib.parse.urlparse(url)
- url_path = parsed_url.path
- # Decode percent-encoded characters in the path segment
- filename = urllib.parse.unquote(os.path.basename(url_path))
-
- # If filename couldn't be extracted, use Content-Disposition header
- if not filename:
- content_disposition = response.headers.get("Content-Disposition")
- if content_disposition:
- filename_match = re.search(r'filename="?(.+)"?', content_disposition)
- if filename_match:
- filename = filename_match.group(1)
-
- # If still no filename, generate a unique one
- if not filename:
- unique_name = str(uuid4())
- filename = f"{unique_name}"
-
- # Guess MIME type from filename first, then URL
- mimetype, _ = mimetypes.guess_type(filename)
- if mimetype is None:
- mimetype, _ = mimetypes.guess_type(url)
- if mimetype is None:
- # If guessing fails, use Content-Type from response headers
- mimetype = response.headers.get("Content-Type", "application/octet-stream")
-
- # Use python-magic to guess MIME type if still unknown or generic
- if mimetype == "application/octet-stream" and magic is not None:
- with contextlib.suppress(magic.MagicException):
- mimetype = magic.from_buffer(response.content[:1024], mime=True)
-
- extension = os.path.splitext(filename)[1]
-
- # Ensure filename has an extension
- if not extension:
- extension = mimetypes.guess_extension(mimetype) or ".bin"
- filename = f"{filename}{extension}"
-
- return FileInfo(
- filename=filename,
- extension=extension,
- mimetype=mimetype,
- size=int(response.headers.get("Content-Length", -1)),
- )
diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py
index f14fe1eca95..87ff0317971 100644
--- a/api/controllers/console/app/app.py
+++ b/api/controllers/console/app/app.py
@@ -13,7 +13,6 @@ from werkzeug.exceptions import BadRequest, Forbidden, NotFound
from configs import dify_config
from controllers.common.app_access import resolve_app_access_filter
from controllers.common.fields import RedirectUrlResponse, SimpleResultResponse
-from controllers.common.helpers import FileInfo
from controllers.common.schema import (
query_params_from_model,
query_params_from_request,
@@ -39,6 +38,7 @@ from controllers.console.wraps import (
with_current_user,
with_current_user_id,
)
+from core.file.remote_file_metadata import FileInfo
from core.ops.ops_trace_manager import OpsTraceManager
from core.rag.entities import PreProcessingRule, Rule, Segmentation
from core.rag.retrieval.retrieval_methods import RetrievalMethod
diff --git a/api/controllers/console/explore/trial.py b/api/controllers/console/explore/trial.py
index d5388c7b542..e481c80b756 100644
--- a/api/controllers/console/explore/trial.py
+++ b/api/controllers/console/explore/trial.py
@@ -47,7 +47,7 @@ from controllers.console.explore.error import (
)
from controllers.console.explore.wraps import TrialAppResource
from controllers.console.files import FILE_UPLOAD_PARAMS, upload_file_from_request
-from controllers.console.remote_files import RemoteFileUploadPayload, upload_remote_file_from_request
+from controllers.console.remote_files import RemoteFileUploadPayload, upload_remote_file
from controllers.console.wraps import cloud_edition_billing_resource_check, model_validate, with_current_user
from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError
from core.app.apps.base_app_queue_manager import AppQueueManager
@@ -472,13 +472,15 @@ class TrialAppRemoteFileUploadApi(TrialAppResource):
@console_ns.expect(console_ns.models[RemoteFileUploadPayload.__name__])
@console_ns.response(201, "File uploaded successfully", console_ns.models[FileWithSignedUrl.__name__])
@with_current_user
- def post(self, current_user: Account, app_model: App):
+ @model_validate(RemoteFileUploadPayload)
+ def post(self, payload: RemoteFileUploadPayload, current_user: Account, app_model: App):
"""Upload a remote file into the tenant that owns the trial app."""
- remote_file = upload_remote_file_from_request(
+ remote_file = upload_remote_file(
+ url=payload.url,
current_user=current_user,
resource_tenant_id=app_model.tenant_id,
)
- return remote_file.model_dump(mode="json"), 201
+ return dump_response(FileWithSignedUrl, remote_file), 201
class TrialAppWorkflowRunApi(TrialAppResource):
diff --git a/api/controllers/console/remote_files.py b/api/controllers/console/remote_files.py
index e3aefb72595..e7e4eef2f53 100644
--- a/api/controllers/console/remote_files.py
+++ b/api/controllers/console/remote_files.py
@@ -1,4 +1,3 @@
-import httpx
from flask import request
from flask_restx import Resource
from pydantic import BaseModel, Field
@@ -6,24 +5,41 @@ from pydantic import BaseModel, Field
import services
from controllers.common import helpers
from controllers.common.errors import (
+ BlockedFileExtensionError,
FileTooLargeError,
- RemoteFileUploadError,
+ RemoteFileAccessDeniedError,
+ RemoteFileInvalidResponseError,
+ RemoteFileInvalidUrlError,
+ RemoteFileNotFoundError,
+ RemoteFileUnavailableError,
+ RemoteFileUrlBlockedError,
UnsupportedFileTypeError,
)
-from controllers.common.schema import register_response_schema_models, register_schema_models
+from controllers.common.schema import JsonResponseWithStatus, register_response_schema_models, register_schema_models
from controllers.console import console_ns
-from controllers.console.wraps import with_current_user
-from core.file import remote_fetcher
-from extensions.ext_database import db
+from controllers.console.flask_admission import console_account_admission
+from controllers.console.wraps import model_validate, with_current_user
+from extensions.ext_application_services import application_services
from fields.file_fields import FileWithSignedUrl, RemoteFileInfo
-from graphon.file import helpers as file_helpers
+from libs.helper import dump_response
from libs.login import login_required
+from machinery.context import RequestContext
from models import Account
-from services.file_service import FileService
+from services.remote_file_service import (
+ RemoteFileAccessDeniedError as RemoteFileAccessDeniedServiceError,
+)
+from services.remote_file_service import (
+ RemoteFileInvalidResponseError as RemoteFileInvalidResponseServiceError,
+)
+from services.remote_file_service import RemoteFileInvalidUrlError as RemoteFileInvalidUrlServiceError
+from services.remote_file_service import RemoteFileNotFoundError as RemoteFileNotFoundServiceError
+from services.remote_file_service import RemoteFileUnavailableError as RemoteFileUnavailableServiceError
+from services.remote_file_service import RemoteFileUploadResult
+from services.remote_file_service import RemoteFileUrlBlockedError as RemoteFileUrlBlockedServiceError
class RemoteFileUploadPayload(BaseModel):
- url: str = Field(..., description="URL to fetch")
+ url: str = Field(description="URL to fetch", json_schema_extra={"format": "uri"})
register_schema_models(console_ns, RemoteFileUploadPayload)
@@ -32,84 +48,96 @@ register_response_schema_models(console_ns, FileWithSignedUrl, RemoteFileInfo)
@console_ns.route("/remote-files/<path:url>")
class GetRemoteFileInfo(Resource):
+ @console_ns.doc(
+ responses={
+ 400: "Invalid, blocked, or inaccessible remote file URL",
+ 404: "Remote file not found",
+ 502: "Remote file unavailable or returned an invalid response",
+ 500: "Internal server error",
+ }
+ )
@console_ns.response(200, "Success", console_ns.models[RemoteFileInfo.__name__])
- @login_required
- def get(self, url: str):
+ @console_account_admission()
+ def get(self, _request_context: RequestContext, url: str):
decoded_url = helpers.decode_remote_url(url, request.query_string)
- resp = remote_fetcher.make_request("HEAD", decoded_url)
- if resp.status_code != httpx.codes.OK:
- resp = remote_fetcher.make_request("GET", decoded_url, timeout=3)
- resp.raise_for_status()
- return RemoteFileInfo(
- file_type=resp.headers.get("Content-Type", "application/octet-stream"),
- file_length=int(resp.headers.get("Content-Length", 0)),
- ).model_dump(mode="json")
+ try:
+ file_info = application_services().remote_files.fetch_info(url=decoded_url)
+ except RemoteFileInvalidUrlServiceError as error:
+ raise RemoteFileInvalidUrlError from error
+ except RemoteFileUrlBlockedServiceError as error:
+ raise RemoteFileUrlBlockedError from error
+ except RemoteFileNotFoundServiceError as error:
+ raise RemoteFileNotFoundError from error
+ except RemoteFileAccessDeniedServiceError as error:
+ raise RemoteFileAccessDeniedError from error
+ except RemoteFileUnavailableServiceError as error:
+ raise RemoteFileUnavailableError from error
+ except RemoteFileInvalidResponseServiceError as error:
+ raise RemoteFileInvalidResponseError from error
+
+ return dump_response(
+ RemoteFileInfo,
+ {
+ "file_type": file_info.content_type,
+ "file_length": file_info.content_length if file_info.content_length is not None else 0,
+ },
+ )
-def upload_remote_file_from_request(
+def upload_remote_file(
*,
+ url: str,
current_user: Account,
resource_tenant_id: str | None = None,
-) -> FileWithSignedUrl:
- payload = RemoteFileUploadPayload.model_validate(console_ns.payload)
- """Validate the JSON request, fetch its remote file, and persist it under the requested tenant."""
- url = payload.url
-
- # Try to fetch remote file metadata/content first
- try:
- resp = remote_fetcher.make_request("HEAD", url=url)
- if resp.status_code != httpx.codes.OK:
- resp = remote_fetcher.make_request("GET", url=url, timeout=3, follow_redirects=True)
- if resp.status_code != httpx.codes.OK:
- # Normalize into a user-friendly error message expected by tests
- raise RemoteFileUploadError(f"Failed to fetch file from {url}: {resp.text}")
- except httpx.RequestError as e:
- raise RemoteFileUploadError(f"Failed to fetch file from {url}: {str(e)}")
-
- file_info = helpers.guess_file_info_from_response(resp)
-
- # Enforce file size limit with 400 (Bad Request) per tests' expectation
- if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):
- raise FileTooLargeError()
-
- # Load content if needed
- content = resp.content if resp.request.method == "GET" else remote_fetcher.make_request("GET", url).content
-
+) -> RemoteFileUploadResult:
+ """Fetch a remote file and persist it under the requested tenant."""
try:
- upload_file = FileService(db.engine).upload_file(
- filename=file_info.filename,
- content=content,
- mimetype=file_info.mimetype,
+ return application_services().remote_files.upload_from_url(
+ url=url,
user=current_user,
tenant_id=resource_tenant_id,
- source_url=url,
)
- except services.errors.file.FileTooLargeError as file_too_large_error:
- raise FileTooLargeError(file_too_large_error.description)
- except services.errors.file.UnsupportedFileTypeError:
- raise UnsupportedFileTypeError()
-
- return FileWithSignedUrl(
- id=upload_file.id,
- name=upload_file.name,
- size=upload_file.size,
- extension=upload_file.extension,
- url=file_helpers.get_signed_file_url(upload_file_id=upload_file.id),
- mime_type=upload_file.mime_type,
- created_by=upload_file.created_by,
- created_at=int(upload_file.created_at.timestamp()),
- )
+ except RemoteFileInvalidUrlServiceError as error:
+ raise RemoteFileInvalidUrlError from error
+ except RemoteFileUrlBlockedServiceError as error:
+ raise RemoteFileUrlBlockedError from error
+ except RemoteFileNotFoundServiceError as error:
+ raise RemoteFileNotFoundError from error
+ except RemoteFileAccessDeniedServiceError as error:
+ raise RemoteFileAccessDeniedError from error
+ except RemoteFileUnavailableServiceError as error:
+ raise RemoteFileUnavailableError from error
+ except RemoteFileInvalidResponseServiceError as error:
+ raise RemoteFileInvalidResponseError from error
+ except services.errors.file.FileTooLargeError as error:
+ raise FileTooLargeError(error.description or "File size exceeded.") from error
+ except services.errors.file.UnsupportedFileTypeError as error:
+ raise UnsupportedFileTypeError from error
+ except services.errors.file.BlockedFileExtensionError as error:
+ raise BlockedFileExtensionError(error.description) from error
@console_ns.route("/remote-files/upload")
class RemoteFileUpload(Resource):
+ @console_ns.doc(
+ responses={
+ 400: "Invalid, blocked, or inaccessible remote file URL",
+ 404: "Remote file not found",
+ 413: "File too large",
+ 415: "Unsupported file type",
+ 422: "Request payload validation failed",
+ 502: "Remote file unavailable or returned an invalid response",
+ 500: "Internal server error",
+ }
+ )
@console_ns.expect(console_ns.models[RemoteFileUploadPayload.__name__])
@console_ns.response(201, "File uploaded successfully", console_ns.models[FileWithSignedUrl.__name__])
@login_required
@with_current_user
- def post(self, current_user: Account):
- remote_file = upload_remote_file_from_request(current_user=current_user)
- return (
- remote_file.model_dump(mode="json"),
- 201,
+ @model_validate(RemoteFileUploadPayload)
+ def post(self, payload: RemoteFileUploadPayload, current_user: Account) -> JsonResponseWithStatus:
+ remote_file = upload_remote_file(
+ url=payload.url,
+ current_user=current_user,
)
+ return dump_response(FileWithSignedUrl, remote_file), 201
diff --git a/api/controllers/web/human_input_file_upload.py b/api/controllers/web/human_input_file_upload.py
index 5203951a993..dcddec8f35b 100644
--- a/api/controllers/web/human_input_file_upload.py
+++ b/api/controllers/web/human_input_file_upload.py
@@ -13,7 +13,6 @@ from pydantic import BaseModel, ConfigDict, Field, HttpUrl
from sqlalchemy.orm import sessionmaker
import services
-from controllers.common import helpers
from controllers.common.errors import (
BlockedFileExtensionError,
FileTooLargeError,
@@ -24,6 +23,7 @@ from controllers.common.errors import (
)
from controllers.common.schema import register_schema_models
from controllers.web import web_ns
+from core.file.remote_file_metadata import guess_file_info_from_response
from core.helper import ssrf_proxy
from extensions.ext_database import db
from fields.file_fields import FileResponse, FileWithSignedUrl
@@ -155,7 +155,7 @@ def _upload_remote_file(context, url: str):
except httpx.RequestError as exc:
raise RemoteFileUploadError(f"Failed to fetch file from {url}: {str(exc)}")
- file_info = helpers.guess_file_info_from_response(resp)
+ file_info = guess_file_info_from_response(resp)
if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):
raise FileTooLargeError()
diff --git a/api/controllers/web/remote_files.py b/api/controllers/web/remote_files.py
index ab099077b79..51004073c91 100644
--- a/api/controllers/web/remote_files.py
+++ b/api/controllers/web/remote_files.py
@@ -1,21 +1,34 @@
-import httpx
from flask import request
-from pydantic import BaseModel, Field, HttpUrl
+from pydantic import BaseModel, Field
import services
from controllers.common import helpers
from controllers.common.errors import (
+ BlockedFileExtensionError,
FileTooLargeError,
- RemoteFileUploadError,
+ RemoteFileAccessDeniedError,
+ RemoteFileInvalidResponseError,
+ RemoteFileInvalidUrlError,
+ RemoteFileNotFoundError,
+ RemoteFileUnavailableError,
+ RemoteFileUrlBlockedError,
UnsupportedFileTypeError,
)
from controllers.console.wraps import model_validate
-from core.file import remote_fetcher
-from extensions.ext_database import db
+from extensions.ext_application_services import application_services
from fields.file_fields import FileWithSignedUrl, RemoteFileInfo
-from graphon.file import helpers as file_helpers
+from libs.helper import dump_response
from models.model import App, EndUser
-from services.file_service import FileService
+from services.remote_file_service import (
+ RemoteFileAccessDeniedError as RemoteFileAccessDeniedServiceError,
+)
+from services.remote_file_service import (
+ RemoteFileInvalidResponseError as RemoteFileInvalidResponseServiceError,
+)
+from services.remote_file_service import RemoteFileInvalidUrlError as RemoteFileInvalidUrlServiceError
+from services.remote_file_service import RemoteFileNotFoundError as RemoteFileNotFoundServiceError
+from services.remote_file_service import RemoteFileUnavailableError as RemoteFileUnavailableServiceError
+from services.remote_file_service import RemoteFileUrlBlockedError as RemoteFileUrlBlockedServiceError
from ..common.schema import register_response_schema_models, register_schema_models
from . import web_ns
@@ -23,7 +36,7 @@ from .wraps import WebApiResource
class RemoteFileUploadPayload(BaseModel):
- url: HttpUrl = Field(description="Remote file URL")
+ url: str = Field(description="Remote file URL", json_schema_extra={"format": "uri"})
register_schema_models(web_ns, RemoteFileUploadPayload)
@@ -37,9 +50,10 @@ class RemoteFileInfoApi(WebApiResource):
@web_ns.doc(
responses={
200: "Remote file information retrieved successfully",
- 400: "Bad request - invalid URL",
+ 400: "Invalid, blocked, or inaccessible remote file URL",
404: "Remote file not found",
- 500: "Failed to fetch remote file",
+ 502: "Remote file unavailable or returned an invalid response",
+ 500: "Internal server error",
}
)
@web_ns.response(200, "Remote file info", web_ns.models[RemoteFileInfo.__name__])
@@ -61,15 +75,28 @@ class RemoteFileInfoApi(WebApiResource):
HTTPException: If the remote file cannot be accessed
"""
decoded_url = helpers.decode_remote_url(url, request.query_string)
- resp = remote_fetcher.make_request("HEAD", decoded_url)
- if resp.status_code != httpx.codes.OK:
- # failed back to get method
- resp = remote_fetcher.make_request("GET", decoded_url, timeout=3)
- resp.raise_for_status()
- return RemoteFileInfo(
- file_type=resp.headers.get("Content-Type", "application/octet-stream"),
- file_length=int(resp.headers.get("Content-Length", -1)),
- ).model_dump(mode="json")
+ try:
+ file_info = application_services().remote_files.fetch_info(url=decoded_url)
+ except RemoteFileInvalidUrlServiceError as error:
+ raise RemoteFileInvalidUrlError from error
+ except RemoteFileUrlBlockedServiceError as error:
+ raise RemoteFileUrlBlockedError from error
+ except RemoteFileNotFoundServiceError as error:
+ raise RemoteFileNotFoundError from error
+ except RemoteFileAccessDeniedServiceError as error:
+ raise RemoteFileAccessDeniedError from error
+ except RemoteFileUnavailableServiceError as error:
+ raise RemoteFileUnavailableError from error
+ except RemoteFileInvalidResponseServiceError as error:
+ raise RemoteFileInvalidResponseError from error
+
+ return dump_response(
+ RemoteFileInfo,
+ {
+ "file_type": file_info.content_type,
+ "file_length": file_info.content_length if file_info.content_length is not None else -1,
+ },
+ )
@web_ns.route("/remote-files/upload")
@@ -79,10 +106,13 @@ class RemoteFileUploadApi(WebApiResource):
@web_ns.doc(
responses={
201: "Remote file uploaded successfully",
- 400: "Bad request - invalid URL or parameters",
+ 400: "Invalid, blocked, or inaccessible remote file URL",
+ 404: "Remote file not found",
413: "File too large",
415: "Unsupported file type",
- 500: "Failed to fetch remote file",
+ 422: "Request payload validation failed",
+ 502: "Remote file unavailable or returned an invalid response",
+ 500: "Internal server error",
}
)
@web_ns.response(201, "Remote file uploaded", web_ns.models[FileWithSignedUrl.__name__])
@@ -106,48 +136,37 @@ class RemoteFileUploadApi(WebApiResource):
int: HTTP status code 201 for success
Raises:
- RemoteFileUploadError: Failed to fetch file from remote URL
+ RemoteFileInvalidUrlError: Remote file URL is invalid
+ RemoteFileUrlBlockedError: Remote file URL is blocked
+ RemoteFileNotFoundError: Remote file does not exist
+ RemoteFileAccessDeniedError: Remote file requires authorization
+ RemoteFileUnavailableError: Remote file is unavailable
+ RemoteFileInvalidResponseError: Remote file response is invalid
FileTooLargeError: File exceeds size limit
UnsupportedFileTypeError: File type not supported
"""
- url = str(payload.url)
-
- try:
- resp = remote_fetcher.make_request("HEAD", url=url)
- if resp.status_code != httpx.codes.OK:
- resp = remote_fetcher.make_request("GET", url=url, timeout=3, follow_redirects=True)
- if resp.status_code != httpx.codes.OK:
- raise RemoteFileUploadError(f"Failed to fetch file from {url}: {resp.text}")
- except httpx.RequestError as e:
- raise RemoteFileUploadError(f"Failed to fetch file from {url}: {str(e)}")
-
- file_info = helpers.guess_file_info_from_response(resp)
-
- if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):
- raise FileTooLargeError
-
- content = resp.content if resp.request.method == "GET" else remote_fetcher.make_request("GET", url).content
-
try:
- upload_file = FileService(db.engine).upload_file(
- filename=file_info.filename,
- content=content,
- mimetype=file_info.mimetype,
+ remote_file = application_services().remote_files.upload_from_url(
+ url=payload.url,
user=end_user,
- source_url=url,
)
- except services.errors.file.FileTooLargeError as file_too_large_error:
- raise FileTooLargeError(file_too_large_error.description)
- except services.errors.file.UnsupportedFileTypeError:
- raise UnsupportedFileTypeError
-
- return FileWithSignedUrl(
- id=upload_file.id,
- name=upload_file.name,
- size=upload_file.size,
- extension=upload_file.extension,
- url=file_helpers.get_signed_file_url(upload_file_id=upload_file.id),
- mime_type=upload_file.mime_type,
- created_by=upload_file.created_by,
- created_at=int(upload_file.created_at.timestamp()),
- ).model_dump(mode="json"), 201
+ except RemoteFileInvalidUrlServiceError as error:
+ raise RemoteFileInvalidUrlError from error
+ except RemoteFileUrlBlockedServiceError as error:
+ raise RemoteFileUrlBlockedError from error
+ except RemoteFileNotFoundServiceError as error:
+ raise RemoteFileNotFoundError from error
+ except RemoteFileAccessDeniedServiceError as error:
+ raise RemoteFileAccessDeniedError from error
+ except RemoteFileUnavailableServiceError as error:
+ raise RemoteFileUnavailableError from error
+ except RemoteFileInvalidResponseServiceError as error:
+ raise RemoteFileInvalidResponseError from error
+ except services.errors.file.FileTooLargeError as error:
+ raise FileTooLargeError(error.description or "File size exceeded.") from error
+ except services.errors.file.UnsupportedFileTypeError as error:
+ raise UnsupportedFileTypeError from error
+ except services.errors.file.BlockedFileExtensionError as error:
+ raise BlockedFileExtensionError(error.description) from error
+
+ return dump_response(FileWithSignedUrl, remote_file), 201
diff --git a/api/core/file/remote_file_metadata.py b/api/core/file/remote_file_metadata.py
new file mode 100644
index 00000000000..d4d14860fcc
--- /dev/null
+++ b/api/core/file/remote_file_metadata.py
@@ -0,0 +1,94 @@
+import contextlib
+import mimetypes
+import os
+import platform
+import re
+import urllib.parse
+import warnings
+from uuid import uuid4
+
+import httpx
+
+try:
+ import magic
+except ImportError:
+ if platform.system() == "Windows":
+ warnings.warn(
+ "To use python-magic guess MIMETYPE, you need to run `pip install python-magic-bin`", stacklevel=2
+ )
+ elif platform.system() == "Darwin":
+ warnings.warn("To use python-magic guess MIMETYPE, you need to run `brew install libmagic`", stacklevel=2)
+ elif platform.system() == "Linux":
+ warnings.warn(
+ "To use python-magic guess MIMETYPE, you need to run `sudo apt-get install libmagic1`", stacklevel=2
+ )
+ else:
+ warnings.warn("To use python-magic guess MIMETYPE, you need to install `libmagic`", stacklevel=2)
+ magic = None # type: ignore[assignment]
+
+from pydantic import BaseModel
+
+
+class FileInfo(BaseModel):
+ filename: str
+ extension: str
+ mimetype: str
+ size: int
+
+
+class InvalidRemoteFileMetadataError(ValueError):
+ pass
+
+
+def guess_file_info_from_response(response: httpx.Response) -> FileInfo:
+ url = str(response.url)
+ # Try to extract filename from URL
+ parsed_url = urllib.parse.urlparse(url)
+ url_path = parsed_url.path
+ # Decode percent-encoded characters in the path segment
+ filename = urllib.parse.unquote(os.path.basename(url_path))
+
+ # If filename couldn't be extracted, use Content-Disposition header
+ if not filename:
+ content_disposition = response.headers.get("Content-Disposition")
+ if content_disposition:
+ filename_match = re.search(r'filename="?(.+)"?', content_disposition)
+ if filename_match:
+ filename = filename_match.group(1)
+
+ # If still no filename, generate a unique one
+ if not filename:
+ unique_name = str(uuid4())
+ filename = f"{unique_name}"
+
+ # Guess MIME type from filename first, then URL
+ mimetype, _ = mimetypes.guess_type(filename)
+ if mimetype is None:
+ mimetype, _ = mimetypes.guess_type(url)
+ if mimetype is None:
+ # If guessing fails, use Content-Type from response headers
+ mimetype = response.headers.get("Content-Type", "application/octet-stream")
+
+ # Use python-magic to guess MIME type if still unknown or generic
+ if mimetype == "application/octet-stream" and magic is not None:
+ with contextlib.suppress(magic.MagicException):
+ mimetype = magic.from_buffer(response.content[:1024], mime=True)
+
+ extension = os.path.splitext(filename)[1]
+
+ # Ensure filename has an extension
+ if not extension:
+ extension = mimetypes.guess_extension(mimetype) or ".bin"
+ filename = f"{filename}{extension}"
+
+ try:
+ size = int(response.headers.get("Content-Length", -1))
+ except ValueError as error:
+ raise InvalidRemoteFileMetadataError("The Content-Length header is invalid") from error
+
+ return FileInfo(
+ filename=filename,
+ extension=extension,
+ mimetype=mimetype,
+ size=size,
+ )
diff --git a/api/extensions/ext_application_services.py b/api/extensions/ext_application_services.py
index 1ceaf4a4fcc..986fc5a79c4 100644
--- a/api/extensions/ext_application_services.py
+++ b/api/extensions/ext_application_services.py
@@ -129,6 +129,7 @@ from services.recommended_app_catalog_gateway import (
RemoteRecommendedAppCatalogGateway,
)
from services.recommended_app_query_service import RecommendedAppQueryService
+from services.remote_file_service import RemoteFileService
from services.retention.workflow_run.archive_download_adapters import (
dispatch_workflow_run_archive_download_task,
sign_workflow_run_archive_download_url,
@@ -218,6 +219,7 @@ class ApplicationServices:
step_by_step_tour: StepByStepTourService
partner_tenant_bindings: PartnerTenantBindingService
recommended_app_queries: RecommendedAppQueryService
+ remote_files: RemoteFileService
trial_app_usage: TrialAppUsageRecorder
workflow_run_archives: WorkflowRunArchiveService
workspace_queries: WorkspaceQueryService
@@ -502,6 +504,9 @@ def build_application_services(
trial_apps=TrialAppQueryRepository(session_factory=database_client),
trial_enabled=trial_app_enabled,
),
+ remote_files=RemoteFileService(
+ files=FileService(session_factory=database_client),
+ ),
trial_app_usage=TrialAppUsageRepository(session_factory=database_client),
workflow_run_archives=WorkflowRunArchiveService(
bundles=WorkflowRunArchiveBundleQueryRepository(session_factory=database_client),
diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md
index 79313ba2ebb..4a63869ccd9 100644
--- a/api/openapi/markdown/console-openapi.md
+++ b/api/openapi/markdown/console-openapi.md
@@ -8393,6 +8393,13 @@ Update account-level Step-by-step Tour state
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 201 | File uploaded successfully | **application/json**: [FileWithSignedUrl](#filewithsignedurl)<br> |
+| 400 | Invalid, blocked, or inaccessible remote file URL | |
+| 404 | Remote file not found | |
+| 413 | File too large | |
+| 415 | Unsupported file type | |
+| 422 | Request payload validation failed | |
+| 500 | Internal server error | |
+| 502 | Remote file unavailable or returned an invalid response | |
### [GET] /remote-files/{url}
#### Parameters
@@ -8406,6 +8413,10 @@ Update account-level Step-by-step Tour state
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Success | **application/json**: [RemoteFileInfo](#remotefileinfo)<br> |
+| 400 | Invalid, blocked, or inaccessible remote file URL | |
+| 404 | Remote file not found | |
+| 500 | Internal server error | |
+| 502 | Remote file unavailable or returned an invalid response | |
### [POST] /reset-password
#### Request Body
@@ -21347,7 +21358,7 @@ Model class for provider quota configuration.
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
-| url | string | URL to fetch | Yes |
+| url | string (uri) | URL to fetch | Yes |
#### ReplaceUserAccessPolicies
diff --git a/api/openapi/markdown/web-openapi.md b/api/openapi/markdown/web-openapi.md
index 1cb85bf66d8..1f6d7a8e723 100644
--- a/api/openapi/markdown/web-openapi.md
+++ b/api/openapi/markdown/web-openapi.md
@@ -638,7 +638,12 @@ Returns:
int: HTTP status code 201 for success
Raises:
- RemoteFileUploadError: Failed to fetch file from remote URL
+ RemoteFileInvalidUrlError: Remote file URL is invalid
+ RemoteFileUrlBlockedError: Remote file URL is blocked
+ RemoteFileNotFoundError: Remote file does not exist
+ RemoteFileAccessDeniedError: Remote file requires authorization
+ RemoteFileUnavailableError: Remote file is unavailable
+ RemoteFileInvalidResponseError: Remote file response is invalid
FileTooLargeError: File exceeds size limit
UnsupportedFileTypeError: File type not supported
@@ -653,10 +658,13 @@ Raises:
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 201 | Remote file uploaded successfully | **application/json**: [FileWithSignedUrl](#filewithsignedurl)<br> |
-| 400 | Bad request - invalid URL or parameters | |
+| 400 | Invalid, blocked, or inaccessible remote file URL | |
+| 404 | Remote file not found | |
| 413 | File too large | |
| 415 | Unsupported file type | |
-| 500 | Failed to fetch remote file | |
+| 422 | Request payload validation failed | |
+| 500 | Internal server error | |
+| 502 | Remote file unavailable or returned an invalid response | |
### [GET] /remote-files/{url}
**Get information about a remote file**
@@ -687,9 +695,10 @@ Raises:
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 200 | Remote file information retrieved successfully | **application/json**: [RemoteFileInfo](#remotefileinfo)<br> |
-| 400 | Bad request - invalid URL | |
+| 400 | Invalid, blocked, or inaccessible remote file URL | |
| 404 | Remote file not found | |
-| 500 | Failed to fetch remote file | |
+| 500 | Internal server error | |
+| 502 | Remote file unavailable or returned an invalid response | |
### [GET] /saved-messages
Retrieve paginated list of saved messages for a completion application.
diff --git a/api/services/remote_file_service.py b/api/services/remote_file_service.py
new file mode 100644
index 00000000000..1ccb63ebb9a
--- /dev/null
+++ b/api/services/remote_file_service.py
@@ -0,0 +1,185 @@
+import urllib.parse
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Literal
+
+import httpx
+
+from core.file import remote_fetcher
+from core.file.remote_file_metadata import InvalidRemoteFileMetadataError, guess_file_info_from_response
+from core.helper.ssrf_proxy import MaxRetriesExceededError
+from core.tools.errors import ToolSSRFError
+from graphon.file import helpers as file_helpers
+from models import Account
+from models.model import EndUser
+from services.errors.file import FileTooLargeError
+from services.file_service import FileService
+
+
+@dataclass(frozen=True, slots=True)
+class RemoteFileInfoResult:
+ content_type: str
+ content_length: int | None
+
+
+@dataclass(frozen=True, slots=True)
+class RemoteFileUploadResult:
+ id: str
+ name: str
+ size: int
+ extension: str
+ url: str
+ mime_type: str | None
+ created_by: str
+ created_at: datetime
+
+
+class RemoteFileError(Exception):
+ pass
+
+
+class RemoteFileInvalidUrlError(RemoteFileError):
+ pass
+
+
+class RemoteFileUrlBlockedError(RemoteFileError):
+ pass
+
+
+class RemoteFileNotFoundError(RemoteFileError):
+ pass
+
+
+class RemoteFileAccessDeniedError(RemoteFileError):
+ pass
+
+
+class RemoteFileUnavailableError(RemoteFileError):
+ pass
+
+
+class RemoteFileInvalidResponseError(RemoteFileError):
+ pass
+
+
+class RemoteFileService:
+ def __init__(self, *, files: FileService) -> None:
+ self._files = files
+
+ def fetch_info(self, *, url: str) -> RemoteFileInfoResult:
+ response = self._request("HEAD", url=url)
+ if response.status_code != httpx.codes.OK:
+ response = self._request("GET", url=url, timeout=3)
+ self._ensure_success(response)
+
+ content_length = response.headers.get("Content-Length")
+ try:
+ parsed_content_length = int(content_length) if content_length is not None else None
+ except ValueError as error:
+ raise RemoteFileInvalidResponseError("The remote response has an invalid Content-Length header") from error
+
+ return RemoteFileInfoResult(
+ content_type=response.headers.get("Content-Type", "application/octet-stream"),
+ content_length=parsed_content_length,
+ )
+
+ def upload_from_url(
+ self,
+ *,
+ url: str,
+ user: Account | EndUser,
+ tenant_id: str | None = None,
+ ) -> RemoteFileUploadResult:
+ response = self._fetch_for_upload(url=url)
+ try:
+ file_info = guess_file_info_from_response(response)
+ except InvalidRemoteFileMetadataError as error:
+ raise RemoteFileInvalidResponseError("The remote response contains invalid file metadata") from error
+ except ValueError as error:
+ # Unclassified parser failures are server bugs, not invalid request parameters.
+ raise RuntimeError("Unexpected remote file metadata parsing failure") from error
+
+ if any(separator in file_info.filename for separator in ("/", "\\")):
+ raise RemoteFileInvalidResponseError("The remote response contains an invalid filename")
+
+ if not self._files.is_file_size_within_limit(
+ extension=file_info.extension,
+ file_size=file_info.size,
+ ):
+ raise FileTooLargeError()
+
+ if response.request.method == "GET":
+ content = response.content
+ else:
+ content = self._fetch_content(url=url)
+
+ upload_file = self._files.upload_file(
+ filename=file_info.filename,
+ content=content,
+ mimetype=file_info.mimetype,
+ user=user,
+ tenant_id=tenant_id,
+ source_url=url,
+ )
+ return RemoteFileUploadResult(
+ id=upload_file.id,
+ name=upload_file.name,
+ size=upload_file.size,
+ extension=upload_file.extension,
+ url=file_helpers.get_signed_file_url(upload_file_id=upload_file.id),
+ mime_type=upload_file.mime_type,
+ created_by=upload_file.created_by,
+ created_at=upload_file.created_at,
+ )
+
+ @staticmethod
+ def _request(
+ method: Literal["GET", "HEAD"],
+ *,
+ url: str,
+ **kwargs: Any,
+ ) -> httpx.Response:
+ try:
+ parsed_url = urllib.parse.urlsplit(url)
+ port = parsed_url.port
+ except ValueError as error:
+ raise RemoteFileInvalidUrlError("The remote file URL is invalid") from error
+
+ if parsed_url.scheme not in {"http", "https"} or parsed_url.hostname is None or port == 0:
+ raise RemoteFileInvalidUrlError("The remote file URL is invalid")
+
+ try:
+ return remote_fetcher.make_request(method, url=url, **kwargs)
+ except httpx.InvalidURL as error:
+ raise RemoteFileInvalidUrlError("The remote file URL is invalid") from error
+ except ToolSSRFError as error:
+ raise RemoteFileUrlBlockedError("The remote file URL was blocked by SSRF protection") from error
+ except (MaxRetriesExceededError, httpx.RequestError) as error:
+ raise RemoteFileUnavailableError("The remote file request failed") from error
+
+ @classmethod
+ def _fetch_for_upload(cls, *, url: str) -> httpx.Response:
+ response = cls._request("HEAD", url=url)
+ if response.status_code != httpx.codes.OK:
+ response = cls._request("GET", url=url, timeout=3, follow_redirects=True)
+
+ cls._ensure_success(response)
+ return response
+
+ @staticmethod
+ def _ensure_success(response: httpx.Response) -> None:
+ if response.status_code == httpx.codes.OK:
+ return
+
+ if response.status_code in {httpx.codes.NOT_FOUND, httpx.codes.GONE}:
+ raise RemoteFileNotFoundError("The remote file does not exist")
+ if response.status_code in {httpx.codes.UNAUTHORIZED, httpx.codes.FORBIDDEN}:
+ raise RemoteFileAccessDeniedError("The remote file cannot be accessed anonymously")
+
+ raise RemoteFileUnavailableError(f"The remote file request returned HTTP {response.status_code}")
+
+ @classmethod
+ def _fetch_content(cls, *, url: str) -> bytes:
+ response = cls._request("GET", url=url)
+ cls._ensure_success(response)
+ return response.content
diff --git a/api/tests/unit_tests/controllers/common/test_errors.py b/api/tests/unit_tests/controllers/common/test_errors.py
index 25a9fe5b668..ada7e6d379e 100644
--- a/api/tests/unit_tests/controllers/common/test_errors.py
+++ b/api/tests/unit_tests/controllers/common/test_errors.py
@@ -1,12 +1,21 @@
+import pytest
+
from controllers.common.errors import (
BlockedFileExtensionError,
FilenameNotExistsError,
FileTooLargeError,
NoFileUploadedError,
+ RemoteFileAccessDeniedError,
+ RemoteFileInvalidResponseError,
+ RemoteFileInvalidUrlError,
+ RemoteFileNotFoundError,
+ RemoteFileUnavailableError,
RemoteFileUploadError,
+ RemoteFileUrlBlockedError,
TooManyFilesError,
UnsupportedFileTypeError,
)
+from libs.exception import BaseHTTPException
class TestFilenameNotExistsError:
@@ -14,7 +23,13 @@ class TestFilenameNotExistsError:
error = FilenameNotExistsError()
assert error.code == 400
+ assert error.error_code == "filename_not_exists_error"
assert error.description == "The specified filename does not exist."
+ assert error.data == {
+ "code": "filename_not_exists_error",
+ "message": "The specified filename does not exist.",
+ "status": 400,
+ }
class TestRemoteFileUploadError:
@@ -22,7 +37,53 @@ class TestRemoteFileUploadError:
error = RemoteFileUploadError()
assert error.code == 400
+ assert error.error_code == "remote_file_upload_error"
assert error.description == "Error uploading remote file."
+ assert error.data == {
+ "code": "remote_file_upload_error",
+ "message": "Error uploading remote file.",
+ "status": 400,
+ }
+
+
+ ("error_type", "error_code", "description", "status"),
+ [
+ (RemoteFileInvalidUrlError, "remote_file_invalid_url", "The remote file URL is invalid.", 400),
+ (RemoteFileUrlBlockedError, "remote_file_url_blocked", "The remote file URL is not allowed.", 400),
+ (RemoteFileNotFoundError, "remote_file_not_found", "The remote file could not be found.", 404),
+ (
+ RemoteFileAccessDeniedError,
+ "remote_file_access_denied",
+ "The remote file cannot be accessed without authorization.",
+ 400,
+ ),
+ (
+ RemoteFileUnavailableError,
+ "remote_file_unavailable",
+ "The remote file is temporarily unavailable.",
+ 502,
+ ),
+ (
+ RemoteFileInvalidResponseError,
+ "remote_file_invalid_response",
+ "The remote file server returned an invalid response.",
+ 502,
+ ),
+ ],
+)
+def test_remote_file_errors(
+ error_type: type[BaseHTTPException],
+ error_code: str,
+ description: str,
+ status: int,
+) -> None:
+ error = error_type()
+
+ assert error.code == status
+ assert error.error_code == error_code
+ assert error.description == description
+ assert error.data == {"code": error_code, "message": description, "status": status}
class TestFileTooLargeError:
diff --git a/api/tests/unit_tests/controllers/common/test_helpers.py b/api/tests/unit_tests/controllers/common/test_helpers.py
index 376a7a90c55..b06bd1e3ea2 100644
--- a/api/tests/unit_tests/controllers/common/test_helpers.py
+++ b/api/tests/unit_tests/controllers/common/test_helpers.py
@@ -1,188 +1,30 @@
-from uuid import UUID
+import urllib.parse
-import httpx
import pytest
-from controllers.common import helpers
-from controllers.common.helpers import FileInfo, guess_file_info_from_response
-
-
-def make_response(
- url="https://example.com/file.txt",
- headers=None,
- content=None,
-):
- return httpx.Response(
- 200,
- request=httpx.Request("GET", url),
- headers=headers or {},
- content=content or b"",
- )
-
-
-class TestGuessFileInfoFromResponse:
- def test_filename_from_url(self):
- response = make_response(
- url="https://example.com/test.pdf",
- content=b"Hello World",
- )
-
- info = guess_file_info_from_response(response)
-
- assert info.filename == "test.pdf"
- assert info.extension == ".pdf"
- assert info.mimetype == "application/pdf"
-
- def test_filename_from_content_disposition(self):
- headers = {
- "Content-Disposition": "attachment; filename=myfile.csv",
- "Content-Type": "text/csv",
- }
- response = make_response(
- url="https://example.com/",
- headers=headers,
- content=b"Hello World",
- )
-
- info = guess_file_info_from_response(response)
-
- assert info.filename == "myfile.csv"
- assert info.extension == ".csv"
- assert info.mimetype == "text/csv"
-
- @pytest.mark.parametrize(
- ("magic_available", "expected_ext"),
- [
- (True, "txt"),
- (False, "bin"),
- ],
- )
- def test_generated_filename_when_missing(self, monkeypatch: pytest.MonkeyPatch, magic_available, expected_ext):
- if magic_available:
- if helpers.magic is None:
- pytest.skip("python-magic is not installed, cannot run 'magic_available=True' test variant")
- else:
- monkeypatch.setattr(helpers, "magic", None)
-
- response = make_response(
- url="https://example.com/",
- content=b"Hello World",
- )
-
- info = guess_file_info_from_response(response)
-
- name, ext = info.filename.split(".")
- UUID(name)
- assert ext == expected_ext
-
- def test_mimetype_from_header_when_unknown(self):
- headers = {"Content-Type": "application/json"}
- response = make_response(
- url="https://example.com/file.unknown",
- headers=headers,
- content=b'{"a": 1}',
- )
-
- info = guess_file_info_from_response(response)
-
- assert info.mimetype == "application/json"
-
- def test_extension_added_when_missing(self):
- headers = {"Content-Type": "image/png"}
- response = make_response(
- url="https://example.com/image",
- headers=headers,
- content=b"fakepngdata",
- )
-
- info = guess_file_info_from_response(response)
-
- assert info.extension == ".png"
- assert info.filename.endswith(".png")
-
- def test_content_length_used_as_size(self):
- headers = {
- "Content-Length": "1234",
- "Content-Type": "text/plain",
- }
- response = make_response(
- url="https://example.com/a.txt",
- headers=headers,
- content=b"a" * 1234,
- )
-
- info = guess_file_info_from_response(response)
-
- assert info.size == 1234
-
- def test_size_minus_one_when_header_missing(self):
- response = make_response(url="https://example.com/a.txt")
-
- info = guess_file_info_from_response(response)
-
- assert info.size == -1
-
- def test_fallback_to_bin_extension(self):
- headers = {"Content-Type": "application/octet-stream"}
- response = make_response(
- url="https://example.com/download",
- headers=headers,
- content=b"\x00\x01\x02\x03",
- )
-
- info = guess_file_info_from_response(response)
-
- assert info.extension == ".bin"
- assert info.filename.endswith(".bin")
-
- def test_return_type(self):
- response = make_response()
-
- info = guess_file_info_from_response(response)
-
- assert isinstance(info, FileInfo)
-
-
-class TestMagicImportWarnings:
- @pytest.mark.parametrize(
- ("platform_name", "expected_message"),
- [
- ("Windows", "pip install python-magic-bin"),
- ("Darwin", "brew install libmagic"),
- ("Linux", "sudo apt-get install libmagic1"),
- ("Other", "install `libmagic`"),
- ],
- )
- def test_magic_import_warning_per_platform(
- self,
- monkeypatch: pytest.MonkeyPatch,
- platform_name,
- expected_message,
- ):
- import builtins
- import importlib
-
- # Force ImportError when "magic" is imported
- real_import = builtins.__import__
-
- def fake_import(name, *args, **kwargs):
- if name == "magic":
- raise ImportError("No module named magic")
- return real_import(name, *args, **kwargs)
-
- monkeypatch.setattr(builtins, "__import__", fake_import)
- monkeypatch.setattr("platform.system", lambda: platform_name)
-
- # Remove helpers so it imports fresh
- import sys
-
- original_helpers = sys.modules.get(helpers.__name__)
- sys.modules.pop(helpers.__name__, None)
-
- try:
- with pytest.warns(UserWarning, match="To use python-magic") as warning:
- imported_helpers = importlib.import_module(helpers.__name__)
- assert expected_message in str(warning[0].message)
- finally:
- if original_helpers is not None:
- sys.modules[helpers.__name__] = original_helpers
+from controllers.common.helpers import decode_remote_url
+
+
+ ("url", "query_string", "expected"),
+ [
+ (
+ urllib.parse.quote("https://example.com/report.txt", safe=""),
+ b"download=1",
+ "https://example.com/report.txt?download=1",
+ ),
+ (
+ urllib.parse.quote("https://example.com/report.txt?version=1", safe=""),
+ "download=1",
+ "https://example.com/report.txt?version=1&download=1",
+ ),
+ ("https://example.com/report.txt?", "download=1", "https://example.com/report.txt?download=1"),
+ ("https://example.com/report.txt", b"", "https://example.com/report.txt"),
+ ],
+)
+def test_decode_remote_url(url: str, query_string: bytes | str, expected: str) -> None:
+ assert decode_remote_url(url, query_string) == expected
+
+
+def test_decode_remote_url_defers_invalid_url_validation() -> None:
+ assert decode_remote_url("http://[invalid", "download=1") == "http://[invalid?download=1"
diff --git a/api/tests/unit_tests/controllers/console/explore/test_trial.py b/api/tests/unit_tests/controllers/console/explore/test_trial.py
index 2f2966b3844..03d93348382 100644
--- a/api/tests/unit_tests/controllers/console/explore/test_trial.py
+++ b/api/tests/unit_tests/controllers/console/explore/test_trial.py
@@ -252,17 +252,23 @@ class TestTrialAppRemoteFileUploadApi:
method = unwrap(api.post)
app_model = _app(app_id="app-1", mode=AppMode.CHAT, tenant_id="app-tenant-id")
remote_file = MagicMock()
- remote_file.model_dump.return_value = {"id": "upload-file-id"}
+ payload = module.RemoteFileUploadPayload(url="https://example.com/file.txt")
with (
app.test_request_context("/", method="POST", json={"url": "https://example.com/file.txt"}),
- patch.object(module, "upload_remote_file_from_request", return_value=remote_file) as upload,
+ patch.object(module, "upload_remote_file", return_value=remote_file) as upload,
+ patch.object(module, "dump_response", return_value={"id": "upload-file-id"}) as dump,
):
- response, status = method(api, account, app_model)
+ response, status = method(api, payload, account, app_model)
assert status == 201
assert response == {"id": "upload-file-id"}
- upload.assert_called_once_with(current_user=account, resource_tenant_id="app-tenant-id")
+ upload.assert_called_once_with(
+ url="https://example.com/file.txt",
+ current_user=account,
+ resource_tenant_id="app-tenant-id",
+ )
+ dump.assert_called_once_with(module.FileWithSignedUrl, remote_file)
class TestTrialAppWorkflowRunApi(_UsesSQLiteSession):
diff --git a/api/tests/unit_tests/controllers/console/test_remote_files.py b/api/tests/unit_tests/controllers/console/test_remote_files.py
index 6eeb68f79ac..c0bd04614ab 100644
--- a/api/tests/unit_tests/controllers/console/test_remote_files.py
+++ b/api/tests/unit_tests/controllers/console/test_remote_files.py
@@ -1,349 +1,260 @@
from __future__ import annotations
import urllib.parse
+from collections.abc import Generator
+from contextlib import contextmanager
from datetime import UTC, datetime
from inspect import unwrap
from types import SimpleNamespace
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
-import httpx
import pytest
from flask import Flask
-from controllers.common.errors import FileTooLargeError, RemoteFileUploadError, UnsupportedFileTypeError
+from controllers.common.errors import (
+ BlockedFileExtensionError,
+ FileTooLargeError,
+ RemoteFileAccessDeniedError,
+ RemoteFileInvalidResponseError,
+ RemoteFileInvalidUrlError,
+ RemoteFileNotFoundError,
+ RemoteFileUnavailableError,
+ RemoteFileUrlBlockedError,
+ UnsupportedFileTypeError,
+)
from controllers.console import remote_files as remote_files_module
-from extensions.storage.storage_type import StorageType
+from libs.exception import BaseHTTPException
+from machinery.context import RequestContext
from models import Account
from models.account import AccountStatus, TenantAccountRole
-from models.enums import CreatorUserRole
-from models.model import UploadFile
+from services.errors.file import (
+ BlockedFileExtensionError as ServiceBlockedFileExtensionError,
+)
from services.errors.file import FileTooLargeError as ServiceFileTooLargeError
from services.errors.file import UnsupportedFileTypeError as ServiceUnsupportedFileTypeError
-
-
-def _make_account(account_id: str = "u1") -> Account:
+from services.remote_file_service import (
+ RemoteFileAccessDeniedError as RemoteFileAccessDeniedServiceError,
+)
+from services.remote_file_service import RemoteFileError, RemoteFileInfoResult, RemoteFileUploadResult
+from services.remote_file_service import (
+ RemoteFileInvalidResponseError as RemoteFileInvalidResponseServiceError,
+)
+from services.remote_file_service import RemoteFileInvalidUrlError as RemoteFileInvalidUrlServiceError
+from services.remote_file_service import RemoteFileNotFoundError as RemoteFileNotFoundServiceError
+from services.remote_file_service import RemoteFileUnavailableError as RemoteFileUnavailableServiceError
+from services.remote_file_service import RemoteFileUrlBlockedError as RemoteFileUrlBlockedServiceError
+
+REMOTE_FILE_ERROR_CASES: tuple[tuple[type[RemoteFileError], type[BaseHTTPException], str, int], ...] = (
+ (RemoteFileInvalidUrlServiceError, RemoteFileInvalidUrlError, "remote_file_invalid_url", 400),
+ (RemoteFileUrlBlockedServiceError, RemoteFileUrlBlockedError, "remote_file_url_blocked", 400),
+ (RemoteFileNotFoundServiceError, RemoteFileNotFoundError, "remote_file_not_found", 404),
+ (RemoteFileAccessDeniedServiceError, RemoteFileAccessDeniedError, "remote_file_access_denied", 400),
+ (RemoteFileUnavailableServiceError, RemoteFileUnavailableError, "remote_file_unavailable", 502),
+ (
+ RemoteFileInvalidResponseServiceError,
+ RemoteFileInvalidResponseError,
+ "remote_file_invalid_response",
+ 502,
+ ),
+)
+
+
+def _account() -> Account:
account = Account(
name="Test User",
- email=f"{account_id}@example.com",
status=AccountStatus.ACTIVE,
)
- account.id = account_id
+ account.id = "account-1"
account.role = TenantAccountRole.OWNER
return account
-def _upload_file(
- *,
- file_id: str = "file-1",
- name: str = "report.txt",
- size: int = 16,
- extension: str = ".txt",
- mime_type: str = "text/plain",
- created_at: datetime | None = None,
-) -> UploadFile:
- upload_file = UploadFile(
- tenant_id="tenant-1",
- storage_type=StorageType.LOCAL,
- key=f"upload/{name}",
- name=name,
- size=size,
- extension=extension,
- mime_type=mime_type,
- created_by_role=CreatorUserRole.ACCOUNT,
- created_by="u1",
- created_at=created_at or datetime(2024, 1, 1, tzinfo=UTC),
- used=False,
+def _upload_result() -> RemoteFileUploadResult:
+ return RemoteFileUploadResult(
+ id="file-1",
+ name="report.txt",
+ size=16,
+ extension="txt",
+ url="https://signed.example/file-1",
+ mime_type="text/plain",
+ created_by="account-1",
+ created_at=datetime(2024, 1, 1, tzinfo=UTC),
)
- upload_file.id = file_id
- return upload_file
-class _FakeResponse:
- def __init__(
- self,
- *,
- status_code: int = 200,
- headers: dict[str, str] | None = None,
- method: str = "GET",
- content: bytes = b"",
- text: str = "",
- error: Exception | None = None,
- ) -> None:
- self.status_code = status_code
- self.headers = headers or {}
- self.request = SimpleNamespace(method=method)
- self.content = content
- self.text = text
- self._error = error
-
- def raise_for_status(self) -> None:
- if self._error:
- raise self._error
-
-
-def _mock_upload_dependencies(
- monkeypatch: pytest.MonkeyPatch,
- *,
- file_size_within_limit: bool = True,
-):
- file_info = SimpleNamespace(
- filename="report.txt",
- extension=".txt",
- mimetype="text/plain",
- size=3,
- )
- monkeypatch.setattr(
- remote_files_module.helpers,
- "guess_file_info_from_response",
- MagicMock(return_value=file_info),
- )
-
- file_service_cls = MagicMock()
- file_service_cls.is_file_size_within_limit.return_value = file_size_within_limit
- monkeypatch.setattr(remote_files_module, "FileService", file_service_cls)
- current_user = _make_account()
- monkeypatch.setattr(remote_files_module, "db", SimpleNamespace(engine=object()))
- monkeypatch.setattr(
- remote_files_module.file_helpers,
- "get_signed_file_url",
- lambda upload_file_id: f"https://signed.example/{upload_file_id}",
+def _request_context() -> RequestContext:
+ return RequestContext(
+ request_id="request-1",
+ trace_id="trace-1",
+ account_id="account-1",
+ active_workspace_id="tenant-1",
)
- return file_service_cls, current_user
-
-
-def test_get_remote_file_info_uses_head_when_successful(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:
- api = remote_files_module.GetRemoteFileInfo()
- handler = unwrap(api.get)
- decoded_url = "https://example.com/test.txt"
- encoded_url = urllib.parse.quote(decoded_url, safe="")
-
- head_resp = _FakeResponse(
- status_code=200,
- headers={"Content-Type": "text/plain", "Content-Length": "128"},
- method="HEAD",
- )
- make_request = MagicMock(return_value=head_resp)
- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)
-
- with app.test_request_context(method="GET"):
- payload = handler(api, url=encoded_url)
-
- assert payload == {"file_type": "text/plain", "file_length": 128}
- make_request.assert_called_once_with("HEAD", decoded_url)
-
-
-def test_get_remote_file_info_preserves_unencoded_target_query(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:
- api = remote_files_module.GetRemoteFileInfo()
- handler = unwrap(api.get)
- target_url = "http://example.com/api/aiagent/httpview/txt"
- query = "fileNameKey=cankao1_ce4305bc-be20-4c5d-8732-de1741d28e27"
- head_resp = _FakeResponse(
- status_code=200,
- headers={"Content-Type": "text/plain", "Content-Length": "128"},
- method="HEAD",
- )
- make_request = MagicMock(return_value=head_resp)
- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)
-
- with app.test_request_context(f"/remote-files/{target_url}?{query}", method="GET"):
- payload = handler(api, url=target_url)
-
- assert payload == {"file_type": "text/plain", "file_length": 128}
- make_request.assert_called_once_with("HEAD", f"{target_url}?{query}")
-
-
-def test_get_remote_file_info_falls_back_to_get_and_uses_default_headers(
- app: Flask, monkeypatch: pytest.MonkeyPatch
-) -> None:
- api = remote_files_module.GetRemoteFileInfo()
- handler = unwrap(api.get)
- decoded_url = "https://example.com/test.txt"
- encoded_url = urllib.parse.quote(decoded_url, safe="")
-
- make_request = MagicMock(
- side_effect=[
- _FakeResponse(status_code=503),
- _FakeResponse(status_code=200, headers={}, method="GET"),
- ]
- )
- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)
-
- with app.test_request_context(method="GET"):
- payload = handler(api, url=encoded_url)
-
- assert payload == {"file_type": "application/octet-stream", "file_length": 0}
- assert make_request.call_args_list[0].args == ("HEAD", decoded_url)
- assert make_request.call_args_list[1].args == ("GET", decoded_url)
- assert make_request.call_args_list[1].kwargs == {"timeout": 3}
-
-
-def test_remote_file_upload_success_when_fetch_falls_back_to_get(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:
- api = remote_files_module.RemoteFileUpload()
- handler = unwrap(api.post)
- url = "https://example.com/report.txt"
-
- get_resp = _FakeResponse(status_code=200, method="GET", content=b"fallback-content")
- make_request = MagicMock(side_effect=[_FakeResponse(status_code=404), get_resp])
- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)
-
- file_service_cls, current_user = _mock_upload_dependencies(monkeypatch)
- upload_file = _upload_file()
- file_service_cls.return_value.upload_file.return_value = upload_file
-
- with app.test_request_context(method="POST", json={"url": url}):
- payload, status = handler(api, current_user)
-
- assert status == 201
- assert payload["id"] == "file-1"
- assert payload["url"] == "https://signed.example/file-1"
- assert make_request.call_args_list[0].args == ("HEAD",)
- assert make_request.call_args_list[0].kwargs == {"url": url}
- assert make_request.call_args_list[1].args == ("GET",)
- assert make_request.call_args_list[1].kwargs == {"url": url, "timeout": 3, "follow_redirects": True}
- file_service_cls.return_value.upload_file.assert_called_once_with(
- filename="report.txt",
- content=b"fallback-content",
- mimetype="text/plain",
- user=current_user,
- tenant_id=None,
- source_url=url,
- )
-
-
-def test_remote_file_upload_assigns_resource_tenant(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:
- url = "https://example.com/report.txt"
- response = _FakeResponse(status_code=200, method="GET", content=b"content")
- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", MagicMock(return_value=response))
-
- file_service_cls, current_user = _mock_upload_dependencies(monkeypatch)
- file_service_cls.return_value.upload_file.return_value = _upload_file(size=7)
-
- with app.test_request_context(method="POST", json={"url": url}):
- remote_files_module.upload_remote_file_from_request(
- current_user=current_user,
- resource_tenant_id="app-tenant-id",
+@contextmanager
+def _patch_application_services(remote_files: MagicMock) -> Generator[None]:
+ with patch.object(
+ remote_files_module,
+ "application_services",
+ return_value=SimpleNamespace(remote_files=remote_files),
+ ):
+ yield
+
+
+class TestGetRemoteFileInfo:
+ def test_decodes_url_and_preserves_console_missing_length_default(self, app: Flask) -> None:
+ api = remote_files_module.GetRemoteFileInfo()
+ handler = unwrap(api.get)
+ remote_files = MagicMock()
+ remote_files.fetch_info.return_value = RemoteFileInfoResult(
+ content_type="text/plain",
+ content_length=None,
)
+ target_url = "https://example.com/file?name=report.txt"
+ encoded_url = urllib.parse.quote(target_url, safe="")
- file_service_cls.return_value.upload_file.assert_called_once_with(
- filename="report.txt",
- content=b"content",
- mimetype="text/plain",
- user=current_user,
- tenant_id="app-tenant-id",
- source_url=url,
- )
-
+ with (
+ app.test_request_context(f"/remote-files/{encoded_url}", method="GET"),
+ _patch_application_services(remote_files),
+ ):
+ response = handler(api, _request_context(), encoded_url)
-def test_remote_file_upload_fetches_content_with_second_get_when_head_succeeds(
- app, monkeypatch: pytest.MonkeyPatch
-) -> None:
- api = remote_files_module.RemoteFileUpload()
- handler = unwrap(api.post)
- url = "https://example.com/photo.jpg"
-
- head_resp = _FakeResponse(status_code=200, method="HEAD", content=b"head-content")
- extra_get_resp = _FakeResponse(status_code=200, method="GET", content=b"downloaded-content")
- make_request = MagicMock(side_effect=[head_resp, extra_get_resp])
- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)
-
- file_service_cls, current_user = _mock_upload_dependencies(monkeypatch)
- upload_file = _upload_file(
- file_id="file-2",
- name="photo.jpg",
- size=18,
- extension=".jpg",
- mime_type="image/jpeg",
- created_at=datetime(2024, 1, 2, tzinfo=UTC),
- )
- file_service_cls.return_value.upload_file.return_value = upload_file
-
- with app.test_request_context(method="POST", json={"url": url}):
- payload, status = handler(api, current_user)
-
- assert status == 201
- assert payload["id"] == "file-2"
- assert make_request.call_args_list[1].args == ("GET", url)
- assert file_service_cls.return_value.upload_file.call_args.kwargs["content"] == b"downloaded-content"
+ assert response == {"file_type": "text/plain", "file_length": 0}
+ remote_files.fetch_info.assert_called_once_with(url=target_url)
+ @pytest.mark.parametrize(("service_error_type", "http_error", "error_code", "status"), REMOTE_FILE_ERROR_CASES)
+ def test_translates_remote_file_errors(
+ self,
+ app: Flask,
+ service_error_type: type[RemoteFileError],
+ http_error: type[BaseHTTPException],
+ error_code: str,
+ status: int,
+ ) -> None:
+ api = remote_files_module.GetRemoteFileInfo()
+ handler = unwrap(api.get)
+ service_error = service_error_type("sensitive upstream details")
+ remote_files = MagicMock()
+ remote_files.fetch_info.side_effect = service_error
+
+ with (
+ app.test_request_context("/remote-files/url", method="GET"),
+ _patch_application_services(remote_files),
+ pytest.raises(http_error) as error_info,
+ ):
+ handler(api, _request_context(), "url")
+
+ assert error_info.value.__cause__ is service_error
+ assert error_info.value.data is not None
+ assert error_info.value.data["code"] == error_code
+ assert error_info.value.data["status"] == status
+ assert "sensitive upstream details" not in error_info.value.data["message"]
+
+
+class TestRemoteFileUpload:
+ def test_controller_delegates_with_validated_payload(self, app: Flask) -> None:
+ api = remote_files_module.RemoteFileUpload()
+ handler = unwrap(api.post)
+ account = _account()
+ remote_file = _upload_result()
+
+ with (
+ app.test_request_context("/remote-files/upload", method="POST"),
+ patch.object(remote_files_module, "upload_remote_file", return_value=remote_file) as upload,
+ ):
+ response, status = handler(
+ api,
+ remote_files_module.RemoteFileUploadPayload(url="https://example.com/report.txt"),
+ account,
+ )
+
+ assert status == 201
+ assert response["created_at"] == 1704067200
+ upload.assert_called_once_with(
+ url="https://example.com/report.txt",
+ current_user=account,
+ )
-def test_remote_file_upload_raises_when_fallback_get_still_not_ok(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:
- api = remote_files_module.RemoteFileUpload()
- handler = unwrap(api.post)
- url = "https://example.com/fail.txt"
+ def test_helper_preserves_explicit_resource_tenant(self, app: Flask) -> None:
+ account = _account()
+ remote_files = MagicMock()
+ remote_files.upload_from_url.return_value = _upload_result()
+
+ with (
+ app.test_request_context("/remote-files/upload", method="POST"),
+ _patch_application_services(remote_files),
+ ):
+ result = remote_files_module.upload_remote_file(
+ url="https://example.com/report.txt",
+ current_user=account,
+ resource_tenant_id="app-tenant-id",
+ )
+
+ assert result == _upload_result()
+ remote_files.upload_from_url.assert_called_once_with(
+ url="https://example.com/report.txt",
+ user=account,
+ tenant_id="app-tenant-id",
+ )
- make_request = MagicMock(
- side_effect=[
- _FakeResponse(status_code=500),
- _FakeResponse(status_code=502, text="bad gateway"),
- ]
+ @pytest.mark.parametrize(("service_error_type", "http_error", "error_code", "status"), REMOTE_FILE_ERROR_CASES)
+ def test_helper_translates_remote_file_errors(
+ self,
+ app: Flask,
+ service_error_type: type[RemoteFileError],
+ http_error: type[BaseHTTPException],
+ error_code: str,
+ status: int,
+ ) -> None:
+ service_error = service_error_type("sensitive upstream details")
+ remote_files = MagicMock()
+ remote_files.upload_from_url.side_effect = service_error
+
+ with (
+ app.test_request_context("/remote-files/upload", method="POST"),
+ _patch_application_services(remote_files),
+ pytest.raises(http_error) as error_info,
+ ):
+ remote_files_module.upload_remote_file(
+ url="https://example.com/report.txt",
+ current_user=_account(),
+ )
+
+ assert error_info.value.__cause__ is service_error
+ assert error_info.value.data is not None
+ assert error_info.value.data["code"] == error_code
+ assert error_info.value.data["status"] == status
+ assert "sensitive upstream details" not in error_info.value.data["message"]
+
+ @pytest.mark.parametrize(
+ ("service_error", "http_error"),
+ [
+ (ServiceFileTooLargeError("size exceeded"), FileTooLargeError),
+ (ServiceUnsupportedFileTypeError(), UnsupportedFileTypeError),
+ (
+ ServiceBlockedFileExtensionError("File extension '.exe' is not allowed"),
+ BlockedFileExtensionError,
+ ),
+ ],
)
- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)
-
- with app.test_request_context(method="POST", json={"url": url}):
- with pytest.raises(RemoteFileUploadError, match=f"Failed to fetch file from {url}: bad gateway"):
- handler(api, _make_account())
-
-
-def test_remote_file_upload_raises_on_httpx_request_error(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:
- api = remote_files_module.RemoteFileUpload()
- handler = unwrap(api.post)
- url = "https://example.com/fail.txt"
-
- request = httpx.Request("HEAD", url)
- make_request = MagicMock(side_effect=httpx.RequestError("network down", request=request))
- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)
-
- with app.test_request_context(method="POST", json={"url": url}):
- with pytest.raises(RemoteFileUploadError, match=f"Failed to fetch file from {url}: network down"):
- handler(api, _make_account())
-
-
-def test_remote_file_upload_rejects_oversized_file(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:
- api = remote_files_module.RemoteFileUpload()
- handler = unwrap(api.post)
- url = "https://example.com/large.bin"
-
- make_request = MagicMock(return_value=_FakeResponse(status_code=200, method="GET", content=b"payload"))
- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)
-
- _, current_user = _mock_upload_dependencies(monkeypatch, file_size_within_limit=False)
-
- with app.test_request_context(method="POST", json={"url": url}):
- with pytest.raises(FileTooLargeError):
- handler(api, current_user)
-
-
-def test_remote_file_upload_translates_service_file_too_large_error(
- app: Flask, monkeypatch: pytest.MonkeyPatch
-) -> None:
- api = remote_files_module.RemoteFileUpload()
- handler = unwrap(api.post)
- url = "https://example.com/large.bin"
-
- make_request = MagicMock(return_value=_FakeResponse(status_code=200, method="GET", content=b"payload"))
- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)
- file_service_cls, current_user = _mock_upload_dependencies(monkeypatch)
- file_service_cls.return_value.upload_file.side_effect = ServiceFileTooLargeError("size exceeded")
-
- with app.test_request_context(method="POST", json={"url": url}):
- with pytest.raises(FileTooLargeError, match="size exceeded"):
- handler(api, current_user)
-
-
-def test_remote_file_upload_translates_service_unsupported_type_error(
- app: Flask, monkeypatch: pytest.MonkeyPatch
-) -> None:
- api = remote_files_module.RemoteFileUpload()
- handler = unwrap(api.post)
- url = "https://example.com/file.exe"
-
- make_request = MagicMock(return_value=_FakeResponse(status_code=200, method="GET", content=b"payload"))
- monkeypatch.setattr(remote_files_module.remote_fetcher, "make_request", make_request)
- file_service_cls, current_user = _mock_upload_dependencies(monkeypatch)
- file_service_cls.return_value.upload_file.side_effect = ServiceUnsupportedFileTypeError()
-
- with app.test_request_context(method="POST", json={"url": url}):
- with pytest.raises(UnsupportedFileTypeError):
- handler(api, current_user)
+ def test_helper_translates_file_errors(
+ self,
+ app: Flask,
+ service_error: Exception,
+ http_error: type[Exception],
+ ) -> None:
+ remote_files = MagicMock()
+ remote_files.upload_from_url.side_effect = service_error
+
+ with (
+ app.test_request_context("/remote-files/upload", method="POST"),
+ _patch_application_services(remote_files),
+ pytest.raises(http_error) as error_info,
+ ):
+ remote_files_module.upload_remote_file(
+ url="https://example.com/report.txt",
+ current_user=_account(),
+ )
+
+ assert error_info.value.__cause__ is service_error
diff --git a/api/tests/unit_tests/controllers/web/test_human_input_file_upload.py b/api/tests/unit_tests/controllers/web/test_human_input_file_upload.py
index 81e361d494c..3dfa2be777d 100644
--- a/api/tests/unit_tests/controllers/web/test_human_input_file_upload.py
+++ b/api/tests/unit_tests/controllers/web/test_human_input_file_upload.py
@@ -210,7 +210,7 @@ def test_remote_upload_records_form_file_link(
ssrf_proxy.head.return_value = response
monkeypatch.setattr(upload_module, "ssrf_proxy", ssrf_proxy)
monkeypatch.setattr(
- upload_module.helpers,
+ upload_module,
"guess_file_info_from_response",
lambda _response: SimpleNamespace(filename="sample.txt", extension="txt", mimetype="text/plain", size=6),
)
diff --git a/api/tests/unit_tests/controllers/web/test_pydantic_models.py b/api/tests/unit_tests/controllers/web/test_pydantic_models.py
index bceb65b89f6..c2d5b24b3c6 100644
--- a/api/tests/unit_tests/controllers/web/test_pydantic_models.py
+++ b/api/tests/unit_tests/controllers/web/test_pydantic_models.py
@@ -260,9 +260,10 @@ class TestRemoteFileUploadPayload:
p = RemoteFileUploadPayload(url="https://example.com/file.pdf")
assert str(p.url) == "https://example.com/file.pdf"
- def test_invalid_url(self) -> None:
- with pytest.raises(ValidationError):
- RemoteFileUploadPayload(url="not-a-url")
+ def test_url_syntax_is_validated_by_remote_file_service(self) -> None:
+ payload = RemoteFileUploadPayload(url="not-a-url")
+
+ assert payload.url == "not-a-url"
def test_url_required(self) -> None:
with pytest.raises(ValidationError):
diff --git a/api/tests/unit_tests/controllers/web/test_remote_files.py b/api/tests/unit_tests/controllers/web/test_remote_files.py
index 68cb0e6b6bb..791ec2770fb 100644
--- a/api/tests/unit_tests/controllers/web/test_remote_files.py
+++ b/api/tests/unit_tests/controllers/web/test_remote_files.py
@@ -1,21 +1,60 @@
-"""Unit tests for controllers.web.remote_files endpoints."""
-
from __future__ import annotations
import urllib.parse
-from datetime import datetime
+from collections.abc import Generator
+from contextlib import contextmanager
+from datetime import UTC, datetime
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from flask import Flask
-from sqlalchemy import Engine
-from controllers.common.errors import FileTooLargeError, RemoteFileUploadError
-from controllers.web.remote_files import RemoteFileInfoApi, RemoteFileUploadApi
-from extensions.storage.storage_type import StorageType
-from models.enums import CreatorUserRole, EndUserType
-from models.model import App, AppMode, EndUser, UploadFile
+from controllers.common.errors import (
+ BlockedFileExtensionError,
+ FileTooLargeError,
+ RemoteFileAccessDeniedError,
+ RemoteFileInvalidResponseError,
+ RemoteFileInvalidUrlError,
+ RemoteFileNotFoundError,
+ RemoteFileUnavailableError,
+ RemoteFileUrlBlockedError,
+ UnsupportedFileTypeError,
+)
+from controllers.web import remote_files as remote_files_module
+from libs.exception import BaseHTTPException
+from models.enums import EndUserType
+from models.model import App, AppMode, EndUser
+from services.errors.file import (
+ BlockedFileExtensionError as ServiceBlockedFileExtensionError,
+)
+from services.errors.file import FileTooLargeError as ServiceFileTooLargeError
+from services.errors.file import UnsupportedFileTypeError as ServiceUnsupportedFileTypeError
+from services.remote_file_service import (
+ RemoteFileAccessDeniedError as RemoteFileAccessDeniedServiceError,
+)
+from services.remote_file_service import RemoteFileError, RemoteFileInfoResult, RemoteFileUploadResult
+from services.remote_file_service import (
+ RemoteFileInvalidResponseError as RemoteFileInvalidResponseServiceError,
+)
+from services.remote_file_service import RemoteFileInvalidUrlError as RemoteFileInvalidUrlServiceError
+from services.remote_file_service import RemoteFileNotFoundError as RemoteFileNotFoundServiceError
+from services.remote_file_service import RemoteFileUnavailableError as RemoteFileUnavailableServiceError
+from services.remote_file_service import RemoteFileUrlBlockedError as RemoteFileUrlBlockedServiceError
+
+REMOTE_FILE_ERROR_CASES: tuple[tuple[type[RemoteFileError], type[BaseHTTPException], str, int], ...] = (
+ (RemoteFileInvalidUrlServiceError, RemoteFileInvalidUrlError, "remote_file_invalid_url", 400),
+ (RemoteFileUrlBlockedServiceError, RemoteFileUrlBlockedError, "remote_file_url_blocked", 400),
+ (RemoteFileNotFoundServiceError, RemoteFileNotFoundError, "remote_file_not_found", 404),
+ (RemoteFileAccessDeniedServiceError, RemoteFileAccessDeniedError, "remote_file_access_denied", 400),
+ (RemoteFileUnavailableServiceError, RemoteFileUnavailableError, "remote_file_unavailable", 502),
+ (
+ RemoteFileInvalidResponseServiceError,
+ RemoteFileInvalidResponseError,
+ "remote_file_invalid_response",
+ 502,
+ ),
+)
def _app_model() -> App:
@@ -41,163 +80,172 @@ def _end_user() -> EndUser:
)
-def _upload_file() -> UploadFile:
- upload_file = UploadFile(
- tenant_id="tenant-1",
- storage_type=StorageType.LOCAL,
- key="upload/file.pdf",
- name="file.pdf",
- size=100,
- extension="pdf",
- mime_type="application/pdf",
- created_by_role=CreatorUserRole.END_USER,
+def _upload_result() -> RemoteFileUploadResult:
+ return RemoteFileUploadResult(
+ id="file-1",
+ name="report.txt",
+ size=16,
+ extension="txt",
+ url="https://signed.example/file-1",
+ mime_type="text/plain",
created_by="eu-1",
- created_at=datetime(2024, 1, 1),
- used=False,
+ created_at=datetime(2024, 1, 1, tzinfo=UTC),
)
- upload_file.id = "f-1"
- return upload_file
-
-
-# ---------------------------------------------------------------------------
-# RemoteFileInfoApi
-# ---------------------------------------------------------------------------
-class TestRemoteFileInfoApi:
- @patch("controllers.web.remote_files.remote_fetcher")
- def test_head_success(self, mock_proxy: MagicMock, app: Flask) -> None:
- mock_resp = MagicMock()
- mock_resp.status_code = 200
- mock_resp.headers = {"Content-Type": "application/pdf", "Content-Length": "1024"}
- mock_proxy.make_request.return_value = mock_resp
-
- with app.test_request_context("/remote-files/https%3A%2F%2Fexample.com%2Ffile.pdf"):
- result = RemoteFileInfoApi().get(_app_model(), _end_user(), "https%3A%2F%2Fexample.com%2Ffile.pdf")
-
- assert result["file_type"] == "application/pdf"
- assert result["file_length"] == 1024
- mock_proxy.make_request.assert_called_once_with("HEAD", "https://example.com/file.pdf")
-
- @patch("controllers.web.remote_files.remote_fetcher")
- def test_preserves_unencoded_target_query(self, mock_proxy: MagicMock, app: Flask) -> None:
- mock_resp = MagicMock()
- mock_resp.status_code = 200
- mock_resp.headers = {"Content-Type": "text/plain", "Content-Length": "128"}
- mock_proxy.make_request.return_value = mock_resp
-
- target_url = "http://example.com/api/aiagent/httpview/txt"
- query = "fileNameKey=cankao1_ce4305bc-be20-4c5d-8732-de1741d28e27"
-
- with app.test_request_context(f"/remote-files/{target_url}?{query}"):
- result = RemoteFileInfoApi().get(_app_model(), _end_user(), target_url)
-
- assert result["file_type"] == "text/plain"
- mock_proxy.make_request.assert_called_once_with("HEAD", f"{target_url}?{query}")
-
- @patch("controllers.web.remote_files.remote_fetcher")
- def test_preserves_encoded_target_query(self, mock_proxy: MagicMock, app: Flask) -> None:
- mock_resp = MagicMock()
- mock_resp.status_code = 200
- mock_resp.headers = {"Content-Type": "text/plain", "Content-Length": "128"}
- mock_proxy.make_request.return_value = mock_resp
-
- target_url = "http://example.com/api/aiagent/httpview/txt?fileNameKey=cankao1"
- encoded_url = urllib.parse.quote(target_url, safe="")
- with app.test_request_context(f"/remote-files/{encoded_url}"):
- result = RemoteFileInfoApi().get(_app_model(), _end_user(), encoded_url)
- assert result["file_type"] == "text/plain"
- mock_proxy.make_request.assert_called_once_with("HEAD", target_url)
+@contextmanager
+def _patch_application_services(remote_files: MagicMock) -> Generator[None]:
+ with patch.object(
+ remote_files_module,
+ "application_services",
+ return_value=SimpleNamespace(remote_files=remote_files),
+ ):
+ yield
- @patch("controllers.web.remote_files.remote_fetcher")
- def test_fallback_to_get(self, mock_proxy: MagicMock, app: Flask) -> None:
- head_resp = MagicMock()
- head_resp.status_code = 405 # Method not allowed
- get_resp = MagicMock()
- get_resp.status_code = 200
- get_resp.headers = {"Content-Type": "text/plain", "Content-Length": "42"}
- get_resp.raise_for_status = MagicMock()
- mock_proxy.make_request.side_effect = [head_resp, get_resp]
- with app.test_request_context("/remote-files/https%3A%2F%2Fexample.com%2Ffile.txt"):
- result = RemoteFileInfoApi().get(_app_model(), _end_user(), "https%3A%2F%2Fexample.com%2Ffile.txt")
+class TestRemoteFileInfoApi:
+ def test_decodes_url_and_preserves_web_missing_length_default(self, app: Flask) -> None:
+ remote_files = MagicMock()
+ remote_files.fetch_info.return_value = RemoteFileInfoResult(
+ content_type="text/plain",
+ content_length=None,
+ )
+ target_url = "http://example.com/api/file?name=report.txt"
+ encoded_url = urllib.parse.quote(target_url, safe="")
- assert result["file_type"] == "text/plain"
- assert mock_proxy.make_request.call_args_list[1].args == ("GET", "https://example.com/file.txt")
+ with (
+ app.test_request_context(f"/remote-files/{encoded_url}"),
+ _patch_application_services(remote_files),
+ ):
+ result = remote_files_module.RemoteFileInfoApi().get(_app_model(), _end_user(), encoded_url)
+ assert result == {"file_type": "text/plain", "file_length": -1}
+ remote_files.fetch_info.assert_called_once_with(url=target_url)
-# ---------------------------------------------------------------------------
-# RemoteFileUploadApi
-# ---------------------------------------------------------------------------
-class TestRemoteFileUploadApi:
- @patch("controllers.web.remote_files.file_helpers.get_signed_file_url", return_value="https://signed-url")
- @patch("controllers.web.remote_files.FileService")
- @patch("controllers.web.remote_files.helpers.guess_file_info_from_response")
- @patch("controllers.web.remote_files.remote_fetcher")
- @patch("controllers.web.remote_files.db")
- def test_upload_success(
+ @pytest.mark.parametrize(("service_error_type", "http_error", "error_code", "status"), REMOTE_FILE_ERROR_CASES)
+ def test_translates_remote_file_errors(
self,
- mock_db: MagicMock,
- mock_proxy: MagicMock,
- mock_guess: MagicMock,
- mock_file_svc_cls: MagicMock,
- mock_signed: MagicMock,
app: Flask,
- sqlite_engine: Engine,
+ service_error_type: type[RemoteFileError],
+ http_error: type[BaseHTTPException],
+ error_code: str,
+ status: int,
) -> None:
- mock_db.engine = sqlite_engine
- head_resp = MagicMock()
- head_resp.status_code = 200
- head_resp.content = b"pdf-content"
- head_resp.request.method = "HEAD"
- get_resp = MagicMock()
- get_resp.content = b"pdf-content"
- mock_proxy.make_request.side_effect = [head_resp, get_resp]
-
- mock_guess.return_value = SimpleNamespace(
- filename="file.pdf", extension="pdf", mimetype="application/pdf", size=100
- )
- mock_file_svc_cls.is_file_size_within_limit.return_value = True
+ service_error = service_error_type("sensitive upstream details")
+ remote_files = MagicMock()
+ remote_files.fetch_info.side_effect = service_error
+
+ with (
+ app.test_request_context("/remote-files/url"),
+ _patch_application_services(remote_files),
+ pytest.raises(http_error) as error_info,
+ ):
+ remote_files_module.RemoteFileInfoApi().get(_app_model(), _end_user(), "url")
+
+ assert error_info.value.__cause__ is service_error
+ assert error_info.value.data is not None
+ assert error_info.value.data["code"] == error_code
+ assert error_info.value.data["status"] == status
+ assert "sensitive upstream details" not in error_info.value.data["message"]
- mock_file_svc_cls.return_value.upload_file.return_value = _upload_file()
- with app.test_request_context(
- "/remote-files/upload", method="POST", json={"url": "https://example.com/file.pdf"}
+class TestRemoteFileUploadApi:
+ def test_upload_delegates_without_overriding_web_tenant(self, app: Flask) -> None:
+ remote_files = MagicMock()
+ remote_files.upload_from_url.return_value = _upload_result()
+ end_user = _end_user()
+ url = "https://example.com/report.txt"
+
+ with (
+ app.test_request_context("/remote-files/upload", method="POST", json={"url": url}),
+ _patch_application_services(remote_files),
):
- result, status = RemoteFileUploadApi().post(_app_model(), _end_user())
+ response, status = remote_files_module.RemoteFileUploadApi().post(_app_model(), end_user)
assert status == 201
- assert result["id"] == "f-1"
+ assert response == {
+ "id": "file-1",
+ "name": "report.txt",
+ "size": 16,
+ "extension": "txt",
+ "url": "https://signed.example/file-1",
+ "mime_type": "text/plain",
+ "created_by": "eu-1",
+ "created_at": 1704067200,
+ }
+ remote_files.upload_from_url.assert_called_once_with(url=url, user=end_user)
+
+ def test_uses_complete_message_for_empty_file_size_error(self, app: Flask) -> None:
+ remote_files = MagicMock()
+ remote_files.upload_from_url.side_effect = ServiceFileTooLargeError()
+
+ with (
+ app.test_request_context(
+ "/remote-files/upload",
+ method="POST",
+ json={"url": "https://example.com/report.txt"},
+ ),
+ _patch_application_services(remote_files),
+ pytest.raises(FileTooLargeError) as error_info,
+ ):
+ remote_files_module.RemoteFileUploadApi().post(_app_model(), _end_user())
+
+ assert error_info.value.description == "File size exceeded."
- @patch("controllers.web.remote_files.FileService.is_file_size_within_limit", return_value=False)
- @patch("controllers.web.remote_files.helpers.guess_file_info_from_response")
- @patch("controllers.web.remote_files.remote_fetcher")
- def test_file_too_large(
+ @pytest.mark.parametrize(("service_error_type", "http_error", "error_code", "status"), REMOTE_FILE_ERROR_CASES)
+ def test_translates_remote_file_errors(
self,
- mock_proxy: MagicMock,
- mock_guess: MagicMock,
- mock_size_check: MagicMock,
app: Flask,
+ service_error_type: type[RemoteFileError],
+ http_error: type[BaseHTTPException],
+ error_code: str,
+ status: int,
) -> None:
- head_resp = MagicMock()
- head_resp.status_code = 200
- mock_proxy.make_request.return_value = head_resp
- mock_guess.return_value = SimpleNamespace(
- filename="big.zip", extension="zip", mimetype="application/zip", size=999999999
- )
-
- with app.test_request_context(
- "/remote-files/upload", method="POST", json={"url": "https://example.com/big.zip"}
+ service_error = service_error_type("sensitive upstream details")
+ remote_files = MagicMock()
+ remote_files.upload_from_url.side_effect = service_error
+
+ with (
+ app.test_request_context(
+ "/remote-files/upload",
+ method="POST",
+ json={"url": "https://example.com/report.txt"},
+ ),
+ _patch_application_services(remote_files),
+ pytest.raises(http_error) as error_info,
):
- with pytest.raises(FileTooLargeError):
- RemoteFileUploadApi().post(_app_model(), _end_user())
-
- @patch("controllers.web.remote_files.remote_fetcher")
- def test_fetch_failure_raises(self, mock_proxy: MagicMock, app: Flask) -> None:
- import httpx
-
- mock_proxy.make_request.side_effect = httpx.RequestError("connection failed")
+ remote_files_module.RemoteFileUploadApi().post(_app_model(), _end_user())
+
+ assert error_info.value.__cause__ is service_error
+ assert error_info.value.data is not None
+ assert error_info.value.data["code"] == error_code
+ assert error_info.value.data["status"] == status
+ assert "sensitive upstream details" not in error_info.value.data["message"]
+
+ @pytest.mark.parametrize(
+ ("service_error", "http_error"),
+ [
+ (ServiceFileTooLargeError("size exceeded"), FileTooLargeError),
+ (ServiceUnsupportedFileTypeError(), UnsupportedFileTypeError),
+ (
+ ServiceBlockedFileExtensionError("File extension '.exe' is not allowed"),
+ BlockedFileExtensionError,
+ ),
+ ],
+ )
+ def test_translates_file_errors(self, app: Flask, service_error: Exception, http_error: type[Exception]) -> None:
+ remote_files = MagicMock()
+ remote_files.upload_from_url.side_effect = service_error
+
+ with (
+ app.test_request_context(
+ "/remote-files/upload",
+ method="POST",
+ json={"url": "https://example.com/report.txt"},
+ ),
+ _patch_application_services(remote_files),
+ pytest.raises(http_error) as error_info,
+ ):
+ remote_files_module.RemoteFileUploadApi().post(_app_model(), _end_user())
- with app.test_request_context("/remote-files/upload", method="POST", json={"url": "https://example.com/bad"}):
- with pytest.raises(RemoteFileUploadError):
- RemoteFileUploadApi().post(_app_model(), _end_user())
+ assert error_info.value.__cause__ is service_error
diff --git a/api/tests/unit_tests/core/file/test_remote_file_metadata.py b/api/tests/unit_tests/core/file/test_remote_file_metadata.py
new file mode 100644
index 00000000000..2e9425d65dc
--- /dev/null
+++ b/api/tests/unit_tests/core/file/test_remote_file_metadata.py
@@ -0,0 +1,209 @@
+from collections.abc import Mapping, Sequence
+from types import ModuleType
+from uuid import UUID
+
+import httpx
+import pytest
+
+from core.file import remote_file_metadata
+from core.file.remote_file_metadata import FileInfo, InvalidRemoteFileMetadataError, guess_file_info_from_response
+
+
+def make_response(
+ url: str = "https://example.com/file.txt",
+ headers: dict[str, str] | None = None,
+ content: bytes | None = None,
+) -> httpx.Response:
+ return httpx.Response(
+ 200,
+ request=httpx.Request("GET", url),
+ headers=headers or {},
+ content=content or b"",
+ )
+
+
+class TestGuessFileInfoFromResponse:
+ def test_filename_from_url(self) -> None:
+ response = make_response(
+ url="https://example.com/test.pdf",
+ content=b"Hello World",
+ )
+
+ info = guess_file_info_from_response(response)
+
+ assert info.filename == "test.pdf"
+ assert info.extension == ".pdf"
+ assert info.mimetype == "application/pdf"
+
+ def test_filename_from_content_disposition(self) -> None:
+ headers = {
+ "Content-Disposition": "attachment; filename=myfile.csv",
+ "Content-Type": "text/csv",
+ }
+ response = make_response(
+ url="https://example.com/",
+ headers=headers,
+ content=b"Hello World",
+ )
+
+ info = guess_file_info_from_response(response)
+
+ assert info.filename == "myfile.csv"
+ assert info.extension == ".csv"
+ assert info.mimetype == "text/csv"
+
+ @pytest.mark.parametrize(
+ ("magic_available", "expected_ext"),
+ [
+ (True, "txt"),
+ (False, "bin"),
+ ],
+ )
+ def test_generated_filename_when_missing(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ magic_available: bool,
+ expected_ext: str,
+ ) -> None:
+ if magic_available:
+ if remote_file_metadata.magic is None:
+ pytest.skip("python-magic is not installed, cannot run 'magic_available=True' test variant")
+ else:
+ monkeypatch.setattr(remote_file_metadata, "magic", None)
+
+ response = make_response(
+ url="https://example.com/",
+ content=b"Hello World",
+ )
+
+ info = guess_file_info_from_response(response)
+
+ name, ext = info.filename.split(".")
+ UUID(name)
+ assert ext == expected_ext
+
+ def test_mimetype_from_header_when_unknown(self) -> None:
+ headers = {"Content-Type": "application/json"}
+ response = make_response(
+ url="https://example.com/file.unknown",
+ headers=headers,
+ content=b'{"a": 1}',
+ )
+
+ info = guess_file_info_from_response(response)
+
+ assert info.mimetype == "application/json"
+
+ def test_extension_added_when_missing(self) -> None:
+ headers = {"Content-Type": "image/png"}
+ response = make_response(
+ url="https://example.com/image",
+ headers=headers,
+ content=b"fakepngdata",
+ )
+
+ info = guess_file_info_from_response(response)
+
+ assert info.extension == ".png"
+ assert info.filename.endswith(".png")
+
+ def test_content_length_used_as_size(self) -> None:
+ headers = {
+ "Content-Length": "1234",
+ "Content-Type": "text/plain",
+ }
+ response = make_response(
+ url="https://example.com/a.txt",
+ headers=headers,
+ content=b"a" * 1234,
+ )
+
+ info = guess_file_info_from_response(response)
+
+ assert info.size == 1234
+
+ def test_size_minus_one_when_header_missing(self) -> None:
+ response = make_response(url="https://example.com/a.txt")
+
+ info = guess_file_info_from_response(response)
+
+ assert info.size == -1
+
+ def test_invalid_content_length_raises_typed_metadata_error(self) -> None:
+ response = make_response(headers={"Content-Length": "not-a-number"})
+
+ with pytest.raises(InvalidRemoteFileMetadataError) as error_info:
+ guess_file_info_from_response(response)
+
+ assert isinstance(error_info.value.__cause__, ValueError)
+
+ def test_fallback_to_bin_extension(self) -> None:
+ headers = {"Content-Type": "application/octet-stream"}
+ response = make_response(
+ url="https://example.com/download",
+ headers=headers,
+ content=b"\x00\x01\x02\x03",
+ )
+
+ info = guess_file_info_from_response(response)
+
+ assert info.extension == ".bin"
+ assert info.filename.endswith(".bin")
+
+ def test_return_type(self) -> None:
+ response = make_response()
+
+ info = guess_file_info_from_response(response)
+
+ assert isinstance(info, FileInfo)
+
+
+class TestMagicImportWarnings:
+ @pytest.mark.parametrize(
+ ("platform_name", "expected_message"),
+ [
+ ("Windows", "pip install python-magic-bin"),
+ ("Darwin", "brew install libmagic"),
+ ("Linux", "sudo apt-get install libmagic1"),
+ ("Other", "install `libmagic`"),
+ ],
+ )
+ def test_magic_import_warning_per_platform(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ platform_name: str,
+ expected_message: str,
+ ) -> None:
+ import builtins
+ import importlib
+
+ # Force ImportError when "magic" is imported
+ real_import = builtins.__import__
+
+ def fake_import(
+ name: str,
+ global_vars: Mapping[str, object] | None = None,
+ local_vars: Mapping[str, object] | None = None,
+ fromlist: Sequence[str] = (),
+ level: int = 0,
+ ) -> ModuleType:
+ if name == "magic":
+ raise ImportError("No module named magic")
+ return real_import(name, global_vars, local_vars, fromlist, level)
+
+ monkeypatch.setattr(builtins, "__import__", fake_import)
+ monkeypatch.setattr("platform.system", lambda: platform_name)
+
+ # Remove the metadata module so it imports fresh
+ import sys
+
+ original_metadata = sys.modules.get(remote_file_metadata.__name__)
+ sys.modules.pop(remote_file_metadata.__name__, None)
+
+ try:
+ with pytest.warns(UserWarning, match="To use python-magic") as warning:
+ importlib.import_module(remote_file_metadata.__name__)
+ assert expected_message in str(warning[0].message)
+ finally:
+ if original_metadata is not None:
+ sys.modules[remote_file_metadata.__name__] = original_metadata
diff --git a/api/tests/unit_tests/services/test_remote_file_service.py b/api/tests/unit_tests/services/test_remote_file_service.py
new file mode 100644
index 00000000000..cea00fd8ade
--- /dev/null
+++ b/api/tests/unit_tests/services/test_remote_file_service.py
@@ -0,0 +1,357 @@
+from datetime import UTC, datetime
+from types import SimpleNamespace
+from unittest.mock import MagicMock, call, patch
+
+import httpx
+import pytest
+
+from core.file.remote_file_metadata import FileInfo, InvalidRemoteFileMetadataError
+from core.helper.ssrf_proxy import MaxRetriesExceededError
+from core.tools.errors import ToolSSRFError
+from models.model import Account
+from services.errors.file import FileTooLargeError
+from services.file_service import FileService
+from services.remote_file_service import (
+ RemoteFileAccessDeniedError,
+ RemoteFileError,
+ RemoteFileInvalidResponseError,
+ RemoteFileInvalidUrlError,
+ RemoteFileNotFoundError,
+ RemoteFileService,
+ RemoteFileUnavailableError,
+ RemoteFileUploadResult,
+ RemoteFileUrlBlockedError,
+)
+
+REMOTE_URL = "https://example.com/files/report.pdf"
+
+
+def _response(
+ method: str,
+ status_code: int,
+ *,
+ content: bytes = b"",
+ headers: dict[str, str] | None = None,
+) -> httpx.Response:
+ return httpx.Response(
+ status_code,
+ content=content,
+ headers=headers,
+ request=httpx.Request(method, REMOTE_URL),
+ )
+
+
+def _account() -> Account:
+ account = Account(name="Test Account", email="[email protected]")
+ account.id = "account-id"
+ return account
+
+
+def file_service() -> MagicMock:
+ service = MagicMock(spec=FileService)
+ service.is_file_size_within_limit.return_value = True
+ return service
+
+
+def remote_file_service(file_service: MagicMock) -> RemoteFileService:
+ return RemoteFileService(files=file_service)
+
+
+def test_fetch_info_uses_successful_head_response(remote_file_service: RemoteFileService) -> None:
+ response = _response(
+ "HEAD",
+ httpx.codes.OK,
+ headers={"Content-Type": "application/pdf", "Content-Length": "42"},
+ )
+
+ with patch("services.remote_file_service.remote_fetcher.make_request", return_value=response) as make_request:
+ result = remote_file_service.fetch_info(url=REMOTE_URL)
+
+ assert result.content_type == "application/pdf"
+ assert result.content_length == 42
+ make_request.assert_called_once_with("HEAD", url=REMOTE_URL)
+
+
+def test_fetch_info_falls_back_to_get(remote_file_service: RemoteFileService) -> None:
+ head_response = _response("HEAD", httpx.codes.METHOD_NOT_ALLOWED)
+ get_response = _response("GET", httpx.codes.OK)
+
+ with patch(
+ "services.remote_file_service.remote_fetcher.make_request",
+ side_effect=[head_response, get_response],
+ ) as make_request:
+ result = remote_file_service.fetch_info(url=REMOTE_URL)
+
+ assert result.content_type == "application/octet-stream"
+ assert result.content_length is None
+ assert make_request.call_args_list == [call("HEAD", url=REMOTE_URL), call("GET", url=REMOTE_URL, timeout=3)]
+
+
+ ("source_error", "service_error"),
+ [
+ (httpx.InvalidURL("invalid URL"), RemoteFileInvalidUrlError),
+ (ToolSSRFError("blocked URL and internal proxy details"), RemoteFileUrlBlockedError),
+ (MaxRetriesExceededError("maximum retries reached"), RemoteFileUnavailableError),
+ (
+ httpx.ConnectError("network down", request=httpx.Request("HEAD", REMOTE_URL)),
+ RemoteFileUnavailableError,
+ ),
+ ],
+)
+def test_fetch_info_translates_known_request_failures(
+ remote_file_service: RemoteFileService,
+ source_error: Exception,
+ service_error: type[RemoteFileError],
+) -> None:
+ with patch("services.remote_file_service.remote_fetcher.make_request", side_effect=source_error):
+ with pytest.raises(service_error) as error_info:
+ remote_file_service.fetch_info(url=REMOTE_URL)
+
+ assert error_info.value.__cause__ is source_error
+
+
[email protected]("url", ["not-a-url", "ftp://example.com/report.pdf", "http://", "http://[invalid"])
+def test_fetch_info_rejects_invalid_url_without_requesting_it(
+ remote_file_service: RemoteFileService,
+ url: str,
+) -> None:
+ with patch("services.remote_file_service.remote_fetcher.make_request") as make_request:
+ with pytest.raises(RemoteFileInvalidUrlError):
+ remote_file_service.fetch_info(url=url)
+
+ make_request.assert_not_called()
+
+
+def test_fetch_info_does_not_translate_unknown_failure(remote_file_service: RemoteFileService) -> None:
+ source_error = RuntimeError("database failure while resolving a signed file URL")
+
+ with patch("services.remote_file_service.remote_fetcher.make_request", side_effect=source_error):
+ with pytest.raises(RuntimeError) as error_info:
+ remote_file_service.fetch_info(url=REMOTE_URL)
+
+ assert error_info.value is source_error
+
+
+ ("status_code", "service_error"),
+ [
+ (httpx.codes.NOT_FOUND, RemoteFileNotFoundError),
+ (httpx.codes.GONE, RemoteFileNotFoundError),
+ (httpx.codes.UNAUTHORIZED, RemoteFileAccessDeniedError),
+ (httpx.codes.FORBIDDEN, RemoteFileAccessDeniedError),
+ (httpx.codes.BAD_GATEWAY, RemoteFileUnavailableError),
+ ],
+)
+def test_fetch_info_classifies_remote_status(
+ remote_file_service: RemoteFileService,
+ status_code: int,
+ service_error: type[RemoteFileError],
+) -> None:
+ request = httpx.Request("HEAD", REMOTE_URL)
+ response = httpx.Response(status_code, request=request, text="upstream response must not reach the client")
+
+ with patch("services.remote_file_service.remote_fetcher.make_request", return_value=response):
+ with pytest.raises(service_error):
+ remote_file_service.fetch_info(url=REMOTE_URL)
+
+
+def test_fetch_info_rejects_invalid_content_length(remote_file_service: RemoteFileService) -> None:
+ response = _response("HEAD", httpx.codes.OK, headers={"Content-Length": "not-a-number"})
+
+ with patch("services.remote_file_service.remote_fetcher.make_request", return_value=response):
+ with pytest.raises(RemoteFileInvalidResponseError) as error_info:
+ remote_file_service.fetch_info(url=REMOTE_URL)
+
+ assert isinstance(error_info.value.__cause__, ValueError)
+
+
+def test_upload_reuses_get_fallback_content_and_returns_signed_result(
+ remote_file_service: RemoteFileService,
+ file_service: MagicMock,
+) -> None:
+ account = _account()
+ head_response = _response("HEAD", httpx.codes.METHOD_NOT_ALLOWED)
+ get_response = _response("GET", httpx.codes.OK, content=b"remote content")
+ file_info = FileInfo(filename="report.pdf", extension=".pdf", mimetype="application/pdf", size=14)
+ created_at = datetime(2026, 9, 2, tzinfo=UTC)
+ upload_file = SimpleNamespace(
+ id="upload-id",
+ name="report.pdf",
+ size=14,
+ extension="pdf",
+ mime_type="application/pdf",
+ created_by=account.id,
+ created_at=created_at,
+ )
+ file_service.upload_file.return_value = upload_file
+
+ with (
+ patch(
+ "services.remote_file_service.remote_fetcher.make_request",
+ side_effect=[head_response, get_response],
+ ) as make_request,
+ patch("services.remote_file_service.guess_file_info_from_response", return_value=file_info),
+ patch(
+ "services.remote_file_service.file_helpers.get_signed_file_url",
+ return_value="https://example.com/signed/upload-id",
+ ) as get_signed_file_url,
+ ):
+ result = remote_file_service.upload_from_url(
+ url=REMOTE_URL,
+ user=account,
+ tenant_id="tenant-id",
+ )
+
+ assert make_request.call_args_list == [
+ call("HEAD", url=REMOTE_URL),
+ call("GET", url=REMOTE_URL, timeout=3, follow_redirects=True),
+ ]
+ file_service.is_file_size_within_limit.assert_called_once_with(extension=".pdf", file_size=14)
+ file_service.upload_file.assert_called_once_with(
+ filename="report.pdf",
+ content=b"remote content",
+ mimetype="application/pdf",
+ user=account,
+ tenant_id="tenant-id",
+ source_url=REMOTE_URL,
+ )
+ get_signed_file_url.assert_called_once_with(upload_file_id="upload-id")
+ assert result == RemoteFileUploadResult(
+ id="upload-id",
+ name="report.pdf",
+ size=14,
+ extension="pdf",
+ url="https://example.com/signed/upload-id",
+ mime_type="application/pdf",
+ created_by="account-id",
+ created_at=created_at,
+ )
+
+
+def test_upload_fetches_content_after_successful_head(
+ remote_file_service: RemoteFileService,
+ file_service: MagicMock,
+) -> None:
+ account = _account()
+ head_response = _response("HEAD", httpx.codes.OK)
+ content_response = _response("GET", httpx.codes.OK, content=b"downloaded content")
+ file_info = FileInfo(filename="report.pdf", extension=".pdf", mimetype="application/pdf", size=18)
+ file_service.upload_file.return_value = SimpleNamespace(
+ id="upload-id",
+ name="report.pdf",
+ size=18,
+ extension="pdf",
+ mime_type="application/pdf",
+ created_by=account.id,
+ created_at=datetime(2026, 9, 2, tzinfo=UTC),
+ )
+
+ with (
+ patch(
+ "services.remote_file_service.remote_fetcher.make_request",
+ side_effect=[head_response, content_response],
+ ) as make_request,
+ patch("services.remote_file_service.guess_file_info_from_response", return_value=file_info),
+ patch("services.remote_file_service.file_helpers.get_signed_file_url", return_value="signed-url"),
+ ):
+ remote_file_service.upload_from_url(url=REMOTE_URL, user=account)
+
+ assert make_request.call_args_list == [call("HEAD", url=REMOTE_URL), call("GET", url=REMOTE_URL)]
+ assert file_service.upload_file.call_args.kwargs["content"] == b"downloaded content"
+ assert file_service.upload_file.call_args.kwargs["tenant_id"] is None
+
+
+def test_upload_rejects_failed_content_download(
+ remote_file_service: RemoteFileService,
+ file_service: MagicMock,
+) -> None:
+ head_response = _response("HEAD", httpx.codes.OK)
+ content_response = _response("GET", httpx.codes.BAD_GATEWAY, content=b"bad gateway")
+ file_info = FileInfo(filename="report.pdf", extension=".pdf", mimetype="application/pdf", size=14)
+
+ with (
+ patch(
+ "services.remote_file_service.remote_fetcher.make_request",
+ side_effect=[head_response, content_response],
+ ),
+ patch("services.remote_file_service.guess_file_info_from_response", return_value=file_info),
+ ):
+ with pytest.raises(RemoteFileUnavailableError):
+ remote_file_service.upload_from_url(url=REMOTE_URL, user=_account())
+
+ file_service.upload_file.assert_not_called()
+
+
+def test_upload_rejects_invalid_remote_metadata(
+ remote_file_service: RemoteFileService,
+ file_service: MagicMock,
+) -> None:
+ response = _response("GET", httpx.codes.OK, content=b"remote content")
+ metadata_error = InvalidRemoteFileMetadataError("invalid Content-Length")
+
+ with (
+ patch("services.remote_file_service.remote_fetcher.make_request", return_value=response),
+ patch("services.remote_file_service.guess_file_info_from_response", side_effect=metadata_error),
+ ):
+ with pytest.raises(RemoteFileInvalidResponseError) as error_info:
+ remote_file_service.upload_from_url(url=REMOTE_URL, user=_account())
+
+ assert error_info.value.__cause__ is metadata_error
+ file_service.upload_file.assert_not_called()
+
+
+def test_upload_does_not_translate_unknown_metadata_failure(
+ remote_file_service: RemoteFileService,
+ file_service: MagicMock,
+) -> None:
+ response = _response("GET", httpx.codes.OK, content=b"remote content")
+ source_error = ValueError("unexpected parser bug")
+
+ with (
+ patch("services.remote_file_service.remote_fetcher.make_request", return_value=response),
+ patch("services.remote_file_service.guess_file_info_from_response", side_effect=source_error),
+ ):
+ with pytest.raises(RuntimeError) as error_info:
+ remote_file_service.upload_from_url(url=REMOTE_URL, user=_account())
+
+ assert error_info.value.__cause__ is source_error
+ file_service.upload_file.assert_not_called()
+
+
+def test_upload_rejects_remote_filename_with_path_separator(
+ remote_file_service: RemoteFileService,
+ file_service: MagicMock,
+) -> None:
+ response = _response("GET", httpx.codes.OK, content=b"remote content")
+ file_info = FileInfo(filename="folder/report.pdf", extension=".pdf", mimetype="application/pdf", size=14)
+
+ with (
+ patch("services.remote_file_service.remote_fetcher.make_request", return_value=response),
+ patch("services.remote_file_service.guess_file_info_from_response", return_value=file_info),
+ ):
+ with pytest.raises(RemoteFileInvalidResponseError):
+ remote_file_service.upload_from_url(url=REMOTE_URL, user=_account())
+
+ file_service.upload_file.assert_not_called()
+
+
+def test_upload_rejects_file_that_exceeds_size_limit(
+ remote_file_service: RemoteFileService,
+ file_service: MagicMock,
+) -> None:
+ response = _response("GET", httpx.codes.OK, content=b"remote content")
+ file_info = FileInfo(filename="report.pdf", extension=".pdf", mimetype="application/pdf", size=1024)
+ file_service.is_file_size_within_limit.return_value = False
+
+ with (
+ patch("services.remote_file_service.remote_fetcher.make_request", return_value=response),
+ patch("services.remote_file_service.guess_file_info_from_response", return_value=file_info),
+ ):
+ with pytest.raises(FileTooLargeError):
+ remote_file_service.upload_from_url(url=REMOTE_URL, user=_account())
+
+ file_service.is_file_size_within_limit.assert_called_once_with(extension=".pdf", file_size=1024)
+ file_service.upload_file.assert_not_called()
diff --git a/packages/contracts/generated/api/console/remote-files/types.gen.ts b/packages/contracts/generated/api/console/remote-files/types.gen.ts
index 7a7e3ffa2ed..3edb44fe765 100644
--- a/packages/contracts/generated/api/console/remote-files/types.gen.ts
+++ b/packages/contracts/generated/api/console/remote-files/types.gen.ts
@@ -31,6 +31,16 @@ export type PostRemoteFilesUploadData = {
url: '/remote-files/upload'
}
+export type PostRemoteFilesUploadErrors = {
+ 400: unknown
+ 404: unknown
+ 413: unknown
+ 415: unknown
+ 422: unknown
+ 500: unknown
+ 502: unknown
+}
+
export type PostRemoteFilesUploadResponses = {
201: FileWithSignedUrl
}
@@ -47,6 +57,13 @@ export type GetRemoteFilesByUrlData = {
url: '/remote-files/{url}'
}
+export type GetRemoteFilesByUrlErrors = {
+ 400: unknown
+ 404: unknown
+ 500: unknown
+ 502: unknown
+}
+
export type GetRemoteFilesByUrlResponses = {
200: RemoteFileInfo
}
diff --git a/packages/contracts/generated/api/console/remote-files/zod.gen.ts b/packages/contracts/generated/api/console/remote-files/zod.gen.ts
index f3892990592..f132f9b564a 100644
--- a/packages/contracts/generated/api/console/remote-files/zod.gen.ts
+++ b/packages/contracts/generated/api/console/remote-files/zod.gen.ts
@@ -6,7 +6,7 @@ import * as z from 'zod'
* RemoteFileUploadPayload
*/
export const zRemoteFileUploadPayload = z.object({
- url: z.string(),
+ url: z.url(),
})
/**
diff --git a/packages/contracts/generated/api/console/trial-apps/zod.gen.ts b/packages/contracts/generated/api/console/trial-apps/zod.gen.ts
index 395446a2002..9cdf7a94ca5 100644
--- a/packages/contracts/generated/api/console/trial-apps/zod.gen.ts
+++ b/packages/contracts/generated/api/console/trial-apps/zod.gen.ts
@@ -64,7 +64,7 @@ export const zSuggestedQuestionsResponse = z.object({
* RemoteFileUploadPayload
*/
export const zRemoteFileUploadPayload = z.object({
- url: z.string(),
+ url: z.url(),
})
/**
diff --git a/packages/contracts/generated/api/web/orpc.gen.ts b/packages/contracts/generated/api/web/orpc.gen.ts
index 583c974803d..d235d56a60f 100644
--- a/packages/contracts/generated/api/web/orpc.gen.ts
+++ b/packages/contracts/generated/api/web/orpc.gen.ts
@@ -798,14 +798,19 @@ export const passport = {
* int: HTTP status code 201 for success
*
* Raises:
- * RemoteFileUploadError: Failed to fetch file from remote URL
+ * RemoteFileInvalidUrlError: Remote file URL is invalid
+ * RemoteFileUrlBlockedError: Remote file URL is blocked
+ * RemoteFileNotFoundError: Remote file does not exist
+ * RemoteFileAccessDeniedError: Remote file requires authorization
+ * RemoteFileUnavailableError: Remote file is unavailable
+ * RemoteFileInvalidResponseError: Remote file response is invalid
* FileTooLargeError: File exceeds size limit
* UnsupportedFileTypeError: File type not supported
*/
export const post19 = oc
.route({
description:
- 'Upload a file from a remote URL\nDownloads a file from the provided remote URL and uploads it\nto the platform storage for use in web applications.\n\nArgs:\n app_model: The associated application model\n end_user: The end user making the request\n\nJSON Parameters:\n url: The remote URL to download the file from (required)\n\nReturns:\n dict: File information including ID, signed URL, and metadata\n int: HTTP status code 201 for success\n\nRaises:\n RemoteFileUploadError: Failed to fetch file from remote URL\n FileTooLargeError: File exceeds size limit\n UnsupportedFileTypeError: File type not supported',
+ 'Upload a file from a remote URL\nDownloads a file from the provided remote URL and uploads it\nto the platform storage for use in web applications.\n\nArgs:\n app_model: The associated application model\n end_user: The end user making the request\n\nJSON Parameters:\n url: The remote URL to download the file from (required)\n\nReturns:\n dict: File information including ID, signed URL, and metadata\n int: HTTP status code 201 for success\n\nRaises:\n RemoteFileInvalidUrlError: Remote file URL is invalid\n RemoteFileUrlBlockedError: Remote file URL is blocked\n RemoteFileNotFoundError: Remote file does not exist\n RemoteFileAccessDeniedError: Remote file requires authorization\n RemoteFileUnavailableError: Remote file is unavailable\n RemoteFileInvalidResponseError: Remote file response is invalid\n FileTooLargeError: File exceeds size limit\n UnsupportedFileTypeError: File type not supported',
inputStructure: 'detailed',
method: 'POST',
operationId: 'postRemoteFilesUpload',
diff --git a/packages/contracts/generated/api/web/types.gen.ts b/packages/contracts/generated/api/web/types.gen.ts
index 8db116c5fef..09672af5c97 100644
--- a/packages/contracts/generated/api/web/types.gen.ts
+++ b/packages/contracts/generated/api/web/types.gen.ts
@@ -1695,9 +1695,12 @@ export type PostRemoteFilesUploadData = {
export type PostRemoteFilesUploadErrors = {
400: unknown
+ 404: unknown
413: unknown
415: unknown
+ 422: unknown
500: unknown
+ 502: unknown
}
export type PostRemoteFilesUploadResponses = {
@@ -1720,6 +1723,7 @@ export type GetRemoteFilesByUrlErrors = {
400: unknown
404: unknown
500: unknown
+ 502: unknown
}
export type GetRemoteFilesByUrlResponses = {
diff --git a/packages/contracts/generated/api/web/zod.gen.ts b/packages/contracts/generated/api/web/zod.gen.ts
index 7b6aed9e8b3..d6c3a3a31d9 100644
--- a/packages/contracts/generated/api/web/zod.gen.ts
+++ b/packages/contracts/generated/api/web/zod.gen.ts
@@ -478,7 +478,7 @@ export const zRemoteFileInfo = z.object({
* RemoteFileUploadPayload
*/
export const zRemoteFileUploadPayload = z.object({
- url: z.url().min(1).max(2083),
+ url: z.url(),
})
/**