Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2354 [FEAT] Support presigned S3 URLs for file uploads in API deployments (#1485)

Praveen Kumarcbf6931fb532
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py
index a2a49e998..434751560 100644
--- a/backend/api_v2/api_deployment_views.py
+++ b/backend/api_v2/api_deployment_views.py
@@ -60,7 +60,8 @@ class DeploymentExecution(views.APIView):
data=request.data, context={"api": api, "api_key": api_key}
)
serializer.is_valid(raise_exception=True)
- file_objs = serializer.validated_data.get(ApiExecution.FILES_FORM_DATA)
+ file_objs = serializer.validated_data.get(ApiExecution.FILES_FORM_DATA, [])
+ presigned_urls = serializer.validated_data.get(ApiExecution.PRESIGNED_URLS, [])
timeout = serializer.validated_data.get(ApiExecution.TIMEOUT_FORM_DATA)
include_metadata = serializer.validated_data.get(ApiExecution.INCLUDE_METADATA)
include_metrics = serializer.validated_data.get(ApiExecution.INCLUDE_METRICS)
@@ -69,6 +70,9 @@ class DeploymentExecution(views.APIView):
llm_profile_id = serializer.validated_data.get(ApiExecution.LLM_PROFILE_ID)
hitl_queue_name = serializer.validated_data.get(ApiExecution.HITL_QUEUE_NAME)
+ if presigned_urls:
+ DeploymentHelper.load_presigned_files(presigned_urls, file_objs)
+
response = DeploymentHelper.execute_workflow(
organization_name=org_name,
api=api,
diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.py
index 2c4fdb1ec..82d2e8f6a 100644
--- a/backend/api_v2/deployment_helper.py
+++ b/backend/api_v2/deployment_helper.py
@@ -1,9 +1,12 @@
import logging
+from io import BytesIO
from typing import Any
-from urllib.parse import urlencode
+from urllib.parse import urlencode, urlparse
+import requests
from configuration.models import Configuration
-from django.core.files.uploadedfile import UploadedFile
+from django.conf import settings
+from django.core.files.uploadedfile import InMemoryUploadedFile, UploadedFile
from rest_framework.request import Request
from rest_framework.serializers import Serializer
from rest_framework.utils.serializer_helpers import ReturnDict
@@ -25,6 +28,7 @@ from api_v2.exceptions import (
APINotFound,
InactiveAPI,
InvalidAPIRequest,
+ PresignedURLFetchError,
)
from api_v2.key_helper import KeyHelper
from api_v2.models import APIDeployment, APIKey
@@ -249,3 +253,149 @@ class DeploymentHelper(BaseAPIKeyValidator):
execution_id=execution_id
)
return execution_response
+
+ @staticmethod
+ def fetch_presigned_file(url: str) -> InMemoryUploadedFile:
+ """Fetch a file from a presigned URL and convert it to an uploaded file.
+
+ Args:
+ url (str): The presigned URL to fetch the file from
+
+ Returns:
+ InMemoryUploadedFile: The fetched file as an uploaded file object
+
+ Raises:
+ PresignedURLFetchError: If the file cannot be fetched
+ """
+ parsed_url = urlparse(url)
+ sanitized_url = parsed_url._replace(query="").geturl() # For logging
+ file_stream = None
+
+ try:
+ max_bytes = settings.API_DEPL_PRESIGNED_URL_MAX_FILE_SIZE_MB * 1024 * 1024
+
+ file_stream = BytesIO()
+ downloaded = 0
+ content_type = "" # Default content type
+
+ # Download the file with streaming
+ with requests.get(
+ url, stream=True, timeout=(5, 30), allow_redirects=False
+ ) as resp:
+ resp.raise_for_status()
+
+ # Store content type for later use
+ content_type = resp.headers.get("Content-Type", "")
+
+ # Check Content-Length header if available
+ content_length = resp.headers.get("Content-Length")
+ if content_length:
+ try:
+ if int(content_length) > max_bytes:
+ raise PresignedURLFetchError(
+ url=sanitized_url,
+ error_message=f"File too large ({content_length} bytes). Max allowed: {max_bytes} bytes",
+ status_code=413, # Payload Too Large
+ )
+ except ValueError:
+ # Non-integer Content-Length; ignore and fall back to stream enforcement
+ pass
+
+ # Stream the body with an upper bound to prevent memory exhaustion
+ for chunk in resp.iter_content(chunk_size=1024 * 1024):
+ if not chunk:
+ continue
+
+ downloaded += len(chunk)
+ if downloaded > max_bytes:
+ raise PresignedURLFetchError(
+ url=sanitized_url,
+ error_message=f"File exceeds maximum allowed size of {max_bytes} bytes",
+ status_code=413, # Payload Too Large
+ )
+
+ file_stream.write(chunk)
+
+ # Reset stream position to beginning for reading
+ file_stream.seek(0)
+
+ # Extract filename from URL path
+ filename = (
+ parsed_url.path.split("/")[-1] if parsed_url.path else "unknown_file"
+ )
+
+ # Use octet-stream for generic or missing content types
+ if content_type in ["", "application/octet-stream", "binary/octet-stream"]:
+ content_type = "application/octet-stream"
+ logger.warning(
+ f"Could not detect MIME type for file '{filename}' from URL '{sanitized_url}'"
+ )
+
+ logger.info(
+ f"Fetched file '{filename}' with MIME type '{content_type}' from presigned URL {sanitized_url}"
+ )
+
+ # Return as an InMemoryUploadedFile
+ return InMemoryUploadedFile(
+ file=file_stream,
+ field_name="file",
+ name=filename,
+ content_type=content_type,
+ size=downloaded,
+ charset=None,
+ )
+
+ except requests.RequestException as e:
+ if (
+ isinstance(e, requests.exceptions.HTTPError)
+ and hasattr(e, "response")
+ and e.response is not None
+ ):
+ status_code = e.response.status_code
+ error_msg = f"{e.response.status_code} {e.response.reason}"
+ elif isinstance(
+ e, (requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout)
+ ):
+ status_code = 504 # Gateway Timeout
+ error_msg = f"Request timed out: {str(e)}"
+ logger.error(f"Timeout error fetching presigned URL {sanitized_url}: {e}")
+ elif isinstance(e, requests.exceptions.ConnectionError):
+ status_code = 502 # Bad Gateway
+ error_msg = f"Connection error: {str(e)}"
+ logger.error(
+ f"Connection error fetching presigned URL {sanitized_url}: {e}"
+ )
+ else:
+ status_code = 400
+ error_msg = str(e)
+ logger.error(f"Error fetching presigned URL {sanitized_url}: {e}")
+
+ raise PresignedURLFetchError(
+ url=sanitized_url, error_message=error_msg, status_code=status_code
+ )
+
+ finally:
+ if file_stream:
+ try:
+ file_stream.close()
+ except Exception as e:
+ logger.warning(f"Failed to close file stream: {str(e)}")
+
+ @staticmethod
+ def load_presigned_files(
+ presigned_urls: list[str], file_objs: list[UploadedFile]
+ ) -> None:
+ """Load files from presigned URLs and append them to file_objs.
+
+ This method processes each URL individually, fetches the file,
+ and adds it to the provided file_objs list.
+
+ Note: URL validation is assumed to be already done by the serializer.
+
+ Args:
+ presigned_urls (list[str]): List of presigned URLs to fetch files from
+ file_objs (list[UploadedFile]): List to append the fetched files to
+ """
+ for url in presigned_urls:
+ uploaded_file = DeploymentHelper.fetch_presigned_file(url)
+ file_objs.append(uploaded_file)
diff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.py
index 2fc9b26fa..aa949a03d 100644
--- a/backend/api_v2/serializers.py
+++ b/backend/api_v2/serializers.py
@@ -1,7 +1,9 @@
+import logging
import re
import uuid
from collections import OrderedDict
from typing import Any
+from urllib.parse import urlparse
from django.core.validators import RegexValidator
from pipeline_v2.models import Pipeline
@@ -15,6 +17,7 @@ from rest_framework.serializers import (
ListField,
ModelSerializer,
Serializer,
+ URLField,
ValidationError,
)
from tags.serializers import TagParamsSerializer
@@ -27,6 +30,8 @@ from api_v2.constants import ApiExecution
from api_v2.models import APIDeployment, APIKey
from backend.serializers import AuditSerializer
+logger = logging.getLogger(__name__)
+
class APIDeploymentSerializer(IntegrityErrorMixin, AuditSerializer):
class Meta:
@@ -206,6 +211,8 @@ class ExecutionRequestSerializer(TagParamsSerializer):
If not provided, the default profile will be used.
hitl_queue_name (str, optional): Document class name for manual review queue.
If not provided, uses API name as document class.
+ presigned_urls (list): List of presigned URLs to fetch files from.
+ URLs are validated for HTTPS and S3 endpoint requirements.
"""
MAX_FILES_ALLOWED = 32
@@ -216,6 +223,8 @@ class ExecutionRequestSerializer(TagParamsSerializer):
include_metadata = BooleanField(default=False)
include_metrics = BooleanField(default=False)
use_file_history = BooleanField(default=False)
+
+ presigned_urls = ListField(child=URLField(), required=False)
llm_profile_id = CharField(required=False, allow_null=True, allow_blank=True)
hitl_queue_name = CharField(required=False, allow_null=True, allow_blank=True)
@@ -247,26 +256,81 @@ class ExecutionRequestSerializer(TagParamsSerializer):
raise ValidationError(
"Queue name cannot have repeating underscores or hyphens."
)
-
return value
files = ListField(
child=FileField(),
- required=True,
- allow_empty=False,
- error_messages={
- "required": "At least one file must be provided.",
- "empty": "The file list cannot be empty.",
- },
+ required=False,
+ allow_empty=True,
)
- def validate_files(self, value):
- """Validate the files field."""
- if len(value) == 0:
- raise ValidationError("The file list cannot be empty.")
- if len(value) > self.MAX_FILES_ALLOWED:
- raise ValidationError(f"Maximum '{self.MAX_FILES_ALLOWED}' files allowed.")
- return value
+ def _validate_presigned_url(self, url: str) -> bool:
+ """Validate presigned URL for security and compatibility.
+
+ Args:
+ url (str): The presigned URL to validate
+
+ Returns:
+ bool: True if URL is valid
+
+ Raises:
+ ValidationError: If the URL is invalid or not secure
+ """
+ parsed_url = urlparse(url)
+ scheme = parsed_url.scheme.lower()
+ host = (parsed_url.hostname or "").lower()
+
+ # Require HTTPS for security
+ if scheme != "https":
+ raise ValidationError(
+ {
+ "presigned_urls": f"Only HTTPS presigned URLs are allowed. URL scheme found: {scheme}"
+ }
+ )
+
+ # Only allow S3 endpoints
+ is_aws = host.endswith(".amazonaws.com")
+ looks_like_s3 = (
+ host == "s3.amazonaws.com"
+ or host.endswith(".s3.amazonaws.com")
+ or re.match(r"(^|.*\.)s3[.-]([a-z0-9-]+)\.amazonaws\.com$", host) is not None
+ )
+
+ if not (is_aws and looks_like_s3):
+ raise ValidationError(
+ {
+ "presigned_urls": f"URL host '{host}' is not a valid S3 endpoint. Only S3 pre-signed URLs are supported currently."
+ }
+ )
+
+ return True
+
+ def validate_presigned_urls(self, urls):
+ """Validate presigned URLs for proper format and endpoint requirements."""
+ if not urls:
+ return urls
+
+ for url in urls:
+ self._validate_presigned_url(url)
+
+ return urls
+
+ def validate(self, data):
+ """Validate all parameters including presigned URLs."""
+ data = super().validate(data)
+
+ files = data.get("files", [])
+ urls = data.get("presigned_urls", [])
+ total = len(files) + len(urls)
+
+ if total == 0:
+ raise ValidationError("You must provide at least one file or presigned URL.")
+
+ if total > self.MAX_FILES_ALLOWED:
+ raise ValidationError(
+ f"You can upload a maximum of {self.MAX_FILES_ALLOWED} files in total (uploaded or via presigned URLs)."
+ )
+ return data
def validate_llm_profile_id(self, value):
"""Validate that the llm_profile_id belongs to the API key owner."""
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 8ac0c2671..cb18de0ca 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -103,6 +103,11 @@ API_DEPLOYMENT_PATH_PREFIX = os.environ.get(
"API_DEPLOYMENT_PATH_PREFIX", "deployment"
).strip("/")
+# Maximum file size for presigned URLs in API deployments (in MB)
+API_DEPL_PRESIGNED_URL_MAX_FILE_SIZE_MB = int(
+ os.environ.get("API_DEPL_PRESIGNED_URL_MAX_FILE_SIZE_MB", 20)
+)
+
DB_NAME = os.environ.get("DB_NAME", "unstract_db")
DB_USER = os.environ.get("DB_USER", "unstract_dev")
DB_HOST = os.environ.get("DB_HOST", "backend-db-1")