LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2354 [FEAT] Support presigned S3 URLs for file uploads in API deployments (#1485)
Praveen Kumarcbf6931fb532
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.pyindex a2a49e998..434751560 100644--- a/backend/api_v2/api_deployment_views.py+++ b/backend/api_v2/api_deployment_views.py@@ -60,7 +60,8 @@ class DeploymentExecution(views.APIView):data=request.data, context={"api": api, "api_key": api_key})serializer.is_valid(raise_exception=True)- file_objs = serializer.validated_data.get(ApiExecution.FILES_FORM_DATA)+ file_objs = serializer.validated_data.get(ApiExecution.FILES_FORM_DATA, [])+ presigned_urls = serializer.validated_data.get(ApiExecution.PRESIGNED_URLS, [])timeout = serializer.validated_data.get(ApiExecution.TIMEOUT_FORM_DATA)include_metadata = serializer.validated_data.get(ApiExecution.INCLUDE_METADATA)include_metrics = serializer.validated_data.get(ApiExecution.INCLUDE_METRICS)@@ -69,6 +70,9 @@ class DeploymentExecution(views.APIView):llm_profile_id = serializer.validated_data.get(ApiExecution.LLM_PROFILE_ID)hitl_queue_name = serializer.validated_data.get(ApiExecution.HITL_QUEUE_NAME)+ if presigned_urls:+ DeploymentHelper.load_presigned_files(presigned_urls, file_objs)+response = DeploymentHelper.execute_workflow(organization_name=org_name,api=api,diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.pyindex 2c4fdb1ec..82d2e8f6a 100644--- a/backend/api_v2/deployment_helper.py+++ b/backend/api_v2/deployment_helper.py@@ -1,9 +1,12 @@import logging+from io import BytesIOfrom typing import Any-from urllib.parse import urlencode+from urllib.parse import urlencode, urlparse+import requestsfrom configuration.models import Configuration-from django.core.files.uploadedfile import UploadedFile+from django.conf import settings+from django.core.files.uploadedfile import InMemoryUploadedFile, UploadedFilefrom rest_framework.request import Requestfrom rest_framework.serializers import Serializerfrom rest_framework.utils.serializer_helpers import ReturnDict@@ -25,6 +28,7 @@ from api_v2.exceptions import (APINotFound,InactiveAPI,InvalidAPIRequest,+ PresignedURLFetchError,)from api_v2.key_helper import KeyHelperfrom api_v2.models import APIDeployment, APIKey@@ -249,3 +253,149 @@ class DeploymentHelper(BaseAPIKeyValidator):execution_id=execution_id)return execution_response++ @staticmethod+ def fetch_presigned_file(url: str) -> InMemoryUploadedFile:+ """Fetch a file from a presigned URL and convert it to an uploaded file.++ Args:+ url (str): The presigned URL to fetch the file from++ Returns:+ InMemoryUploadedFile: The fetched file as an uploaded file object++ Raises:+ PresignedURLFetchError: If the file cannot be fetched+ """+ parsed_url = urlparse(url)+ sanitized_url = parsed_url._replace(query="").geturl() # For logging+ file_stream = None++ try:+ max_bytes = settings.API_DEPL_PRESIGNED_URL_MAX_FILE_SIZE_MB * 1024 * 1024++ file_stream = BytesIO()+ downloaded = 0+ content_type = "" # Default content type++ # Download the file with streaming+ with requests.get(+ url, stream=True, timeout=(5, 30), allow_redirects=False+ ) as resp:+ resp.raise_for_status()++ # Store content type for later use+ content_type = resp.headers.get("Content-Type", "")++ # Check Content-Length header if available+ content_length = resp.headers.get("Content-Length")+ if content_length:+ try:+ if int(content_length) > max_bytes:+ raise PresignedURLFetchError(+ url=sanitized_url,+ error_message=f"File too large ({content_length} bytes). Max allowed: {max_bytes} bytes",+ status_code=413, # Payload Too Large+ )+ except ValueError:+ # Non-integer Content-Length; ignore and fall back to stream enforcement+ pass++ # Stream the body with an upper bound to prevent memory exhaustion+ for chunk in resp.iter_content(chunk_size=1024 * 1024):+ if not chunk:+ continue++ downloaded += len(chunk)+ if downloaded > max_bytes:+ raise PresignedURLFetchError(+ url=sanitized_url,+ error_message=f"File exceeds maximum allowed size of {max_bytes} bytes",+ status_code=413, # Payload Too Large+ )++ file_stream.write(chunk)++ # Reset stream position to beginning for reading+ file_stream.seek(0)++ # Extract filename from URL path+ filename = (+ parsed_url.path.split("/")[-1] if parsed_url.path else "unknown_file"+ )++ # Use octet-stream for generic or missing content types+ if content_type in ["", "application/octet-stream", "binary/octet-stream"]:+ content_type = "application/octet-stream"+ logger.warning(+ f"Could not detect MIME type for file '{filename}' from URL '{sanitized_url}'"+ )++ logger.info(+ f"Fetched file '{filename}' with MIME type '{content_type}' from presigned URL {sanitized_url}"+ )++ # Return as an InMemoryUploadedFile+ return InMemoryUploadedFile(+ file=file_stream,+ field_name="file",+ name=filename,+ content_type=content_type,+ size=downloaded,+ charset=None,+ )++ except requests.RequestException as e:+ if (+ isinstance(e, requests.exceptions.HTTPError)+ and hasattr(e, "response")+ and e.response is not None+ ):+ status_code = e.response.status_code+ error_msg = f"{e.response.status_code} {e.response.reason}"+ elif isinstance(+ e, (requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout)+ ):+ status_code = 504 # Gateway Timeout+ error_msg = f"Request timed out: {str(e)}"+ logger.error(f"Timeout error fetching presigned URL {sanitized_url}: {e}")+ elif isinstance(e, requests.exceptions.ConnectionError):+ status_code = 502 # Bad Gateway+ error_msg = f"Connection error: {str(e)}"+ logger.error(+ f"Connection error fetching presigned URL {sanitized_url}: {e}"+ )+ else:+ status_code = 400+ error_msg = str(e)+ logger.error(f"Error fetching presigned URL {sanitized_url}: {e}")++ raise PresignedURLFetchError(+ url=sanitized_url, error_message=error_msg, status_code=status_code+ )++ finally:+ if file_stream:+ try:+ file_stream.close()+ except Exception as e:+ logger.warning(f"Failed to close file stream: {str(e)}")++ @staticmethod+ def load_presigned_files(+ presigned_urls: list[str], file_objs: list[UploadedFile]+ ) -> None:+ """Load files from presigned URLs and append them to file_objs.++ This method processes each URL individually, fetches the file,+ and adds it to the provided file_objs list.++ Note: URL validation is assumed to be already done by the serializer.++ Args:+ presigned_urls (list[str]): List of presigned URLs to fetch files from+ file_objs (list[UploadedFile]): List to append the fetched files to+ """+ for url in presigned_urls:+ uploaded_file = DeploymentHelper.fetch_presigned_file(url)+ file_objs.append(uploaded_file)diff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.pyindex 2fc9b26fa..aa949a03d 100644--- a/backend/api_v2/serializers.py+++ b/backend/api_v2/serializers.py@@ -1,7 +1,9 @@+import loggingimport reimport uuidfrom collections import OrderedDictfrom typing import Any+from urllib.parse import urlparsefrom django.core.validators import RegexValidatorfrom pipeline_v2.models import Pipeline@@ -15,6 +17,7 @@ from rest_framework.serializers import (ListField,ModelSerializer,Serializer,+ URLField,ValidationError,)from tags.serializers import TagParamsSerializer@@ -27,6 +30,8 @@ from api_v2.constants import ApiExecutionfrom api_v2.models import APIDeployment, APIKeyfrom backend.serializers import AuditSerializer+logger = logging.getLogger(__name__)+class APIDeploymentSerializer(IntegrityErrorMixin, AuditSerializer):class Meta:@@ -206,6 +211,8 @@ class ExecutionRequestSerializer(TagParamsSerializer):If not provided, the default profile will be used.hitl_queue_name (str, optional): Document class name for manual review queue.If not provided, uses API name as document class.+ presigned_urls (list): List of presigned URLs to fetch files from.+ URLs are validated for HTTPS and S3 endpoint requirements."""MAX_FILES_ALLOWED = 32@@ -216,6 +223,8 @@ class ExecutionRequestSerializer(TagParamsSerializer):include_metadata = BooleanField(default=False)include_metrics = BooleanField(default=False)use_file_history = BooleanField(default=False)++ presigned_urls = ListField(child=URLField(), required=False)llm_profile_id = CharField(required=False, allow_null=True, allow_blank=True)hitl_queue_name = CharField(required=False, allow_null=True, allow_blank=True)@@ -247,26 +256,81 @@ class ExecutionRequestSerializer(TagParamsSerializer):raise ValidationError("Queue name cannot have repeating underscores or hyphens.")-return valuefiles = ListField(child=FileField(),- required=True,- allow_empty=False,- error_messages={- "required": "At least one file must be provided.",- "empty": "The file list cannot be empty.",- },+ required=False,+ allow_empty=True,)- def validate_files(self, value):- """Validate the files field."""- if len(value) == 0:- raise ValidationError("The file list cannot be empty.")- if len(value) > self.MAX_FILES_ALLOWED:- raise ValidationError(f"Maximum '{self.MAX_FILES_ALLOWED}' files allowed.")- return value+ def _validate_presigned_url(self, url: str) -> bool:+ """Validate presigned URL for security and compatibility.++ Args:+ url (str): The presigned URL to validate++ Returns:+ bool: True if URL is valid++ Raises:+ ValidationError: If the URL is invalid or not secure+ """+ parsed_url = urlparse(url)+ scheme = parsed_url.scheme.lower()+ host = (parsed_url.hostname or "").lower()++ # Require HTTPS for security+ if scheme != "https":+ raise ValidationError(+ {+ "presigned_urls": f"Only HTTPS presigned URLs are allowed. URL scheme found: {scheme}"+ }+ )++ # Only allow S3 endpoints+ is_aws = host.endswith(".amazonaws.com")+ looks_like_s3 = (+ host == "s3.amazonaws.com"+ or host.endswith(".s3.amazonaws.com")+ or re.match(r"(^|.*\.)s3[.-]([a-z0-9-]+)\.amazonaws\.com$", host) is not None+ )++ if not (is_aws and looks_like_s3):+ raise ValidationError(+ {+ "presigned_urls": f"URL host '{host}' is not a valid S3 endpoint. Only S3 pre-signed URLs are supported currently."+ }+ )++ return True++ def validate_presigned_urls(self, urls):+ """Validate presigned URLs for proper format and endpoint requirements."""+ if not urls:+ return urls++ for url in urls:+ self._validate_presigned_url(url)++ return urls++ def validate(self, data):+ """Validate all parameters including presigned URLs."""+ data = super().validate(data)++ files = data.get("files", [])+ urls = data.get("presigned_urls", [])+ total = len(files) + len(urls)++ if total == 0:+ raise ValidationError("You must provide at least one file or presigned URL.")++ if total > self.MAX_FILES_ALLOWED:+ raise ValidationError(+ f"You can upload a maximum of {self.MAX_FILES_ALLOWED} files in total (uploaded or via presigned URLs)."+ )+ return datadef validate_llm_profile_id(self, value):"""Validate that the llm_profile_id belongs to the API key owner."""diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 8ac0c2671..cb18de0ca 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -103,6 +103,11 @@ API_DEPLOYMENT_PATH_PREFIX = os.environ.get("API_DEPLOYMENT_PATH_PREFIX", "deployment").strip("/")+# Maximum file size for presigned URLs in API deployments (in MB)+API_DEPL_PRESIGNED_URL_MAX_FILE_SIZE_MB = int(+ os.environ.get("API_DEPL_PRESIGNED_URL_MAX_FILE_SIZE_MB", 20)+)+DB_NAME = os.environ.get("DB_NAME", "unstract_db")DB_USER = os.environ.get("DB_USER", "unstract_dev")DB_HOST = os.environ.get("DB_HOST", "backend-db-1")