LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2445 [FEAT] Prevent unsupported file in API and ETL workflow-executions by validating file extension and MIME type (#1361)
alif277dcf1827c
diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.pyindex f6c6af62b..df89d5bd7 100644--- a/backend/api_v2/deployment_helper.py+++ b/backend/api_v2/deployment_helper.py@@ -170,6 +170,7 @@ class DeploymentHelper(BaseAPIKeyValidator):execution_id = workflow_execution.idhash_values_of_files = SourceConnector.add_input_file_to_api_storage(+ pipeline_id=pipeline_id,workflow_id=workflow_id,execution_id=execution_id,file_objs=file_objs,diff --git a/backend/workflow_manager/endpoint_v2/constants.py b/backend/workflow_manager/endpoint_v2/constants.pyindex 75f5434cd..1968ebb12 100644--- a/backend/workflow_manager/endpoint_v2/constants.py+++ b/backend/workflow_manager/endpoint_v2/constants.py@@ -1,3 +1,7 @@+import os+from fnmatch import fnmatch++class TableColumns:CREATED_BY = "created_by"CREATED_AT = "created_at"@@ -80,22 +84,38 @@ class FilePattern:"*.bmp","*.tif","*.tiff",+ "*.webp",]- UNSUPPORTED_FILE_EXTENSIONS = [- # Archive formats- "*.zip",- "*.tar",- "*.gz",- "*.tar.gz",- "*.xz",- "*.tar.xz",- "*.7z",- "*.rar",- "*.bz2",- "*.tar.bz2",- "*.tgz",- "*.tbz2",- ]+ SPREADSHEETS = ["*.xls", "*.xlsx", "*.ods"]+ PRESENTATIONS = ["*.ppt", "*.pptx", "*.odp"]+ OPEN_DOCS = ["*.odt"]+ DATA_FILES = ["*.csv", "*.json"]+ OTHER_FILES = ["*.cdfv2"]++ @classmethod+ def get_supported_extensions(cls) -> list[str]:+ """Aggregate all supported file extensions."""+ return [+ *cls.PDF_DOCUMENTS,+ *cls.TEXT_DOCUMENTS,+ *cls.IMAGES,+ *cls.SPREADSHEETS,+ *cls.PRESENTATIONS,+ *cls.OPEN_DOCS,+ *cls.DATA_FILES,+ *cls.OTHER_FILES,+ ]++ @classmethod+ def is_supported(cls, filename: str) -> bool:+ """Check if the file extension of the given filename is supported."""+ _, ext = os.path.splitext(filename)+ if not ext:+ return True # allow files with no extension+ return any(+ fnmatch(filename.lower(), pattern)+ for pattern in cls.get_supported_extensions()+ )class SourceConstant:diff --git a/backend/workflow_manager/endpoint_v2/enums.py b/backend/workflow_manager/endpoint_v2/enums.pynew file mode 100644index 000000000..cd6cd9f58--- /dev/null+++ b/backend/workflow_manager/endpoint_v2/enums.py@@ -0,0 +1,28 @@+from enum import Enum+++class AllowedFileTypes(Enum):+ PLAIN_TEXT = "text/plain"+ PDF = "application/pdf"+ JPEG = "image/jpeg"+ PNG = "image/png"+ TIFF = "image/tiff"+ BMP = "image/bmp"+ GIF = "image/gif"+ WEBP = "image/webp"+ DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"+ DOC = "application/msword"+ XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"+ XLS = "application/vnd.ms-excel"+ PPTX = "application/vnd.openxmlformats-officedocument.presentationml.presentation"+ PPT = "application/vnd.ms-powerpoint"+ ODT = "application/vnd.oasis.opendocument.text"+ ODS = "application/vnd.oasis.opendocument.spreadsheet"+ ODP = "application/vnd.oasis.opendocument.presentation"+ CDFV2 = "application/CDFV2"+ JSON = "application/json"+ CSV = "text/csv"++ @classmethod+ def is_allowed(cls, mime_type: str) -> bool:+ return mime_type in cls._value2member_map_diff --git a/backend/workflow_manager/endpoint_v2/exceptions.py b/backend/workflow_manager/endpoint_v2/exceptions.pyindex e994627c8..dc2ac95c3 100644--- a/backend/workflow_manager/endpoint_v2/exceptions.py+++ b/backend/workflow_manager/endpoint_v2/exceptions.py@@ -112,3 +112,8 @@ class UnstractQueueException(APIException):class SourceFileOrInfilePathNotFound(APIException):status_code = 500default_detail = "Unable to obtain file (Source or Infile) for execution."+++class UnsupportedMimeTypeError(APIException):+ status_code = 400+ default_detail = "Unsupported MIME type."diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex 29fe05ab9..8df0d0953 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -2,12 +2,14 @@ import fnmatchimport loggingimport osimport shutil+import uuidfrom hashlib import sha256from io import BytesIOfrom itertools import islicefrom typing import Anyimport fsspec+import magicfrom connector_processor.constants import ConnectorKeysfrom connector_v2.models import ConnectorInstancefrom django.core.files.uploadedfile import UploadedFile@@ -22,6 +24,7 @@ from workflow_manager.endpoint_v2.constants import (SourceKey,)from workflow_manager.endpoint_v2.dto import FileHash, SourceConfig+from workflow_manager.endpoint_v2.enums import AllowedFileTypesfrom workflow_manager.endpoint_v2.exceptions import (InvalidInputDirectory,InvalidSourceConnectionType,@@ -29,6 +32,7 @@ from workflow_manager.endpoint_v2.exceptions import (OrganizationIdNotFound,SourceConnectorNotConfigured,SourceFileOrInfilePathNotFound,+ UnsupportedMimeTypeError,)from workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.file_execution.models import WorkflowFileExecution@@ -40,7 +44,7 @@ from workflow_manager.workflow_v2.models.workflow import Workflowfrom unstract.connectors.filesystems.unstract_file_system import UnstractFileSystemfrom unstract.filesystem import FileStorageType, FileSystemfrom unstract.sdk.file_storage import FileStorage-from unstract.workflow_execution.enums import LogState+from unstract.workflow_execution.enums import LogStage, LogStatelogger = logging.getLogger(__name__)@@ -495,13 +499,7 @@ class SourceConnector(BaseConnector):Returns:bool: True if file format is supported, False otherwise"""- file_lower = file_name.lower()- matches_blocked = any(- fnmatch.fnmatchcase(file_lower, ext.lower())- for ext in FilePattern.UNSUPPORTED_FILE_EXTENSIONS- )-- if matches_blocked:+ if not FilePattern.is_supported(file_name):message = f"Skipping '{file_name}' as it has an unsupported file format"logger.debug(message)self.workflow_log.publish_log(message)@@ -699,11 +697,20 @@ class SourceConnector(BaseConnector):first_iteration = Trueinput_log = ""- file_hash = sha256()+ file_content_hash = sha256()with source_fs.open(input_file_path, "rb") as remote_file:while chunk := remote_file.read(self.READ_CHUNK_SIZE):- file_hash.update(chunk)+ file_content_hash.update(chunk)if first_iteration:+ # Detect MIME type using first chunk+ mime_type = magic.from_buffer(chunk, mime=True)+ logger.info(+ f"Detected MIME type: {mime_type} for file {input_file_path}"+ )+ if not AllowedFileTypes.is_allowed(mime_type):+ raise UnsupportedMimeTypeError(+ f"Unsupported MIME type '{mime_type}' for file '{input_file_path}'"+ )input_log = (chunk[:500].decode("utf-8", errors="replace") + "...(truncated)")@@ -717,7 +724,8 @@ class SourceConnector(BaseConnector):# This function is typically relevant for extracted text content,# may not be necessary for PDFs, images, or other non-text formats.self.publish_input_file_content(input_file_path, input_log)- hash_value_of_file_content = file_hash.hexdigest()+ hash_value_of_file_content = file_content_hash.hexdigest()+ file_hash.mime_type = mime_typelogger.info(f"hash_value_of_file {source_file_path} is : {hash_value_of_file_content}")@@ -880,6 +888,7 @@ class SourceConnector(BaseConnector):@classmethoddef add_input_file_to_api_storage(cls,+ pipeline_id: str,workflow_id: str,execution_id: str,file_objs: list[UploadedFile],@@ -896,16 +905,47 @@ class SourceConnector(BaseConnector):Returns:dict[str, FileHash]: Dict containing file name and its corresponding hash"""+ org_schema = UserContext.get_organization_identifier()+ workflow_log = WorkflowLog(+ execution_id=execution_id,+ log_stage=LogStage.SOURCE,+ pipeline_id=pipeline_id,+ organization_id=org_schema,+ )+ workflow_log.publish_log(+ "Staging files in API storage for validation and processing."+ )api_storage_dir = cls.get_api_storage_dir_path(workflow_id=workflow_id, execution_id=execution_id)workflow: Workflow = Workflow.objects.get(id=workflow_id)file_hashes: dict[str, FileHash] = {}unique_file_hashes: set[str] = set()+ connection_type = WorkflowEndpoint.ConnectionType.APIfor file in file_objs:file_name = file.namedestination_path = os.path.join(api_storage_dir, file_name)+ mime_type = file.content_type+ logger.info(f"Detected MIME type: {mime_type} for file {file_name}")+ if not AllowedFileTypes.is_allowed(mime_type):+ log_message = f"Skipping file '{file_name}' to stage due to unsupported MIME type '{mime_type}'"+ workflow_log.log_info(logger=logger, message=log_message)+ # Generate a clearly marked temporary hash to avoid reading the file content+ # Helps to prevent duplicate entries in file executions+ fake_hash = f"temp-hash-{uuid.uuid4().hex}"+ file_hash = FileHash(+ file_path=destination_path,+ source_connection_type=connection_type,+ file_name=file_name,+ file_hash=fake_hash,+ is_executed=True,+ file_size=file.size,+ mime_type=mime_type,+ )+ file_hashes.update({file_name: file_hash})+ continue+file_system = FileSystem(FileStorageType.API_EXECUTION)file_storage = file_system.get_file_storage()file_hash = sha256()@@ -913,13 +953,11 @@ class SourceConnector(BaseConnector):file_hash.update(chunk)file_storage.write(path=destination_path, mode="ab", data=chunk)file_hash = file_hash.hexdigest()- connection_type = WorkflowEndpoint.ConnectionType.API# Skip duplicate filesif file_hash in unique_file_hashes:- logger.info(- f"[Matched Files] Skipping execution of duplicate file: {file_name}"- )+ log_message = f"Skipping file '{file_name}' — duplicate detected within the current request. Already staged for processing."+ workflow_log.log_info(logger=logger, message=log_message)continueunique_file_hashes.add(file_hash)@@ -936,7 +974,7 @@ class SourceConnector(BaseConnector):file_hash=file_hash,is_executed=is_executed,file_size=file.size,- mime_type=file.content_type,+ mime_type=mime_type,)file_hashes.update({file_name: file_hash})return file_hashesdiff --git a/backend/workflow_manager/utils/workflow_log.py b/backend/workflow_manager/utils/workflow_log.pyindex 854be42c6..d7ccad856 100644--- a/backend/workflow_manager/utils/workflow_log.py+++ b/backend/workflow_manager/utils/workflow_log.py@@ -45,9 +45,19 @@ class WorkflowLog:)LogPublisher.publish(self.messaging_channel, log_details)- def log_error(self, logger: logging.Logger, message: str) -> None:+ def log_error(self, logger: logging.Logger, message: str, **kwargs) -> None:+ """Publishes an error log message to the configured logger and to the+ websocket channel.++ Args:+ logger (logging.Logger): The logger to use for logging.+ message (str): The log message to be published.++ Returns:+ None+ """self.publish_log(message, level=LogLevel.ERROR)- logger.error(message, exc_info=True, stack_info=True)+ logger.error(message, **kwargs)def log_info(self, logger: logging.Logger, message: str) -> None:"""Publishes an info log message to the configured logger and to thediff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.pyindex 60adb880b..3d8d450bc 100644--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py@@ -38,6 +38,8 @@ from workflow_manager.endpoint_v2.dto import (FileHash,SourceConfig,)+from workflow_manager.endpoint_v2.enums import AllowedFileTypes+from workflow_manager.endpoint_v2.exceptions import UnsupportedMimeTypeErrorfrom workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.endpoint_v2.result_cache_utils import ResultCacheUtilsfrom workflow_manager.endpoint_v2.source import SourceConnector@@ -488,7 +490,9 @@ class FileExecutionTasks:organization_id=file_data.organization_id,pipeline_id=str(workflow_execution.pipeline_id),)- workflow_log.log_error(logger=logger, message=error_msg)+ workflow_log.log_error(+ logger=logger, message=error_msg, exc_info=True, stack_info=True+ )result = FinalOutputResult(output=None, metadata=None, error=error_msg)return cls._build_final_result(workflow_execution=workflow_execution,@@ -500,8 +504,13 @@ class FileExecutionTasks:destination=destination,)except Exception as error:- error_msg = f"File execution failed: {error}"- workflow_log.log_error(logger=logger, message=error_msg)+ if isinstance(error, UnsupportedMimeTypeError):+ error_msg = str(error)+ else:+ error_msg = f"File execution failed: {error}"+ workflow_log.log_error(+ logger=logger, message=error_msg, exc_info=True, stack_info=True+ )workflow_file_execution.update_status(status=ExecutionStatus.ERROR, execution_error=error_msg[:500])@@ -668,6 +677,12 @@ class FileExecutionTasks:try:logger.info(f"Preparing file for processing: {file_hash.file_name}")+ if file_hash.mime_type and not AllowedFileTypes.is_allowed(+ file_hash.mime_type+ ):+ raise UnsupportedMimeTypeError(+ f"Unsupported MIME type '{file_hash.mime_type}'"+ )content_hash = source.add_file_to_volume(workflow_file_execution=workflow_file_exec,tags=workflow_execution.tag_names,@@ -676,13 +691,32 @@ class FileExecutionTasks:file_hash.file_hash = content_hashworkflow_file_exec.update(file_hash=content_hash)return content_hash+ except UnsupportedMimeTypeError as error:+ error_msg = f"Unsupported MIME type: {error}"+ logger_message = f"Skipping file {file_hash.file_name} due to {error_msg}"+ workflow_log.log_error(logger=logger, message=logger_message)++ # TODO: (Optional) Handle unsupported MIME types in file_history+ # Goal: Record failure to prevent retries, but cache_key is missing+ # Constraints:+ # - cache_key required (NOT NULL) but unavailable (file not fully read)+ # Solutions:+ # 1. Allow null cache_key (schema change)+ # 2. Generate fallback hash (e.g., f"UNSUPPORTED_{file_hash.file_name}")+ # 3. Read full file for hash (performance cost)+ # Action: Requires team decision on preferred approach.+ raiseexcept FileNotFoundError as error:error_msg = f"File not Found in execution dir: {error}"- workflow_log.log_error(logger=logger, message=error_msg)+ workflow_log.log_error(+ logger=logger, message=error_msg, exc_info=True, stack_info=True+ )raise WorkflowExecutionError(error_msg) from errorexcept Exception as error:error_msg = f"File preparation failed: {error}"- workflow_log.log_error(logger=logger, message=error_msg)+ workflow_log.log_error(+ logger=logger, message=error_msg, exc_info=True, stack_info=True+ )raise WorkflowExecutionError(error_msg) from error@classmethoddiff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.pyindex 6098af0b3..c8bcafb26 100644--- a/backend/workflow_manager/workflow_v2/views.py+++ b/backend/workflow_manager/workflow_v2/views.py@@ -155,6 +155,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):if file_objs and execution_id and workflow_id:use_file_history = Falsehashes_of_files = SourceConnector.add_input_file_to_api_storage(+ pipeline_id=pipeline_guid,workflow_id=workflow_id,execution_id=execution_id,file_objs=file_objs,