Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2445 [FEAT] Prevent unsupported file in API and ETL workflow-executions by validating file extension and MIME type (#1361)

alif277dcf1827c
diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.py
index f6c6af62b..df89d5bd7 100644
--- a/backend/api_v2/deployment_helper.py
+++ b/backend/api_v2/deployment_helper.py
@@ -170,6 +170,7 @@ class DeploymentHelper(BaseAPIKeyValidator):
execution_id = workflow_execution.id
hash_values_of_files = SourceConnector.add_input_file_to_api_storage(
+ pipeline_id=pipeline_id,
workflow_id=workflow_id,
execution_id=execution_id,
file_objs=file_objs,
diff --git a/backend/workflow_manager/endpoint_v2/constants.py b/backend/workflow_manager/endpoint_v2/constants.py
index 75f5434cd..1968ebb12 100644
--- a/backend/workflow_manager/endpoint_v2/constants.py
+++ b/backend/workflow_manager/endpoint_v2/constants.py
@@ -1,3 +1,7 @@
+import os
+from fnmatch import fnmatch
+
+
class TableColumns:
CREATED_BY = "created_by"
CREATED_AT = "created_at"
@@ -80,22 +84,38 @@ class FilePattern:
"*.bmp",
"*.tif",
"*.tiff",
+ "*.webp",
]
- UNSUPPORTED_FILE_EXTENSIONS = [
- # Archive formats
- "*.zip",
- "*.tar",
- "*.gz",
- "*.tar.gz",
- "*.xz",
- "*.tar.xz",
- "*.7z",
- "*.rar",
- "*.bz2",
- "*.tar.bz2",
- "*.tgz",
- "*.tbz2",
- ]
+ SPREADSHEETS = ["*.xls", "*.xlsx", "*.ods"]
+ PRESENTATIONS = ["*.ppt", "*.pptx", "*.odp"]
+ OPEN_DOCS = ["*.odt"]
+ DATA_FILES = ["*.csv", "*.json"]
+ OTHER_FILES = ["*.cdfv2"]
+
+ @classmethod
+ def get_supported_extensions(cls) -> list[str]:
+ """Aggregate all supported file extensions."""
+ return [
+ *cls.PDF_DOCUMENTS,
+ *cls.TEXT_DOCUMENTS,
+ *cls.IMAGES,
+ *cls.SPREADSHEETS,
+ *cls.PRESENTATIONS,
+ *cls.OPEN_DOCS,
+ *cls.DATA_FILES,
+ *cls.OTHER_FILES,
+ ]
+
+ @classmethod
+ def is_supported(cls, filename: str) -> bool:
+ """Check if the file extension of the given filename is supported."""
+ _, ext = os.path.splitext(filename)
+ if not ext:
+ return True # allow files with no extension
+ return any(
+ fnmatch(filename.lower(), pattern)
+ for pattern in cls.get_supported_extensions()
+ )
class SourceConstant:
diff --git a/backend/workflow_manager/endpoint_v2/enums.py b/backend/workflow_manager/endpoint_v2/enums.py
new file mode 100644
index 000000000..cd6cd9f58
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/enums.py
@@ -0,0 +1,28 @@
+from enum import Enum
+
+
+class AllowedFileTypes(Enum):
+ PLAIN_TEXT = "text/plain"
+ PDF = "application/pdf"
+ JPEG = "image/jpeg"
+ PNG = "image/png"
+ TIFF = "image/tiff"
+ BMP = "image/bmp"
+ GIF = "image/gif"
+ WEBP = "image/webp"
+ DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+ DOC = "application/msword"
+ XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+ XLS = "application/vnd.ms-excel"
+ PPTX = "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+ PPT = "application/vnd.ms-powerpoint"
+ ODT = "application/vnd.oasis.opendocument.text"
+ ODS = "application/vnd.oasis.opendocument.spreadsheet"
+ ODP = "application/vnd.oasis.opendocument.presentation"
+ CDFV2 = "application/CDFV2"
+ JSON = "application/json"
+ CSV = "text/csv"
+
+ @classmethod
+ def is_allowed(cls, mime_type: str) -> bool:
+ return mime_type in cls._value2member_map_
diff --git a/backend/workflow_manager/endpoint_v2/exceptions.py b/backend/workflow_manager/endpoint_v2/exceptions.py
index e994627c8..dc2ac95c3 100644
--- a/backend/workflow_manager/endpoint_v2/exceptions.py
+++ b/backend/workflow_manager/endpoint_v2/exceptions.py
@@ -112,3 +112,8 @@ class UnstractQueueException(APIException):
class SourceFileOrInfilePathNotFound(APIException):
status_code = 500
default_detail = "Unable to obtain file (Source or Infile) for execution."
+
+
+class UnsupportedMimeTypeError(APIException):
+ status_code = 400
+ default_detail = "Unsupported MIME type."
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index 29fe05ab9..8df0d0953 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -2,12 +2,14 @@ import fnmatch
import logging
import os
import shutil
+import uuid
from hashlib import sha256
from io import BytesIO
from itertools import islice
from typing import Any
import fsspec
+import magic
from connector_processor.constants import ConnectorKeys
from connector_v2.models import ConnectorInstance
from django.core.files.uploadedfile import UploadedFile
@@ -22,6 +24,7 @@ from workflow_manager.endpoint_v2.constants import (
SourceKey,
)
from workflow_manager.endpoint_v2.dto import FileHash, SourceConfig
+from workflow_manager.endpoint_v2.enums import AllowedFileTypes
from workflow_manager.endpoint_v2.exceptions import (
InvalidInputDirectory,
InvalidSourceConnectionType,
@@ -29,6 +32,7 @@ from workflow_manager.endpoint_v2.exceptions import (
OrganizationIdNotFound,
SourceConnectorNotConfigured,
SourceFileOrInfilePathNotFound,
+ UnsupportedMimeTypeError,
)
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.file_execution.models import WorkflowFileExecution
@@ -40,7 +44,7 @@ from workflow_manager.workflow_v2.models.workflow import Workflow
from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
from unstract.filesystem import FileStorageType, FileSystem
from unstract.sdk.file_storage import FileStorage
-from unstract.workflow_execution.enums import LogState
+from unstract.workflow_execution.enums import LogStage, LogState
logger = logging.getLogger(__name__)
@@ -495,13 +499,7 @@ class SourceConnector(BaseConnector):
Returns:
bool: True if file format is supported, False otherwise
"""
- file_lower = file_name.lower()
- matches_blocked = any(
- fnmatch.fnmatchcase(file_lower, ext.lower())
- for ext in FilePattern.UNSUPPORTED_FILE_EXTENSIONS
- )
-
- if matches_blocked:
+ if not FilePattern.is_supported(file_name):
message = f"Skipping '{file_name}' as it has an unsupported file format"
logger.debug(message)
self.workflow_log.publish_log(message)
@@ -699,11 +697,20 @@ class SourceConnector(BaseConnector):
first_iteration = True
input_log = ""
- file_hash = sha256()
+ file_content_hash = sha256()
with source_fs.open(input_file_path, "rb") as remote_file:
while chunk := remote_file.read(self.READ_CHUNK_SIZE):
- file_hash.update(chunk)
+ file_content_hash.update(chunk)
if first_iteration:
+ # Detect MIME type using first chunk
+ mime_type = magic.from_buffer(chunk, mime=True)
+ logger.info(
+ f"Detected MIME type: {mime_type} for file {input_file_path}"
+ )
+ if not AllowedFileTypes.is_allowed(mime_type):
+ raise UnsupportedMimeTypeError(
+ f"Unsupported MIME type '{mime_type}' for file '{input_file_path}'"
+ )
input_log = (
chunk[:500].decode("utf-8", errors="replace") + "...(truncated)"
)
@@ -717,7 +724,8 @@ class SourceConnector(BaseConnector):
# This function is typically relevant for extracted text content,
# may not be necessary for PDFs, images, or other non-text formats.
self.publish_input_file_content(input_file_path, input_log)
- hash_value_of_file_content = file_hash.hexdigest()
+ hash_value_of_file_content = file_content_hash.hexdigest()
+ file_hash.mime_type = mime_type
logger.info(
f"hash_value_of_file {source_file_path} is : {hash_value_of_file_content}"
)
@@ -880,6 +888,7 @@ class SourceConnector(BaseConnector):
@classmethod
def add_input_file_to_api_storage(
cls,
+ pipeline_id: str,
workflow_id: str,
execution_id: str,
file_objs: list[UploadedFile],
@@ -896,16 +905,47 @@ class SourceConnector(BaseConnector):
Returns:
dict[str, FileHash]: Dict containing file name and its corresponding hash
"""
+ org_schema = UserContext.get_organization_identifier()
+ workflow_log = WorkflowLog(
+ execution_id=execution_id,
+ log_stage=LogStage.SOURCE,
+ pipeline_id=pipeline_id,
+ organization_id=org_schema,
+ )
+ workflow_log.publish_log(
+ "Staging files in API storage for validation and processing."
+ )
api_storage_dir = cls.get_api_storage_dir_path(
workflow_id=workflow_id, execution_id=execution_id
)
workflow: Workflow = Workflow.objects.get(id=workflow_id)
file_hashes: dict[str, FileHash] = {}
unique_file_hashes: set[str] = set()
+ connection_type = WorkflowEndpoint.ConnectionType.API
for file in file_objs:
file_name = file.name
destination_path = os.path.join(api_storage_dir, file_name)
+ mime_type = file.content_type
+ logger.info(f"Detected MIME type: {mime_type} for file {file_name}")
+ if not AllowedFileTypes.is_allowed(mime_type):
+ log_message = f"Skipping file '{file_name}' to stage due to unsupported MIME type '{mime_type}'"
+ workflow_log.log_info(logger=logger, message=log_message)
+ # Generate a clearly marked temporary hash to avoid reading the file content
+ # Helps to prevent duplicate entries in file executions
+ fake_hash = f"temp-hash-{uuid.uuid4().hex}"
+ file_hash = FileHash(
+ file_path=destination_path,
+ source_connection_type=connection_type,
+ file_name=file_name,
+ file_hash=fake_hash,
+ is_executed=True,
+ file_size=file.size,
+ mime_type=mime_type,
+ )
+ file_hashes.update({file_name: file_hash})
+ continue
+
file_system = FileSystem(FileStorageType.API_EXECUTION)
file_storage = file_system.get_file_storage()
file_hash = sha256()
@@ -913,13 +953,11 @@ class SourceConnector(BaseConnector):
file_hash.update(chunk)
file_storage.write(path=destination_path, mode="ab", data=chunk)
file_hash = file_hash.hexdigest()
- connection_type = WorkflowEndpoint.ConnectionType.API
# Skip duplicate files
if file_hash in unique_file_hashes:
- logger.info(
- f"[Matched Files] Skipping execution of duplicate file: {file_name}"
- )
+ log_message = f"Skipping file '{file_name}' — duplicate detected within the current request. Already staged for processing."
+ workflow_log.log_info(logger=logger, message=log_message)
continue
unique_file_hashes.add(file_hash)
@@ -936,7 +974,7 @@ class SourceConnector(BaseConnector):
file_hash=file_hash,
is_executed=is_executed,
file_size=file.size,
- mime_type=file.content_type,
+ mime_type=mime_type,
)
file_hashes.update({file_name: file_hash})
return file_hashes
diff --git a/backend/workflow_manager/utils/workflow_log.py b/backend/workflow_manager/utils/workflow_log.py
index 854be42c6..d7ccad856 100644
--- a/backend/workflow_manager/utils/workflow_log.py
+++ b/backend/workflow_manager/utils/workflow_log.py
@@ -45,9 +45,19 @@ class WorkflowLog:
)
LogPublisher.publish(self.messaging_channel, log_details)
- def log_error(self, logger: logging.Logger, message: str) -> None:
+ def log_error(self, logger: logging.Logger, message: str, **kwargs) -> None:
+ """Publishes an error log message to the configured logger and to the
+ websocket channel.
+
+ Args:
+ logger (logging.Logger): The logger to use for logging.
+ message (str): The log message to be published.
+
+ Returns:
+ None
+ """
self.publish_log(message, level=LogLevel.ERROR)
- logger.error(message, exc_info=True, stack_info=True)
+ logger.error(message, **kwargs)
def log_info(self, logger: logging.Logger, message: str) -> None:
"""Publishes an info log message to the configured logger and to the
diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
index 60adb880b..3d8d450bc 100644
--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py
+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
@@ -38,6 +38,8 @@ from workflow_manager.endpoint_v2.dto import (
FileHash,
SourceConfig,
)
+from workflow_manager.endpoint_v2.enums import AllowedFileTypes
+from workflow_manager.endpoint_v2.exceptions import UnsupportedMimeTypeError
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.endpoint_v2.result_cache_utils import ResultCacheUtils
from workflow_manager.endpoint_v2.source import SourceConnector
@@ -488,7 +490,9 @@ class FileExecutionTasks:
organization_id=file_data.organization_id,
pipeline_id=str(workflow_execution.pipeline_id),
)
- workflow_log.log_error(logger=logger, message=error_msg)
+ workflow_log.log_error(
+ logger=logger, message=error_msg, exc_info=True, stack_info=True
+ )
result = FinalOutputResult(output=None, metadata=None, error=error_msg)
return cls._build_final_result(
workflow_execution=workflow_execution,
@@ -500,8 +504,13 @@ class FileExecutionTasks:
destination=destination,
)
except Exception as error:
- error_msg = f"File execution failed: {error}"
- workflow_log.log_error(logger=logger, message=error_msg)
+ if isinstance(error, UnsupportedMimeTypeError):
+ error_msg = str(error)
+ else:
+ error_msg = f"File execution failed: {error}"
+ workflow_log.log_error(
+ logger=logger, message=error_msg, exc_info=True, stack_info=True
+ )
workflow_file_execution.update_status(
status=ExecutionStatus.ERROR, execution_error=error_msg[:500]
)
@@ -668,6 +677,12 @@ class FileExecutionTasks:
try:
logger.info(f"Preparing file for processing: {file_hash.file_name}")
+ if file_hash.mime_type and not AllowedFileTypes.is_allowed(
+ file_hash.mime_type
+ ):
+ raise UnsupportedMimeTypeError(
+ f"Unsupported MIME type '{file_hash.mime_type}'"
+ )
content_hash = source.add_file_to_volume(
workflow_file_execution=workflow_file_exec,
tags=workflow_execution.tag_names,
@@ -676,13 +691,32 @@ class FileExecutionTasks:
file_hash.file_hash = content_hash
workflow_file_exec.update(file_hash=content_hash)
return content_hash
+ except UnsupportedMimeTypeError as error:
+ error_msg = f"Unsupported MIME type: {error}"
+ logger_message = f"Skipping file {file_hash.file_name} due to {error_msg}"
+ workflow_log.log_error(logger=logger, message=logger_message)
+
+ # TODO: (Optional) Handle unsupported MIME types in file_history
+ # Goal: Record failure to prevent retries, but cache_key is missing
+ # Constraints:
+ # - cache_key required (NOT NULL) but unavailable (file not fully read)
+ # Solutions:
+ # 1. Allow null cache_key (schema change)
+ # 2. Generate fallback hash (e.g., f"UNSUPPORTED_{file_hash.file_name}")
+ # 3. Read full file for hash (performance cost)
+ # Action: Requires team decision on preferred approach.
+ raise
except FileNotFoundError as error:
error_msg = f"File not Found in execution dir: {error}"
- workflow_log.log_error(logger=logger, message=error_msg)
+ workflow_log.log_error(
+ logger=logger, message=error_msg, exc_info=True, stack_info=True
+ )
raise WorkflowExecutionError(error_msg) from error
except Exception as error:
error_msg = f"File preparation failed: {error}"
- workflow_log.log_error(logger=logger, message=error_msg)
+ workflow_log.log_error(
+ logger=logger, message=error_msg, exc_info=True, stack_info=True
+ )
raise WorkflowExecutionError(error_msg) from error
@classmethod
diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.py
index 6098af0b3..c8bcafb26 100644
--- a/backend/workflow_manager/workflow_v2/views.py
+++ b/backend/workflow_manager/workflow_v2/views.py
@@ -155,6 +155,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
if file_objs and execution_id and workflow_id:
use_file_history = False
hashes_of_files = SourceConnector.add_input_file_to_api_storage(
+ pipeline_id=pipeline_guid,
workflow_id=workflow_id,
execution_id=execution_id,
file_objs=file_objs,