Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Added and validate for unsupported extesnions while list files from source (#1152)

ali3c2068d3ed64
diff --git a/backend/workflow_manager/endpoint_v2/constants.py b/backend/workflow_manager/endpoint_v2/constants.py
index 9a89763f6..75f5434cd 100644
--- a/backend/workflow_manager/endpoint_v2/constants.py
+++ b/backend/workflow_manager/endpoint_v2/constants.py
@@ -81,6 +81,21 @@ class FilePattern:
"*.tif",
"*.tiff",
]
+ UNSUPPORTED_FILE_EXTENSIONS = [
+ # Archive formats
+ "*.zip",
+ "*.tar",
+ "*.gz",
+ "*.tar.gz",
+ "*.xz",
+ "*.tar.xz",
+ "*.7z",
+ "*.rar",
+ "*.bz2",
+ "*.tar.bz2",
+ "*.tgz",
+ "*.tbz2",
+ ]
class SourceConstant:
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index a03932146..123e09759 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -330,11 +330,50 @@ class SourceConnector(BaseConnector):
return matched_files, count
def _should_process_file(self, file: str, patterns: list[str]) -> bool:
- """Check if the file should be processed based on the patterns."""
- return bool(file) and any(
- fnmatch.fnmatchcase(file.lower(), pattern.lower()) for pattern in patterns
+ """
+ Check if the file should be processed based on the patterns.
+
+ Args:
+ file: The filename to check
+ patterns: List of patterns to match against
+
+ Returns:
+ bool: True if file should be processed, False otherwise
+ """
+ if not file:
+ return False
+
+ file_lower = file.lower()
+ matches_pattern = any(
+ fnmatch.fnmatchcase(file_lower, pattern.lower()) for pattern in patterns
)
+ return matches_pattern and self._is_valid_pattern(file)
+
+ def _is_valid_pattern(self, file_name: str) -> bool:
+ """
+ Check if the file has a supported format.
+
+ Args:
+ file_name: The filename to check
+
+ Returns:
+ bool: True if file format is supported, False otherwise
+ """
+ file_lower = file_name.lower()
+ matches_blocked = any(
+ fnmatch.fnmatchcase(file_lower, ext.lower())
+ for ext in FilePattern.UNSUPPORTED_FILE_EXTENSIONS
+ )
+
+ if matches_blocked:
+ message = f"Skipping '{file_name}' as it has an unsupported file format"
+ logger.debug(message)
+ self.execution_service.publish_log(message)
+ return False
+
+ return True
+
def _is_new_file(
self, file_path: str, file_content: bytes, workflow: Workflow
) -> bool: