LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Added and validate for unsupported extesnions while list files from source (#1152)
ali3c2068d3ed64
diff --git a/backend/workflow_manager/endpoint_v2/constants.py b/backend/workflow_manager/endpoint_v2/constants.pyindex 9a89763f6..75f5434cd 100644--- a/backend/workflow_manager/endpoint_v2/constants.py+++ b/backend/workflow_manager/endpoint_v2/constants.py@@ -81,6 +81,21 @@ class FilePattern:"*.tif","*.tiff",]+ UNSUPPORTED_FILE_EXTENSIONS = [+ # Archive formats+ "*.zip",+ "*.tar",+ "*.gz",+ "*.tar.gz",+ "*.xz",+ "*.tar.xz",+ "*.7z",+ "*.rar",+ "*.bz2",+ "*.tar.bz2",+ "*.tgz",+ "*.tbz2",+ ]class SourceConstant:diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex a03932146..123e09759 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -330,11 +330,50 @@ class SourceConnector(BaseConnector):return matched_files, countdef _should_process_file(self, file: str, patterns: list[str]) -> bool:- """Check if the file should be processed based on the patterns."""- return bool(file) and any(- fnmatch.fnmatchcase(file.lower(), pattern.lower()) for pattern in patterns+ """+ Check if the file should be processed based on the patterns.++ Args:+ file: The filename to check+ patterns: List of patterns to match against++ Returns:+ bool: True if file should be processed, False otherwise+ """+ if not file:+ return False++ file_lower = file.lower()+ matches_pattern = any(+ fnmatch.fnmatchcase(file_lower, pattern.lower()) for pattern in patterns)+ return matches_pattern and self._is_valid_pattern(file)++ def _is_valid_pattern(self, file_name: str) -> bool:+ """+ Check if the file has a supported format.++ Args:+ file_name: The filename to check++ Returns:+ bool: True if file format is supported, False otherwise+ """+ file_lower = file_name.lower()+ matches_blocked = any(+ fnmatch.fnmatchcase(file_lower, ext.lower())+ for ext in FilePattern.UNSUPPORTED_FILE_EXTENSIONS+ )++ if matches_blocked:+ message = f"Skipping '{file_name}' as it has an unsupported file format"+ logger.debug(message)+ self.execution_service.publish_log(message)+ return False++ return True+def _is_new_file(self, file_path: str, file_content: bytes, workflow: Workflow) -> bool: