Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

feat: Changes to accept multiple folders for file source (#603)

Chandrasekharan Mccad74834cd0
diff --git a/backend/workflow_manager/endpoint/constants.py b/backend/workflow_manager/endpoint/constants.py
index 428d45ce5..6a23900a0 100644
--- a/backend/workflow_manager/endpoint/constants.py
+++ b/backend/workflow_manager/endpoint/constants.py
@@ -44,7 +44,7 @@ class SourceKey:
FILE_EXTENSIONS = "fileExtensions"
PROCESS_SUB_DIRECTORIES = "processSubDirectories"
MAX_FILES = "maxFiles"
- ROOT_FOLDER = "rootFolder"
+ FOLDERS = "folders"
class DestinationKey:
diff --git a/backend/workflow_manager/endpoint/exceptions.py b/backend/workflow_manager/endpoint/exceptions.py
index 2781b46aa..9a1b03488 100644
--- a/backend/workflow_manager/endpoint/exceptions.py
+++ b/backend/workflow_manager/endpoint/exceptions.py
@@ -1,9 +1,21 @@
+from typing import Optional
+
from rest_framework.exceptions import APIException
class InvalidInputDirectory(APIException):
status_code = 400
- default_detail = "The provided directory is invalid."
+ default_detail = "The provided path is not a valid directory."
+
+ def __init__(
+ self,
+ dir: Optional[str] = None,
+ detail: Optional[str] = None,
+ code: Optional[str] = None,
+ ):
+ if dir:
+ detail = self.default_detail.replace("path", f"path '{dir}'")
+ super().__init__(detail, code)
class InvalidSourceConnectionType(APIException):
diff --git a/backend/workflow_manager/endpoint/migrations/0004_allow_multi_source_inputs_for_fs.py b/backend/workflow_manager/endpoint/migrations/0004_allow_multi_source_inputs_for_fs.py
new file mode 100644
index 000000000..53c47f62c
--- /dev/null
+++ b/backend/workflow_manager/endpoint/migrations/0004_allow_multi_source_inputs_for_fs.py
@@ -0,0 +1,93 @@
+# Generated by Django 4.2.1 on 2024-08-21 02:56
+
+from typing import Any, Optional
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("endpoint", "0003_alter_workflowendpoint_connection_type"),
+ ]
+
+ def migrate_to_support_multiple_folder_input(apps: Any, schema_editor: Any) -> None:
+ """Migrates the data for multiple folders as source input.
+
+ Converts a string field `rootFolder` in `configuration` JSON to
+ accept a list of values. Empty strings are converted into '/' to
+ handle reads from the root of the file system.
+
+ Parameters:
+ apps (Any): The registry of installed applications.
+ schema_editor (Any): The schema editor for the database operation.
+
+ Returns:
+ None: This method does not return anything.
+ """
+ WorkflowEndpoint = apps.get_model("endpoint", "WorkflowEndpoint")
+
+ endpoints_to_update = WorkflowEndpoint.objects.filter(endpoint_type="SOURCE")
+ for endpoint in endpoints_to_update:
+ configuration: Optional[dict[str, Any]] = endpoint.configuration
+ # Skip in case of partially initialized sources
+ if not configuration:
+ continue
+ root_folder = configuration.get("rootFolder", "")
+
+ # Convert rootFolder to folders list
+ if not root_folder:
+ folders = ["/"]
+ else:
+ folders = [root_folder]
+
+ # Update the configuration JSON
+ configuration["folders"] = folders
+
+ # Save the updated configuration back to the model
+ endpoint.configuration = configuration
+ endpoint.save()
+
+ def migrate_to_support_single_folder_input(apps: Any, schema_editor: Any) -> None:
+ """Reverses the migration by converting folders list back to a string.
+
+ Takes the first element of the `folders` list and reassigns it to
+ `rootFolder`. If the list is empty, assigns an empty string to `rootFolder`.
+
+ Parameters:
+ apps (Any): The registry of installed applications.
+ schema_editor (Any): The schema editor for the database operation.
+
+ Returns:
+ None: This method does not return anything.
+ """
+ WorkflowEndpoint = apps.get_model("endpoint", "WorkflowEndpoint")
+
+ endpoints_to_update = WorkflowEndpoint.objects.filter(endpoint_type="SOURCE")
+
+ for endpoint in endpoints_to_update:
+ configuration: Optional[dict[str, Any]] = endpoint.configuration
+ # Skip in case of partially initialized sources
+ if not configuration:
+ continue
+ folders = configuration.get("folders", [])
+
+ # Convert the folders list back to a single rootFolder string
+ if folders:
+ root_folder = folders[0]
+ else:
+ root_folder = ""
+
+ # Update the configuration JSON
+ configuration["rootFolder"] = root_folder
+
+ # Save the updated configuration back to the model
+ endpoint.configuration = configuration
+ endpoint.save()
+
+ operations = [
+ migrations.RunPython(
+ migrate_to_support_multiple_folder_input,
+ reverse_code=migrate_to_support_single_folder_input,
+ ),
+ ]
diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.py
index 251b11568..c71a3c26b 100644
--- a/backend/workflow_manager/endpoint/source.py
+++ b/backend/workflow_manager/endpoint/source.py
@@ -143,7 +143,9 @@ class SourceConnector(BaseConnector):
connector: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector.connector_metadata
source_configurations: dict[str, Any] = self.endpoint.configuration
- required_patterns = source_configurations.get(SourceKey.FILE_EXTENSIONS, [])
+ required_patterns = list(
+ source_configurations.get(SourceKey.FILE_EXTENSIONS, [])
+ )
recursive = bool(
source_configurations.get(SourceKey.PROCESS_SUB_DIRECTORIES, False)
)
@@ -153,37 +155,56 @@ class SourceConnector(BaseConnector):
)
)
root_dir_path = connector_settings.get(ConnectorKeys.PATH, "")
-
- input_directory = str(source_configurations.get(SourceKey.ROOT_FOLDER, ""))
+ folders_to_process = list(source_configurations.get(SourceKey.FOLDERS, ["/"]))
+ # Process from root in case its user provided list is empty
+ if not folders_to_process:
+ folders_to_process = ["/"]
+ logger.info(f"Folders to process: {folders_to_process}")
source_fs = self.get_fs_connector(
settings=connector_settings, connector_id=connector.connector_id
)
- input_directory = source_fs.get_connector_root_dir(
- input_dir=input_directory, root_path=root_dir_path
- )
- logger.debug(f"source input directory {input_directory}")
- if not isinstance(required_patterns, list):
- required_patterns = [required_patterns]
-
source_fs_fsspec = source_fs.get_fsspec_fs()
-
+ # Checking if folders exist at source before processing
+ # TODO: Validate while receiving this input configuration as well
+ for input_directory in folders_to_process:
+ # TODO: Move to connector class for better error handling
+ try:
+ is_directory = source_fs_fsspec.isdir(input_directory)
+ except Exception as e:
+ raise InvalidInputDirectory(
+ detail=f"Error while validating path '{input_directory}': {str(e)}"
+ )
+ if not is_directory:
+ raise InvalidInputDirectory(dir=input_directory)
+
+ total_files_to_process = 0
+ total_matched_files = {}
patterns = self.valid_file_patterns(required_patterns=required_patterns)
- is_directory = source_fs_fsspec.isdir(input_directory)
- if not is_directory:
- raise InvalidInputDirectory()
- matched_files, count = self._get_matched_files(
- source_fs_fsspec, input_directory, patterns, recursive, limit
+ for input_directory in folders_to_process:
+ input_directory = source_fs.get_connector_root_dir(
+ input_dir=input_directory, root_path=root_dir_path
+ )
+ logger.debug(f"Listing files from: {input_directory}")
+ matched_files, count = self._get_matched_files(
+ source_fs_fsspec, input_directory, patterns, recursive, limit
+ )
+ logger.info(f"Matched '{count}' files from '{input_directory}'")
+ total_matched_files.update(matched_files)
+ total_files_to_process += count
+ self.publish_input_output_list_file_logs(
+ folders_to_process, total_matched_files, total_files_to_process
)
- self.publish_input_output_list_file_logs(input_directory, matched_files, count)
- return matched_files, count
+ return total_matched_files, total_files_to_process
def publish_input_output_list_file_logs(
- self, input_directory: str, matched_files: dict[str, FileHash], count: int
+ self, folders: list[str], matched_files: dict[str, FileHash], count: int
) -> None:
if not self.execution_service:
return None
- input_log = f"##Input folder:\n\n `{os.path.basename(input_directory)}`\n\n"
+
+ folders_list = "\n".join(f"- `{folder.strip()}`" for folder in folders)
+ input_log = f"##Folders to process:\n\n{folders_list}\n\n"
self.execution_service.publish_update_log(
state=LogState.INPUT_UPDATE, message=input_log
)
diff --git a/backend/workflow_manager/endpoint/static/src/file.json b/backend/workflow_manager/endpoint/static/src/file.json
index d603b7e53..23bf3dc00 100644
--- a/backend/workflow_manager/endpoint/static/src/file.json
+++ b/backend/workflow_manager/endpoint/static/src/file.json
@@ -4,17 +4,21 @@
"type": "object",
"required": [],
"properties": {
- "rootFolder": {
- "type": "string",
- "title": "Folder to process",
- "default": "",
- "description": "The root folder to start processing files from. Leave it empty to use the root folder"
+ "folders": {
+ "type": "array",
+ "title": "Folders to process",
+ "description": "Folders in the connected filesystem to start processing files from. Leave it empty or pass '/' to specify the root folder. Separate inputs with ⏎ Enter.",
+ "items": {
+ "type": "string",
+ "title": "Folder Path"
+ },
+ "uniqueItems": true
},
"processSubDirectories": {
"type": "boolean",
- "title": "Process sub directories",
+ "title": "Process sub-folders",
"default": true,
- "description": "Process sub directories recursively"
+ "description": "Process sub folders recursively"
},
"fileExtensions": {
"type": "array",
diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.py
index 90db646e4..05cfd9007 100644
--- a/backend/workflow_manager/workflow/execution.py
+++ b/backend/workflow_manager/workflow/execution.py
@@ -3,10 +3,8 @@ import time
from typing import Optional
from account.constants import Common
-from api.exceptions import InvalidAPIRequest
from django.db import connection
from platform_settings.platform_auth_service import PlatformAuthenticationService
-from tool_instance.constants import JsonSchemaKey
from tool_instance.models import ToolInstance
from tool_instance.tool_processor import ToolProcessor
from unstract.tool_registry.dto import Tool
@@ -105,13 +103,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
self.compilation_result = self.compile_workflow(execution_id=self.execution_id)
- def _initiate_api_execution(
- self, tool_instance: ToolInstance, execution_path: Optional[str]
- ) -> None:
- if not execution_path:
- raise InvalidAPIRequest("File shouldn't be empty")
- tool_instance.metadata[JsonSchemaKey.ROOT_FOLDER] = execution_path
-
@staticmethod
def create_workflow_execution(
workflow_id: str,
diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.py
index ccc257331..f21bb3a37 100644
--- a/backend/workflow_manager/workflow/workflow_helper.py
+++ b/backend/workflow_manager/workflow/workflow_helper.py
@@ -154,10 +154,11 @@ class WorkflowHelper:
)
break
except Exception as error:
- error_message = str(error)
error_raised += 1
- log_message = f"Error processing file {file_path}: {error_message}"
- execution_service.publish_log(message=log_message, level=LogLevel.ERROR)
+ error_message = f"Error processing file {file_path}. {error}"
+ execution_service.publish_log(
+ message=error_message, level=LogLevel.ERROR
+ )
if error_raised and error_raised >= total_files:
execution_service.update_execution(
ExecutionStatus.ERROR, error=error_message
@@ -199,11 +200,8 @@ class WorkflowHelper:
except StopExecution:
raise
except Exception as e:
- execution_service.publish_log(
- f"Error processing file {input_file}: {str(e)}",
- level=LogLevel.ERROR,
- )
- error = str(e)
+ error = f"Error processing file {input_file}: {str(e)}"
+ execution_service.publish_log(error, level=LogLevel.ERROR)
execution_service.publish_update_log(
LogState.RUNNING,
f"Processing output for {file_name}",
@@ -302,6 +300,7 @@ class WorkflowHelper:
)
except Exception as e:
logger.error(f"Error executing workflow {workflow}: {e}")
+ logger.error(f"Error {traceback.format_exc()}")
WorkflowHelper._update_pipeline_status(
pipeline_id=pipeline_id, workflow_execution=workflow_execution
)
@@ -310,7 +309,7 @@ class WorkflowHelper:
str(workflow_execution.id),
workflow_execution.status,
log_id=str(execution_service.execution_log_id),
- error=workflow_execution.error_message,
+ error=workflow_execution.error_message or str(e),
mode=workflow_execution.execution_mode,
)
finally:
diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.py
index fd64cba50..2f07a2219 100644
--- a/backend/workflow_manager/workflow_v2/execution.py
+++ b/backend/workflow_manager/workflow_v2/execution.py
@@ -3,9 +3,7 @@ import time
from typing import Optional
from account_v2.constants import Common
-from api_v2.exceptions import InvalidAPIRequest
from platform_settings_v2.platform_auth_service import PlatformAuthenticationService
-from tool_instance_v2.constants import JsonSchemaKey
from tool_instance_v2.models import ToolInstance
from tool_instance_v2.tool_processor import ToolProcessor
from unstract.tool_registry.dto import Tool
@@ -108,13 +106,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
self.compilation_result = self.compile_workflow(execution_id=self.execution_id)
- def _initiate_api_execution(
- self, tool_instance: ToolInstance, execution_path: Optional[str]
- ) -> None:
- if not execution_path:
- raise InvalidAPIRequest("File shouldn't be empty")
- tool_instance.metadata[JsonSchemaKey.ROOT_FOLDER] = execution_path
-
@staticmethod
def create_workflow_execution(
workflow_id: str,
diff --git a/tools/structure/src/constants.py b/tools/structure/src/constants.py
index 20a8edfd1..ae5dfb133 100644
--- a/tools/structure/src/constants.py
+++ b/tools/structure/src/constants.py
@@ -1,5 +1,4 @@
class SettingsKeys:
- ROOT_FOLDER = "rootFolder"
TOOL_INSTANCE_ID = "tool_instance_id"
PROMPT_REGISTRY_ID = "prompt_registry_id"
PROMPT_HOST = "PROMPT_HOST"