LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
feat: Changes to accept multiple folders for file source (#603)
Chandrasekharan Mccad74834cd0
diff --git a/backend/workflow_manager/endpoint/constants.py b/backend/workflow_manager/endpoint/constants.pyindex 428d45ce5..6a23900a0 100644--- a/backend/workflow_manager/endpoint/constants.py+++ b/backend/workflow_manager/endpoint/constants.py@@ -44,7 +44,7 @@ class SourceKey:FILE_EXTENSIONS = "fileExtensions"PROCESS_SUB_DIRECTORIES = "processSubDirectories"MAX_FILES = "maxFiles"- ROOT_FOLDER = "rootFolder"+ FOLDERS = "folders"class DestinationKey:diff --git a/backend/workflow_manager/endpoint/exceptions.py b/backend/workflow_manager/endpoint/exceptions.pyindex 2781b46aa..9a1b03488 100644--- a/backend/workflow_manager/endpoint/exceptions.py+++ b/backend/workflow_manager/endpoint/exceptions.py@@ -1,9 +1,21 @@+from typing import Optional+from rest_framework.exceptions import APIExceptionclass InvalidInputDirectory(APIException):status_code = 400- default_detail = "The provided directory is invalid."+ default_detail = "The provided path is not a valid directory."++ def __init__(+ self,+ dir: Optional[str] = None,+ detail: Optional[str] = None,+ code: Optional[str] = None,+ ):+ if dir:+ detail = self.default_detail.replace("path", f"path '{dir}'")+ super().__init__(detail, code)class InvalidSourceConnectionType(APIException):diff --git a/backend/workflow_manager/endpoint/migrations/0004_allow_multi_source_inputs_for_fs.py b/backend/workflow_manager/endpoint/migrations/0004_allow_multi_source_inputs_for_fs.pynew file mode 100644index 000000000..53c47f62c--- /dev/null+++ b/backend/workflow_manager/endpoint/migrations/0004_allow_multi_source_inputs_for_fs.py@@ -0,0 +1,93 @@+# Generated by Django 4.2.1 on 2024-08-21 02:56++from typing import Any, Optional++from django.db import migrations+++class Migration(migrations.Migration):++ dependencies = [+ ("endpoint", "0003_alter_workflowendpoint_connection_type"),+ ]++ def migrate_to_support_multiple_folder_input(apps: Any, schema_editor: Any) -> None:+ """Migrates the data for multiple folders as source input.++ Converts a string field `rootFolder` in `configuration` JSON to+ accept a list of values. Empty strings are converted into '/' to+ handle reads from the root of the file system.++ Parameters:+ apps (Any): The registry of installed applications.+ schema_editor (Any): The schema editor for the database operation.++ Returns:+ None: This method does not return anything.+ """+ WorkflowEndpoint = apps.get_model("endpoint", "WorkflowEndpoint")++ endpoints_to_update = WorkflowEndpoint.objects.filter(endpoint_type="SOURCE")+ for endpoint in endpoints_to_update:+ configuration: Optional[dict[str, Any]] = endpoint.configuration+ # Skip in case of partially initialized sources+ if not configuration:+ continue+ root_folder = configuration.get("rootFolder", "")++ # Convert rootFolder to folders list+ if not root_folder:+ folders = ["/"]+ else:+ folders = [root_folder]++ # Update the configuration JSON+ configuration["folders"] = folders++ # Save the updated configuration back to the model+ endpoint.configuration = configuration+ endpoint.save()++ def migrate_to_support_single_folder_input(apps: Any, schema_editor: Any) -> None:+ """Reverses the migration by converting folders list back to a string.++ Takes the first element of the `folders` list and reassigns it to+ `rootFolder`. If the list is empty, assigns an empty string to `rootFolder`.++ Parameters:+ apps (Any): The registry of installed applications.+ schema_editor (Any): The schema editor for the database operation.++ Returns:+ None: This method does not return anything.+ """+ WorkflowEndpoint = apps.get_model("endpoint", "WorkflowEndpoint")++ endpoints_to_update = WorkflowEndpoint.objects.filter(endpoint_type="SOURCE")++ for endpoint in endpoints_to_update:+ configuration: Optional[dict[str, Any]] = endpoint.configuration+ # Skip in case of partially initialized sources+ if not configuration:+ continue+ folders = configuration.get("folders", [])++ # Convert the folders list back to a single rootFolder string+ if folders:+ root_folder = folders[0]+ else:+ root_folder = ""++ # Update the configuration JSON+ configuration["rootFolder"] = root_folder++ # Save the updated configuration back to the model+ endpoint.configuration = configuration+ endpoint.save()++ operations = [+ migrations.RunPython(+ migrate_to_support_multiple_folder_input,+ reverse_code=migrate_to_support_single_folder_input,+ ),+ ]diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.pyindex 251b11568..c71a3c26b 100644--- a/backend/workflow_manager/endpoint/source.py+++ b/backend/workflow_manager/endpoint/source.py@@ -143,7 +143,9 @@ class SourceConnector(BaseConnector):connector: ConnectorInstance = self.endpoint.connector_instanceconnector_settings: dict[str, Any] = connector.connector_metadatasource_configurations: dict[str, Any] = self.endpoint.configuration- required_patterns = source_configurations.get(SourceKey.FILE_EXTENSIONS, [])+ required_patterns = list(+ source_configurations.get(SourceKey.FILE_EXTENSIONS, [])+ )recursive = bool(source_configurations.get(SourceKey.PROCESS_SUB_DIRECTORIES, False))@@ -153,37 +155,56 @@ class SourceConnector(BaseConnector):))root_dir_path = connector_settings.get(ConnectorKeys.PATH, "")-- input_directory = str(source_configurations.get(SourceKey.ROOT_FOLDER, ""))+ folders_to_process = list(source_configurations.get(SourceKey.FOLDERS, ["/"]))+ # Process from root in case its user provided list is empty+ if not folders_to_process:+ folders_to_process = ["/"]+ logger.info(f"Folders to process: {folders_to_process}")source_fs = self.get_fs_connector(settings=connector_settings, connector_id=connector.connector_id)- input_directory = source_fs.get_connector_root_dir(- input_dir=input_directory, root_path=root_dir_path- )- logger.debug(f"source input directory {input_directory}")- if not isinstance(required_patterns, list):- required_patterns = [required_patterns]-source_fs_fsspec = source_fs.get_fsspec_fs()-+ # Checking if folders exist at source before processing+ # TODO: Validate while receiving this input configuration as well+ for input_directory in folders_to_process:+ # TODO: Move to connector class for better error handling+ try:+ is_directory = source_fs_fsspec.isdir(input_directory)+ except Exception as e:+ raise InvalidInputDirectory(+ detail=f"Error while validating path '{input_directory}': {str(e)}"+ )+ if not is_directory:+ raise InvalidInputDirectory(dir=input_directory)++ total_files_to_process = 0+ total_matched_files = {}patterns = self.valid_file_patterns(required_patterns=required_patterns)- is_directory = source_fs_fsspec.isdir(input_directory)- if not is_directory:- raise InvalidInputDirectory()- matched_files, count = self._get_matched_files(- source_fs_fsspec, input_directory, patterns, recursive, limit+ for input_directory in folders_to_process:+ input_directory = source_fs.get_connector_root_dir(+ input_dir=input_directory, root_path=root_dir_path+ )+ logger.debug(f"Listing files from: {input_directory}")+ matched_files, count = self._get_matched_files(+ source_fs_fsspec, input_directory, patterns, recursive, limit+ )+ logger.info(f"Matched '{count}' files from '{input_directory}'")+ total_matched_files.update(matched_files)+ total_files_to_process += count+ self.publish_input_output_list_file_logs(+ folders_to_process, total_matched_files, total_files_to_process)- self.publish_input_output_list_file_logs(input_directory, matched_files, count)- return matched_files, count+ return total_matched_files, total_files_to_processdef publish_input_output_list_file_logs(- self, input_directory: str, matched_files: dict[str, FileHash], count: int+ self, folders: list[str], matched_files: dict[str, FileHash], count: int) -> None:if not self.execution_service:return None- input_log = f"##Input folder:\n\n `{os.path.basename(input_directory)}`\n\n"++ folders_list = "\n".join(f"- `{folder.strip()}`" for folder in folders)+ input_log = f"##Folders to process:\n\n{folders_list}\n\n"self.execution_service.publish_update_log(state=LogState.INPUT_UPDATE, message=input_log)diff --git a/backend/workflow_manager/endpoint/static/src/file.json b/backend/workflow_manager/endpoint/static/src/file.jsonindex d603b7e53..23bf3dc00 100644--- a/backend/workflow_manager/endpoint/static/src/file.json+++ b/backend/workflow_manager/endpoint/static/src/file.json@@ -4,17 +4,21 @@"type": "object","required": [],"properties": {- "rootFolder": {- "type": "string",- "title": "Folder to process",- "default": "",- "description": "The root folder to start processing files from. Leave it empty to use the root folder"+ "folders": {+ "type": "array",+ "title": "Folders to process",+ "description": "Folders in the connected filesystem to start processing files from. Leave it empty or pass '/' to specify the root folder. Separate inputs with ⏎ Enter.",+ "items": {+ "type": "string",+ "title": "Folder Path"+ },+ "uniqueItems": true},"processSubDirectories": {"type": "boolean",- "title": "Process sub directories",+ "title": "Process sub-folders","default": true,- "description": "Process sub directories recursively"+ "description": "Process sub folders recursively"},"fileExtensions": {"type": "array",diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.pyindex 90db646e4..05cfd9007 100644--- a/backend/workflow_manager/workflow/execution.py+++ b/backend/workflow_manager/workflow/execution.py@@ -3,10 +3,8 @@ import timefrom typing import Optionalfrom account.constants import Common-from api.exceptions import InvalidAPIRequestfrom django.db import connectionfrom platform_settings.platform_auth_service import PlatformAuthenticationService-from tool_instance.constants import JsonSchemaKeyfrom tool_instance.models import ToolInstancefrom tool_instance.tool_processor import ToolProcessorfrom unstract.tool_registry.dto import Tool@@ -105,13 +103,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):self.compilation_result = self.compile_workflow(execution_id=self.execution_id)- def _initiate_api_execution(- self, tool_instance: ToolInstance, execution_path: Optional[str]- ) -> None:- if not execution_path:- raise InvalidAPIRequest("File shouldn't be empty")- tool_instance.metadata[JsonSchemaKey.ROOT_FOLDER] = execution_path-@staticmethoddef create_workflow_execution(workflow_id: str,diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.pyindex ccc257331..f21bb3a37 100644--- a/backend/workflow_manager/workflow/workflow_helper.py+++ b/backend/workflow_manager/workflow/workflow_helper.py@@ -154,10 +154,11 @@ class WorkflowHelper:)breakexcept Exception as error:- error_message = str(error)error_raised += 1- log_message = f"Error processing file {file_path}: {error_message}"- execution_service.publish_log(message=log_message, level=LogLevel.ERROR)+ error_message = f"Error processing file {file_path}. {error}"+ execution_service.publish_log(+ message=error_message, level=LogLevel.ERROR+ )if error_raised and error_raised >= total_files:execution_service.update_execution(ExecutionStatus.ERROR, error=error_message@@ -199,11 +200,8 @@ class WorkflowHelper:except StopExecution:raiseexcept Exception as e:- execution_service.publish_log(- f"Error processing file {input_file}: {str(e)}",- level=LogLevel.ERROR,- )- error = str(e)+ error = f"Error processing file {input_file}: {str(e)}"+ execution_service.publish_log(error, level=LogLevel.ERROR)execution_service.publish_update_log(LogState.RUNNING,f"Processing output for {file_name}",@@ -302,6 +300,7 @@ class WorkflowHelper:)except Exception as e:logger.error(f"Error executing workflow {workflow}: {e}")+ logger.error(f"Error {traceback.format_exc()}")WorkflowHelper._update_pipeline_status(pipeline_id=pipeline_id, workflow_execution=workflow_execution)@@ -310,7 +309,7 @@ class WorkflowHelper:str(workflow_execution.id),workflow_execution.status,log_id=str(execution_service.execution_log_id),- error=workflow_execution.error_message,+ error=workflow_execution.error_message or str(e),mode=workflow_execution.execution_mode,)finally:diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.pyindex fd64cba50..2f07a2219 100644--- a/backend/workflow_manager/workflow_v2/execution.py+++ b/backend/workflow_manager/workflow_v2/execution.py@@ -3,9 +3,7 @@ import timefrom typing import Optionalfrom account_v2.constants import Common-from api_v2.exceptions import InvalidAPIRequestfrom platform_settings_v2.platform_auth_service import PlatformAuthenticationService-from tool_instance_v2.constants import JsonSchemaKeyfrom tool_instance_v2.models import ToolInstancefrom tool_instance_v2.tool_processor import ToolProcessorfrom unstract.tool_registry.dto import Tool@@ -108,13 +106,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):self.compilation_result = self.compile_workflow(execution_id=self.execution_id)- def _initiate_api_execution(- self, tool_instance: ToolInstance, execution_path: Optional[str]- ) -> None:- if not execution_path:- raise InvalidAPIRequest("File shouldn't be empty")- tool_instance.metadata[JsonSchemaKey.ROOT_FOLDER] = execution_path-@staticmethoddef create_workflow_execution(workflow_id: str,diff --git a/tools/structure/src/constants.py b/tools/structure/src/constants.pyindex 20a8edfd1..ae5dfb133 100644--- a/tools/structure/src/constants.py+++ b/tools/structure/src/constants.py@@ -1,5 +1,4 @@class SettingsKeys:- ROOT_FOLDER = "rootFolder"TOOL_INSTANCE_ID = "tool_instance_id"PROMPT_REGISTRY_ID = "prompt_registry_id"PROMPT_HOST = "PROMPT_HOST"