LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Feat/percentage mrq (#578)
vishnuszipstack6055e91d8bef
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.pyindex 5a65cc779..21b2a21d0 100644--- a/backend/workflow_manager/endpoint/destination.py+++ b/backend/workflow_manager/endpoint/destination.py@@ -20,6 +20,7 @@ from workflow_manager.endpoint.constants import (WorkflowFileType,)from workflow_manager.endpoint.database_utils import DatabaseUtils+from workflow_manager.endpoint.dto import FileHashfrom workflow_manager.endpoint.exceptions import (DestinationConnectorNotConfigured,InvalidDestinationConnectionType,@@ -30,6 +31,7 @@ from workflow_manager.endpoint.exceptions import (from workflow_manager.endpoint.models import WorkflowEndpointfrom workflow_manager.endpoint.queue_utils import QueueResult, QueueUtilsfrom workflow_manager.workflow.enums import ExecutionStatus+from workflow_manager.workflow.execution import WorkflowExecutionServiceHelperfrom workflow_manager.workflow.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow.models.file_history import FileHistoryfrom workflow_manager.workflow.models.workflow import Workflow@@ -49,7 +51,12 @@ class DestinationConnector(BaseConnector):the destination connector."""- def __init__(self, workflow: Workflow, execution_id: str) -> None:+ def __init__(+ self,+ workflow: Workflow,+ execution_id: str,+ execution_service: Optional[WorkflowExecutionServiceHelper] = None,+ ) -> None:"""Initialize a DestinationConnector object.Args:@@ -62,6 +69,7 @@ class DestinationConnector(BaseConnector):self.execution_id = execution_idself.api_results: list[dict[str, Any]] = []self.queue_results: list[dict[str, Any]] = []+ self.execution_service = execution_servicedef _get_endpoint_for_workflow(self,@@ -123,12 +131,27 @@ class DestinationConnector(BaseConnector):):raise DestinationConnectorNotConfigured()+ def _push_data_to_queue(+ self,+ file_name: str,+ workflow: Workflow,+ input_file_path: str,+ ) -> None:+ result = self.get_result()+ meta_data = self.get_metadata()+ self._push_to_queue(+ file_name=file_name,+ workflow=workflow,+ result=result,+ input_file_path=input_file_path,+ meta_data=meta_data,+ )+def handle_output(self,file_name: str,- file_hash: str,+ file_hash: FileHash,workflow: Workflow,- file_history: Optional[FileHistory] = None,error: Optional[str] = None,input_file_path: Optional[str] = None,) -> None:@@ -140,12 +163,19 @@ class DestinationConnector(BaseConnector):if connection_type == WorkflowEndpoint.ConnectionType.API:self._handle_api_result(file_name=file_name, error=error, result=result)return+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=file_hash.file_hash+ )if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:self.copy_output_to_output_directory()elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:- self.insert_into_db(- file_history=file_history, input_file_path=input_file_path- )+ if (+ file_hash.file_destination+ == WorkflowEndpoint.ConnectionType.MANUALREVIEW+ ):+ self._push_data_to_queue(file_name, workflow, input_file_path)+ else:+ self.insert_into_db(input_file_path=input_file_path)elif connection_type == WorkflowEndpoint.ConnectionType.API:result = self.get_result(file_history)meta_data = self.get_metadata(file_history)@@ -153,24 +183,14 @@ class DestinationConnector(BaseConnector):file_name=file_name, error=error, result=result, meta_data=meta_data)elif connection_type == WorkflowEndpoint.ConnectionType.MANUALREVIEW:- if file_history:- logger.info(- f"Already processed result, skip adding to MRQ. "- f"WorkflowID{workflow.id}"- )- return- result = self.get_result(file_history)- meta_data = self.get_metadata(file_history)- self._push_to_queue(- file_name=file_name,- workflow=workflow,- result=result,- input_file_path=input_file_path,- meta_data=meta_data,+ self._push_data_to_queue(file_name, workflow, input_file_path)+ if self.execution_service:+ self.execution_service.publish_log(+ message=f"File {file_name} processed successfully")if not file_history:FileHistoryHelper.create_file_history(- cache_key=file_hash,+ cache_key=file_hash.file_hash,workflow=workflow,status=ExecutionStatus.COMPLETED,result=result,@@ -224,9 +244,7 @@ class DestinationConnector(BaseConnector):with open(source_path, "rb") as source_file:destination_fsspec.write_bytes(normalized_path, source_file.read())- def insert_into_db(- self, file_history: Optional[FileHistory], input_file_path: str- ) -> None:+ def insert_into_db(self, input_file_path: str) -> None:"""Insert data into the database."""connector_instance: ConnectorInstance = self.endpoint.connector_instanceconnector_settings: dict[str, Any] = connector_instance.metadata@@ -250,7 +268,7 @@ class DestinationConnector(BaseConnector):destination_configurations.get(DestinationKey.EXECUTION_ID, "execution_id"))- data = self.get_result(file_history)+ data = self.get_result()values = DatabaseUtils.get_columns_and_values(column_mode_str=column_mode,data=data,@@ -367,7 +385,7 @@ class DestinationConnector(BaseConnector):# assume it's a plain stringreturn original_string- def get_result(self, file_history: Optional[FileHistory]) -> Optional[Any]:+ def get_result(self, file_history: Optional[FileHistory] = None) -> Optional[Any]:"""Get result data from the output file.Returns:@@ -403,7 +421,7 @@ class DestinationConnector(BaseConnector):return resultdef get_metadata(- self, file_history: Optional[FileHistory]+ self, file_history: Optional[FileHistory] = None) -> Optional[dict[str, Any]]:"""Get meta_data from the output file.diff --git a/backend/workflow_manager/endpoint/dto.py b/backend/workflow_manager/endpoint/dto.pynew file mode 100644index 000000000..0e7aa6fd9--- /dev/null+++ b/backend/workflow_manager/endpoint/dto.py@@ -0,0 +1,36 @@+import json+from dataclasses import dataclass+from typing import Any, Optional+++@dataclass+class FileHash:+ file_path: str+ file_hash: str+ file_name: str+ source_connection_type: str+ file_destination: Optional[tuple[str, str]] = (+ None # To which destination this file wants to go for MRQ percentage+ )+ is_executed: bool = False++ def to_json(self) -> dict[str, Any]:+ return {+ "file_path": self.file_path,+ "file_hash": self.file_hash,+ "file_name": self.file_name,+ "source_connection_type": self.source_connection_type,+ "file_destination": self.file_destination,+ "is_executed": self.is_executed,+ }++ @staticmethod+ def from_json(json_str_or_dict: Any) -> "FileHash":+ """Deserialize a JSON string or dictionary to a FileHash instance."""+ if isinstance(json_str_or_dict, dict):+ # If already a dictionary, assume it's in the right format+ data = json_str_or_dict+ else:+ # Otherwise, assume it's a JSON string+ data = json.loads(json_str_or_dict)+ return FileHash(**data)diff --git a/backend/workflow_manager/endpoint/exceptions.py b/backend/workflow_manager/endpoint/exceptions.pyindex 7028eec28..2781b46aa 100644--- a/backend/workflow_manager/endpoint/exceptions.py+++ b/backend/workflow_manager/endpoint/exceptions.py@@ -41,6 +41,14 @@ class FileHashNotFound(APIException):default_detail = "Internal server error: File hash not found."+class FileHashMismatched(APIException):+ status_code = 400+ default_detail = (+ "The file's hash does not match the expected value. "+ "The file may have been altered."+ )++class ToolMetadataNotFound(APIException):status_code = 500default_detail = "Internal server error: Tool metadata not found."diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.pyindex d2707de0f..251b11568 100644--- a/backend/workflow_manager/endpoint/source.py+++ b/backend/workflow_manager/endpoint/source.py@@ -4,6 +4,7 @@ import osimport shutilfrom hashlib import md5, sha256from io import BytesIO+from itertools import islicefrom typing import Any, Optionalimport fsspec@@ -21,7 +22,9 @@ from workflow_manager.endpoint.constants import (SourceKey,WorkflowFileType,)+from workflow_manager.endpoint.dto import FileHashfrom workflow_manager.endpoint.exceptions import (+ FileHashMismatched,FileHashNotFound,InvalidInputDirectory,InvalidSourceConnectionType,@@ -31,6 +34,7 @@ from workflow_manager.endpoint.exceptions import ()from workflow_manager.endpoint.models import WorkflowEndpointfrom workflow_manager.workflow.execution import WorkflowExecutionServiceHelper+from workflow_manager.workflow.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow.models.workflow import Workflowlogger = logging.getLogger(__name__)@@ -119,25 +123,22 @@ class SourceConnector(BaseConnector):wildcard.extend(patterns.get(pattern, []))return wildcard- def list_file_from_api_storage(self) -> list[str]:+ def list_file_from_api_storage(+ self, file_hashes: dict[str, FileHash]+ ) -> tuple[dict[str, FileHash], int]:"""List all files from the api_storage_dir directory."""- files: list[str] = []- if not self.api_storage_dir:- return files- for file in os.listdir(self.api_storage_dir):- file_path = os.path.join(self.api_storage_dir, file)- if os.path.isfile(file_path):- files.append(file_path)- return files-- def list_files_from_file_connector(self) -> list[str]:+ return file_hashes, len(file_hashes)++ def list_files_from_file_connector(self) -> tuple[dict[str, FileHash], int]:"""_summary_Raises:InvalidDirectory: _description_Returns:- list[str]: _description_+ tuple[dict[str, FileHash], int]: A dictionary of matched file paths+ and their corresponding FileHash objects, along with the total count+ of matched files."""connector: ConnectorInstance = self.endpoint.connector_instanceconnector_settings: dict[str, Any] = connector.connector_metadata@@ -171,14 +172,14 @@ class SourceConnector(BaseConnector):is_directory = source_fs_fsspec.isdir(input_directory)if not is_directory:raise InvalidInputDirectory()- matched_files = self._get_matched_files(+ matched_files, count = self._get_matched_files(source_fs_fsspec, input_directory, patterns, recursive, limit)- self.publish_input_output_list_file_logs(input_directory, matched_files)- return matched_files+ self.publish_input_output_list_file_logs(input_directory, matched_files, count)+ return matched_files, countdef publish_input_output_list_file_logs(- self, input_directory: str, matched_files: list[str]+ self, input_directory: str, matched_files: dict[str, FileHash], count: int) -> None:if not self.execution_service:return None@@ -186,7 +187,7 @@ class SourceConnector(BaseConnector):self.execution_service.publish_update_log(state=LogState.INPUT_UPDATE, message=input_log)- output_log = self._matched_files_component_log(matched_files)+ output_log = self._matched_files_component_log(matched_files, count)self.execution_service.publish_update_log(state=LogState.OUTPUT_UPDATE, message=output_log)@@ -205,12 +206,14 @@ class SourceConnector(BaseConnector):state=LogState.OUTPUT_UPDATE, message=output_log_message)- def _matched_files_component_log(self, matched_files: list[str]) -> str:+ def _matched_files_component_log(+ self, matched_files: dict[str, FileHash], count: int+ ) -> str:output_log = "### Matched files \n```text\n\n\n"- for file in matched_files[:20]:- output_log += f"{file}\n"+ for file_path in islice(matched_files.keys(), 20):+ output_log += f"{file_path}\n"output_log += "```\n\n"- output_log += f"""Total matched files: {len(matched_files)}+ output_log += f"""Total matched files: {count}\n\nPlease note that only the first 20 files are shown.\n\n"""return output_log@@ -221,13 +224,13 @@ class SourceConnector(BaseConnector):patterns: list[str],recursive: bool,limit: int,- ) -> list[str]:- """Get a list of matched files based on patterns in a directory.+ ) -> tuple[dict[str, FileHash], int]:+ """Get a dictionary of matched files based on patterns in a directory.This method searches for files in the specified `input_directory` that- match any of the given `patterns`.- The search can be performed recursively if `recursive` is set to True.- The number of matched files returned is limited by `limit`.+ match any of the given `patterns`. The search can be performed recursively+ if `recursive` is set to True. The number of matched files returned is+ limited by `limit`.Args:source_fs (Any): The file system object used for searching.@@ -237,44 +240,81 @@ class SourceConnector(BaseConnector):limit (int): The maximum number of matched files to return.Returns:- list[str]: A list of matched file paths.+ tuple[dict[str, FileHash], int]: A dictionary of matched file paths+ and their corresponding FileHash objects, along with the total count+ of matched files."""- matched_files = []+ matched_files: dict[str, FileHash] = {}count = 0max_depth = int(SourceConstant.MAX_RECURSIVE_DEPTH) if recursive else 1for root, dirs, files in source_fs.walk(input_directory, maxdepth=max_depth):- if count >= limit:- breakfor file in files:- if not file:- continueif count >= limit:break- if any(- fnmatch.fnmatchcase(file.lower(), pattern.lower())- for pattern in patterns- ):- file_path = os.path.join(root, file)- file_path = f"{file_path}"- matched_files.append(file_path)- count += 1-- return matched_files-- def list_files_from_source(self) -> list[str]:+ if self._should_process_file(file, patterns):+ file_path = str(os.path.join(root, file))+ if self._is_new_file(+ file_path=file_path, workflow=self.endpoint.workflow+ ):+ matched_files[file_path] = self._create_file_hash(file_path)+ count += 1++ return matched_files, count++ def _should_process_file(self, file: str, patterns: list[str]) -> bool:+ """Check if the file should be processed based on the patterns."""+ return bool(file) and any(+ fnmatch.fnmatchcase(file.lower(), pattern.lower()) for pattern in patterns+ )++ def _is_new_file(self, file_path: str, workflow: Workflow) -> bool:+ """Check if the file is new or already processed."""+ file_content = self.get_file_content(input_file_path=file_path)+ file_hash = self.get_hash_value(file_content)+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=file_hash+ )++ if file_history and file_history.is_completed():+ self.execution_service.publish_log(+ f"Skipping file {file_path} as it has already been processed. "+ "Clear the cache to process it again."+ )+ return False++ return True++ def _create_file_hash(self, file_path: str) -> FileHash:+ """Create a FileHash object for the matched file."""+ file_name = os.path.basename(file_path)+ file_content = self.get_file_content(input_file_path=file_path)+ file_hash = self.get_hash_value(file_content)+ connection_type = self.endpoint.connection_type++ return FileHash(+ file_path=file_path,+ source_connection_type=connection_type,+ file_name=file_name,+ file_hash=file_hash,+ )++ def list_files_from_source(+ self, file_hashes: dict[str, FileHash] = {}+ ) -> tuple[dict[str, FileHash], int]:"""List files from source connector.Args:api_storage_dir (Optional[str], optional): API storage directoryReturns:- list[str]: list of files+ tuple[dict[str, FileHash], int]: A dictionary of FileHashes,+ along with the total count of matched files."""connection_type = self.endpoint.connection_typeif connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:return self.list_files_from_file_connector()elif connection_type == WorkflowEndpoint.ConnectionType.API:- return self.list_file_from_api_storage()+ return self.list_file_from_api_storage(file_hashes)raise InvalidSourceConnectionType()@classmethod@@ -303,44 +343,86 @@ class SourceConnector(BaseConnector):else:raise ValueError(f"Unsupported hash_method: {hash_method}")+ def get_file_content(self, input_file_path: str, chunk_size: int = 8192) -> bytes:+ """Read the content of a file from a remote filesystem in chunks.++ Args:+ input_file_path (str): The path of the input file.+ chunk_size (int): The size of the chunks to read at a time+ (default is 8192 bytes).++ Returns:+ bytes: The content of the file.+ """+ connector: ConnectorInstance = self.endpoint.connector_instance+ connector_settings: dict[str, Any] = connector.connector_metadata+ source_fs = self.get_fsspec(+ settings=connector_settings, connector_id=connector.connector_id+ )+ file_content = bytearray() # Use bytearray for efficient byte concatenation+ with source_fs.open(input_file_path, "rb") as remote_file:+ while chunk := remote_file.read(chunk_size):+ file_content.extend(chunk)++ return bytes(file_content)++ def get_hash_value(self, file_content: bytes) -> str:+ """Generate a hash value from the file content.++ Args:+ file_content (bytes): The content of the file.++ Returns:+ str: The hash value of the file content.+ """+ return self.hash_str(file_content)++ def copy_file_to_infile_dir(self, source_file_path: str, infile_path: str) -> None:+ """Copy the source file to the infile directory.++ Args:+ source_file_path (str): The path of the source file.+ infile_path (str): The destination path in the infile directory.+ """+ shutil.copyfile(source_file_path, infile_path)+ logger.info(f"File copied from {source_file_path} to {infile_path}")+def add_input_from_connector_to_volume(self, input_file_path: str) -> str:"""Add input file to execution directory.Args:input_file_path (str): The path of the input file.+Returns:str: The hash value of the file content.+Raises:- FileHashNotFound: If the hash value of the file content- is not found.+ FileHashNotFound: If the hash value of the file content is not found."""- connector: ConnectorInstance = self.endpoint.connector_instance- connector_settings: dict[str, Any] = connector.connector_metadatasource_file_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)source_file = f"file://{source_file_path}"- source_fs = self.get_fsspec(- settings=connector_settings, connector_id=connector.connector_id+ # Get file content and hash value+ file_content = self.get_file_content(input_file_path)+ hash_value_of_file_content = self.get_hash_value(file_content)++ logger.info(+ f"hash_value_of_file {source_file} is : {hash_value_of_file_content}")- with (- source_fs.open(input_file_path, "rb") as remote_file,- fsspec.open(source_file, "wb") as local_file,- ):- file_content = remote_file.read()- hash_value_of_file_content = self.hash_str(file_content)- logger.info(- f"hash_value_of_file {source_file} is "- f": {hash_value_of_file_content}"- )- input_log = (- file_content[:500].decode("utf-8", errors="replace") + "...(truncated)"- )- self.publish_input_file_content(input_file_path, input_log)+ input_log = (+ file_content[:500].decode("utf-8", errors="replace") + "...(truncated)"+ )+ self.publish_input_file_content(input_file_path, input_log)++ with fsspec.open(source_file, "wb") as local_file:local_file.write(file_content)- shutil.copyfile(source_file_path, infile_path)- logger.info(f"{input_file_path} is added in to execution directory")++ # Copy file to infile directory+ self.copy_file_to_infile_dir(source_file_path, infile_path)++ logger.info(f"{input_file_path} is added to execution directory")return hash_value_of_file_contentdef add_input_from_api_storage_to_volume(self, input_file_path: str) -> None:@@ -351,7 +433,7 @@ class SourceConnector(BaseConnector):shutil.copyfile(input_file_path, source_path)def add_file_to_volume(- self, input_file_path: str, hash_values_of_files: dict[str, str] = {}+ self, input_file_path: str, file_hash: FileHash) -> tuple[str, str]:"""Add input file to execution directory.@@ -370,18 +452,20 @@ class SourceConnector(BaseConnector):file_content_hash = self.add_input_from_connector_to_volume(input_file_path=input_file_path,)+ if file_content_hash != file_hash.file_hash:+ raise FileHashMismatched()elif connection_type == WorkflowEndpoint.ConnectionType.API:self.add_input_from_api_storage_to_volume(input_file_path=input_file_path)- if file_name not in hash_values_of_files:+ if file_name != file_hash.file_name:raise FileHashNotFound()- file_content_hash = hash_values_of_files[file_name]+ file_content_hash = file_hash.file_hashelse:raise InvalidSourceConnectionType()self.add_metadata_to_volume(input_file_path=input_file_path, source_hash=file_content_hash)- return file_name, file_content_hash+ return file_namedef handle_final_result(self,@@ -419,18 +503,21 @@ class SourceConnector(BaseConnector):@classmethoddef add_input_file_to_api_storage(cls, workflow_id: str, execution_id: str, file_objs: list[UploadedFile]- ) -> dict[str, str]:+ ) -> dict[str, FileHash]:"""Add input file to api storage.Args:workflow_id (str): workflow idexecution_id (str): execution_idfile_objs (list[UploadedFile]): api file objects+ returns:+ dict[str, FileHash]: file hashes"""api_storage_dir = cls.get_api_storage_dir_path(workflow_id=workflow_id, execution_id=execution_id)- file_hashes: dict[str, str] = {}+ workflow: Workflow = Workflow.objects.get(id=workflow_id)+ file_hashes: dict[str, FileHash] = {}for file in file_objs:file_name = file.namedestination_path = os.path.join(api_storage_dir, file_name)@@ -440,7 +527,22 @@ class SourceConnector(BaseConnector):for chunk in file.chunks():buffer.extend(chunk)f.write(buffer)- file_hashes.update({file_name: cls.hash_str(buffer)})+ file_hash = cls.hash_str(buffer)+ connection_type = WorkflowEndpoint.ConnectionType.API+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=file_hash+ )+ is_executed = (+ True if file_history and file_history.is_completed() else False+ )+ file_hash = FileHash(+ file_path=destination_path,+ source_connection_type=connection_type,+ file_name=file_name,+ file_hash=file_hash,+ is_executed=is_executed,+ )+ file_hashes.update({file_name: file_hash})return file_hashes@classmethoddiff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.pyindex c9efb4690..1f126469d 100644--- a/backend/workflow_manager/workflow/execution.py+++ b/backend/workflow_manager/workflow/execution.py@@ -21,7 +21,6 @@ from workflow_manager.workflow.enums import ExecutionStatusfrom workflow_manager.workflow.exceptions import WorkflowExecutionErrorfrom workflow_manager.workflow.models import Workflow, WorkflowExecutionfrom workflow_manager.workflow.models.execution import EXECUTION_ERROR_LENGTH-from workflow_manager.workflow.models.file_history import FileHistorylogger = logging.getLogger(__name__)@@ -289,7 +288,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):self,file_name: str,single_step: bool,- file_history: Optional[FileHistory] = None,) -> tuple[bool, bool]:"""Executes the input file.@@ -297,29 +295,16 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):file_name (str): The name of the file to be executed.single_step (bool): Flag indicating whether to execute insingle step mode.- file_history (Optional[FileHistory], optional):- The file history object. Defaults to None.Returns:tuple[bool, bool]: Flag indicating whether the file was executedand skipped."""execution_type = ExecutionType.COMPLETE- is_executed = False- is_skipped = Falseif single_step:execution_type = ExecutionType.STEP- if not (file_history and file_history.is_completed()):- self.execute_uncached_input(file_name=file_name, single_step=single_step)- self.publish_log(f"Tool executed successfully for {file_name}")- is_executed = True- else:- self.publish_log(- f"Skipping file {file_name} as it is already processed."- "Clear the cache to process it again"- )- is_skipped = True+ self.execute_uncached_input(file_name=file_name, single_step=single_step)+ self.publish_log(f"Tool executed successfully for {file_name}")self._handle_execution_type(execution_type)- return is_executed, is_skippeddef execute_uncached_input(self, file_name: str, single_step: bool) -> None:"""Executes the uncached input file.diff --git a/backend/workflow_manager/workflow/views.py b/backend/workflow_manager/workflow/views.pyindex fc6c691c4..7c775a1a8 100644--- a/backend/workflow_manager/workflow/views.py+++ b/backend/workflow_manager/workflow/views.py@@ -17,6 +17,7 @@ from tool_instance.tool_processor import ToolProcessorfrom unstract.tool_registry.dto import Toolfrom utils.filtering import FilterHelperfrom workflow_manager.endpoint.destination import DestinationConnector+from workflow_manager.endpoint.dto import FileHashfrom workflow_manager.endpoint.endpoint_utils import WorkflowEndpointUtilsfrom workflow_manager.endpoint.source import SourceConnectorfrom workflow_manager.workflow.constants import WorkflowKey@@ -173,7 +174,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):include_metadata = (request.data.get("include_metadata", "false").lower() == "true")- hashes_of_files = {}+ hashes_of_files: dict[str, FileHash] = {}if file_objs and execution_id and workflow_id:hashes_of_files = SourceConnector.add_input_file_to_api_storage(workflow_id=workflow_id,@@ -217,7 +218,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_action: Optional[str] = None,execution_id: Optional[str] = None,pipeline_guid: Optional[str] = None,- hash_values_of_files: dict[str, str] = {},+ hash_values_of_files: dict[str, FileHash] = {},include_metadata: bool = False,) -> ExecutionResponse:if execution_action is not None:diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.pyindex 9f35d4fee..2ea03366a 100644--- a/backend/workflow_manager/workflow/workflow_helper.py+++ b/backend/workflow_manager/workflow/workflow_helper.py@@ -25,6 +25,7 @@ from unstract.workflow_execution.exceptions import StopExecutionfrom utils.cache_service import CacheServicefrom utils.local_context import StateStorefrom workflow_manager.endpoint.destination import DestinationConnector+from workflow_manager.endpoint.dto import FileHashfrom workflow_manager.endpoint.source import SourceConnectorfrom workflow_manager.workflow.constants import (CeleryConfigurations,@@ -44,6 +45,7 @@ from workflow_manager.workflow.execution import WorkflowExecutionServiceHelperfrom workflow_manager.workflow.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow.models.execution import WorkflowExecutionfrom workflow_manager.workflow.models.workflow import Workflow+from workflow_manager.workflow.utils import WorkflowUtillogger = logging.getLogger(__name__)@@ -112,10 +114,9 @@ class WorkflowHelper:destination: DestinationConnector,execution_service: WorkflowExecutionServiceHelper,single_step: bool,- hash_values_of_files: dict[str, str] = {},+ hash_values_of_files: dict[str, FileHash] = {},) -> WorkflowExecution:- input_files = source.list_files_from_source()- total_files = len(input_files)+ input_files, total_files = source.list_files_from_source(hash_values_of_files)error_message = Noneprocessed_files = 0error_raised = 0@@ -123,19 +124,27 @@ class WorkflowHelper:execution_service.update_execution(ExecutionStatus.EXECUTING, increment_attempt=True)- for index, input_file in enumerate(input_files):+ if total_files > 0:+ q_file_no_list = WorkflowUtil.get_q_no_list(workflow, total_files)++ for index, (file_path, file_hash) in enumerate(input_files.items()):file_number = index + 1+ file_hash = WorkflowUtil.add_file_destination_filehash(+ file_number,+ q_file_no_list,+ file_hash,+ )try:is_executed, error = WorkflowHelper.process_file(current_file_idx=file_number,total_files=total_files,- input_file=input_file,+ input_file=file_hash.file_path,workflow=workflow,source=source,destination=destination,execution_service=execution_service,single_step=single_step,- hash_values_of_files=hash_values_of_files,+ file_hash=file_hash,)if is_executed:processed_files += 1@@ -149,6 +158,8 @@ class WorkflowHelper:except Exception as error:error_message = str(error)error_raised += 1+ log_message = f"Error processing file {file_path}: {error_message}"+ execution_service.publish_log(message=log_message, level=LogLevel.ERROR)if error_raised and error_raised >= total_files:execution_service.update_execution(ExecutionStatus.ERROR, error=error_message@@ -171,27 +182,22 @@ class WorkflowHelper:destination: DestinationConnector,execution_service: WorkflowExecutionServiceHelper,single_step: bool,- hash_values_of_files: dict[str, str],+ file_hash: FileHash,) -> tuple[bool, Optional[str]]:- file_history = Noneerror = Noneis_executed = False- file_name, file_hash = source.add_file_to_volume(- input_file_path=input_file,- hash_values_of_files=hash_values_of_files,+ file_name = source.add_file_to_volume(+ input_file_path=input_file, file_hash=file_hash)try:execution_service.initiate_tool_execution(current_file_idx, total_files, file_name, single_step)- file_history = FileHistoryHelper.get_file_history(- workflow=workflow, cache_key=file_hash- )- is_executed = execution_service.execute_input_file(- file_name=file_name,- single_step=single_step,- file_history=file_history,- )+ if not file_hash.is_executed:+ execution_service.execute_input_file(+ file_name=file_name,+ single_step=single_step,+ )except StopExecution:raiseexcept Exception as e:@@ -209,7 +215,6 @@ class WorkflowHelper:file_name=file_name,file_hash=file_hash,workflow=workflow,- file_history=file_history,error=error,input_file_path=input_file,)@@ -234,7 +239,7 @@ class WorkflowHelper:@staticmethoddef run_workflow(workflow: Workflow,- hash_values_of_files: dict[str, str] = {},+ hash_values_of_files: dict[str, FileHash] = {},organization_id: Optional[str] = None,pipeline_id: Optional[str] = None,scheduled: bool = False,@@ -269,7 +274,11 @@ class WorkflowHelper:execution_id=execution_id,execution_service=execution_service,)- destination = DestinationConnector(workflow=workflow, execution_id=execution_id)+ destination = DestinationConnector(+ workflow=workflow,+ execution_id=execution_id,+ execution_service=execution_service,+ )# Validating endpointssource.validate()destination.validate()@@ -378,7 +387,7 @@ class WorkflowHelper:def execute_workflow_async(workflow_id: str,execution_id: str,- hash_values_of_files: dict[str, str],+ hash_values_of_files: dict[str, FileHash],timeout: int = -1,pipeline_id: Optional[str] = None,include_metadata: bool = False,@@ -396,12 +405,15 @@ class WorkflowHelper:ExecutionResponse: Existing status of execution"""try:+ file_hash_in_str = {+ key: value.to_json() for key, value in hash_values_of_files.items()+ }org_schema = connection.tenant.schema_namelog_events_id = StateStore.get(Common.LOG_EVENTS_ID)async_execution = WorkflowHelper.execute_bin.delay(org_schema,workflow_id,- hash_values_of_files=hash_values_of_files,+ hash_values_of_files=file_hash_in_str,execution_id=execution_id,pipeline_id=pipeline_id,log_events_id=log_events_id,@@ -459,7 +471,7 @@ class WorkflowHelper:schema_name: str,workflow_id: str,execution_id: str,- hash_values_of_files: dict[str, str],+ hash_values_of_files: dict[str, dict[str, Any]],scheduled: bool = False,execution_mode: Optional[tuple[str, str]] = None,pipeline_id: Optional[str] = None,@@ -486,6 +498,10 @@ class WorkflowHelper:Returns:dict[str, list[Any]]: Returns a dict with result from workflow execution"""+ hash_values = {+ key: FileHash.from_json(value)+ for key, value in hash_values_of_files.items()+ }task_id = current_task.request.idtenant: Organization = (get_tenant_model().objects.filter(schema_name=schema_name).first()@@ -516,7 +532,7 @@ class WorkflowHelper:scheduled=scheduled,workflow_execution=workflow_execution,execution_mode=execution_mode,- hash_values_of_files=hash_values_of_files,+ hash_values_of_files=hash_values,include_metadata=include_metadata,).resultreturn result@@ -526,7 +542,7 @@ class WorkflowHelper:workflow: Workflow,execution_id: Optional[str] = None,pipeline_id: Optional[str] = None,- hash_values_of_files: dict[str, str] = {},+ hash_values_of_files: dict[str, FileHash] = {},include_metadata: bool = False,) -> ExecutionResponse:if pipeline_id:@@ -593,7 +609,7 @@ class WorkflowHelper:workflow: Workflow,execution_action: str,execution_id: Optional[str] = None,- hash_values_of_files: dict[str, str] = {},+ hash_values_of_files: dict[str, FileHash] = {},include_metadata: bool = False,) -> ExecutionResponse:if execution_action is Workflow.ExecutionAction.START.value: # type: ignorediff --git a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsxindex 00f925625..b5026d09a 100644--- a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx+++ b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx@@ -8,6 +8,14 @@ import { ConfigureFormsLayout } from "../configure-forms-layout/ConfigureFormsLaimport { ManageFiles } from "../../input-output/manage-files/ManageFiles";import usePostHogEvents from "../../../hooks/usePostHogEvents";+let DBRules;++try {+ DBRules =+ require("../../../plugins/manual-review/db-rules/DBRules.jsx").DBRules;+} catch {+ // The component will remain null of it is not available+}function ConfigureConnectorModal({open,setOpen,@@ -29,7 +37,7 @@ function ConfigureConnectorModal({}) {const [activeKey, setActiveKey] = useState("1");useEffect(() => {- if (connectorMetadata) {+ if (connectorMetadata && connType === "FILESYSTEM") {setActiveKey("2"); // If connector is already configured} else {setActiveKey("1"); // default value@@ -37,10 +45,12 @@ function ConfigureConnectorModal({}, [open, connectorMetadata]);const { setPostHogCustomEvent, posthogConnectorEventText } =usePostHogEvents();- const tabItems = [++ const [tabItems, setTabItems] = useState([{key: "1",label: "Settings",+ visible: true,},{key: "2",@@ -48,11 +58,64 @@ function ConfigureConnectorModal({disabled:!connectorId ||connDetails?.connector_id !== selectedId ||- connType === "DATABASE" ||- connType === "MANUALREVIEW",+ connType === "DATABASE",+ visible: false,},- ];+ ]);+ const setUpdatedTabOptions = (tabOption) => {+ setTabItems((prevTabOptions) => {+ // Check if inputOption already exists in prevTabOptions+ if (prevTabOptions.some((opt) => opt.key === tabOption.key)) {+ return prevTabOptions; // Return previous state unchanged+ } else {+ // Create a new array with the existing options and the new option+ const updatedTabOptions = [...prevTabOptions, tabOption];+ return updatedTabOptions;+ }+ });+ };+ useEffect(() => {+ const updatedTabItems = tabItems.map((item) => {+ if (item.key === "2") {+ item.visible = connType === "FILESYSTEM";+ } else if (item.key === "MANUALREVIEW") {+ item.disabled =+ !connectorId || connDetails?.connector_id !== selectedId;+ item.visible = connType === "DATABASE" || connType === "MANUALREVIEW";+ } else {+ item.visible = true;+ }+ return item;+ });+ setTabItems(updatedTabItems);+ }, [open]);++ useEffect(() => {+ const updatedTabItems = tabItems.map((item) => {+ if (item.key === "MANUALREVIEW") {+ item.disabled =+ !connectorId || connDetails?.connector_id !== selectedId;+ }+ return item;+ });+ setTabItems(updatedTabItems);+ }, [connectorMetadata]);++ useEffect(() => {+ try {+ const tabOption =+ require("../../../plugins/manual-review/connector-config-tab-mrq/ConnectorConfigTabMRQ").mrqTabs;+ if (tabOption) {+ tabOption["disabled"] =+ !connectorId || connDetails?.connector_id !== selectedId;+ tabOption["visible"] = false;+ setUpdatedTabOptions(tabOption);+ }+ } catch {+ // The component will remain null of it is not available+ }+ }, []);const handleSelectItem = (e) => {const id = e.key;setSelectedId(id?.toString());@@ -103,7 +166,7 @@ function ConfigureConnectorModal({<Col span={20} className="conn-modal-col conn-modal-form-pad-left"><TabsactiveKey={activeKey}- items={tabItems}+ items={tabItems.filter((item) => item?.visible !== false)}onChange={onTabChange}moreIcon={<></>}/>@@ -124,7 +187,12 @@ function ConfigureConnectorModal({selectedItemName={selectedItemName}/>)}- {activeKey === "2" && <ManageFiles selectedItem={connectorId} />}+ {activeKey === "2" && connType === "FILESYSTEM" && (+ <ManageFiles selectedItem={connectorId} />+ )}+ {activeKey === "MANUALREVIEW" && (+ <DBRules connDetails={connDetails} />+ )}</Col></Row></div>