Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Feat/percentage mrq (#578)

vishnuszipstack6055e91d8bef
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.py
index 5a65cc779..21b2a21d0 100644
--- a/backend/workflow_manager/endpoint/destination.py
+++ b/backend/workflow_manager/endpoint/destination.py
@@ -20,6 +20,7 @@ from workflow_manager.endpoint.constants import (
WorkflowFileType,
)
from workflow_manager.endpoint.database_utils import DatabaseUtils
+from workflow_manager.endpoint.dto import FileHash
from workflow_manager.endpoint.exceptions import (
DestinationConnectorNotConfigured,
InvalidDestinationConnectionType,
@@ -30,6 +31,7 @@ from workflow_manager.endpoint.exceptions import (
from workflow_manager.endpoint.models import WorkflowEndpoint
from workflow_manager.endpoint.queue_utils import QueueResult, QueueUtils
from workflow_manager.workflow.enums import ExecutionStatus
+from workflow_manager.workflow.execution import WorkflowExecutionServiceHelper
from workflow_manager.workflow.file_history_helper import FileHistoryHelper
from workflow_manager.workflow.models.file_history import FileHistory
from workflow_manager.workflow.models.workflow import Workflow
@@ -49,7 +51,12 @@ class DestinationConnector(BaseConnector):
the destination connector.
"""
- def __init__(self, workflow: Workflow, execution_id: str) -> None:
+ def __init__(
+ self,
+ workflow: Workflow,
+ execution_id: str,
+ execution_service: Optional[WorkflowExecutionServiceHelper] = None,
+ ) -> None:
"""Initialize a DestinationConnector object.
Args:
@@ -62,6 +69,7 @@ class DestinationConnector(BaseConnector):
self.execution_id = execution_id
self.api_results: list[dict[str, Any]] = []
self.queue_results: list[dict[str, Any]] = []
+ self.execution_service = execution_service
def _get_endpoint_for_workflow(
self,
@@ -123,12 +131,27 @@ class DestinationConnector(BaseConnector):
):
raise DestinationConnectorNotConfigured()
+ def _push_data_to_queue(
+ self,
+ file_name: str,
+ workflow: Workflow,
+ input_file_path: str,
+ ) -> None:
+ result = self.get_result()
+ meta_data = self.get_metadata()
+ self._push_to_queue(
+ file_name=file_name,
+ workflow=workflow,
+ result=result,
+ input_file_path=input_file_path,
+ meta_data=meta_data,
+ )
+
def handle_output(
self,
file_name: str,
- file_hash: str,
+ file_hash: FileHash,
workflow: Workflow,
- file_history: Optional[FileHistory] = None,
error: Optional[str] = None,
input_file_path: Optional[str] = None,
) -> None:
@@ -140,12 +163,19 @@ class DestinationConnector(BaseConnector):
if connection_type == WorkflowEndpoint.ConnectionType.API:
self._handle_api_result(file_name=file_name, error=error, result=result)
return
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=file_hash.file_hash
+ )
if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
self.copy_output_to_output_directory()
elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:
- self.insert_into_db(
- file_history=file_history, input_file_path=input_file_path
- )
+ if (
+ file_hash.file_destination
+ == WorkflowEndpoint.ConnectionType.MANUALREVIEW
+ ):
+ self._push_data_to_queue(file_name, workflow, input_file_path)
+ else:
+ self.insert_into_db(input_file_path=input_file_path)
elif connection_type == WorkflowEndpoint.ConnectionType.API:
result = self.get_result(file_history)
meta_data = self.get_metadata(file_history)
@@ -153,24 +183,14 @@ class DestinationConnector(BaseConnector):
file_name=file_name, error=error, result=result, meta_data=meta_data
)
elif connection_type == WorkflowEndpoint.ConnectionType.MANUALREVIEW:
- if file_history:
- logger.info(
- f"Already processed result, skip adding to MRQ. "
- f"WorkflowID{workflow.id}"
- )
- return
- result = self.get_result(file_history)
- meta_data = self.get_metadata(file_history)
- self._push_to_queue(
- file_name=file_name,
- workflow=workflow,
- result=result,
- input_file_path=input_file_path,
- meta_data=meta_data,
+ self._push_data_to_queue(file_name, workflow, input_file_path)
+ if self.execution_service:
+ self.execution_service.publish_log(
+ message=f"File {file_name} processed successfully"
)
if not file_history:
FileHistoryHelper.create_file_history(
- cache_key=file_hash,
+ cache_key=file_hash.file_hash,
workflow=workflow,
status=ExecutionStatus.COMPLETED,
result=result,
@@ -224,9 +244,7 @@ class DestinationConnector(BaseConnector):
with open(source_path, "rb") as source_file:
destination_fsspec.write_bytes(normalized_path, source_file.read())
- def insert_into_db(
- self, file_history: Optional[FileHistory], input_file_path: str
- ) -> None:
+ def insert_into_db(self, input_file_path: str) -> None:
"""Insert data into the database."""
connector_instance: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector_instance.metadata
@@ -250,7 +268,7 @@ class DestinationConnector(BaseConnector):
destination_configurations.get(DestinationKey.EXECUTION_ID, "execution_id")
)
- data = self.get_result(file_history)
+ data = self.get_result()
values = DatabaseUtils.get_columns_and_values(
column_mode_str=column_mode,
data=data,
@@ -367,7 +385,7 @@ class DestinationConnector(BaseConnector):
# assume it's a plain string
return original_string
- def get_result(self, file_history: Optional[FileHistory]) -> Optional[Any]:
+ def get_result(self, file_history: Optional[FileHistory] = None) -> Optional[Any]:
"""Get result data from the output file.
Returns:
@@ -403,7 +421,7 @@ class DestinationConnector(BaseConnector):
return result
def get_metadata(
- self, file_history: Optional[FileHistory]
+ self, file_history: Optional[FileHistory] = None
) -> Optional[dict[str, Any]]:
"""Get meta_data from the output file.
diff --git a/backend/workflow_manager/endpoint/dto.py b/backend/workflow_manager/endpoint/dto.py
new file mode 100644
index 000000000..0e7aa6fd9
--- /dev/null
+++ b/backend/workflow_manager/endpoint/dto.py
@@ -0,0 +1,36 @@
+import json
+from dataclasses import dataclass
+from typing import Any, Optional
+
+
+@dataclass
+class FileHash:
+ file_path: str
+ file_hash: str
+ file_name: str
+ source_connection_type: str
+ file_destination: Optional[tuple[str, str]] = (
+ None # To which destination this file wants to go for MRQ percentage
+ )
+ is_executed: bool = False
+
+ def to_json(self) -> dict[str, Any]:
+ return {
+ "file_path": self.file_path,
+ "file_hash": self.file_hash,
+ "file_name": self.file_name,
+ "source_connection_type": self.source_connection_type,
+ "file_destination": self.file_destination,
+ "is_executed": self.is_executed,
+ }
+
+ @staticmethod
+ def from_json(json_str_or_dict: Any) -> "FileHash":
+ """Deserialize a JSON string or dictionary to a FileHash instance."""
+ if isinstance(json_str_or_dict, dict):
+ # If already a dictionary, assume it's in the right format
+ data = json_str_or_dict
+ else:
+ # Otherwise, assume it's a JSON string
+ data = json.loads(json_str_or_dict)
+ return FileHash(**data)
diff --git a/backend/workflow_manager/endpoint/exceptions.py b/backend/workflow_manager/endpoint/exceptions.py
index 7028eec28..2781b46aa 100644
--- a/backend/workflow_manager/endpoint/exceptions.py
+++ b/backend/workflow_manager/endpoint/exceptions.py
@@ -41,6 +41,14 @@ class FileHashNotFound(APIException):
default_detail = "Internal server error: File hash not found."
+class FileHashMismatched(APIException):
+ status_code = 400
+ default_detail = (
+ "The file's hash does not match the expected value. "
+ "The file may have been altered."
+ )
+
+
class ToolMetadataNotFound(APIException):
status_code = 500
default_detail = "Internal server error: Tool metadata not found."
diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.py
index d2707de0f..251b11568 100644
--- a/backend/workflow_manager/endpoint/source.py
+++ b/backend/workflow_manager/endpoint/source.py
@@ -4,6 +4,7 @@ import os
import shutil
from hashlib import md5, sha256
from io import BytesIO
+from itertools import islice
from typing import Any, Optional
import fsspec
@@ -21,7 +22,9 @@ from workflow_manager.endpoint.constants import (
SourceKey,
WorkflowFileType,
)
+from workflow_manager.endpoint.dto import FileHash
from workflow_manager.endpoint.exceptions import (
+ FileHashMismatched,
FileHashNotFound,
InvalidInputDirectory,
InvalidSourceConnectionType,
@@ -31,6 +34,7 @@ from workflow_manager.endpoint.exceptions import (
)
from workflow_manager.endpoint.models import WorkflowEndpoint
from workflow_manager.workflow.execution import WorkflowExecutionServiceHelper
+from workflow_manager.workflow.file_history_helper import FileHistoryHelper
from workflow_manager.workflow.models.workflow import Workflow
logger = logging.getLogger(__name__)
@@ -119,25 +123,22 @@ class SourceConnector(BaseConnector):
wildcard.extend(patterns.get(pattern, []))
return wildcard
- def list_file_from_api_storage(self) -> list[str]:
+ def list_file_from_api_storage(
+ self, file_hashes: dict[str, FileHash]
+ ) -> tuple[dict[str, FileHash], int]:
"""List all files from the api_storage_dir directory."""
- files: list[str] = []
- if not self.api_storage_dir:
- return files
- for file in os.listdir(self.api_storage_dir):
- file_path = os.path.join(self.api_storage_dir, file)
- if os.path.isfile(file_path):
- files.append(file_path)
- return files
-
- def list_files_from_file_connector(self) -> list[str]:
+ return file_hashes, len(file_hashes)
+
+ def list_files_from_file_connector(self) -> tuple[dict[str, FileHash], int]:
"""_summary_
Raises:
InvalidDirectory: _description_
Returns:
- list[str]: _description_
+ tuple[dict[str, FileHash], int]: A dictionary of matched file paths
+ and their corresponding FileHash objects, along with the total count
+ of matched files.
"""
connector: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector.connector_metadata
@@ -171,14 +172,14 @@ class SourceConnector(BaseConnector):
is_directory = source_fs_fsspec.isdir(input_directory)
if not is_directory:
raise InvalidInputDirectory()
- matched_files = self._get_matched_files(
+ matched_files, count = self._get_matched_files(
source_fs_fsspec, input_directory, patterns, recursive, limit
)
- self.publish_input_output_list_file_logs(input_directory, matched_files)
- return matched_files
+ self.publish_input_output_list_file_logs(input_directory, matched_files, count)
+ return matched_files, count
def publish_input_output_list_file_logs(
- self, input_directory: str, matched_files: list[str]
+ self, input_directory: str, matched_files: dict[str, FileHash], count: int
) -> None:
if not self.execution_service:
return None
@@ -186,7 +187,7 @@ class SourceConnector(BaseConnector):
self.execution_service.publish_update_log(
state=LogState.INPUT_UPDATE, message=input_log
)
- output_log = self._matched_files_component_log(matched_files)
+ output_log = self._matched_files_component_log(matched_files, count)
self.execution_service.publish_update_log(
state=LogState.OUTPUT_UPDATE, message=output_log
)
@@ -205,12 +206,14 @@ class SourceConnector(BaseConnector):
state=LogState.OUTPUT_UPDATE, message=output_log_message
)
- def _matched_files_component_log(self, matched_files: list[str]) -> str:
+ def _matched_files_component_log(
+ self, matched_files: dict[str, FileHash], count: int
+ ) -> str:
output_log = "### Matched files \n```text\n\n\n"
- for file in matched_files[:20]:
- output_log += f"{file}\n"
+ for file_path in islice(matched_files.keys(), 20):
+ output_log += f"{file_path}\n"
output_log += "```\n\n"
- output_log += f"""Total matched files: {len(matched_files)}
+ output_log += f"""Total matched files: {count}
\n\nPlease note that only the first 20 files are shown.\n\n"""
return output_log
@@ -221,13 +224,13 @@ class SourceConnector(BaseConnector):
patterns: list[str],
recursive: bool,
limit: int,
- ) -> list[str]:
- """Get a list of matched files based on patterns in a directory.
+ ) -> tuple[dict[str, FileHash], int]:
+ """Get a dictionary of matched files based on patterns in a directory.
This method searches for files in the specified `input_directory` that
- match any of the given `patterns`.
- The search can be performed recursively if `recursive` is set to True.
- The number of matched files returned is limited by `limit`.
+ match any of the given `patterns`. The search can be performed recursively
+ if `recursive` is set to True. The number of matched files returned is
+ limited by `limit`.
Args:
source_fs (Any): The file system object used for searching.
@@ -237,44 +240,81 @@ class SourceConnector(BaseConnector):
limit (int): The maximum number of matched files to return.
Returns:
- list[str]: A list of matched file paths.
+ tuple[dict[str, FileHash], int]: A dictionary of matched file paths
+ and their corresponding FileHash objects, along with the total count
+ of matched files.
"""
- matched_files = []
+ matched_files: dict[str, FileHash] = {}
count = 0
max_depth = int(SourceConstant.MAX_RECURSIVE_DEPTH) if recursive else 1
for root, dirs, files in source_fs.walk(input_directory, maxdepth=max_depth):
- if count >= limit:
- break
for file in files:
- if not file:
- continue
if count >= limit:
break
- if any(
- fnmatch.fnmatchcase(file.lower(), pattern.lower())
- for pattern in patterns
- ):
- file_path = os.path.join(root, file)
- file_path = f"{file_path}"
- matched_files.append(file_path)
- count += 1
-
- return matched_files
-
- def list_files_from_source(self) -> list[str]:
+ if self._should_process_file(file, patterns):
+ file_path = str(os.path.join(root, file))
+ if self._is_new_file(
+ file_path=file_path, workflow=self.endpoint.workflow
+ ):
+ matched_files[file_path] = self._create_file_hash(file_path)
+ count += 1
+
+ return matched_files, count
+
+ def _should_process_file(self, file: str, patterns: list[str]) -> bool:
+ """Check if the file should be processed based on the patterns."""
+ return bool(file) and any(
+ fnmatch.fnmatchcase(file.lower(), pattern.lower()) for pattern in patterns
+ )
+
+ def _is_new_file(self, file_path: str, workflow: Workflow) -> bool:
+ """Check if the file is new or already processed."""
+ file_content = self.get_file_content(input_file_path=file_path)
+ file_hash = self.get_hash_value(file_content)
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=file_hash
+ )
+
+ if file_history and file_history.is_completed():
+ self.execution_service.publish_log(
+ f"Skipping file {file_path} as it has already been processed. "
+ "Clear the cache to process it again."
+ )
+ return False
+
+ return True
+
+ def _create_file_hash(self, file_path: str) -> FileHash:
+ """Create a FileHash object for the matched file."""
+ file_name = os.path.basename(file_path)
+ file_content = self.get_file_content(input_file_path=file_path)
+ file_hash = self.get_hash_value(file_content)
+ connection_type = self.endpoint.connection_type
+
+ return FileHash(
+ file_path=file_path,
+ source_connection_type=connection_type,
+ file_name=file_name,
+ file_hash=file_hash,
+ )
+
+ def list_files_from_source(
+ self, file_hashes: dict[str, FileHash] = {}
+ ) -> tuple[dict[str, FileHash], int]:
"""List files from source connector.
Args:
api_storage_dir (Optional[str], optional): API storage directory
Returns:
- list[str]: list of files
+ tuple[dict[str, FileHash], int]: A dictionary of FileHashes,
+ along with the total count of matched files.
"""
connection_type = self.endpoint.connection_type
if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
return self.list_files_from_file_connector()
elif connection_type == WorkflowEndpoint.ConnectionType.API:
- return self.list_file_from_api_storage()
+ return self.list_file_from_api_storage(file_hashes)
raise InvalidSourceConnectionType()
@classmethod
@@ -303,44 +343,86 @@ class SourceConnector(BaseConnector):
else:
raise ValueError(f"Unsupported hash_method: {hash_method}")
+ def get_file_content(self, input_file_path: str, chunk_size: int = 8192) -> bytes:
+ """Read the content of a file from a remote filesystem in chunks.
+
+ Args:
+ input_file_path (str): The path of the input file.
+ chunk_size (int): The size of the chunks to read at a time
+ (default is 8192 bytes).
+
+ Returns:
+ bytes: The content of the file.
+ """
+ connector: ConnectorInstance = self.endpoint.connector_instance
+ connector_settings: dict[str, Any] = connector.connector_metadata
+ source_fs = self.get_fsspec(
+ settings=connector_settings, connector_id=connector.connector_id
+ )
+ file_content = bytearray() # Use bytearray for efficient byte concatenation
+ with source_fs.open(input_file_path, "rb") as remote_file:
+ while chunk := remote_file.read(chunk_size):
+ file_content.extend(chunk)
+
+ return bytes(file_content)
+
+ def get_hash_value(self, file_content: bytes) -> str:
+ """Generate a hash value from the file content.
+
+ Args:
+ file_content (bytes): The content of the file.
+
+ Returns:
+ str: The hash value of the file content.
+ """
+ return self.hash_str(file_content)
+
+ def copy_file_to_infile_dir(self, source_file_path: str, infile_path: str) -> None:
+ """Copy the source file to the infile directory.
+
+ Args:
+ source_file_path (str): The path of the source file.
+ infile_path (str): The destination path in the infile directory.
+ """
+ shutil.copyfile(source_file_path, infile_path)
+ logger.info(f"File copied from {source_file_path} to {infile_path}")
+
def add_input_from_connector_to_volume(self, input_file_path: str) -> str:
"""Add input file to execution directory.
Args:
input_file_path (str): The path of the input file.
+
Returns:
str: The hash value of the file content.
+
Raises:
- FileHashNotFound: If the hash value of the file content
- is not found.
+ FileHashNotFound: If the hash value of the file content is not found.
"""
- connector: ConnectorInstance = self.endpoint.connector_instance
- connector_settings: dict[str, Any] = connector.connector_metadata
source_file_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)
infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
source_file = f"file://{source_file_path}"
- source_fs = self.get_fsspec(
- settings=connector_settings, connector_id=connector.connector_id
+ # Get file content and hash value
+ file_content = self.get_file_content(input_file_path)
+ hash_value_of_file_content = self.get_hash_value(file_content)
+
+ logger.info(
+ f"hash_value_of_file {source_file} is : {hash_value_of_file_content}"
)
- with (
- source_fs.open(input_file_path, "rb") as remote_file,
- fsspec.open(source_file, "wb") as local_file,
- ):
- file_content = remote_file.read()
- hash_value_of_file_content = self.hash_str(file_content)
- logger.info(
- f"hash_value_of_file {source_file} is "
- f": {hash_value_of_file_content}"
- )
- input_log = (
- file_content[:500].decode("utf-8", errors="replace") + "...(truncated)"
- )
- self.publish_input_file_content(input_file_path, input_log)
+ input_log = (
+ file_content[:500].decode("utf-8", errors="replace") + "...(truncated)"
+ )
+ self.publish_input_file_content(input_file_path, input_log)
+
+ with fsspec.open(source_file, "wb") as local_file:
local_file.write(file_content)
- shutil.copyfile(source_file_path, infile_path)
- logger.info(f"{input_file_path} is added in to execution directory")
+
+ # Copy file to infile directory
+ self.copy_file_to_infile_dir(source_file_path, infile_path)
+
+ logger.info(f"{input_file_path} is added to execution directory")
return hash_value_of_file_content
def add_input_from_api_storage_to_volume(self, input_file_path: str) -> None:
@@ -351,7 +433,7 @@ class SourceConnector(BaseConnector):
shutil.copyfile(input_file_path, source_path)
def add_file_to_volume(
- self, input_file_path: str, hash_values_of_files: dict[str, str] = {}
+ self, input_file_path: str, file_hash: FileHash
) -> tuple[str, str]:
"""Add input file to execution directory.
@@ -370,18 +452,20 @@ class SourceConnector(BaseConnector):
file_content_hash = self.add_input_from_connector_to_volume(
input_file_path=input_file_path,
)
+ if file_content_hash != file_hash.file_hash:
+ raise FileHashMismatched()
elif connection_type == WorkflowEndpoint.ConnectionType.API:
self.add_input_from_api_storage_to_volume(input_file_path=input_file_path)
- if file_name not in hash_values_of_files:
+ if file_name != file_hash.file_name:
raise FileHashNotFound()
- file_content_hash = hash_values_of_files[file_name]
+ file_content_hash = file_hash.file_hash
else:
raise InvalidSourceConnectionType()
self.add_metadata_to_volume(
input_file_path=input_file_path, source_hash=file_content_hash
)
- return file_name, file_content_hash
+ return file_name
def handle_final_result(
self,
@@ -419,18 +503,21 @@ class SourceConnector(BaseConnector):
@classmethod
def add_input_file_to_api_storage(
cls, workflow_id: str, execution_id: str, file_objs: list[UploadedFile]
- ) -> dict[str, str]:
+ ) -> dict[str, FileHash]:
"""Add input file to api storage.
Args:
workflow_id (str): workflow id
execution_id (str): execution_id
file_objs (list[UploadedFile]): api file objects
+ returns:
+ dict[str, FileHash]: file hashes
"""
api_storage_dir = cls.get_api_storage_dir_path(
workflow_id=workflow_id, execution_id=execution_id
)
- file_hashes: dict[str, str] = {}
+ workflow: Workflow = Workflow.objects.get(id=workflow_id)
+ file_hashes: dict[str, FileHash] = {}
for file in file_objs:
file_name = file.name
destination_path = os.path.join(api_storage_dir, file_name)
@@ -440,7 +527,22 @@ class SourceConnector(BaseConnector):
for chunk in file.chunks():
buffer.extend(chunk)
f.write(buffer)
- file_hashes.update({file_name: cls.hash_str(buffer)})
+ file_hash = cls.hash_str(buffer)
+ connection_type = WorkflowEndpoint.ConnectionType.API
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=file_hash
+ )
+ is_executed = (
+ True if file_history and file_history.is_completed() else False
+ )
+ file_hash = FileHash(
+ file_path=destination_path,
+ source_connection_type=connection_type,
+ file_name=file_name,
+ file_hash=file_hash,
+ is_executed=is_executed,
+ )
+ file_hashes.update({file_name: file_hash})
return file_hashes
@classmethod
diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.py
index c9efb4690..1f126469d 100644
--- a/backend/workflow_manager/workflow/execution.py
+++ b/backend/workflow_manager/workflow/execution.py
@@ -21,7 +21,6 @@ from workflow_manager.workflow.enums import ExecutionStatus
from workflow_manager.workflow.exceptions import WorkflowExecutionError
from workflow_manager.workflow.models import Workflow, WorkflowExecution
from workflow_manager.workflow.models.execution import EXECUTION_ERROR_LENGTH
-from workflow_manager.workflow.models.file_history import FileHistory
logger = logging.getLogger(__name__)
@@ -289,7 +288,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
self,
file_name: str,
single_step: bool,
- file_history: Optional[FileHistory] = None,
) -> tuple[bool, bool]:
"""Executes the input file.
@@ -297,29 +295,16 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
file_name (str): The name of the file to be executed.
single_step (bool): Flag indicating whether to execute in
single step mode.
- file_history (Optional[FileHistory], optional):
- The file history object. Defaults to None.
Returns:
tuple[bool, bool]: Flag indicating whether the file was executed
and skipped.
"""
execution_type = ExecutionType.COMPLETE
- is_executed = False
- is_skipped = False
if single_step:
execution_type = ExecutionType.STEP
- if not (file_history and file_history.is_completed()):
- self.execute_uncached_input(file_name=file_name, single_step=single_step)
- self.publish_log(f"Tool executed successfully for {file_name}")
- is_executed = True
- else:
- self.publish_log(
- f"Skipping file {file_name} as it is already processed."
- "Clear the cache to process it again"
- )
- is_skipped = True
+ self.execute_uncached_input(file_name=file_name, single_step=single_step)
+ self.publish_log(f"Tool executed successfully for {file_name}")
self._handle_execution_type(execution_type)
- return is_executed, is_skipped
def execute_uncached_input(self, file_name: str, single_step: bool) -> None:
"""Executes the uncached input file.
diff --git a/backend/workflow_manager/workflow/views.py b/backend/workflow_manager/workflow/views.py
index fc6c691c4..7c775a1a8 100644
--- a/backend/workflow_manager/workflow/views.py
+++ b/backend/workflow_manager/workflow/views.py
@@ -17,6 +17,7 @@ from tool_instance.tool_processor import ToolProcessor
from unstract.tool_registry.dto import Tool
from utils.filtering import FilterHelper
from workflow_manager.endpoint.destination import DestinationConnector
+from workflow_manager.endpoint.dto import FileHash
from workflow_manager.endpoint.endpoint_utils import WorkflowEndpointUtils
from workflow_manager.endpoint.source import SourceConnector
from workflow_manager.workflow.constants import WorkflowKey
@@ -173,7 +174,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
include_metadata = (
request.data.get("include_metadata", "false").lower() == "true"
)
- hashes_of_files = {}
+ hashes_of_files: dict[str, FileHash] = {}
if file_objs and execution_id and workflow_id:
hashes_of_files = SourceConnector.add_input_file_to_api_storage(
workflow_id=workflow_id,
@@ -217,7 +218,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_action: Optional[str] = None,
execution_id: Optional[str] = None,
pipeline_guid: Optional[str] = None,
- hash_values_of_files: dict[str, str] = {},
+ hash_values_of_files: dict[str, FileHash] = {},
include_metadata: bool = False,
) -> ExecutionResponse:
if execution_action is not None:
diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.py
index 9f35d4fee..2ea03366a 100644
--- a/backend/workflow_manager/workflow/workflow_helper.py
+++ b/backend/workflow_manager/workflow/workflow_helper.py
@@ -25,6 +25,7 @@ from unstract.workflow_execution.exceptions import StopExecution
from utils.cache_service import CacheService
from utils.local_context import StateStore
from workflow_manager.endpoint.destination import DestinationConnector
+from workflow_manager.endpoint.dto import FileHash
from workflow_manager.endpoint.source import SourceConnector
from workflow_manager.workflow.constants import (
CeleryConfigurations,
@@ -44,6 +45,7 @@ from workflow_manager.workflow.execution import WorkflowExecutionServiceHelper
from workflow_manager.workflow.file_history_helper import FileHistoryHelper
from workflow_manager.workflow.models.execution import WorkflowExecution
from workflow_manager.workflow.models.workflow import Workflow
+from workflow_manager.workflow.utils import WorkflowUtil
logger = logging.getLogger(__name__)
@@ -112,10 +114,9 @@ class WorkflowHelper:
destination: DestinationConnector,
execution_service: WorkflowExecutionServiceHelper,
single_step: bool,
- hash_values_of_files: dict[str, str] = {},
+ hash_values_of_files: dict[str, FileHash] = {},
) -> WorkflowExecution:
- input_files = source.list_files_from_source()
- total_files = len(input_files)
+ input_files, total_files = source.list_files_from_source(hash_values_of_files)
error_message = None
processed_files = 0
error_raised = 0
@@ -123,19 +124,27 @@ class WorkflowHelper:
execution_service.update_execution(
ExecutionStatus.EXECUTING, increment_attempt=True
)
- for index, input_file in enumerate(input_files):
+ if total_files > 0:
+ q_file_no_list = WorkflowUtil.get_q_no_list(workflow, total_files)
+
+ for index, (file_path, file_hash) in enumerate(input_files.items()):
file_number = index + 1
+ file_hash = WorkflowUtil.add_file_destination_filehash(
+ file_number,
+ q_file_no_list,
+ file_hash,
+ )
try:
is_executed, error = WorkflowHelper.process_file(
current_file_idx=file_number,
total_files=total_files,
- input_file=input_file,
+ input_file=file_hash.file_path,
workflow=workflow,
source=source,
destination=destination,
execution_service=execution_service,
single_step=single_step,
- hash_values_of_files=hash_values_of_files,
+ file_hash=file_hash,
)
if is_executed:
processed_files += 1
@@ -149,6 +158,8 @@ class WorkflowHelper:
except Exception as error:
error_message = str(error)
error_raised += 1
+ log_message = f"Error processing file {file_path}: {error_message}"
+ execution_service.publish_log(message=log_message, level=LogLevel.ERROR)
if error_raised and error_raised >= total_files:
execution_service.update_execution(
ExecutionStatus.ERROR, error=error_message
@@ -171,27 +182,22 @@ class WorkflowHelper:
destination: DestinationConnector,
execution_service: WorkflowExecutionServiceHelper,
single_step: bool,
- hash_values_of_files: dict[str, str],
+ file_hash: FileHash,
) -> tuple[bool, Optional[str]]:
- file_history = None
error = None
is_executed = False
- file_name, file_hash = source.add_file_to_volume(
- input_file_path=input_file,
- hash_values_of_files=hash_values_of_files,
+ file_name = source.add_file_to_volume(
+ input_file_path=input_file, file_hash=file_hash
)
try:
execution_service.initiate_tool_execution(
current_file_idx, total_files, file_name, single_step
)
- file_history = FileHistoryHelper.get_file_history(
- workflow=workflow, cache_key=file_hash
- )
- is_executed = execution_service.execute_input_file(
- file_name=file_name,
- single_step=single_step,
- file_history=file_history,
- )
+ if not file_hash.is_executed:
+ execution_service.execute_input_file(
+ file_name=file_name,
+ single_step=single_step,
+ )
except StopExecution:
raise
except Exception as e:
@@ -209,7 +215,6 @@ class WorkflowHelper:
file_name=file_name,
file_hash=file_hash,
workflow=workflow,
- file_history=file_history,
error=error,
input_file_path=input_file,
)
@@ -234,7 +239,7 @@ class WorkflowHelper:
@staticmethod
def run_workflow(
workflow: Workflow,
- hash_values_of_files: dict[str, str] = {},
+ hash_values_of_files: dict[str, FileHash] = {},
organization_id: Optional[str] = None,
pipeline_id: Optional[str] = None,
scheduled: bool = False,
@@ -269,7 +274,11 @@ class WorkflowHelper:
execution_id=execution_id,
execution_service=execution_service,
)
- destination = DestinationConnector(workflow=workflow, execution_id=execution_id)
+ destination = DestinationConnector(
+ workflow=workflow,
+ execution_id=execution_id,
+ execution_service=execution_service,
+ )
# Validating endpoints
source.validate()
destination.validate()
@@ -378,7 +387,7 @@ class WorkflowHelper:
def execute_workflow_async(
workflow_id: str,
execution_id: str,
- hash_values_of_files: dict[str, str],
+ hash_values_of_files: dict[str, FileHash],
timeout: int = -1,
pipeline_id: Optional[str] = None,
include_metadata: bool = False,
@@ -396,12 +405,15 @@ class WorkflowHelper:
ExecutionResponse: Existing status of execution
"""
try:
+ file_hash_in_str = {
+ key: value.to_json() for key, value in hash_values_of_files.items()
+ }
org_schema = connection.tenant.schema_name
log_events_id = StateStore.get(Common.LOG_EVENTS_ID)
async_execution = WorkflowHelper.execute_bin.delay(
org_schema,
workflow_id,
- hash_values_of_files=hash_values_of_files,
+ hash_values_of_files=file_hash_in_str,
execution_id=execution_id,
pipeline_id=pipeline_id,
log_events_id=log_events_id,
@@ -459,7 +471,7 @@ class WorkflowHelper:
schema_name: str,
workflow_id: str,
execution_id: str,
- hash_values_of_files: dict[str, str],
+ hash_values_of_files: dict[str, dict[str, Any]],
scheduled: bool = False,
execution_mode: Optional[tuple[str, str]] = None,
pipeline_id: Optional[str] = None,
@@ -486,6 +498,10 @@ class WorkflowHelper:
Returns:
dict[str, list[Any]]: Returns a dict with result from workflow execution
"""
+ hash_values = {
+ key: FileHash.from_json(value)
+ for key, value in hash_values_of_files.items()
+ }
task_id = current_task.request.id
tenant: Organization = (
get_tenant_model().objects.filter(schema_name=schema_name).first()
@@ -516,7 +532,7 @@ class WorkflowHelper:
scheduled=scheduled,
workflow_execution=workflow_execution,
execution_mode=execution_mode,
- hash_values_of_files=hash_values_of_files,
+ hash_values_of_files=hash_values,
include_metadata=include_metadata,
).result
return result
@@ -526,7 +542,7 @@ class WorkflowHelper:
workflow: Workflow,
execution_id: Optional[str] = None,
pipeline_id: Optional[str] = None,
- hash_values_of_files: dict[str, str] = {},
+ hash_values_of_files: dict[str, FileHash] = {},
include_metadata: bool = False,
) -> ExecutionResponse:
if pipeline_id:
@@ -593,7 +609,7 @@ class WorkflowHelper:
workflow: Workflow,
execution_action: str,
execution_id: Optional[str] = None,
- hash_values_of_files: dict[str, str] = {},
+ hash_values_of_files: dict[str, FileHash] = {},
include_metadata: bool = False,
) -> ExecutionResponse:
if execution_action is Workflow.ExecutionAction.START.value: # type: ignore
diff --git a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx
index 00f925625..b5026d09a 100644
--- a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx
+++ b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx
@@ -8,6 +8,14 @@ import { ConfigureFormsLayout } from "../configure-forms-layout/ConfigureFormsLa
import { ManageFiles } from "../../input-output/manage-files/ManageFiles";
import usePostHogEvents from "../../../hooks/usePostHogEvents";
+let DBRules;
+
+try {
+ DBRules =
+ require("../../../plugins/manual-review/db-rules/DBRules.jsx").DBRules;
+} catch {
+ // The component will remain null of it is not available
+}
function ConfigureConnectorModal({
open,
setOpen,
@@ -29,7 +37,7 @@ function ConfigureConnectorModal({
}) {
const [activeKey, setActiveKey] = useState("1");
useEffect(() => {
- if (connectorMetadata) {
+ if (connectorMetadata && connType === "FILESYSTEM") {
setActiveKey("2"); // If connector is already configured
} else {
setActiveKey("1"); // default value
@@ -37,10 +45,12 @@ function ConfigureConnectorModal({
}, [open, connectorMetadata]);
const { setPostHogCustomEvent, posthogConnectorEventText } =
usePostHogEvents();
- const tabItems = [
+
+ const [tabItems, setTabItems] = useState([
{
key: "1",
label: "Settings",
+ visible: true,
},
{
key: "2",
@@ -48,11 +58,64 @@ function ConfigureConnectorModal({
disabled:
!connectorId ||
connDetails?.connector_id !== selectedId ||
- connType === "DATABASE" ||
- connType === "MANUALREVIEW",
+ connType === "DATABASE",
+ visible: false,
},
- ];
+ ]);
+ const setUpdatedTabOptions = (tabOption) => {
+ setTabItems((prevTabOptions) => {
+ // Check if inputOption already exists in prevTabOptions
+ if (prevTabOptions.some((opt) => opt.key === tabOption.key)) {
+ return prevTabOptions; // Return previous state unchanged
+ } else {
+ // Create a new array with the existing options and the new option
+ const updatedTabOptions = [...prevTabOptions, tabOption];
+ return updatedTabOptions;
+ }
+ });
+ };
+ useEffect(() => {
+ const updatedTabItems = tabItems.map((item) => {
+ if (item.key === "2") {
+ item.visible = connType === "FILESYSTEM";
+ } else if (item.key === "MANUALREVIEW") {
+ item.disabled =
+ !connectorId || connDetails?.connector_id !== selectedId;
+ item.visible = connType === "DATABASE" || connType === "MANUALREVIEW";
+ } else {
+ item.visible = true;
+ }
+ return item;
+ });
+ setTabItems(updatedTabItems);
+ }, [open]);
+
+ useEffect(() => {
+ const updatedTabItems = tabItems.map((item) => {
+ if (item.key === "MANUALREVIEW") {
+ item.disabled =
+ !connectorId || connDetails?.connector_id !== selectedId;
+ }
+ return item;
+ });
+ setTabItems(updatedTabItems);
+ }, [connectorMetadata]);
+
+ useEffect(() => {
+ try {
+ const tabOption =
+ require("../../../plugins/manual-review/connector-config-tab-mrq/ConnectorConfigTabMRQ").mrqTabs;
+ if (tabOption) {
+ tabOption["disabled"] =
+ !connectorId || connDetails?.connector_id !== selectedId;
+ tabOption["visible"] = false;
+ setUpdatedTabOptions(tabOption);
+ }
+ } catch {
+ // The component will remain null of it is not available
+ }
+ }, []);
const handleSelectItem = (e) => {
const id = e.key;
setSelectedId(id?.toString());
@@ -103,7 +166,7 @@ function ConfigureConnectorModal({
<Col span={20} className="conn-modal-col conn-modal-form-pad-left">
<Tabs
activeKey={activeKey}
- items={tabItems}
+ items={tabItems.filter((item) => item?.visible !== false)}
onChange={onTabChange}
moreIcon={<></>}
/>
@@ -124,7 +187,12 @@ function ConfigureConnectorModal({
selectedItemName={selectedItemName}
/>
)}
- {activeKey === "2" && <ManageFiles selectedItem={connectorId} />}
+ {activeKey === "2" && connType === "FILESYSTEM" && (
+ <ManageFiles selectedItem={connectorId} />
+ )}
+ {activeKey === "MANUALREVIEW" && (
+ <DBRules connDetails={connDetails} />
+ )}
</Col>
</Row>
</div>