Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

workflow manager endpoint v2 delta changes (#688)

vishnuszipstack398a44ed6e82
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index 7839e1c8c..6f3ec1d75 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -20,6 +20,7 @@ from workflow_manager.endpoint_v2.constants import (
WorkflowFileType,
)
from workflow_manager.endpoint_v2.database_utils import DatabaseUtils
+from workflow_manager.endpoint_v2.dto import FileHash
from workflow_manager.endpoint_v2.exceptions import (
DestinationConnectorNotConfigured,
InvalidDestinationConnectionType,
@@ -30,6 +31,7 @@ from workflow_manager.endpoint_v2.exceptions import (
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.endpoint_v2.queue_utils import QueueResult, QueueUtils
from workflow_manager.workflow_v2.enums import ExecutionStatus
+from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper
from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
from workflow_manager.workflow_v2.models.file_history import FileHistory
from workflow_manager.workflow_v2.models.workflow import Workflow
@@ -49,7 +51,12 @@ class DestinationConnector(BaseConnector):
the destination connector.
"""
- def __init__(self, workflow: Workflow, execution_id: str) -> None:
+ def __init__(
+ self,
+ workflow: Workflow,
+ execution_id: str,
+ execution_service: Optional[WorkflowExecutionServiceHelper] = None,
+ ) -> None:
"""Initialize a DestinationConnector object.
Args:
@@ -62,6 +69,7 @@ class DestinationConnector(BaseConnector):
self.execution_id = execution_id
self.api_results: list[dict[str, Any]] = []
self.queue_results: list[dict[str, Any]] = []
+ self.execution_service = execution_service
def _get_endpoint_for_workflow(
self,
@@ -123,12 +131,27 @@ class DestinationConnector(BaseConnector):
):
raise DestinationConnectorNotConfigured()
+ def _push_data_to_queue(
+ self,
+ file_name: str,
+ workflow: Workflow,
+ input_file_path: str,
+ ) -> None:
+ result = self.get_result()
+ meta_data = self.get_metadata()
+ self._push_to_queue(
+ file_name=file_name,
+ workflow=workflow,
+ result=result,
+ input_file_path=input_file_path,
+ meta_data=meta_data,
+ )
+
def handle_output(
self,
file_name: str,
- file_hash: str,
+ file_hash: FileHash,
workflow: Workflow,
- file_history: Optional[FileHistory] = None,
error: Optional[str] = None,
input_file_path: Optional[str] = None,
) -> None:
@@ -140,10 +163,19 @@ class DestinationConnector(BaseConnector):
if connection_type == WorkflowEndpoint.ConnectionType.API:
self._handle_api_result(file_name=file_name, error=error, result=result)
return
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=file_hash.file_hash
+ )
if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
self.copy_output_to_output_directory()
elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:
- self.insert_into_db(file_history)
+ if (
+ file_hash.file_destination
+ == WorkflowEndpoint.ConnectionType.MANUALREVIEW
+ ):
+ self._push_data_to_queue(file_name, workflow, input_file_path)
+ else:
+ self.insert_into_db(input_file_path=input_file_path)
elif connection_type == WorkflowEndpoint.ConnectionType.API:
result = self.get_result(file_history)
metadata = self.get_metadata(file_history)
@@ -151,18 +183,14 @@ class DestinationConnector(BaseConnector):
file_name=file_name, error=error, result=result, metadata=metadata
)
elif connection_type == WorkflowEndpoint.ConnectionType.MANUALREVIEW:
- result = self.get_result(file_history)
- metadata = self.get_metadata(file_history)
- self._push_to_queue(
- file_name=file_name,
- workflow=workflow,
- result=result,
- input_file_path=input_file_path,
- metadata=metadata,
+ self._push_data_to_queue(file_name, workflow, input_file_path)
+ if self.execution_service:
+ self.execution_service.publish_log(
+ message=f"File '{file_name}' processed successfully"
)
if not file_history:
FileHistoryHelper.create_file_history(
- cache_key=file_hash,
+ cache_key=file_hash.file_hash,
workflow=workflow,
status=ExecutionStatus.COMPLETED,
result=result,
@@ -216,7 +244,7 @@ class DestinationConnector(BaseConnector):
with open(source_path, "rb") as source_file:
destination_fsspec.write_bytes(normalized_path, source_file.read())
- def insert_into_db(self, file_history: Optional[FileHistory]) -> None:
+ def insert_into_db(self, input_file_path: str) -> None:
"""Insert data into the database."""
connector_instance: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector_instance.metadata
@@ -233,8 +261,18 @@ class DestinationConnector(BaseConnector):
single_column_name = str(
destination_configurations.get(DestinationKey.SINGLE_COLUMN_NAME, "data")
)
-
- data = self.get_result(file_history)
+ file_path_name = str(
+ destination_configurations.get(DestinationKey.FILE_PATH, "file_path")
+ )
+ execution_id_name = str(
+ destination_configurations.get(DestinationKey.EXECUTION_ID, "execution_id")
+ )
+ data = self.get_result()
+ # If data is None, don't execute CREATE or INSERT query
+ if not data:
+ return
+ # Remove metadata from result
+ data.pop("metadata", None)
values = DatabaseUtils.get_columns_and_values(
column_mode_str=column_mode,
data=data,
@@ -242,15 +280,16 @@ class DestinationConnector(BaseConnector):
include_agent=include_agent,
agent_name=agent_name,
single_column_name=single_column_name,
+ file_path_name=file_path_name,
+ execution_id_name=execution_id_name,
+ file_path=input_file_path,
+ execution_id=self.execution_id,
)
db_class = DatabaseUtils.get_db_class(
connector_id=connector_instance.connector_id,
connector_settings=connector_settings,
)
engine = db_class.get_engine()
- # If data is None, don't execute CREATE or INSERT query
- if data is None:
- return
DatabaseUtils.create_table_if_not_exists(
db_class=db_class,
engine=engine,
@@ -347,7 +386,7 @@ class DestinationConnector(BaseConnector):
# assume it's a plain string
return original_string
- def get_result(self, file_history: Optional[FileHistory]) -> Optional[Any]:
+ def get_result(self, file_history: Optional[FileHistory] = None) -> Optional[Any]:
"""Get result data from the output file.
Returns:
@@ -383,7 +422,7 @@ class DestinationConnector(BaseConnector):
return result
def get_metadata(
- self, file_history: Optional[FileHistory]
+ self, file_history: Optional[FileHistory] = None
) -> Optional[dict[str, Any]]:
"""Get metadata from the output file.
@@ -477,7 +516,7 @@ class DestinationConnector(BaseConnector):
workflow: Workflow,
result: Optional[str] = None,
input_file_path: Optional[str] = None,
- metadata: Optional[dict[str, Any]] = None,
+ meta_data: Optional[dict[str, Any]] = None,
) -> None:
"""Handle the Manual Review QUEUE result.
@@ -492,7 +531,7 @@ class DestinationConnector(BaseConnector):
input_file_path (Optional[str], optional):
The path to the input file.
Defaults to None.
- metadata (Optional[dict[str, Any]], optional):
+ meta_data (Optional[dict[str, Any]], optional):
A dictionary containing additional
metadata related to the file. Defaults to None.
@@ -508,18 +547,23 @@ class DestinationConnector(BaseConnector):
settings=connector_settings, connector_id=connector.connector_id
)
with source_fs.open(input_file_path, "rb") as remote_file:
+ whisper_hash = None
file_content = remote_file.read()
# Convert file content to a base64 encoded string
file_content_base64 = base64.b64encode(file_content).decode("utf-8")
- q_name = f"review_queue_{self.organization_id}_{workflow.workflow_name}"
+ q_name = f"review_queue_{self.organization_id}_{workflow.id}"
+ if meta_data:
+ whisper_hash = meta_data.get("whisper-hash")
+ else:
+ whisper_hash = None
queue_result = QueueResult(
file=file_name,
- whisper_hash=metadata["whisper-hash"],
status=QueueResultStatus.SUCCESS,
result=result,
workflow_id=str(self.workflow_id),
file_content=file_content_base64,
- )
+ whisper_hash=whisper_hash,
+ ).to_dict()
# Convert the result dictionary to a JSON string
queue_result_json = json.dumps(queue_result)
conn = QueueUtils.get_queue_inst()
diff --git a/backend/workflow_manager/endpoint_v2/dto.py b/backend/workflow_manager/endpoint_v2/dto.py
new file mode 100644
index 000000000..0e7aa6fd9
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/dto.py
@@ -0,0 +1,36 @@
+import json
+from dataclasses import dataclass
+from typing import Any, Optional
+
+
+@dataclass
+class FileHash:
+ file_path: str
+ file_hash: str
+ file_name: str
+ source_connection_type: str
+ file_destination: Optional[tuple[str, str]] = (
+ None # To which destination this file wants to go for MRQ percentage
+ )
+ is_executed: bool = False
+
+ def to_json(self) -> dict[str, Any]:
+ return {
+ "file_path": self.file_path,
+ "file_hash": self.file_hash,
+ "file_name": self.file_name,
+ "source_connection_type": self.source_connection_type,
+ "file_destination": self.file_destination,
+ "is_executed": self.is_executed,
+ }
+
+ @staticmethod
+ def from_json(json_str_or_dict: Any) -> "FileHash":
+ """Deserialize a JSON string or dictionary to a FileHash instance."""
+ if isinstance(json_str_or_dict, dict):
+ # If already a dictionary, assume it's in the right format
+ data = json_str_or_dict
+ else:
+ # Otherwise, assume it's a JSON string
+ data = json.loads(json_str_or_dict)
+ return FileHash(**data)
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index dfa5aea5d..4ba76a488 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -4,6 +4,7 @@ import os
import shutil
from hashlib import md5, sha256
from io import BytesIO
+from itertools import islice
from typing import Any, Optional
import fsspec
@@ -21,7 +22,9 @@ from workflow_manager.endpoint_v2.constants import (
SourceKey,
WorkflowFileType,
)
+from workflow_manager.endpoint_v2.dto import FileHash
from workflow_manager.endpoint_v2.exceptions import (
+ FileHashMismatched,
FileHashNotFound,
InvalidInputDirectory,
InvalidSourceConnectionType,
@@ -31,6 +34,7 @@ from workflow_manager.endpoint_v2.exceptions import (
)
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper
+from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
from workflow_manager.workflow_v2.models.workflow import Workflow
logger = logging.getLogger(__name__)
@@ -58,10 +62,18 @@ class SourceConnector(BaseConnector):
organization_id: Optional[str] = None,
execution_service: Optional[WorkflowExecutionServiceHelper] = None,
) -> None:
- """Initialize a SourceConnector object.
+ """Create a SourceConnector.
Args:
- workflow (Workflow): _description_
+ workflow (Workflow): Associated workflow instance
+ execution_id (str): UUID of the current execution
+ organization_id (Optional[str]): Organization ID. Defaults to None.
+ execution_service (Optional[WorkflowExecutionServiceHelper]): Instance of
+ WorkflowExecutionServiceHelper that helps with WF execution.
+ Defaults to None. This is not used in case of execution by API.
+
+ Raises:
+ OrganizationIdNotFound: _description_
"""
organization_id = organization_id or UserContext.get_organization_identifier()
if not organization_id:
@@ -120,30 +132,29 @@ class SourceConnector(BaseConnector):
wildcard.extend(patterns.get(pattern, []))
return wildcard
- def list_file_from_api_storage(self) -> list[str]:
+ def list_file_from_api_storage(
+ self, file_hashes: dict[str, FileHash]
+ ) -> tuple[dict[str, FileHash], int]:
"""List all files from the api_storage_dir directory."""
- files: list[str] = []
- if not self.api_storage_dir:
- return files
- for file in os.listdir(self.api_storage_dir):
- file_path = os.path.join(self.api_storage_dir, file)
- if os.path.isfile(file_path):
- files.append(file_path)
- return files
-
- def list_files_from_file_connector(self) -> list[str]:
+ return file_hashes, len(file_hashes)
+
+ def list_files_from_file_connector(self) -> tuple[dict[str, FileHash], int]:
"""_summary_
Raises:
InvalidDirectory: _description_
Returns:
- list[str]: _description_
+ tuple[dict[str, FileHash], int]: A dictionary of matched file paths
+ and their corresponding FileHash objects, along with the total count
+ of matched files.
"""
connector: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector.connector_metadata
source_configurations: dict[str, Any] = self.endpoint.configuration
- required_patterns = source_configurations.get(SourceKey.FILE_EXTENSIONS, [])
+ required_patterns = list(
+ source_configurations.get(SourceKey.FILE_EXTENSIONS, [])
+ )
recursive = bool(
source_configurations.get(SourceKey.PROCESS_SUB_DIRECTORIES, False)
)
@@ -153,65 +164,109 @@ class SourceConnector(BaseConnector):
)
)
root_dir_path = connector_settings.get(ConnectorKeys.PATH, "")
-
- input_directory = str(source_configurations.get(SourceKey.ROOT_FOLDER, ""))
+ folders_to_process = list(source_configurations.get(SourceKey.FOLDERS, ["/"]))
+ # Process from root in case its user provided list is empty
+ if not folders_to_process:
+ folders_to_process = ["/"]
+ patterns = self.valid_file_patterns(required_patterns=required_patterns)
+ self.publish_user_sys_log(
+ f"Matching for patterns '{', '.join(patterns)}' from "
+ f"'{', '.join(folders_to_process)}'"
+ )
source_fs = self.get_fs_connector(
settings=connector_settings, connector_id=connector.connector_id
)
- input_directory = source_fs.get_connector_root_dir(
- input_dir=input_directory, root_path=root_dir_path
+ source_fs_fsspec = source_fs.get_fsspec_fs()
+ # Checking if folders exist at source before processing
+ # TODO: Validate while receiving this input configuration as well
+ for input_directory in folders_to_process:
+ # TODO: Move to connector class for better error handling
+ try:
+ input_directory = source_fs.get_connector_root_dir(
+ input_dir=input_directory, root_path=root_dir_path
+ )
+ if not source_fs_fsspec.isdir(input_directory):
+ raise InvalidInputDirectory(dir=input_directory)
+ except Exception as e:
+ msg = f"Error while validating path '{input_directory}'. {str(e)}"
+ self.publish_user_sys_log(msg)
+ if isinstance(e, InvalidInputDirectory):
+ raise
+ raise InvalidInputDirectory(detail=msg)
+
+ total_files_to_process = 0
+ total_matched_files = {}
+
+ for input_directory in folders_to_process:
+ input_directory = source_fs.get_connector_root_dir(
+ input_dir=input_directory, root_path=root_dir_path
+ )
+ logger.debug(f"Listing files from: {input_directory}")
+ matched_files, count = self._get_matched_files(
+ source_fs_fsspec, input_directory, patterns, recursive, limit
+ )
+ self.publish_user_sys_log(
+ f"Matched '{count}' files from '{input_directory}'"
+ )
+ total_matched_files.update(matched_files)
+ total_files_to_process += count
+ self.publish_input_output_list_file_logs(
+ folders_to_process, total_matched_files, total_files_to_process
)
- logger.debug(f"source input directory {input_directory}")
- if not isinstance(required_patterns, list):
- required_patterns = [required_patterns]
+ return total_matched_files, total_files_to_process
- source_fs_fsspec = source_fs.get_fsspec_fs()
+ def publish_user_sys_log(self, msg: str) -> None:
+ """Publishes log to the user and system.
- patterns = self.valid_file_patterns(required_patterns=required_patterns)
- is_directory = source_fs_fsspec.isdir(input_directory)
- if not is_directory:
- raise InvalidInputDirectory()
- matched_files = self._get_matched_files(
- source_fs_fsspec, input_directory, patterns, recursive, limit
- )
- self.publish_input_output_list_file_logs(input_directory, matched_files)
- return matched_files
+ Pushes logs messages to the configured logger and to the
+ websocket channel if the `execution_service` is configured.
+
+ Args:
+ msg (str): Message to log
+ """
+ logger.info(msg)
+ if self.execution_service:
+ self.execution_service.publish_log(msg)
def publish_input_output_list_file_logs(
- self, input_directory: str, matched_files: list[str]
+ self, folders: list[str], matched_files: dict[str, FileHash], count: int
) -> None:
if not self.execution_service:
return None
- input_log = f"##Input folder:\n\n `{os.path.basename(input_directory)}`\n\n"
+
+ folders_list = "\n".join(f"- `{folder.strip()}`" for folder in folders)
+ input_log = f"##Folders to process:\n\n{folders_list}\n\n"
self.execution_service.publish_update_log(
state=LogState.INPUT_UPDATE, message=input_log
)
- output_log = self._matched_files_component_log(matched_files)
+ output_log = self._matched_files_component_log(matched_files, count)
self.execution_service.publish_update_log(
state=LogState.OUTPUT_UPDATE, message=output_log
)
def publish_input_file_content(self, input_file_path: str, input_text: str) -> None:
- if self.execution_service:
- output_log_message = f"##Input text:\n\n```text\n{input_text}\n```\n\n"
- input_log_message = (
- "##Input file:\n\n```text\n"
- f"{os.path.basename(input_file_path)}\n```\n\n"
- )
- self.execution_service.publish_update_log(
- state=LogState.INPUT_UPDATE, message=input_log_message
- )
- self.execution_service.publish_update_log(
- state=LogState.OUTPUT_UPDATE, message=output_log_message
- )
+ if not self.execution_service:
+ return None
+ output_log_message = f"##Input text:\n\n```text\n{input_text}\n```\n\n"
+ input_log_message = (
+ "##Input file:\n\n```text\n" f"{os.path.basename(input_file_path)}\n```\n\n"
+ )
+ self.execution_service.publish_update_log(
+ state=LogState.INPUT_UPDATE, message=input_log_message
+ )
+ self.execution_service.publish_update_log(
+ state=LogState.OUTPUT_UPDATE, message=output_log_message
+ )
- def _matched_files_component_log(self, matched_files: list[str]) -> str:
+ def _matched_files_component_log(
+ self, matched_files: dict[str, FileHash], count: int
+ ) -> str:
output_log = "### Matched files \n```text\n\n\n"
- for file in matched_files[:20]:
- output_log += f"{file}\n"
+ for file_path in islice(matched_files.keys(), 20):
+ output_log += f"- {file_path}\n"
output_log += "```\n\n"
- output_log += f"""Total matched files: {len(matched_files)}
+ output_log += f"""Total matched files: {count}
\n\nPlease note that only the first 20 files are shown.\n\n"""
return output_log
@@ -222,13 +277,13 @@ class SourceConnector(BaseConnector):
patterns: list[str],
recursive: bool,
limit: int,
- ) -> list[str]:
- """Get a list of matched files based on patterns in a directory.
+ ) -> tuple[dict[str, FileHash], int]:
+ """Get a dictionary of matched files based on patterns in a directory.
This method searches for files in the specified `input_directory` that
- match any of the given `patterns`.
- The search can be performed recursively if `recursive` is set to True.
- The number of matched files returned is limited by `limit`.
+ match any of the given `patterns`. The search can be performed recursively
+ if `recursive` is set to True. The number of matched files returned is
+ limited by `limit`.
Args:
source_fs (Any): The file system object used for searching.
@@ -238,41 +293,81 @@ class SourceConnector(BaseConnector):
limit (int): The maximum number of matched files to return.
Returns:
- list[str]: A list of matched file paths.
+ tuple[dict[str, FileHash], int]: A dictionary of matched file paths
+ and their corresponding FileHash objects, along with the total count
+ of matched files.
"""
- matched_files = []
+ matched_files: dict[str, FileHash] = {}
count = 0
max_depth = int(SourceConstant.MAX_RECURSIVE_DEPTH) if recursive else 1
for root, dirs, files in source_fs.walk(input_directory, maxdepth=max_depth):
- if count >= limit:
- break
for file in files:
- if not file:
- continue
if count >= limit:
break
- if any(fnmatch.fnmatch(file, pattern) for pattern in patterns):
- file_path = os.path.join(root, file)
- file_path = f"{file_path}"
- matched_files.append(file_path)
- count += 1
+ if self._should_process_file(file, patterns):
+ file_path = str(os.path.join(root, file))
+ if self._is_new_file(
+ file_path=file_path, workflow=self.endpoint.workflow
+ ):
+ matched_files[file_path] = self._create_file_hash(file_path)
+ count += 1
+
+ return matched_files, count
+
+ def _should_process_file(self, file: str, patterns: list[str]) -> bool:
+ """Check if the file should be processed based on the patterns."""
+ return bool(file) and any(
+ fnmatch.fnmatchcase(file.lower(), pattern.lower()) for pattern in patterns
+ )
- return matched_files
+ def _is_new_file(self, file_path: str, workflow: Workflow) -> bool:
+ """Check if the file is new or already processed."""
+ file_content = self.get_file_content(input_file_path=file_path)
+ file_hash = self.get_hash_value(file_content)
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=file_hash
+ )
- def list_files_from_source(self) -> list[str]:
+ if file_history and file_history.is_completed():
+ self.execution_service.publish_log(
+ f"Skipping file {file_path} as it has already been processed. "
+ "Clear the file markers to process it again."
+ )
+ return False
+
+ return True
+
+ def _create_file_hash(self, file_path: str) -> FileHash:
+ """Create a FileHash object for the matched file."""
+ file_name = os.path.basename(file_path)
+ file_content = self.get_file_content(input_file_path=file_path)
+ file_hash = self.get_hash_value(file_content)
+ connection_type = self.endpoint.connection_type
+
+ return FileHash(
+ file_path=file_path,
+ source_connection_type=connection_type,
+ file_name=file_name,
+ file_hash=file_hash,
+ )
+
+ def list_files_from_source(
+ self, file_hashes: dict[str, FileHash] = {}
+ ) -> tuple[dict[str, FileHash], int]:
"""List files from source connector.
Args:
api_storage_dir (Optional[str], optional): API storage directory
Returns:
- list[str]: list of files
+ tuple[dict[str, FileHash], int]: A dictionary of FileHashes,
+ along with the total count of matched files.
"""
connection_type = self.endpoint.connection_type
if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
return self.list_files_from_file_connector()
elif connection_type == WorkflowEndpoint.ConnectionType.API:
- return self.list_file_from_api_storage()
+ return self.list_file_from_api_storage(file_hashes)
raise InvalidSourceConnectionType()
@classmethod
@@ -301,44 +396,86 @@ class SourceConnector(BaseConnector):
else:
raise ValueError(f"Unsupported hash_method: {hash_method}")
+ def get_file_content(self, input_file_path: str, chunk_size: int = 8192) -> bytes:
+ """Read the content of a file from a remote filesystem in chunks.
+
+ Args:
+ input_file_path (str): The path of the input file.
+ chunk_size (int): The size of the chunks to read at a time
+ (default is 8192 bytes).
+
+ Returns:
+ bytes: The content of the file.
+ """
+ connector: ConnectorInstance = self.endpoint.connector_instance
+ connector_settings: dict[str, Any] = connector.connector_metadata
+ source_fs = self.get_fsspec(
+ settings=connector_settings, connector_id=connector.connector_id
+ )
+ file_content = bytearray() # Use bytearray for efficient byte concatenation
+ with source_fs.open(input_file_path, "rb") as remote_file:
+ while chunk := remote_file.read(chunk_size):
+ file_content.extend(chunk)
+
+ return bytes(file_content)
+
+ def get_hash_value(self, file_content: bytes) -> str:
+ """Generate a hash value from the file content.
+
+ Args:
+ file_content (bytes): The content of the file.
+
+ Returns:
+ str: The hash value of the file content.
+ """
+ return self.hash_str(file_content)
+
+ def copy_file_to_infile_dir(self, source_file_path: str, infile_path: str) -> None:
+ """Copy the source file to the infile directory.
+
+ Args:
+ source_file_path (str): The path of the source file.
+ infile_path (str): The destination path in the infile directory.
+ """
+ shutil.copyfile(source_file_path, infile_path)
+ logger.info(f"File copied from {source_file_path} to {infile_path}")
+
def add_input_from_connector_to_volume(self, input_file_path: str) -> str:
"""Add input file to execution directory.
Args:
input_file_path (str): The path of the input file.
+
Returns:
str: The hash value of the file content.
+
Raises:
- FileHashNotFound: If the hash value of the file content
- is not found.
+ FileHashNotFound: If the hash value of the file content is not found.
"""
- connector: ConnectorInstance = self.endpoint.connector_instance
- connector_settings: dict[str, Any] = connector.connector_metadata
source_file_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)
infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
source_file = f"file://{source_file_path}"
- source_fs = self.get_fsspec(
- settings=connector_settings, connector_id=connector.connector_id
+ # Get file content and hash value
+ file_content = self.get_file_content(input_file_path)
+ hash_value_of_file_content = self.get_hash_value(file_content)
+
+ logger.info(
+ f"hash_value_of_file {source_file} is : {hash_value_of_file_content}"
)
- with (
- source_fs.open(input_file_path, "rb") as remote_file,
- fsspec.open(source_file, "wb") as local_file,
- ):
- file_content = remote_file.read()
- hash_value_of_file_content = self.hash_str(file_content)
- logger.info(
- f"hash_value_of_file {source_file} is "
- f": {hash_value_of_file_content}"
- )
- input_log = (
- file_content[:500].decode("utf-8", errors="replace") + "...(truncated)"
- )
- self.publish_input_file_content(input_file_path, input_log)
+ input_log = (
+ file_content[:500].decode("utf-8", errors="replace") + "...(truncated)"
+ )
+ self.publish_input_file_content(input_file_path, input_log)
+
+ with fsspec.open(source_file, "wb") as local_file:
local_file.write(file_content)
- shutil.copyfile(source_file_path, infile_path)
- logger.info(f"{input_file_path} is added in to execution directory")
+
+ # Copy file to infile directory
+ self.copy_file_to_infile_dir(source_file_path, infile_path)
+
+ logger.info(f"{input_file_path} is added to execution directory")
return hash_value_of_file_content
def add_input_from_api_storage_to_volume(self, input_file_path: str) -> None:
@@ -349,7 +486,7 @@ class SourceConnector(BaseConnector):
shutil.copyfile(input_file_path, source_path)
def add_file_to_volume(
- self, input_file_path: str, hash_values_of_files: dict[str, str] = {}
+ self, input_file_path: str, file_hash: FileHash
) -> tuple[str, str]:
"""Add input file to execution directory.
@@ -368,18 +505,20 @@ class SourceConnector(BaseConnector):
file_content_hash = self.add_input_from_connector_to_volume(
input_file_path=input_file_path,
)
+ if file_content_hash != file_hash.file_hash:
+ raise FileHashMismatched()
elif connection_type == WorkflowEndpoint.ConnectionType.API:
self.add_input_from_api_storage_to_volume(input_file_path=input_file_path)
- if file_name not in hash_values_of_files:
+ if file_name != file_hash.file_name:
raise FileHashNotFound()
- file_content_hash = hash_values_of_files[file_name]
+ file_content_hash = file_hash.file_hash
else:
raise InvalidSourceConnectionType()
self.add_metadata_to_volume(
input_file_path=input_file_path, source_hash=file_content_hash
)
- return file_name, file_content_hash
+ return file_name
def handle_final_result(
self,
@@ -417,7 +556,7 @@ class SourceConnector(BaseConnector):
@classmethod
def add_input_file_to_api_storage(
cls, workflow_id: str, execution_id: str, file_objs: list[UploadedFile]
- ) -> dict[str, str]:
+ ) -> dict[str, FileHash]:
"""Add input file to api storage.
Args:
@@ -430,7 +569,8 @@ class SourceConnector(BaseConnector):
api_storage_dir = cls.get_api_storage_dir_path(
workflow_id=workflow_id, execution_id=execution_id
)
- file_hashes: dict[str, str] = {}
+ workflow: Workflow = Workflow.objects.get(id=workflow_id)
+ file_hashes: dict[str, FileHash] = {}
for file in file_objs:
file_name = file.name
destination_path = os.path.join(api_storage_dir, file_name)
@@ -440,7 +580,22 @@ class SourceConnector(BaseConnector):
for chunk in file.chunks():
buffer.extend(chunk)
f.write(buffer)
- file_hashes.update({file_name: cls.hash_str(buffer)})
+ file_hash = cls.hash_str(buffer)
+ connection_type = WorkflowEndpoint.ConnectionType.API
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=file_hash
+ )
+ is_executed = (
+ True if file_history and file_history.is_completed() else False
+ )
+ file_hash = FileHash(
+ file_path=destination_path,
+ source_connection_type=connection_type,
+ file_name=file_name,
+ file_hash=file_hash,
+ is_executed=is_executed,
+ )
+ file_hashes.update({file_name: file_hash})
return file_hashes
@classmethod