LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
workflow manager endpoint v2 delta changes (#688)
vishnuszipstack398a44ed6e82
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex 7839e1c8c..6f3ec1d75 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -20,6 +20,7 @@ from workflow_manager.endpoint_v2.constants import (WorkflowFileType,)from workflow_manager.endpoint_v2.database_utils import DatabaseUtils+from workflow_manager.endpoint_v2.dto import FileHashfrom workflow_manager.endpoint_v2.exceptions import (DestinationConnectorNotConfigured,InvalidDestinationConnectionType,@@ -30,6 +31,7 @@ from workflow_manager.endpoint_v2.exceptions import (from workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.endpoint_v2.queue_utils import QueueResult, QueueUtilsfrom workflow_manager.workflow_v2.enums import ExecutionStatus+from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelperfrom workflow_manager.workflow_v2.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow_v2.models.file_history import FileHistoryfrom workflow_manager.workflow_v2.models.workflow import Workflow@@ -49,7 +51,12 @@ class DestinationConnector(BaseConnector):the destination connector."""- def __init__(self, workflow: Workflow, execution_id: str) -> None:+ def __init__(+ self,+ workflow: Workflow,+ execution_id: str,+ execution_service: Optional[WorkflowExecutionServiceHelper] = None,+ ) -> None:"""Initialize a DestinationConnector object.Args:@@ -62,6 +69,7 @@ class DestinationConnector(BaseConnector):self.execution_id = execution_idself.api_results: list[dict[str, Any]] = []self.queue_results: list[dict[str, Any]] = []+ self.execution_service = execution_servicedef _get_endpoint_for_workflow(self,@@ -123,12 +131,27 @@ class DestinationConnector(BaseConnector):):raise DestinationConnectorNotConfigured()+ def _push_data_to_queue(+ self,+ file_name: str,+ workflow: Workflow,+ input_file_path: str,+ ) -> None:+ result = self.get_result()+ meta_data = self.get_metadata()+ self._push_to_queue(+ file_name=file_name,+ workflow=workflow,+ result=result,+ input_file_path=input_file_path,+ meta_data=meta_data,+ )+def handle_output(self,file_name: str,- file_hash: str,+ file_hash: FileHash,workflow: Workflow,- file_history: Optional[FileHistory] = None,error: Optional[str] = None,input_file_path: Optional[str] = None,) -> None:@@ -140,10 +163,19 @@ class DestinationConnector(BaseConnector):if connection_type == WorkflowEndpoint.ConnectionType.API:self._handle_api_result(file_name=file_name, error=error, result=result)return+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=file_hash.file_hash+ )if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:self.copy_output_to_output_directory()elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:- self.insert_into_db(file_history)+ if (+ file_hash.file_destination+ == WorkflowEndpoint.ConnectionType.MANUALREVIEW+ ):+ self._push_data_to_queue(file_name, workflow, input_file_path)+ else:+ self.insert_into_db(input_file_path=input_file_path)elif connection_type == WorkflowEndpoint.ConnectionType.API:result = self.get_result(file_history)metadata = self.get_metadata(file_history)@@ -151,18 +183,14 @@ class DestinationConnector(BaseConnector):file_name=file_name, error=error, result=result, metadata=metadata)elif connection_type == WorkflowEndpoint.ConnectionType.MANUALREVIEW:- result = self.get_result(file_history)- metadata = self.get_metadata(file_history)- self._push_to_queue(- file_name=file_name,- workflow=workflow,- result=result,- input_file_path=input_file_path,- metadata=metadata,+ self._push_data_to_queue(file_name, workflow, input_file_path)+ if self.execution_service:+ self.execution_service.publish_log(+ message=f"File '{file_name}' processed successfully")if not file_history:FileHistoryHelper.create_file_history(- cache_key=file_hash,+ cache_key=file_hash.file_hash,workflow=workflow,status=ExecutionStatus.COMPLETED,result=result,@@ -216,7 +244,7 @@ class DestinationConnector(BaseConnector):with open(source_path, "rb") as source_file:destination_fsspec.write_bytes(normalized_path, source_file.read())- def insert_into_db(self, file_history: Optional[FileHistory]) -> None:+ def insert_into_db(self, input_file_path: str) -> None:"""Insert data into the database."""connector_instance: ConnectorInstance = self.endpoint.connector_instanceconnector_settings: dict[str, Any] = connector_instance.metadata@@ -233,8 +261,18 @@ class DestinationConnector(BaseConnector):single_column_name = str(destination_configurations.get(DestinationKey.SINGLE_COLUMN_NAME, "data"))-- data = self.get_result(file_history)+ file_path_name = str(+ destination_configurations.get(DestinationKey.FILE_PATH, "file_path")+ )+ execution_id_name = str(+ destination_configurations.get(DestinationKey.EXECUTION_ID, "execution_id")+ )+ data = self.get_result()+ # If data is None, don't execute CREATE or INSERT query+ if not data:+ return+ # Remove metadata from result+ data.pop("metadata", None)values = DatabaseUtils.get_columns_and_values(column_mode_str=column_mode,data=data,@@ -242,15 +280,16 @@ class DestinationConnector(BaseConnector):include_agent=include_agent,agent_name=agent_name,single_column_name=single_column_name,+ file_path_name=file_path_name,+ execution_id_name=execution_id_name,+ file_path=input_file_path,+ execution_id=self.execution_id,)db_class = DatabaseUtils.get_db_class(connector_id=connector_instance.connector_id,connector_settings=connector_settings,)engine = db_class.get_engine()- # If data is None, don't execute CREATE or INSERT query- if data is None:- returnDatabaseUtils.create_table_if_not_exists(db_class=db_class,engine=engine,@@ -347,7 +386,7 @@ class DestinationConnector(BaseConnector):# assume it's a plain stringreturn original_string- def get_result(self, file_history: Optional[FileHistory]) -> Optional[Any]:+ def get_result(self, file_history: Optional[FileHistory] = None) -> Optional[Any]:"""Get result data from the output file.Returns:@@ -383,7 +422,7 @@ class DestinationConnector(BaseConnector):return resultdef get_metadata(- self, file_history: Optional[FileHistory]+ self, file_history: Optional[FileHistory] = None) -> Optional[dict[str, Any]]:"""Get metadata from the output file.@@ -477,7 +516,7 @@ class DestinationConnector(BaseConnector):workflow: Workflow,result: Optional[str] = None,input_file_path: Optional[str] = None,- metadata: Optional[dict[str, Any]] = None,+ meta_data: Optional[dict[str, Any]] = None,) -> None:"""Handle the Manual Review QUEUE result.@@ -492,7 +531,7 @@ class DestinationConnector(BaseConnector):input_file_path (Optional[str], optional):The path to the input file.Defaults to None.- metadata (Optional[dict[str, Any]], optional):+ meta_data (Optional[dict[str, Any]], optional):A dictionary containing additionalmetadata related to the file. Defaults to None.@@ -508,18 +547,23 @@ class DestinationConnector(BaseConnector):settings=connector_settings, connector_id=connector.connector_id)with source_fs.open(input_file_path, "rb") as remote_file:+ whisper_hash = Nonefile_content = remote_file.read()# Convert file content to a base64 encoded stringfile_content_base64 = base64.b64encode(file_content).decode("utf-8")- q_name = f"review_queue_{self.organization_id}_{workflow.workflow_name}"+ q_name = f"review_queue_{self.organization_id}_{workflow.id}"+ if meta_data:+ whisper_hash = meta_data.get("whisper-hash")+ else:+ whisper_hash = Nonequeue_result = QueueResult(file=file_name,- whisper_hash=metadata["whisper-hash"],status=QueueResultStatus.SUCCESS,result=result,workflow_id=str(self.workflow_id),file_content=file_content_base64,- )+ whisper_hash=whisper_hash,+ ).to_dict()# Convert the result dictionary to a JSON stringqueue_result_json = json.dumps(queue_result)conn = QueueUtils.get_queue_inst()diff --git a/backend/workflow_manager/endpoint_v2/dto.py b/backend/workflow_manager/endpoint_v2/dto.pynew file mode 100644index 000000000..0e7aa6fd9--- /dev/null+++ b/backend/workflow_manager/endpoint_v2/dto.py@@ -0,0 +1,36 @@+import json+from dataclasses import dataclass+from typing import Any, Optional+++@dataclass+class FileHash:+ file_path: str+ file_hash: str+ file_name: str+ source_connection_type: str+ file_destination: Optional[tuple[str, str]] = (+ None # To which destination this file wants to go for MRQ percentage+ )+ is_executed: bool = False++ def to_json(self) -> dict[str, Any]:+ return {+ "file_path": self.file_path,+ "file_hash": self.file_hash,+ "file_name": self.file_name,+ "source_connection_type": self.source_connection_type,+ "file_destination": self.file_destination,+ "is_executed": self.is_executed,+ }++ @staticmethod+ def from_json(json_str_or_dict: Any) -> "FileHash":+ """Deserialize a JSON string or dictionary to a FileHash instance."""+ if isinstance(json_str_or_dict, dict):+ # If already a dictionary, assume it's in the right format+ data = json_str_or_dict+ else:+ # Otherwise, assume it's a JSON string+ data = json.loads(json_str_or_dict)+ return FileHash(**data)diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex dfa5aea5d..4ba76a488 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -4,6 +4,7 @@ import osimport shutilfrom hashlib import md5, sha256from io import BytesIO+from itertools import islicefrom typing import Any, Optionalimport fsspec@@ -21,7 +22,9 @@ from workflow_manager.endpoint_v2.constants import (SourceKey,WorkflowFileType,)+from workflow_manager.endpoint_v2.dto import FileHashfrom workflow_manager.endpoint_v2.exceptions import (+ FileHashMismatched,FileHashNotFound,InvalidInputDirectory,InvalidSourceConnectionType,@@ -31,6 +34,7 @@ from workflow_manager.endpoint_v2.exceptions import ()from workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper+from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow_v2.models.workflow import Workflowlogger = logging.getLogger(__name__)@@ -58,10 +62,18 @@ class SourceConnector(BaseConnector):organization_id: Optional[str] = None,execution_service: Optional[WorkflowExecutionServiceHelper] = None,) -> None:- """Initialize a SourceConnector object.+ """Create a SourceConnector.Args:- workflow (Workflow): _description_+ workflow (Workflow): Associated workflow instance+ execution_id (str): UUID of the current execution+ organization_id (Optional[str]): Organization ID. Defaults to None.+ execution_service (Optional[WorkflowExecutionServiceHelper]): Instance of+ WorkflowExecutionServiceHelper that helps with WF execution.+ Defaults to None. This is not used in case of execution by API.++ Raises:+ OrganizationIdNotFound: _description_"""organization_id = organization_id or UserContext.get_organization_identifier()if not organization_id:@@ -120,30 +132,29 @@ class SourceConnector(BaseConnector):wildcard.extend(patterns.get(pattern, []))return wildcard- def list_file_from_api_storage(self) -> list[str]:+ def list_file_from_api_storage(+ self, file_hashes: dict[str, FileHash]+ ) -> tuple[dict[str, FileHash], int]:"""List all files from the api_storage_dir directory."""- files: list[str] = []- if not self.api_storage_dir:- return files- for file in os.listdir(self.api_storage_dir):- file_path = os.path.join(self.api_storage_dir, file)- if os.path.isfile(file_path):- files.append(file_path)- return files-- def list_files_from_file_connector(self) -> list[str]:+ return file_hashes, len(file_hashes)++ def list_files_from_file_connector(self) -> tuple[dict[str, FileHash], int]:"""_summary_Raises:InvalidDirectory: _description_Returns:- list[str]: _description_+ tuple[dict[str, FileHash], int]: A dictionary of matched file paths+ and their corresponding FileHash objects, along with the total count+ of matched files."""connector: ConnectorInstance = self.endpoint.connector_instanceconnector_settings: dict[str, Any] = connector.connector_metadatasource_configurations: dict[str, Any] = self.endpoint.configuration- required_patterns = source_configurations.get(SourceKey.FILE_EXTENSIONS, [])+ required_patterns = list(+ source_configurations.get(SourceKey.FILE_EXTENSIONS, [])+ )recursive = bool(source_configurations.get(SourceKey.PROCESS_SUB_DIRECTORIES, False))@@ -153,65 +164,109 @@ class SourceConnector(BaseConnector):))root_dir_path = connector_settings.get(ConnectorKeys.PATH, "")-- input_directory = str(source_configurations.get(SourceKey.ROOT_FOLDER, ""))+ folders_to_process = list(source_configurations.get(SourceKey.FOLDERS, ["/"]))+ # Process from root in case its user provided list is empty+ if not folders_to_process:+ folders_to_process = ["/"]+ patterns = self.valid_file_patterns(required_patterns=required_patterns)+ self.publish_user_sys_log(+ f"Matching for patterns '{', '.join(patterns)}' from "+ f"'{', '.join(folders_to_process)}'"+ )source_fs = self.get_fs_connector(settings=connector_settings, connector_id=connector.connector_id)- input_directory = source_fs.get_connector_root_dir(- input_dir=input_directory, root_path=root_dir_path+ source_fs_fsspec = source_fs.get_fsspec_fs()+ # Checking if folders exist at source before processing+ # TODO: Validate while receiving this input configuration as well+ for input_directory in folders_to_process:+ # TODO: Move to connector class for better error handling+ try:+ input_directory = source_fs.get_connector_root_dir(+ input_dir=input_directory, root_path=root_dir_path+ )+ if not source_fs_fsspec.isdir(input_directory):+ raise InvalidInputDirectory(dir=input_directory)+ except Exception as e:+ msg = f"Error while validating path '{input_directory}'. {str(e)}"+ self.publish_user_sys_log(msg)+ if isinstance(e, InvalidInputDirectory):+ raise+ raise InvalidInputDirectory(detail=msg)++ total_files_to_process = 0+ total_matched_files = {}++ for input_directory in folders_to_process:+ input_directory = source_fs.get_connector_root_dir(+ input_dir=input_directory, root_path=root_dir_path+ )+ logger.debug(f"Listing files from: {input_directory}")+ matched_files, count = self._get_matched_files(+ source_fs_fsspec, input_directory, patterns, recursive, limit+ )+ self.publish_user_sys_log(+ f"Matched '{count}' files from '{input_directory}'"+ )+ total_matched_files.update(matched_files)+ total_files_to_process += count+ self.publish_input_output_list_file_logs(+ folders_to_process, total_matched_files, total_files_to_process)- logger.debug(f"source input directory {input_directory}")- if not isinstance(required_patterns, list):- required_patterns = [required_patterns]+ return total_matched_files, total_files_to_process- source_fs_fsspec = source_fs.get_fsspec_fs()+ def publish_user_sys_log(self, msg: str) -> None:+ """Publishes log to the user and system.- patterns = self.valid_file_patterns(required_patterns=required_patterns)- is_directory = source_fs_fsspec.isdir(input_directory)- if not is_directory:- raise InvalidInputDirectory()- matched_files = self._get_matched_files(- source_fs_fsspec, input_directory, patterns, recursive, limit- )- self.publish_input_output_list_file_logs(input_directory, matched_files)- return matched_files+ Pushes logs messages to the configured logger and to the+ websocket channel if the `execution_service` is configured.++ Args:+ msg (str): Message to log+ """+ logger.info(msg)+ if self.execution_service:+ self.execution_service.publish_log(msg)def publish_input_output_list_file_logs(- self, input_directory: str, matched_files: list[str]+ self, folders: list[str], matched_files: dict[str, FileHash], count: int) -> None:if not self.execution_service:return None- input_log = f"##Input folder:\n\n `{os.path.basename(input_directory)}`\n\n"++ folders_list = "\n".join(f"- `{folder.strip()}`" for folder in folders)+ input_log = f"##Folders to process:\n\n{folders_list}\n\n"self.execution_service.publish_update_log(state=LogState.INPUT_UPDATE, message=input_log)- output_log = self._matched_files_component_log(matched_files)+ output_log = self._matched_files_component_log(matched_files, count)self.execution_service.publish_update_log(state=LogState.OUTPUT_UPDATE, message=output_log)def publish_input_file_content(self, input_file_path: str, input_text: str) -> None:- if self.execution_service:- output_log_message = f"##Input text:\n\n```text\n{input_text}\n```\n\n"- input_log_message = (- "##Input file:\n\n```text\n"- f"{os.path.basename(input_file_path)}\n```\n\n"- )- self.execution_service.publish_update_log(- state=LogState.INPUT_UPDATE, message=input_log_message- )- self.execution_service.publish_update_log(- state=LogState.OUTPUT_UPDATE, message=output_log_message- )+ if not self.execution_service:+ return None+ output_log_message = f"##Input text:\n\n```text\n{input_text}\n```\n\n"+ input_log_message = (+ "##Input file:\n\n```text\n" f"{os.path.basename(input_file_path)}\n```\n\n"+ )+ self.execution_service.publish_update_log(+ state=LogState.INPUT_UPDATE, message=input_log_message+ )+ self.execution_service.publish_update_log(+ state=LogState.OUTPUT_UPDATE, message=output_log_message+ )- def _matched_files_component_log(self, matched_files: list[str]) -> str:+ def _matched_files_component_log(+ self, matched_files: dict[str, FileHash], count: int+ ) -> str:output_log = "### Matched files \n```text\n\n\n"- for file in matched_files[:20]:- output_log += f"{file}\n"+ for file_path in islice(matched_files.keys(), 20):+ output_log += f"- {file_path}\n"output_log += "```\n\n"- output_log += f"""Total matched files: {len(matched_files)}+ output_log += f"""Total matched files: {count}\n\nPlease note that only the first 20 files are shown.\n\n"""return output_log@@ -222,13 +277,13 @@ class SourceConnector(BaseConnector):patterns: list[str],recursive: bool,limit: int,- ) -> list[str]:- """Get a list of matched files based on patterns in a directory.+ ) -> tuple[dict[str, FileHash], int]:+ """Get a dictionary of matched files based on patterns in a directory.This method searches for files in the specified `input_directory` that- match any of the given `patterns`.- The search can be performed recursively if `recursive` is set to True.- The number of matched files returned is limited by `limit`.+ match any of the given `patterns`. The search can be performed recursively+ if `recursive` is set to True. The number of matched files returned is+ limited by `limit`.Args:source_fs (Any): The file system object used for searching.@@ -238,41 +293,81 @@ class SourceConnector(BaseConnector):limit (int): The maximum number of matched files to return.Returns:- list[str]: A list of matched file paths.+ tuple[dict[str, FileHash], int]: A dictionary of matched file paths+ and their corresponding FileHash objects, along with the total count+ of matched files."""- matched_files = []+ matched_files: dict[str, FileHash] = {}count = 0max_depth = int(SourceConstant.MAX_RECURSIVE_DEPTH) if recursive else 1for root, dirs, files in source_fs.walk(input_directory, maxdepth=max_depth):- if count >= limit:- breakfor file in files:- if not file:- continueif count >= limit:break- if any(fnmatch.fnmatch(file, pattern) for pattern in patterns):- file_path = os.path.join(root, file)- file_path = f"{file_path}"- matched_files.append(file_path)- count += 1+ if self._should_process_file(file, patterns):+ file_path = str(os.path.join(root, file))+ if self._is_new_file(+ file_path=file_path, workflow=self.endpoint.workflow+ ):+ matched_files[file_path] = self._create_file_hash(file_path)+ count += 1++ return matched_files, count++ def _should_process_file(self, file: str, patterns: list[str]) -> bool:+ """Check if the file should be processed based on the patterns."""+ return bool(file) and any(+ fnmatch.fnmatchcase(file.lower(), pattern.lower()) for pattern in patterns+ )- return matched_files+ def _is_new_file(self, file_path: str, workflow: Workflow) -> bool:+ """Check if the file is new or already processed."""+ file_content = self.get_file_content(input_file_path=file_path)+ file_hash = self.get_hash_value(file_content)+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=file_hash+ )- def list_files_from_source(self) -> list[str]:+ if file_history and file_history.is_completed():+ self.execution_service.publish_log(+ f"Skipping file {file_path} as it has already been processed. "+ "Clear the file markers to process it again."+ )+ return False++ return True++ def _create_file_hash(self, file_path: str) -> FileHash:+ """Create a FileHash object for the matched file."""+ file_name = os.path.basename(file_path)+ file_content = self.get_file_content(input_file_path=file_path)+ file_hash = self.get_hash_value(file_content)+ connection_type = self.endpoint.connection_type++ return FileHash(+ file_path=file_path,+ source_connection_type=connection_type,+ file_name=file_name,+ file_hash=file_hash,+ )++ def list_files_from_source(+ self, file_hashes: dict[str, FileHash] = {}+ ) -> tuple[dict[str, FileHash], int]:"""List files from source connector.Args:api_storage_dir (Optional[str], optional): API storage directoryReturns:- list[str]: list of files+ tuple[dict[str, FileHash], int]: A dictionary of FileHashes,+ along with the total count of matched files."""connection_type = self.endpoint.connection_typeif connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:return self.list_files_from_file_connector()elif connection_type == WorkflowEndpoint.ConnectionType.API:- return self.list_file_from_api_storage()+ return self.list_file_from_api_storage(file_hashes)raise InvalidSourceConnectionType()@classmethod@@ -301,44 +396,86 @@ class SourceConnector(BaseConnector):else:raise ValueError(f"Unsupported hash_method: {hash_method}")+ def get_file_content(self, input_file_path: str, chunk_size: int = 8192) -> bytes:+ """Read the content of a file from a remote filesystem in chunks.++ Args:+ input_file_path (str): The path of the input file.+ chunk_size (int): The size of the chunks to read at a time+ (default is 8192 bytes).++ Returns:+ bytes: The content of the file.+ """+ connector: ConnectorInstance = self.endpoint.connector_instance+ connector_settings: dict[str, Any] = connector.connector_metadata+ source_fs = self.get_fsspec(+ settings=connector_settings, connector_id=connector.connector_id+ )+ file_content = bytearray() # Use bytearray for efficient byte concatenation+ with source_fs.open(input_file_path, "rb") as remote_file:+ while chunk := remote_file.read(chunk_size):+ file_content.extend(chunk)++ return bytes(file_content)++ def get_hash_value(self, file_content: bytes) -> str:+ """Generate a hash value from the file content.++ Args:+ file_content (bytes): The content of the file.++ Returns:+ str: The hash value of the file content.+ """+ return self.hash_str(file_content)++ def copy_file_to_infile_dir(self, source_file_path: str, infile_path: str) -> None:+ """Copy the source file to the infile directory.++ Args:+ source_file_path (str): The path of the source file.+ infile_path (str): The destination path in the infile directory.+ """+ shutil.copyfile(source_file_path, infile_path)+ logger.info(f"File copied from {source_file_path} to {infile_path}")+def add_input_from_connector_to_volume(self, input_file_path: str) -> str:"""Add input file to execution directory.Args:input_file_path (str): The path of the input file.+Returns:str: The hash value of the file content.+Raises:- FileHashNotFound: If the hash value of the file content- is not found.+ FileHashNotFound: If the hash value of the file content is not found."""- connector: ConnectorInstance = self.endpoint.connector_instance- connector_settings: dict[str, Any] = connector.connector_metadatasource_file_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)source_file = f"file://{source_file_path}"- source_fs = self.get_fsspec(- settings=connector_settings, connector_id=connector.connector_id+ # Get file content and hash value+ file_content = self.get_file_content(input_file_path)+ hash_value_of_file_content = self.get_hash_value(file_content)++ logger.info(+ f"hash_value_of_file {source_file} is : {hash_value_of_file_content}")- with (- source_fs.open(input_file_path, "rb") as remote_file,- fsspec.open(source_file, "wb") as local_file,- ):- file_content = remote_file.read()- hash_value_of_file_content = self.hash_str(file_content)- logger.info(- f"hash_value_of_file {source_file} is "- f": {hash_value_of_file_content}"- )- input_log = (- file_content[:500].decode("utf-8", errors="replace") + "...(truncated)"- )- self.publish_input_file_content(input_file_path, input_log)+ input_log = (+ file_content[:500].decode("utf-8", errors="replace") + "...(truncated)"+ )+ self.publish_input_file_content(input_file_path, input_log)++ with fsspec.open(source_file, "wb") as local_file:local_file.write(file_content)- shutil.copyfile(source_file_path, infile_path)- logger.info(f"{input_file_path} is added in to execution directory")++ # Copy file to infile directory+ self.copy_file_to_infile_dir(source_file_path, infile_path)++ logger.info(f"{input_file_path} is added to execution directory")return hash_value_of_file_contentdef add_input_from_api_storage_to_volume(self, input_file_path: str) -> None:@@ -349,7 +486,7 @@ class SourceConnector(BaseConnector):shutil.copyfile(input_file_path, source_path)def add_file_to_volume(- self, input_file_path: str, hash_values_of_files: dict[str, str] = {}+ self, input_file_path: str, file_hash: FileHash) -> tuple[str, str]:"""Add input file to execution directory.@@ -368,18 +505,20 @@ class SourceConnector(BaseConnector):file_content_hash = self.add_input_from_connector_to_volume(input_file_path=input_file_path,)+ if file_content_hash != file_hash.file_hash:+ raise FileHashMismatched()elif connection_type == WorkflowEndpoint.ConnectionType.API:self.add_input_from_api_storage_to_volume(input_file_path=input_file_path)- if file_name not in hash_values_of_files:+ if file_name != file_hash.file_name:raise FileHashNotFound()- file_content_hash = hash_values_of_files[file_name]+ file_content_hash = file_hash.file_hashelse:raise InvalidSourceConnectionType()self.add_metadata_to_volume(input_file_path=input_file_path, source_hash=file_content_hash)- return file_name, file_content_hash+ return file_namedef handle_final_result(self,@@ -417,7 +556,7 @@ class SourceConnector(BaseConnector):@classmethoddef add_input_file_to_api_storage(cls, workflow_id: str, execution_id: str, file_objs: list[UploadedFile]- ) -> dict[str, str]:+ ) -> dict[str, FileHash]:"""Add input file to api storage.Args:@@ -430,7 +569,8 @@ class SourceConnector(BaseConnector):api_storage_dir = cls.get_api_storage_dir_path(workflow_id=workflow_id, execution_id=execution_id)- file_hashes: dict[str, str] = {}+ workflow: Workflow = Workflow.objects.get(id=workflow_id)+ file_hashes: dict[str, FileHash] = {}for file in file_objs:file_name = file.namedestination_path = os.path.join(api_storage_dir, file_name)@@ -440,7 +580,22 @@ class SourceConnector(BaseConnector):for chunk in file.chunks():buffer.extend(chunk)f.write(buffer)- file_hashes.update({file_name: cls.hash_str(buffer)})+ file_hash = cls.hash_str(buffer)+ connection_type = WorkflowEndpoint.ConnectionType.API+ file_history = FileHistoryHelper.get_file_history(+ workflow=workflow, cache_key=file_hash+ )+ is_executed = (+ True if file_history and file_history.is_completed() else False+ )+ file_hash = FileHash(+ file_path=destination_path,+ source_connection_type=connection_type,+ file_name=file_name,+ file_hash=file_hash,+ is_executed=is_executed,+ )+ file_hashes.update({file_name: file_hash})return file_hashes@classmethod