Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Un 2271 optimization of workflow file listing (#1221)

ali648ee6d15b31
diff --git a/backend/connector_v2/models.py b/backend/connector_v2/models.py
index cb28b1332..b2613ba73 100644
--- a/backend/connector_v2/models.py
+++ b/backend/connector_v2/models.py
@@ -15,7 +15,6 @@ from utils.models.organization_mixin import (
DefaultOrganizationManagerMixin,
DefaultOrganizationMixin,
)
-from workflow_manager.workflow_v2.models import Workflow
from backend.constants import FieldLengthConstants as FLC
@@ -43,7 +42,7 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):
max_length=CONNECTOR_NAME_SIZE, null=False, blank=False
)
workflow = models.ForeignKey(
- Workflow,
+ "workflow_v2.Workflow",
on_delete=models.CASCADE,
related_name="connector_workflow",
null=False,
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index 4105e0912..1ef31ee93 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -135,6 +135,43 @@ class DestinationConnector(BaseConnector):
):
raise DestinationConnectorNotConfigured()
+ # Validate database connection if it's a database destination
+ if connection_type == WorkflowEndpoint.ConnectionType.DATABASE and connector:
+ try:
+ # Get database class and test connection
+ db_class = DatabaseUtils.get_db_class(
+ connector_id=connector.connector_id,
+ connector_settings=connector.metadata,
+ )
+ engine = db_class.get_engine()
+ if hasattr(engine, "close"):
+ engine.close()
+ except Exception as e:
+ logger.error(f"Database connection failed: {str(e)}")
+ raise
+
+ def _handle_hitl(
+ self,
+ file_name: str,
+ file_hash: FileHash,
+ workflow: Workflow,
+ input_file_path: str,
+ file_execution_id: str,
+ ) -> bool:
+ """Handles HITL processing, returning True if data was pushed to the queue."""
+ execution_result = self.get_result()
+ if WorkflowUtil.validate_db_rule(
+ execution_result, workflow, file_hash.file_destination
+ ):
+ self._push_data_to_queue(
+ file_name=file_name,
+ workflow=workflow,
+ input_file_path=input_file_path,
+ file_execution_id=file_execution_id,
+ )
+ return True
+ return False
+
def _push_data_to_queue(
self,
file_name: str,
@@ -180,17 +217,13 @@ class DestinationConnector(BaseConnector):
if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
self.copy_output_to_output_directory()
elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:
- result = self.get_result(file_history)
- if WorkflowUtil.validate_db_rule(
- result, workflow, file_hash.file_destination
+ if not self._handle_hitl(
+ file_name=file_name,
+ file_hash=file_hash,
+ workflow=workflow,
+ input_file_path=input_file_path,
+ file_execution_id=file_execution_id,
):
- self._push_data_to_queue(
- file_name,
- workflow,
- input_file_path,
- file_execution_id,
- )
- else:
self.insert_into_db(input_file_path=input_file_path)
elif connection_type == WorkflowEndpoint.ConnectionType.API:
result = self.get_result(file_history)
@@ -212,7 +245,7 @@ class DestinationConnector(BaseConnector):
if use_file_history and not file_history:
FileHistoryHelper.create_file_history(
- cache_key=file_hash.file_hash,
+ file_hash=file_hash,
workflow=workflow,
status=ExecutionStatus.COMPLETED,
result=result,
@@ -316,29 +349,53 @@ class DestinationConnector(BaseConnector):
file_path=input_file_path,
execution_id=self.execution_id,
)
- db_class = DatabaseUtils.get_db_class(
- connector_id=connector_instance.connector_id,
- connector_settings=connector_settings,
- )
- engine = db_class.get_engine()
- DatabaseUtils.create_table_if_not_exists(
- db_class=db_class,
- engine=engine,
- table_name=table_name,
- database_entry=values,
- )
- sql_columns_and_values = DatabaseUtils.get_sql_query_data(
- conn_cls=db_class,
- table_name=table_name,
- values=values,
- )
- DatabaseUtils.execute_write_query(
- db_class=db_class,
- engine=engine,
- table_name=table_name,
- sql_keys=list(sql_columns_and_values.keys()),
- sql_values=list(sql_columns_and_values.values()),
- )
+ engine = None
+ try:
+ db_class = DatabaseUtils.get_db_class(
+ connector_id=connector_instance.connector_id,
+ connector_settings=connector_settings,
+ )
+ engine = db_class.get_engine()
+ DatabaseUtils.create_table_if_not_exists(
+ db_class=db_class,
+ engine=engine,
+ table_name=table_name,
+ database_entry=values,
+ )
+ sql_columns_and_values = DatabaseUtils.get_sql_query_data(
+ conn_cls=db_class,
+ table_name=table_name,
+ values=values,
+ )
+ DatabaseUtils.execute_write_query(
+ db_class=db_class,
+ engine=engine,
+ table_name=table_name,
+ sql_keys=list(sql_columns_and_values.keys()),
+ sql_values=list(sql_columns_and_values.values()),
+ )
+ except ConnectorError as e:
+ error_msg = f"Database connection failed for {input_file_path}: {str(e)}"
+ logger.error(error_msg)
+ raise
+ except Exception as e:
+ error_msg = (
+ f"Failed to insert data into database for {input_file_path}: {str(e)}"
+ )
+ logger.error(error_msg)
+ raise
+ finally:
+ self._close_engine(engine, input_file_path)
+
+ def _close_engine(self, engine: Any, input_file_path: str) -> None:
+ """Safely close database engine."""
+ if engine:
+ try:
+ engine.close()
+ except Exception as e:
+ logger.error(
+ f"Failed to close database engine for {input_file_path}: {str(e)}"
+ )
def _handle_api_result(
self,
@@ -488,6 +545,14 @@ class DestinationConnector(BaseConnector):
file_storage = file_system.get_file_storage()
if file_storage.exists(self.execution_dir):
file_storage.rm(self.execution_dir, recursive=True)
+
+ def delete_execution_and_api_storage_dir(self) -> None:
+ """Delete the execution and api storage directories.
+
+ Returns:
+ None
+ """
+ self.delete_execution_directory()
self.delete_api_storage_dir(self.workflow_id, self.execution_id)
@classmethod
diff --git a/backend/workflow_manager/endpoint_v2/dto.py b/backend/workflow_manager/endpoint_v2/dto.py
index 6c701f2e9..9576f376a 100644
--- a/backend/workflow_manager/endpoint_v2/dto.py
+++ b/backend/workflow_manager/endpoint_v2/dto.py
@@ -6,11 +6,13 @@ from typing import Any, Optional
@dataclass
class FileHash:
file_path: str
- file_hash: str
file_name: str
source_connection_type: str
+ file_hash: Optional[str] = None
file_size: Optional[int] = None
+ provider_file_uuid: Optional[str] = None
mime_type: Optional[str] = None
+ fs_metadata: Optional[dict[str, Any]] = None
file_destination: Optional[tuple[str, str]] = (
None # To which destination this file wants to go for MRQ percentage
)
@@ -26,6 +28,8 @@ class FileHash:
"is_executed": self.is_executed,
"file_size": self.file_size,
"mime_type": self.mime_type,
+ "provider_file_uuid": self.provider_file_uuid,
+ "fs_metadata": self.fs_metadata,
}
@staticmethod
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index 619621b7f..75fb52139 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -2,16 +2,16 @@ import fnmatch
import logging
import os
import shutil
-from hashlib import md5, sha256
+from hashlib import sha256
from io import BytesIO
from itertools import islice
from typing import Any, Optional
import fsspec
-import magic
from connector_processor.constants import ConnectorKeys
from connector_v2.models import ConnectorInstance
from django.core.files.uploadedfile import UploadedFile
+from unstract.sdk.file_storage import FileStorage
from unstract.workflow_execution.enums import LogState
from utils.user_context import UserContext
from workflow_manager.endpoint_v2.base_connector import BaseConnector
@@ -25,8 +25,6 @@ from workflow_manager.endpoint_v2.constants import (
)
from workflow_manager.endpoint_v2.dto import FileHash
from workflow_manager.endpoint_v2.exceptions import (
- FileHashMismatched,
- FileHashNotFound,
InvalidInputDirectory,
InvalidSourceConnectionType,
MissingSourceConnectionType,
@@ -37,8 +35,10 @@ from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.file_execution.models import WorkflowFileExecution
from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper
from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
+from workflow_manager.workflow_v2.models.file_history import FileHistory
from workflow_manager.workflow_v2.models.workflow import Workflow
+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
from unstract.filesystem import FileStorageType, FileSystem
logger = logging.getLogger(__name__)
@@ -59,6 +59,8 @@ class SourceConnector(BaseConnector):
workflow (Workflow): The workflow associated with the source connector.
"""
+ READ_CHUNK_SIZE = 4194304 # Chunk size for reading files
+
def __init__(
self,
workflow: Workflow,
@@ -208,7 +210,7 @@ class SourceConnector(BaseConnector):
)
logger.debug(f"Listing files from: {input_directory}")
matched_files, count = self._get_matched_files(
- source_fs_fsspec, input_directory, patterns, recursive, limit
+ source_fs, input_directory, patterns, recursive, limit
)
self.publish_user_sys_log(
f"Matched '{count}' files from '{input_directory}'"
@@ -277,7 +279,7 @@ class SourceConnector(BaseConnector):
def _get_matched_files(
self,
- source_fs: Any,
+ source_fs: UnstractFileSystem,
input_directory: str,
patterns: list[str],
recursive: bool,
@@ -305,28 +307,23 @@ class SourceConnector(BaseConnector):
matched_files: dict[str, FileHash] = {}
count = 0
max_depth = int(SourceConstant.MAX_RECURSIVE_DEPTH) if recursive else 1
-
- for root, dirs, files in source_fs.walk(input_directory, maxdepth=max_depth):
+ fs_fsspec = source_fs.get_fsspec_fs()
+ for root, dirs, files in fs_fsspec.walk(input_directory, maxdepth=max_depth):
for file in files:
if count >= limit:
break
if self._should_process_file(file, patterns):
file_path = str(os.path.join(root, file))
- file_content, file_size = self.get_file_content(
- input_file_path=file_path
- )
if self._is_new_file(
file_path=file_path,
- file_content=file_content,
workflow=self.endpoint.workflow,
+ source_fs=source_fs,
):
matched_files[file_path] = self._create_file_hash(
file_path=file_path,
- file_content=file_content,
- file_size=file_size,
+ source_fs=source_fs,
)
count += 1
-
return matched_files, count
def _should_process_file(self, file: str, patterns: list[str]) -> bool:
@@ -375,14 +372,10 @@ class SourceConnector(BaseConnector):
return True
def _is_new_file(
- self, file_path: str, file_content: bytes, workflow: Workflow
+ self, file_path: str, workflow: Workflow, source_fs: UnstractFileSystem
) -> bool:
"""Check if the file is new or already processed."""
- file_hash = self.get_file_content_hash(file_content)
- file_history = FileHistoryHelper.get_file_history(
- workflow=workflow, cache_key=file_hash
- )
-
+ file_history = self._get_file_history(workflow, source_fs, file_path)
# In case of ETL pipelines, its necessary to skip files which have
# already been processed
if (
@@ -398,21 +391,69 @@ class SourceConnector(BaseConnector):
return True
+ def _get_file_history(
+ self, workflow: Workflow, source_fs: UnstractFileSystem, file_path: str
+ ) -> Optional[FileHistory]:
+ """Retrieve file history using provider UUID or legacy cache key."""
+
+ provider_file_uuid = source_fs.get_file_system_uuid(file_path)
+
+ if provider_file_uuid:
+ logger.info(
+ f"Checking file history for provider UUID: {provider_file_uuid}"
+ )
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, provider_file_uuid=provider_file_uuid
+ )
+
+ if file_history:
+ return file_history # Early return if history exists
+
+ # The provider_file_uuid was recently integrated,
+ # so we also check the cache_key for backward compatibility.
+ # This ensures older files without a provider UUID
+ # can still be identified.
+ # In the future, this check can be removed as file history validation
+ # is already handled during file execution.
+ file_content_hash = self.get_file_content_hash(source_fs, file_path)
+ logger.info(
+ f"Checking file history for legacy cache key: {file_content_hash}"
+ )
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=file_content_hash
+ )
+
+ if file_history and file_history.is_completed():
+ file_history.update(provider_file_uuid=provider_file_uuid)
+
+ # Fallback for connectors that do not support provider_file_uuid
+ else:
+ file_content_hash = self.get_file_content_hash(source_fs, file_path)
+ logger.info(
+ f"Checking file history for legacy cache key: {file_content_hash}"
+ )
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=file_content_hash
+ )
+
+ return file_history
+
def _create_file_hash(
- self, file_path: str, file_content: bytes, file_size: Optional[int]
+ self, file_path: str, source_fs: UnstractFileSystem
) -> FileHash:
"""Create a FileHash object for the matched file."""
file_name = os.path.basename(file_path)
- file_hash = self.get_file_content_hash(file_content)
+ provider_file_uuid = source_fs.get_file_system_uuid(file_path)
+ file_size = source_fs.get_file_size(file_path)
+ fs_metadata = source_fs.get_file_metadata(file_path)
connection_type = self.endpoint.connection_type
- file_type = magic.from_buffer(file_content, mime=True)
return FileHash(
file_path=file_path,
source_connection_type=connection_type,
file_name=file_name,
- file_hash=file_hash,
file_size=file_size,
- mime_type=file_type,
+ provider_file_uuid=provider_file_uuid,
+ fs_metadata=fs_metadata,
)
def list_files_from_source(
@@ -433,76 +474,25 @@ class SourceConnector(BaseConnector):
return self.list_file_from_api_storage(file_hashes)
raise InvalidSourceConnectionType()
- @classmethod
- def hash_str(cls, string_to_hash: Any, hash_method: str = "sha256") -> str:
- """Computes the hash for a given input string.
-
- Useful to hash strings needed for caching and other purposes.
- Hash method defaults to "md5"
-
- Args:
- string_to_hash (str): String to be hashed
- hash_method (str): Hash hash_method to use, supported ones
- - "md5"
-
- Returns:
- str: Hashed string
- """
- if hash_method == "md5":
- if isinstance(string_to_hash, bytes):
- return str(md5(string_to_hash).hexdigest())
- return str(md5(string_to_hash.encode()).hexdigest())
- elif hash_method == "sha256":
- if isinstance(string_to_hash, (bytes, bytearray)):
- return str(sha256(string_to_hash).hexdigest())
- return str(sha256(string_to_hash.encode()).hexdigest())
- else:
- raise ValueError(f"Unsupported hash_method: {hash_method}")
-
- def get_file_content(
- self, input_file_path: str, chunk_size: int = 4194304
- ) -> tuple[bytes, Optional[int]]:
- """Read the content of a file from a remote filesystem in chunks.
-
- Args:
- input_file_path (str): The path of the input file.
- chunk_size (int): The size of the chunks to read at a time
- (default is 4194304 bytes = 4 MB).
-
- Returns:
- tuple[bytes, int]:
- A tuple containing the content of the file as bytes and
- its size in bytes.
- """
- connector: ConnectorInstance = self.endpoint.connector_instance
- connector_settings: dict[str, Any] = connector.connector_metadata
- source_fs = self.get_fsspec(
- settings=connector_settings, connector_id=connector.connector_id
- )
-
- # Get file size
- file_metadata = source_fs.stat(input_file_path)
- file_size: Optional[int] = file_metadata.get("size")
- if file_size is None:
- logger.warning(f"File size for {input_file_path} could not be determined.")
-
- file_content = bytearray() # Use bytearray for efficient byte concatenation
- with source_fs.open(input_file_path, "rb") as remote_file:
- while chunk := remote_file.read(chunk_size):
- file_content.extend(chunk)
-
- return bytes(file_content), file_size
-
- def get_file_content_hash(self, file_content: bytes) -> str:
+ def get_file_content_hash(
+ self, source_fs: UnstractFileSystem, file_path: str
+ ) -> str:
"""Generate a hash value from the file content.
Args:
- file_content (bytes): The content of the file.
+ source_fs (UnstractFileSystem): The file system object used for
+ reading the file.
+ file_path (str): The path of the file.
Returns:
str: The hash value of the file content.
"""
- return self.hash_str(file_content)
+ file_content_hash = sha256()
+ source = source_fs.get_fsspec_fs()
+ with source.open(file_path, "rb") as remote_file:
+ while chunk := remote_file.read(self.READ_CHUNK_SIZE):
+ file_content_hash.update(chunk)
+ return file_content_hash.hexdigest()
def copy_file_to_infile_dir(self, source_file_path: str, infile_path: str) -> None:
"""Copy the source file to the infile directory.
@@ -514,11 +504,11 @@ class SourceConnector(BaseConnector):
shutil.copyfile(source_file_path, infile_path)
logger.info(f"File copied from {source_file_path} to {infile_path}")
- def add_input_from_connector_to_volume(self, input_file_path: str) -> str:
+ def add_input_from_connector_to_volume(self, file_hash: FileHash) -> str:
"""Add input file to execution directory.
Args:
- input_file_path (str): The path of the input file.
+ file_hash (FileHash): The file hash object.
Returns:
str: The hash value of the file content.
@@ -526,32 +516,49 @@ class SourceConnector(BaseConnector):
Raises:
FileHashNotFound: If the hash value of the file content is not found.
"""
- source_file_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)
- infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
- source_file = f"file://{source_file_path}"
+ input_file_path = file_hash.file_path
+ source_file_path = self.source_file
+ infile_path = self.infile
+ connector: ConnectorInstance = self.endpoint.connector_instance
+ connector_settings: dict[str, Any] = connector.connector_metadata
+ source_fs = self.get_fsspec(
+ settings=connector_settings, connector_id=connector.connector_id
+ )
- # Get file content and hash value
- file_content, _ = self.get_file_content(input_file_path)
- hash_value_of_file_content = self.get_file_content_hash(file_content)
+ workflow_file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
+ workflow_file_storage = workflow_file_system.get_file_storage()
- logger.info(
- f"hash_value_of_file {source_file} is : {hash_value_of_file_content}"
- )
+ first_iteration = True
+ input_log = ""
+ file_hash = sha256()
+ with source_fs.open(input_file_path, "rb") as remote_file:
+ while chunk := remote_file.read(self.READ_CHUNK_SIZE):
+ file_hash.update(chunk)
+ if first_iteration:
+ input_log = (
+ chunk[:500].decode("utf-8", errors="replace") + "...(truncated)"
+ )
+ first_iteration = False
+ # write the chunk in to execution directory
+ workflow_file_storage.write(
+ path=source_file_path, mode="ab", data=chunk
+ )
+ workflow_file_storage.write(path=infile_path, mode="ab", data=chunk)
- input_log = (
- file_content[:500].decode("utf-8", errors="replace") + "...(truncated)"
- )
+ # publish input file content
+ # TODO: Consider removing this since the input is not extracted text.
+ # This function is typically relevant for extracted text content,
+ # may not be necessary for PDFs, images, or other non-text formats.
self.publish_input_file_content(input_file_path, input_log)
-
- file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
- file_storage = file_system.get_file_storage()
- file_storage.write(path=source_file_path, mode="wb", data=file_content)
- file_storage.write(path=infile_path, mode="wb", data=file_content)
+ hash_value_of_file_content = file_hash.hexdigest()
+ logger.info(
+ f"hash_value_of_file {source_file_path} is : {hash_value_of_file_content}"
+ )
logger.info(f"{input_file_path} is added to execution directory")
return hash_value_of_file_content
- def add_input_from_api_storage_to_volume(self, input_file_path: str) -> None:
+ def add_input_from_api_storage_to_volume(self, input_file_path: str) -> str:
"""Add input file to execution directory from api storage."""
infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
source_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)
@@ -560,22 +567,23 @@ class SourceConnector(BaseConnector):
api_file_storage = api_file_system.get_file_storage()
workflow_file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
workflow_file_storage = workflow_file_system.get_file_storage()
- self._copy_file_to_destination(
+ file_content_hash = self._copy_file_to_destination(
source_storage=api_file_storage,
destination_storage=workflow_file_storage,
source_path=input_file_path,
destination_paths=[infile_path, source_path],
)
+ logger.info(f"File {input_file_path} added to execution directory")
+ return file_content_hash
# TODO: replace it with method from SDK Utils
def _copy_file_to_destination(
self,
- source_storage: Any,
- destination_storage: Any,
+ source_storage: FileStorage,
+ destination_storage: FileStorage,
source_path: str,
destination_paths: list[str],
- chunk_size: int = 4194304,
- ) -> None:
+ ) -> str:
"""
Copy a file from a source storage to one or more paths in a
destination storage.
@@ -592,49 +600,43 @@ class SourceConnector(BaseConnector):
source_path (str): The path of the file in the source storage.
destination_paths (list[str]): A list of paths where the file will be
copied in the destination storage.
- chunk_size (int, optional): The number of bytes to read per chunk.
- (default is 4194304 bytes = 4 MB).
+ Returns:
+ str: The SHA-256 hash of the file content.
"""
seek_position = 0 # Start from the beginning
- end_of_file = False
-
+ file_content_hash = sha256()
# Loop to read and write in chunks until the end of the file
- while not end_of_file:
- # Read a chunk from the source file
- chunk = source_storage.read(
- path=source_path,
- mode="rb",
- seek_position=seek_position,
- length=chunk_size,
- )
- # Check if the end of the file has been reached
- if not chunk:
- end_of_file = True
- else:
- # Write the chunk to each destination path
- for destination_file in destination_paths:
- destination_storage.write(
- path=destination_file,
- mode="ab",
- seek_position=seek_position,
- data=chunk,
- )
+ while chunk := source_storage.read(
+ path=source_path,
+ mode="rb",
+ seek_position=seek_position,
+ length=self.READ_CHUNK_SIZE,
+ ):
+ file_content_hash.update(chunk)
+ # Write the chunk to each destination path
+ for destination_file in destination_paths:
+ destination_storage.write(
+ path=destination_file,
+ mode="ab",
+ data=chunk,
+ )
- # Update the seek position
- seek_position += len(chunk)
+ # Update the seek position
+ seek_position += len(chunk)
+ return file_content_hash.hexdigest()
def add_file_to_volume(
self,
- input_file_path: str,
workflow_file_execution: WorkflowFileExecution,
- tags: list[str] = [],
+ tags: list[str],
+ file_hash: FileHash,
) -> str:
"""Add input file to execution directory.
Args:
- input_file_path (str): source file
workflow_file_execution: WorkflowFileExecution model
tags (list[str]): Tag names associated with the workflow execution.
+ file_hash: FileHash model
Raises:
InvalidSource: _description_
@@ -643,18 +645,15 @@ class SourceConnector(BaseConnector):
str: file_name
"""
connection_type = self.endpoint.connection_type
- file_name = os.path.basename(input_file_path)
+ input_file_path = file_hash.file_path
if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
file_content_hash = self.add_input_from_connector_to_volume(
- input_file_path=input_file_path,
+ file_hash=file_hash
)
- if file_content_hash != workflow_file_execution.file_hash:
- raise FileHashMismatched()
elif connection_type == WorkflowEndpoint.ConnectionType.API:
- self.add_input_from_api_storage_to_volume(input_file_path=input_file_path)
- if file_name != workflow_file_execution.file_name:
- raise FileHashNotFound()
- file_content_hash = workflow_file_execution.file_hash
+ file_content_hash = self.add_input_from_api_storage_to_volume(
+ input_file_path=input_file_path
+ )
else:
raise InvalidSourceConnectionType()
@@ -664,7 +663,7 @@ class SourceConnector(BaseConnector):
source_hash=file_content_hash,
tags=tags,
)
- return file_name
+ return file_content_hash
def handle_final_result(
self,
@@ -729,11 +728,11 @@ class SourceConnector(BaseConnector):
file_system = FileSystem(FileStorageType.API_EXECUTION)
file_storage = file_system.get_file_storage()
- buffer = bytearray()
- for chunk in file.chunks():
- buffer.extend(chunk)
- file_storage.write(path=destination_path, mode="wb", data=buffer)
- file_hash = cls.hash_str(buffer)
+ file_hash = sha256()
+ for chunk in file.chunks(chunk_size=cls.READ_CHUNK_SIZE):
+ file_hash.update(chunk)
+ file_storage.write(path=destination_path, mode="ab", data=chunk)
+ file_hash = file_hash.hexdigest()
connection_type = WorkflowEndpoint.ConnectionType.API
file_history = None
diff --git a/backend/workflow_manager/file_execution/migrations/0005_workflowfileexecution_fs_metadata_and_more.py b/backend/workflow_manager/file_execution/migrations/0005_workflowfileexecution_fs_metadata_and_more.py
new file mode 100644
index 000000000..9ff5b951c
--- /dev/null
+++ b/backend/workflow_manager/file_execution/migrations/0005_workflowfileexecution_fs_metadata_and_more.py
@@ -0,0 +1,51 @@
+# Generated by Django 4.2.1 on 2025-04-02 13:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("file_execution", "0004_alter_workflowfileexecution_status"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="workflowfileexecution",
+ name="fs_metadata",
+ field=models.JSONField(
+ db_comment="Complete metadata of the file retrieved from the file system.",
+ null=True,
+ ),
+ ),
+ migrations.AddField(
+ model_name="workflowfileexecution",
+ name="provider_file_uuid",
+ field=models.CharField(
+ db_comment="Unique identifier assigned by the file storage provider",
+ max_length=64,
+ null=True,
+ ),
+ ),
+ migrations.AlterField(
+ model_name="workflowfileexecution",
+ name="file_hash",
+ field=models.CharField(
+ db_comment="Hash of the file content", max_length=64, null=True
+ ),
+ ),
+ migrations.AddIndex(
+ model_name="workflowfileexecution",
+ index=models.Index(
+ fields=["workflow_execution", "provider_file_uuid"],
+ name="workflow_exec_p_uuid_idx",
+ ),
+ ),
+ migrations.AddConstraint(
+ model_name="workflowfileexecution",
+ constraint=models.UniqueConstraint(
+ fields=("workflow_execution", "provider_file_uuid", "file_path"),
+ name="unique_workflow_provider_uuid_path",
+ ),
+ ),
+ ]
diff --git a/backend/workflow_manager/file_execution/models.py b/backend/workflow_manager/file_execution/models.py
index 9c58a9158..20850909c 100644
--- a/backend/workflow_manager/file_execution/models.py
+++ b/backend/workflow_manager/file_execution/models.py
@@ -1,12 +1,13 @@
import uuid
from datetime import timedelta
-from typing import Optional
+from typing import Any
from django.db import models
from utils.common_utils import CommonUtils
from utils.models.base_model import BaseModel
+from workflow_manager.endpoint_v2.dto import FileHash
+from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.workflow_v2.enums import ExecutionStatus
-from workflow_manager.workflow_v2.models.execution import WorkflowExecution
FILE_NAME_LENGTH = 255
FILE_PATH_LENGTH = 255
@@ -17,46 +18,59 @@ MIME_TYPE_LENGTH = 128
class WorkflowFileExecutionManager(models.Manager):
def get_or_create_file_execution(
self,
- workflow_execution: WorkflowExecution,
- file_name: str,
- file_size: int,
- file_hash: str,
- file_path: Optional[str] = None,
- mime_type: Optional[str] = None,
+ workflow_execution: Any,
+ file_hash: FileHash,
+ connection_type: WorkflowEndpoint.ConnectionType,
) -> "WorkflowFileExecution":
"""
Retrieves or creates a new input file record for a workflow execution.
Args:
- workflow_execution: The `WorkflowExecution` object associated with this file
- file_name: The name of the input file
- file_size: The size of the file in bytes
- file_hash: The hash of the file content
- file_path: (Optional) The full path of the input file
- mime_type: (Optional) MIME type of the file
+ workflow_execution: The `WorkflowExecution` object
+ associated with this file.
+ file_hash: The `FileHash` object containing file metadata.
+ file_path: (Optional) The full path of the input file.
- return:
- The `WorkflowFileExecution` object
+ Returns:
+ The `WorkflowFileExecution` object.
"""
- execution_file: WorkflowFileExecution
- execution_file, is_created = self.get_or_create(
- workflow_execution=workflow_execution,
- file_hash=file_hash,
- file_path=file_path,
- )
+ is_api = connection_type == WorkflowEndpoint.ConnectionType.API
+ # Determine file path based on connection type
+ execution_file_path = file_hash.file_path if not is_api else None
+
+ lookup_fields = {
+ "workflow_execution": workflow_execution,
+ "file_path": execution_file_path,
+ }
+
+ if file_hash.file_hash:
+ lookup_fields["file_hash"] = file_hash.file_hash
+ elif file_hash.provider_file_uuid:
+ lookup_fields["provider_file_uuid"] = file_hash.provider_file_uuid
+
+ execution_file, is_created = self.get_or_create(**lookup_fields)
if is_created:
- execution_file.file_name = file_name
- execution_file.file_size = file_size
- execution_file.mime_type = mime_type
- execution_file.save()
+ self._update_execution_file(execution_file, file_hash)
+
return execution_file
+ def _update_execution_file(
+ self, execution_file: "WorkflowFileExecution", file_hash: FileHash
+ ) -> None:
+ """Updates the attributes of a newly created WorkflowFileExecution object."""
+ execution_file.file_name = file_hash.file_name
+ execution_file.file_size = file_hash.file_size
+ execution_file.mime_type = file_hash.mime_type
+ execution_file.provider_file_uuid = file_hash.provider_file_uuid
+ execution_file.fs_metadata = file_hash.fs_metadata
+ execution_file.save()
+
class WorkflowFileExecution(BaseModel):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
workflow_execution = models.ForeignKey(
- WorkflowExecution,
+ "workflow_v2.WorkflowExecution",
on_delete=models.CASCADE,
db_index=True,
editable=False,
@@ -73,7 +87,16 @@ class WorkflowFileExecution(BaseModel):
null=True, db_comment="Size of the file in bytes"
)
file_hash = models.CharField(
- max_length=HASH_LENGTH, db_comment="Hash of the file content"
+ max_length=HASH_LENGTH, null=True, db_comment="Hash of the file content"
+ )
+ provider_file_uuid = models.CharField(
+ max_length=HASH_LENGTH,
+ null=True,
+ db_comment="Unique identifier assigned by the file storage provider",
+ )
+ fs_metadata = models.JSONField(
+ null=True,
+ db_comment="Complete metadata of the file retrieved from the file system.",
)
mime_type = models.CharField(
max_length=MIME_TYPE_LENGTH,
@@ -167,10 +190,52 @@ class WorkflowFileExecution(BaseModel):
fields=["workflow_execution", "file_hash"],
name="workflow_file_hash_idx",
),
+ models.Index(
+ fields=["workflow_execution", "provider_file_uuid"],
+ name="workflow_exec_p_uuid_idx",
+ ),
]
constraints = [
models.UniqueConstraint(
fields=["workflow_execution", "file_hash", "file_path"],
name="unique_workflow_file_hash_path",
),
+ models.UniqueConstraint(
+ fields=["workflow_execution", "provider_file_uuid", "file_path"],
+ name="unique_workflow_provider_uuid_path",
+ ),
]
+
+ @property
+ def is_completed(self) -> bool:
+ """Check if the execution status is completed.
+
+ Returns:
+ bool: True if the execution status is completed, False otherwise.
+ """
+ return self.status is not None and self.status == ExecutionStatus.COMPLETED
+
+ def update(
+ self,
+ file_hash: str = None,
+ fs_metadata: dict[str, Any] = None,
+ ) -> None:
+ """
+ Updates the file execution details.
+
+ Args:
+ file_hash: (Optional) Hash of the file content
+
+ Returns:
+ None
+ """
+ update_fields = []
+
+ if file_hash is not None:
+ self.file_hash = file_hash
+ update_fields.append("file_hash")
+ if fs_metadata is not None:
+ self.fs_metadata = fs_metadata
+ update_fields.append("fs_metadata")
+ if update_fields: # Save only if there's an actual update
+ self.save(update_fields=update_fields)
diff --git a/backend/workflow_manager/workflow_v2/file_history_helper.py b/backend/workflow_manager/workflow_v2/file_history_helper.py
index 820146004..7d9c26872 100644
--- a/backend/workflow_manager/workflow_v2/file_history_helper.py
+++ b/backend/workflow_manager/workflow_v2/file_history_helper.py
@@ -2,6 +2,7 @@ import logging
from typing import Any, Optional
from django.db.utils import IntegrityError
+from workflow_manager.endpoint_v2.dto import FileHash
from workflow_manager.workflow_v2.enums import ExecutionStatus
from workflow_manager.workflow_v2.models.file_history import FileHistory
from workflow_manager.workflow_v2.models.workflow import Workflow
@@ -14,36 +15,41 @@ class FileHistoryHelper:
@staticmethod
def get_file_history(
- workflow: Workflow, cache_key: Optional[str] = None
+ workflow: Workflow,
+ cache_key: Optional[str] = None,
+ provider_file_uuid: Optional[str] = None,
) -> Optional[FileHistory]:
"""Retrieve a file history record based on the cache key.
Args:
cache_key (Optional[str]): The cache key to search for.
+ provider_file_uuid (Optional[str]): The provider file UUID to search for.
Returns:
Optional[FileHistory]: The matching file history record, if found.
"""
- if not cache_key:
+
+ if not cache_key and not provider_file_uuid:
return None
try:
- file_history: FileHistory = FileHistory.objects.get(
- cache_key=cache_key, workflow=workflow
- )
+ if not cache_key:
+ return FileHistory.objects.get(
+ provider_file_uuid=provider_file_uuid, workflow=workflow
+ )
+ return FileHistory.objects.get(cache_key=cache_key, workflow=workflow)
except FileHistory.DoesNotExist:
return None
- return file_history
@staticmethod
def create_file_history(
- cache_key: str,
+ file_hash: FileHash,
workflow: Workflow,
status: ExecutionStatus,
result: Any,
metadata: Any,
error: Optional[str] = None,
file_name: Optional[str] = None,
- ) -> FileHistory:
+ ) -> None:
"""Create a new file history record.
Args:
@@ -51,31 +57,23 @@ class FileHistoryHelper:
workflow (Workflow): The associated workflow.
status (ExecutionStatus): The execution status.
result (Any): The result from the execution.
-
- Returns:
- FileHistory: The newly created file history record.
"""
try:
- file_history: FileHistory = FileHistory.objects.create(
+ FileHistory.objects.create(
workflow=workflow,
- cache_key=cache_key,
+ cache_key=file_hash.file_hash,
+ provider_file_uuid=file_hash.provider_file_uuid,
status=status,
result=str(result),
metadata=str(metadata),
error=str(error) if error else "",
)
- except IntegrityError:
+ except IntegrityError as e:
# TODO: Need to find why duplicate insert is coming
logger.warning(
- "Trying to insert duplication data for filename %s for workflow %s",
- file_name,
- workflow,
+ f"Trying to insert duplication data for filename {file_name} "
+ f"for workflow {workflow}. Error: {str(e)} with metadata {metadata}",
)
- file_history = FileHistoryHelper.get_file_history(
- workflow=workflow, cache_key=cache_key
- )
-
- return file_history
@staticmethod
def clear_history_for_workflow(
diff --git a/backend/workflow_manager/workflow_v2/migrations/0011_remove_filehistory_unique_workflow_cachekey_and_more.py b/backend/workflow_manager/workflow_v2/migrations/0011_remove_filehistory_unique_workflow_cachekey_and_more.py
new file mode 100644
index 000000000..a4ed9c412
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/migrations/0011_remove_filehistory_unique_workflow_cachekey_and_more.py
@@ -0,0 +1,42 @@
+# Generated by Django 4.2.1 on 2025-04-02 13:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("workflow_v2", "0010_executionlog_wf_execution_and_more"),
+ ]
+
+ operations = [
+ migrations.RemoveConstraint(
+ model_name="filehistory",
+ name="unique_workflow_cacheKey",
+ ),
+ migrations.AddField(
+ model_name="filehistory",
+ name="provider_file_uuid",
+ field=models.CharField(
+ db_comment="Unique identifier assigned by the file storage provider",
+ max_length=64,
+ null=True,
+ ),
+ ),
+ migrations.AddConstraint(
+ model_name="filehistory",
+ constraint=models.UniqueConstraint(
+ condition=models.Q(("cache_key__isnull", False)),
+ fields=("workflow", "cache_key"),
+ name="unique_workflow_cacheKey",
+ ),
+ ),
+ migrations.AddConstraint(
+ model_name="filehistory",
+ constraint=models.UniqueConstraint(
+ condition=models.Q(("provider_file_uuid__isnull", False)),
+ fields=("workflow", "provider_file_uuid"),
+ name="unique_workflow_providerFileUUID",
+ ),
+ ),
+ ]
diff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.py
index dcf3043e3..67a099b1e 100644
--- a/backend/workflow_manager/workflow_v2/models/execution.py
+++ b/backend/workflow_manager/workflow_v2/models/execution.py
@@ -104,21 +104,21 @@ class WorkflowExecution(BaseModel):
return list(self.tags.values_list("name", flat=True))
@property
- def workflow(self) -> Optional[str]:
- """Obtains the workflow associated to this execution."""
+ def workflow_name(self) -> Optional[str]:
+ """Obtains the workflow's name associated to this execution."""
try:
- return Workflow.objects.get(id=self.workflow_id)
+ return Workflow.objects.get(id=self.workflow_id).workflow_name
except ObjectDoesNotExist:
logger.warning(
- f"Expected workflow '{self.workflow_id}' to exist but missing"
+ f"Expected workflow ID '{self.workflow_id}' to exist but missing"
)
return None
@property
- def workflow_name(self) -> Optional[str]:
- """Obtains the workflow's name associated to this execution."""
+ def workflow(self) -> Optional[Workflow]:
+ """Obtains the workflow associated to this execution."""
try:
- return Workflow.objects.get(id=self.workflow_id).workflow_name
+ return Workflow.objects.get(id=self.workflow_id)
except ObjectDoesNotExist:
logger.warning(
f"Expected workflow ID '{self.workflow_id}' to exist but missing"
diff --git a/backend/workflow_manager/workflow_v2/models/execution_log.py b/backend/workflow_manager/workflow_v2/models/execution_log.py
index 0093d6529..acecfe66a 100644
--- a/backend/workflow_manager/workflow_v2/models/execution_log.py
+++ b/backend/workflow_manager/workflow_v2/models/execution_log.py
@@ -2,8 +2,6 @@ import uuid
from django.db import models
from utils.models.base_model import BaseModel
-from workflow_manager.file_execution.models import WorkflowFileExecution
-from workflow_manager.workflow_v2.models import WorkflowExecution
class ExecutionLog(BaseModel):
@@ -17,7 +15,7 @@ class ExecutionLog(BaseModel):
null=True,
)
wf_execution = models.ForeignKey(
- WorkflowExecution,
+ "workflow_v2.WorkflowExecution",
on_delete=models.CASCADE,
editable=False,
db_comment="Foreign key from WorkflowExecution model",
@@ -26,7 +24,7 @@ class ExecutionLog(BaseModel):
blank=True,
)
file_execution = models.ForeignKey(
- WorkflowFileExecution,
+ "file_execution.WorkflowFileExecution",
on_delete=models.CASCADE,
db_index=True,
editable=False,
diff --git a/backend/workflow_manager/workflow_v2/models/file_history.py b/backend/workflow_manager/workflow_v2/models/file_history.py
index afbf8587f..17a270228 100644
--- a/backend/workflow_manager/workflow_v2/models/file_history.py
+++ b/backend/workflow_manager/workflow_v2/models/file_history.py
@@ -1,4 +1,5 @@
import uuid
+from typing import Optional
from django.db import models
from utils.models.base_model import BaseModel
@@ -22,6 +23,11 @@ class FileHistory(BaseModel):
max_length=HASH_LENGTH,
db_comment="Hash value of file contents, WF and tool modified times",
)
+ provider_file_uuid = models.CharField(
+ max_length=HASH_LENGTH,
+ null=True,
+ db_comment="Unique identifier assigned by the file storage provider",
+ )
workflow = models.ForeignKey(
Workflow,
on_delete=models.CASCADE,
@@ -47,5 +53,32 @@ class FileHistory(BaseModel):
models.UniqueConstraint(
fields=["workflow", "cache_key"],
name="unique_workflow_cacheKey",
+ condition=models.Q(cache_key__isnull=False),
+ ),
+ models.UniqueConstraint(
+ fields=["workflow", "provider_file_uuid"],
+ name="unique_workflow_providerFileUUID",
+ condition=models.Q(provider_file_uuid__isnull=False),
),
]
+
+ def update(
+ self,
+ provider_file_uuid: Optional[str] = None,
+ ) -> None:
+ """
+ Updates the file execution details.
+
+ Args:
+ provider_file_uuid: (Optional) UUID of the file in the storage provider
+
+ Returns:
+ None
+ """
+ update_fields = []
+
+ if provider_file_uuid is not None:
+ self.provider_file_uuid = provider_file_uuid
+ update_fields.append("provider_file_uuid")
+ if update_fields: # Save only if there's an actual update
+ self.save(update_fields=update_fields)
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index 1417ff2ec..fffbed3cf 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -27,7 +27,6 @@ from utils.local_context import StateStore
from utils.user_context import UserContext
from workflow_manager.endpoint_v2.destination import DestinationConnector
from workflow_manager.endpoint_v2.dto import FileHash
-from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.endpoint_v2.source import SourceConnector
from workflow_manager.file_execution.models import WorkflowFileExecution
from workflow_manager.workflow_v2.constants import (
@@ -42,6 +41,7 @@ from workflow_manager.workflow_v2.exceptions import (
InvalidRequest,
TaskDoesNotExistError,
WorkflowDoesNotExistError,
+ WorkflowExecutionError,
WorkflowExecutionNotExist,
)
from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper
@@ -109,25 +109,6 @@ class WorkflowHelper:
workflow_execution_service.build()
return workflow_execution_service
- @staticmethod
- def _get_or_create_workflow_execution_file(
- execution_service: WorkflowExecutionServiceHelper,
- file_hash: FileHash,
- source: SourceConnector,
- ) -> WorkflowFileExecution:
- is_api = source.endpoint.connection_type == WorkflowEndpoint.ConnectionType.API
- # Determine file path based on connection type
- execution_file_path = file_hash.file_path if not is_api else None
- # Create or retrieve the workflow execution file
- return WorkflowFileExecution.objects.get_or_create_file_execution(
- workflow_execution=execution_service.workflow_execution,
- file_name=file_hash.file_name,
- file_hash=file_hash.file_hash,
- file_path=execution_file_path,
- file_size=file_hash.file_size,
- mime_type=file_hash.mime_type,
- )
-
@classmethod
def process_input_files(
cls,
@@ -146,16 +127,12 @@ class WorkflowHelper:
execution_service.update_execution(
ExecutionStatus.EXECUTING, increment_attempt=True
)
- if total_files > 0:
- q_file_no_list = WorkflowUtil.get_q_no_list(workflow, total_files)
+ q_file_no_list = (
+ WorkflowUtil.get_q_no_list(workflow, total_files) if total_files > 0 else []
+ )
for index, (file_name, file_hash) in enumerate(input_files.items()):
# Get workflow execution file
- workflow_execution_file = cls._get_or_create_workflow_execution_file(
- execution_service=execution_service,
- file_hash=file_hash,
- source=source,
- )
file_number = index + 1
file_hash = WorkflowUtil.add_file_destination_filehash(
file_number,
@@ -173,44 +150,31 @@ class WorkflowHelper:
execution_service=execution_service,
single_step=single_step,
file_hash=file_hash,
- workflow_file_execution=workflow_execution_file,
)
if error:
failed_files += 1
- workflow_execution_file.update_status(
- status=ExecutionStatus.ERROR,
- execution_error=error,
- )
else:
successful_files += 1
- workflow_execution_file.update_status(ExecutionStatus.COMPLETED)
except StopExecution as e:
execution_service.update_execution(
ExecutionStatus.STOPPED, error=str(e)
)
- workflow_execution_file.update_status(
- status=ExecutionStatus.STOPPED, execution_error=str(e)
- )
break
except Exception as e:
failed_files += 1
error_message = f"Error processing file '{file_name}'. {e}"
logger.error(error_message, stack_info=True, exc_info=True)
- workflow_execution_file.update_status(
- status=ExecutionStatus.ERROR,
- execution_error=error_message,
- )
execution_service.publish_log(
message=error_message, level=LogLevel.ERROR
)
# TODO: Store only generic WF errors here (concerning all failed files)
# TODO: Review if we need partial success
- if failed_files and failed_files >= total_files:
- execution_service.update_execution(
- ExecutionStatus.ERROR, error=error_message
- )
- else:
- execution_service.update_execution(ExecutionStatus.COMPLETED)
+ final_status = (
+ ExecutionStatus.ERROR
+ if total_files > 0 and failed_files >= total_files
+ else ExecutionStatus.COMPLETED
+ )
+ execution_service.update_execution(final_status, error=error_message)
workflow_execution = execution_service.get_execution_instance()
@@ -222,8 +186,9 @@ class WorkflowHelper:
return workflow_execution
- @staticmethod
+ @classmethod
def _process_file(
+ cls,
current_file_idx: int,
total_files: int,
input_file: str,
@@ -233,18 +198,46 @@ class WorkflowHelper:
execution_service: WorkflowExecutionServiceHelper,
single_step: bool,
file_hash: FileHash,
- workflow_file_execution: WorkflowFileExecution,
) -> Optional[str]:
- error: Optional[str] = None
+
+ # Delete previous execution directory to avoid conflicts
+ destination.delete_execution_directory()
+
# Multiple run_ids are linked to an execution_id
# Each run_id corresponds to workflow runs for a single file
# It should e uuid of workflow_file_execution
+ workflow_file_execution: WorkflowFileExecution = (
+ WorkflowFileExecution.objects.get_or_create_file_execution(
+ workflow_execution=execution_service.workflow_execution,
+ file_hash=file_hash,
+ connection_type=source.endpoint.connection_type,
+ )
+ )
+ workflow_file_execution.update_status(status=ExecutionStatus.EXECUTING)
file_execution_id = str(workflow_file_execution.id)
- file_name = source.add_file_to_volume(
- input_file_path=input_file,
+ file_name = file_hash.file_name
+
+ # This will add the file to the volume
+ file_content_hash = source.add_file_to_volume(
workflow_file_execution=workflow_file_execution,
tags=execution_service.tags,
+ file_hash=file_hash,
)
+
+ # Update file_hash after adding to volume
+ file_hash.file_hash = file_content_hash
+ workflow_file_execution.update(file_hash=file_content_hash)
+
+ # Ensure no duplicate file processing
+ if FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=file_content_hash
+ ):
+ logger.info(f"Skipping '{file_name}', already processed.")
+ execution_service.publish_log(
+ f"Skipping '{file_name}', already processed.", LogLevel.INFO
+ )
+ return None
+
try:
execution_service.file_execution_id = file_execution_id
execution_service.initiate_tool_execution(
@@ -259,35 +252,55 @@ class WorkflowHelper:
workflow_file_execution=workflow_file_execution,
)
except StopExecution:
+ workflow_file_execution.update_status(status=ExecutionStatus.STOPPED)
raise
+ except WorkflowExecutionError as e:
+ error_message = f"Error processing file '{file_name}': {e}"
+ execution_service.publish_log(error_message, level=LogLevel.ERROR)
+ error = error_message
except Exception as e:
- error = f"Error processing file '{os.path.basename(input_file)}'. {str(e)}"
- execution_service.publish_log(error, level=LogLevel.ERROR)
+ error_message = f"Unexpected error processing file '{file_name}': {str(e)}"
+ logger.error(error_message, stack_info=True, exc_info=True)
+ execution_service.publish_log(error_message, level=LogLevel.ERROR)
+ error = error_message
# Handling error based on destination and continuing for other files
+ else:
+ error = None
execution_service.publish_update_log(
LogState.RUNNING,
f"Processing output for {file_name}",
LogComponent.DESTINATION,
)
- destination.handle_output(
- file_name=file_name,
- file_hash=file_hash,
- workflow=workflow,
- input_file_path=input_file,
- error=error,
- use_file_history=execution_service.use_file_history,
- file_execution_id=file_execution_id,
- )
-
- execution_service.log_total_cost_per_file(
- run_id=file_execution_id, file_name=file_name
- )
-
- execution_service.publish_update_log(
- LogState.SUCCESS,
- f"{file_name}'s output is processed successfully",
- LogComponent.DESTINATION,
- )
+ try:
+ destination.handle_output(
+ file_name=file_name,
+ file_hash=file_hash,
+ workflow=workflow,
+ input_file_path=input_file,
+ error=error,
+ use_file_history=execution_service.use_file_history,
+ file_execution_id=file_execution_id,
+ )
+ except Exception as e:
+ error_message = (
+ f"Error during output processing for '{file_name}': {str(e)}"
+ )
+ logger.error(error_message, stack_info=True, exc_info=True)
+ execution_service.publish_log(error_message, level=LogLevel.ERROR)
+ error = error_message
+ finally:
+ execution_service.log_total_cost_per_file(
+ run_id=file_execution_id, file_name=file_name
+ )
+ execution_service.publish_update_log(
+ LogState.SUCCESS,
+ f"{file_name}'s output is processed successfully",
+ LogComponent.DESTINATION,
+ )
+ workflow_file_execution.update_status(
+ status=ExecutionStatus.ERROR if error else ExecutionStatus.COMPLETED,
+ execution_error=error if error else None,
+ )
return error
@staticmethod
@@ -344,11 +357,11 @@ class WorkflowHelper:
execution_id=execution_id,
execution_service=execution_service,
)
- # Validating endpoints
- source.validate()
- destination.validate()
- # Execution Process
try:
+ # Validating endpoints
+ source.validate()
+ destination.validate()
+ # Execution Process
input_files, total_files = source.list_files_from_source(
hash_values_of_files
)
@@ -387,7 +400,7 @@ class WorkflowHelper:
finally:
# TODO: Handle error gracefully during delete
# Mark status as an ERROR correctly
- destination.delete_execution_directory()
+ destination.delete_execution_and_api_storage_dir()
@staticmethod
def _update_pipeline_status(
diff --git a/unstract/connectors/src/unstract/connectors/databases/exceptions.py b/unstract/connectors/src/unstract/connectors/databases/exceptions.py
index a9bc4da2b..b63f09eb4 100644
--- a/unstract/connectors/src/unstract/connectors/databases/exceptions.py
+++ b/unstract/connectors/src/unstract/connectors/databases/exceptions.py
@@ -111,3 +111,9 @@ class ColumnMissingException(UnstractDBConnectorException):
f"as per the destination DB configuration.\n"
)
super().__init__(detail=default_detail)
+
+
+class OperationalException(UnstractDBConnectorException):
+ def __init__(self, detail: Any, database: str) -> None:
+ default_detail = f"Error creating/writing to {database}. Operational error. "
+ super().__init__(detail=default_detail)
diff --git a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py
index 3c2b5c4bd..976b2dbd2 100644
--- a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py
+++ b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py
@@ -9,6 +9,13 @@ from unstract.connectors.databases.unstract_db import UnstractDB
class PostgreSQL(UnstractDB, PsycoPgHandler):
+ # Connection timeout settings (in seconds)
+ CONNECT_TIMEOUT = 30 # Time to establish connection
+ STATEMENT_TIMEOUT = 300 # Time for query execution (5 minutes)
+ KEEPALIVE_IDLE = 30 # Time before sending keepalive
+ KEEPALIVE_INTERVAL = 10 # Time between keepalive probes
+ KEEPALIVE_COUNT = 3 # Number of keepalive failures before dropping
+
def __init__(self, settings: dict[str, Any]):
super().__init__("PostgreSQL")
@@ -60,17 +67,51 @@ class PostgreSQL(UnstractDB, PsycoPgHandler):
return True
def get_engine(self) -> connection:
+ # Set timeouts via options
+ timeout_options = (
+ f"-c connect_timeout={self.CONNECT_TIMEOUT} "
+ f"-c statement_timeout={self.STATEMENT_TIMEOUT * 1000} "
+ f"-c tcp_keepalives_idle={self.KEEPALIVE_IDLE} "
+ f"-c tcp_keepalives_interval={self.KEEPALIVE_INTERVAL} "
+ f"-c tcp_keepalives_count={self.KEEPALIVE_COUNT}"
+ )
+
+ # Base connection parameters
+ conn_params = {
+ "keepalives": 1,
+ "keepalives_idle": self.KEEPALIVE_IDLE,
+ "keepalives_interval": self.KEEPALIVE_INTERVAL,
+ "keepalives_count": self.KEEPALIVE_COUNT,
+ "connect_timeout": self.CONNECT_TIMEOUT,
+ "application_name": "unstract_connector",
+ }
+
+ # Determine SSL mode based on connection URL
+ if self.connection_url and (
+ "neon.tech" in self.connection_url or "amazonaws.com" in self.connection_url
+ ):
+ # Cloud hosted PostgreSQL (Neon, AWS RDS etc)
+ conn_params.update({"sslmode": "verify-full", "sslrootcert": "system"})
+ else:
+ # Standard PostgreSQL - use basic SSL if available
+ conn_params["sslmode"] = "prefer"
+
if self.connection_url:
- con = psycopg2.connect(dsn=self.connection_url)
+ conn_params.update({"dsn": self.connection_url, "options": timeout_options})
+ con = psycopg2.connect(**conn_params)
else:
- con = psycopg2.connect(
- host=self.host,
- port=self.port,
- database=self.database,
- user=self.user,
- password=self.password,
- options=f"-c search_path={self.schema}",
+ conn_params.update(
+ {
+ "host": self.host,
+ "port": self.port,
+ "database": self.database,
+ "user": self.user,
+ "password": self.password,
+ "options": f"{timeout_options} -c search_path={self.schema}",
+ }
)
+ con = psycopg2.connect(**conn_params)
+
return con
def execute_query(
diff --git a/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py b/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py
index 34b089cd3..08bae5564 100644
--- a/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py
+++ b/unstract/connectors/src/unstract/connectors/databases/psycopg_handler.py
@@ -8,6 +8,7 @@ from unstract.connectors.databases.exceptions import (
FeatureNotSupportedException,
InvalidSchemaException,
InvalidSyntaxException,
+ OperationalException,
UnderfinedTableException,
ValueTooLongException,
)
@@ -62,3 +63,6 @@ class PsycoPgHandler:
schema=schema,
table_name=table_name,
) from e
+ except PsycopgError.OperationalError as e:
+ logger.error(f"Operational error in creating/inserting data: {e.pgerror}")
+ raise OperationalException(detail=e.pgerror, database=database) from e
diff --git a/unstract/connectors/src/unstract/connectors/exceptions.py b/unstract/connectors/src/unstract/connectors/exceptions.py
index cfeeb6f4c..827da3b8e 100644
--- a/unstract/connectors/src/unstract/connectors/exceptions.py
+++ b/unstract/connectors/src/unstract/connectors/exceptions.py
@@ -55,3 +55,11 @@ class PermissionDeniedError(ConnectorError):
Args:
ConnectorError (Class): Inherits class ConnectorError
"""
+
+
+class FileHashNotFound(ConnectorError):
+ """Handles file hash not found error from fs connector.
+
+ Args:
+ ConnectorError (Class): Inherits class ConnectorError
+ """
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py
index e4cc47ab1..4419b9324 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py
@@ -1,6 +1,6 @@
import logging
import os
-from typing import Any
+from typing import Any, Optional
import azure.core.exceptions as AzureException
from adlfs import AzureBlobFileSystem
@@ -67,6 +67,25 @@ class AzureCloudStorageFS(UnstractFileSystem):
def get_fsspec_fs(self) -> AzureBlobFileSystem:
return self.azure_fs
+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:
+ """
+ Extracts a unique file hash from metadata.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec.
+
+ Returns:
+ Optional[str]: The file hash in hexadecimal format or None if not found.
+ """
+ # Extracts content_md5 (Bytearray) for Azure Blob Storage
+ content_md5 = metadata.get("content_settings", {}).get("content_md5")
+ if content_md5:
+ return content_md5.hex()
+ logger.error(
+ f"[Azure Blob Storage] File hash not found for the metadata: {metadata}"
+ )
+ return None
+
def test_credentials(self) -> bool:
"""To test credentials for Azure Cloud Storage."""
try:
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/box/box.py b/unstract/connectors/src/unstract/connectors/filesystems/box/box.py
index 2989e7916..133641280 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/box/box.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/box/box.py
@@ -2,7 +2,7 @@ import json
import logging
import os
from json import JSONDecodeError
-from typing import Any
+from typing import Any, Optional
from boxfs import BoxFileSystem
from boxsdk import JWTAuth
@@ -110,6 +110,19 @@ class BoxFS(UnstractFileSystem):
def get_fsspec_fs(self) -> BoxFileSystem:
return self.box_fs
+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:
+ """
+ Extracts a unique file hash from metadata.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec.
+
+ Returns:
+ Optional[str]: The file hash in hexadecimal format or None if not found.
+ """
+ logger.error(f"[Box] File hash not found for the metadata: {metadata}")
+ return None
+
def test_credentials(self) -> bool:
"""To test credentials for the Box connector."""
is_dir = False
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py
index 025ede70c..07569898c 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py
@@ -1,7 +1,8 @@
+import base64
import json
import logging
import os
-from typing import Any
+from typing import Any, Optional
from gcsfs import GCSFileSystem
@@ -61,6 +62,23 @@ class GoogleCloudStorageFS(UnstractFileSystem):
def get_fsspec_fs(self) -> GCSFileSystem:
return self.gcs_fs
+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:
+ """
+ Extracts a unique file hash from metadata.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.
+
+ Returns:
+ Optional[str]: The file hash in hexadecimal format or None if not found.
+ """
+ # Extracts md5Hash (Base64) for GCS
+ file_hash = metadata.get("md5Hash")
+ if file_hash:
+ return base64.b64decode(file_hash).hex()
+ logger.error(f"[GCS] File hash not found for the metadata: {metadata}")
+ return None
+
def test_credentials(self) -> bool:
"""To test credentials for Google Cloud Storage."""
try:
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py
index 1743246dd..649dfb8c6 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py
@@ -2,7 +2,7 @@ import json
import logging
import os
from pathlib import Path
-from typing import Any
+from typing import Any, Optional
import google.api_core.exceptions as GoogleApiException
from oauth2client.client import OAuth2Credentials
@@ -98,6 +98,19 @@ class GoogleDriveFS(UnstractFileSystem):
def get_fsspec_fs(self) -> GDriveFileSystem:
return self.drive
+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:
+ """
+ Extracts a unique file hash from metadata.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec.
+
+ Returns:
+ Optional[str]: The file hash in hexadecimal format or None if not found.
+ """
+ logger.error(f"[Google Drive] File hash not found for the metadata: {metadata}")
+ return None
+
def test_credentials(self) -> bool:
"""To test credentials for Google Drive."""
is_dir = False
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py b/unstract/connectors/src/unstract/connectors/filesystems/http/http.py
index bf0a29dd6..b5e859209 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/http/http.py
@@ -1,6 +1,6 @@
import logging
import os
-from typing import Any
+from typing import Any, Optional
import aiohttp
from fsspec.implementations.http import HTTPFileSystem
@@ -69,6 +69,19 @@ class HttpFS(UnstractFileSystem):
def get_fsspec_fs(self) -> HTTPFileSystem:
return self.http_fs
+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:
+ """
+ Extracts a unique file hash from metadata.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec.
+
+ Returns:
+ Optional[str]: The file hash in hexadecimal format or None if not found.
+ """
+ logger.error(f"[HTTP] File hash not found for the metadata: {metadata}")
+ return None
+
def test_credentials(self) -> bool:
"""To test credentials for HTTP(S)."""
is_dir = False
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py
index 783803fe3..ce15a2c44 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py
@@ -59,6 +59,19 @@ class LocalStorageFS(UnstractFileSystem):
def get_fsspec_fs(self) -> Any:
return self.local
+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:
+ """
+ Extracts a unique file hash from metadata.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec.
+
+ Returns:
+ Optional[str]: The file hash in hexadecimal format or None if not found.
+ """
+ logger.error(f"[LocalStorage] File hash not found for the metadata: {metadata}")
+ return None
+
def test_credentials(self, *args, **kwargs) -> bool: # type:ignore
"""To test credentials for LocalStorage."""
is_dir = False
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
index 7f99a6711..2c51ee589 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
@@ -1,6 +1,6 @@
import logging
import os
-from typing import Any
+from typing import Any, Optional
from s3fs.core import S3FileSystem
@@ -70,6 +70,30 @@ class MinioFS(UnstractFileSystem):
def can_read() -> bool:
return True
+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:
+ """
+ Extracts a unique file hash from metadata.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec.
+
+ Returns:
+ Optional[str]: The file hash in hexadecimal format or None if not found.
+ """
+ # Extracts ETag for MinIO
+ file_hash = metadata.get("ETag")
+ if file_hash:
+ file_hash = file_hash.strip('"')
+ if "-" in file_hash:
+ logger.warning(
+ f"[S3/MinIO] Multipart upload detected. ETag may not be an "
+ f"MD5 hash. Full metadata: {metadata}"
+ )
+ return None
+ return file_hash
+ logger.error(f"[MinIO] File hash not found for the metadata: {metadata}")
+ return None
+
def get_fsspec_fs(self) -> S3FileSystem:
return self.s3
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py
index feef4d62a..c8b9d290d 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/sftp/sftp.py
@@ -1,6 +1,6 @@
import logging
import os
-from typing import Any
+from typing import Any, Optional
from fsspec.implementations.sftp import SFTPFileSystem
@@ -65,6 +65,19 @@ class SftpFS(UnstractFileSystem):
def get_fsspec_fs(self) -> SFTPFileSystem:
return self.sftp_fs
+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:
+ """
+ Extracts a unique file hash from metadata.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec.
+
+ Returns:
+ Optional[str]: The file hash in hexadecimal format or None if not found.
+ """
+ logger.error(f"[SFTP] File hash not found for the metadata: {metadata}")
+ return None
+
# TODO: Check if this method can be removed, and use it from parent class
# (class UnstractFileSystem)
@staticmethod
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py
index b2162420c..fcaf3409c 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/unstract_file_system.py
@@ -1,7 +1,9 @@
+import base64
import logging
import os
from abc import ABC, abstractmethod
-from typing import Any
+from datetime import date, datetime
+from typing import Any, Optional
from fsspec import AbstractFileSystem
@@ -75,11 +77,121 @@ class UnstractFileSystem(UnstractConnector, ABC):
"""Override to test credentials for a connector."""
pass
+ @abstractmethod
+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:
+ """
+ Extracts a unique file hash from metadata.
+
+ This method must be implemented in subclasses.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.
+
+ Returns:
+ Optional[str]: The file hash in hexadecimal format or None if not found.
+ """
+ pass
+
@staticmethod
def get_connector_root_dir(input_dir: str, **kwargs: Any) -> str:
"""Override to get root dir of a connector."""
return f"{input_dir.strip('/')}/"
+ def get_file_size(self, file_path: str) -> Optional[int]:
+ """Get the size of a file."""
+ fs_fsspec = self.get_fsspec_fs()
+ metadata = fs_fsspec.stat(file_path)
+ if not metadata or not isinstance(metadata, dict):
+ return None
+ return metadata.get("size")
+
+ def _serialize_metadata_value(self, value: Any, depth: int = 0) -> Any:
+ """
+ Recursively serialize metadata values to ensure JSON compatibility.
+
+ Args:
+ value: Any value that needs to be serialized
+ depth: Current recursion depth (default: 0)
+
+ Returns:
+ JSON serializable version of the value
+ """
+ # Prevent infinite recursion
+ if depth > 10: # Max recursion depth
+ return str(value)
+
+ if value is None:
+ return None
+ elif isinstance(value, (str, int, float, bool)):
+ return value
+ elif isinstance(value, (datetime, date)):
+ return value.isoformat()
+ elif isinstance(value, (bytes, bytearray)):
+ return base64.b64encode(value).decode("utf-8")
+ elif isinstance(value, dict):
+ return {
+ k: self._serialize_metadata_value(v, depth + 1)
+ for k, v in value.items()
+ }
+ elif isinstance(value, (list, tuple)):
+ return [self._serialize_metadata_value(v, depth + 1) for v in value]
+ else:
+ # For custom objects like ContentSettings, convert to dict if possible
+ if hasattr(value, "__dict__"):
+ try:
+ return self._serialize_metadata_value(value.__dict__, depth + 1)
+ except Exception:
+ return str(value)
+ # Last resort: convert to string
+ return str(value)
+
+ def get_file_metadata(self, file_path: str) -> dict[str, Any]:
+ """
+ Get the metadata of a file.
+
+ Args:
+ file_path (str): Path of the file.
+
+ Returns:
+ dict[str, Any]: Metadata of the file or empty dict
+ if metadata cannot be retrieved.
+ All values are guaranteed to be JSON serializable.
+ """
+ try:
+ fs_fsspec = self.get_fsspec_fs()
+ metadata = fs_fsspec.stat(file_path)
+ if not metadata or not isinstance(metadata, dict):
+ return {}
+
+ # Recursively serialize all metadata values
+ serialized_metadata = {}
+ for key, value in metadata.items():
+ serialized_metadata[key] = self._serialize_metadata_value(value)
+
+ return serialized_metadata
+ except Exception as e:
+ logger.error(
+ f"Error getting file system metadata for {file_path}: {str(e)}"
+ )
+ return {}
+
+ def get_file_system_uuid(self, file_path: str) -> Optional[str]:
+ """
+ Get the UUID of a file.
+
+ Args:
+ file_path (str): Path of the file.
+
+ Returns:
+ Optional[str]: UUID of the file in hex format or None if not found.
+ """
+ fs_fsspec = self.get_fsspec_fs()
+ metadata = fs_fsspec.stat(file_path)
+ file_hash = self.extract_metadata_file_hash(metadata)
+ if not file_hash:
+ logger.error(f"File hash not found for {file_path}.")
+ return file_hash
+
def create_dir_if_not_exists(self, input_dir: str) -> None:
"""Method to create dir of a connector if not exists.
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py
index e7d5a7d37..8930855e5 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py
@@ -1,6 +1,6 @@
import logging
import os
-from typing import Any
+from typing import Any, Optional
from dropbox.exceptions import ApiError as DropBoxApiError
from dropbox.exceptions import DropboxException
@@ -63,6 +63,24 @@ class DropboxFS(UnstractFileSystem):
def get_fsspec_fs(self) -> DropboxDriveFileSystem:
return self.dropbox_fs
+ def extract_metadata_file_hash(self, metadata: dict[str, Any]) -> Optional[str]:
+ """
+ Extracts a unique file hash from metadata.
+
+ Args:
+ metadata (dict): Metadata dictionary obtained from fsspec or cloud API.
+
+ Returns:
+ Optional[str]: The file hash in hexadecimal format or None if not found.
+ """
+ file_hash = metadata.get("content_hash")
+ if file_hash:
+ return file_hash
+ logger.error(
+ f"[Dropbox] File hash (content_hash) not found for the metadata: {metadata}"
+ )
+ return None
+
def test_credentials(self) -> bool:
"""To test credentials for Dropbox."""
try: