Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

58.4 KB
Raw Blame
import fnmatch
import logging
import os
import shutil
import tempfile
from collections.abc import Collection
from hashlib import sha256
from io import BytesIO
from itertools import islice
from typing import Any

import fsspec
import magic
from connector_processor.constants import ConnectorKeys
from connector_v2.models import ConnectorInstance
from django.core.files.uploadedfile import UploadedFile
from django.db.models import Q
from utils.user_context import UserContext
from workflow_manager.endpoint_v2.base_connector import BaseConnector
from workflow_manager.endpoint_v2.constants import (
    FilePattern,
    FileProcessingOrder,
    FileSystemConnector,
    FileType,
    SourceConstant,
    SourceKey,
)
from workflow_manager.endpoint_v2.dto import (
    FileExecutionResult,
    FileHash,
    SourceConfig,
)
from workflow_manager.endpoint_v2.enums import (
    INCONCLUSIVE_MIME_TYPES,
    AllowedFileTypes,
    resolve_inconclusive_mime_type,
)
from workflow_manager.endpoint_v2.exceptions import (
    InvalidInputDirectory,
    InvalidSourceConnectionType,
    MissingSourceConnectionType,
    OrganizationIdNotFound,
    SourceConnectorNotConfigured,
    SourceFileOrInfilePathNotFound,
    UnsupportedMimeTypeError,
)
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.endpoint_v2.result_cache_utils import ResultCacheUtils
from workflow_manager.file_execution.models import WorkflowFileExecution
from workflow_manager.utils.workflow_log import WorkflowLog
from workflow_manager.workflow_v2.enums import ExecutionStatus
from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
from workflow_manager.workflow_v2.models.execution import WorkflowExecution
from workflow_manager.workflow_v2.models.file_history import FileHistory
from workflow_manager.workflow_v2.models.workflow import Workflow

from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
from unstract.core.mime_gate import identify_zip_container
from unstract.filesystem import FileStorageType, FileSystem
from unstract.sdk1.file_storage import FileStorage
from unstract.workflow_execution.enums import LogLevel, LogStage, LogState

logger = logging.getLogger(__name__)


# TODO: Inherit from SourceConnector for different sources - File, API .etc.
class SourceConnector(BaseConnector):
    """A class representing a source connector for a workflow.

    This class extends the BaseConnector class and provides methods for
    interacting with different types of source connectors,
    such as file system connectors and API connectors.
    It allows listing files from the source connector,
    adding files to the execution volume, and retrieving JSON schemas for
    different types of connectors.

    Attributes:
        workflow (Workflow): The workflow associated with the source connector.
    """

    READ_CHUNK_SIZE = 4194304  # Chunk size for reading files
    # Most formats are identifiable from their leading bytes.
    MIME_DETECT_CHUNK_SIZE = 8192
    # Shared, so the API and connector paths agree on what counts as undecided.
    CONTAINER_MIME_TYPES = INCONCLUSIVE_MIME_TYPES

    def __init__(
        self,
        workflow: Workflow,
        execution_id: str,
        workflow_log: WorkflowLog,
        use_file_history: bool,
        organization_id: str | None = None,
        file_execution_id: str | None = None,
    ) -> None:
        """Create a SourceConnector.

        Args:
            workflow (Workflow): Associated workflow instance
            execution_id (str): UUID of the current execution
            organization_id (Optional[str]): Organization ID. Defaults to None.
            execution_service (Optional[WorkflowExecutionServiceHelper]): Instance of
                WorkflowExecutionServiceHelper that helps with WF execution.
                Defaults to None. This is not used in case of execution by API.

        Raises:
            OrganizationIdNotFound: _description_
        """
        organization_id = organization_id or UserContext.get_organization_identifier()
        if not organization_id:
            raise OrganizationIdNotFound()
        super().__init__(workflow.id, execution_id, organization_id, file_execution_id)
        self.endpoint = self._get_endpoint_for_workflow(workflow=workflow)
        self.workflow = workflow
        self.execution_id = execution_id
        self.organization_id = organization_id
        self.hash_value_of_file_content: str | None = None
        self.workflow_log = workflow_log
        self.use_file_history = use_file_history

    def _get_endpoint_for_workflow(
        self,
        workflow: Workflow,
    ) -> WorkflowEndpoint:
        """Get WorkflowEndpoint instance.

        Args:
            workflow (Workflow): Workflow

        Returns:
            WorkflowEndpoint: _description_
        """
        endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(
            workflow=workflow,
            endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
        )
        return endpoint

    def validate(self) -> None:
        connection_type = self.endpoint.connection_type
        connector: ConnectorInstance = self.endpoint.connector_instance

        if connection_type is None:
            error_msg = "Missing source connection type"
            self.workflow_log.log_error(logger, error_msg)
            raise MissingSourceConnectionType()

        if connection_type not in WorkflowEndpoint.ConnectionType.values:
            error_msg = f"Invalid source connection type: {connection_type}"
            self.workflow_log.log_error(logger, error_msg)
            raise InvalidSourceConnectionType()

        if connection_type != WorkflowEndpoint.ConnectionType.API and connector is None:
            error_msg = "Source connector not configured"
            self.workflow_log.log_error(logger, error_msg)
            raise SourceConnectorNotConfigured()

    def valid_file_patterns(self, required_patterns: list[Any]) -> list[str]:
        patterns = {
            FileType.PDF_DOCUMENTS: FilePattern.PDF_DOCUMENTS,
            FileType.TEXT_DOCUMENTS: FilePattern.TEXT_DOCUMENTS,
            FileType.IMAGES: FilePattern.IMAGES,
        }
        wildcard = []
        if not required_patterns:
            wildcard.append("*")
        else:
            for pattern in required_patterns:
                wildcard.extend(patterns.get(pattern, []))
        return wildcard

    def list_file_from_api_storage(
        self, file_hashes: dict[str, FileHash]
    ) -> dict[str, FileHash]:
        """List all files from the api_storage_dir directory."""
        return file_hashes

    def list_files_from_file_connector(self) -> dict[str, FileHash]:
        """_summary_

        Raises:
            InvalidDirectory: _description_

        Returns:
            dict[str, FileHash]: A dictionary of matched file paths
            and their corresponding FileHash objects.
        """
        connector: ConnectorInstance = self.endpoint.connector_instance
        connector_settings: dict[str, Any] = connector.connector_metadata
        source_configuration: dict[str, Any] = self.endpoint.configuration
        logger.info("Source configuration: %s", source_configuration)
        required_patterns = list(source_configuration.get(SourceKey.FILE_EXTENSIONS, []))
        recursive = bool(
            source_configuration.get(SourceKey.PROCESS_SUB_DIRECTORIES, False)
        )
        limit = int(
            source_configuration.get(SourceKey.MAX_FILES, FileSystemConnector.MAX_FILES)
        )
        file_processing_order = FileProcessingOrder.from_value(
            source_configuration.get(SourceKey.FILE_PROCESSING_ORDER)
        )
        root_dir_path = connector_settings.get(ConnectorKeys.PATH, "")
        folders_to_process = list(source_configuration.get(SourceKey.FOLDERS, ["/"]))
        # Process from root in case its user provided list is empty
        if not folders_to_process:
            folders_to_process = ["/"]
        patterns = self.valid_file_patterns(required_patterns=required_patterns)
        self.workflow_log.publish_log(
            f"Matching for patterns '{', '.join(patterns)}' from "
            f"'{', '.join(folders_to_process)}' to process upto {limit} files "
            "recursively"
            if recursive
            else ""
        )

        source_fs = self.get_fs_connector(
            settings=connector_settings, connector_id=connector.connector_id
        )
        source_fs_fsspec = source_fs.get_fsspec_fs()
        # Checking if folders exist at source before processing
        # TODO: Validate while receiving this input configuration as well
        valid_directories = []
        for input_directory in folders_to_process:
            # TODO: Move to connector class for better error handling
            try:
                input_directory = source_fs.get_connector_root_dir(
                    input_dir=input_directory, root_path=root_dir_path
                )
                if not source_fs_fsspec.isdir(input_directory):
                    raise InvalidInputDirectory(dir=input_directory)
                valid_directories.append(input_directory)
            except Exception as e:
                msg = f"Error while validating path '{input_directory}'. {str(e)}"
                self.publish_user_sys_log(msg)
                if isinstance(e, InvalidInputDirectory):
                    raise
                raise InvalidInputDirectory(detail=msg)

        if file_processing_order == FileProcessingOrder.UNORDERED:
            total_matched_files = self._process_without_sorting(
                source_fs,
                valid_directories,
                patterns,
                recursive,
                limit,
            )
        else:
            total_matched_files = self._process_with_sorting(
                source_fs,
                valid_directories,
                patterns,
                recursive,
                limit,
                file_processing_order,
            )

        self.publish_input_output_list_file_logs(folders_to_process, total_matched_files)
        return total_matched_files

    def publish_user_sys_log(self, msg: str) -> None:
        """Publishes log to the user and system.

        Pushes logs messages to the configured logger and to the
        websocket channel if the `execution_service` is configured.

        Args:
            msg (str): Message to log
        """
        logger.info(msg)
        self.workflow_log.publish_log(message=msg)

    def publish_input_output_list_file_logs(
        self, folders: list[str], matched_files: dict[str, FileHash]
    ) -> None:
        folders_list = "\n".join(f"- `{folder.strip()}`" for folder in folders)
        input_log = f"## Folders to process:\n\n{folders_list}\n\n"
        self.workflow_log.publish_update_log(
            state=LogState.INPUT_UPDATE, message=input_log
        )
        output_log = self._matched_files_component_log(matched_files)
        self.workflow_log.publish_update_log(
            state=LogState.OUTPUT_UPDATE, message=output_log
        )

    def publish_input_file_content(self, input_file_path: str, input_text: str) -> None:
        output_log_message = f"## Input text:\n\n```text\n{input_text}\n```\n\n"
        input_log_message = (
            f"## Input file:\n\n```text\n{os.path.basename(input_file_path)}\n```\n\n"
        )
        self.workflow_log.publish_update_log(
            state=LogState.INPUT_UPDATE, message=input_log_message
        )
        self.workflow_log.publish_update_log(
            state=LogState.OUTPUT_UPDATE, message=output_log_message
        )

    def _matched_files_component_log(self, matched_files: dict[str, FileHash]) -> str:
        output_log = "### Matched files \n```text\n\n\n"
        for file_path in islice(matched_files.keys(), 20):
            output_log += f"- {file_path}\n"
        output_log += "```\n\n"
        output_log += f"""Total matched files: {len(matched_files)}
            \n\nPlease note that only the first 20 files are shown.\n\n"""
        return output_log

    def _collect_and_sort_files(
        self,
        source_fs: UnstractFileSystem,
        valid_directories: list[str],
        recursive: bool,
        file_processing_order: str,
    ) -> list[dict[str, Any]]:
        """Collect files from all directories and apply global sorting.

        Used when file_processing_order is not UNORDERED.
        """
        all_files_metadata = []
        max_depth = int(SourceConstant.MAX_RECURSIVE_DEPTH) if recursive else 1
        max_files_for_sorting = SourceConstant.MAX_FILES_FOR_SORTING
        total_collected = 0

        # Collect files from all directories
        for input_directory in valid_directories:
            logger.debug(f"Collecting files from: {input_directory}")

            # Calculate remaining limit for this directory
            remaining_limit = max_files_for_sorting - total_collected
            if remaining_limit <= 0:
                break

            try:
                files_metadata = source_fs.list_files(
                    directory=input_directory,
                    max_depth=max_depth,
                    include_dirs=False,
                    limit=remaining_limit,
                )
                all_files_metadata.extend(files_metadata)
                total_collected += len(files_metadata)

                # Report errors
                user_errors = source_fs.report_errors_to_user()
                for error_msg in user_errors[:5]:
                    self.workflow_log.publish_log(error_msg, level=LogLevel.ERROR)
                if len(user_errors) > 5:
                    self.workflow_log.publish_log(
                        f"... and {len(user_errors) - 5} more errors",
                        level=LogLevel.ERROR,
                    )

                # Check if we've hit the limit
                if total_collected >= max_files_for_sorting:
                    break

            except Exception as e:
                error_msg = f"Failed to collect files from {input_directory}"
                logger.error(f"{error_msg}: {e}")
                self.workflow_log.publish_log(error_msg, level=LogLevel.ERROR)

        # Log warning if collection limit was reached
        if total_collected >= max_files_for_sorting:
            warning_msg = (
                f"File collection limit of '{max_files_for_sorting}' files reached. "
                "Ordering may not reflect all available files from source file system"
            )
            logger.warning(warning_msg)
            self.workflow_log.publish_log(warning_msg, level=LogLevel.WARN)

        # Apply sorting
        if file_processing_order == FileProcessingOrder.OLDEST_FIRST:
            sorted_files = source_fs.sort_files_by_modified_date(
                all_files_metadata, ascending=True
            )
            order_desc = "FIFO (oldest first)"
        else:
            sorted_files = source_fs.sort_files_by_modified_date(
                all_files_metadata, ascending=False
            )
            order_desc = "LIFO (newest first)"

        self.workflow_log.publish_log(
            f"Collected {len(all_files_metadata)} files, processing in {order_desc} order"
        )

        # Debug logging for first few files in sorted order
        if sorted_files and logger.isEnabledFor(logging.DEBUG):
            logger.debug("File processing order (first 10):")
            for i, file_meta in enumerate(sorted_files[:10]):
                file_path = file_meta.get("name", "unknown")
                modified_date = source_fs.extract_modified_date(file_meta)
                date_str = modified_date.isoformat() if modified_date else "unknown"
                logger.debug(f"  {i + 1}. {file_path} (modified: {date_str})")

        return sorted_files

    def _process_with_sorting(
        self,
        source_fs: UnstractFileSystem,
        valid_directories: list[str],
        patterns: list[str],
        recursive: bool,
        limit: int,
        file_processing_order: str,
    ) -> dict[str, FileHash]:
        """In case ordering is expected, collect and sort all files."""
        matched_files = {}
        unique_file_paths: set[str] = set()

        # Collect and sort all files
        sorted_files = self._collect_and_sort_files(
            source_fs, valid_directories, recursive, file_processing_order
        )

        self._process_file_fs_directory(
            fs_metadata_list=sorted_files,
            count=0,
            limit=limit,
            unique_file_paths=unique_file_paths,
            matched_files=matched_files,
            patterns=patterns,
            source_fs=source_fs,
            dirs=[],  # Not needed for pre-collected files
        )

        return matched_files

    def _process_without_sorting(
        self,
        source_fs: UnstractFileSystem,
        valid_directories: list[str],
        patterns: list[str],
        recursive: bool,
        limit: int,
    ) -> dict[str, FileHash]:
        """Process files directory by directory while checking for limit."""
        total_matched_files = {}
        total_files_to_process = 0
        unique_file_paths: set[str] = set()

        for input_directory in valid_directories:
            logger.debug(f"Listing files from:  {input_directory}")

            remaining_limit = limit - total_files_to_process
            if remaining_limit <= 0:
                break

            matched_files = self._get_matched_files(
                source_fs,
                input_directory,
                patterns,
                recursive,
                remaining_limit,
                unique_file_paths,
            )
            matched_file_count = len(matched_files)
            self.publish_user_sys_log(
                f"Matched '{matched_file_count}' files from '{input_directory}'"
            )
            total_matched_files.update(matched_files)
            total_files_to_process += matched_file_count

            if total_files_to_process >= limit:
                msg = f"Maximum limit of '{limit}' files to process reached"
                self.workflow_log.publish_log(msg)
                logger.info(msg)
                break

        return total_matched_files

    def _get_matched_files(
        self,
        source_fs: UnstractFileSystem,
        input_directory: str,
        patterns: list[str],
        recursive: bool,
        limit: int,
        unique_file_paths: set[str],
    ) -> dict[str, FileHash]:
        """Get a dictionary of matched files based on patterns in a directory.

        This method searches for files in the specified `input_directory` that
        match any of the given `patterns`. The search can be performed recursively
        if `recursive` is set to True. The number of matched files returned is
        limited by `limit`.

        Args:
            source_fs (Any): The file system object used for searching.
            input_directory (str): The directory to search for files.
            patterns (list[str]): The patterns to match against file names.
            recursive (bool): Whether to perform a recursive search.
            limit (int): The maximum number of matched files to return.

        Returns:
            dict[str, FileHash]: A dictionary of matched file paths
            and their corresponding FileHash objects.
        """
        matched_files: dict[str, FileHash] = {}
        count = 0

        # Get max depth for recursive traversal
        max_depth = int(SourceConstant.MAX_RECURSIVE_DEPTH) if recursive else 1

        fs_fsspec = source_fs.get_fsspec_fs()

        def _on_walk_error(error: Exception) -> None:
            logger.warning(f"Failed to list directory: {error}")

        for _root, dirs, files in fs_fsspec.walk(
            input_directory, maxdepth=max_depth, detail=True, on_error=_on_walk_error
        ):
            fs_metadata_list: list[dict[str, Any]] = list(files.values())

            count = self._process_file_fs_directory(
                fs_metadata_list=fs_metadata_list,
                count=count,
                limit=limit,
                unique_file_paths=unique_file_paths,
                matched_files=matched_files,
                patterns=patterns,
                source_fs=source_fs,
                dirs=dirs,
            )

        return matched_files

    def _process_file_fs_directory(
        self,
        fs_metadata_list: list[dict[str, Any]],
        count: int,
        limit: int,
        unique_file_paths: set[str],
        matched_files: dict[str, FileHash],
        patterns: list[str],
        source_fs: UnstractFileSystem,
        dirs: Collection[str],
    ) -> int:
        for fs_metadata in fs_metadata_list:
            if count >= limit:
                msg = f"Maximum limit of '{limit}' files to process reached"
                self.workflow_log.publish_log(msg)
                logger.info(msg)
                break

            file_path: str | None = fs_metadata.get("name")
            file_size = fs_metadata.get("size", 0)

            if not file_path or self._is_directory(
                source_fs, file_path, fs_metadata, dirs
            ):
                continue

            file_hash = self._create_file_hash(
                file_path=file_path,
                source_fs=source_fs,
                file_size=file_size,
                fs_metadata=fs_metadata,
            )
            if self._should_skip_file(file_hash, patterns):
                continue

            # Skip duplicate files
            if self._is_duplicate(file_hash, unique_file_paths):
                msg = f"Skipping execution of duplicate file '{file_path}'"
                self.workflow_log.publish_log(msg)
                logger.info(msg)
                continue
            self._update_unique_file_paths(file_hash, unique_file_paths)

            matched_files[file_path] = file_hash
            count += 1
        return count

    def _is_file_being_processed(self, file_hash: FileHash) -> bool:
        """Check if file is currently being processed or should be skipped
        Uses direct database queries instead of FileExecutionStatusTracker

        Args:
            file_hash (FileHash): The file hash to check

        Returns:
            bool: True if file should be skipped (being processed or already completed), False otherwise
        """
        active_executions = self._get_active_workflow_executions()
        logger.info(
            f"Found {len(active_executions)} active executions for workflow {self.workflow.id}"
        )

        for execution in active_executions:
            if self._has_blocking_file_execution(execution, file_hash):
                return True

        return False

    def _get_active_workflow_executions(self):
        """Get active executions for this workflow with organization filtering for security."""
        organization = UserContext.get_organization()
        return WorkflowExecution.objects.filter(
            workflow=self.workflow,
            workflow__organization_id=organization.id,  # Security: Organization isolation
            status__in=[ExecutionStatus.EXECUTING.value, ExecutionStatus.PENDING.value],
        )

    def _has_blocking_file_execution(self, execution, file_hash: FileHash) -> bool:
        """Check if there's a blocking file execution that prevents processing."""
        try:
            # Try to find blocking file execution using file_hash
            file_exec = self._find_blocking_file_execution_by_hash(execution, file_hash)
            if file_exec:
                self._log_duplicate_file_execution_skip(file_hash, file_exec, execution)
                return True

            # Try to find blocking file execution using provider_file_uuid
            file_exec = self._find_blocking_file_execution_by_provider_uuid(
                execution, file_hash
            )
            if file_exec:
                self._log_duplicate_file_execution_skip(file_hash, file_exec, execution)
                return True

        except Exception as e:
            logger.warning(
                f"Error checking file execution status for {file_hash.file_name}: {e}\n"
                "Allowing its execution to continue anyway"
            )

        return False

    def _find_blocking_file_execution_by_hash(self, execution, file_hash: FileHash):
        """Find blocking file execution by file hash if it exists."""
        if not file_hash.file_hash:
            return None

        try:
            return WorkflowFileExecution.objects.get(
                workflow_execution=execution,
                file_hash=file_hash.file_hash,
                file_path=file_hash.file_path,
                status__in=[
                    status.value
                    for status in ExecutionStatus.get_skip_processing_statuses()
                ],
            )
        except WorkflowFileExecution.DoesNotExist:
            return None

    def _find_blocking_file_execution_by_provider_uuid(
        self, execution, file_hash: FileHash
    ):
        """Find blocking file execution by provider UUID if it exists."""
        if not file_hash.provider_file_uuid:
            return None

        try:
            return WorkflowFileExecution.objects.get(
                workflow_execution=execution,
                provider_file_uuid=file_hash.provider_file_uuid,
                file_path=file_hash.file_path,
                status__in=[
                    status.value
                    for status in ExecutionStatus.get_skip_processing_statuses()
                ],
            )
        except WorkflowFileExecution.DoesNotExist:
            return None

    def _log_duplicate_file_execution_skip(
        self, file_hash: FileHash, file_exec, execution
    ):
        """Log message when skipping file due to duplicate/blocking execution."""
        message = (
            f"Skipping file '{file_hash.file_name}' - status: {file_exec.status} "
            f"in execution {execution.id}, file execution id: {file_exec.id}"
        )
        self.workflow_log.publish_log(message)
        logger.info(message)

    def _should_skip_file(
        self,
        file_hash: FileHash,
        patterns: list[str],
    ) -> bool:
        """Check if the given file should be skipped.

        Args:
            file_hash (FileHash): The hash of the file.
            patterns (list[str]): The patterns to match against file names.

        Returns:
            bool: True if the file should be skipped, False otherwise.
        """
        file_name = os.path.basename(file_hash.file_path)

        # Existing pattern check
        if not self._should_process_file(file_name, patterns):
            return True

        # Existing file history check
        if not self._is_new_file(file_hash=file_hash, workflow=self.endpoint.workflow):
            return True

        # NEW: Check if file is being processed
        if self._is_file_being_processed(file_hash):
            return True

        return False

    def _is_duplicate(self, file_hash: FileHash, unique_file_paths: set[str]) -> bool:
        return (
            file_hash.file_path in unique_file_paths
            or file_hash.file_name in unique_file_paths
        )

    def _update_unique_file_paths(
        self, file_hash: FileHash, unique_file_paths: set[str]
    ) -> None:
        if file_hash.file_path:
            unique_file_paths.add(file_hash.file_path)
        elif file_hash.file_name:
            unique_file_paths.add(file_hash.file_name)

    def _is_directory(
        self,
        source_fs: UnstractFileSystem,
        file_path: str,
        metadata: dict[str, Any],
        dirs: Collection[str],
    ) -> bool:
        """Check if the given path is a directory.

        Args:
            source_fs (UnstractFileSystem): The file system object used for
                reading the file.
            file_path (str): The path of the file.
            metadata (dict[str, Any]): The metadata of the file.
            dirs (Collection[str]): The collection of directory names.

        Returns:
            bool: True if the file is a directory, False otherwise.
        """
        try:
            # Check if the path is a directory using metadata first.
            # Some connectors incorrectly label directories as files, so if metadata is inconclusive or fails,
            # fall back to other checks: directory listing, path suffix ("/"), or zero file size.
            if source_fs.is_dir_by_metadata(metadata):
                logger.info(
                    f"[Directory Check] '{file_path}' identified as a directory via metadata."
                )
                return True
        except NotImplementedError:
            logger.debug(
                f"[Directory Check] Metadata-based check not implemented for '{file_path}'."
            )
        except Exception as e:
            logger.warning(
                f"[Directory Check] Error while checking metadata for '{file_path}': {e}"
            )

        file_name = os.path.basename(file_path)

        # Fallback 1: Check if the file is explicitly listed in directory entries
        if file_name in dirs:
            logger.info(
                f"[Directory Check] '{file_path}' identified as a directory via checking list of directories."
            )
            return True

        # Fallback 2: Check if the path ends with a slash
        if file_path.endswith("/"):
            logger.info(
                f"[Directory Check] '{file_path}' identified as a directory based on path suffix '/'."
            )
            return True

        # Fallback 3: Check if the file has size zero
        if source_fs.get_file_size(metadata=metadata) == 0:
            logger.info(
                f"[Directory Check] '{file_path}' identified as a directory based on file size = 0."
            )
            return True

        return False

    def _should_process_file(self, file: str, patterns: list[str]) -> bool:
        """Check if the file should be processed based on the patterns.

        Args:
            file: The filename to check
            patterns: List of patterns to match against

        Returns:
            bool: True if file should be processed, False otherwise
        """
        if not file:
            return False

        file_lower = file.lower()
        matches_pattern = any(
            fnmatch.fnmatchcase(file_lower, pattern.lower()) for pattern in patterns
        )

        return matches_pattern and self._is_valid_pattern(file)

    def _is_valid_pattern(self, file_name: str) -> bool:
        """Check if the file has a supported format.

        Args:
            file_name: The filename to check

        Returns:
            bool: True if file format is supported, False otherwise
        """
        if not FilePattern.is_supported(file_name):
            message = f"Skipping '{file_name}' as it has an unsupported file format"
            logger.debug(message)
            self.workflow_log.publish_log(message)
            return False

        return True

    def _is_new_file(
        self,
        file_hash: FileHash,
        workflow: Workflow,
    ) -> bool:
        """Check if the file is new or already processed."""
        # Always treat the file as new if history usage is not enforced
        if not self.use_file_history:
            return True

        # Always treat the file as new if neither identifier is available
        if not file_hash.provider_file_uuid and not file_hash.file_hash:
            return True

        current_file_path = file_hash.file_path
        file_history = self._get_file_history(file_hash=file_hash, workflow=workflow)

        # No history or incomplete history means the file is new
        if not file_history or not file_history.is_completed():
            return True

        # Note: To enforce content-only deduplication (ignoring file path), introduce a
        # `use_content_deduplication_only` flag. If enabled, apply the check here and return False accordingly.

        # Compare file paths
        if file_history.file_path and file_history.file_path != current_file_path:
            return True

        # Default: file has been processed with the same path
        self._log_file_skipped(current_file_path)
        return False

    def _log_file_skipped(self, file_path: str):
        """Log a message indicating that a file has been skipped.

        Args:
            file_path (str): The path of the file.
        """
        msg = f"Skipping file '{file_path}' as it has already been processed."
        self.workflow_log.publish_log(
            msg + " Clear the file markers to process it again."
        )
        logger.info(msg)

    def _get_file_history(
        self,
        file_hash: FileHash,
        workflow: Workflow,
    ) -> FileHistory | None:
        """Retrieve file history using provider UUID or legacy cache key."""
        provider_file_uuid = file_hash.provider_file_uuid
        # Note: For content-only deduplication, use the `use_content_deduplication_only` flag to fetch file history
        # without including file path as a filter.

        if provider_file_uuid:
            logger.info(f"Checking file history for provider UUID: {provider_file_uuid}")
            return FileHistoryHelper.get_file_history(
                workflow=workflow,
                provider_file_uuid=provider_file_uuid,
                file_path=file_hash.file_path,
                workflow_log=self.workflow_log,
            )
        return None

    def _get_file_execution_by_file_hash(
        self,
        file_hash: FileHash,
        workflow: Workflow,
    ) -> WorkflowFileExecution | None:
        """Retrieve file execution by file hash."""
        # Build base query conditions
        base_conditions = Q(workflow_execution__workflow=workflow)
        content_conditions = Q()
        if file_hash.provider_file_uuid:
            content_conditions |= Q(provider_file_uuid=file_hash.provider_file_uuid)
        if file_hash.file_hash:
            content_conditions |= Q(file_hash=file_hash.file_hash)

        # Filter file executions based on conditions
        conditions = base_conditions & content_conditions
        file_execution = WorkflowFileExecution.objects.filter(conditions).first()
        if file_execution:
            return file_execution
        return None

    def _create_file_hash(
        self,
        file_path: str,
        source_fs: UnstractFileSystem,
        file_size: int,
        fs_metadata: dict[str, Any],
    ) -> FileHash:
        """Create a FileHash object for the matched file."""
        file_name = os.path.basename(file_path)
        provider_file_uuid = source_fs.get_file_system_uuid(
            file_path=file_path, metadata=fs_metadata
        )
        serialized_metadata = source_fs.serialize_metadata_value(value=fs_metadata)
        connection_type = self.endpoint.connection_type
        return FileHash(
            file_path=file_path,
            source_connection_type=connection_type,
            file_name=file_name,
            file_size=file_size,
            provider_file_uuid=provider_file_uuid,
            fs_metadata=serialized_metadata,
        )

    def list_files_from_source(
        self, file_hashes: dict[str, FileHash] = {}
    ) -> tuple[dict[str, FileHash], int]:
        """List files from source connector.

        Args:
            api_storage_dir (Optional[str], optional): API storage directory
        Returns:
            tuple[dict[str, FileHash], int]: A dictionary of FileHashes,
            along with the total count of matched files.
        """
        connection_type = self.endpoint.connection_type
        if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
            files = self.list_files_from_file_connector()
        elif connection_type == WorkflowEndpoint.ConnectionType.API:
            files = self.list_file_from_api_storage(file_hashes)
        else:
            raise InvalidSourceConnectionType()
        # TODO: move this to where file is listed at source.
        for index, file_hash in enumerate(files.values(), start=1):
            file_hash.file_number = index

        return files, len(files)

    def get_file_content_hash(self, source_fs: UnstractFileSystem, file_path: str) -> str:
        """Generate a hash value from the file content.

        Args:
            source_fs (UnstractFileSystem): The file system object used for
                reading the file.
            file_path (str): The path of the file.

        Returns:
            str: The hash value of the file content.
        """
        file_content_hash = sha256()
        source = source_fs.get_fsspec_fs()
        with source.open(file_path, "rb") as remote_file:
            while chunk := remote_file.read(self.READ_CHUNK_SIZE):
                file_content_hash.update(chunk)
        return file_content_hash.hexdigest()

    def copy_file_to_infile_dir(self, source_file_path: str, infile_path: str) -> None:
        """Copy the source file to the infile directory.

        Args:
            source_file_path (str): The path of the source file.
            infile_path (str): The destination path in the infile directory.
        """
        shutil.copyfile(source_file_path, infile_path)
        logger.info(f"File copied from {source_file_path} to {infile_path}")

    def add_input_from_connector_to_volume(self, file_hash: FileHash) -> str:
        """Add input file to execution directory.

        Args:
            file_hash (FileHash): The file hash object.

        Returns:
            str: The hash value of the file content.

        Raises:
            FileHashNotFound: If the hash value of the file content is not found.
        """
        logger.info(
            f"Adding input file from source connector to execution directory: {file_hash.file_name}"
        )
        input_file_path = file_hash.file_path
        source_file_path = self.source_file
        infile_path = self.infile
        if not source_file_path or not infile_path:
            raise SourceFileOrInfilePathNotFound()
        connector: ConnectorInstance = self.endpoint.connector_instance
        connector_settings: dict[str, Any] = connector.connector_metadata
        source_fs = self.get_fsspec(
            settings=connector_settings, connector_id=connector.connector_id
        )

        workflow_file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
        workflow_file_storage = workflow_file_system.get_file_storage()

        first_iteration = True
        input_log = ""
        file_content_hash = sha256()
        with source_fs.open(input_file_path, "rb") as remote_file:
            while chunk := remote_file.read(self.READ_CHUNK_SIZE):
                file_content_hash.update(chunk)
                if first_iteration:
                    # Detect MIME type using first chunk
                    mime_type = resolve_inconclusive_mime_type(
                        magic.from_buffer(chunk, mime=True), chunk
                    )
                    logger.info(
                        f"Detected MIME type: {mime_type} for file {input_file_path}"
                    )
                    if not AllowedFileTypes.is_allowed(mime_type):
                        raise UnsupportedMimeTypeError(
                            f"Unsupported MIME type '{mime_type}' for file '{input_file_path}'"
                        )
                    input_log = (
                        chunk[:500].decode("utf-8", errors="replace") + "...(truncated)"
                    )
                    first_iteration = False
                # write the chunk in to execution directory
                workflow_file_storage.write(path=source_file_path, mode="ab", data=chunk)
                workflow_file_storage.write(path=infile_path, mode="ab", data=chunk)

        # publish input file content
        # TODO: Consider removing this since the input is not extracted text.
        # This function is typically relevant for extracted text content,
        # may not be necessary for PDFs, images, or other non-text formats.
        self.publish_input_file_content(input_file_path, input_log)
        hash_value_of_file_content = file_content_hash.hexdigest()
        file_hash.mime_type = mime_type
        logger.info(
            f"hash_value_of_file {source_file_path} is : {hash_value_of_file_content}"
        )

        logger.info(f"{input_file_path} is added to execution directory")
        return hash_value_of_file_content

    def add_input_from_api_storage_to_volume(self, input_file_path: str) -> str:
        """Add input file to execution directory from api storage."""
        logger.info(
            f"Adding input file from api storage to execution directory: {input_file_path}"
        )
        source_file_path = self.source_file
        infile_path = self.infile
        if not source_file_path or not infile_path:
            raise SourceFileOrInfilePathNotFound()

        api_file_system = FileSystem(FileStorageType.API_EXECUTION)
        api_file_storage = api_file_system.get_file_storage()
        workflow_file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
        workflow_file_storage = workflow_file_system.get_file_storage()
        file_content_hash = self._copy_file_to_destination(
            source_storage=api_file_storage,
            destination_storage=workflow_file_storage,
            source_path=input_file_path,
            destination_paths=[infile_path, source_file_path],
        )
        logger.info(f"File {input_file_path} added to execution directory")
        return file_content_hash

    # TODO: replace it with method from SDK Utils
    def _copy_file_to_destination(
        self,
        source_storage: FileStorage,
        destination_storage: FileStorage,
        source_path: str,
        destination_paths: list[str],
    ) -> str:
        """Copy a file from a source storage to one or more paths in a
        destination storage.

        This function reads the source file in chunks and writes each chunk to
        the specified destination paths. The function will continue until the
        entire source file is copied.

        Args:
            source_storage (FileStorage): The storage object from which
                the file is read.
            destination_storage (FileStorage): The storage object to which
                the file is written.
            source_path (str): The path of the file in the source storage.
            destination_paths (list[str]): A list of paths where the file will be
                copied in the destination storage.

        Returns:
            str: The SHA-256 hash of the file content.
        """
        seek_position = 0  # Start from the beginning
        file_content_hash = sha256()
        # Loop to read and write in chunks until the end of the file
        while chunk := source_storage.read(
            path=source_path,
            mode="rb",
            seek_position=seek_position,
            length=self.READ_CHUNK_SIZE,
        ):
            file_content_hash.update(chunk)
            # Write the chunk to each destination path
            for destination_file in destination_paths:
                destination_storage.write(
                    path=destination_file,
                    mode="ab",
                    data=chunk,
                )

            # Update the seek position
            seek_position += len(chunk)
        return file_content_hash.hexdigest()

    def add_file_to_volume(
        self,
        workflow_file_execution: WorkflowFileExecution,
        tags: list[str],
        file_hash: FileHash,
        llm_profile_id: str | None = None,
        custom_data: dict[str, Any] | None = None,
    ) -> str:
        """Add input file to execution directory.

        Args:
            workflow_file_execution: WorkflowFileExecution model
            tags (list[str]): Tag names associated with the workflow execution.
            file_hash: FileHash model
            llm_profile_id (str, optional): LLM profile ID for overriding tool settings.

        Raises:
            InvalidSource: _description_

        Returns:
            str: file_name
        """
        logger.info(f"Adding input file to execution directory: {file_hash.file_name}")
        connection_type = self.endpoint.connection_type
        input_file_path = file_hash.file_path
        if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
            file_content_hash = self.add_input_from_connector_to_volume(
                file_hash=file_hash
            )
        elif connection_type == WorkflowEndpoint.ConnectionType.API:
            file_content_hash = self.add_input_from_api_storage_to_volume(
                input_file_path=input_file_path
            )
            # Filehash created from Django requests files might be different
            # from file_content_hash created from file content here.
            # So use file_content_hash only when file_hash is not available
            file_content_hash = (
                file_hash.file_hash if file_hash.file_hash else file_content_hash
            )
        else:
            raise InvalidSourceConnectionType()

        self.add_metadata_to_volume(
            input_file_path=input_file_path,
            file_execution_id=workflow_file_execution.id,
            source_hash=file_content_hash,
            tags=tags,
            llm_profile_id=llm_profile_id,
            custom_data=custom_data,
        )
        return file_content_hash

    def handle_final_result(
        self,
        results: list[dict[str, Any]],
        file_name: str,
        result: str | None,
    ) -> None:
        connection_type = self.endpoint.connection_type
        if connection_type == WorkflowEndpoint.ConnectionType.API:
            results.append({"file": file_name, "result": result})

    def load_file(self, input_file_path: str) -> tuple[str, BytesIO]:
        """Load file contnt and file name based on the file path.

        Args:
            input_file_path (str): source file

        Raises:
            InvalidSource: _description_

        Returns:
            tuple[str, BytesIO]: file_name , file content
        """
        connector: ConnectorInstance = self.endpoint.connector_instance
        connector_settings: dict[str, Any] = connector.connector_metadata
        source_fs: fsspec.AbstractFileSystem = self.get_fsspec(
            settings=connector_settings, connector_id=connector.connector_id
        )
        with source_fs.open(input_file_path, "rb") as remote_file:
            file_content = remote_file.read()
            file_stream = BytesIO(file_content)

        return os.path.basename(input_file_path), file_stream

    @classmethod
    def _detect_uploaded_file_mime_type(cls, file: UploadedFile) -> str | None:
        """Detect an uploaded file's MIME type from its own bytes.

        The multipart Content-Type is supplied by the caller and never verified,
        so it cannot be used to decide what is allowed into API storage.

        Returns None for an empty upload, which has no type to judge.
        """
        sample = file.read(cls.MIME_DETECT_CHUNK_SIZE)
        file.seek(0)
        if not sample:
            # An empty upload is its own failure downstream, not a type error.
            return None

        mime_type = magic.from_buffer(sample, mime=True)
        if mime_type in cls.CONTAINER_MIME_TYPES:
            mime_type = cls._detect_container_mime_type(file, fallback=mime_type)
        return resolve_inconclusive_mime_type(mime_type, sample)

    @classmethod
    def _detect_container_mime_type(cls, file: UploadedFile, fallback: str) -> str:
        """Resolve a wrapper by classifying the file from a path.

        A path rather than a buffer, because libmagic will not name a zip
        container from one; an in-memory upload is spilled to a temp file.
        """
        temporary_file_path = getattr(file, "temporary_file_path", None)
        if temporary_file_path is not None:
            return cls._classify_file_path(temporary_file_path(), fallback)

        with tempfile.NamedTemporaryFile(suffix=".upload") as spill:
            for chunk in file.chunks(chunk_size=cls.READ_CHUNK_SIZE):
                spill.write(chunk)
            spill.flush()
            file.seek(0)
            return cls._classify_file_path(spill.name, fallback)

    @classmethod
    def _classify_file_path(cls, path: str, fallback: str) -> str:
        """Classify a staged path, looking inside a zip when that is all we get."""
        mime_type = magic.from_file(path, mime=True)
        if mime_type == "application/zip":
            inside = identify_zip_container(path)
            return inside or mime_type
        return mime_type or fallback

    @classmethod
    def _report_rejected_file(
        cls,
        workflow_log: WorkflowLog,
        workflow_id: str,
        execution_id: str,
        file: UploadedFile,
        file_name: str,
        mime_type: str | None,
        log_message: str,
        position: int,
    ) -> None:
        """Record a rejection for the caller without risking the whole request.

        Runs per file inside the staging loop, so nothing here may escape and
        cost the caller the files already staged.
        """
        workflow_log.log_error(logger=logger, message=log_message)
        try:
            ResultCacheUtils.update_api_results(
                workflow_id=workflow_id,
                execution_id=execution_id,
                api_result=FileExecutionResult(file=file_name, error=log_message),
            )
        except Exception:
            logger.exception(
                f"Could not record the rejection of '{file_name}' for the caller"
            )

        try:
            cls._persist_rejected_file_execution(
                execution_id=execution_id,
                file=file,
                file_name=file_name,
                mime_type=mime_type,
                log_message=log_message,
                position=position,
            )
        except Exception:
            logger.exception(
                f"Could not persist a file execution row for rejected '{file_name}'"
            )

    @classmethod
    def _persist_rejected_file_execution(
        cls,
        execution_id: str,
        file: UploadedFile,
        file_name: str,
        mime_type: str | None,
        log_message: str,
        position: int,
    ) -> None:
        """Record a rejected file as a terminal ERROR file execution.

        The cached result is short-lived, so the row is what keeps the rejection
        both auditable and countable.
        """
        workflow_execution = WorkflowExecution.objects.get(pk=execution_id)

        # The row's identity rests on this alone, since the API path has no
        # file_path to tell two rejections apart. Content, name and the upload's
        # position in the request, so every rejected part gets its own record
        # even when two are byte-identical and share a name. Not a plain content
        # hash on purpose: this file was never processed and must not look to
        # file history as though it were.
        digest = sha256()
        for chunk in file.chunks(chunk_size=cls.READ_CHUNK_SIZE):
            digest.update(chunk)
        file.seek(0)
        row_identity = sha256(
            digest.digest() + file_name.encode("utf-8") + str(position).encode("ascii")
        ).hexdigest()

        file_execution = WorkflowFileExecution.objects.get_or_create_file_execution(
            workflow_execution=workflow_execution,
            file_hash=FileHash(
                file_path=None,
                source_connection_type=WorkflowEndpoint.ConnectionType.API,
                file_name=file_name,
                file_hash=row_identity,
                file_size=file.size,
                mime_type=mime_type,
            ),
            is_api=True,
        )
        file_execution.update_status(
            status=ExecutionStatus.ERROR, execution_error=log_message
        )

    @classmethod
    def add_input_file_to_api_storage(
        cls,
        pipeline_id: str,
        workflow_id: str,
        execution_id: str,
        file_objs: list[UploadedFile],
        use_file_history: bool = False,
    ) -> dict[str, FileHash]:
        """Add input file to api storage.

        Args:
            workflow_id (str): UUID of the worklfow
            execution_id (str): UUID of the execution
            file_objs (list[UploadedFile]): List of uploaded files
            use_file_history (bool): Use FileHistory table to return results on already
                processed files. Defaults to False
        Returns:
            dict[str, FileHash]: Dict containing file name and its corresponding hash
        """
        org_schema = UserContext.get_organization_identifier()
        workflow_log = WorkflowLog(
            execution_id=execution_id,
            log_stage=LogStage.SOURCE,
            pipeline_id=pipeline_id,
            organization_id=org_schema,
        )
        workflow_log.publish_log(
            "Staging files in API storage for validation and processing."
        )
        api_storage_dir = cls.get_api_storage_dir_path(
            workflow_id=workflow_id, execution_id=execution_id
        )
        workflow: Workflow = Workflow.objects.get(id=workflow_id)
        file_hashes: dict[str, FileHash] = {}
        unique_file_hashes: set[str] = set()
        connection_type = WorkflowEndpoint.ConnectionType.API
        for position, file in enumerate(file_objs):
            file_name = file.name
            destination_path = os.path.join(api_storage_dir, file_name)

            try:
                mime_type = cls._detect_uploaded_file_mime_type(file)
            except (OSError, ValueError, magic.MagicException):
                # Narrow on purpose: these are faults in THIS upload's bytes, so
                # failing the one file is right. A broken libmagic database or an
                # unreadable temp dir hits every file in every request, and
                # swallowing that would tell each caller their files are invalid
                # while the platform is down - it must propagate and fail loudly.
                log_message = (
                    f"Rejecting file '{file_name}': could not determine its type"
                )
                logger.exception(log_message)
                cls._report_rejected_file(
                    workflow_log,
                    workflow_id,
                    execution_id,
                    file,
                    file_name,
                    None,
                    log_message,
                    position,
                )
                continue

            logger.info(f"Detected MIME type: {mime_type} for file {file_name}")

            if mime_type is not None and not AllowedFileTypes.is_allowed(mime_type):
                log_message = (
                    f"Rejecting file '{file_name}' with unsupported MIME type "
                    f"'{mime_type}'"
                )
                # Rejected files are never dispatched, so nothing downstream will
                # report on them - surface the failure in the API response here.
                cls._report_rejected_file(
                    workflow_log,
                    workflow_id,
                    execution_id,
                    file,
                    file_name,
                    mime_type,
                    log_message,
                    position,
                )
                continue

            file_system = FileSystem(FileStorageType.API_EXECUTION)
            file_storage = file_system.get_file_storage()
            file_hash = sha256()
            for chunk in file.chunks(chunk_size=cls.READ_CHUNK_SIZE):
                file_hash.update(chunk)
                file_storage.write(path=destination_path, mode="ab", data=chunk)
            file_hash = file_hash.hexdigest()

            # Skip duplicate files
            if file_hash in unique_file_hashes:
                log_message = f"Skipping file '{file_name}' — duplicate detected within the current request. Already staged for processing."
                workflow_log.log_info(logger=logger, message=log_message)
                continue
            unique_file_hashes.add(file_hash)

            file_history = None
            if use_file_history:
                file_history = FileHistoryHelper.get_file_history(
                    workflow=workflow, cache_key=file_hash
                )
            is_executed = True if file_history and file_history.is_completed() else False
            file_hash = FileHash(
                file_path=destination_path,
                source_connection_type=connection_type,
                file_name=file_name,
                file_hash=file_hash,
                is_executed=is_executed,
                file_size=file.size,
                mime_type=mime_type,
            )
            file_hashes.update({file_name: file_hash})
        return file_hashes

    @classmethod
    def create_endpoint_for_workflow(
        cls,
        workflow: Workflow,
    ) -> None:
        """Creating WorkflowEndpoint entity."""
        endpoint = WorkflowEndpoint(
            workflow=workflow,
            endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
        )
        endpoint.save()

    @classmethod
    def get_json_schema_for_api(cls) -> dict[str, Any]:
        """Json schema for api.

        Returns:
            dict[str, Any]: _description_
        """
        schema_path = os.path.join(os.path.dirname(__file__), "static", "src", "api.json")
        return cls.get_json_schema(file_path=schema_path)

    @classmethod
    def get_json_schema_for_file_system(cls) -> dict[str, Any]:
        """Json schema for Filesystem.

        Returns:
            dict[str, Any]: _description_
        """
        schema_path = os.path.join(
            os.path.dirname(__file__), "static", "src", "file.json"
        )
        return cls.get_json_schema(file_path=schema_path)

    def get_config(self) -> SourceConfig:
        """Get serializable configuration for the source connector.

        Returns:
            SourceConfig: Configuration containing all necessary data to reconstruct the connector
        """
        source_config = SourceConfig(
            workflow_id=self.workflow.id,
            execution_id=self.execution_id,
            organization_id=self.organization_id,
            use_file_history=self.use_file_history,
        )
        return source_config

    @classmethod
    def from_config(
        cls, workflow_log: WorkflowLog, config: SourceConfig
    ) -> "SourceConnector":
        """Create a SourceConnector instance from configuration.

        Args:
            workflow_log (WorkflowLog): Workflow log instance
            config (SourceConfig): Configuration containing all necessary data to reconstruct the connector

        Returns:
            SourceConnector: New instance
        """
        # Reconstruct workflow
        workflow = Workflow.objects.get(id=config.workflow_id)

        # Create source connector instance
        source = cls(
            workflow=workflow,
            execution_id=config.execution_id,
            workflow_log=workflow_log,
            use_file_history=config.use_file_history,
            organization_id=config.organization_id,
            file_execution_id=config.file_execution_id,
        )

        return source