Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

38.1 KB
Raw Blame
"""File Discovery Service

This module provides the StreamingFileDiscovery service, moved to shared/processing
to avoid circular imports and provide clean separation of concerns.
"""

import time
from collections.abc import Collection
from typing import Any

from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
from unstract.core.data_models import ConnectionType, FileHashData, FileOperationConstants
from unstract.core.file_operations import FileOperations

from ..infrastructure.logging import WorkerLogger
from .filter_pipeline import FilterPipeline

logger = WorkerLogger.get_logger(__name__)


class StreamingFileDiscovery:
    """Streams files from directories with early filtering and termination.

    This class replaces the inefficient "discover-all-then-filter" approach with
    a streaming system that:
    1. Walks directories incrementally
    2. Applies ALL filters as files are discovered
    3. Stops immediately when limit is reached
    4. Uses batch processing for efficient API calls
    """

    def __init__(
        self,
        source_fs: UnstractFileSystem,
        api_client,  # Removed type hint to avoid import
        workflow_id: str,
        execution_id: str,
        organization_id: str,
        connector_id: str | None = None,
    ):
        """Initialize streaming file discovery.

        Args:
            source_fs: Filesystem connector
            api_client: API client for backend communication
            workflow_id: Workflow ID for filtering
            execution_id: Current execution ID
            organization_id: Organization ID
            connector_id: Optional connector ID for metadata
        """
        self.source_fs = source_fs
        self.api_client = api_client
        self.workflow_id = workflow_id
        self.execution_id = execution_id
        self.organization_id = organization_id
        self.connector_id = connector_id
        self.fs_fsspec = source_fs.get_fsspec_fs()

    def discover_files_streaming(
        self,
        directories: list[str],
        patterns: list[str],
        recursive: bool,
        file_hard_limit: int,
        filter_pipeline: FilterPipeline,
        batch_size: int = 100,
    ) -> tuple[dict[str, FileHashData], int]:
        """Discover files using streaming with early filtering.

        This is the main entry point that replaces the old _get_matched_files.
        It discovers files incrementally and applies all filters immediately.

        Args:
            directories: List of directories to search
            patterns: File patterns to match
            recursive: Whether to search recursively
            file_hard_limit: Maximum files to return (hard stop)
            filter_pipeline: Pipeline of filters to apply
            batch_size: Size of batches for processing

        Returns:
            Tuple of (matched_files, count)
        """
        start_time = time.time()

        matched_files: dict[str, FileHashData] = {}
        batch_buffer: list[tuple[str, dict[str, Any]]] = []

        # Metrics tracking
        metrics = {
            "total_files_discovered": 0,
            "files_pattern_matched": 0,
            "files_after_filtering": 0,
            "batches_processed": 0,
            "directories_walked": 0,
        }

        # Calculate max depth for recursive search
        max_depth = int(FileOperationConstants.MAX_RECURSIVE_DEPTH) if recursive else 1

        logger.info(
            f"[StreamingDiscovery] Starting streaming discovery for {len(directories)} directories "
            f"with limit={file_hard_limit}, batch_size={batch_size}, recursive={recursive}, patterns={patterns}, "
            f"max_depth={max_depth}"
        )

        try:
            for directory in directories:
                # Check if we've reached the limit
                if len(matched_files) >= file_hard_limit:
                    logger.info(
                        f"[StreamingDiscovery] Reached file limit ({file_hard_limit}), "
                        f"stopping discovery early"
                    )
                    break

                logger.info(f"[StreamingDiscovery] Processing directory: {directory}")

                def _on_walk_error(error: Exception) -> None:
                    logger.warning(
                        f"[StreamingDiscovery] Failed to list directory: {error}"
                    )

                # Walk directory with max depth control
                for _root, dirs, files in self.fs_fsspec.walk(
                    directory, maxdepth=max_depth, detail=True, on_error=_on_walk_error
                ):
                    metrics["directories_walked"] += 1

                    # Check limit before processing directory
                    if len(matched_files) >= file_hard_limit:
                        break

                    fs_metadata_list: list[dict[str, Any]] = list(files.values())

                    # Process files in this directory
                    for fs_metadata in fs_metadata_list:
                        # Early termination check
                        if len(matched_files) >= file_hard_limit:
                            break

                        file_path = fs_metadata.get("name")
                        if not file_path:
                            logger.info(
                                f"DEBUG: [StreamingDiscovery] Skipping item with no name: {fs_metadata}"
                            )
                            continue

                        # Log detailed file metadata for debugging
                        file_type = fs_metadata.get("type", "unknown")
                        file_size = fs_metadata.get("size", "unknown")
                        logger.info(
                            f"DEBUG: [StreamingDiscovery] Discovered item: '{file_path}' (type: {file_type}, size: {file_size})"
                        )

                        metrics["total_files_discovered"] += 1

                        # Skip directories with detailed logging
                        is_directory = self._is_directory(file_path, fs_metadata, dirs)
                        if is_directory:
                            logger.info(
                                f"DEBUG: [StreamingDiscovery] Skipping directory: {file_path}"
                            )
                            continue

                        logger.info(
                            f"DEBUG: [StreamingDiscovery] File passed directory check: {file_path}"
                        )

                        # Apply pattern filter first (cheapest)
                        pattern_matches = self._matches_patterns(file_path, patterns)
                        if not pattern_matches:
                            logger.info(
                                f"DEBUG: [StreamingDiscovery] File failed pattern match: {file_path} (patterns: {patterns})"
                            )
                            continue

                        logger.info(
                            f"DEBUG: [StreamingDiscovery] File passed pattern match: {file_path}"
                        )
                        metrics["files_pattern_matched"] += 1

                        # Add to batch buffer
                        batch_buffer.append((file_path, fs_metadata))
                        logger.info(
                            f"DEBUG: [StreamingDiscovery] Added to batch buffer: {file_path} (buffer size: {len(batch_buffer)})"
                        )

                        # Process batch when full
                        if len(batch_buffer) >= batch_size:
                            self._process_batch(
                                batch_buffer,
                                matched_files,
                                filter_pipeline,
                                file_hard_limit,
                            )
                            metrics["batches_processed"] += 1
                            batch_buffer = []

                            # Check if we've reached limit after batch processing
                            if len(matched_files) >= file_hard_limit:
                                logger.info(
                                    "[StreamingDiscovery] Reached limit after batch processing"
                                )
                                break

            # Process remaining files in buffer
            if batch_buffer and len(matched_files) < file_hard_limit:
                self._process_batch(
                    batch_buffer, matched_files, filter_pipeline, file_hard_limit
                )
                metrics["batches_processed"] += 1

            # Ensure we never exceed the hard limit
            if len(matched_files) > file_hard_limit:
                logger.info(
                    f"[StreamingDiscovery] Trimming results from {len(matched_files)} to {file_hard_limit}"
                )
                matched_files = dict(list(matched_files.items())[:file_hard_limit])

            final_count = len(matched_files)
            metrics["files_after_filtering"] = final_count
            elapsed_time = time.time() - start_time

            # Log comprehensive metrics
            logger.info(
                f"[StreamingDiscovery] 🎯 Discovery complete in {elapsed_time:.2f}s:\n"
                f"  • Directories walked: {metrics['directories_walked']}\n"
                f"  • Total files discovered: {metrics['total_files_discovered']}\n"
                f"  • Files matching patterns: {metrics['files_pattern_matched']}\n"
                f"  • Files after all filters: {metrics['files_after_filtering']}\n"
                f"  • Batches processed: {metrics['batches_processed']}\n"
                f"  • Hard limit: {file_hard_limit}\n"
                f"  • Early termination: {'Yes' if final_count >= file_hard_limit else 'No'}"
            )

            # Performance metrics
            if metrics["total_files_discovered"] > 0:
                filter_efficiency = (
                    (metrics["total_files_discovered"] - final_count)
                    / metrics["total_files_discovered"]
                    * 100
                )
                logger.info(
                    f"[StreamingDiscovery] 📊 Performance metrics:\n"
                    f"  • Filter efficiency: {filter_efficiency:.1f}% files filtered out\n"
                    f"  • Discovery rate: {metrics['total_files_discovered'] / elapsed_time:.0f} files/sec\n"
                    f"  • Final rate: {final_count / elapsed_time:.0f} accepted files/sec"
                )

            # Create cache entries for files that will be processed to prevent race conditions
            try:
                from ..workflow.execution.active_file_manager import ActiveFileManager

                cache_stats = ActiveFileManager.create_cache_entries_simple(
                    files_to_cache=matched_files,  # Create cache entries for all final files
                    workflow_id=self.workflow_id,
                    execution_id=self.execution_id,
                    logger_instance=logger,
                )

                logger.info(
                    f"[StreamingDiscovery] 🔒 Created {cache_stats.get('cache_created', 0)} cache entries "
                    f"for race condition prevention"
                )

                # Return original matched_files since FilterPipeline already applied all filtering
                return matched_files, final_count

            except Exception as cache_error:
                logger.warning(
                    f"[StreamingDiscovery] Cache creation failed (proceeding anyway): {cache_error}"
                )
                return matched_files, final_count

        except Exception as e:
            logger.error(
                f"[StreamingDiscovery] Error during file discovery: {e}", exc_info=True
            )
            # Return what we've collected so far
            return matched_files, len(matched_files)

    def _process_batch(
        self,
        batch_buffer: list[tuple[str, dict[str, Any]]],
        matched_files: dict[str, FileHashData],
        filter_pipeline: FilterPipeline,
        file_hard_limit: int,
    ) -> None:
        """Process a batch of files through the filter pipeline.

        Args:
            batch_buffer: Buffer of (file_path, metadata) tuples
            matched_files: Dictionary to add accepted files to
            filter_pipeline: Pipeline of filters to apply
            file_hard_limit: Maximum number of files to collect
        """
        if not batch_buffer:
            return

        logger.info(f"[StreamingDiscovery] Processing batch of {len(batch_buffer)} files")

        # Log the files being processed in this batch for debugging
        batch_files = [file_path for file_path, _ in batch_buffer]
        logger.info(f"DEBUG: [StreamingDiscovery] Batch files: {batch_files}")

        # Convert batch to FileHashData objects with defensive validation
        file_hash_batch: dict[str, FileHashData] = {}
        for file_path, fs_metadata in batch_buffer:
            try:
                # Validate file path and extract file name
                if not file_path or not isinstance(file_path, str):
                    logger.warning(
                        f"[StreamingDiscovery] Skipping invalid file path: {file_path}"
                    )
                    continue

                # Extract and validate file name
                import os

                file_name = os.path.basename(file_path.rstrip("/"))
                if not file_name or file_name in ["", ".", ".."]:
                    logger.warning(
                        f"[StreamingDiscovery] Skipping file with invalid name: '{file_path}' -> '{file_name}'"
                    )
                    continue

                # Check if this looks like a directory path
                if file_path.endswith("/") or not file_name:
                    logger.info(
                        f"DEBUG: [StreamingDiscovery] Skipping directory-like path: {file_path}"
                    )
                    continue

                logger.info(
                    f"DEBUG: [StreamingDiscovery] Processing file: '{file_path}' (name: '{file_name}', size: {fs_metadata.get('size', 0)})"
                )

                # Add connector_id to metadata if available
                if self.connector_id:
                    fs_metadata = fs_metadata.copy()
                    fs_metadata["connector_id"] = self.connector_id

                # Create FileHashData object with proper error handling
                file_hash = FileOperations.create_file_hash_from_backend_logic(
                    file_path=file_path,
                    source_fs=self.source_fs,
                    source_connection_type=ConnectionType.FILESYSTEM,
                    file_size=fs_metadata.get("size", 0),
                    fs_metadata=fs_metadata,
                    compute_content_hash=False,  # Only use provider_file_uuid
                )

                file_hash_batch[file_path] = file_hash
                logger.info(
                    f"DEBUG: [StreamingDiscovery] Successfully created FileHashData for: {file_path}"
                )

            except ValueError as ve:
                logger.error(
                    f"[StreamingDiscovery] FileHashData creation failed for '{file_path}': {ve}"
                )
                logger.info(f"DEBUG: [StreamingDiscovery] File metadata: {fs_metadata}")
                continue
            except Exception as e:
                logger.error(
                    f"[StreamingDiscovery] Unexpected error processing '{file_path}': {e}",
                    exc_info=True,
                )
                continue

        # Apply filter pipeline to batch
        filtered_batch = filter_pipeline.apply_filters(
            files=file_hash_batch,
            workflow_id=self.workflow_id,
            execution_id=self.execution_id,
            api_client=self.api_client,
            organization_id=self.organization_id,
        )

        logger.info(
            f"[StreamingDiscovery] Batch processing complete: {len(batch_buffer)} raw → {len(file_hash_batch)} valid → {len(filtered_batch)} filtered"
        )

        # Add filtered files to results (respecting limit)
        added_count = 0
        for file_path, file_hash in filtered_batch.items():
            if len(matched_files) >= file_hard_limit:
                break
            matched_files[file_path] = file_hash
            added_count += 1

        if added_count > 0:
            logger.info(
                f"DEBUG: [StreamingDiscovery] Added {added_count} files to final results"
            )

    def _is_directory(
        self, file_path: str, metadata: dict[str, Any], dirs: Collection[str]
    ) -> bool:
        """Check if path is a directory using multiple detection methods.

        Args:
            file_path: Path to check
            metadata: File metadata from fsspec
            dirs: Collection of directory names from walk

        Returns:
            True if path is a directory
        """
        import os

        if not file_path:
            return False

        # 1. Check if path ends with directory separator
        if file_path.endswith("/") or file_path.endswith("\\"):
            logger.info(
                f"DEBUG: [StreamingDiscovery] Directory detected by path suffix: {file_path}"
            )
            return True

        # 2. Check if basename is in dirs list from walk
        basename = os.path.basename(file_path)
        if basename in dirs:
            logger.info(
                f"DEBUG: [StreamingDiscovery] Directory detected in dirs list: {file_path}"
            )
            return True

        # 3. Check metadata type with broader detection
        file_type = metadata.get("type", "").lower()
        if file_type in ["directory", "dir", "folder", "d"]:
            logger.info(
                f"DEBUG: [StreamingDiscovery] Directory detected by metadata type '{file_type}': {file_path}"
            )
            return True

        # 4. Check size - directories often have size 0 or None
        file_size = metadata.get("size")
        if file_size is None and metadata.get("type") != "file":
            logger.info(
                f"DEBUG: [StreamingDiscovery] Possible directory (no size, not file type): {file_path}"
            )
            return True

        # 5. Check for common directory characteristics
        if not basename or basename in [".", ".."]:
            logger.info(
                f"DEBUG: [StreamingDiscovery] Directory detected by special name: {file_path}"
            )
            return True

        # 6. Try connector-specific directory check
        try:
            if hasattr(self.source_fs, "is_dir_by_metadata"):
                is_dir = self.source_fs.is_dir_by_metadata(metadata)
                if is_dir:
                    logger.info(
                        f"DEBUG: [StreamingDiscovery] Directory detected by connector-specific check: {file_path}"
                    )
                return is_dir
            else:
                is_dir = self.fs_fsspec.isdir(file_path)
                if is_dir:
                    logger.info(
                        f"DEBUG: [StreamingDiscovery] Directory detected by fsspec.isdir: {file_path}"
                    )
                return is_dir
        except Exception as e:
            logger.info(
                f"DEBUG: [StreamingDiscovery] Directory check failed for {file_path}: {e}"
            )

        # 7. Final check: if no file extension and metadata suggests it might be a directory
        if "." not in basename and file_size == 0:
            logger.info(
                f"DEBUG: [StreamingDiscovery] Possible directory (no extension, zero size): {file_path}"
            )
            # Don't return True here, just log - this is too aggressive

        return False

    def _matches_patterns(self, file_path: str, patterns: list[str]) -> bool:
        """Check if file matches any of the patterns.

        Args:
            file_path: File path to check
            patterns: List of file patterns

        Returns:
            True if file matches any pattern
        """
        if not patterns or patterns == ["*"]:
            return True

        import fnmatch
        import os

        file_name = os.path.basename(file_path)
        for pattern in patterns:
            # Case-insensitive matching
            if fnmatch.fnmatch(file_name.lower(), pattern.lower()):
                return True

        return False


class OrderedFileDiscovery:
    """Ordered file discovery with sorting and chunked filtering.

    This class handles ORDERED file processing (FIFO/LIFO) by:
    1. Loading all file metadata into memory (up to MAX_FILES_FOR_SORTING)
    2. Sorting by modification time (oldest/newest first)
    3. Filtering in chunks using FilterPipeline to avoid overwhelming backend APIs

    This provides the same memory behavior as backend source.py while maintaining
    clean architecture separation.
    """

    def __init__(
        self,
        source_fs: UnstractFileSystem,
        api_client,  # Removed type hint to avoid import
        workflow_id: str,
        execution_id: str,
        organization_id: str,
        use_file_history: bool = True,
        connector_id: str | None = None,
    ):
        """Initialize ordered file discovery.

        Args:
            source_fs: Filesystem connector
            api_client: API client for backend communication
            workflow_id: Workflow ID for filtering
            execution_id: Current execution ID
            organization_id: Organization ID
            use_file_history: Whether to use file history filtering
            connector_id: Optional connector ID for metadata
        """
        self.source_fs = source_fs
        self.api_client = api_client
        self.workflow_id = workflow_id
        self.execution_id = execution_id
        self.organization_id = organization_id
        self.use_file_history = use_file_history
        self.connector_id = connector_id
        self.fs_fsspec = source_fs.get_fsspec_fs()

    def discover_files_ordered(
        self,
        directories: list[str],
        patterns: list[str],
        recursive: bool,
        file_hard_limit: int,
        file_processing_order: str,  # FileProcessingOrder enum value
        batch_size: int = 100,
    ) -> tuple[dict[str, FileHashData], int]:
        """Discover files with ordering (OLDEST_FIRST/NEWEST_FIRST).

        Memory behavior: Load all files → Sort → Filter in chunks
        This mirrors the backend logic exactly.

        Args:
            directories: List of directories to search
            patterns: File patterns to match
            recursive: Whether to search recursively
            file_hard_limit: Maximum files to return (hard stop)
            file_processing_order: Order to process files (OLDEST_FIRST/NEWEST_FIRST)
            batch_size: Size of filter processing chunks

        Returns:
            Tuple of (matched_files, count)
        """
        start_time = time.time()

        # Metrics tracking for comprehensive analysis
        metrics = {
            "total_files_collected": 0,
            "directories_processed": 0,
            "files_matching_patterns": 0,
            "files_after_filtering": 0,
            "batches_processed": 0,
            "collection_time": 0.0,
            "filtering_time": 0.0,
        }

        logger.info(
            f"[OrderedDiscovery] Starting ordered file discovery for {len(directories)} directories "
            f"with limit={file_hard_limit}, batch_size={batch_size}, recursive={recursive}, "
            f"patterns={patterns}, order={file_processing_order}"
        )

        try:
            # Step 1: Collect and sort all files (load into memory)
            collection_start = time.time()
            sorted_files = self._collect_and_sort_files(
                directories, recursive, file_processing_order, metrics
            )
            metrics["collection_time"] = time.time() - collection_start

            logger.info(
                f"[OrderedDiscovery] Collected {len(sorted_files)} files for ordered processing "
                f"in {metrics['collection_time']:.2f}s"
            )

            if not sorted_files:
                # Log comprehensive metrics even for empty results
                elapsed_time = time.time() - start_time
                logger.info(
                    f"[OrderedDiscovery] 🎯 Discovery complete in {elapsed_time:.2f}s:\n"
                    f"  • Directories processed: {metrics['directories_processed']}\n"
                    f"  • Total files collected: {metrics['total_files_collected']}\n"
                    f"  • Files matching patterns: 0\n"
                    f"  • Files after all filters: 0\n"
                    f"  • Batches processed: 0\n"
                    f"  • Hard limit: {file_hard_limit}\n"
                    f"  • Processing order: {file_processing_order}\n"
                    f"  • Early termination: No"
                )
                return {}, 0

            # Step 2: Process in chunks with FilterPipeline
            filtering_start = time.time()
            matched_files, total_processed = self._process_sorted_files_in_chunks(
                sorted_files, patterns, file_hard_limit, batch_size, metrics
            )
            metrics["filtering_time"] = time.time() - filtering_start
            final_count = total_processed

            # Update final metrics
            elapsed_time = time.time() - start_time
            metrics["files_after_filtering"] = final_count

            # Log comprehensive metrics
            logger.info(
                f"[OrderedDiscovery] 🎯 Discovery complete in {elapsed_time:.2f}s:\n"
                f"  • Directories processed: {metrics['directories_processed']}\n"
                f"  • Total files collected: {metrics['total_files_collected']}\n"
                f"  • Files matching patterns: {metrics['files_matching_patterns']}\n"
                f"  • Files after all filters: {metrics['files_after_filtering']}\n"
                f"  • Batches processed: {metrics['batches_processed']}\n"
                f"  • Hard limit: {file_hard_limit}\n"
                f"  • Processing order: {file_processing_order}\n"
                f"  • Early termination: {'Yes' if final_count >= file_hard_limit else 'No'}"
            )

            # Performance metrics
            if metrics["total_files_collected"] > 0:
                filter_efficiency = (
                    (metrics["total_files_collected"] - final_count)
                    / metrics["total_files_collected"]
                    * 100
                )
                collection_rate = (
                    metrics["total_files_collected"] / metrics["collection_time"]
                    if metrics["collection_time"] > 0
                    else 0
                )
                processing_rate = final_count / elapsed_time if elapsed_time > 0 else 0

                logger.info(
                    f"[OrderedDiscovery] 📊 Performance metrics:\n"
                    f"  • Filter efficiency: {filter_efficiency:.1f}% files filtered out\n"
                    f"  • Collection rate: {collection_rate:.0f} files/sec\n"
                    f"  • Processing rate: {processing_rate:.0f} accepted files/sec\n"
                    f"  • Phase timing - Collection: {metrics['collection_time']:.1f}s, "
                    f"Filtering: {metrics['filtering_time']:.1f}s"
                )

            return matched_files, total_processed

        except Exception as e:
            elapsed_time = time.time() - start_time
            logger.error(
                f"[OrderedDiscovery] Error during ordered file discovery after {elapsed_time:.1f}s: {e}",
                exc_info=True,
            )
            # Return partial results if available
            return {}, 0

    def _collect_and_sort_files(
        self,
        directories: list[str],
        recursive: bool,
        file_processing_order: str,
        metrics: dict[str, Any],
    ) -> list[dict[str, Any]]:
        """Collect files from all directories and sort globally.

        This matches the backend logic from source.py:282-370 exactly.

        Args:
            directories: List of directories to search
            recursive: Whether to search recursively
            file_processing_order: Order to sort files
            metrics: Metrics dictionary to populate

        Returns:
            List of file metadata dictionaries sorted by modified date
        """
        all_files_metadata = []
        max_depth = FileOperationConstants.MAX_RECURSIVE_DEPTH if recursive else 1
        max_files_for_sorting = FileOperationConstants.MAX_FILES_FOR_SORTING
        total_collected = 0

        # Collect files from all directories (matching backend logic)
        for directory in directories:
            logger.debug(f"[OrderedDiscovery] Collecting files from: {directory}")
            metrics["directories_processed"] += 1

            # Calculate remaining limit for this directory
            remaining_limit = max_files_for_sorting - total_collected
            if remaining_limit <= 0:
                logger.debug(
                    "[OrderedDiscovery] Reached collection limit, stopping directory processing"
                )
                break

            try:
                files_metadata = self.source_fs.list_files(
                    directory=directory,
                    max_depth=max_depth,
                    include_dirs=False,
                    limit=remaining_limit,
                )
                all_files_metadata.extend(files_metadata)
                total_collected += len(files_metadata)

                logger.debug(
                    f"[OrderedDiscovery] Collected {len(files_metadata)} files from {directory} "
                    f"(total: {total_collected}/{max_files_for_sorting})"
                )

                # Check if we've hit the limit
                if total_collected >= max_files_for_sorting:
                    logger.warning(
                        f"[OrderedDiscovery] File collection limit of '{max_files_for_sorting}' reached. "
                        "Ordering may not reflect all available files."
                    )
                    break

            except Exception as e:
                error_msg = f"Failed to collect files from {directory}"
                logger.error(f"[OrderedDiscovery] {error_msg}: {e}")
                continue

        # Update metrics with collection results
        metrics["total_files_collected"] = total_collected

        # Apply sorting (matching backend logic)
        if file_processing_order == "oldest_first":
            sorted_files = self.source_fs.sort_files_by_modified_date(
                all_files_metadata, ascending=True
            )
            order_desc = "FIFO (oldest first)"
        else:  # newest_first
            sorted_files = self.source_fs.sort_files_by_modified_date(
                all_files_metadata, ascending=False
            )
            order_desc = "LIFO (newest first)"

        logger.info(
            f"[OrderedDiscovery] Collected {len(all_files_metadata)} files, processing in {order_desc} order"
        )

        return sorted_files

    def _process_sorted_files_in_chunks(
        self,
        sorted_files: list[dict[str, Any]],
        patterns: list[str],
        file_hard_limit: int,
        batch_size: int,
        metrics: dict[str, Any],
    ) -> tuple[dict[str, FileHashData], int]:
        """Process sorted files in chunks using FilterPipeline.

        This avoids sending 40K files to backend APIs at once by processing
        in manageable chunks of ~100 files each.

        Args:
            sorted_files: Pre-sorted list of file metadata
            patterns: File patterns for filtering
            file_hard_limit: Maximum files to return
            batch_size: Size of each processing chunk
            metrics: Metrics dictionary to populate

        Returns:
            Tuple of (matched_files, total_count)
        """
        from .filter_pipeline import create_standard_pipeline

        matched_files = {}
        total_processed = 0
        files_pattern_matched = 0

        # Create FilterPipeline for chunked processing
        filter_pipeline = create_standard_pipeline(
            use_file_history=self.use_file_history,
            enable_active_filtering=True,  # Always enable for ordered processing
        )

        logger.info(
            f"[OrderedDiscovery] Processing {len(sorted_files)} sorted files in chunks of {batch_size}"
        )

        for i in range(0, len(sorted_files), batch_size):
            # Check if we've reached the limit
            if total_processed >= file_hard_limit:
                logger.info(
                    f"[OrderedDiscovery] Hard limit of '{file_hard_limit}' files reached"
                )
                break

            # Get current chunk
            chunk_files = sorted_files[i : i + batch_size]
            chunk_num = (i // batch_size) + 1
            metrics["batches_processed"] += 1

            logger.debug(
                f"[OrderedDiscovery] Processing chunk {chunk_num}: files {i + 1}-{min(i + batch_size, len(sorted_files))}"
            )

            # Convert metadata to FileHashData for this chunk (with pattern filtering)
            chunk_file_dict = {}
            chunk_pattern_matched = 0

            for file_metadata in chunk_files:
                file_path = file_metadata.get("name")
                if not file_path:
                    continue

                # Create FileHashData from metadata
                file_hash_data = self._create_file_hash_from_metadata(
                    file_path, file_metadata
                )

                # Apply pattern matching (same as backend logic)
                if not self._should_process_file(file_hash_data.file_name, patterns):
                    logger.debug(
                        f"[OrderedDiscovery] File failed pattern match: {file_path} (patterns: {patterns})"
                    )
                    continue

                chunk_file_dict[file_path] = file_hash_data
                chunk_pattern_matched += 1

            files_pattern_matched += chunk_pattern_matched

            if not chunk_file_dict:
                logger.debug(
                    f"[OrderedDiscovery] Chunk {chunk_num}: No files passed pattern matching"
                )
                continue

            # Apply FilterPipeline to this chunk (DeduplicationFilter, FileHistoryFilter, ActiveFileFilter)
            filtered_chunk = filter_pipeline.apply_filters(
                files=chunk_file_dict,
                workflow_id=self.workflow_id,
                execution_id=self.execution_id,
                api_client=self.api_client,
                organization_id=self.organization_id,
            )

            # Add filtered files to results (respecting hard limit)
            chunk_accepted = 0
            for file_path, file_hash_data in filtered_chunk.items():
                if total_processed >= file_hard_limit:
                    break
                matched_files[file_path] = file_hash_data
                total_processed += 1
                chunk_accepted += 1

            logger.debug(
                f"[OrderedDiscovery] Chunk {chunk_num}: {len(chunk_files)} → {chunk_pattern_matched} → {len(filtered_chunk)} → {chunk_accepted} files "
                f"(total: {total_processed})"
            )

        # Update metrics
        metrics["files_matching_patterns"] = files_pattern_matched

        logger.info(
            f"[OrderedDiscovery] Ordered processing complete: {len(sorted_files)} → {files_pattern_matched} → {total_processed} files matched"
        )

        # Create cache entries for files that will be processed to prevent race conditions
        try:
            from ..workflow.execution.active_file_manager import ActiveFileManager

            cache_stats = ActiveFileManager.create_cache_entries_simple(
                files_to_cache=matched_files,  # Create cache entries for all final files
                workflow_id=self.workflow_id,
                execution_id=self.execution_id,
                logger_instance=logger,
            )

            logger.info(
                f"[OrderedDiscovery] 🔒 Created {cache_stats.get('cache_created', 0)} cache entries "
                f"for race condition prevention"
            )

            # Return original matched_files since FilterPipeline already applied all filtering
            return matched_files, total_processed

        except Exception as cache_error:
            logger.warning(
                f"[OrderedDiscovery] Cache creation failed (proceeding anyway): {cache_error}"
            )
            return matched_files, total_processed

    def _create_file_hash_from_metadata(
        self, file_path: str, file_metadata: dict[str, Any]
    ) -> FileHashData:
        """Create FileHashData from file metadata.

        Args:
            file_path: Path to the file
            file_metadata: File metadata dictionary

        Returns:
            FileHashData: File hash data object
        """
        file_name = file_path.split("/")[-1]  # Get basename
        file_size = file_metadata.get("size", 0)
        provider_file_uuid = self.source_fs.get_file_system_uuid(
            file_path=file_path, metadata=file_metadata
        )
        serialized_metadata = self.source_fs.serialize_metadata_value(value=file_metadata)

        return FileHashData(
            file_path=file_path,
            source_connection_type=ConnectionType.FILESYSTEM,
            file_name=file_name,
            file_size=file_size,
            provider_file_uuid=provider_file_uuid,
            fs_metadata=serialized_metadata,
        )

    def _should_process_file(self, file_name: str, patterns: list[str]) -> bool:
        """Check if file matches the given patterns.

        Args:
            file_name: Name of the file to check
            patterns: List of patterns to match against

        Returns:
            bool: True if file should be processed, False otherwise
        """
        if not patterns:
            return True

        # Use the existing pattern matching logic from StreamingFileDiscovery
        import fnmatch
        import os

        file_name = os.path.basename(file_name)
        for pattern in patterns:
            # Case-insensitive matching
            if fnmatch.fnmatch(file_name.lower(), pattern.lower()):
                return True

        return False