Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

15.7 KB
Raw Blame
"""File Processing Utilities for Worker Tasks

This module provides standardized file processing operations, batching,
validation, and conversion utilities used across worker implementations.
"""

import time
from typing import Any

from unstract.core.data_models import FileHashData

from ...infrastructure.logging import WorkerLogger

logger = WorkerLogger.get_logger(__name__)


class FileProcessingUtils:
    """Centralized file processing operations and utilities."""

    @staticmethod
    def convert_file_hash_data(
        hash_values_of_files: dict[str, Any] | None,
    ) -> dict[str, FileHashData]:
        """Standardized file hash conversion with comprehensive error handling.

        Args:
            hash_values_of_files: Raw file hash data from API

        Returns:
            Dictionary of converted FileHashData objects

        Note:
            This consolidates the repeated file conversion logic found across
            api-deployment, general, and file_processing workers.
        """
        if not hash_values_of_files:
            logger.warning("No file hash data provided for conversion")
            return {}

        converted_files = {}
        conversion_errors = []

        for file_key, file_data in hash_values_of_files.items():
            try:
                if isinstance(file_data, dict):
                    # Convert dictionary to FileHashData
                    converted_files[file_key] = FileHashData.from_dict(file_data)

                elif isinstance(file_data, FileHashData):
                    # Already converted
                    converted_files[file_key] = file_data

                else:
                    # Attempt manual conversion for other types
                    logger.warning(
                        f"Unexpected file data type for {file_key}: {type(file_data)}. "
                        "Attempting manual conversion."
                    )
                    converted_files[file_key] = FileHashData(
                        file_name=str(file_data),
                        file_path="",  # Will be populated later
                        file_hash=file_key,
                    )

            except Exception as e:
                error_msg = f"Failed to convert file data for {file_key}: {e}"
                logger.error(error_msg)
                conversion_errors.append(error_msg)
                continue

        if conversion_errors:
            logger.warning(
                f"File conversion completed with {len(conversion_errors)} errors. "
                f"Successfully converted {len(converted_files)} files."
            )

        return converted_files

    @staticmethod
    def create_file_batches(
        files: dict[str, Any],
        organization_id: str | None = None,
        api_client=None,
        batch_size_env_var: str = "MAX_PARALLEL_FILE_BATCHES",
        default_batch_size: int = 1,
    ) -> list[list[tuple[str, Any]]]:
        """Standardized file batching algorithm used across workers with organization-specific config.

        Args:
            files: Dictionary of files to batch
            organization_id: Organization ID for configuration lookup
            api_client: Internal API client for configuration access
            batch_size_env_var: Environment variable for batch size config (fallback)
            default_batch_size: Default batch size if all else fails

        Returns:
            List of file batches, each batch is a list of (key, value) tuples

        Note:
            This consolidates the math.ceil batching logic found in
            api-deployment and general workers, now with organization-specific configuration support.
        """
        if not files:
            logger.warning("No files provided for batching")
            return []

        # Get batch size using internal API client (consistent with other worker operations)
        batch_size = FileProcessingUtils._get_batch_size_via_api(
            organization_id=organization_id,
            api_client=api_client,
            env_var_name=batch_size_env_var,
            default_value=default_batch_size,
        )

        # Convert to list of items
        file_items = list(files.items())
        # calculate number of files
        num_files = len(file_items)
        # Target number of batches (can't exceed number of files)
        num_batches = min(batch_size, num_files)

        logger.info(
            f"Arranging {num_files} files into {num_batches} batches "
            f"(max_batch_size={batch_size})"
        )

        # Arrange files in batches
        batches = FileProcessingUtils._arrange_files_in_batches(
            file_items=file_items, num_files=num_files, num_batches=num_batches
        )

        return batches

    @staticmethod
    def _arrange_files_in_batches(
        file_items: list[tuple[str, Any]],
        num_files: int,
        num_batches: int,
    ) -> list[list[tuple[str, Any]]]:
        """Arrange files in batches using round-robin distribution for even workload.

        Distributes files evenly across batches in round-robin fashion to ensure
        balanced workload, especially when files vary in size or complexity.

        Note:
            There is an optimization opportunity here to use weighted round-robin distribution
            to balance large files with small files across batches for more even workload
            distribution based on actual processing requirements rather than just file count.

        Args:
            file_items: List of file items to batch
            num_files: Total number of files
            num_batches: Number of batches to create

        Returns:
            List of file batches with files distributed evenly
        """
        # Initialize empty batches
        batches = [[] for _ in range(num_batches)]

        # Distribute files in round-robin fashion
        for index, file_item in enumerate(file_items):
            batch_index = index % num_batches
            batches[batch_index].append(file_item)

        # Remove any empty batches (shouldn't happen, but safety check)
        batches = [batch for batch in batches if batch]

        logger.info(
            f"Created {len(batches)} batches from {num_files} files "
            f"(round-robin distribution)"
        )

        return batches

    @staticmethod
    def validate_file_data(
        file_data: Any, operation_name: str, required_fields: list[str] | None = None
    ) -> dict[str, Any]:
        """Common file validation logic with standardized error handling.

        Args:
            file_data: File data to validate
            operation_name: Name of operation for logging context
            required_fields: List of required field names

        Returns:
            Validated file data dictionary

        Raises:
            ValueError: If validation fails

        Note:
            This consolidates validation patterns found across multiple workers.
        """
        if not file_data:
            raise ValueError(f"{operation_name}: No file data provided")

        # Convert to dict if it's a FileHashData object
        if isinstance(file_data, FileHashData):
            file_dict = file_data.__dict__
        elif isinstance(file_data, dict):
            file_dict = file_data.copy()
        else:
            raise ValueError(
                f"{operation_name}: Invalid file data type: {type(file_data)}"
            )

        # Validate required fields
        if required_fields:
            missing_fields = [
                field for field in required_fields if not file_dict.get(field)
            ]
            if missing_fields:
                raise ValueError(
                    f"{operation_name}: Missing required fields: {missing_fields}"
                )

        # Standardize file name handling
        file_name = file_dict.get("file_name")
        if not file_name or file_name == "unknown":
            logger.warning(
                f"{operation_name}: File missing or unknown name, "
                f"generating timestamp-based name"
            )
            file_dict["file_name"] = f"unknown_file_{int(time.time())}"

        # Validate execution ID
        execution_id = file_dict.get("file_execution_id")
        if not execution_id:
            logger.warning(f"{operation_name}: File missing execution ID: {file_dict}")

        return file_dict

    @staticmethod
    def extract_file_metadata(
        files: dict[str, FileHashData], include_sensitive: bool = False
    ) -> dict[str, dict[str, Any]]:
        """Extract standardized metadata from file collection.

        Args:
            files: Dictionary of FileHashData objects
            include_sensitive: Whether to include potentially sensitive data

        Returns:
            Dictionary of metadata per file

        Note:
            This provides consistent metadata extraction used in logging
            and debugging across workers.
        """
        metadata = {}

        for file_key, file_data in files.items():
            file_metadata = {
                "file_name": getattr(file_data, "file_name", "unknown"),
                "file_size": getattr(file_data, "file_size", 0),
                "file_type": getattr(file_data, "file_type", "unknown"),
                "created_at": getattr(file_data, "created_at", None),
            }

            if include_sensitive:
                file_metadata.update(
                    {
                        "file_path": getattr(file_data, "file_path", ""),
                        "file_hash": getattr(file_data, "file_hash", ""),
                    }
                )

            metadata[file_key] = file_metadata

        return metadata

    @staticmethod
    def _get_batch_size_via_api(
        organization_id: str | None = None,
        api_client=None,
        env_var_name: str = "MAX_PARALLEL_FILE_BATCHES",
        default_value: int = 1,
    ) -> int:
        """Get batch size using internal API client (unified approach).

        This replaces the complex infrastructure config client with direct API calls,
        maintaining the same fallback behavior as the backend Configuration.get_value_by_organization().

        Args:
            organization_id: Organization ID for configuration lookup
            api_client: Internal API client for configuration access
            env_var_name: Environment variable for batch size config (fallback)
            default_value: Default batch size if all else fails

        Returns:
            Batch size (guaranteed to be >= 1)
        """
        # Try organization-specific configuration via internal API
        if api_client and organization_id:
            try:
                response = api_client.get_configuration(
                    config_key=env_var_name,
                    organization_id=organization_id,
                )

                if (
                    response.get("success")
                    and response.get("data", {}).get("value") is not None
                ):
                    config_value = int(response["data"]["value"])
                    if config_value >= 1:
                        logger.info(
                            f"Using organization configuration for {organization_id} {env_var_name}: {config_value}"
                        )
                        return config_value

            except Exception as e:
                logger.warning(f"Failed to get organization config, falling back: {e}")

        # Fall back to environment variable
        import os

        try:
            env_value = int(os.getenv(env_var_name, str(default_value)))
            if env_value >= 1:
                logger.info(f"Using environment variable {env_var_name}: {env_value}")
                return env_value
        except (ValueError, TypeError):
            logger.warning(f"Invalid {env_var_name} environment variable")

        # Final fallback to backend default (matching backend logic)
        try:
            final_value = int(os.getenv(env_var_name, str(default_value)))
            final_value = max(1, final_value)
            logger.info(f"Using final environment fallback: {final_value}")
            return final_value
        except (ValueError, TypeError):
            logger.info(f"Using absolute fallback: {default_value}")
            return 1

    @staticmethod
    def create_file_processing_summary(
        total_files: int,
        successful_files: int,
        failed_files: int,
        skipped_files: int = 0,
        duration_seconds: float | None = None,
    ) -> str:
        """Create standardized file processing summary string.

        Args:
            total_files: Total number of files processed
            successful_files: Number of successfully processed files
            failed_files: Number of failed files
            skipped_files: Number of skipped files
            duration_seconds: Optional processing duration

        Returns:
            Formatted summary string

        Note:
            This provides consistent result reporting across all workers.
        """
        summary_parts = [
            f"total={total_files}",
            f"success={successful_files}",
            f"failed={failed_files}",
        ]

        if skipped_files > 0:
            summary_parts.append(f"skipped={skipped_files}")

        if duration_seconds is not None:
            summary_parts.append(f"duration={duration_seconds:.2f}s")

        success_rate = (successful_files / total_files * 100) if total_files > 0 else 0
        summary_parts.append(f"success_rate={success_rate:.1f}%")

        return " - ".join(summary_parts)

    @staticmethod
    def handle_file_format_variations(
        files_data: dict | list | tuple | Any,
    ) -> dict[str, Any]:
        """Handle various file data format variations found across workers.

        Args:
            files_data: File data in various possible formats

        Returns:
            Normalized dictionary format

        Note:
            This consolidates the complex file format handling found in
            file_processing worker's _process_individual_files method.
        """
        if isinstance(files_data, dict):
            return files_data

        elif isinstance(files_data, (list, tuple)):
            # Convert list/tuple to dict with index as key
            normalized = {}
            for i, item in enumerate(files_data):
                if isinstance(item, dict):
                    # Use file_name as key if available, otherwise use index
                    key = item.get("file_name", f"file_{i}")
                    normalized[key] = item
                else:
                    normalized[f"file_{i}"] = {"file_data": item}
            return normalized

        else:
            # Single item - wrap in dict
            logger.warning(
                f"Unexpected file data type: {type(files_data)}. "
                "Wrapping as single file."
            )
            return {"single_file": {"file_data": files_data}}


class FileProcessingMixin:
    """Mixin class to add file processing utilities to worker tasks."""

    def convert_files(
        self, hash_values_of_files: dict[str, Any] | None
    ) -> dict[str, FileHashData]:
        """Convert file hash data using standardized logic."""
        return FileProcessingUtils.convert_file_hash_data(hash_values_of_files)

    def create_batches(
        self, files: dict[str, Any], **kwargs
    ) -> list[list[tuple[str, Any]]]:
        """Create file batches using standardized algorithm."""
        return FileProcessingUtils.create_file_batches(files, **kwargs)

    def validate_file(
        self, file_data: Any, operation_name: str, **kwargs
    ) -> dict[str, Any]:
        """Validate file data using standardized logic."""
        return FileProcessingUtils.validate_file_data(file_data, operation_name, **kwargs)

    def create_summary(self, **kwargs) -> str:
        """Create processing summary using standardized format."""
        return FileProcessingUtils.create_file_processing_summary(**kwargs)