Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

8.3 KB
Raw Blame
"""Source Connector for Workflow Input Handling

This module provides specialized source connector for handling workflow inputs,
extracted from the backend workflow_manager/endpoint_v2/source.py to work without Django.

Handles:
- Filesystem source input
- API source input
- Source validation
- File listing and processing
"""

from dataclasses import dataclass
from typing import Any

from shared.infrastructure.logging.logger import WorkerLogger
from shared.workflow.logger_helper import WorkflowLoggerHelper

from unstract.core.data_models import ConnectionType as CoreConnectionType

logger = WorkerLogger.get_logger(__name__)


@dataclass
class SourceConfig:
    """Worker-compatible SourceConfig implementation."""

    connection_type: str
    settings: dict[str, Any] = None
    # Connector instance fields from backend API
    connector_id: str | None = None
    connector_settings: dict[str, Any] = None
    connector_name: str | None = None

    def __post_init__(self):
        if self.settings is None:
            self.settings = {}
        if self.connector_settings is None:
            self.connector_settings = {}

    def get_core_connection_type(self) -> CoreConnectionType:
        """Convert string connection_type to CoreConnectionType enum."""
        try:
            # Use the enum directly for consistent mapping
            connection_type_upper = self.connection_type.upper()

            # Try to get enum member by value
            for connection_type_enum in CoreConnectionType:
                if connection_type_enum.value == connection_type_upper:
                    return connection_type_enum

            # Fallback: handle legacy/unknown types
            logger.warning(
                f"Unknown connection type '{self.connection_type}', defaulting to FILESYSTEM"
            )
            return CoreConnectionType.FILESYSTEM

        except Exception as e:
            logger.error(
                f"Failed to convert connection type '{self.connection_type}' to enum: {e}"
            )
            return CoreConnectionType.FILESYSTEM

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> "SourceConfig":
        """Create SourceConfig from dictionary data."""
        return cls(
            connection_type=data.get("connection_type", ""),
            settings=data.get("settings", {}),
            connector_id=data.get("connector_id"),
            connector_settings=data.get("connector_settings", {}),
            connector_name=data.get("connector_name"),
        )


class WorkerSourceConnector:
    """Worker-compatible source connector following production patterns.

    This class replicates the functionality of backend SourceConnector
    from workflow_manager/endpoint_v2/source.py without Django dependencies.
    """

    # Local connection types for workflow source handling
    class ConnectionType:
        # Standard connection types from CoreConnectionType
        FILESYSTEM = CoreConnectionType.FILESYSTEM.value
        API = CoreConnectionType.API.value
        DATABASE = CoreConnectionType.DATABASE.value

        # Workflow-specific connection type for API execution storage
        API_STORAGE = "API_STORAGE"

    def __init__(self, config: SourceConfig, workflow_log=None):
        self.config = config
        self.connection_type = config.connection_type
        self.settings = config.settings
        self.workflow_log = workflow_log
        self.logger_helper = WorkflowLoggerHelper(workflow_log)

        # Store connector instance details
        self.connector_id = config.connector_id
        self.connector_settings = config.connector_settings
        self.connector_name = config.connector_name

    @classmethod
    def from_config(cls, workflow_log, config: SourceConfig):
        """Create source connector from config (matching Django backend interface)."""
        return cls(config, workflow_log)

    def get_fsspec_fs(self):
        """Get fsspec filesystem for the source connector.

        This method replicates backend logic for getting filesystem access.
        """
        try:
            if self.connection_type == self.ConnectionType.API_STORAGE:
                # API storage uses workflow execution storage
                from unstract.filesystem import FileStorageType, FileSystem

                file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
                return file_system.get_file_storage()

            if not self.connector_id or not self.connector_settings:
                error_msg = (
                    "Source connector not configured - missing connector_id or settings"
                )
                self.logger_helper.log_error(logger, error_msg)
                raise Exception(error_msg)

            # Get the connector instance using connectorkit
            from unstract.connectors.connectorkit import Connectorkit

            connectorkit = Connectorkit()
            connector_class = connectorkit.get_connector_class_by_connector_id(
                self.connector_id
            )
            connector_instance = connector_class(self.connector_settings)

            # Get fsspec filesystem
            fs = connector_instance.get_fsspec_fs()
            return fs

        except Exception as e:
            error_msg = f"Failed to initialize source connector filesystem: {str(e)}"
            self.logger_helper.log_error(logger, error_msg)
            logger.error(error_msg)
            raise

    def read_file_content(self, file_path: str) -> bytes:
        """Read file content from source connector.

        Args:
            file_path: Path to the file in source storage

        Returns:
            File content as bytes
        """
        fs = self.get_fsspec_fs()

        if self.connection_type == self.ConnectionType.API_STORAGE:
            # For API storage, use the filesystem's read method
            return fs.read(file_path, mode="rb")
        else:
            # For other connectors, use fsspec open
            with fs.open(file_path, "rb") as f:
                return f.read()

    def list_files(
        self, input_directory: str, file_pattern: str = None
    ) -> list[dict[str, Any]]:
        """List files from source connector.

        Args:
            input_directory: Directory to list files from
            file_pattern: Optional glob pattern to filter files

        Returns:
            List of file information dictionaries
        """
        fs = self.get_fsspec_fs()
        # Implementation would list files using fsspec
        # This is a simplified version
        try:
            files = []
            if self.connection_type == self.ConnectionType.API_STORAGE:
                # Use filesystem listing
                file_paths = fs.list(input_directory)
            else:
                # Use fsspec listing
                file_paths = fs.ls(input_directory, detail=False)

            for file_path in file_paths:
                files.append(
                    {
                        "name": file_path.split("/")[-1],
                        "path": file_path,
                    }
                )

            return files
        except Exception as e:
            error_msg = f"Failed to list files from source connector directory '{input_directory}': {str(e)}"
            self.logger_helper.log_error(logger, error_msg)
            logger.error(error_msg)
            return []

    def validate(self) -> None:
        """Validate source connector configuration."""
        connection_type = self.connection_type

        if connection_type not in [
            self.ConnectionType.FILESYSTEM,
            self.ConnectionType.API,
            self.ConnectionType.API_STORAGE,
        ]:
            error_msg = f"Invalid source connection type: {connection_type}"
            self.logger_helper.log_error(logger, error_msg)
            raise Exception(error_msg)

        if connection_type == self.ConnectionType.FILESYSTEM:
            if not self.connector_id or not self.connector_settings:
                error_msg = "Filesystem source requires connector configuration"
                self.logger_helper.log_error(logger, error_msg)
                raise Exception(error_msg)

    def get_config(self) -> SourceConfig:
        """Get serializable configuration for the source connector."""
        return self.config


# Alias for backward compatibility
SourceConnector = WorkerSourceConnector