Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

7.6 KB
Raw Blame
# mypy: ignore-errors
import json
import logging
from typing import Any

from connector_auth_v2.constants import ConnectorAuthKey
from connector_auth_v2.pipeline.common import ConnectorAuthHelper
from connector_v2.constants import ConnectorInstanceKey as CIKey

from backend.exceptions import UnstractFSException
from connector_processor.constants import ConnectorKeys
from connector_processor.exceptions import (
    InvalidConnectorID,
    InvalidConnectorMode,
    OAuthTimeOut,
    TestConnectorInputError,
)
from unstract.connectors.base import UnstractConnector
from unstract.connectors.connectorkit import Connectorkit
from unstract.connectors.enums import ConnectorMode
from unstract.connectors.exceptions import ConnectorError, FSAccessDeniedError
from unstract.connectors.filesystems.ucs import UnstractCloudStorage

logger = logging.getLogger(__name__)


def import_optional_connector(module_path: str, class_name: str):
    """Import connector class with graceful error handling."""
    try:
        module = __import__(module_path, fromlist=[class_name])
        connector_class = getattr(module, class_name)
        logger.debug(f"Successfully imported {class_name}")
        return connector_class
    except ImportError as e:
        logger.debug(f"Failed to import {class_name}: {e}")
        return None


# Import optional connectors
RedisQueue = import_optional_connector("unstract.connectors.queues.redis", "RedisQueue")
# TODO(UN-2261): Oracle temporarily excluded due to missing wallet support
OracleDB = import_optional_connector(
    "unstract.connectors.databases.oracle_db", "OracleDB"
)


def fetch_connectors_by_key_value(
    key: str, value: Any, connector_mode: ConnectorMode | None = None
) -> list[UnstractConnector]:
    """Fetches a list of connectors that have an attribute matching key and
    value.
    """
    logger.info(f"Fetching connector list for {key} with {value}")
    connector_kit = Connectorkit()
    connectors = connector_kit.get_connectors_list(mode=connector_mode)
    return [iterate for iterate in connectors if iterate[key] == value]


class ConnectorProcessor:
    @staticmethod
    def get_json_schema(connector_id: str) -> dict:
        """Function to return JSON Schema for Connectors."""
        schema_details: dict = {}
        if connector_id == UnstractCloudStorage.get_id():
            return schema_details
        updated_connectors = fetch_connectors_by_key_value(ConnectorKeys.ID, connector_id)
        if len(updated_connectors) == 0:
            raise InvalidConnectorID(
                f"Invalid connector ID '{connector_id}' while fetching JSON Schema"
            )

        connector = updated_connectors[0]
        schema_details[ConnectorKeys.OAUTH] = connector.get(ConnectorKeys.OAUTH)
        schema_details[ConnectorKeys.SOCIAL_AUTH_URL] = connector.get(
            ConnectorKeys.SOCIAL_AUTH_URL
        )
        try:
            schema_details[ConnectorKeys.JSON_SCHEMA] = json.loads(
                connector.get(ConnectorKeys.JSON_SCHEMA)
            )
        except Exception as exc:
            logger.error(f"Error occurred decoding JSON for {connector_id}: {exc}")
            raise exc

        return schema_details

    @staticmethod
    def get_all_supported_connectors(
        type: str | None = None, connector_mode: ConnectorMode | None = None
    ) -> list[dict]:
        """Function to return list of all supported connectors except PCS."""
        supported_connectors = []
        updated_connectors = []

        # TODO: Remove RedisQueue from the list of connectors and use separately instead
        # HACK: Connectors that are marked active but not supported explicitly
        unsupported_connectors = [
            connector.get_id() for connector in filter(None, [RedisQueue, OracleDB])
        ]

        if type == ConnectorKeys.INPUT:
            updated_connectors = fetch_connectors_by_key_value(
                ConnectorKeys.CAN_READ, True, connector_mode=connector_mode
            )
        elif type == ConnectorKeys.OUTPUT:
            updated_connectors = fetch_connectors_by_key_value(
                ConnectorKeys.CAN_WRITE, True, connector_mode=connector_mode
            )
        else:
            # When type is None, get all connectors directly
            connector_kit = Connectorkit()
            updated_connectors = connector_kit.get_connectors_list(mode=connector_mode)

        for connector in updated_connectors:
            if connector.get(ConnectorKeys.ID) in unsupported_connectors:
                continue
            mode = connector.get(CIKey.CONNECTOR_MODE)
            supported_connectors.append(
                {
                    ConnectorKeys.ID: connector.get(ConnectorKeys.ID),
                    ConnectorKeys.NAME: connector.get(ConnectorKeys.NAME),
                    ConnectorKeys.DESCRIPTION: connector.get(ConnectorKeys.DESCRIPTION),
                    ConnectorKeys.ICON: connector.get(ConnectorKeys.ICON),
                    ConnectorKeys.CAN_READ: connector.get(ConnectorKeys.CAN_READ),
                    ConnectorKeys.CAN_WRITE: connector.get(ConnectorKeys.CAN_WRITE),
                    CIKey.CONNECTOR_MODE: getattr(mode, "value", None),
                    ConnectorKeys.DOC_URL: connector.get(ConnectorKeys.DOC_URL, ""),
                }
            )

        return supported_connectors

    @staticmethod
    def test_connectors(connector_id: str, credentials: dict[str, Any]) -> bool:
        logger.info(f"Testing connector: {connector_id}")
        connector: dict[str, Any] = fetch_connectors_by_key_value(
            ConnectorKeys.ID, connector_id
        )[0]
        if connector.get(ConnectorKeys.OAUTH):
            oauth_key = credentials.get(ConnectorAuthKey.OAUTH_KEY)
            if oauth_key:  # Only fetch from cache if oauth_key is provided
                try:
                    credentials = ConnectorAuthHelper.get_oauth_creds_from_cache(
                        cache_key=oauth_key, delete_key=False
                    )
                except Exception as exc:
                    logger.error(
                        "Error while testing file based OAuth supported "
                        "connectors: %s",
                        exc,
                    )
                    raise OAuthTimeOut()

        try:
            connector_impl = Connectorkit().get_connector_by_id(connector_id, credentials)
            test_result = connector_impl.test_credentials()
            logger.info(f"{connector_id} test result: {test_result}")
            return test_result
        except FSAccessDeniedError as e:
            raise UnstractFSException(core_err=e) from e
        except ConnectorError as e:
            raise TestConnectorInputError(core_err=e) from e

    def get_connector_data_with_key(connector_id: str, key_value: str) -> Any:
        """Generic Function to get connector data with provided key."""
        updated_connectors = fetch_connectors_by_key_value("id", connector_id)
        if len(updated_connectors) == 0:
            raise InvalidConnectorID(
                f"Invalid connector ID '{connector_id}' while invoking utility"
            )
        return fetch_connectors_by_key_value("id", connector_id)[0].get(key_value)

    @staticmethod
    def validate_connector_mode(connector_mode: str) -> ConnectorMode:
        """Validate the connector mode.

        Parameters:
        - connector_mode (str): The connector mode to validate.

        Returns:
        - ConnectorMode: The validated connector mode.

        Raises:
        - InValidConnectorMode: If the connector mode is not valid.
        """
        try:
            connector_mode = ConnectorMode(connector_mode)
        except ValueError:
            raise InvalidConnectorMode
        return connector_mode