LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
# mypy: ignore-errors
import json
import logging
from typing import Any
from connector_auth_v2.constants import ConnectorAuthKey
from connector_auth_v2.pipeline.common import ConnectorAuthHelper
from connector_v2.constants import ConnectorInstanceKey as CIKey
from backend.exceptions import UnstractFSException
from connector_processor.constants import ConnectorKeys
from connector_processor.exceptions import (
InvalidConnectorID,
InvalidConnectorMode,
OAuthTimeOut,
TestConnectorInputError,
)
from unstract.connectors.base import UnstractConnector
from unstract.connectors.connectorkit import Connectorkit
from unstract.connectors.enums import ConnectorMode
from unstract.connectors.exceptions import ConnectorError, FSAccessDeniedError
from unstract.connectors.filesystems.ucs import UnstractCloudStorage
logger = logging.getLogger(__name__)
def import_optional_connector(module_path: str, class_name: str):
"""Import connector class with graceful error handling."""
try:
module = __import__(module_path, fromlist=[class_name])
connector_class = getattr(module, class_name)
logger.debug(f"Successfully imported {class_name}")
return connector_class
except ImportError as e:
logger.debug(f"Failed to import {class_name}: {e}")
return None
# Import optional connectors
RedisQueue = import_optional_connector("unstract.connectors.queues.redis", "RedisQueue")
# TODO(UN-2261): Oracle temporarily excluded due to missing wallet support
OracleDB = import_optional_connector(
"unstract.connectors.databases.oracle_db", "OracleDB"
)
def fetch_connectors_by_key_value(
key: str, value: Any, connector_mode: ConnectorMode | None = None
) -> list[UnstractConnector]:
"""Fetches a list of connectors that have an attribute matching key and
value.
"""
logger.info(f"Fetching connector list for {key} with {value}")
connector_kit = Connectorkit()
connectors = connector_kit.get_connectors_list(mode=connector_mode)
return [iterate for iterate in connectors if iterate[key] == value]
class ConnectorProcessor:
@staticmethod
def get_json_schema(connector_id: str) -> dict:
"""Function to return JSON Schema for Connectors."""
schema_details: dict = {}
if connector_id == UnstractCloudStorage.get_id():
return schema_details
updated_connectors = fetch_connectors_by_key_value(ConnectorKeys.ID, connector_id)
if len(updated_connectors) == 0:
raise InvalidConnectorID(
f"Invalid connector ID '{connector_id}' while fetching JSON Schema"
)
connector = updated_connectors[0]
schema_details[ConnectorKeys.OAUTH] = connector.get(ConnectorKeys.OAUTH)
schema_details[ConnectorKeys.SOCIAL_AUTH_URL] = connector.get(
ConnectorKeys.SOCIAL_AUTH_URL
)
try:
schema_details[ConnectorKeys.JSON_SCHEMA] = json.loads(
connector.get(ConnectorKeys.JSON_SCHEMA)
)
except Exception as exc:
logger.error(f"Error occurred decoding JSON for {connector_id}: {exc}")
raise exc
return schema_details
@staticmethod
def get_all_supported_connectors(
type: str | None = None, connector_mode: ConnectorMode | None = None
) -> list[dict]:
"""Function to return list of all supported connectors except PCS."""
supported_connectors = []
updated_connectors = []
# TODO: Remove RedisQueue from the list of connectors and use separately instead
# HACK: Connectors that are marked active but not supported explicitly
unsupported_connectors = [
connector.get_id() for connector in filter(None, [RedisQueue, OracleDB])
]
if type == ConnectorKeys.INPUT:
updated_connectors = fetch_connectors_by_key_value(
ConnectorKeys.CAN_READ, True, connector_mode=connector_mode
)
elif type == ConnectorKeys.OUTPUT:
updated_connectors = fetch_connectors_by_key_value(
ConnectorKeys.CAN_WRITE, True, connector_mode=connector_mode
)
else:
# When type is None, get all connectors directly
connector_kit = Connectorkit()
updated_connectors = connector_kit.get_connectors_list(mode=connector_mode)
for connector in updated_connectors:
if connector.get(ConnectorKeys.ID) in unsupported_connectors:
continue
mode = connector.get(CIKey.CONNECTOR_MODE)
supported_connectors.append(
{
ConnectorKeys.ID: connector.get(ConnectorKeys.ID),
ConnectorKeys.NAME: connector.get(ConnectorKeys.NAME),
ConnectorKeys.DESCRIPTION: connector.get(ConnectorKeys.DESCRIPTION),
ConnectorKeys.ICON: connector.get(ConnectorKeys.ICON),
ConnectorKeys.CAN_READ: connector.get(ConnectorKeys.CAN_READ),
ConnectorKeys.CAN_WRITE: connector.get(ConnectorKeys.CAN_WRITE),
CIKey.CONNECTOR_MODE: getattr(mode, "value", None),
ConnectorKeys.DOC_URL: connector.get(ConnectorKeys.DOC_URL, ""),
}
)
return supported_connectors
@staticmethod
def test_connectors(connector_id: str, credentials: dict[str, Any]) -> bool:
logger.info(f"Testing connector: {connector_id}")
connector: dict[str, Any] = fetch_connectors_by_key_value(
ConnectorKeys.ID, connector_id
)[0]
if connector.get(ConnectorKeys.OAUTH):
oauth_key = credentials.get(ConnectorAuthKey.OAUTH_KEY)
if oauth_key: # Only fetch from cache if oauth_key is provided
try:
credentials = ConnectorAuthHelper.get_oauth_creds_from_cache(
cache_key=oauth_key, delete_key=False
)
except Exception as exc:
logger.error(
"Error while testing file based OAuth supported "
"connectors: %s",
exc,
)
raise OAuthTimeOut()
try:
connector_impl = Connectorkit().get_connector_by_id(connector_id, credentials)
test_result = connector_impl.test_credentials()
logger.info(f"{connector_id} test result: {test_result}")
return test_result
except FSAccessDeniedError as e:
raise UnstractFSException(core_err=e) from e
except ConnectorError as e:
raise TestConnectorInputError(core_err=e) from e
def get_connector_data_with_key(connector_id: str, key_value: str) -> Any:
"""Generic Function to get connector data with provided key."""
updated_connectors = fetch_connectors_by_key_value("id", connector_id)
if len(updated_connectors) == 0:
raise InvalidConnectorID(
f"Invalid connector ID '{connector_id}' while invoking utility"
)
return fetch_connectors_by_key_value("id", connector_id)[0].get(key_value)
@staticmethod
def validate_connector_mode(connector_mode: str) -> ConnectorMode:
"""Validate the connector mode.
Parameters:
- connector_mode (str): The connector mode to validate.
Returns:
- ConnectorMode: The validated connector mode.
Raises:
- InValidConnectorMode: If the connector mode is not valid.
"""
try:
connector_mode = ConnectorMode(connector_mode)
except ValueError:
raise InvalidConnectorMode
return connector_mode