Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

12.2 KB
Raw Blame
"""Client Plugin Registry for Workers

This registry system allows dynamic loading of API client plugins based on
Django settings configuration. This eliminates the need for conditional imports
and try/except blocks while maintaining clean separation between OSS and cloud features.

The registry follows the same pattern as the main plugin registry but is
specifically designed for API client extensions.
"""

import logging
from typing import Any

logger = logging.getLogger(__name__)


class APIClientPlugin:
    """Base class for API client plugins."""

    name: str = ""
    description: str = ""
    version: str = "1.0.0"

    def __init__(self, config: Any):
        """Initialize the plugin with configuration."""
        self.config = config

    def close(self):
        """Clean up plugin resources."""
        pass


class ClientPluginRegistry:
    """Registry for API client plugins loaded from Django settings."""

    def __init__(self):
        self._plugins: dict[str, type[APIClientPlugin]] = {}
        self._instances: dict[str, APIClientPlugin] = {}
        self._initialized = False

    def initialize_from_settings(self):
        """Initialize plugins from environment configuration (Django not required in workers)."""
        if self._initialized:
            return

        try:
            self._initialize_worker_plugins()
        except Exception as e:
            logger.error(
                f"Failed to initialize client plugins from settings: {e}", exc_info=True
            )

        self._initialized = True

    def _initialize_worker_plugins(self):
        """Initialize plugins for workers environment (no Django dependencies)."""
        # Auto-discover plugins from plugin directories
        worker_plugins = self._discover_worker_plugins()

        for plugin_name, plugin_config in worker_plugins.items():
            if not plugin_config.get("enabled", False):
                continue

            try:
                self._load_plugin_from_config_worker(plugin_name, plugin_config)
                logger.debug(f"Loaded worker client plugin: {plugin_name}")
            except Exception as e:
                logger.debug(f"Failed to load worker client plugin {plugin_name}: {e}")

    def _discover_worker_plugins(self) -> dict[str, dict[str, Any]]:
        """Auto-discover plugins from plugin directories."""
        import importlib.util
        import os

        discovered_plugins = {}

        # Get plugins directory
        workers_dir = os.path.dirname(__file__)
        plugins_dir = os.path.join(workers_dir, "plugins")

        if not os.path.exists(plugins_dir):
            logger.debug(f"Plugins directory not found: {plugins_dir}")
            return discovered_plugins

        logger.debug(f"Scanning for plugins in: {plugins_dir}")

        # Scan plugin directories
        for item in os.listdir(plugins_dir):
            plugin_path = os.path.join(plugins_dir, item)

            # Skip files, only process directories
            if not os.path.isdir(plugin_path):
                continue

            # Skip __pycache__ and other system directories
            if item.startswith("__") or item.startswith("."):
                continue

            init_file = os.path.join(plugin_path, "__init__.py")
            if not os.path.exists(init_file):
                logger.debug(f"Skipping {item}: no __init__.py found")
                continue

            try:
                # Load the plugin's __init__.py to check for CLIENT_PLUGIN_CONFIG
                spec = importlib.util.spec_from_file_location(
                    f"plugins.{item}", init_file
                )
                module = importlib.util.module_from_spec(spec)
                spec.loader.exec_module(module)

                # Check if the plugin has CLIENT_PLUGIN_CONFIG
                if hasattr(module, "CLIENT_PLUGIN_CONFIG"):
                    config = module.CLIENT_PLUGIN_CONFIG
                    # Use directory name as plugin name (standard convention)
                    discovered_plugins[item] = config
                    logger.debug(f"Discovered plugin: {item} with config: {config}")
                else:
                    logger.debug(f"Plugin {item} has no CLIENT_PLUGIN_CONFIG, skipping")

            except Exception as e:
                logger.debug(f"Failed to load plugin {item}: {e}")

        logger.debug(
            f"Discovered {len(discovered_plugins)} plugins: {list(discovered_plugins.keys())}"
        )
        return discovered_plugins

    def _load_plugin_from_config_worker(self, plugin_name: str, config: dict[str, Any]):
        """Load a plugin from configuration for workers (handles relative imports better)."""
        plugin_path = config.get("plugin_path")
        if not plugin_path:
            raise ValueError(f"Plugin {plugin_name} missing plugin_path")

        try:
            # For workers, use importlib with proper path handling
            import glob
            import importlib
            import os

            # Convert plugin path to file path
            module_path, class_name = plugin_path.rsplit(".", 1)
            relative_path_base = module_path.replace(".", os.sep)

            # Get absolute path from workers directory
            workers_dir = os.path.dirname(__file__)

            # Search for both .py and .so files (supports Nuitka-compiled modules)
            # Try .py first, then .so files
            plugin_file_path = None

            # Try .py file first
            py_path = os.path.join(workers_dir, relative_path_base + ".py")
            if os.path.exists(py_path):
                plugin_file_path = py_path
            else:
                # Try .so files (Nuitka compiled modules: .cpython-312-*.so)
                so_pattern = os.path.join(workers_dir, relative_path_base + ".*.so")
                so_files = glob.glob(so_pattern)
                if so_files:
                    # Use the first .so file found
                    plugin_file_path = so_files[0]
                    logger.debug(f"Found compiled module: {plugin_file_path}")

            if plugin_file_path:
                # Use importlib.util for file-based import
                import importlib.util

                spec = importlib.util.spec_from_file_location(
                    module_path, plugin_file_path
                )
                module = importlib.util.module_from_spec(spec)

                # Execute the module to load the class
                spec.loader.exec_module(module)
                plugin_class = getattr(module, class_name)

                # Validate plugin class
                if not issubclass(plugin_class, APIClientPlugin):
                    raise TypeError(
                        f"Plugin {plugin_name} must inherit from APIClientPlugin"
                    )

                # Register the plugin class
                self._plugins[plugin_name] = plugin_class
                logger.debug(
                    f"Successfully loaded plugin class {class_name} from {plugin_file_path}"
                )
            else:
                logger.debug(
                    f"Plugin file not found for {module_path} "
                    f"(searched for .py and .so files)"
                )

        except Exception as e:
            logger.debug(f"Failed to load plugin {plugin_name} using worker loader: {e}")
            # Fall back to standard import if file-based loading fails
            try:
                module_path, class_name = plugin_path.rsplit(".", 1)
                module = __import__(module_path, fromlist=[class_name])
                plugin_class = getattr(module, class_name)

                if not issubclass(plugin_class, APIClientPlugin):
                    raise TypeError(
                        f"Plugin {plugin_name} must inherit from APIClientPlugin"
                    )

                self._plugins[plugin_name] = plugin_class
                logger.debug(f"Loaded plugin {plugin_name} using fallback import")
            except Exception as fallback_error:
                raise Exception(
                    f"Both file-based and import-based loading failed: {e}, {fallback_error}"
                )

    def _load_plugin_from_config(self, plugin_name: str, config: dict[str, Any]):
        """Load a plugin from configuration."""
        plugin_path = config.get("plugin_path")
        if not plugin_path:
            raise ValueError(f"Plugin {plugin_name} missing plugin_path")

        # Import the plugin module
        module_path, class_name = plugin_path.rsplit(".", 1)
        module = __import__(module_path, fromlist=[class_name])
        plugin_class = getattr(module, class_name)

        # Validate plugin class
        if not issubclass(plugin_class, APIClientPlugin):
            raise TypeError(f"Plugin {plugin_name} must inherit from APIClientPlugin")

        # Register the plugin class
        self._plugins[plugin_name] = plugin_class

    def register_plugin_class(self, name: str, plugin_class: type[APIClientPlugin]):
        """Manually register a plugin class."""
        if not issubclass(plugin_class, APIClientPlugin):
            raise TypeError(f"Plugin {name} must inherit from APIClientPlugin")

        self._plugins[name] = plugin_class
        logger.debug(f"Manually registered client plugin: {name}")

    def get_plugin_instance(
        self, name: str, config: Any = None
    ) -> APIClientPlugin | None:
        """Get or create a plugin instance."""
        self.initialize_from_settings()

        # Return cached instance if available
        if name in self._instances:
            return self._instances[name]

        # Create new instance if plugin class is registered
        if name in self._plugins:
            try:
                plugin_class = self._plugins[name]
                instance = plugin_class(config)
                self._instances[name] = instance
                logger.debug(f"Created instance for client plugin: {name}")
                return instance
            except Exception as e:
                logger.error(f"Failed to create instance for client plugin {name}: {e}")
                return None

        return None

    def has_plugin(self, name: str) -> bool:
        """Check if a plugin is available."""
        self.initialize_from_settings()
        return name in self._plugins

    def list_available_plugins(self) -> list[dict[str, Any]]:
        """List all available client plugins."""
        self.initialize_from_settings()

        plugins = []
        for name, plugin_class in self._plugins.items():
            plugins.append(
                {
                    "name": name,
                    "description": getattr(plugin_class, "description", ""),
                    "version": getattr(plugin_class, "version", "1.0.0"),
                    "enabled": True,
                }
            )

        return plugins

    def close_all_instances(self):
        """Close all plugin instances."""
        for instance in self._instances.values():
            try:
                instance.close()
            except Exception as e:
                logger.warning(f"Error closing client plugin instance: {e}")

        self._instances.clear()

    def clear(self):
        """Clear all plugins and instances."""
        self.close_all_instances()
        self._plugins.clear()
        self._initialized = False


# Global registry instance
_client_plugin_registry = ClientPluginRegistry()


def get_client_plugin(name: str, config: Any = None) -> APIClientPlugin | None:
    """Get a client plugin instance by name."""
    return _client_plugin_registry.get_plugin_instance(name, config)


def has_client_plugin(name: str) -> bool:
    """Check if a client plugin is available."""
    return _client_plugin_registry.has_plugin(name)


def list_client_plugins() -> list[dict[str, Any]]:
    """List all available client plugins."""
    return _client_plugin_registry.list_available_plugins()


def register_client_plugin(name: str, plugin_class: type[APIClientPlugin]):
    """Register a client plugin class."""
    _client_plugin_registry.register_plugin_class(name, plugin_class)


def close_all_client_plugins():
    """Close all client plugin instances."""
    _client_plugin_registry.close_all_instances()


def initialize_client_plugins():
    """Initialize client plugins from settings."""
    _client_plugin_registry.initialize_from_settings()