Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2930 [MISC] Add plugin infrastructure to unstract-core (#1618)

Chandrasekharan M808040c63f29
diff --git a/.gitignore b/.gitignore
index 8cb0de037..c89b40911 100644
--- a/.gitignore
+++ b/.gitignore
@@ -620,29 +620,6 @@ $RECYCLE.BIN/
### Unstract ###
-# Authentication Plugins
-backend/plugins/authentication/*
-!backend/plugins/authentication/auth_sample
-
-# Processor Plugins
-backend/plugins/processor/*
-
-# Subscription Plugins
-backend/plugins/subscription/*
-
-
-# API Deployment Plugins
-backend/plugins/api/**
-
-# Notification Plugin
-backend/plugins/notification/**
-
-# Configuration Plugin
-backend/plugins/configuration/**
-
-# Verticals Usage Plugin
-backend/plugins/verticals_usage/**
-
# BE pluggable-apps
backend/pluggable_apps/*
diff --git a/backend/account_v2/subscription_loader.py b/backend/account_v2/subscription_loader.py
deleted file mode 100644
index 8f323c29b..000000000
--- a/backend/account_v2/subscription_loader.py
+++ /dev/null
@@ -1,129 +0,0 @@
-import logging
-import os
-from importlib import import_module
-from typing import Any
-
-from django.apps import apps
-from django.utils import timezone
-
-logger = logging.getLogger(__name__)
-
-
-class SubscriptionConfig:
- """Loader config for subscription plugins."""
-
- PLUGINS_APP = "plugins"
- PLUGIN_DIR = "subscription"
- MODULE = "module"
- METADATA = "metadata"
- METADATA_NAME = "name"
- METADATA_SERVICE_CLASS = "service_class"
- METADATA_IS_ACTIVE = "is_active"
-
-
-# Cache for loaded plugins to avoid repeated loading
-_subscription_plugins_cache: list[Any] = []
-_plugins_loaded = False
-
-
-def load_plugins() -> list[Any]:
- """Iterate through the subscription plugins and register them."""
- global _subscription_plugins_cache, _plugins_loaded
-
- # Return cached plugins if already loaded
- if _plugins_loaded:
- return _subscription_plugins_cache
-
- plugins_app = apps.get_app_config(SubscriptionConfig.PLUGINS_APP)
- package_path = plugins_app.module.__package__
- subscription_dir = os.path.join(plugins_app.path, SubscriptionConfig.PLUGIN_DIR)
- subscription_package_path = f"{package_path}.{SubscriptionConfig.PLUGIN_DIR}"
- subscription_plugins: list[Any] = []
-
- if not os.path.exists(subscription_dir):
- _subscription_plugins_cache = subscription_plugins
- _plugins_loaded = True
- return subscription_plugins
-
- for item in os.listdir(subscription_dir):
- # Loads a plugin if it is in a directory.
- if os.path.isdir(os.path.join(subscription_dir, item)):
- subscription_module_name = item
- # Loads a plugin if it is a shared library.
- # Module name is extracted from shared library name.
- # `subscription.platform_architecture.so` will be file name and
- # `subscription` will be the module name.
- elif item.endswith(".so"):
- subscription_module_name = item.split(".")[0]
- else:
- continue
- try:
- full_module_path = f"{subscription_package_path}.{subscription_module_name}"
- module = import_module(full_module_path)
- metadata = getattr(module, SubscriptionConfig.METADATA, {})
-
- if metadata.get(SubscriptionConfig.METADATA_IS_ACTIVE, False):
- subscription_plugins.append(
- {
- SubscriptionConfig.MODULE: module,
- SubscriptionConfig.METADATA: module.metadata,
- }
- )
- name = metadata.get(
- SubscriptionConfig.METADATA_NAME,
- getattr(module, "__name__", "unknown"),
- )
- is_active = metadata.get(SubscriptionConfig.METADATA_IS_ACTIVE, False)
- logger.info(
- "Loaded subscription plugin: %s, is_active: %s", name, is_active
- )
- else:
- logger.info(
- "subscription plugin %s is not active.",
- subscription_module_name,
- )
- except ModuleNotFoundError as exception:
- logger.error(
- "Error while importing subscription plugin: %s",
- exception,
- )
-
- if len(subscription_plugins) == 0:
- logger.info("No subscription plugins found.")
-
- # Cache the results for future requests
- _subscription_plugins_cache = subscription_plugins
- _plugins_loaded = True
-
- return subscription_plugins
-
-
-def validate_etl_run(org_id: str) -> bool:
- """Method to check subscription status before ETL runs.
-
- Args:
- org_id: The ID of the organization.
-
- Returns:
- A boolean indicating whether the pre-run check passed or not.
- """
- try:
- from pluggable_apps.subscription_v2.subscription_helper import (
- SubscriptionHelper,
- )
- except ModuleNotFoundError:
- logger.error("Subscription plugin not found.")
- return False
-
- org_plans = SubscriptionHelper.get_subscription(org_id)
- if not org_plans or not org_plans.is_active:
- return False
-
- if org_plans.is_paid:
- return True
-
- if timezone.now() >= org_plans.end_date:
- logger.debug(f"Trial expired for org {org_id}")
- return False
-
- return True
diff --git a/backend/adapter_processor_v2/views.py b/backend/adapter_processor_v2/views.py
index 78f288063..046f80bd4 100644
--- a/backend/adapter_processor_v2/views.py
+++ b/backend/adapter_processor_v2/views.py
@@ -342,37 +342,40 @@ class AdapterInstanceViewSet(ModelViewSet):
# Send email notifications to newly shared users
if response.status_code == 200 and AdapterKeys.SHARED_USERS in request.data:
try:
- from plugins.notification.constants import ResourceType
- from plugins.notification.sharing_notification import (
- SharingNotificationService,
- )
-
- adapter.refresh_from_db()
- new_shared_users = set(adapter.shared_users.all())
- newly_shared_users = new_shared_users - current_shared_users
-
- if newly_shared_users:
- # Map adapter type to specific resource type
- adapter_type_to_resource = {
- "LLM": ResourceType.LLM.value,
- "EMBEDDING": ResourceType.EMBEDDING.value,
- "VECTOR_DB": ResourceType.VECTOR_DB.value,
- "X2TEXT": ResourceType.X2TEXT.value,
- }
-
- resource_type = adapter_type_to_resource.get(
- adapter.adapter_type, ResourceType.LLM.value
- )
+ from plugins import get_plugin
+
+ notification_plugin = get_plugin("notification")
+ if notification_plugin:
+ from plugins.notification.constants import ResourceType
+
+ adapter.refresh_from_db()
+ new_shared_users = set(adapter.shared_users.all())
+ newly_shared_users = new_shared_users - current_shared_users
+
+ if newly_shared_users:
+ # Map adapter type to specific resource type
+ adapter_type_to_resource = {
+ "LLM": ResourceType.LLM.value,
+ "EMBEDDING": ResourceType.EMBEDDING.value,
+ "VECTOR_DB": ResourceType.VECTOR_DB.value,
+ "X2TEXT": ResourceType.X2TEXT.value,
+ }
+
+ resource_type = adapter_type_to_resource.get(
+ adapter.adapter_type, ResourceType.LLM.value
+ )
- notification_service = SharingNotificationService()
- notification_service.send_sharing_notification(
- resource_type=resource_type,
- resource_name=adapter.adapter_name,
- resource_id=str(adapter.id),
- shared_by=request.user,
- shared_to=list(newly_shared_users),
- resource_instance=adapter,
- )
+ # Get notification service from plugin
+ service_class = notification_plugin["service_class"]
+ notification_service = service_class()
+ notification_service.send_sharing_notification(
+ resource_type=resource_type,
+ resource_name=adapter.adapter_name,
+ resource_id=str(adapter.id),
+ shared_by=request.user,
+ shared_to=list(newly_shared_users),
+ resource_instance=adapter,
+ )
except Exception as e:
logger.exception(f"Failed to send sharing notification: {e}")
diff --git a/backend/api_v2/api_deployment_dto_registry.py b/backend/api_v2/api_deployment_dto_registry.py
index 981a6520e..b40df3e45 100644
--- a/backend/api_v2/api_deployment_dto_registry.py
+++ b/backend/api_v2/api_deployment_dto_registry.py
@@ -1,7 +1,7 @@
import logging
from typing import Any
-from plugins.api.dto import metadata
+from plugins import get_plugin
from api_v2.postman_collection.dto import PostmanCollection
@@ -13,10 +13,16 @@ class ApiDeploymentDTORegistry:
@classmethod
def load_dto(cls) -> Any | None:
- class_name = PostmanCollection.__name__
- if metadata.get(class_name):
- return metadata[class_name].class_name
- return PostmanCollection # Return as soon as we find a valid DTO
+ """Load DTO from plugin or return default PostmanCollection.
+
+ Checks if the api_dto plugin is available and gets the Postman DTO
+ via the service, otherwise returns the base PostmanCollection class.
+ """
+ plugin = get_plugin("api_dto")
+ if plugin:
+ service = plugin["service_class"]()
+ return service.get_postman_dto()
+ return PostmanCollection
@classmethod
def get_dto(cls) -> type | None:
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py
index dc4d77108..7174547e0 100644
--- a/backend/api_v2/api_deployment_views.py
+++ b/backend/api_v2/api_deployment_views.py
@@ -7,6 +7,7 @@ from configuration.models import Configuration
from django.db.models import QuerySet
from django.http import HttpResponse
from permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg
+from plugins import get_plugin
from prompt_studio.prompt_studio_registry_v2.models import PromptStudioRegistry
from rest_framework import serializers, status, views, viewsets
from rest_framework.decorators import action
@@ -33,15 +34,12 @@ from api_v2.serializers import (
SharedUserListSerializer,
)
-try:
- from plugins.notification.constants import ResourceType
- from plugins.notification.sharing_notification import SharingNotificationService
+# Check if notification plugin is available
+notification_plugin = get_plugin("notification")
- NOTIFICATION_PLUGIN_AVAILABLE = True
- sharing_notification_service = SharingNotificationService()
-except ImportError:
- NOTIFICATION_PLUGIN_AVAILABLE = False
- sharing_notification_service = None
+# Import constants from notification plugin if available
+if notification_plugin:
+ from plugins.notification.constants import ResourceType
logger = logging.getLogger(__name__)
@@ -319,7 +317,7 @@ class APIDeploymentViewSet(viewsets.ModelViewSet):
if (
response.status_code == 200
and "shared_users" in request.data
- and NOTIFICATION_PLUGIN_AVAILABLE
+ and notification_plugin
):
try:
instance.refresh_from_db()
@@ -327,7 +325,9 @@ class APIDeploymentViewSet(viewsets.ModelViewSet):
newly_shared_users = new_shared_users - current_shared_users
if newly_shared_users:
- notification_service = SharingNotificationService()
+ # Get notification service from plugin
+ service_class = notification_plugin["service_class"]
+ notification_service = service_class()
notification_service.send_sharing_notification(
resource_type=ResourceType.API_DEPLOYMENT.value,
resource_name=instance.display_name,
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 26f74c747..b6cbb2599 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -308,7 +308,7 @@ SHARED_APPS = (
"drf_yasg",
"docs",
# Plugins
- "plugins",
+ "plugins.apps.PluginsConfig",
"feature_flag",
"django_celery_beat",
# For additional helper commands
diff --git a/backend/configuration/config_registry.py b/backend/configuration/config_registry.py
index 189126631..ce4f91fca 100644
--- a/backend/configuration/config_registry.py
+++ b/backend/configuration/config_registry.py
@@ -5,12 +5,9 @@ cloud-specific configuration keys from plugins when available.
"""
import logging
-import os
from importlib import import_module
from typing import Any
-from django.apps import apps
-
from configuration.constants import ConfigPluginConstants
from configuration.enums import ConfigKey, ConfigSpec
@@ -26,52 +23,46 @@ def _load_cloud_config_specs() -> dict[str, ConfigSpec]:
cloud_specs = {}
try:
- # Check if plugins app exists
- auth_app = apps.get_app_config(ConfigPluginConstants.PLUGINS_APP)
- config_dir = os.path.join(auth_app.path, ConfigPluginConstants.CONFIG_PLUGIN_DIR)
+ from plugins import get_plugin
+
+ # Try to get the configuration plugin
+ plugin = get_plugin(ConfigPluginConstants.CONFIG_PLUGIN_DIR)
+
+ if not plugin:
+ logger.debug("Configuration plugin not found")
+ return cloud_specs
- # Check if configuration plugin directory exists
- if not os.path.exists(config_dir):
- logger.debug("Configuration plugin directory not found")
+ # Check if plugin is active
+ metadata = plugin.get(ConfigPluginConstants.METADATA, {})
+ if not metadata.get(ConfigPluginConstants.METADATA_IS_ACTIVE, False):
+ logger.warning("Cloud configuration plugin is not active")
return cloud_specs
- # Try to import the configuration plugin package
- config_package_path = (
- f"{auth_app.module.__package__}.{ConfigPluginConstants.CONFIG_PLUGIN_DIR}"
+ # Get the plugin module and import the config specs submodule
+ plugin_module = plugin.get("module")
+ if not plugin_module:
+ logger.warning("Configuration plugin module not found")
+ return cloud_specs
+
+ # Import the cloud_config submodule to get config specs
+ base_module = (
+ getattr(plugin_module, "__package__", None) or plugin_module.__name__
+ )
+ config_module_path = f"{base_module}.{ConfigPluginConstants.CONFIG_MODULE}"
+ config_module = import_module(config_module_path)
+ config_specs = getattr(config_module, ConfigPluginConstants.CONFIG_SPECS, {})
+
+ cloud_specs.update(config_specs)
+ logger.info(
+ "Loaded cloud configuration plugin: %s with %d config keys",
+ metadata.get(ConfigPluginConstants.METADATA_NAME, "Unknown"),
+ len(config_specs),
)
- try:
- # Import the plugin package to get metadata from __init__.py
- plugin_module = import_module(config_package_path)
- metadata = getattr(plugin_module, ConfigPluginConstants.CONFIG_METADATA, {})
-
- if metadata.get(ConfigPluginConstants.METADATA_IS_ACTIVE, False):
- # Import the cloud_config module to get config specs
- full_module_path = (
- f"{config_package_path}.{ConfigPluginConstants.CONFIG_MODULE}"
- )
- config_module = import_module(full_module_path)
- config_specs = getattr(
- config_module, ConfigPluginConstants.CONFIG_SPECS, {}
- )
- cloud_specs.update(config_specs)
- logger.info(
- "Loaded cloud configuration plugin: %s with %d config keys",
- metadata.get(ConfigPluginConstants.METADATA_NAME, "Unknown"),
- len(config_specs),
- )
- else:
- logger.debug("Cloud configuration plugin not active")
-
- except ModuleNotFoundError as e:
- logger.debug("Cloud configuration module not found: %s", e)
- except Exception as e:
- logger.error("Error loading cloud configuration plugin: %s", e)
-
- except LookupError:
- logger.debug("Plugins app not configured")
+ except ImportError as e:
+ logger.debug("Could not import plugins module: %s", e)
except Exception as e:
- logger.error("Error accessing plugins app: %s", e)
+ logger.error("Error loading cloud configuration plugin: %s", e)
return cloud_specs
diff --git a/backend/pipeline_v2/views.py b/backend/pipeline_v2/views.py
index 396729f39..02625a52f 100644
--- a/backend/pipeline_v2/views.py
+++ b/backend/pipeline_v2/views.py
@@ -10,6 +10,7 @@ from django.db import IntegrityError
from django.db.models import QuerySet
from django.http import HttpResponse
from permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg
+from plugins import get_plugin
from rest_framework import serializers, status, viewsets
from rest_framework.decorators import action
from rest_framework.request import Request
@@ -32,15 +33,13 @@ from pipeline_v2.serializers.execute import (
)
from pipeline_v2.serializers.sharing import SharedUserListSerializer
-try:
- from plugins.notification.constants import ResourceType
- from plugins.notification.sharing_notification import SharingNotificationService
+# Check if notification plugin is available
+notification_plugin = get_plugin("notification")
+NOTIFICATION_PLUGIN_AVAILABLE = notification_plugin is not None
- NOTIFICATION_PLUGIN_AVAILABLE = True
- sharing_notification_service = SharingNotificationService()
-except ImportError:
- NOTIFICATION_PLUGIN_AVAILABLE = False
- sharing_notification_service = None
+# Import constants from notification plugin if available
+if NOTIFICATION_PLUGIN_AVAILABLE:
+ from plugins.notification.constants import ResourceType
logger = logging.getLogger(__name__)
@@ -149,8 +148,10 @@ class PipelineViewSet(viewsets.ModelViewSet):
resource_type = ResourceType.TASK.value
if newly_shared_users:
- # Only send notifications if there are newly shared users
- sharing_notification_service.send_sharing_notification(
+ # Get notification service from plugin and send notification
+ service_class = notification_plugin["service_class"]
+ notification_service = service_class()
+ notification_service.send_sharing_notification(
resource_type=resource_type,
resource_name=instance.pipeline_name,
resource_id=str(instance.id),
diff --git a/backend/plugins/.gitignore b/backend/plugins/.gitignore
index 5d66e53bd..8d0b5901c 100644
--- a/backend/plugins/.gitignore
+++ b/backend/plugins/.gitignore
@@ -1 +1,11 @@
-modifier
+# Ignore all plugin implementation directories
+# Keep only infrastructure files (__init__.py, plugin_manager.py, etc.)
+*/
+
+# But allow infrastructure files to be tracked
+!__init__.py
+!apps.py
+
+# TODO: Make these as proper plugins
+!authentication/auth_sample
+!workflow_manager
diff --git a/backend/plugins/README.md b/backend/plugins/README.md
index b03ee2c78..c3e7cb4bb 100644
--- a/backend/plugins/README.md
+++ b/backend/plugins/README.md
@@ -1,5 +1,3 @@
# Plugins
-## Authentication
-
-Enhance the authentication capabilities of the `account/authentication_controller.py` module by incorporating additional modules.
+Read [unstract-core's README](../../unstract/core/src/unstract/core/plugins/README.md) for more details on adding and using plugins.
diff --git a/backend/plugins/__init__.py b/backend/plugins/__init__.py
index e69de29bb..feb126f1b 100644
--- a/backend/plugins/__init__.py
+++ b/backend/plugins/__init__.py
@@ -0,0 +1,60 @@
+"""Backend plugins initialization.
+
+This module initializes the Django plugin manager for all backend plugins.
+"""
+
+import logging
+from pathlib import Path
+
+from unstract.core.django import DjangoPluginManager
+
+logger = logging.getLogger(__name__)
+
+# Initialize plugin manager singleton
+_plugin_manager = None
+
+
+def get_plugin_manager() -> DjangoPluginManager:
+ """Get or initialize the plugin manager singleton.
+
+ Note: Plugins are loaded by PluginsConfig.ready() after Django initialization.
+ This function only creates the manager instance without loading plugins.
+
+ Returns:
+ DjangoPluginManager: The plugin manager instance
+ """
+ global _plugin_manager
+ if _plugin_manager is None:
+ plugins_dir = Path(__file__).parent
+ _plugin_manager = DjangoPluginManager(
+ plugins_dir=plugins_dir,
+ plugins_pkg="plugins",
+ logger=logger,
+ )
+ return _plugin_manager
+
+
+def get_plugin(plugin_name: str) -> dict:
+ """Get plugin metadata by name (simplified single-line access).
+
+ This is a convenience function that combines get_plugin_manager()
+ and plugin_manager.get_plugin() into one call.
+
+ Args:
+ plugin_name: Name of the plugin to retrieve
+
+ Returns:
+ Dictionary containing plugin metadata (version, module, service_class, etc.)
+ or empty dict if plugin not found
+
+ Example:
+ >>> from plugins import get_plugin
+ >>> plugin = get_plugin("subscription_usage")
+ >>> if plugin:
+ ... service = plugin["service_class"]()
+ """
+ manager = get_plugin_manager()
+ return manager.get_plugin(plugin_name)
+
+
+__all__ = ["get_plugin_manager", "get_plugin"]
diff --git a/backend/plugins/api/dto/__init__.py b/backend/plugins/api/dto/__init__.py
index b84a87d20..18b88861f 100644
--- a/backend/plugins/api/dto/__init__.py
+++ b/backend/plugins/api/dto/__init__.py
@@ -1,12 +1,31 @@
-from dataclasses import dataclass
-from typing import Any
+"""API DTO extensions for cloud deployments.
+This plugin extends OSS DTOs with cloud-specific features like
+highlight data in Postman collections.
+"""
-@dataclass
-class MetadataDto:
- name: str
- class_name: Any
- is_active: bool
+from .postman_collection import HighlightPostmanDto
-metadata = {}
+class ApiDtoService:
+ """Service for providing cloud-specific API DTOs."""
+
+ def get_postman_dto(self):
+ """Get the Postman collection DTO class for cloud features.
+
+ Returns:
+ Class: HighlightPostmanDto class for creating Postman collections
+ with cloud-specific features like highlight data API.
+ """
+ return HighlightPostmanDto
+
+
+metadata = {
+ "version": "1.0.0",
+ "name": "api_dto",
+ "is_active": True,
+ "description": "API DTO extensions for cloud-specific features",
+ "service_class": ApiDtoService,
+}
+
+__all__ = ["metadata", "ApiDtoService"]
diff --git a/backend/plugins/apps.py b/backend/plugins/apps.py
new file mode 100644
index 000000000..0b5e416df
--- /dev/null
+++ b/backend/plugins/apps.py
@@ -0,0 +1,34 @@
+"""Django AppConfig for backend plugins."""
+
+import logging
+
+from django.apps import AppConfig
+
+logger = logging.getLogger(__name__)
+
+
+class PluginsConfig(AppConfig):
+ """Django AppConfig for the plugins package.
+
+ This loads plugins after Django is fully initialized to avoid
+ AppRegistryNotReady errors.
+ """
+
+ name = "plugins"
+ verbose_name = "Backend Plugins"
+
+ def ready(self) -> None:
+ """Load plugins after Django apps are ready.
+
+ This method is called by Django after all apps are loaded,
+ ensuring that models and other Django components are available.
+ """
+ from plugins import get_plugin_manager
+
+ try:
+ plugin_manager = get_plugin_manager()
+ plugin_manager.load_plugins()
+ logger.info("Backend plugins loaded successfully")
+ except Exception as e:
+ logger.error(f"Failed to load plugins: {e}", exc_info=True)
+ # Don't raise - allow Django to continue even if plugins fail
diff --git a/backend/prompt_studio/modifier_loader.py b/backend/prompt_studio/modifier_loader.py
deleted file mode 100644
index e78d18113..000000000
--- a/backend/prompt_studio/modifier_loader.py
+++ /dev/null
@@ -1,89 +0,0 @@
-import logging
-import os
-from importlib import import_module
-from typing import Any
-
-from django.apps import apps
-
-logger = logging.getLogger(__name__)
-
-
-class ModifierConfig:
- """Loader config for extraction plugins."""
-
- PLUGINS_APP = "plugins"
- PLUGIN_DIR = "modifier"
- MODULE = "module"
- METADATA = "metadata"
- METADATA_NAME = "name"
- METADATA_SERVICE_CLASS = "service_class"
- METADATA_IS_ACTIVE = "is_active"
-
-
-# Cache for loaded plugins to avoid repeated loading
-_modifier_plugins_cache: list[Any] = []
-_plugins_loaded = False
-
-
-def load_plugins() -> list[Any]:
- """Iterate through the extraction plugins and register them."""
- global _modifier_plugins_cache, _plugins_loaded
-
- # Return cached plugins if already loaded
- if _plugins_loaded:
- return _modifier_plugins_cache
-
- plugins_app = apps.get_app_config(ModifierConfig.PLUGINS_APP)
- package_path = plugins_app.module.__package__
- modifier_dir = os.path.join(plugins_app.path, ModifierConfig.PLUGIN_DIR)
- modifier_package_path = f"{package_path}.{ModifierConfig.PLUGIN_DIR}"
- modifier_plugins: list[Any] = []
-
- if not os.path.exists(modifier_dir):
- _modifier_plugins_cache = modifier_plugins
- _plugins_loaded = True
- return modifier_plugins
-
- for item in os.listdir(modifier_dir):
- # Loads a plugin if it is in a directory.
- if os.path.isdir(os.path.join(modifier_dir, item)):
- modifier_module_name = item
- # Loads a plugin if it is a shared library.
- # Module name is extracted from shared library name.
- elif item.endswith(".so"):
- modifier_module_name = item.split(".")[0]
- else:
- continue
- try:
- full_module_path = f"{modifier_package_path}.{modifier_module_name}"
- module = import_module(full_module_path)
- metadata = getattr(module, ModifierConfig.METADATA, {})
-
- if metadata.get(ModifierConfig.METADATA_IS_ACTIVE, False):
- modifier_plugins.append(
- {
- ModifierConfig.MODULE: module,
- ModifierConfig.METADATA: module.metadata,
- }
- )
- logger.info(
- "Loaded modifier plugin: %s, is_active: %s",
- module.metadata[ModifierConfig.METADATA_NAME],
- module.metadata[ModifierConfig.METADATA_IS_ACTIVE],
- )
- else:
- logger.info(
- "modifier plugin %s is not active.",
- modifier_module_name,
- )
- except ModuleNotFoundError:
- logger.warning("No prompt modifier plugins loaded")
-
- if len(modifier_plugins) == 0:
- logger.info("No modifier plugins found.")
-
- # Cache the results for future requests
- _modifier_plugins_cache = modifier_plugins
- _plugins_loaded = True
-
- return modifier_plugins
diff --git a/backend/prompt_studio/processor_loader.py b/backend/prompt_studio/processor_loader.py
deleted file mode 100644
index 8c22dd82e..000000000
--- a/backend/prompt_studio/processor_loader.py
+++ /dev/null
@@ -1,103 +0,0 @@
-import logging
-import os
-from importlib import import_module
-from typing import Any
-
-from django.apps import apps
-
-logger = logging.getLogger(__name__)
-
-
-class ProcessorConfig:
- """Loader config for processor plugins."""
-
- PLUGINS_APP = "plugins"
- PLUGIN_DIR = "processor"
- MODULE = "module"
- METADATA = "metadata"
- METADATA_NAME = "name"
- METADATA_SERVICE_CLASS = "service_class"
- METADATA_IS_ACTIVE = "is_active"
-
-
-# Cache for loaded plugins to avoid repeated loading
-_processor_plugins_cache: list[Any] = []
-_plugins_loaded = False
-
-
-def load_plugins() -> list[Any]:
- """Iterate through the processor plugins and register them."""
- global _processor_plugins_cache, _plugins_loaded
-
- # Return cached plugins if already loaded
- if _plugins_loaded:
- return _processor_plugins_cache
-
- plugins_app = apps.get_app_config(ProcessorConfig.PLUGINS_APP)
- package_path = plugins_app.module.__package__
- processor_dir = os.path.join(plugins_app.path, ProcessorConfig.PLUGIN_DIR)
- processor_package_path = f"{package_path}.{ProcessorConfig.PLUGIN_DIR}"
- processor_plugins: list[Any] = []
-
- if not os.path.exists(processor_dir):
- logger.info("No processor directory found at %s.", processor_dir)
- return []
-
- for item in os.listdir(processor_dir):
- # Loads a plugin if it is in a directory.
- if os.path.isdir(os.path.join(processor_dir, item)):
- processor_module_name = item
- # Loads a plugin if it is a shared library.
- # Module name is extracted from shared library name.
- # `processor.platform_architecture.so` will be file name and
- # `processor` will be the module name.
- elif item.endswith(".so"):
- processor_module_name = item.split(".")[0]
- else:
- continue
- try:
- full_module_path = f"{processor_package_path}.{processor_module_name}"
- module = import_module(full_module_path)
- metadata = getattr(module, ProcessorConfig.METADATA, {})
-
- if metadata.get(ProcessorConfig.METADATA_IS_ACTIVE, False):
- processor_plugins.append(
- {
- ProcessorConfig.MODULE: module,
- ProcessorConfig.METADATA: module.metadata,
- }
- )
- logger.info(
- "Loaded processor plugin: %s, is_active: %s",
- module.metadata[ProcessorConfig.METADATA_NAME],
- module.metadata[ProcessorConfig.METADATA_IS_ACTIVE],
- )
- else:
- logger.info(
- "Processor plugin %s is not active.",
- processor_module_name,
- )
- except ModuleNotFoundError as exception:
- logger.error(
- "Error while importing processor plugin: %s",
- exception,
- )
-
- if len(processor_plugins) == 0:
- logger.info("No processor plugins found.")
-
- # Cache the results for future requests
- _processor_plugins_cache = processor_plugins
- _plugins_loaded = True
-
- return processor_plugins
-
-
-def get_plugin_class_by_name(name: str, plugins: list[Any]) -> Any:
- """Retrieve a specific plugin class by name."""
- for plugin in plugins:
- metadata = plugin[ProcessorConfig.METADATA]
- if metadata.get(ProcessorConfig.METADATA_NAME) == name:
- return metadata.get(ProcessorConfig.METADATA_SERVICE_CLASS)
- logger.warning("Plugin with name '%s' not found.", name)
- return None
diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
index 7dd2fc8be..91f8f0740 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
@@ -12,15 +12,13 @@ from adapter_processor_v2.constants import AdapterKeys
from adapter_processor_v2.models import AdapterInstance
from django.conf import settings
from django.db.models.manager import BaseManager
+from plugins import get_plugin
from rest_framework.request import Request
from utils.file_storage.constants import FileStorageKeys
from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper
from utils.local_context import StateStore
+from utils.subscription_usage_decorator import track_subscription_usage_if_available
-from prompt_studio.modifier_loader import ModifierConfig
-from prompt_studio.modifier_loader import load_plugins as load_modifier_plugins
-from prompt_studio.processor_loader import get_plugin_class_by_name
-from prompt_studio.processor_loader import load_plugins as load_processor_plugins
from prompt_studio.prompt_profile_manager_v2.models import ProfileManager
from prompt_studio.prompt_profile_manager_v2.profile_manager_helper import (
ProfileManagerHelper,
@@ -91,14 +89,10 @@ ERROR_MSG = "User %s doesn't have access to adapter %s"
logger = logging.getLogger(__name__)
-modifier_plugins = load_modifier_plugins()
-
class PromptStudioHelper:
"""Helper class for Custom tool operations."""
- processor_plugins = load_processor_plugins()
-
@staticmethod
def create_default_profile_manager(user: User, tool_id: uuid) -> None:
"""Create a default profile manager for a given user and tool.
@@ -292,11 +286,11 @@ class PromptStudioHelper:
choices = f.read()
f.close()
response: dict[str, Any] = json.loads(choices)
- for modifier_plugin in modifier_plugins:
- cls = modifier_plugin[ModifierConfig.METADATA][
- ModifierConfig.METADATA_SERVICE_CLASS
- ]
- response = cls.update_select_choices(default_choices=response)
+ # Update select choices with payload modifier plugin if available
+ payload_modifier_plugin = get_plugin("payload_modifier")
+ if payload_modifier_plugin:
+ modifier_service = payload_modifier_plugin["service_class"]()
+ response = modifier_service.update_select_choices(default_choices=response)
return response
@staticmethod
@@ -328,6 +322,7 @@ class PromptStudioHelper:
return prompt_instances
@staticmethod
+ @track_subscription_usage_if_available(file_execution_id_param="run_id")
def index_document(
tool_id: str,
file_name: str,
@@ -487,17 +482,15 @@ class PromptStudioHelper:
@staticmethod
def summarize(file_name, org_id, document_id, run_id, tool, doc_id) -> str:
- cls = get_plugin_class_by_name(
- name="summarizer",
- plugins=PromptStudioHelper.processor_plugins,
- )
+ summarizer_plugin = get_plugin("summarizer")
usage_kwargs: dict[Any, Any] = dict()
usage_kwargs[ToolStudioPromptKeys.RUN_ID] = run_id
prompts: list[ToolStudioPrompt] = PromptStudioHelper.fetch_prompt_from_tool(
tool.tool_id
)
- if cls:
- summarize_file_path = cls.process(
+ if summarizer_plugin:
+ summarizer_service = summarizer_plugin["service_class"]()
+ summarize_file_path = summarizer_service.process(
tool_id=str(tool.tool_id),
file_name=file_name,
org_id=org_id,
@@ -523,6 +516,7 @@ class PromptStudioHelper:
return summarize_file_path
@staticmethod
+ @track_subscription_usage_if_available(file_execution_id_param="run_id")
def prompt_responder(
tool_id: str,
org_id: str,
@@ -591,10 +585,12 @@ class PromptStudioHelper:
):
prompt_instance = PromptStudioHelper._fetch_prompt_from_id(id)
+ # Check if payload modifier plugin is available for table/record operations
+ payload_modifier_plugin = get_plugin("payload_modifier")
if (
prompt_instance.enforce_type == TSPKeys.TABLE
or prompt_instance.enforce_type == TSPKeys.RECORD
- ) and not modifier_plugins:
+ ) and not payload_modifier_plugin:
raise OperationNotSupported()
prompt_name = prompt_instance.prompt_key
@@ -1038,11 +1034,11 @@ class PromptStudioHelper:
org_id, user_id, tool_id, doc_name, TSPKeys.EXTRACT
)
)
- for modifier_plugin in modifier_plugins:
- cls = modifier_plugin[ModifierConfig.METADATA][
- ModifierConfig.METADATA_SERVICE_CLASS
- ]
- output = cls.update(
+ # Update output with payload modifier plugin if available
+ payload_modifier_plugin = get_plugin("payload_modifier")
+ if payload_modifier_plugin:
+ modifier_service = payload_modifier_plugin["service_class"]()
+ output = modifier_service.update(
output=output,
tool_id=tool_id,
prompt_id=str(prompt.prompt_id),
diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.py
index b6495404b..9743e348e 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/views.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py
@@ -11,6 +11,7 @@ from django.http import HttpRequest, HttpResponse
from file_management.constants import FileInformationKey as FileKey
from file_management.exceptions import FileNotFound
from permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg
+from plugins import get_plugin
from rest_framework import status, viewsets
from rest_framework.decorators import action
from rest_framework.request import Request
@@ -21,8 +22,6 @@ from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFil
from utils.user_context import UserContext
from utils.user_session import UserSessionUtils
-from prompt_studio.processor_loader import get_plugin_class_by_name
-from prompt_studio.processor_loader import load_plugins as load_processor_plugins
from prompt_studio.prompt_profile_manager_v2.constants import (
ProfileManagerErrors,
ProfileManagerKeys,
@@ -88,8 +87,6 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
serializer_class = CustomToolSerializer
- processor_plugins = load_processor_plugins()
-
def get_permissions(self) -> list[Any]:
if self.action == "destroy":
return [IsOwner()]
@@ -162,35 +159,37 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
# Send email notifications to newly shared users
if response.status_code == 200 and "shared_users" in request.data:
- from plugins.notification.constants import ResourceType
- from plugins.notification.sharing_notification import (
- SharingNotificationService,
- )
-
- # Refresh the object to get updated shared_users
- custom_tool.refresh_from_db()
- updated_shared_users = set(custom_tool.shared_users.all())
-
- # Find newly added users (not previously shared)
- newly_shared_users = updated_shared_users - current_shared_users
-
- if newly_shared_users:
- notification_service = SharingNotificationService()
- try:
- notification_service.send_sharing_notification(
- resource_type=ResourceType.TEXT_EXTRACTOR.value,
- resource_name=custom_tool.tool_name,
- resource_id=str(custom_tool.tool_id),
- shared_by=request.user,
- shared_to=list(newly_shared_users),
- resource_instance=custom_tool,
- )
- except Exception as e:
- # Log error but don't fail the request
- logger.exception(
- f"Failed to send sharing notification for "
- f"custom tool {custom_tool.tool_id}: {str(e)}"
- )
+ from plugins import get_plugin
+
+ notification_plugin = get_plugin("notification")
+ if notification_plugin:
+ from plugins.notification.constants import ResourceType
+
+ # Refresh the object to get updated shared_users
+ custom_tool.refresh_from_db()
+ updated_shared_users = set(custom_tool.shared_users.all())
+
+ # Find newly added users (not previously shared)
+ newly_shared_users = updated_shared_users - current_shared_users
+
+ if newly_shared_users:
+ service_class = notification_plugin["service_class"]
+ notification_service = service_class()
+ try:
+ notification_service.send_sharing_notification(
+ resource_type=ResourceType.TEXT_EXTRACTOR.value,
+ resource_name=custom_tool.tool_name,
+ resource_id=str(custom_tool.tool_id),
+ shared_by=request.user,
+ shared_to=list(newly_shared_users),
+ resource_instance=custom_tool,
+ )
+ except Exception as e:
+ # Log error but don't fail the request
+ logger.exception(
+ f"Failed to send sharing notification for "
+ f"custom tool {custom_tool.tool_id}: {str(e)}"
+ )
return response
@@ -437,14 +436,14 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
document: DocumentManager = DocumentManager.objects.get(pk=document_id)
file_name: str = document.document_name
view_type: str = serializer.validated_data.get("view_type")
- file_converter = get_plugin_class_by_name(
- name="file_converter",
- plugins=self.processor_plugins,
- )
+ file_converter_plugin = get_plugin("file_converter")
allowed_content_types = FileKey.FILE_UPLOAD_ALLOWED_MIME
- if file_converter:
- allowed_content_types = file_converter.get_extented_file_information_key()
+ if file_converter_plugin:
+ file_converter_service = file_converter_plugin["service_class"]()
+ allowed_content_types = (
+ file_converter_service.get_extented_file_information_key()
+ )
filename_without_extension = file_name.rsplit(".", 1)[0]
if view_type == FileViewTypes.EXTRACT:
file_name = (
@@ -472,10 +471,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
serializer = FileUploadIdeSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
uploaded_files: Any = serializer.validated_data.get("file")
- file_converter = get_plugin_class_by_name(
- name="file_converter",
- plugins=self.processor_plugins,
- )
+ file_converter_plugin = get_plugin("file_converter")
documents = []
for uploaded_file in uploaded_files:
@@ -484,8 +480,9 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
file_data = uploaded_file
file_type = uploaded_file.content_type
# Convert non-PDF files
- if file_converter and file_type != "application/pdf":
- file_data, file_name = file_converter.process_file(
+ if file_converter_plugin and file_type != "application/pdf":
+ file_converter_service = file_converter_plugin["service_class"]()
+ file_data, file_name = file_converter_service.process_file(
uploaded_file, file_name
)
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
index 8832055cb..724010ce1 100644
--- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
+++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
@@ -5,9 +5,8 @@ from account_v2.models import User
from adapter_processor_v2.models import AdapterInstance
from django.conf import settings
from django.db import IntegrityError
+from plugins import get_plugin
-from prompt_studio.modifier_loader import ModifierConfig
-from prompt_studio.modifier_loader import load_plugins as load_modifier_plugins
from prompt_studio.prompt_profile_manager_v2.models import ProfileManager
from prompt_studio.prompt_studio_core_v2.models import CustomTool
from prompt_studio.prompt_studio_core_v2.prompt_studio_helper import PromptStudioHelper
@@ -28,7 +27,6 @@ from .models import PromptStudioRegistry
from .serializers import PromptStudioRegistrySerializer
logger = logging.getLogger(__name__)
-modifier_plugins = load_modifier_plugins()
class PromptStudioRegistryHelper:
@@ -354,11 +352,10 @@ class PromptStudioRegistryHelper:
or prompt.enforce_type == PromptStudioRegistryKeys.RECORD
or prompt.enforce_type == PromptStudioRegistryKeys.LINE_ITEM
):
- for modifier_plugin in modifier_plugins:
- cls = modifier_plugin[ModifierConfig.METADATA][
- ModifierConfig.METADATA_SERVICE_CLASS
- ]
- output = cls.update(
+ payload_modifier_plugin = get_plugin("payload_modifier")
+ if payload_modifier_plugin:
+ modifier_service = payload_modifier_plugin["service_class"]()
+ output = modifier_service.update(
output=output,
tool_id=tool.tool_id,
prompt_id=prompt.prompt_id,
diff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.py
index a532c1453..ad8c94306 100644
--- a/backend/scheduler/tasks.py
+++ b/backend/scheduler/tasks.py
@@ -3,7 +3,6 @@ import logging
import traceback
from typing import Any
-from account_v2.subscription_loader import load_plugins, validate_etl_run
from celery import shared_task
from django_celery_beat.models import CrontabSchedule, PeriodicTask
from pipeline_v2.models import Pipeline
@@ -12,7 +11,6 @@ from utils.user_context import UserContext
from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper
logger = logging.getLogger(__name__)
-subscription_loader = load_plugins()
def create_or_update_periodic_task(
@@ -95,20 +93,25 @@ def execute_pipeline_task_v2(
f"Executing pipeline: {pipeline_id}, "
f"workflow: {workflow}, pipeline name: {pipeline_name}"
)
- if (
- subscription_loader
- and subscription_loader[0]
- and not validate_etl_run(organization_id)
- ):
- try:
- logger.info(
- f"Subscription expired for '{organization_id}', "
- f"disabling pipeline: {pipeline_id}"
- )
- disable_task(pipeline_id)
- except Exception as e:
- logger.warning(f"Failed to disable task: {pipeline_id}. Error: {e}")
- return
+ # Check subscription status before ETL run (cloud-specific)
+ try:
+ from pluggable_apps.subscription_v2.subscription_helper import (
+ SubscriptionHelper,
+ )
+
+ if not SubscriptionHelper.validate_etl_run(organization_id):
+ try:
+ logger.info(
+ f"Subscription expired for '{organization_id}', "
+ f"disabling pipeline: {pipeline_id}"
+ )
+ disable_task(pipeline_id)
+ except Exception as e:
+ logger.warning(f"Failed to disable task: {pipeline_id}. Error: {e}")
+ return
+ except ModuleNotFoundError:
+ # Subscription module not available (OSS deployment), skip validation
+ pass
PipelineProcessor.update_pipeline(pipeline_id, Pipeline.PipelineStatus.INPROGRESS)
# Mark the File in file history to avoid duplicate execution
# only for ETL and TASK execution
diff --git a/backend/utils/subscription_usage_decorator.py b/backend/utils/subscription_usage_decorator.py
new file mode 100644
index 000000000..5a84a2e87
--- /dev/null
+++ b/backend/utils/subscription_usage_decorator.py
@@ -0,0 +1,139 @@
+"""Decorator and utilities for automatic subscription usage tracking.
+
+This module provides a decorator to automatically handle subscription usage
+commit/discard operations, reducing boilerplate code and ensuring consistent
+behavior across the codebase.
+"""
+
+import functools
+import logging
+from collections.abc import Callable
+from datetime import datetime
+from typing import Any
+
+from plugins import get_plugin
+
+from utils.user_session import UserSessionUtils
+
+logger = logging.getLogger(__name__)
+
+
+def track_subscription_usage_if_available(
+ file_execution_id_param: str,
+ org_id_param: str = "org_id",
+ extract_from_request: bool = False,
+) -> Callable:
+ """Decorator to automatically track subscription usage.
+
+ Automatically commits subscription usage on successful function execution
+ and discards it on failure. This eliminates repetitive try/except blocks
+ for subscription tracking.
+
+ Args:
+ file_execution_id_param: Name of the parameter containing file_execution_id
+ org_id_param: Name of the parameter containing org_id (default: "org_id")
+ extract_from_request: If True, extracts org_id from Django request object
+ using UserSessionUtils.get_organization_id()
+
+ Returns:
+ Decorated function
+
+ Example:
+ @track_subscription_usage_if_available(file_execution_id_param="run_id", extract_from_request=True)
+ def index_document(self, request, pk=None):
+ # Your code here - subscription usage tracked automatically
+ return result
+
+ Note:
+ - Subscription tracking errors are logged but don't affect main function
+ - If identifiers are missing, tracking is silently skipped
+ - Works with both regular functions and Django view methods
+ """
+
+ def decorator(func: Callable) -> Callable:
+ @functools.wraps(func)
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
+ # Extract file_execution_id
+ file_execution_id = kwargs.get(file_execution_id_param)
+
+ # Extract org_id
+ if extract_from_request:
+ # Handle both function(request) and method(self, request) patterns
+ request = kwargs.get("request")
+ if not request and len(args) >= 2:
+ # Assume args[0] is self, args[1] is request for methods
+ request = args[1]
+ elif not request and len(args) >= 1 and "self" not in kwargs:
+ # Assume args[0] is request for functions
+ request = args[0]
+
+ if request:
+ org_id = UserSessionUtils.get_organization_id(request)
+ else:
+ org_id = None
+ logger.warning(
+ f"Could not extract request object in {func.__name__}, "
+ "subscription tracking skipped"
+ )
+ else:
+ org_id = kwargs.get(org_id_param)
+
+ # Skip tracking if identifiers are missing
+ if not file_execution_id or not org_id:
+ logger.debug(
+ f"Skipping subscription tracking in {func.__name__}: "
+ f"file_execution_id={file_execution_id}, org_id={org_id}"
+ )
+ return func(*args, **kwargs)
+
+ try:
+ # Execute main function
+ result = func(*args, **kwargs)
+
+ # Commit subscription usage on success
+ subscription_usage_plugin = get_plugin("subscription_usage")
+ if subscription_usage_plugin:
+ try:
+ service = subscription_usage_plugin["service_class"]()
+ service.commit_subscription_usage(
+ org_id=org_id,
+ date_key=datetime.now().date(),
+ file_execution_id=file_execution_id,
+ )
+ logger.info(
+ f"Committed subscription usage for file_execution_id {file_execution_id}"
+ )
+ except Exception as commit_error:
+ logger.error(
+ f"Error committing subscription usage for "
+ f"file_execution_id {file_execution_id}: {commit_error}"
+ )
+
+ return result
+
+ except Exception:
+ # Discard subscription usage on failure
+ subscription_usage_plugin = get_plugin("subscription_usage")
+ if subscription_usage_plugin:
+ try:
+ service = subscription_usage_plugin["service_class"]()
+ service.discard_subscription_usage(
+ org_id=org_id,
+ date_key=datetime.now().date(),
+ file_execution_id=file_execution_id,
+ )
+ logger.info(
+ f"Discarded subscription usage for file_execution_id {file_execution_id}"
+ )
+ except Exception as discard_error:
+ logger.error(
+ f"Error discarding subscription usage for "
+ f"file_execution_id {file_execution_id}: {discard_error}"
+ )
+
+ # Re-raise original exception
+ raise
+
+ return wrapper
+
+ return decorator
diff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
index c8b8ca0a0..a9b3ab871 100644
--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py
+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py
@@ -369,6 +369,25 @@ class FileExecutionTasks:
successful_files=total_successful,
failed_files=total_failed,
)
+
+ # TODO: Remove related code when v1 workers are deprecated and removed
+ from plugins import get_plugin
+
+ subscription_usage_plugin = get_plugin("subscription_usage")
+ if subscription_usage_plugin:
+ try:
+ service = subscription_usage_plugin["service_class"]()
+ service.handle_workflow_execution_completion(
+ organization_id=organization_id,
+ execution_id=execution_id,
+ final_status=final_status,
+ total_successful=total_successful,
+ )
+ except Exception as e:
+ logger.error(
+ f"Error in subscription usage plugin completion handler: {e}"
+ )
+
logger.info(
f"Execution completed for execution id: '{execution_id}' "
f"with status: '{final_status}' "
diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.py
index 3304952de..7bf448f04 100644
--- a/backend/workflow_manager/workflow_v2/views.py
+++ b/backend/workflow_manager/workflow_v2/views.py
@@ -11,6 +11,7 @@ from django.views.decorators.csrf import csrf_exempt
from permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg
from pipeline_v2.models import Pipeline
from pipeline_v2.pipeline_processor import PipelineProcessor
+from plugins import get_plugin
from rest_framework import serializers, status, viewsets
from rest_framework.decorators import action, api_view
from rest_framework.request import Request
@@ -20,17 +21,6 @@ from rest_framework.views import APIView
from utils.filtering import FilterHelper
from utils.organization_utils import filter_queryset_by_organization, resolve_organization
-try:
- from plugins.notification.constants import ResourceType
- from plugins.notification.sharing_notification import SharingNotificationService
-
- NOTIFICATION_PLUGIN_AVAILABLE = True
- sharing_notification_service = SharingNotificationService()
-except ImportError:
- NOTIFICATION_PLUGIN_AVAILABLE = False
- sharing_notification_service = None
-
-
from backend.constants import RequestKey
from unstract.core.data_models import FileHistoryCreateRequest
from workflow_manager.endpoint_v2.destination import DestinationConnector
@@ -68,6 +58,14 @@ from workflow_manager.workflow_v2.workflow_helper import (
WorkflowSchemaHelper,
)
+# Check if notification plugin is available
+notification_plugin = get_plugin("notification")
+NOTIFICATION_PLUGIN_AVAILABLE = notification_plugin is not None
+
+# Import constants from notification plugin if available
+if NOTIFICATION_PLUGIN_AVAILABLE:
+ from plugins.notification.constants import ResourceType
+
logger = logging.getLogger(__name__)
@@ -169,8 +167,10 @@ class WorkflowViewSet(viewsets.ModelViewSet):
newly_shared_users = new_shared_users - current_shared_users
if newly_shared_users:
- # Send notification to newly shared users
- sharing_notification_service.send_sharing_notification(
+ # Get notification service from plugin and send notification
+ service_class = notification_plugin["service_class"]
+ notification_service = service_class()
+ notification_service.send_sharing_notification(
resource_type=ResourceType.WORKFLOW.value,
resource_name=workflow.workflow_name,
resource_id=str(workflow.id),
@@ -858,11 +858,11 @@ def file_history_batch_lookup_internal(request):
)
logger.info(
- f"DEBUG: FileHistoryBatch - File {i+1}: {file_data.get('file_path', 'NO_PATH')}"
+ f"DEBUG: FileHistoryBatch - File {i + 1}: {file_data.get('file_path', 'NO_PATH')}"
f" (identifier: {identifier})"
)
logger.info(
- f"DEBUG: FileHistoryBatch - File {i+1} data: cache_key={file_data.get('cache_key')}, "
+ f"DEBUG: FileHistoryBatch - File {i + 1} data: cache_key={file_data.get('cache_key')}, "
f"provider_file_uuid={file_data.get('provider_file_uuid')}, "
f"file_path={file_data.get('file_path')}"
)
@@ -879,7 +879,7 @@ def file_history_batch_lookup_internal(request):
# Legacy mapping (may have collisions)
file_identifiers[file_data["cache_key"]] = identifier
logger.info(
- f"DEBUG: FileHistoryBatch - File {i+1}: Using cache_key={file_data['cache_key']}, "
+ f"DEBUG: FileHistoryBatch - File {i + 1}: Using cache_key={file_data['cache_key']}, "
f"mapped to identifier={identifier}"
)
elif file_data.get("provider_file_uuid"):
@@ -891,12 +891,12 @@ def file_history_batch_lookup_internal(request):
# Legacy mapping (may have collisions)
file_identifiers[file_data["provider_file_uuid"]] = identifier
logger.info(
- f"DEBUG: FileHistoryBatch - File {i+1}: Using provider_file_uuid={file_data['provider_file_uuid']}, "
+ f"DEBUG: FileHistoryBatch - File {i + 1}: Using provider_file_uuid={file_data['provider_file_uuid']}, "
f"mapped to identifier={identifier}"
)
else:
logger.warning(
- f"DEBUG: FileHistoryBatch - File {i+1}: No cache_key or provider_file_uuid!"
+ f"DEBUG: FileHistoryBatch - File {i + 1}: No cache_key or provider_file_uuid!"
)
continue
@@ -910,18 +910,18 @@ def file_history_batch_lookup_internal(request):
# Combine: match either exact path OR legacy null path
filters &= cache_key_filters & (path_filters | fallback_filters)
logger.info(
- f"DEBUG: FileHistoryBatch - File {i+1}: Added file_path constraint={file_data['file_path']} "
+ f"DEBUG: FileHistoryBatch - File {i + 1}: Added file_path constraint={file_data['file_path']} "
f"with legacy fallback (file_path__isnull=True)"
)
else:
# No file_path provided, only search legacy records
filters &= cache_key_filters & Q(file_path__isnull=True)
logger.info(
- f"DEBUG: FileHistoryBatch - File {i+1}: No file_path provided, using legacy fallback only"
+ f"DEBUG: FileHistoryBatch - File {i + 1}: No file_path provided, using legacy fallback only"
)
logger.info(
- f"DEBUG: FileHistoryBatch - File {i+1}: Final query filters: {filters}"
+ f"DEBUG: FileHistoryBatch - File {i + 1}: Final query filters: {filters}"
)
queries.append(filters)
@@ -955,7 +955,7 @@ def file_history_batch_lookup_internal(request):
)
for i, fh in enumerate(file_histories):
logger.info(
- f"DEBUG: FileHistoryBatch - DB Record {i+1}: "
+ f"DEBUG: FileHistoryBatch - DB Record {i + 1}: "
f"provider_file_uuid={fh.provider_file_uuid}, "
f"cache_key={fh.cache_key}, "
f"file_path={fh.file_path}, "
@@ -988,7 +988,7 @@ def file_history_batch_lookup_internal(request):
for i, fh in enumerate(file_histories):
logger.info(
- f"DEBUG: FileHistoryBatch - Processing DB record {i+1}: "
+ f"DEBUG: FileHistoryBatch - Processing DB record {i + 1}: "
f"cache_key={fh.cache_key}, provider_file_uuid={fh.provider_file_uuid}, file_path={fh.file_path}"
)
@@ -1000,7 +1000,7 @@ def file_history_batch_lookup_internal(request):
if composite_key in composite_file_map:
matched_identifiers.append(composite_file_map[composite_key])
logger.info(
- f"DEBUG: FileHistoryBatch - DB record {i+1}: Matched by composite cache_key={composite_key} "
+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: Matched by composite cache_key={composite_key} "
f"-> identifier={composite_file_map[composite_key]}"
)
@@ -1009,7 +1009,7 @@ def file_history_batch_lookup_internal(request):
if composite_key in composite_file_map:
matched_identifiers.append(composite_file_map[composite_key])
logger.info(
- f"DEBUG: FileHistoryBatch - DB record {i+1}: Matched by composite uuid={composite_key} "
+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: Matched by composite uuid={composite_key} "
f"-> identifier={composite_file_map[composite_key]}"
)
@@ -1018,13 +1018,13 @@ def file_history_batch_lookup_internal(request):
if fh.cache_key and fh.cache_key in file_identifiers:
matched_identifiers.append(file_identifiers[fh.cache_key])
logger.info(
- f"DEBUG: FileHistoryBatch - DB record {i+1}: Matched by legacy cache_key={fh.cache_key} "
+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: Matched by legacy cache_key={fh.cache_key} "
f"-> identifier={file_identifiers[fh.cache_key]}"
)
elif fh.provider_file_uuid and fh.provider_file_uuid in file_identifiers:
matched_identifiers.append(file_identifiers[fh.provider_file_uuid])
logger.info(
- f"DEBUG: FileHistoryBatch - DB record {i+1}: Matched by legacy provider_file_uuid={fh.provider_file_uuid} "
+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: Matched by legacy provider_file_uuid={fh.provider_file_uuid} "
f"-> identifier={file_identifiers[fh.provider_file_uuid]}"
)
@@ -1037,7 +1037,7 @@ def file_history_batch_lookup_internal(request):
potential_matches.append((req_identifier, req_data))
logger.info(
- f"DEBUG: FileHistoryBatch - DB record {i+1}: Found {len(potential_matches)} potential UUID matches"
+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: Found {len(potential_matches)} potential UUID matches"
)
# Try to match by file path
@@ -1046,7 +1046,7 @@ def file_history_batch_lookup_internal(request):
if fh.file_path == req_path:
matched_identifiers.append(req_identifier)
logger.info(
- f"DEBUG: FileHistoryBatch - DB record {i+1}: Matched by manual path comparison "
+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: Matched by manual path comparison "
f"db_path={fh.file_path} == req_path={req_path} -> identifier={req_identifier}"
)
break
@@ -1054,7 +1054,7 @@ def file_history_batch_lookup_internal(request):
# If still no exact path match, but we have UUID matches, log for fallback handling
if not matched_identifiers and potential_matches:
logger.warning(
- f"DEBUG: FileHistoryBatch - DB record {i+1}: UUID collision detected! "
+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: UUID collision detected! "
f"DB path={fh.file_path} doesn't match any request paths: "
f"{[req_data.get('file_path') for _, req_data in potential_matches]}"
)
@@ -1062,7 +1062,7 @@ def file_history_batch_lookup_internal(request):
# Process all matched identifiers
if not matched_identifiers:
logger.warning(
- f"DEBUG: FileHistoryBatch - DB record {i+1}: NO MATCH FOUND! "
+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: NO MATCH FOUND! "
f"cache_key={fh.cache_key}, provider_file_uuid={fh.provider_file_uuid}, file_path={fh.file_path}"
)
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index e106dde2d..99a4f55fe 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -236,6 +236,24 @@ class WorkflowHelper:
status=ExecutionStatus.ERROR,
error=f"Error while processing files: {str(e)}",
)
+
+ # TODO: Remove related code when v1 workers are deprecated and removed
+ from plugins import get_plugin
+
+ organization_id = workflow_execution.workflow.organization.organization_id
+ subscription_usage_plugin = get_plugin("subscription_usage")
+ if subscription_usage_plugin:
+ try:
+ service = subscription_usage_plugin["service_class"]()
+ service.handle_workflow_execution_failure(
+ organization_id=organization_id,
+ execution_id=str(workflow_execution.id),
+ )
+ except Exception as e:
+ logger.error(
+ f"Error in subscription usage plugin failure handler: {e}"
+ )
+
logger.error(
f"Execution {workflow_execution.id} failed: {str(e)}", exc_info=True
)
diff --git a/platform-service/.gitignore b/platform-service/.gitignore
index 4b3ab3003..3b35241ca 100644
--- a/platform-service/.gitignore
+++ b/platform-service/.gitignore
@@ -1,162 +1,2 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-# Usually these files are written by a python script from a template
-# before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-# For a library or package, you might want to ignore these files since the code is
-# intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-# However, in case of collaboration, if having platform-specific dependencies or dependencies
-# having no cross-platform support, pipenv may install dependencies that don't work, or not
-# install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-# This is especially recommended for binary packages to ensure reproducibility, and is more
-# commonly ignored for libraries.
-# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-# in version control.
-# https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-.pdm-python
-.pdm-build/
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-# and can be added to the global gitignore or merged into this file. For a more nuclear
-# option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+# Plugins
+src/unstract/platform_service/plugins
diff --git a/platform-service/src/unstract/platform_service/config.py b/platform-service/src/unstract/platform_service/config.py
index f6ae73583..08fff4130 100644
--- a/platform-service/src/unstract/platform_service/config.py
+++ b/platform-service/src/unstract/platform_service/config.py
@@ -1,10 +1,15 @@
import logging
import os
+from pathlib import Path
from dotenv import load_dotenv
from flask import Flask
-from unstract.core.flask import register_error_handlers, register_request_id_middleware
+from unstract.core.flask import (
+ PluginManager,
+ register_error_handlers,
+ register_request_id_middleware,
+)
from unstract.core.flask.logging import setup_logging
from unstract.platform_service.constants import LogLevel
from unstract.platform_service.controller import api
@@ -41,4 +46,10 @@ def create_app() -> Flask:
options=f"-c application_name={Env.APPLICATION_NAME}",
)
+ # Load plugins (cloud plugins will be overlaid during build)
+ plugins_dir = Path(__file__).parent / "plugins"
+ plugins_pkg = "unstract.platform_service.plugins"
+ manager = PluginManager(app, plugins_dir, plugins_pkg)
+ manager.load_plugins()
+
return app
diff --git a/platform-service/src/unstract/platform_service/controller/platform.py b/platform-service/src/unstract/platform_service/controller/platform.py
index 049989997..26a1e7493 100644
--- a/platform-service/src/unstract/platform_service/controller/platform.py
+++ b/platform-service/src/unstract/platform_service/controller/platform.py
@@ -8,6 +8,7 @@ from cryptography.fernet import Fernet, InvalidToken
from flask import Blueprint, Request, jsonify, make_response, request
from flask import current_app as app
+from unstract.core.flask import PluginManager
from unstract.core.flask.exceptions import APIError
from unstract.platform_service.constants import DBTable
from unstract.platform_service.env import Env
@@ -163,79 +164,19 @@ def page_usage() -> Any:
result["status"] = "OK"
result["unique_id"] = usage_id
- # Check if the subscription_usage table exists
- check_table_query = f"""
- SELECT EXISTS (
- SELECT FROM information_schema.tables
- WHERE table_schema = '{Env.DB_SCHEMA}'
- AND table_name = '{DBTable.SUBSCRIPTION_USAGE}'
- );
- """
- app.logger.info("Checking if subscription_usage table exists")
-
- table_exists = db.execute_sql(check_table_query).fetchone()[0]
-
- if table_exists:
- app.logger.info("subscription_usage table exists")
-
- # Query to get subscription_id and stripe_subscription_id
- # from Subscription table.
- subscription_query = f"""
- SELECT id, stripe_subscription_id
- FROM \"{Env.DB_SCHEMA}\".{DBTable.SUBSCRIPTION}
- WHERE organization_id = %s AND is_active = TRUE;
- """
- subscription_query_params = (org_id,)
- app.logger.info(
- "Executing subscription query for organization %s", org_id
- )
-
- subscription_result = db.execute_sql(
- subscription_query, subscription_query_params
- ).fetchone()
-
- if subscription_result:
- subscription_id, stripe_subscription_id = subscription_result
- app.logger.info(
- "Found subscription: id=%s, stripe_subscription_id=%s",
- subscription_id,
- stripe_subscription_id,
+ # Cloud-only: Handle subscription usage via plugin
+ usage_plugin = PluginManager().get_plugin("subscription_usage")
+ if usage_plugin:
+ try:
+ handler = usage_plugin["entrypoint_cls"]()
+ handler.handle_subscription_usage(
+ org_id=org_id,
+ page_count=page_count,
+ run_id=run_id,
+ current_time=current_time,
)
-
- stripe_subscription_id = (
- stripe_subscription_id if stripe_subscription_id else "trial"
- )
-
- # Insert or update data in the subscription_usage table
- subscription_usage_query = f"""
- INSERT INTO
- \"{Env.DB_SCHEMA}\".{DBTable.SUBSCRIPTION_USAGE} (
- subscription_id, stripe_subscription_id,
- organization_id, pages_processed, record_date,
- created_at, updated_at
- ) VALUES (%s, %s, %s, %s, %s, %s, %s)
- ON CONFLICT (
- subscription_id, stripe_subscription_id, record_date
- ) DO UPDATE
- SET pages_processed =
- {DBTable.SUBSCRIPTION_USAGE}.pages_processed
- + EXCLUDED.pages_processed;
- """
- subscription_usage_params = (
- subscription_id,
- stripe_subscription_id,
- org_id,
- page_count,
- current_time.date(),
- current_time,
- current_time,
- )
- app.logger.info("Executing subscription usage insert/update query.")
-
- db.execute_sql(subscription_usage_query, subscription_usage_params)
- app.logger.info("Subscription usage updated for %s", org_id)
- else:
- app.logger.warning("No active subscription found for %s", org_id)
+ except Exception as e:
+ app.logger.exception(f"Error from subscription usage plugin: {e}")
return make_response(result, 200)
except Exception as e:
diff --git a/prompt-service/.gitignore b/prompt-service/.gitignore
index df775c6bf..ae2f699ad 100644
--- a/prompt-service/.gitignore
+++ b/prompt-service/.gitignore
@@ -1,164 +1,2 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-# Usually these files are written by a python script from a template
-# before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-# For a library or package, you might want to ignore these files since the code is
-# intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-# However, in case of collaboration, if having platform-specific dependencies or dependencies
-# having no cross-platform support, pipenv may install dependencies that don't work, or not
-# install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-# This is especially recommended for binary packages to ensure reproducibility, and is more
-# commonly ignored for libraries.
-# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-# in version control.
-# https://pdm-project.org/#use-with-ide
-.pdm.toml
-.pdm-python
-.pdm-build/
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-# and can be added to the global gitignore or merged into this file. For a more nuclear
-# option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-
-plugins
+# Plugins
+src/unstract/prompt_service/plugins
diff --git a/prompt-service/src/unstract/prompt_service/config.py b/prompt-service/src/unstract/prompt_service/config.py
index 7d72d92ff..4a43da7c0 100644
--- a/prompt-service/src/unstract/prompt_service/config.py
+++ b/prompt-service/src/unstract/prompt_service/config.py
@@ -1,15 +1,19 @@
import logging
import warnings
from os import environ as env
+from pathlib import Path
from dotenv import load_dotenv
from flask import Flask
-from unstract.core.flask import register_error_handlers, register_request_id_middleware
+from unstract.core.flask import (
+ PluginManager,
+ register_error_handlers,
+ register_request_id_middleware,
+)
from unstract.core.flask.logging import setup_logging
from unstract.flags.feature_flag import check_feature_flag_status
from unstract.prompt_service.controllers import api
-from unstract.prompt_service.helpers.plugin import plugin_loader
if check_feature_flag_status("sdk1"):
from unstract.sdk1.constants import LogLevel
@@ -42,7 +46,11 @@ def create_app() -> Flask:
app.logger.info("Initializing Flask application...")
# Load plugins
- plugin_loader(app)
+ plugins_dir = Path(__file__).parent / "plugins"
+ plugins_pkg = "unstract.prompt_service.plugins"
+ manager = PluginManager(app, plugins_dir, plugins_pkg)
+ manager.load_plugins()
+
register_request_id_middleware(app)
register_error_handlers(app)
app.register_blueprint(api)
diff --git a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py
index 974ba8ebf..4ef219b22 100644
--- a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py
+++ b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py
@@ -5,13 +5,13 @@ from typing import Any
from flask import Blueprint, request
from flask import current_app as app
+from unstract.core.flask import PluginManager
from unstract.core.flask.exceptions import APIError
from unstract.flags.feature_flag import check_feature_flag_status
from unstract.prompt_service.constants import PromptServiceConstants as PSKeys
from unstract.prompt_service.constants import RetrievalStrategy, RunLevel
from unstract.prompt_service.exceptions import BadRequest
from unstract.prompt_service.helpers.auth import AuthHelper
-from unstract.prompt_service.helpers.plugin import PluginManager
from unstract.prompt_service.helpers.prompt_ide_base_tool import PromptServiceBaseTool
from unstract.prompt_service.helpers.usage import UsageHelper
from unstract.prompt_service.services.answer_prompt import AnswerPromptService
diff --git a/prompt-service/src/unstract/prompt_service/helpers/plugin.py b/prompt-service/src/unstract/prompt_service/helpers/plugin.py
deleted file mode 100644
index 8cd6e9aba..000000000
--- a/prompt-service/src/unstract/prompt_service/helpers/plugin.py
+++ /dev/null
@@ -1,78 +0,0 @@
-import importlib
-import os
-from pathlib import Path
-from typing import Any
-
-from flask import Flask
-
-
-class PluginManager:
- _instance = None
-
- def __new__(cls, app: Flask | None = None) -> "PluginManager":
- if cls._instance is None:
- # Only create the instance if it doesn't already exist
- cls._instance = super().__new__(cls)
- cls._instance.plugins = {}
- cls._instance.plugins_dir = Path(__file__).parent.parent / "plugins"
- cls._instance.plugins_pkg = "unstract.prompt_service.plugins"
- # Always update `app` if provided
- if app:
- cls._instance.app = app
- return cls._instance
-
- def load_plugins(self) -> None:
- """Loads plugins found in the plugins root dir."""
- if not self.plugins_dir.exists():
- self.app.logger.warning(
- f"Plugins directory not found: {self.plugins_dir}. Skipping."
- )
- return
-
- self.app.logger.info(f"Loading plugins from: {self.plugins_dir}")
-
- for pkg in os.listdir(os.fspath(self.plugins_dir)):
- if pkg.endswith(".so"):
- pkg = pkg.split(".")[0]
- pkg_anchor = f"{self.plugins_pkg}.{pkg}.src"
- try:
- module = importlib.import_module(pkg_anchor)
- except ImportError as e:
- self.app.logger.error(f"Failed to load plugin ({pkg}): {str(e)}")
- continue
-
- metadata = getattr(module, "metadata", None)
- if not metadata:
- self.app.logger.warning(f"Skipping plugin ({pkg}): No metadata found.")
- continue
-
- if metadata.get("disable", False):
- self.app.logger.info(
- f"Skipping disabled plugin: {pkg}"
- f" v{metadata.get('version', 'unknown')}"
- )
- continue
-
- try:
- self.plugins[metadata["name"]] = {
- "version": metadata["version"],
- "entrypoint_cls": metadata["entrypoint_cls"],
- "exception_cls": metadata["exception_cls"],
- }
- if blueprint := metadata.get("blueprint"):
- self.app.register_blueprint(blueprint)
- self.app.logger.info(f"✔ Loaded plugin: {pkg} v{metadata['version']}")
- except KeyError as e:
- self.app.logger.error(f"Invalid metadata for plugin '{pkg}': {str(e)}")
-
- if not self.plugins:
- self.app.logger.warning("⚠ No plugins loaded.")
-
- def get_plugin(self, name: str) -> dict[str, Any]:
- """Get the plugin metadata by name."""
- return self.plugins.get(name, {})
-
-
-def plugin_loader(app: Flask) -> None:
- manager = PluginManager(app)
- manager.load_plugins()
diff --git a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py
index 2200c125a..0a5419135 100644
--- a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py
+++ b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py
@@ -6,12 +6,12 @@ from urllib.parse import urlparse
from flask import current_app as app
+from unstract.core.flask import PluginManager
from unstract.core.flask.exceptions import APIError
from unstract.flags.feature_flag import check_feature_flag_status
from unstract.prompt_service.constants import ExecutionSource, FileStorageKeys, RunLevel
from unstract.prompt_service.constants import PromptServiceConstants as PSKeys
from unstract.prompt_service.exceptions import RateLimitError
-from unstract.prompt_service.helpers.plugin import PluginManager
from unstract.prompt_service.helpers.postprocessor import postprocess_data
from unstract.prompt_service.utils.env_loader import get_env_or_die
from unstract.prompt_service.utils.json_repair_helper import (
@@ -222,6 +222,12 @@ class AnswerPromptService:
storage_type=StorageType.SHARED_TEMPORARY,
env_name=FileStorageKeys.TEMPORARY_REMOTE_STORAGE,
)
+ logger.info(
+ f"Initializing highlight plugin with: "
+ f"file_path={file_path}, "
+ f"execution_source={execution_source}, "
+ f"fs_instance={fs_instance}"
+ )
highlight_data = highlight_data_plugin["entrypoint_cls"](
file_path=file_path,
fs_instance=fs_instance,
diff --git a/unstract/core/src/unstract/core/cache/redis_client.py b/unstract/core/src/unstract/core/cache/redis_client.py
index 5480aca8f..12ababdba 100644
--- a/unstract/core/src/unstract/core/cache/redis_client.py
+++ b/unstract/core/src/unstract/core/cache/redis_client.py
@@ -252,6 +252,7 @@ class RedisClient:
REDIS_HOST: Redis host (default: localhost)
REDIS_PORT: Redis port (default: 6379)
REDIS_USER: Redis username (optional)
+ REDIS_USERNAME: Redis username (optional, alternative to REDIS_USER)
REDIS_PASSWORD: Redis password (optional)
REDIS_DB: Redis database number (default: 0)
@@ -263,7 +264,7 @@ class RedisClient:
return cls(
host=os.getenv("REDIS_HOST", "localhost"),
port=int(os.getenv("REDIS_PORT", "6379")),
- username=os.getenv("REDIS_USER"),
+ username=os.getenv("REDIS_USER") or os.getenv("REDIS_USERNAME"),
password=os.getenv("REDIS_PASSWORD"),
db=int(os.getenv("REDIS_DB", "0")),
)
diff --git a/unstract/core/src/unstract/core/django/__init__.py b/unstract/core/src/unstract/core/django/__init__.py
new file mode 100644
index 000000000..60ee5f46c
--- /dev/null
+++ b/unstract/core/src/unstract/core/django/__init__.py
@@ -0,0 +1 @@
+from .plugin import DjangoPluginManager, PluginManager, plugin_loader # noqa: F401
diff --git a/unstract/core/src/unstract/core/django/plugin.py b/unstract/core/src/unstract/core/django/plugin.py
new file mode 100644
index 000000000..6449b0037
--- /dev/null
+++ b/unstract/core/src/unstract/core/django/plugin.py
@@ -0,0 +1,163 @@
+"""Django-specific plugin manager wrapper.
+
+This module provides Django integration for the generic PluginManager,
+handling Django-specific features like app registry integration and Django
+logging.
+"""
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from unstract.core.plugins import PluginManager as GenericPluginManager
+
+
+class DjangoPluginManager:
+ """Django-specific plugin manager wrapper.
+
+ Wraps the generic PluginManager with Django-specific functionality like
+ Django app integration and logging.
+ """
+
+ _instance = None
+
+ def __new__(
+ cls,
+ plugins_dir: Path | str,
+ plugins_pkg: str,
+ logger: logging.Logger | None = None,
+ ) -> "DjangoPluginManager":
+ """Create or return the singleton DjangoPluginManager instance.
+
+ Args:
+ plugins_dir: Directory containing plugins
+ plugins_pkg: Python package path for plugins (e.g., 'myapp.plugins')
+ logger: Logger instance (defaults to module logger)
+
+ Returns:
+ DjangoPluginManager singleton instance
+ """
+ if cls._instance is None:
+ cls._instance = super().__new__(cls)
+ cls._instance._initialized = False
+
+ # Initialize or update plugin manager if parameters change
+ if plugins_dir and plugins_pkg:
+ plugins_dir_path = (
+ Path(plugins_dir) if isinstance(plugins_dir, str) else plugins_dir
+ )
+
+ if (
+ not cls._instance._initialized
+ or cls._instance._plugins_dir != plugins_dir_path
+ or cls._instance._plugins_pkg != plugins_pkg
+ ):
+ cls._instance._plugins_dir = plugins_dir_path
+ cls._instance._plugins_pkg = plugins_pkg
+ cls._instance._logger = logger or logging.getLogger(__name__)
+ cls._instance._init_manager()
+
+ return cls._instance
+
+ def _init_manager(self) -> None:
+ """Initialize the generic plugin manager."""
+ self._manager = GenericPluginManager(
+ plugins_dir=self._plugins_dir,
+ plugins_pkg=self._plugins_pkg,
+ logger=self._logger,
+ use_singleton=True,
+ registration_callback=None, # Django doesn't need special registration
+ )
+ self._initialized = True
+
+ def load_plugins(self) -> None:
+ """Load plugins using the generic manager."""
+ if not self._initialized:
+ raise RuntimeError(
+ "DjangoPluginManager not initialized. "
+ "Call with plugins_dir and plugins_pkg first."
+ )
+ self._manager.load_plugins()
+
+ def get_plugin(self, name: str) -> dict[str, Any]:
+ """Get plugin metadata by name.
+
+ Args:
+ name: Plugin name to retrieve
+
+ Returns:
+ Dictionary containing plugin metadata
+ """
+ if not self._initialized:
+ return {}
+ return self._manager.get_plugin(name)
+
+ def has_plugin(self, name: str) -> bool:
+ """Check if a plugin is loaded.
+
+ Args:
+ name: Plugin name to check
+
+ Returns:
+ bool: True if plugin exists
+ """
+ if not self._initialized:
+ return False
+ return self._manager.has_plugin(name)
+
+ def get_all_plugins(self) -> dict[str, dict[str, Any]]:
+ """Get all loaded plugins.
+
+ Returns:
+ Dictionary mapping plugin names to their metadata
+ """
+ if not self._initialized:
+ return {}
+ return self._manager.get_all_plugins()
+
+ @property
+ def plugins(self) -> dict[str, dict[str, Any]]:
+ """Get all loaded plugins."""
+ return self.get_all_plugins()
+
+
+# Maintain backward compatibility with common class name
+PluginManager = DjangoPluginManager
+
+
+def plugin_loader(
+ plugins_dir: Path | str,
+ plugins_pkg: str,
+ logger: logging.Logger | None = None,
+) -> DjangoPluginManager:
+ """Load plugins for a Django application.
+
+ Convenience function to create a DjangoPluginManager instance and load plugins.
+
+ Args:
+ plugins_dir: Directory containing plugins
+ plugins_pkg: Python package path for plugins (e.g., 'backend.plugins')
+ logger: Logger instance (optional)
+
+ Returns:
+ DjangoPluginManager: The plugin manager instance
+
+ Example:
+ # In your Django app initialization (e.g., apps.py or __init__.py):
+ from pathlib import Path
+ from unstract.core.django import plugin_loader
+
+ # Load plugins from backend/plugins directory
+ plugins_dir = Path(__file__).parent / 'plugins'
+ manager = plugin_loader(plugins_dir, 'backend.plugins')
+ manager.load_plugins()
+
+ # Later, check for plugins:
+ if manager.has_plugin('subscription_usage'):
+ plugin = manager.get_plugin('subscription_usage')
+ service = plugin['service_class']()
+ service.commit_usage(...)
+ """
+ manager = DjangoPluginManager(plugins_dir, plugins_pkg, logger)
+ manager.load_plugins()
+ return manager
diff --git a/unstract/core/src/unstract/core/flask/__init__.py b/unstract/core/src/unstract/core/flask/__init__.py
index 94eae362e..d03add0f4 100644
--- a/unstract/core/src/unstract/core/flask/__init__.py
+++ b/unstract/core/src/unstract/core/flask/__init__.py
@@ -1,2 +1,3 @@
from .exceptions import register_error_handlers # noqa: F401
from .middleware import register_request_id_middleware # noqa: F401
+from .plugin import PluginManager, plugin_loader # noqa: F401
diff --git a/unstract/core/src/unstract/core/flask/plugin.py b/unstract/core/src/unstract/core/flask/plugin.py
new file mode 100644
index 000000000..51e263927
--- /dev/null
+++ b/unstract/core/src/unstract/core/flask/plugin.py
@@ -0,0 +1,175 @@
+"""Flask-specific plugin manager wrapper.
+
+This module provides Flask integration for the generic PluginManager,
+handling Flask-specific features like blueprint registration and app.logger
+integration.
+"""
+
+import threading
+from pathlib import Path
+from typing import Any
+
+from flask import Flask
+
+from unstract.core.plugins import PluginManager as GenericPluginManager
+
+
+class FlaskPluginManager:
+ """Flask-specific plugin manager wrapper.
+
+ Wraps the generic PluginManager with Flask-specific functionality like
+ blueprint registration and Flask app integration.
+ """
+
+ _instance = None
+ _lock = threading.Lock()
+
+ def __new__(
+ cls,
+ app: Flask | None = None,
+ plugins_dir: Path | None = None,
+ plugins_pkg: str | None = None,
+ ) -> "FlaskPluginManager":
+ """Create or return the singleton FlaskPluginManager instance.
+
+ Args:
+ app: Flask application instance
+ plugins_dir: Directory containing plugins
+ plugins_pkg: Python package path for plugins
+
+ Returns:
+ FlaskPluginManager singleton instance
+ """
+ with cls._lock:
+ # Check inside lock to handle race condition
+ if cls._instance is None:
+ cls._instance = super().__new__(cls)
+ cls._instance._initialized = False
+
+ # Update app if provided
+ if app:
+ cls._instance.app = app
+
+ # Initialize or update plugin manager if parameters change
+ if (
+ plugins_dir
+ and plugins_pkg
+ and hasattr(cls._instance, "app")
+ and (
+ not cls._instance._initialized
+ or cls._instance._plugins_dir != plugins_dir
+ or cls._instance._plugins_pkg != plugins_pkg
+ )
+ ):
+ cls._instance._plugins_dir = plugins_dir
+ cls._instance._plugins_pkg = plugins_pkg
+ cls._instance._init_manager()
+
+ return cls._instance
+
+ def _init_manager(self) -> None:
+ """Initialize the generic plugin manager with Flask-specific callback."""
+
+ def flask_registration_callback(plugin_data: dict[str, Any]) -> None:
+ """Flask-specific registration callback for blueprints."""
+ metadata = plugin_data.get("metadata", {})
+ if blueprint := metadata.get("blueprint"):
+ self.app.register_blueprint(blueprint)
+ self.app.logger.debug(
+ f"Registered blueprint for plugin: {metadata.get('name', 'unknown')}"
+ )
+
+ self._manager = GenericPluginManager(
+ plugins_dir=self._plugins_dir,
+ plugins_pkg=self._plugins_pkg,
+ logger=self.app.logger,
+ use_singleton=True,
+ registration_callback=flask_registration_callback,
+ )
+ self._initialized = True
+
+ def load_plugins(self) -> None:
+ """Load plugins using the generic manager."""
+ if not self._initialized:
+ raise RuntimeError(
+ "FlaskPluginManager not initialized. "
+ "Call with app, plugins_dir, and plugins_pkg first."
+ )
+ self._manager.load_plugins()
+
+ def get_plugin(self, name: str) -> dict[str, Any]:
+ """Get plugin metadata by name.
+
+ Args:
+ name: Plugin name to retrieve
+
+ Returns:
+ Dictionary containing plugin metadata
+ """
+ if not self._initialized:
+ return {}
+ return self._manager.get_plugin(name)
+
+ def has_plugin(self, name: str) -> bool:
+ """Check if a plugin is loaded.
+
+ Args:
+ name: Plugin name to check
+
+ Returns:
+ bool: True if plugin exists
+ """
+ if not self._initialized:
+ return False
+ return self._manager.has_plugin(name)
+
+ @property
+ def plugins(self) -> dict[str, dict[str, Any]]:
+ """Get all loaded plugins."""
+ if not self._initialized:
+ return {}
+ return self._manager.get_all_plugins()
+
+
+# Maintain backward compatibility with old class name
+PluginManager = FlaskPluginManager
+
+
+def plugin_loader(
+ app: Flask,
+ plugins_dir: Path | None = None,
+ plugins_pkg: str | None = None,
+) -> None:
+ """Load plugins for a Flask application.
+
+ Convenience function to create a PluginManager instance and load plugins.
+
+ Args:
+ app: Flask application instance
+ plugins_dir: Directory containing plugins (required)
+ plugins_pkg: Python package path for plugins (required)
+
+ Raises:
+ ValueError: If plugins_dir or plugins_pkg is not provided
+
+ Example:
+ from pathlib import Path
+ from unstract.core.flask import plugin_loader
+
+ def create_app():
+ app = Flask(__name__)
+ plugins_dir = Path(__file__).parent / "plugins"
+ plugin_loader(app, plugins_dir, "unstract.myservice.plugins")
+ return app
+ """
+ if not plugins_dir or not plugins_pkg:
+ raise ValueError(
+ "Both 'plugins_dir' and 'plugins_pkg' are required for plugin_loader.\n\n"
+ "Example usage:\n"
+ " from pathlib import Path\n"
+ " plugins_dir = Path(__file__).parent / 'plugins'\n"
+ " plugin_loader(app, plugins_dir, 'unstract.myservice.plugins')"
+ )
+
+ manager = PluginManager(app, plugins_dir, plugins_pkg)
+ manager.load_plugins()
diff --git a/unstract/core/src/unstract/core/plugins/.gitignore b/unstract/core/src/unstract/core/plugins/.gitignore
new file mode 100644
index 000000000..f81789a40
--- /dev/null
+++ b/unstract/core/src/unstract/core/plugins/.gitignore
@@ -0,0 +1,7 @@
+# Ignore all enterprise plugins (copied from cloud repo at build time)
+*
+
+# Keep core plugin infrastructure in OSS
+!__init__.py
+!plugin_manager.py
+!README.md
diff --git a/unstract/core/src/unstract/core/plugins/README.md b/unstract/core/src/unstract/core/plugins/README.md
new file mode 100644
index 000000000..aaa2113df
--- /dev/null
+++ b/unstract/core/src/unstract/core/plugins/README.md
@@ -0,0 +1,122 @@
+# Unstract Plugin System
+
+## Plugin Metadata Format
+
+Every plugin must define a `metadata` dictionary in its `__init__.py` file. The plugin manager reads this metadata to discover, validate, and load plugins.
+
+### Required Fields
+
+| Field | Type | Description | Example |
+|-------|------|-------------|---------|
+| `name` | `str` | Unique plugin identifier | `"my_plugin"` |
+| `version` | `str` | Semantic version string | `"1.0.0"` |
+
+### Optional Fields
+
+| Field | Type | Description | Used In |
+|-------|------|-------------|---------|
+| `description` | `str` | Human-readable description | All |
+| `is_active` | `bool` | Enable/disable flag (default: `True`) | All |
+| `disable` | `bool` | Explicit disable flag (checked first) | All |
+| `entrypoint_cls` | `class` | Main handler class for the plugin | Flask|
+| `exception_cls` | `class` | Custom exception class | Flask |
+| `service_class` | `callable` | Service factory (usually lambda) | Django Backend |
+| `blueprint` | `Blueprint` | Flask blueprint for routes | Flask only |
+
+---
+
+## Creating a New Plugin
+
+### 1. Define Plugin Metadata
+
+**File:** `<service>/plugins/my_plugin/__init__.py`
+
+```python
+"""My custom plugin."""
+
+from .service import MyService
+
+metadata = {
+ # Required
+ "name": "my_plugin",
+ "version": "1.0.0",
+
+ # Optional
+ "description": "Description of what this plugin does",
+ "is_active": True,
+ "service_class": lambda: MyService(),
+}
+```
+
+### 2. Use the Plugin
+
+**Django:**
+```python
+from backend.plugins import get_plugin
+
+plugin = get_plugin("my_plugin")
+result = plugin.some_method()
+```
+
+**Flask:**
+```python
+from unstract.core.flask import plugin_loader
+
+# In create_app()
+plugin_manager = plugin_loader(app, plugin_dirs=["service/plugins"])
+
+# In controller
+plugin = plugin_manager.get_plugin("my_plugin")
+handler = plugin["entrypoint_cls"]()
+result = handler.process()
+```
+
+---
+
+## Plugin Loading Process
+
+1. **Scans** the configured plugins directory
+2. **Skips** items starting with `__` (like `__pycache__`)
+3. **Imports** the plugin module (supports `.so` compiled extensions)
+4. **Validates** that `metadata` dictionary exists
+5. **Checks** if disabled via `metadata.get("disable", False)` or `not metadata.get("is_active", True)`
+6. **Registers** plugin with all metadata fields
+7. **Calls** registration callback if provided (e.g., for Flask blueprint registration)
+
+## Using Shared Libraries in Plugins
+
+This directory (`unstract.core.plugins`) can contain shared library code that can be used by plugins across all Unstract services (backend, platform-service, prompt-service, workers).
+
+### Import Pattern
+
+Plugins can import shared code as subpackages from `unstract.core.plugins`:
+
+```python
+# In your plugin code (e.g., backend/plugins/my_plugin/service.py)
+from unstract.core.plugins.xxx import yyy
+```
+
+### Directory Structure
+
+```
+unstract/core/src/unstract/core/plugins/
+├── README.md # This file
+├── __init__.py # Package initialization
+├── plugin_manager.py # Core plugin discovery and loading
+```
+
+### When to Add Shared Code Here
+
+Add code to `unstract.core.plugins` when:
+
+- ✅ Multiple plugins need the same functionality
+- ✅ The code is framework-agnostic (works in Django, Flask, Workers)
+- ✅ You want to avoid code duplication across services
+
+Keep code in service-specific plugins when:
+
+- ❌ Logic is specific to one service only
+- ❌ Code depends on service-specific frameworks (Django ORM, Flask blueprints)
+- ❌ Business logic is unique to that plugin
+
+---
diff --git a/unstract/core/src/unstract/core/plugins/__init__.py b/unstract/core/src/unstract/core/plugins/__init__.py
new file mode 100644
index 000000000..365e95b58
--- /dev/null
+++ b/unstract/core/src/unstract/core/plugins/__init__.py
@@ -0,0 +1,12 @@
+"""Plugin system for Unstract.
+
+This module provides the core plugin infrastructure for loading and managing
+enterprise plugins that are injected at build time from the cloud repository.
+
+The PluginManager supports framework-agnostic plugin loading with Flask and
+Django-specific wrappers available in unstract.core.flask and unstract.core.django.
+"""
+
+from .plugin_manager import PluginManager
+
+__all__ = ["PluginManager"]
diff --git a/unstract/core/src/unstract/core/plugins/plugin_manager.py b/unstract/core/src/unstract/core/plugins/plugin_manager.py
new file mode 100644
index 000000000..1b091d9cf
--- /dev/null
+++ b/unstract/core/src/unstract/core/plugins/plugin_manager.py
@@ -0,0 +1,282 @@
+"""Framework-agnostic plugin manager for loading and managing plugins dynamically.
+
+This module provides a generic PluginManager class that can be used across
+different frameworks (Flask, Django, etc.) to load and manage plugins with
+consistent behavior.
+"""
+
+import importlib
+import logging
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+
+class PluginManager:
+ """Generic plugin manager for loading and managing application plugins.
+
+ This class provides framework-agnostic plugin loading with support for:
+ - Dynamic plugin discovery from directories
+ - Metadata validation
+ - Optional singleton pattern
+ - Framework-specific registration callbacks
+ - Compiled (.so) and Python module support
+ """
+
+ _instance = None
+ _use_singleton = True
+
+ def __init__(
+ self,
+ plugins_dir: Path | str,
+ plugins_pkg: str,
+ logger: logging.Logger | None = None,
+ use_singleton: bool = True,
+ registration_callback: Callable[[dict[str, Any]], None] | None = None,
+ plugin_submodule: str | None = None,
+ ):
+ """Initialize the PluginManager.
+
+ Args:
+ plugins_dir: Directory containing plugins
+ plugins_pkg: Python package path for plugins (e.g., 'myapp.plugins')
+ logger: Logger instance for logging (defaults to module logger)
+ use_singleton: Whether to use singleton pattern (default: True)
+ registration_callback: Optional callback for framework-specific registration
+ (e.g., Flask blueprint registration, Django URL patterns)
+ plugin_submodule: Submodule path within each plugin (e.g., 'src').
+ Set to None to import plugin directory directly.
+ Default: None (imports from plugin root).
+ """
+ self.plugins_dir = (
+ Path(plugins_dir) if isinstance(plugins_dir, str) else plugins_dir
+ )
+ self.plugins_pkg = plugins_pkg
+ self.logger = logger or logging.getLogger(__name__)
+ self.plugins: dict[str, dict[str, Any]] = {}
+ self.registration_callback = registration_callback
+ self._use_singleton = use_singleton
+ self.plugin_submodule = plugin_submodule
+
+ def __new__(cls, *args, **kwargs):
+ """Create singleton instance if use_singleton is True."""
+ use_singleton = kwargs.get("use_singleton", True)
+
+ if use_singleton:
+ if cls._instance is None:
+ cls._instance = super().__new__(cls)
+ return cls._instance
+ else:
+ # Create new instance if singleton is disabled
+ return super().__new__(cls)
+
+ def _find_plugin_modules(
+ self, base_path: Path, max_depth: int = 2
+ ) -> list[tuple[Path, str]]:
+ """Recursively find plugin directories containing valid metadata.
+
+ Scans the plugin directory tree up to max_depth levels to find directories
+ with __init__.py that contain plugin metadata. This supports both flat
+ plugin structures (plugins/plugin_name/) and nested structures
+ (plugins/category/plugin_name/).
+
+ Args:
+ base_path: Root directory to start searching from
+ max_depth: Maximum directory depth to search (default: 2)
+ depth=0: only base_path
+ depth=1: base_path + immediate subdirs
+ depth=2: base_path + subdirs + their subdirs
+
+ Returns:
+ List of tuples (plugin_path, module_name) where:
+ - plugin_path: Path to the plugin directory
+ - module_name: Dot-separated module path relative to plugins_pkg
+ (e.g., 'base_plugin.nested_plugin' or 'base_plugin')
+
+ """
+ plugins = []
+
+ def _scan_directory(path: Path, depth: int = 0, rel_parts: tuple[str, ...] = ()):
+ """Recursively scan directories for plugins.
+
+ Args:
+ path: Current directory being scanned
+ depth: Current recursion depth
+ rel_parts: Tuple of path components relative to base_path
+ """
+ if depth > max_depth:
+ return
+
+ try:
+ items = sorted(path.iterdir())
+ except OSError as e:
+ self.logger.warning(
+ f"Cannot access directory {path} for plugin discovery: {e}"
+ )
+ return
+
+ for item in items:
+ # Skip special Python directories and build artifacts
+ if item.name.startswith("__") or item.name in (
+ "build",
+ "dist",
+ "egg-info",
+ ".pytest_cache",
+ ".mypy_cache",
+ "node_modules",
+ ):
+ continue
+
+ # Handle .so files (compiled extensions) at any level
+ if item.name.endswith(".so"):
+ pkg_name = item.name.split(".")[0]
+ module_path = ".".join(rel_parts + (pkg_name,))
+ plugins.append((item, module_path))
+ continue
+
+ # Only process directories
+ if not item.is_dir():
+ continue
+
+ # Build relative module path
+ new_rel_parts = rel_parts + (item.name,)
+
+ # Check if this directory has __init__.py
+ init_file = item / "__init__.py"
+ if init_file.exists():
+ # This could be a plugin - add it to candidates
+ module_path = ".".join(new_rel_parts)
+ plugins.append((item, module_path))
+
+ # Continue scanning subdirectories if we haven't reached max depth
+ if depth < max_depth:
+ _scan_directory(item, depth + 1, new_rel_parts)
+
+ _scan_directory(base_path)
+ return plugins
+
+ def load_plugins(self) -> None:
+ """Load plugins found in the plugins directory.
+
+ Scans the plugins directory recursively for valid plugin packages,
+ validates their metadata, and optionally calls registration callback
+ for framework-specific integration (e.g., Flask blueprints, Django URL patterns).
+
+ Supports both flat plugin structures (plugins/plugin_name/) and nested
+ structures (plugins/category/plugin_name/) up to max_depth levels.
+ """
+ if not self.plugins_dir or not self.plugins_dir.exists():
+ self.logger.warning(
+ f"Plugins directory not found: {self.plugins_dir}. Skipping plugin loading."
+ )
+ return
+
+ self.logger.info(f"Loading plugins from: {self.plugins_dir}")
+
+ # Recursively discover all potential plugin directories
+ plugin_candidates = self._find_plugin_modules(self.plugins_dir, max_depth=2)
+
+ for plugin_path, module_name in plugin_candidates:
+ # Build import path with optional submodule
+ if self.plugin_submodule:
+ pkg_anchor = f"{self.plugins_pkg}.{module_name}.{self.plugin_submodule}"
+ else:
+ pkg_anchor = f"{self.plugins_pkg}.{module_name}"
+
+ # Try to import the plugin module
+ try:
+ module = importlib.import_module(pkg_anchor)
+ except ImportError as e:
+ self.logger.debug(
+ f"Could not import {pkg_anchor} (might not be a plugin): {str(e)}"
+ )
+ continue
+
+ # Validate plugin metadata
+ metadata = getattr(module, "metadata", None)
+ if not metadata:
+ # Not a plugin - just a regular directory with __init__.py
+ continue
+
+ # Skip disabled plugins
+ if metadata.get("disable", False) or not metadata.get("is_active", True):
+ self.logger.info(
+ f"Skipping disabled plugin: {module_name} "
+ f"v{metadata.get('version', 'unknown')}"
+ )
+ continue
+
+ # Register plugin
+ try:
+ plugin_name = metadata["name"]
+ plugin_data = {
+ "version": metadata.get("version", "unknown"),
+ "module": module,
+ "metadata": metadata,
+ }
+
+ # Add optional fields if present
+ if "entrypoint_cls" in metadata:
+ plugin_data["entrypoint_cls"] = metadata["entrypoint_cls"]
+ if "exception_cls" in metadata:
+ plugin_data["exception_cls"] = metadata["exception_cls"]
+ if "service_class" in metadata:
+ plugin_data["service_class"] = metadata["service_class"]
+
+ self.plugins[plugin_name] = plugin_data
+
+ # Call framework-specific registration callback if provided
+ if self.registration_callback:
+ self.registration_callback(plugin_data)
+
+ self.logger.info(
+ f"✔ Loaded plugin: {plugin_name} v{plugin_data['version']}"
+ )
+
+ except KeyError as e:
+ self.logger.error(
+ f"Invalid metadata for plugin '{module_name}': {str(e)}"
+ )
+
+ # Log appropriate message based on whether plugins were loaded
+ if not self.plugins:
+ self.logger.info("No plugins loaded (OSS mode).")
+
+ def get_plugin(self, name: str) -> dict[str, Any]:
+ """Get plugin metadata by name.
+
+ Args:
+ name: Plugin name to retrieve
+
+ Returns:
+ Dictionary containing plugin metadata (version, module, entrypoint_cls, etc.)
+ or empty dict if plugin not found
+ """
+ return self.plugins.get(name, {})
+
+ def has_plugin(self, name: str) -> bool:
+ """Check if a plugin is loaded.
+
+ Args:
+ name: Plugin name to check
+
+ Returns:
+ bool: True if plugin is loaded, False otherwise
+ """
+ return name in self.plugins
+
+ def get_all_plugins(self) -> dict[str, dict[str, Any]]:
+ """Get all loaded plugins.
+
+ Returns:
+ Dictionary mapping plugin names to their metadata
+ """
+ return self.plugins.copy()
+
+ def reload_plugins(self) -> None:
+ """Reload all plugins from the plugins directory.
+
+ Clears existing plugins and reloads them from disk.
+ """
+ self.plugins.clear()
+ self.load_plugins()
diff --git a/workers/callback/tasks.py b/workers/callback/tasks.py
index a7496c3d4..cccfb78c3 100644
--- a/workers/callback/tasks.py
+++ b/workers/callback/tasks.py
@@ -66,6 +66,7 @@ class CallbackContext:
self.pipeline_type: str | None = None
self.pipeline_data: dict[str, Any] | None = None
self.api_client: InternalAPIClient | None = None
+ self.file_executions: list[dict[str, Any]] = []
def _initialize_performance_managers():
@@ -719,6 +720,7 @@ def _extract_callback_parameters(
workflow_definition = execution_data.get("workflow_definition", {})
organization_context = execution_data.get("organization_context", {})
source_config = execution_data.get("source_config", {})
+ context.file_executions = execution_data.get("file_executions", [])
# 3. Extract parameters with kwargs as fast path, execution data as fallback
context.pipeline_id = kwargs.get("pipeline_id") or execution_info.get(
@@ -754,6 +756,7 @@ def _extract_callback_parameters(
workflow_definition = execution_data.get("workflow_definition", {})
organization_context = execution_data.get("organization_context", {})
source_config = execution_data.get("source_config", {})
+ context.file_executions = execution_data.get("file_executions", [])
# Extract organization ID from execution data
context.organization_id = organization_context.get(
@@ -772,13 +775,6 @@ def _extract_callback_parameters(
context.pipeline_id = kwargs.get("pipeline_id") or execution_info.get(
"pipeline_id"
)
- if not context.pipeline_id:
- execution_response = api_client.get_workflow_execution(
- execution_id=context.execution_id,
- file_execution=False,
- )
- execution_info = execution_response.data.get("execution", {})
- context.pipeline_id = execution_info.get("pipeline_id")
finally:
# Clean up temporary client
if hasattr(temp_api_client, "close"):
@@ -833,7 +829,9 @@ def _extract_callback_parameters(
f"Extracted from kwargs: pipeline_id={kwargs.get('pipeline_id')}, org_id={kwargs.get('organization_id')}"
)
logger.info(
- f"Extracted from execution: workflow_id={context.workflow_id}, is_api={is_api_deployment}, pipeline_type={context.pipeline_type}"
+ f"Extracted from execution: workflow_id={context.workflow_id}, "
+ f"is_api={is_api_deployment}, pipeline_type={context.pipeline_type} "
+ f"file_executions={len(context.file_executions)}"
)
except Exception as e:
@@ -1268,6 +1266,97 @@ def _cleanup_execution_resources(context: CallbackContext) -> dict[str, Any]:
}
+def _track_subscription_usage_if_available(
+ context: CallbackContext,
+ execution_status: str,
+) -> dict[str, Any] | None:
+ """Track subscription usage for successful file executions if plugin available.
+
+ This is a non-blocking operation - errors are logged but do not fail the workflow.
+ Only tracks usage for completed workflows with successful file executions.
+
+ Args:
+ context: Callback execution context with organization and execution IDs
+ execution_status: Final execution status (should be COMPLETED for tracking)
+
+ Returns:
+ dict with status, committed_count, message if tracking attempted
+ None if plugin not available or workflow not completed
+ """
+ # Only track for completed workflows
+ if execution_status != ExecutionStatus.COMPLETED.value:
+ logger.debug(
+ f"Skipping subscription tracking for execution {context.execution_id} "
+ f"with status {execution_status}"
+ )
+ return None
+
+ try:
+ # Try to get subscription_usage client plugin
+ from client_plugin_registry import get_client_plugin
+
+ subscription_plugin = get_client_plugin("subscription_usage")
+
+ if not subscription_plugin:
+ logger.debug(
+ f"Subscription usage plugin not found for execution {context.execution_id} (OSS mode)"
+ )
+ return None
+
+ # Extract successful file execution IDs directly from context.file_executions
+ # Status in database is "COMPLETED" for successful executions
+ successful_file_exec_ids = [
+ file_exec.get("id")
+ for file_exec in context.file_executions
+ if isinstance(file_exec, dict)
+ and file_exec.get("status") == "COMPLETED"
+ and file_exec.get("id")
+ and not file_exec.get("execution_error")
+ ]
+
+ if not successful_file_exec_ids:
+ logger.debug(
+ f"No successful file executions to track for execution {context.execution_id}"
+ )
+ return {
+ "status": "skipped",
+ "committed_count": 0,
+ "message": "No successful file executions",
+ }
+
+ logger.info(
+ f"Committing subscription usage for {len(successful_file_exec_ids)} "
+ f"successful file executions in execution {context.execution_id}"
+ )
+
+ subscription_tracking_result = (
+ subscription_plugin.commit_batch_subscription_usage(
+ organization_id=context.organization_id,
+ file_execution_ids=successful_file_exec_ids,
+ )
+ )
+
+ logger.info(
+ f"Subscription usage committed: "
+ f"{subscription_tracking_result.get('committed_count', 0)}/{len(successful_file_exec_ids)} "
+ f"file executions (status: {subscription_tracking_result.get('status')})"
+ )
+
+ return subscription_tracking_result
+
+ except Exception as tracking_error:
+ logger.error(
+ "Failed to commit subscription usage for execution "
+ f"{context.execution_id} (continuing execution): {tracking_error}",
+ exc_info=True,
+ )
+ return {
+ "status": "error",
+ "committed_count": 0,
+ "error": str(tracking_error),
+ }
+
+
def _process_batch_callback_core(
task_instance, results, *args, **kwargs
) -> dict[str, Any]:
@@ -1332,6 +1421,12 @@ def _process_batch_callback_core(
context, execution_status, is_api_deployment=False
)
+ # Track subscription usage if plugin is present
+ subscription_tracking_result = _track_subscription_usage_if_available(
+ context=context,
+ execution_status=execution_status,
+ )
+
# Add missing UI logs for cost and final workflow status (matching backend behavior)
_publish_final_workflow_ui_logs(
context=context,
@@ -1351,6 +1446,7 @@ def _process_batch_callback_core(
"expected_files": expected_files,
"execution_update_result": execution_update_result,
"pipeline_result": pipeline_result,
+ "subscription_tracking_result": subscription_tracking_result,
"cleanup_result": cleanup_result,
"pipeline_id": context.pipeline_id,
"unified_callback": True,
@@ -1565,6 +1661,7 @@ def process_batch_callback_api(
context.pipeline_name = pipeline_name
context.pipeline_type = pipeline_type
context.api_client = api_client
+ context.file_executions = execution_context.get("file_executions", [])
context.pipeline_data = {
"is_api": True
} # Mark as API execution for cleanup logic
@@ -1584,6 +1681,12 @@ def process_batch_callback_api(
context=context, final_status=execution_status, is_api_deployment=True
)
+ # Track subscription usage if plugin is present
+ subscription_tracking_result = _track_subscription_usage_if_available(
+ context=context,
+ execution_status=execution_status,
+ )
+
# Handle notifications using unified function
notification_result = _handle_notifications_unified(
api_client=api_client,
@@ -1612,6 +1715,7 @@ def process_batch_callback_api(
"execution_update": execution_update_result,
"pipeline_update": pipeline_result,
"notifications": notification_result,
+ "subscription_tracking_result": subscription_tracking_result,
"cleanup_result": cleanup_result,
"optimization": {
"method": "unified_callback_functions",
diff --git a/workers/pyproject.toml b/workers/pyproject.toml
index 4c320131d..0783784eb 100644
--- a/workers/pyproject.toml
+++ b/workers/pyproject.toml
@@ -49,7 +49,8 @@ dev = [
"black>=23.7.0",
"isort>=5.12.0",
"flake8>=6.0.0",
- "mypy>=1.5.0"
+ "mypy>=1.5.0",
+ "debugpy>=1.8.14",
]
test = [
diff --git a/workers/uv.lock b/workers/uv.lock
index 319eacaad..6432ed650 100644
--- a/workers/uv.lock
+++ b/workers/uv.lock
@@ -1,5 +1,5 @@
version = 1
-revision = 1
+revision = 3
requires-python = ">=3.12"
resolution-markers = [
"python_full_version >= '3.13'",
@@ -818,6 +818,27 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686 },
]
+[[package]]
+name = "debugpy"
+version = "1.8.17"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/15/ad/71e708ff4ca377c4230530d6a7aa7992592648c122a2cd2b321cf8b35a76/debugpy-1.8.17.tar.gz", hash = "sha256:fd723b47a8c08892b1a16b2c6239a8b96637c62a59b94bb5dab4bac592a58a8e", size = 1644129, upload-time = "2025-09-17T16:33:20.633Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/08/2b/9d8e65beb2751876c82e1aceb32f328c43ec872711fa80257c7674f45650/debugpy-1.8.17-cp312-cp312-macosx_15_0_universal2.whl", hash = "sha256:f14467edef672195c6f6b8e27ce5005313cb5d03c9239059bc7182b60c176e2d", size = 2549522, upload-time = "2025-09-17T16:33:38.466Z" },
+ { url = "https://files.pythonhosted.org/packages/b4/78/eb0d77f02971c05fca0eb7465b18058ba84bd957062f5eec82f941ac792a/debugpy-1.8.17-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:24693179ef9dfa20dca8605905a42b392be56d410c333af82f1c5dff807a64cc", size = 4309417, upload-time = "2025-09-17T16:33:41.299Z" },
+ { url = "https://files.pythonhosted.org/packages/37/42/c40f1d8cc1fed1e75ea54298a382395b8b937d923fcf41ab0797a554f555/debugpy-1.8.17-cp312-cp312-win32.whl", hash = "sha256:6a4e9dacf2cbb60d2514ff7b04b4534b0139facbf2abdffe0639ddb6088e59cf", size = 5277130, upload-time = "2025-09-17T16:33:43.554Z" },
+ { url = "https://files.pythonhosted.org/packages/72/22/84263b205baad32b81b36eac076de0cdbe09fe2d0637f5b32243dc7c925b/debugpy-1.8.17-cp312-cp312-win_amd64.whl", hash = "sha256:e8f8f61c518952fb15f74a302e068b48d9c4691768ade433e4adeea961993464", size = 5319053, upload-time = "2025-09-17T16:33:53.033Z" },
+ { url = "https://files.pythonhosted.org/packages/50/76/597e5cb97d026274ba297af8d89138dfd9e695767ba0e0895edb20963f40/debugpy-1.8.17-cp313-cp313-macosx_15_0_universal2.whl", hash = "sha256:857c1dd5d70042502aef1c6d1c2801211f3ea7e56f75e9c335f434afb403e464", size = 2538386, upload-time = "2025-09-17T16:33:54.594Z" },
+ { url = "https://files.pythonhosted.org/packages/5f/60/ce5c34fcdfec493701f9d1532dba95b21b2f6394147234dce21160bd923f/debugpy-1.8.17-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:3bea3b0b12f3946e098cce9b43c3c46e317b567f79570c3f43f0b96d00788088", size = 4292100, upload-time = "2025-09-17T16:33:56.353Z" },
+ { url = "https://files.pythonhosted.org/packages/e8/95/7873cf2146577ef71d2a20bf553f12df865922a6f87b9e8ee1df04f01785/debugpy-1.8.17-cp313-cp313-win32.whl", hash = "sha256:e34ee844c2f17b18556b5bbe59e1e2ff4e86a00282d2a46edab73fd7f18f4a83", size = 5277002, upload-time = "2025-09-17T16:33:58.231Z" },
+ { url = "https://files.pythonhosted.org/packages/46/11/18c79a1cee5ff539a94ec4aa290c1c069a5580fd5cfd2fb2e282f8e905da/debugpy-1.8.17-cp313-cp313-win_amd64.whl", hash = "sha256:6c5cd6f009ad4fca8e33e5238210dc1e5f42db07d4b6ab21ac7ffa904a196420", size = 5319047, upload-time = "2025-09-17T16:34:00.586Z" },
+ { url = "https://files.pythonhosted.org/packages/de/45/115d55b2a9da6de812696064ceb505c31e952c5d89c4ed1d9bb983deec34/debugpy-1.8.17-cp314-cp314-macosx_15_0_universal2.whl", hash = "sha256:045290c010bcd2d82bc97aa2daf6837443cd52f6328592698809b4549babcee1", size = 2536899, upload-time = "2025-09-17T16:34:02.657Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/73/2aa00c7f1f06e997ef57dc9b23d61a92120bec1437a012afb6d176585197/debugpy-1.8.17-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:b69b6bd9dba6a03632534cdf67c760625760a215ae289f7489a452af1031fe1f", size = 4268254, upload-time = "2025-09-17T16:34:04.486Z" },
+ { url = "https://files.pythonhosted.org/packages/86/b5/ed3e65c63c68a6634e3ba04bd10255c8e46ec16ebed7d1c79e4816d8a760/debugpy-1.8.17-cp314-cp314-win32.whl", hash = "sha256:5c59b74aa5630f3a5194467100c3b3d1c77898f9ab27e3f7dc5d40fc2f122670", size = 5277203, upload-time = "2025-09-17T16:34:06.65Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/26/394276b71c7538445f29e792f589ab7379ae70fd26ff5577dfde71158e96/debugpy-1.8.17-cp314-cp314-win_amd64.whl", hash = "sha256:893cba7bb0f55161de4365584b025f7064e1f88913551bcd23be3260b231429c", size = 5318493, upload-time = "2025-09-17T16:34:08.483Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/d0/89247ec250369fc76db477720a26b2fce7ba079ff1380e4ab4529d2fe233/debugpy-1.8.17-py2.py3-none-any.whl", hash = "sha256:60c7dca6571efe660ccb7a9508d73ca14b8796c4ed484c2002abba714226cfef", size = 5283210, upload-time = "2025-09-17T16:34:25.835Z" },
+]
+
[[package]]
name = "decorator"
version = "5.2.1"
@@ -3038,7 +3059,7 @@ dependencies = [
{ name = "pinecone-plugin-interface" },
{ name = "python-dateutil" },
{ name = "typing-extensions" },
- { name = "urllib3", marker = "python_full_version < '4.0'" },
+ { name = "urllib3", marker = "python_full_version < '4'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/bc/9d/07a7f2136ce04cabd21d69c057dc2915867082b0047e6873e424388d4475/pinecone-7.0.1.tar.gz", hash = "sha256:49ff7b0f5be4a2ddec5aaa709758a9f2df56baa58ad46507d081409e246a81ec", size = 207930 }
wheels = [
@@ -4669,6 +4690,7 @@ deploy = [
]
dev = [
{ name = "black" },
+ { name = "debugpy" },
{ name = "flake8" },
{ name = "isort" },
{ name = "mypy" },
@@ -4716,6 +4738,7 @@ deploy = [
]
dev = [
{ name = "black", specifier = ">=23.7.0" },
+ { name = "debugpy", specifier = ">=1.8.14" },
{ name = "flake8", specifier = ">=6.0.0" },
{ name = "isort", specifier = ">=5.12.0" },
{ name = "mypy", specifier = ">=1.5.0" },