LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2930 [MISC] Add plugin infrastructure to unstract-core (#1618)
Chandrasekharan M808040c63f29
diff --git a/.gitignore b/.gitignoreindex 8cb0de037..c89b40911 100644--- a/.gitignore+++ b/.gitignore@@ -620,29 +620,6 @@ $RECYCLE.BIN/### Unstract ###-# Authentication Plugins-backend/plugins/authentication/*-!backend/plugins/authentication/auth_sample--# Processor Plugins-backend/plugins/processor/*--# Subscription Plugins-backend/plugins/subscription/*---# API Deployment Plugins-backend/plugins/api/**--# Notification Plugin-backend/plugins/notification/**--# Configuration Plugin-backend/plugins/configuration/**--# Verticals Usage Plugin-backend/plugins/verticals_usage/**-# BE pluggable-appsbackend/pluggable_apps/*diff --git a/backend/account_v2/subscription_loader.py b/backend/account_v2/subscription_loader.pydeleted file mode 100644index 8f323c29b..000000000--- a/backend/account_v2/subscription_loader.py+++ /dev/null@@ -1,129 +0,0 @@-import logging-import os-from importlib import import_module-from typing import Any--from django.apps import apps-from django.utils import timezone--logger = logging.getLogger(__name__)---class SubscriptionConfig:- """Loader config for subscription plugins."""-- PLUGINS_APP = "plugins"- PLUGIN_DIR = "subscription"- MODULE = "module"- METADATA = "metadata"- METADATA_NAME = "name"- METADATA_SERVICE_CLASS = "service_class"- METADATA_IS_ACTIVE = "is_active"---# Cache for loaded plugins to avoid repeated loading-_subscription_plugins_cache: list[Any] = []-_plugins_loaded = False---def load_plugins() -> list[Any]:- """Iterate through the subscription plugins and register them."""- global _subscription_plugins_cache, _plugins_loaded-- # Return cached plugins if already loaded- if _plugins_loaded:- return _subscription_plugins_cache-- plugins_app = apps.get_app_config(SubscriptionConfig.PLUGINS_APP)- package_path = plugins_app.module.__package__- subscription_dir = os.path.join(plugins_app.path, SubscriptionConfig.PLUGIN_DIR)- subscription_package_path = f"{package_path}.{SubscriptionConfig.PLUGIN_DIR}"- subscription_plugins: list[Any] = []-- if not os.path.exists(subscription_dir):- _subscription_plugins_cache = subscription_plugins- _plugins_loaded = True- return subscription_plugins-- for item in os.listdir(subscription_dir):- # Loads a plugin if it is in a directory.- if os.path.isdir(os.path.join(subscription_dir, item)):- subscription_module_name = item- # Loads a plugin if it is a shared library.- # Module name is extracted from shared library name.- # `subscription.platform_architecture.so` will be file name and- # `subscription` will be the module name.- elif item.endswith(".so"):- subscription_module_name = item.split(".")[0]- else:- continue- try:- full_module_path = f"{subscription_package_path}.{subscription_module_name}"- module = import_module(full_module_path)- metadata = getattr(module, SubscriptionConfig.METADATA, {})-- if metadata.get(SubscriptionConfig.METADATA_IS_ACTIVE, False):- subscription_plugins.append(- {- SubscriptionConfig.MODULE: module,- SubscriptionConfig.METADATA: module.metadata,- }- )- name = metadata.get(- SubscriptionConfig.METADATA_NAME,- getattr(module, "__name__", "unknown"),- )- is_active = metadata.get(SubscriptionConfig.METADATA_IS_ACTIVE, False)- logger.info(- "Loaded subscription plugin: %s, is_active: %s", name, is_active- )- else:- logger.info(- "subscription plugin %s is not active.",- subscription_module_name,- )- except ModuleNotFoundError as exception:- logger.error(- "Error while importing subscription plugin: %s",- exception,- )-- if len(subscription_plugins) == 0:- logger.info("No subscription plugins found.")-- # Cache the results for future requests- _subscription_plugins_cache = subscription_plugins- _plugins_loaded = True-- return subscription_plugins---def validate_etl_run(org_id: str) -> bool:- """Method to check subscription status before ETL runs.-- Args:- org_id: The ID of the organization.-- Returns:- A boolean indicating whether the pre-run check passed or not.- """- try:- from pluggable_apps.subscription_v2.subscription_helper import (- SubscriptionHelper,- )- except ModuleNotFoundError:- logger.error("Subscription plugin not found.")- return False-- org_plans = SubscriptionHelper.get_subscription(org_id)- if not org_plans or not org_plans.is_active:- return False-- if org_plans.is_paid:- return True-- if timezone.now() >= org_plans.end_date:- logger.debug(f"Trial expired for org {org_id}")- return False-- return Truediff --git a/backend/adapter_processor_v2/views.py b/backend/adapter_processor_v2/views.pyindex 78f288063..046f80bd4 100644--- a/backend/adapter_processor_v2/views.py+++ b/backend/adapter_processor_v2/views.py@@ -342,37 +342,40 @@ class AdapterInstanceViewSet(ModelViewSet):# Send email notifications to newly shared usersif response.status_code == 200 and AdapterKeys.SHARED_USERS in request.data:try:- from plugins.notification.constants import ResourceType- from plugins.notification.sharing_notification import (- SharingNotificationService,- )-- adapter.refresh_from_db()- new_shared_users = set(adapter.shared_users.all())- newly_shared_users = new_shared_users - current_shared_users-- if newly_shared_users:- # Map adapter type to specific resource type- adapter_type_to_resource = {- "LLM": ResourceType.LLM.value,- "EMBEDDING": ResourceType.EMBEDDING.value,- "VECTOR_DB": ResourceType.VECTOR_DB.value,- "X2TEXT": ResourceType.X2TEXT.value,- }-- resource_type = adapter_type_to_resource.get(- adapter.adapter_type, ResourceType.LLM.value- )+ from plugins import get_plugin++ notification_plugin = get_plugin("notification")+ if notification_plugin:+ from plugins.notification.constants import ResourceType++ adapter.refresh_from_db()+ new_shared_users = set(adapter.shared_users.all())+ newly_shared_users = new_shared_users - current_shared_users++ if newly_shared_users:+ # Map adapter type to specific resource type+ adapter_type_to_resource = {+ "LLM": ResourceType.LLM.value,+ "EMBEDDING": ResourceType.EMBEDDING.value,+ "VECTOR_DB": ResourceType.VECTOR_DB.value,+ "X2TEXT": ResourceType.X2TEXT.value,+ }++ resource_type = adapter_type_to_resource.get(+ adapter.adapter_type, ResourceType.LLM.value+ )- notification_service = SharingNotificationService()- notification_service.send_sharing_notification(- resource_type=resource_type,- resource_name=adapter.adapter_name,- resource_id=str(adapter.id),- shared_by=request.user,- shared_to=list(newly_shared_users),- resource_instance=adapter,- )+ # Get notification service from plugin+ service_class = notification_plugin["service_class"]+ notification_service = service_class()+ notification_service.send_sharing_notification(+ resource_type=resource_type,+ resource_name=adapter.adapter_name,+ resource_id=str(adapter.id),+ shared_by=request.user,+ shared_to=list(newly_shared_users),+ resource_instance=adapter,+ )except Exception as e:logger.exception(f"Failed to send sharing notification: {e}")diff --git a/backend/api_v2/api_deployment_dto_registry.py b/backend/api_v2/api_deployment_dto_registry.pyindex 981a6520e..b40df3e45 100644--- a/backend/api_v2/api_deployment_dto_registry.py+++ b/backend/api_v2/api_deployment_dto_registry.py@@ -1,7 +1,7 @@import loggingfrom typing import Any-from plugins.api.dto import metadata+from plugins import get_pluginfrom api_v2.postman_collection.dto import PostmanCollection@@ -13,10 +13,16 @@ class ApiDeploymentDTORegistry:@classmethoddef load_dto(cls) -> Any | None:- class_name = PostmanCollection.__name__- if metadata.get(class_name):- return metadata[class_name].class_name- return PostmanCollection # Return as soon as we find a valid DTO+ """Load DTO from plugin or return default PostmanCollection.++ Checks if the api_dto plugin is available and gets the Postman DTO+ via the service, otherwise returns the base PostmanCollection class.+ """+ plugin = get_plugin("api_dto")+ if plugin:+ service = plugin["service_class"]()+ return service.get_postman_dto()+ return PostmanCollection@classmethoddef get_dto(cls) -> type | None:diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.pyindex dc4d77108..7174547e0 100644--- a/backend/api_v2/api_deployment_views.py+++ b/backend/api_v2/api_deployment_views.py@@ -7,6 +7,7 @@ from configuration.models import Configurationfrom django.db.models import QuerySetfrom django.http import HttpResponsefrom permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg+from plugins import get_pluginfrom prompt_studio.prompt_studio_registry_v2.models import PromptStudioRegistryfrom rest_framework import serializers, status, views, viewsetsfrom rest_framework.decorators import action@@ -33,15 +34,12 @@ from api_v2.serializers import (SharedUserListSerializer,)-try:- from plugins.notification.constants import ResourceType- from plugins.notification.sharing_notification import SharingNotificationService+# Check if notification plugin is available+notification_plugin = get_plugin("notification")- NOTIFICATION_PLUGIN_AVAILABLE = True- sharing_notification_service = SharingNotificationService()-except ImportError:- NOTIFICATION_PLUGIN_AVAILABLE = False- sharing_notification_service = None+# Import constants from notification plugin if available+if notification_plugin:+ from plugins.notification.constants import ResourceTypelogger = logging.getLogger(__name__)@@ -319,7 +317,7 @@ class APIDeploymentViewSet(viewsets.ModelViewSet):if (response.status_code == 200and "shared_users" in request.data- and NOTIFICATION_PLUGIN_AVAILABLE+ and notification_plugin):try:instance.refresh_from_db()@@ -327,7 +325,9 @@ class APIDeploymentViewSet(viewsets.ModelViewSet):newly_shared_users = new_shared_users - current_shared_usersif newly_shared_users:- notification_service = SharingNotificationService()+ # Get notification service from plugin+ service_class = notification_plugin["service_class"]+ notification_service = service_class()notification_service.send_sharing_notification(resource_type=ResourceType.API_DEPLOYMENT.value,resource_name=instance.display_name,diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 26f74c747..b6cbb2599 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -308,7 +308,7 @@ SHARED_APPS = ("drf_yasg","docs",# Plugins- "plugins",+ "plugins.apps.PluginsConfig","feature_flag","django_celery_beat",# For additional helper commandsdiff --git a/backend/configuration/config_registry.py b/backend/configuration/config_registry.pyindex 189126631..ce4f91fca 100644--- a/backend/configuration/config_registry.py+++ b/backend/configuration/config_registry.py@@ -5,12 +5,9 @@ cloud-specific configuration keys from plugins when available."""import logging-import osfrom importlib import import_modulefrom typing import Any-from django.apps import apps-from configuration.constants import ConfigPluginConstantsfrom configuration.enums import ConfigKey, ConfigSpec@@ -26,52 +23,46 @@ def _load_cloud_config_specs() -> dict[str, ConfigSpec]:cloud_specs = {}try:- # Check if plugins app exists- auth_app = apps.get_app_config(ConfigPluginConstants.PLUGINS_APP)- config_dir = os.path.join(auth_app.path, ConfigPluginConstants.CONFIG_PLUGIN_DIR)+ from plugins import get_plugin++ # Try to get the configuration plugin+ plugin = get_plugin(ConfigPluginConstants.CONFIG_PLUGIN_DIR)++ if not plugin:+ logger.debug("Configuration plugin not found")+ return cloud_specs- # Check if configuration plugin directory exists- if not os.path.exists(config_dir):- logger.debug("Configuration plugin directory not found")+ # Check if plugin is active+ metadata = plugin.get(ConfigPluginConstants.METADATA, {})+ if not metadata.get(ConfigPluginConstants.METADATA_IS_ACTIVE, False):+ logger.warning("Cloud configuration plugin is not active")return cloud_specs- # Try to import the configuration plugin package- config_package_path = (- f"{auth_app.module.__package__}.{ConfigPluginConstants.CONFIG_PLUGIN_DIR}"+ # Get the plugin module and import the config specs submodule+ plugin_module = plugin.get("module")+ if not plugin_module:+ logger.warning("Configuration plugin module not found")+ return cloud_specs++ # Import the cloud_config submodule to get config specs+ base_module = (+ getattr(plugin_module, "__package__", None) or plugin_module.__name__+ )+ config_module_path = f"{base_module}.{ConfigPluginConstants.CONFIG_MODULE}"+ config_module = import_module(config_module_path)+ config_specs = getattr(config_module, ConfigPluginConstants.CONFIG_SPECS, {})++ cloud_specs.update(config_specs)+ logger.info(+ "Loaded cloud configuration plugin: %s with %d config keys",+ metadata.get(ConfigPluginConstants.METADATA_NAME, "Unknown"),+ len(config_specs),)- try:- # Import the plugin package to get metadata from __init__.py- plugin_module = import_module(config_package_path)- metadata = getattr(plugin_module, ConfigPluginConstants.CONFIG_METADATA, {})-- if metadata.get(ConfigPluginConstants.METADATA_IS_ACTIVE, False):- # Import the cloud_config module to get config specs- full_module_path = (- f"{config_package_path}.{ConfigPluginConstants.CONFIG_MODULE}"- )- config_module = import_module(full_module_path)- config_specs = getattr(- config_module, ConfigPluginConstants.CONFIG_SPECS, {}- )- cloud_specs.update(config_specs)- logger.info(- "Loaded cloud configuration plugin: %s with %d config keys",- metadata.get(ConfigPluginConstants.METADATA_NAME, "Unknown"),- len(config_specs),- )- else:- logger.debug("Cloud configuration plugin not active")-- except ModuleNotFoundError as e:- logger.debug("Cloud configuration module not found: %s", e)- except Exception as e:- logger.error("Error loading cloud configuration plugin: %s", e)-- except LookupError:- logger.debug("Plugins app not configured")+ except ImportError as e:+ logger.debug("Could not import plugins module: %s", e)except Exception as e:- logger.error("Error accessing plugins app: %s", e)+ logger.error("Error loading cloud configuration plugin: %s", e)return cloud_specsdiff --git a/backend/pipeline_v2/views.py b/backend/pipeline_v2/views.pyindex 396729f39..02625a52f 100644--- a/backend/pipeline_v2/views.py+++ b/backend/pipeline_v2/views.py@@ -10,6 +10,7 @@ from django.db import IntegrityErrorfrom django.db.models import QuerySetfrom django.http import HttpResponsefrom permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg+from plugins import get_pluginfrom rest_framework import serializers, status, viewsetsfrom rest_framework.decorators import actionfrom rest_framework.request import Request@@ -32,15 +33,13 @@ from pipeline_v2.serializers.execute import ()from pipeline_v2.serializers.sharing import SharedUserListSerializer-try:- from plugins.notification.constants import ResourceType- from plugins.notification.sharing_notification import SharingNotificationService+# Check if notification plugin is available+notification_plugin = get_plugin("notification")+NOTIFICATION_PLUGIN_AVAILABLE = notification_plugin is not None- NOTIFICATION_PLUGIN_AVAILABLE = True- sharing_notification_service = SharingNotificationService()-except ImportError:- NOTIFICATION_PLUGIN_AVAILABLE = False- sharing_notification_service = None+# Import constants from notification plugin if available+if NOTIFICATION_PLUGIN_AVAILABLE:+ from plugins.notification.constants import ResourceTypelogger = logging.getLogger(__name__)@@ -149,8 +148,10 @@ class PipelineViewSet(viewsets.ModelViewSet):resource_type = ResourceType.TASK.valueif newly_shared_users:- # Only send notifications if there are newly shared users- sharing_notification_service.send_sharing_notification(+ # Get notification service from plugin and send notification+ service_class = notification_plugin["service_class"]+ notification_service = service_class()+ notification_service.send_sharing_notification(resource_type=resource_type,resource_name=instance.pipeline_name,resource_id=str(instance.id),diff --git a/backend/plugins/.gitignore b/backend/plugins/.gitignoreindex 5d66e53bd..8d0b5901c 100644--- a/backend/plugins/.gitignore+++ b/backend/plugins/.gitignore@@ -1 +1,11 @@-modifier+# Ignore all plugin implementation directories+# Keep only infrastructure files (__init__.py, plugin_manager.py, etc.)+*/++# But allow infrastructure files to be tracked+!__init__.py+!apps.py++# TODO: Make these as proper plugins+!authentication/auth_sample+!workflow_managerdiff --git a/backend/plugins/README.md b/backend/plugins/README.mdindex b03ee2c78..c3e7cb4bb 100644--- a/backend/plugins/README.md+++ b/backend/plugins/README.md@@ -1,5 +1,3 @@# Plugins-## Authentication--Enhance the authentication capabilities of the `account/authentication_controller.py` module by incorporating additional modules.+Read [unstract-core's README](../../unstract/core/src/unstract/core/plugins/README.md) for more details on adding and using plugins.diff --git a/backend/plugins/__init__.py b/backend/plugins/__init__.pyindex e69de29bb..feb126f1b 100644--- a/backend/plugins/__init__.py+++ b/backend/plugins/__init__.py@@ -0,0 +1,60 @@+"""Backend plugins initialization.++This module initializes the Django plugin manager for all backend plugins.+"""++import logging+from pathlib import Path++from unstract.core.django import DjangoPluginManager++logger = logging.getLogger(__name__)++# Initialize plugin manager singleton+_plugin_manager = None+++def get_plugin_manager() -> DjangoPluginManager:+ """Get or initialize the plugin manager singleton.++ Note: Plugins are loaded by PluginsConfig.ready() after Django initialization.+ This function only creates the manager instance without loading plugins.++ Returns:+ DjangoPluginManager: The plugin manager instance+ """+ global _plugin_manager+ if _plugin_manager is None:+ plugins_dir = Path(__file__).parent+ _plugin_manager = DjangoPluginManager(+ plugins_dir=plugins_dir,+ plugins_pkg="plugins",+ logger=logger,+ )+ return _plugin_manager+++def get_plugin(plugin_name: str) -> dict:+ """Get plugin metadata by name (simplified single-line access).++ This is a convenience function that combines get_plugin_manager()+ and plugin_manager.get_plugin() into one call.++ Args:+ plugin_name: Name of the plugin to retrieve++ Returns:+ Dictionary containing plugin metadata (version, module, service_class, etc.)+ or empty dict if plugin not found++ Example:+ >>> from plugins import get_plugin+ >>> plugin = get_plugin("subscription_usage")+ >>> if plugin:+ ... service = plugin["service_class"]()+ """+ manager = get_plugin_manager()+ return manager.get_plugin(plugin_name)+++__all__ = ["get_plugin_manager", "get_plugin"]diff --git a/backend/plugins/api/dto/__init__.py b/backend/plugins/api/dto/__init__.pyindex b84a87d20..18b88861f 100644--- a/backend/plugins/api/dto/__init__.py+++ b/backend/plugins/api/dto/__init__.py@@ -1,12 +1,31 @@-from dataclasses import dataclass-from typing import Any+"""API DTO extensions for cloud deployments.+This plugin extends OSS DTOs with cloud-specific features like+highlight data in Postman collections.+"""-@dataclass-class MetadataDto:- name: str- class_name: Any- is_active: bool+from .postman_collection import HighlightPostmanDto-metadata = {}+class ApiDtoService:+ """Service for providing cloud-specific API DTOs."""++ def get_postman_dto(self):+ """Get the Postman collection DTO class for cloud features.++ Returns:+ Class: HighlightPostmanDto class for creating Postman collections+ with cloud-specific features like highlight data API.+ """+ return HighlightPostmanDto+++metadata = {+ "version": "1.0.0",+ "name": "api_dto",+ "is_active": True,+ "description": "API DTO extensions for cloud-specific features",+ "service_class": ApiDtoService,+}++__all__ = ["metadata", "ApiDtoService"]diff --git a/backend/plugins/apps.py b/backend/plugins/apps.pynew file mode 100644index 000000000..0b5e416df--- /dev/null+++ b/backend/plugins/apps.py@@ -0,0 +1,34 @@+"""Django AppConfig for backend plugins."""++import logging++from django.apps import AppConfig++logger = logging.getLogger(__name__)+++class PluginsConfig(AppConfig):+ """Django AppConfig for the plugins package.++ This loads plugins after Django is fully initialized to avoid+ AppRegistryNotReady errors.+ """++ name = "plugins"+ verbose_name = "Backend Plugins"++ def ready(self) -> None:+ """Load plugins after Django apps are ready.++ This method is called by Django after all apps are loaded,+ ensuring that models and other Django components are available.+ """+ from plugins import get_plugin_manager++ try:+ plugin_manager = get_plugin_manager()+ plugin_manager.load_plugins()+ logger.info("Backend plugins loaded successfully")+ except Exception as e:+ logger.error(f"Failed to load plugins: {e}", exc_info=True)+ # Don't raise - allow Django to continue even if plugins faildiff --git a/backend/prompt_studio/modifier_loader.py b/backend/prompt_studio/modifier_loader.pydeleted file mode 100644index e78d18113..000000000--- a/backend/prompt_studio/modifier_loader.py+++ /dev/null@@ -1,89 +0,0 @@-import logging-import os-from importlib import import_module-from typing import Any--from django.apps import apps--logger = logging.getLogger(__name__)---class ModifierConfig:- """Loader config for extraction plugins."""-- PLUGINS_APP = "plugins"- PLUGIN_DIR = "modifier"- MODULE = "module"- METADATA = "metadata"- METADATA_NAME = "name"- METADATA_SERVICE_CLASS = "service_class"- METADATA_IS_ACTIVE = "is_active"---# Cache for loaded plugins to avoid repeated loading-_modifier_plugins_cache: list[Any] = []-_plugins_loaded = False---def load_plugins() -> list[Any]:- """Iterate through the extraction plugins and register them."""- global _modifier_plugins_cache, _plugins_loaded-- # Return cached plugins if already loaded- if _plugins_loaded:- return _modifier_plugins_cache-- plugins_app = apps.get_app_config(ModifierConfig.PLUGINS_APP)- package_path = plugins_app.module.__package__- modifier_dir = os.path.join(plugins_app.path, ModifierConfig.PLUGIN_DIR)- modifier_package_path = f"{package_path}.{ModifierConfig.PLUGIN_DIR}"- modifier_plugins: list[Any] = []-- if not os.path.exists(modifier_dir):- _modifier_plugins_cache = modifier_plugins- _plugins_loaded = True- return modifier_plugins-- for item in os.listdir(modifier_dir):- # Loads a plugin if it is in a directory.- if os.path.isdir(os.path.join(modifier_dir, item)):- modifier_module_name = item- # Loads a plugin if it is a shared library.- # Module name is extracted from shared library name.- elif item.endswith(".so"):- modifier_module_name = item.split(".")[0]- else:- continue- try:- full_module_path = f"{modifier_package_path}.{modifier_module_name}"- module = import_module(full_module_path)- metadata = getattr(module, ModifierConfig.METADATA, {})-- if metadata.get(ModifierConfig.METADATA_IS_ACTIVE, False):- modifier_plugins.append(- {- ModifierConfig.MODULE: module,- ModifierConfig.METADATA: module.metadata,- }- )- logger.info(- "Loaded modifier plugin: %s, is_active: %s",- module.metadata[ModifierConfig.METADATA_NAME],- module.metadata[ModifierConfig.METADATA_IS_ACTIVE],- )- else:- logger.info(- "modifier plugin %s is not active.",- modifier_module_name,- )- except ModuleNotFoundError:- logger.warning("No prompt modifier plugins loaded")-- if len(modifier_plugins) == 0:- logger.info("No modifier plugins found.")-- # Cache the results for future requests- _modifier_plugins_cache = modifier_plugins- _plugins_loaded = True-- return modifier_pluginsdiff --git a/backend/prompt_studio/processor_loader.py b/backend/prompt_studio/processor_loader.pydeleted file mode 100644index 8c22dd82e..000000000--- a/backend/prompt_studio/processor_loader.py+++ /dev/null@@ -1,103 +0,0 @@-import logging-import os-from importlib import import_module-from typing import Any--from django.apps import apps--logger = logging.getLogger(__name__)---class ProcessorConfig:- """Loader config for processor plugins."""-- PLUGINS_APP = "plugins"- PLUGIN_DIR = "processor"- MODULE = "module"- METADATA = "metadata"- METADATA_NAME = "name"- METADATA_SERVICE_CLASS = "service_class"- METADATA_IS_ACTIVE = "is_active"---# Cache for loaded plugins to avoid repeated loading-_processor_plugins_cache: list[Any] = []-_plugins_loaded = False---def load_plugins() -> list[Any]:- """Iterate through the processor plugins and register them."""- global _processor_plugins_cache, _plugins_loaded-- # Return cached plugins if already loaded- if _plugins_loaded:- return _processor_plugins_cache-- plugins_app = apps.get_app_config(ProcessorConfig.PLUGINS_APP)- package_path = plugins_app.module.__package__- processor_dir = os.path.join(plugins_app.path, ProcessorConfig.PLUGIN_DIR)- processor_package_path = f"{package_path}.{ProcessorConfig.PLUGIN_DIR}"- processor_plugins: list[Any] = []-- if not os.path.exists(processor_dir):- logger.info("No processor directory found at %s.", processor_dir)- return []-- for item in os.listdir(processor_dir):- # Loads a plugin if it is in a directory.- if os.path.isdir(os.path.join(processor_dir, item)):- processor_module_name = item- # Loads a plugin if it is a shared library.- # Module name is extracted from shared library name.- # `processor.platform_architecture.so` will be file name and- # `processor` will be the module name.- elif item.endswith(".so"):- processor_module_name = item.split(".")[0]- else:- continue- try:- full_module_path = f"{processor_package_path}.{processor_module_name}"- module = import_module(full_module_path)- metadata = getattr(module, ProcessorConfig.METADATA, {})-- if metadata.get(ProcessorConfig.METADATA_IS_ACTIVE, False):- processor_plugins.append(- {- ProcessorConfig.MODULE: module,- ProcessorConfig.METADATA: module.metadata,- }- )- logger.info(- "Loaded processor plugin: %s, is_active: %s",- module.metadata[ProcessorConfig.METADATA_NAME],- module.metadata[ProcessorConfig.METADATA_IS_ACTIVE],- )- else:- logger.info(- "Processor plugin %s is not active.",- processor_module_name,- )- except ModuleNotFoundError as exception:- logger.error(- "Error while importing processor plugin: %s",- exception,- )-- if len(processor_plugins) == 0:- logger.info("No processor plugins found.")-- # Cache the results for future requests- _processor_plugins_cache = processor_plugins- _plugins_loaded = True-- return processor_plugins---def get_plugin_class_by_name(name: str, plugins: list[Any]) -> Any:- """Retrieve a specific plugin class by name."""- for plugin in plugins:- metadata = plugin[ProcessorConfig.METADATA]- if metadata.get(ProcessorConfig.METADATA_NAME) == name:- return metadata.get(ProcessorConfig.METADATA_SERVICE_CLASS)- logger.warning("Plugin with name '%s' not found.", name)- return Nonediff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.pyindex 7dd2fc8be..91f8f0740 100644--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py@@ -12,15 +12,13 @@ from adapter_processor_v2.constants import AdapterKeysfrom adapter_processor_v2.models import AdapterInstancefrom django.conf import settingsfrom django.db.models.manager import BaseManager+from plugins import get_pluginfrom rest_framework.request import Requestfrom utils.file_storage.constants import FileStorageKeysfrom utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelperfrom utils.local_context import StateStore+from utils.subscription_usage_decorator import track_subscription_usage_if_available-from prompt_studio.modifier_loader import ModifierConfig-from prompt_studio.modifier_loader import load_plugins as load_modifier_plugins-from prompt_studio.processor_loader import get_plugin_class_by_name-from prompt_studio.processor_loader import load_plugins as load_processor_pluginsfrom prompt_studio.prompt_profile_manager_v2.models import ProfileManagerfrom prompt_studio.prompt_profile_manager_v2.profile_manager_helper import (ProfileManagerHelper,@@ -91,14 +89,10 @@ ERROR_MSG = "User %s doesn't have access to adapter %s"logger = logging.getLogger(__name__)-modifier_plugins = load_modifier_plugins()-class PromptStudioHelper:"""Helper class for Custom tool operations."""- processor_plugins = load_processor_plugins()-@staticmethoddef create_default_profile_manager(user: User, tool_id: uuid) -> None:"""Create a default profile manager for a given user and tool.@@ -292,11 +286,11 @@ class PromptStudioHelper:choices = f.read()f.close()response: dict[str, Any] = json.loads(choices)- for modifier_plugin in modifier_plugins:- cls = modifier_plugin[ModifierConfig.METADATA][- ModifierConfig.METADATA_SERVICE_CLASS- ]- response = cls.update_select_choices(default_choices=response)+ # Update select choices with payload modifier plugin if available+ payload_modifier_plugin = get_plugin("payload_modifier")+ if payload_modifier_plugin:+ modifier_service = payload_modifier_plugin["service_class"]()+ response = modifier_service.update_select_choices(default_choices=response)return response@staticmethod@@ -328,6 +322,7 @@ class PromptStudioHelper:return prompt_instances@staticmethod+ @track_subscription_usage_if_available(file_execution_id_param="run_id")def index_document(tool_id: str,file_name: str,@@ -487,17 +482,15 @@ class PromptStudioHelper:@staticmethoddef summarize(file_name, org_id, document_id, run_id, tool, doc_id) -> str:- cls = get_plugin_class_by_name(- name="summarizer",- plugins=PromptStudioHelper.processor_plugins,- )+ summarizer_plugin = get_plugin("summarizer")usage_kwargs: dict[Any, Any] = dict()usage_kwargs[ToolStudioPromptKeys.RUN_ID] = run_idprompts: list[ToolStudioPrompt] = PromptStudioHelper.fetch_prompt_from_tool(tool.tool_id)- if cls:- summarize_file_path = cls.process(+ if summarizer_plugin:+ summarizer_service = summarizer_plugin["service_class"]()+ summarize_file_path = summarizer_service.process(tool_id=str(tool.tool_id),file_name=file_name,org_id=org_id,@@ -523,6 +516,7 @@ class PromptStudioHelper:return summarize_file_path@staticmethod+ @track_subscription_usage_if_available(file_execution_id_param="run_id")def prompt_responder(tool_id: str,org_id: str,@@ -591,10 +585,12 @@ class PromptStudioHelper:):prompt_instance = PromptStudioHelper._fetch_prompt_from_id(id)+ # Check if payload modifier plugin is available for table/record operations+ payload_modifier_plugin = get_plugin("payload_modifier")if (prompt_instance.enforce_type == TSPKeys.TABLEor prompt_instance.enforce_type == TSPKeys.RECORD- ) and not modifier_plugins:+ ) and not payload_modifier_plugin:raise OperationNotSupported()prompt_name = prompt_instance.prompt_key@@ -1038,11 +1034,11 @@ class PromptStudioHelper:org_id, user_id, tool_id, doc_name, TSPKeys.EXTRACT))- for modifier_plugin in modifier_plugins:- cls = modifier_plugin[ModifierConfig.METADATA][- ModifierConfig.METADATA_SERVICE_CLASS- ]- output = cls.update(+ # Update output with payload modifier plugin if available+ payload_modifier_plugin = get_plugin("payload_modifier")+ if payload_modifier_plugin:+ modifier_service = payload_modifier_plugin["service_class"]()+ output = modifier_service.update(output=output,tool_id=tool_id,prompt_id=str(prompt.prompt_id),diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.pyindex b6495404b..9743e348e 100644--- a/backend/prompt_studio/prompt_studio_core_v2/views.py+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py@@ -11,6 +11,7 @@ from django.http import HttpRequest, HttpResponsefrom file_management.constants import FileInformationKey as FileKeyfrom file_management.exceptions import FileNotFoundfrom permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg+from plugins import get_pluginfrom rest_framework import status, viewsetsfrom rest_framework.decorators import actionfrom rest_framework.request import Request@@ -21,8 +22,6 @@ from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFilfrom utils.user_context import UserContextfrom utils.user_session import UserSessionUtils-from prompt_studio.processor_loader import get_plugin_class_by_name-from prompt_studio.processor_loader import load_plugins as load_processor_pluginsfrom prompt_studio.prompt_profile_manager_v2.constants import (ProfileManagerErrors,ProfileManagerKeys,@@ -88,8 +87,6 @@ class PromptStudioCoreView(viewsets.ModelViewSet):serializer_class = CustomToolSerializer- processor_plugins = load_processor_plugins()-def get_permissions(self) -> list[Any]:if self.action == "destroy":return [IsOwner()]@@ -162,35 +159,37 @@ class PromptStudioCoreView(viewsets.ModelViewSet):# Send email notifications to newly shared usersif response.status_code == 200 and "shared_users" in request.data:- from plugins.notification.constants import ResourceType- from plugins.notification.sharing_notification import (- SharingNotificationService,- )-- # Refresh the object to get updated shared_users- custom_tool.refresh_from_db()- updated_shared_users = set(custom_tool.shared_users.all())-- # Find newly added users (not previously shared)- newly_shared_users = updated_shared_users - current_shared_users-- if newly_shared_users:- notification_service = SharingNotificationService()- try:- notification_service.send_sharing_notification(- resource_type=ResourceType.TEXT_EXTRACTOR.value,- resource_name=custom_tool.tool_name,- resource_id=str(custom_tool.tool_id),- shared_by=request.user,- shared_to=list(newly_shared_users),- resource_instance=custom_tool,- )- except Exception as e:- # Log error but don't fail the request- logger.exception(- f"Failed to send sharing notification for "- f"custom tool {custom_tool.tool_id}: {str(e)}"- )+ from plugins import get_plugin++ notification_plugin = get_plugin("notification")+ if notification_plugin:+ from plugins.notification.constants import ResourceType++ # Refresh the object to get updated shared_users+ custom_tool.refresh_from_db()+ updated_shared_users = set(custom_tool.shared_users.all())++ # Find newly added users (not previously shared)+ newly_shared_users = updated_shared_users - current_shared_users++ if newly_shared_users:+ service_class = notification_plugin["service_class"]+ notification_service = service_class()+ try:+ notification_service.send_sharing_notification(+ resource_type=ResourceType.TEXT_EXTRACTOR.value,+ resource_name=custom_tool.tool_name,+ resource_id=str(custom_tool.tool_id),+ shared_by=request.user,+ shared_to=list(newly_shared_users),+ resource_instance=custom_tool,+ )+ except Exception as e:+ # Log error but don't fail the request+ logger.exception(+ f"Failed to send sharing notification for "+ f"custom tool {custom_tool.tool_id}: {str(e)}"+ )return response@@ -437,14 +436,14 @@ class PromptStudioCoreView(viewsets.ModelViewSet):document: DocumentManager = DocumentManager.objects.get(pk=document_id)file_name: str = document.document_nameview_type: str = serializer.validated_data.get("view_type")- file_converter = get_plugin_class_by_name(- name="file_converter",- plugins=self.processor_plugins,- )+ file_converter_plugin = get_plugin("file_converter")allowed_content_types = FileKey.FILE_UPLOAD_ALLOWED_MIME- if file_converter:- allowed_content_types = file_converter.get_extented_file_information_key()+ if file_converter_plugin:+ file_converter_service = file_converter_plugin["service_class"]()+ allowed_content_types = (+ file_converter_service.get_extented_file_information_key()+ )filename_without_extension = file_name.rsplit(".", 1)[0]if view_type == FileViewTypes.EXTRACT:file_name = (@@ -472,10 +471,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):serializer = FileUploadIdeSerializer(data=request.data)serializer.is_valid(raise_exception=True)uploaded_files: Any = serializer.validated_data.get("file")- file_converter = get_plugin_class_by_name(- name="file_converter",- plugins=self.processor_plugins,- )+ file_converter_plugin = get_plugin("file_converter")documents = []for uploaded_file in uploaded_files:@@ -484,8 +480,9 @@ class PromptStudioCoreView(viewsets.ModelViewSet):file_data = uploaded_filefile_type = uploaded_file.content_type# Convert non-PDF files- if file_converter and file_type != "application/pdf":- file_data, file_name = file_converter.process_file(+ if file_converter_plugin and file_type != "application/pdf":+ file_converter_service = file_converter_plugin["service_class"]()+ file_data, file_name = file_converter_service.process_file(uploaded_file, file_name)diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.pyindex 8832055cb..724010ce1 100644--- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py+++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py@@ -5,9 +5,8 @@ from account_v2.models import Userfrom adapter_processor_v2.models import AdapterInstancefrom django.conf import settingsfrom django.db import IntegrityError+from plugins import get_plugin-from prompt_studio.modifier_loader import ModifierConfig-from prompt_studio.modifier_loader import load_plugins as load_modifier_pluginsfrom prompt_studio.prompt_profile_manager_v2.models import ProfileManagerfrom prompt_studio.prompt_studio_core_v2.models import CustomToolfrom prompt_studio.prompt_studio_core_v2.prompt_studio_helper import PromptStudioHelper@@ -28,7 +27,6 @@ from .models import PromptStudioRegistryfrom .serializers import PromptStudioRegistrySerializerlogger = logging.getLogger(__name__)-modifier_plugins = load_modifier_plugins()class PromptStudioRegistryHelper:@@ -354,11 +352,10 @@ class PromptStudioRegistryHelper:or prompt.enforce_type == PromptStudioRegistryKeys.RECORDor prompt.enforce_type == PromptStudioRegistryKeys.LINE_ITEM):- for modifier_plugin in modifier_plugins:- cls = modifier_plugin[ModifierConfig.METADATA][- ModifierConfig.METADATA_SERVICE_CLASS- ]- output = cls.update(+ payload_modifier_plugin = get_plugin("payload_modifier")+ if payload_modifier_plugin:+ modifier_service = payload_modifier_plugin["service_class"]()+ output = modifier_service.update(output=output,tool_id=tool.tool_id,prompt_id=prompt.prompt_id,diff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.pyindex a532c1453..ad8c94306 100644--- a/backend/scheduler/tasks.py+++ b/backend/scheduler/tasks.py@@ -3,7 +3,6 @@ import loggingimport tracebackfrom typing import Any-from account_v2.subscription_loader import load_plugins, validate_etl_runfrom celery import shared_taskfrom django_celery_beat.models import CrontabSchedule, PeriodicTaskfrom pipeline_v2.models import Pipeline@@ -12,7 +11,6 @@ from utils.user_context import UserContextfrom workflow_manager.workflow_v2.workflow_helper import WorkflowHelperlogger = logging.getLogger(__name__)-subscription_loader = load_plugins()def create_or_update_periodic_task(@@ -95,20 +93,25 @@ def execute_pipeline_task_v2(f"Executing pipeline: {pipeline_id}, "f"workflow: {workflow}, pipeline name: {pipeline_name}")- if (- subscription_loader- and subscription_loader[0]- and not validate_etl_run(organization_id)- ):- try:- logger.info(- f"Subscription expired for '{organization_id}', "- f"disabling pipeline: {pipeline_id}"- )- disable_task(pipeline_id)- except Exception as e:- logger.warning(f"Failed to disable task: {pipeline_id}. Error: {e}")- return+ # Check subscription status before ETL run (cloud-specific)+ try:+ from pluggable_apps.subscription_v2.subscription_helper import (+ SubscriptionHelper,+ )++ if not SubscriptionHelper.validate_etl_run(organization_id):+ try:+ logger.info(+ f"Subscription expired for '{organization_id}', "+ f"disabling pipeline: {pipeline_id}"+ )+ disable_task(pipeline_id)+ except Exception as e:+ logger.warning(f"Failed to disable task: {pipeline_id}. Error: {e}")+ return+ except ModuleNotFoundError:+ # Subscription module not available (OSS deployment), skip validation+ passPipelineProcessor.update_pipeline(pipeline_id, Pipeline.PipelineStatus.INPROGRESS)# Mark the File in file history to avoid duplicate execution# only for ETL and TASK executiondiff --git a/backend/utils/subscription_usage_decorator.py b/backend/utils/subscription_usage_decorator.pynew file mode 100644index 000000000..5a84a2e87--- /dev/null+++ b/backend/utils/subscription_usage_decorator.py@@ -0,0 +1,139 @@+"""Decorator and utilities for automatic subscription usage tracking.++This module provides a decorator to automatically handle subscription usage+commit/discard operations, reducing boilerplate code and ensuring consistent+behavior across the codebase.+"""++import functools+import logging+from collections.abc import Callable+from datetime import datetime+from typing import Any++from plugins import get_plugin++from utils.user_session import UserSessionUtils++logger = logging.getLogger(__name__)+++def track_subscription_usage_if_available(+ file_execution_id_param: str,+ org_id_param: str = "org_id",+ extract_from_request: bool = False,+) -> Callable:+ """Decorator to automatically track subscription usage.++ Automatically commits subscription usage on successful function execution+ and discards it on failure. This eliminates repetitive try/except blocks+ for subscription tracking.++ Args:+ file_execution_id_param: Name of the parameter containing file_execution_id+ org_id_param: Name of the parameter containing org_id (default: "org_id")+ extract_from_request: If True, extracts org_id from Django request object+ using UserSessionUtils.get_organization_id()++ Returns:+ Decorated function++ Example:+ @track_subscription_usage_if_available(file_execution_id_param="run_id", extract_from_request=True)+ def index_document(self, request, pk=None):+ # Your code here - subscription usage tracked automatically+ return result++ Note:+ - Subscription tracking errors are logged but don't affect main function+ - If identifiers are missing, tracking is silently skipped+ - Works with both regular functions and Django view methods+ """++ def decorator(func: Callable) -> Callable:+ @functools.wraps(func)+ def wrapper(*args: Any, **kwargs: Any) -> Any:+ # Extract file_execution_id+ file_execution_id = kwargs.get(file_execution_id_param)++ # Extract org_id+ if extract_from_request:+ # Handle both function(request) and method(self, request) patterns+ request = kwargs.get("request")+ if not request and len(args) >= 2:+ # Assume args[0] is self, args[1] is request for methods+ request = args[1]+ elif not request and len(args) >= 1 and "self" not in kwargs:+ # Assume args[0] is request for functions+ request = args[0]++ if request:+ org_id = UserSessionUtils.get_organization_id(request)+ else:+ org_id = None+ logger.warning(+ f"Could not extract request object in {func.__name__}, "+ "subscription tracking skipped"+ )+ else:+ org_id = kwargs.get(org_id_param)++ # Skip tracking if identifiers are missing+ if not file_execution_id or not org_id:+ logger.debug(+ f"Skipping subscription tracking in {func.__name__}: "+ f"file_execution_id={file_execution_id}, org_id={org_id}"+ )+ return func(*args, **kwargs)++ try:+ # Execute main function+ result = func(*args, **kwargs)++ # Commit subscription usage on success+ subscription_usage_plugin = get_plugin("subscription_usage")+ if subscription_usage_plugin:+ try:+ service = subscription_usage_plugin["service_class"]()+ service.commit_subscription_usage(+ org_id=org_id,+ date_key=datetime.now().date(),+ file_execution_id=file_execution_id,+ )+ logger.info(+ f"Committed subscription usage for file_execution_id {file_execution_id}"+ )+ except Exception as commit_error:+ logger.error(+ f"Error committing subscription usage for "+ f"file_execution_id {file_execution_id}: {commit_error}"+ )++ return result++ except Exception:+ # Discard subscription usage on failure+ subscription_usage_plugin = get_plugin("subscription_usage")+ if subscription_usage_plugin:+ try:+ service = subscription_usage_plugin["service_class"]()+ service.discard_subscription_usage(+ org_id=org_id,+ date_key=datetime.now().date(),+ file_execution_id=file_execution_id,+ )+ logger.info(+ f"Discarded subscription usage for file_execution_id {file_execution_id}"+ )+ except Exception as discard_error:+ logger.error(+ f"Error discarding subscription usage for "+ f"file_execution_id {file_execution_id}: {discard_error}"+ )++ # Re-raise original exception+ raise++ return wrapper++ return decoratordiff --git a/backend/workflow_manager/workflow_v2/file_execution_tasks.py b/backend/workflow_manager/workflow_v2/file_execution_tasks.pyindex c8b8ca0a0..a9b3ab871 100644--- a/backend/workflow_manager/workflow_v2/file_execution_tasks.py+++ b/backend/workflow_manager/workflow_v2/file_execution_tasks.py@@ -369,6 +369,25 @@ class FileExecutionTasks:successful_files=total_successful,failed_files=total_failed,)++ # TODO: Remove related code when v1 workers are deprecated and removed+ from plugins import get_plugin++ subscription_usage_plugin = get_plugin("subscription_usage")+ if subscription_usage_plugin:+ try:+ service = subscription_usage_plugin["service_class"]()+ service.handle_workflow_execution_completion(+ organization_id=organization_id,+ execution_id=execution_id,+ final_status=final_status,+ total_successful=total_successful,+ )+ except Exception as e:+ logger.error(+ f"Error in subscription usage plugin completion handler: {e}"+ )+logger.info(f"Execution completed for execution id: '{execution_id}' "f"with status: '{final_status}' "diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.pyindex 3304952de..7bf448f04 100644--- a/backend/workflow_manager/workflow_v2/views.py+++ b/backend/workflow_manager/workflow_v2/views.py@@ -11,6 +11,7 @@ from django.views.decorators.csrf import csrf_exemptfrom permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrgfrom pipeline_v2.models import Pipelinefrom pipeline_v2.pipeline_processor import PipelineProcessor+from plugins import get_pluginfrom rest_framework import serializers, status, viewsetsfrom rest_framework.decorators import action, api_viewfrom rest_framework.request import Request@@ -20,17 +21,6 @@ from rest_framework.views import APIViewfrom utils.filtering import FilterHelperfrom utils.organization_utils import filter_queryset_by_organization, resolve_organization-try:- from plugins.notification.constants import ResourceType- from plugins.notification.sharing_notification import SharingNotificationService-- NOTIFICATION_PLUGIN_AVAILABLE = True- sharing_notification_service = SharingNotificationService()-except ImportError:- NOTIFICATION_PLUGIN_AVAILABLE = False- sharing_notification_service = None--from backend.constants import RequestKeyfrom unstract.core.data_models import FileHistoryCreateRequestfrom workflow_manager.endpoint_v2.destination import DestinationConnector@@ -68,6 +58,14 @@ from workflow_manager.workflow_v2.workflow_helper import (WorkflowSchemaHelper,)+# Check if notification plugin is available+notification_plugin = get_plugin("notification")+NOTIFICATION_PLUGIN_AVAILABLE = notification_plugin is not None++# Import constants from notification plugin if available+if NOTIFICATION_PLUGIN_AVAILABLE:+ from plugins.notification.constants import ResourceType+logger = logging.getLogger(__name__)@@ -169,8 +167,10 @@ class WorkflowViewSet(viewsets.ModelViewSet):newly_shared_users = new_shared_users - current_shared_usersif newly_shared_users:- # Send notification to newly shared users- sharing_notification_service.send_sharing_notification(+ # Get notification service from plugin and send notification+ service_class = notification_plugin["service_class"]+ notification_service = service_class()+ notification_service.send_sharing_notification(resource_type=ResourceType.WORKFLOW.value,resource_name=workflow.workflow_name,resource_id=str(workflow.id),@@ -858,11 +858,11 @@ def file_history_batch_lookup_internal(request):)logger.info(- f"DEBUG: FileHistoryBatch - File {i+1}: {file_data.get('file_path', 'NO_PATH')}"+ f"DEBUG: FileHistoryBatch - File {i + 1}: {file_data.get('file_path', 'NO_PATH')}"f" (identifier: {identifier})")logger.info(- f"DEBUG: FileHistoryBatch - File {i+1} data: cache_key={file_data.get('cache_key')}, "+ f"DEBUG: FileHistoryBatch - File {i + 1} data: cache_key={file_data.get('cache_key')}, "f"provider_file_uuid={file_data.get('provider_file_uuid')}, "f"file_path={file_data.get('file_path')}")@@ -879,7 +879,7 @@ def file_history_batch_lookup_internal(request):# Legacy mapping (may have collisions)file_identifiers[file_data["cache_key"]] = identifierlogger.info(- f"DEBUG: FileHistoryBatch - File {i+1}: Using cache_key={file_data['cache_key']}, "+ f"DEBUG: FileHistoryBatch - File {i + 1}: Using cache_key={file_data['cache_key']}, "f"mapped to identifier={identifier}")elif file_data.get("provider_file_uuid"):@@ -891,12 +891,12 @@ def file_history_batch_lookup_internal(request):# Legacy mapping (may have collisions)file_identifiers[file_data["provider_file_uuid"]] = identifierlogger.info(- f"DEBUG: FileHistoryBatch - File {i+1}: Using provider_file_uuid={file_data['provider_file_uuid']}, "+ f"DEBUG: FileHistoryBatch - File {i + 1}: Using provider_file_uuid={file_data['provider_file_uuid']}, "f"mapped to identifier={identifier}")else:logger.warning(- f"DEBUG: FileHistoryBatch - File {i+1}: No cache_key or provider_file_uuid!"+ f"DEBUG: FileHistoryBatch - File {i + 1}: No cache_key or provider_file_uuid!")continue@@ -910,18 +910,18 @@ def file_history_batch_lookup_internal(request):# Combine: match either exact path OR legacy null pathfilters &= cache_key_filters & (path_filters | fallback_filters)logger.info(- f"DEBUG: FileHistoryBatch - File {i+1}: Added file_path constraint={file_data['file_path']} "+ f"DEBUG: FileHistoryBatch - File {i + 1}: Added file_path constraint={file_data['file_path']} "f"with legacy fallback (file_path__isnull=True)")else:# No file_path provided, only search legacy recordsfilters &= cache_key_filters & Q(file_path__isnull=True)logger.info(- f"DEBUG: FileHistoryBatch - File {i+1}: No file_path provided, using legacy fallback only"+ f"DEBUG: FileHistoryBatch - File {i + 1}: No file_path provided, using legacy fallback only")logger.info(- f"DEBUG: FileHistoryBatch - File {i+1}: Final query filters: {filters}"+ f"DEBUG: FileHistoryBatch - File {i + 1}: Final query filters: {filters}")queries.append(filters)@@ -955,7 +955,7 @@ def file_history_batch_lookup_internal(request):)for i, fh in enumerate(file_histories):logger.info(- f"DEBUG: FileHistoryBatch - DB Record {i+1}: "+ f"DEBUG: FileHistoryBatch - DB Record {i + 1}: "f"provider_file_uuid={fh.provider_file_uuid}, "f"cache_key={fh.cache_key}, "f"file_path={fh.file_path}, "@@ -988,7 +988,7 @@ def file_history_batch_lookup_internal(request):for i, fh in enumerate(file_histories):logger.info(- f"DEBUG: FileHistoryBatch - Processing DB record {i+1}: "+ f"DEBUG: FileHistoryBatch - Processing DB record {i + 1}: "f"cache_key={fh.cache_key}, provider_file_uuid={fh.provider_file_uuid}, file_path={fh.file_path}")@@ -1000,7 +1000,7 @@ def file_history_batch_lookup_internal(request):if composite_key in composite_file_map:matched_identifiers.append(composite_file_map[composite_key])logger.info(- f"DEBUG: FileHistoryBatch - DB record {i+1}: Matched by composite cache_key={composite_key} "+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: Matched by composite cache_key={composite_key} "f"-> identifier={composite_file_map[composite_key]}")@@ -1009,7 +1009,7 @@ def file_history_batch_lookup_internal(request):if composite_key in composite_file_map:matched_identifiers.append(composite_file_map[composite_key])logger.info(- f"DEBUG: FileHistoryBatch - DB record {i+1}: Matched by composite uuid={composite_key} "+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: Matched by composite uuid={composite_key} "f"-> identifier={composite_file_map[composite_key]}")@@ -1018,13 +1018,13 @@ def file_history_batch_lookup_internal(request):if fh.cache_key and fh.cache_key in file_identifiers:matched_identifiers.append(file_identifiers[fh.cache_key])logger.info(- f"DEBUG: FileHistoryBatch - DB record {i+1}: Matched by legacy cache_key={fh.cache_key} "+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: Matched by legacy cache_key={fh.cache_key} "f"-> identifier={file_identifiers[fh.cache_key]}")elif fh.provider_file_uuid and fh.provider_file_uuid in file_identifiers:matched_identifiers.append(file_identifiers[fh.provider_file_uuid])logger.info(- f"DEBUG: FileHistoryBatch - DB record {i+1}: Matched by legacy provider_file_uuid={fh.provider_file_uuid} "+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: Matched by legacy provider_file_uuid={fh.provider_file_uuid} "f"-> identifier={file_identifiers[fh.provider_file_uuid]}")@@ -1037,7 +1037,7 @@ def file_history_batch_lookup_internal(request):potential_matches.append((req_identifier, req_data))logger.info(- f"DEBUG: FileHistoryBatch - DB record {i+1}: Found {len(potential_matches)} potential UUID matches"+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: Found {len(potential_matches)} potential UUID matches")# Try to match by file path@@ -1046,7 +1046,7 @@ def file_history_batch_lookup_internal(request):if fh.file_path == req_path:matched_identifiers.append(req_identifier)logger.info(- f"DEBUG: FileHistoryBatch - DB record {i+1}: Matched by manual path comparison "+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: Matched by manual path comparison "f"db_path={fh.file_path} == req_path={req_path} -> identifier={req_identifier}")break@@ -1054,7 +1054,7 @@ def file_history_batch_lookup_internal(request):# If still no exact path match, but we have UUID matches, log for fallback handlingif not matched_identifiers and potential_matches:logger.warning(- f"DEBUG: FileHistoryBatch - DB record {i+1}: UUID collision detected! "+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: UUID collision detected! "f"DB path={fh.file_path} doesn't match any request paths: "f"{[req_data.get('file_path') for _, req_data in potential_matches]}")@@ -1062,7 +1062,7 @@ def file_history_batch_lookup_internal(request):# Process all matched identifiersif not matched_identifiers:logger.warning(- f"DEBUG: FileHistoryBatch - DB record {i+1}: NO MATCH FOUND! "+ f"DEBUG: FileHistoryBatch - DB record {i + 1}: NO MATCH FOUND! "f"cache_key={fh.cache_key}, provider_file_uuid={fh.provider_file_uuid}, file_path={fh.file_path}")diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex e106dde2d..99a4f55fe 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -236,6 +236,24 @@ class WorkflowHelper:status=ExecutionStatus.ERROR,error=f"Error while processing files: {str(e)}",)++ # TODO: Remove related code when v1 workers are deprecated and removed+ from plugins import get_plugin++ organization_id = workflow_execution.workflow.organization.organization_id+ subscription_usage_plugin = get_plugin("subscription_usage")+ if subscription_usage_plugin:+ try:+ service = subscription_usage_plugin["service_class"]()+ service.handle_workflow_execution_failure(+ organization_id=organization_id,+ execution_id=str(workflow_execution.id),+ )+ except Exception as e:+ logger.error(+ f"Error in subscription usage plugin failure handler: {e}"+ )+logger.error(f"Execution {workflow_execution.id} failed: {str(e)}", exc_info=True)diff --git a/platform-service/.gitignore b/platform-service/.gitignoreindex 4b3ab3003..3b35241ca 100644--- a/platform-service/.gitignore+++ b/platform-service/.gitignore@@ -1,162 +1,2 @@-# Byte-compiled / optimized / DLL files-__pycache__/-*.py[cod]-*$py.class--# C extensions-*.so--# Distribution / packaging-.Python-build/-develop-eggs/-dist/-downloads/-eggs/-.eggs/-lib/-lib64/-parts/-sdist/-var/-wheels/-share/python-wheels/-*.egg-info/-.installed.cfg-*.egg-MANIFEST--# PyInstaller-# Usually these files are written by a python script from a template-# before PyInstaller builds the exe, so as to inject date/other infos into it.-*.manifest-*.spec--# Installer logs-pip-log.txt-pip-delete-this-directory.txt--# Unit test / coverage reports-htmlcov/-.tox/-.nox/-.coverage-.coverage.*-.cache-nosetests.xml-coverage.xml-*.cover-*.py,cover-.hypothesis/-.pytest_cache/-cover/--# Translations-*.mo-*.pot--# Django stuff:-*.log-local_settings.py-db.sqlite3-db.sqlite3-journal--# Flask stuff:-instance/-.webassets-cache--# Scrapy stuff:-.scrapy--# Sphinx documentation-docs/_build/--# PyBuilder-.pybuilder/-target/--# Jupyter Notebook-.ipynb_checkpoints--# IPython-profile_default/-ipython_config.py--# pyenv-# For a library or package, you might want to ignore these files since the code is-# intended to run in multiple environments; otherwise, check them in:-# .python-version--# pipenv-# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.-# However, in case of collaboration, if having platform-specific dependencies or dependencies-# having no cross-platform support, pipenv may install dependencies that don't work, or not-# install all needed dependencies.-#Pipfile.lock--# poetry-# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.-# This is especially recommended for binary packages to ensure reproducibility, and is more-# commonly ignored for libraries.-# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control-#poetry.lock--# pdm-# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.-#pdm.lock-# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it-# in version control.-# https://pdm.fming.dev/#use-with-ide-.pdm.toml-.pdm-python-.pdm-build/--# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm-__pypackages__/--# Celery stuff-celerybeat-schedule-celerybeat.pid--# SageMath parsed files-*.sage.py--# Environments-.env-.venv-env/-venv/-ENV/-env.bak/-venv.bak/--# Spyder project settings-.spyderproject-.spyproject--# Rope project settings-.ropeproject--# mkdocs documentation-/site--# mypy-.mypy_cache/-.dmypy.json-dmypy.json--# Pyre type checker-.pyre/--# pytype static type analyzer-.pytype/--# Cython debug symbols-cython_debug/--# PyCharm-# JetBrains specific template is maintained in a separate JetBrains.gitignore that can-# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore-# and can be added to the global gitignore or merged into this file. For a more nuclear-# option (not recommended) you can uncomment the following to ignore the entire idea folder.-#.idea/+# Plugins+src/unstract/platform_service/pluginsdiff --git a/platform-service/src/unstract/platform_service/config.py b/platform-service/src/unstract/platform_service/config.pyindex f6ae73583..08fff4130 100644--- a/platform-service/src/unstract/platform_service/config.py+++ b/platform-service/src/unstract/platform_service/config.py@@ -1,10 +1,15 @@import loggingimport os+from pathlib import Pathfrom dotenv import load_dotenvfrom flask import Flask-from unstract.core.flask import register_error_handlers, register_request_id_middleware+from unstract.core.flask import (+ PluginManager,+ register_error_handlers,+ register_request_id_middleware,+)from unstract.core.flask.logging import setup_loggingfrom unstract.platform_service.constants import LogLevelfrom unstract.platform_service.controller import api@@ -41,4 +46,10 @@ def create_app() -> Flask:options=f"-c application_name={Env.APPLICATION_NAME}",)+ # Load plugins (cloud plugins will be overlaid during build)+ plugins_dir = Path(__file__).parent / "plugins"+ plugins_pkg = "unstract.platform_service.plugins"+ manager = PluginManager(app, plugins_dir, plugins_pkg)+ manager.load_plugins()+return appdiff --git a/platform-service/src/unstract/platform_service/controller/platform.py b/platform-service/src/unstract/platform_service/controller/platform.pyindex 049989997..26a1e7493 100644--- a/platform-service/src/unstract/platform_service/controller/platform.py+++ b/platform-service/src/unstract/platform_service/controller/platform.py@@ -8,6 +8,7 @@ from cryptography.fernet import Fernet, InvalidTokenfrom flask import Blueprint, Request, jsonify, make_response, requestfrom flask import current_app as app+from unstract.core.flask import PluginManagerfrom unstract.core.flask.exceptions import APIErrorfrom unstract.platform_service.constants import DBTablefrom unstract.platform_service.env import Env@@ -163,79 +164,19 @@ def page_usage() -> Any:result["status"] = "OK"result["unique_id"] = usage_id- # Check if the subscription_usage table exists- check_table_query = f"""- SELECT EXISTS (- SELECT FROM information_schema.tables- WHERE table_schema = '{Env.DB_SCHEMA}'- AND table_name = '{DBTable.SUBSCRIPTION_USAGE}'- );- """- app.logger.info("Checking if subscription_usage table exists")-- table_exists = db.execute_sql(check_table_query).fetchone()[0]-- if table_exists:- app.logger.info("subscription_usage table exists")-- # Query to get subscription_id and stripe_subscription_id- # from Subscription table.- subscription_query = f"""- SELECT id, stripe_subscription_id- FROM \"{Env.DB_SCHEMA}\".{DBTable.SUBSCRIPTION}- WHERE organization_id = %s AND is_active = TRUE;- """- subscription_query_params = (org_id,)- app.logger.info(- "Executing subscription query for organization %s", org_id- )-- subscription_result = db.execute_sql(- subscription_query, subscription_query_params- ).fetchone()-- if subscription_result:- subscription_id, stripe_subscription_id = subscription_result- app.logger.info(- "Found subscription: id=%s, stripe_subscription_id=%s",- subscription_id,- stripe_subscription_id,+ # Cloud-only: Handle subscription usage via plugin+ usage_plugin = PluginManager().get_plugin("subscription_usage")+ if usage_plugin:+ try:+ handler = usage_plugin["entrypoint_cls"]()+ handler.handle_subscription_usage(+ org_id=org_id,+ page_count=page_count,+ run_id=run_id,+ current_time=current_time,)-- stripe_subscription_id = (- stripe_subscription_id if stripe_subscription_id else "trial"- )-- # Insert or update data in the subscription_usage table- subscription_usage_query = f"""- INSERT INTO- \"{Env.DB_SCHEMA}\".{DBTable.SUBSCRIPTION_USAGE} (- subscription_id, stripe_subscription_id,- organization_id, pages_processed, record_date,- created_at, updated_at- ) VALUES (%s, %s, %s, %s, %s, %s, %s)- ON CONFLICT (- subscription_id, stripe_subscription_id, record_date- ) DO UPDATE- SET pages_processed =- {DBTable.SUBSCRIPTION_USAGE}.pages_processed- + EXCLUDED.pages_processed;- """- subscription_usage_params = (- subscription_id,- stripe_subscription_id,- org_id,- page_count,- current_time.date(),- current_time,- current_time,- )- app.logger.info("Executing subscription usage insert/update query.")-- db.execute_sql(subscription_usage_query, subscription_usage_params)- app.logger.info("Subscription usage updated for %s", org_id)- else:- app.logger.warning("No active subscription found for %s", org_id)+ except Exception as e:+ app.logger.exception(f"Error from subscription usage plugin: {e}")return make_response(result, 200)except Exception as e:diff --git a/prompt-service/.gitignore b/prompt-service/.gitignoreindex df775c6bf..ae2f699ad 100644--- a/prompt-service/.gitignore+++ b/prompt-service/.gitignore@@ -1,164 +1,2 @@-# Byte-compiled / optimized / DLL files-__pycache__/-*.py[cod]-*$py.class--# C extensions-*.so--# Distribution / packaging-.Python-build/-develop-eggs/-dist/-downloads/-eggs/-.eggs/-lib/-lib64/-parts/-sdist/-var/-wheels/-share/python-wheels/-*.egg-info/-.installed.cfg-*.egg-MANIFEST--# PyInstaller-# Usually these files are written by a python script from a template-# before PyInstaller builds the exe, so as to inject date/other infos into it.-*.manifest-*.spec--# Installer logs-pip-log.txt-pip-delete-this-directory.txt--# Unit test / coverage reports-htmlcov/-.tox/-.nox/-.coverage-.coverage.*-.cache-nosetests.xml-coverage.xml-*.cover-*.py,cover-.hypothesis/-.pytest_cache/-cover/--# Translations-*.mo-*.pot--# Django stuff:-*.log-local_settings.py-db.sqlite3-db.sqlite3-journal--# Flask stuff:-instance/-.webassets-cache--# Scrapy stuff:-.scrapy--# Sphinx documentation-docs/_build/--# PyBuilder-.pybuilder/-target/--# Jupyter Notebook-.ipynb_checkpoints--# IPython-profile_default/-ipython_config.py--# pyenv-# For a library or package, you might want to ignore these files since the code is-# intended to run in multiple environments; otherwise, check them in:-# .python-version--# pipenv-# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.-# However, in case of collaboration, if having platform-specific dependencies or dependencies-# having no cross-platform support, pipenv may install dependencies that don't work, or not-# install all needed dependencies.-#Pipfile.lock--# poetry-# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.-# This is especially recommended for binary packages to ensure reproducibility, and is more-# commonly ignored for libraries.-# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control-#poetry.lock--# pdm-# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.-#pdm.lock-# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it-# in version control.-# https://pdm-project.org/#use-with-ide-.pdm.toml-.pdm-python-.pdm-build/--# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm-__pypackages__/--# Celery stuff-celerybeat-schedule-celerybeat.pid--# SageMath parsed files-*.sage.py--# Environments-.env-.venv-env/-venv/-ENV/-env.bak/-venv.bak/--# Spyder project settings-.spyderproject-.spyproject--# Rope project settings-.ropeproject--# mkdocs documentation-/site--# mypy-.mypy_cache/-.dmypy.json-dmypy.json--# Pyre type checker-.pyre/--# pytype static type analyzer-.pytype/--# Cython debug symbols-cython_debug/--# PyCharm-# JetBrains specific template is maintained in a separate JetBrains.gitignore that can-# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore-# and can be added to the global gitignore or merged into this file. For a more nuclear-# option (not recommended) you can uncomment the following to ignore the entire idea folder.-#.idea/--plugins+# Plugins+src/unstract/prompt_service/pluginsdiff --git a/prompt-service/src/unstract/prompt_service/config.py b/prompt-service/src/unstract/prompt_service/config.pyindex 7d72d92ff..4a43da7c0 100644--- a/prompt-service/src/unstract/prompt_service/config.py+++ b/prompt-service/src/unstract/prompt_service/config.py@@ -1,15 +1,19 @@import loggingimport warningsfrom os import environ as env+from pathlib import Pathfrom dotenv import load_dotenvfrom flask import Flask-from unstract.core.flask import register_error_handlers, register_request_id_middleware+from unstract.core.flask import (+ PluginManager,+ register_error_handlers,+ register_request_id_middleware,+)from unstract.core.flask.logging import setup_loggingfrom unstract.flags.feature_flag import check_feature_flag_statusfrom unstract.prompt_service.controllers import api-from unstract.prompt_service.helpers.plugin import plugin_loaderif check_feature_flag_status("sdk1"):from unstract.sdk1.constants import LogLevel@@ -42,7 +46,11 @@ def create_app() -> Flask:app.logger.info("Initializing Flask application...")# Load plugins- plugin_loader(app)+ plugins_dir = Path(__file__).parent / "plugins"+ plugins_pkg = "unstract.prompt_service.plugins"+ manager = PluginManager(app, plugins_dir, plugins_pkg)+ manager.load_plugins()+register_request_id_middleware(app)register_error_handlers(app)app.register_blueprint(api)diff --git a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.pyindex 974ba8ebf..4ef219b22 100644--- a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py+++ b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py@@ -5,13 +5,13 @@ from typing import Anyfrom flask import Blueprint, requestfrom flask import current_app as app+from unstract.core.flask import PluginManagerfrom unstract.core.flask.exceptions import APIErrorfrom unstract.flags.feature_flag import check_feature_flag_statusfrom unstract.prompt_service.constants import PromptServiceConstants as PSKeysfrom unstract.prompt_service.constants import RetrievalStrategy, RunLevelfrom unstract.prompt_service.exceptions import BadRequestfrom unstract.prompt_service.helpers.auth import AuthHelper-from unstract.prompt_service.helpers.plugin import PluginManagerfrom unstract.prompt_service.helpers.prompt_ide_base_tool import PromptServiceBaseToolfrom unstract.prompt_service.helpers.usage import UsageHelperfrom unstract.prompt_service.services.answer_prompt import AnswerPromptServicediff --git a/prompt-service/src/unstract/prompt_service/helpers/plugin.py b/prompt-service/src/unstract/prompt_service/helpers/plugin.pydeleted file mode 100644index 8cd6e9aba..000000000--- a/prompt-service/src/unstract/prompt_service/helpers/plugin.py+++ /dev/null@@ -1,78 +0,0 @@-import importlib-import os-from pathlib import Path-from typing import Any--from flask import Flask---class PluginManager:- _instance = None-- def __new__(cls, app: Flask | None = None) -> "PluginManager":- if cls._instance is None:- # Only create the instance if it doesn't already exist- cls._instance = super().__new__(cls)- cls._instance.plugins = {}- cls._instance.plugins_dir = Path(__file__).parent.parent / "plugins"- cls._instance.plugins_pkg = "unstract.prompt_service.plugins"- # Always update `app` if provided- if app:- cls._instance.app = app- return cls._instance-- def load_plugins(self) -> None:- """Loads plugins found in the plugins root dir."""- if not self.plugins_dir.exists():- self.app.logger.warning(- f"Plugins directory not found: {self.plugins_dir}. Skipping."- )- return-- self.app.logger.info(f"Loading plugins from: {self.plugins_dir}")-- for pkg in os.listdir(os.fspath(self.plugins_dir)):- if pkg.endswith(".so"):- pkg = pkg.split(".")[0]- pkg_anchor = f"{self.plugins_pkg}.{pkg}.src"- try:- module = importlib.import_module(pkg_anchor)- except ImportError as e:- self.app.logger.error(f"Failed to load plugin ({pkg}): {str(e)}")- continue-- metadata = getattr(module, "metadata", None)- if not metadata:- self.app.logger.warning(f"Skipping plugin ({pkg}): No metadata found.")- continue-- if metadata.get("disable", False):- self.app.logger.info(- f"Skipping disabled plugin: {pkg}"- f" v{metadata.get('version', 'unknown')}"- )- continue-- try:- self.plugins[metadata["name"]] = {- "version": metadata["version"],- "entrypoint_cls": metadata["entrypoint_cls"],- "exception_cls": metadata["exception_cls"],- }- if blueprint := metadata.get("blueprint"):- self.app.register_blueprint(blueprint)- self.app.logger.info(f"✔ Loaded plugin: {pkg} v{metadata['version']}")- except KeyError as e:- self.app.logger.error(f"Invalid metadata for plugin '{pkg}': {str(e)}")-- if not self.plugins:- self.app.logger.warning("⚠ No plugins loaded.")-- def get_plugin(self, name: str) -> dict[str, Any]:- """Get the plugin metadata by name."""- return self.plugins.get(name, {})---def plugin_loader(app: Flask) -> None:- manager = PluginManager(app)- manager.load_plugins()diff --git a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py b/prompt-service/src/unstract/prompt_service/services/answer_prompt.pyindex 2200c125a..0a5419135 100644--- a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py+++ b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py@@ -6,12 +6,12 @@ from urllib.parse import urlparsefrom flask import current_app as app+from unstract.core.flask import PluginManagerfrom unstract.core.flask.exceptions import APIErrorfrom unstract.flags.feature_flag import check_feature_flag_statusfrom unstract.prompt_service.constants import ExecutionSource, FileStorageKeys, RunLevelfrom unstract.prompt_service.constants import PromptServiceConstants as PSKeysfrom unstract.prompt_service.exceptions import RateLimitError-from unstract.prompt_service.helpers.plugin import PluginManagerfrom unstract.prompt_service.helpers.postprocessor import postprocess_datafrom unstract.prompt_service.utils.env_loader import get_env_or_diefrom unstract.prompt_service.utils.json_repair_helper import (@@ -222,6 +222,12 @@ class AnswerPromptService:storage_type=StorageType.SHARED_TEMPORARY,env_name=FileStorageKeys.TEMPORARY_REMOTE_STORAGE,)+ logger.info(+ f"Initializing highlight plugin with: "+ f"file_path={file_path}, "+ f"execution_source={execution_source}, "+ f"fs_instance={fs_instance}"+ )highlight_data = highlight_data_plugin["entrypoint_cls"](file_path=file_path,fs_instance=fs_instance,diff --git a/unstract/core/src/unstract/core/cache/redis_client.py b/unstract/core/src/unstract/core/cache/redis_client.pyindex 5480aca8f..12ababdba 100644--- a/unstract/core/src/unstract/core/cache/redis_client.py+++ b/unstract/core/src/unstract/core/cache/redis_client.py@@ -252,6 +252,7 @@ class RedisClient:REDIS_HOST: Redis host (default: localhost)REDIS_PORT: Redis port (default: 6379)REDIS_USER: Redis username (optional)+ REDIS_USERNAME: Redis username (optional, alternative to REDIS_USER)REDIS_PASSWORD: Redis password (optional)REDIS_DB: Redis database number (default: 0)@@ -263,7 +264,7 @@ class RedisClient:return cls(host=os.getenv("REDIS_HOST", "localhost"),port=int(os.getenv("REDIS_PORT", "6379")),- username=os.getenv("REDIS_USER"),+ username=os.getenv("REDIS_USER") or os.getenv("REDIS_USERNAME"),password=os.getenv("REDIS_PASSWORD"),db=int(os.getenv("REDIS_DB", "0")),)diff --git a/unstract/core/src/unstract/core/django/__init__.py b/unstract/core/src/unstract/core/django/__init__.pynew file mode 100644index 000000000..60ee5f46c--- /dev/null+++ b/unstract/core/src/unstract/core/django/__init__.py@@ -0,0 +1 @@+from .plugin import DjangoPluginManager, PluginManager, plugin_loader # noqa: F401diff --git a/unstract/core/src/unstract/core/django/plugin.py b/unstract/core/src/unstract/core/django/plugin.pynew file mode 100644index 000000000..6449b0037--- /dev/null+++ b/unstract/core/src/unstract/core/django/plugin.py@@ -0,0 +1,163 @@+"""Django-specific plugin manager wrapper.++This module provides Django integration for the generic PluginManager,+handling Django-specific features like app registry integration and Django+logging.+"""++import logging+from pathlib import Path+from typing import Any++from unstract.core.plugins import PluginManager as GenericPluginManager+++class DjangoPluginManager:+ """Django-specific plugin manager wrapper.++ Wraps the generic PluginManager with Django-specific functionality like+ Django app integration and logging.+ """++ _instance = None++ def __new__(+ cls,+ plugins_dir: Path | str,+ plugins_pkg: str,+ logger: logging.Logger | None = None,+ ) -> "DjangoPluginManager":+ """Create or return the singleton DjangoPluginManager instance.++ Args:+ plugins_dir: Directory containing plugins+ plugins_pkg: Python package path for plugins (e.g., 'myapp.plugins')+ logger: Logger instance (defaults to module logger)++ Returns:+ DjangoPluginManager singleton instance+ """+ if cls._instance is None:+ cls._instance = super().__new__(cls)+ cls._instance._initialized = False++ # Initialize or update plugin manager if parameters change+ if plugins_dir and plugins_pkg:+ plugins_dir_path = (+ Path(plugins_dir) if isinstance(plugins_dir, str) else plugins_dir+ )++ if (+ not cls._instance._initialized+ or cls._instance._plugins_dir != plugins_dir_path+ or cls._instance._plugins_pkg != plugins_pkg+ ):+ cls._instance._plugins_dir = plugins_dir_path+ cls._instance._plugins_pkg = plugins_pkg+ cls._instance._logger = logger or logging.getLogger(__name__)+ cls._instance._init_manager()++ return cls._instance++ def _init_manager(self) -> None:+ """Initialize the generic plugin manager."""+ self._manager = GenericPluginManager(+ plugins_dir=self._plugins_dir,+ plugins_pkg=self._plugins_pkg,+ logger=self._logger,+ use_singleton=True,+ registration_callback=None, # Django doesn't need special registration+ )+ self._initialized = True++ def load_plugins(self) -> None:+ """Load plugins using the generic manager."""+ if not self._initialized:+ raise RuntimeError(+ "DjangoPluginManager not initialized. "+ "Call with plugins_dir and plugins_pkg first."+ )+ self._manager.load_plugins()++ def get_plugin(self, name: str) -> dict[str, Any]:+ """Get plugin metadata by name.++ Args:+ name: Plugin name to retrieve++ Returns:+ Dictionary containing plugin metadata+ """+ if not self._initialized:+ return {}+ return self._manager.get_plugin(name)++ def has_plugin(self, name: str) -> bool:+ """Check if a plugin is loaded.++ Args:+ name: Plugin name to check++ Returns:+ bool: True if plugin exists+ """+ if not self._initialized:+ return False+ return self._manager.has_plugin(name)++ def get_all_plugins(self) -> dict[str, dict[str, Any]]:+ """Get all loaded plugins.++ Returns:+ Dictionary mapping plugin names to their metadata+ """+ if not self._initialized:+ return {}+ return self._manager.get_all_plugins()++ @property+ def plugins(self) -> dict[str, dict[str, Any]]:+ """Get all loaded plugins."""+ return self.get_all_plugins()+++# Maintain backward compatibility with common class name+PluginManager = DjangoPluginManager+++def plugin_loader(+ plugins_dir: Path | str,+ plugins_pkg: str,+ logger: logging.Logger | None = None,+) -> DjangoPluginManager:+ """Load plugins for a Django application.++ Convenience function to create a DjangoPluginManager instance and load plugins.++ Args:+ plugins_dir: Directory containing plugins+ plugins_pkg: Python package path for plugins (e.g., 'backend.plugins')+ logger: Logger instance (optional)++ Returns:+ DjangoPluginManager: The plugin manager instance++ Example:+ # In your Django app initialization (e.g., apps.py or __init__.py):+ from pathlib import Path+ from unstract.core.django import plugin_loader++ # Load plugins from backend/plugins directory+ plugins_dir = Path(__file__).parent / 'plugins'+ manager = plugin_loader(plugins_dir, 'backend.plugins')+ manager.load_plugins()++ # Later, check for plugins:+ if manager.has_plugin('subscription_usage'):+ plugin = manager.get_plugin('subscription_usage')+ service = plugin['service_class']()+ service.commit_usage(...)+ """+ manager = DjangoPluginManager(plugins_dir, plugins_pkg, logger)+ manager.load_plugins()+ return managerdiff --git a/unstract/core/src/unstract/core/flask/__init__.py b/unstract/core/src/unstract/core/flask/__init__.pyindex 94eae362e..d03add0f4 100644--- a/unstract/core/src/unstract/core/flask/__init__.py+++ b/unstract/core/src/unstract/core/flask/__init__.py@@ -1,2 +1,3 @@from .exceptions import register_error_handlers # noqa: F401from .middleware import register_request_id_middleware # noqa: F401+from .plugin import PluginManager, plugin_loader # noqa: F401diff --git a/unstract/core/src/unstract/core/flask/plugin.py b/unstract/core/src/unstract/core/flask/plugin.pynew file mode 100644index 000000000..51e263927--- /dev/null+++ b/unstract/core/src/unstract/core/flask/plugin.py@@ -0,0 +1,175 @@+"""Flask-specific plugin manager wrapper.++This module provides Flask integration for the generic PluginManager,+handling Flask-specific features like blueprint registration and app.logger+integration.+"""++import threading+from pathlib import Path+from typing import Any++from flask import Flask++from unstract.core.plugins import PluginManager as GenericPluginManager+++class FlaskPluginManager:+ """Flask-specific plugin manager wrapper.++ Wraps the generic PluginManager with Flask-specific functionality like+ blueprint registration and Flask app integration.+ """++ _instance = None+ _lock = threading.Lock()++ def __new__(+ cls,+ app: Flask | None = None,+ plugins_dir: Path | None = None,+ plugins_pkg: str | None = None,+ ) -> "FlaskPluginManager":+ """Create or return the singleton FlaskPluginManager instance.++ Args:+ app: Flask application instance+ plugins_dir: Directory containing plugins+ plugins_pkg: Python package path for plugins++ Returns:+ FlaskPluginManager singleton instance+ """+ with cls._lock:+ # Check inside lock to handle race condition+ if cls._instance is None:+ cls._instance = super().__new__(cls)+ cls._instance._initialized = False++ # Update app if provided+ if app:+ cls._instance.app = app++ # Initialize or update plugin manager if parameters change+ if (+ plugins_dir+ and plugins_pkg+ and hasattr(cls._instance, "app")+ and (+ not cls._instance._initialized+ or cls._instance._plugins_dir != plugins_dir+ or cls._instance._plugins_pkg != plugins_pkg+ )+ ):+ cls._instance._plugins_dir = plugins_dir+ cls._instance._plugins_pkg = plugins_pkg+ cls._instance._init_manager()++ return cls._instance++ def _init_manager(self) -> None:+ """Initialize the generic plugin manager with Flask-specific callback."""++ def flask_registration_callback(plugin_data: dict[str, Any]) -> None:+ """Flask-specific registration callback for blueprints."""+ metadata = plugin_data.get("metadata", {})+ if blueprint := metadata.get("blueprint"):+ self.app.register_blueprint(blueprint)+ self.app.logger.debug(+ f"Registered blueprint for plugin: {metadata.get('name', 'unknown')}"+ )++ self._manager = GenericPluginManager(+ plugins_dir=self._plugins_dir,+ plugins_pkg=self._plugins_pkg,+ logger=self.app.logger,+ use_singleton=True,+ registration_callback=flask_registration_callback,+ )+ self._initialized = True++ def load_plugins(self) -> None:+ """Load plugins using the generic manager."""+ if not self._initialized:+ raise RuntimeError(+ "FlaskPluginManager not initialized. "+ "Call with app, plugins_dir, and plugins_pkg first."+ )+ self._manager.load_plugins()++ def get_plugin(self, name: str) -> dict[str, Any]:+ """Get plugin metadata by name.++ Args:+ name: Plugin name to retrieve++ Returns:+ Dictionary containing plugin metadata+ """+ if not self._initialized:+ return {}+ return self._manager.get_plugin(name)++ def has_plugin(self, name: str) -> bool:+ """Check if a plugin is loaded.++ Args:+ name: Plugin name to check++ Returns:+ bool: True if plugin exists+ """+ if not self._initialized:+ return False+ return self._manager.has_plugin(name)++ @property+ def plugins(self) -> dict[str, dict[str, Any]]:+ """Get all loaded plugins."""+ if not self._initialized:+ return {}+ return self._manager.get_all_plugins()+++# Maintain backward compatibility with old class name+PluginManager = FlaskPluginManager+++def plugin_loader(+ app: Flask,+ plugins_dir: Path | None = None,+ plugins_pkg: str | None = None,+) -> None:+ """Load plugins for a Flask application.++ Convenience function to create a PluginManager instance and load plugins.++ Args:+ app: Flask application instance+ plugins_dir: Directory containing plugins (required)+ plugins_pkg: Python package path for plugins (required)++ Raises:+ ValueError: If plugins_dir or plugins_pkg is not provided++ Example:+ from pathlib import Path+ from unstract.core.flask import plugin_loader++ def create_app():+ app = Flask(__name__)+ plugins_dir = Path(__file__).parent / "plugins"+ plugin_loader(app, plugins_dir, "unstract.myservice.plugins")+ return app+ """+ if not plugins_dir or not plugins_pkg:+ raise ValueError(+ "Both 'plugins_dir' and 'plugins_pkg' are required for plugin_loader.\n\n"+ "Example usage:\n"+ " from pathlib import Path\n"+ " plugins_dir = Path(__file__).parent / 'plugins'\n"+ " plugin_loader(app, plugins_dir, 'unstract.myservice.plugins')"+ )++ manager = PluginManager(app, plugins_dir, plugins_pkg)+ manager.load_plugins()diff --git a/unstract/core/src/unstract/core/plugins/.gitignore b/unstract/core/src/unstract/core/plugins/.gitignorenew file mode 100644index 000000000..f81789a40--- /dev/null+++ b/unstract/core/src/unstract/core/plugins/.gitignore@@ -0,0 +1,7 @@+# Ignore all enterprise plugins (copied from cloud repo at build time)+*++# Keep core plugin infrastructure in OSS+!__init__.py+!plugin_manager.py+!README.mddiff --git a/unstract/core/src/unstract/core/plugins/README.md b/unstract/core/src/unstract/core/plugins/README.mdnew file mode 100644index 000000000..aaa2113df--- /dev/null+++ b/unstract/core/src/unstract/core/plugins/README.md@@ -0,0 +1,122 @@+# Unstract Plugin System++## Plugin Metadata Format++Every plugin must define a `metadata` dictionary in its `__init__.py` file. The plugin manager reads this metadata to discover, validate, and load plugins.++### Required Fields++| Field | Type | Description | Example |+|-------|------|-------------|---------|+| `name` | `str` | Unique plugin identifier | `"my_plugin"` |+| `version` | `str` | Semantic version string | `"1.0.0"` |++### Optional Fields++| Field | Type | Description | Used In |+|-------|------|-------------|---------|+| `description` | `str` | Human-readable description | All |+| `is_active` | `bool` | Enable/disable flag (default: `True`) | All |+| `disable` | `bool` | Explicit disable flag (checked first) | All |+| `entrypoint_cls` | `class` | Main handler class for the plugin | Flask|+| `exception_cls` | `class` | Custom exception class | Flask |+| `service_class` | `callable` | Service factory (usually lambda) | Django Backend |+| `blueprint` | `Blueprint` | Flask blueprint for routes | Flask only |++---++## Creating a New Plugin++### 1. Define Plugin Metadata++**File:** `<service>/plugins/my_plugin/__init__.py`++```python+"""My custom plugin."""++from .service import MyService++metadata = {+ # Required+ "name": "my_plugin",+ "version": "1.0.0",++ # Optional+ "description": "Description of what this plugin does",+ "is_active": True,+ "service_class": lambda: MyService(),+}+```++### 2. Use the Plugin++**Django:**+```python+from backend.plugins import get_plugin++plugin = get_plugin("my_plugin")+result = plugin.some_method()+```++**Flask:**+```python+from unstract.core.flask import plugin_loader++# In create_app()+plugin_manager = plugin_loader(app, plugin_dirs=["service/plugins"])++# In controller+plugin = plugin_manager.get_plugin("my_plugin")+handler = plugin["entrypoint_cls"]()+result = handler.process()+```++---++## Plugin Loading Process++1. **Scans** the configured plugins directory+2. **Skips** items starting with `__` (like `__pycache__`)+3. **Imports** the plugin module (supports `.so` compiled extensions)+4. **Validates** that `metadata` dictionary exists+5. **Checks** if disabled via `metadata.get("disable", False)` or `not metadata.get("is_active", True)`+6. **Registers** plugin with all metadata fields+7. **Calls** registration callback if provided (e.g., for Flask blueprint registration)++## Using Shared Libraries in Plugins++This directory (`unstract.core.plugins`) can contain shared library code that can be used by plugins across all Unstract services (backend, platform-service, prompt-service, workers).++### Import Pattern++Plugins can import shared code as subpackages from `unstract.core.plugins`:++```python+# In your plugin code (e.g., backend/plugins/my_plugin/service.py)+from unstract.core.plugins.xxx import yyy+```++### Directory Structure++```+unstract/core/src/unstract/core/plugins/+├── README.md # This file+├── __init__.py # Package initialization+├── plugin_manager.py # Core plugin discovery and loading+```++### When to Add Shared Code Here++Add code to `unstract.core.plugins` when:++- ✅ Multiple plugins need the same functionality+- ✅ The code is framework-agnostic (works in Django, Flask, Workers)+- ✅ You want to avoid code duplication across services++Keep code in service-specific plugins when:++- ❌ Logic is specific to one service only+- ❌ Code depends on service-specific frameworks (Django ORM, Flask blueprints)+- ❌ Business logic is unique to that plugin++---diff --git a/unstract/core/src/unstract/core/plugins/__init__.py b/unstract/core/src/unstract/core/plugins/__init__.pynew file mode 100644index 000000000..365e95b58--- /dev/null+++ b/unstract/core/src/unstract/core/plugins/__init__.py@@ -0,0 +1,12 @@+"""Plugin system for Unstract.++This module provides the core plugin infrastructure for loading and managing+enterprise plugins that are injected at build time from the cloud repository.++The PluginManager supports framework-agnostic plugin loading with Flask and+Django-specific wrappers available in unstract.core.flask and unstract.core.django.+"""++from .plugin_manager import PluginManager++__all__ = ["PluginManager"]diff --git a/unstract/core/src/unstract/core/plugins/plugin_manager.py b/unstract/core/src/unstract/core/plugins/plugin_manager.pynew file mode 100644index 000000000..1b091d9cf--- /dev/null+++ b/unstract/core/src/unstract/core/plugins/plugin_manager.py@@ -0,0 +1,282 @@+"""Framework-agnostic plugin manager for loading and managing plugins dynamically.++This module provides a generic PluginManager class that can be used across+different frameworks (Flask, Django, etc.) to load and manage plugins with+consistent behavior.+"""++import importlib+import logging+from collections.abc import Callable+from pathlib import Path+from typing import Any+++class PluginManager:+ """Generic plugin manager for loading and managing application plugins.++ This class provides framework-agnostic plugin loading with support for:+ - Dynamic plugin discovery from directories+ - Metadata validation+ - Optional singleton pattern+ - Framework-specific registration callbacks+ - Compiled (.so) and Python module support+ """++ _instance = None+ _use_singleton = True++ def __init__(+ self,+ plugins_dir: Path | str,+ plugins_pkg: str,+ logger: logging.Logger | None = None,+ use_singleton: bool = True,+ registration_callback: Callable[[dict[str, Any]], None] | None = None,+ plugin_submodule: str | None = None,+ ):+ """Initialize the PluginManager.++ Args:+ plugins_dir: Directory containing plugins+ plugins_pkg: Python package path for plugins (e.g., 'myapp.plugins')+ logger: Logger instance for logging (defaults to module logger)+ use_singleton: Whether to use singleton pattern (default: True)+ registration_callback: Optional callback for framework-specific registration+ (e.g., Flask blueprint registration, Django URL patterns)+ plugin_submodule: Submodule path within each plugin (e.g., 'src').+ Set to None to import plugin directory directly.+ Default: None (imports from plugin root).+ """+ self.plugins_dir = (+ Path(plugins_dir) if isinstance(plugins_dir, str) else plugins_dir+ )+ self.plugins_pkg = plugins_pkg+ self.logger = logger or logging.getLogger(__name__)+ self.plugins: dict[str, dict[str, Any]] = {}+ self.registration_callback = registration_callback+ self._use_singleton = use_singleton+ self.plugin_submodule = plugin_submodule++ def __new__(cls, *args, **kwargs):+ """Create singleton instance if use_singleton is True."""+ use_singleton = kwargs.get("use_singleton", True)++ if use_singleton:+ if cls._instance is None:+ cls._instance = super().__new__(cls)+ return cls._instance+ else:+ # Create new instance if singleton is disabled+ return super().__new__(cls)++ def _find_plugin_modules(+ self, base_path: Path, max_depth: int = 2+ ) -> list[tuple[Path, str]]:+ """Recursively find plugin directories containing valid metadata.++ Scans the plugin directory tree up to max_depth levels to find directories+ with __init__.py that contain plugin metadata. This supports both flat+ plugin structures (plugins/plugin_name/) and nested structures+ (plugins/category/plugin_name/).++ Args:+ base_path: Root directory to start searching from+ max_depth: Maximum directory depth to search (default: 2)+ depth=0: only base_path+ depth=1: base_path + immediate subdirs+ depth=2: base_path + subdirs + their subdirs++ Returns:+ List of tuples (plugin_path, module_name) where:+ - plugin_path: Path to the plugin directory+ - module_name: Dot-separated module path relative to plugins_pkg+ (e.g., 'base_plugin.nested_plugin' or 'base_plugin')++ """+ plugins = []++ def _scan_directory(path: Path, depth: int = 0, rel_parts: tuple[str, ...] = ()):+ """Recursively scan directories for plugins.++ Args:+ path: Current directory being scanned+ depth: Current recursion depth+ rel_parts: Tuple of path components relative to base_path+ """+ if depth > max_depth:+ return++ try:+ items = sorted(path.iterdir())+ except OSError as e:+ self.logger.warning(+ f"Cannot access directory {path} for plugin discovery: {e}"+ )+ return++ for item in items:+ # Skip special Python directories and build artifacts+ if item.name.startswith("__") or item.name in (+ "build",+ "dist",+ "egg-info",+ ".pytest_cache",+ ".mypy_cache",+ "node_modules",+ ):+ continue++ # Handle .so files (compiled extensions) at any level+ if item.name.endswith(".so"):+ pkg_name = item.name.split(".")[0]+ module_path = ".".join(rel_parts + (pkg_name,))+ plugins.append((item, module_path))+ continue++ # Only process directories+ if not item.is_dir():+ continue++ # Build relative module path+ new_rel_parts = rel_parts + (item.name,)++ # Check if this directory has __init__.py+ init_file = item / "__init__.py"+ if init_file.exists():+ # This could be a plugin - add it to candidates+ module_path = ".".join(new_rel_parts)+ plugins.append((item, module_path))++ # Continue scanning subdirectories if we haven't reached max depth+ if depth < max_depth:+ _scan_directory(item, depth + 1, new_rel_parts)++ _scan_directory(base_path)+ return plugins++ def load_plugins(self) -> None:+ """Load plugins found in the plugins directory.++ Scans the plugins directory recursively for valid plugin packages,+ validates their metadata, and optionally calls registration callback+ for framework-specific integration (e.g., Flask blueprints, Django URL patterns).++ Supports both flat plugin structures (plugins/plugin_name/) and nested+ structures (plugins/category/plugin_name/) up to max_depth levels.+ """+ if not self.plugins_dir or not self.plugins_dir.exists():+ self.logger.warning(+ f"Plugins directory not found: {self.plugins_dir}. Skipping plugin loading."+ )+ return++ self.logger.info(f"Loading plugins from: {self.plugins_dir}")++ # Recursively discover all potential plugin directories+ plugin_candidates = self._find_plugin_modules(self.plugins_dir, max_depth=2)++ for plugin_path, module_name in plugin_candidates:+ # Build import path with optional submodule+ if self.plugin_submodule:+ pkg_anchor = f"{self.plugins_pkg}.{module_name}.{self.plugin_submodule}"+ else:+ pkg_anchor = f"{self.plugins_pkg}.{module_name}"++ # Try to import the plugin module+ try:+ module = importlib.import_module(pkg_anchor)+ except ImportError as e:+ self.logger.debug(+ f"Could not import {pkg_anchor} (might not be a plugin): {str(e)}"+ )+ continue++ # Validate plugin metadata+ metadata = getattr(module, "metadata", None)+ if not metadata:+ # Not a plugin - just a regular directory with __init__.py+ continue++ # Skip disabled plugins+ if metadata.get("disable", False) or not metadata.get("is_active", True):+ self.logger.info(+ f"Skipping disabled plugin: {module_name} "+ f"v{metadata.get('version', 'unknown')}"+ )+ continue++ # Register plugin+ try:+ plugin_name = metadata["name"]+ plugin_data = {+ "version": metadata.get("version", "unknown"),+ "module": module,+ "metadata": metadata,+ }++ # Add optional fields if present+ if "entrypoint_cls" in metadata:+ plugin_data["entrypoint_cls"] = metadata["entrypoint_cls"]+ if "exception_cls" in metadata:+ plugin_data["exception_cls"] = metadata["exception_cls"]+ if "service_class" in metadata:+ plugin_data["service_class"] = metadata["service_class"]++ self.plugins[plugin_name] = plugin_data++ # Call framework-specific registration callback if provided+ if self.registration_callback:+ self.registration_callback(plugin_data)++ self.logger.info(+ f"✔ Loaded plugin: {plugin_name} v{plugin_data['version']}"+ )++ except KeyError as e:+ self.logger.error(+ f"Invalid metadata for plugin '{module_name}': {str(e)}"+ )++ # Log appropriate message based on whether plugins were loaded+ if not self.plugins:+ self.logger.info("No plugins loaded (OSS mode).")++ def get_plugin(self, name: str) -> dict[str, Any]:+ """Get plugin metadata by name.++ Args:+ name: Plugin name to retrieve++ Returns:+ Dictionary containing plugin metadata (version, module, entrypoint_cls, etc.)+ or empty dict if plugin not found+ """+ return self.plugins.get(name, {})++ def has_plugin(self, name: str) -> bool:+ """Check if a plugin is loaded.++ Args:+ name: Plugin name to check++ Returns:+ bool: True if plugin is loaded, False otherwise+ """+ return name in self.plugins++ def get_all_plugins(self) -> dict[str, dict[str, Any]]:+ """Get all loaded plugins.++ Returns:+ Dictionary mapping plugin names to their metadata+ """+ return self.plugins.copy()++ def reload_plugins(self) -> None:+ """Reload all plugins from the plugins directory.++ Clears existing plugins and reloads them from disk.+ """+ self.plugins.clear()+ self.load_plugins()diff --git a/workers/callback/tasks.py b/workers/callback/tasks.pyindex a7496c3d4..cccfb78c3 100644--- a/workers/callback/tasks.py+++ b/workers/callback/tasks.py@@ -66,6 +66,7 @@ class CallbackContext:self.pipeline_type: str | None = Noneself.pipeline_data: dict[str, Any] | None = Noneself.api_client: InternalAPIClient | None = None+ self.file_executions: list[dict[str, Any]] = []def _initialize_performance_managers():@@ -719,6 +720,7 @@ def _extract_callback_parameters(workflow_definition = execution_data.get("workflow_definition", {})organization_context = execution_data.get("organization_context", {})source_config = execution_data.get("source_config", {})+ context.file_executions = execution_data.get("file_executions", [])# 3. Extract parameters with kwargs as fast path, execution data as fallbackcontext.pipeline_id = kwargs.get("pipeline_id") or execution_info.get(@@ -754,6 +756,7 @@ def _extract_callback_parameters(workflow_definition = execution_data.get("workflow_definition", {})organization_context = execution_data.get("organization_context", {})source_config = execution_data.get("source_config", {})+ context.file_executions = execution_data.get("file_executions", [])# Extract organization ID from execution datacontext.organization_id = organization_context.get(@@ -772,13 +775,6 @@ def _extract_callback_parameters(context.pipeline_id = kwargs.get("pipeline_id") or execution_info.get("pipeline_id")- if not context.pipeline_id:- execution_response = api_client.get_workflow_execution(- execution_id=context.execution_id,- file_execution=False,- )- execution_info = execution_response.data.get("execution", {})- context.pipeline_id = execution_info.get("pipeline_id")finally:# Clean up temporary clientif hasattr(temp_api_client, "close"):@@ -833,7 +829,9 @@ def _extract_callback_parameters(f"Extracted from kwargs: pipeline_id={kwargs.get('pipeline_id')}, org_id={kwargs.get('organization_id')}")logger.info(- f"Extracted from execution: workflow_id={context.workflow_id}, is_api={is_api_deployment}, pipeline_type={context.pipeline_type}"+ f"Extracted from execution: workflow_id={context.workflow_id}, "+ f"is_api={is_api_deployment}, pipeline_type={context.pipeline_type} "+ f"file_executions={len(context.file_executions)}")except Exception as e:@@ -1268,6 +1266,97 @@ def _cleanup_execution_resources(context: CallbackContext) -> dict[str, Any]:}+def _track_subscription_usage_if_available(+ context: CallbackContext,+ execution_status: str,+) -> dict[str, Any] | None:+ """Track subscription usage for successful file executions if plugin available.++ This is a non-blocking operation - errors are logged but do not fail the workflow.+ Only tracks usage for completed workflows with successful file executions.++ Args:+ context: Callback execution context with organization and execution IDs+ execution_status: Final execution status (should be COMPLETED for tracking)++ Returns:+ dict with status, committed_count, message if tracking attempted+ None if plugin not available or workflow not completed+ """+ # Only track for completed workflows+ if execution_status != ExecutionStatus.COMPLETED.value:+ logger.debug(+ f"Skipping subscription tracking for execution {context.execution_id} "+ f"with status {execution_status}"+ )+ return None++ try:+ # Try to get subscription_usage client plugin+ from client_plugin_registry import get_client_plugin++ subscription_plugin = get_client_plugin("subscription_usage")++ if not subscription_plugin:+ logger.debug(+ f"Subscription usage plugin not found for execution {context.execution_id} (OSS mode)"+ )+ return None++ # Extract successful file execution IDs directly from context.file_executions+ # Status in database is "COMPLETED" for successful executions+ successful_file_exec_ids = [+ file_exec.get("id")+ for file_exec in context.file_executions+ if isinstance(file_exec, dict)+ and file_exec.get("status") == "COMPLETED"+ and file_exec.get("id")+ and not file_exec.get("execution_error")+ ]++ if not successful_file_exec_ids:+ logger.debug(+ f"No successful file executions to track for execution {context.execution_id}"+ )+ return {+ "status": "skipped",+ "committed_count": 0,+ "message": "No successful file executions",+ }++ logger.info(+ f"Committing subscription usage for {len(successful_file_exec_ids)} "+ f"successful file executions in execution {context.execution_id}"+ )++ subscription_tracking_result = (+ subscription_plugin.commit_batch_subscription_usage(+ organization_id=context.organization_id,+ file_execution_ids=successful_file_exec_ids,+ )+ )++ logger.info(+ f"Subscription usage committed: "+ f"{subscription_tracking_result.get('committed_count', 0)}/{len(successful_file_exec_ids)} "+ f"file executions (status: {subscription_tracking_result.get('status')})"+ )++ return subscription_tracking_result++ except Exception as tracking_error:+ logger.error(+ "Failed to commit subscription usage for execution "+ f"{context.execution_id} (continuing execution): {tracking_error}",+ exc_info=True,+ )+ return {+ "status": "error",+ "committed_count": 0,+ "error": str(tracking_error),+ }++def _process_batch_callback_core(task_instance, results, *args, **kwargs) -> dict[str, Any]:@@ -1332,6 +1421,12 @@ def _process_batch_callback_core(context, execution_status, is_api_deployment=False)+ # Track subscription usage if plugin is present+ subscription_tracking_result = _track_subscription_usage_if_available(+ context=context,+ execution_status=execution_status,+ )+# Add missing UI logs for cost and final workflow status (matching backend behavior)_publish_final_workflow_ui_logs(context=context,@@ -1351,6 +1446,7 @@ def _process_batch_callback_core("expected_files": expected_files,"execution_update_result": execution_update_result,"pipeline_result": pipeline_result,+ "subscription_tracking_result": subscription_tracking_result,"cleanup_result": cleanup_result,"pipeline_id": context.pipeline_id,"unified_callback": True,@@ -1565,6 +1661,7 @@ def process_batch_callback_api(context.pipeline_name = pipeline_namecontext.pipeline_type = pipeline_typecontext.api_client = api_client+ context.file_executions = execution_context.get("file_executions", [])context.pipeline_data = {"is_api": True} # Mark as API execution for cleanup logic@@ -1584,6 +1681,12 @@ def process_batch_callback_api(context=context, final_status=execution_status, is_api_deployment=True)+ # Track subscription usage if plugin is present+ subscription_tracking_result = _track_subscription_usage_if_available(+ context=context,+ execution_status=execution_status,+ )+# Handle notifications using unified functionnotification_result = _handle_notifications_unified(api_client=api_client,@@ -1612,6 +1715,7 @@ def process_batch_callback_api("execution_update": execution_update_result,"pipeline_update": pipeline_result,"notifications": notification_result,+ "subscription_tracking_result": subscription_tracking_result,"cleanup_result": cleanup_result,"optimization": {"method": "unified_callback_functions",diff --git a/workers/pyproject.toml b/workers/pyproject.tomlindex 4c320131d..0783784eb 100644--- a/workers/pyproject.toml+++ b/workers/pyproject.toml@@ -49,7 +49,8 @@ dev = ["black>=23.7.0","isort>=5.12.0","flake8>=6.0.0",- "mypy>=1.5.0"+ "mypy>=1.5.0",+ "debugpy>=1.8.14",]test = [diff --git a/workers/uv.lock b/workers/uv.lockindex 319eacaad..6432ed650 100644--- a/workers/uv.lock+++ b/workers/uv.lock@@ -1,5 +1,5 @@version = 1-revision = 1+revision = 3requires-python = ">=3.12"resolution-markers = ["python_full_version >= '3.13'",@@ -818,6 +818,27 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686 },]+[[package]]+name = "debugpy"+version = "1.8.17"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/15/ad/71e708ff4ca377c4230530d6a7aa7992592648c122a2cd2b321cf8b35a76/debugpy-1.8.17.tar.gz", hash = "sha256:fd723b47a8c08892b1a16b2c6239a8b96637c62a59b94bb5dab4bac592a58a8e", size = 1644129, upload-time = "2025-09-17T16:33:20.633Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/08/2b/9d8e65beb2751876c82e1aceb32f328c43ec872711fa80257c7674f45650/debugpy-1.8.17-cp312-cp312-macosx_15_0_universal2.whl", hash = "sha256:f14467edef672195c6f6b8e27ce5005313cb5d03c9239059bc7182b60c176e2d", size = 2549522, upload-time = "2025-09-17T16:33:38.466Z" },+ { url = "https://files.pythonhosted.org/packages/b4/78/eb0d77f02971c05fca0eb7465b18058ba84bd957062f5eec82f941ac792a/debugpy-1.8.17-cp312-cp312-manylinux_2_34_x86_64.whl", hash = "sha256:24693179ef9dfa20dca8605905a42b392be56d410c333af82f1c5dff807a64cc", size = 4309417, upload-time = "2025-09-17T16:33:41.299Z" },+ { url = "https://files.pythonhosted.org/packages/37/42/c40f1d8cc1fed1e75ea54298a382395b8b937d923fcf41ab0797a554f555/debugpy-1.8.17-cp312-cp312-win32.whl", hash = "sha256:6a4e9dacf2cbb60d2514ff7b04b4534b0139facbf2abdffe0639ddb6088e59cf", size = 5277130, upload-time = "2025-09-17T16:33:43.554Z" },+ { url = "https://files.pythonhosted.org/packages/72/22/84263b205baad32b81b36eac076de0cdbe09fe2d0637f5b32243dc7c925b/debugpy-1.8.17-cp312-cp312-win_amd64.whl", hash = "sha256:e8f8f61c518952fb15f74a302e068b48d9c4691768ade433e4adeea961993464", size = 5319053, upload-time = "2025-09-17T16:33:53.033Z" },+ { url = "https://files.pythonhosted.org/packages/50/76/597e5cb97d026274ba297af8d89138dfd9e695767ba0e0895edb20963f40/debugpy-1.8.17-cp313-cp313-macosx_15_0_universal2.whl", hash = "sha256:857c1dd5d70042502aef1c6d1c2801211f3ea7e56f75e9c335f434afb403e464", size = 2538386, upload-time = "2025-09-17T16:33:54.594Z" },+ { url = "https://files.pythonhosted.org/packages/5f/60/ce5c34fcdfec493701f9d1532dba95b21b2f6394147234dce21160bd923f/debugpy-1.8.17-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:3bea3b0b12f3946e098cce9b43c3c46e317b567f79570c3f43f0b96d00788088", size = 4292100, upload-time = "2025-09-17T16:33:56.353Z" },+ { url = "https://files.pythonhosted.org/packages/e8/95/7873cf2146577ef71d2a20bf553f12df865922a6f87b9e8ee1df04f01785/debugpy-1.8.17-cp313-cp313-win32.whl", hash = "sha256:e34ee844c2f17b18556b5bbe59e1e2ff4e86a00282d2a46edab73fd7f18f4a83", size = 5277002, upload-time = "2025-09-17T16:33:58.231Z" },+ { url = "https://files.pythonhosted.org/packages/46/11/18c79a1cee5ff539a94ec4aa290c1c069a5580fd5cfd2fb2e282f8e905da/debugpy-1.8.17-cp313-cp313-win_amd64.whl", hash = "sha256:6c5cd6f009ad4fca8e33e5238210dc1e5f42db07d4b6ab21ac7ffa904a196420", size = 5319047, upload-time = "2025-09-17T16:34:00.586Z" },+ { url = "https://files.pythonhosted.org/packages/de/45/115d55b2a9da6de812696064ceb505c31e952c5d89c4ed1d9bb983deec34/debugpy-1.8.17-cp314-cp314-macosx_15_0_universal2.whl", hash = "sha256:045290c010bcd2d82bc97aa2daf6837443cd52f6328592698809b4549babcee1", size = 2536899, upload-time = "2025-09-17T16:34:02.657Z" },+ { url = "https://files.pythonhosted.org/packages/5a/73/2aa00c7f1f06e997ef57dc9b23d61a92120bec1437a012afb6d176585197/debugpy-1.8.17-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:b69b6bd9dba6a03632534cdf67c760625760a215ae289f7489a452af1031fe1f", size = 4268254, upload-time = "2025-09-17T16:34:04.486Z" },+ { url = "https://files.pythonhosted.org/packages/86/b5/ed3e65c63c68a6634e3ba04bd10255c8e46ec16ebed7d1c79e4816d8a760/debugpy-1.8.17-cp314-cp314-win32.whl", hash = "sha256:5c59b74aa5630f3a5194467100c3b3d1c77898f9ab27e3f7dc5d40fc2f122670", size = 5277203, upload-time = "2025-09-17T16:34:06.65Z" },+ { url = "https://files.pythonhosted.org/packages/b0/26/394276b71c7538445f29e792f589ab7379ae70fd26ff5577dfde71158e96/debugpy-1.8.17-cp314-cp314-win_amd64.whl", hash = "sha256:893cba7bb0f55161de4365584b025f7064e1f88913551bcd23be3260b231429c", size = 5318493, upload-time = "2025-09-17T16:34:08.483Z" },+ { url = "https://files.pythonhosted.org/packages/b0/d0/89247ec250369fc76db477720a26b2fce7ba079ff1380e4ab4529d2fe233/debugpy-1.8.17-py2.py3-none-any.whl", hash = "sha256:60c7dca6571efe660ccb7a9508d73ca14b8796c4ed484c2002abba714226cfef", size = 5283210, upload-time = "2025-09-17T16:34:25.835Z" },+]+[[package]]name = "decorator"version = "5.2.1"@@ -3038,7 +3059,7 @@ dependencies = [{ name = "pinecone-plugin-interface" },{ name = "python-dateutil" },{ name = "typing-extensions" },- { name = "urllib3", marker = "python_full_version < '4.0'" },+ { name = "urllib3", marker = "python_full_version < '4'" },]sdist = { url = "https://files.pythonhosted.org/packages/bc/9d/07a7f2136ce04cabd21d69c057dc2915867082b0047e6873e424388d4475/pinecone-7.0.1.tar.gz", hash = "sha256:49ff7b0f5be4a2ddec5aaa709758a9f2df56baa58ad46507d081409e246a81ec", size = 207930 }wheels = [@@ -4669,6 +4690,7 @@ deploy = []dev = [{ name = "black" },+ { name = "debugpy" },{ name = "flake8" },{ name = "isort" },{ name = "mypy" },@@ -4716,6 +4738,7 @@ deploy = []dev = [{ name = "black", specifier = ">=23.7.0" },+ { name = "debugpy", specifier = ">=1.8.14" },{ name = "flake8", specifier = ">=6.0.0" },{ name = "isort", specifier = ">=5.12.0" },{ name = "mypy", specifier = ">=1.5.0" },