Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Added support for processor plugins

Deepakdaef5ce1a73f
diff --git a/.gitignore b/.gitignore
index 9b9ca8f4a..9e300b9e3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -607,6 +607,9 @@ $RECYCLE.BIN/
backend/plugins/authentication/*
!backend/plugins/authentication/auth_sample
+# Processor Plugins
+backend/plugins/processor/*
+
# Tool registry
unstract/tool-registry/src/unstract/tool_registry/*.json
unstract/tool-registry/tests/*.yaml
@@ -627,5 +630,3 @@ tools/*/sdks/
tools/*/data_dir/
docker/workflow_data/
-
-
diff --git a/backend/prompt_studio/processor_loader.py b/backend/prompt_studio/processor_loader.py
new file mode 100644
index 000000000..28d12f8d9
--- /dev/null
+++ b/backend/prompt_studio/processor_loader.py
@@ -0,0 +1,76 @@
+import logging
+import os
+from importlib import import_module
+from typing import Any
+
+from django.apps import apps
+
+logger = logging.getLogger(__name__)
+
+
+class ProcessorConfig:
+ """Loader config for processor plugins."""
+
+ PLUGINS_APP = "plugins"
+ PLUGIN_DIR = "processor"
+ MODULE = "module"
+ METADATA = "metadata"
+ METADATA_NAME = "name"
+ METADATA_SERVICE_CLASS = "service_class"
+ METADATA_IS_ACTIVE = "is_active"
+
+
+def load_plugins() -> list[Any]:
+ """Iterate through the processor plugins and register them."""
+ plugins_app = apps.get_app_config(ProcessorConfig.PLUGINS_APP)
+ package_path = plugins_app.module.__package__
+ processor_dir = os.path.join(plugins_app.path, ProcessorConfig.PLUGIN_DIR)
+ processor_package_path = f"{package_path}.{ProcessorConfig.PLUGIN_DIR}"
+ processor_plugins: list[Any] = []
+
+ for item in os.listdir(processor_dir):
+ # Loads a plugin if it is in a directory.
+ if os.path.isdir(os.path.join(processor_dir, item)):
+ processor_module_name = item
+ # Loads a plugin if it is a shared library.
+ # Module name is extracted from shared library name.
+ # `processor.platform_architecture.so` will be file name and
+ # `processor` will be the module name.
+ elif item.endswith(".so"):
+ processor_module_name = item.split(".")[0]
+ else:
+ continue
+ try:
+ full_module_path = (
+ f"{processor_package_path}.{processor_module_name}"
+ )
+ module = import_module(full_module_path)
+ metadata = getattr(module, ProcessorConfig.METADATA, {})
+
+ if metadata.get(ProcessorConfig.METADATA_IS_ACTIVE, False):
+ processor_plugins.append(
+ {
+ ProcessorConfig.MODULE: module,
+ ProcessorConfig.METADATA: module.metadata,
+ }
+ )
+ logger.info(
+ "Loaded processor plugin: %s, is_active: %s",
+ module.metadata[ProcessorConfig.METADATA_NAME],
+ module.metadata[ProcessorConfig.METADATA_IS_ACTIVE],
+ )
+ else:
+ logger.info(
+ "Processor plugin %s is not active.",
+ processor_module_name,
+ )
+ except ModuleNotFoundError as exception:
+ logger.error(
+ "Error while importing processor plugin: %s",
+ exception,
+ )
+
+ if len(processor_plugins) == 0:
+ logger.info("No processor plugins found.")
+
+ return processor_plugins
diff --git a/backend/prompt_studio/prompt_studio_core/views.py b/backend/prompt_studio/prompt_studio_core/views.py
index 5be5fa54c..cbd0c1890 100644
--- a/backend/prompt_studio/prompt_studio_core/views.py
+++ b/backend/prompt_studio/prompt_studio_core/views.py
@@ -6,6 +6,7 @@ from django.db import IntegrityError
from django.db.models import QuerySet
from django.http import HttpRequest
from permissions.permission import IsOwner
+from prompt_studio.processor_loader import ProcessorConfig, load_plugins
from prompt_studio.prompt_studio.exceptions import FilenameMissingError
from prompt_studio.prompt_studio_core.constants import (
ToolStudioErrors,
@@ -41,6 +42,8 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
permission_classes = [IsOwner]
serializer_class = CustomToolSerializer
+ processor_plugins = load_plugins()
+
def get_queryset(self) -> Optional[QuerySet]:
filter_args = FilterHelper.build_filter_args(
self.request,
@@ -139,6 +142,18 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
org_id=request.org_id,
user_id=request.user.user_id,
)
+
+ for processor_plugin in self.processor_plugins:
+ cls = processor_plugin[ProcessorConfig.METADATA][
+ ProcessorConfig.METADATA_SERVICE_CLASS
+ ]
+ cls.process(
+ tool_id=tool_id,
+ file_name=file_name,
+ org_id=request.org_id,
+ user_id=request.user.user_id,
+ )
+
if unique_id:
return Response(
{"message": "Document indexed successfully."},