LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Added support for processor plugins
Deepakdaef5ce1a73f
diff --git a/.gitignore b/.gitignoreindex 9b9ca8f4a..9e300b9e3 100644--- a/.gitignore+++ b/.gitignore@@ -607,6 +607,9 @@ $RECYCLE.BIN/backend/plugins/authentication/*!backend/plugins/authentication/auth_sample+# Processor Plugins+backend/plugins/processor/*+# Tool registryunstract/tool-registry/src/unstract/tool_registry/*.jsonunstract/tool-registry/tests/*.yaml@@ -627,5 +630,3 @@ tools/*/sdks/tools/*/data_dir/docker/workflow_data/--diff --git a/backend/prompt_studio/processor_loader.py b/backend/prompt_studio/processor_loader.pynew file mode 100644index 000000000..28d12f8d9--- /dev/null+++ b/backend/prompt_studio/processor_loader.py@@ -0,0 +1,76 @@+import logging+import os+from importlib import import_module+from typing import Any++from django.apps import apps++logger = logging.getLogger(__name__)+++class ProcessorConfig:+ """Loader config for processor plugins."""++ PLUGINS_APP = "plugins"+ PLUGIN_DIR = "processor"+ MODULE = "module"+ METADATA = "metadata"+ METADATA_NAME = "name"+ METADATA_SERVICE_CLASS = "service_class"+ METADATA_IS_ACTIVE = "is_active"+++def load_plugins() -> list[Any]:+ """Iterate through the processor plugins and register them."""+ plugins_app = apps.get_app_config(ProcessorConfig.PLUGINS_APP)+ package_path = plugins_app.module.__package__+ processor_dir = os.path.join(plugins_app.path, ProcessorConfig.PLUGIN_DIR)+ processor_package_path = f"{package_path}.{ProcessorConfig.PLUGIN_DIR}"+ processor_plugins: list[Any] = []++ for item in os.listdir(processor_dir):+ # Loads a plugin if it is in a directory.+ if os.path.isdir(os.path.join(processor_dir, item)):+ processor_module_name = item+ # Loads a plugin if it is a shared library.+ # Module name is extracted from shared library name.+ # `processor.platform_architecture.so` will be file name and+ # `processor` will be the module name.+ elif item.endswith(".so"):+ processor_module_name = item.split(".")[0]+ else:+ continue+ try:+ full_module_path = (+ f"{processor_package_path}.{processor_module_name}"+ )+ module = import_module(full_module_path)+ metadata = getattr(module, ProcessorConfig.METADATA, {})++ if metadata.get(ProcessorConfig.METADATA_IS_ACTIVE, False):+ processor_plugins.append(+ {+ ProcessorConfig.MODULE: module,+ ProcessorConfig.METADATA: module.metadata,+ }+ )+ logger.info(+ "Loaded processor plugin: %s, is_active: %s",+ module.metadata[ProcessorConfig.METADATA_NAME],+ module.metadata[ProcessorConfig.METADATA_IS_ACTIVE],+ )+ else:+ logger.info(+ "Processor plugin %s is not active.",+ processor_module_name,+ )+ except ModuleNotFoundError as exception:+ logger.error(+ "Error while importing processor plugin: %s",+ exception,+ )++ if len(processor_plugins) == 0:+ logger.info("No processor plugins found.")++ return processor_pluginsdiff --git a/backend/prompt_studio/prompt_studio_core/views.py b/backend/prompt_studio/prompt_studio_core/views.pyindex 5be5fa54c..cbd0c1890 100644--- a/backend/prompt_studio/prompt_studio_core/views.py+++ b/backend/prompt_studio/prompt_studio_core/views.py@@ -6,6 +6,7 @@ from django.db import IntegrityErrorfrom django.db.models import QuerySetfrom django.http import HttpRequestfrom permissions.permission import IsOwner+from prompt_studio.processor_loader import ProcessorConfig, load_pluginsfrom prompt_studio.prompt_studio.exceptions import FilenameMissingErrorfrom prompt_studio.prompt_studio_core.constants import (ToolStudioErrors,@@ -41,6 +42,8 @@ class PromptStudioCoreView(viewsets.ModelViewSet):permission_classes = [IsOwner]serializer_class = CustomToolSerializer+ processor_plugins = load_plugins()+def get_queryset(self) -> Optional[QuerySet]:filter_args = FilterHelper.build_filter_args(self.request,@@ -139,6 +142,18 @@ class PromptStudioCoreView(viewsets.ModelViewSet):org_id=request.org_id,user_id=request.user.user_id,)++ for processor_plugin in self.processor_plugins:+ cls = processor_plugin[ProcessorConfig.METADATA][+ ProcessorConfig.METADATA_SERVICE_CLASS+ ]+ cls.process(+ tool_id=tool_id,+ file_name=file_name,+ org_id=request.org_id,+ user_id=request.user.user_id,+ )+if unique_id:return Response({"message": "Document indexed successfully."},