LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
removed process text functionality to highlight as it is not required… (#1180)
vishnuszipstackdf3d44e2845f
diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.pyindex fe3ad9f1d..62cc2c23c 100644--- a/backend/prompt_studio/prompt_studio_core_v2/views.py+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py@@ -219,10 +219,6 @@ class PromptStudioCoreView(viewsets.ModelViewSet):)document: DocumentManager = DocumentManager.objects.get(pk=document_id)file_name: str = document.document_name- text_processor = get_plugin_class_by_name(- name="text_processor",- plugins=self.processor_plugins,- )# Generate a run_idrun_id = CommonUtils.generate_uuid()unique_id = PromptStudioHelper.index_document(@@ -232,7 +228,6 @@ class PromptStudioCoreView(viewsets.ModelViewSet):user_id=tool.created_by.user_id,document_id=document_id,run_id=run_id,- text_processor=text_processor,)usage_kwargs: dict[Any, Any] = dict()@@ -282,10 +277,6 @@ class PromptStudioCoreView(viewsets.ModelViewSet):if not run_id:# Generate a run_idrun_id = CommonUtils.generate_uuid()- text_processor = get_plugin_class_by_name(- name="text_processor",- plugins=self.processor_plugins,- )response: dict[str, Any] = PromptStudioHelper.prompt_responder(id=id,tool_id=tool_id,@@ -294,7 +285,6 @@ class PromptStudioCoreView(viewsets.ModelViewSet):document_id=document_id,run_id=run_id,profile_manager_id=profile_manager,- text_processor=text_processor,)return Response(response, status=status.HTTP_200_OK)@@ -318,17 +308,12 @@ class PromptStudioCoreView(viewsets.ModelViewSet):if not run_id:# Generate a run_idrun_id = CommonUtils.generate_uuid()- text_processor = get_plugin_class_by_name(- name="text_processor",- plugins=self.processor_plugins,- )response: dict[str, Any] = PromptStudioHelper.prompt_responder(tool_id=tool_id,org_id=UserSessionUtils.get_organization_id(request),user_id=custom_tool.created_by.user_id,document_id=document_id,run_id=run_id,- text_processor=text_processor,)return Response(response, status=status.HTTP_200_OK)diff --git a/tools/structure/src/main.py b/tools/structure/src/main.pyindex d748ca7c0..70f983bea 100644--- a/tools/structure/src/main.py+++ b/tools/structure/src/main.py@@ -3,7 +3,7 @@ import loggingimport osimport sysfrom pathlib import Path-from typing import Any, Callable, Optional+from typing import Anyfrom constants import SettingsKeys # type: ignore [attr-defined]from unstract.sdk.constants import LogLevel, LogState, MetadataKey, ToolEnv, UsageKwargs@@ -123,14 +123,6 @@ class StructureTool(BaseTool):usage_kwargs[UsageKwargs.RUN_ID] = self.file_execution_idusage_kwargs[UsageKwargs.FILE_NAME] = self.source_file_name- process_text: Optional[Callable[[str], str]] = None- try:- from helper import process_text # type: ignore [attr-defined]- except ImportError:- self.stream_log(- f"Function to higlight context is not found. {PAID_FEATURE_MSG}",- level=LogLevel.WARN,- )if tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION]:index.index(tool_id=tool_id,@@ -144,7 +136,6 @@ class StructureTool(BaseTool):output_file_path=tool_data_dir / SettingsKeys.EXTRACT,reindex=True,usage_kwargs=usage_kwargs,- process_text=process_text,tags=self.tags,enable_highlight=enable_highlight,**({"fs": self.workflow_filestorage}),@@ -184,7 +175,6 @@ class StructureTool(BaseTool):output_file_path=tool_data_dir / SettingsKeys.EXTRACT,reindex=reindex,usage_kwargs=usage_kwargs,- process_text=process_text,tags=self.tags,enable_highlight=enable_highlight,**({"fs": self.workflow_filestorage}),