LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
[FEATURE] Changes required for highlighting (#555)
Deepak Kb6057e8f451b
diff --git a/backend/api/api_deployment_views.py b/backend/api/api_deployment_views.pyindex 7217117ca..ca65f00e2 100644--- a/backend/api/api_deployment_views.py+++ b/backend/api/api_deployment_views.py@@ -74,6 +74,7 @@ class DeploymentExecution(views.APIView):self, request: Request, org_name: str, api_name: str, api: APIDeployment) -> Response:execution_id = request.query_params.get("execution_id")+ include_metadata = request.query_params.get("include_metadata", False)if not execution_id:raise InvalidAPIRequest("execution_id shouldn't be empty")response: ExecutionResponse = DeploymentHelper.get_execution_status(@@ -87,6 +88,11 @@ class DeploymentExecution(views.APIView):},status=status.HTTP_422_UNPROCESSABLE_ENTITY,)+ if (+ response.execution_status == CeleryTaskState.SUCCESS.value+ and not include_metadata+ ):+ response.remove_result_metadata_keys()return Response({"status": response.execution_status, "message": response.result},status=status.HTTP_200_OK,diff --git a/backend/api/deployment_helper.py b/backend/api/deployment_helper.pyindex 3866af51d..cca63fb9c 100644--- a/backend/api/deployment_helper.py+++ b/backend/api/deployment_helper.py@@ -174,11 +174,13 @@ class DeploymentHelper(BaseAPIKeyValidator):hash_values_of_files=hash_values_of_files,timeout=timeout,execution_id=execution_id,- include_metadata=include_metadata,)result.status_api = DeploymentHelper.construct_status_endpoint(api_endpoint=api.api_endpoint, execution_id=execution_id)+ if not include_metadata:+ result.remove_result_metadata_keys()+ cls._send_notification(api=api, result=result)except Exception as error:DestinationConnector.delete_api_storage_dir(workflow_id=workflow_id, execution_id=execution_iddiff --git a/backend/file_management/file_management_helper.py b/backend/file_management/file_management_helper.pyindex 6293f2583..ce538068b 100644--- a/backend/file_management/file_management_helper.py+++ b/backend/file_management/file_management_helper.py@@ -223,15 +223,22 @@ class FileManagerHelper:) -> bool:fs = file_system.get_fsspec_fs()base_path = FileManagerHelper._get_base_path(file_system, path)-base_file_name, _ = os.path.splitext(file_name)- file_name_txt = base_file_name + ".txt"-- for directory in directories:- file_path = str(Path(base_path) / directory / file_name_txt)+ pattern = f"{base_file_name}.*"+ file_paths = FileManagerHelper._find_files(fs, base_path, directories, pattern)+ for file_path in file_paths:FileManagerHelper._delete_file(fs, file_path)return True+ @staticmethod+ def _find_files(fs, base_path: str, directories: list[str], pattern: str):+ file_paths = []+ for directory in directories:+ directory_path = str(Path(base_path) / directory)+ for file in fs.glob(f"{directory_path}/{pattern}"):+ file_paths.append(file)+ return file_paths+@staticmethoddef handle_sub_directory_for_tenants(org_id: str, user_id: str, tool_id: str, is_create: booldiff --git a/backend/prompt_studio/processor_loader.py b/backend/prompt_studio/processor_loader.pyindex 2a371a458..0bd8a7ddd 100644--- a/backend/prompt_studio/processor_loader.py+++ b/backend/prompt_studio/processor_loader.py@@ -72,3 +72,13 @@ def load_plugins() -> list[Any]:logger.info("No processor plugins found.")return processor_plugins+++def get_plugin_class_by_name(name: str, plugins: list[Any]) -> Any:+ """Retrieve a specific plugin class by name."""+ for plugin in plugins:+ metadata = plugin[ProcessorConfig.METADATA]+ if metadata.get(ProcessorConfig.METADATA_NAME) == name:+ return metadata.get(ProcessorConfig.METADATA_SERVICE_CLASS)+ logger.warning("Plugin with name '%s' not found.", name)+ return Nonediff --git a/backend/prompt_studio/prompt_studio_core/constants.py b/backend/prompt_studio/prompt_studio_core/constants.pyindex 213fc066f..38c48dbd5 100644--- a/backend/prompt_studio/prompt_studio_core/constants.py+++ b/backend/prompt_studio/prompt_studio_core/constants.py@@ -62,7 +62,6 @@ class ToolStudioPromptKeys:WORD = "word"SYNONYMS = "synonyms"OUTPUTS = "outputs"- ASSERT_PROMPT = "assert_prompt"SECTION = "section"DEFAULT = "default"REINDEX = "reindex"@@ -89,6 +88,7 @@ class ToolStudioPromptKeys:CONTEXT = "context"METADATA = "metadata"INCLUDE_METADATA = "include_metadata"+ PLATFORM_POSTAMBLE = "platform_postamble"class FileViewTypes:diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.pyindex befabc54d..a81797d70 100644--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py@@ -3,7 +3,7 @@ import loggingimport osimport uuidfrom pathlib import Path-from typing import Any, Optional+from typing import Any, Callable, Optionalfrom account.constants import Commonfrom account.models import User@@ -291,6 +291,7 @@ class PromptStudioHelper:document_id: str,is_summary: bool = False,run_id: str = None,+ text_processor: Optional[type[Any]] = None,) -> Any:"""Method to index a document.@@ -340,7 +341,9 @@ class PromptStudioHelper:# Need to check the user who created profile manager# has access to adapters configured in profile managerPromptStudioHelper.validate_profile_manager_owner_access(default_profile)-+ process_text = None+ if text_processor:+ process_text = text_processor.processdoc_id = PromptStudioHelper.dynamic_indexer(profile_manager=default_profile,tool_id=tool_id,@@ -351,6 +354,7 @@ class PromptStudioHelper:reindex=True,run_id=run_id,user_id=user_id,+ process_text=process_text,)logger.info(f"[{tool_id}] Indexing successful for doc: {file_name}")@@ -372,6 +376,7 @@ class PromptStudioHelper:id: Optional[str] = None,run_id: str = None,profile_manager_id: Optional[str] = None,+ text_processor: Optional[type[Any]] = None,) -> Any:"""Execute chain/single run of the prompts. Makes a call to promptservice and returns the dict of response.@@ -398,19 +403,26 @@ class PromptStudioHelper:if id:return PromptStudioHelper._execute_single_prompt(- id,- doc_path,- doc_name,- tool_id,- org_id,- user_id,- document_id,- run_id,- profile_manager_id,+ id=id,+ doc_path=doc_path,+ doc_name=doc_name,+ tool_id=tool_id,+ org_id=org_id,+ user_id=user_id,+ document_id=document_id,+ run_id=run_id,+ profile_manager_id=profile_manager_id,+ text_processor=text_processor,)else:return PromptStudioHelper._execute_prompts_in_single_pass(- doc_path, tool_id, org_id, user_id, document_id, run_id+ doc_path=doc_path,+ tool_id=tool_id,+ org_id=org_id,+ user_id=user_id,+ document_id=document_id,+ run_id=run_id,+ text_processor=text_processor,)@staticmethod@@ -424,6 +436,7 @@ class PromptStudioHelper:document_id,run_id,profile_manager_id,+ text_processor: Optional[type[Any]] = None,):prompt_instance = PromptStudioHelper._fetch_prompt_from_id(id)prompt_name = prompt_instance.prompt_key@@ -458,7 +471,9 @@ class PromptStudioHelper:LogLevels.RUN,"Invoking prompt service",)-+ process_text = None+ if text_processor:+ process_text = text_processor.processtry:response = PromptStudioHelper._fetch_response(doc_path=doc_path,@@ -470,9 +485,15 @@ class PromptStudioHelper:run_id=run_id,profile_manager_id=profile_manager_id,user_id=user_id,+ process_text=process_text,)return PromptStudioHelper._handle_response(- response, run_id, prompts, document_id, False, profile_manager_id+ response=response,+ run_id=run_id,+ prompts=prompts,+ document_id=document_id,+ is_single_pass=False,+ profile_manager_id=profile_manager_id,)except Exception as e:logger.error(@@ -495,7 +516,13 @@ class PromptStudioHelper:@staticmethoddef _execute_prompts_in_single_pass(- doc_path, tool_id, org_id, user_id, document_id, run_id+ doc_path,+ tool_id,+ org_id,+ user_id,+ document_id,+ run_id,+ text_processor: Optional[type[Any]] = None,):prompts = PromptStudioHelper.fetch_prompt_from_tool(tool_id)prompts = [@@ -513,7 +540,9 @@ class PromptStudioHelper:LogLevels.RUN,"Executing prompts in single pass",)-+ process_text = None+ if text_processor:+ process_text = text_processor.processtry:tool = prompts[0].tool_idresponse = PromptStudioHelper._fetch_single_pass_response(@@ -524,9 +553,14 @@ class PromptStudioHelper:document_id=document_id,run_id=run_id,user_id=user_id,+ process_text=process_text,)return PromptStudioHelper._handle_response(- response, run_id, prompts, document_id, True+ response=response,+ run_id=run_id,+ prompts=prompts,+ document_id=document_id,+ is_single_pass=True,)except Exception as e:logger.error(@@ -556,7 +590,12 @@ class PromptStudioHelper:@staticmethoddef _handle_response(- response, run_id, prompts, document_id, is_single_pass, profile_manager_id=None+ response,+ run_id,+ prompts,+ document_id,+ is_single_pass,+ profile_manager_id=None,):if response.get("status") == IndexingStatus.PENDING_STATUS.value:return {@@ -586,6 +625,7 @@ class PromptStudioHelper:run_id: str,user_id: str,profile_manager_id: Optional[str] = None,+ process_text: Optional[Callable[[str], str]] = None,) -> Any:"""Utility function to invoke prompt service. Used internally.@@ -656,6 +696,7 @@ class PromptStudioHelper:is_summary=tool.summarize_as_source,run_id=run_id,user_id=user_id,+ process_text=process_text,)if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:return {@@ -713,6 +754,9 @@ class PromptStudioHelper:tool_settings[TSPKeys.PREAMBLE] = tool.preambletool_settings[TSPKeys.POSTAMBLE] = tool.postambletool_settings[TSPKeys.GRAMMAR] = grammar_list+ tool_settings[TSPKeys.PLATFORM_POSTAMBLE] = getattr(+ settings, TSPKeys.PLATFORM_POSTAMBLE.upper(), ""+ )tool_id = str(tool.tool_id)@@ -760,6 +804,7 @@ class PromptStudioHelper:is_summary: bool = False,reindex: bool = False,run_id: str = None,+ process_text: Optional[Callable[[str], str]] = None,) -> Any:"""Used to index a file based on the passed arguments.@@ -842,6 +887,7 @@ class PromptStudioHelper:reindex=reindex,output_file_path=extract_file_path,usage_kwargs=usage_kwargs.copy(),+ process_text=process_text,)PromptStudioIndexHelper.handle_index_manager(@@ -875,6 +921,7 @@ class PromptStudioHelper:user_id: str,document_id: str,run_id: str = None,+ process_text: Optional[Callable[[str], str]] = None,) -> Any:tool_id: str = str(tool.tool_id)outputs: list[dict[str, Any]] = []@@ -910,6 +957,7 @@ class PromptStudioHelper:document_id=document_id,run_id=run_id,user_id=user_id,+ process_text=process_text,)if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:return {diff --git a/backend/prompt_studio/prompt_studio_core/views.py b/backend/prompt_studio/prompt_studio_core/views.pyindex 18915ac24..7b197fa39 100644--- a/backend/prompt_studio/prompt_studio_core/views.py+++ b/backend/prompt_studio/prompt_studio_core/views.py@@ -9,7 +9,7 @@ from django.http import HttpRequestfrom file_management.exceptions import FileNotFoundfrom file_management.file_management_helper import FileManagerHelperfrom permissions.permission import IsOwner, IsOwnerOrSharedUser-from prompt_studio.processor_loader import ProcessorConfig, load_plugins+from prompt_studio.processor_loader import get_plugin_class_by_name, load_pluginsfrom prompt_studio.prompt_profile_manager.constants import (ProfileManagerErrors,ProfileManagerKeys,@@ -219,6 +219,10 @@ class PromptStudioCoreView(viewsets.ModelViewSet):)document: DocumentManager = DocumentManager.objects.get(pk=document_id)file_name: str = document.document_name+ text_processor = get_plugin_class_by_name(+ name="text_processor",+ plugins=self.processor_plugins,+ )# Generate a run_idrun_id = CommonUtils.generate_uuid()unique_id = PromptStudioHelper.index_document(@@ -228,22 +232,23 @@ class PromptStudioCoreView(viewsets.ModelViewSet):user_id=tool.created_by.user_id,document_id=document_id,run_id=run_id,+ text_processor=text_processor,)usage_kwargs: dict[Any, Any] = dict()usage_kwargs[ToolStudioPromptKeys.RUN_ID] = run_id- for processor_plugin in self.processor_plugins:- cls = processor_plugin[ProcessorConfig.METADATA][- ProcessorConfig.METADATA_SERVICE_CLASS- ]- cls.process(- tool_id=str(tool.tool_id),- file_name=file_name,- org_id=UserSessionUtils.get_organization_id(request),- user_id=tool.created_by.user_id,- document_id=document_id,- usage_kwargs=usage_kwargs.copy(),- )+ cls = get_plugin_class_by_name(+ name="summarizer",+ plugins=self.processor_plugins,+ )+ cls.process(+ tool_id=str(tool.tool_id),+ file_name=file_name,+ org_id=UserSessionUtils.get_organization_id(request),+ user_id=tool.created_by.user_id,+ document_id=document_id,+ usage_kwargs=usage_kwargs.copy(),+ )if unique_id:return Response(@@ -276,7 +281,10 @@ class PromptStudioCoreView(viewsets.ModelViewSet):if not run_id:# Generate a run_idrun_id = CommonUtils.generate_uuid()-+ text_processor = get_plugin_class_by_name(+ name="text_processor",+ plugins=self.processor_plugins,+ )response: dict[str, Any] = PromptStudioHelper.prompt_responder(id=id,tool_id=tool_id,@@ -285,6 +293,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):document_id=document_id,run_id=run_id,profile_manager_id=profile_manager,+ text_processor=text_processor,)return Response(response, status=status.HTTP_200_OK)@@ -308,12 +317,17 @@ class PromptStudioCoreView(viewsets.ModelViewSet):if not run_id:# Generate a run_idrun_id = CommonUtils.generate_uuid()+ text_processor = get_plugin_class_by_name(+ name="text_processor",+ plugins=self.processor_plugins,+ )response: dict[str, Any] = PromptStudioHelper.prompt_responder(tool_id=tool_id,org_id=UserSessionUtils.get_organization_id(request),user_id=custom_tool.created_by.user_id,document_id=document_id,run_id=run_id,+ text_processor=text_processor,)return Response(response, status=status.HTTP_200_OK)@@ -491,7 +505,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):# Delete the filesFileManagerHelper.delete_file(file_system, path, file_name)# Directories to delete the text files- directories = ["extract/", "summarize/"]+ directories = ["extract/", "extract/metadata/", "summarize/"]FileManagerHelper.delete_related_files(file_system, path, file_name, directories)diff --git a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.pyindex c9b3672a8..f431e4539 100644--- a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py+++ b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py@@ -1,4 +1,3 @@-import jsonimport loggingfrom typing import Any, Optional@@ -106,9 +105,9 @@ class OutputManagerHelper:continueif not is_single_pass_extract:- context = json.dumps(context.get(prompt.prompt_key))+ context = context.get(prompt.prompt_key)- output = json.dumps(outputs.get(prompt.prompt_key))+ output = outputs.get(prompt.prompt_key)profile_manager = default_profileeval_metrics = outputs.get(f"{prompt.prompt_key}__evaluation", [])diff --git a/backend/prompt_studio/prompt_studio_registry/constants.py b/backend/prompt_studio/prompt_studio_registry/constants.pyindex 7fea2c73c..6cd809ae9 100644--- a/backend/prompt_studio/prompt_studio_registry/constants.py+++ b/backend/prompt_studio/prompt_studio_registry/constants.py@@ -92,6 +92,7 @@ class JsonSchemaKey:SUMMARIZE_PROMPT = "summarize_prompt"SUMMARIZE_AS_SOURCE = "summarize_as_source"ENABLE_HIGHLIGHT = "enable_highlight"+ PLATFORM_POSTAMBLE = "platform_postamble"class SpecKey:diff --git a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.pyindex 8a9877a85..adb062984 100644--- a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py+++ b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py@@ -266,6 +266,9 @@ class PromptStudioRegistryHelper:tool.single_pass_extraction_mode)tool_settings[JsonSchemaKey.ENABLE_HIGHLIGHT] = tool.enable_highlight+ tool_settings[JsonSchemaKey.PLATFORM_POSTAMBLE] = getattr(+ settings, JsonSchemaKey.PLATFORM_POSTAMBLE.upper(), ""+ )for prompt in prompts:if prompt.prompt_type == JsonSchemaKey.NOTES or not prompt.active:diff --git a/backend/sample.env b/backend/sample.envindex bb8267dfd..fd45516ae 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -90,9 +90,9 @@ PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data# Structure Tool Image (Runs prompt studio exported tools)# https://hub.docker.com/r/unstract/tool-structure-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.36"+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.37"STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"-STRUCTURE_TOOL_IMAGE_TAG="0.0.36"+STRUCTURE_TOOL_IMAGE_TAG="0.0.37"# Feature FlagsEVALUATION_SERVER_IP=unstract-fliptdiff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.pyindex 5b5a590ba..b922f4e64 100644--- a/backend/scheduler/tasks.py+++ b/backend/scheduler/tasks.py@@ -117,6 +117,7 @@ def execute_pipeline_task(execution_response = WorkflowHelper.complete_execution(workflow, execution_id, pipepline_id)+ execution_response.remove_result_metadata_keys()logger.info(f"Execution response: {execution_response}")logger.info(f"Execution completed for pipeline: {name}")except Exception as e:@@ -166,6 +167,7 @@ def execute_pipeline_task_v2(execution_response = WorkflowHelper.complete_execution(workflow, execution_id, pipeline_id)+ execution_response.remove_result_metadata_keys()logger.info(f"Execution response for pipeline {pipeline_name} of organization "f"{organization_id}: {execution_response}"diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.pyindex 21b2a21d0..61dd658ab 100644--- a/backend/workflow_manager/endpoint/destination.py+++ b/backend/workflow_manager/endpoint/destination.py@@ -267,8 +267,12 @@ class DestinationConnector(BaseConnector):execution_id_name = str(destination_configurations.get(DestinationKey.EXECUTION_ID, "execution_id"))-data = self.get_result()+ # If data is None, don't execute CREATE or INSERT query+ if not data:+ return+ # Remove metadata from result+ data.pop("metadata", None)values = DatabaseUtils.get_columns_and_values(column_mode_str=column_mode,data=data,@@ -286,9 +290,6 @@ class DestinationConnector(BaseConnector):connector_settings=connector_settings,)engine = db_class.get_engine()- # If data is None, don't execute CREATE or INSERT query- if data is None:- returnDatabaseUtils.create_table_if_not_exists(db_class=db_class,engine=engine,diff --git a/backend/workflow_manager/workflow/dto.py b/backend/workflow_manager/workflow/dto.pyindex 1f16d7df3..66ea68976 100644--- a/backend/workflow_manager/workflow/dto.py+++ b/backend/workflow_manager/workflow/dto.py@@ -41,6 +41,58 @@ class ExecutionResponse:self.message = self.message or Noneself.status_api = self.status_api or None+ def remove_result_metadata_keys(self, keys_to_remove: list[str] = []) -> None:+ """Removes specified keys from the 'metadata' dictionary within each+ 'result' dictionary in the 'result' list attribute of the instance. If+ 'keys_to_remove' is empty, the 'metadata' key itself is removed.++ Args:+ keys_to_remove (List[str]): List of keys to be removed from 'metadata'.+ """+ if not isinstance(self.result, list):+ return++ for item in self.result:+ if not isinstance(item, dict):+ break++ result = item.get("result")+ if not isinstance(result, dict):+ break++ self._remove_specific_keys(result=result, keys_to_remove=keys_to_remove)++ def _remove_specific_keys(self, result: dict, keys_to_remove: list[str]) -> None:+ """Removes specified keys from the 'metadata' dictionary within the+ provided 'result' dictionary. If 'keys_to_remove' is empty, the+ 'metadata' dictionary is cleared.++ Args:+ result (dict): The dictionary containing the 'metadata' key.+ keys_to_remove (List[str]): List of keys to be removed from 'metadata'.+ """+ metadata = result.get("metadata", {})+ if keys_to_remove:+ for key in keys_to_remove:+ metadata.pop(key, None)+ else:+ metadata = {}+ self._update_metadata(result=result, metadata=metadata)++ def _update_metadata(self, result: dict, metadata: dict) -> None:+ """Updates the 'metadata' key in the provided 'result' dictionary. If+ 'metadata' is empty, removes the 'metadata' key from 'result'.++ Args:+ result (dict): The dictionary to be updated.+ metadata (dict): The new metadata dictionary to be set. If empty, 'metadata'+ is removed.+ """+ if metadata:+ result["metadata"] = metadata+ else:+ result.pop("metadata", None)+@dataclassclass AsyncResultData:diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.pyindex 1f126469d..c94d7eb70 100644--- a/backend/workflow_manager/workflow/execution.py+++ b/backend/workflow_manager/workflow/execution.py@@ -36,7 +36,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):scheduled: bool = False,mode: tuple[str, str] = WorkflowExecution.Mode.INSTANT,workflow_execution: Optional[WorkflowExecution] = None,- include_metadata: bool = False,) -> None:tool_instances_as_dto = []for tool_instance in tool_instances:@@ -58,7 +57,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):tool_instances=tool_instances_as_dto,platform_service_api_key=str(platform_key.key),ignore_processed_entities=False,- include_metadata=include_metadata,)if not workflow_execution:# Use pipline_id for pipelines / API deploymentdiff --git a/backend/workflow_manager/workflow/views.py b/backend/workflow_manager/workflow/views.pyindex 7c775a1a8..a572025f3 100644--- a/backend/workflow_manager/workflow/views.py+++ b/backend/workflow_manager/workflow/views.py@@ -171,9 +171,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_id = serializer.get_execution_id(serializer.validated_data)execution_action = serializer.get_execution_action(serializer.validated_data)file_objs = request.FILES.getlist("files")- include_metadata = (- request.data.get("include_metadata", "false").lower() == "true"- )hashes_of_files: dict[str, FileHash] = {}if file_objs and execution_id and workflow_id:hashes_of_files = SourceConnector.add_input_file_to_api_storage(@@ -192,7 +189,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_id=execution_id,pipeline_guid=pipeline_guid,hash_values_of_files=hashes_of_files,- include_metadata=include_metadata,)if (execution_response.execution_status == "ERROR"@@ -219,7 +215,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_id: Optional[str] = None,pipeline_guid: Optional[str] = None,hash_values_of_files: dict[str, FileHash] = {},- include_metadata: bool = False,) -> ExecutionResponse:if execution_action is not None:# Step execution@@ -228,7 +223,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):execution_action,execution_id=execution_id,hash_values_of_files=hash_values_of_files,- include_metadata=include_metadata,)elif pipeline_guid:# pipeline execution@@ -246,7 +240,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):workflow=workflow,execution_id=execution_id,hash_values_of_files=hash_values_of_files,- include_metadata=include_metadata,)return execution_responsediff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.pyindex 2ea03366a..199c5005e 100644--- a/backend/workflow_manager/workflow/workflow_helper.py+++ b/backend/workflow_manager/workflow/workflow_helper.py@@ -91,7 +91,6 @@ class WorkflowHelper:scheduled: bool,execution_mode: tuple[str, str],workflow_execution: Optional[WorkflowExecution],- include_metadata: bool = False,) -> WorkflowExecutionServiceHelper:workflow_execution_service = WorkflowExecutionServiceHelper(organization_id=organization_id,@@ -102,7 +101,6 @@ class WorkflowHelper:scheduled=scheduled,mode=execution_mode,workflow_execution=workflow_execution,- include_metadata=include_metadata,)workflow_execution_service.build()return workflow_execution_service@@ -246,7 +244,6 @@ class WorkflowHelper:single_step: bool = False,workflow_execution: Optional[WorkflowExecution] = None,execution_mode: Optional[tuple[str, str]] = None,- include_metadata: bool = False,) -> ExecutionResponse:tool_instances: list[ToolInstance] = (ToolInstanceHelper.get_tool_instances_by_workflow(@@ -265,7 +262,6 @@ class WorkflowHelper:scheduled=scheduled,execution_mode=execution_mode,workflow_execution=workflow_execution,- include_metadata=include_metadata,)execution_id = execution_service.execution_idsource = SourceConnector(@@ -390,7 +386,6 @@ class WorkflowHelper:hash_values_of_files: dict[str, FileHash],timeout: int = -1,pipeline_id: Optional[str] = None,- include_metadata: bool = False,) -> ExecutionResponse:"""Adding a workflow to the queue for execution.@@ -399,7 +394,6 @@ class WorkflowHelper:execution_id (str): Execution IDtimeout (int): Celery timeout (timeout -1 : async execution)pipeline_id (Optional[str], optional): Optional pipeline. Defaults to None.- include_metadata (bool): Whether to include metadata in the prompt outputReturns:ExecutionResponse: Existing status of execution@@ -417,7 +411,6 @@ class WorkflowHelper:execution_id=execution_id,pipeline_id=pipeline_id,log_events_id=log_events_id,- include_metadata=include_metadata,)if timeout > -1:async_execution.wait(@@ -431,12 +424,13 @@ class WorkflowHelper:workflow_execution = WorkflowExecutionServiceHelper.get_execution_by_id(execution_id=execution_id)- return ExecutionResponse(+ execution_response = ExecutionResponse(workflow_id,execution_id,workflow_execution.status,result=task_result,)+ return execution_responseexcept celery_exceptions.TimeoutError:return ExecutionResponse(workflow_id,@@ -475,7 +469,6 @@ class WorkflowHelper:scheduled: bool = False,execution_mode: Optional[tuple[str, str]] = None,pipeline_id: Optional[str] = None,- include_metadata: bool = False,**kwargs: dict[str, Any],) -> Optional[list[Any]]:"""Asynchronous Execution By celery.@@ -489,7 +482,6 @@ class WorkflowHelper:execution_mode (Optional[WorkflowExecution.Mode]): WorkflowExecution ModeDefaults to Nonepipeline_id (Optional[str], optional): Id of pipeline. Defaults to None- include_metadata (bool): Whether to include metadata in the prompt outputKwargs:log_events_id (str): Session ID of the user,@@ -525,7 +517,7 @@ class WorkflowHelper:WorkflowExecutionServiceHelper.update_execution_task(execution_id=execution_id, task_id=task_id)- result = WorkflowHelper.run_workflow(+ execution_response = WorkflowHelper.run_workflow(workflow=workflow,organization_id=schema_name,pipeline_id=pipeline_id,@@ -533,9 +525,8 @@ class WorkflowHelper:workflow_execution=workflow_execution,execution_mode=execution_mode,hash_values_of_files=hash_values,- include_metadata=include_metadata,- ).result- return result+ )+ return execution_response.result@staticmethoddef complete_execution(@@ -543,7 +534,6 @@ class WorkflowHelper:execution_id: Optional[str] = None,pipeline_id: Optional[str] = None,hash_values_of_files: dict[str, FileHash] = {},- include_metadata: bool = False,) -> ExecutionResponse:if pipeline_id:logger.info(f"Executing pipeline: {pipeline_id}")@@ -582,7 +572,6 @@ class WorkflowHelper:workflow=workflow,workflow_execution=workflow_execution,hash_values_of_files=hash_values_of_files,- include_metadata=include_metadata,)except WorkflowExecution.DoesNotExist:return WorkflowHelper.create_and_make_execution_response(@@ -610,7 +599,6 @@ class WorkflowHelper:execution_action: str,execution_id: Optional[str] = None,hash_values_of_files: dict[str, FileHash] = {},- include_metadata: bool = False,) -> ExecutionResponse:if execution_action is Workflow.ExecutionAction.START.value: # type: ignoreif execution_id is None:@@ -624,7 +612,6 @@ class WorkflowHelper:single_step=True,workflow_execution=workflow_execution,hash_values_of_files=hash_values_of_files,- include_metadata=include_metadata,)except WorkflowExecution.DoesNotExist:return WorkflowHelper.create_and_make_execution_response(diff --git a/prompt-service/src/unstract/prompt_service/authentication_middleware.py b/prompt-service/src/unstract/prompt_service/authentication_middleware.pyindex a9fce59fa..fbf03c37b 100644--- a/prompt-service/src/unstract/prompt_service/authentication_middleware.py+++ b/prompt-service/src/unstract/prompt_service/authentication_middleware.py@@ -1,17 +1,16 @@from typing import Any, Optionalfrom flask import Request, current_app-from peewee import PostgresqlDatabase+from unstract.prompt_service.config import dbfrom unstract.prompt_service.constants import DBTableV2, FeatureFlagfrom unstract.flags.feature_flag import check_feature_flag_statusclass AuthenticationMiddleware:- be_db: PostgresqlDatabase- @classmethod- def validate_bearer_token(cls, token: Optional[str]) -> bool:+ @staticmethod+ def validate_bearer_token(token: Optional[str]) -> bool:try:if token is None:current_app.logger.error("Authentication failed. Empty bearer token")@@ -23,7 +22,7 @@ class AuthenticationMiddleware:platform_key_table = "account_platformkey"query = f"SELECT * FROM {platform_key_table} WHERE key = '{token}'"- cursor = cls.be_db.execute_sql(query)+ cursor = db.execute_sql(query)result_row = cursor.fetchone()cursor.close()if not result_row or len(result_row) == 0:@@ -54,8 +53,8 @@ class AuthenticationMiddleware:return Falsereturn True- @classmethod- def get_token_from_auth_header(cls, request: Request) -> Optional[str]:+ @staticmethod+ def get_token_from_auth_header(request: Request) -> Optional[str]:try:bearer_token = request.headers.get("Authorization")if not bearer_token:@@ -66,8 +65,8 @@ class AuthenticationMiddleware:current_app.logger.info(f"Exception while getting token {e}")return None- @classmethod- def get_account_from_bearer_token(cls, token: Optional[str]) -> str:+ @staticmethod+ def get_account_from_bearer_token(token: Optional[str]) -> str:if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):platform_key_table = DBTableV2.PLATFORM_KEYorganization_table = DBTableV2.ORGANIZATION@@ -83,9 +82,9 @@ class AuthenticationMiddleware:schema_name: str = AuthenticationMiddleware.execute_query(query_org)return schema_name- @classmethod- def execute_query(cls, query: str) -> Any:- cursor = cls.be_db.execute_sql(query)+ @staticmethod+ def execute_query(query: str) -> Any:+ cursor = db.execute_sql(query)result_row = cursor.fetchone()cursor.close()if not result_row or len(result_row) == 0:diff --git a/prompt-service/src/unstract/prompt_service/config.py b/prompt-service/src/unstract/prompt_service/config.pyindex 22073db6b..5ead148de 100644--- a/prompt-service/src/unstract/prompt_service/config.py+++ b/prompt-service/src/unstract/prompt_service/config.py@@ -4,6 +4,7 @@ from os import environ as envfrom dotenv import load_dotenvfrom flask import Flask+from peewee import PostgresqlDatabasefrom unstract.prompt_service.constants import LogLevelload_dotenv()@@ -27,6 +28,15 @@ dictConfig(})+db = PostgresqlDatabase(None)+++def get_env_or_die(env_key: str) -> str:+ env_value = env.get(env_key)+ if not env_value:+ raise ValueError(f"Env variable {env_key} is required")+ return env_value+def create_app() -> Flask:app = Flask("prompt-service")@@ -37,4 +47,22 @@ def create_app() -> Flask:app.logger.setLevel(logging.INFO)else:app.logger.setLevel(logging.WARNING)++ # Load required environment variables+ db_host = get_env_or_die("PG_BE_HOST")+ db_port = get_env_or_die("PG_BE_PORT")+ db_user = get_env_or_die("PG_BE_USERNAME")+ db_pass = get_env_or_die("PG_BE_PASSWORD")+ db_name = get_env_or_die("PG_BE_DATABASE")++ # Initialize and connect to the database+ db.init(+ database=db_name,+ user=db_user,+ password=db_pass,+ host=db_host,+ port=db_port,+ )+ db.connect()+return appdiff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.pyindex cad39731e..4e41a3a94 100644--- a/prompt-service/src/unstract/prompt_service/constants.py+++ b/prompt-service/src/unstract/prompt_service/constants.py@@ -58,6 +58,10 @@ class PromptServiceContants:OUTPUT = "output"CONTEXT = "context"INCLUDE_METADATA = "include_metadata"+ EPILOGUE = "epilogue"+ PLATFORM_POSTAMBLE = "platform_postamble"+ EXTRACT_EPILOGUE = "extract-epilogue"+ CLEAN_CONTEXT = "clean-context"class LogLevel(Enum):diff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.pyindex 6d2b4e927..b6f7bfc66 100644--- a/prompt-service/src/unstract/prompt_service/helper.py+++ b/prompt-service/src/unstract/prompt_service/helper.py@@ -1,12 +1,14 @@import importlibimport os+from json import JSONDecodeErrorfrom logging import Loggerfrom pathlib import Path-from typing import Any+from typing import Any, Optionalfrom dotenv import load_dotenv-from flask import Flask, current_app+from flask import Flask, current_app, jsonfrom unstract.prompt_service.authentication_middleware import AuthenticationMiddleware+from unstract.prompt_service.config import dbfrom unstract.prompt_service.constants import PromptServiceContants as PSKeysfrom unstract.prompt_service.exceptions import APIError, RateLimitErrorfrom unstract.sdk.exceptions import RateLimitError as SdkRateLimitError@@ -14,21 +16,11 @@ from unstract.sdk.llm import LLMload_dotenv()+# Global variable to store plugins+plugins: dict[str, dict[str, Any]] = {}-class EnvLoader:- @staticmethod- def get_env_or_die(env_key: str) -> str:- env_value = os.environ.get(env_key)- if env_value is None or env_value == "":- raise ValueError(f"Env variable {env_key} is required")- return env_value--class PluginException(Exception):- """All exceptions raised from a plugin."""---def plugin_loader(app: Flask) -> dict[str, dict[str, Any]]:+def plugin_loader(app: Flask) -> None:"""Loads plugins found in the plugins root dir.Each plugin:@@ -43,14 +35,14 @@ def plugin_loader(app: Flask) -> dict[str, dict[str, Any]]:"disable": bool}"""+ global pluginsplugins_dir: Path = Path(os.path.dirname(__file__)) / "plugins"plugins_pkg = "unstract.prompt_service.plugins"if not plugins_dir.exists():app.logger.info(f"Plugins dir not found: {plugins_dir}. Skipping.")- return {}+ return- plugins: dict[str, dict[str, Any]] = {}app.logger.info(f"Loading plugins from: {plugins_dir}")for pkg in os.listdir(os.fspath(plugins_dir)):@@ -85,10 +77,40 @@ def plugin_loader(app: Flask) -> dict[str, dict[str, Any]]:if not plugins:app.logger.info("No plugins found.")- return plugins+ initialize_plugin_endpoints(app=app)-def query_usage_metadata(db, token: str, metadata: dict[str, Any]) -> dict[str, Any]:+def get_cleaned_context(context: str) -> str:+ clean_context_plugin: dict[str, Any] = plugins.get(PSKeys.CLEAN_CONTEXT, {})+ if clean_context_plugin:+ return clean_context_plugin["entrypoint_cls"].run(context=context)+ return context+++def initialize_plugin_endpoints(app: Flask) -> None:+ """Enables plugins if available."""+ single_pass_extration_plugin: dict[str, Any] = plugins.get(+ PSKeys.SINGLE_PASS_EXTRACTION, {}+ )+ summarize_plugin: dict[str, Any] = plugins.get(PSKeys.SUMMARIZE, {})+ simple_prompt_studio: dict[str, Any] = plugins.get(PSKeys.SIMPLE_PROMPT_STUDIO, {})+ if single_pass_extration_plugin:+ single_pass_extration_plugin["entrypoint_cls"](+ app=app,+ challenge_plugin=plugins.get(PSKeys.CHALLENGE, {}),+ get_cleaned_context=get_cleaned_context,+ )+ if summarize_plugin:+ summarize_plugin["entrypoint_cls"](+ app=app,+ )+ if simple_prompt_studio:+ simple_prompt_studio["entrypoint_cls"](+ app=app,+ )+++def query_usage_metadata(token: str, metadata: dict[str, Any]) -> dict[str, Any]:org_id: str = AuthenticationMiddleware.get_account_from_bearer_token(token)run_id: str = metadata["run_id"]query: str = f"""@@ -188,17 +210,21 @@ def construct_and_run_prompt(llm: LLM,context: str,prompt: str,-) -> tuple[str, dict[str, Any]]:+ metadata: dict[str, Any],+) -> str:prompt = construct_prompt(preamble=tool_settings.get(PSKeys.PREAMBLE, ""),prompt=output[prompt],postamble=tool_settings.get(PSKeys.POSTAMBLE, ""),grammar_list=tool_settings.get(PSKeys.GRAMMAR, []),context=context,+ platform_postamble=tool_settings.get(PSKeys.PLATFORM_POSTAMBLE, ""),)return run_completion(llm=llm,prompt=prompt,+ metadata=metadata,+ prompt_key=output[PSKeys.NAME],)@@ -208,30 +234,10 @@ def construct_prompt(postamble: str,grammar_list: list[dict[str, Any]],context: str,+ platform_postamble: str,) -> str:- logger: Logger = current_app.logger- # Let's cleanup the context. Remove if 3 consecutive newlines are found- context_lines = context.split("\n")- new_context_lines = []- empty_line_count = 0- for line in context_lines:- if line.strip() == "":- empty_line_count += 1- else:- if empty_line_count >= 3:- empty_line_count = 3- for i in range(empty_line_count):- new_context_lines.append("")- empty_line_count = 0- new_context_lines.append(line.rstrip())- context = "\n".join(new_context_lines)- logger.info(- f"Old context length: {len(context_lines)}, "- f"New context length: {len(new_context_lines)}"- )-prompt = (- f"{preamble}\n\nContext:\n---------------{context}\n"+ f"{preamble}\n\nContext:\n---------------\n{context}\n"f"-----------------\n\nQuestion or Instruction: {prompt}\n")if grammar_list is not None and len(grammar_list) > 0:@@ -246,20 +252,39 @@ def construct_prompt(if len(synonyms) > 0 and word != "":prompt += f'\nNote: You can consider that the word {word} is same as \{", ".join(synonyms)} in both the quesiton and the context.' # noqa- prompt += f"\n\n{postamble}"- prompt += "\n\nAnswer:"+ if platform_postamble:+ platform_postamble += "\n\n"+ prompt += f"\n\n{postamble}\n\n{platform_postamble}Answer:"return promptdef run_completion(llm: LLM,prompt: str,-) -> tuple[str, dict[str, Any]]:+ metadata: Optional[dict[str, str]] = None,+ prompt_key: Optional[str] = None,+) -> str:logger: Logger = current_app.loggertry:- completion = llm.complete(prompt)-+ extract_epilogue_plugin: dict[str, Any] = plugins.get(+ PSKeys.EXTRACT_EPILOGUE, {}+ )+ extract_epilogue = None+ if extract_epilogue_plugin:+ extract_epilogue = extract_epilogue_plugin["entrypoint_cls"].run+ completion = llm.complete(+ prompt=prompt,+ process_text=extract_epilogue,+ )answer: str = completion[PSKeys.RESPONSE].text+ epilogue = completion.get(PSKeys.EPILOGUE)+ if all([metadata, epilogue, prompt_key]):+ try:+ logger.info(f"Epilogue extracted from LLM: {epilogue}")+ epilogue = json.loads(epilogue)+ except JSONDecodeError:+ logger.error(f"Failed to convert epilogue to JSON: {epilogue}")+ metadata.setdefault(PSKeys.EPILOGUE, {})[prompt_key] = epiloguereturn answer# TODO: Catch and handle specific exception hereexcept SdkRateLimitError as e:diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.pyindex d0cd36ae6..40c85c2f5 100644--- a/prompt-service/src/unstract/prompt_service/main.py+++ b/prompt-service/src/unstract/prompt_service/main.py@@ -4,7 +4,6 @@ from enum import Enumfrom json import JSONDecodeErrorfrom typing import Any, Optional-import peeweefrom flask import json, jsonify, requestfrom llama_index.core.vector_stores import ExactMatchFilter, MetadataFiltersfrom unstract.prompt_service.authentication_middleware import AuthenticationMiddleware@@ -13,10 +12,11 @@ from unstract.prompt_service.constants import PromptServiceContants as PSKeysfrom unstract.prompt_service.constants import RunLevelfrom unstract.prompt_service.exceptions import APIError, ErrorResponse, NoPayloadErrorfrom unstract.prompt_service.helper import (- EnvLoader,construct_and_run_prompt,extract_variable,+ get_cleaned_context,plugin_loader,+ plugins,query_usage_metadata,run_completion,)@@ -31,9 +31,6 @@ from werkzeug.exceptions import HTTPExceptionfrom unstract.core.pubsub_helper import LogPublisher-POS_TEXT_PATH = "/tmp/pos.txt"-USE_UNSTRACT_PROMPT = True-MAX_RETRIES = 3NO_CONTEXT_ERROR = ("Couldn't fetch context from vector DB. ""This happens usually due to a delay by the Vector DB "@@ -41,28 +38,9 @@ NO_CONTEXT_ERROR = ("Please try again after some time")-PG_BE_HOST = EnvLoader.get_env_or_die("PG_BE_HOST")-PG_BE_PORT = EnvLoader.get_env_or_die("PG_BE_PORT")-PG_BE_USERNAME = EnvLoader.get_env_or_die("PG_BE_USERNAME")-PG_BE_PASSWORD = EnvLoader.get_env_or_die("PG_BE_PASSWORD")-PG_BE_DATABASE = EnvLoader.get_env_or_die("PG_BE_DATABASE")--be_db = peewee.PostgresqlDatabase(- PG_BE_DATABASE,- user=PG_BE_USERNAME,- password=PG_BE_PASSWORD,- host=PG_BE_HOST,- port=PG_BE_PORT,-)-be_db.init(PG_BE_DATABASE)-be_db.connect()--AuthenticationMiddleware.be_db = be_db-app = create_app()---plugins: dict[str, dict[str, Any]] = plugin_loader(app)+# Load plugins+plugin_loader(app)def _publish_log(@@ -108,9 +86,6 @@ def prompt_processor() -> Any:file_hash = payload.get(PSKeys.FILE_HASH)doc_name = str(payload.get(PSKeys.FILE_NAME, ""))log_events_id: str = payload.get(PSKeys.LOG_EVENTS_ID, "")- include_metadata = (- request.args.get(PSKeys.INCLUDE_METADATA, "false").lower() == "true"- )structured_output: dict[str, Any] = {}metadata: Optional[dict[str, Any]] = {PSKeys.RUN_ID: run_id,@@ -290,8 +265,11 @@ def prompt_processor() -> Any:llm=llm,context=context,prompt="promptx",+ metadata=metadata,+ )+ metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = get_cleaned_context(+ context)- metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = contextelse:answer = "NA"_publish_log(@@ -316,8 +294,11 @@ def prompt_processor() -> Any:llm=llm,vector_index=vector_index,retrieval_type=retrieval_strategy,+ metadata=metadata,+ )+ metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = get_cleaned_context(+ context)- metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = contextelse:app.logger.info("Invalid retrieval strategy passed: %s",@@ -611,11 +592,8 @@ def prompt_processor() -> Any:RunLevel.RUN,"Execution complete",)- if include_metadata:- metadata = query_usage_metadata(db=be_db, token=platform_key, metadata=metadata)- response = {PSKeys.METADATA: metadata, PSKeys.OUTPUT: structured_output}- else:- response = {PSKeys.OUTPUT: structured_output}+ metadata = query_usage_metadata(token=platform_key, metadata=metadata)+ response = {PSKeys.METADATA: metadata, PSKeys.OUTPUT: structured_output}return response@@ -626,6 +604,7 @@ def run_retrieval( # type:ignorellm: LLM,vector_index,retrieval_type: str,+ metadata: dict[str, Any],) -> tuple[str, str]:prompt = output[PSKeys.PROMPTX]if retrieval_type == PSKeys.SUBQUESTION:@@ -656,6 +635,7 @@ def run_retrieval( # type:ignorellm=llm,context=context,prompt="promptx",+ metadata=metadata,)return (answer, context)@@ -685,30 +665,6 @@ def _retrieve_context(output, doc_id, vector_index, answer) -> str:return text-def enable_plugins() -> None:- """Enables plugins if available."""- single_pass_extration_plugin: dict[str, Any] = plugins.get(- PSKeys.SINGLE_PASS_EXTRACTION, {}- )- summarize_plugin: dict[str, Any] = plugins.get(PSKeys.SUMMARIZE, {})- simple_prompt_studio: dict[str, Any] = plugins.get(PSKeys.SIMPLE_PROMPT_STUDIO, {})- if single_pass_extration_plugin:- single_pass_extration_plugin["entrypoint_cls"](- app=app, challenge_plugin=plugins.get(PSKeys.CHALLENGE, {})- )- if summarize_plugin:- summarize_plugin["entrypoint_cls"](- app=app,- )- if simple_prompt_studio:- simple_prompt_studio["entrypoint_cls"](- app=app,- )---enable_plugins()--def log_exceptions(e: HTTPException):"""Helper method to log exceptions.diff --git a/tools/structure/src/constants.py b/tools/structure/src/constants.pyindex 72780fe2b..1f79c109e 100644--- a/tools/structure/src/constants.py+++ b/tools/structure/src/constants.py@@ -67,4 +67,6 @@ class SettingsKeys:FILE_HASH = "file_hash"ENABLE_HIGHLIGHT = "enable_highlight"NAME = "name"- INCLUDE_METADATA = "include_metadata"+ METADATA = "metadata"+ EPILOGUE = "epilogue"+ HIGHLIGHT_DATA = "highlight_data"diff --git a/tools/structure/src/main.py b/tools/structure/src/main.pyindex 36678e74e..929d3fe73 100644--- a/tools/structure/src/main.py+++ b/tools/structure/src/main.py@@ -27,7 +27,6 @@ class StructureTool(BaseTool):prompt_registry_id: str = settings[SettingsKeys.PROMPT_REGISTRY_ID]challenge_llm: str = settings[SettingsKeys.CHALLENGE_LLM_ADAPTER_ID]enable_challenge: bool = settings[SettingsKeys.ENABLE_CHALLENGE]- include_metadata: bool = settings.get(SettingsKeys.INCLUDE_METADATA, False)summarize_as_source: bool = settings[SettingsKeys.SUMMARIZE_AS_SOURCE]single_pass_extraction_mode: bool = settings[SettingsKeys.SINGLE_PASS_EXTRACTION_MODE@@ -80,7 +79,6 @@ class StructureTool(BaseTool):SettingsKeys.FILE_HASH: file_hash,SettingsKeys.FILE_NAME: file_name,}- params = {SettingsKeys.INCLUDE_METADATA: include_metadata}# TODO: Need to split extraction and indexing# to avoid unwanted indexingself.stream_log("Indexing document")@@ -90,6 +88,12 @@ class StructureTool(BaseTool):self.get_exec_metadata.get(MetadataKey.SOURCE_NAME),)+ process_text = None+ try:+ from helper import process_text # type: ignore [attr-defined]+ except ImportError:+ self.stream_log("Function 'process_text' is not found")+if tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION]:index.index(tool_id=tool_id,@@ -104,6 +108,7 @@ class StructureTool(BaseTool):reindex=True,usage_kwargs=usage_kwargs,enable_highlight=enable_highlight,+ process_text=process_text,)if summarize_as_source:summarize_file_hash = self._summarize_and_index(@@ -120,7 +125,6 @@ class StructureTool(BaseTool):self.stream_log("Fetching response for single pass extraction")prompt_service_resp = responder.single_pass_extraction(payload=payload,- params=params,)else:try:@@ -142,6 +146,7 @@ class StructureTool(BaseTool):reindex=reindex,usage_kwargs=usage_kwargs,enable_highlight=enable_highlight,+ process_text=process_text,)if summarize_as_source:@@ -163,7 +168,6 @@ class StructureTool(BaseTool):self.stream_log("Fetching responses for prompts...")prompt_service_resp = responder.answer_prompt(payload=payload,- params=params,)# TODO: Make use of dataclasses@@ -174,6 +178,22 @@ class StructureTool(BaseTool):)structured_output = prompt_service_resp[SettingsKeys.STRUCTURE_OUTPUT]+ structured_output_dict = json.loads(structured_output)++ if not summarize_as_source:+ metadata = structured_output_dict[SettingsKeys.METADATA]+ epilogue = metadata.pop(SettingsKeys.EPILOGUE, None)+ if epilogue:+ try:+ from helper import transform_dict # type: ignore [attr-defined]++ highlight_data = transform_dict(epilogue, tool_data_dir)+ metadata[SettingsKeys.HIGHLIGHT_DATA] = highlight_data+ except ImportError:+ self.stream_log("Function 'transform_dict' is not found")+ # Update the dictionary with modified metadata+ structured_output_dict[SettingsKeys.METADATA] = metadata+ structured_output = json.dumps(structured_output_dict)# Update GUIinput_log = (@@ -191,11 +211,11 @@ class StructureTool(BaseTool):output_path = Path(output_dir) / f"{Path(source_name).stem}.json"with open(output_path, "w", encoding="utf-8") as f:f.write(structured_output)- except (FileNotFoundError, PermissionError, OSError) as e:+ except OSError as e:self.stream_error_and_exit(f"Error creating output file: {e}")except json.JSONDecodeError as e:self.stream_error_and_exit(f"Error encoding JSON: {e}")- self.write_tool_result(data=json.loads(structured_output))+ self.write_tool_result(data=structured_output_dict)def _summarize_and_index(self,diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.pyindex eef11fffc..a506b9707 100644--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py@@ -12,7 +12,3 @@ class ToolCommandKey:SPEC = "spec"VARIABLES = "variables"ICON = "icon"---class Common:- INCLUDE_METADATA = "include_metadata"diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.pyindex a603a590b..d878ce090 100644--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py@@ -1,6 +1,6 @@from typing import Any, Optional-from unstract.tool_sandbox.constants import Common, ToolCommandKey, UnstractWorker+from unstract.tool_sandbox.constants import ToolCommandKey, UnstractWorkerfrom unstract.tool_sandbox.helper import ToolSandboxHelper@@ -16,7 +16,6 @@ class ToolSandbox:tool_instance_id: Optional[str] = None,environment_variables: dict[str, Any] = {},messaging_channel: Optional[str] = None,- include_metadata: bool = False,):"""PLATFORM_SERVICE_API_KEY should be available in the environment."""self.messaging_channel = str(messaging_channel)@@ -31,11 +30,9 @@ class ToolSandbox:self.tool_instance_id = tool_instance_idself.image_name = image_nameself.image_tag = image_tag- self.include_metadata = include_metadataself.settings: dict[str, Any] = {}def set_tool_instance_settings(self, tool_settings: dict[str, Any]) -> None:- tool_settings[Common.INCLUDE_METADATA] = self.include_metadataself.settings = tool_settingsdef get_tool_uid(self) -> str:diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.pyindex d5fe76804..f76ca885c 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py@@ -28,7 +28,6 @@ class ToolsUtils:workflow: WorkflowDto,platform_service_api_key: str,ignore_processed_entities: bool = False,- include_metadata: bool = False,) -> None:self.redis = redisself.tool_registry = ToolRegistry()@@ -53,7 +52,6 @@ class ToolsUtils:self.llmw_max_polls = ToolsUtils.get_env(ToolRV.ADAPTER_LLMW_MAX_POLLS, raise_exception=False)- self.include_metadata = include_metadatadef set_messaging_channel(self, messaging_channel: str) -> None:self.messaging_channel = messaging_channel@@ -137,7 +135,6 @@ class ToolsUtils:image_tag=image_tag,environment_variables=tool_envs,messaging_channel=self.messaging_channel,- include_metadata=self.include_metadata,)tool_sandbox.set_tool_instance_settings(tool_instance.metadata)tool_sandboxes.append(tool_sandbox)diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.pyindex 270ebb670..a49021f98 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py@@ -44,7 +44,6 @@ class WorkflowExecutionService:tool_instances: list[ToolInstance],platform_service_api_key: str,ignore_processed_entities: bool = False,- include_metadata: bool = False,) -> None:self.organization_id = organization_idself.workflow_id = workflow_id@@ -56,7 +55,6 @@ class WorkflowExecutionService:workflow=workflow,platform_service_api_key=platform_service_api_key,ignore_processed_entities=False,- include_metadata=include_metadata,)self.tool_sandboxes: list[ToolSandbox] = []self.ignore_processed_entities = ignore_processed_entities