Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[FEATURE] Changes required for highlighting (#555)

Deepak Kb6057e8f451b
diff --git a/backend/api/api_deployment_views.py b/backend/api/api_deployment_views.py
index 7217117ca..ca65f00e2 100644
--- a/backend/api/api_deployment_views.py
+++ b/backend/api/api_deployment_views.py
@@ -74,6 +74,7 @@ class DeploymentExecution(views.APIView):
self, request: Request, org_name: str, api_name: str, api: APIDeployment
) -> Response:
execution_id = request.query_params.get("execution_id")
+ include_metadata = request.query_params.get("include_metadata", False)
if not execution_id:
raise InvalidAPIRequest("execution_id shouldn't be empty")
response: ExecutionResponse = DeploymentHelper.get_execution_status(
@@ -87,6 +88,11 @@ class DeploymentExecution(views.APIView):
},
status=status.HTTP_422_UNPROCESSABLE_ENTITY,
)
+ if (
+ response.execution_status == CeleryTaskState.SUCCESS.value
+ and not include_metadata
+ ):
+ response.remove_result_metadata_keys()
return Response(
{"status": response.execution_status, "message": response.result},
status=status.HTTP_200_OK,
diff --git a/backend/api/deployment_helper.py b/backend/api/deployment_helper.py
index 3866af51d..cca63fb9c 100644
--- a/backend/api/deployment_helper.py
+++ b/backend/api/deployment_helper.py
@@ -174,11 +174,13 @@ class DeploymentHelper(BaseAPIKeyValidator):
hash_values_of_files=hash_values_of_files,
timeout=timeout,
execution_id=execution_id,
- include_metadata=include_metadata,
)
result.status_api = DeploymentHelper.construct_status_endpoint(
api_endpoint=api.api_endpoint, execution_id=execution_id
)
+ if not include_metadata:
+ result.remove_result_metadata_keys()
+ cls._send_notification(api=api, result=result)
except Exception as error:
DestinationConnector.delete_api_storage_dir(
workflow_id=workflow_id, execution_id=execution_id
diff --git a/backend/file_management/file_management_helper.py b/backend/file_management/file_management_helper.py
index 6293f2583..ce538068b 100644
--- a/backend/file_management/file_management_helper.py
+++ b/backend/file_management/file_management_helper.py
@@ -223,15 +223,22 @@ class FileManagerHelper:
) -> bool:
fs = file_system.get_fsspec_fs()
base_path = FileManagerHelper._get_base_path(file_system, path)
-
base_file_name, _ = os.path.splitext(file_name)
- file_name_txt = base_file_name + ".txt"
-
- for directory in directories:
- file_path = str(Path(base_path) / directory / file_name_txt)
+ pattern = f"{base_file_name}.*"
+ file_paths = FileManagerHelper._find_files(fs, base_path, directories, pattern)
+ for file_path in file_paths:
FileManagerHelper._delete_file(fs, file_path)
return True
+ @staticmethod
+ def _find_files(fs, base_path: str, directories: list[str], pattern: str):
+ file_paths = []
+ for directory in directories:
+ directory_path = str(Path(base_path) / directory)
+ for file in fs.glob(f"{directory_path}/{pattern}"):
+ file_paths.append(file)
+ return file_paths
+
@staticmethod
def handle_sub_directory_for_tenants(
org_id: str, user_id: str, tool_id: str, is_create: bool
diff --git a/backend/prompt_studio/processor_loader.py b/backend/prompt_studio/processor_loader.py
index 2a371a458..0bd8a7ddd 100644
--- a/backend/prompt_studio/processor_loader.py
+++ b/backend/prompt_studio/processor_loader.py
@@ -72,3 +72,13 @@ def load_plugins() -> list[Any]:
logger.info("No processor plugins found.")
return processor_plugins
+
+
+def get_plugin_class_by_name(name: str, plugins: list[Any]) -> Any:
+ """Retrieve a specific plugin class by name."""
+ for plugin in plugins:
+ metadata = plugin[ProcessorConfig.METADATA]
+ if metadata.get(ProcessorConfig.METADATA_NAME) == name:
+ return metadata.get(ProcessorConfig.METADATA_SERVICE_CLASS)
+ logger.warning("Plugin with name '%s' not found.", name)
+ return None
diff --git a/backend/prompt_studio/prompt_studio_core/constants.py b/backend/prompt_studio/prompt_studio_core/constants.py
index 213fc066f..38c48dbd5 100644
--- a/backend/prompt_studio/prompt_studio_core/constants.py
+++ b/backend/prompt_studio/prompt_studio_core/constants.py
@@ -62,7 +62,6 @@ class ToolStudioPromptKeys:
WORD = "word"
SYNONYMS = "synonyms"
OUTPUTS = "outputs"
- ASSERT_PROMPT = "assert_prompt"
SECTION = "section"
DEFAULT = "default"
REINDEX = "reindex"
@@ -89,6 +88,7 @@ class ToolStudioPromptKeys:
CONTEXT = "context"
METADATA = "metadata"
INCLUDE_METADATA = "include_metadata"
+ PLATFORM_POSTAMBLE = "platform_postamble"
class FileViewTypes:
diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
index befabc54d..a81797d70 100644
--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
@@ -3,7 +3,7 @@ import logging
import os
import uuid
from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Callable, Optional
from account.constants import Common
from account.models import User
@@ -291,6 +291,7 @@ class PromptStudioHelper:
document_id: str,
is_summary: bool = False,
run_id: str = None,
+ text_processor: Optional[type[Any]] = None,
) -> Any:
"""Method to index a document.
@@ -340,7 +341,9 @@ class PromptStudioHelper:
# Need to check the user who created profile manager
# has access to adapters configured in profile manager
PromptStudioHelper.validate_profile_manager_owner_access(default_profile)
-
+ process_text = None
+ if text_processor:
+ process_text = text_processor.process
doc_id = PromptStudioHelper.dynamic_indexer(
profile_manager=default_profile,
tool_id=tool_id,
@@ -351,6 +354,7 @@ class PromptStudioHelper:
reindex=True,
run_id=run_id,
user_id=user_id,
+ process_text=process_text,
)
logger.info(f"[{tool_id}] Indexing successful for doc: {file_name}")
@@ -372,6 +376,7 @@ class PromptStudioHelper:
id: Optional[str] = None,
run_id: str = None,
profile_manager_id: Optional[str] = None,
+ text_processor: Optional[type[Any]] = None,
) -> Any:
"""Execute chain/single run of the prompts. Makes a call to prompt
service and returns the dict of response.
@@ -398,19 +403,26 @@ class PromptStudioHelper:
if id:
return PromptStudioHelper._execute_single_prompt(
- id,
- doc_path,
- doc_name,
- tool_id,
- org_id,
- user_id,
- document_id,
- run_id,
- profile_manager_id,
+ id=id,
+ doc_path=doc_path,
+ doc_name=doc_name,
+ tool_id=tool_id,
+ org_id=org_id,
+ user_id=user_id,
+ document_id=document_id,
+ run_id=run_id,
+ profile_manager_id=profile_manager_id,
+ text_processor=text_processor,
)
else:
return PromptStudioHelper._execute_prompts_in_single_pass(
- doc_path, tool_id, org_id, user_id, document_id, run_id
+ doc_path=doc_path,
+ tool_id=tool_id,
+ org_id=org_id,
+ user_id=user_id,
+ document_id=document_id,
+ run_id=run_id,
+ text_processor=text_processor,
)
@staticmethod
@@ -424,6 +436,7 @@ class PromptStudioHelper:
document_id,
run_id,
profile_manager_id,
+ text_processor: Optional[type[Any]] = None,
):
prompt_instance = PromptStudioHelper._fetch_prompt_from_id(id)
prompt_name = prompt_instance.prompt_key
@@ -458,7 +471,9 @@ class PromptStudioHelper:
LogLevels.RUN,
"Invoking prompt service",
)
-
+ process_text = None
+ if text_processor:
+ process_text = text_processor.process
try:
response = PromptStudioHelper._fetch_response(
doc_path=doc_path,
@@ -470,9 +485,15 @@ class PromptStudioHelper:
run_id=run_id,
profile_manager_id=profile_manager_id,
user_id=user_id,
+ process_text=process_text,
)
return PromptStudioHelper._handle_response(
- response, run_id, prompts, document_id, False, profile_manager_id
+ response=response,
+ run_id=run_id,
+ prompts=prompts,
+ document_id=document_id,
+ is_single_pass=False,
+ profile_manager_id=profile_manager_id,
)
except Exception as e:
logger.error(
@@ -495,7 +516,13 @@ class PromptStudioHelper:
@staticmethod
def _execute_prompts_in_single_pass(
- doc_path, tool_id, org_id, user_id, document_id, run_id
+ doc_path,
+ tool_id,
+ org_id,
+ user_id,
+ document_id,
+ run_id,
+ text_processor: Optional[type[Any]] = None,
):
prompts = PromptStudioHelper.fetch_prompt_from_tool(tool_id)
prompts = [
@@ -513,7 +540,9 @@ class PromptStudioHelper:
LogLevels.RUN,
"Executing prompts in single pass",
)
-
+ process_text = None
+ if text_processor:
+ process_text = text_processor.process
try:
tool = prompts[0].tool_id
response = PromptStudioHelper._fetch_single_pass_response(
@@ -524,9 +553,14 @@ class PromptStudioHelper:
document_id=document_id,
run_id=run_id,
user_id=user_id,
+ process_text=process_text,
)
return PromptStudioHelper._handle_response(
- response, run_id, prompts, document_id, True
+ response=response,
+ run_id=run_id,
+ prompts=prompts,
+ document_id=document_id,
+ is_single_pass=True,
)
except Exception as e:
logger.error(
@@ -556,7 +590,12 @@ class PromptStudioHelper:
@staticmethod
def _handle_response(
- response, run_id, prompts, document_id, is_single_pass, profile_manager_id=None
+ response,
+ run_id,
+ prompts,
+ document_id,
+ is_single_pass,
+ profile_manager_id=None,
):
if response.get("status") == IndexingStatus.PENDING_STATUS.value:
return {
@@ -586,6 +625,7 @@ class PromptStudioHelper:
run_id: str,
user_id: str,
profile_manager_id: Optional[str] = None,
+ process_text: Optional[Callable[[str], str]] = None,
) -> Any:
"""Utility function to invoke prompt service. Used internally.
@@ -656,6 +696,7 @@ class PromptStudioHelper:
is_summary=tool.summarize_as_source,
run_id=run_id,
user_id=user_id,
+ process_text=process_text,
)
if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:
return {
@@ -713,6 +754,9 @@ class PromptStudioHelper:
tool_settings[TSPKeys.PREAMBLE] = tool.preamble
tool_settings[TSPKeys.POSTAMBLE] = tool.postamble
tool_settings[TSPKeys.GRAMMAR] = grammar_list
+ tool_settings[TSPKeys.PLATFORM_POSTAMBLE] = getattr(
+ settings, TSPKeys.PLATFORM_POSTAMBLE.upper(), ""
+ )
tool_id = str(tool.tool_id)
@@ -760,6 +804,7 @@ class PromptStudioHelper:
is_summary: bool = False,
reindex: bool = False,
run_id: str = None,
+ process_text: Optional[Callable[[str], str]] = None,
) -> Any:
"""Used to index a file based on the passed arguments.
@@ -842,6 +887,7 @@ class PromptStudioHelper:
reindex=reindex,
output_file_path=extract_file_path,
usage_kwargs=usage_kwargs.copy(),
+ process_text=process_text,
)
PromptStudioIndexHelper.handle_index_manager(
@@ -875,6 +921,7 @@ class PromptStudioHelper:
user_id: str,
document_id: str,
run_id: str = None,
+ process_text: Optional[Callable[[str], str]] = None,
) -> Any:
tool_id: str = str(tool.tool_id)
outputs: list[dict[str, Any]] = []
@@ -910,6 +957,7 @@ class PromptStudioHelper:
document_id=document_id,
run_id=run_id,
user_id=user_id,
+ process_text=process_text,
)
if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:
return {
diff --git a/backend/prompt_studio/prompt_studio_core/views.py b/backend/prompt_studio/prompt_studio_core/views.py
index 18915ac24..7b197fa39 100644
--- a/backend/prompt_studio/prompt_studio_core/views.py
+++ b/backend/prompt_studio/prompt_studio_core/views.py
@@ -9,7 +9,7 @@ from django.http import HttpRequest
from file_management.exceptions import FileNotFound
from file_management.file_management_helper import FileManagerHelper
from permissions.permission import IsOwner, IsOwnerOrSharedUser
-from prompt_studio.processor_loader import ProcessorConfig, load_plugins
+from prompt_studio.processor_loader import get_plugin_class_by_name, load_plugins
from prompt_studio.prompt_profile_manager.constants import (
ProfileManagerErrors,
ProfileManagerKeys,
@@ -219,6 +219,10 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
)
document: DocumentManager = DocumentManager.objects.get(pk=document_id)
file_name: str = document.document_name
+ text_processor = get_plugin_class_by_name(
+ name="text_processor",
+ plugins=self.processor_plugins,
+ )
# Generate a run_id
run_id = CommonUtils.generate_uuid()
unique_id = PromptStudioHelper.index_document(
@@ -228,22 +232,23 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
user_id=tool.created_by.user_id,
document_id=document_id,
run_id=run_id,
+ text_processor=text_processor,
)
usage_kwargs: dict[Any, Any] = dict()
usage_kwargs[ToolStudioPromptKeys.RUN_ID] = run_id
- for processor_plugin in self.processor_plugins:
- cls = processor_plugin[ProcessorConfig.METADATA][
- ProcessorConfig.METADATA_SERVICE_CLASS
- ]
- cls.process(
- tool_id=str(tool.tool_id),
- file_name=file_name,
- org_id=UserSessionUtils.get_organization_id(request),
- user_id=tool.created_by.user_id,
- document_id=document_id,
- usage_kwargs=usage_kwargs.copy(),
- )
+ cls = get_plugin_class_by_name(
+ name="summarizer",
+ plugins=self.processor_plugins,
+ )
+ cls.process(
+ tool_id=str(tool.tool_id),
+ file_name=file_name,
+ org_id=UserSessionUtils.get_organization_id(request),
+ user_id=tool.created_by.user_id,
+ document_id=document_id,
+ usage_kwargs=usage_kwargs.copy(),
+ )
if unique_id:
return Response(
@@ -276,7 +281,10 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
if not run_id:
# Generate a run_id
run_id = CommonUtils.generate_uuid()
-
+ text_processor = get_plugin_class_by_name(
+ name="text_processor",
+ plugins=self.processor_plugins,
+ )
response: dict[str, Any] = PromptStudioHelper.prompt_responder(
id=id,
tool_id=tool_id,
@@ -285,6 +293,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
document_id=document_id,
run_id=run_id,
profile_manager_id=profile_manager,
+ text_processor=text_processor,
)
return Response(response, status=status.HTTP_200_OK)
@@ -308,12 +317,17 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
if not run_id:
# Generate a run_id
run_id = CommonUtils.generate_uuid()
+ text_processor = get_plugin_class_by_name(
+ name="text_processor",
+ plugins=self.processor_plugins,
+ )
response: dict[str, Any] = PromptStudioHelper.prompt_responder(
tool_id=tool_id,
org_id=UserSessionUtils.get_organization_id(request),
user_id=custom_tool.created_by.user_id,
document_id=document_id,
run_id=run_id,
+ text_processor=text_processor,
)
return Response(response, status=status.HTTP_200_OK)
@@ -491,7 +505,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
# Delete the files
FileManagerHelper.delete_file(file_system, path, file_name)
# Directories to delete the text files
- directories = ["extract/", "summarize/"]
+ directories = ["extract/", "extract/metadata/", "summarize/"]
FileManagerHelper.delete_related_files(
file_system, path, file_name, directories
)
diff --git a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
index c9b3672a8..f431e4539 100644
--- a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
+++ b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
@@ -1,4 +1,3 @@
-import json
import logging
from typing import Any, Optional
@@ -106,9 +105,9 @@ class OutputManagerHelper:
continue
if not is_single_pass_extract:
- context = json.dumps(context.get(prompt.prompt_key))
+ context = context.get(prompt.prompt_key)
- output = json.dumps(outputs.get(prompt.prompt_key))
+ output = outputs.get(prompt.prompt_key)
profile_manager = default_profile
eval_metrics = outputs.get(f"{prompt.prompt_key}__evaluation", [])
diff --git a/backend/prompt_studio/prompt_studio_registry/constants.py b/backend/prompt_studio/prompt_studio_registry/constants.py
index 7fea2c73c..6cd809ae9 100644
--- a/backend/prompt_studio/prompt_studio_registry/constants.py
+++ b/backend/prompt_studio/prompt_studio_registry/constants.py
@@ -92,6 +92,7 @@ class JsonSchemaKey:
SUMMARIZE_PROMPT = "summarize_prompt"
SUMMARIZE_AS_SOURCE = "summarize_as_source"
ENABLE_HIGHLIGHT = "enable_highlight"
+ PLATFORM_POSTAMBLE = "platform_postamble"
class SpecKey:
diff --git a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py
index 8a9877a85..adb062984 100644
--- a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py
+++ b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py
@@ -266,6 +266,9 @@ class PromptStudioRegistryHelper:
tool.single_pass_extraction_mode
)
tool_settings[JsonSchemaKey.ENABLE_HIGHLIGHT] = tool.enable_highlight
+ tool_settings[JsonSchemaKey.PLATFORM_POSTAMBLE] = getattr(
+ settings, JsonSchemaKey.PLATFORM_POSTAMBLE.upper(), ""
+ )
for prompt in prompts:
if prompt.prompt_type == JsonSchemaKey.NOTES or not prompt.active:
diff --git a/backend/sample.env b/backend/sample.env
index bb8267dfd..fd45516ae 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -90,9 +90,9 @@ PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data
# Structure Tool Image (Runs prompt studio exported tools)
# https://hub.docker.com/r/unstract/tool-structure
-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.36"
+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.37"
STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"
-STRUCTURE_TOOL_IMAGE_TAG="0.0.36"
+STRUCTURE_TOOL_IMAGE_TAG="0.0.37"
# Feature Flags
EVALUATION_SERVER_IP=unstract-flipt
diff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.py
index 5b5a590ba..b922f4e64 100644
--- a/backend/scheduler/tasks.py
+++ b/backend/scheduler/tasks.py
@@ -117,6 +117,7 @@ def execute_pipeline_task(
execution_response = WorkflowHelper.complete_execution(
workflow, execution_id, pipepline_id
)
+ execution_response.remove_result_metadata_keys()
logger.info(f"Execution response: {execution_response}")
logger.info(f"Execution completed for pipeline: {name}")
except Exception as e:
@@ -166,6 +167,7 @@ def execute_pipeline_task_v2(
execution_response = WorkflowHelper.complete_execution(
workflow, execution_id, pipeline_id
)
+ execution_response.remove_result_metadata_keys()
logger.info(
f"Execution response for pipeline {pipeline_name} of organization "
f"{organization_id}: {execution_response}"
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.py
index 21b2a21d0..61dd658ab 100644
--- a/backend/workflow_manager/endpoint/destination.py
+++ b/backend/workflow_manager/endpoint/destination.py
@@ -267,8 +267,12 @@ class DestinationConnector(BaseConnector):
execution_id_name = str(
destination_configurations.get(DestinationKey.EXECUTION_ID, "execution_id")
)
-
data = self.get_result()
+ # If data is None, don't execute CREATE or INSERT query
+ if not data:
+ return
+ # Remove metadata from result
+ data.pop("metadata", None)
values = DatabaseUtils.get_columns_and_values(
column_mode_str=column_mode,
data=data,
@@ -286,9 +290,6 @@ class DestinationConnector(BaseConnector):
connector_settings=connector_settings,
)
engine = db_class.get_engine()
- # If data is None, don't execute CREATE or INSERT query
- if data is None:
- return
DatabaseUtils.create_table_if_not_exists(
db_class=db_class,
engine=engine,
diff --git a/backend/workflow_manager/workflow/dto.py b/backend/workflow_manager/workflow/dto.py
index 1f16d7df3..66ea68976 100644
--- a/backend/workflow_manager/workflow/dto.py
+++ b/backend/workflow_manager/workflow/dto.py
@@ -41,6 +41,58 @@ class ExecutionResponse:
self.message = self.message or None
self.status_api = self.status_api or None
+ def remove_result_metadata_keys(self, keys_to_remove: list[str] = []) -> None:
+ """Removes specified keys from the 'metadata' dictionary within each
+ 'result' dictionary in the 'result' list attribute of the instance. If
+ 'keys_to_remove' is empty, the 'metadata' key itself is removed.
+
+ Args:
+ keys_to_remove (List[str]): List of keys to be removed from 'metadata'.
+ """
+ if not isinstance(self.result, list):
+ return
+
+ for item in self.result:
+ if not isinstance(item, dict):
+ break
+
+ result = item.get("result")
+ if not isinstance(result, dict):
+ break
+
+ self._remove_specific_keys(result=result, keys_to_remove=keys_to_remove)
+
+ def _remove_specific_keys(self, result: dict, keys_to_remove: list[str]) -> None:
+ """Removes specified keys from the 'metadata' dictionary within the
+ provided 'result' dictionary. If 'keys_to_remove' is empty, the
+ 'metadata' dictionary is cleared.
+
+ Args:
+ result (dict): The dictionary containing the 'metadata' key.
+ keys_to_remove (List[str]): List of keys to be removed from 'metadata'.
+ """
+ metadata = result.get("metadata", {})
+ if keys_to_remove:
+ for key in keys_to_remove:
+ metadata.pop(key, None)
+ else:
+ metadata = {}
+ self._update_metadata(result=result, metadata=metadata)
+
+ def _update_metadata(self, result: dict, metadata: dict) -> None:
+ """Updates the 'metadata' key in the provided 'result' dictionary. If
+ 'metadata' is empty, removes the 'metadata' key from 'result'.
+
+ Args:
+ result (dict): The dictionary to be updated.
+ metadata (dict): The new metadata dictionary to be set. If empty, 'metadata'
+ is removed.
+ """
+ if metadata:
+ result["metadata"] = metadata
+ else:
+ result.pop("metadata", None)
+
@dataclass
class AsyncResultData:
diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.py
index 1f126469d..c94d7eb70 100644
--- a/backend/workflow_manager/workflow/execution.py
+++ b/backend/workflow_manager/workflow/execution.py
@@ -36,7 +36,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
scheduled: bool = False,
mode: tuple[str, str] = WorkflowExecution.Mode.INSTANT,
workflow_execution: Optional[WorkflowExecution] = None,
- include_metadata: bool = False,
) -> None:
tool_instances_as_dto = []
for tool_instance in tool_instances:
@@ -58,7 +57,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
tool_instances=tool_instances_as_dto,
platform_service_api_key=str(platform_key.key),
ignore_processed_entities=False,
- include_metadata=include_metadata,
)
if not workflow_execution:
# Use pipline_id for pipelines / API deployment
diff --git a/backend/workflow_manager/workflow/views.py b/backend/workflow_manager/workflow/views.py
index 7c775a1a8..a572025f3 100644
--- a/backend/workflow_manager/workflow/views.py
+++ b/backend/workflow_manager/workflow/views.py
@@ -171,9 +171,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_id = serializer.get_execution_id(serializer.validated_data)
execution_action = serializer.get_execution_action(serializer.validated_data)
file_objs = request.FILES.getlist("files")
- include_metadata = (
- request.data.get("include_metadata", "false").lower() == "true"
- )
hashes_of_files: dict[str, FileHash] = {}
if file_objs and execution_id and workflow_id:
hashes_of_files = SourceConnector.add_input_file_to_api_storage(
@@ -192,7 +189,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_id=execution_id,
pipeline_guid=pipeline_guid,
hash_values_of_files=hashes_of_files,
- include_metadata=include_metadata,
)
if (
execution_response.execution_status == "ERROR"
@@ -219,7 +215,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_id: Optional[str] = None,
pipeline_guid: Optional[str] = None,
hash_values_of_files: dict[str, FileHash] = {},
- include_metadata: bool = False,
) -> ExecutionResponse:
if execution_action is not None:
# Step execution
@@ -228,7 +223,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution_action,
execution_id=execution_id,
hash_values_of_files=hash_values_of_files,
- include_metadata=include_metadata,
)
elif pipeline_guid:
# pipeline execution
@@ -246,7 +240,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):
workflow=workflow,
execution_id=execution_id,
hash_values_of_files=hash_values_of_files,
- include_metadata=include_metadata,
)
return execution_response
diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.py
index 2ea03366a..199c5005e 100644
--- a/backend/workflow_manager/workflow/workflow_helper.py
+++ b/backend/workflow_manager/workflow/workflow_helper.py
@@ -91,7 +91,6 @@ class WorkflowHelper:
scheduled: bool,
execution_mode: tuple[str, str],
workflow_execution: Optional[WorkflowExecution],
- include_metadata: bool = False,
) -> WorkflowExecutionServiceHelper:
workflow_execution_service = WorkflowExecutionServiceHelper(
organization_id=organization_id,
@@ -102,7 +101,6 @@ class WorkflowHelper:
scheduled=scheduled,
mode=execution_mode,
workflow_execution=workflow_execution,
- include_metadata=include_metadata,
)
workflow_execution_service.build()
return workflow_execution_service
@@ -246,7 +244,6 @@ class WorkflowHelper:
single_step: bool = False,
workflow_execution: Optional[WorkflowExecution] = None,
execution_mode: Optional[tuple[str, str]] = None,
- include_metadata: bool = False,
) -> ExecutionResponse:
tool_instances: list[ToolInstance] = (
ToolInstanceHelper.get_tool_instances_by_workflow(
@@ -265,7 +262,6 @@ class WorkflowHelper:
scheduled=scheduled,
execution_mode=execution_mode,
workflow_execution=workflow_execution,
- include_metadata=include_metadata,
)
execution_id = execution_service.execution_id
source = SourceConnector(
@@ -390,7 +386,6 @@ class WorkflowHelper:
hash_values_of_files: dict[str, FileHash],
timeout: int = -1,
pipeline_id: Optional[str] = None,
- include_metadata: bool = False,
) -> ExecutionResponse:
"""Adding a workflow to the queue for execution.
@@ -399,7 +394,6 @@ class WorkflowHelper:
execution_id (str): Execution ID
timeout (int): Celery timeout (timeout -1 : async execution)
pipeline_id (Optional[str], optional): Optional pipeline. Defaults to None.
- include_metadata (bool): Whether to include metadata in the prompt output
Returns:
ExecutionResponse: Existing status of execution
@@ -417,7 +411,6 @@ class WorkflowHelper:
execution_id=execution_id,
pipeline_id=pipeline_id,
log_events_id=log_events_id,
- include_metadata=include_metadata,
)
if timeout > -1:
async_execution.wait(
@@ -431,12 +424,13 @@ class WorkflowHelper:
workflow_execution = WorkflowExecutionServiceHelper.get_execution_by_id(
execution_id=execution_id
)
- return ExecutionResponse(
+ execution_response = ExecutionResponse(
workflow_id,
execution_id,
workflow_execution.status,
result=task_result,
)
+ return execution_response
except celery_exceptions.TimeoutError:
return ExecutionResponse(
workflow_id,
@@ -475,7 +469,6 @@ class WorkflowHelper:
scheduled: bool = False,
execution_mode: Optional[tuple[str, str]] = None,
pipeline_id: Optional[str] = None,
- include_metadata: bool = False,
**kwargs: dict[str, Any],
) -> Optional[list[Any]]:
"""Asynchronous Execution By celery.
@@ -489,7 +482,6 @@ class WorkflowHelper:
execution_mode (Optional[WorkflowExecution.Mode]): WorkflowExecution Mode
Defaults to None
pipeline_id (Optional[str], optional): Id of pipeline. Defaults to None
- include_metadata (bool): Whether to include metadata in the prompt output
Kwargs:
log_events_id (str): Session ID of the user,
@@ -525,7 +517,7 @@ class WorkflowHelper:
WorkflowExecutionServiceHelper.update_execution_task(
execution_id=execution_id, task_id=task_id
)
- result = WorkflowHelper.run_workflow(
+ execution_response = WorkflowHelper.run_workflow(
workflow=workflow,
organization_id=schema_name,
pipeline_id=pipeline_id,
@@ -533,9 +525,8 @@ class WorkflowHelper:
workflow_execution=workflow_execution,
execution_mode=execution_mode,
hash_values_of_files=hash_values,
- include_metadata=include_metadata,
- ).result
- return result
+ )
+ return execution_response.result
@staticmethod
def complete_execution(
@@ -543,7 +534,6 @@ class WorkflowHelper:
execution_id: Optional[str] = None,
pipeline_id: Optional[str] = None,
hash_values_of_files: dict[str, FileHash] = {},
- include_metadata: bool = False,
) -> ExecutionResponse:
if pipeline_id:
logger.info(f"Executing pipeline: {pipeline_id}")
@@ -582,7 +572,6 @@ class WorkflowHelper:
workflow=workflow,
workflow_execution=workflow_execution,
hash_values_of_files=hash_values_of_files,
- include_metadata=include_metadata,
)
except WorkflowExecution.DoesNotExist:
return WorkflowHelper.create_and_make_execution_response(
@@ -610,7 +599,6 @@ class WorkflowHelper:
execution_action: str,
execution_id: Optional[str] = None,
hash_values_of_files: dict[str, FileHash] = {},
- include_metadata: bool = False,
) -> ExecutionResponse:
if execution_action is Workflow.ExecutionAction.START.value: # type: ignore
if execution_id is None:
@@ -624,7 +612,6 @@ class WorkflowHelper:
single_step=True,
workflow_execution=workflow_execution,
hash_values_of_files=hash_values_of_files,
- include_metadata=include_metadata,
)
except WorkflowExecution.DoesNotExist:
return WorkflowHelper.create_and_make_execution_response(
diff --git a/prompt-service/src/unstract/prompt_service/authentication_middleware.py b/prompt-service/src/unstract/prompt_service/authentication_middleware.py
index a9fce59fa..fbf03c37b 100644
--- a/prompt-service/src/unstract/prompt_service/authentication_middleware.py
+++ b/prompt-service/src/unstract/prompt_service/authentication_middleware.py
@@ -1,17 +1,16 @@
from typing import Any, Optional
from flask import Request, current_app
-from peewee import PostgresqlDatabase
+from unstract.prompt_service.config import db
from unstract.prompt_service.constants import DBTableV2, FeatureFlag
from unstract.flags.feature_flag import check_feature_flag_status
class AuthenticationMiddleware:
- be_db: PostgresqlDatabase
- @classmethod
- def validate_bearer_token(cls, token: Optional[str]) -> bool:
+ @staticmethod
+ def validate_bearer_token(token: Optional[str]) -> bool:
try:
if token is None:
current_app.logger.error("Authentication failed. Empty bearer token")
@@ -23,7 +22,7 @@ class AuthenticationMiddleware:
platform_key_table = "account_platformkey"
query = f"SELECT * FROM {platform_key_table} WHERE key = '{token}'"
- cursor = cls.be_db.execute_sql(query)
+ cursor = db.execute_sql(query)
result_row = cursor.fetchone()
cursor.close()
if not result_row or len(result_row) == 0:
@@ -54,8 +53,8 @@ class AuthenticationMiddleware:
return False
return True
- @classmethod
- def get_token_from_auth_header(cls, request: Request) -> Optional[str]:
+ @staticmethod
+ def get_token_from_auth_header(request: Request) -> Optional[str]:
try:
bearer_token = request.headers.get("Authorization")
if not bearer_token:
@@ -66,8 +65,8 @@ class AuthenticationMiddleware:
current_app.logger.info(f"Exception while getting token {e}")
return None
- @classmethod
- def get_account_from_bearer_token(cls, token: Optional[str]) -> str:
+ @staticmethod
+ def get_account_from_bearer_token(token: Optional[str]) -> str:
if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
platform_key_table = DBTableV2.PLATFORM_KEY
organization_table = DBTableV2.ORGANIZATION
@@ -83,9 +82,9 @@ class AuthenticationMiddleware:
schema_name: str = AuthenticationMiddleware.execute_query(query_org)
return schema_name
- @classmethod
- def execute_query(cls, query: str) -> Any:
- cursor = cls.be_db.execute_sql(query)
+ @staticmethod
+ def execute_query(query: str) -> Any:
+ cursor = db.execute_sql(query)
result_row = cursor.fetchone()
cursor.close()
if not result_row or len(result_row) == 0:
diff --git a/prompt-service/src/unstract/prompt_service/config.py b/prompt-service/src/unstract/prompt_service/config.py
index 22073db6b..5ead148de 100644
--- a/prompt-service/src/unstract/prompt_service/config.py
+++ b/prompt-service/src/unstract/prompt_service/config.py
@@ -4,6 +4,7 @@ from os import environ as env
from dotenv import load_dotenv
from flask import Flask
+from peewee import PostgresqlDatabase
from unstract.prompt_service.constants import LogLevel
load_dotenv()
@@ -27,6 +28,15 @@ dictConfig(
}
)
+db = PostgresqlDatabase(None)
+
+
+def get_env_or_die(env_key: str) -> str:
+ env_value = env.get(env_key)
+ if not env_value:
+ raise ValueError(f"Env variable {env_key} is required")
+ return env_value
+
def create_app() -> Flask:
app = Flask("prompt-service")
@@ -37,4 +47,22 @@ def create_app() -> Flask:
app.logger.setLevel(logging.INFO)
else:
app.logger.setLevel(logging.WARNING)
+
+ # Load required environment variables
+ db_host = get_env_or_die("PG_BE_HOST")
+ db_port = get_env_or_die("PG_BE_PORT")
+ db_user = get_env_or_die("PG_BE_USERNAME")
+ db_pass = get_env_or_die("PG_BE_PASSWORD")
+ db_name = get_env_or_die("PG_BE_DATABASE")
+
+ # Initialize and connect to the database
+ db.init(
+ database=db_name,
+ user=db_user,
+ password=db_pass,
+ host=db_host,
+ port=db_port,
+ )
+ db.connect()
+
return app
diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.py
index cad39731e..4e41a3a94 100644
--- a/prompt-service/src/unstract/prompt_service/constants.py
+++ b/prompt-service/src/unstract/prompt_service/constants.py
@@ -58,6 +58,10 @@ class PromptServiceContants:
OUTPUT = "output"
CONTEXT = "context"
INCLUDE_METADATA = "include_metadata"
+ EPILOGUE = "epilogue"
+ PLATFORM_POSTAMBLE = "platform_postamble"
+ EXTRACT_EPILOGUE = "extract-epilogue"
+ CLEAN_CONTEXT = "clean-context"
class LogLevel(Enum):
diff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.py
index 6d2b4e927..b6f7bfc66 100644
--- a/prompt-service/src/unstract/prompt_service/helper.py
+++ b/prompt-service/src/unstract/prompt_service/helper.py
@@ -1,12 +1,14 @@
import importlib
import os
+from json import JSONDecodeError
from logging import Logger
from pathlib import Path
-from typing import Any
+from typing import Any, Optional
from dotenv import load_dotenv
-from flask import Flask, current_app
+from flask import Flask, current_app, json
from unstract.prompt_service.authentication_middleware import AuthenticationMiddleware
+from unstract.prompt_service.config import db
from unstract.prompt_service.constants import PromptServiceContants as PSKeys
from unstract.prompt_service.exceptions import APIError, RateLimitError
from unstract.sdk.exceptions import RateLimitError as SdkRateLimitError
@@ -14,21 +16,11 @@ from unstract.sdk.llm import LLM
load_dotenv()
+# Global variable to store plugins
+plugins: dict[str, dict[str, Any]] = {}
-class EnvLoader:
- @staticmethod
- def get_env_or_die(env_key: str) -> str:
- env_value = os.environ.get(env_key)
- if env_value is None or env_value == "":
- raise ValueError(f"Env variable {env_key} is required")
- return env_value
-
-class PluginException(Exception):
- """All exceptions raised from a plugin."""
-
-
-def plugin_loader(app: Flask) -> dict[str, dict[str, Any]]:
+def plugin_loader(app: Flask) -> None:
"""Loads plugins found in the plugins root dir.
Each plugin:
@@ -43,14 +35,14 @@ def plugin_loader(app: Flask) -> dict[str, dict[str, Any]]:
"disable": bool
}
"""
+ global plugins
plugins_dir: Path = Path(os.path.dirname(__file__)) / "plugins"
plugins_pkg = "unstract.prompt_service.plugins"
if not plugins_dir.exists():
app.logger.info(f"Plugins dir not found: {plugins_dir}. Skipping.")
- return {}
+ return
- plugins: dict[str, dict[str, Any]] = {}
app.logger.info(f"Loading plugins from: {plugins_dir}")
for pkg in os.listdir(os.fspath(plugins_dir)):
@@ -85,10 +77,40 @@ def plugin_loader(app: Flask) -> dict[str, dict[str, Any]]:
if not plugins:
app.logger.info("No plugins found.")
- return plugins
+ initialize_plugin_endpoints(app=app)
-def query_usage_metadata(db, token: str, metadata: dict[str, Any]) -> dict[str, Any]:
+def get_cleaned_context(context: str) -> str:
+ clean_context_plugin: dict[str, Any] = plugins.get(PSKeys.CLEAN_CONTEXT, {})
+ if clean_context_plugin:
+ return clean_context_plugin["entrypoint_cls"].run(context=context)
+ return context
+
+
+def initialize_plugin_endpoints(app: Flask) -> None:
+ """Enables plugins if available."""
+ single_pass_extration_plugin: dict[str, Any] = plugins.get(
+ PSKeys.SINGLE_PASS_EXTRACTION, {}
+ )
+ summarize_plugin: dict[str, Any] = plugins.get(PSKeys.SUMMARIZE, {})
+ simple_prompt_studio: dict[str, Any] = plugins.get(PSKeys.SIMPLE_PROMPT_STUDIO, {})
+ if single_pass_extration_plugin:
+ single_pass_extration_plugin["entrypoint_cls"](
+ app=app,
+ challenge_plugin=plugins.get(PSKeys.CHALLENGE, {}),
+ get_cleaned_context=get_cleaned_context,
+ )
+ if summarize_plugin:
+ summarize_plugin["entrypoint_cls"](
+ app=app,
+ )
+ if simple_prompt_studio:
+ simple_prompt_studio["entrypoint_cls"](
+ app=app,
+ )
+
+
+def query_usage_metadata(token: str, metadata: dict[str, Any]) -> dict[str, Any]:
org_id: str = AuthenticationMiddleware.get_account_from_bearer_token(token)
run_id: str = metadata["run_id"]
query: str = f"""
@@ -188,17 +210,21 @@ def construct_and_run_prompt(
llm: LLM,
context: str,
prompt: str,
-) -> tuple[str, dict[str, Any]]:
+ metadata: dict[str, Any],
+) -> str:
prompt = construct_prompt(
preamble=tool_settings.get(PSKeys.PREAMBLE, ""),
prompt=output[prompt],
postamble=tool_settings.get(PSKeys.POSTAMBLE, ""),
grammar_list=tool_settings.get(PSKeys.GRAMMAR, []),
context=context,
+ platform_postamble=tool_settings.get(PSKeys.PLATFORM_POSTAMBLE, ""),
)
return run_completion(
llm=llm,
prompt=prompt,
+ metadata=metadata,
+ prompt_key=output[PSKeys.NAME],
)
@@ -208,30 +234,10 @@ def construct_prompt(
postamble: str,
grammar_list: list[dict[str, Any]],
context: str,
+ platform_postamble: str,
) -> str:
- logger: Logger = current_app.logger
- # Let's cleanup the context. Remove if 3 consecutive newlines are found
- context_lines = context.split("\n")
- new_context_lines = []
- empty_line_count = 0
- for line in context_lines:
- if line.strip() == "":
- empty_line_count += 1
- else:
- if empty_line_count >= 3:
- empty_line_count = 3
- for i in range(empty_line_count):
- new_context_lines.append("")
- empty_line_count = 0
- new_context_lines.append(line.rstrip())
- context = "\n".join(new_context_lines)
- logger.info(
- f"Old context length: {len(context_lines)}, "
- f"New context length: {len(new_context_lines)}"
- )
-
prompt = (
- f"{preamble}\n\nContext:\n---------------{context}\n"
+ f"{preamble}\n\nContext:\n---------------\n{context}\n"
f"-----------------\n\nQuestion or Instruction: {prompt}\n"
)
if grammar_list is not None and len(grammar_list) > 0:
@@ -246,20 +252,39 @@ def construct_prompt(
if len(synonyms) > 0 and word != "":
prompt += f'\nNote: You can consider that the word {word} is same as \
{", ".join(synonyms)} in both the quesiton and the context.' # noqa
- prompt += f"\n\n{postamble}"
- prompt += "\n\nAnswer:"
+ if platform_postamble:
+ platform_postamble += "\n\n"
+ prompt += f"\n\n{postamble}\n\n{platform_postamble}Answer:"
return prompt
def run_completion(
llm: LLM,
prompt: str,
-) -> tuple[str, dict[str, Any]]:
+ metadata: Optional[dict[str, str]] = None,
+ prompt_key: Optional[str] = None,
+) -> str:
logger: Logger = current_app.logger
try:
- completion = llm.complete(prompt)
-
+ extract_epilogue_plugin: dict[str, Any] = plugins.get(
+ PSKeys.EXTRACT_EPILOGUE, {}
+ )
+ extract_epilogue = None
+ if extract_epilogue_plugin:
+ extract_epilogue = extract_epilogue_plugin["entrypoint_cls"].run
+ completion = llm.complete(
+ prompt=prompt,
+ process_text=extract_epilogue,
+ )
answer: str = completion[PSKeys.RESPONSE].text
+ epilogue = completion.get(PSKeys.EPILOGUE)
+ if all([metadata, epilogue, prompt_key]):
+ try:
+ logger.info(f"Epilogue extracted from LLM: {epilogue}")
+ epilogue = json.loads(epilogue)
+ except JSONDecodeError:
+ logger.error(f"Failed to convert epilogue to JSON: {epilogue}")
+ metadata.setdefault(PSKeys.EPILOGUE, {})[prompt_key] = epilogue
return answer
# TODO: Catch and handle specific exception here
except SdkRateLimitError as e:
diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.py
index d0cd36ae6..40c85c2f5 100644
--- a/prompt-service/src/unstract/prompt_service/main.py
+++ b/prompt-service/src/unstract/prompt_service/main.py
@@ -4,7 +4,6 @@ from enum import Enum
from json import JSONDecodeError
from typing import Any, Optional
-import peewee
from flask import json, jsonify, request
from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters
from unstract.prompt_service.authentication_middleware import AuthenticationMiddleware
@@ -13,10 +12,11 @@ from unstract.prompt_service.constants import PromptServiceContants as PSKeys
from unstract.prompt_service.constants import RunLevel
from unstract.prompt_service.exceptions import APIError, ErrorResponse, NoPayloadError
from unstract.prompt_service.helper import (
- EnvLoader,
construct_and_run_prompt,
extract_variable,
+ get_cleaned_context,
plugin_loader,
+ plugins,
query_usage_metadata,
run_completion,
)
@@ -31,9 +31,6 @@ from werkzeug.exceptions import HTTPException
from unstract.core.pubsub_helper import LogPublisher
-POS_TEXT_PATH = "/tmp/pos.txt"
-USE_UNSTRACT_PROMPT = True
-MAX_RETRIES = 3
NO_CONTEXT_ERROR = (
"Couldn't fetch context from vector DB. "
"This happens usually due to a delay by the Vector DB "
@@ -41,28 +38,9 @@ NO_CONTEXT_ERROR = (
"Please try again after some time"
)
-PG_BE_HOST = EnvLoader.get_env_or_die("PG_BE_HOST")
-PG_BE_PORT = EnvLoader.get_env_or_die("PG_BE_PORT")
-PG_BE_USERNAME = EnvLoader.get_env_or_die("PG_BE_USERNAME")
-PG_BE_PASSWORD = EnvLoader.get_env_or_die("PG_BE_PASSWORD")
-PG_BE_DATABASE = EnvLoader.get_env_or_die("PG_BE_DATABASE")
-
-be_db = peewee.PostgresqlDatabase(
- PG_BE_DATABASE,
- user=PG_BE_USERNAME,
- password=PG_BE_PASSWORD,
- host=PG_BE_HOST,
- port=PG_BE_PORT,
-)
-be_db.init(PG_BE_DATABASE)
-be_db.connect()
-
-AuthenticationMiddleware.be_db = be_db
-
app = create_app()
-
-
-plugins: dict[str, dict[str, Any]] = plugin_loader(app)
+# Load plugins
+plugin_loader(app)
def _publish_log(
@@ -108,9 +86,6 @@ def prompt_processor() -> Any:
file_hash = payload.get(PSKeys.FILE_HASH)
doc_name = str(payload.get(PSKeys.FILE_NAME, ""))
log_events_id: str = payload.get(PSKeys.LOG_EVENTS_ID, "")
- include_metadata = (
- request.args.get(PSKeys.INCLUDE_METADATA, "false").lower() == "true"
- )
structured_output: dict[str, Any] = {}
metadata: Optional[dict[str, Any]] = {
PSKeys.RUN_ID: run_id,
@@ -290,8 +265,11 @@ def prompt_processor() -> Any:
llm=llm,
context=context,
prompt="promptx",
+ metadata=metadata,
+ )
+ metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = get_cleaned_context(
+ context
)
- metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = context
else:
answer = "NA"
_publish_log(
@@ -316,8 +294,11 @@ def prompt_processor() -> Any:
llm=llm,
vector_index=vector_index,
retrieval_type=retrieval_strategy,
+ metadata=metadata,
+ )
+ metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = get_cleaned_context(
+ context
)
- metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = context
else:
app.logger.info(
"Invalid retrieval strategy passed: %s",
@@ -611,11 +592,8 @@ def prompt_processor() -> Any:
RunLevel.RUN,
"Execution complete",
)
- if include_metadata:
- metadata = query_usage_metadata(db=be_db, token=platform_key, metadata=metadata)
- response = {PSKeys.METADATA: metadata, PSKeys.OUTPUT: structured_output}
- else:
- response = {PSKeys.OUTPUT: structured_output}
+ metadata = query_usage_metadata(token=platform_key, metadata=metadata)
+ response = {PSKeys.METADATA: metadata, PSKeys.OUTPUT: structured_output}
return response
@@ -626,6 +604,7 @@ def run_retrieval( # type:ignore
llm: LLM,
vector_index,
retrieval_type: str,
+ metadata: dict[str, Any],
) -> tuple[str, str]:
prompt = output[PSKeys.PROMPTX]
if retrieval_type == PSKeys.SUBQUESTION:
@@ -656,6 +635,7 @@ def run_retrieval( # type:ignore
llm=llm,
context=context,
prompt="promptx",
+ metadata=metadata,
)
return (answer, context)
@@ -685,30 +665,6 @@ def _retrieve_context(output, doc_id, vector_index, answer) -> str:
return text
-def enable_plugins() -> None:
- """Enables plugins if available."""
- single_pass_extration_plugin: dict[str, Any] = plugins.get(
- PSKeys.SINGLE_PASS_EXTRACTION, {}
- )
- summarize_plugin: dict[str, Any] = plugins.get(PSKeys.SUMMARIZE, {})
- simple_prompt_studio: dict[str, Any] = plugins.get(PSKeys.SIMPLE_PROMPT_STUDIO, {})
- if single_pass_extration_plugin:
- single_pass_extration_plugin["entrypoint_cls"](
- app=app, challenge_plugin=plugins.get(PSKeys.CHALLENGE, {})
- )
- if summarize_plugin:
- summarize_plugin["entrypoint_cls"](
- app=app,
- )
- if simple_prompt_studio:
- simple_prompt_studio["entrypoint_cls"](
- app=app,
- )
-
-
-enable_plugins()
-
-
def log_exceptions(e: HTTPException):
"""Helper method to log exceptions.
diff --git a/tools/structure/src/constants.py b/tools/structure/src/constants.py
index 72780fe2b..1f79c109e 100644
--- a/tools/structure/src/constants.py
+++ b/tools/structure/src/constants.py
@@ -67,4 +67,6 @@ class SettingsKeys:
FILE_HASH = "file_hash"
ENABLE_HIGHLIGHT = "enable_highlight"
NAME = "name"
- INCLUDE_METADATA = "include_metadata"
+ METADATA = "metadata"
+ EPILOGUE = "epilogue"
+ HIGHLIGHT_DATA = "highlight_data"
diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py
index 36678e74e..929d3fe73 100644
--- a/tools/structure/src/main.py
+++ b/tools/structure/src/main.py
@@ -27,7 +27,6 @@ class StructureTool(BaseTool):
prompt_registry_id: str = settings[SettingsKeys.PROMPT_REGISTRY_ID]
challenge_llm: str = settings[SettingsKeys.CHALLENGE_LLM_ADAPTER_ID]
enable_challenge: bool = settings[SettingsKeys.ENABLE_CHALLENGE]
- include_metadata: bool = settings.get(SettingsKeys.INCLUDE_METADATA, False)
summarize_as_source: bool = settings[SettingsKeys.SUMMARIZE_AS_SOURCE]
single_pass_extraction_mode: bool = settings[
SettingsKeys.SINGLE_PASS_EXTRACTION_MODE
@@ -80,7 +79,6 @@ class StructureTool(BaseTool):
SettingsKeys.FILE_HASH: file_hash,
SettingsKeys.FILE_NAME: file_name,
}
- params = {SettingsKeys.INCLUDE_METADATA: include_metadata}
# TODO: Need to split extraction and indexing
# to avoid unwanted indexing
self.stream_log("Indexing document")
@@ -90,6 +88,12 @@ class StructureTool(BaseTool):
self.get_exec_metadata.get(MetadataKey.SOURCE_NAME),
)
+ process_text = None
+ try:
+ from helper import process_text # type: ignore [attr-defined]
+ except ImportError:
+ self.stream_log("Function 'process_text' is not found")
+
if tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION]:
index.index(
tool_id=tool_id,
@@ -104,6 +108,7 @@ class StructureTool(BaseTool):
reindex=True,
usage_kwargs=usage_kwargs,
enable_highlight=enable_highlight,
+ process_text=process_text,
)
if summarize_as_source:
summarize_file_hash = self._summarize_and_index(
@@ -120,7 +125,6 @@ class StructureTool(BaseTool):
self.stream_log("Fetching response for single pass extraction")
prompt_service_resp = responder.single_pass_extraction(
payload=payload,
- params=params,
)
else:
try:
@@ -142,6 +146,7 @@ class StructureTool(BaseTool):
reindex=reindex,
usage_kwargs=usage_kwargs,
enable_highlight=enable_highlight,
+ process_text=process_text,
)
if summarize_as_source:
@@ -163,7 +168,6 @@ class StructureTool(BaseTool):
self.stream_log("Fetching responses for prompts...")
prompt_service_resp = responder.answer_prompt(
payload=payload,
- params=params,
)
# TODO: Make use of dataclasses
@@ -174,6 +178,22 @@ class StructureTool(BaseTool):
)
structured_output = prompt_service_resp[SettingsKeys.STRUCTURE_OUTPUT]
+ structured_output_dict = json.loads(structured_output)
+
+ if not summarize_as_source:
+ metadata = structured_output_dict[SettingsKeys.METADATA]
+ epilogue = metadata.pop(SettingsKeys.EPILOGUE, None)
+ if epilogue:
+ try:
+ from helper import transform_dict # type: ignore [attr-defined]
+
+ highlight_data = transform_dict(epilogue, tool_data_dir)
+ metadata[SettingsKeys.HIGHLIGHT_DATA] = highlight_data
+ except ImportError:
+ self.stream_log("Function 'transform_dict' is not found")
+ # Update the dictionary with modified metadata
+ structured_output_dict[SettingsKeys.METADATA] = metadata
+ structured_output = json.dumps(structured_output_dict)
# Update GUI
input_log = (
@@ -191,11 +211,11 @@ class StructureTool(BaseTool):
output_path = Path(output_dir) / f"{Path(source_name).stem}.json"
with open(output_path, "w", encoding="utf-8") as f:
f.write(structured_output)
- except (FileNotFoundError, PermissionError, OSError) as e:
+ except OSError as e:
self.stream_error_and_exit(f"Error creating output file: {e}")
except json.JSONDecodeError as e:
self.stream_error_and_exit(f"Error encoding JSON: {e}")
- self.write_tool_result(data=json.loads(structured_output))
+ self.write_tool_result(data=structured_output_dict)
def _summarize_and_index(
self,
diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py
index eef11fffc..a506b9707 100644
--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py
+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/constants.py
@@ -12,7 +12,3 @@ class ToolCommandKey:
SPEC = "spec"
VARIABLES = "variables"
ICON = "icon"
-
-
-class Common:
- INCLUDE_METADATA = "include_metadata"
diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py
index a603a590b..d878ce090 100644
--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py
+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py
@@ -1,6 +1,6 @@
from typing import Any, Optional
-from unstract.tool_sandbox.constants import Common, ToolCommandKey, UnstractWorker
+from unstract.tool_sandbox.constants import ToolCommandKey, UnstractWorker
from unstract.tool_sandbox.helper import ToolSandboxHelper
@@ -16,7 +16,6 @@ class ToolSandbox:
tool_instance_id: Optional[str] = None,
environment_variables: dict[str, Any] = {},
messaging_channel: Optional[str] = None,
- include_metadata: bool = False,
):
"""PLATFORM_SERVICE_API_KEY should be available in the environment."""
self.messaging_channel = str(messaging_channel)
@@ -31,11 +30,9 @@ class ToolSandbox:
self.tool_instance_id = tool_instance_id
self.image_name = image_name
self.image_tag = image_tag
- self.include_metadata = include_metadata
self.settings: dict[str, Any] = {}
def set_tool_instance_settings(self, tool_settings: dict[str, Any]) -> None:
- tool_settings[Common.INCLUDE_METADATA] = self.include_metadata
self.settings = tool_settings
def get_tool_uid(self) -> str:
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
index d5fe76804..f76ca885c 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
@@ -28,7 +28,6 @@ class ToolsUtils:
workflow: WorkflowDto,
platform_service_api_key: str,
ignore_processed_entities: bool = False,
- include_metadata: bool = False,
) -> None:
self.redis = redis
self.tool_registry = ToolRegistry()
@@ -53,7 +52,6 @@ class ToolsUtils:
self.llmw_max_polls = ToolsUtils.get_env(
ToolRV.ADAPTER_LLMW_MAX_POLLS, raise_exception=False
)
- self.include_metadata = include_metadata
def set_messaging_channel(self, messaging_channel: str) -> None:
self.messaging_channel = messaging_channel
@@ -137,7 +135,6 @@ class ToolsUtils:
image_tag=image_tag,
environment_variables=tool_envs,
messaging_channel=self.messaging_channel,
- include_metadata=self.include_metadata,
)
tool_sandbox.set_tool_instance_settings(tool_instance.metadata)
tool_sandboxes.append(tool_sandbox)
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
index 270ebb670..a49021f98 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
@@ -44,7 +44,6 @@ class WorkflowExecutionService:
tool_instances: list[ToolInstance],
platform_service_api_key: str,
ignore_processed_entities: bool = False,
- include_metadata: bool = False,
) -> None:
self.organization_id = organization_id
self.workflow_id = workflow_id
@@ -56,7 +55,6 @@ class WorkflowExecutionService:
workflow=workflow,
platform_service_api_key=platform_service_api_key,
ignore_processed_entities=False,
- include_metadata=include_metadata,
)
self.tool_sandboxes: list[ToolSandbox] = []
self.ignore_processed_entities = ignore_processed_entities