Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Multitenancy v2 changes for prompt studio apps - Prompt studio core, Registry (#720)

harini-venkataraman88659df1567d
diff --git a/backend/prompt_studio/prompt_studio_core_v2/constants.py b/backend/prompt_studio/prompt_studio_core_v2/constants.py
index 213fc066f..afba6153d 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/constants.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/constants.py
@@ -62,7 +62,6 @@ class ToolStudioPromptKeys:
WORD = "word"
SYNONYMS = "synonyms"
OUTPUTS = "outputs"
- ASSERT_PROMPT = "assert_prompt"
SECTION = "section"
DEFAULT = "default"
REINDEX = "reindex"
@@ -89,6 +88,12 @@ class ToolStudioPromptKeys:
CONTEXT = "context"
METADATA = "metadata"
INCLUDE_METADATA = "include_metadata"
+ TXT_EXTENTION = ".txt"
+ TABLE = "table"
+ EXTRACT = "extract"
+ PLATFORM_POSTAMBLE = "platform_postamble"
+ SUMMARIZE_AS_SOURCE = "summarize_as_source"
+ VARIABLE_MAP = "variable_map"
class FileViewTypes:
diff --git a/backend/prompt_studio/prompt_studio_core_v2/exceptions.py b/backend/prompt_studio/prompt_studio_core_v2/exceptions.py
index 273d6eabe..9d9d0664d 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/exceptions.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/exceptions.py
@@ -1,3 +1,5 @@
+from typing import Optional
+
from prompt_studio.prompt_profile_manager_v2.constants import ProfileManagerKeys
from prompt_studio.prompt_studio_core_v2.constants import ToolStudioErrors
from rest_framework.exceptions import APIException
@@ -86,3 +88,10 @@ class PromptNotRun(APIException):
"it can be used as a variable in another prompt. "
"Please execute the prompt first and try again."
)
+
+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):
+ if detail is not None:
+ self.detail = detail
+ if code is not None:
+ self.code = code
+ super().__init__(detail, code)
diff --git a/backend/prompt_studio/prompt_studio_core_v2/models.py b/backend/prompt_studio/prompt_studio_core_v2/models.py
index 87abd4766..9cefe3fa2 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/models.py
@@ -125,6 +125,10 @@ class CustomTool(DefaultOrganizationMixin, BaseModel):
db_comment="Flag to enable or disable document highlighting", default=False
)
+ enable_highlight = models.BooleanField(
+ db_comment="Flag to enable or disable document highlighting", default=False
+ )
+
# Introduced field to establish M2M relation between users and custom_tool.
# This will introduce intermediary table which relates both the models.
shared_users = models.ManyToManyField(User, related_name="shared_custom_tools")
diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
index d0d7890cd..2a5e9ac97 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
@@ -3,7 +3,7 @@ import logging
import os
import uuid
from pathlib import Path
-from typing import Any, Optional
+from typing import Any, Callable, Optional
from account_v2.constants import Common
from account_v2.models import User
@@ -12,6 +12,8 @@ from adapter_processor_v2.models import AdapterInstance
from django.conf import settings
from django.db.models.manager import BaseManager
from file_management.file_management_helper import FileManagerHelper
+from prompt_studio.modifier_loader import ModifierConfig
+from prompt_studio.modifier_loader import load_plugins as load_modifier_plugins
from prompt_studio.prompt_profile_manager_v2.models import ProfileManager
from prompt_studio.prompt_profile_manager_v2.profile_manager_helper import (
ProfileManagerHelper,
@@ -29,11 +31,15 @@ from prompt_studio.prompt_studio_core_v2.exceptions import (
EmptyPromptError,
IndexingAPIError,
NoPromptsFound,
+ OperationNotSupported,
PermissionError,
ToolNotValid,
)
from prompt_studio.prompt_studio_core_v2.models import CustomTool
from prompt_studio.prompt_studio_core_v2.prompt_ide_base_tool import PromptIdeBaseTool
+from prompt_studio.prompt_studio_core_v2.prompt_variable_service import (
+ PromptStudioVariableService,
+)
from prompt_studio.prompt_studio_document_manager_v2.models import DocumentManager
from prompt_studio.prompt_studio_index_manager_v2.prompt_studio_index_helper import ( # noqa: E501
PromptStudioIndexHelper,
@@ -56,6 +62,8 @@ ERROR_MSG = "User %s doesn't have access to adapter %s"
logger = logging.getLogger(__name__)
+modifier_loader = load_modifier_plugins()
+
class PromptStudioHelper:
"""Helper class for Custom tool operations."""
@@ -293,6 +301,7 @@ class PromptStudioHelper:
document_id: str,
is_summary: bool = False,
run_id: str = None,
+ text_processor: Optional[type[Any]] = None,
) -> Any:
"""Method to index a document.
@@ -342,7 +351,9 @@ class PromptStudioHelper:
# Need to check the user who created profile manager
# has access to adapters configured in profile manager
PromptStudioHelper.validate_profile_manager_owner_access(default_profile)
-
+ process_text = None
+ if text_processor:
+ process_text = text_processor.process
doc_id = PromptStudioHelper.dynamic_indexer(
profile_manager=default_profile,
tool_id=tool_id,
@@ -353,6 +364,7 @@ class PromptStudioHelper:
reindex=True,
run_id=run_id,
user_id=user_id,
+ process_text=process_text,
)
logger.info(f"[{tool_id}] Indexing successful for doc: {file_name}")
@@ -374,6 +386,7 @@ class PromptStudioHelper:
id: Optional[str] = None,
run_id: str = None,
profile_manager_id: Optional[str] = None,
+ text_processor: Optional[type[Any]] = None,
) -> Any:
"""Execute chain/single run of the prompts. Makes a call to prompt
service and returns the dict of response.
@@ -400,19 +413,27 @@ class PromptStudioHelper:
if id:
return PromptStudioHelper._execute_single_prompt(
- id,
- doc_path,
- doc_name,
- tool_id,
- org_id,
- user_id,
- document_id,
- run_id,
- profile_manager_id,
+ id=id,
+ doc_path=doc_path,
+ doc_name=doc_name,
+ tool_id=tool_id,
+ org_id=org_id,
+ user_id=user_id,
+ document_id=document_id,
+ run_id=run_id,
+ profile_manager_id=profile_manager_id,
+ text_processor=text_processor,
)
else:
return PromptStudioHelper._execute_prompts_in_single_pass(
- doc_path, tool_id, org_id, user_id, document_id, run_id
+ doc_path=doc_path,
+ doc_name=doc_name,
+ tool_id=tool_id,
+ org_id=org_id,
+ user_id=user_id,
+ document_id=document_id,
+ run_id=run_id,
+ text_processor=text_processor,
)
@staticmethod
@@ -426,8 +447,13 @@ class PromptStudioHelper:
document_id,
run_id,
profile_manager_id,
+ text_processor: Optional[type[Any]] = None,
):
prompt_instance = PromptStudioHelper._fetch_prompt_from_id(id)
+
+ if prompt_instance.enforce_type == TSPKeys.TABLE and not modifier_loader:
+ raise OperationNotSupported()
+
prompt_name = prompt_instance.prompt_key
PromptStudioHelper._publish_log(
{
@@ -460,7 +486,9 @@ class PromptStudioHelper:
LogLevels.RUN,
"Invoking prompt service",
)
-
+ process_text = None
+ if text_processor:
+ process_text = text_processor.process
try:
response = PromptStudioHelper._fetch_response(
doc_path=doc_path,
@@ -472,9 +500,15 @@ class PromptStudioHelper:
run_id=run_id,
profile_manager_id=profile_manager_id,
user_id=user_id,
+ process_text=process_text,
)
return PromptStudioHelper._handle_response(
- response, run_id, prompts, document_id, False, profile_manager_id
+ response=response,
+ run_id=run_id,
+ prompts=prompts,
+ document_id=document_id,
+ is_single_pass=False,
+ profile_manager_id=profile_manager_id,
)
except Exception as e:
logger.error(
@@ -497,13 +531,22 @@ class PromptStudioHelper:
@staticmethod
def _execute_prompts_in_single_pass(
- doc_path, tool_id, org_id, user_id, document_id, run_id
+ doc_path,
+ doc_name,
+ tool_id,
+ org_id,
+ user_id,
+ document_id,
+ run_id,
+ text_processor: Optional[type[Any]] = None,
):
prompts = PromptStudioHelper.fetch_prompt_from_tool(tool_id)
prompts = [
prompt
for prompt in prompts
- if prompt.prompt_type != TSPKeys.NOTES and prompt.active
+ if prompt.prompt_type != TSPKeys.NOTES
+ and prompt.active
+ and prompt.enforce_type != TSPKeys.TABLE
]
if not prompts:
logger.error(f"[{tool_id or 'NA'}] No prompts found for id: {id}")
@@ -515,20 +558,28 @@ class PromptStudioHelper:
LogLevels.RUN,
"Executing prompts in single pass",
)
-
+ process_text = None
+ if text_processor:
+ process_text = text_processor.process
try:
tool = prompts[0].tool_id
response = PromptStudioHelper._fetch_single_pass_response(
file_path=doc_path,
+ doc_name=doc_name,
tool=tool,
prompts=prompts,
org_id=org_id,
document_id=document_id,
run_id=run_id,
user_id=user_id,
+ process_text=process_text,
)
return PromptStudioHelper._handle_response(
- response, run_id, prompts, document_id, True
+ response=response,
+ run_id=run_id,
+ prompts=prompts,
+ document_id=document_id,
+ is_single_pass=True,
)
except Exception as e:
logger.error(
@@ -556,9 +607,27 @@ class PromptStudioHelper:
)
return str(Path(doc_path) / doc_name)
+ @staticmethod
+ def _get_extract_or_summary_document_path(
+ org_id, user_id, tool_id, doc_name, doc_type
+ ) -> str:
+ doc_path = FileManagerHelper.handle_sub_directory_for_tenants(
+ org_id=org_id,
+ user_id=user_id,
+ tool_id=tool_id,
+ is_create=False,
+ )
+ extracted_doc_name = Path(doc_name).stem + TSPKeys.TXT_EXTENTION
+ return str(Path(doc_path) / doc_type / extracted_doc_name)
+
@staticmethod
def _handle_response(
- response, run_id, prompts, document_id, is_single_pass, profile_manager_id=None
+ response,
+ run_id,
+ prompts,
+ document_id,
+ is_single_pass,
+ profile_manager_id=None,
):
if response.get("status") == IndexingStatus.PENDING_STATUS.value:
return {
@@ -566,16 +635,15 @@ class PromptStudioHelper:
"message": IndexingStatus.DOCUMENT_BEING_INDEXED.value,
}
- OutputManagerHelper.handle_prompt_output_update(
+ return OutputManagerHelper.handle_prompt_output_update(
run_id=run_id,
prompts=prompts,
outputs=response["output"],
document_id=document_id,
is_single_pass_extract=is_single_pass,
profile_manager_id=profile_manager_id,
- context=response["metadata"].get("context"),
+ metadata=response["metadata"],
)
- return response
@staticmethod
def _fetch_response(
@@ -588,6 +656,7 @@ class PromptStudioHelper:
run_id: str,
user_id: str,
profile_manager_id: Optional[str] = None,
+ process_text: Optional[Callable[[str], str]] = None,
) -> Any:
"""Utility function to invoke prompt service. Used internally.
@@ -658,13 +727,14 @@ class PromptStudioHelper:
is_summary=tool.summarize_as_source,
run_id=run_id,
user_id=user_id,
+ process_text=process_text,
)
if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:
return {
"status": IndexingStatus.PENDING_STATUS.value,
"message": IndexingStatus.DOCUMENT_BEING_INDEXED.value,
}
-
+ tool_id = str(tool.tool_id)
output: dict[str, Any] = {}
outputs: list[dict[str, Any]] = []
grammer_dict = {}
@@ -704,6 +774,14 @@ class PromptStudioHelper:
attr_val = getattr(prompt, attr)
output[TSPKeys.EVAL_SETTINGS][attr] = attr_val
+ output = PromptStudioHelper.fetch_table_settings_if_enabled(
+ doc_name, prompt, org_id, user_id, tool_id, output
+ )
+ variable_map = PromptStudioVariableService.frame_variable_replacement_map(
+ doc_id=document_id, prompt_object=prompt
+ )
+ if variable_map:
+ output[TSPKeys.VARIABLE_MAP] = variable_map
outputs.append(output)
tool_settings = {}
@@ -712,11 +790,13 @@ class PromptStudioHelper:
tool_settings[TSPKeys.SINGLE_PASS_EXTRACTION_MODE] = (
tool.single_pass_extraction_mode
)
+ tool_settings[TSPKeys.SUMMARIZE_AS_SOURCE] = tool.summarize_as_source
tool_settings[TSPKeys.PREAMBLE] = tool.preamble
tool_settings[TSPKeys.POSTAMBLE] = tool.postamble
tool_settings[TSPKeys.GRAMMAR] = grammar_list
-
- tool_id = str(tool.tool_id)
+ tool_settings[TSPKeys.PLATFORM_POSTAMBLE] = getattr(
+ settings, TSPKeys.PLATFORM_POSTAMBLE.upper(), ""
+ )
file_hash = ToolUtils.get_hash_from_file(file_path=doc_path)
@@ -751,6 +831,36 @@ class PromptStudioHelper:
output_response = json.loads(answer["structure_output"])
return output_response
+ @staticmethod
+ def fetch_table_settings_if_enabled(
+ doc_name: str,
+ prompt: ToolStudioPrompt,
+ org_id: str,
+ user_id: str,
+ tool_id: str,
+ output: dict[str, Any],
+ ) -> dict[str, Any]:
+
+ if prompt.enforce_type == TSPKeys.TABLE:
+ extract_doc_path: str = (
+ PromptStudioHelper._get_extract_or_summary_document_path(
+ org_id, user_id, tool_id, doc_name, TSPKeys.EXTRACT
+ )
+ )
+ for modifier_plugin in modifier_loader:
+ cls = modifier_plugin[ModifierConfig.METADATA][
+ ModifierConfig.METADATA_SERVICE_CLASS
+ ]
+ output = cls.update(
+ output=output,
+ tool_id=tool_id,
+ prompt_id=str(prompt.prompt_id),
+ prompt=prompt.prompt,
+ input_file=extract_doc_path,
+ )
+
+ return output
+
@staticmethod
def dynamic_indexer(
profile_manager: ProfileManager,
@@ -762,6 +872,7 @@ class PromptStudioHelper:
is_summary: bool = False,
reindex: bool = False,
run_id: str = None,
+ process_text: Optional[Callable[[str], str]] = None,
) -> Any:
"""Used to index a file based on the passed arguments.
@@ -796,6 +907,7 @@ class PromptStudioHelper:
try:
usage_kwargs = {"run_id": run_id}
+ # Orginal file name with which file got uploaded in prompt studio
usage_kwargs["file_name"] = filename
util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)
tool_index = Index(tool=util)
@@ -841,6 +953,7 @@ class PromptStudioHelper:
reindex=reindex,
output_file_path=extract_file_path,
usage_kwargs=usage_kwargs.copy(),
+ process_text=process_text,
)
PromptStudioIndexHelper.handle_index_manager(
@@ -870,11 +983,13 @@ class PromptStudioHelper:
def _fetch_single_pass_response(
tool: CustomTool,
file_path: str,
+ doc_name: str,
prompts: list[ToolStudioPrompt],
org_id: str,
user_id: str,
document_id: str,
run_id: str = None,
+ process_text: Optional[Callable[[str], str]] = None,
) -> Any:
tool_id: str = str(tool.tool_id)
outputs: list[dict[str, Any]] = []
@@ -910,6 +1025,7 @@ class PromptStudioHelper:
document_id=document_id,
run_id=run_id,
user_id=user_id,
+ process_text=process_text,
)
if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:
return {
@@ -955,6 +1071,7 @@ class PromptStudioHelper:
TSPKeys.TOOL_ID: tool_id,
TSPKeys.RUN_ID: run_id,
TSPKeys.FILE_HASH: file_hash,
+ TSPKeys.FILE_NAME: doc_name,
Common.LOG_EVENTS_ID: StateStore.get(Common.LOG_EVENTS_ID),
}
@@ -972,7 +1089,7 @@ class PromptStudioHelper:
if answer["status"] == "ERROR":
error_message = answer.get("error", None)
raise AnswerFetchError(
- f"Error while fetching response for prompt. {error_message}"
+ f"Error while fetching response for prompt(s). {error_message}"
)
output_response = json.loads(answer["structure_output"])
return output_response
diff --git a/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json b/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json
index 1e9e22ef4..4fa10995f 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json
+++ b/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json
@@ -13,7 +13,8 @@
"email":"email",
"date":"date",
"boolean":"boolean",
- "json":"json"
+ "json":"json",
+ "table":"table"
},
"output_processing":{
"DEFAULT":"Default"
diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.py
index 86d4c5564..2b36f3f53 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/views.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py
@@ -9,8 +9,11 @@ from django.http import HttpRequest
from file_management.exceptions import FileNotFound
from file_management.file_management_helper import FileManagerHelper
from permissions.permission import IsOwner, IsOwnerOrSharedUser
-from prompt_studio.processor_loader import ProcessorConfig, load_plugins
-from prompt_studio.prompt_profile_manager_v2.constants import ProfileManagerErrors
+from prompt_studio.processor_loader import get_plugin_class_by_name, load_plugins
+from prompt_studio.prompt_profile_manager_v2.constants import (
+ ProfileManagerErrors,
+ ProfileManagerKeys,
+)
from prompt_studio.prompt_profile_manager_v2.models import ProfileManager
from prompt_studio.prompt_profile_manager_v2.serializers import ProfileManagerSerializer
from prompt_studio.prompt_studio_core_v2.constants import (
@@ -24,6 +27,7 @@ from prompt_studio.prompt_studio_core_v2.document_indexing_service import (
)
from prompt_studio.prompt_studio_core_v2.exceptions import (
IndexingAPIError,
+ MaxProfilesReachedError,
ToolDeleteError,
)
from prompt_studio.prompt_studio_core_v2.prompt_studio_helper import PromptStudioHelper
@@ -215,6 +219,10 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
)
document: DocumentManager = DocumentManager.objects.get(pk=document_id)
file_name: str = document.document_name
+ text_processor = get_plugin_class_by_name(
+ name="text_processor",
+ plugins=self.processor_plugins,
+ )
# Generate a run_id
run_id = CommonUtils.generate_uuid()
unique_id = PromptStudioHelper.index_document(
@@ -224,23 +232,24 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
user_id=tool.created_by.user_id,
document_id=document_id,
run_id=run_id,
+ text_processor=text_processor,
)
usage_kwargs: dict[Any, Any] = dict()
usage_kwargs[ToolStudioPromptKeys.RUN_ID] = run_id
- for processor_plugin in self.processor_plugins:
- cls = processor_plugin[ProcessorConfig.METADATA][
- ProcessorConfig.METADATA_SERVICE_CLASS
- ]
- if cls:
- cls.process(
- tool_id=str(tool.tool_id),
- file_name=file_name,
- org_id=UserSessionUtils.get_organization_id(request),
- user_id=tool.created_by.user_id,
- document_id=document_id,
- usage_kwargs=usage_kwargs.copy(),
- )
+ cls = get_plugin_class_by_name(
+ name="summarizer",
+ plugins=self.processor_plugins,
+ )
+ if cls:
+ cls.process(
+ tool_id=str(tool.tool_id),
+ file_name=file_name,
+ org_id=UserSessionUtils.get_organization_id(request),
+ user_id=tool.created_by.user_id,
+ document_id=document_id,
+ usage_kwargs=usage_kwargs.copy(),
+ )
if unique_id:
return Response(
@@ -273,7 +282,10 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
if not run_id:
# Generate a run_id
run_id = CommonUtils.generate_uuid()
-
+ text_processor = get_plugin_class_by_name(
+ name="text_processor",
+ plugins=self.processor_plugins,
+ )
response: dict[str, Any] = PromptStudioHelper.prompt_responder(
id=id,
tool_id=tool_id,
@@ -282,6 +294,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
document_id=document_id,
run_id=run_id,
profile_manager_id=profile_manager,
+ text_processor=text_processor,
)
return Response(response, status=status.HTTP_200_OK)
@@ -305,12 +318,17 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
if not run_id:
# Generate a run_id
run_id = CommonUtils.generate_uuid()
+ text_processor = get_plugin_class_by_name(
+ name="text_processor",
+ plugins=self.processor_plugins,
+ )
response: dict[str, Any] = PromptStudioHelper.prompt_responder(
tool_id=tool_id,
org_id=UserSessionUtils.get_organization_id(request),
user_id=custom_tool.created_by.user_id,
document_id=document_id,
run_id=run_id,
+ text_processor=text_processor,
)
return Response(response, status=status.HTTP_200_OK)
@@ -346,6 +364,16 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
serializer = ProfileManagerSerializer(data=request.data, context=context)
serializer.is_valid(raise_exception=True)
+ # Check for the maximum number of profiles constraint
+ prompt_studio_tool = serializer.validated_data[
+ ProfileManagerKeys.PROMPT_STUDIO_TOOL
+ ]
+ profile_count = ProfileManager.objects.filter(
+ prompt_studio_tool=prompt_studio_tool
+ ).count()
+
+ if profile_count >= ProfileManagerKeys.MAX_PROFILE_COUNT:
+ raise MaxProfilesReachedError()
try:
self.perform_create(serializer)
except IntegrityError:
@@ -478,7 +506,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
# Delete the files
FileManagerHelper.delete_file(file_system, path, file_name)
# Directories to delete the text files
- directories = ["extract/", "summarize/"]
+ directories = ["extract/", "extract/metadata/", "summarize/"]
FileManagerHelper.delete_related_files(
file_system, path, file_name, directories
)
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/constants.py b/backend/prompt_studio/prompt_studio_registry_v2/constants.py
index 6dbde1c09..28f3999da 100644
--- a/backend/prompt_studio/prompt_studio_registry_v2/constants.py
+++ b/backend/prompt_studio/prompt_studio_registry_v2/constants.py
@@ -9,6 +9,7 @@ class PromptStudioRegistryKeys:
PROMPT_REGISTRY_ID = "prompt_registry_id"
FILE_NAME = "file_name"
UNDEFINED = "undefined"
+ TABLE = "table"
class PromptStudioRegistryErrors:
@@ -95,6 +96,7 @@ class JsonSchemaKey:
SUMMARIZE_PROMPT = "summarize_prompt"
SUMMARIZE_AS_SOURCE = "summarize_as_source"
ENABLE_HIGHLIGHT = "enable_highlight"
+ PLATFORM_POSTAMBLE = "platform_postamble"
class SpecKey:
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/exceptions.py b/backend/prompt_studio/prompt_studio_registry_v2/exceptions.py
index 12a182991..93086d815 100644
--- a/backend/prompt_studio/prompt_studio_registry_v2/exceptions.py
+++ b/backend/prompt_studio/prompt_studio_registry_v2/exceptions.py
@@ -19,8 +19,9 @@ class ToolSaveError(APIException):
class EmptyToolExportError(APIException):
status_code = 500
default_detail = (
- "Empty Prompt Studio project without prompts cannot be exported. "
- "Try adding a prompt and executing it."
+ "Prompt Studio project without prompts cannot be exported. "
+ "Please ensure there is at least one prompt and "
+ "it is active before exporting."
)
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/fields.py b/backend/prompt_studio/prompt_studio_registry_v2/fields.py
index fcf8330b7..6e790fb3c 100644
--- a/backend/prompt_studio/prompt_studio_registry_v2/fields.py
+++ b/backend/prompt_studio/prompt_studio_registry_v2/fields.py
@@ -5,27 +5,25 @@ from django.db import models
logger = logging.getLogger(__name__)
-class ToolBaseJSONField(models.JSONField):
+class ToolPropertyJSONField(models.JSONField):
def from_db_value(self, value, expression, connection): # type: ignore
metadata = super().from_db_value(value, expression, connection)
return metadata
-# TODO: Investigate if ToolBaseJSONField can replace the need for ToolPropertyJSONField,
-# ToolSpecJSONField, ToolVariablesJSONField, and ToolMetadataJSONField classes.
-
-
-class ToolPropertyJSONField(ToolBaseJSONField, models.JSONField):
- pass
-
-
-class ToolSpecJSONField(ToolBaseJSONField, models.JSONField):
- pass
+class ToolSpecJSONField(models.JSONField):
+ def from_db_value(self, value, expression, connection): # type: ignore
+ metadata = super().from_db_value(value, expression, connection)
+ return metadata
-class ToolVariablesJSONField(ToolBaseJSONField, models.JSONField):
- pass
+class ToolVariablesJSONField(models.JSONField):
+ def from_db_value(self, value, expression, connection): # type: ignore
+ metadata = super().from_db_value(value, expression, connection)
+ return metadata
-class ToolMetadataJSONField(ToolBaseJSONField, models.JSONField):
- pass
+class ToolMetadataJSONField(models.JSONField):
+ def from_db_value(self, value, expression, connection): # type: ignore
+ metadata = super().from_db_value(value, expression, connection)
+ return metadata
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
index e9d267e4d..3854e5874 100644
--- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
+++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py
@@ -5,6 +5,8 @@ from account_v2.models import User
from adapter_processor_v2.models import AdapterInstance
from django.conf import settings
from django.db import IntegrityError
+from prompt_studio.modifier_loader import ModifierConfig
+from prompt_studio.modifier_loader import load_plugins as load_modifier_plugins
from prompt_studio.prompt_profile_manager_v2.models import ProfileManager
from prompt_studio.prompt_studio_core_v2.models import CustomTool
from prompt_studio.prompt_studio_core_v2.prompt_studio_helper import PromptStudioHelper
@@ -14,7 +16,7 @@ from prompt_studio.prompt_studio_output_manager_v2.models import (
from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt
from unstract.tool_registry.dto import Properties, Spec, Tool
-from .constants import JsonSchemaKey
+from .constants import JsonSchemaKey, PromptStudioRegistryKeys
from .exceptions import (
EmptyToolExportError,
InternalError,
@@ -25,6 +27,7 @@ from .models import PromptStudioRegistry
from .serializers import PromptStudioRegistrySerializer
logger = logging.getLogger(__name__)
+modifier_loader = load_modifier_plugins()
class PromptStudioRegistryHelper:
@@ -121,25 +124,14 @@ class PromptStudioRegistryHelper:
f"ID {prompt_registry_id}: {e} "
)
return None
- # The below properties are introduced after 0.20.0
- # So defaulting to 0.20.0 if the properties are not found
- image_url = prompt_registry_tool.tool_metadata.get(
- JsonSchemaKey.IMAGE_URL, "docker:unstract/tool-structure:0.0.20"
- )
- image_name = prompt_registry_tool.tool_metadata.get(
- JsonSchemaKey.IMAGE_NAME, "unstract/tool-structure"
- )
- image_tag = prompt_registry_tool.tool_metadata.get(
- JsonSchemaKey.IMAGE_TAG, "0.0.20"
- )
return Tool(
tool_uid=prompt_registry_tool.prompt_registry_id,
properties=Properties.from_dict(prompt_registry_tool.tool_property),
spec=Spec.from_dict(prompt_registry_tool.tool_spec),
icon=prompt_registry_tool.icon,
- image_url=image_url,
- image_name=image_name,
- image_tag=image_tag,
+ image_url=settings.STRUCTURE_TOOL_IMAGE_URL,
+ image_name=settings.STRUCTURE_TOOL_IMAGE_NAME,
+ image_tag=settings.STRUCTURE_TOOL_IMAGE_TAG,
)
@staticmethod
@@ -178,7 +170,6 @@ class PromptStudioRegistryHelper:
obj, created = PromptStudioRegistry.objects.update_or_create(
custom_tool=custom_tool,
created_by=custom_tool.created_by,
- modified_by=custom_tool.modified_by,
defaults={
"name": custom_tool.tool_name,
"tool_property": properties.to_dict(),
@@ -192,7 +183,7 @@ class PromptStudioRegistryHelper:
logger.info(f"PSR {obj.prompt_registry_id} was created")
else:
logger.info(f"PSR {obj.prompt_registry_id} was updated")
-
+ obj.modified_by = custom_tool.modified_by
obj.shared_to_org = shared_with_org
if not shared_with_org:
obj.shared_users.clear()
@@ -244,9 +235,6 @@ class PromptStudioRegistryHelper:
export_metadata[JsonSchemaKey.DESCRIPTION] = tool.description
export_metadata[JsonSchemaKey.AUTHOR] = tool.author
export_metadata[JsonSchemaKey.TOOL_ID] = str(tool.tool_id)
- export_metadata[JsonSchemaKey.IMAGE_URL] = settings.STRUCTURE_TOOL_IMAGE_URL
- export_metadata[JsonSchemaKey.IMAGE_NAME] = settings.STRUCTURE_TOOL_IMAGE_NAME
- export_metadata[JsonSchemaKey.IMAGE_TAG] = settings.STRUCTURE_TOOL_IMAGE_TAG
default_llm_profile = ProfileManager.get_default_llm_profile(tool)
challenge_llm_instance: Optional[AdapterInstance] = tool.challenge_llm
@@ -282,8 +270,14 @@ class PromptStudioRegistryHelper:
tool_settings[JsonSchemaKey.ENABLE_SINGLE_PASS_EXTRACTION] = (
tool.single_pass_extraction_mode
)
+ tool_settings[JsonSchemaKey.ENABLE_HIGHLIGHT] = tool.enable_highlight
+ tool_settings[JsonSchemaKey.PLATFORM_POSTAMBLE] = getattr(
+ settings, JsonSchemaKey.PLATFORM_POSTAMBLE.upper(), ""
+ )
for prompt in prompts:
+ if prompt.prompt_type == JsonSchemaKey.NOTES or not prompt.active:
+ continue
if not prompt.prompt:
invalidated_prompts.append(prompt.prompt_key)
@@ -299,8 +293,6 @@ class PromptStudioRegistryHelper:
invalidated_outputs.append(prompt.prompt_key)
continue
- if prompt.prompt_type == JsonSchemaKey.NOTES:
- continue
if not prompt.profile_manager:
prompt.profile_manager = default_llm_profile
@@ -333,6 +325,19 @@ class PromptStudioRegistryHelper:
output[JsonSchemaKey.SECTION] = prompt.profile_manager.section
output[JsonSchemaKey.REINDEX] = prompt.profile_manager.reindex
output[JsonSchemaKey.EMBEDDING_SUFFIX] = embedding_suffix
+
+ if prompt.enforce_type == PromptStudioRegistryKeys.TABLE:
+ for modifier_plugin in modifier_loader:
+ cls = modifier_plugin[ModifierConfig.METADATA][
+ ModifierConfig.METADATA_SERVICE_CLASS
+ ]
+ output = cls.update(
+ output=output,
+ tool_id=tool.tool_id,
+ prompt_id=prompt.prompt_id,
+ prompt=prompt.prompt,
+ )
+
outputs.append(output)
output = {}
vector_db = ""
@@ -341,6 +346,9 @@ class PromptStudioRegistryHelper:
llm = ""
embedding_model = ""
+ if not outputs:
+ raise EmptyToolExportError()
+
if invalidated_prompts:
raise InValidCustomToolError(
f"Cannot export tool. Prompt(s): {', '.join(invalidated_prompts)} "
diff --git a/backend/prompt_studio/prompt_studio_v2/controller.py b/backend/prompt_studio/prompt_studio_v2/controller.py
index 30cacfe29..a2eb12630 100644
--- a/backend/prompt_studio/prompt_studio_v2/controller.py
+++ b/backend/prompt_studio/prompt_studio_v2/controller.py
@@ -1,5 +1,6 @@
import logging
+from django.db import models
from prompt_studio.prompt_studio_v2.constants import ToolStudioPromptKeys
from prompt_studio.prompt_studio_v2.helper import PromptStudioHelper
from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt
@@ -12,7 +13,7 @@ logger = logging.getLogger(__name__)
class PromptStudioController:
- def reorder_prompts(self, request: Request) -> Response:
+ def reorder_prompts(self, request: Request, prompt_model: models.Model) -> Response:
"""Reorder the sequence of prompts based on the start and end sequence
numbers.
@@ -46,6 +47,7 @@ class PromptStudioController:
prompt_id=prompt_id,
start_sequence_number=start_sequence_number,
end_sequence_number=end_sequence_number,
+ prompt_model=prompt_model,
)
logger.info("Re-ordering completed successfully.")
diff --git a/backend/prompt_studio/prompt_studio_v2/helper.py b/backend/prompt_studio/prompt_studio_v2/helper.py
index 8491332a4..442ef463f 100644
--- a/backend/prompt_studio/prompt_studio_v2/helper.py
+++ b/backend/prompt_studio/prompt_studio_v2/helper.py
@@ -1,6 +1,6 @@
import logging
-from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt
+from django.db import models
logger = logging.getLogger(__name__)
@@ -8,7 +8,10 @@ logger = logging.getLogger(__name__)
class PromptStudioHelper:
@staticmethod
def reorder_prompts_helper(
- prompt_id: str, start_sequence_number: int, end_sequence_number: int
+ prompt_id: str,
+ start_sequence_number: int,
+ end_sequence_number: int,
+ prompt_model: models.Model,
) -> list[dict[str, int]]:
"""Helper method to reorder prompts based on sequence numbers.
@@ -16,13 +19,14 @@ class PromptStudioHelper:
prompt_id (str): The ID of the prompt to be reordered.
start_sequence_number (int): The initial sequence number of the prompt.
end_sequence_number (int): The new sequence number of the prompt.
+ is_sps (bool): Flag to determine the prompt model to use.
Returns:
list[dict[str, int]]: A list of updated prompt data with their IDs
and new sequence numbers.
"""
- prompt_instance: ToolStudioPrompt = ToolStudioPrompt.objects.get(pk=prompt_id)
- filtered_prompts_data = []
+
+ prompt_instance = prompt_model.objects.get(pk=prompt_id)
tool_id = prompt_instance.tool_id
# Determine the direction of sequence adjustment based on start
@@ -51,9 +55,9 @@ class PromptStudioHelper:
}
increment = True
- # Call helper method to update sequence numbers and get filtered prompt data
+ # Update sequence numbers and get filtered prompt data
filtered_prompts_data = PromptStudioHelper.update_sequence_numbers(
- filters, increment
+ filters, increment, prompt_model
)
# Update the sequence number of the moved prompt
@@ -71,7 +75,9 @@ class PromptStudioHelper:
return filtered_prompts_data
@staticmethod
- def update_sequence_numbers(filters: dict, increment: bool) -> list[dict[str, int]]:
+ def update_sequence_numbers(
+ filters: dict, increment: bool, prompt_model: models.Model
+ ) -> list[dict[str, int]]:
"""Update the sequence numbers for prompts based on the provided
filters and increment flag.
@@ -79,23 +85,21 @@ class PromptStudioHelper:
filters (dict): The filter criteria for selecting prompts.
increment (bool): Whether to increment (True) or decrement (False)
the sequence numbers.
+ prompt_model: The model class for the prompts
+ (either ToolStudioPrompt or SPSPrompt).
Returns:
list[dict[str, int]]: A list of updated prompt data with their IDs
and new sequence numbers.
"""
- # Filter prompts based on the provided filters
- filtered_prompts = ToolStudioPrompt.objects.filter(**filters)
+ filtered_prompts = prompt_model.objects.filter(**filters)
# List to hold updated prompt data
filtered_prompts_data = []
# Prepare updates and collect data
for prompt in filtered_prompts:
- if increment:
- prompt.sequence_number += 1
- else:
- prompt.sequence_number -= 1
+ prompt.sequence_number += 1 if increment else -1
# Append prompt data to the list
filtered_prompts_data.append(
@@ -106,6 +110,6 @@ class PromptStudioHelper:
)
# Bulk update the sequence numbers
- ToolStudioPrompt.objects.bulk_update(filtered_prompts, ["sequence_number"])
+ prompt_model.objects.bulk_update(filtered_prompts, ["sequence_number"])
return filtered_prompts_data
diff --git a/backend/prompt_studio/prompt_studio_v2/models.py b/backend/prompt_studio/prompt_studio_v2/models.py
index 7e65d3c84..e3b092606 100644
--- a/backend/prompt_studio/prompt_studio_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_v2/models.py
@@ -20,6 +20,7 @@ class ToolStudioPrompt(BaseModel):
DATE = "date", "Response sent as date"
BOOLEAN = "boolean", "Response sent as boolean"
JSON = "json", "Response sent as json"
+ TABLE = "table", "Response sent as table"
class PromptType(models.TextChoices):
PROMPT = "PROMPT", "Response sent as Text"
@@ -45,8 +46,8 @@ class ToolStudioPrompt(BaseModel):
)
tool_id = models.ForeignKey(
CustomTool,
- on_delete=models.CASCADE,
- related_name="tool_studio_prompts",
+ on_delete=models.SET_NULL,
+ related_name="mapped_prompt",
null=True,
blank=True,
)
diff --git a/backend/prompt_studio/prompt_studio_v2/urls.py b/backend/prompt_studio/prompt_studio_v2/urls.py
index 3d8c3d74c..23e5f0243 100644
--- a/backend/prompt_studio/prompt_studio_v2/urls.py
+++ b/backend/prompt_studio/prompt_studio_v2/urls.py
@@ -22,7 +22,7 @@ urlpatterns = format_suffix_patterns(
name="tool-studio-prompt-detail",
),
path(
- "prompt/reorder",
+ "prompt/reorder/",
reorder_prompts,
name="reorder_prompts",
),
diff --git a/backend/prompt_studio/prompt_studio_v2/views.py b/backend/prompt_studio/prompt_studio_v2/views.py
index ccf63aa0d..ad9062674 100644
--- a/backend/prompt_studio/prompt_studio_v2/views.py
+++ b/backend/prompt_studio/prompt_studio_v2/views.py
@@ -53,4 +53,4 @@ class ToolStudioPromptView(viewsets.ModelViewSet):
Response: The HTTP response indicating the status of the reorder operation.
"""
prompt_studio_controller = PromptStudioController()
- return prompt_studio_controller.reorder_prompts(request)
+ return prompt_studio_controller.reorder_prompts(request, ToolStudioPrompt)