LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Multitenancy v2 changes for prompt studio apps - Prompt studio core, Registry (#720)
harini-venkataraman88659df1567d
diff --git a/backend/prompt_studio/prompt_studio_core_v2/constants.py b/backend/prompt_studio/prompt_studio_core_v2/constants.pyindex 213fc066f..afba6153d 100644--- a/backend/prompt_studio/prompt_studio_core_v2/constants.py+++ b/backend/prompt_studio/prompt_studio_core_v2/constants.py@@ -62,7 +62,6 @@ class ToolStudioPromptKeys:WORD = "word"SYNONYMS = "synonyms"OUTPUTS = "outputs"- ASSERT_PROMPT = "assert_prompt"SECTION = "section"DEFAULT = "default"REINDEX = "reindex"@@ -89,6 +88,12 @@ class ToolStudioPromptKeys:CONTEXT = "context"METADATA = "metadata"INCLUDE_METADATA = "include_metadata"+ TXT_EXTENTION = ".txt"+ TABLE = "table"+ EXTRACT = "extract"+ PLATFORM_POSTAMBLE = "platform_postamble"+ SUMMARIZE_AS_SOURCE = "summarize_as_source"+ VARIABLE_MAP = "variable_map"class FileViewTypes:diff --git a/backend/prompt_studio/prompt_studio_core_v2/exceptions.py b/backend/prompt_studio/prompt_studio_core_v2/exceptions.pyindex 273d6eabe..9d9d0664d 100644--- a/backend/prompt_studio/prompt_studio_core_v2/exceptions.py+++ b/backend/prompt_studio/prompt_studio_core_v2/exceptions.py@@ -1,3 +1,5 @@+from typing import Optional+from prompt_studio.prompt_profile_manager_v2.constants import ProfileManagerKeysfrom prompt_studio.prompt_studio_core_v2.constants import ToolStudioErrorsfrom rest_framework.exceptions import APIException@@ -86,3 +88,10 @@ class PromptNotRun(APIException):"it can be used as a variable in another prompt. ""Please execute the prompt first and try again.")++ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):+ if detail is not None:+ self.detail = detail+ if code is not None:+ self.code = code+ super().__init__(detail, code)diff --git a/backend/prompt_studio/prompt_studio_core_v2/models.py b/backend/prompt_studio/prompt_studio_core_v2/models.pyindex 87abd4766..9cefe3fa2 100644--- a/backend/prompt_studio/prompt_studio_core_v2/models.py+++ b/backend/prompt_studio/prompt_studio_core_v2/models.py@@ -125,6 +125,10 @@ class CustomTool(DefaultOrganizationMixin, BaseModel):db_comment="Flag to enable or disable document highlighting", default=False)+ enable_highlight = models.BooleanField(+ db_comment="Flag to enable or disable document highlighting", default=False+ )+# Introduced field to establish M2M relation between users and custom_tool.# This will introduce intermediary table which relates both the models.shared_users = models.ManyToManyField(User, related_name="shared_custom_tools")diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.pyindex d0d7890cd..2a5e9ac97 100644--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py@@ -3,7 +3,7 @@ import loggingimport osimport uuidfrom pathlib import Path-from typing import Any, Optional+from typing import Any, Callable, Optionalfrom account_v2.constants import Commonfrom account_v2.models import User@@ -12,6 +12,8 @@ from adapter_processor_v2.models import AdapterInstancefrom django.conf import settingsfrom django.db.models.manager import BaseManagerfrom file_management.file_management_helper import FileManagerHelper+from prompt_studio.modifier_loader import ModifierConfig+from prompt_studio.modifier_loader import load_plugins as load_modifier_pluginsfrom prompt_studio.prompt_profile_manager_v2.models import ProfileManagerfrom prompt_studio.prompt_profile_manager_v2.profile_manager_helper import (ProfileManagerHelper,@@ -29,11 +31,15 @@ from prompt_studio.prompt_studio_core_v2.exceptions import (EmptyPromptError,IndexingAPIError,NoPromptsFound,+ OperationNotSupported,PermissionError,ToolNotValid,)from prompt_studio.prompt_studio_core_v2.models import CustomToolfrom prompt_studio.prompt_studio_core_v2.prompt_ide_base_tool import PromptIdeBaseTool+from prompt_studio.prompt_studio_core_v2.prompt_variable_service import (+ PromptStudioVariableService,+)from prompt_studio.prompt_studio_document_manager_v2.models import DocumentManagerfrom prompt_studio.prompt_studio_index_manager_v2.prompt_studio_index_helper import ( # noqa: E501PromptStudioIndexHelper,@@ -56,6 +62,8 @@ ERROR_MSG = "User %s doesn't have access to adapter %s"logger = logging.getLogger(__name__)+modifier_loader = load_modifier_plugins()+class PromptStudioHelper:"""Helper class for Custom tool operations."""@@ -293,6 +301,7 @@ class PromptStudioHelper:document_id: str,is_summary: bool = False,run_id: str = None,+ text_processor: Optional[type[Any]] = None,) -> Any:"""Method to index a document.@@ -342,7 +351,9 @@ class PromptStudioHelper:# Need to check the user who created profile manager# has access to adapters configured in profile managerPromptStudioHelper.validate_profile_manager_owner_access(default_profile)-+ process_text = None+ if text_processor:+ process_text = text_processor.processdoc_id = PromptStudioHelper.dynamic_indexer(profile_manager=default_profile,tool_id=tool_id,@@ -353,6 +364,7 @@ class PromptStudioHelper:reindex=True,run_id=run_id,user_id=user_id,+ process_text=process_text,)logger.info(f"[{tool_id}] Indexing successful for doc: {file_name}")@@ -374,6 +386,7 @@ class PromptStudioHelper:id: Optional[str] = None,run_id: str = None,profile_manager_id: Optional[str] = None,+ text_processor: Optional[type[Any]] = None,) -> Any:"""Execute chain/single run of the prompts. Makes a call to promptservice and returns the dict of response.@@ -400,19 +413,27 @@ class PromptStudioHelper:if id:return PromptStudioHelper._execute_single_prompt(- id,- doc_path,- doc_name,- tool_id,- org_id,- user_id,- document_id,- run_id,- profile_manager_id,+ id=id,+ doc_path=doc_path,+ doc_name=doc_name,+ tool_id=tool_id,+ org_id=org_id,+ user_id=user_id,+ document_id=document_id,+ run_id=run_id,+ profile_manager_id=profile_manager_id,+ text_processor=text_processor,)else:return PromptStudioHelper._execute_prompts_in_single_pass(- doc_path, tool_id, org_id, user_id, document_id, run_id+ doc_path=doc_path,+ doc_name=doc_name,+ tool_id=tool_id,+ org_id=org_id,+ user_id=user_id,+ document_id=document_id,+ run_id=run_id,+ text_processor=text_processor,)@staticmethod@@ -426,8 +447,13 @@ class PromptStudioHelper:document_id,run_id,profile_manager_id,+ text_processor: Optional[type[Any]] = None,):prompt_instance = PromptStudioHelper._fetch_prompt_from_id(id)++ if prompt_instance.enforce_type == TSPKeys.TABLE and not modifier_loader:+ raise OperationNotSupported()+prompt_name = prompt_instance.prompt_keyPromptStudioHelper._publish_log({@@ -460,7 +486,9 @@ class PromptStudioHelper:LogLevels.RUN,"Invoking prompt service",)-+ process_text = None+ if text_processor:+ process_text = text_processor.processtry:response = PromptStudioHelper._fetch_response(doc_path=doc_path,@@ -472,9 +500,15 @@ class PromptStudioHelper:run_id=run_id,profile_manager_id=profile_manager_id,user_id=user_id,+ process_text=process_text,)return PromptStudioHelper._handle_response(- response, run_id, prompts, document_id, False, profile_manager_id+ response=response,+ run_id=run_id,+ prompts=prompts,+ document_id=document_id,+ is_single_pass=False,+ profile_manager_id=profile_manager_id,)except Exception as e:logger.error(@@ -497,13 +531,22 @@ class PromptStudioHelper:@staticmethoddef _execute_prompts_in_single_pass(- doc_path, tool_id, org_id, user_id, document_id, run_id+ doc_path,+ doc_name,+ tool_id,+ org_id,+ user_id,+ document_id,+ run_id,+ text_processor: Optional[type[Any]] = None,):prompts = PromptStudioHelper.fetch_prompt_from_tool(tool_id)prompts = [promptfor prompt in prompts- if prompt.prompt_type != TSPKeys.NOTES and prompt.active+ if prompt.prompt_type != TSPKeys.NOTES+ and prompt.active+ and prompt.enforce_type != TSPKeys.TABLE]if not prompts:logger.error(f"[{tool_id or 'NA'}] No prompts found for id: {id}")@@ -515,20 +558,28 @@ class PromptStudioHelper:LogLevels.RUN,"Executing prompts in single pass",)-+ process_text = None+ if text_processor:+ process_text = text_processor.processtry:tool = prompts[0].tool_idresponse = PromptStudioHelper._fetch_single_pass_response(file_path=doc_path,+ doc_name=doc_name,tool=tool,prompts=prompts,org_id=org_id,document_id=document_id,run_id=run_id,user_id=user_id,+ process_text=process_text,)return PromptStudioHelper._handle_response(- response, run_id, prompts, document_id, True+ response=response,+ run_id=run_id,+ prompts=prompts,+ document_id=document_id,+ is_single_pass=True,)except Exception as e:logger.error(@@ -556,9 +607,27 @@ class PromptStudioHelper:)return str(Path(doc_path) / doc_name)+ @staticmethod+ def _get_extract_or_summary_document_path(+ org_id, user_id, tool_id, doc_name, doc_type+ ) -> str:+ doc_path = FileManagerHelper.handle_sub_directory_for_tenants(+ org_id=org_id,+ user_id=user_id,+ tool_id=tool_id,+ is_create=False,+ )+ extracted_doc_name = Path(doc_name).stem + TSPKeys.TXT_EXTENTION+ return str(Path(doc_path) / doc_type / extracted_doc_name)+@staticmethoddef _handle_response(- response, run_id, prompts, document_id, is_single_pass, profile_manager_id=None+ response,+ run_id,+ prompts,+ document_id,+ is_single_pass,+ profile_manager_id=None,):if response.get("status") == IndexingStatus.PENDING_STATUS.value:return {@@ -566,16 +635,15 @@ class PromptStudioHelper:"message": IndexingStatus.DOCUMENT_BEING_INDEXED.value,}- OutputManagerHelper.handle_prompt_output_update(+ return OutputManagerHelper.handle_prompt_output_update(run_id=run_id,prompts=prompts,outputs=response["output"],document_id=document_id,is_single_pass_extract=is_single_pass,profile_manager_id=profile_manager_id,- context=response["metadata"].get("context"),+ metadata=response["metadata"],)- return response@staticmethoddef _fetch_response(@@ -588,6 +656,7 @@ class PromptStudioHelper:run_id: str,user_id: str,profile_manager_id: Optional[str] = None,+ process_text: Optional[Callable[[str], str]] = None,) -> Any:"""Utility function to invoke prompt service. Used internally.@@ -658,13 +727,14 @@ class PromptStudioHelper:is_summary=tool.summarize_as_source,run_id=run_id,user_id=user_id,+ process_text=process_text,)if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:return {"status": IndexingStatus.PENDING_STATUS.value,"message": IndexingStatus.DOCUMENT_BEING_INDEXED.value,}-+ tool_id = str(tool.tool_id)output: dict[str, Any] = {}outputs: list[dict[str, Any]] = []grammer_dict = {}@@ -704,6 +774,14 @@ class PromptStudioHelper:attr_val = getattr(prompt, attr)output[TSPKeys.EVAL_SETTINGS][attr] = attr_val+ output = PromptStudioHelper.fetch_table_settings_if_enabled(+ doc_name, prompt, org_id, user_id, tool_id, output+ )+ variable_map = PromptStudioVariableService.frame_variable_replacement_map(+ doc_id=document_id, prompt_object=prompt+ )+ if variable_map:+ output[TSPKeys.VARIABLE_MAP] = variable_mapoutputs.append(output)tool_settings = {}@@ -712,11 +790,13 @@ class PromptStudioHelper:tool_settings[TSPKeys.SINGLE_PASS_EXTRACTION_MODE] = (tool.single_pass_extraction_mode)+ tool_settings[TSPKeys.SUMMARIZE_AS_SOURCE] = tool.summarize_as_sourcetool_settings[TSPKeys.PREAMBLE] = tool.preambletool_settings[TSPKeys.POSTAMBLE] = tool.postambletool_settings[TSPKeys.GRAMMAR] = grammar_list-- tool_id = str(tool.tool_id)+ tool_settings[TSPKeys.PLATFORM_POSTAMBLE] = getattr(+ settings, TSPKeys.PLATFORM_POSTAMBLE.upper(), ""+ )file_hash = ToolUtils.get_hash_from_file(file_path=doc_path)@@ -751,6 +831,36 @@ class PromptStudioHelper:output_response = json.loads(answer["structure_output"])return output_response+ @staticmethod+ def fetch_table_settings_if_enabled(+ doc_name: str,+ prompt: ToolStudioPrompt,+ org_id: str,+ user_id: str,+ tool_id: str,+ output: dict[str, Any],+ ) -> dict[str, Any]:++ if prompt.enforce_type == TSPKeys.TABLE:+ extract_doc_path: str = (+ PromptStudioHelper._get_extract_or_summary_document_path(+ org_id, user_id, tool_id, doc_name, TSPKeys.EXTRACT+ )+ )+ for modifier_plugin in modifier_loader:+ cls = modifier_plugin[ModifierConfig.METADATA][+ ModifierConfig.METADATA_SERVICE_CLASS+ ]+ output = cls.update(+ output=output,+ tool_id=tool_id,+ prompt_id=str(prompt.prompt_id),+ prompt=prompt.prompt,+ input_file=extract_doc_path,+ )++ return output+@staticmethoddef dynamic_indexer(profile_manager: ProfileManager,@@ -762,6 +872,7 @@ class PromptStudioHelper:is_summary: bool = False,reindex: bool = False,run_id: str = None,+ process_text: Optional[Callable[[str], str]] = None,) -> Any:"""Used to index a file based on the passed arguments.@@ -796,6 +907,7 @@ class PromptStudioHelper:try:usage_kwargs = {"run_id": run_id}+ # Orginal file name with which file got uploaded in prompt studiousage_kwargs["file_name"] = filenameutil = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)tool_index = Index(tool=util)@@ -841,6 +953,7 @@ class PromptStudioHelper:reindex=reindex,output_file_path=extract_file_path,usage_kwargs=usage_kwargs.copy(),+ process_text=process_text,)PromptStudioIndexHelper.handle_index_manager(@@ -870,11 +983,13 @@ class PromptStudioHelper:def _fetch_single_pass_response(tool: CustomTool,file_path: str,+ doc_name: str,prompts: list[ToolStudioPrompt],org_id: str,user_id: str,document_id: str,run_id: str = None,+ process_text: Optional[Callable[[str], str]] = None,) -> Any:tool_id: str = str(tool.tool_id)outputs: list[dict[str, Any]] = []@@ -910,6 +1025,7 @@ class PromptStudioHelper:document_id=document_id,run_id=run_id,user_id=user_id,+ process_text=process_text,)if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:return {@@ -955,6 +1071,7 @@ class PromptStudioHelper:TSPKeys.TOOL_ID: tool_id,TSPKeys.RUN_ID: run_id,TSPKeys.FILE_HASH: file_hash,+ TSPKeys.FILE_NAME: doc_name,Common.LOG_EVENTS_ID: StateStore.get(Common.LOG_EVENTS_ID),}@@ -972,7 +1089,7 @@ class PromptStudioHelper:if answer["status"] == "ERROR":error_message = answer.get("error", None)raise AnswerFetchError(- f"Error while fetching response for prompt. {error_message}"+ f"Error while fetching response for prompt(s). {error_message}")output_response = json.loads(answer["structure_output"])return output_responsediff --git a/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json b/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.jsonindex 1e9e22ef4..4fa10995f 100644--- a/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json+++ b/backend/prompt_studio/prompt_studio_core_v2/static/select_choices.json@@ -13,7 +13,8 @@"email":"email","date":"date","boolean":"boolean",- "json":"json"+ "json":"json",+ "table":"table"},"output_processing":{"DEFAULT":"Default"diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.pyindex 86d4c5564..2b36f3f53 100644--- a/backend/prompt_studio/prompt_studio_core_v2/views.py+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py@@ -9,8 +9,11 @@ from django.http import HttpRequestfrom file_management.exceptions import FileNotFoundfrom file_management.file_management_helper import FileManagerHelperfrom permissions.permission import IsOwner, IsOwnerOrSharedUser-from prompt_studio.processor_loader import ProcessorConfig, load_plugins-from prompt_studio.prompt_profile_manager_v2.constants import ProfileManagerErrors+from prompt_studio.processor_loader import get_plugin_class_by_name, load_plugins+from prompt_studio.prompt_profile_manager_v2.constants import (+ ProfileManagerErrors,+ ProfileManagerKeys,+)from prompt_studio.prompt_profile_manager_v2.models import ProfileManagerfrom prompt_studio.prompt_profile_manager_v2.serializers import ProfileManagerSerializerfrom prompt_studio.prompt_studio_core_v2.constants import (@@ -24,6 +27,7 @@ from prompt_studio.prompt_studio_core_v2.document_indexing_service import ()from prompt_studio.prompt_studio_core_v2.exceptions import (IndexingAPIError,+ MaxProfilesReachedError,ToolDeleteError,)from prompt_studio.prompt_studio_core_v2.prompt_studio_helper import PromptStudioHelper@@ -215,6 +219,10 @@ class PromptStudioCoreView(viewsets.ModelViewSet):)document: DocumentManager = DocumentManager.objects.get(pk=document_id)file_name: str = document.document_name+ text_processor = get_plugin_class_by_name(+ name="text_processor",+ plugins=self.processor_plugins,+ )# Generate a run_idrun_id = CommonUtils.generate_uuid()unique_id = PromptStudioHelper.index_document(@@ -224,23 +232,24 @@ class PromptStudioCoreView(viewsets.ModelViewSet):user_id=tool.created_by.user_id,document_id=document_id,run_id=run_id,+ text_processor=text_processor,)usage_kwargs: dict[Any, Any] = dict()usage_kwargs[ToolStudioPromptKeys.RUN_ID] = run_id- for processor_plugin in self.processor_plugins:- cls = processor_plugin[ProcessorConfig.METADATA][- ProcessorConfig.METADATA_SERVICE_CLASS- ]- if cls:- cls.process(- tool_id=str(tool.tool_id),- file_name=file_name,- org_id=UserSessionUtils.get_organization_id(request),- user_id=tool.created_by.user_id,- document_id=document_id,- usage_kwargs=usage_kwargs.copy(),- )+ cls = get_plugin_class_by_name(+ name="summarizer",+ plugins=self.processor_plugins,+ )+ if cls:+ cls.process(+ tool_id=str(tool.tool_id),+ file_name=file_name,+ org_id=UserSessionUtils.get_organization_id(request),+ user_id=tool.created_by.user_id,+ document_id=document_id,+ usage_kwargs=usage_kwargs.copy(),+ )if unique_id:return Response(@@ -273,7 +282,10 @@ class PromptStudioCoreView(viewsets.ModelViewSet):if not run_id:# Generate a run_idrun_id = CommonUtils.generate_uuid()-+ text_processor = get_plugin_class_by_name(+ name="text_processor",+ plugins=self.processor_plugins,+ )response: dict[str, Any] = PromptStudioHelper.prompt_responder(id=id,tool_id=tool_id,@@ -282,6 +294,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):document_id=document_id,run_id=run_id,profile_manager_id=profile_manager,+ text_processor=text_processor,)return Response(response, status=status.HTTP_200_OK)@@ -305,12 +318,17 @@ class PromptStudioCoreView(viewsets.ModelViewSet):if not run_id:# Generate a run_idrun_id = CommonUtils.generate_uuid()+ text_processor = get_plugin_class_by_name(+ name="text_processor",+ plugins=self.processor_plugins,+ )response: dict[str, Any] = PromptStudioHelper.prompt_responder(tool_id=tool_id,org_id=UserSessionUtils.get_organization_id(request),user_id=custom_tool.created_by.user_id,document_id=document_id,run_id=run_id,+ text_processor=text_processor,)return Response(response, status=status.HTTP_200_OK)@@ -346,6 +364,16 @@ class PromptStudioCoreView(viewsets.ModelViewSet):serializer = ProfileManagerSerializer(data=request.data, context=context)serializer.is_valid(raise_exception=True)+ # Check for the maximum number of profiles constraint+ prompt_studio_tool = serializer.validated_data[+ ProfileManagerKeys.PROMPT_STUDIO_TOOL+ ]+ profile_count = ProfileManager.objects.filter(+ prompt_studio_tool=prompt_studio_tool+ ).count()++ if profile_count >= ProfileManagerKeys.MAX_PROFILE_COUNT:+ raise MaxProfilesReachedError()try:self.perform_create(serializer)except IntegrityError:@@ -478,7 +506,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):# Delete the filesFileManagerHelper.delete_file(file_system, path, file_name)# Directories to delete the text files- directories = ["extract/", "summarize/"]+ directories = ["extract/", "extract/metadata/", "summarize/"]FileManagerHelper.delete_related_files(file_system, path, file_name, directories)diff --git a/backend/prompt_studio/prompt_studio_registry_v2/constants.py b/backend/prompt_studio/prompt_studio_registry_v2/constants.pyindex 6dbde1c09..28f3999da 100644--- a/backend/prompt_studio/prompt_studio_registry_v2/constants.py+++ b/backend/prompt_studio/prompt_studio_registry_v2/constants.py@@ -9,6 +9,7 @@ class PromptStudioRegistryKeys:PROMPT_REGISTRY_ID = "prompt_registry_id"FILE_NAME = "file_name"UNDEFINED = "undefined"+ TABLE = "table"class PromptStudioRegistryErrors:@@ -95,6 +96,7 @@ class JsonSchemaKey:SUMMARIZE_PROMPT = "summarize_prompt"SUMMARIZE_AS_SOURCE = "summarize_as_source"ENABLE_HIGHLIGHT = "enable_highlight"+ PLATFORM_POSTAMBLE = "platform_postamble"class SpecKey:diff --git a/backend/prompt_studio/prompt_studio_registry_v2/exceptions.py b/backend/prompt_studio/prompt_studio_registry_v2/exceptions.pyindex 12a182991..93086d815 100644--- a/backend/prompt_studio/prompt_studio_registry_v2/exceptions.py+++ b/backend/prompt_studio/prompt_studio_registry_v2/exceptions.py@@ -19,8 +19,9 @@ class ToolSaveError(APIException):class EmptyToolExportError(APIException):status_code = 500default_detail = (- "Empty Prompt Studio project without prompts cannot be exported. "- "Try adding a prompt and executing it."+ "Prompt Studio project without prompts cannot be exported. "+ "Please ensure there is at least one prompt and "+ "it is active before exporting.")diff --git a/backend/prompt_studio/prompt_studio_registry_v2/fields.py b/backend/prompt_studio/prompt_studio_registry_v2/fields.pyindex fcf8330b7..6e790fb3c 100644--- a/backend/prompt_studio/prompt_studio_registry_v2/fields.py+++ b/backend/prompt_studio/prompt_studio_registry_v2/fields.py@@ -5,27 +5,25 @@ from django.db import modelslogger = logging.getLogger(__name__)-class ToolBaseJSONField(models.JSONField):+class ToolPropertyJSONField(models.JSONField):def from_db_value(self, value, expression, connection): # type: ignoremetadata = super().from_db_value(value, expression, connection)return metadata-# TODO: Investigate if ToolBaseJSONField can replace the need for ToolPropertyJSONField,-# ToolSpecJSONField, ToolVariablesJSONField, and ToolMetadataJSONField classes.---class ToolPropertyJSONField(ToolBaseJSONField, models.JSONField):- pass---class ToolSpecJSONField(ToolBaseJSONField, models.JSONField):- pass+class ToolSpecJSONField(models.JSONField):+ def from_db_value(self, value, expression, connection): # type: ignore+ metadata = super().from_db_value(value, expression, connection)+ return metadata-class ToolVariablesJSONField(ToolBaseJSONField, models.JSONField):- pass+class ToolVariablesJSONField(models.JSONField):+ def from_db_value(self, value, expression, connection): # type: ignore+ metadata = super().from_db_value(value, expression, connection)+ return metadata-class ToolMetadataJSONField(ToolBaseJSONField, models.JSONField):- pass+class ToolMetadataJSONField(models.JSONField):+ def from_db_value(self, value, expression, connection): # type: ignore+ metadata = super().from_db_value(value, expression, connection)+ return metadatadiff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.pyindex e9d267e4d..3854e5874 100644--- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py+++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py@@ -5,6 +5,8 @@ from account_v2.models import Userfrom adapter_processor_v2.models import AdapterInstancefrom django.conf import settingsfrom django.db import IntegrityError+from prompt_studio.modifier_loader import ModifierConfig+from prompt_studio.modifier_loader import load_plugins as load_modifier_pluginsfrom prompt_studio.prompt_profile_manager_v2.models import ProfileManagerfrom prompt_studio.prompt_studio_core_v2.models import CustomToolfrom prompt_studio.prompt_studio_core_v2.prompt_studio_helper import PromptStudioHelper@@ -14,7 +16,7 @@ from prompt_studio.prompt_studio_output_manager_v2.models import (from prompt_studio.prompt_studio_v2.models import ToolStudioPromptfrom unstract.tool_registry.dto import Properties, Spec, Tool-from .constants import JsonSchemaKey+from .constants import JsonSchemaKey, PromptStudioRegistryKeysfrom .exceptions import (EmptyToolExportError,InternalError,@@ -25,6 +27,7 @@ from .models import PromptStudioRegistryfrom .serializers import PromptStudioRegistrySerializerlogger = logging.getLogger(__name__)+modifier_loader = load_modifier_plugins()class PromptStudioRegistryHelper:@@ -121,25 +124,14 @@ class PromptStudioRegistryHelper:f"ID {prompt_registry_id}: {e} ")return None- # The below properties are introduced after 0.20.0- # So defaulting to 0.20.0 if the properties are not found- image_url = prompt_registry_tool.tool_metadata.get(- JsonSchemaKey.IMAGE_URL, "docker:unstract/tool-structure:0.0.20"- )- image_name = prompt_registry_tool.tool_metadata.get(- JsonSchemaKey.IMAGE_NAME, "unstract/tool-structure"- )- image_tag = prompt_registry_tool.tool_metadata.get(- JsonSchemaKey.IMAGE_TAG, "0.0.20"- )return Tool(tool_uid=prompt_registry_tool.prompt_registry_id,properties=Properties.from_dict(prompt_registry_tool.tool_property),spec=Spec.from_dict(prompt_registry_tool.tool_spec),icon=prompt_registry_tool.icon,- image_url=image_url,- image_name=image_name,- image_tag=image_tag,+ image_url=settings.STRUCTURE_TOOL_IMAGE_URL,+ image_name=settings.STRUCTURE_TOOL_IMAGE_NAME,+ image_tag=settings.STRUCTURE_TOOL_IMAGE_TAG,)@staticmethod@@ -178,7 +170,6 @@ class PromptStudioRegistryHelper:obj, created = PromptStudioRegistry.objects.update_or_create(custom_tool=custom_tool,created_by=custom_tool.created_by,- modified_by=custom_tool.modified_by,defaults={"name": custom_tool.tool_name,"tool_property": properties.to_dict(),@@ -192,7 +183,7 @@ class PromptStudioRegistryHelper:logger.info(f"PSR {obj.prompt_registry_id} was created")else:logger.info(f"PSR {obj.prompt_registry_id} was updated")-+ obj.modified_by = custom_tool.modified_byobj.shared_to_org = shared_with_orgif not shared_with_org:obj.shared_users.clear()@@ -244,9 +235,6 @@ class PromptStudioRegistryHelper:export_metadata[JsonSchemaKey.DESCRIPTION] = tool.descriptionexport_metadata[JsonSchemaKey.AUTHOR] = tool.authorexport_metadata[JsonSchemaKey.TOOL_ID] = str(tool.tool_id)- export_metadata[JsonSchemaKey.IMAGE_URL] = settings.STRUCTURE_TOOL_IMAGE_URL- export_metadata[JsonSchemaKey.IMAGE_NAME] = settings.STRUCTURE_TOOL_IMAGE_NAME- export_metadata[JsonSchemaKey.IMAGE_TAG] = settings.STRUCTURE_TOOL_IMAGE_TAGdefault_llm_profile = ProfileManager.get_default_llm_profile(tool)challenge_llm_instance: Optional[AdapterInstance] = tool.challenge_llm@@ -282,8 +270,14 @@ class PromptStudioRegistryHelper:tool_settings[JsonSchemaKey.ENABLE_SINGLE_PASS_EXTRACTION] = (tool.single_pass_extraction_mode)+ tool_settings[JsonSchemaKey.ENABLE_HIGHLIGHT] = tool.enable_highlight+ tool_settings[JsonSchemaKey.PLATFORM_POSTAMBLE] = getattr(+ settings, JsonSchemaKey.PLATFORM_POSTAMBLE.upper(), ""+ )for prompt in prompts:+ if prompt.prompt_type == JsonSchemaKey.NOTES or not prompt.active:+ continueif not prompt.prompt:invalidated_prompts.append(prompt.prompt_key)@@ -299,8 +293,6 @@ class PromptStudioRegistryHelper:invalidated_outputs.append(prompt.prompt_key)continue- if prompt.prompt_type == JsonSchemaKey.NOTES:- continueif not prompt.profile_manager:prompt.profile_manager = default_llm_profile@@ -333,6 +325,19 @@ class PromptStudioRegistryHelper:output[JsonSchemaKey.SECTION] = prompt.profile_manager.sectionoutput[JsonSchemaKey.REINDEX] = prompt.profile_manager.reindexoutput[JsonSchemaKey.EMBEDDING_SUFFIX] = embedding_suffix++ if prompt.enforce_type == PromptStudioRegistryKeys.TABLE:+ for modifier_plugin in modifier_loader:+ cls = modifier_plugin[ModifierConfig.METADATA][+ ModifierConfig.METADATA_SERVICE_CLASS+ ]+ output = cls.update(+ output=output,+ tool_id=tool.tool_id,+ prompt_id=prompt.prompt_id,+ prompt=prompt.prompt,+ )+outputs.append(output)output = {}vector_db = ""@@ -341,6 +346,9 @@ class PromptStudioRegistryHelper:llm = ""embedding_model = ""+ if not outputs:+ raise EmptyToolExportError()+if invalidated_prompts:raise InValidCustomToolError(f"Cannot export tool. Prompt(s): {', '.join(invalidated_prompts)} "diff --git a/backend/prompt_studio/prompt_studio_v2/controller.py b/backend/prompt_studio/prompt_studio_v2/controller.pyindex 30cacfe29..a2eb12630 100644--- a/backend/prompt_studio/prompt_studio_v2/controller.py+++ b/backend/prompt_studio/prompt_studio_v2/controller.py@@ -1,5 +1,6 @@import logging+from django.db import modelsfrom prompt_studio.prompt_studio_v2.constants import ToolStudioPromptKeysfrom prompt_studio.prompt_studio_v2.helper import PromptStudioHelperfrom prompt_studio.prompt_studio_v2.models import ToolStudioPrompt@@ -12,7 +13,7 @@ logger = logging.getLogger(__name__)class PromptStudioController:- def reorder_prompts(self, request: Request) -> Response:+ def reorder_prompts(self, request: Request, prompt_model: models.Model) -> Response:"""Reorder the sequence of prompts based on the start and end sequencenumbers.@@ -46,6 +47,7 @@ class PromptStudioController:prompt_id=prompt_id,start_sequence_number=start_sequence_number,end_sequence_number=end_sequence_number,+ prompt_model=prompt_model,)logger.info("Re-ordering completed successfully.")diff --git a/backend/prompt_studio/prompt_studio_v2/helper.py b/backend/prompt_studio/prompt_studio_v2/helper.pyindex 8491332a4..442ef463f 100644--- a/backend/prompt_studio/prompt_studio_v2/helper.py+++ b/backend/prompt_studio/prompt_studio_v2/helper.py@@ -1,6 +1,6 @@import logging-from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt+from django.db import modelslogger = logging.getLogger(__name__)@@ -8,7 +8,10 @@ logger = logging.getLogger(__name__)class PromptStudioHelper:@staticmethoddef reorder_prompts_helper(- prompt_id: str, start_sequence_number: int, end_sequence_number: int+ prompt_id: str,+ start_sequence_number: int,+ end_sequence_number: int,+ prompt_model: models.Model,) -> list[dict[str, int]]:"""Helper method to reorder prompts based on sequence numbers.@@ -16,13 +19,14 @@ class PromptStudioHelper:prompt_id (str): The ID of the prompt to be reordered.start_sequence_number (int): The initial sequence number of the prompt.end_sequence_number (int): The new sequence number of the prompt.+ is_sps (bool): Flag to determine the prompt model to use.Returns:list[dict[str, int]]: A list of updated prompt data with their IDsand new sequence numbers."""- prompt_instance: ToolStudioPrompt = ToolStudioPrompt.objects.get(pk=prompt_id)- filtered_prompts_data = []++ prompt_instance = prompt_model.objects.get(pk=prompt_id)tool_id = prompt_instance.tool_id# Determine the direction of sequence adjustment based on start@@ -51,9 +55,9 @@ class PromptStudioHelper:}increment = True- # Call helper method to update sequence numbers and get filtered prompt data+ # Update sequence numbers and get filtered prompt datafiltered_prompts_data = PromptStudioHelper.update_sequence_numbers(- filters, increment+ filters, increment, prompt_model)# Update the sequence number of the moved prompt@@ -71,7 +75,9 @@ class PromptStudioHelper:return filtered_prompts_data@staticmethod- def update_sequence_numbers(filters: dict, increment: bool) -> list[dict[str, int]]:+ def update_sequence_numbers(+ filters: dict, increment: bool, prompt_model: models.Model+ ) -> list[dict[str, int]]:"""Update the sequence numbers for prompts based on the providedfilters and increment flag.@@ -79,23 +85,21 @@ class PromptStudioHelper:filters (dict): The filter criteria for selecting prompts.increment (bool): Whether to increment (True) or decrement (False)the sequence numbers.+ prompt_model: The model class for the prompts+ (either ToolStudioPrompt or SPSPrompt).Returns:list[dict[str, int]]: A list of updated prompt data with their IDsand new sequence numbers."""- # Filter prompts based on the provided filters- filtered_prompts = ToolStudioPrompt.objects.filter(**filters)+ filtered_prompts = prompt_model.objects.filter(**filters)# List to hold updated prompt datafiltered_prompts_data = []# Prepare updates and collect datafor prompt in filtered_prompts:- if increment:- prompt.sequence_number += 1- else:- prompt.sequence_number -= 1+ prompt.sequence_number += 1 if increment else -1# Append prompt data to the listfiltered_prompts_data.append(@@ -106,6 +110,6 @@ class PromptStudioHelper:)# Bulk update the sequence numbers- ToolStudioPrompt.objects.bulk_update(filtered_prompts, ["sequence_number"])+ prompt_model.objects.bulk_update(filtered_prompts, ["sequence_number"])return filtered_prompts_datadiff --git a/backend/prompt_studio/prompt_studio_v2/models.py b/backend/prompt_studio/prompt_studio_v2/models.pyindex 7e65d3c84..e3b092606 100644--- a/backend/prompt_studio/prompt_studio_v2/models.py+++ b/backend/prompt_studio/prompt_studio_v2/models.py@@ -20,6 +20,7 @@ class ToolStudioPrompt(BaseModel):DATE = "date", "Response sent as date"BOOLEAN = "boolean", "Response sent as boolean"JSON = "json", "Response sent as json"+ TABLE = "table", "Response sent as table"class PromptType(models.TextChoices):PROMPT = "PROMPT", "Response sent as Text"@@ -45,8 +46,8 @@ class ToolStudioPrompt(BaseModel):)tool_id = models.ForeignKey(CustomTool,- on_delete=models.CASCADE,- related_name="tool_studio_prompts",+ on_delete=models.SET_NULL,+ related_name="mapped_prompt",null=True,blank=True,)diff --git a/backend/prompt_studio/prompt_studio_v2/urls.py b/backend/prompt_studio/prompt_studio_v2/urls.pyindex 3d8c3d74c..23e5f0243 100644--- a/backend/prompt_studio/prompt_studio_v2/urls.py+++ b/backend/prompt_studio/prompt_studio_v2/urls.py@@ -22,7 +22,7 @@ urlpatterns = format_suffix_patterns(name="tool-studio-prompt-detail",),path(- "prompt/reorder",+ "prompt/reorder/",reorder_prompts,name="reorder_prompts",),diff --git a/backend/prompt_studio/prompt_studio_v2/views.py b/backend/prompt_studio/prompt_studio_v2/views.pyindex ccf63aa0d..ad9062674 100644--- a/backend/prompt_studio/prompt_studio_v2/views.py+++ b/backend/prompt_studio/prompt_studio_v2/views.py@@ -53,4 +53,4 @@ class ToolStudioPromptView(viewsets.ModelViewSet):Response: The HTTP response indicating the status of the reorder operation."""prompt_studio_controller = PromptStudioController()- return prompt_studio_controller.reorder_prompts(request)+ return prompt_studio_controller.reorder_prompts(request, ToolStudioPrompt)