LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Integrating Prompt Service V2 APIs to backend & tools (#1202)
harini-venkataraman6caab41846a6
diff --git a/backend/prompt_studio/prompt_studio_core_v2/constants.py b/backend/prompt_studio/prompt_studio_core_v2/constants.pyindex 9dad3bb82..ad3451175 100644--- a/backend/prompt_studio/prompt_studio_core_v2/constants.py+++ b/backend/prompt_studio/prompt_studio_core_v2/constants.py@@ -147,3 +147,26 @@ class ExecutionSource(Enum):"""IDE = "ide"+++class IndexingConstants:+ TOOL_ID = "tool_id"+ EMBEDDING_INSTANCE_ID = "embedding_instance_id"+ VECTOR_DB_INSTANCE_ID = "vector_db_instance_id"+ X2TEXT_INSTANCE_ID = "x2text_instance_id"+ FILE_PATH = "file_path"+ CHUNK_SIZE = "chunk_size"+ CHUNK_OVERLAP = "chunk_overlap"+ REINDEX = "reindex"+ FILE_HASH = "file_hash"+ OUTPUT_FILE_PATH = "output_file_path"+ ENABLE_HIGHLIGHT = "enable_highlight"+ USAGE_KWARGS = "usage_kwargs"+ PROCESS_TEXT = "process_text"+ EXTRACTED_TEXT = "extracted_text"+ TAGS = "tags"+ EXECUTION_SOURCE = "execution_source"+ DOC_ID = "doc_id"+ TOOL_EXECUTION_METATADA = "tool_execution_metadata"+ EXECUTION_DATA_DIR = "execution_data_dir"+ RUN_ID = "run_id"diff --git a/backend/prompt_studio/prompt_studio_core_v2/exceptions.py b/backend/prompt_studio/prompt_studio_core_v2/exceptions.pyindex 4f45a5e96..32cf43a0f 100644--- a/backend/prompt_studio/prompt_studio_core_v2/exceptions.py+++ b/backend/prompt_studio/prompt_studio_core_v2/exceptions.py@@ -19,6 +19,10 @@ class IndexingAPIError(APIException):status_code = 500default_detail = "Error while indexing file"+ def __init__(self, detail: Optional[str] = None, status_code: int = 500):+ super().__init__(detail)+ self.status_code = status_code+class AnswerFetchError(APIException):status_code = 500diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.pyindex e681e2d8c..4238c0936 100644--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py@@ -4,7 +4,7 @@ import osimport timeimport uuidfrom pathlib import Path-from typing import Any, Callable, Optional+from typing import Any, Optionalfrom account_v2.constants import Commonfrom account_v2.models import User@@ -14,15 +14,16 @@ from django.conf import settingsfrom django.db.models.manager import BaseManagerfrom prompt_studio.modifier_loader import ModifierConfigfrom prompt_studio.modifier_loader import load_plugins as load_modifier_plugins+from prompt_studio.processor_loader import get_plugin_class_by_name+from prompt_studio.processor_loader import load_plugins as load_processor_pluginsfrom prompt_studio.prompt_profile_manager_v2.models import ProfileManagerfrom prompt_studio.prompt_profile_manager_v2.profile_manager_helper import (ProfileManagerHelper,)-from prompt_studio.prompt_studio_core_v2.constants import (- ExecutionSource,- IndexingStatus,- LogLevels,-)+from prompt_studio.prompt_studio_core_v2.constants import ExecutionSource+from prompt_studio.prompt_studio_core_v2.constants import IndexingConstants as IKeys+from prompt_studio.prompt_studio_core_v2.constants import IndexingStatus, LogLevels+from prompt_studio.prompt_studio_core_v2.constants import ToolStudioPromptKeysfrom prompt_studio.prompt_studio_core_v2.constants import (ToolStudioPromptKeys as TSPKeys,)@@ -54,17 +55,18 @@ from prompt_studio.prompt_studio_output_manager_v2.output_manager_helper importfrom prompt_studio.prompt_studio_v2.models import ToolStudioPromptfrom unstract.sdk.constants import LogLevelfrom unstract.sdk.exceptions import IndexingError, SdkError-from unstract.sdk.file_storage import FileStorage, FileStorageProviderfrom unstract.sdk.file_storage.constants import StorageTypefrom unstract.sdk.file_storage.env_helper import EnvHelper-from unstract.sdk.index import Indexfrom unstract.sdk.prompt import PromptTool+from unstract.sdk.utils.indexing_utils import IndexingUtilsfrom utils.file_storage.constants import FileStorageKeysfrom utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelperfrom utils.local_context import StateStorefrom unstract.core.pubsub_helper import LogPublisher+logger = logging.getLogger(__name__)+CHOICES_JSON = "/static/select_choices.json"ERROR_MSG = "User %s doesn't have access to adapter %s"@@ -76,6 +78,8 @@ modifier_plugins = load_modifier_plugins()class PromptStudioHelper:"""Helper class for Custom tool operations."""+ processor_plugins = load_processor_plugins()+@staticmethoddef create_default_profile_manager(user: User, tool_id: uuid) -> None:"""Create a default profile manager for a given user and tool.@@ -314,7 +318,6 @@ class PromptStudioHelper:document_id: str,is_summary: bool = False,run_id: str = None,- text_processor: Optional[type[Any]] = None,) -> Any:"""Method to index a document.@@ -331,61 +334,99 @@ class PromptStudioHelper:IndexingError"""tool: CustomTool = CustomTool.objects.get(pk=tool_id)+ file_path = PromptStudioFileHelper.get_or_create_prompt_studio_subdirectory(+ org_id,+ is_create=False,+ user_id=user_id,+ tool_id=tool_id,+ )+ file_path = str(Path(file_path) / file_name)+if is_summary:profile_manager: ProfileManager = ProfileManager.objects.get(prompt_studio_tool=tool, is_summarize_llm=True)+ profile_manager.chunk_size == 0default_profile = profile_manager- file_path = file_nameelse:default_profile = ProfileManager.get_default_llm_profile(tool)- file_path = PromptStudioFileHelper.get_or_create_prompt_studio_subdirectory(- org_id,- is_create=False,- user_id=user_id,- tool_id=tool_id,- )- file_path = str(Path(file_path) / file_name)if not tool:logger.error(f"No tool instance found for the ID {tool_id}")raise ToolNotValid()- start_time = time.time()- logger.info(f"[{tool_id}] Indexing started for doc: {file_name}")- PromptStudioHelper._publish_log(- {"tool_id": tool_id, "run_id": run_id, "doc_name": file_name},- LogLevels.INFO,- LogLevels.RUN,- "Indexing started",- )-# Validate the status of adapter in profile managerPromptStudioHelper.validate_adapter_status(default_profile)# Need to check the user who created profile manager# has access to adapters configured in profile managerPromptStudioHelper.validate_profile_manager_owner_access(default_profile)- process_text = None- if text_processor:- process_text = text_processor.processfs_instance = EnvHelper.get_storage(storage_type=StorageType.PERMANENT,env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,)- doc_id = PromptStudioHelper.dynamic_indexer(+ util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)+ doc_id = IndexingUtils.generate_index_key(+ vector_db=str(default_profile.vector_store.id),+ embedding=str(default_profile.embedding_model.id),+ x2text=str(default_profile.x2text.id),+ chunk_size=str(default_profile.chunk_size),+ chunk_overlap=str(default_profile.chunk_overlap),+ file_path=file_path,+ file_hash=None,+ fs=fs_instance,+ tool=util,+ )+ extracted_text = PromptStudioHelper.dynamic_extractor(+ profile_manager=default_profile,+ file_path=file_path,+ org_id=org_id,+ document_id=document_id,+ run_id=run_id,+ enable_highlight=tool.enable_highlight,+ doc_id=doc_id,+ reindex=True,+ )+ if is_summary:+ summarize_file_path = PromptStudioHelper.summarize(+ file_name, org_id, document_id, is_summary, run_id, tool, doc_id+ )+ summarize_doc_id = IndexingUtils.generate_index_key(+ vector_db=str(default_profile.vector_store.id),+ embedding=str(default_profile.embedding_model.id),+ x2text=str(default_profile.x2text.id),+ chunk_size=str(default_profile.chunk_size),+ chunk_overlap=str(default_profile.chunk_overlap),+ file_path=summarize_file_path,+ fs=fs_instance,+ tool=util,+ )+ PromptStudioIndexHelper.handle_index_manager(+ document_id=document_id,+ is_summary=is_summary,+ profile_manager=profile_manager,+ doc_id=summarize_doc_id,+ )+ start_time = time.time()+ logger.info(f"[{tool_id}] Indexing started for doc: {file_name}")+ PromptStudioHelper._publish_log(+ {"tool_id": tool_id, "run_id": run_id, "doc_name": file_name},+ LogLevels.INFO,+ LogLevels.RUN,+ "Indexing started",+ )+ PromptStudioHelper.dynamic_indexer(profile_manager=default_profile,tool_id=tool_id,file_path=file_path,org_id=org_id,document_id=document_id,- is_summary=is_summary,reindex=True,run_id=run_id,user_id=user_id,- process_text=process_text,- fs=fs_instance,enable_highlight=tool.enable_highlight,+ extracted_text=extracted_text,+ doc_id_key=doc_id,)elapsed_time = time.time() - start_time@@ -400,8 +441,43 @@ class PromptStudioHelper:LogLevels.RUN,f"Indexing successful, took {elapsed_time:.3f}s",)+ logger.info(f"Indexing successful : {doc_id}")+ return doc_id- return doc_id.get("output")+ @staticmethod+ def summarize(+ file_name, org_id, document_id, is_summary, run_id, tool, doc_id+ ) -> str:+ cls = get_plugin_class_by_name(+ name="summarizer",+ plugins=PromptStudioHelper.processor_plugins,+ )+ usage_kwargs: dict[Any, Any] = dict()+ usage_kwargs[ToolStudioPromptKeys.RUN_ID] = run_id+ prompts: list[ToolStudioPrompt] = PromptStudioHelper.fetch_prompt_from_tool(+ tool.tool_id+ )+ if cls:+ summarize_file_path = cls.process(+ tool_id=str(tool.tool_id),+ file_name=file_name,+ org_id=org_id,+ user_id=tool.created_by.user_id,+ usage_kwargs=usage_kwargs.copy(),+ prompts=prompts,+ )+ profile_manager: ProfileManager = ProfileManager.objects.get(+ prompt_studio_tool=tool, is_summarize_llm=True+ )+ default_profile = profile_manager+ default_profile.chunk_size = 0+ PromptStudioIndexHelper.handle_index_manager(+ document_id=document_id,+ is_summary=is_summary,+ profile_manager=default_profile,+ doc_id=doc_id,+ )+ return summarize_file_path@staticmethoddef prompt_responder(@@ -412,7 +488,6 @@ class PromptStudioHelper:id: Optional[str] = None,run_id: str = None,profile_manager_id: Optional[str] = None,- text_processor: Optional[type[Any]] = None,) -> Any:"""Execute chain/single run of the prompts. Makes a call to promptservice and returns the dict of response.@@ -448,7 +523,6 @@ class PromptStudioHelper:document_id=document_id,run_id=run_id,profile_manager_id=profile_manager_id,- text_processor=text_processor,)else:return PromptStudioHelper._execute_prompts_in_single_pass(@@ -456,10 +530,8 @@ class PromptStudioHelper:doc_name=doc_name,tool_id=tool_id,org_id=org_id,- user_id=user_id,document_id=document_id,run_id=run_id,- text_processor=text_processor,)@staticmethod@@ -473,7 +545,6 @@ class PromptStudioHelper:document_id,run_id,profile_manager_id,- text_processor: Optional[type[Any]] = None,):prompt_instance = PromptStudioHelper._fetch_prompt_from_id(id)@@ -498,12 +569,6 @@ class PromptStudioHelper:prompts = [prompt_instance]tool = prompt_instance.tool_id- if tool.summarize_as_source:- directory, filename = os.path.split(doc_path)- doc_path = os.path.join(- directory, TSPKeys.SUMMARIZE, os.path.splitext(filename)[0] + ".txt"- )-PromptStudioHelper._publish_log({"tool_id": tool_id,@@ -515,9 +580,6 @@ class PromptStudioHelper:LogLevels.RUN,"Invoking prompt service",)- process_text = None- if text_processor:- process_text = text_processor.processtry:response = PromptStudioHelper._fetch_response(doc_path=doc_path,@@ -529,7 +591,6 @@ class PromptStudioHelper:run_id=run_id,profile_manager_id=profile_manager_id,user_id=user_id,- process_text=process_text,)return PromptStudioHelper._handle_response(response=response,@@ -564,10 +625,8 @@ class PromptStudioHelper:doc_name,tool_id,org_id,- user_id,document_id,run_id,- text_processor: Optional[type[Any]] = None,):prompts = PromptStudioHelper.fetch_prompt_from_tool(tool_id)prompts = [@@ -588,21 +647,16 @@ class PromptStudioHelper:LogLevels.RUN,"Executing prompts in single pass",)- process_text = None- if text_processor:- process_text = text_processor.processtry:tool = prompts[0].tool_idresponse = PromptStudioHelper._fetch_single_pass_response(- file_path=doc_path,+ input_file_path=doc_path,doc_name=doc_name,tool=tool,prompts=prompts,org_id=org_id,document_id=document_id,run_id=run_id,- user_id=user_id,- process_text=process_text,)return PromptStudioHelper._handle_response(response=response,@@ -686,7 +740,6 @@ class PromptStudioHelper:run_id: str,user_id: str,profile_manager_id: Optional[str] = None,- process_text: Optional[Callable[[str], str]] = None,) -> Any:"""Utility function to invoke prompt service. Used internally.@@ -751,18 +804,78 @@ class PromptStudioHelper:storage_type=StorageType.PERMANENT,env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,)+ file_path = doc_path+ directory, filename = os.path.split(doc_path)+ doc_path = os.path.join(+ directory, "extract", os.path.splitext(filename)[0] + ".txt"+ )+ is_summary = tool.summarize_as_source+ logger.info(f"Summary status : {is_summary}")+ util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)+ logger.info(+ f"Passing file_path for fetching answer "+ f"{file_path} : extraction path {doc_path}"+ )+ doc_id = IndexingUtils.generate_index_key(+ vector_db=str(profile_manager.vector_store.id),+ embedding=str(profile_manager.embedding_model.id),+ x2text=str(profile_manager.x2text.id),+ chunk_size=str(profile_manager.chunk_size),+ chunk_overlap=str(profile_manager.chunk_overlap),+ file_path=file_path,+ file_hash=None,+ fs=fs_instance,+ tool=util,+ )+ logger.info(f"Extracting text from {file_path} for {doc_id}")+ extracted_text = PromptStudioHelper.dynamic_extractor(+ profile_manager=profile_manager,+ file_path=file_path,+ org_id=org_id,+ document_id=document_id,+ run_id=run_id,+ enable_highlight=tool.enable_highlight,+ doc_id=doc_id,+ )+ logger.info(f"Extracted text from {file_path} for {doc_id}")+ if is_summary:+ profile_manager.chunk_size = 0+ doc_path = Path(doc_path) # Convert string to Path object+ doc_path = str(+ doc_path.parent.parent / "summarize" / (doc_path.stem + ".txt")+ )+ summarize_file_path = PromptStudioHelper.summarize(+ filename, org_id, document_id, is_summary, run_id, tool, doc_id+ )+ summarize_doc_id = IndexingUtils.generate_index_key(+ vector_db=str(default_profile.vector_store.id),+ embedding=str(default_profile.embedding_model.id),+ x2text=str(default_profile.x2text.id),+ chunk_size=str(default_profile.chunk_size),+ chunk_overlap=str(default_profile.chunk_overlap),+ file_path=summarize_file_path,+ fs=fs_instance,+ tool=util,+ )+ PromptStudioIndexHelper.handle_index_manager(+ document_id=document_id,+ is_summary=is_summary,+ profile_manager=profile_manager,+ doc_id=summarize_doc_id,+ )+ logger.info("Summary enabled, set chunk to zero..")+ logger.info(f"Indexing document {doc_path} for {doc_id}")index_result = PromptStudioHelper.dynamic_indexer(profile_manager=profile_manager,file_path=doc_path,tool_id=str(tool.tool_id),org_id=org_id,document_id=document_id,- is_summary=tool.summarize_as_source,run_id=run_id,user_id=user_id,- process_text=process_text,- fs=fs_instance,enable_highlight=tool.enable_highlight,+ extracted_text=extracted_text,+ doc_id_key=doc_id,)if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:return {@@ -787,6 +900,7 @@ class PromptStudioHelper:output[TSPKeys.PROMPT] = prompt.promptoutput[TSPKeys.ACTIVE] = prompt.activeoutput[TSPKeys.REQUIRED] = prompt.required+ logger.info(f"Chunk size set to {profile_manager.chunk_size} ")output[TSPKeys.CHUNK_SIZE] = profile_manager.chunk_sizeoutput[TSPKeys.VECTOR_DB] = vector_dboutput[TSPKeys.EMBEDDING] = embedding_model@@ -848,8 +962,6 @@ class PromptStudioHelper:TSPKeys.EXECUTION_SOURCE: ExecutionSource.IDE.value,}- util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)-responder = PromptTool(tool=util,prompt_host=settings.PROMPT_HOST,@@ -860,9 +972,7 @@ class PromptStudioHelper:answer = responder.answer_prompt(payload=payload, params=include_metadata, headers=headers)- # TODO: Make use of dataclassesif answer["status"] == "ERROR":- # TODO: Publish to FE logs from hereerror_message = answer.get("error", "")raise AnswerFetchError("Error while fetching response for "@@ -914,12 +1024,11 @@ class PromptStudioHelper:org_id: str,document_id: str,user_id: str,- is_summary: bool = False,+ extracted_text: str,reindex: bool = False,run_id: str = None,- process_text: Optional[Callable[[str], str]] = None,- fs: FileStorage = FileStorage(provider=FileStorageProvider.LOCAL),enable_highlight: bool = False,+ doc_id_key: Optional[str] = None,) -> Any:"""Used to index a file based on the passed arguments.@@ -939,35 +1048,32 @@ class PromptStudioHelper:Returns:str: Index key for the combination of arguments"""++ if profile_manager.chunk_size == 0:+ PromptStudioIndexHelper.handle_index_manager(+ document_id=document_id,+ profile_manager=profile_manager,+ doc_id=doc_id_key,+ )+ logger.info("Skipping addition of nodes to VectoDB since chunk size is 0")+ return {+ "status": IndexingStatus.COMPLETED_STATUS.value,+ "output": doc_id_key,+ }+embedding_model = str(profile_manager.embedding_model.id)vector_db = str(profile_manager.vector_store.id)x2text_adapter = str(profile_manager.x2text.id)- extract_file_path: Optional[str] = Nonedirectory, filename = os.path.split(file_path)- if not is_summary:- extract_file_path = os.path.join(- directory, "extract", os.path.splitext(filename)[0] + ".txt"- )- else:- profile_manager.chunk_size = 0-+ file_path = os.path.join(+ directory, "extract", os.path.splitext(filename)[0] + ".txt"+ )try:usage_kwargs = {"run_id": run_id}# Orginal file name with which file got uploaded in prompt studiousage_kwargs["file_name"] = filename- util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)- tool_index = Index(tool=util)- doc_id_key = tool_index.generate_index_key(- vector_db=vector_db,- embedding=embedding_model,- x2text=x2text_adapter,- chunk_size=str(profile_manager.chunk_size),- chunk_overlap=str(profile_manager.chunk_overlap),- file_path=file_path,- file_hash=None,- fs=fs,- )+if not reindex:indexed_doc_id = DocumentIndexingService.get_indexed_document_id(org_id=org_id, user_id=user_id, doc_id_key=doc_id_key@@ -990,25 +1096,44 @@ class PromptStudioHelper:DocumentIndexingService.set_document_indexing(org_id=org_id, user_id=user_id, doc_id_key=doc_id_key)- doc_id: str = tool_index.index(- tool_id=tool_id,- embedding_instance_id=embedding_model,- vector_db_instance_id=vector_db,- x2text_instance_id=x2text_adapter,- file_path=file_path,- chunk_size=profile_manager.chunk_size,- chunk_overlap=profile_manager.chunk_overlap,- reindex=reindex,- output_file_path=extract_file_path,- usage_kwargs=usage_kwargs.copy(),- process_text=process_text,- fs=fs,- enable_highlight=enable_highlight,+ logger.info(f"Invoking prompt service for indexing : {doc_id_key}")+ payload = {+ IKeys.TOOL_ID: tool_id,+ IKeys.EMBEDDING_INSTANCE_ID: embedding_model,+ IKeys.VECTOR_DB_INSTANCE_ID: vector_db,+ IKeys.X2TEXT_INSTANCE_ID: x2text_adapter,+ IKeys.FILE_PATH: file_path,+ IKeys.FILE_HASH: None,+ IKeys.CHUNK_OVERLAP: profile_manager.chunk_overlap,+ IKeys.CHUNK_SIZE: profile_manager.chunk_size,+ IKeys.REINDEX: reindex,+ IKeys.ENABLE_HIGHLIGHT: enable_highlight,+ IKeys.USAGE_KWARGS: usage_kwargs.copy(),+ IKeys.EXTRACTED_TEXT: extracted_text,+ IKeys.RUN_ID: run_id,+ Common.LOG_EVENTS_ID: StateStore.get(Common.LOG_EVENTS_ID),+ TSPKeys.EXECUTION_SOURCE: ExecutionSource.IDE.value,+ }++ util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)++ responder = PromptTool(+ tool=util,+ prompt_host=settings.PROMPT_HOST,+ prompt_port=settings.PROMPT_PORT,)+ headers = {Common.X_REQUEST_ID: StateStore.get(Common.REQUEST_ID)}+ response = responder.index(payload=payload, headers=headers)+ try:+ response_text = PromptStudioHelper.handle_response(response)+ doc_id = json.loads(response_text).get("doc_id")++ except IndexingAPIError as e:+ logger.error(f"Failed to index document: {str(e)}")+ raise IndexingAPIError(f"Failed to index document: {str(e)}") from ePromptStudioIndexHelper.handle_index_manager(document_id=document_id,- is_summary=is_summary,profile_manager=profile_manager,doc_id=doc_id,)@@ -1032,14 +1157,12 @@ class PromptStudioHelper:@staticmethoddef _fetch_single_pass_response(tool: CustomTool,- file_path: str,+ input_file_path: str,doc_name: str,prompts: list[ToolStudioPrompt],org_id: str,- user_id: str,document_id: str,run_id: str = None,- process_text: Optional[Callable[[str], str]] = None,) -> Any:tool_id: str = str(tool.tool_id)outputs: list[dict[str, Any]] = []@@ -1058,7 +1181,7 @@ class PromptStudioHelper:# has access to adapters configured in profile managerPromptStudioHelper.validate_profile_manager_owner_access(default_profile)default_profile.chunk_size = 0 # To retrive full context-+ util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)if prompt_grammar:for word, synonyms in prompt_grammar.items():grammar.append({TSPKeys.WORD: word, TSPKeys.SYNONYMS: synonyms})@@ -1070,25 +1193,31 @@ class PromptStudioHelper:storage_type=StorageType.PERMANENT,env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,)- index_result = PromptStudioHelper.dynamic_indexer(+ directory, filename = os.path.split(input_file_path)+ file_path = os.path.join(+ directory, "extract", os.path.splitext(filename)[0] + ".txt"+ )+ doc_id = IndexingUtils.generate_index_key(+ vector_db=str(default_profile.vector_store.id),+ embedding=str(default_profile.embedding_model.id),+ x2text=str(default_profile.x2text.id),+ chunk_size=str(default_profile.chunk_size),+ chunk_overlap=str(default_profile.chunk_overlap),+ file_path=input_file_path,+ file_hash=None,+ fs=fs_instance,+ tool=util,+ )+ PromptStudioHelper.dynamic_extractor(profile_manager=default_profile,- file_path=file_path,- tool_id=tool_id,+ file_path=input_file_path,org_id=org_id,- is_summary=tool.summarize_as_source,document_id=document_id,run_id=run_id,- user_id=user_id,- process_text=process_text,- fs=fs_instance,enable_highlight=tool.enable_highlight,+ doc_id=doc_id,)- if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:- return {- "status": IndexingStatus.PENDING_STATUS.value,- "message": IndexingStatus.DOCUMENT_BEING_INDEXED.value,- }-+ # Indexing is not needed as Single pass is always non chunked.vector_db = str(default_profile.vector_store.id)embedding_model = str(default_profile.embedding_model.id)llm = str(default_profile.llm.id)@@ -1122,9 +1251,11 @@ class PromptStudioHelper:if tool.summarize_as_source:path = Path(file_path)- file_path = str(path.parent / TSPKeys.SUMMARIZE / (path.stem + ".txt"))+ file_path = str(+ path.parent.parent / TSPKeys.SUMMARIZE / (path.stem + ".txt")+ )file_hash = fs_instance.get_hash_from_file(path=file_path)-+ logger.info("payload constructued, calling prompt service..")payload = {TSPKeys.TOOL_SETTINGS: tool_settings,TSPKeys.OUTPUTS: outputs,@@ -1137,8 +1268,6 @@ class PromptStudioHelper:TSPKeys.EXECUTION_SOURCE: ExecutionSource.IDE.value,}- util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)-responder = PromptTool(tool=util,prompt_host=settings.PROMPT_HOST,@@ -1147,11 +1276,13 @@ class PromptStudioHelper:include_metadata = {TSPKeys.INCLUDE_METADATA: True}headers = {Common.X_REQUEST_ID: StateStore.get(Common.REQUEST_ID)}answer = responder.single_pass_extraction(- payload=payload, params=include_metadata, headers=headers+ payload=payload,+ params=include_metadata,+ headers=headers,)- # TODO: Make use of dataclassesif answer["status"] == "ERROR":error_message = answer.get("error", None)+ logger.info(f"{str(answer)}")raise AnswerFetchError(f"Error while fetching response for prompt(s). {error_message}")@@ -1165,3 +1296,89 @@ class PromptStudioHelper:return toolexcept CustomTool.DoesNotExist:return None++ @staticmethod+ def dynamic_extractor(+ file_path: str,+ enable_highlight: bool,+ run_id: str,+ org_id: str,+ profile_manager: ProfileManager,+ document_id: str,+ doc_id: str,+ reindex: Optional[bool] = False,+ ) -> str:+ x2text = str(profile_manager.x2text.id)+ is_extracted: bool = False+ extract_file_path: Optional[str] = None+ extracted_text = ""+ directory, filename = os.path.split(file_path)+ extract_file_path = os.path.join(+ directory, "extract", os.path.splitext(filename)[0] + ".txt"+ )+ usage_kwargs = {"run_id": run_id}+ # Orginal file name with which file got uploaded in prompt studio+ usage_kwargs["file_name"] = filename+ if is_extracted and not reindex:+ is_extracted = PromptStudioIndexHelper.check_extraction_status(+ document_id=document_id,+ profile_manager=profile_manager,+ doc_id=doc_id,+ )+ fs_instance = EnvHelper.get_storage(+ storage_type=StorageType.PERMANENT,+ env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,+ )+ try:+ extracted_text = fs_instance.read(path=extract_file_path, mode="r")+ logger.info("Extracted text found. Reading from file..")+ return extracted_text+ except FileNotFoundError as e:+ logger.warning(+ f"File not found for extraction. {extract_file_path}. {e}"+ "Continuing extraction.."+ )+ extracted_text = None+ payload = {+ IKeys.X2TEXT_INSTANCE_ID: x2text,+ IKeys.FILE_PATH: file_path,+ IKeys.ENABLE_HIGHLIGHT: enable_highlight,+ IKeys.USAGE_KWARGS: usage_kwargs.copy(),+ IKeys.RUN_ID: run_id,+ Common.LOG_EVENTS_ID: StateStore.get(Common.LOG_EVENTS_ID),+ TSPKeys.EXECUTION_SOURCE: ExecutionSource.IDE.value,+ IKeys.OUTPUT_FILE_PATH: extract_file_path,+ }++ util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)++ responder = PromptTool(+ tool=util,+ prompt_host=settings.PROMPT_HOST,+ prompt_port=settings.PROMPT_PORT,+ )+ headers = {Common.X_REQUEST_ID: StateStore.get(Common.REQUEST_ID)}+ response = responder.extract(payload=payload, headers=headers)+ try:+ response_data = PromptStudioHelper.handle_response(response)+ extracted_text = json.loads(response_data)+ PromptStudioIndexHelper.mark_extraction_status(+ document_id=document_id,+ profile_manager=profile_manager,+ doc_id=doc_id,+ )+ except IndexingAPIError as e:+ logger.error(f"Failed to extract document: {str(e)}")+ raise IndexingAPIError(f"Failed to extract document: {str(e)}") from e++ return extracted_text++ @staticmethod+ def handle_response(response: dict) -> dict:+ """Handles API responses stored in dictionary format."""+ status_code = response.get("status_code")+ if status_code == 200:+ return response.get("structure_output")+ else:+ error_message = response.get("error", "Unknown error")+ raise IndexingAPIError(f"Error while fetching response. {error_message}")diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.pyindex 62cc2c23c..6d259afbf 100644--- a/backend/prompt_studio/prompt_studio_core_v2/views.py+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py@@ -9,7 +9,8 @@ from django.http import HttpRequestfrom file_management.constants import FileInformationKey as FileKeyfrom file_management.exceptions import FileNotFoundfrom permissions.permission import IsOwner, IsOwnerOrSharedUser-from prompt_studio.processor_loader import get_plugin_class_by_name, load_plugins+from prompt_studio.processor_loader import get_plugin_class_by_name+from prompt_studio.processor_loader import load_plugins as load_processor_pluginsfrom prompt_studio.prompt_profile_manager_v2.constants import (ProfileManagerErrors,ProfileManagerKeys,@@ -75,7 +76,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):serializer_class = CustomToolSerializer- processor_plugins = load_plugins()+ processor_plugins = load_processor_plugins()def get_permissions(self) -> list[Any]:if self.action == "destroy":@@ -221,6 +222,8 @@ class PromptStudioCoreView(viewsets.ModelViewSet):file_name: str = document.document_name# Generate a run_idrun_id = CommonUtils.generate_uuid()+ is_summary = tool.summarize_context+unique_id = PromptStudioHelper.index_document(tool_id=str(tool.tool_id),file_name=file_name,@@ -228,24 +231,8 @@ class PromptStudioCoreView(viewsets.ModelViewSet):user_id=tool.created_by.user_id,document_id=document_id,run_id=run_id,+ is_summary=is_summary,)-- usage_kwargs: dict[Any, Any] = dict()- usage_kwargs[ToolStudioPromptKeys.RUN_ID] = run_id- cls = get_plugin_class_by_name(- name="summarizer",- plugins=self.processor_plugins,- )- if cls:- cls.process(- tool_id=str(tool.tool_id),- file_name=file_name,- org_id=UserSessionUtils.get_organization_id(request),- user_id=tool.created_by.user_id,- document_id=document_id,- usage_kwargs=usage_kwargs.copy(),- )-if unique_id:return Response({"message": "Document indexed successfully."},diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/0002_indexmanager_extraction_status.py b/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/0002_indexmanager_extraction_status.pynew file mode 100644index 000000000..b1b0789c1--- /dev/null+++ b/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/0002_indexmanager_extraction_status.py@@ -0,0 +1,20 @@+# Generated by Django 4.2.1 on 2025-03-20 12:12++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("prompt_studio_index_manager_v2", "0001_initial"),+ ]++ operations = [+ migrations.AddField(+ model_name="indexmanager",+ name="extraction_status",+ field=models.JSONField(+ db_comment="Extraction status for documents", default=dict+ ),+ ),+ ]diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/models.py b/backend/prompt_studio/prompt_studio_index_manager_v2/models.pyindex e58ff87b0..328cd6031 100644--- a/backend/prompt_studio/prompt_studio_index_manager_v2/models.py+++ b/backend/prompt_studio/prompt_studio_index_manager_v2/models.py@@ -62,6 +62,12 @@ class IndexManager(BaseModel):null=False,blank=False,)+ extraction_status = models.JSONField(+ db_comment="Extraction status for documents",+ null=False,+ blank=False,+ default=dict,+ )created_by = models.ForeignKey(User,diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py b/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.pyindex 84747d52b..255045588 100644--- a/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py+++ b/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py@@ -15,9 +15,9 @@ class PromptStudioIndexHelper:@staticmethoddef handle_index_manager(document_id: str,- is_summary: bool,profile_manager: ProfileManager,doc_id: str,+ is_summary: bool = False,) -> IndexManager:try:@@ -67,3 +67,105 @@ class PromptStudioIndexHelper:except Exception as e:transaction.rollback()raise IndexingAPIError("Error updating indexing status") from e++ @staticmethod+ def mark_extraction_status(+ document_id: str, profile_manager: ProfileManager, doc_id: str+ ) -> bool:+ """+ Marks the extraction status for a given document.++ Args:+ document_id (str): ID of the document in DocumentManager.+ profile_manager (ProfileManager): ProfileManager instance for context.+ doc_id (str): Unique identifier for the document within extraction status.++ Returns:+ bool: True if the status is successfully updated, False otherwise.++ """+ try:+ with transaction.atomic():+ document = DocumentManager.objects.get(pk=document_id)++ args = {+ "document_manager": document,+ "profile_manager": profile_manager,+ }++ index_manager, created = IndexManager.objects.get_or_create(**args)++ index_manager.extraction_status = index_manager.extraction_status or {}++ index_manager.extraction_status[doc_id] = True+ logger.info(+ f"Index manager {index_manager} {index_manager.index_ids_history}"+ )+ index_manager.save(update_fields=["extraction_status"])++ if created:+ logger.info(+ f"IndexManager entry created "+ f"for document: {document_id} with {doc_id}"+ )+ else:+ logger.info(+ f"Updated extraction status "+ f"for document: {document_id} with {doc_id}"+ )+ return True++ except DocumentManager.DoesNotExist:+ logger.error(f"Document with ID {document_id} does not exist.")+ raise IndexingAPIError(+ "Error occured while extracting. Please contact admin."+ )++ except Exception as e:+ logger.error(f"Unexpected error updating extraction status: {e}")+ raise IndexingAPIError(f"Error updating indexing status {str(e)}") from e++ @staticmethod+ def check_extraction_status(+ document_id: str, profile_manager: ProfileManager, doc_id: str+ ) -> bool:+ """+ Checks if the extraction status is already marked as complete+ for the given document and doc_id.++ Args:+ document_id (str): ID of the document in DocumentManager.+ profile_manager (ProfileManager): ProfileManager instance for context.+ doc_id (str): Unique identifier for the document within extraction status.++ Returns:+ bool: True if extraction is complete, False otherwise.+ """+ try:+ index_manager = IndexManager.objects.filter(+ document_manager=document_id, profile_manager=profile_manager+ ).first()++ if not index_manager:+ logger.info(f"No IndexManager record found for document: {document_id}")+ return False++ extraction_status = index_manager.extraction_status or {}+ is_extracted = extraction_status.get(doc_id, False)++ if is_extracted:+ logger.info(+ f"Extraction is already marked as complete "+ f"for document: {document_id} with {doc_id}"+ )+ else:+ logger.info(+ f"Extraction is NOT yet marked as complete "+ f"for document: {document_id} with {doc_id}"+ )++ return is_extracted++ except Exception as e:+ logger.error(f"Unexpected error while checking extraction status: {e}")+ raise IndexingAPIError(f"Error checking extraction status {str(e)}") from ediff --git a/prompt-service/pdm.lock b/prompt-service/pdm.lockindex 2bf185058..613e945cd 100644--- a/prompt-service/pdm.lock+++ b/prompt-service/pdm.lock@@ -2687,7 +2687,7 @@ files = [[[package]]name = "openai"-version = "1.70.0"+version = "1.71.0"requires_python = ">=3.8"summary = "The official Python library for the openai API"groups = ["default"]@@ -2702,8 +2702,8 @@ dependencies = ["typing-extensions<5,>=4.11",]files = [- {file = "openai-1.70.0-py3-none-any.whl", hash = "sha256:f6438d053fd8b2e05fd6bef70871e832d9bbdf55e119d0ac5b92726f1ae6f614"},- {file = "openai-1.70.0.tar.gz", hash = "sha256:e52a8d54c3efeb08cf58539b5b21a5abef25368b5432965e4de88cdf4e091b2b"},+ {file = "openai-1.71.0-py3-none-any.whl", hash = "sha256:e1c643738f1fff1af52bce6ef06a7716c95d089281e7011777179614f32937aa"},+ {file = "openai-1.71.0.tar.gz", hash = "sha256:52b20bb990a1780f9b0b8ccebac93416343ebd3e4e714e3eff730336833ca207"},][[package]]@@ -4577,66 +4577,69 @@ files = [[[package]]name = "yarl"-version = "1.18.3"+version = "1.19.0"requires_python = ">=3.9"summary = "Yet another URL library"groups = ["default"]dependencies = ["idna>=2.0","multidict>=4.0",- "propcache>=0.2.0",-]-files = [- {file = "yarl-1.18.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7df647e8edd71f000a5208fe6ff8c382a1de8edfbccdbbfe649d263de07d8c34"},- {file = "yarl-1.18.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c69697d3adff5aa4f874b19c0e4ed65180ceed6318ec856ebc423aa5850d84f7"},- {file = "yarl-1.18.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:602d98f2c2d929f8e697ed274fbadc09902c4025c5a9963bf4e9edfc3ab6f7ed"},- {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c654d5207c78e0bd6d749f6dae1dcbbfde3403ad3a4b11f3c5544d9906969dde"},- {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5094d9206c64181d0f6e76ebd8fb2f8fe274950a63890ee9e0ebfd58bf9d787b"},- {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35098b24e0327fc4ebdc8ffe336cee0a87a700c24ffed13161af80124b7dc8e5"},- {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3236da9272872443f81fedc389bace88408f64f89f75d1bdb2256069a8730ccc"},- {file = "yarl-1.18.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2c08cc9b16f4f4bc522771d96734c7901e7ebef70c6c5c35dd0f10845270bcd"},- {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:80316a8bd5109320d38eef8833ccf5f89608c9107d02d2a7f985f98ed6876990"},- {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c1e1cc06da1491e6734f0ea1e6294ce00792193c463350626571c287c9a704db"},- {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62"},- {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e3b9fd71836999aad54084906f8663dffcd2a7fb5cdafd6c37713b2e72be1760"},- {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:757e81cae69244257d125ff31663249b3013b5dc0a8520d73694aed497fb195b"},- {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b1771de9944d875f1b98a745bc547e684b863abf8f8287da8466cf470ef52690"},- {file = "yarl-1.18.3-cp310-cp310-win32.whl", hash = "sha256:8874027a53e3aea659a6d62751800cf6e63314c160fd607489ba5c2edd753cf6"},- {file = "yarl-1.18.3-cp310-cp310-win_amd64.whl", hash = "sha256:93b2e109287f93db79210f86deb6b9bbb81ac32fc97236b16f7433db7fc437d8"},- {file = "yarl-1.18.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8503ad47387b8ebd39cbbbdf0bf113e17330ffd339ba1144074da24c545f0069"},- {file = "yarl-1.18.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:02ddb6756f8f4517a2d5e99d8b2f272488e18dd0bfbc802f31c16c6c20f22193"},- {file = "yarl-1.18.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:67a283dd2882ac98cc6318384f565bffc751ab564605959df4752d42483ad889"},- {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d980e0325b6eddc81331d3f4551e2a333999fb176fd153e075c6d1c2530aa8a8"},- {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b643562c12680b01e17239be267bc306bbc6aac1f34f6444d1bded0c5ce438ca"},- {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c017a3b6df3a1bd45b9fa49a0f54005e53fbcad16633870104b66fa1a30a29d8"},- {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75674776d96d7b851b6498f17824ba17849d790a44d282929c42dbb77d4f17ae"},- {file = "yarl-1.18.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ccaa3a4b521b780a7e771cc336a2dba389a0861592bbce09a476190bb0c8b4b3"},- {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2d06d3005e668744e11ed80812e61efd77d70bb7f03e33c1598c301eea20efbb"},- {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:9d41beda9dc97ca9ab0b9888cb71f7539124bc05df02c0cff6e5acc5a19dcc6e"},- {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ba23302c0c61a9999784e73809427c9dbedd79f66a13d84ad1b1943802eaaf59"},- {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6748dbf9bfa5ba1afcc7556b71cda0d7ce5f24768043a02a58846e4a443d808d"},- {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0b0cad37311123211dc91eadcb322ef4d4a66008d3e1bdc404808992260e1a0e"},- {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0fb2171a4486bb075316ee754c6d8382ea6eb8b399d4ec62fde2b591f879778a"},- {file = "yarl-1.18.3-cp311-cp311-win32.whl", hash = "sha256:61b1a825a13bef4a5f10b1885245377d3cd0bf87cba068e1d9a88c2ae36880e1"},- {file = "yarl-1.18.3-cp311-cp311-win_amd64.whl", hash = "sha256:b9d60031cf568c627d028239693fd718025719c02c9f55df0a53e587aab951b5"},- {file = "yarl-1.18.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:61e5e68cb65ac8f547f6b5ef933f510134a6bf31bb178be428994b0cb46c2a04"},- {file = "yarl-1.18.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719"},- {file = "yarl-1.18.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a440a2a624683108a1b454705ecd7afc1c3438a08e890a1513d468671d90a04e"},- {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09c7907c8548bcd6ab860e5f513e727c53b4a714f459b084f6580b49fa1b9cee"},- {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4f6450109834af88cb4cc5ecddfc5380ebb9c228695afc11915a0bf82116789"},- {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9ca04806f3be0ac6d558fffc2fdf8fcef767e0489d2684a21912cc4ed0cd1b8"},- {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77a6e85b90a7641d2e07184df5557132a337f136250caafc9ccaa4a2a998ca2c"},- {file = "yarl-1.18.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6333c5a377c8e2f5fae35e7b8f145c617b02c939d04110c76f29ee3676b5f9a5"},- {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0b3c92fa08759dbf12b3a59579a4096ba9af8dd344d9a813fc7f5070d86bbab1"},- {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:4ac515b860c36becb81bb84b667466885096b5fc85596948548b667da3bf9f24"},- {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:045b8482ce9483ada4f3f23b3774f4e1bf4f23a2d5c912ed5170f68efb053318"},- {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:a4bb030cf46a434ec0225bddbebd4b89e6471814ca851abb8696170adb163985"},- {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:54d6921f07555713b9300bee9c50fb46e57e2e639027089b1d795ecd9f7fa910"},- {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1d407181cfa6e70077df3377938c08012d18893f9f20e92f7d2f314a437c30b1"},- {file = "yarl-1.18.3-cp39-cp39-win32.whl", hash = "sha256:ac36703a585e0929b032fbaab0707b75dc12703766d0b53486eabd5139ebadd5"},- {file = "yarl-1.18.3-cp39-cp39-win_amd64.whl", hash = "sha256:ba87babd629f8af77f557b61e49e7c7cac36f22f871156b91e10a6e9d4f829e9"},- {file = "yarl-1.18.3-py3-none-any.whl", hash = "sha256:b57f4f58099328dfb26c6a771d09fb20dbbae81d20cfb66141251ea063bd101b"},- {file = "yarl-1.18.3.tar.gz", hash = "sha256:ac1801c45cbf77b6c99242eeff4fffb5e4e73a800b5c4ad4fc0be5def634d2e1"},+ "propcache>=0.2.1",+]+files = [+ {file = "yarl-1.19.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0bae32f8ebd35c04d6528cedb4a26b8bf25339d3616b04613b97347f919b76d3"},+ {file = "yarl-1.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8015a076daf77823e7ebdcba474156587391dab4e70c732822960368c01251e6"},+ {file = "yarl-1.19.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9973ac95327f5d699eb620286c39365990b240031672b5c436a4cd00539596c5"},+ {file = "yarl-1.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd4b5fbd7b9dde785cfeb486b8cca211a0b138d4f3a7da27db89a25b3c482e5c"},+ {file = "yarl-1.19.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:75460740005de5a912b19f657848aef419387426a40f581b1dc9fac0eb9addb5"},+ {file = "yarl-1.19.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:57abd66ca913f2cfbb51eb3dbbbac3648f1f6983f614a4446e0802e241441d2a"},+ {file = "yarl-1.19.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:46ade37911b7c99ce28a959147cb28bffbd14cea9e7dd91021e06a8d2359a5aa"},+ {file = "yarl-1.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8346ec72ada749a6b5d82bff7be72578eab056ad7ec38c04f668a685abde6af0"},+ {file = "yarl-1.19.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e4cb14a6ee5b6649ccf1c6d648b4da9220e8277d4d4380593c03cc08d8fe937"},+ {file = "yarl-1.19.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:66fc1c2926a73a2fb46e4b92e3a6c03904d9bc3a0b65e01cb7d2b84146a8bd3b"},+ {file = "yarl-1.19.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:5a70201dd1e0a4304849b6445a9891d7210604c27e67da59091d5412bc19e51c"},+ {file = "yarl-1.19.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4807aab1bdeab6ae6f296be46337a260ae4b1f3a8c2fcd373e236b4b2b46efd"},+ {file = "yarl-1.19.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ae584afe81a1de4c1bb06672481050f0d001cad13163e3c019477409f638f9b7"},+ {file = "yarl-1.19.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:30eaf4459df6e91f21b2999d1ee18f891bcd51e3cbe1de301b4858c84385895b"},+ {file = "yarl-1.19.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0e617d45d03c8dec0dfce6f51f3e1b8a31aa81aaf4a4d1442fdb232bcf0c6d8c"},+ {file = "yarl-1.19.0-cp310-cp310-win32.whl", hash = "sha256:32ba32d0fa23893fd8ea8d05bdb05de6eb19d7f2106787024fd969f4ba5466cb"},+ {file = "yarl-1.19.0-cp310-cp310-win_amd64.whl", hash = "sha256:545575ecfcd465891b51546c2bcafdde0acd2c62c2097d8d71902050b20e4922"},+ {file = "yarl-1.19.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:163ff326680de5f6d4966954cf9e3fe1bf980f5fee2255e46e89b8cf0f3418b5"},+ {file = "yarl-1.19.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a626c4d9cca298d1be8625cff4b17004a9066330ac82d132bbda64a4c17c18d3"},+ {file = "yarl-1.19.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:961c3e401ea7f13d02b8bb7cb0c709152a632a6e14cdc8119e9c6ee5596cd45d"},+ {file = "yarl-1.19.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a39d7b807ab58e633ed760f80195cbd145b58ba265436af35f9080f1810dfe64"},+ {file = "yarl-1.19.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4228978fb59c6b10f60124ba8e311c26151e176df364e996f3f8ff8b93971b5"},+ {file = "yarl-1.19.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9ba536b17ecf3c74a94239ec1137a3ad3caea8c0e4deb8c8d2ffe847d870a8c5"},+ {file = "yarl-1.19.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a251e00e445d2e9df7b827c9843c0b87f58a3254aaa3f162fb610747491fe00f"},+ {file = "yarl-1.19.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9b92431d8b4d4ca5ccbfdbac95b05a3a6cd70cd73aa62f32f9627acfde7549c"},+ {file = "yarl-1.19.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec2f56edaf476f70b5831bbd59700b53d9dd011b1f77cd4846b5ab5c5eafdb3f"},+ {file = "yarl-1.19.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:acf9b92c4245ac8b59bc7ec66a38d3dcb8d1f97fac934672529562bb824ecadb"},+ {file = "yarl-1.19.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:57711f1465c06fee8825b95c0b83e82991e6d9425f9a042c3c19070a70ac92bf"},+ {file = "yarl-1.19.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:528e86f5b1de0ad8dd758ddef4e0ed24f5d946d4a1cef80ffb2d4fca4e10f122"},+ {file = "yarl-1.19.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:3b77173663e075d9e5a57e09d711e9da2f3266be729ecca0b8ae78190990d260"},+ {file = "yarl-1.19.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d8717924cf0a825b62b1a96fc7d28aab7f55a81bf5338b8ef41d7a76ab9223e9"},+ {file = "yarl-1.19.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0df9f0221a78d858793f40cbea3915c29f969c11366646a92ca47e080a14f881"},+ {file = "yarl-1.19.0-cp311-cp311-win32.whl", hash = "sha256:8b3ade62678ee2c7c10dcd6be19045135e9badad53108f7d2ed14896ee396045"},+ {file = "yarl-1.19.0-cp311-cp311-win_amd64.whl", hash = "sha256:0626ee31edb23ac36bdffe607231de2cca055ad3a5e2dc5da587ef8bc6a321bc"},+ {file = "yarl-1.19.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:85ac908cd5a97bbd3048cca9f1bf37b932ea26c3885099444f34b0bf5d5e9fa6"},+ {file = "yarl-1.19.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6ba0931b559f1345df48a78521c31cfe356585670e8be22af84a33a39f7b9221"},+ {file = "yarl-1.19.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5bc503e1c1fee1b86bcb58db67c032957a52cae39fe8ddd95441f414ffbab83e"},+ {file = "yarl-1.19.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d995122dcaf180fd4830a9aa425abddab7c0246107c21ecca2fa085611fa7ce9"},+ {file = "yarl-1.19.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:217f69e60a14da4eed454a030ea8283f8fbd01a7d6d81e57efb865856822489b"},+ {file = "yarl-1.19.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aad67c8f13a4b79990082f72ef09c078a77de2b39899aabf3960a48069704973"},+ {file = "yarl-1.19.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dff065a1a8ed051d7e641369ba1ad030d5a707afac54cf4ede7069b959898835"},+ {file = "yarl-1.19.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ada882e26b16ee651ab6544ce956f2f4beaed38261238f67c2a96db748e17741"},+ {file = "yarl-1.19.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:67a56b1acc7093451ea2de0687aa3bd4e58d6b4ef6cbeeaad137b45203deaade"},+ {file = "yarl-1.19.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:e97d2f0a06b39e231e59ebab0e6eec45c7683b339e8262299ac952707bdf7688"},+ {file = "yarl-1.19.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:a5288adb7c59d0f54e4ad58d86fb06d4b26e08a59ed06d00a1aac978c0e32884"},+ {file = "yarl-1.19.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1efbf4d03e6eddf5da27752e0b67a8e70599053436e9344d0969532baa99df53"},+ {file = "yarl-1.19.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:f228f42f29cc87db67020f7d71624102b2c837686e55317b16e1d3ef2747a993"},+ {file = "yarl-1.19.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:c515f7dd60ca724e4c62b34aeaa603188964abed2eb66bb8e220f7f104d5a187"},+ {file = "yarl-1.19.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4815ec6d3d68a96557fa71bd36661b45ac773fb50e5cfa31a7e843edb098f060"},+ {file = "yarl-1.19.0-cp39-cp39-win32.whl", hash = "sha256:9fac2dd1c5ecb921359d9546bc23a6dcc18c6acd50c6d96f118188d68010f497"},+ {file = "yarl-1.19.0-cp39-cp39-win_amd64.whl", hash = "sha256:5864f539ce86b935053bfa18205fa08ce38e9a40ea4d51b19ce923345f0ed5db"},+ {file = "yarl-1.19.0-py3-none-any.whl", hash = "sha256:a727101eb27f66727576630d02985d8a065d09cd0b5fcbe38a5793f71b2a97ef"},+ {file = "yarl-1.19.0.tar.gz", hash = "sha256:01e02bb80ae0dbed44273c304095295106e1d9470460e773268a27d11e594892"},][[package]]diff --git a/prompt-service/pyproject.toml b/prompt-service/pyproject.tomlindex 7f857ab7d..725f4282c 100644--- a/prompt-service/pyproject.toml+++ b/prompt-service/pyproject.toml@@ -51,9 +51,6 @@ test = [][tool.pdm.scripts]-prompt_service.cmd = "flask --app src/unstract/prompt_service/main.py run --port 3003"+prompt_service.cmd = "flask --app src/unstract/prompt_service/run.py run --port 3003"prompt_service.env_file = ".env"prompt_service.help = "Runs the Unstract prompt service"-prompt_service_v2.cmd = "flask --app src/unstract/prompt_service_v2/run.py run --port 3003"-prompt_service_v2.env_file = ".env"-prompt_service_v2.help = "Runs the Unstract prompt service v2"diff --git a/prompt-service/sample.env b/prompt-service/sample.envindex f31edd9f7..cb0aeb73e 100644--- a/prompt-service/sample.env+++ b/prompt-service/sample.env@@ -37,3 +37,8 @@ FLIPT_SERVICE_AVAILABLE=FalsePERMANENT_REMOTE_STORAGE='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'TEMPORARY_REMOTE_STORAGE='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'REMOTE_PROMPT_STUDIO_FILE_PATH="unstract/prompt-studio-data/"+++#X2Text Service+X2TEXT_HOST=http://unstract-x2text-service+X2TEXT_PORT=3004diff --git a/prompt-service/src/unstract/prompt_service/authentication_middleware.py b/prompt-service/src/unstract/prompt_service/authentication_middleware.pydeleted file mode 100644index 52297d8e1..000000000--- a/prompt-service/src/unstract/prompt_service/authentication_middleware.py+++ /dev/null@@ -1,78 +0,0 @@-from typing import Optional--from flask import Request, current_app-from unstract.prompt_service.config import db-from unstract.prompt_service.constants import DBTableV2-from unstract.prompt_service.db_utils import DBUtils-from unstract.prompt_service.env_manager import EnvLoader--DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract")---class AuthenticationMiddleware:-- @staticmethod- def validate_bearer_token(token: Optional[str]) -> bool:- try:- if token is None:- current_app.logger.error("Authentication failed. Empty bearer token")- return False-- platform_key_table = f'"{DB_SCHEMA}".{DBTableV2.PLATFORM_KEY}'-- query = f"SELECT * FROM {platform_key_table} WHERE key = '{token}'"- cursor = db.execute_sql(query)- result_row = cursor.fetchone()- cursor.close()- if not result_row or len(result_row) == 0:- current_app.logger.error(- f"Authentication failed. bearer token not found {token}"- )- return False- platform_key = str(result_row[1])- is_active = bool(result_row[2])- if not is_active:- current_app.logger.error(- f"Token is not active. Activate \- before using it. token {token}"- )- return False- if platform_key != token:- current_app.logger.error(- f"Authentication failed. Invalid bearer token: {token}"- )- return False-- except Exception as e:- current_app.logger.error(- f"Error while validating bearer token: {e}",- stack_info=True,- exc_info=True,- )- return False- return True-- @staticmethod- def get_token_from_auth_header(request: Request) -> Optional[str]:- try:- bearer_token = request.headers.get("Authorization")- if not bearer_token:- return None- token: str = bearer_token.strip().replace("Bearer ", "")- return token- except Exception as e:- current_app.logger.info(f"Exception while getting token {e}")- return None-- @staticmethod- def get_account_from_bearer_token(token: Optional[str]) -> str:- platform_key_table = DBTableV2.PLATFORM_KEY- organization_table = DBTableV2.ORGANIZATION-- query = f"SELECT organization_id FROM {platform_key_table} WHERE key='{token}'"- organization = DBUtils.execute_query(query)- query_org = (- f"SELECT schema_name FROM {organization_table} WHERE id='{organization}'"- )- schema_name: str = DBUtils.execute_query(query_org)- return schema_namediff --git a/prompt-service/src/unstract/prompt_service/config.py b/prompt-service/src/unstract/prompt_service/config.pyindex ad9220a39..f5e993c06 100644--- a/prompt-service/src/unstract/prompt_service/config.py+++ b/prompt-service/src/unstract/prompt_service/config.py@@ -1,53 +1,30 @@import logging-from logging.config import dictConfigfrom os import environ as envfrom dotenv import load_dotenvfrom flask import Flask-from peewee import PostgresqlDatabase+from unstract.prompt_service.controllers import api+from unstract.prompt_service.extensions import db+from unstract.prompt_service.helpers.lifecycle import register_lifecycle_hooks+from unstract.prompt_service.helpers.plugin import plugin_loader+from unstract.prompt_service.utils.env_loader import get_env_or_diefrom unstract.sdk.constants import LogLevel-load_dotenv()--dictConfig(- {- "version": 1,- "formatters": {- "default": {- "format": "[%(asctime)s] %(levelname)s in %(module)s: %(message)s",- }- },- "handlers": {- "wsgi": {- "class": "logging.StreamHandler",- "stream": "ext://flask.logging.wsgi_errors_stream",- "formatter": "default",- }- },- "root": {"level": "INFO", "handlers": ["wsgi"]},- }-)--db = PostgresqlDatabase(None)+from unstract.core.flask import register_error_handlers, register_request_id_middleware+from unstract.core.flask.logging import setup_logging--def get_env_or_die(env_key: str) -> str:- env_value = env.get(env_key)- if not env_value:- raise ValueError(f"Env variable {env_key} is required")- return env_value+load_dotenv()def create_app() -> Flask:- app = Flask("prompt-service")- log_level = env.get("LOG_LEVEL", LogLevel.WARN.value)- if log_level == LogLevel.DEBUG.value:- app.logger.setLevel(logging.DEBUG)- elif log_level == LogLevel.INFO.value:- app.logger.setLevel(logging.INFO)- else:- app.logger.setLevel(logging.WARNING)+ """Creates and configures the Flask application."""+ log_level = env.get("LOG_LEVEL", LogLevel.INFO.value).upper()+ setup_logging(log_level)+ log_level = getattr(logging, log_level, logging.INFO)+ app = Flask("prompt-service")+ app.logger.setLevel(log_level)+ app.logger.info("Initializing Flask application...")# Load required environment variablesdb_host = get_env_or_die("PG_BE_HOST")db_port = get_env_or_die("PG_BE_PORT")@@ -65,4 +42,13 @@ def create_app() -> Flask:port=db_port,options=f"-c application_name={application_name}",)++ # Load plugins+ plugin_loader(app)+ register_lifecycle_hooks(app)+ register_request_id_middleware(app)+ register_error_handlers(app)+ app.register_blueprint(api)++ app.logger.info("Flask app created successfully.")return appdiff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.pyindex 2f007d731..2ad922183 100644--- a/prompt-service/src/unstract/prompt_service/constants.py+++ b/prompt-service/src/unstract/prompt_service/constants.py@@ -1,7 +1,9 @@from enum import Enum-class PromptServiceContants:+class PromptServiceConstants:+ """Constants used in the prompt service."""+WORD = "word"SYNONYMS = "synonyms"OUTPUTS = "outputs"@@ -80,6 +82,16 @@ class PromptServiceContants:LINE_ITEM = "line-item"LINE_NUMBERS = "line_numbers"WHISPER_HASH = "whisper_hash"+ PAID_FEATURE_MSG = (+ "It is a cloud / enterprise feature. If you have purchased a plan and still "+ "face this issue, please contact support"+ )+ NO_CONTEXT_ERROR = (+ "Couldn't fetch context from vector DB. "+ "This happens usually due to a delay by the Vector DB "+ "provider to confirm writes to DB. "+ "Please try again after some time"+ )class RunLevel(Enum):@@ -105,15 +117,59 @@ class DBTableV2:class FileStorageKeys:+ """File storage keys."""+PERMANENT_REMOTE_STORAGE = "PERMANENT_REMOTE_STORAGE"TEMPORARY_REMOTE_STORAGE = "TEMPORARY_REMOTE_STORAGE"class FileStorageType(Enum):+ """File storage type."""+PERMANENT = "permanent"TEMPORARY = "temporary"class ExecutionSource(Enum):+ """Execution source."""+IDE = "ide"TOOL = "tool"+++class VariableType(str, Enum):+ """Type of variable."""++ STATIC = "STATIC"+ DYNAMIC = "DYNAMIC"+++class VariableConstants:+ """Constants for variable extraction."""++ VARIABLE_REGEX = "{{(.+?)}}"+ DYNAMIC_VARIABLE_DATA_REGEX = r"\[(.*?)\]"+ DYNAMIC_VARIABLE_URL_REGEX = r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))" # noqa: E501+++class IndexingConstants:+ TOOL_ID = "tool_id"+ EMBEDDING_INSTANCE_ID = "embedding_instance_id"+ VECTOR_DB_INSTANCE_ID = "vector_db_instance_id"+ X2TEXT_INSTANCE_ID = "x2text_instance_id"+ FILE_PATH = "file_path"+ CHUNK_SIZE = "chunk_size"+ CHUNK_OVERLAP = "chunk_overlap"+ REINDEX = "reindex"+ FILE_HASH = "file_hash"+ OUTPUT_FILE_PATH = "output_file_path"+ ENABLE_HIGHLIGHT = "enable_highlight"+ USAGE_KWARGS = "usage_kwargs"+ PROCESS_TEXT = "process_text"+ EXTRACTED_TEXT = "extracted_text"+ TAGS = "tags"+ EXECUTION_SOURCE = "execution_source"+ DOC_ID = "doc_id"+ TOOL_EXECUTION_METATADA = "tool_execution_metadata"+ EXECUTION_DATA_DIR = "execution_data_dir"+ METADATA_FILE = "METADATA.json"diff --git a/prompt-service/src/unstract/prompt_service_v2/controllers/__init__.py b/prompt-service/src/unstract/prompt_service/controllers/__init__.pysimilarity index 100%rename from prompt-service/src/unstract/prompt_service_v2/controllers/__init__.pyrename to prompt-service/src/unstract/prompt_service/controllers/__init__.pydiff --git a/prompt-service/src/unstract/prompt_service_v2/controllers/answer_prompt.py b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.pysimilarity index 84%rename from prompt-service/src/unstract/prompt_service_v2/controllers/answer_prompt.pyrename to prompt-service/src/unstract/prompt_service/controllers/answer_prompt.pyindex 308c19267..1184f9e45 100644--- a/prompt-service/src/unstract/prompt_service_v2/controllers/answer_prompt.py+++ b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py@@ -9,21 +9,19 @@ from typing import Anyfrom flask import Blueprintfrom flask import current_app as appfrom flask import request-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys-from unstract.prompt_service_v2.constants import RunLevel-from unstract.prompt_service_v2.exceptions import BadRequest-from unstract.prompt_service_v2.helpers.auth import AuthHelper-from unstract.prompt_service_v2.helpers.plugin import PluginManager-from unstract.prompt_service_v2.helpers.prompt_ide_base_tool import (- PromptServiceBaseTool,-)-from unstract.prompt_service_v2.helpers.usage import UsageHelper-from unstract.prompt_service_v2.services.answer_prompt import AnswerPromptService-from unstract.prompt_service_v2.services.retrieval import RetrievalService-from unstract.prompt_service_v2.services.variable_replacement import (+from unstract.prompt_service.constants import PromptServiceConstants as PSKeys+from unstract.prompt_service.constants import RunLevel+from unstract.prompt_service.exceptions import BadRequest+from unstract.prompt_service.helpers.auth import AuthHelper+from unstract.prompt_service.helpers.plugin import PluginManager+from unstract.prompt_service.helpers.prompt_ide_base_tool import PromptServiceBaseTool+from unstract.prompt_service.helpers.usage import UsageHelper+from unstract.prompt_service.services.answer_prompt import AnswerPromptService+from unstract.prompt_service.services.retrieval import RetrievalService+from unstract.prompt_service.services.variable_replacement import (VariableReplacementService,)-from unstract.prompt_service_v2.utils.log import publish_log+from unstract.prompt_service.utils.log import publish_logfrom unstract.sdk.adapters.llm.no_op.src.no_op_custom_llm import NoOpCustomLLMfrom unstract.sdk.constants import LogLevelfrom unstract.sdk.embedding import Embedding@@ -46,11 +44,11 @@ def prompt_processor() -> Any:raise BadRequesttool_settings = payload.get(PSKeys.TOOL_SETTINGS, {})enable_challenge = tool_settings.get(PSKeys.ENABLE_CHALLENGE, False)- challenge_llm = None- # TODO: Rename "outputs" to "prompts" in payload+ # Rename "outputs" to "prompts" in payloadprompts = payload.get(PSKeys.OUTPUTS, [])tool_id: str = payload.get(PSKeys.TOOL_ID, "")run_id: str = payload.get(PSKeys.RUN_ID, "")+ execution_id: str = payload.get(PSKeys.EXECUTION_ID, "")file_hash = payload.get(PSKeys.FILE_HASH)file_path = payload.get(PSKeys.FILE_PATH)doc_name = str(payload.get(PSKeys.FILE_NAME, ""))@@ -73,7 +71,7 @@ def prompt_processor() -> Any:RunLevel.RUN,f"Preparing to execute {len(prompts)} prompt(s)",)- # TODO: Rename "output" to "prompt"+ # Rename "output" to "prompt"for output in prompts: # type:ignorevariable_names.append(output[PSKeys.NAME])metadata[PSKeys.REQUIRED_FIELDS][output[PSKeys.NAME]] = output.get(@@ -84,6 +82,7 @@ def prompt_processor() -> Any:prompt_name = output[PSKeys.NAME]prompt_text = output[PSKeys.PROMPT]chunk_size = output[PSKeys.CHUNK_SIZE]+ app.logger.info(f"[{tool_id}] chunk size: {chunk_size}")util = PromptServiceBaseTool(platform_key=platform_key)index = Index(tool=util, run_id=run_id, capture_metrics=True)if VariableReplacementService.is_variables_present(prompt_text=prompt_text):@@ -137,7 +136,7 @@ def prompt_processor() -> Any:)try:- usage_kwargs = {"run_id": run_id}+ usage_kwargs = {"run_id": run_id, "execution_id": execution_id}adapter_instance_id = output[PSKeys.LLM]llm = LLM(tool=util,@@ -244,90 +243,54 @@ def prompt_processor() -> Any:raise etry:- if chunk_size == 0:- # We can do this only for chunkless indexes- context: set[str] = RetrievalService.fetch_context_from_vector_db(- index=index,- output=output,- doc_id=doc_id,- tool_id=tool_id,- doc_name=doc_name,- prompt_name=prompt_name,- log_events_id=log_events_id,- usage_kwargs=usage_kwargs,- )- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.INFO,- RunLevel.RUN,- "Retrieving answer from LLM",- )- answer = AnswerPromptService.construct_and_run_prompt(++ answer = "NA"+ publish_log(+ log_events_id,+ {+ "tool_id": tool_id,+ "prompt_key": prompt_name,+ "doc_name": doc_name,+ },+ LogLevel.INFO,+ RunLevel.RUN,+ "Retrieving context from adapter",+ )++ retrieval_strategy = output.get(PSKeys.RETRIEVAL_STRATEGY)++ if retrieval_strategy in {PSKeys.SIMPLE, PSKeys.SUBQUESTION}:+ app.logger.info(f"[{tool_id}] Performing retrieval for : {file_path}")+ answer, context = RetrievalService.perform_retrieval(tool_settings=tool_settings,output=output,+ doc_id=doc_id,llm=llm,- context="\n".join(context),- prompt="promptx",+ vector_db=vector_db,+ retrieval_type=retrieval_strategy,metadata=metadata,- file_path=file_path,+ chunk_size=chunk_size,execution_source=execution_source,+ file_path=file_path,)- metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = (- AnswerPromptService.get_cleaned_context(context)- )+ metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = contextelse:- answer = "NA"- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.INFO,- RunLevel.RUN,- "Retrieving context from adapter",+ app.logger.info(+ "Invalid retrieval strategy passed: %s",+ retrieval_strategy,)- retrieval_strategy = output.get(PSKeys.RETRIEVAL_STRATEGY)-- if retrieval_strategy in {PSKeys.SIMPLE, PSKeys.SUBQUESTION}:- vector_index = vector_db.get_vector_store_index()- answer, context = RetrievalService.run_retrieval(- tool_settings=tool_settings,- output=output,- doc_id=doc_id,- llm=llm,- vector_index=vector_index,- retrieval_type=retrieval_strategy,- metadata=metadata,- execution_source=execution_source,- )- metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = (- AnswerPromptService.get_cleaned_context(context)- )- else:- app.logger.info(- "Invalid retrieval strategy passed: %s",- retrieval_strategy,- )-- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.DEBUG,- RunLevel.RUN,- "Retrieved context from adapter",- )+ publish_log(+ log_events_id,+ {+ "tool_id": tool_id,+ "prompt_key": prompt_name,+ "doc_name": doc_name,+ },+ LogLevel.DEBUG,+ RunLevel.RUN,+ "Retrieved context from adapter",+ )publish_log(log_events_id,@@ -345,7 +308,7 @@ def prompt_processor() -> Any:if answer.lower() == "na":structured_output[output[PSKeys.NAME]] = Noneelse:- # TODO: Extract these prompts as constants after pkging+ # Extract these prompts as constants after pkgingprompt = f"Extract the number from the following \text:\n{answer}\n\nOutput just the number. \If the number is expressed in millions \@@ -446,7 +409,7 @@ def prompt_processor() -> Any:f"Candidate JSON: {answer}")app.logger.info(err_msg, LogLevel.ERROR)- # TODO: Format log message after unifying these types+ # Format log message after unifying these typespublish_log(log_events_id,{@@ -557,7 +520,7 @@ def prompt_processor() -> Any:try:evaluator = eval_plugin["entrypoint_cls"]("",- "\n".join(context),+ "".join(context),"","",output,diff --git a/prompt-service/src/unstract/prompt_service_v2/controllers/extraction.py b/prompt-service/src/unstract/prompt_service/controllers/extraction.pysimilarity index 81%rename from prompt-service/src/unstract/prompt_service_v2/controllers/extraction.pyrename to prompt-service/src/unstract/prompt_service/controllers/extraction.pyindex aecabdc43..4dcff8c47 100644--- a/prompt-service/src/unstract/prompt_service_v2/controllers/extraction.py+++ b/prompt-service/src/unstract/prompt_service/controllers/extraction.py@@ -1,11 +1,11 @@from typing import Any, Optionalfrom flask import Blueprint, request-from unstract.prompt_service_v2.constants import IndexingConstants as IKeys-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys-from unstract.prompt_service_v2.helpers.auth import AuthHelper-from unstract.prompt_service_v2.services.extraction import ExtractionService-from unstract.prompt_service_v2.utils.request import validate_request_payload+from unstract.prompt_service.constants import IndexingConstants as IKeys+from unstract.prompt_service.constants import PromptServiceConstants as PSKeys+from unstract.prompt_service.helpers.auth import AuthHelper+from unstract.prompt_service.services.extraction import ExtractionService+from unstract.prompt_service.utils.request import validate_request_payloadextraction_bp = Blueprint("extract", __name__)@@ -14,7 +14,6 @@ REQUIRED_FIELDS = ["file_path","execution_source","run_id",- "tags",]diff --git a/prompt-service/src/unstract/prompt_service_v2/controllers/health.py b/prompt-service/src/unstract/prompt_service/controllers/health.pysimilarity index 100%rename from prompt-service/src/unstract/prompt_service_v2/controllers/health.pyrename to prompt-service/src/unstract/prompt_service/controllers/health.pydiff --git a/prompt-service/src/unstract/prompt_service_v2/controllers/indexing.py b/prompt-service/src/unstract/prompt_service/controllers/indexing.pysimilarity index 88%rename from prompt-service/src/unstract/prompt_service_v2/controllers/indexing.pyrename to prompt-service/src/unstract/prompt_service/controllers/indexing.pyindex 339b4a790..0fb1147d4 100644--- a/prompt-service/src/unstract/prompt_service_v2/controllers/indexing.py+++ b/prompt-service/src/unstract/prompt_service/controllers/indexing.py@@ -2,17 +2,17 @@ import loggingfrom typing import Any, Optionalfrom flask import Blueprint, request-from unstract.prompt_service_v2.constants import IndexingConstants as IKeys-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys-from unstract.prompt_service_v2.dto import (+from unstract.prompt_service.constants import IndexingConstants as IKeys+from unstract.prompt_service.constants import PromptServiceConstants as PSKeys+from unstract.prompt_service.dto import (ChunkingConfig,FileInfo,InstanceIdentifiers,ProcessingOptions,)-from unstract.prompt_service_v2.helpers.auth import AuthHelper-from unstract.prompt_service_v2.services.indexing import IndexingService-from unstract.prompt_service_v2.utils.request import validate_request_payload+from unstract.prompt_service.helpers.auth import AuthHelper+from unstract.prompt_service.services.indexing import IndexingService+from unstract.prompt_service.utils.request import validate_request_payloadindexing_bp = Blueprint("index", __name__)logger = logging.getLogger(__name__)diff --git a/prompt-service/src/unstract/prompt_service_v2/core/index_v2.py b/prompt-service/src/unstract/prompt_service/core/index_v2.pysimilarity index 96%rename from prompt-service/src/unstract/prompt_service_v2/core/index_v2.pyrename to prompt-service/src/unstract/prompt_service/core/index_v2.pyindex 17da0b56a..e3c1342ad 100644--- a/prompt-service/src/unstract/prompt_service_v2/core/index_v2.py+++ b/prompt-service/src/unstract/prompt_service/core/index_v2.py@@ -1,6 +1,6 @@import jsonimport logging-from typing import Optional+from typing import Any, Optionalfrom llama_index.core import Documentfrom llama_index.core.vector_stores import (@@ -10,7 +10,7 @@ from llama_index.core.vector_stores import (VectorStoreQuery,VectorStoreQueryResult,)-from unstract.prompt_service_v2.dto import (+from unstract.prompt_service.dto import (ChunkingConfig,FileInfo,InstanceIdentifiers,@@ -43,7 +43,6 @@ class Index:run_id: Optional[str] = None,capture_metrics: bool = False,):- # TODO: Inherit from StreamMixin and avoid using BaseToolself.tool = toolself._run_id = run_idself._capture_metrics = capture_metrics@@ -154,6 +153,7 @@ class Index:vector_db: VectorDB,doc_id: str,extracted_text: str,+ doc_id_found: str,):if isinstance(vector_db.get_vector_db(@@ -168,12 +168,12 @@ class Index:full_text = [{"section": "full",- "text_contents": extracted_text,+ "text_contents": str(extracted_text),}]# Convert raw text to llama index usage Documentdocuments = self._prepare_documents(doc_id, full_text)- if self.processing_options.reindex:+ if self.processing_options.reindex and doc_id_found:self.delete_nodes(vector_db, doc_id)self._trigger_indexing(vector_db, documents)return doc_id@@ -206,7 +206,7 @@ class Index:)raise SdkError(f"Error deleting nodes for {doc_id}: {e}") from e- def _prepare_documents(self, doc_id: str, full_text: str) -> list:+ def _prepare_documents(self, doc_id: str, full_text: Any) -> list:documents = []try:for item in full_text:diff --git a/prompt-service/src/unstract/prompt_service_v2/core/retrievers/base_retriever.py b/prompt-service/src/unstract/prompt_service/core/retrievers/base_retriever.pysimilarity index 66%rename from prompt-service/src/unstract/prompt_service_v2/core/retrievers/base_retriever.pyrename to prompt-service/src/unstract/prompt_service/core/retrievers/base_retriever.pyindex 5dc26bd65..cf560f92a 100644--- a/prompt-service/src/unstract/prompt_service_v2/core/retrievers/base_retriever.py+++ b/prompt-service/src/unstract/prompt_service/core/retrievers/base_retriever.py@@ -1,8 +1,18 @@+from typing import Optional++from unstract.sdk.llm import LLMfrom unstract.sdk.vector_db import VectorDBclass BaseRetriever:- def __init__(self, vector_db: VectorDB, prompt: str, doc_id: str, top_k: int):+ def __init__(+ self,+ vector_db: VectorDB,+ prompt: str,+ doc_id: str,+ top_k: int,+ llm: Optional[LLM] = None,+ ):"""Initialize the Retrieval class.Args:@@ -15,6 +25,7 @@ class BaseRetriever:self.prompt = promptself.doc_id = doc_idself.top_k = top_k+ self.llm = llm._llm_instance if llm else None@staticmethoddef retrieve() -> set[str]:diff --git a/prompt-service/src/unstract/prompt_service_v2/core/retrievers/simple.py b/prompt-service/src/unstract/prompt_service/core/retrievers/simple.pysimilarity index 78%rename from prompt-service/src/unstract/prompt_service_v2/core/retrievers/simple.pyrename to prompt-service/src/unstract/prompt_service/core/retrievers/simple.pyindex 183e638e2..211558c68 100644--- a/prompt-service/src/unstract/prompt_service_v2/core/retrievers/simple.py+++ b/prompt-service/src/unstract/prompt_service/core/retrievers/simple.py@@ -1,15 +1,28 @@-import loggingimport time+from flask import current_app as appfrom llama_index.core import VectorStoreIndexfrom llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters-from unstract.prompt_service_v2.core.retrievers.base_retriever import BaseRetriever--logger = logging.getLogger(__name__)+from unstract.prompt_service.core.retrievers.base_retriever import BaseRetrieverclass SimpleRetriever(BaseRetriever):def retrieve(self) -> set[str]:+ context = self._simple_retrieval()+ app.logger.info(f"context {context}")+ if not context:+ # UN-1288 For Pinecone, we are seeing an inconsistent case where+ # query with doc_id fails even though indexing just happened.+ # This causes the following retrieve to return no text.+ # To rule out any lag on the Pinecone vector DB write,+ # the following sleep is added+ # Note: This will not fix the issue. Since this issue is inconsistent+ # and not reproducible easily, this is just a safety net.+ time.sleep(2)+ context = self._simple_retrieval()+ return context++ def _simple_retrieval(self):vector_query_engine: VectorStoreIndex = self.vector_db.get_vector_store_index()retriever = vector_query_engine.as_retriever(similarity_top_k=self.top_k,@@ -22,24 +35,13 @@ class SimpleRetriever(BaseRetriever):nodes = retriever.retrieve(self.prompt)context: set[str] = set()for node in nodes:- # ToDo: May have to fine-tune this value for node score or keep it+ # May have to fine-tune this value for node score or keep it# configurable at the adapter levelif node.score > 0:context.add(node.get_content())else:- logger.info(+ app.logger.info("Node score is less than 0. "f"Ignored: {node.node_id} with score {node.score}")-- if not context:- # UN-1288 For Pinecone, we are seeing an inconsistent case where- # query with doc_id fails even though indexing just happened.- # This causes the following retrieve to return no text.- # To rule out any lag on the Pinecone vector DB write,- # the following sleep is added- # Note: This will not fix the issue. Since this issue is inconsistent- # and not reproducible easily, this is just a safety net.- time.sleep(2)- context = self.retrieve()return contextdiff --git a/prompt-service/src/unstract/prompt_service_v2/core/retrievers/subquestion.py b/prompt-service/src/unstract/prompt_service/core/retrievers/subquestion.pysimilarity index 63%rename from prompt-service/src/unstract/prompt_service_v2/core/retrievers/subquestion.pyrename to prompt-service/src/unstract/prompt_service/core/retrievers/subquestion.pyindex 4e5eefabe..4961d91af 100644--- a/prompt-service/src/unstract/prompt_service_v2/core/retrievers/subquestion.py+++ b/prompt-service/src/unstract/prompt_service/core/retrievers/subquestion.py@@ -1,9 +1,10 @@import loggingfrom llama_index.core.query_engine import SubQuestionQueryEngine+from llama_index.core.schema import QueryBundlefrom llama_index.core.tools import QueryEngineTool, ToolMetadata-from unstract.prompt_service_v2.core.retrievers.base_retriever import BaseRetriever-from unstract.prompt_service_v2.exceptions import RetrievalError+from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever+from unstract.prompt_service.exceptions import RetrievalErrorlogger = logging.getLogger(__name__)@@ -19,27 +20,32 @@ class SubquestionRetriever(BaseRetriever):set[str]: A set of text chunks retrieved from the database."""try:+ logger.info("Initialising vector query engine...")vector_query_engine = (- self.vector_db.get_vector_store_index().as_query_engine()+ self.vector_db.get_vector_store_index().as_query_engine(+ llm=self.llm, similarity_top_k=self.top_k+ )+ )+ logger.info(+ f"Retrieving chunks for {self.doc_id} using SubQuestionQueryEngine.")-query_engine_tools = [QueryEngineTool(query_engine=vector_query_engine,- metadata=ToolMetadata(name=self.doc_id),+ metadata=ToolMetadata(+ name=self.doc_id, description=f"Nodes for {self.doc_id}"+ ),),]+ query_bundle = QueryBundle(query_str=self.prompt)query_engine = SubQuestionQueryEngine.from_defaults(query_engine_tools=query_engine_tools,use_async=True,+ llm=self.llm,)- response = query_engine.query(- prompt=self.prompt,- top_k=self.top_k,- return_full_response=True,- )+ response = query_engine.query(str_or_query_bundle=query_bundle)chunks: set[str] = {node.text for node in response.source_nodes}logger.info(f"Successfully retrieved {len(chunks)} chunks.")diff --git a/prompt-service/src/unstract/prompt_service_v2/dto.py b/prompt-service/src/unstract/prompt_service/dto.pysimilarity index 100%rename from prompt-service/src/unstract/prompt_service_v2/dto.pyrename to prompt-service/src/unstract/prompt_service/dto.pydiff --git a/prompt-service/src/unstract/prompt_service/env_manager.py b/prompt-service/src/unstract/prompt_service/env_manager.pydeleted file mode 100644index c51f5cf80..000000000--- a/prompt-service/src/unstract/prompt_service/env_manager.py+++ /dev/null@@ -1,11 +0,0 @@-import os-from typing import Optional---class EnvLoader:- @staticmethod- def get_env_or_die(env_key: str, default: Optional[str] = None) -> str:- env_value = os.environ.get(env_key, default=default)- if env_value is None or env_value == "":- raise ValueError(f"Env variable {env_key} is required")- return env_valuediff --git a/prompt-service/src/unstract/prompt_service/exceptions.py b/prompt-service/src/unstract/prompt_service/exceptions.pyindex b7e898254..3df7d16ac 100644--- a/prompt-service/src/unstract/prompt_service/exceptions.py+++ b/prompt-service/src/unstract/prompt_service/exceptions.py@@ -46,7 +46,7 @@ class APIError(HTTPException):return str(self.message)-class NoPayloadError(APIError):+class BadRequest(APIError):code = 400message = "Bad Request / No payload"@@ -54,3 +54,20 @@ class NoPayloadError(APIError):class RateLimitError(APIError):code = 429message = "Running into rate limit errors, please try again later"+++class MissingFieldError(APIError):+ """Custom error for missing fields."""++ def __init__(self, missing_fields: list[str]):+ message = f"Missing required fields: {', '.join(missing_fields)}"+ super().__init__(message=message)+++class RetrievalError(APIError):+ """Custom exception raised for errors during retrieval from VectorDB."""++ DEFAULT_MESSAGE = (+ "Error while retrieving data from the VectorDB. "+ "Please contact the admin for further assistance."+ )diff --git a/prompt-service/src/unstract/prompt_service_v2/extensions.py b/prompt-service/src/unstract/prompt_service/extensions.pysimilarity index 100%rename from prompt-service/src/unstract/prompt_service_v2/extensions.pyrename to prompt-service/src/unstract/prompt_service/extensions.pydiff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.pydeleted file mode 100644index 00559efb9..000000000--- a/prompt-service/src/unstract/prompt_service/helper.py+++ /dev/null@@ -1,459 +0,0 @@-import importlib-import os-from logging import Logger-from pathlib import Path-from typing import Any, Optional--from dotenv import load_dotenv-from flask import Flask, current_app-from unstract.prompt_service.config import db-from unstract.prompt_service.constants import (- DBTableV2,- ExecutionSource,- FileStorageKeys,-)-from unstract.prompt_service.constants import PromptServiceContants as PSKeys-from unstract.prompt_service.db_utils import DBUtils-from unstract.prompt_service.env_manager import EnvLoader-from unstract.prompt_service.exceptions import APIError, RateLimitError-from unstract.sdk.exceptions import RateLimitError as SdkRateLimitError-from unstract.sdk.exceptions import SdkError-from unstract.sdk.file_storage import FileStorage, FileStorageProvider-from unstract.sdk.file_storage.constants import StorageType-from unstract.sdk.file_storage.env_helper import EnvHelper-from unstract.sdk.llm import LLM--PAID_FEATURE_MSG = (- "It is a cloud / enterprise feature. If you have purchased a plan and still "- "face this issue, please contact support"-)--load_dotenv()--# Global variable to store plugins-plugins: dict[str, dict[str, Any]] = {}---def plugin_loader(app: Flask) -> None:- """Loads plugins found in the plugins root dir.-- Each plugin:- * needs to be a package dir AND- * should contain a src/__init__.py which exports below:-- metadata: dict[str, Any] = {- "version": str,- "name": str,- "entrypoint_cls": class,- "exception_cls": class,- "disable": bool- }- """- global plugins- plugins_dir: Path = Path(os.path.dirname(__file__)) / "plugins"- plugins_pkg = "unstract.prompt_service.plugins"-- if not plugins_dir.exists():- app.logger.info(f"Plugins dir not found: {plugins_dir}. Skipping.")- return-- app.logger.info(f"Loading plugins from: {plugins_dir}")-- for pkg in os.listdir(os.fspath(plugins_dir)):- if pkg.endswith(".so"):- pkg = pkg.split(".")[0]- pkg_anchor = f"{plugins_pkg}.{pkg}.src"- try:- module = importlib.import_module(pkg_anchor)- except ImportError as e:- app.logger.error(f"Failed to load plugin ({pkg}): {str(e)}")- continue-- if not hasattr(module, "metadata"):- app.logger.warn(f"Plugin metadata not found: {pkg}")- continue-- metadata: dict[str, Any] = module.metadata- if metadata.get("disable", False):- app.logger.info(f"Ignore disabled plugin: {pkg} v{metadata['version']}")- continue-- try:- plugins[metadata["name"]] = {- "version": metadata["version"],- "entrypoint_cls": metadata["entrypoint_cls"],- "exception_cls": metadata["exception_cls"],- }- app.logger.info(f"Loaded plugin: {pkg} v{metadata['version']}")- except KeyError as e:- app.logger.error(f"Invalid metadata for plugin '{pkg}': {str(e)}")-- if not plugins:- app.logger.info("No plugins found.")-- initialize_plugin_endpoints(app=app)---def get_cleaned_context(context: set[str]) -> list[str]:- clean_context_plugin: dict[str, Any] = plugins.get(PSKeys.CLEAN_CONTEXT, {})- if clean_context_plugin:- return clean_context_plugin["entrypoint_cls"].run(context=context)- return list(context)---def initialize_plugin_endpoints(app: Flask) -> None:- """Enables plugins if available."""- single_pass_extration_plugin: dict[str, Any] = plugins.get(- PSKeys.SINGLE_PASS_EXTRACTION, {}- )- summarize_plugin: dict[str, Any] = plugins.get(PSKeys.SUMMARIZE, {})- simple_prompt_studio: dict[str, Any] = plugins.get(PSKeys.SIMPLE_PROMPT_STUDIO, {})- if single_pass_extration_plugin:- single_pass_extration_plugin["entrypoint_cls"](- app=app,- challenge_plugin=plugins.get(PSKeys.CHALLENGE, {}),- get_cleaned_context=get_cleaned_context,- highlight_data_plugin=plugins.get(PSKeys.HIGHLIGHT_DATA_PLUGIN, {}),- )- if summarize_plugin:- summarize_plugin["entrypoint_cls"](- app=app,- )- if simple_prompt_studio:- simple_prompt_studio["entrypoint_cls"](- app=app,- )---def query_usage_metadata(token: str, metadata: dict[str, Any]) -> dict[str, Any]:- DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract")- organization_uid, org_id = DBUtils.get_organization_from_bearer_token(token)- run_id: str = metadata["run_id"]- query: str = f"""- SELECT- usage_type,- llm_usage_reason,- model_name,- SUM(prompt_tokens) AS input_tokens,- SUM(completion_tokens) AS output_tokens,- SUM(total_tokens) AS total_tokens,- SUM(embedding_tokens) AS embedding_tokens,- SUM(cost_in_dollars) AS cost_in_dollars- FROM "{DB_SCHEMA}"."{DBTableV2.TOKEN_USAGE}"- WHERE run_id = %s and organization_id = %s- GROUP BY usage_type, llm_usage_reason, model_name;- """- logger: Logger = current_app.logger- try:- with db.atomic():- logger.info(- "Querying usage metadata for org_id: %s, run_id: %s", org_id, run_id- )- cursor = db.execute_sql(query, (run_id, organization_uid))- results: list[tuple] = cursor.fetchall()- # Process results as needed- for row in results:- key, item = _get_key_and_item(row)- # Initialize the key as an empty list if it doesn't exist- if key not in metadata:- metadata[key] = []- # Append the item to the list associated with the key- metadata[key].append(item)- except Exception as e:- logger.error(f"Error executing querying usage metadata: {e}")- return metadata---def _get_key_and_item(row: tuple) -> tuple[str, dict[str, Any]]:- (- usage_type,- llm_usage_reason,- model_name,- input_tokens,- output_tokens,- total_tokens,- embedding_tokens,- cost_in_dollars,- ) = row- cost_in_dollars: str = _format_float_positional(cost_in_dollars)- key: str = usage_type- item: dict[str, Any] = {- "model_name": model_name,- "cost_in_dollars": cost_in_dollars,- }- if llm_usage_reason:- key = f"{llm_usage_reason}_{key}"- item["input_tokens"] = input_tokens- item["output_tokens"] = output_tokens- item["total_tokens"] = total_tokens- else:- item["embedding_tokens"] = embedding_tokens- return key, item---def _format_float_positional(value: float, precision: int = 10) -> str:- formatted: str = f"{value:.{precision}f}"- return formatted.rstrip("0").rstrip(".") if "." in formatted else formatted---def extract_variable(- structured_output: dict[str, Any],- variable_names: list[Any],- output: dict[str, Any],- promptx: str,-) -> str:- logger: Logger = current_app.logger- for variable_name in variable_names:- if promptx.find(f"%{variable_name}%") >= 0:- if variable_name in structured_output:- promptx = promptx.replace(- f"%{variable_name}%",- str(structured_output[variable_name]),- )- else:- raise ValueError(- f"Variable {variable_name} not found " "in structured output"- )-- if promptx != output[PSKeys.PROMPT]:- logger.info(f"Prompt after variable replacement: {promptx}")- return promptx---def construct_and_run_prompt(- tool_settings: dict[str, Any],- output: dict[str, Any],- llm: LLM,- context: str,- prompt: str,- metadata: dict[str, Any],- file_path: str = "",- execution_source: Optional[str] = ExecutionSource.IDE.value,-) -> str:- platform_postamble = tool_settings.get(PSKeys.PLATFORM_POSTAMBLE, "")- summarize_as_source = tool_settings.get(PSKeys.SUMMARIZE_AS_SOURCE)- enable_highlight = tool_settings.get(PSKeys.ENABLE_HIGHLIGHT, False)- if not enable_highlight or summarize_as_source:- platform_postamble = ""- prompt = construct_prompt(- preamble=tool_settings.get(PSKeys.PREAMBLE, ""),- prompt=output[prompt],- postamble=tool_settings.get(PSKeys.POSTAMBLE, ""),- grammar_list=tool_settings.get(PSKeys.GRAMMAR, []),- context=context,- platform_postamble=platform_postamble,- )- return run_completion(- llm=llm,- prompt=prompt,- metadata=metadata,- prompt_key=output[PSKeys.NAME],- prompt_type=output.get(PSKeys.TYPE, PSKeys.TEXT),- enable_highlight=enable_highlight,- file_path=file_path,- execution_source=execution_source,- )---def construct_prompt(- preamble: str,- prompt: str,- postamble: str,- grammar_list: list[dict[str, Any]],- context: str,- platform_postamble: str,-) -> str:- prompt = f"{preamble}\n\nQuestion or Instruction: {prompt}"- if grammar_list is not None and len(grammar_list) > 0:- prompt += "\n"- for grammar in grammar_list:- word = ""- synonyms = []- if PSKeys.WORD in grammar:- word = grammar[PSKeys.WORD]- if PSKeys.SYNONYMS in grammar:- synonyms = grammar[PSKeys.SYNONYMS]- if len(synonyms) > 0 and word != "":- prompt += f'\nNote: You can consider that the word {word} is same as \- {", ".join(synonyms)} in both the quesiton and the context.' # noqa- if platform_postamble:- platform_postamble += "\n\n"- prompt += (- f"\n\n{postamble}\n\nContext:\n---------------\n{context}\n"- f"-----------------\n\n{platform_postamble}Answer:"- )- return prompt---def run_completion(- llm: LLM,- prompt: str,- metadata: Optional[dict[str, str]] = None,- prompt_key: Optional[str] = None,- prompt_type: Optional[str] = PSKeys.TEXT,- enable_highlight: bool = False,- file_path: str = "",- execution_source: Optional[str] = None,-) -> str:- logger: Logger = current_app.logger- try:- highlight_data_plugin: dict[str, Any] = plugins.get(- PSKeys.HIGHLIGHT_DATA_PLUGIN, {}- )- highlight_data = None- if highlight_data_plugin and enable_highlight:- fs_instance: FileStorage = FileStorage(FileStorageProvider.LOCAL)- if execution_source == ExecutionSource.IDE.value:- fs_instance = EnvHelper.get_storage(- storage_type=StorageType.PERMANENT,- env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,- )- if execution_source == ExecutionSource.TOOL.value:- fs_instance = EnvHelper.get_storage(- storage_type=StorageType.SHARED_TEMPORARY,- env_name=FileStorageKeys.TEMPORARY_REMOTE_STORAGE,- )- highlight_data = highlight_data_plugin["entrypoint_cls"](- logger=current_app.logger,- file_path=file_path,- fs_instance=fs_instance,- ).run-- completion = llm.complete(- prompt=prompt,- process_text=highlight_data,- extract_json=prompt_type.lower() != PSKeys.TEXT,- )- answer: str = completion[PSKeys.RESPONSE].text- highlight_data = completion.get(PSKeys.HIGHLIGHT_DATA, [])- confidence_data = completion.get(PSKeys.CONFIDENCE_DATA)- line_numbers = completion.get(PSKeys.LINE_NUMBERS, [])- whisper_hash = completion.get(PSKeys.WHISPER_HASH, "")- if metadata is not None and prompt_key:- metadata.setdefault(PSKeys.HIGHLIGHT_DATA, {})[prompt_key] = highlight_data- metadata.setdefault(PSKeys.LINE_NUMBERS, {})[prompt_key] = line_numbers- metadata[PSKeys.WHISPER_HASH] = whisper_hash- if confidence_data:- metadata.setdefault(PSKeys.CONFIDENCE_DATA, {})[- prompt_key- ] = confidence_data- return answer- # TODO: Catch and handle specific exception here- except SdkRateLimitError as e:- raise RateLimitError(f"Rate limit error. {str(e)}") from e- except SdkError as e:- logger.error(f"Error fetching response for prompt: {e}.")- # TODO: Publish this error as a FE update- raise APIError(str(e)) from e---def extract_table(- output: dict[str, Any],- plugins: dict[str, dict[str, Any]],- structured_output: dict[str, Any],- llm: LLM,- enforce_type: str,- execution_source: str,-) -> dict[str, Any]:- table_settings = output[PSKeys.TABLE_SETTINGS]- table_extractor: dict[str, Any] = plugins.get("table-extractor", {})- if not table_extractor:- raise APIError(- "Unable to extract table details. "- "Please contact admin to resolve this issue."- )- fs_instance: FileStorage = FileStorage(FileStorageProvider.LOCAL)- if execution_source == ExecutionSource.IDE.value:- fs_instance = EnvHelper.get_storage(- storage_type=StorageType.PERMANENT,- env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,- )- if execution_source == ExecutionSource.TOOL.value:- fs_instance = EnvHelper.get_storage(- storage_type=StorageType.TEMPORARY,- env_name=FileStorageKeys.TEMPORARY_REMOTE_STORAGE,- )- try:- answer = table_extractor["entrypoint_cls"].extract_large_table(- llm=llm,- table_settings=table_settings,- enforce_type=enforce_type,- fs_instance=fs_instance,- )- structured_output[output[PSKeys.NAME]] = answer- # We do not support summary and eval for table.- # Hence returning the result- return structured_output- except table_extractor["exception_cls"] as e:- msg = f"Couldn't extract table. {e}"- raise APIError(message=msg)---def extract_line_item(- tool_settings: dict[str, Any],- output: dict[str, Any],- plugins: dict[str, dict[str, Any]],- structured_output: dict[str, Any],- llm: LLM,- file_path: str,- metadata: Optional[dict[str, str]],- execution_source: str,-) -> dict[str, Any]:- line_item_extraction_plugin: dict[str, Any] = plugins.get(- "line-item-extraction", {}- )- if not line_item_extraction_plugin:- raise APIError(PAID_FEATURE_MSG)-- extract_file_path = file_path- if execution_source == ExecutionSource.IDE.value:- # Adjust file path to read from the extract folder- base_name = os.path.splitext(os.path.basename(file_path))[0]- extract_file_path = os.path.join(- os.path.dirname(file_path), "extract", f"{base_name}.txt"- )-- # Read file content into context- fs_instance: FileStorage = FileStorage(FileStorageProvider.LOCAL)- if execution_source == ExecutionSource.IDE.value:- fs_instance = EnvHelper.get_storage(- storage_type=StorageType.PERMANENT,- env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,- )- if execution_source == ExecutionSource.TOOL.value:- fs_instance = EnvHelper.get_storage(- storage_type=StorageType.SHARED_TEMPORARY,- env_name=FileStorageKeys.TEMPORARY_REMOTE_STORAGE,- )-- if not fs_instance.exists(extract_file_path):- raise FileNotFoundError(- f"The file at path '{extract_file_path}' does not exist."- )- context = fs_instance.read(path=extract_file_path, encoding="utf-8", mode="r")-- prompt = construct_prompt(- preamble=tool_settings.get(PSKeys.PREAMBLE, ""),- prompt=output["promptx"],- postamble=tool_settings.get(PSKeys.POSTAMBLE, ""),- grammar_list=tool_settings.get(PSKeys.GRAMMAR, []),- context=context,- platform_postamble="",- )-- try:- line_item_extraction = line_item_extraction_plugin["entrypoint_cls"](- llm=llm,- tool_settings=tool_settings,- output=output,- prompt=prompt,- structured_output=structured_output,- logger=current_app.logger,- )- answer = line_item_extraction.run()- structured_output[output[PSKeys.NAME]] = answer- metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = [context]- return structured_output- except line_item_extraction_plugin["exception_cls"] as e:- msg = f"Couldn't extract table. {e}"- raise APIError(message=msg)diff --git a/prompt-service/src/unstract/prompt_service_v2/__init__.py b/prompt-service/src/unstract/prompt_service/helpers/__init__.pysimilarity index 100%rename from prompt-service/src/unstract/prompt_service_v2/__init__.pyrename to prompt-service/src/unstract/prompt_service/helpers/__init__.pydiff --git a/prompt-service/src/unstract/prompt_service_v2/helpers/auth.py b/prompt-service/src/unstract/prompt_service/helpers/auth.pysimilarity index 92%rename from prompt-service/src/unstract/prompt_service_v2/helpers/auth.pyrename to prompt-service/src/unstract/prompt_service/helpers/auth.pyindex fdeca58c0..457a8f3f8 100644--- a/prompt-service/src/unstract/prompt_service_v2/helpers/auth.py+++ b/prompt-service/src/unstract/prompt_service/helpers/auth.py@@ -3,10 +3,10 @@ from typing import Any, Optionalfrom flask import Requestfrom flask import current_app as appfrom flask import request-from unstract.prompt_service_v2.constants import DBTableV2-from unstract.prompt_service_v2.extensions import db-from unstract.prompt_service_v2.utils.db_utils import DBUtils-from unstract.prompt_service_v2.utils.env_loader import get_env_or_die+from unstract.prompt_service.constants import DBTableV2+from unstract.prompt_service.extensions import db+from unstract.prompt_service.utils.db_utils import DBUtils+from unstract.prompt_service.utils.env_loader import get_env_or_dieDB_SCHEMA = get_env_or_die("DB_SCHEMA", "unstract")diff --git a/prompt-service/src/unstract/prompt_service_v2/helpers/lifecycle.py b/prompt-service/src/unstract/prompt_service/helpers/lifecycle.pysimilarity index 93%rename from prompt-service/src/unstract/prompt_service_v2/helpers/lifecycle.pyrename to prompt-service/src/unstract/prompt_service/helpers/lifecycle.pyindex 503f7a1a5..c1d2121d1 100644--- a/prompt-service/src/unstract/prompt_service_v2/helpers/lifecycle.py+++ b/prompt-service/src/unstract/prompt_service/helpers/lifecycle.py@@ -1,7 +1,7 @@from typing import Anyfrom flask import Flask, request-from unstract.prompt_service_v2.extensions import db+from unstract.prompt_service.extensions import dbdef register_lifecycle_hooks(app: Flask):diff --git a/prompt-service/src/unstract/prompt_service_v2/helpers/plugin.py b/prompt-service/src/unstract/prompt_service/helpers/plugin.pysimilarity index 97%rename from prompt-service/src/unstract/prompt_service_v2/helpers/plugin.pyrename to prompt-service/src/unstract/prompt_service/helpers/plugin.pyindex 365c31f00..6553cbda1 100644--- a/prompt-service/src/unstract/prompt_service_v2/helpers/plugin.py+++ b/prompt-service/src/unstract/prompt_service/helpers/plugin.py@@ -4,7 +4,7 @@ from pathlib import Pathfrom typing import Any, Optionalfrom flask import Flask-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys+from unstract.prompt_service.constants import PromptServiceConstants as PSKeysclass PluginManager:@@ -16,7 +16,7 @@ class PluginManager:cls._instance = super().__new__(cls)cls._instance.plugins = {}cls._instance.plugins_dir = Path(__file__).parent.parent / "plugins"- cls._instance.plugins_pkg = "unstract.prompt_service_v2.plugins"+ cls._instance.plugins_pkg = "unstract.prompt_service.plugins"# Always update `app` if providedif app:cls._instance.app = appdiff --git a/prompt-service/src/unstract/prompt_service_v2/helpers/prompt_ide_base_tool.py b/prompt-service/src/unstract/prompt_service/helpers/prompt_ide_base_tool.pysimilarity index 94%rename from prompt-service/src/unstract/prompt_service_v2/helpers/prompt_ide_base_tool.pyrename to prompt-service/src/unstract/prompt_service/helpers/prompt_ide_base_tool.pyindex 0650fde70..55040d044 100644--- a/prompt-service/src/unstract/prompt_service_v2/helpers/prompt_ide_base_tool.py+++ b/prompt-service/src/unstract/prompt_service/helpers/prompt_ide_base_tool.py@@ -3,14 +3,14 @@ import warningsfrom typing import Optionalfrom flask import current_app-from unstract.prompt_service_v2.constants import PromptServiceConstants+from unstract.prompt_service.constants import PromptServiceConstantsfrom unstract.sdk.constants import LogLevelfrom unstract.sdk.tool.stream import StreamMixinfrom unstract.sdk.utils.common_utils import PY_TO_UNSTRACT_LOG_LEVELclass PromptServiceBaseTool(StreamMixin):- # TODO: Remove unused log_level arg+ # Remove unused log_level argdef __init__(self, log_level: Optional[LogLevel] = None, platform_key: str = "") -> None:diff --git a/prompt-service/src/unstract/prompt_service/helpers/retrieval.py b/prompt-service/src/unstract/prompt_service/helpers/retrieval.pynew file mode 100644index 000000000..75d486285--- /dev/null+++ b/prompt-service/src/unstract/prompt_service/helpers/retrieval.py@@ -0,0 +1,45 @@+from typing import Any++from unstract.prompt_service.constants import PromptServiceConstants as PSKeys+from unstract.prompt_service.core.retrievers.simple import SimpleRetriever+from unstract.prompt_service.core.retrievers.subquestion import SubquestionRetriever+from unstract.prompt_service.utils.file_utils import FileUtils+from unstract.sdk.llm import LLM+from unstract.sdk.vector_db import VectorDB+++class RetrievalHelper:+ @staticmethod+ def run_retrieval( # type:ignore+ output: dict[str, Any],+ doc_id: str,+ llm: LLM,+ vector_db: VectorDB,+ retrieval_type: str,+ ) -> set[str]:+ context: set[str] = set()+ prompt = output[PSKeys.PROMPTX]+ top_k = output[PSKeys.SIMILARITY_TOP_K]+ if retrieval_type == PSKeys.SUBQUESTION:+ context = SubquestionRetriever(+ vector_db=vector_db, doc_id=doc_id, prompt=prompt, top_k=top_k, llm=llm+ ).retrieve()+ if retrieval_type == PSKeys.SIMPLE:+ context = SimpleRetriever(+ vector_db=vector_db, doc_id=doc_id, prompt=prompt, top_k=top_k+ ).retrieve()++ return context++ @staticmethod+ def retrieve_complete_context(execution_source: str, file_path: str) -> str:+ """+ Loads full context from raw file for zero chunk size retrieval+ Args:+ path (str): Path to the directory containing text file.++ Returns:+ str: context from extracted file.+ """+ fs_instance = FileUtils.get_fs_instance(execution_source=execution_source)+ return fs_instance.read(path=file_path, mode="r")diff --git a/prompt-service/src/unstract/prompt_service_v2/helpers/usage.py b/prompt-service/src/unstract/prompt_service/helpers/usage.pysimilarity index 92%rename from prompt-service/src/unstract/prompt_service_v2/helpers/usage.pyrename to prompt-service/src/unstract/prompt_service/helpers/usage.pyindex f1b2be7d3..097d01174 100644--- a/prompt-service/src/unstract/prompt_service_v2/helpers/usage.py+++ b/prompt-service/src/unstract/prompt_service/helpers/usage.py@@ -2,10 +2,10 @@ from logging import Loggerfrom typing import Anyfrom flask import current_app as app-from unstract.prompt_service_v2.constants import DBTableV2-from unstract.prompt_service_v2.extensions import db-from unstract.prompt_service_v2.utils.db_utils import DBUtils-from unstract.prompt_service_v2.utils.env_loader import get_env_or_die+from unstract.prompt_service.constants import DBTableV2+from unstract.prompt_service.extensions import db+from unstract.prompt_service.utils.db_utils import DBUtils+from unstract.prompt_service.utils.env_loader import get_env_or_dieclass UsageHelper:diff --git a/prompt-service/src/unstract/prompt_service_v2/helpers/variable_replacement.py b/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.pysimilarity index 96%rename from prompt-service/src/unstract/prompt_service_v2/helpers/variable_replacement.pyrename to prompt-service/src/unstract/prompt_service/helpers/variable_replacement.pyindex ec1632520..d5f4e02b1 100644--- a/prompt-service/src/unstract/prompt_service_v2/helpers/variable_replacement.py+++ b/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py@@ -4,8 +4,8 @@ from functools import lru_cachefrom typing import Anyfrom flask import current_app as app-from unstract.prompt_service_v2.constants import VariableConstants, VariableType-from unstract.prompt_service_v2.utils.request import HTTPMethod, make_http_request+from unstract.prompt_service.constants import VariableConstants, VariableType+from unstract.prompt_service.utils.request import HTTPMethod, make_http_requestclass VariableReplacementHelper:diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.pydeleted file mode 100644index e418e6f5a..000000000--- a/prompt-service/src/unstract/prompt_service/main.py+++ /dev/null@@ -1,966 +0,0 @@-import time-import traceback-from json import JSONDecodeError-from typing import Any, Optional--from flask import json, jsonify, request-from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters-from unstract.prompt_service.authentication_middleware import AuthenticationMiddleware-from unstract.prompt_service.config import create_app, db-from unstract.prompt_service.constants import PromptServiceContants as PSKeys-from unstract.prompt_service.constants import RunLevel-from unstract.prompt_service.exceptions import APIError, ErrorResponse, NoPayloadError-from unstract.prompt_service.helper import (- construct_and_run_prompt,- extract_line_item,- extract_table,- extract_variable,- get_cleaned_context,- plugin_loader,- plugins,- query_usage_metadata,- run_completion,-)-from unstract.prompt_service.prompt_ide_base_tool import PromptServiceBaseTool-from unstract.prompt_service.utils.log import publish_log-from unstract.prompt_service.variable_extractor.base import VariableExtractor-from unstract.sdk.adapters.llm.no_op.src.no_op_custom_llm import NoOpCustomLLM-from unstract.sdk.constants import LogLevel-from unstract.sdk.embedding import Embedding-from unstract.sdk.exceptions import SdkError-from unstract.sdk.index import Index-from unstract.sdk.llm import LLM-from unstract.sdk.vector_db import VectorDB-from werkzeug.exceptions import HTTPException--USE_UNSTRACT_PROMPT = True-MAX_RETRIES = 3--NO_CONTEXT_ERROR = (- "Couldn't fetch context from vector DB. "- "This happens usually due to a delay by the Vector DB "- "provider to confirm writes to DB. "- "Please try again after some time"-)--app = create_app()-# Load plugins-plugin_loader(app)--[email protected]_request-def before_request() -> None:- if db.is_closed():- db.connect(reuse_if_open=True)--[email protected]_request-def after_request(exception: Any) -> None:- # Close the connection after each request- if not db.is_closed():- db.close()--[email protected]_request-def log_request_info():- app.logger.info(f"Request Path: {request.path} | Method: {request.method}")--[email protected]_request-def log_response_info(response):- app.logger.info(f"Response Status: {response.status}")- return response---def authentication_middleware(func: Any) -> Any:- def wrapper(*args: Any, **kwargs: Any) -> Any:- token = AuthenticationMiddleware.get_token_from_auth_header(request)- # Check if bearer token exists and validate it- if not token or not AuthenticationMiddleware.validate_bearer_token(token):- return "Unauthorized", 401-- return func(*args, **kwargs)-- return wrapper--[email protected]("/health", methods=["GET"], endpoint="health_check")-def health_check() -> str:- return "OK"--- "/answer-prompt",- endpoint="answer_prompt",- methods=["POST"],-)-@authentication_middleware-def prompt_processor() -> Any:- platform_key = AuthenticationMiddleware.get_token_from_auth_header(request)- payload: dict[Any, Any] = request.json- if not payload:- raise NoPayloadError- tool_settings = payload.get(PSKeys.TOOL_SETTINGS, {})- enable_challenge = tool_settings.get(PSKeys.ENABLE_CHALLENGE, False)- challenge_llm = None- # TODO: Rename "outputs" to "prompts" in payload- prompts = payload.get(PSKeys.OUTPUTS, [])- tool_id: str = payload.get(PSKeys.TOOL_ID, "")- run_id: str = payload.get(PSKeys.RUN_ID, "")- execution_id: str = payload.get(PSKeys.EXECUTION_ID, "")- file_hash = payload.get(PSKeys.FILE_HASH)- file_path = payload.get(PSKeys.FILE_PATH)- doc_name = str(payload.get(PSKeys.FILE_NAME, ""))- log_events_id: str = payload.get(PSKeys.LOG_EVENTS_ID, "")- structured_output: dict[str, Any] = {}- metadata: dict[str, Any] = {- PSKeys.RUN_ID: run_id,- PSKeys.FILE_NAME: doc_name,- PSKeys.CONTEXT: {},- PSKeys.REQUIRED_FIELDS: {},- }- metrics: dict = {}- variable_names: list[str] = []- # Identifier for source of invocation- execution_source = payload.get(PSKeys.EXECUTION_SOURCE, "")- publish_log(- log_events_id,- {"tool_id": tool_id, "run_id": run_id, "doc_name": doc_name},- LogLevel.DEBUG,- RunLevel.RUN,- f"Preparing to execute {len(prompts)} prompt(s)",- )- # TODO: Rename "output" to "prompt"- for output in prompts: # type:ignore- variable_names.append(output[PSKeys.NAME])- metadata[PSKeys.REQUIRED_FIELDS][output[PSKeys.NAME]] = output.get(- PSKeys.REQUIRED, None- )-- for output in prompts: # type:ignore- prompt_name = output[PSKeys.NAME]- prompt_text = output[PSKeys.PROMPT]- chunk_size = output[PSKeys.CHUNK_SIZE]- util = PromptServiceBaseTool(platform_key=platform_key)- index = Index(tool=util, run_id=run_id, capture_metrics=True)- if VariableExtractor.is_variables_present(prompt_text=prompt_text):- prompt_text = VariableExtractor.replace_variables_in_prompt(- prompt=output,- structured_output=structured_output,- log_events_id=log_events_id,- tool_id=tool_id,- prompt_name=prompt_name,- doc_name=doc_name,- )-- app.logger.info(f"[{tool_id}] Executing prompt: {prompt_name}")- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.DEBUG,- RunLevel.RUN,- "Executing prompt",- )-- # Finding and replacing the variables in the prompt- # The variables are in the form %variable_name%-- output[PSKeys.PROMPTX] = extract_variable(- structured_output, variable_names, output, prompt_text- )-- doc_id = index.generate_index_key(- file_hash=file_hash,- vector_db=output[PSKeys.VECTOR_DB],- embedding=output[PSKeys.EMBEDDING],- x2text=output[PSKeys.X2TEXT_ADAPTER],- chunk_size=str(output[PSKeys.CHUNK_SIZE]),- chunk_overlap=str(output[PSKeys.CHUNK_OVERLAP]),- )- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.DEBUG,- RunLevel.RUN,- "Retrieved document ID",- )-- try:- usage_kwargs = {"run_id": run_id, "execution_id": execution_id}- adapter_instance_id = output[PSKeys.LLM]- llm = LLM(- tool=util,- adapter_instance_id=adapter_instance_id,- usage_kwargs={- **usage_kwargs,- PSKeys.LLM_USAGE_REASON: PSKeys.EXTRACTION,- },- capture_metrics=True,- )-- embedding = Embedding(- tool=util,- adapter_instance_id=output[PSKeys.EMBEDDING],- usage_kwargs=usage_kwargs.copy(),- )-- vector_db = VectorDB(- tool=util,- adapter_instance_id=output[PSKeys.VECTOR_DB],- embedding=embedding,- )- except SdkError as e:- msg = f"Couldn't fetch adapter. {e}"- app.logger.error(msg)- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.ERROR,- RunLevel.RUN,- "Unable to obtain LLM / embedding / vectorDB",- )- return APIError(message=msg)-- if output[PSKeys.TYPE] == PSKeys.TABLE or output[PSKeys.TYPE] == PSKeys.RECORD:- try:- structured_output = extract_table(- output=output,- plugins=plugins,- structured_output=structured_output,- llm=llm,- enforce_type=output[PSKeys.TYPE],- execution_source=execution_source,- )- metadata = query_usage_metadata(token=platform_key, metadata=metadata)- response = {- PSKeys.METADATA: metadata,- PSKeys.OUTPUT: structured_output,- PSKeys.METRICS: metrics,- }- return response- except APIError as api_error:- app.logger.error(- "Failed to extract table for the prompt %s: %s",- output[PSKeys.NAME],- str(api_error),- )- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.ERROR,- RunLevel.TABLE_EXTRACTION,- "Error while extracting table for the prompt",- )- raise api_error- elif output[PSKeys.TYPE] == PSKeys.LINE_ITEM:- try:- structured_output = extract_line_item(- tool_settings=tool_settings,- output=output,- plugins=plugins,- structured_output=structured_output,- llm=llm,- file_path=file_path,- metadata=metadata,- execution_source=execution_source,- )- continue- except APIError as e:- app.logger.error(- "Failed to extract line-item for the prompt %s: %s",- output[PSKeys.NAME],- str(e),- )- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.ERROR,- RunLevel.RUN,- "Error while extracting line-item for the prompt",- )- raise e-- try:- if chunk_size == 0:- # We can do this only for chunkless indexes- context: set[str] = fetch_context_from_vector_db(- index=index,- output=output,- doc_id=doc_id,- tool_id=tool_id,- doc_name=doc_name,- prompt_name=prompt_name,- log_events_id=log_events_id,- usage_kwargs=usage_kwargs,- )- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.INFO,- RunLevel.RUN,- "Retrieving answer from LLM",- )- answer = construct_and_run_prompt(- tool_settings=tool_settings,- output=output,- llm=llm,- context="\n".join(context),- prompt="promptx",- metadata=metadata,- file_path=file_path,- execution_source=execution_source,- )- metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = get_cleaned_context(- context- )- else:- answer = "NA"- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.INFO,- RunLevel.RUN,- "Retrieving context from adapter",- )-- retrieval_strategy = output.get(PSKeys.RETRIEVAL_STRATEGY)-- if retrieval_strategy in {PSKeys.SIMPLE, PSKeys.SUBQUESTION}:- vector_index = vector_db.get_vector_store_index()- answer, context = run_retrieval(- tool_settings=tool_settings,- output=output,- doc_id=doc_id,- llm=llm,- vector_index=vector_index,- retrieval_type=retrieval_strategy,- metadata=metadata,- execution_source=execution_source,- )- metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = get_cleaned_context(- context- )- else:- app.logger.info(- "Invalid retrieval strategy passed: %s",- retrieval_strategy,- )-- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.DEBUG,- RunLevel.RUN,- "Retrieved context from adapter",- )-- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.INFO,- RunLevel.RUN,- f"Processing prompt type: {output[PSKeys.TYPE]}",- )-- if output[PSKeys.TYPE] == PSKeys.NUMBER:- if answer.lower() == "na":- structured_output[output[PSKeys.NAME]] = None- else:- # TODO: Extract these prompts as constants after pkging- prompt = f"Extract the number from the following \- text:\n{answer}\n\nOutput just the number. \- If the number is expressed in millions \- or thousands, expand the number to its numeric value \- The number should be directly assignable\- to a numeric variable.\- It should not have any commas, \- percentages or other grouping \- characters. No explanation is required.\- If you cannot extract the number, output 0."- answer = run_completion(- llm=llm,- prompt=prompt,- )- try:- structured_output[output[PSKeys.NAME]] = float(answer)- except Exception as e:- app.logger.info(- f"Error parsing response (to numeric, float): {e}",- LogLevel.ERROR,- )- structured_output[output[PSKeys.NAME]] = None- # No-op adapter always returns a string data and- # to keep this response uniform- # through all enforce types- # we add this check, if not for this,- # type casting to float raises- # an error and we return None.- if isinstance(- llm.get_llm(adapter_instance_id=adapter_instance_id),- NoOpCustomLLM,- ):- structured_output[output[PSKeys.NAME]] = answer- elif output[PSKeys.TYPE] == PSKeys.EMAIL:- if answer.lower() == "na":- structured_output[output[PSKeys.NAME]] = None- else:- prompt = f'Extract the email from the following text:\n{answer}\n\nOutput just the email. \- The email should be directly assignable to a string variable. \- No explanation is required. If you cannot extract the email, output "NA".' # noqa- answer = run_completion(- llm=llm,- prompt=prompt,- )- structured_output[output[PSKeys.NAME]] = answer- elif output[PSKeys.TYPE] == PSKeys.DATE:- if answer.lower() == "na":- structured_output[output[PSKeys.NAME]] = None- else:- prompt = f'Extract the date from the following text:\n{answer}\n\nOutput just the date.\- The date should be in ISO date time format. No explanation is required. \- The date should be directly assignable to a date variable. \- If you cannot convert the string into a date, output "NA".' # noqa- answer = run_completion(- llm=llm,- prompt=prompt,- )- structured_output[output[PSKeys.NAME]] = answer-- elif output[PSKeys.TYPE] == PSKeys.BOOLEAN:- if answer.lower() == "na":- structured_output[output[PSKeys.NAME]] = None- else:- prompt = f'Extract yes/no from the following text:\n{answer}\n\n\- Output in single word.\- If the context is trying to convey that the answer is true, \- then return "yes", else return "no".'- answer = run_completion(- llm=llm,- prompt=prompt,- )- if answer.lower() == "yes":- structured_output[output[PSKeys.NAME]] = True- else:- structured_output[output[PSKeys.NAME]] = False- elif output[PSKeys.TYPE] == PSKeys.JSON:- if answer.lower() == "[]" or answer.lower() == "na":- structured_output[output[PSKeys.NAME]] = None- else:- try:- structured_output[output[PSKeys.NAME]] = json.loads(answer)- except JSONDecodeError:- prompt = f"Convert the following text into valid JSON string: \- \n{answer}\n\n The JSON string should be able to be parsed \- into a Python dictionary. \- Output just the JSON string. No explanation is required. \- If you cannot extract the JSON string, output {{}}"- try:- answer = run_completion(- llm=llm,- prompt=prompt,- prompt_type=PSKeys.JSON,- )- structured_output[output[PSKeys.NAME]] = json.loads(answer)- except JSONDecodeError as e:- err_msg = (- f"Error parsing response (to json): {e}\n"- f"Candidate JSON: {answer}"- )- app.logger.info(err_msg, LogLevel.ERROR)- # TODO: Format log message after unifying these types- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.INFO,- RunLevel.RUN,- "Unable to parse JSON response from LLM, try using our"- " cloud / enterprise feature of 'line-item', "- "'record' or 'table' type",- )- structured_output[output[PSKeys.NAME]] = {}-- else:- structured_output[output[PSKeys.NAME]] = answer-- # If there is a trailing '\n' remove it- if isinstance(structured_output[output[PSKeys.NAME]], str):- structured_output[output[PSKeys.NAME]] = structured_output[- output[PSKeys.NAME]- ].rstrip("\n")-- # Challenge condition- if enable_challenge:- challenge_plugin: dict[str, Any] = plugins.get(PSKeys.CHALLENGE, {})- try:- if challenge_plugin:- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.INFO,- RunLevel.CHALLENGE,- "Challenging response",- )- challenge_llm = LLM(- tool=util,- adapter_instance_id=tool_settings[PSKeys.CHALLENGE_LLM],- usage_kwargs={- **usage_kwargs,- PSKeys.LLM_USAGE_REASON: PSKeys.CHALLENGE,- },- capture_metrics=True,- )- challenge = challenge_plugin["entrypoint_cls"](- llm=llm,- challenge_llm=challenge_llm,- run_id=run_id,- context="\n".join(context),- tool_settings=tool_settings,- output=output,- structured_output=structured_output,- logger=app.logger,- platform_key=platform_key,- metadata=metadata,- )- # Will inline replace the structured output passed.- challenge.run()- else:- app.logger.info(- "No challenge plugin found to evaluate prompt: %s",- output[PSKeys.NAME],- )- except challenge_plugin["exception_cls"] as e:- app.logger.error(- "Failed to challenge prompt %s: %s",- output[PSKeys.NAME],- str(e),- )- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.ERROR,- RunLevel.CHALLENGE,- "Error while challenging response",- )-- #- # Evaluate the prompt.- #- if (- PSKeys.EVAL_SETTINGS in output- and output[PSKeys.EVAL_SETTINGS][PSKeys.EVAL_SETTINGS_EVALUATE]- ):- eval_plugin: dict[str, Any] = plugins.get("evaluation", {})- if eval_plugin:- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.INFO,- RunLevel.EVAL,- "Evaluating response",- )- try:- evaluator = eval_plugin["entrypoint_cls"](- "",- "\n".join(context),- "",- "",- output,- structured_output,- app.logger,- platform_key,- )- # Will inline replace the structured output passed.- evaluator.run()- except eval_plugin["exception_cls"] as e:- app.logger.error(- f"Failed to evaluate prompt {output[PSKeys.NAME]}: {str(e)}"- )- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.ERROR,- RunLevel.EVAL,- "Error while evaluation",- )- else:- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.DEBUG,- RunLevel.EVAL,- "Evaluation completed",- )- else:- app.logger.info(- f"No eval plugin found to evaluate prompt: {output[PSKeys.NAME]}" # noqa: E501- )- finally:- challenge_metrics = (- {f"{challenge_llm.get_usage_reason()}_llm": challenge_llm.get_metrics()}- if enable_challenge and challenge_llm- else {}- )- metrics.setdefault(prompt_name, {}).update(- {- "context_retrieval": index.get_metrics(),- f"{llm.get_usage_reason()}_llm": llm.get_metrics(),- **challenge_metrics,- }- )- vector_db.close()- publish_log(- log_events_id,- {"tool_id": tool_id, "doc_name": doc_name},- LogLevel.INFO,- RunLevel.RUN,- "Sanitizing null values",- )- for k, v in structured_output.items():- if isinstance(v, str) and v.lower() == "na":- structured_output[k] = None- elif isinstance(v, list):- for i in range(len(v)):- if isinstance(v[i], str) and v[i].lower() == "na":- v[i] = None- elif isinstance(v[i], dict):- for k1, v1 in v[i].items():- if isinstance(v1, str) and v1.lower() == "na":- v[i][k1] = None- elif isinstance(v, dict):- for k1, v1 in v.items():- if isinstance(v1, str) and v1.lower() == "na":- v[k1] = None-- publish_log(- log_events_id,- {"tool_id": tool_id, "doc_name": doc_name},- LogLevel.INFO,- RunLevel.RUN,- "Execution complete",- )- metadata = query_usage_metadata(token=platform_key, metadata=metadata)- response = {- PSKeys.METADATA: metadata,- PSKeys.OUTPUT: structured_output,- PSKeys.METRICS: metrics,- }- return response---def fetch_context_from_vector_db(- index: Index,- output: dict[str, Any],- doc_id: str,- tool_id: str,- doc_name: str,- prompt_name: str,- log_events_id: str,- usage_kwargs: dict[str, Any],-) -> set[str]:- """- Fetches context from the index for the given document ID. Implements a retry- mechanism with logging and raises an error if context retrieval fails.-- Args:- index: The index object to query.- output: Dictionary containing keys like embedding and vector DB instance ID.- doc_id: The document ID to query.- tool_id: Identifier for the tool in use.- doc_name: Name of the document being queried.- prompt_name: Name of the prompt being executed.- log_events_id: Unique ID for logging events.- usage_kwargs: Additional usage parameters.-- Raises:- APIError: If context retrieval fails after retrying.- """- context: set[str] = set()- try:- retrieved_context = index.query_index(- embedding_instance_id=output[PSKeys.EMBEDDING],- vector_db_instance_id=output[PSKeys.VECTOR_DB],- doc_id=doc_id,- usage_kwargs=usage_kwargs,- )-- if retrieved_context:- context.add(retrieved_context)- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.DEBUG,- RunLevel.RUN,- "Fetched context from vector DB",- )- else:- # Handle lag in vector DB write (e.g., Pinecone issue)- time.sleep(2)- retrieved_context = index.query_index(- embedding_instance_id=output[PSKeys.EMBEDDING],- vector_db_instance_id=output[PSKeys.VECTOR_DB],- doc_id=doc_id,- usage_kwargs=usage_kwargs,- )-- if retrieved_context is None:- msg = NO_CONTEXT_ERROR- app.logger.error(- f"{msg} {output[PSKeys.VECTOR_DB]} for doc_id {doc_id}"- )- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.ERROR,- RunLevel.RUN,- msg,- )- raise APIError(message=msg)- context.add(retrieved_context)- except SdkError as e:- msg = f"Unable to fetch context from vector DB. {str(e)}"- app.logger.error(- f"{msg}. VectorDB: {output[PSKeys.VECTOR_DB]}, doc_id: {doc_id}"- )- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.ERROR,- RunLevel.RUN,- msg,- )- raise APIError(message=msg, code=e.status_code)- return context---def run_retrieval( # type:ignore- tool_settings: dict[str, Any],- output: dict[str, Any],- doc_id: str,- llm: LLM,- vector_index,- retrieval_type: str,- metadata: dict[str, Any],- execution_source: Optional[str] = None,-) -> tuple[str, set[str]]:- context: set[str] = set()- prompt = output[PSKeys.PROMPTX]- if retrieval_type == PSKeys.SUBQUESTION:- subq_prompt: str = (- f"I am sending you a verbose prompt \n \n Prompt : {prompt} \n \n"- "Generate set of specific subquestions "- "from the prompt which can be used to retrive "- "relevant context from vector db. "- "Use your logical abilities to "- " only generate as many subquestions as necessary "- " — fewer subquestions if the prompt is simpler. "- "Decide the minimum limit for subquestions "- "based on the complexity input prompt and set the maximum limit"- "for the subquestions to 10."- "Ensure that each subquestion is distinct and relevant"- "to the the original query. "- "Do not add subquestions for details"- "not mentioned in the original prompt."- " The goal is to maximize retrieval accuracy"- " using these subquestions. Use your logical abilities to ensure "- " that each subquestion targets a distinct aspect of the original query."- " Please note that, there are cases where the "- "response might have a list of answers. The subquestions must not miss out "- "any values in these cases. "- "Output should be a list of comma seperated "- "subquestion prompts. Do not change this format. \n \n "- " Subquestions : "- )- subquestions = run_completion(- llm=llm,- prompt=subq_prompt,- )- subquestion_list = subquestions.split(",")- for each_subq in subquestion_list:- retrieved_context = _retrieve_context(- output, doc_id, vector_index, each_subq- )- context.update(retrieved_context)-- if retrieval_type == PSKeys.SIMPLE:-- context = _retrieve_context(output, doc_id, vector_index, prompt)-- if not context:- # UN-1288 For Pinecone, we are seeing an inconsistent case where- # query with doc_id fails even though indexing just happened.- # This causes the following retrieve to return no text.- # To rule out any lag on the Pinecone vector DB write,- # the following sleep is added- # Note: This will not fix the issue. Since this issue is inconsistent- # and not reproducible easily, this is just a safety net.- time.sleep(2)- context = _retrieve_context(output, doc_id, vector_index, prompt)-- answer = construct_and_run_prompt( # type:ignore- tool_settings=tool_settings,- output=output,- llm=llm,- context="\n".join(context),- prompt="promptx",- metadata=metadata,- execution_source=execution_source,- )-- return (answer, context)---def _retrieve_context(output, doc_id, vector_index, answer) -> set[str]:- retriever = vector_index.as_retriever(- similarity_top_k=output[PSKeys.SIMILARITY_TOP_K],- filters=MetadataFilters(- filters=[- ExactMatchFilter(key="doc_id", value=doc_id),- # TODO: Enable after adding section in GUI- # ExactMatchFilter(- # key="section", value=output["section"]- ],- ),- )- nodes = retriever.retrieve(answer)- context: set[str] = set()- for node in nodes:- # ToDo: May have to fine-tune this value for node score or keep it- # configurable at the adapter level- if node.score > 0:- context.add(node.get_content())- else:- app.logger.info(- "Node score is less than 0. "- f"Ignored: {node.node_id} with score {node.score}"- )- return context---def log_exceptions(e: HTTPException):- """Helper method to log exceptions.-- Args:- e (HTTPException): Exception to log- """- code = 500- if hasattr(e, "code"):- code = e.code or code-- if code >= 500:- message = "{method} {url} {status}\n\n{error}\n\n````{tb}````".format(- method=request.method,- url=request.url,- status=code,- error=str(e),- tb=traceback.format_exc(),- )- else:- message = "{method} {url} {status} {error}".format(- method=request.method,- url=request.url,- status=code,- error=str(e),- )- app.logger.error(message)--[email protected](HTTPException)-def handle_http_exception(e: HTTPException):- """Return JSON instead of HTML for HTTP errors."""- log_exceptions(e)- if isinstance(e, APIError):- return jsonify(e.to_dict()), e.code- else:- response = e.get_response()- response.data = json.dumps(- ErrorResponse(error=e.description, name=e.name, code=e.code)- )- response.content_type = "application/json"- return response--[email protected](Exception)-def handle_uncaught_exception(e):- """Handler for uncaught exceptions.-- Args:- e (Exception): Any uncaught exception- """- # pass through HTTP errors- if isinstance(e, HTTPException):- return handle_http_exception(e)-- log_exceptions(e)- return handle_http_exception(APIError())diff --git a/prompt-service/src/unstract/prompt_service/prompt_ide_base_tool.py b/prompt-service/src/unstract/prompt_service/prompt_ide_base_tool.pydeleted file mode 100644index 42a65c94e..000000000--- a/prompt-service/src/unstract/prompt_service/prompt_ide_base_tool.py+++ /dev/null@@ -1,60 +0,0 @@-import os-import warnings-from typing import Optional--from flask import current_app-from unstract.prompt_service.constants import PromptServiceContants-from unstract.sdk.constants import LogLevel-from unstract.sdk.tool.stream import StreamMixin-from unstract.sdk.utils.common_utils import PY_TO_UNSTRACT_LOG_LEVEL---class PromptServiceBaseTool(StreamMixin):- # TODO: Remove unused log_level arg- def __init__(- self, log_level: Optional[LogLevel] = None, platform_key: str = ""- ) -> None:- """- Args:- tool (UnstractAbstractTool): Instance of UnstractAbstractTool.- Deprecated, will be removed in future versions- Notes:- - PLATFORM_SERVICE_API_KEY environment variable is required.- """- if log_level:- warnings.warn(- "The 'log_level' argument is deprecated and will be "- "removed in a future version.",- DeprecationWarning,- stacklevel=2, # ensures the warning points to the caller of the method- )- self.log_level = PY_TO_UNSTRACT_LOG_LEVEL.get(- current_app.logger.getEffectiveLevel(), LogLevel.INFO- )- self.platform_key = platform_key- super().__init__(log_level=self.log_level)-- def get_env_or_die(self, env_key: str) -> str:- """Returns the value of an env variable.-- If its empty or None, raises an error and exits-- Args:- env_key (str): Key to retrieve-- Returns:- str: Value of the env- """- # HACK: Adding platform key for multitenancy- if env_key == PromptServiceContants.PLATFORM_SERVICE_API_KEY:- if not self.platform_key:- current_app.logger.error(f"{env_key} is required")- else:- key: str = self.platform_key- return key- else:- env_value = os.environ.get(env_key)- if env_value is None or env_value == "":- current_app.logger.error(f"Env variable {env_key} is required")- return env_value # type:ignore- return ""diff --git a/prompt-service/src/unstract/prompt_service/run.py b/prompt-service/src/unstract/prompt_service/run.pynew file mode 100644index 000000000..b50e97250--- /dev/null+++ b/prompt-service/src/unstract/prompt_service/run.py@@ -0,0 +1,3 @@+from unstract.prompt_service.config import create_app++app = create_app()diff --git a/prompt-service/src/unstract/prompt_service_v2/helpers/__init__.py b/prompt-service/src/unstract/prompt_service/services/__init__.pysimilarity index 100%rename from prompt-service/src/unstract/prompt_service_v2/helpers/__init__.pyrename to prompt-service/src/unstract/prompt_service/services/__init__.pydiff --git a/prompt-service/src/unstract/prompt_service_v2/services/answer_prompt.py b/prompt-service/src/unstract/prompt_service/services/answer_prompt.pysimilarity index 95%rename from prompt-service/src/unstract/prompt_service_v2/services/answer_prompt.pyrename to prompt-service/src/unstract/prompt_service/services/answer_prompt.pyindex ebd9534fd..a4b703e3e 100644--- a/prompt-service/src/unstract/prompt_service_v2/services/answer_prompt.py+++ b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py@@ -3,10 +3,10 @@ from logging import Loggerfrom typing import Any, Optionalfrom flask import current_app as app+from unstract.prompt_service.constants import ExecutionSource, FileStorageKeys+from unstract.prompt_service.constants import PromptServiceConstants as PSKeysfrom unstract.prompt_service.exceptions import RateLimitError-from unstract.prompt_service_v2.constants import ExecutionSource, FileStorageKeys-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys-from unstract.prompt_service_v2.helpers.plugin import PluginManager+from unstract.prompt_service.helpers.plugin import PluginManagerfrom unstract.sdk.exceptions import RateLimitError as SdkRateLimitErrorfrom unstract.sdk.exceptions import SdkErrorfrom unstract.sdk.file_storage import FileStorage, FileStorageProvider@@ -19,15 +19,6 @@ from unstract.core.flask.exceptions import APIErrorclass AnswerPromptService:- @staticmethod- def get_cleaned_context(context: set[str]) -> list[str]:- clean_context_plugin: dict[str, Any] = PluginManager().get_plugin(- PSKeys.CLEAN_CONTEXT- )- if clean_context_plugin:- return clean_context_plugin["entrypoint_cls"].run(context=context)- return list(context)-@staticmethoddef extract_variable(structured_output: dict[str, Any],diff --git a/prompt-service/src/unstract/prompt_service_v2/services/extraction.py b/prompt-service/src/unstract/prompt_service/services/extraction.pysimilarity index 85%rename from prompt-service/src/unstract/prompt_service_v2/services/extraction.pyrename to prompt-service/src/unstract/prompt_service/services/extraction.pyindex b43894bb6..866e325f8 100644--- a/prompt-service/src/unstract/prompt_service_v2/services/extraction.py+++ b/prompt-service/src/unstract/prompt_service/services/extraction.py@@ -1,16 +1,14 @@from typing import Any, Optional-from unstract.prompt_service_v2.constants import ExecutionSource-from unstract.prompt_service_v2.constants import IndexingConstants as IKeys-from unstract.prompt_service_v2.helpers.prompt_ide_base_tool import (- PromptServiceBaseTool,-)+from unstract.prompt_service.constants import ExecutionSource+from unstract.prompt_service.constants import IndexingConstants as IKeys+from unstract.prompt_service.helpers.prompt_ide_base_tool import PromptServiceBaseTool+from unstract.prompt_service.utils.file_utils import FileUtilsfrom unstract.sdk.adapters.exceptions import AdapterErrorfrom unstract.sdk.adapters.x2text.constants import X2TextConstantsfrom unstract.sdk.adapters.x2text.llm_whisperer.src import LLMWhispererfrom unstract.sdk.adapters.x2text.llm_whisperer_v2.src import LLMWhispererV2from unstract.sdk.exceptions import X2TextError-from unstract.sdk.file_storage import FileStorage, FileStorageProviderfrom unstract.sdk.utils import ToolUtilsfrom unstract.sdk.utils.common_utils import log_elapsedfrom unstract.sdk.x2txt import TextExtractionResult, X2Text@@ -28,7 +26,6 @@ class ExtractionService:output_file_path: Optional[str] = None,enable_highlight: bool = False,usage_kwargs: dict[Any, Any] = {},- fs: FileStorage = FileStorage(FileStorageProvider.LOCAL),tags: Optional[list[str]] = None,execution_source: Optional[str] = None,tool_exec_metadata: Optional[dict[str, Any]] = None,@@ -40,6 +37,7 @@ class ExtractionService:x2text = X2Text(tool=util, adapter_instance_id=x2text_instance_id, usage_kwargs=usage_kwargs)+ fs = FileUtils.get_fs_instance(execution_source=execution_source)try:if enable_highlight and (isinstance(x2text.x2text_instance, LLMWhisperer)@@ -92,8 +90,3 @@ class ExtractionService:json_to_dump=metadata,fs=fs,)-- # TODO : Handle runId- # TODO: Revisit passage of tool through SDK- # TODO : Revisit usgae of update_exec_metadata- # TODO : Process Text callablediff --git a/prompt-service/src/unstract/prompt_service_v2/services/indexing.py b/prompt-service/src/unstract/prompt_service/services/indexing.pysimilarity index 85%rename from prompt-service/src/unstract/prompt_service_v2/services/indexing.pyrename to prompt-service/src/unstract/prompt_service/services/indexing.pyindex 09361887b..9205294c8 100644--- a/prompt-service/src/unstract/prompt_service_v2/services/indexing.py+++ b/prompt-service/src/unstract/prompt_service/services/indexing.py@@ -1,17 +1,16 @@import logging-from unstract.prompt_service_v2.core.index_v2 import Index-from unstract.prompt_service_v2.dto import (+from flask import current_app as app+from unstract.prompt_service.core.index_v2 import Index+from unstract.prompt_service.dto import (ChunkingConfig,FileInfo,InstanceIdentifiers,ProcessingOptions,)-from unstract.prompt_service_v2.exceptions import APIError-from unstract.prompt_service_v2.helpers.prompt_ide_base_tool import (- PromptServiceBaseTool,-)-from unstract.prompt_service_v2.utils.file_utils import FileUtils+from unstract.prompt_service.exceptions import APIError+from unstract.prompt_service.helpers.prompt_ide_base_tool import PromptServiceBaseTool+from unstract.prompt_service.utils.file_utils import FileUtilsfrom unstract.sdk.embedding import Embeddingfrom unstract.sdk.utils.indexing_utils import IndexingUtilsfrom unstract.sdk.vector_db import VectorDB@@ -72,6 +71,7 @@ class IndexingService:vector_db=vector_db,)if doc_id_found:+ app.logger.info("Doc ID found: %s", doc_id_found)return doc_id# Index and return doc_id@@ -79,6 +79,7 @@ class IndexingService:vector_db=vector_db,doc_id=doc_id,extracted_text=extracted_text,+ doc_id_found=doc_id_found,)return doc_idexcept Exception as e:diff --git a/prompt-service/src/unstract/prompt_service_v2/services/retrieval_v2.py b/prompt-service/src/unstract/prompt_service/services/retrieval.pysimilarity index 62%rename from prompt-service/src/unstract/prompt_service_v2/services/retrieval_v2.pyrename to prompt-service/src/unstract/prompt_service/services/retrieval.pyindex 63211573e..1e6cff78b 100644--- a/prompt-service/src/unstract/prompt_service_v2/services/retrieval_v2.py+++ b/prompt-service/src/unstract/prompt_service/services/retrieval.py@@ -1,10 +1,10 @@-from typing import Any, Optional+from typing import Any-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys-from unstract.prompt_service_v2.core.retrievers.simple import SimpleRetriever-from unstract.prompt_service_v2.core.retrievers.subquestion import SubquestionRetriever-from unstract.prompt_service_v2.services.answer_prompt import AnswerPromptService-from unstract.prompt_service_v2.utils.file_utils import FileUtils+from unstract.prompt_service.constants import PromptServiceConstants as PSKeys+from unstract.prompt_service.core.retrievers.simple import SimpleRetriever+from unstract.prompt_service.core.retrievers.subquestion import SubquestionRetriever+from unstract.prompt_service.services.answer_prompt import AnswerPromptService+from unstract.prompt_service.utils.file_utils import FileUtilsfrom unstract.sdk.llm import LLMfrom unstract.sdk.vector_db import VectorDB@@ -22,71 +22,73 @@ class RetrievalService:chunk_size: int,execution_source: str,file_path: str,- ) -> set[str]:- context: set[str] = set()-+ ) -> tuple[str, list[str]]:+ context: list[str]if chunk_size == 0:context = RetrievalService.retrieve_complete_context(execution_source=execution_source, file_path=file_path)else:context = RetrievalService.run_retrieval(- tool_settings=tool_settings,output=output,doc_id=doc_id,llm=llm,vector_db=vector_db,retrieval_type=retrieval_type,- metadata=metadata,- execution_source=execution_source,)-- return context+ answer = AnswerPromptService.construct_and_run_prompt( # type:ignore+ tool_settings=tool_settings,+ output=output,+ llm=llm,+ context="\n".join(context),+ prompt="promptx",+ metadata=metadata,+ execution_source=execution_source,+ file_path=file_path,+ )+ return answer, context@staticmethoddef run_retrieval( # type:ignore- tool_settings: dict[str, Any],output: dict[str, Any],doc_id: str,llm: LLM,vector_db: VectorDB,retrieval_type: str,- metadata: dict[str, Any],- execution_source: Optional[str] = None,- ) -> tuple[str, set[str]]:- context: set[str] = set()+ ) -> list[str]:+ context: set[str]prompt = output[PSKeys.PROMPTX]top_k = output[PSKeys.SIMILARITY_TOP_K]if retrieval_type == PSKeys.SUBQUESTION:context = SubquestionRetriever(- vector_db=vector_db, doc_id=doc_id, prompt=prompt, top_k=top_k+ vector_db=vector_db,+ doc_id=doc_id,+ prompt=prompt,+ top_k=top_k,+ llm=llm,).retrieve()if retrieval_type == PSKeys.SIMPLE:context = SimpleRetriever(- vector_db=vector_db, doc_id=doc_id, prompt=prompt, top_k=top_k+ vector_db=vector_db,+ doc_id=doc_id,+ prompt=prompt,+ top_k=top_k,+ llm=llm,).retrieve()- answer = AnswerPromptService.construct_and_run_prompt( # type:ignore- tool_settings=tool_settings,- output=output,- llm=llm,- context="\n".join(context),- prompt="promptx",- metadata=metadata,- execution_source=execution_source,- )-- return (answer, context)+ return list(context)@staticmethod- def retrieve_complete_context(execution_source: str, file_path: str) -> str:+ def retrieve_complete_context(execution_source: str, file_path: str) -> list[str]:"""Loads full context from raw file for zero chunk size retrievalArgs:- path (str): Path to the directory containing text file.+ execution_source (str): Source of execution.+ file_path (str): Path to the directory containing text file.Returns:- str: context from extracted file.+ list[str]: context from extracted file."""fs_instance = FileUtils.get_fs_instance(execution_source=execution_source)- return fs_instance.read(path=file_path, mode="r")+ context = fs_instance.read(path=file_path, mode="r")+ return [context]diff --git a/prompt-service/src/unstract/prompt_service_v2/services/variable_replacement.py b/prompt-service/src/unstract/prompt_service/services/variable_replacement.pysimilarity index 94%rename from prompt-service/src/unstract/prompt_service_v2/services/variable_replacement.pyrename to prompt-service/src/unstract/prompt_service/services/variable_replacement.pyindex 3492a418a..13d5987e8 100644--- a/prompt-service/src/unstract/prompt_service_v2/services/variable_replacement.py+++ b/prompt-service/src/unstract/prompt_service/services/variable_replacement.py@@ -1,12 +1,12 @@from typing import Anyfrom flask import current_app as app-from unstract.prompt_service.utils.log import publish_log-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys-from unstract.prompt_service_v2.constants import RunLevel, VariableType-from unstract.prompt_service_v2.helpers.variable_replacement import (+from unstract.prompt_service.constants import PromptServiceConstants as PSKeys+from unstract.prompt_service.constants import RunLevel, VariableType+from unstract.prompt_service.helpers.variable_replacement import (VariableReplacementHelper,)+from unstract.prompt_service.utils.log import publish_logfrom unstract.sdk.constants import LogLeveldiff --git a/prompt-service/src/unstract/prompt_service_v2/tests/conftest.py b/prompt-service/src/unstract/prompt_service/tests/conftest.pysimilarity index 77%rename from prompt-service/src/unstract/prompt_service_v2/tests/conftest.pyrename to prompt-service/src/unstract/prompt_service/tests/conftest.pyindex e13f7f486..b37f29e07 100644--- a/prompt-service/src/unstract/prompt_service_v2/tests/conftest.py+++ b/prompt-service/src/unstract/prompt_service/tests/conftest.py@@ -2,8 +2,8 @@ import pytestfrom dotenv import load_dotenvfrom flask import Flaskfrom flask_wtf.csrf import CSRFProtect-from unstract.prompt_service_v2.controllers.extraction import extraction_bp-from unstract.prompt_service_v2.controllers.indexing import indexing_bp+from unstract.prompt_service.controllers.extraction import extraction_bp+from unstract.prompt_service.controllers.indexing import indexing_bp# Load test environment variablesload_dotenv(".env.test")@@ -12,7 +12,6 @@ load_dotenv(".env.test")@pytest.fixturedef client():app = Flask(__name__)-app.register_blueprint(indexing_bp)app.register_blueprint(extraction_bp)app.config.update(@@ -27,4 +26,3 @@ def client():csrf.init_app(app) # Compliantwith app.test_client() as client:yield client- # TODO Add teardown code herediff --git a/prompt-service/src/unstract/prompt_service_v2/tests/integration/input/sample1.pdf b/prompt-service/src/unstract/prompt_service/tests/integration/input/sample1.pdfsimilarity index 100%rename from prompt-service/src/unstract/prompt_service_v2/tests/integration/input/sample1.pdfrename to prompt-service/src/unstract/prompt_service/tests/integration/input/sample1.pdfdiff --git a/prompt-service/src/unstract/prompt_service_v2/tests/integration/test_api_endpoints.py b/prompt-service/src/unstract/prompt_service/tests/integration/test_api_endpoints.pysimilarity index 97%rename from prompt-service/src/unstract/prompt_service_v2/tests/integration/test_api_endpoints.pyrename to prompt-service/src/unstract/prompt_service/tests/integration/test_api_endpoints.pyindex ed4b16d66..7192d1817 100644--- a/prompt-service/src/unstract/prompt_service_v2/tests/integration/test_api_endpoints.py+++ b/prompt-service/src/unstract/prompt_service/tests/integration/test_api_endpoints.py@@ -3,7 +3,7 @@ import osfrom dotenv import load_dotenvfrom flask.testing import FlaskClient-from unstract.prompt_service_v2.helpers.auth import AuthHelper+from unstract.prompt_service.helpers.auth import AuthHelperload_dotenv(".env.test")diff --git a/prompt-service/src/unstract/prompt_service_v2/tests/sample.env.test b/prompt-service/src/unstract/prompt_service/tests/sample.env.testsimilarity index 91%rename from prompt-service/src/unstract/prompt_service_v2/tests/sample.env.testrename to prompt-service/src/unstract/prompt_service/tests/sample.env.testindex 0fd829f42..7554cc215 100644--- a/prompt-service/src/unstract/prompt_service_v2/tests/sample.env.test+++ b/prompt-service/src/unstract/prompt_service/tests/sample.env.test@@ -10,8 +10,8 @@ TEST_EXECUTION_SOURCE=ideTEST_RUN_ID=mock_run_idTEST_EXTRACTED_TEXT=mock_extracted_textTEST_TAGS=mock_tag1,mock_tag2-TEST_SOURCE_FILE_PATH=/unstract/prompt-service/src/unstract/prompt_service_v2/tests/integration/input/sample1.pdf-TEST_OUTPUT_FILE_PATH=/unstract/prompt-service/src/unstract/prompt_service_v2/tests/integration/output/output.txt+TEST_SOURCE_FILE_PATH=/unstract/prompt-service/src/unstract/prompt_service/tests/integration/input/sample1.pdf+TEST_OUTPUT_FILE_PATH=/unstract/prompt-service/src/unstract/prompt_service/tests/integration/output/output.txt# Platform Service# Platform service is expected to be up and running for this integration test to rundiff --git a/prompt-service/src/unstract/prompt_service_v2/services/__init__.py b/prompt-service/src/unstract/prompt_service/utils/__init__.pysimilarity index 100%rename from prompt-service/src/unstract/prompt_service_v2/services/__init__.pyrename to prompt-service/src/unstract/prompt_service/utils/__init__.pydiff --git a/prompt-service/src/unstract/prompt_service/db_utils.py b/prompt-service/src/unstract/prompt_service/utils/db_utils.pysimilarity index 89%rename from prompt-service/src/unstract/prompt_service/db_utils.pyrename to prompt-service/src/unstract/prompt_service/utils/db_utils.pyindex f20e34ddb..f650f6187 100644--- a/prompt-service/src/unstract/prompt_service/db_utils.py+++ b/prompt-service/src/unstract/prompt_service/utils/db_utils.py@@ -1,10 +1,10 @@from typing import Any, Optional-from unstract.prompt_service.config import dbfrom unstract.prompt_service.constants import DBTableV2-from unstract.prompt_service.env_manager import EnvLoader+from unstract.prompt_service.extensions import db+from unstract.prompt_service.utils.env_loader import get_env_or_die-DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract")+DB_SCHEMA = get_env_or_die("DB_SCHEMA", "unstract")class DBUtils:diff --git a/prompt-service/src/unstract/prompt_service_v2/utils/env_loader.py b/prompt-service/src/unstract/prompt_service/utils/env_loader.pysimilarity index 100%rename from prompt-service/src/unstract/prompt_service_v2/utils/env_loader.pyrename to prompt-service/src/unstract/prompt_service/utils/env_loader.pydiff --git a/prompt-service/src/unstract/prompt_service_v2/utils/file_utils.py b/prompt-service/src/unstract/prompt_service/utils/file_utils.pysimilarity index 93%rename from prompt-service/src/unstract/prompt_service_v2/utils/file_utils.pyrename to prompt-service/src/unstract/prompt_service/utils/file_utils.pyindex ce73926f5..1ab7145e4 100644--- a/prompt-service/src/unstract/prompt_service_v2/utils/file_utils.py+++ b/prompt-service/src/unstract/prompt_service/utils/file_utils.py@@ -1,4 +1,4 @@-from unstract.prompt_service_v2.constants import ExecutionSource, FileStorageKeys+from unstract.prompt_service.constants import ExecutionSource, FileStorageKeysfrom unstract.sdk.file_storage import FileStoragefrom unstract.sdk.file_storage.constants import StorageTypefrom unstract.sdk.file_storage.env_helper import EnvHelperdiff --git a/prompt-service/src/unstract/prompt_service/utils/request.py b/prompt-service/src/unstract/prompt_service/utils/request.pyindex c7006af38..b6c6848ba 100644--- a/prompt-service/src/unstract/prompt_service/utils/request.py+++ b/prompt-service/src/unstract/prompt_service/utils/request.py@@ -1,12 +1,10 @@-import loggingfrom enum import Enumfrom typing import Any, Optionalimport requests as pyrequests+from flask import current_app as appfrom requests.exceptions import RequestException-from unstract.prompt_service.exceptions import APIError--logger = logging.getLogger(__name__)+from unstract.prompt_service.exceptions import APIError, BadRequest, MissingFieldErrorclass HTTPMethod(str, Enum):@@ -43,8 +41,18 @@ def make_http_request()return return_valexcept RequestException as e:- logger.error(f"HTTP request error: {e}")+ app.logger.error(f"HTTP request error: {e}")raise APIError(f"Error occured while invoking POST API Variable : {str(e)}")except Exception as e:- logger.error(f"An unexpected error occurred: {e}")+ app.logger.error(f"An unexpected error occurred: {e}")raise e+++def validate_request_payload(payload, required_fields):+ if not payload:+ raise BadRequest()++ # Validate required fields+ missing_fields = [field for field in required_fields if field not in payload]+ if missing_fields:+ raise MissingFieldError(missing_fields)diff --git a/prompt-service/src/unstract/prompt_service/utils/request_id_filter.py b/prompt-service/src/unstract/prompt_service/utils/request_id_filter.pynew file mode 100644index 000000000..649cff547--- /dev/null+++ b/prompt-service/src/unstract/prompt_service/utils/request_id_filter.py@@ -0,0 +1,11 @@+import logging++from flask import g+++class RequestIDFilter(logging.Filter):+ """Filter to inject request ID into log records."""++ def filter(self, record):+ record.request_id = getattr(g, "request_id", "N/A") if g else "N/A"+ return Truediff --git a/prompt-service/src/unstract/prompt_service/variable_extractor/base.py b/prompt-service/src/unstract/prompt_service/variable_extractor/base.pydeleted file mode 100644index dbcde7a39..000000000--- a/prompt-service/src/unstract/prompt_service/variable_extractor/base.py+++ /dev/null@@ -1,114 +0,0 @@-from typing import Any--from flask import current_app as app-from unstract.prompt_service.constants import PromptServiceContants as PSKeys-from unstract.prompt_service.constants import RunLevel-from unstract.prompt_service.utils.log import publish_log-from unstract.sdk.constants import LogLevel--from .constants import VariableType-from .prompt_variable_service import VariableService---class VariableExtractor:-- @staticmethod- def execute_variable_replacement(- prompt_text: str, variable_map: dict[str, Any]- ) -> str:- variables: list[str] = VariableService.extract_variables_from_prompt(- prompt_text=prompt_text- )- for variable in variables:- variable_type = VariableService.identify_variable_type(variable=variable)- if variable_type == VariableType.STATIC:- prompt_text = VariableService.replace_static_variable(- prompt=prompt_text,- structured_output=variable_map,- variable=variable,- )-- if variable_type == VariableType.DYNAMIC:- prompt_text = VariableService.replace_dynamic_variable(- prompt=prompt_text,- variable=variable,- structured_output=variable_map,- )- return prompt_text-- @staticmethod- def is_variables_present(prompt_text: str) -> bool:- """Determines if variables are present in the prompt.-- Args:- prompt (str): Prompt to check-- Returns:- bool: True if variables are present else False- """- return bool(len(VariableService.extract_variables_from_prompt(prompt_text)))-- @staticmethod- def replace_variables_in_prompt(- prompt: dict[str, Any],- structured_output: dict[str, Any],- log_events_id: str,- tool_id: str,- prompt_name: str,- doc_name: str,- ) -> str:- """Replaces variables in prompt.-- Args:- prompt (dict[str, Any]): Dict representing the prompt card- structured_output (dict[str, Any]): Structured data used for variable- replacement when variable map is not present in `prompt`.- log_events_id (str): UUID for the WS communication- tool_id (str): UUID for the prompt studio project- prompt_name (str): Name of the prompt being run- doc_name (str): Name of the document being run with- Returns:- prompt_text (str): Prompt with variables replaced- """- app.logger.info(f"[{tool_id}] Replacing variables in prompt : {prompt_name}")- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.DEBUG,- RunLevel.RUN,- "Replacing variables in prompt",- )- # Initialized with the prompt because- # it is used in finally block- prompt_text = prompt[PSKeys.PROMPT]- try:- variable_map = prompt[PSKeys.VARIABLE_MAP]- prompt_text = VariableExtractor.execute_variable_replacement(- prompt_text=prompt[PSKeys.PROMPT], variable_map=variable_map- )- except KeyError:- # Executed incase of structured tool and- # APIs where we do not set the variable map- prompt_text = VariableExtractor.execute_variable_replacement(- prompt_text=prompt_text, variable_map=structured_output- )- finally:- app.logger.info(- f"[{tool_id}] Prompt after variable replacement: {prompt_text}"- )- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.DEBUG,- RunLevel.RUN,- f"Prompt after variable replacement:{prompt_text} ",- )- return prompt_textdiff --git a/prompt-service/src/unstract/prompt_service/variable_extractor/constants.py b/prompt-service/src/unstract/prompt_service/variable_extractor/constants.pydeleted file mode 100644index b456315f4..000000000--- a/prompt-service/src/unstract/prompt_service/variable_extractor/constants.py+++ /dev/null@@ -1,13 +0,0 @@-from enum import Enum---class VariableType(str, Enum):- STATIC = "STATIC"- DYNAMIC = "DYNAMIC"---class VariableConstants:-- VARIABLE_REGEX = "{{(.+?)}}"- DYNAMIC_VARIABLE_DATA_REGEX = r"\[(.*?)\]"- DYNAMIC_VARIABLE_URL_REGEX = r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))" # noqa: E501diff --git a/prompt-service/src/unstract/prompt_service/variable_extractor/prompt_variable_service.py b/prompt-service/src/unstract/prompt_service/variable_extractor/prompt_variable_service.pydeleted file mode 100644index d27f4525a..000000000--- a/prompt-service/src/unstract/prompt_service/variable_extractor/prompt_variable_service.py+++ /dev/null@@ -1,121 +0,0 @@-import json-import logging-import re-from functools import lru_cache-from typing import Any--from unstract.prompt_service.utils.request import HTTPMethod, make_http_request--from .constants import VariableConstants, VariableType--logger = logging.getLogger(__name__)---class VariableService:-- @staticmethod- def replace_static_variable(- prompt: str, structured_output: dict[str, Any], variable: str- ) -> str:- output_value = VariableService.check_static_variable_run_status(- structure_output=structured_output, variable=variable- )- if not output_value:- return prompt- static_variable_marker_string = "".join(["{{", variable, "}}"])-- replaced_prompt: str = VariableService.replace_generic_string_value(- prompt=prompt, variable=static_variable_marker_string, value=output_value- )-- return replaced_prompt-- @staticmethod- def check_static_variable_run_status(- structure_output: dict[str, Any], variable: str- ) -> Any:- output = None- try:- output = structure_output[variable]- return output- except KeyError:- logger.warn(- f"Prompt with {variable} is not executed yet."- " Unable to replace the variable"- )- return output-- @staticmethod- def replace_generic_string_value(prompt: str, variable: str, value: Any) -> str:- formatted_value: str = value- if not isinstance(value, str):- formatted_value = VariableService.handle_json_and_str_types(value)- replaced_prompt = prompt.replace(variable, formatted_value)- return replaced_prompt-- @staticmethod- def handle_json_and_str_types(value: Any) -> str:- try:- formatted_value = json.dumps(value)- except ValueError:- formatted_value = str(value)- return formatted_value-- @staticmethod- def identify_variable_type(variable: str) -> VariableType:- variable_type: VariableType- pattern = re.compile(VariableConstants.DYNAMIC_VARIABLE_URL_REGEX)- if re.findall(pattern, variable):- variable_type = VariableType.DYNAMIC- else:- variable_type = VariableType.STATIC- return variable_type-- @staticmethod- def replace_dynamic_variable(- prompt: str, variable: str, structured_output: dict[str, Any]- ) -> str:- url = re.search(VariableConstants.DYNAMIC_VARIABLE_URL_REGEX, variable).group(0)- data = re.findall(VariableConstants.DYNAMIC_VARIABLE_DATA_REGEX, variable)[0]- output_value = VariableService.check_static_variable_run_status(- structure_output=structured_output, variable=data- )- if not output_value:- return prompt- api_response: Any = VariableService.fetch_dynamic_variable_value(- url=url, data=output_value- )- formatted_api_response: str = VariableService.handle_json_and_str_types(- api_response- )- static_variable_marker_string = "".join(["{{", variable, "}}"])- replaced_prompt: str = VariableService.replace_generic_string_value(- prompt=prompt,- variable=static_variable_marker_string,- value=formatted_api_response,- )- return replaced_prompt-- @staticmethod- @lru_cache(maxsize=128)- def extract_variables_from_prompt(prompt_text: str) -> list[str]:- variable: list[str] = []- variable = re.findall(VariableConstants.VARIABLE_REGEX, prompt_text)- return variable-- @staticmethod- def fetch_dynamic_variable_value(url: str, data: str) -> Any:-- # This prototype method currently supports- # only endpoints that do not require authentication.- # Additionally, it only accepts plain text- # inputs for POST requests in this version.- # Future versions may include support for- # authentication and other input formats.-- verb: HTTPMethod = HTTPMethod.POST- headers = {"Content-Type": "text/plain"}- response: Any = make_http_request(- verb=verb, url=url, data=data, headers=headers- )- return responsediff --git a/prompt-service/src/unstract/prompt_service_v2/config.py b/prompt-service/src/unstract/prompt_service_v2/config.pydeleted file mode 100644index 391996a7f..000000000--- a/prompt-service/src/unstract/prompt_service_v2/config.py+++ /dev/null@@ -1,54 +0,0 @@-import logging-from os import environ as env--from dotenv import load_dotenv-from flask import Flask-from unstract.prompt_service_v2.controllers import api-from unstract.prompt_service_v2.extensions import db-from unstract.prompt_service_v2.helpers.lifecycle import register_lifecycle_hooks-from unstract.prompt_service_v2.helpers.plugin import plugin_loader-from unstract.prompt_service_v2.utils.env_loader import get_env_or_die-from unstract.sdk.constants import LogLevel--from unstract.core.flask import register_error_handlers, register_request_id_middleware-from unstract.core.flask.logging import setup_logging--load_dotenv()---def create_app() -> Flask:- """Creates and configures the Flask application."""-- log_level = env.get("LOG_LEVEL", LogLevel.INFO.value).upper()- setup_logging(log_level)- log_level = getattr(logging, log_level, logging.INFO)- app = Flask("prompt-service")- app.logger.setLevel(log_level)- app.logger.info("Initializing Flask application...")- # Load required environment variables- db_host = get_env_or_die("PG_BE_HOST")- db_port = get_env_or_die("PG_BE_PORT")- db_user = get_env_or_die("PG_BE_USERNAME")- db_pass = get_env_or_die("PG_BE_PASSWORD")- db_name = get_env_or_die("PG_BE_DATABASE")- application_name = env.get("APPLICATION_NAME", "unstract-prompt-service")-- # Initialize and connect to the database- db.init(- database=db_name,- user=db_user,- password=db_pass,- host=db_host,- port=db_port,- options=f"-c application_name={application_name}",- )-- # Load plugins- plugin_loader(app)- register_lifecycle_hooks(app)- register_request_id_middleware(app)- register_error_handlers(app)- app.register_blueprint(api)-- app.logger.info("Flask app created successfully.")- return appdiff --git a/prompt-service/src/unstract/prompt_service_v2/constants.py b/prompt-service/src/unstract/prompt_service_v2/constants.pydeleted file mode 100644index 22bc3bc87..000000000--- a/prompt-service/src/unstract/prompt_service_v2/constants.py+++ /dev/null@@ -1,174 +0,0 @@-from enum import Enum---class PromptServiceConstants:- """Constants used in the prompt service."""-- WORD = "word"- SYNONYMS = "synonyms"- OUTPUTS = "outputs"- TOOL_ID = "tool_id"- RUN_ID = "run_id"- FILE_NAME = "file_name"- FILE_HASH = "file_hash"- NAME = "name"- ACTIVE = "active"- PROMPT = "prompt"- CHUNK_SIZE = "chunk-size"- PROMPTX = "promptx"- VECTOR_DB = "vector-db"- EMBEDDING = "embedding"- X2TEXT_ADAPTER = "x2text_adapter"- CHUNK_OVERLAP = "chunk-overlap"- LLM = "llm"- IS_ASSERT = "is_assert"- ASSERTION_FAILURE_PROMPT = "assertion_failure_prompt"- RETRIEVAL_STRATEGY = "retrieval-strategy"- SIMPLE = "simple"- SUBQUESTION = "subquestion"- TYPE = "type"- NUMBER = "number"- EMAIL = "email"- DATE = "date"- BOOLEAN = "boolean"- JSON = "json"- PREAMBLE = "preamble"- SIMILARITY_TOP_K = "similarity-top-k"- PROMPT_TOKENS = "prompt_tokens"- COMPLETION_TOKENS = "completion_tokens"- TOTAL_TOKENS = "total_tokens"- RESPONSE = "response"- POSTAMBLE = "postamble"- GRAMMAR = "grammar"- PLATFORM_SERVICE_API_KEY = "PLATFORM_SERVICE_API_KEY"- EMBEDDING_SUFFIX = "embedding_suffix"- EVAL_SETTINGS = "eval_settings"- EVAL_SETTINGS_EVALUATE = "evaluate"- EVAL_SETTINGS_MONITOR_LLM = "monitor_llm"- EVAL_SETTINGS_EXCLUDE_FAILED = "exclude_failed"- TOOL_SETTINGS = "tool_settings"- LOG_EVENTS_ID = "log_events_id"- CHALLENGE_LLM = "challenge_llm"- CHALLENGE = "challenge"- ENABLE_CHALLENGE = "enable_challenge"- EXTRACTION = "extraction"- SUMMARIZE = "summarize"- SINGLE_PASS_EXTRACTION = "single-pass-extraction"- SIMPLE_PROMPT_STUDIO = "simple-prompt-studio"- LLM_USAGE_REASON = "llm_usage_reason"- METADATA = "metadata"- OUTPUT = "output"- CONTEXT = "context"- INCLUDE_METADATA = "include_metadata"- TABLE = "table"- TABLE_SETTINGS = "table_settings"- EPILOGUE = "epilogue"- PLATFORM_POSTAMBLE = "platform_postamble"- HIGHLIGHT_DATA_PLUGIN = "highlight-data"- CLEAN_CONTEXT = "clean-context"- SUMMARIZE_AS_SOURCE = "summarize_as_source"- VARIABLE_MAP = "variable_map"- RECORD = "record"- TEXT = "text"- ENABLE_HIGHLIGHT = "enable_highlight"- FILE_PATH = "file_path"- HIGHLIGHT_DATA = "highlight_data"- CONFIDENCE_DATA = "confidence_data"- REQUIRED_FIELDS = "required_fields"- REQUIRED = "required"- EXECUTION_SOURCE = "execution_source"- METRICS = "metrics"- LINE_ITEM = "line-item"- LINE_NUMBERS = "line_numbers"- WHISPER_HASH = "whisper_hash"- PAID_FEATURE_MSG = (- "It is a cloud / enterprise feature. If you have purchased a plan and still "- "face this issue, please contact support"- )- NO_CONTEXT_ERROR = (- "Couldn't fetch context from vector DB. "- "This happens usually due to a delay by the Vector DB "- "provider to confirm writes to DB. "- "Please try again after some time"- )---class RunLevel(Enum):- """Different stages of prompt execution.-- Comprises of prompt run and response evaluation stages.- """-- RUN = "RUN"- EVAL = "EVAL"- CHALLENGE = "CHALLENGE"- TABLE_EXTRACTION = "TABLE_EXTRACTION"---class DBTableV2:- """Database tables."""-- ORGANIZATION = "organization"- ADAPTER_INSTANCE = "adapter_instance"- PROMPT_STUDIO_REGISTRY = "prompt_studio_registry"- PLATFORM_KEY = "platform_key"- TOKEN_USAGE = "usage"---class FileStorageKeys:- """File storage keys."""-- PERMANENT_REMOTE_STORAGE = "PERMANENT_REMOTE_STORAGE"- TEMPORARY_REMOTE_STORAGE = "TEMPORARY_REMOTE_STORAGE"---class FileStorageType(Enum):- """File storage type."""-- PERMANENT = "permanent"- TEMPORARY = "temporary"---class ExecutionSource(Enum):- """Execution source."""-- IDE = "ide"- TOOL = "tool"---class VariableType(str, Enum):- """Type of variable."""-- STATIC = "STATIC"- DYNAMIC = "DYNAMIC"---class VariableConstants:- """Constants for variable extraction."""-- VARIABLE_REGEX = "{{(.+?)}}"- DYNAMIC_VARIABLE_DATA_REGEX = r"\[(.*?)\]"- DYNAMIC_VARIABLE_URL_REGEX = r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))" # noqa: E501---class IndexingConstants:- TOOL_ID = "tool_id"- EMBEDDING_INSTANCE_ID = "embedding_instance_id"- VECTOR_DB_INSTANCE_ID = "vector_db_instance_id"- X2TEXT_INSTANCE_ID = "x2text_instance_id"- FILE_PATH = "file_path"- CHUNK_SIZE = "chunk_size"- CHUNK_OVERLAP = "chunk_overlap"- REINDEX = "reindex"- FILE_HASH = "file_hash"- OUTPUT_FILE_PATH = "output_file_path"- ENABLE_HIGHLIGHT = "enable_highlight"- USAGE_KWARGS = "usage_kwargs"- PROCESS_TEXT = "process_text"- EXTRACTED_TEXT = "extracted_text"- TAGS = "tags"- EXECUTION_SOURCE = "execution_source"- DOC_ID = "doc_id"- TOOL_EXECUTION_METATADA = "tool_execution_metadata"- EXECUTION_DATA_DIR = "execution_data_dir"- METADATA_FILE = "METADATA.json"diff --git a/prompt-service/src/unstract/prompt_service_v2/exceptions.py b/prompt-service/src/unstract/prompt_service_v2/exceptions.pydeleted file mode 100644index 96b87a614..000000000--- a/prompt-service/src/unstract/prompt_service_v2/exceptions.py+++ /dev/null@@ -1,28 +0,0 @@-from unstract.core.flask.exceptions import APIError---class BadRequest(APIError):- code = 400- message = "Bad Request / No payload"---class RateLimitError(APIError):- code = 429- message = "Running into rate limit errors, please try again later"---class MissingFieldError(APIError):- """Custom error for missing fields."""-- def __init__(self, missing_fields: list[str]):- message = f"Missing required fields: {', '.join(missing_fields)}"- super().__init__(message=message)---class RetrievalError(APIError):- """Custom exception raised for errors during retrieval from VectorDB."""-- DEFAULT_MESSAGE = (- "Error while retrieving data from the VectorDB. "- "Please contact the admin for further assistance."- )diff --git a/prompt-service/src/unstract/prompt_service_v2/run.py b/prompt-service/src/unstract/prompt_service_v2/run.pydeleted file mode 100644index f33606904..000000000--- a/prompt-service/src/unstract/prompt_service_v2/run.py+++ /dev/null@@ -1,3 +0,0 @@-from unstract.prompt_service_v2.config import create_app--app = create_app()diff --git a/prompt-service/src/unstract/prompt_service_v2/services/retrieval.py b/prompt-service/src/unstract/prompt_service_v2/services/retrieval.pydeleted file mode 100644index dbcb19e03..000000000--- a/prompt-service/src/unstract/prompt_service_v2/services/retrieval.py+++ /dev/null@@ -1,219 +0,0 @@-import time-from typing import Any, Optional--from flask import current_app as app-from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters-from unstract.prompt_service.exceptions import APIError-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys-from unstract.prompt_service_v2.constants import RunLevel-from unstract.prompt_service_v2.services.answer_prompt import AnswerPromptService-from unstract.prompt_service_v2.utils.log import publish_log-from unstract.sdk.constants import LogLevel-from unstract.sdk.exceptions import SdkError-from unstract.sdk.index import Index-from unstract.sdk.llm import LLM---class RetrievalService:-- @staticmethod- def run_retrieval( # type:ignore- tool_settings: dict[str, Any],- output: dict[str, Any],- doc_id: str,- llm: LLM,- vector_index,- retrieval_type: str,- metadata: dict[str, Any],- execution_source: Optional[str] = None,- ) -> tuple[str, set[str]]:- context: set[str] = set()- prompt = output[PSKeys.PROMPTX]- if retrieval_type == PSKeys.SUBQUESTION:- subq_prompt: str = (- f"I am sending you a verbose prompt \n \n Prompt : {prompt} \n \n"- "Generate set of specific subquestions "- "from the prompt which can be used to retrive "- "relevant context from vector db. "- "Use your logical abilities to "- " only generate as many subquestions as necessary "- " — fewer subquestions if the prompt is simpler. "- "Decide the minimum limit for subquestions "- "based on the complexity input prompt and set the maximum limit"- "for the subquestions to 10."- "Ensure that each subquestion is distinct and relevant"- "to the the original query. "- "Do not add subquestions for details"- "not mentioned in the original prompt."- " The goal is to maximize retrieval accuracy"- " using these subquestions. Use your logical abilities to ensure "- " that each subquestion targets a distinct aspect of the original"- " query. Please note that, there are cases where the "- "response might have a list of answers. The subquestions must not"- " miss out any values in these cases. "- "Output should be a list of comma seperated "- "subquestion prompts. Do not change this format. \n \n "- " Subquestions : "- )- subquestions = AnswerPromptService.run_completion(- llm=llm,- prompt=subq_prompt,- )- subquestion_list = subquestions.split(",")- for each_subq in subquestion_list:- retrieved_context = RetrievalService._retrieve_context(- output, doc_id, vector_index, each_subq- )- context.update(retrieved_context)-- if retrieval_type == PSKeys.SIMPLE:-- context = RetrievalService._retrieve_context(- output, doc_id, vector_index, prompt- )-- if not context:- # UN-1288 For Pinecone, we are seeing an inconsistent case where- # query with doc_id fails even though indexing just happened.- # This causes the following retrieve to return no text.- # To rule out any lag on the Pinecone vector DB write,- # the following sleep is added- # Note: This will not fix the issue. Since this issue is inconsistent- # and not reproducible easily, this is just a safety net.- time.sleep(2)- context = RetrievalService._retrieve_context(- output, doc_id, vector_index, prompt- )-- answer = AnswerPromptService.construct_and_run_prompt( # type:ignore- tool_settings=tool_settings,- output=output,- llm=llm,- context="\n".join(context),- prompt="promptx",- metadata=metadata,- execution_source=execution_source,- )-- return (answer, context)-- @staticmethod- def _retrieve_context(output, doc_id, vector_index, answer) -> set[str]:- retriever = vector_index.as_retriever(- similarity_top_k=output[PSKeys.SIMILARITY_TOP_K],- filters=MetadataFilters(- filters=[- ExactMatchFilter(key="doc_id", value=doc_id),- # TODO: Enable after adding section in GUI- # ExactMatchFilter(- # key="section", value=output["section"]- ],- ),- )- nodes = retriever.retrieve(answer)- context: set[str] = set()- for node in nodes:- # ToDo: May have to fine-tune this value for node score or keep it- # configurable at the adapter level- if node.score > 0:- context.add(node.get_content())- else:- app.logger.info(- "Node score is less than 0. "- f"Ignored: {node.node_id} with score {node.score}"- )- return context-- @staticmethod- def fetch_context_from_vector_db(- index: Index,- output: dict[str, Any],- doc_id: str,- tool_id: str,- doc_name: str,- prompt_name: str,- log_events_id: str,- usage_kwargs: dict[str, Any],- ) -> set[str]:- """- Fetches context from the index for the given document ID. Implements a retry- mechanism with logging and raises an error if context retrieval fails.- Args:- index: The index object to query.- output: Dictionary containing keys like embedding and vector DB instance ID.- doc_id: The document ID to query.- tool_id: Identifier for the tool in use.- doc_name: Name of the document being queried.- prompt_name: Name of the prompt being executed.- log_events_id: Unique ID for logging events.- usage_kwargs: Additional usage parameters.- Raises:- APIError: If context retrieval fails after retrying.- """- context: set[str] = set()- try:- retrieved_context = index.query_index(- embedding_instance_id=output[PSKeys.EMBEDDING],- vector_db_instance_id=output[PSKeys.VECTOR_DB],- doc_id=doc_id,- usage_kwargs=usage_kwargs,- )-- if retrieved_context:- context.add(retrieved_context)- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.DEBUG,- RunLevel.RUN,- "Fetched context from vector DB",- )- else:- # Handle lag in vector DB write (e.g., Pinecone issue)- time.sleep(2)- retrieved_context = index.query_index(- embedding_instance_id=output[PSKeys.EMBEDDING],- vector_db_instance_id=output[PSKeys.VECTOR_DB],- doc_id=doc_id,- usage_kwargs=usage_kwargs,- )-- if retrieved_context is None:- msg = PSKeys.NO_CONTEXT_ERROR- app.logger.error(- f"{msg} {output[PSKeys.VECTOR_DB]} for doc_id {doc_id}"- )- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.ERROR,- RunLevel.RUN,- msg,- )- raise APIError(message=msg)- except SdkError as e:- msg = f"Unable to fetch context from vector DB. {str(e)}"- app.logger.error(- f"{msg}. VectorDB: {output[PSKeys.VECTOR_DB]}, doc_id: {doc_id}"- )- publish_log(- log_events_id,- {- "tool_id": tool_id,- "prompt_key": prompt_name,- "doc_name": doc_name,- },- LogLevel.ERROR,- RunLevel.RUN,- msg,- )- raise APIError(message=msg, code=e.status_code)- return contextdiff --git a/prompt-service/src/unstract/prompt_service_v2/utils/__init__.py b/prompt-service/src/unstract/prompt_service_v2/utils/__init__.pydeleted file mode 100644index e69de29bb..000000000diff --git a/prompt-service/src/unstract/prompt_service_v2/utils/db_utils.py b/prompt-service/src/unstract/prompt_service_v2/utils/db_utils.pydeleted file mode 100644index f1c47e8c3..000000000--- a/prompt-service/src/unstract/prompt_service_v2/utils/db_utils.py+++ /dev/null@@ -1,46 +0,0 @@-from typing import Any, Optional--from unstract.prompt_service_v2.constants import DBTableV2-from unstract.prompt_service_v2.extensions import db-from unstract.prompt_service_v2.utils.env_loader import get_env_or_die--DB_SCHEMA = get_env_or_die("DB_SCHEMA", "unstract")---class DBUtils:-- @classmethod- def get_organization_from_bearer_token(- cls, token: str- ) -> tuple[Optional[int], str]:- """Retrieve organization ID and identifier using a bearer token.-- Args:- token (str): The bearer token (platform key).-- Returns:- tuple[int, str]: organization uid and organization identifier- """- platform_key_table = f'"{DB_SCHEMA}".{DBTableV2.PLATFORM_KEY}'- organization_table = f'"{DB_SCHEMA}".{DBTableV2.ORGANIZATION}'-- organization_uid: Optional[int] = cls.execute_query(- f"SELECT organization_id FROM {platform_key_table} WHERE key=%s", (token,)- )- if organization_uid is None:- return None, None-- organization_identifier: Optional[str] = cls.execute_query(- f"SELECT organization_id FROM {organization_table} WHERE id=%s",- (organization_uid,),- )- return organization_uid, organization_identifier-- @classmethod- def execute_query(cls, query: str, params: tuple = ()) -> Any:- cursor = db.execute_sql(query, params)- result_row = cursor.fetchone()- cursor.close()- if not result_row or len(result_row) == 0:- return None- return result_row[0]diff --git a/prompt-service/src/unstract/prompt_service_v2/utils/log.py b/prompt-service/src/unstract/prompt_service_v2/utils/log.pydeleted file mode 100644index cb19ef53c..000000000--- a/prompt-service/src/unstract/prompt_service_v2/utils/log.py+++ /dev/null@@ -1,27 +0,0 @@-from unstract.prompt_service_v2.constants import RunLevel-from unstract.sdk.constants import LogLevel--from unstract.core.pubsub_helper import LogPublisher---def publish_log(- log_events_id: str,- component: dict[str, str],- level: LogLevel,- state: RunLevel,- message: str,-) -> None:- """Publishes a log to the web socket.-- Args:- log_events_id (str): UUID for the connection- component (dict[str, str]): Dict of tool_id, doc_name and prompt_name to- log context- level (LogLevel): Log level, one of INFO, WARNING, DEBUG, ERROR- state (RunLevel): Run level, one of EVAL, RUN, CHALLENGE, TABLE_EXTRACTION- message (str): Message to log- """- LogPublisher.publish(- log_events_id,- LogPublisher.log_prompt(component, level.value, state.value, message),- )diff --git a/prompt-service/src/unstract/prompt_service_v2/utils/request.py b/prompt-service/src/unstract/prompt_service_v2/utils/request.pydeleted file mode 100644index 3de8b4846..000000000--- a/prompt-service/src/unstract/prompt_service_v2/utils/request.py+++ /dev/null@@ -1,62 +0,0 @@-from enum import Enum-from typing import Any, Optional--import requests as pyrequests-from flask import current_app as app-from requests.exceptions import RequestException-from unstract.prompt_service_v2.exceptions import (- APIError,- BadRequest,- MissingFieldError,-)---class HTTPMethod(str, Enum):- GET = "GET"- POST = "POST"- DELETE = "DELETE"- PUT = "PUT"- PATCH = "PATCH"---def make_http_request(- verb: HTTPMethod,- url: str,- data: Optional[dict[str, Any]] = None,- headers: Optional[dict[str, Any]] = None,- params: Optional[dict[str, Any]] = None,-) -> str:- """Generic helper function to help make a HTTP request."""- try:- if verb == HTTPMethod.GET:- response = pyrequests.get(url, params=params, headers=headers)- elif verb == HTTPMethod.POST:- response = pyrequests.post(url, json=data, params=params, headers=headers)- elif verb == HTTPMethod.DELETE:- response = pyrequests.delete(url, params=params, headers=headers)- else:- raise ValueError("Invalid HTTP verb. Supported verbs: GET, POST, DELETE")-- response.raise_for_status()- return_val: str = (- response.json()- if response.headers.get("content-type") == "application/json"- else response.text- )- return return_val- except RequestException as e:- app.logger.error(f"HTTP request error: {e}")- raise APIError(f"Error occured while invoking POST API Variable : {str(e)}")- except Exception as e:- app.logger.error(f"An unexpected error occurred: {e}")- raise e---def validate_request_payload(payload, REQUIRED_FIELDS):- if not payload:- raise BadRequest()-- # Validate required fields- missing_fields = [field for field in REQUIRED_FIELDS if field not in payload]- if missing_fields:- raise MissingFieldError(missing_fields)diff --git a/tools/structure/.gitignore b/tools/structure/.gitignorenew file mode 100644index 000000000..be0e458d4--- /dev/null+++ b/tools/structure/.gitignore@@ -0,0 +1 @@+helper.pydiff --git a/tools/structure/src/constants.py b/tools/structure/src/constants.pyindex 32b097a4f..8d71ae163 100644--- a/tools/structure/src/constants.py+++ b/tools/structure/src/constants.py@@ -78,3 +78,27 @@ class SettingsKeys:METRICS = "metrics"INDEXING = "indexing"EXECUTION_ID = "execution_id"+++class IndexingConstants:+ TOOL_ID = "tool_id"+ EMBEDDING_INSTANCE_ID = "embedding_instance_id"+ VECTOR_DB_INSTANCE_ID = "vector_db_instance_id"+ X2TEXT_INSTANCE_ID = "x2text_instance_id"+ FILE_PATH = "file_path"+ CHUNK_SIZE = "chunk_size"+ CHUNK_OVERLAP = "chunk_overlap"+ REINDEX = "reindex"+ FILE_HASH = "file_hash"+ OUTPUT_FILE_PATH = "output_file_path"+ ENABLE_HIGHLIGHT = "enable_highlight"+ USAGE_KWARGS = "usage_kwargs"+ PROCESS_TEXT = "process_text"+ EXTRACTED_TEXT = "extracted_text"+ TAGS = "tags"+ EXECUTION_SOURCE = "execution_source"+ DOC_ID = "doc_id"+ TOOL_EXECUTION_METATADA = "tool_execution_metadata"+ EXECUTION_DATA_DIR = "execution_data_dir"+ RUN_ID = "run_id"+ EXECUTION_ID = "execution_id"diff --git a/tools/structure/src/helpers.py b/tools/structure/src/helpers.pynew file mode 100644index 000000000..e29b5f31c--- /dev/null+++ b/tools/structure/src/helpers.py@@ -0,0 +1,93 @@+from typing import Any, Optional++from constants import IndexingConstants as IKeys+from constants import SettingsKeys # type: ignore [attr-defined]+from unstract.sdk.prompt import PromptTool+from unstract.sdk.tool.base import BaseTool+++class StructureToolHelper:+ @staticmethod+ def dynamic_extraction(+ file_path: str,+ enable_highlight: bool,+ usage_kwargs: dict[str, Any],+ run_id: str,+ tool_settings: dict[str, Any],+ extract_file_path: str,+ tool: BaseTool,+ execution_run_data_folder: str,+ ) -> str:++ x2text = tool_settings[SettingsKeys.X2TEXT_ADAPTER]+ tool.stream_log(f"Extracting text from {file_path} into {extract_file_path}")+ payload = {+ IKeys.X2TEXT_INSTANCE_ID: x2text,+ IKeys.FILE_PATH: file_path,+ IKeys.ENABLE_HIGHLIGHT: enable_highlight,+ IKeys.USAGE_KWARGS: usage_kwargs.copy(),+ IKeys.RUN_ID: run_id,+ IKeys.EXECUTION_SOURCE: SettingsKeys.TOOL,+ IKeys.OUTPUT_FILE_PATH: str(extract_file_path),+ IKeys.TAGS: tool.tags,+ IKeys.TOOL_EXECUTION_METATADA: tool.get_exec_metadata,+ IKeys.EXECUTION_DATA_DIR: str(execution_run_data_folder),+ }++ tool.stream_log(f"Payload constructed : {payload}")+ responder = PromptTool(+ tool=tool,+ prompt_host=tool.get_env_or_die(SettingsKeys.PROMPT_HOST),+ prompt_port=tool.get_env_or_die(SettingsKeys.PROMPT_PORT),+ )+ tool.stream_log(f"responder : {responder}")+ extracted_text = responder.extract(payload=payload)++ return extracted_text++ @staticmethod+ def dynamic_indexing(+ file_path: str,+ tool_settings: dict[str, Any],+ run_id: str,+ tool: BaseTool,+ execution_run_data_folder: str,+ reindex: bool,+ usage_kwargs: dict[str, Any],+ enable_highlight: bool,+ chunk_size: int,+ chunk_overlap: int,+ file_hash: Optional[str] = None,+ tool_id: str = None,+ extracted_text: str = None,+ ) -> str:++ x2text = tool_settings[SettingsKeys.X2TEXT_ADAPTER]++ payload = {+ IKeys.TOOL_ID: tool_id,+ IKeys.EMBEDDING_INSTANCE_ID: tool_settings[SettingsKeys.EMBEDDING],+ IKeys.VECTOR_DB_INSTANCE_ID: tool_settings[SettingsKeys.VECTOR_DB],+ IKeys.X2TEXT_INSTANCE_ID: x2text,+ IKeys.FILE_HASH: file_hash,+ IKeys.CHUNK_SIZE: chunk_size,+ IKeys.CHUNK_OVERLAP: chunk_overlap,+ IKeys.REINDEX: reindex,+ IKeys.FILE_PATH: str(file_path),+ IKeys.ENABLE_HIGHLIGHT: enable_highlight,+ IKeys.USAGE_KWARGS: usage_kwargs.copy(),+ IKeys.RUN_ID: run_id,+ IKeys.EXECUTION_SOURCE: SettingsKeys.TOOL,+ IKeys.TAGS: tool.tags,+ IKeys.TOOL_EXECUTION_METATADA: tool.get_exec_metadata,+ IKeys.EXECUTION_DATA_DIR: str(execution_run_data_folder),+ IKeys.EXTRACTED_TEXT: extracted_text,+ }++ responder = PromptTool(+ tool=tool,+ prompt_host=tool.get_env_or_die(SettingsKeys.PROMPT_HOST),+ prompt_port=tool.get_env_or_die(SettingsKeys.PROMPT_PORT),+ )+ doc_id = responder.index(payload=payload)+ return doc_iddiff --git a/tools/structure/src/main.py b/tools/structure/src/main.pyindex 84cbaf46e..d87851dc6 100644--- a/tools/structure/src/main.py+++ b/tools/structure/src/main.py@@ -6,8 +6,8 @@ from pathlib import Pathfrom typing import Anyfrom constants import SettingsKeys # type: ignore [attr-defined]+from helpers import StructureToolHelper as STHelperfrom unstract.sdk.constants import LogLevel, LogState, MetadataKey, ToolEnv, UsageKwargs-from unstract.sdk.index import Indexfrom unstract.sdk.prompt import PromptToolfrom unstract.sdk.tool.base import BaseToolfrom unstract.sdk.tool.entrypoint import ToolEntrypoint@@ -98,14 +98,9 @@ class StructureTool(BaseTool):self.get_env_or_die(ToolEnv.EXECUTION_DATA_DIR))- index = Index(- tool=self,- run_id=self.file_execution_id,- capture_metrics=True,- )index_metrics = {}extracted_input_file = str(execution_run_data_folder / SettingsKeys.EXTRACT)- # TODO : Resolve and pass log events ID+ # Resolve and pass log events IDpayload = {SettingsKeys.RUN_ID: self.file_execution_id,SettingsKeys.EXECUTION_ID: self.execution_id,@@ -117,32 +112,23 @@ class StructureTool(BaseTool):SettingsKeys.FILE_PATH: extracted_input_file,SettingsKeys.EXECUTION_SOURCE: SettingsKeys.TOOL,}- # TODO: Need to split extraction and indexing- # to avoid unwanted indexing- self.stream_log(f"Indexing document '{self.source_file_name}'")+ self.stream_log(f"Extracting document '{self.source_file_name}'")usage_kwargs: dict[Any, Any] = dict()usage_kwargs[UsageKwargs.RUN_ID] = self.file_execution_idusage_kwargs[UsageKwargs.FILE_NAME] = self.source_file_nameusage_kwargs[UsageKwargs.EXECUTION_ID] = self.execution_id+ extracted_text = STHelper.dynamic_extraction(+ file_path=input_file,+ enable_highlight=enable_highlight,+ usage_kwargs=usage_kwargs,+ run_id=self.file_execution_id,+ tool_settings=tool_settings,+ extract_file_path=tool_data_dir / SettingsKeys.EXTRACT,+ tool=self,+ execution_run_data_folder=str(execution_run_data_folder),+ )if tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION]:- index.index(- tool_id=tool_id,- embedding_instance_id=tool_settings[SettingsKeys.EMBEDDING],- vector_db_instance_id=tool_settings[SettingsKeys.VECTOR_DB],- x2text_instance_id=tool_settings[SettingsKeys.X2TEXT_ADAPTER],- file_path=input_file,- file_hash=file_hash,- chunk_size=tool_settings[SettingsKeys.CHUNK_SIZE],- chunk_overlap=tool_settings[SettingsKeys.CHUNK_OVERLAP],- output_file_path=tool_data_dir / SettingsKeys.EXTRACT,- reindex=True,- usage_kwargs=usage_kwargs,- tags=self.tags,- enable_highlight=enable_highlight,- **({"fs": self.workflow_filestorage}),- )- index_metrics = {SettingsKeys.INDEXING: index.get_metrics()}if summarize_as_source:summarize_file_hash = self._summarize_and_index(tool_id=tool_id,@@ -150,10 +136,27 @@ class StructureTool(BaseTool):tool_data_dir=tool_data_dir,responder=responder,outputs=outputs,- index=index,usage_kwargs=usage_kwargs,)payload[SettingsKeys.FILE_HASH] = summarize_file_hash+ # STHelper.dynamic_indexing(+ # tool_settings=tool_settings,+ # run_id=self.file_execution_id,+ # file_path=tool_data_dir / SettingsKeys.EXTRACT,+ # tool=self,+ # execution_run_data_folder=execution_run_data_folder,+ # chunk_overlap=tool_settings[SettingsKeys.CHUNK_OVERLAP],+ # chunk_size=tool_settings[SettingsKeys.CHUNK_SIZE],+ # reIndex=True,+ # usage_kwargs=usage_kwargs,+ # enable_highlight=enable_highlight,+ # extracted_text=extracted_text,+ # tool_id=tool_metadata[SettingsKeys.TOOL_ID],+ # file_hash=file_hash,+ # )+ # Handle metrics for single pass extraction+ # index_metrics = {SettingsKeys.INDEXING: index.get_metrics()}+self.stream_log("Fetching response for single pass extraction")prompt_service_resp = responder.single_pass_extraction(payload=payload,@@ -164,28 +167,6 @@ class StructureTool(BaseTool):# indexed to get the output in required pathreindex = Truefor output in outputs:- if reindex or not summarize_as_source:- index.index(- tool_id=tool_metadata[SettingsKeys.TOOL_ID],- embedding_instance_id=output[SettingsKeys.EMBEDDING],- vector_db_instance_id=output[SettingsKeys.VECTOR_DB],- x2text_instance_id=output[SettingsKeys.X2TEXT_ADAPTER],- file_path=input_file,- file_hash=file_hash,- chunk_size=output[SettingsKeys.CHUNK_SIZE],- chunk_overlap=output[SettingsKeys.CHUNK_OVERLAP],- output_file_path=tool_data_dir / SettingsKeys.EXTRACT,- reindex=reindex,- usage_kwargs=usage_kwargs,- tags=self.tags,- enable_highlight=enable_highlight,- **({"fs": self.workflow_filestorage}),- )- index_metrics[output[SettingsKeys.NAME]] = {- SettingsKeys.INDEXING: index.get_metrics()- }- index.clear_metrics()-if summarize_as_source:summarize_file_hash = self._summarize_and_index(tool_id=tool_id,@@ -193,12 +174,34 @@ class StructureTool(BaseTool):tool_data_dir=tool_data_dir,responder=responder,outputs=outputs,- index=index,usage_kwargs=usage_kwargs,)payload[SettingsKeys.OUTPUTS] = outputspayload[SettingsKeys.FILE_HASH] = summarize_file_hashbreak+ if reindex or not summarize_as_source:+ self.stream_log("Sucessfully extracted text, indexing..")+ STHelper.dynamic_indexing(+ tool_settings=tool_settings,+ run_id=self.file_execution_id,+ file_path=tool_data_dir / SettingsKeys.EXTRACT,+ tool=self,+ execution_run_data_folder=str(execution_run_data_folder),+ chunk_overlap=output[SettingsKeys.CHUNK_OVERLAP],+ reindex=reindex,+ usage_kwargs=usage_kwargs,+ enable_highlight=enable_highlight,+ chunk_size=output[SettingsKeys.CHUNK_SIZE],+ tool_id=tool_metadata[SettingsKeys.TOOL_ID],+ file_hash=file_hash,+ extracted_text=extracted_text,+ )+ # Handle metrics for single pass extraction+ # index_metrics[output[SettingsKeys.NAME]] = {+ # SettingsKeys.INDEXING: index.get_metrics()+ # }+ # index.clear_metrics()+reindex = Falseexcept Exception as e:self.stream_log(@@ -206,7 +209,6 @@ class StructureTool(BaseTool):)raise- # TODO : Make this snippet pluggable and introduce pluggablity for tools.for output in outputs:if SettingsKeys.TABLE_SETTINGS in output:table_settings = output[SettingsKeys.TABLE_SETTINGS]@@ -218,7 +220,6 @@ class StructureTool(BaseTool):payload=payload,)- # TODO: Make use of dataclassesif prompt_service_resp[SettingsKeys.STATUS] != SettingsKeys.OK:self.stream_error_and_exit(f"Failed to fetch responses for "@@ -284,8 +285,9 @@ class StructureTool(BaseTool):tool_data_dir: Path,responder: PromptTool,outputs: dict[str, Any],- index: Index,usage_kwargs: dict[Any, Any] = {},+ enable_highlight: bool = False,+ execution_run_data_folder: str = None,) -> str:"""Summarizes the context of the file and indexes the summarizedcontent.@@ -311,6 +313,9 @@ class StructureTool(BaseTool):summarize_file_path = tool_data_dir / SettingsKeys.SUMMARIZEsummarized_context = ""+ self.stream_log(+ f"Checking if summarized context exists at {summarize_file_path}"+ )if self.workflow_filestorage.exists(summarize_file_path):summarized_context = self.workflow_filestorage.read(path=summarize_file_path, mode="r"@@ -350,18 +355,20 @@ class StructureTool(BaseTool):summarize_file_hash: str = self.workflow_filestorage.get_hash_from_file(path=summarize_file_path)- index.index(- tool_id=tool_id,- embedding_instance_id=embedding_instance_id,- vector_db_instance_id=vector_db_instance_id,- x2text_instance_id=x2text_instance_id,+ STHelper.dynamic_indexing(file_path=summarize_file_path,+ tool_settings=tool_settings,+ run_id=self.file_execution_id,+ execution_run_data_folder=execution_run_data_folder,+ reindex=True,+ usage_kwargs=usage_kwargs,+ enable_highlight=enable_highlight,file_hash=summarize_file_hash,- chunk_size=0,+ tool_id=tool_id,+ tool=self,chunk_overlap=0,- usage_kwargs=usage_kwargs,- tags=self.tags,- **({"fs": self.workflow_filestorage}),+ chunk_size=0,+ extracted_text=summarized_context,)return summarize_file_hash