Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Integrating Prompt Service V2 APIs to backend & tools (#1202)

harini-venkataraman6caab41846a6
diff --git a/backend/prompt_studio/prompt_studio_core_v2/constants.py b/backend/prompt_studio/prompt_studio_core_v2/constants.py
index 9dad3bb82..ad3451175 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/constants.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/constants.py
@@ -147,3 +147,26 @@ class ExecutionSource(Enum):
"""
IDE = "ide"
+
+
+class IndexingConstants:
+ TOOL_ID = "tool_id"
+ EMBEDDING_INSTANCE_ID = "embedding_instance_id"
+ VECTOR_DB_INSTANCE_ID = "vector_db_instance_id"
+ X2TEXT_INSTANCE_ID = "x2text_instance_id"
+ FILE_PATH = "file_path"
+ CHUNK_SIZE = "chunk_size"
+ CHUNK_OVERLAP = "chunk_overlap"
+ REINDEX = "reindex"
+ FILE_HASH = "file_hash"
+ OUTPUT_FILE_PATH = "output_file_path"
+ ENABLE_HIGHLIGHT = "enable_highlight"
+ USAGE_KWARGS = "usage_kwargs"
+ PROCESS_TEXT = "process_text"
+ EXTRACTED_TEXT = "extracted_text"
+ TAGS = "tags"
+ EXECUTION_SOURCE = "execution_source"
+ DOC_ID = "doc_id"
+ TOOL_EXECUTION_METATADA = "tool_execution_metadata"
+ EXECUTION_DATA_DIR = "execution_data_dir"
+ RUN_ID = "run_id"
diff --git a/backend/prompt_studio/prompt_studio_core_v2/exceptions.py b/backend/prompt_studio/prompt_studio_core_v2/exceptions.py
index 4f45a5e96..32cf43a0f 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/exceptions.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/exceptions.py
@@ -19,6 +19,10 @@ class IndexingAPIError(APIException):
status_code = 500
default_detail = "Error while indexing file"
+ def __init__(self, detail: Optional[str] = None, status_code: int = 500):
+ super().__init__(detail)
+ self.status_code = status_code
+
class AnswerFetchError(APIException):
status_code = 500
diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
index e681e2d8c..4238c0936 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
@@ -4,7 +4,7 @@ import os
import time
import uuid
from pathlib import Path
-from typing import Any, Callable, Optional
+from typing import Any, Optional
from account_v2.constants import Common
from account_v2.models import User
@@ -14,15 +14,16 @@ from django.conf import settings
from django.db.models.manager import BaseManager
from prompt_studio.modifier_loader import ModifierConfig
from prompt_studio.modifier_loader import load_plugins as load_modifier_plugins
+from prompt_studio.processor_loader import get_plugin_class_by_name
+from prompt_studio.processor_loader import load_plugins as load_processor_plugins
from prompt_studio.prompt_profile_manager_v2.models import ProfileManager
from prompt_studio.prompt_profile_manager_v2.profile_manager_helper import (
ProfileManagerHelper,
)
-from prompt_studio.prompt_studio_core_v2.constants import (
- ExecutionSource,
- IndexingStatus,
- LogLevels,
-)
+from prompt_studio.prompt_studio_core_v2.constants import ExecutionSource
+from prompt_studio.prompt_studio_core_v2.constants import IndexingConstants as IKeys
+from prompt_studio.prompt_studio_core_v2.constants import IndexingStatus, LogLevels
+from prompt_studio.prompt_studio_core_v2.constants import ToolStudioPromptKeys
from prompt_studio.prompt_studio_core_v2.constants import (
ToolStudioPromptKeys as TSPKeys,
)
@@ -54,17 +55,18 @@ from prompt_studio.prompt_studio_output_manager_v2.output_manager_helper import
from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt
from unstract.sdk.constants import LogLevel
from unstract.sdk.exceptions import IndexingError, SdkError
-from unstract.sdk.file_storage import FileStorage, FileStorageProvider
from unstract.sdk.file_storage.constants import StorageType
from unstract.sdk.file_storage.env_helper import EnvHelper
-from unstract.sdk.index import Index
from unstract.sdk.prompt import PromptTool
+from unstract.sdk.utils.indexing_utils import IndexingUtils
from utils.file_storage.constants import FileStorageKeys
from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper
from utils.local_context import StateStore
from unstract.core.pubsub_helper import LogPublisher
+logger = logging.getLogger(__name__)
+
CHOICES_JSON = "/static/select_choices.json"
ERROR_MSG = "User %s doesn't have access to adapter %s"
@@ -76,6 +78,8 @@ modifier_plugins = load_modifier_plugins()
class PromptStudioHelper:
"""Helper class for Custom tool operations."""
+ processor_plugins = load_processor_plugins()
+
@staticmethod
def create_default_profile_manager(user: User, tool_id: uuid) -> None:
"""Create a default profile manager for a given user and tool.
@@ -314,7 +318,6 @@ class PromptStudioHelper:
document_id: str,
is_summary: bool = False,
run_id: str = None,
- text_processor: Optional[type[Any]] = None,
) -> Any:
"""Method to index a document.
@@ -331,61 +334,99 @@ class PromptStudioHelper:
IndexingError
"""
tool: CustomTool = CustomTool.objects.get(pk=tool_id)
+ file_path = PromptStudioFileHelper.get_or_create_prompt_studio_subdirectory(
+ org_id,
+ is_create=False,
+ user_id=user_id,
+ tool_id=tool_id,
+ )
+ file_path = str(Path(file_path) / file_name)
+
if is_summary:
profile_manager: ProfileManager = ProfileManager.objects.get(
prompt_studio_tool=tool, is_summarize_llm=True
)
+ profile_manager.chunk_size == 0
default_profile = profile_manager
- file_path = file_name
else:
default_profile = ProfileManager.get_default_llm_profile(tool)
- file_path = PromptStudioFileHelper.get_or_create_prompt_studio_subdirectory(
- org_id,
- is_create=False,
- user_id=user_id,
- tool_id=tool_id,
- )
- file_path = str(Path(file_path) / file_name)
if not tool:
logger.error(f"No tool instance found for the ID {tool_id}")
raise ToolNotValid()
- start_time = time.time()
- logger.info(f"[{tool_id}] Indexing started for doc: {file_name}")
- PromptStudioHelper._publish_log(
- {"tool_id": tool_id, "run_id": run_id, "doc_name": file_name},
- LogLevels.INFO,
- LogLevels.RUN,
- "Indexing started",
- )
-
# Validate the status of adapter in profile manager
PromptStudioHelper.validate_adapter_status(default_profile)
# Need to check the user who created profile manager
# has access to adapters configured in profile manager
PromptStudioHelper.validate_profile_manager_owner_access(default_profile)
- process_text = None
- if text_processor:
- process_text = text_processor.process
fs_instance = EnvHelper.get_storage(
storage_type=StorageType.PERMANENT,
env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,
)
- doc_id = PromptStudioHelper.dynamic_indexer(
+ util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)
+ doc_id = IndexingUtils.generate_index_key(
+ vector_db=str(default_profile.vector_store.id),
+ embedding=str(default_profile.embedding_model.id),
+ x2text=str(default_profile.x2text.id),
+ chunk_size=str(default_profile.chunk_size),
+ chunk_overlap=str(default_profile.chunk_overlap),
+ file_path=file_path,
+ file_hash=None,
+ fs=fs_instance,
+ tool=util,
+ )
+ extracted_text = PromptStudioHelper.dynamic_extractor(
+ profile_manager=default_profile,
+ file_path=file_path,
+ org_id=org_id,
+ document_id=document_id,
+ run_id=run_id,
+ enable_highlight=tool.enable_highlight,
+ doc_id=doc_id,
+ reindex=True,
+ )
+ if is_summary:
+ summarize_file_path = PromptStudioHelper.summarize(
+ file_name, org_id, document_id, is_summary, run_id, tool, doc_id
+ )
+ summarize_doc_id = IndexingUtils.generate_index_key(
+ vector_db=str(default_profile.vector_store.id),
+ embedding=str(default_profile.embedding_model.id),
+ x2text=str(default_profile.x2text.id),
+ chunk_size=str(default_profile.chunk_size),
+ chunk_overlap=str(default_profile.chunk_overlap),
+ file_path=summarize_file_path,
+ fs=fs_instance,
+ tool=util,
+ )
+ PromptStudioIndexHelper.handle_index_manager(
+ document_id=document_id,
+ is_summary=is_summary,
+ profile_manager=profile_manager,
+ doc_id=summarize_doc_id,
+ )
+ start_time = time.time()
+ logger.info(f"[{tool_id}] Indexing started for doc: {file_name}")
+ PromptStudioHelper._publish_log(
+ {"tool_id": tool_id, "run_id": run_id, "doc_name": file_name},
+ LogLevels.INFO,
+ LogLevels.RUN,
+ "Indexing started",
+ )
+ PromptStudioHelper.dynamic_indexer(
profile_manager=default_profile,
tool_id=tool_id,
file_path=file_path,
org_id=org_id,
document_id=document_id,
- is_summary=is_summary,
reindex=True,
run_id=run_id,
user_id=user_id,
- process_text=process_text,
- fs=fs_instance,
enable_highlight=tool.enable_highlight,
+ extracted_text=extracted_text,
+ doc_id_key=doc_id,
)
elapsed_time = time.time() - start_time
@@ -400,8 +441,43 @@ class PromptStudioHelper:
LogLevels.RUN,
f"Indexing successful, took {elapsed_time:.3f}s",
)
+ logger.info(f"Indexing successful : {doc_id}")
+ return doc_id
- return doc_id.get("output")
+ @staticmethod
+ def summarize(
+ file_name, org_id, document_id, is_summary, run_id, tool, doc_id
+ ) -> str:
+ cls = get_plugin_class_by_name(
+ name="summarizer",
+ plugins=PromptStudioHelper.processor_plugins,
+ )
+ usage_kwargs: dict[Any, Any] = dict()
+ usage_kwargs[ToolStudioPromptKeys.RUN_ID] = run_id
+ prompts: list[ToolStudioPrompt] = PromptStudioHelper.fetch_prompt_from_tool(
+ tool.tool_id
+ )
+ if cls:
+ summarize_file_path = cls.process(
+ tool_id=str(tool.tool_id),
+ file_name=file_name,
+ org_id=org_id,
+ user_id=tool.created_by.user_id,
+ usage_kwargs=usage_kwargs.copy(),
+ prompts=prompts,
+ )
+ profile_manager: ProfileManager = ProfileManager.objects.get(
+ prompt_studio_tool=tool, is_summarize_llm=True
+ )
+ default_profile = profile_manager
+ default_profile.chunk_size = 0
+ PromptStudioIndexHelper.handle_index_manager(
+ document_id=document_id,
+ is_summary=is_summary,
+ profile_manager=default_profile,
+ doc_id=doc_id,
+ )
+ return summarize_file_path
@staticmethod
def prompt_responder(
@@ -412,7 +488,6 @@ class PromptStudioHelper:
id: Optional[str] = None,
run_id: str = None,
profile_manager_id: Optional[str] = None,
- text_processor: Optional[type[Any]] = None,
) -> Any:
"""Execute chain/single run of the prompts. Makes a call to prompt
service and returns the dict of response.
@@ -448,7 +523,6 @@ class PromptStudioHelper:
document_id=document_id,
run_id=run_id,
profile_manager_id=profile_manager_id,
- text_processor=text_processor,
)
else:
return PromptStudioHelper._execute_prompts_in_single_pass(
@@ -456,10 +530,8 @@ class PromptStudioHelper:
doc_name=doc_name,
tool_id=tool_id,
org_id=org_id,
- user_id=user_id,
document_id=document_id,
run_id=run_id,
- text_processor=text_processor,
)
@staticmethod
@@ -473,7 +545,6 @@ class PromptStudioHelper:
document_id,
run_id,
profile_manager_id,
- text_processor: Optional[type[Any]] = None,
):
prompt_instance = PromptStudioHelper._fetch_prompt_from_id(id)
@@ -498,12 +569,6 @@ class PromptStudioHelper:
prompts = [prompt_instance]
tool = prompt_instance.tool_id
- if tool.summarize_as_source:
- directory, filename = os.path.split(doc_path)
- doc_path = os.path.join(
- directory, TSPKeys.SUMMARIZE, os.path.splitext(filename)[0] + ".txt"
- )
-
PromptStudioHelper._publish_log(
{
"tool_id": tool_id,
@@ -515,9 +580,6 @@ class PromptStudioHelper:
LogLevels.RUN,
"Invoking prompt service",
)
- process_text = None
- if text_processor:
- process_text = text_processor.process
try:
response = PromptStudioHelper._fetch_response(
doc_path=doc_path,
@@ -529,7 +591,6 @@ class PromptStudioHelper:
run_id=run_id,
profile_manager_id=profile_manager_id,
user_id=user_id,
- process_text=process_text,
)
return PromptStudioHelper._handle_response(
response=response,
@@ -564,10 +625,8 @@ class PromptStudioHelper:
doc_name,
tool_id,
org_id,
- user_id,
document_id,
run_id,
- text_processor: Optional[type[Any]] = None,
):
prompts = PromptStudioHelper.fetch_prompt_from_tool(tool_id)
prompts = [
@@ -588,21 +647,16 @@ class PromptStudioHelper:
LogLevels.RUN,
"Executing prompts in single pass",
)
- process_text = None
- if text_processor:
- process_text = text_processor.process
try:
tool = prompts[0].tool_id
response = PromptStudioHelper._fetch_single_pass_response(
- file_path=doc_path,
+ input_file_path=doc_path,
doc_name=doc_name,
tool=tool,
prompts=prompts,
org_id=org_id,
document_id=document_id,
run_id=run_id,
- user_id=user_id,
- process_text=process_text,
)
return PromptStudioHelper._handle_response(
response=response,
@@ -686,7 +740,6 @@ class PromptStudioHelper:
run_id: str,
user_id: str,
profile_manager_id: Optional[str] = None,
- process_text: Optional[Callable[[str], str]] = None,
) -> Any:
"""Utility function to invoke prompt service. Used internally.
@@ -751,18 +804,78 @@ class PromptStudioHelper:
storage_type=StorageType.PERMANENT,
env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,
)
+ file_path = doc_path
+ directory, filename = os.path.split(doc_path)
+ doc_path = os.path.join(
+ directory, "extract", os.path.splitext(filename)[0] + ".txt"
+ )
+ is_summary = tool.summarize_as_source
+ logger.info(f"Summary status : {is_summary}")
+ util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)
+ logger.info(
+ f"Passing file_path for fetching answer "
+ f"{file_path} : extraction path {doc_path}"
+ )
+ doc_id = IndexingUtils.generate_index_key(
+ vector_db=str(profile_manager.vector_store.id),
+ embedding=str(profile_manager.embedding_model.id),
+ x2text=str(profile_manager.x2text.id),
+ chunk_size=str(profile_manager.chunk_size),
+ chunk_overlap=str(profile_manager.chunk_overlap),
+ file_path=file_path,
+ file_hash=None,
+ fs=fs_instance,
+ tool=util,
+ )
+ logger.info(f"Extracting text from {file_path} for {doc_id}")
+ extracted_text = PromptStudioHelper.dynamic_extractor(
+ profile_manager=profile_manager,
+ file_path=file_path,
+ org_id=org_id,
+ document_id=document_id,
+ run_id=run_id,
+ enable_highlight=tool.enable_highlight,
+ doc_id=doc_id,
+ )
+ logger.info(f"Extracted text from {file_path} for {doc_id}")
+ if is_summary:
+ profile_manager.chunk_size = 0
+ doc_path = Path(doc_path) # Convert string to Path object
+ doc_path = str(
+ doc_path.parent.parent / "summarize" / (doc_path.stem + ".txt")
+ )
+ summarize_file_path = PromptStudioHelper.summarize(
+ filename, org_id, document_id, is_summary, run_id, tool, doc_id
+ )
+ summarize_doc_id = IndexingUtils.generate_index_key(
+ vector_db=str(default_profile.vector_store.id),
+ embedding=str(default_profile.embedding_model.id),
+ x2text=str(default_profile.x2text.id),
+ chunk_size=str(default_profile.chunk_size),
+ chunk_overlap=str(default_profile.chunk_overlap),
+ file_path=summarize_file_path,
+ fs=fs_instance,
+ tool=util,
+ )
+ PromptStudioIndexHelper.handle_index_manager(
+ document_id=document_id,
+ is_summary=is_summary,
+ profile_manager=profile_manager,
+ doc_id=summarize_doc_id,
+ )
+ logger.info("Summary enabled, set chunk to zero..")
+ logger.info(f"Indexing document {doc_path} for {doc_id}")
index_result = PromptStudioHelper.dynamic_indexer(
profile_manager=profile_manager,
file_path=doc_path,
tool_id=str(tool.tool_id),
org_id=org_id,
document_id=document_id,
- is_summary=tool.summarize_as_source,
run_id=run_id,
user_id=user_id,
- process_text=process_text,
- fs=fs_instance,
enable_highlight=tool.enable_highlight,
+ extracted_text=extracted_text,
+ doc_id_key=doc_id,
)
if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:
return {
@@ -787,6 +900,7 @@ class PromptStudioHelper:
output[TSPKeys.PROMPT] = prompt.prompt
output[TSPKeys.ACTIVE] = prompt.active
output[TSPKeys.REQUIRED] = prompt.required
+ logger.info(f"Chunk size set to {profile_manager.chunk_size} ")
output[TSPKeys.CHUNK_SIZE] = profile_manager.chunk_size
output[TSPKeys.VECTOR_DB] = vector_db
output[TSPKeys.EMBEDDING] = embedding_model
@@ -848,8 +962,6 @@ class PromptStudioHelper:
TSPKeys.EXECUTION_SOURCE: ExecutionSource.IDE.value,
}
- util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)
-
responder = PromptTool(
tool=util,
prompt_host=settings.PROMPT_HOST,
@@ -860,9 +972,7 @@ class PromptStudioHelper:
answer = responder.answer_prompt(
payload=payload, params=include_metadata, headers=headers
)
- # TODO: Make use of dataclasses
if answer["status"] == "ERROR":
- # TODO: Publish to FE logs from here
error_message = answer.get("error", "")
raise AnswerFetchError(
"Error while fetching response for "
@@ -914,12 +1024,11 @@ class PromptStudioHelper:
org_id: str,
document_id: str,
user_id: str,
- is_summary: bool = False,
+ extracted_text: str,
reindex: bool = False,
run_id: str = None,
- process_text: Optional[Callable[[str], str]] = None,
- fs: FileStorage = FileStorage(provider=FileStorageProvider.LOCAL),
enable_highlight: bool = False,
+ doc_id_key: Optional[str] = None,
) -> Any:
"""Used to index a file based on the passed arguments.
@@ -939,35 +1048,32 @@ class PromptStudioHelper:
Returns:
str: Index key for the combination of arguments
"""
+
+ if profile_manager.chunk_size == 0:
+ PromptStudioIndexHelper.handle_index_manager(
+ document_id=document_id,
+ profile_manager=profile_manager,
+ doc_id=doc_id_key,
+ )
+ logger.info("Skipping addition of nodes to VectoDB since chunk size is 0")
+ return {
+ "status": IndexingStatus.COMPLETED_STATUS.value,
+ "output": doc_id_key,
+ }
+
embedding_model = str(profile_manager.embedding_model.id)
vector_db = str(profile_manager.vector_store.id)
x2text_adapter = str(profile_manager.x2text.id)
- extract_file_path: Optional[str] = None
directory, filename = os.path.split(file_path)
- if not is_summary:
- extract_file_path = os.path.join(
- directory, "extract", os.path.splitext(filename)[0] + ".txt"
- )
- else:
- profile_manager.chunk_size = 0
-
+ file_path = os.path.join(
+ directory, "extract", os.path.splitext(filename)[0] + ".txt"
+ )
try:
usage_kwargs = {"run_id": run_id}
# Orginal file name with which file got uploaded in prompt studio
usage_kwargs["file_name"] = filename
- util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)
- tool_index = Index(tool=util)
- doc_id_key = tool_index.generate_index_key(
- vector_db=vector_db,
- embedding=embedding_model,
- x2text=x2text_adapter,
- chunk_size=str(profile_manager.chunk_size),
- chunk_overlap=str(profile_manager.chunk_overlap),
- file_path=file_path,
- file_hash=None,
- fs=fs,
- )
+
if not reindex:
indexed_doc_id = DocumentIndexingService.get_indexed_document_id(
org_id=org_id, user_id=user_id, doc_id_key=doc_id_key
@@ -990,25 +1096,44 @@ class PromptStudioHelper:
DocumentIndexingService.set_document_indexing(
org_id=org_id, user_id=user_id, doc_id_key=doc_id_key
)
- doc_id: str = tool_index.index(
- tool_id=tool_id,
- embedding_instance_id=embedding_model,
- vector_db_instance_id=vector_db,
- x2text_instance_id=x2text_adapter,
- file_path=file_path,
- chunk_size=profile_manager.chunk_size,
- chunk_overlap=profile_manager.chunk_overlap,
- reindex=reindex,
- output_file_path=extract_file_path,
- usage_kwargs=usage_kwargs.copy(),
- process_text=process_text,
- fs=fs,
- enable_highlight=enable_highlight,
+ logger.info(f"Invoking prompt service for indexing : {doc_id_key}")
+ payload = {
+ IKeys.TOOL_ID: tool_id,
+ IKeys.EMBEDDING_INSTANCE_ID: embedding_model,
+ IKeys.VECTOR_DB_INSTANCE_ID: vector_db,
+ IKeys.X2TEXT_INSTANCE_ID: x2text_adapter,
+ IKeys.FILE_PATH: file_path,
+ IKeys.FILE_HASH: None,
+ IKeys.CHUNK_OVERLAP: profile_manager.chunk_overlap,
+ IKeys.CHUNK_SIZE: profile_manager.chunk_size,
+ IKeys.REINDEX: reindex,
+ IKeys.ENABLE_HIGHLIGHT: enable_highlight,
+ IKeys.USAGE_KWARGS: usage_kwargs.copy(),
+ IKeys.EXTRACTED_TEXT: extracted_text,
+ IKeys.RUN_ID: run_id,
+ Common.LOG_EVENTS_ID: StateStore.get(Common.LOG_EVENTS_ID),
+ TSPKeys.EXECUTION_SOURCE: ExecutionSource.IDE.value,
+ }
+
+ util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)
+
+ responder = PromptTool(
+ tool=util,
+ prompt_host=settings.PROMPT_HOST,
+ prompt_port=settings.PROMPT_PORT,
)
+ headers = {Common.X_REQUEST_ID: StateStore.get(Common.REQUEST_ID)}
+ response = responder.index(payload=payload, headers=headers)
+ try:
+ response_text = PromptStudioHelper.handle_response(response)
+ doc_id = json.loads(response_text).get("doc_id")
+
+ except IndexingAPIError as e:
+ logger.error(f"Failed to index document: {str(e)}")
+ raise IndexingAPIError(f"Failed to index document: {str(e)}") from e
PromptStudioIndexHelper.handle_index_manager(
document_id=document_id,
- is_summary=is_summary,
profile_manager=profile_manager,
doc_id=doc_id,
)
@@ -1032,14 +1157,12 @@ class PromptStudioHelper:
@staticmethod
def _fetch_single_pass_response(
tool: CustomTool,
- file_path: str,
+ input_file_path: str,
doc_name: str,
prompts: list[ToolStudioPrompt],
org_id: str,
- user_id: str,
document_id: str,
run_id: str = None,
- process_text: Optional[Callable[[str], str]] = None,
) -> Any:
tool_id: str = str(tool.tool_id)
outputs: list[dict[str, Any]] = []
@@ -1058,7 +1181,7 @@ class PromptStudioHelper:
# has access to adapters configured in profile manager
PromptStudioHelper.validate_profile_manager_owner_access(default_profile)
default_profile.chunk_size = 0 # To retrive full context
-
+ util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)
if prompt_grammar:
for word, synonyms in prompt_grammar.items():
grammar.append({TSPKeys.WORD: word, TSPKeys.SYNONYMS: synonyms})
@@ -1070,25 +1193,31 @@ class PromptStudioHelper:
storage_type=StorageType.PERMANENT,
env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,
)
- index_result = PromptStudioHelper.dynamic_indexer(
+ directory, filename = os.path.split(input_file_path)
+ file_path = os.path.join(
+ directory, "extract", os.path.splitext(filename)[0] + ".txt"
+ )
+ doc_id = IndexingUtils.generate_index_key(
+ vector_db=str(default_profile.vector_store.id),
+ embedding=str(default_profile.embedding_model.id),
+ x2text=str(default_profile.x2text.id),
+ chunk_size=str(default_profile.chunk_size),
+ chunk_overlap=str(default_profile.chunk_overlap),
+ file_path=input_file_path,
+ file_hash=None,
+ fs=fs_instance,
+ tool=util,
+ )
+ PromptStudioHelper.dynamic_extractor(
profile_manager=default_profile,
- file_path=file_path,
- tool_id=tool_id,
+ file_path=input_file_path,
org_id=org_id,
- is_summary=tool.summarize_as_source,
document_id=document_id,
run_id=run_id,
- user_id=user_id,
- process_text=process_text,
- fs=fs_instance,
enable_highlight=tool.enable_highlight,
+ doc_id=doc_id,
)
- if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:
- return {
- "status": IndexingStatus.PENDING_STATUS.value,
- "message": IndexingStatus.DOCUMENT_BEING_INDEXED.value,
- }
-
+ # Indexing is not needed as Single pass is always non chunked.
vector_db = str(default_profile.vector_store.id)
embedding_model = str(default_profile.embedding_model.id)
llm = str(default_profile.llm.id)
@@ -1122,9 +1251,11 @@ class PromptStudioHelper:
if tool.summarize_as_source:
path = Path(file_path)
- file_path = str(path.parent / TSPKeys.SUMMARIZE / (path.stem + ".txt"))
+ file_path = str(
+ path.parent.parent / TSPKeys.SUMMARIZE / (path.stem + ".txt")
+ )
file_hash = fs_instance.get_hash_from_file(path=file_path)
-
+ logger.info("payload constructued, calling prompt service..")
payload = {
TSPKeys.TOOL_SETTINGS: tool_settings,
TSPKeys.OUTPUTS: outputs,
@@ -1137,8 +1268,6 @@ class PromptStudioHelper:
TSPKeys.EXECUTION_SOURCE: ExecutionSource.IDE.value,
}
- util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)
-
responder = PromptTool(
tool=util,
prompt_host=settings.PROMPT_HOST,
@@ -1147,11 +1276,13 @@ class PromptStudioHelper:
include_metadata = {TSPKeys.INCLUDE_METADATA: True}
headers = {Common.X_REQUEST_ID: StateStore.get(Common.REQUEST_ID)}
answer = responder.single_pass_extraction(
- payload=payload, params=include_metadata, headers=headers
+ payload=payload,
+ params=include_metadata,
+ headers=headers,
)
- # TODO: Make use of dataclasses
if answer["status"] == "ERROR":
error_message = answer.get("error", None)
+ logger.info(f"{str(answer)}")
raise AnswerFetchError(
f"Error while fetching response for prompt(s). {error_message}"
)
@@ -1165,3 +1296,89 @@ class PromptStudioHelper:
return tool
except CustomTool.DoesNotExist:
return None
+
+ @staticmethod
+ def dynamic_extractor(
+ file_path: str,
+ enable_highlight: bool,
+ run_id: str,
+ org_id: str,
+ profile_manager: ProfileManager,
+ document_id: str,
+ doc_id: str,
+ reindex: Optional[bool] = False,
+ ) -> str:
+ x2text = str(profile_manager.x2text.id)
+ is_extracted: bool = False
+ extract_file_path: Optional[str] = None
+ extracted_text = ""
+ directory, filename = os.path.split(file_path)
+ extract_file_path = os.path.join(
+ directory, "extract", os.path.splitext(filename)[0] + ".txt"
+ )
+ usage_kwargs = {"run_id": run_id}
+ # Orginal file name with which file got uploaded in prompt studio
+ usage_kwargs["file_name"] = filename
+ if is_extracted and not reindex:
+ is_extracted = PromptStudioIndexHelper.check_extraction_status(
+ document_id=document_id,
+ profile_manager=profile_manager,
+ doc_id=doc_id,
+ )
+ fs_instance = EnvHelper.get_storage(
+ storage_type=StorageType.PERMANENT,
+ env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,
+ )
+ try:
+ extracted_text = fs_instance.read(path=extract_file_path, mode="r")
+ logger.info("Extracted text found. Reading from file..")
+ return extracted_text
+ except FileNotFoundError as e:
+ logger.warning(
+ f"File not found for extraction. {extract_file_path}. {e}"
+ "Continuing extraction.."
+ )
+ extracted_text = None
+ payload = {
+ IKeys.X2TEXT_INSTANCE_ID: x2text,
+ IKeys.FILE_PATH: file_path,
+ IKeys.ENABLE_HIGHLIGHT: enable_highlight,
+ IKeys.USAGE_KWARGS: usage_kwargs.copy(),
+ IKeys.RUN_ID: run_id,
+ Common.LOG_EVENTS_ID: StateStore.get(Common.LOG_EVENTS_ID),
+ TSPKeys.EXECUTION_SOURCE: ExecutionSource.IDE.value,
+ IKeys.OUTPUT_FILE_PATH: extract_file_path,
+ }
+
+ util = PromptIdeBaseTool(log_level=LogLevel.INFO, org_id=org_id)
+
+ responder = PromptTool(
+ tool=util,
+ prompt_host=settings.PROMPT_HOST,
+ prompt_port=settings.PROMPT_PORT,
+ )
+ headers = {Common.X_REQUEST_ID: StateStore.get(Common.REQUEST_ID)}
+ response = responder.extract(payload=payload, headers=headers)
+ try:
+ response_data = PromptStudioHelper.handle_response(response)
+ extracted_text = json.loads(response_data)
+ PromptStudioIndexHelper.mark_extraction_status(
+ document_id=document_id,
+ profile_manager=profile_manager,
+ doc_id=doc_id,
+ )
+ except IndexingAPIError as e:
+ logger.error(f"Failed to extract document: {str(e)}")
+ raise IndexingAPIError(f"Failed to extract document: {str(e)}") from e
+
+ return extracted_text
+
+ @staticmethod
+ def handle_response(response: dict) -> dict:
+ """Handles API responses stored in dictionary format."""
+ status_code = response.get("status_code")
+ if status_code == 200:
+ return response.get("structure_output")
+ else:
+ error_message = response.get("error", "Unknown error")
+ raise IndexingAPIError(f"Error while fetching response. {error_message}")
diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.py
index 62cc2c23c..6d259afbf 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/views.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py
@@ -9,7 +9,8 @@ from django.http import HttpRequest
from file_management.constants import FileInformationKey as FileKey
from file_management.exceptions import FileNotFound
from permissions.permission import IsOwner, IsOwnerOrSharedUser
-from prompt_studio.processor_loader import get_plugin_class_by_name, load_plugins
+from prompt_studio.processor_loader import get_plugin_class_by_name
+from prompt_studio.processor_loader import load_plugins as load_processor_plugins
from prompt_studio.prompt_profile_manager_v2.constants import (
ProfileManagerErrors,
ProfileManagerKeys,
@@ -75,7 +76,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
serializer_class = CustomToolSerializer
- processor_plugins = load_plugins()
+ processor_plugins = load_processor_plugins()
def get_permissions(self) -> list[Any]:
if self.action == "destroy":
@@ -221,6 +222,8 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
file_name: str = document.document_name
# Generate a run_id
run_id = CommonUtils.generate_uuid()
+ is_summary = tool.summarize_context
+
unique_id = PromptStudioHelper.index_document(
tool_id=str(tool.tool_id),
file_name=file_name,
@@ -228,24 +231,8 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
user_id=tool.created_by.user_id,
document_id=document_id,
run_id=run_id,
+ is_summary=is_summary,
)
-
- usage_kwargs: dict[Any, Any] = dict()
- usage_kwargs[ToolStudioPromptKeys.RUN_ID] = run_id
- cls = get_plugin_class_by_name(
- name="summarizer",
- plugins=self.processor_plugins,
- )
- if cls:
- cls.process(
- tool_id=str(tool.tool_id),
- file_name=file_name,
- org_id=UserSessionUtils.get_organization_id(request),
- user_id=tool.created_by.user_id,
- document_id=document_id,
- usage_kwargs=usage_kwargs.copy(),
- )
-
if unique_id:
return Response(
{"message": "Document indexed successfully."},
diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/0002_indexmanager_extraction_status.py b/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/0002_indexmanager_extraction_status.py
new file mode 100644
index 000000000..b1b0789c1
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/0002_indexmanager_extraction_status.py
@@ -0,0 +1,20 @@
+# Generated by Django 4.2.1 on 2025-03-20 12:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("prompt_studio_index_manager_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="indexmanager",
+ name="extraction_status",
+ field=models.JSONField(
+ db_comment="Extraction status for documents", default=dict
+ ),
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/models.py b/backend/prompt_studio/prompt_studio_index_manager_v2/models.py
index e58ff87b0..328cd6031 100644
--- a/backend/prompt_studio/prompt_studio_index_manager_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_index_manager_v2/models.py
@@ -62,6 +62,12 @@ class IndexManager(BaseModel):
null=False,
blank=False,
)
+ extraction_status = models.JSONField(
+ db_comment="Extraction status for documents",
+ null=False,
+ blank=False,
+ default=dict,
+ )
created_by = models.ForeignKey(
User,
diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py b/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py
index 84747d52b..255045588 100644
--- a/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py
+++ b/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py
@@ -15,9 +15,9 @@ class PromptStudioIndexHelper:
@staticmethod
def handle_index_manager(
document_id: str,
- is_summary: bool,
profile_manager: ProfileManager,
doc_id: str,
+ is_summary: bool = False,
) -> IndexManager:
try:
@@ -67,3 +67,105 @@ class PromptStudioIndexHelper:
except Exception as e:
transaction.rollback()
raise IndexingAPIError("Error updating indexing status") from e
+
+ @staticmethod
+ def mark_extraction_status(
+ document_id: str, profile_manager: ProfileManager, doc_id: str
+ ) -> bool:
+ """
+ Marks the extraction status for a given document.
+
+ Args:
+ document_id (str): ID of the document in DocumentManager.
+ profile_manager (ProfileManager): ProfileManager instance for context.
+ doc_id (str): Unique identifier for the document within extraction status.
+
+ Returns:
+ bool: True if the status is successfully updated, False otherwise.
+
+ """
+ try:
+ with transaction.atomic():
+ document = DocumentManager.objects.get(pk=document_id)
+
+ args = {
+ "document_manager": document,
+ "profile_manager": profile_manager,
+ }
+
+ index_manager, created = IndexManager.objects.get_or_create(**args)
+
+ index_manager.extraction_status = index_manager.extraction_status or {}
+
+ index_manager.extraction_status[doc_id] = True
+ logger.info(
+ f"Index manager {index_manager} {index_manager.index_ids_history}"
+ )
+ index_manager.save(update_fields=["extraction_status"])
+
+ if created:
+ logger.info(
+ f"IndexManager entry created "
+ f"for document: {document_id} with {doc_id}"
+ )
+ else:
+ logger.info(
+ f"Updated extraction status "
+ f"for document: {document_id} with {doc_id}"
+ )
+ return True
+
+ except DocumentManager.DoesNotExist:
+ logger.error(f"Document with ID {document_id} does not exist.")
+ raise IndexingAPIError(
+ "Error occured while extracting. Please contact admin."
+ )
+
+ except Exception as e:
+ logger.error(f"Unexpected error updating extraction status: {e}")
+ raise IndexingAPIError(f"Error updating indexing status {str(e)}") from e
+
+ @staticmethod
+ def check_extraction_status(
+ document_id: str, profile_manager: ProfileManager, doc_id: str
+ ) -> bool:
+ """
+ Checks if the extraction status is already marked as complete
+ for the given document and doc_id.
+
+ Args:
+ document_id (str): ID of the document in DocumentManager.
+ profile_manager (ProfileManager): ProfileManager instance for context.
+ doc_id (str): Unique identifier for the document within extraction status.
+
+ Returns:
+ bool: True if extraction is complete, False otherwise.
+ """
+ try:
+ index_manager = IndexManager.objects.filter(
+ document_manager=document_id, profile_manager=profile_manager
+ ).first()
+
+ if not index_manager:
+ logger.info(f"No IndexManager record found for document: {document_id}")
+ return False
+
+ extraction_status = index_manager.extraction_status or {}
+ is_extracted = extraction_status.get(doc_id, False)
+
+ if is_extracted:
+ logger.info(
+ f"Extraction is already marked as complete "
+ f"for document: {document_id} with {doc_id}"
+ )
+ else:
+ logger.info(
+ f"Extraction is NOT yet marked as complete "
+ f"for document: {document_id} with {doc_id}"
+ )
+
+ return is_extracted
+
+ except Exception as e:
+ logger.error(f"Unexpected error while checking extraction status: {e}")
+ raise IndexingAPIError(f"Error checking extraction status {str(e)}") from e
diff --git a/prompt-service/pdm.lock b/prompt-service/pdm.lock
index 2bf185058..613e945cd 100644
--- a/prompt-service/pdm.lock
+++ b/prompt-service/pdm.lock
@@ -2687,7 +2687,7 @@ files = [
[[package]]
name = "openai"
-version = "1.70.0"
+version = "1.71.0"
requires_python = ">=3.8"
summary = "The official Python library for the openai API"
groups = ["default"]
@@ -2702,8 +2702,8 @@ dependencies = [
"typing-extensions<5,>=4.11",
]
files = [
- {file = "openai-1.70.0-py3-none-any.whl", hash = "sha256:f6438d053fd8b2e05fd6bef70871e832d9bbdf55e119d0ac5b92726f1ae6f614"},
- {file = "openai-1.70.0.tar.gz", hash = "sha256:e52a8d54c3efeb08cf58539b5b21a5abef25368b5432965e4de88cdf4e091b2b"},
+ {file = "openai-1.71.0-py3-none-any.whl", hash = "sha256:e1c643738f1fff1af52bce6ef06a7716c95d089281e7011777179614f32937aa"},
+ {file = "openai-1.71.0.tar.gz", hash = "sha256:52b20bb990a1780f9b0b8ccebac93416343ebd3e4e714e3eff730336833ca207"},
]
[[package]]
@@ -4577,66 +4577,69 @@ files = [
[[package]]
name = "yarl"
-version = "1.18.3"
+version = "1.19.0"
requires_python = ">=3.9"
summary = "Yet another URL library"
groups = ["default"]
dependencies = [
"idna>=2.0",
"multidict>=4.0",
- "propcache>=0.2.0",
-]
-files = [
- {file = "yarl-1.18.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7df647e8edd71f000a5208fe6ff8c382a1de8edfbccdbbfe649d263de07d8c34"},
- {file = "yarl-1.18.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c69697d3adff5aa4f874b19c0e4ed65180ceed6318ec856ebc423aa5850d84f7"},
- {file = "yarl-1.18.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:602d98f2c2d929f8e697ed274fbadc09902c4025c5a9963bf4e9edfc3ab6f7ed"},
- {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c654d5207c78e0bd6d749f6dae1dcbbfde3403ad3a4b11f3c5544d9906969dde"},
- {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5094d9206c64181d0f6e76ebd8fb2f8fe274950a63890ee9e0ebfd58bf9d787b"},
- {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35098b24e0327fc4ebdc8ffe336cee0a87a700c24ffed13161af80124b7dc8e5"},
- {file = "yarl-1.18.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3236da9272872443f81fedc389bace88408f64f89f75d1bdb2256069a8730ccc"},
- {file = "yarl-1.18.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2c08cc9b16f4f4bc522771d96734c7901e7ebef70c6c5c35dd0f10845270bcd"},
- {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:80316a8bd5109320d38eef8833ccf5f89608c9107d02d2a7f985f98ed6876990"},
- {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c1e1cc06da1491e6734f0ea1e6294ce00792193c463350626571c287c9a704db"},
- {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62"},
- {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e3b9fd71836999aad54084906f8663dffcd2a7fb5cdafd6c37713b2e72be1760"},
- {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:757e81cae69244257d125ff31663249b3013b5dc0a8520d73694aed497fb195b"},
- {file = "yarl-1.18.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b1771de9944d875f1b98a745bc547e684b863abf8f8287da8466cf470ef52690"},
- {file = "yarl-1.18.3-cp310-cp310-win32.whl", hash = "sha256:8874027a53e3aea659a6d62751800cf6e63314c160fd607489ba5c2edd753cf6"},
- {file = "yarl-1.18.3-cp310-cp310-win_amd64.whl", hash = "sha256:93b2e109287f93db79210f86deb6b9bbb81ac32fc97236b16f7433db7fc437d8"},
- {file = "yarl-1.18.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8503ad47387b8ebd39cbbbdf0bf113e17330ffd339ba1144074da24c545f0069"},
- {file = "yarl-1.18.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:02ddb6756f8f4517a2d5e99d8b2f272488e18dd0bfbc802f31c16c6c20f22193"},
- {file = "yarl-1.18.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:67a283dd2882ac98cc6318384f565bffc751ab564605959df4752d42483ad889"},
- {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d980e0325b6eddc81331d3f4551e2a333999fb176fd153e075c6d1c2530aa8a8"},
- {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b643562c12680b01e17239be267bc306bbc6aac1f34f6444d1bded0c5ce438ca"},
- {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c017a3b6df3a1bd45b9fa49a0f54005e53fbcad16633870104b66fa1a30a29d8"},
- {file = "yarl-1.18.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75674776d96d7b851b6498f17824ba17849d790a44d282929c42dbb77d4f17ae"},
- {file = "yarl-1.18.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ccaa3a4b521b780a7e771cc336a2dba389a0861592bbce09a476190bb0c8b4b3"},
- {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2d06d3005e668744e11ed80812e61efd77d70bb7f03e33c1598c301eea20efbb"},
- {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:9d41beda9dc97ca9ab0b9888cb71f7539124bc05df02c0cff6e5acc5a19dcc6e"},
- {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ba23302c0c61a9999784e73809427c9dbedd79f66a13d84ad1b1943802eaaf59"},
- {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6748dbf9bfa5ba1afcc7556b71cda0d7ce5f24768043a02a58846e4a443d808d"},
- {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0b0cad37311123211dc91eadcb322ef4d4a66008d3e1bdc404808992260e1a0e"},
- {file = "yarl-1.18.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0fb2171a4486bb075316ee754c6d8382ea6eb8b399d4ec62fde2b591f879778a"},
- {file = "yarl-1.18.3-cp311-cp311-win32.whl", hash = "sha256:61b1a825a13bef4a5f10b1885245377d3cd0bf87cba068e1d9a88c2ae36880e1"},
- {file = "yarl-1.18.3-cp311-cp311-win_amd64.whl", hash = "sha256:b9d60031cf568c627d028239693fd718025719c02c9f55df0a53e587aab951b5"},
- {file = "yarl-1.18.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:61e5e68cb65ac8f547f6b5ef933f510134a6bf31bb178be428994b0cb46c2a04"},
- {file = "yarl-1.18.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719"},
- {file = "yarl-1.18.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a440a2a624683108a1b454705ecd7afc1c3438a08e890a1513d468671d90a04e"},
- {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09c7907c8548bcd6ab860e5f513e727c53b4a714f459b084f6580b49fa1b9cee"},
- {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4f6450109834af88cb4cc5ecddfc5380ebb9c228695afc11915a0bf82116789"},
- {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9ca04806f3be0ac6d558fffc2fdf8fcef767e0489d2684a21912cc4ed0cd1b8"},
- {file = "yarl-1.18.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77a6e85b90a7641d2e07184df5557132a337f136250caafc9ccaa4a2a998ca2c"},
- {file = "yarl-1.18.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6333c5a377c8e2f5fae35e7b8f145c617b02c939d04110c76f29ee3676b5f9a5"},
- {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0b3c92fa08759dbf12b3a59579a4096ba9af8dd344d9a813fc7f5070d86bbab1"},
- {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:4ac515b860c36becb81bb84b667466885096b5fc85596948548b667da3bf9f24"},
- {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:045b8482ce9483ada4f3f23b3774f4e1bf4f23a2d5c912ed5170f68efb053318"},
- {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:a4bb030cf46a434ec0225bddbebd4b89e6471814ca851abb8696170adb163985"},
- {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:54d6921f07555713b9300bee9c50fb46e57e2e639027089b1d795ecd9f7fa910"},
- {file = "yarl-1.18.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1d407181cfa6e70077df3377938c08012d18893f9f20e92f7d2f314a437c30b1"},
- {file = "yarl-1.18.3-cp39-cp39-win32.whl", hash = "sha256:ac36703a585e0929b032fbaab0707b75dc12703766d0b53486eabd5139ebadd5"},
- {file = "yarl-1.18.3-cp39-cp39-win_amd64.whl", hash = "sha256:ba87babd629f8af77f557b61e49e7c7cac36f22f871156b91e10a6e9d4f829e9"},
- {file = "yarl-1.18.3-py3-none-any.whl", hash = "sha256:b57f4f58099328dfb26c6a771d09fb20dbbae81d20cfb66141251ea063bd101b"},
- {file = "yarl-1.18.3.tar.gz", hash = "sha256:ac1801c45cbf77b6c99242eeff4fffb5e4e73a800b5c4ad4fc0be5def634d2e1"},
+ "propcache>=0.2.1",
+]
+files = [
+ {file = "yarl-1.19.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0bae32f8ebd35c04d6528cedb4a26b8bf25339d3616b04613b97347f919b76d3"},
+ {file = "yarl-1.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8015a076daf77823e7ebdcba474156587391dab4e70c732822960368c01251e6"},
+ {file = "yarl-1.19.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9973ac95327f5d699eb620286c39365990b240031672b5c436a4cd00539596c5"},
+ {file = "yarl-1.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd4b5fbd7b9dde785cfeb486b8cca211a0b138d4f3a7da27db89a25b3c482e5c"},
+ {file = "yarl-1.19.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:75460740005de5a912b19f657848aef419387426a40f581b1dc9fac0eb9addb5"},
+ {file = "yarl-1.19.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:57abd66ca913f2cfbb51eb3dbbbac3648f1f6983f614a4446e0802e241441d2a"},
+ {file = "yarl-1.19.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:46ade37911b7c99ce28a959147cb28bffbd14cea9e7dd91021e06a8d2359a5aa"},
+ {file = "yarl-1.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8346ec72ada749a6b5d82bff7be72578eab056ad7ec38c04f668a685abde6af0"},
+ {file = "yarl-1.19.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e4cb14a6ee5b6649ccf1c6d648b4da9220e8277d4d4380593c03cc08d8fe937"},
+ {file = "yarl-1.19.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:66fc1c2926a73a2fb46e4b92e3a6c03904d9bc3a0b65e01cb7d2b84146a8bd3b"},
+ {file = "yarl-1.19.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:5a70201dd1e0a4304849b6445a9891d7210604c27e67da59091d5412bc19e51c"},
+ {file = "yarl-1.19.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4807aab1bdeab6ae6f296be46337a260ae4b1f3a8c2fcd373e236b4b2b46efd"},
+ {file = "yarl-1.19.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ae584afe81a1de4c1bb06672481050f0d001cad13163e3c019477409f638f9b7"},
+ {file = "yarl-1.19.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:30eaf4459df6e91f21b2999d1ee18f891bcd51e3cbe1de301b4858c84385895b"},
+ {file = "yarl-1.19.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0e617d45d03c8dec0dfce6f51f3e1b8a31aa81aaf4a4d1442fdb232bcf0c6d8c"},
+ {file = "yarl-1.19.0-cp310-cp310-win32.whl", hash = "sha256:32ba32d0fa23893fd8ea8d05bdb05de6eb19d7f2106787024fd969f4ba5466cb"},
+ {file = "yarl-1.19.0-cp310-cp310-win_amd64.whl", hash = "sha256:545575ecfcd465891b51546c2bcafdde0acd2c62c2097d8d71902050b20e4922"},
+ {file = "yarl-1.19.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:163ff326680de5f6d4966954cf9e3fe1bf980f5fee2255e46e89b8cf0f3418b5"},
+ {file = "yarl-1.19.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a626c4d9cca298d1be8625cff4b17004a9066330ac82d132bbda64a4c17c18d3"},
+ {file = "yarl-1.19.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:961c3e401ea7f13d02b8bb7cb0c709152a632a6e14cdc8119e9c6ee5596cd45d"},
+ {file = "yarl-1.19.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a39d7b807ab58e633ed760f80195cbd145b58ba265436af35f9080f1810dfe64"},
+ {file = "yarl-1.19.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4228978fb59c6b10f60124ba8e311c26151e176df364e996f3f8ff8b93971b5"},
+ {file = "yarl-1.19.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9ba536b17ecf3c74a94239ec1137a3ad3caea8c0e4deb8c8d2ffe847d870a8c5"},
+ {file = "yarl-1.19.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a251e00e445d2e9df7b827c9843c0b87f58a3254aaa3f162fb610747491fe00f"},
+ {file = "yarl-1.19.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9b92431d8b4d4ca5ccbfdbac95b05a3a6cd70cd73aa62f32f9627acfde7549c"},
+ {file = "yarl-1.19.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec2f56edaf476f70b5831bbd59700b53d9dd011b1f77cd4846b5ab5c5eafdb3f"},
+ {file = "yarl-1.19.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:acf9b92c4245ac8b59bc7ec66a38d3dcb8d1f97fac934672529562bb824ecadb"},
+ {file = "yarl-1.19.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:57711f1465c06fee8825b95c0b83e82991e6d9425f9a042c3c19070a70ac92bf"},
+ {file = "yarl-1.19.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:528e86f5b1de0ad8dd758ddef4e0ed24f5d946d4a1cef80ffb2d4fca4e10f122"},
+ {file = "yarl-1.19.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:3b77173663e075d9e5a57e09d711e9da2f3266be729ecca0b8ae78190990d260"},
+ {file = "yarl-1.19.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d8717924cf0a825b62b1a96fc7d28aab7f55a81bf5338b8ef41d7a76ab9223e9"},
+ {file = "yarl-1.19.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0df9f0221a78d858793f40cbea3915c29f969c11366646a92ca47e080a14f881"},
+ {file = "yarl-1.19.0-cp311-cp311-win32.whl", hash = "sha256:8b3ade62678ee2c7c10dcd6be19045135e9badad53108f7d2ed14896ee396045"},
+ {file = "yarl-1.19.0-cp311-cp311-win_amd64.whl", hash = "sha256:0626ee31edb23ac36bdffe607231de2cca055ad3a5e2dc5da587ef8bc6a321bc"},
+ {file = "yarl-1.19.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:85ac908cd5a97bbd3048cca9f1bf37b932ea26c3885099444f34b0bf5d5e9fa6"},
+ {file = "yarl-1.19.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6ba0931b559f1345df48a78521c31cfe356585670e8be22af84a33a39f7b9221"},
+ {file = "yarl-1.19.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5bc503e1c1fee1b86bcb58db67c032957a52cae39fe8ddd95441f414ffbab83e"},
+ {file = "yarl-1.19.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d995122dcaf180fd4830a9aa425abddab7c0246107c21ecca2fa085611fa7ce9"},
+ {file = "yarl-1.19.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:217f69e60a14da4eed454a030ea8283f8fbd01a7d6d81e57efb865856822489b"},
+ {file = "yarl-1.19.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aad67c8f13a4b79990082f72ef09c078a77de2b39899aabf3960a48069704973"},
+ {file = "yarl-1.19.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dff065a1a8ed051d7e641369ba1ad030d5a707afac54cf4ede7069b959898835"},
+ {file = "yarl-1.19.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ada882e26b16ee651ab6544ce956f2f4beaed38261238f67c2a96db748e17741"},
+ {file = "yarl-1.19.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:67a56b1acc7093451ea2de0687aa3bd4e58d6b4ef6cbeeaad137b45203deaade"},
+ {file = "yarl-1.19.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:e97d2f0a06b39e231e59ebab0e6eec45c7683b339e8262299ac952707bdf7688"},
+ {file = "yarl-1.19.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:a5288adb7c59d0f54e4ad58d86fb06d4b26e08a59ed06d00a1aac978c0e32884"},
+ {file = "yarl-1.19.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1efbf4d03e6eddf5da27752e0b67a8e70599053436e9344d0969532baa99df53"},
+ {file = "yarl-1.19.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:f228f42f29cc87db67020f7d71624102b2c837686e55317b16e1d3ef2747a993"},
+ {file = "yarl-1.19.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:c515f7dd60ca724e4c62b34aeaa603188964abed2eb66bb8e220f7f104d5a187"},
+ {file = "yarl-1.19.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4815ec6d3d68a96557fa71bd36661b45ac773fb50e5cfa31a7e843edb098f060"},
+ {file = "yarl-1.19.0-cp39-cp39-win32.whl", hash = "sha256:9fac2dd1c5ecb921359d9546bc23a6dcc18c6acd50c6d96f118188d68010f497"},
+ {file = "yarl-1.19.0-cp39-cp39-win_amd64.whl", hash = "sha256:5864f539ce86b935053bfa18205fa08ce38e9a40ea4d51b19ce923345f0ed5db"},
+ {file = "yarl-1.19.0-py3-none-any.whl", hash = "sha256:a727101eb27f66727576630d02985d8a065d09cd0b5fcbe38a5793f71b2a97ef"},
+ {file = "yarl-1.19.0.tar.gz", hash = "sha256:01e02bb80ae0dbed44273c304095295106e1d9470460e773268a27d11e594892"},
]
[[package]]
diff --git a/prompt-service/pyproject.toml b/prompt-service/pyproject.toml
index 7f857ab7d..725f4282c 100644
--- a/prompt-service/pyproject.toml
+++ b/prompt-service/pyproject.toml
@@ -51,9 +51,6 @@ test = [
]
[tool.pdm.scripts]
-prompt_service.cmd = "flask --app src/unstract/prompt_service/main.py run --port 3003"
+prompt_service.cmd = "flask --app src/unstract/prompt_service/run.py run --port 3003"
prompt_service.env_file = ".env"
prompt_service.help = "Runs the Unstract prompt service"
-prompt_service_v2.cmd = "flask --app src/unstract/prompt_service_v2/run.py run --port 3003"
-prompt_service_v2.env_file = ".env"
-prompt_service_v2.help = "Runs the Unstract prompt service v2"
diff --git a/prompt-service/sample.env b/prompt-service/sample.env
index f31edd9f7..cb0aeb73e 100644
--- a/prompt-service/sample.env
+++ b/prompt-service/sample.env
@@ -37,3 +37,8 @@ FLIPT_SERVICE_AVAILABLE=False
PERMANENT_REMOTE_STORAGE='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'
TEMPORARY_REMOTE_STORAGE='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'
REMOTE_PROMPT_STUDIO_FILE_PATH="unstract/prompt-studio-data/"
+
+
+#X2Text Service
+X2TEXT_HOST=http://unstract-x2text-service
+X2TEXT_PORT=3004
diff --git a/prompt-service/src/unstract/prompt_service/authentication_middleware.py b/prompt-service/src/unstract/prompt_service/authentication_middleware.py
deleted file mode 100644
index 52297d8e1..000000000
--- a/prompt-service/src/unstract/prompt_service/authentication_middleware.py
+++ /dev/null
@@ -1,78 +0,0 @@
-from typing import Optional
-
-from flask import Request, current_app
-from unstract.prompt_service.config import db
-from unstract.prompt_service.constants import DBTableV2
-from unstract.prompt_service.db_utils import DBUtils
-from unstract.prompt_service.env_manager import EnvLoader
-
-DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract")
-
-
-class AuthenticationMiddleware:
-
- @staticmethod
- def validate_bearer_token(token: Optional[str]) -> bool:
- try:
- if token is None:
- current_app.logger.error("Authentication failed. Empty bearer token")
- return False
-
- platform_key_table = f'"{DB_SCHEMA}".{DBTableV2.PLATFORM_KEY}'
-
- query = f"SELECT * FROM {platform_key_table} WHERE key = '{token}'"
- cursor = db.execute_sql(query)
- result_row = cursor.fetchone()
- cursor.close()
- if not result_row or len(result_row) == 0:
- current_app.logger.error(
- f"Authentication failed. bearer token not found {token}"
- )
- return False
- platform_key = str(result_row[1])
- is_active = bool(result_row[2])
- if not is_active:
- current_app.logger.error(
- f"Token is not active. Activate \
- before using it. token {token}"
- )
- return False
- if platform_key != token:
- current_app.logger.error(
- f"Authentication failed. Invalid bearer token: {token}"
- )
- return False
-
- except Exception as e:
- current_app.logger.error(
- f"Error while validating bearer token: {e}",
- stack_info=True,
- exc_info=True,
- )
- return False
- return True
-
- @staticmethod
- def get_token_from_auth_header(request: Request) -> Optional[str]:
- try:
- bearer_token = request.headers.get("Authorization")
- if not bearer_token:
- return None
- token: str = bearer_token.strip().replace("Bearer ", "")
- return token
- except Exception as e:
- current_app.logger.info(f"Exception while getting token {e}")
- return None
-
- @staticmethod
- def get_account_from_bearer_token(token: Optional[str]) -> str:
- platform_key_table = DBTableV2.PLATFORM_KEY
- organization_table = DBTableV2.ORGANIZATION
-
- query = f"SELECT organization_id FROM {platform_key_table} WHERE key='{token}'"
- organization = DBUtils.execute_query(query)
- query_org = (
- f"SELECT schema_name FROM {organization_table} WHERE id='{organization}'"
- )
- schema_name: str = DBUtils.execute_query(query_org)
- return schema_name
diff --git a/prompt-service/src/unstract/prompt_service/config.py b/prompt-service/src/unstract/prompt_service/config.py
index ad9220a39..f5e993c06 100644
--- a/prompt-service/src/unstract/prompt_service/config.py
+++ b/prompt-service/src/unstract/prompt_service/config.py
@@ -1,53 +1,30 @@
import logging
-from logging.config import dictConfig
from os import environ as env
from dotenv import load_dotenv
from flask import Flask
-from peewee import PostgresqlDatabase
+from unstract.prompt_service.controllers import api
+from unstract.prompt_service.extensions import db
+from unstract.prompt_service.helpers.lifecycle import register_lifecycle_hooks
+from unstract.prompt_service.helpers.plugin import plugin_loader
+from unstract.prompt_service.utils.env_loader import get_env_or_die
from unstract.sdk.constants import LogLevel
-load_dotenv()
-
-dictConfig(
- {
- "version": 1,
- "formatters": {
- "default": {
- "format": "[%(asctime)s] %(levelname)s in %(module)s: %(message)s",
- }
- },
- "handlers": {
- "wsgi": {
- "class": "logging.StreamHandler",
- "stream": "ext://flask.logging.wsgi_errors_stream",
- "formatter": "default",
- }
- },
- "root": {"level": "INFO", "handlers": ["wsgi"]},
- }
-)
-
-db = PostgresqlDatabase(None)
+from unstract.core.flask import register_error_handlers, register_request_id_middleware
+from unstract.core.flask.logging import setup_logging
-
-def get_env_or_die(env_key: str) -> str:
- env_value = env.get(env_key)
- if not env_value:
- raise ValueError(f"Env variable {env_key} is required")
- return env_value
+load_dotenv()
def create_app() -> Flask:
- app = Flask("prompt-service")
- log_level = env.get("LOG_LEVEL", LogLevel.WARN.value)
- if log_level == LogLevel.DEBUG.value:
- app.logger.setLevel(logging.DEBUG)
- elif log_level == LogLevel.INFO.value:
- app.logger.setLevel(logging.INFO)
- else:
- app.logger.setLevel(logging.WARNING)
+ """Creates and configures the Flask application."""
+ log_level = env.get("LOG_LEVEL", LogLevel.INFO.value).upper()
+ setup_logging(log_level)
+ log_level = getattr(logging, log_level, logging.INFO)
+ app = Flask("prompt-service")
+ app.logger.setLevel(log_level)
+ app.logger.info("Initializing Flask application...")
# Load required environment variables
db_host = get_env_or_die("PG_BE_HOST")
db_port = get_env_or_die("PG_BE_PORT")
@@ -65,4 +42,13 @@ def create_app() -> Flask:
port=db_port,
options=f"-c application_name={application_name}",
)
+
+ # Load plugins
+ plugin_loader(app)
+ register_lifecycle_hooks(app)
+ register_request_id_middleware(app)
+ register_error_handlers(app)
+ app.register_blueprint(api)
+
+ app.logger.info("Flask app created successfully.")
return app
diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.py
index 2f007d731..2ad922183 100644
--- a/prompt-service/src/unstract/prompt_service/constants.py
+++ b/prompt-service/src/unstract/prompt_service/constants.py
@@ -1,7 +1,9 @@
from enum import Enum
-class PromptServiceContants:
+class PromptServiceConstants:
+ """Constants used in the prompt service."""
+
WORD = "word"
SYNONYMS = "synonyms"
OUTPUTS = "outputs"
@@ -80,6 +82,16 @@ class PromptServiceContants:
LINE_ITEM = "line-item"
LINE_NUMBERS = "line_numbers"
WHISPER_HASH = "whisper_hash"
+ PAID_FEATURE_MSG = (
+ "It is a cloud / enterprise feature. If you have purchased a plan and still "
+ "face this issue, please contact support"
+ )
+ NO_CONTEXT_ERROR = (
+ "Couldn't fetch context from vector DB. "
+ "This happens usually due to a delay by the Vector DB "
+ "provider to confirm writes to DB. "
+ "Please try again after some time"
+ )
class RunLevel(Enum):
@@ -105,15 +117,59 @@ class DBTableV2:
class FileStorageKeys:
+ """File storage keys."""
+
PERMANENT_REMOTE_STORAGE = "PERMANENT_REMOTE_STORAGE"
TEMPORARY_REMOTE_STORAGE = "TEMPORARY_REMOTE_STORAGE"
class FileStorageType(Enum):
+ """File storage type."""
+
PERMANENT = "permanent"
TEMPORARY = "temporary"
class ExecutionSource(Enum):
+ """Execution source."""
+
IDE = "ide"
TOOL = "tool"
+
+
+class VariableType(str, Enum):
+ """Type of variable."""
+
+ STATIC = "STATIC"
+ DYNAMIC = "DYNAMIC"
+
+
+class VariableConstants:
+ """Constants for variable extraction."""
+
+ VARIABLE_REGEX = "{{(.+?)}}"
+ DYNAMIC_VARIABLE_DATA_REGEX = r"\[(.*?)\]"
+ DYNAMIC_VARIABLE_URL_REGEX = r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))" # noqa: E501
+
+
+class IndexingConstants:
+ TOOL_ID = "tool_id"
+ EMBEDDING_INSTANCE_ID = "embedding_instance_id"
+ VECTOR_DB_INSTANCE_ID = "vector_db_instance_id"
+ X2TEXT_INSTANCE_ID = "x2text_instance_id"
+ FILE_PATH = "file_path"
+ CHUNK_SIZE = "chunk_size"
+ CHUNK_OVERLAP = "chunk_overlap"
+ REINDEX = "reindex"
+ FILE_HASH = "file_hash"
+ OUTPUT_FILE_PATH = "output_file_path"
+ ENABLE_HIGHLIGHT = "enable_highlight"
+ USAGE_KWARGS = "usage_kwargs"
+ PROCESS_TEXT = "process_text"
+ EXTRACTED_TEXT = "extracted_text"
+ TAGS = "tags"
+ EXECUTION_SOURCE = "execution_source"
+ DOC_ID = "doc_id"
+ TOOL_EXECUTION_METATADA = "tool_execution_metadata"
+ EXECUTION_DATA_DIR = "execution_data_dir"
+ METADATA_FILE = "METADATA.json"
diff --git a/prompt-service/src/unstract/prompt_service_v2/controllers/__init__.py b/prompt-service/src/unstract/prompt_service/controllers/__init__.py
similarity index 100%
rename from prompt-service/src/unstract/prompt_service_v2/controllers/__init__.py
rename to prompt-service/src/unstract/prompt_service/controllers/__init__.py
diff --git a/prompt-service/src/unstract/prompt_service_v2/controllers/answer_prompt.py b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py
similarity index 84%
rename from prompt-service/src/unstract/prompt_service_v2/controllers/answer_prompt.py
rename to prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py
index 308c19267..1184f9e45 100644
--- a/prompt-service/src/unstract/prompt_service_v2/controllers/answer_prompt.py
+++ b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py
@@ -9,21 +9,19 @@ from typing import Any
from flask import Blueprint
from flask import current_app as app
from flask import request
-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys
-from unstract.prompt_service_v2.constants import RunLevel
-from unstract.prompt_service_v2.exceptions import BadRequest
-from unstract.prompt_service_v2.helpers.auth import AuthHelper
-from unstract.prompt_service_v2.helpers.plugin import PluginManager
-from unstract.prompt_service_v2.helpers.prompt_ide_base_tool import (
- PromptServiceBaseTool,
-)
-from unstract.prompt_service_v2.helpers.usage import UsageHelper
-from unstract.prompt_service_v2.services.answer_prompt import AnswerPromptService
-from unstract.prompt_service_v2.services.retrieval import RetrievalService
-from unstract.prompt_service_v2.services.variable_replacement import (
+from unstract.prompt_service.constants import PromptServiceConstants as PSKeys
+from unstract.prompt_service.constants import RunLevel
+from unstract.prompt_service.exceptions import BadRequest
+from unstract.prompt_service.helpers.auth import AuthHelper
+from unstract.prompt_service.helpers.plugin import PluginManager
+from unstract.prompt_service.helpers.prompt_ide_base_tool import PromptServiceBaseTool
+from unstract.prompt_service.helpers.usage import UsageHelper
+from unstract.prompt_service.services.answer_prompt import AnswerPromptService
+from unstract.prompt_service.services.retrieval import RetrievalService
+from unstract.prompt_service.services.variable_replacement import (
VariableReplacementService,
)
-from unstract.prompt_service_v2.utils.log import publish_log
+from unstract.prompt_service.utils.log import publish_log
from unstract.sdk.adapters.llm.no_op.src.no_op_custom_llm import NoOpCustomLLM
from unstract.sdk.constants import LogLevel
from unstract.sdk.embedding import Embedding
@@ -46,11 +44,11 @@ def prompt_processor() -> Any:
raise BadRequest
tool_settings = payload.get(PSKeys.TOOL_SETTINGS, {})
enable_challenge = tool_settings.get(PSKeys.ENABLE_CHALLENGE, False)
- challenge_llm = None
- # TODO: Rename "outputs" to "prompts" in payload
+ # Rename "outputs" to "prompts" in payload
prompts = payload.get(PSKeys.OUTPUTS, [])
tool_id: str = payload.get(PSKeys.TOOL_ID, "")
run_id: str = payload.get(PSKeys.RUN_ID, "")
+ execution_id: str = payload.get(PSKeys.EXECUTION_ID, "")
file_hash = payload.get(PSKeys.FILE_HASH)
file_path = payload.get(PSKeys.FILE_PATH)
doc_name = str(payload.get(PSKeys.FILE_NAME, ""))
@@ -73,7 +71,7 @@ def prompt_processor() -> Any:
RunLevel.RUN,
f"Preparing to execute {len(prompts)} prompt(s)",
)
- # TODO: Rename "output" to "prompt"
+ # Rename "output" to "prompt"
for output in prompts: # type:ignore
variable_names.append(output[PSKeys.NAME])
metadata[PSKeys.REQUIRED_FIELDS][output[PSKeys.NAME]] = output.get(
@@ -84,6 +82,7 @@ def prompt_processor() -> Any:
prompt_name = output[PSKeys.NAME]
prompt_text = output[PSKeys.PROMPT]
chunk_size = output[PSKeys.CHUNK_SIZE]
+ app.logger.info(f"[{tool_id}] chunk size: {chunk_size}")
util = PromptServiceBaseTool(platform_key=platform_key)
index = Index(tool=util, run_id=run_id, capture_metrics=True)
if VariableReplacementService.is_variables_present(prompt_text=prompt_text):
@@ -137,7 +136,7 @@ def prompt_processor() -> Any:
)
try:
- usage_kwargs = {"run_id": run_id}
+ usage_kwargs = {"run_id": run_id, "execution_id": execution_id}
adapter_instance_id = output[PSKeys.LLM]
llm = LLM(
tool=util,
@@ -244,90 +243,54 @@ def prompt_processor() -> Any:
raise e
try:
- if chunk_size == 0:
- # We can do this only for chunkless indexes
- context: set[str] = RetrievalService.fetch_context_from_vector_db(
- index=index,
- output=output,
- doc_id=doc_id,
- tool_id=tool_id,
- doc_name=doc_name,
- prompt_name=prompt_name,
- log_events_id=log_events_id,
- usage_kwargs=usage_kwargs,
- )
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.INFO,
- RunLevel.RUN,
- "Retrieving answer from LLM",
- )
- answer = AnswerPromptService.construct_and_run_prompt(
+
+ answer = "NA"
+ publish_log(
+ log_events_id,
+ {
+ "tool_id": tool_id,
+ "prompt_key": prompt_name,
+ "doc_name": doc_name,
+ },
+ LogLevel.INFO,
+ RunLevel.RUN,
+ "Retrieving context from adapter",
+ )
+
+ retrieval_strategy = output.get(PSKeys.RETRIEVAL_STRATEGY)
+
+ if retrieval_strategy in {PSKeys.SIMPLE, PSKeys.SUBQUESTION}:
+ app.logger.info(f"[{tool_id}] Performing retrieval for : {file_path}")
+ answer, context = RetrievalService.perform_retrieval(
tool_settings=tool_settings,
output=output,
+ doc_id=doc_id,
llm=llm,
- context="\n".join(context),
- prompt="promptx",
+ vector_db=vector_db,
+ retrieval_type=retrieval_strategy,
metadata=metadata,
- file_path=file_path,
+ chunk_size=chunk_size,
execution_source=execution_source,
+ file_path=file_path,
)
- metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = (
- AnswerPromptService.get_cleaned_context(context)
- )
+ metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = context
else:
- answer = "NA"
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.INFO,
- RunLevel.RUN,
- "Retrieving context from adapter",
+ app.logger.info(
+ "Invalid retrieval strategy passed: %s",
+ retrieval_strategy,
)
- retrieval_strategy = output.get(PSKeys.RETRIEVAL_STRATEGY)
-
- if retrieval_strategy in {PSKeys.SIMPLE, PSKeys.SUBQUESTION}:
- vector_index = vector_db.get_vector_store_index()
- answer, context = RetrievalService.run_retrieval(
- tool_settings=tool_settings,
- output=output,
- doc_id=doc_id,
- llm=llm,
- vector_index=vector_index,
- retrieval_type=retrieval_strategy,
- metadata=metadata,
- execution_source=execution_source,
- )
- metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = (
- AnswerPromptService.get_cleaned_context(context)
- )
- else:
- app.logger.info(
- "Invalid retrieval strategy passed: %s",
- retrieval_strategy,
- )
-
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.DEBUG,
- RunLevel.RUN,
- "Retrieved context from adapter",
- )
+ publish_log(
+ log_events_id,
+ {
+ "tool_id": tool_id,
+ "prompt_key": prompt_name,
+ "doc_name": doc_name,
+ },
+ LogLevel.DEBUG,
+ RunLevel.RUN,
+ "Retrieved context from adapter",
+ )
publish_log(
log_events_id,
@@ -345,7 +308,7 @@ def prompt_processor() -> Any:
if answer.lower() == "na":
structured_output[output[PSKeys.NAME]] = None
else:
- # TODO: Extract these prompts as constants after pkging
+ # Extract these prompts as constants after pkging
prompt = f"Extract the number from the following \
text:\n{answer}\n\nOutput just the number. \
If the number is expressed in millions \
@@ -446,7 +409,7 @@ def prompt_processor() -> Any:
f"Candidate JSON: {answer}"
)
app.logger.info(err_msg, LogLevel.ERROR)
- # TODO: Format log message after unifying these types
+ # Format log message after unifying these types
publish_log(
log_events_id,
{
@@ -557,7 +520,7 @@ def prompt_processor() -> Any:
try:
evaluator = eval_plugin["entrypoint_cls"](
"",
- "\n".join(context),
+ "".join(context),
"",
"",
output,
diff --git a/prompt-service/src/unstract/prompt_service_v2/controllers/extraction.py b/prompt-service/src/unstract/prompt_service/controllers/extraction.py
similarity index 81%
rename from prompt-service/src/unstract/prompt_service_v2/controllers/extraction.py
rename to prompt-service/src/unstract/prompt_service/controllers/extraction.py
index aecabdc43..4dcff8c47 100644
--- a/prompt-service/src/unstract/prompt_service_v2/controllers/extraction.py
+++ b/prompt-service/src/unstract/prompt_service/controllers/extraction.py
@@ -1,11 +1,11 @@
from typing import Any, Optional
from flask import Blueprint, request
-from unstract.prompt_service_v2.constants import IndexingConstants as IKeys
-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys
-from unstract.prompt_service_v2.helpers.auth import AuthHelper
-from unstract.prompt_service_v2.services.extraction import ExtractionService
-from unstract.prompt_service_v2.utils.request import validate_request_payload
+from unstract.prompt_service.constants import IndexingConstants as IKeys
+from unstract.prompt_service.constants import PromptServiceConstants as PSKeys
+from unstract.prompt_service.helpers.auth import AuthHelper
+from unstract.prompt_service.services.extraction import ExtractionService
+from unstract.prompt_service.utils.request import validate_request_payload
extraction_bp = Blueprint("extract", __name__)
@@ -14,7 +14,6 @@ REQUIRED_FIELDS = [
"file_path",
"execution_source",
"run_id",
- "tags",
]
diff --git a/prompt-service/src/unstract/prompt_service_v2/controllers/health.py b/prompt-service/src/unstract/prompt_service/controllers/health.py
similarity index 100%
rename from prompt-service/src/unstract/prompt_service_v2/controllers/health.py
rename to prompt-service/src/unstract/prompt_service/controllers/health.py
diff --git a/prompt-service/src/unstract/prompt_service_v2/controllers/indexing.py b/prompt-service/src/unstract/prompt_service/controllers/indexing.py
similarity index 88%
rename from prompt-service/src/unstract/prompt_service_v2/controllers/indexing.py
rename to prompt-service/src/unstract/prompt_service/controllers/indexing.py
index 339b4a790..0fb1147d4 100644
--- a/prompt-service/src/unstract/prompt_service_v2/controllers/indexing.py
+++ b/prompt-service/src/unstract/prompt_service/controllers/indexing.py
@@ -2,17 +2,17 @@ import logging
from typing import Any, Optional
from flask import Blueprint, request
-from unstract.prompt_service_v2.constants import IndexingConstants as IKeys
-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys
-from unstract.prompt_service_v2.dto import (
+from unstract.prompt_service.constants import IndexingConstants as IKeys
+from unstract.prompt_service.constants import PromptServiceConstants as PSKeys
+from unstract.prompt_service.dto import (
ChunkingConfig,
FileInfo,
InstanceIdentifiers,
ProcessingOptions,
)
-from unstract.prompt_service_v2.helpers.auth import AuthHelper
-from unstract.prompt_service_v2.services.indexing import IndexingService
-from unstract.prompt_service_v2.utils.request import validate_request_payload
+from unstract.prompt_service.helpers.auth import AuthHelper
+from unstract.prompt_service.services.indexing import IndexingService
+from unstract.prompt_service.utils.request import validate_request_payload
indexing_bp = Blueprint("index", __name__)
logger = logging.getLogger(__name__)
diff --git a/prompt-service/src/unstract/prompt_service_v2/core/index_v2.py b/prompt-service/src/unstract/prompt_service/core/index_v2.py
similarity index 96%
rename from prompt-service/src/unstract/prompt_service_v2/core/index_v2.py
rename to prompt-service/src/unstract/prompt_service/core/index_v2.py
index 17da0b56a..e3c1342ad 100644
--- a/prompt-service/src/unstract/prompt_service_v2/core/index_v2.py
+++ b/prompt-service/src/unstract/prompt_service/core/index_v2.py
@@ -1,6 +1,6 @@
import json
import logging
-from typing import Optional
+from typing import Any, Optional
from llama_index.core import Document
from llama_index.core.vector_stores import (
@@ -10,7 +10,7 @@ from llama_index.core.vector_stores import (
VectorStoreQuery,
VectorStoreQueryResult,
)
-from unstract.prompt_service_v2.dto import (
+from unstract.prompt_service.dto import (
ChunkingConfig,
FileInfo,
InstanceIdentifiers,
@@ -43,7 +43,6 @@ class Index:
run_id: Optional[str] = None,
capture_metrics: bool = False,
):
- # TODO: Inherit from StreamMixin and avoid using BaseTool
self.tool = tool
self._run_id = run_id
self._capture_metrics = capture_metrics
@@ -154,6 +153,7 @@ class Index:
vector_db: VectorDB,
doc_id: str,
extracted_text: str,
+ doc_id_found: str,
):
if isinstance(
vector_db.get_vector_db(
@@ -168,12 +168,12 @@ class Index:
full_text = [
{
"section": "full",
- "text_contents": extracted_text,
+ "text_contents": str(extracted_text),
}
]
# Convert raw text to llama index usage Document
documents = self._prepare_documents(doc_id, full_text)
- if self.processing_options.reindex:
+ if self.processing_options.reindex and doc_id_found:
self.delete_nodes(vector_db, doc_id)
self._trigger_indexing(vector_db, documents)
return doc_id
@@ -206,7 +206,7 @@ class Index:
)
raise SdkError(f"Error deleting nodes for {doc_id}: {e}") from e
- def _prepare_documents(self, doc_id: str, full_text: str) -> list:
+ def _prepare_documents(self, doc_id: str, full_text: Any) -> list:
documents = []
try:
for item in full_text:
diff --git a/prompt-service/src/unstract/prompt_service_v2/core/retrievers/base_retriever.py b/prompt-service/src/unstract/prompt_service/core/retrievers/base_retriever.py
similarity index 66%
rename from prompt-service/src/unstract/prompt_service_v2/core/retrievers/base_retriever.py
rename to prompt-service/src/unstract/prompt_service/core/retrievers/base_retriever.py
index 5dc26bd65..cf560f92a 100644
--- a/prompt-service/src/unstract/prompt_service_v2/core/retrievers/base_retriever.py
+++ b/prompt-service/src/unstract/prompt_service/core/retrievers/base_retriever.py
@@ -1,8 +1,18 @@
+from typing import Optional
+
+from unstract.sdk.llm import LLM
from unstract.sdk.vector_db import VectorDB
class BaseRetriever:
- def __init__(self, vector_db: VectorDB, prompt: str, doc_id: str, top_k: int):
+ def __init__(
+ self,
+ vector_db: VectorDB,
+ prompt: str,
+ doc_id: str,
+ top_k: int,
+ llm: Optional[LLM] = None,
+ ):
"""Initialize the Retrieval class.
Args:
@@ -15,6 +25,7 @@ class BaseRetriever:
self.prompt = prompt
self.doc_id = doc_id
self.top_k = top_k
+ self.llm = llm._llm_instance if llm else None
@staticmethod
def retrieve() -> set[str]:
diff --git a/prompt-service/src/unstract/prompt_service_v2/core/retrievers/simple.py b/prompt-service/src/unstract/prompt_service/core/retrievers/simple.py
similarity index 78%
rename from prompt-service/src/unstract/prompt_service_v2/core/retrievers/simple.py
rename to prompt-service/src/unstract/prompt_service/core/retrievers/simple.py
index 183e638e2..211558c68 100644
--- a/prompt-service/src/unstract/prompt_service_v2/core/retrievers/simple.py
+++ b/prompt-service/src/unstract/prompt_service/core/retrievers/simple.py
@@ -1,15 +1,28 @@
-import logging
import time
+from flask import current_app as app
from llama_index.core import VectorStoreIndex
from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters
-from unstract.prompt_service_v2.core.retrievers.base_retriever import BaseRetriever
-
-logger = logging.getLogger(__name__)
+from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever
class SimpleRetriever(BaseRetriever):
def retrieve(self) -> set[str]:
+ context = self._simple_retrieval()
+ app.logger.info(f"context {context}")
+ if not context:
+ # UN-1288 For Pinecone, we are seeing an inconsistent case where
+ # query with doc_id fails even though indexing just happened.
+ # This causes the following retrieve to return no text.
+ # To rule out any lag on the Pinecone vector DB write,
+ # the following sleep is added
+ # Note: This will not fix the issue. Since this issue is inconsistent
+ # and not reproducible easily, this is just a safety net.
+ time.sleep(2)
+ context = self._simple_retrieval()
+ return context
+
+ def _simple_retrieval(self):
vector_query_engine: VectorStoreIndex = self.vector_db.get_vector_store_index()
retriever = vector_query_engine.as_retriever(
similarity_top_k=self.top_k,
@@ -22,24 +35,13 @@ class SimpleRetriever(BaseRetriever):
nodes = retriever.retrieve(self.prompt)
context: set[str] = set()
for node in nodes:
- # ToDo: May have to fine-tune this value for node score or keep it
+ # May have to fine-tune this value for node score or keep it
# configurable at the adapter level
if node.score > 0:
context.add(node.get_content())
else:
- logger.info(
+ app.logger.info(
"Node score is less than 0. "
f"Ignored: {node.node_id} with score {node.score}"
)
-
- if not context:
- # UN-1288 For Pinecone, we are seeing an inconsistent case where
- # query with doc_id fails even though indexing just happened.
- # This causes the following retrieve to return no text.
- # To rule out any lag on the Pinecone vector DB write,
- # the following sleep is added
- # Note: This will not fix the issue. Since this issue is inconsistent
- # and not reproducible easily, this is just a safety net.
- time.sleep(2)
- context = self.retrieve()
return context
diff --git a/prompt-service/src/unstract/prompt_service_v2/core/retrievers/subquestion.py b/prompt-service/src/unstract/prompt_service/core/retrievers/subquestion.py
similarity index 63%
rename from prompt-service/src/unstract/prompt_service_v2/core/retrievers/subquestion.py
rename to prompt-service/src/unstract/prompt_service/core/retrievers/subquestion.py
index 4e5eefabe..4961d91af 100644
--- a/prompt-service/src/unstract/prompt_service_v2/core/retrievers/subquestion.py
+++ b/prompt-service/src/unstract/prompt_service/core/retrievers/subquestion.py
@@ -1,9 +1,10 @@
import logging
from llama_index.core.query_engine import SubQuestionQueryEngine
+from llama_index.core.schema import QueryBundle
from llama_index.core.tools import QueryEngineTool, ToolMetadata
-from unstract.prompt_service_v2.core.retrievers.base_retriever import BaseRetriever
-from unstract.prompt_service_v2.exceptions import RetrievalError
+from unstract.prompt_service.core.retrievers.base_retriever import BaseRetriever
+from unstract.prompt_service.exceptions import RetrievalError
logger = logging.getLogger(__name__)
@@ -19,27 +20,32 @@ class SubquestionRetriever(BaseRetriever):
set[str]: A set of text chunks retrieved from the database.
"""
try:
+ logger.info("Initialising vector query engine...")
vector_query_engine = (
- self.vector_db.get_vector_store_index().as_query_engine()
+ self.vector_db.get_vector_store_index().as_query_engine(
+ llm=self.llm, similarity_top_k=self.top_k
+ )
+ )
+ logger.info(
+ f"Retrieving chunks for {self.doc_id} using SubQuestionQueryEngine."
)
-
query_engine_tools = [
QueryEngineTool(
query_engine=vector_query_engine,
- metadata=ToolMetadata(name=self.doc_id),
+ metadata=ToolMetadata(
+ name=self.doc_id, description=f"Nodes for {self.doc_id}"
+ ),
),
]
+ query_bundle = QueryBundle(query_str=self.prompt)
query_engine = SubQuestionQueryEngine.from_defaults(
query_engine_tools=query_engine_tools,
use_async=True,
+ llm=self.llm,
)
- response = query_engine.query(
- prompt=self.prompt,
- top_k=self.top_k,
- return_full_response=True,
- )
+ response = query_engine.query(str_or_query_bundle=query_bundle)
chunks: set[str] = {node.text for node in response.source_nodes}
logger.info(f"Successfully retrieved {len(chunks)} chunks.")
diff --git a/prompt-service/src/unstract/prompt_service_v2/dto.py b/prompt-service/src/unstract/prompt_service/dto.py
similarity index 100%
rename from prompt-service/src/unstract/prompt_service_v2/dto.py
rename to prompt-service/src/unstract/prompt_service/dto.py
diff --git a/prompt-service/src/unstract/prompt_service/env_manager.py b/prompt-service/src/unstract/prompt_service/env_manager.py
deleted file mode 100644
index c51f5cf80..000000000
--- a/prompt-service/src/unstract/prompt_service/env_manager.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import os
-from typing import Optional
-
-
-class EnvLoader:
- @staticmethod
- def get_env_or_die(env_key: str, default: Optional[str] = None) -> str:
- env_value = os.environ.get(env_key, default=default)
- if env_value is None or env_value == "":
- raise ValueError(f"Env variable {env_key} is required")
- return env_value
diff --git a/prompt-service/src/unstract/prompt_service/exceptions.py b/prompt-service/src/unstract/prompt_service/exceptions.py
index b7e898254..3df7d16ac 100644
--- a/prompt-service/src/unstract/prompt_service/exceptions.py
+++ b/prompt-service/src/unstract/prompt_service/exceptions.py
@@ -46,7 +46,7 @@ class APIError(HTTPException):
return str(self.message)
-class NoPayloadError(APIError):
+class BadRequest(APIError):
code = 400
message = "Bad Request / No payload"
@@ -54,3 +54,20 @@ class NoPayloadError(APIError):
class RateLimitError(APIError):
code = 429
message = "Running into rate limit errors, please try again later"
+
+
+class MissingFieldError(APIError):
+ """Custom error for missing fields."""
+
+ def __init__(self, missing_fields: list[str]):
+ message = f"Missing required fields: {', '.join(missing_fields)}"
+ super().__init__(message=message)
+
+
+class RetrievalError(APIError):
+ """Custom exception raised for errors during retrieval from VectorDB."""
+
+ DEFAULT_MESSAGE = (
+ "Error while retrieving data from the VectorDB. "
+ "Please contact the admin for further assistance."
+ )
diff --git a/prompt-service/src/unstract/prompt_service_v2/extensions.py b/prompt-service/src/unstract/prompt_service/extensions.py
similarity index 100%
rename from prompt-service/src/unstract/prompt_service_v2/extensions.py
rename to prompt-service/src/unstract/prompt_service/extensions.py
diff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.py
deleted file mode 100644
index 00559efb9..000000000
--- a/prompt-service/src/unstract/prompt_service/helper.py
+++ /dev/null
@@ -1,459 +0,0 @@
-import importlib
-import os
-from logging import Logger
-from pathlib import Path
-from typing import Any, Optional
-
-from dotenv import load_dotenv
-from flask import Flask, current_app
-from unstract.prompt_service.config import db
-from unstract.prompt_service.constants import (
- DBTableV2,
- ExecutionSource,
- FileStorageKeys,
-)
-from unstract.prompt_service.constants import PromptServiceContants as PSKeys
-from unstract.prompt_service.db_utils import DBUtils
-from unstract.prompt_service.env_manager import EnvLoader
-from unstract.prompt_service.exceptions import APIError, RateLimitError
-from unstract.sdk.exceptions import RateLimitError as SdkRateLimitError
-from unstract.sdk.exceptions import SdkError
-from unstract.sdk.file_storage import FileStorage, FileStorageProvider
-from unstract.sdk.file_storage.constants import StorageType
-from unstract.sdk.file_storage.env_helper import EnvHelper
-from unstract.sdk.llm import LLM
-
-PAID_FEATURE_MSG = (
- "It is a cloud / enterprise feature. If you have purchased a plan and still "
- "face this issue, please contact support"
-)
-
-load_dotenv()
-
-# Global variable to store plugins
-plugins: dict[str, dict[str, Any]] = {}
-
-
-def plugin_loader(app: Flask) -> None:
- """Loads plugins found in the plugins root dir.
-
- Each plugin:
- * needs to be a package dir AND
- * should contain a src/__init__.py which exports below:
-
- metadata: dict[str, Any] = {
- "version": str,
- "name": str,
- "entrypoint_cls": class,
- "exception_cls": class,
- "disable": bool
- }
- """
- global plugins
- plugins_dir: Path = Path(os.path.dirname(__file__)) / "plugins"
- plugins_pkg = "unstract.prompt_service.plugins"
-
- if not plugins_dir.exists():
- app.logger.info(f"Plugins dir not found: {plugins_dir}. Skipping.")
- return
-
- app.logger.info(f"Loading plugins from: {plugins_dir}")
-
- for pkg in os.listdir(os.fspath(plugins_dir)):
- if pkg.endswith(".so"):
- pkg = pkg.split(".")[0]
- pkg_anchor = f"{plugins_pkg}.{pkg}.src"
- try:
- module = importlib.import_module(pkg_anchor)
- except ImportError as e:
- app.logger.error(f"Failed to load plugin ({pkg}): {str(e)}")
- continue
-
- if not hasattr(module, "metadata"):
- app.logger.warn(f"Plugin metadata not found: {pkg}")
- continue
-
- metadata: dict[str, Any] = module.metadata
- if metadata.get("disable", False):
- app.logger.info(f"Ignore disabled plugin: {pkg} v{metadata['version']}")
- continue
-
- try:
- plugins[metadata["name"]] = {
- "version": metadata["version"],
- "entrypoint_cls": metadata["entrypoint_cls"],
- "exception_cls": metadata["exception_cls"],
- }
- app.logger.info(f"Loaded plugin: {pkg} v{metadata['version']}")
- except KeyError as e:
- app.logger.error(f"Invalid metadata for plugin '{pkg}': {str(e)}")
-
- if not plugins:
- app.logger.info("No plugins found.")
-
- initialize_plugin_endpoints(app=app)
-
-
-def get_cleaned_context(context: set[str]) -> list[str]:
- clean_context_plugin: dict[str, Any] = plugins.get(PSKeys.CLEAN_CONTEXT, {})
- if clean_context_plugin:
- return clean_context_plugin["entrypoint_cls"].run(context=context)
- return list(context)
-
-
-def initialize_plugin_endpoints(app: Flask) -> None:
- """Enables plugins if available."""
- single_pass_extration_plugin: dict[str, Any] = plugins.get(
- PSKeys.SINGLE_PASS_EXTRACTION, {}
- )
- summarize_plugin: dict[str, Any] = plugins.get(PSKeys.SUMMARIZE, {})
- simple_prompt_studio: dict[str, Any] = plugins.get(PSKeys.SIMPLE_PROMPT_STUDIO, {})
- if single_pass_extration_plugin:
- single_pass_extration_plugin["entrypoint_cls"](
- app=app,
- challenge_plugin=plugins.get(PSKeys.CHALLENGE, {}),
- get_cleaned_context=get_cleaned_context,
- highlight_data_plugin=plugins.get(PSKeys.HIGHLIGHT_DATA_PLUGIN, {}),
- )
- if summarize_plugin:
- summarize_plugin["entrypoint_cls"](
- app=app,
- )
- if simple_prompt_studio:
- simple_prompt_studio["entrypoint_cls"](
- app=app,
- )
-
-
-def query_usage_metadata(token: str, metadata: dict[str, Any]) -> dict[str, Any]:
- DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract")
- organization_uid, org_id = DBUtils.get_organization_from_bearer_token(token)
- run_id: str = metadata["run_id"]
- query: str = f"""
- SELECT
- usage_type,
- llm_usage_reason,
- model_name,
- SUM(prompt_tokens) AS input_tokens,
- SUM(completion_tokens) AS output_tokens,
- SUM(total_tokens) AS total_tokens,
- SUM(embedding_tokens) AS embedding_tokens,
- SUM(cost_in_dollars) AS cost_in_dollars
- FROM "{DB_SCHEMA}"."{DBTableV2.TOKEN_USAGE}"
- WHERE run_id = %s and organization_id = %s
- GROUP BY usage_type, llm_usage_reason, model_name;
- """
- logger: Logger = current_app.logger
- try:
- with db.atomic():
- logger.info(
- "Querying usage metadata for org_id: %s, run_id: %s", org_id, run_id
- )
- cursor = db.execute_sql(query, (run_id, organization_uid))
- results: list[tuple] = cursor.fetchall()
- # Process results as needed
- for row in results:
- key, item = _get_key_and_item(row)
- # Initialize the key as an empty list if it doesn't exist
- if key not in metadata:
- metadata[key] = []
- # Append the item to the list associated with the key
- metadata[key].append(item)
- except Exception as e:
- logger.error(f"Error executing querying usage metadata: {e}")
- return metadata
-
-
-def _get_key_and_item(row: tuple) -> tuple[str, dict[str, Any]]:
- (
- usage_type,
- llm_usage_reason,
- model_name,
- input_tokens,
- output_tokens,
- total_tokens,
- embedding_tokens,
- cost_in_dollars,
- ) = row
- cost_in_dollars: str = _format_float_positional(cost_in_dollars)
- key: str = usage_type
- item: dict[str, Any] = {
- "model_name": model_name,
- "cost_in_dollars": cost_in_dollars,
- }
- if llm_usage_reason:
- key = f"{llm_usage_reason}_{key}"
- item["input_tokens"] = input_tokens
- item["output_tokens"] = output_tokens
- item["total_tokens"] = total_tokens
- else:
- item["embedding_tokens"] = embedding_tokens
- return key, item
-
-
-def _format_float_positional(value: float, precision: int = 10) -> str:
- formatted: str = f"{value:.{precision}f}"
- return formatted.rstrip("0").rstrip(".") if "." in formatted else formatted
-
-
-def extract_variable(
- structured_output: dict[str, Any],
- variable_names: list[Any],
- output: dict[str, Any],
- promptx: str,
-) -> str:
- logger: Logger = current_app.logger
- for variable_name in variable_names:
- if promptx.find(f"%{variable_name}%") >= 0:
- if variable_name in structured_output:
- promptx = promptx.replace(
- f"%{variable_name}%",
- str(structured_output[variable_name]),
- )
- else:
- raise ValueError(
- f"Variable {variable_name} not found " "in structured output"
- )
-
- if promptx != output[PSKeys.PROMPT]:
- logger.info(f"Prompt after variable replacement: {promptx}")
- return promptx
-
-
-def construct_and_run_prompt(
- tool_settings: dict[str, Any],
- output: dict[str, Any],
- llm: LLM,
- context: str,
- prompt: str,
- metadata: dict[str, Any],
- file_path: str = "",
- execution_source: Optional[str] = ExecutionSource.IDE.value,
-) -> str:
- platform_postamble = tool_settings.get(PSKeys.PLATFORM_POSTAMBLE, "")
- summarize_as_source = tool_settings.get(PSKeys.SUMMARIZE_AS_SOURCE)
- enable_highlight = tool_settings.get(PSKeys.ENABLE_HIGHLIGHT, False)
- if not enable_highlight or summarize_as_source:
- platform_postamble = ""
- prompt = construct_prompt(
- preamble=tool_settings.get(PSKeys.PREAMBLE, ""),
- prompt=output[prompt],
- postamble=tool_settings.get(PSKeys.POSTAMBLE, ""),
- grammar_list=tool_settings.get(PSKeys.GRAMMAR, []),
- context=context,
- platform_postamble=platform_postamble,
- )
- return run_completion(
- llm=llm,
- prompt=prompt,
- metadata=metadata,
- prompt_key=output[PSKeys.NAME],
- prompt_type=output.get(PSKeys.TYPE, PSKeys.TEXT),
- enable_highlight=enable_highlight,
- file_path=file_path,
- execution_source=execution_source,
- )
-
-
-def construct_prompt(
- preamble: str,
- prompt: str,
- postamble: str,
- grammar_list: list[dict[str, Any]],
- context: str,
- platform_postamble: str,
-) -> str:
- prompt = f"{preamble}\n\nQuestion or Instruction: {prompt}"
- if grammar_list is not None and len(grammar_list) > 0:
- prompt += "\n"
- for grammar in grammar_list:
- word = ""
- synonyms = []
- if PSKeys.WORD in grammar:
- word = grammar[PSKeys.WORD]
- if PSKeys.SYNONYMS in grammar:
- synonyms = grammar[PSKeys.SYNONYMS]
- if len(synonyms) > 0 and word != "":
- prompt += f'\nNote: You can consider that the word {word} is same as \
- {", ".join(synonyms)} in both the quesiton and the context.' # noqa
- if platform_postamble:
- platform_postamble += "\n\n"
- prompt += (
- f"\n\n{postamble}\n\nContext:\n---------------\n{context}\n"
- f"-----------------\n\n{platform_postamble}Answer:"
- )
- return prompt
-
-
-def run_completion(
- llm: LLM,
- prompt: str,
- metadata: Optional[dict[str, str]] = None,
- prompt_key: Optional[str] = None,
- prompt_type: Optional[str] = PSKeys.TEXT,
- enable_highlight: bool = False,
- file_path: str = "",
- execution_source: Optional[str] = None,
-) -> str:
- logger: Logger = current_app.logger
- try:
- highlight_data_plugin: dict[str, Any] = plugins.get(
- PSKeys.HIGHLIGHT_DATA_PLUGIN, {}
- )
- highlight_data = None
- if highlight_data_plugin and enable_highlight:
- fs_instance: FileStorage = FileStorage(FileStorageProvider.LOCAL)
- if execution_source == ExecutionSource.IDE.value:
- fs_instance = EnvHelper.get_storage(
- storage_type=StorageType.PERMANENT,
- env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,
- )
- if execution_source == ExecutionSource.TOOL.value:
- fs_instance = EnvHelper.get_storage(
- storage_type=StorageType.SHARED_TEMPORARY,
- env_name=FileStorageKeys.TEMPORARY_REMOTE_STORAGE,
- )
- highlight_data = highlight_data_plugin["entrypoint_cls"](
- logger=current_app.logger,
- file_path=file_path,
- fs_instance=fs_instance,
- ).run
-
- completion = llm.complete(
- prompt=prompt,
- process_text=highlight_data,
- extract_json=prompt_type.lower() != PSKeys.TEXT,
- )
- answer: str = completion[PSKeys.RESPONSE].text
- highlight_data = completion.get(PSKeys.HIGHLIGHT_DATA, [])
- confidence_data = completion.get(PSKeys.CONFIDENCE_DATA)
- line_numbers = completion.get(PSKeys.LINE_NUMBERS, [])
- whisper_hash = completion.get(PSKeys.WHISPER_HASH, "")
- if metadata is not None and prompt_key:
- metadata.setdefault(PSKeys.HIGHLIGHT_DATA, {})[prompt_key] = highlight_data
- metadata.setdefault(PSKeys.LINE_NUMBERS, {})[prompt_key] = line_numbers
- metadata[PSKeys.WHISPER_HASH] = whisper_hash
- if confidence_data:
- metadata.setdefault(PSKeys.CONFIDENCE_DATA, {})[
- prompt_key
- ] = confidence_data
- return answer
- # TODO: Catch and handle specific exception here
- except SdkRateLimitError as e:
- raise RateLimitError(f"Rate limit error. {str(e)}") from e
- except SdkError as e:
- logger.error(f"Error fetching response for prompt: {e}.")
- # TODO: Publish this error as a FE update
- raise APIError(str(e)) from e
-
-
-def extract_table(
- output: dict[str, Any],
- plugins: dict[str, dict[str, Any]],
- structured_output: dict[str, Any],
- llm: LLM,
- enforce_type: str,
- execution_source: str,
-) -> dict[str, Any]:
- table_settings = output[PSKeys.TABLE_SETTINGS]
- table_extractor: dict[str, Any] = plugins.get("table-extractor", {})
- if not table_extractor:
- raise APIError(
- "Unable to extract table details. "
- "Please contact admin to resolve this issue."
- )
- fs_instance: FileStorage = FileStorage(FileStorageProvider.LOCAL)
- if execution_source == ExecutionSource.IDE.value:
- fs_instance = EnvHelper.get_storage(
- storage_type=StorageType.PERMANENT,
- env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,
- )
- if execution_source == ExecutionSource.TOOL.value:
- fs_instance = EnvHelper.get_storage(
- storage_type=StorageType.TEMPORARY,
- env_name=FileStorageKeys.TEMPORARY_REMOTE_STORAGE,
- )
- try:
- answer = table_extractor["entrypoint_cls"].extract_large_table(
- llm=llm,
- table_settings=table_settings,
- enforce_type=enforce_type,
- fs_instance=fs_instance,
- )
- structured_output[output[PSKeys.NAME]] = answer
- # We do not support summary and eval for table.
- # Hence returning the result
- return structured_output
- except table_extractor["exception_cls"] as e:
- msg = f"Couldn't extract table. {e}"
- raise APIError(message=msg)
-
-
-def extract_line_item(
- tool_settings: dict[str, Any],
- output: dict[str, Any],
- plugins: dict[str, dict[str, Any]],
- structured_output: dict[str, Any],
- llm: LLM,
- file_path: str,
- metadata: Optional[dict[str, str]],
- execution_source: str,
-) -> dict[str, Any]:
- line_item_extraction_plugin: dict[str, Any] = plugins.get(
- "line-item-extraction", {}
- )
- if not line_item_extraction_plugin:
- raise APIError(PAID_FEATURE_MSG)
-
- extract_file_path = file_path
- if execution_source == ExecutionSource.IDE.value:
- # Adjust file path to read from the extract folder
- base_name = os.path.splitext(os.path.basename(file_path))[0]
- extract_file_path = os.path.join(
- os.path.dirname(file_path), "extract", f"{base_name}.txt"
- )
-
- # Read file content into context
- fs_instance: FileStorage = FileStorage(FileStorageProvider.LOCAL)
- if execution_source == ExecutionSource.IDE.value:
- fs_instance = EnvHelper.get_storage(
- storage_type=StorageType.PERMANENT,
- env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,
- )
- if execution_source == ExecutionSource.TOOL.value:
- fs_instance = EnvHelper.get_storage(
- storage_type=StorageType.SHARED_TEMPORARY,
- env_name=FileStorageKeys.TEMPORARY_REMOTE_STORAGE,
- )
-
- if not fs_instance.exists(extract_file_path):
- raise FileNotFoundError(
- f"The file at path '{extract_file_path}' does not exist."
- )
- context = fs_instance.read(path=extract_file_path, encoding="utf-8", mode="r")
-
- prompt = construct_prompt(
- preamble=tool_settings.get(PSKeys.PREAMBLE, ""),
- prompt=output["promptx"],
- postamble=tool_settings.get(PSKeys.POSTAMBLE, ""),
- grammar_list=tool_settings.get(PSKeys.GRAMMAR, []),
- context=context,
- platform_postamble="",
- )
-
- try:
- line_item_extraction = line_item_extraction_plugin["entrypoint_cls"](
- llm=llm,
- tool_settings=tool_settings,
- output=output,
- prompt=prompt,
- structured_output=structured_output,
- logger=current_app.logger,
- )
- answer = line_item_extraction.run()
- structured_output[output[PSKeys.NAME]] = answer
- metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = [context]
- return structured_output
- except line_item_extraction_plugin["exception_cls"] as e:
- msg = f"Couldn't extract table. {e}"
- raise APIError(message=msg)
diff --git a/prompt-service/src/unstract/prompt_service_v2/__init__.py b/prompt-service/src/unstract/prompt_service/helpers/__init__.py
similarity index 100%
rename from prompt-service/src/unstract/prompt_service_v2/__init__.py
rename to prompt-service/src/unstract/prompt_service/helpers/__init__.py
diff --git a/prompt-service/src/unstract/prompt_service_v2/helpers/auth.py b/prompt-service/src/unstract/prompt_service/helpers/auth.py
similarity index 92%
rename from prompt-service/src/unstract/prompt_service_v2/helpers/auth.py
rename to prompt-service/src/unstract/prompt_service/helpers/auth.py
index fdeca58c0..457a8f3f8 100644
--- a/prompt-service/src/unstract/prompt_service_v2/helpers/auth.py
+++ b/prompt-service/src/unstract/prompt_service/helpers/auth.py
@@ -3,10 +3,10 @@ from typing import Any, Optional
from flask import Request
from flask import current_app as app
from flask import request
-from unstract.prompt_service_v2.constants import DBTableV2
-from unstract.prompt_service_v2.extensions import db
-from unstract.prompt_service_v2.utils.db_utils import DBUtils
-from unstract.prompt_service_v2.utils.env_loader import get_env_or_die
+from unstract.prompt_service.constants import DBTableV2
+from unstract.prompt_service.extensions import db
+from unstract.prompt_service.utils.db_utils import DBUtils
+from unstract.prompt_service.utils.env_loader import get_env_or_die
DB_SCHEMA = get_env_or_die("DB_SCHEMA", "unstract")
diff --git a/prompt-service/src/unstract/prompt_service_v2/helpers/lifecycle.py b/prompt-service/src/unstract/prompt_service/helpers/lifecycle.py
similarity index 93%
rename from prompt-service/src/unstract/prompt_service_v2/helpers/lifecycle.py
rename to prompt-service/src/unstract/prompt_service/helpers/lifecycle.py
index 503f7a1a5..c1d2121d1 100644
--- a/prompt-service/src/unstract/prompt_service_v2/helpers/lifecycle.py
+++ b/prompt-service/src/unstract/prompt_service/helpers/lifecycle.py
@@ -1,7 +1,7 @@
from typing import Any
from flask import Flask, request
-from unstract.prompt_service_v2.extensions import db
+from unstract.prompt_service.extensions import db
def register_lifecycle_hooks(app: Flask):
diff --git a/prompt-service/src/unstract/prompt_service_v2/helpers/plugin.py b/prompt-service/src/unstract/prompt_service/helpers/plugin.py
similarity index 97%
rename from prompt-service/src/unstract/prompt_service_v2/helpers/plugin.py
rename to prompt-service/src/unstract/prompt_service/helpers/plugin.py
index 365c31f00..6553cbda1 100644
--- a/prompt-service/src/unstract/prompt_service_v2/helpers/plugin.py
+++ b/prompt-service/src/unstract/prompt_service/helpers/plugin.py
@@ -4,7 +4,7 @@ from pathlib import Path
from typing import Any, Optional
from flask import Flask
-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys
+from unstract.prompt_service.constants import PromptServiceConstants as PSKeys
class PluginManager:
@@ -16,7 +16,7 @@ class PluginManager:
cls._instance = super().__new__(cls)
cls._instance.plugins = {}
cls._instance.plugins_dir = Path(__file__).parent.parent / "plugins"
- cls._instance.plugins_pkg = "unstract.prompt_service_v2.plugins"
+ cls._instance.plugins_pkg = "unstract.prompt_service.plugins"
# Always update `app` if provided
if app:
cls._instance.app = app
diff --git a/prompt-service/src/unstract/prompt_service_v2/helpers/prompt_ide_base_tool.py b/prompt-service/src/unstract/prompt_service/helpers/prompt_ide_base_tool.py
similarity index 94%
rename from prompt-service/src/unstract/prompt_service_v2/helpers/prompt_ide_base_tool.py
rename to prompt-service/src/unstract/prompt_service/helpers/prompt_ide_base_tool.py
index 0650fde70..55040d044 100644
--- a/prompt-service/src/unstract/prompt_service_v2/helpers/prompt_ide_base_tool.py
+++ b/prompt-service/src/unstract/prompt_service/helpers/prompt_ide_base_tool.py
@@ -3,14 +3,14 @@ import warnings
from typing import Optional
from flask import current_app
-from unstract.prompt_service_v2.constants import PromptServiceConstants
+from unstract.prompt_service.constants import PromptServiceConstants
from unstract.sdk.constants import LogLevel
from unstract.sdk.tool.stream import StreamMixin
from unstract.sdk.utils.common_utils import PY_TO_UNSTRACT_LOG_LEVEL
class PromptServiceBaseTool(StreamMixin):
- # TODO: Remove unused log_level arg
+ # Remove unused log_level arg
def __init__(
self, log_level: Optional[LogLevel] = None, platform_key: str = ""
) -> None:
diff --git a/prompt-service/src/unstract/prompt_service/helpers/retrieval.py b/prompt-service/src/unstract/prompt_service/helpers/retrieval.py
new file mode 100644
index 000000000..75d486285
--- /dev/null
+++ b/prompt-service/src/unstract/prompt_service/helpers/retrieval.py
@@ -0,0 +1,45 @@
+from typing import Any
+
+from unstract.prompt_service.constants import PromptServiceConstants as PSKeys
+from unstract.prompt_service.core.retrievers.simple import SimpleRetriever
+from unstract.prompt_service.core.retrievers.subquestion import SubquestionRetriever
+from unstract.prompt_service.utils.file_utils import FileUtils
+from unstract.sdk.llm import LLM
+from unstract.sdk.vector_db import VectorDB
+
+
+class RetrievalHelper:
+ @staticmethod
+ def run_retrieval( # type:ignore
+ output: dict[str, Any],
+ doc_id: str,
+ llm: LLM,
+ vector_db: VectorDB,
+ retrieval_type: str,
+ ) -> set[str]:
+ context: set[str] = set()
+ prompt = output[PSKeys.PROMPTX]
+ top_k = output[PSKeys.SIMILARITY_TOP_K]
+ if retrieval_type == PSKeys.SUBQUESTION:
+ context = SubquestionRetriever(
+ vector_db=vector_db, doc_id=doc_id, prompt=prompt, top_k=top_k, llm=llm
+ ).retrieve()
+ if retrieval_type == PSKeys.SIMPLE:
+ context = SimpleRetriever(
+ vector_db=vector_db, doc_id=doc_id, prompt=prompt, top_k=top_k
+ ).retrieve()
+
+ return context
+
+ @staticmethod
+ def retrieve_complete_context(execution_source: str, file_path: str) -> str:
+ """
+ Loads full context from raw file for zero chunk size retrieval
+ Args:
+ path (str): Path to the directory containing text file.
+
+ Returns:
+ str: context from extracted file.
+ """
+ fs_instance = FileUtils.get_fs_instance(execution_source=execution_source)
+ return fs_instance.read(path=file_path, mode="r")
diff --git a/prompt-service/src/unstract/prompt_service_v2/helpers/usage.py b/prompt-service/src/unstract/prompt_service/helpers/usage.py
similarity index 92%
rename from prompt-service/src/unstract/prompt_service_v2/helpers/usage.py
rename to prompt-service/src/unstract/prompt_service/helpers/usage.py
index f1b2be7d3..097d01174 100644
--- a/prompt-service/src/unstract/prompt_service_v2/helpers/usage.py
+++ b/prompt-service/src/unstract/prompt_service/helpers/usage.py
@@ -2,10 +2,10 @@ from logging import Logger
from typing import Any
from flask import current_app as app
-from unstract.prompt_service_v2.constants import DBTableV2
-from unstract.prompt_service_v2.extensions import db
-from unstract.prompt_service_v2.utils.db_utils import DBUtils
-from unstract.prompt_service_v2.utils.env_loader import get_env_or_die
+from unstract.prompt_service.constants import DBTableV2
+from unstract.prompt_service.extensions import db
+from unstract.prompt_service.utils.db_utils import DBUtils
+from unstract.prompt_service.utils.env_loader import get_env_or_die
class UsageHelper:
diff --git a/prompt-service/src/unstract/prompt_service_v2/helpers/variable_replacement.py b/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py
similarity index 96%
rename from prompt-service/src/unstract/prompt_service_v2/helpers/variable_replacement.py
rename to prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py
index ec1632520..d5f4e02b1 100644
--- a/prompt-service/src/unstract/prompt_service_v2/helpers/variable_replacement.py
+++ b/prompt-service/src/unstract/prompt_service/helpers/variable_replacement.py
@@ -4,8 +4,8 @@ from functools import lru_cache
from typing import Any
from flask import current_app as app
-from unstract.prompt_service_v2.constants import VariableConstants, VariableType
-from unstract.prompt_service_v2.utils.request import HTTPMethod, make_http_request
+from unstract.prompt_service.constants import VariableConstants, VariableType
+from unstract.prompt_service.utils.request import HTTPMethod, make_http_request
class VariableReplacementHelper:
diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.py
deleted file mode 100644
index e418e6f5a..000000000
--- a/prompt-service/src/unstract/prompt_service/main.py
+++ /dev/null
@@ -1,966 +0,0 @@
-import time
-import traceback
-from json import JSONDecodeError
-from typing import Any, Optional
-
-from flask import json, jsonify, request
-from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters
-from unstract.prompt_service.authentication_middleware import AuthenticationMiddleware
-from unstract.prompt_service.config import create_app, db
-from unstract.prompt_service.constants import PromptServiceContants as PSKeys
-from unstract.prompt_service.constants import RunLevel
-from unstract.prompt_service.exceptions import APIError, ErrorResponse, NoPayloadError
-from unstract.prompt_service.helper import (
- construct_and_run_prompt,
- extract_line_item,
- extract_table,
- extract_variable,
- get_cleaned_context,
- plugin_loader,
- plugins,
- query_usage_metadata,
- run_completion,
-)
-from unstract.prompt_service.prompt_ide_base_tool import PromptServiceBaseTool
-from unstract.prompt_service.utils.log import publish_log
-from unstract.prompt_service.variable_extractor.base import VariableExtractor
-from unstract.sdk.adapters.llm.no_op.src.no_op_custom_llm import NoOpCustomLLM
-from unstract.sdk.constants import LogLevel
-from unstract.sdk.embedding import Embedding
-from unstract.sdk.exceptions import SdkError
-from unstract.sdk.index import Index
-from unstract.sdk.llm import LLM
-from unstract.sdk.vector_db import VectorDB
-from werkzeug.exceptions import HTTPException
-
-USE_UNSTRACT_PROMPT = True
-MAX_RETRIES = 3
-
-NO_CONTEXT_ERROR = (
- "Couldn't fetch context from vector DB. "
- "This happens usually due to a delay by the Vector DB "
- "provider to confirm writes to DB. "
- "Please try again after some time"
-)
-
-app = create_app()
-# Load plugins
-plugin_loader(app)
-
-
-def before_request() -> None:
- if db.is_closed():
- db.connect(reuse_if_open=True)
-
-
-def after_request(exception: Any) -> None:
- # Close the connection after each request
- if not db.is_closed():
- db.close()
-
-
-def log_request_info():
- app.logger.info(f"Request Path: {request.path} | Method: {request.method}")
-
-
-def log_response_info(response):
- app.logger.info(f"Response Status: {response.status}")
- return response
-
-
-def authentication_middleware(func: Any) -> Any:
- def wrapper(*args: Any, **kwargs: Any) -> Any:
- token = AuthenticationMiddleware.get_token_from_auth_header(request)
- # Check if bearer token exists and validate it
- if not token or not AuthenticationMiddleware.validate_bearer_token(token):
- return "Unauthorized", 401
-
- return func(*args, **kwargs)
-
- return wrapper
-
-
[email protected]("/health", methods=["GET"], endpoint="health_check")
-def health_check() -> str:
- return "OK"
-
-
- "/answer-prompt",
- endpoint="answer_prompt",
- methods=["POST"],
-)
-@authentication_middleware
-def prompt_processor() -> Any:
- platform_key = AuthenticationMiddleware.get_token_from_auth_header(request)
- payload: dict[Any, Any] = request.json
- if not payload:
- raise NoPayloadError
- tool_settings = payload.get(PSKeys.TOOL_SETTINGS, {})
- enable_challenge = tool_settings.get(PSKeys.ENABLE_CHALLENGE, False)
- challenge_llm = None
- # TODO: Rename "outputs" to "prompts" in payload
- prompts = payload.get(PSKeys.OUTPUTS, [])
- tool_id: str = payload.get(PSKeys.TOOL_ID, "")
- run_id: str = payload.get(PSKeys.RUN_ID, "")
- execution_id: str = payload.get(PSKeys.EXECUTION_ID, "")
- file_hash = payload.get(PSKeys.FILE_HASH)
- file_path = payload.get(PSKeys.FILE_PATH)
- doc_name = str(payload.get(PSKeys.FILE_NAME, ""))
- log_events_id: str = payload.get(PSKeys.LOG_EVENTS_ID, "")
- structured_output: dict[str, Any] = {}
- metadata: dict[str, Any] = {
- PSKeys.RUN_ID: run_id,
- PSKeys.FILE_NAME: doc_name,
- PSKeys.CONTEXT: {},
- PSKeys.REQUIRED_FIELDS: {},
- }
- metrics: dict = {}
- variable_names: list[str] = []
- # Identifier for source of invocation
- execution_source = payload.get(PSKeys.EXECUTION_SOURCE, "")
- publish_log(
- log_events_id,
- {"tool_id": tool_id, "run_id": run_id, "doc_name": doc_name},
- LogLevel.DEBUG,
- RunLevel.RUN,
- f"Preparing to execute {len(prompts)} prompt(s)",
- )
- # TODO: Rename "output" to "prompt"
- for output in prompts: # type:ignore
- variable_names.append(output[PSKeys.NAME])
- metadata[PSKeys.REQUIRED_FIELDS][output[PSKeys.NAME]] = output.get(
- PSKeys.REQUIRED, None
- )
-
- for output in prompts: # type:ignore
- prompt_name = output[PSKeys.NAME]
- prompt_text = output[PSKeys.PROMPT]
- chunk_size = output[PSKeys.CHUNK_SIZE]
- util = PromptServiceBaseTool(platform_key=platform_key)
- index = Index(tool=util, run_id=run_id, capture_metrics=True)
- if VariableExtractor.is_variables_present(prompt_text=prompt_text):
- prompt_text = VariableExtractor.replace_variables_in_prompt(
- prompt=output,
- structured_output=structured_output,
- log_events_id=log_events_id,
- tool_id=tool_id,
- prompt_name=prompt_name,
- doc_name=doc_name,
- )
-
- app.logger.info(f"[{tool_id}] Executing prompt: {prompt_name}")
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.DEBUG,
- RunLevel.RUN,
- "Executing prompt",
- )
-
- # Finding and replacing the variables in the prompt
- # The variables are in the form %variable_name%
-
- output[PSKeys.PROMPTX] = extract_variable(
- structured_output, variable_names, output, prompt_text
- )
-
- doc_id = index.generate_index_key(
- file_hash=file_hash,
- vector_db=output[PSKeys.VECTOR_DB],
- embedding=output[PSKeys.EMBEDDING],
- x2text=output[PSKeys.X2TEXT_ADAPTER],
- chunk_size=str(output[PSKeys.CHUNK_SIZE]),
- chunk_overlap=str(output[PSKeys.CHUNK_OVERLAP]),
- )
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.DEBUG,
- RunLevel.RUN,
- "Retrieved document ID",
- )
-
- try:
- usage_kwargs = {"run_id": run_id, "execution_id": execution_id}
- adapter_instance_id = output[PSKeys.LLM]
- llm = LLM(
- tool=util,
- adapter_instance_id=adapter_instance_id,
- usage_kwargs={
- **usage_kwargs,
- PSKeys.LLM_USAGE_REASON: PSKeys.EXTRACTION,
- },
- capture_metrics=True,
- )
-
- embedding = Embedding(
- tool=util,
- adapter_instance_id=output[PSKeys.EMBEDDING],
- usage_kwargs=usage_kwargs.copy(),
- )
-
- vector_db = VectorDB(
- tool=util,
- adapter_instance_id=output[PSKeys.VECTOR_DB],
- embedding=embedding,
- )
- except SdkError as e:
- msg = f"Couldn't fetch adapter. {e}"
- app.logger.error(msg)
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.ERROR,
- RunLevel.RUN,
- "Unable to obtain LLM / embedding / vectorDB",
- )
- return APIError(message=msg)
-
- if output[PSKeys.TYPE] == PSKeys.TABLE or output[PSKeys.TYPE] == PSKeys.RECORD:
- try:
- structured_output = extract_table(
- output=output,
- plugins=plugins,
- structured_output=structured_output,
- llm=llm,
- enforce_type=output[PSKeys.TYPE],
- execution_source=execution_source,
- )
- metadata = query_usage_metadata(token=platform_key, metadata=metadata)
- response = {
- PSKeys.METADATA: metadata,
- PSKeys.OUTPUT: structured_output,
- PSKeys.METRICS: metrics,
- }
- return response
- except APIError as api_error:
- app.logger.error(
- "Failed to extract table for the prompt %s: %s",
- output[PSKeys.NAME],
- str(api_error),
- )
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.ERROR,
- RunLevel.TABLE_EXTRACTION,
- "Error while extracting table for the prompt",
- )
- raise api_error
- elif output[PSKeys.TYPE] == PSKeys.LINE_ITEM:
- try:
- structured_output = extract_line_item(
- tool_settings=tool_settings,
- output=output,
- plugins=plugins,
- structured_output=structured_output,
- llm=llm,
- file_path=file_path,
- metadata=metadata,
- execution_source=execution_source,
- )
- continue
- except APIError as e:
- app.logger.error(
- "Failed to extract line-item for the prompt %s: %s",
- output[PSKeys.NAME],
- str(e),
- )
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.ERROR,
- RunLevel.RUN,
- "Error while extracting line-item for the prompt",
- )
- raise e
-
- try:
- if chunk_size == 0:
- # We can do this only for chunkless indexes
- context: set[str] = fetch_context_from_vector_db(
- index=index,
- output=output,
- doc_id=doc_id,
- tool_id=tool_id,
- doc_name=doc_name,
- prompt_name=prompt_name,
- log_events_id=log_events_id,
- usage_kwargs=usage_kwargs,
- )
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.INFO,
- RunLevel.RUN,
- "Retrieving answer from LLM",
- )
- answer = construct_and_run_prompt(
- tool_settings=tool_settings,
- output=output,
- llm=llm,
- context="\n".join(context),
- prompt="promptx",
- metadata=metadata,
- file_path=file_path,
- execution_source=execution_source,
- )
- metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = get_cleaned_context(
- context
- )
- else:
- answer = "NA"
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.INFO,
- RunLevel.RUN,
- "Retrieving context from adapter",
- )
-
- retrieval_strategy = output.get(PSKeys.RETRIEVAL_STRATEGY)
-
- if retrieval_strategy in {PSKeys.SIMPLE, PSKeys.SUBQUESTION}:
- vector_index = vector_db.get_vector_store_index()
- answer, context = run_retrieval(
- tool_settings=tool_settings,
- output=output,
- doc_id=doc_id,
- llm=llm,
- vector_index=vector_index,
- retrieval_type=retrieval_strategy,
- metadata=metadata,
- execution_source=execution_source,
- )
- metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = get_cleaned_context(
- context
- )
- else:
- app.logger.info(
- "Invalid retrieval strategy passed: %s",
- retrieval_strategy,
- )
-
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.DEBUG,
- RunLevel.RUN,
- "Retrieved context from adapter",
- )
-
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.INFO,
- RunLevel.RUN,
- f"Processing prompt type: {output[PSKeys.TYPE]}",
- )
-
- if output[PSKeys.TYPE] == PSKeys.NUMBER:
- if answer.lower() == "na":
- structured_output[output[PSKeys.NAME]] = None
- else:
- # TODO: Extract these prompts as constants after pkging
- prompt = f"Extract the number from the following \
- text:\n{answer}\n\nOutput just the number. \
- If the number is expressed in millions \
- or thousands, expand the number to its numeric value \
- The number should be directly assignable\
- to a numeric variable.\
- It should not have any commas, \
- percentages or other grouping \
- characters. No explanation is required.\
- If you cannot extract the number, output 0."
- answer = run_completion(
- llm=llm,
- prompt=prompt,
- )
- try:
- structured_output[output[PSKeys.NAME]] = float(answer)
- except Exception as e:
- app.logger.info(
- f"Error parsing response (to numeric, float): {e}",
- LogLevel.ERROR,
- )
- structured_output[output[PSKeys.NAME]] = None
- # No-op adapter always returns a string data and
- # to keep this response uniform
- # through all enforce types
- # we add this check, if not for this,
- # type casting to float raises
- # an error and we return None.
- if isinstance(
- llm.get_llm(adapter_instance_id=adapter_instance_id),
- NoOpCustomLLM,
- ):
- structured_output[output[PSKeys.NAME]] = answer
- elif output[PSKeys.TYPE] == PSKeys.EMAIL:
- if answer.lower() == "na":
- structured_output[output[PSKeys.NAME]] = None
- else:
- prompt = f'Extract the email from the following text:\n{answer}\n\nOutput just the email. \
- The email should be directly assignable to a string variable. \
- No explanation is required. If you cannot extract the email, output "NA".' # noqa
- answer = run_completion(
- llm=llm,
- prompt=prompt,
- )
- structured_output[output[PSKeys.NAME]] = answer
- elif output[PSKeys.TYPE] == PSKeys.DATE:
- if answer.lower() == "na":
- structured_output[output[PSKeys.NAME]] = None
- else:
- prompt = f'Extract the date from the following text:\n{answer}\n\nOutput just the date.\
- The date should be in ISO date time format. No explanation is required. \
- The date should be directly assignable to a date variable. \
- If you cannot convert the string into a date, output "NA".' # noqa
- answer = run_completion(
- llm=llm,
- prompt=prompt,
- )
- structured_output[output[PSKeys.NAME]] = answer
-
- elif output[PSKeys.TYPE] == PSKeys.BOOLEAN:
- if answer.lower() == "na":
- structured_output[output[PSKeys.NAME]] = None
- else:
- prompt = f'Extract yes/no from the following text:\n{answer}\n\n\
- Output in single word.\
- If the context is trying to convey that the answer is true, \
- then return "yes", else return "no".'
- answer = run_completion(
- llm=llm,
- prompt=prompt,
- )
- if answer.lower() == "yes":
- structured_output[output[PSKeys.NAME]] = True
- else:
- structured_output[output[PSKeys.NAME]] = False
- elif output[PSKeys.TYPE] == PSKeys.JSON:
- if answer.lower() == "[]" or answer.lower() == "na":
- structured_output[output[PSKeys.NAME]] = None
- else:
- try:
- structured_output[output[PSKeys.NAME]] = json.loads(answer)
- except JSONDecodeError:
- prompt = f"Convert the following text into valid JSON string: \
- \n{answer}\n\n The JSON string should be able to be parsed \
- into a Python dictionary. \
- Output just the JSON string. No explanation is required. \
- If you cannot extract the JSON string, output {{}}"
- try:
- answer = run_completion(
- llm=llm,
- prompt=prompt,
- prompt_type=PSKeys.JSON,
- )
- structured_output[output[PSKeys.NAME]] = json.loads(answer)
- except JSONDecodeError as e:
- err_msg = (
- f"Error parsing response (to json): {e}\n"
- f"Candidate JSON: {answer}"
- )
- app.logger.info(err_msg, LogLevel.ERROR)
- # TODO: Format log message after unifying these types
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.INFO,
- RunLevel.RUN,
- "Unable to parse JSON response from LLM, try using our"
- " cloud / enterprise feature of 'line-item', "
- "'record' or 'table' type",
- )
- structured_output[output[PSKeys.NAME]] = {}
-
- else:
- structured_output[output[PSKeys.NAME]] = answer
-
- # If there is a trailing '\n' remove it
- if isinstance(structured_output[output[PSKeys.NAME]], str):
- structured_output[output[PSKeys.NAME]] = structured_output[
- output[PSKeys.NAME]
- ].rstrip("\n")
-
- # Challenge condition
- if enable_challenge:
- challenge_plugin: dict[str, Any] = plugins.get(PSKeys.CHALLENGE, {})
- try:
- if challenge_plugin:
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.INFO,
- RunLevel.CHALLENGE,
- "Challenging response",
- )
- challenge_llm = LLM(
- tool=util,
- adapter_instance_id=tool_settings[PSKeys.CHALLENGE_LLM],
- usage_kwargs={
- **usage_kwargs,
- PSKeys.LLM_USAGE_REASON: PSKeys.CHALLENGE,
- },
- capture_metrics=True,
- )
- challenge = challenge_plugin["entrypoint_cls"](
- llm=llm,
- challenge_llm=challenge_llm,
- run_id=run_id,
- context="\n".join(context),
- tool_settings=tool_settings,
- output=output,
- structured_output=structured_output,
- logger=app.logger,
- platform_key=platform_key,
- metadata=metadata,
- )
- # Will inline replace the structured output passed.
- challenge.run()
- else:
- app.logger.info(
- "No challenge plugin found to evaluate prompt: %s",
- output[PSKeys.NAME],
- )
- except challenge_plugin["exception_cls"] as e:
- app.logger.error(
- "Failed to challenge prompt %s: %s",
- output[PSKeys.NAME],
- str(e),
- )
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.ERROR,
- RunLevel.CHALLENGE,
- "Error while challenging response",
- )
-
- #
- # Evaluate the prompt.
- #
- if (
- PSKeys.EVAL_SETTINGS in output
- and output[PSKeys.EVAL_SETTINGS][PSKeys.EVAL_SETTINGS_EVALUATE]
- ):
- eval_plugin: dict[str, Any] = plugins.get("evaluation", {})
- if eval_plugin:
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.INFO,
- RunLevel.EVAL,
- "Evaluating response",
- )
- try:
- evaluator = eval_plugin["entrypoint_cls"](
- "",
- "\n".join(context),
- "",
- "",
- output,
- structured_output,
- app.logger,
- platform_key,
- )
- # Will inline replace the structured output passed.
- evaluator.run()
- except eval_plugin["exception_cls"] as e:
- app.logger.error(
- f"Failed to evaluate prompt {output[PSKeys.NAME]}: {str(e)}"
- )
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.ERROR,
- RunLevel.EVAL,
- "Error while evaluation",
- )
- else:
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.DEBUG,
- RunLevel.EVAL,
- "Evaluation completed",
- )
- else:
- app.logger.info(
- f"No eval plugin found to evaluate prompt: {output[PSKeys.NAME]}" # noqa: E501
- )
- finally:
- challenge_metrics = (
- {f"{challenge_llm.get_usage_reason()}_llm": challenge_llm.get_metrics()}
- if enable_challenge and challenge_llm
- else {}
- )
- metrics.setdefault(prompt_name, {}).update(
- {
- "context_retrieval": index.get_metrics(),
- f"{llm.get_usage_reason()}_llm": llm.get_metrics(),
- **challenge_metrics,
- }
- )
- vector_db.close()
- publish_log(
- log_events_id,
- {"tool_id": tool_id, "doc_name": doc_name},
- LogLevel.INFO,
- RunLevel.RUN,
- "Sanitizing null values",
- )
- for k, v in structured_output.items():
- if isinstance(v, str) and v.lower() == "na":
- structured_output[k] = None
- elif isinstance(v, list):
- for i in range(len(v)):
- if isinstance(v[i], str) and v[i].lower() == "na":
- v[i] = None
- elif isinstance(v[i], dict):
- for k1, v1 in v[i].items():
- if isinstance(v1, str) and v1.lower() == "na":
- v[i][k1] = None
- elif isinstance(v, dict):
- for k1, v1 in v.items():
- if isinstance(v1, str) and v1.lower() == "na":
- v[k1] = None
-
- publish_log(
- log_events_id,
- {"tool_id": tool_id, "doc_name": doc_name},
- LogLevel.INFO,
- RunLevel.RUN,
- "Execution complete",
- )
- metadata = query_usage_metadata(token=platform_key, metadata=metadata)
- response = {
- PSKeys.METADATA: metadata,
- PSKeys.OUTPUT: structured_output,
- PSKeys.METRICS: metrics,
- }
- return response
-
-
-def fetch_context_from_vector_db(
- index: Index,
- output: dict[str, Any],
- doc_id: str,
- tool_id: str,
- doc_name: str,
- prompt_name: str,
- log_events_id: str,
- usage_kwargs: dict[str, Any],
-) -> set[str]:
- """
- Fetches context from the index for the given document ID. Implements a retry
- mechanism with logging and raises an error if context retrieval fails.
-
- Args:
- index: The index object to query.
- output: Dictionary containing keys like embedding and vector DB instance ID.
- doc_id: The document ID to query.
- tool_id: Identifier for the tool in use.
- doc_name: Name of the document being queried.
- prompt_name: Name of the prompt being executed.
- log_events_id: Unique ID for logging events.
- usage_kwargs: Additional usage parameters.
-
- Raises:
- APIError: If context retrieval fails after retrying.
- """
- context: set[str] = set()
- try:
- retrieved_context = index.query_index(
- embedding_instance_id=output[PSKeys.EMBEDDING],
- vector_db_instance_id=output[PSKeys.VECTOR_DB],
- doc_id=doc_id,
- usage_kwargs=usage_kwargs,
- )
-
- if retrieved_context:
- context.add(retrieved_context)
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.DEBUG,
- RunLevel.RUN,
- "Fetched context from vector DB",
- )
- else:
- # Handle lag in vector DB write (e.g., Pinecone issue)
- time.sleep(2)
- retrieved_context = index.query_index(
- embedding_instance_id=output[PSKeys.EMBEDDING],
- vector_db_instance_id=output[PSKeys.VECTOR_DB],
- doc_id=doc_id,
- usage_kwargs=usage_kwargs,
- )
-
- if retrieved_context is None:
- msg = NO_CONTEXT_ERROR
- app.logger.error(
- f"{msg} {output[PSKeys.VECTOR_DB]} for doc_id {doc_id}"
- )
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.ERROR,
- RunLevel.RUN,
- msg,
- )
- raise APIError(message=msg)
- context.add(retrieved_context)
- except SdkError as e:
- msg = f"Unable to fetch context from vector DB. {str(e)}"
- app.logger.error(
- f"{msg}. VectorDB: {output[PSKeys.VECTOR_DB]}, doc_id: {doc_id}"
- )
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.ERROR,
- RunLevel.RUN,
- msg,
- )
- raise APIError(message=msg, code=e.status_code)
- return context
-
-
-def run_retrieval( # type:ignore
- tool_settings: dict[str, Any],
- output: dict[str, Any],
- doc_id: str,
- llm: LLM,
- vector_index,
- retrieval_type: str,
- metadata: dict[str, Any],
- execution_source: Optional[str] = None,
-) -> tuple[str, set[str]]:
- context: set[str] = set()
- prompt = output[PSKeys.PROMPTX]
- if retrieval_type == PSKeys.SUBQUESTION:
- subq_prompt: str = (
- f"I am sending you a verbose prompt \n \n Prompt : {prompt} \n \n"
- "Generate set of specific subquestions "
- "from the prompt which can be used to retrive "
- "relevant context from vector db. "
- "Use your logical abilities to "
- " only generate as many subquestions as necessary "
- " — fewer subquestions if the prompt is simpler. "
- "Decide the minimum limit for subquestions "
- "based on the complexity input prompt and set the maximum limit"
- "for the subquestions to 10."
- "Ensure that each subquestion is distinct and relevant"
- "to the the original query. "
- "Do not add subquestions for details"
- "not mentioned in the original prompt."
- " The goal is to maximize retrieval accuracy"
- " using these subquestions. Use your logical abilities to ensure "
- " that each subquestion targets a distinct aspect of the original query."
- " Please note that, there are cases where the "
- "response might have a list of answers. The subquestions must not miss out "
- "any values in these cases. "
- "Output should be a list of comma seperated "
- "subquestion prompts. Do not change this format. \n \n "
- " Subquestions : "
- )
- subquestions = run_completion(
- llm=llm,
- prompt=subq_prompt,
- )
- subquestion_list = subquestions.split(",")
- for each_subq in subquestion_list:
- retrieved_context = _retrieve_context(
- output, doc_id, vector_index, each_subq
- )
- context.update(retrieved_context)
-
- if retrieval_type == PSKeys.SIMPLE:
-
- context = _retrieve_context(output, doc_id, vector_index, prompt)
-
- if not context:
- # UN-1288 For Pinecone, we are seeing an inconsistent case where
- # query with doc_id fails even though indexing just happened.
- # This causes the following retrieve to return no text.
- # To rule out any lag on the Pinecone vector DB write,
- # the following sleep is added
- # Note: This will not fix the issue. Since this issue is inconsistent
- # and not reproducible easily, this is just a safety net.
- time.sleep(2)
- context = _retrieve_context(output, doc_id, vector_index, prompt)
-
- answer = construct_and_run_prompt( # type:ignore
- tool_settings=tool_settings,
- output=output,
- llm=llm,
- context="\n".join(context),
- prompt="promptx",
- metadata=metadata,
- execution_source=execution_source,
- )
-
- return (answer, context)
-
-
-def _retrieve_context(output, doc_id, vector_index, answer) -> set[str]:
- retriever = vector_index.as_retriever(
- similarity_top_k=output[PSKeys.SIMILARITY_TOP_K],
- filters=MetadataFilters(
- filters=[
- ExactMatchFilter(key="doc_id", value=doc_id),
- # TODO: Enable after adding section in GUI
- # ExactMatchFilter(
- # key="section", value=output["section"]
- ],
- ),
- )
- nodes = retriever.retrieve(answer)
- context: set[str] = set()
- for node in nodes:
- # ToDo: May have to fine-tune this value for node score or keep it
- # configurable at the adapter level
- if node.score > 0:
- context.add(node.get_content())
- else:
- app.logger.info(
- "Node score is less than 0. "
- f"Ignored: {node.node_id} with score {node.score}"
- )
- return context
-
-
-def log_exceptions(e: HTTPException):
- """Helper method to log exceptions.
-
- Args:
- e (HTTPException): Exception to log
- """
- code = 500
- if hasattr(e, "code"):
- code = e.code or code
-
- if code >= 500:
- message = "{method} {url} {status}\n\n{error}\n\n````{tb}````".format(
- method=request.method,
- url=request.url,
- status=code,
- error=str(e),
- tb=traceback.format_exc(),
- )
- else:
- message = "{method} {url} {status} {error}".format(
- method=request.method,
- url=request.url,
- status=code,
- error=str(e),
- )
- app.logger.error(message)
-
-
[email protected](HTTPException)
-def handle_http_exception(e: HTTPException):
- """Return JSON instead of HTML for HTTP errors."""
- log_exceptions(e)
- if isinstance(e, APIError):
- return jsonify(e.to_dict()), e.code
- else:
- response = e.get_response()
- response.data = json.dumps(
- ErrorResponse(error=e.description, name=e.name, code=e.code)
- )
- response.content_type = "application/json"
- return response
-
-
-def handle_uncaught_exception(e):
- """Handler for uncaught exceptions.
-
- Args:
- e (Exception): Any uncaught exception
- """
- # pass through HTTP errors
- if isinstance(e, HTTPException):
- return handle_http_exception(e)
-
- log_exceptions(e)
- return handle_http_exception(APIError())
diff --git a/prompt-service/src/unstract/prompt_service/prompt_ide_base_tool.py b/prompt-service/src/unstract/prompt_service/prompt_ide_base_tool.py
deleted file mode 100644
index 42a65c94e..000000000
--- a/prompt-service/src/unstract/prompt_service/prompt_ide_base_tool.py
+++ /dev/null
@@ -1,60 +0,0 @@
-import os
-import warnings
-from typing import Optional
-
-from flask import current_app
-from unstract.prompt_service.constants import PromptServiceContants
-from unstract.sdk.constants import LogLevel
-from unstract.sdk.tool.stream import StreamMixin
-from unstract.sdk.utils.common_utils import PY_TO_UNSTRACT_LOG_LEVEL
-
-
-class PromptServiceBaseTool(StreamMixin):
- # TODO: Remove unused log_level arg
- def __init__(
- self, log_level: Optional[LogLevel] = None, platform_key: str = ""
- ) -> None:
- """
- Args:
- tool (UnstractAbstractTool): Instance of UnstractAbstractTool.
- Deprecated, will be removed in future versions
- Notes:
- - PLATFORM_SERVICE_API_KEY environment variable is required.
- """
- if log_level:
- warnings.warn(
- "The 'log_level' argument is deprecated and will be "
- "removed in a future version.",
- DeprecationWarning,
- stacklevel=2, # ensures the warning points to the caller of the method
- )
- self.log_level = PY_TO_UNSTRACT_LOG_LEVEL.get(
- current_app.logger.getEffectiveLevel(), LogLevel.INFO
- )
- self.platform_key = platform_key
- super().__init__(log_level=self.log_level)
-
- def get_env_or_die(self, env_key: str) -> str:
- """Returns the value of an env variable.
-
- If its empty or None, raises an error and exits
-
- Args:
- env_key (str): Key to retrieve
-
- Returns:
- str: Value of the env
- """
- # HACK: Adding platform key for multitenancy
- if env_key == PromptServiceContants.PLATFORM_SERVICE_API_KEY:
- if not self.platform_key:
- current_app.logger.error(f"{env_key} is required")
- else:
- key: str = self.platform_key
- return key
- else:
- env_value = os.environ.get(env_key)
- if env_value is None or env_value == "":
- current_app.logger.error(f"Env variable {env_key} is required")
- return env_value # type:ignore
- return ""
diff --git a/prompt-service/src/unstract/prompt_service/run.py b/prompt-service/src/unstract/prompt_service/run.py
new file mode 100644
index 000000000..b50e97250
--- /dev/null
+++ b/prompt-service/src/unstract/prompt_service/run.py
@@ -0,0 +1,3 @@
+from unstract.prompt_service.config import create_app
+
+app = create_app()
diff --git a/prompt-service/src/unstract/prompt_service_v2/helpers/__init__.py b/prompt-service/src/unstract/prompt_service/services/__init__.py
similarity index 100%
rename from prompt-service/src/unstract/prompt_service_v2/helpers/__init__.py
rename to prompt-service/src/unstract/prompt_service/services/__init__.py
diff --git a/prompt-service/src/unstract/prompt_service_v2/services/answer_prompt.py b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py
similarity index 95%
rename from prompt-service/src/unstract/prompt_service_v2/services/answer_prompt.py
rename to prompt-service/src/unstract/prompt_service/services/answer_prompt.py
index ebd9534fd..a4b703e3e 100644
--- a/prompt-service/src/unstract/prompt_service_v2/services/answer_prompt.py
+++ b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py
@@ -3,10 +3,10 @@ from logging import Logger
from typing import Any, Optional
from flask import current_app as app
+from unstract.prompt_service.constants import ExecutionSource, FileStorageKeys
+from unstract.prompt_service.constants import PromptServiceConstants as PSKeys
from unstract.prompt_service.exceptions import RateLimitError
-from unstract.prompt_service_v2.constants import ExecutionSource, FileStorageKeys
-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys
-from unstract.prompt_service_v2.helpers.plugin import PluginManager
+from unstract.prompt_service.helpers.plugin import PluginManager
from unstract.sdk.exceptions import RateLimitError as SdkRateLimitError
from unstract.sdk.exceptions import SdkError
from unstract.sdk.file_storage import FileStorage, FileStorageProvider
@@ -19,15 +19,6 @@ from unstract.core.flask.exceptions import APIError
class AnswerPromptService:
- @staticmethod
- def get_cleaned_context(context: set[str]) -> list[str]:
- clean_context_plugin: dict[str, Any] = PluginManager().get_plugin(
- PSKeys.CLEAN_CONTEXT
- )
- if clean_context_plugin:
- return clean_context_plugin["entrypoint_cls"].run(context=context)
- return list(context)
-
@staticmethod
def extract_variable(
structured_output: dict[str, Any],
diff --git a/prompt-service/src/unstract/prompt_service_v2/services/extraction.py b/prompt-service/src/unstract/prompt_service/services/extraction.py
similarity index 85%
rename from prompt-service/src/unstract/prompt_service_v2/services/extraction.py
rename to prompt-service/src/unstract/prompt_service/services/extraction.py
index b43894bb6..866e325f8 100644
--- a/prompt-service/src/unstract/prompt_service_v2/services/extraction.py
+++ b/prompt-service/src/unstract/prompt_service/services/extraction.py
@@ -1,16 +1,14 @@
from typing import Any, Optional
-from unstract.prompt_service_v2.constants import ExecutionSource
-from unstract.prompt_service_v2.constants import IndexingConstants as IKeys
-from unstract.prompt_service_v2.helpers.prompt_ide_base_tool import (
- PromptServiceBaseTool,
-)
+from unstract.prompt_service.constants import ExecutionSource
+from unstract.prompt_service.constants import IndexingConstants as IKeys
+from unstract.prompt_service.helpers.prompt_ide_base_tool import PromptServiceBaseTool
+from unstract.prompt_service.utils.file_utils import FileUtils
from unstract.sdk.adapters.exceptions import AdapterError
from unstract.sdk.adapters.x2text.constants import X2TextConstants
from unstract.sdk.adapters.x2text.llm_whisperer.src import LLMWhisperer
from unstract.sdk.adapters.x2text.llm_whisperer_v2.src import LLMWhispererV2
from unstract.sdk.exceptions import X2TextError
-from unstract.sdk.file_storage import FileStorage, FileStorageProvider
from unstract.sdk.utils import ToolUtils
from unstract.sdk.utils.common_utils import log_elapsed
from unstract.sdk.x2txt import TextExtractionResult, X2Text
@@ -28,7 +26,6 @@ class ExtractionService:
output_file_path: Optional[str] = None,
enable_highlight: bool = False,
usage_kwargs: dict[Any, Any] = {},
- fs: FileStorage = FileStorage(FileStorageProvider.LOCAL),
tags: Optional[list[str]] = None,
execution_source: Optional[str] = None,
tool_exec_metadata: Optional[dict[str, Any]] = None,
@@ -40,6 +37,7 @@ class ExtractionService:
x2text = X2Text(
tool=util, adapter_instance_id=x2text_instance_id, usage_kwargs=usage_kwargs
)
+ fs = FileUtils.get_fs_instance(execution_source=execution_source)
try:
if enable_highlight and (
isinstance(x2text.x2text_instance, LLMWhisperer)
@@ -92,8 +90,3 @@ class ExtractionService:
json_to_dump=metadata,
fs=fs,
)
-
- # TODO : Handle runId
- # TODO: Revisit passage of tool through SDK
- # TODO : Revisit usgae of update_exec_metadata
- # TODO : Process Text callable
diff --git a/prompt-service/src/unstract/prompt_service_v2/services/indexing.py b/prompt-service/src/unstract/prompt_service/services/indexing.py
similarity index 85%
rename from prompt-service/src/unstract/prompt_service_v2/services/indexing.py
rename to prompt-service/src/unstract/prompt_service/services/indexing.py
index 09361887b..9205294c8 100644
--- a/prompt-service/src/unstract/prompt_service_v2/services/indexing.py
+++ b/prompt-service/src/unstract/prompt_service/services/indexing.py
@@ -1,17 +1,16 @@
import logging
-from unstract.prompt_service_v2.core.index_v2 import Index
-from unstract.prompt_service_v2.dto import (
+from flask import current_app as app
+from unstract.prompt_service.core.index_v2 import Index
+from unstract.prompt_service.dto import (
ChunkingConfig,
FileInfo,
InstanceIdentifiers,
ProcessingOptions,
)
-from unstract.prompt_service_v2.exceptions import APIError
-from unstract.prompt_service_v2.helpers.prompt_ide_base_tool import (
- PromptServiceBaseTool,
-)
-from unstract.prompt_service_v2.utils.file_utils import FileUtils
+from unstract.prompt_service.exceptions import APIError
+from unstract.prompt_service.helpers.prompt_ide_base_tool import PromptServiceBaseTool
+from unstract.prompt_service.utils.file_utils import FileUtils
from unstract.sdk.embedding import Embedding
from unstract.sdk.utils.indexing_utils import IndexingUtils
from unstract.sdk.vector_db import VectorDB
@@ -72,6 +71,7 @@ class IndexingService:
vector_db=vector_db,
)
if doc_id_found:
+ app.logger.info("Doc ID found: %s", doc_id_found)
return doc_id
# Index and return doc_id
@@ -79,6 +79,7 @@ class IndexingService:
vector_db=vector_db,
doc_id=doc_id,
extracted_text=extracted_text,
+ doc_id_found=doc_id_found,
)
return doc_id
except Exception as e:
diff --git a/prompt-service/src/unstract/prompt_service_v2/services/retrieval_v2.py b/prompt-service/src/unstract/prompt_service/services/retrieval.py
similarity index 62%
rename from prompt-service/src/unstract/prompt_service_v2/services/retrieval_v2.py
rename to prompt-service/src/unstract/prompt_service/services/retrieval.py
index 63211573e..1e6cff78b 100644
--- a/prompt-service/src/unstract/prompt_service_v2/services/retrieval_v2.py
+++ b/prompt-service/src/unstract/prompt_service/services/retrieval.py
@@ -1,10 +1,10 @@
-from typing import Any, Optional
+from typing import Any
-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys
-from unstract.prompt_service_v2.core.retrievers.simple import SimpleRetriever
-from unstract.prompt_service_v2.core.retrievers.subquestion import SubquestionRetriever
-from unstract.prompt_service_v2.services.answer_prompt import AnswerPromptService
-from unstract.prompt_service_v2.utils.file_utils import FileUtils
+from unstract.prompt_service.constants import PromptServiceConstants as PSKeys
+from unstract.prompt_service.core.retrievers.simple import SimpleRetriever
+from unstract.prompt_service.core.retrievers.subquestion import SubquestionRetriever
+from unstract.prompt_service.services.answer_prompt import AnswerPromptService
+from unstract.prompt_service.utils.file_utils import FileUtils
from unstract.sdk.llm import LLM
from unstract.sdk.vector_db import VectorDB
@@ -22,71 +22,73 @@ class RetrievalService:
chunk_size: int,
execution_source: str,
file_path: str,
- ) -> set[str]:
- context: set[str] = set()
-
+ ) -> tuple[str, list[str]]:
+ context: list[str]
if chunk_size == 0:
context = RetrievalService.retrieve_complete_context(
execution_source=execution_source, file_path=file_path
)
else:
context = RetrievalService.run_retrieval(
- tool_settings=tool_settings,
output=output,
doc_id=doc_id,
llm=llm,
vector_db=vector_db,
retrieval_type=retrieval_type,
- metadata=metadata,
- execution_source=execution_source,
)
-
- return context
+ answer = AnswerPromptService.construct_and_run_prompt( # type:ignore
+ tool_settings=tool_settings,
+ output=output,
+ llm=llm,
+ context="\n".join(context),
+ prompt="promptx",
+ metadata=metadata,
+ execution_source=execution_source,
+ file_path=file_path,
+ )
+ return answer, context
@staticmethod
def run_retrieval( # type:ignore
- tool_settings: dict[str, Any],
output: dict[str, Any],
doc_id: str,
llm: LLM,
vector_db: VectorDB,
retrieval_type: str,
- metadata: dict[str, Any],
- execution_source: Optional[str] = None,
- ) -> tuple[str, set[str]]:
- context: set[str] = set()
+ ) -> list[str]:
+ context: set[str]
prompt = output[PSKeys.PROMPTX]
top_k = output[PSKeys.SIMILARITY_TOP_K]
if retrieval_type == PSKeys.SUBQUESTION:
context = SubquestionRetriever(
- vector_db=vector_db, doc_id=doc_id, prompt=prompt, top_k=top_k
+ vector_db=vector_db,
+ doc_id=doc_id,
+ prompt=prompt,
+ top_k=top_k,
+ llm=llm,
).retrieve()
if retrieval_type == PSKeys.SIMPLE:
context = SimpleRetriever(
- vector_db=vector_db, doc_id=doc_id, prompt=prompt, top_k=top_k
+ vector_db=vector_db,
+ doc_id=doc_id,
+ prompt=prompt,
+ top_k=top_k,
+ llm=llm,
).retrieve()
- answer = AnswerPromptService.construct_and_run_prompt( # type:ignore
- tool_settings=tool_settings,
- output=output,
- llm=llm,
- context="\n".join(context),
- prompt="promptx",
- metadata=metadata,
- execution_source=execution_source,
- )
-
- return (answer, context)
+ return list(context)
@staticmethod
- def retrieve_complete_context(execution_source: str, file_path: str) -> str:
+ def retrieve_complete_context(execution_source: str, file_path: str) -> list[str]:
"""
Loads full context from raw file for zero chunk size retrieval
Args:
- path (str): Path to the directory containing text file.
+ execution_source (str): Source of execution.
+ file_path (str): Path to the directory containing text file.
Returns:
- str: context from extracted file.
+ list[str]: context from extracted file.
"""
fs_instance = FileUtils.get_fs_instance(execution_source=execution_source)
- return fs_instance.read(path=file_path, mode="r")
+ context = fs_instance.read(path=file_path, mode="r")
+ return [context]
diff --git a/prompt-service/src/unstract/prompt_service_v2/services/variable_replacement.py b/prompt-service/src/unstract/prompt_service/services/variable_replacement.py
similarity index 94%
rename from prompt-service/src/unstract/prompt_service_v2/services/variable_replacement.py
rename to prompt-service/src/unstract/prompt_service/services/variable_replacement.py
index 3492a418a..13d5987e8 100644
--- a/prompt-service/src/unstract/prompt_service_v2/services/variable_replacement.py
+++ b/prompt-service/src/unstract/prompt_service/services/variable_replacement.py
@@ -1,12 +1,12 @@
from typing import Any
from flask import current_app as app
-from unstract.prompt_service.utils.log import publish_log
-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys
-from unstract.prompt_service_v2.constants import RunLevel, VariableType
-from unstract.prompt_service_v2.helpers.variable_replacement import (
+from unstract.prompt_service.constants import PromptServiceConstants as PSKeys
+from unstract.prompt_service.constants import RunLevel, VariableType
+from unstract.prompt_service.helpers.variable_replacement import (
VariableReplacementHelper,
)
+from unstract.prompt_service.utils.log import publish_log
from unstract.sdk.constants import LogLevel
diff --git a/prompt-service/src/unstract/prompt_service_v2/tests/conftest.py b/prompt-service/src/unstract/prompt_service/tests/conftest.py
similarity index 77%
rename from prompt-service/src/unstract/prompt_service_v2/tests/conftest.py
rename to prompt-service/src/unstract/prompt_service/tests/conftest.py
index e13f7f486..b37f29e07 100644
--- a/prompt-service/src/unstract/prompt_service_v2/tests/conftest.py
+++ b/prompt-service/src/unstract/prompt_service/tests/conftest.py
@@ -2,8 +2,8 @@ import pytest
from dotenv import load_dotenv
from flask import Flask
from flask_wtf.csrf import CSRFProtect
-from unstract.prompt_service_v2.controllers.extraction import extraction_bp
-from unstract.prompt_service_v2.controllers.indexing import indexing_bp
+from unstract.prompt_service.controllers.extraction import extraction_bp
+from unstract.prompt_service.controllers.indexing import indexing_bp
# Load test environment variables
load_dotenv(".env.test")
@@ -12,7 +12,6 @@ load_dotenv(".env.test")
@pytest.fixture
def client():
app = Flask(__name__)
-
app.register_blueprint(indexing_bp)
app.register_blueprint(extraction_bp)
app.config.update(
@@ -27,4 +26,3 @@ def client():
csrf.init_app(app) # Compliant
with app.test_client() as client:
yield client
- # TODO Add teardown code here
diff --git a/prompt-service/src/unstract/prompt_service_v2/tests/integration/input/sample1.pdf b/prompt-service/src/unstract/prompt_service/tests/integration/input/sample1.pdf
similarity index 100%
rename from prompt-service/src/unstract/prompt_service_v2/tests/integration/input/sample1.pdf
rename to prompt-service/src/unstract/prompt_service/tests/integration/input/sample1.pdf
diff --git a/prompt-service/src/unstract/prompt_service_v2/tests/integration/test_api_endpoints.py b/prompt-service/src/unstract/prompt_service/tests/integration/test_api_endpoints.py
similarity index 97%
rename from prompt-service/src/unstract/prompt_service_v2/tests/integration/test_api_endpoints.py
rename to prompt-service/src/unstract/prompt_service/tests/integration/test_api_endpoints.py
index ed4b16d66..7192d1817 100644
--- a/prompt-service/src/unstract/prompt_service_v2/tests/integration/test_api_endpoints.py
+++ b/prompt-service/src/unstract/prompt_service/tests/integration/test_api_endpoints.py
@@ -3,7 +3,7 @@ import os
from dotenv import load_dotenv
from flask.testing import FlaskClient
-from unstract.prompt_service_v2.helpers.auth import AuthHelper
+from unstract.prompt_service.helpers.auth import AuthHelper
load_dotenv(".env.test")
diff --git a/prompt-service/src/unstract/prompt_service_v2/tests/sample.env.test b/prompt-service/src/unstract/prompt_service/tests/sample.env.test
similarity index 91%
rename from prompt-service/src/unstract/prompt_service_v2/tests/sample.env.test
rename to prompt-service/src/unstract/prompt_service/tests/sample.env.test
index 0fd829f42..7554cc215 100644
--- a/prompt-service/src/unstract/prompt_service_v2/tests/sample.env.test
+++ b/prompt-service/src/unstract/prompt_service/tests/sample.env.test
@@ -10,8 +10,8 @@ TEST_EXECUTION_SOURCE=ide
TEST_RUN_ID=mock_run_id
TEST_EXTRACTED_TEXT=mock_extracted_text
TEST_TAGS=mock_tag1,mock_tag2
-TEST_SOURCE_FILE_PATH=/unstract/prompt-service/src/unstract/prompt_service_v2/tests/integration/input/sample1.pdf
-TEST_OUTPUT_FILE_PATH=/unstract/prompt-service/src/unstract/prompt_service_v2/tests/integration/output/output.txt
+TEST_SOURCE_FILE_PATH=/unstract/prompt-service/src/unstract/prompt_service/tests/integration/input/sample1.pdf
+TEST_OUTPUT_FILE_PATH=/unstract/prompt-service/src/unstract/prompt_service/tests/integration/output/output.txt
# Platform Service
# Platform service is expected to be up and running for this integration test to run
diff --git a/prompt-service/src/unstract/prompt_service_v2/services/__init__.py b/prompt-service/src/unstract/prompt_service/utils/__init__.py
similarity index 100%
rename from prompt-service/src/unstract/prompt_service_v2/services/__init__.py
rename to prompt-service/src/unstract/prompt_service/utils/__init__.py
diff --git a/prompt-service/src/unstract/prompt_service/db_utils.py b/prompt-service/src/unstract/prompt_service/utils/db_utils.py
similarity index 89%
rename from prompt-service/src/unstract/prompt_service/db_utils.py
rename to prompt-service/src/unstract/prompt_service/utils/db_utils.py
index f20e34ddb..f650f6187 100644
--- a/prompt-service/src/unstract/prompt_service/db_utils.py
+++ b/prompt-service/src/unstract/prompt_service/utils/db_utils.py
@@ -1,10 +1,10 @@
from typing import Any, Optional
-from unstract.prompt_service.config import db
from unstract.prompt_service.constants import DBTableV2
-from unstract.prompt_service.env_manager import EnvLoader
+from unstract.prompt_service.extensions import db
+from unstract.prompt_service.utils.env_loader import get_env_or_die
-DB_SCHEMA = EnvLoader.get_env_or_die("DB_SCHEMA", "unstract")
+DB_SCHEMA = get_env_or_die("DB_SCHEMA", "unstract")
class DBUtils:
diff --git a/prompt-service/src/unstract/prompt_service_v2/utils/env_loader.py b/prompt-service/src/unstract/prompt_service/utils/env_loader.py
similarity index 100%
rename from prompt-service/src/unstract/prompt_service_v2/utils/env_loader.py
rename to prompt-service/src/unstract/prompt_service/utils/env_loader.py
diff --git a/prompt-service/src/unstract/prompt_service_v2/utils/file_utils.py b/prompt-service/src/unstract/prompt_service/utils/file_utils.py
similarity index 93%
rename from prompt-service/src/unstract/prompt_service_v2/utils/file_utils.py
rename to prompt-service/src/unstract/prompt_service/utils/file_utils.py
index ce73926f5..1ab7145e4 100644
--- a/prompt-service/src/unstract/prompt_service_v2/utils/file_utils.py
+++ b/prompt-service/src/unstract/prompt_service/utils/file_utils.py
@@ -1,4 +1,4 @@
-from unstract.prompt_service_v2.constants import ExecutionSource, FileStorageKeys
+from unstract.prompt_service.constants import ExecutionSource, FileStorageKeys
from unstract.sdk.file_storage import FileStorage
from unstract.sdk.file_storage.constants import StorageType
from unstract.sdk.file_storage.env_helper import EnvHelper
diff --git a/prompt-service/src/unstract/prompt_service/utils/request.py b/prompt-service/src/unstract/prompt_service/utils/request.py
index c7006af38..b6c6848ba 100644
--- a/prompt-service/src/unstract/prompt_service/utils/request.py
+++ b/prompt-service/src/unstract/prompt_service/utils/request.py
@@ -1,12 +1,10 @@
-import logging
from enum import Enum
from typing import Any, Optional
import requests as pyrequests
+from flask import current_app as app
from requests.exceptions import RequestException
-from unstract.prompt_service.exceptions import APIError
-
-logger = logging.getLogger(__name__)
+from unstract.prompt_service.exceptions import APIError, BadRequest, MissingFieldError
class HTTPMethod(str, Enum):
@@ -43,8 +41,18 @@ def make_http_request(
)
return return_val
except RequestException as e:
- logger.error(f"HTTP request error: {e}")
+ app.logger.error(f"HTTP request error: {e}")
raise APIError(f"Error occured while invoking POST API Variable : {str(e)}")
except Exception as e:
- logger.error(f"An unexpected error occurred: {e}")
+ app.logger.error(f"An unexpected error occurred: {e}")
raise e
+
+
+def validate_request_payload(payload, required_fields):
+ if not payload:
+ raise BadRequest()
+
+ # Validate required fields
+ missing_fields = [field for field in required_fields if field not in payload]
+ if missing_fields:
+ raise MissingFieldError(missing_fields)
diff --git a/prompt-service/src/unstract/prompt_service/utils/request_id_filter.py b/prompt-service/src/unstract/prompt_service/utils/request_id_filter.py
new file mode 100644
index 000000000..649cff547
--- /dev/null
+++ b/prompt-service/src/unstract/prompt_service/utils/request_id_filter.py
@@ -0,0 +1,11 @@
+import logging
+
+from flask import g
+
+
+class RequestIDFilter(logging.Filter):
+ """Filter to inject request ID into log records."""
+
+ def filter(self, record):
+ record.request_id = getattr(g, "request_id", "N/A") if g else "N/A"
+ return True
diff --git a/prompt-service/src/unstract/prompt_service/variable_extractor/base.py b/prompt-service/src/unstract/prompt_service/variable_extractor/base.py
deleted file mode 100644
index dbcde7a39..000000000
--- a/prompt-service/src/unstract/prompt_service/variable_extractor/base.py
+++ /dev/null
@@ -1,114 +0,0 @@
-from typing import Any
-
-from flask import current_app as app
-from unstract.prompt_service.constants import PromptServiceContants as PSKeys
-from unstract.prompt_service.constants import RunLevel
-from unstract.prompt_service.utils.log import publish_log
-from unstract.sdk.constants import LogLevel
-
-from .constants import VariableType
-from .prompt_variable_service import VariableService
-
-
-class VariableExtractor:
-
- @staticmethod
- def execute_variable_replacement(
- prompt_text: str, variable_map: dict[str, Any]
- ) -> str:
- variables: list[str] = VariableService.extract_variables_from_prompt(
- prompt_text=prompt_text
- )
- for variable in variables:
- variable_type = VariableService.identify_variable_type(variable=variable)
- if variable_type == VariableType.STATIC:
- prompt_text = VariableService.replace_static_variable(
- prompt=prompt_text,
- structured_output=variable_map,
- variable=variable,
- )
-
- if variable_type == VariableType.DYNAMIC:
- prompt_text = VariableService.replace_dynamic_variable(
- prompt=prompt_text,
- variable=variable,
- structured_output=variable_map,
- )
- return prompt_text
-
- @staticmethod
- def is_variables_present(prompt_text: str) -> bool:
- """Determines if variables are present in the prompt.
-
- Args:
- prompt (str): Prompt to check
-
- Returns:
- bool: True if variables are present else False
- """
- return bool(len(VariableService.extract_variables_from_prompt(prompt_text)))
-
- @staticmethod
- def replace_variables_in_prompt(
- prompt: dict[str, Any],
- structured_output: dict[str, Any],
- log_events_id: str,
- tool_id: str,
- prompt_name: str,
- doc_name: str,
- ) -> str:
- """Replaces variables in prompt.
-
- Args:
- prompt (dict[str, Any]): Dict representing the prompt card
- structured_output (dict[str, Any]): Structured data used for variable
- replacement when variable map is not present in `prompt`.
- log_events_id (str): UUID for the WS communication
- tool_id (str): UUID for the prompt studio project
- prompt_name (str): Name of the prompt being run
- doc_name (str): Name of the document being run with
- Returns:
- prompt_text (str): Prompt with variables replaced
- """
- app.logger.info(f"[{tool_id}] Replacing variables in prompt : {prompt_name}")
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.DEBUG,
- RunLevel.RUN,
- "Replacing variables in prompt",
- )
- # Initialized with the prompt because
- # it is used in finally block
- prompt_text = prompt[PSKeys.PROMPT]
- try:
- variable_map = prompt[PSKeys.VARIABLE_MAP]
- prompt_text = VariableExtractor.execute_variable_replacement(
- prompt_text=prompt[PSKeys.PROMPT], variable_map=variable_map
- )
- except KeyError:
- # Executed incase of structured tool and
- # APIs where we do not set the variable map
- prompt_text = VariableExtractor.execute_variable_replacement(
- prompt_text=prompt_text, variable_map=structured_output
- )
- finally:
- app.logger.info(
- f"[{tool_id}] Prompt after variable replacement: {prompt_text}"
- )
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.DEBUG,
- RunLevel.RUN,
- f"Prompt after variable replacement:{prompt_text} ",
- )
- return prompt_text
diff --git a/prompt-service/src/unstract/prompt_service/variable_extractor/constants.py b/prompt-service/src/unstract/prompt_service/variable_extractor/constants.py
deleted file mode 100644
index b456315f4..000000000
--- a/prompt-service/src/unstract/prompt_service/variable_extractor/constants.py
+++ /dev/null
@@ -1,13 +0,0 @@
-from enum import Enum
-
-
-class VariableType(str, Enum):
- STATIC = "STATIC"
- DYNAMIC = "DYNAMIC"
-
-
-class VariableConstants:
-
- VARIABLE_REGEX = "{{(.+?)}}"
- DYNAMIC_VARIABLE_DATA_REGEX = r"\[(.*?)\]"
- DYNAMIC_VARIABLE_URL_REGEX = r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))" # noqa: E501
diff --git a/prompt-service/src/unstract/prompt_service/variable_extractor/prompt_variable_service.py b/prompt-service/src/unstract/prompt_service/variable_extractor/prompt_variable_service.py
deleted file mode 100644
index d27f4525a..000000000
--- a/prompt-service/src/unstract/prompt_service/variable_extractor/prompt_variable_service.py
+++ /dev/null
@@ -1,121 +0,0 @@
-import json
-import logging
-import re
-from functools import lru_cache
-from typing import Any
-
-from unstract.prompt_service.utils.request import HTTPMethod, make_http_request
-
-from .constants import VariableConstants, VariableType
-
-logger = logging.getLogger(__name__)
-
-
-class VariableService:
-
- @staticmethod
- def replace_static_variable(
- prompt: str, structured_output: dict[str, Any], variable: str
- ) -> str:
- output_value = VariableService.check_static_variable_run_status(
- structure_output=structured_output, variable=variable
- )
- if not output_value:
- return prompt
- static_variable_marker_string = "".join(["{{", variable, "}}"])
-
- replaced_prompt: str = VariableService.replace_generic_string_value(
- prompt=prompt, variable=static_variable_marker_string, value=output_value
- )
-
- return replaced_prompt
-
- @staticmethod
- def check_static_variable_run_status(
- structure_output: dict[str, Any], variable: str
- ) -> Any:
- output = None
- try:
- output = structure_output[variable]
- return output
- except KeyError:
- logger.warn(
- f"Prompt with {variable} is not executed yet."
- " Unable to replace the variable"
- )
- return output
-
- @staticmethod
- def replace_generic_string_value(prompt: str, variable: str, value: Any) -> str:
- formatted_value: str = value
- if not isinstance(value, str):
- formatted_value = VariableService.handle_json_and_str_types(value)
- replaced_prompt = prompt.replace(variable, formatted_value)
- return replaced_prompt
-
- @staticmethod
- def handle_json_and_str_types(value: Any) -> str:
- try:
- formatted_value = json.dumps(value)
- except ValueError:
- formatted_value = str(value)
- return formatted_value
-
- @staticmethod
- def identify_variable_type(variable: str) -> VariableType:
- variable_type: VariableType
- pattern = re.compile(VariableConstants.DYNAMIC_VARIABLE_URL_REGEX)
- if re.findall(pattern, variable):
- variable_type = VariableType.DYNAMIC
- else:
- variable_type = VariableType.STATIC
- return variable_type
-
- @staticmethod
- def replace_dynamic_variable(
- prompt: str, variable: str, structured_output: dict[str, Any]
- ) -> str:
- url = re.search(VariableConstants.DYNAMIC_VARIABLE_URL_REGEX, variable).group(0)
- data = re.findall(VariableConstants.DYNAMIC_VARIABLE_DATA_REGEX, variable)[0]
- output_value = VariableService.check_static_variable_run_status(
- structure_output=structured_output, variable=data
- )
- if not output_value:
- return prompt
- api_response: Any = VariableService.fetch_dynamic_variable_value(
- url=url, data=output_value
- )
- formatted_api_response: str = VariableService.handle_json_and_str_types(
- api_response
- )
- static_variable_marker_string = "".join(["{{", variable, "}}"])
- replaced_prompt: str = VariableService.replace_generic_string_value(
- prompt=prompt,
- variable=static_variable_marker_string,
- value=formatted_api_response,
- )
- return replaced_prompt
-
- @staticmethod
- @lru_cache(maxsize=128)
- def extract_variables_from_prompt(prompt_text: str) -> list[str]:
- variable: list[str] = []
- variable = re.findall(VariableConstants.VARIABLE_REGEX, prompt_text)
- return variable
-
- @staticmethod
- def fetch_dynamic_variable_value(url: str, data: str) -> Any:
-
- # This prototype method currently supports
- # only endpoints that do not require authentication.
- # Additionally, it only accepts plain text
- # inputs for POST requests in this version.
- # Future versions may include support for
- # authentication and other input formats.
-
- verb: HTTPMethod = HTTPMethod.POST
- headers = {"Content-Type": "text/plain"}
- response: Any = make_http_request(
- verb=verb, url=url, data=data, headers=headers
- )
- return response
diff --git a/prompt-service/src/unstract/prompt_service_v2/config.py b/prompt-service/src/unstract/prompt_service_v2/config.py
deleted file mode 100644
index 391996a7f..000000000
--- a/prompt-service/src/unstract/prompt_service_v2/config.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import logging
-from os import environ as env
-
-from dotenv import load_dotenv
-from flask import Flask
-from unstract.prompt_service_v2.controllers import api
-from unstract.prompt_service_v2.extensions import db
-from unstract.prompt_service_v2.helpers.lifecycle import register_lifecycle_hooks
-from unstract.prompt_service_v2.helpers.plugin import plugin_loader
-from unstract.prompt_service_v2.utils.env_loader import get_env_or_die
-from unstract.sdk.constants import LogLevel
-
-from unstract.core.flask import register_error_handlers, register_request_id_middleware
-from unstract.core.flask.logging import setup_logging
-
-load_dotenv()
-
-
-def create_app() -> Flask:
- """Creates and configures the Flask application."""
-
- log_level = env.get("LOG_LEVEL", LogLevel.INFO.value).upper()
- setup_logging(log_level)
- log_level = getattr(logging, log_level, logging.INFO)
- app = Flask("prompt-service")
- app.logger.setLevel(log_level)
- app.logger.info("Initializing Flask application...")
- # Load required environment variables
- db_host = get_env_or_die("PG_BE_HOST")
- db_port = get_env_or_die("PG_BE_PORT")
- db_user = get_env_or_die("PG_BE_USERNAME")
- db_pass = get_env_or_die("PG_BE_PASSWORD")
- db_name = get_env_or_die("PG_BE_DATABASE")
- application_name = env.get("APPLICATION_NAME", "unstract-prompt-service")
-
- # Initialize and connect to the database
- db.init(
- database=db_name,
- user=db_user,
- password=db_pass,
- host=db_host,
- port=db_port,
- options=f"-c application_name={application_name}",
- )
-
- # Load plugins
- plugin_loader(app)
- register_lifecycle_hooks(app)
- register_request_id_middleware(app)
- register_error_handlers(app)
- app.register_blueprint(api)
-
- app.logger.info("Flask app created successfully.")
- return app
diff --git a/prompt-service/src/unstract/prompt_service_v2/constants.py b/prompt-service/src/unstract/prompt_service_v2/constants.py
deleted file mode 100644
index 22bc3bc87..000000000
--- a/prompt-service/src/unstract/prompt_service_v2/constants.py
+++ /dev/null
@@ -1,174 +0,0 @@
-from enum import Enum
-
-
-class PromptServiceConstants:
- """Constants used in the prompt service."""
-
- WORD = "word"
- SYNONYMS = "synonyms"
- OUTPUTS = "outputs"
- TOOL_ID = "tool_id"
- RUN_ID = "run_id"
- FILE_NAME = "file_name"
- FILE_HASH = "file_hash"
- NAME = "name"
- ACTIVE = "active"
- PROMPT = "prompt"
- CHUNK_SIZE = "chunk-size"
- PROMPTX = "promptx"
- VECTOR_DB = "vector-db"
- EMBEDDING = "embedding"
- X2TEXT_ADAPTER = "x2text_adapter"
- CHUNK_OVERLAP = "chunk-overlap"
- LLM = "llm"
- IS_ASSERT = "is_assert"
- ASSERTION_FAILURE_PROMPT = "assertion_failure_prompt"
- RETRIEVAL_STRATEGY = "retrieval-strategy"
- SIMPLE = "simple"
- SUBQUESTION = "subquestion"
- TYPE = "type"
- NUMBER = "number"
- EMAIL = "email"
- DATE = "date"
- BOOLEAN = "boolean"
- JSON = "json"
- PREAMBLE = "preamble"
- SIMILARITY_TOP_K = "similarity-top-k"
- PROMPT_TOKENS = "prompt_tokens"
- COMPLETION_TOKENS = "completion_tokens"
- TOTAL_TOKENS = "total_tokens"
- RESPONSE = "response"
- POSTAMBLE = "postamble"
- GRAMMAR = "grammar"
- PLATFORM_SERVICE_API_KEY = "PLATFORM_SERVICE_API_KEY"
- EMBEDDING_SUFFIX = "embedding_suffix"
- EVAL_SETTINGS = "eval_settings"
- EVAL_SETTINGS_EVALUATE = "evaluate"
- EVAL_SETTINGS_MONITOR_LLM = "monitor_llm"
- EVAL_SETTINGS_EXCLUDE_FAILED = "exclude_failed"
- TOOL_SETTINGS = "tool_settings"
- LOG_EVENTS_ID = "log_events_id"
- CHALLENGE_LLM = "challenge_llm"
- CHALLENGE = "challenge"
- ENABLE_CHALLENGE = "enable_challenge"
- EXTRACTION = "extraction"
- SUMMARIZE = "summarize"
- SINGLE_PASS_EXTRACTION = "single-pass-extraction"
- SIMPLE_PROMPT_STUDIO = "simple-prompt-studio"
- LLM_USAGE_REASON = "llm_usage_reason"
- METADATA = "metadata"
- OUTPUT = "output"
- CONTEXT = "context"
- INCLUDE_METADATA = "include_metadata"
- TABLE = "table"
- TABLE_SETTINGS = "table_settings"
- EPILOGUE = "epilogue"
- PLATFORM_POSTAMBLE = "platform_postamble"
- HIGHLIGHT_DATA_PLUGIN = "highlight-data"
- CLEAN_CONTEXT = "clean-context"
- SUMMARIZE_AS_SOURCE = "summarize_as_source"
- VARIABLE_MAP = "variable_map"
- RECORD = "record"
- TEXT = "text"
- ENABLE_HIGHLIGHT = "enable_highlight"
- FILE_PATH = "file_path"
- HIGHLIGHT_DATA = "highlight_data"
- CONFIDENCE_DATA = "confidence_data"
- REQUIRED_FIELDS = "required_fields"
- REQUIRED = "required"
- EXECUTION_SOURCE = "execution_source"
- METRICS = "metrics"
- LINE_ITEM = "line-item"
- LINE_NUMBERS = "line_numbers"
- WHISPER_HASH = "whisper_hash"
- PAID_FEATURE_MSG = (
- "It is a cloud / enterprise feature. If you have purchased a plan and still "
- "face this issue, please contact support"
- )
- NO_CONTEXT_ERROR = (
- "Couldn't fetch context from vector DB. "
- "This happens usually due to a delay by the Vector DB "
- "provider to confirm writes to DB. "
- "Please try again after some time"
- )
-
-
-class RunLevel(Enum):
- """Different stages of prompt execution.
-
- Comprises of prompt run and response evaluation stages.
- """
-
- RUN = "RUN"
- EVAL = "EVAL"
- CHALLENGE = "CHALLENGE"
- TABLE_EXTRACTION = "TABLE_EXTRACTION"
-
-
-class DBTableV2:
- """Database tables."""
-
- ORGANIZATION = "organization"
- ADAPTER_INSTANCE = "adapter_instance"
- PROMPT_STUDIO_REGISTRY = "prompt_studio_registry"
- PLATFORM_KEY = "platform_key"
- TOKEN_USAGE = "usage"
-
-
-class FileStorageKeys:
- """File storage keys."""
-
- PERMANENT_REMOTE_STORAGE = "PERMANENT_REMOTE_STORAGE"
- TEMPORARY_REMOTE_STORAGE = "TEMPORARY_REMOTE_STORAGE"
-
-
-class FileStorageType(Enum):
- """File storage type."""
-
- PERMANENT = "permanent"
- TEMPORARY = "temporary"
-
-
-class ExecutionSource(Enum):
- """Execution source."""
-
- IDE = "ide"
- TOOL = "tool"
-
-
-class VariableType(str, Enum):
- """Type of variable."""
-
- STATIC = "STATIC"
- DYNAMIC = "DYNAMIC"
-
-
-class VariableConstants:
- """Constants for variable extraction."""
-
- VARIABLE_REGEX = "{{(.+?)}}"
- DYNAMIC_VARIABLE_DATA_REGEX = r"\[(.*?)\]"
- DYNAMIC_VARIABLE_URL_REGEX = r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))" # noqa: E501
-
-
-class IndexingConstants:
- TOOL_ID = "tool_id"
- EMBEDDING_INSTANCE_ID = "embedding_instance_id"
- VECTOR_DB_INSTANCE_ID = "vector_db_instance_id"
- X2TEXT_INSTANCE_ID = "x2text_instance_id"
- FILE_PATH = "file_path"
- CHUNK_SIZE = "chunk_size"
- CHUNK_OVERLAP = "chunk_overlap"
- REINDEX = "reindex"
- FILE_HASH = "file_hash"
- OUTPUT_FILE_PATH = "output_file_path"
- ENABLE_HIGHLIGHT = "enable_highlight"
- USAGE_KWARGS = "usage_kwargs"
- PROCESS_TEXT = "process_text"
- EXTRACTED_TEXT = "extracted_text"
- TAGS = "tags"
- EXECUTION_SOURCE = "execution_source"
- DOC_ID = "doc_id"
- TOOL_EXECUTION_METATADA = "tool_execution_metadata"
- EXECUTION_DATA_DIR = "execution_data_dir"
- METADATA_FILE = "METADATA.json"
diff --git a/prompt-service/src/unstract/prompt_service_v2/exceptions.py b/prompt-service/src/unstract/prompt_service_v2/exceptions.py
deleted file mode 100644
index 96b87a614..000000000
--- a/prompt-service/src/unstract/prompt_service_v2/exceptions.py
+++ /dev/null
@@ -1,28 +0,0 @@
-from unstract.core.flask.exceptions import APIError
-
-
-class BadRequest(APIError):
- code = 400
- message = "Bad Request / No payload"
-
-
-class RateLimitError(APIError):
- code = 429
- message = "Running into rate limit errors, please try again later"
-
-
-class MissingFieldError(APIError):
- """Custom error for missing fields."""
-
- def __init__(self, missing_fields: list[str]):
- message = f"Missing required fields: {', '.join(missing_fields)}"
- super().__init__(message=message)
-
-
-class RetrievalError(APIError):
- """Custom exception raised for errors during retrieval from VectorDB."""
-
- DEFAULT_MESSAGE = (
- "Error while retrieving data from the VectorDB. "
- "Please contact the admin for further assistance."
- )
diff --git a/prompt-service/src/unstract/prompt_service_v2/run.py b/prompt-service/src/unstract/prompt_service_v2/run.py
deleted file mode 100644
index f33606904..000000000
--- a/prompt-service/src/unstract/prompt_service_v2/run.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from unstract.prompt_service_v2.config import create_app
-
-app = create_app()
diff --git a/prompt-service/src/unstract/prompt_service_v2/services/retrieval.py b/prompt-service/src/unstract/prompt_service_v2/services/retrieval.py
deleted file mode 100644
index dbcb19e03..000000000
--- a/prompt-service/src/unstract/prompt_service_v2/services/retrieval.py
+++ /dev/null
@@ -1,219 +0,0 @@
-import time
-from typing import Any, Optional
-
-from flask import current_app as app
-from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters
-from unstract.prompt_service.exceptions import APIError
-from unstract.prompt_service_v2.constants import PromptServiceConstants as PSKeys
-from unstract.prompt_service_v2.constants import RunLevel
-from unstract.prompt_service_v2.services.answer_prompt import AnswerPromptService
-from unstract.prompt_service_v2.utils.log import publish_log
-from unstract.sdk.constants import LogLevel
-from unstract.sdk.exceptions import SdkError
-from unstract.sdk.index import Index
-from unstract.sdk.llm import LLM
-
-
-class RetrievalService:
-
- @staticmethod
- def run_retrieval( # type:ignore
- tool_settings: dict[str, Any],
- output: dict[str, Any],
- doc_id: str,
- llm: LLM,
- vector_index,
- retrieval_type: str,
- metadata: dict[str, Any],
- execution_source: Optional[str] = None,
- ) -> tuple[str, set[str]]:
- context: set[str] = set()
- prompt = output[PSKeys.PROMPTX]
- if retrieval_type == PSKeys.SUBQUESTION:
- subq_prompt: str = (
- f"I am sending you a verbose prompt \n \n Prompt : {prompt} \n \n"
- "Generate set of specific subquestions "
- "from the prompt which can be used to retrive "
- "relevant context from vector db. "
- "Use your logical abilities to "
- " only generate as many subquestions as necessary "
- " — fewer subquestions if the prompt is simpler. "
- "Decide the minimum limit for subquestions "
- "based on the complexity input prompt and set the maximum limit"
- "for the subquestions to 10."
- "Ensure that each subquestion is distinct and relevant"
- "to the the original query. "
- "Do not add subquestions for details"
- "not mentioned in the original prompt."
- " The goal is to maximize retrieval accuracy"
- " using these subquestions. Use your logical abilities to ensure "
- " that each subquestion targets a distinct aspect of the original"
- " query. Please note that, there are cases where the "
- "response might have a list of answers. The subquestions must not"
- " miss out any values in these cases. "
- "Output should be a list of comma seperated "
- "subquestion prompts. Do not change this format. \n \n "
- " Subquestions : "
- )
- subquestions = AnswerPromptService.run_completion(
- llm=llm,
- prompt=subq_prompt,
- )
- subquestion_list = subquestions.split(",")
- for each_subq in subquestion_list:
- retrieved_context = RetrievalService._retrieve_context(
- output, doc_id, vector_index, each_subq
- )
- context.update(retrieved_context)
-
- if retrieval_type == PSKeys.SIMPLE:
-
- context = RetrievalService._retrieve_context(
- output, doc_id, vector_index, prompt
- )
-
- if not context:
- # UN-1288 For Pinecone, we are seeing an inconsistent case where
- # query with doc_id fails even though indexing just happened.
- # This causes the following retrieve to return no text.
- # To rule out any lag on the Pinecone vector DB write,
- # the following sleep is added
- # Note: This will not fix the issue. Since this issue is inconsistent
- # and not reproducible easily, this is just a safety net.
- time.sleep(2)
- context = RetrievalService._retrieve_context(
- output, doc_id, vector_index, prompt
- )
-
- answer = AnswerPromptService.construct_and_run_prompt( # type:ignore
- tool_settings=tool_settings,
- output=output,
- llm=llm,
- context="\n".join(context),
- prompt="promptx",
- metadata=metadata,
- execution_source=execution_source,
- )
-
- return (answer, context)
-
- @staticmethod
- def _retrieve_context(output, doc_id, vector_index, answer) -> set[str]:
- retriever = vector_index.as_retriever(
- similarity_top_k=output[PSKeys.SIMILARITY_TOP_K],
- filters=MetadataFilters(
- filters=[
- ExactMatchFilter(key="doc_id", value=doc_id),
- # TODO: Enable after adding section in GUI
- # ExactMatchFilter(
- # key="section", value=output["section"]
- ],
- ),
- )
- nodes = retriever.retrieve(answer)
- context: set[str] = set()
- for node in nodes:
- # ToDo: May have to fine-tune this value for node score or keep it
- # configurable at the adapter level
- if node.score > 0:
- context.add(node.get_content())
- else:
- app.logger.info(
- "Node score is less than 0. "
- f"Ignored: {node.node_id} with score {node.score}"
- )
- return context
-
- @staticmethod
- def fetch_context_from_vector_db(
- index: Index,
- output: dict[str, Any],
- doc_id: str,
- tool_id: str,
- doc_name: str,
- prompt_name: str,
- log_events_id: str,
- usage_kwargs: dict[str, Any],
- ) -> set[str]:
- """
- Fetches context from the index for the given document ID. Implements a retry
- mechanism with logging and raises an error if context retrieval fails.
- Args:
- index: The index object to query.
- output: Dictionary containing keys like embedding and vector DB instance ID.
- doc_id: The document ID to query.
- tool_id: Identifier for the tool in use.
- doc_name: Name of the document being queried.
- prompt_name: Name of the prompt being executed.
- log_events_id: Unique ID for logging events.
- usage_kwargs: Additional usage parameters.
- Raises:
- APIError: If context retrieval fails after retrying.
- """
- context: set[str] = set()
- try:
- retrieved_context = index.query_index(
- embedding_instance_id=output[PSKeys.EMBEDDING],
- vector_db_instance_id=output[PSKeys.VECTOR_DB],
- doc_id=doc_id,
- usage_kwargs=usage_kwargs,
- )
-
- if retrieved_context:
- context.add(retrieved_context)
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.DEBUG,
- RunLevel.RUN,
- "Fetched context from vector DB",
- )
- else:
- # Handle lag in vector DB write (e.g., Pinecone issue)
- time.sleep(2)
- retrieved_context = index.query_index(
- embedding_instance_id=output[PSKeys.EMBEDDING],
- vector_db_instance_id=output[PSKeys.VECTOR_DB],
- doc_id=doc_id,
- usage_kwargs=usage_kwargs,
- )
-
- if retrieved_context is None:
- msg = PSKeys.NO_CONTEXT_ERROR
- app.logger.error(
- f"{msg} {output[PSKeys.VECTOR_DB]} for doc_id {doc_id}"
- )
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.ERROR,
- RunLevel.RUN,
- msg,
- )
- raise APIError(message=msg)
- except SdkError as e:
- msg = f"Unable to fetch context from vector DB. {str(e)}"
- app.logger.error(
- f"{msg}. VectorDB: {output[PSKeys.VECTOR_DB]}, doc_id: {doc_id}"
- )
- publish_log(
- log_events_id,
- {
- "tool_id": tool_id,
- "prompt_key": prompt_name,
- "doc_name": doc_name,
- },
- LogLevel.ERROR,
- RunLevel.RUN,
- msg,
- )
- raise APIError(message=msg, code=e.status_code)
- return context
diff --git a/prompt-service/src/unstract/prompt_service_v2/utils/__init__.py b/prompt-service/src/unstract/prompt_service_v2/utils/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/prompt-service/src/unstract/prompt_service_v2/utils/db_utils.py b/prompt-service/src/unstract/prompt_service_v2/utils/db_utils.py
deleted file mode 100644
index f1c47e8c3..000000000
--- a/prompt-service/src/unstract/prompt_service_v2/utils/db_utils.py
+++ /dev/null
@@ -1,46 +0,0 @@
-from typing import Any, Optional
-
-from unstract.prompt_service_v2.constants import DBTableV2
-from unstract.prompt_service_v2.extensions import db
-from unstract.prompt_service_v2.utils.env_loader import get_env_or_die
-
-DB_SCHEMA = get_env_or_die("DB_SCHEMA", "unstract")
-
-
-class DBUtils:
-
- @classmethod
- def get_organization_from_bearer_token(
- cls, token: str
- ) -> tuple[Optional[int], str]:
- """Retrieve organization ID and identifier using a bearer token.
-
- Args:
- token (str): The bearer token (platform key).
-
- Returns:
- tuple[int, str]: organization uid and organization identifier
- """
- platform_key_table = f'"{DB_SCHEMA}".{DBTableV2.PLATFORM_KEY}'
- organization_table = f'"{DB_SCHEMA}".{DBTableV2.ORGANIZATION}'
-
- organization_uid: Optional[int] = cls.execute_query(
- f"SELECT organization_id FROM {platform_key_table} WHERE key=%s", (token,)
- )
- if organization_uid is None:
- return None, None
-
- organization_identifier: Optional[str] = cls.execute_query(
- f"SELECT organization_id FROM {organization_table} WHERE id=%s",
- (organization_uid,),
- )
- return organization_uid, organization_identifier
-
- @classmethod
- def execute_query(cls, query: str, params: tuple = ()) -> Any:
- cursor = db.execute_sql(query, params)
- result_row = cursor.fetchone()
- cursor.close()
- if not result_row or len(result_row) == 0:
- return None
- return result_row[0]
diff --git a/prompt-service/src/unstract/prompt_service_v2/utils/log.py b/prompt-service/src/unstract/prompt_service_v2/utils/log.py
deleted file mode 100644
index cb19ef53c..000000000
--- a/prompt-service/src/unstract/prompt_service_v2/utils/log.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from unstract.prompt_service_v2.constants import RunLevel
-from unstract.sdk.constants import LogLevel
-
-from unstract.core.pubsub_helper import LogPublisher
-
-
-def publish_log(
- log_events_id: str,
- component: dict[str, str],
- level: LogLevel,
- state: RunLevel,
- message: str,
-) -> None:
- """Publishes a log to the web socket.
-
- Args:
- log_events_id (str): UUID for the connection
- component (dict[str, str]): Dict of tool_id, doc_name and prompt_name to
- log context
- level (LogLevel): Log level, one of INFO, WARNING, DEBUG, ERROR
- state (RunLevel): Run level, one of EVAL, RUN, CHALLENGE, TABLE_EXTRACTION
- message (str): Message to log
- """
- LogPublisher.publish(
- log_events_id,
- LogPublisher.log_prompt(component, level.value, state.value, message),
- )
diff --git a/prompt-service/src/unstract/prompt_service_v2/utils/request.py b/prompt-service/src/unstract/prompt_service_v2/utils/request.py
deleted file mode 100644
index 3de8b4846..000000000
--- a/prompt-service/src/unstract/prompt_service_v2/utils/request.py
+++ /dev/null
@@ -1,62 +0,0 @@
-from enum import Enum
-from typing import Any, Optional
-
-import requests as pyrequests
-from flask import current_app as app
-from requests.exceptions import RequestException
-from unstract.prompt_service_v2.exceptions import (
- APIError,
- BadRequest,
- MissingFieldError,
-)
-
-
-class HTTPMethod(str, Enum):
- GET = "GET"
- POST = "POST"
- DELETE = "DELETE"
- PUT = "PUT"
- PATCH = "PATCH"
-
-
-def make_http_request(
- verb: HTTPMethod,
- url: str,
- data: Optional[dict[str, Any]] = None,
- headers: Optional[dict[str, Any]] = None,
- params: Optional[dict[str, Any]] = None,
-) -> str:
- """Generic helper function to help make a HTTP request."""
- try:
- if verb == HTTPMethod.GET:
- response = pyrequests.get(url, params=params, headers=headers)
- elif verb == HTTPMethod.POST:
- response = pyrequests.post(url, json=data, params=params, headers=headers)
- elif verb == HTTPMethod.DELETE:
- response = pyrequests.delete(url, params=params, headers=headers)
- else:
- raise ValueError("Invalid HTTP verb. Supported verbs: GET, POST, DELETE")
-
- response.raise_for_status()
- return_val: str = (
- response.json()
- if response.headers.get("content-type") == "application/json"
- else response.text
- )
- return return_val
- except RequestException as e:
- app.logger.error(f"HTTP request error: {e}")
- raise APIError(f"Error occured while invoking POST API Variable : {str(e)}")
- except Exception as e:
- app.logger.error(f"An unexpected error occurred: {e}")
- raise e
-
-
-def validate_request_payload(payload, REQUIRED_FIELDS):
- if not payload:
- raise BadRequest()
-
- # Validate required fields
- missing_fields = [field for field in REQUIRED_FIELDS if field not in payload]
- if missing_fields:
- raise MissingFieldError(missing_fields)
diff --git a/tools/structure/.gitignore b/tools/structure/.gitignore
new file mode 100644
index 000000000..be0e458d4
--- /dev/null
+++ b/tools/structure/.gitignore
@@ -0,0 +1 @@
+helper.py
diff --git a/tools/structure/src/constants.py b/tools/structure/src/constants.py
index 32b097a4f..8d71ae163 100644
--- a/tools/structure/src/constants.py
+++ b/tools/structure/src/constants.py
@@ -78,3 +78,27 @@ class SettingsKeys:
METRICS = "metrics"
INDEXING = "indexing"
EXECUTION_ID = "execution_id"
+
+
+class IndexingConstants:
+ TOOL_ID = "tool_id"
+ EMBEDDING_INSTANCE_ID = "embedding_instance_id"
+ VECTOR_DB_INSTANCE_ID = "vector_db_instance_id"
+ X2TEXT_INSTANCE_ID = "x2text_instance_id"
+ FILE_PATH = "file_path"
+ CHUNK_SIZE = "chunk_size"
+ CHUNK_OVERLAP = "chunk_overlap"
+ REINDEX = "reindex"
+ FILE_HASH = "file_hash"
+ OUTPUT_FILE_PATH = "output_file_path"
+ ENABLE_HIGHLIGHT = "enable_highlight"
+ USAGE_KWARGS = "usage_kwargs"
+ PROCESS_TEXT = "process_text"
+ EXTRACTED_TEXT = "extracted_text"
+ TAGS = "tags"
+ EXECUTION_SOURCE = "execution_source"
+ DOC_ID = "doc_id"
+ TOOL_EXECUTION_METATADA = "tool_execution_metadata"
+ EXECUTION_DATA_DIR = "execution_data_dir"
+ RUN_ID = "run_id"
+ EXECUTION_ID = "execution_id"
diff --git a/tools/structure/src/helpers.py b/tools/structure/src/helpers.py
new file mode 100644
index 000000000..e29b5f31c
--- /dev/null
+++ b/tools/structure/src/helpers.py
@@ -0,0 +1,93 @@
+from typing import Any, Optional
+
+from constants import IndexingConstants as IKeys
+from constants import SettingsKeys # type: ignore [attr-defined]
+from unstract.sdk.prompt import PromptTool
+from unstract.sdk.tool.base import BaseTool
+
+
+class StructureToolHelper:
+ @staticmethod
+ def dynamic_extraction(
+ file_path: str,
+ enable_highlight: bool,
+ usage_kwargs: dict[str, Any],
+ run_id: str,
+ tool_settings: dict[str, Any],
+ extract_file_path: str,
+ tool: BaseTool,
+ execution_run_data_folder: str,
+ ) -> str:
+
+ x2text = tool_settings[SettingsKeys.X2TEXT_ADAPTER]
+ tool.stream_log(f"Extracting text from {file_path} into {extract_file_path}")
+ payload = {
+ IKeys.X2TEXT_INSTANCE_ID: x2text,
+ IKeys.FILE_PATH: file_path,
+ IKeys.ENABLE_HIGHLIGHT: enable_highlight,
+ IKeys.USAGE_KWARGS: usage_kwargs.copy(),
+ IKeys.RUN_ID: run_id,
+ IKeys.EXECUTION_SOURCE: SettingsKeys.TOOL,
+ IKeys.OUTPUT_FILE_PATH: str(extract_file_path),
+ IKeys.TAGS: tool.tags,
+ IKeys.TOOL_EXECUTION_METATADA: tool.get_exec_metadata,
+ IKeys.EXECUTION_DATA_DIR: str(execution_run_data_folder),
+ }
+
+ tool.stream_log(f"Payload constructed : {payload}")
+ responder = PromptTool(
+ tool=tool,
+ prompt_host=tool.get_env_or_die(SettingsKeys.PROMPT_HOST),
+ prompt_port=tool.get_env_or_die(SettingsKeys.PROMPT_PORT),
+ )
+ tool.stream_log(f"responder : {responder}")
+ extracted_text = responder.extract(payload=payload)
+
+ return extracted_text
+
+ @staticmethod
+ def dynamic_indexing(
+ file_path: str,
+ tool_settings: dict[str, Any],
+ run_id: str,
+ tool: BaseTool,
+ execution_run_data_folder: str,
+ reindex: bool,
+ usage_kwargs: dict[str, Any],
+ enable_highlight: bool,
+ chunk_size: int,
+ chunk_overlap: int,
+ file_hash: Optional[str] = None,
+ tool_id: str = None,
+ extracted_text: str = None,
+ ) -> str:
+
+ x2text = tool_settings[SettingsKeys.X2TEXT_ADAPTER]
+
+ payload = {
+ IKeys.TOOL_ID: tool_id,
+ IKeys.EMBEDDING_INSTANCE_ID: tool_settings[SettingsKeys.EMBEDDING],
+ IKeys.VECTOR_DB_INSTANCE_ID: tool_settings[SettingsKeys.VECTOR_DB],
+ IKeys.X2TEXT_INSTANCE_ID: x2text,
+ IKeys.FILE_HASH: file_hash,
+ IKeys.CHUNK_SIZE: chunk_size,
+ IKeys.CHUNK_OVERLAP: chunk_overlap,
+ IKeys.REINDEX: reindex,
+ IKeys.FILE_PATH: str(file_path),
+ IKeys.ENABLE_HIGHLIGHT: enable_highlight,
+ IKeys.USAGE_KWARGS: usage_kwargs.copy(),
+ IKeys.RUN_ID: run_id,
+ IKeys.EXECUTION_SOURCE: SettingsKeys.TOOL,
+ IKeys.TAGS: tool.tags,
+ IKeys.TOOL_EXECUTION_METATADA: tool.get_exec_metadata,
+ IKeys.EXECUTION_DATA_DIR: str(execution_run_data_folder),
+ IKeys.EXTRACTED_TEXT: extracted_text,
+ }
+
+ responder = PromptTool(
+ tool=tool,
+ prompt_host=tool.get_env_or_die(SettingsKeys.PROMPT_HOST),
+ prompt_port=tool.get_env_or_die(SettingsKeys.PROMPT_PORT),
+ )
+ doc_id = responder.index(payload=payload)
+ return doc_id
diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py
index 84cbaf46e..d87851dc6 100644
--- a/tools/structure/src/main.py
+++ b/tools/structure/src/main.py
@@ -6,8 +6,8 @@ from pathlib import Path
from typing import Any
from constants import SettingsKeys # type: ignore [attr-defined]
+from helpers import StructureToolHelper as STHelper
from unstract.sdk.constants import LogLevel, LogState, MetadataKey, ToolEnv, UsageKwargs
-from unstract.sdk.index import Index
from unstract.sdk.prompt import PromptTool
from unstract.sdk.tool.base import BaseTool
from unstract.sdk.tool.entrypoint import ToolEntrypoint
@@ -98,14 +98,9 @@ class StructureTool(BaseTool):
self.get_env_or_die(ToolEnv.EXECUTION_DATA_DIR)
)
- index = Index(
- tool=self,
- run_id=self.file_execution_id,
- capture_metrics=True,
- )
index_metrics = {}
extracted_input_file = str(execution_run_data_folder / SettingsKeys.EXTRACT)
- # TODO : Resolve and pass log events ID
+ # Resolve and pass log events ID
payload = {
SettingsKeys.RUN_ID: self.file_execution_id,
SettingsKeys.EXECUTION_ID: self.execution_id,
@@ -117,32 +112,23 @@ class StructureTool(BaseTool):
SettingsKeys.FILE_PATH: extracted_input_file,
SettingsKeys.EXECUTION_SOURCE: SettingsKeys.TOOL,
}
- # TODO: Need to split extraction and indexing
- # to avoid unwanted indexing
- self.stream_log(f"Indexing document '{self.source_file_name}'")
+ self.stream_log(f"Extracting document '{self.source_file_name}'")
usage_kwargs: dict[Any, Any] = dict()
usage_kwargs[UsageKwargs.RUN_ID] = self.file_execution_id
usage_kwargs[UsageKwargs.FILE_NAME] = self.source_file_name
usage_kwargs[UsageKwargs.EXECUTION_ID] = self.execution_id
+ extracted_text = STHelper.dynamic_extraction(
+ file_path=input_file,
+ enable_highlight=enable_highlight,
+ usage_kwargs=usage_kwargs,
+ run_id=self.file_execution_id,
+ tool_settings=tool_settings,
+ extract_file_path=tool_data_dir / SettingsKeys.EXTRACT,
+ tool=self,
+ execution_run_data_folder=str(execution_run_data_folder),
+ )
if tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION]:
- index.index(
- tool_id=tool_id,
- embedding_instance_id=tool_settings[SettingsKeys.EMBEDDING],
- vector_db_instance_id=tool_settings[SettingsKeys.VECTOR_DB],
- x2text_instance_id=tool_settings[SettingsKeys.X2TEXT_ADAPTER],
- file_path=input_file,
- file_hash=file_hash,
- chunk_size=tool_settings[SettingsKeys.CHUNK_SIZE],
- chunk_overlap=tool_settings[SettingsKeys.CHUNK_OVERLAP],
- output_file_path=tool_data_dir / SettingsKeys.EXTRACT,
- reindex=True,
- usage_kwargs=usage_kwargs,
- tags=self.tags,
- enable_highlight=enable_highlight,
- **({"fs": self.workflow_filestorage}),
- )
- index_metrics = {SettingsKeys.INDEXING: index.get_metrics()}
if summarize_as_source:
summarize_file_hash = self._summarize_and_index(
tool_id=tool_id,
@@ -150,10 +136,27 @@ class StructureTool(BaseTool):
tool_data_dir=tool_data_dir,
responder=responder,
outputs=outputs,
- index=index,
usage_kwargs=usage_kwargs,
)
payload[SettingsKeys.FILE_HASH] = summarize_file_hash
+ # STHelper.dynamic_indexing(
+ # tool_settings=tool_settings,
+ # run_id=self.file_execution_id,
+ # file_path=tool_data_dir / SettingsKeys.EXTRACT,
+ # tool=self,
+ # execution_run_data_folder=execution_run_data_folder,
+ # chunk_overlap=tool_settings[SettingsKeys.CHUNK_OVERLAP],
+ # chunk_size=tool_settings[SettingsKeys.CHUNK_SIZE],
+ # reIndex=True,
+ # usage_kwargs=usage_kwargs,
+ # enable_highlight=enable_highlight,
+ # extracted_text=extracted_text,
+ # tool_id=tool_metadata[SettingsKeys.TOOL_ID],
+ # file_hash=file_hash,
+ # )
+ # Handle metrics for single pass extraction
+ # index_metrics = {SettingsKeys.INDEXING: index.get_metrics()}
+
self.stream_log("Fetching response for single pass extraction")
prompt_service_resp = responder.single_pass_extraction(
payload=payload,
@@ -164,28 +167,6 @@ class StructureTool(BaseTool):
# indexed to get the output in required path
reindex = True
for output in outputs:
- if reindex or not summarize_as_source:
- index.index(
- tool_id=tool_metadata[SettingsKeys.TOOL_ID],
- embedding_instance_id=output[SettingsKeys.EMBEDDING],
- vector_db_instance_id=output[SettingsKeys.VECTOR_DB],
- x2text_instance_id=output[SettingsKeys.X2TEXT_ADAPTER],
- file_path=input_file,
- file_hash=file_hash,
- chunk_size=output[SettingsKeys.CHUNK_SIZE],
- chunk_overlap=output[SettingsKeys.CHUNK_OVERLAP],
- output_file_path=tool_data_dir / SettingsKeys.EXTRACT,
- reindex=reindex,
- usage_kwargs=usage_kwargs,
- tags=self.tags,
- enable_highlight=enable_highlight,
- **({"fs": self.workflow_filestorage}),
- )
- index_metrics[output[SettingsKeys.NAME]] = {
- SettingsKeys.INDEXING: index.get_metrics()
- }
- index.clear_metrics()
-
if summarize_as_source:
summarize_file_hash = self._summarize_and_index(
tool_id=tool_id,
@@ -193,12 +174,34 @@ class StructureTool(BaseTool):
tool_data_dir=tool_data_dir,
responder=responder,
outputs=outputs,
- index=index,
usage_kwargs=usage_kwargs,
)
payload[SettingsKeys.OUTPUTS] = outputs
payload[SettingsKeys.FILE_HASH] = summarize_file_hash
break
+ if reindex or not summarize_as_source:
+ self.stream_log("Sucessfully extracted text, indexing..")
+ STHelper.dynamic_indexing(
+ tool_settings=tool_settings,
+ run_id=self.file_execution_id,
+ file_path=tool_data_dir / SettingsKeys.EXTRACT,
+ tool=self,
+ execution_run_data_folder=str(execution_run_data_folder),
+ chunk_overlap=output[SettingsKeys.CHUNK_OVERLAP],
+ reindex=reindex,
+ usage_kwargs=usage_kwargs,
+ enable_highlight=enable_highlight,
+ chunk_size=output[SettingsKeys.CHUNK_SIZE],
+ tool_id=tool_metadata[SettingsKeys.TOOL_ID],
+ file_hash=file_hash,
+ extracted_text=extracted_text,
+ )
+ # Handle metrics for single pass extraction
+ # index_metrics[output[SettingsKeys.NAME]] = {
+ # SettingsKeys.INDEXING: index.get_metrics()
+ # }
+ # index.clear_metrics()
+
reindex = False
except Exception as e:
self.stream_log(
@@ -206,7 +209,6 @@ class StructureTool(BaseTool):
)
raise
- # TODO : Make this snippet pluggable and introduce pluggablity for tools.
for output in outputs:
if SettingsKeys.TABLE_SETTINGS in output:
table_settings = output[SettingsKeys.TABLE_SETTINGS]
@@ -218,7 +220,6 @@ class StructureTool(BaseTool):
payload=payload,
)
- # TODO: Make use of dataclasses
if prompt_service_resp[SettingsKeys.STATUS] != SettingsKeys.OK:
self.stream_error_and_exit(
f"Failed to fetch responses for "
@@ -284,8 +285,9 @@ class StructureTool(BaseTool):
tool_data_dir: Path,
responder: PromptTool,
outputs: dict[str, Any],
- index: Index,
usage_kwargs: dict[Any, Any] = {},
+ enable_highlight: bool = False,
+ execution_run_data_folder: str = None,
) -> str:
"""Summarizes the context of the file and indexes the summarized
content.
@@ -311,6 +313,9 @@ class StructureTool(BaseTool):
summarize_file_path = tool_data_dir / SettingsKeys.SUMMARIZE
summarized_context = ""
+ self.stream_log(
+ f"Checking if summarized context exists at {summarize_file_path}"
+ )
if self.workflow_filestorage.exists(summarize_file_path):
summarized_context = self.workflow_filestorage.read(
path=summarize_file_path, mode="r"
@@ -350,18 +355,20 @@ class StructureTool(BaseTool):
summarize_file_hash: str = self.workflow_filestorage.get_hash_from_file(
path=summarize_file_path
)
- index.index(
- tool_id=tool_id,
- embedding_instance_id=embedding_instance_id,
- vector_db_instance_id=vector_db_instance_id,
- x2text_instance_id=x2text_instance_id,
+ STHelper.dynamic_indexing(
file_path=summarize_file_path,
+ tool_settings=tool_settings,
+ run_id=self.file_execution_id,
+ execution_run_data_folder=execution_run_data_folder,
+ reindex=True,
+ usage_kwargs=usage_kwargs,
+ enable_highlight=enable_highlight,
file_hash=summarize_file_hash,
- chunk_size=0,
+ tool_id=tool_id,
+ tool=self,
chunk_overlap=0,
- usage_kwargs=usage_kwargs,
- tags=self.tags,
- **({"fs": self.workflow_filestorage}),
+ chunk_size=0,
+ extracted_text=summarized_context,
)
return summarize_file_hash