Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[FIX] Prompt service v2 fixes (#1247)

harini-venkataraman0bfc37f5b984
diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
index fe045df49..f36a08500 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
@@ -859,25 +859,6 @@ class PromptStudioHelper:
doc_path = str(
doc_path.parent.parent / "summarize" / (doc_path.stem + ".txt")
)
- summarize_file_path = PromptStudioHelper.summarize(
- filename, org_id, document_id, is_summary, run_id, tool, doc_id
- )
- summarize_doc_id = IndexingUtils.generate_index_key(
- vector_db=str(default_profile.vector_store.id),
- embedding=str(default_profile.embedding_model.id),
- x2text=str(default_profile.x2text.id),
- chunk_size=str(default_profile.chunk_size),
- chunk_overlap=str(default_profile.chunk_overlap),
- file_path=summarize_file_path,
- fs=fs_instance,
- tool=util,
- )
- PromptStudioIndexHelper.handle_index_manager(
- document_id=document_id,
- is_summary=is_summary,
- profile_manager=profile_manager,
- doc_id=summarize_doc_id,
- )
logger.info("Summary enabled, set chunk to zero..")
logger.info(f"Indexing document {doc_path} for {doc_id}")
index_result = PromptStudioHelper.dynamic_indexer(
diff --git a/backend/sample.env b/backend/sample.env
index e99074eab..102fb92d8 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -75,9 +75,9 @@ PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data
# Structure Tool Image (Runs prompt studio exported tools)
# https://hub.docker.com/r/unstract/tool-structure
-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.73"
+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.74"
STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"
-STRUCTURE_TOOL_IMAGE_TAG="0.0.73"
+STRUCTURE_TOOL_IMAGE_TAG="0.0.74"
# Feature Flags
EVALUATION_SERVER_IP=unstract-flipt
diff --git a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py
index 1184f9e45..f620e24e6 100644
--- a/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py
+++ b/prompt-service/src/unstract/prompt_service/controllers/answer_prompt.py
@@ -64,6 +64,7 @@ def prompt_processor() -> Any:
variable_names: list[str] = []
# Identifier for source of invocation
execution_source = payload.get(PSKeys.EXECUTION_SOURCE, "")
+ context_retrieval_metrics = {}
publish_log(
log_events_id,
{"tool_id": tool_id, "run_id": run_id, "doc_name": doc_name},
@@ -272,6 +273,7 @@ def prompt_processor() -> Any:
chunk_size=chunk_size,
execution_source=execution_source,
file_path=file_path,
+ context_retrieval_metrics=context_retrieval_metrics,
)
metadata[PSKeys.CONTEXT][output[PSKeys.NAME]] = context
else:
@@ -568,7 +570,7 @@ def prompt_processor() -> Any:
)
metrics.setdefault(prompt_name, {}).update(
{
- "context_retrieval": index.get_metrics(),
+ "context_retrieval": context_retrieval_metrics.get(prompt_name, {}),
f"{llm.get_usage_reason()}_llm": llm.get_metrics(),
**challenge_metrics,
}
diff --git a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py
index a4b703e3e..aee5cf1f6 100644
--- a/prompt-service/src/unstract/prompt_service/services/answer_prompt.py
+++ b/prompt-service/src/unstract/prompt_service/services/answer_prompt.py
@@ -1,4 +1,3 @@
-import os
from logging import Logger
from typing import Any, Optional
@@ -228,12 +227,6 @@ class AnswerPromptService:
raise APIError(PSKeys.PAID_FEATURE_MSG)
extract_file_path = file_path
- if execution_source == ExecutionSource.IDE.value:
- # Adjust file path to read from the extract folder
- base_name = os.path.splitext(os.path.basename(file_path))[0]
- extract_file_path = os.path.join(
- os.path.dirname(file_path), "extract", f"{base_name}.txt"
- )
# Read file content into context
fs_instance: FileStorage = FileStorage(FileStorageProvider.LOCAL)
diff --git a/prompt-service/src/unstract/prompt_service/services/extraction.py b/prompt-service/src/unstract/prompt_service/services/extraction.py
index 0793d5ca6..6d188425f 100644
--- a/prompt-service/src/unstract/prompt_service/services/extraction.py
+++ b/prompt-service/src/unstract/prompt_service/services/extraction.py
@@ -1,3 +1,4 @@
+from pathlib import Path
from typing import Any, Optional
from unstract.prompt_service.constants import ExecutionSource
@@ -85,7 +86,7 @@ class ExtractionService:
metadata = {X2TextConstants.WHISPER_HASH: whisper_hash_value}
for key, value in metadata.items():
tool_exec_metadata[key] = value
- metadata_path = execution_run_data_folder / IKeys.METADATA_FILE
+ metadata_path = str(Path(execution_run_data_folder / IKeys.METADATA_FILE))
ToolUtils.dump_json(
file_to_dump=metadata_path,
json_to_dump=metadata,
diff --git a/prompt-service/src/unstract/prompt_service/services/retrieval.py b/prompt-service/src/unstract/prompt_service/services/retrieval.py
index 1e6cff78b..3dc6ea364 100644
--- a/prompt-service/src/unstract/prompt_service/services/retrieval.py
+++ b/prompt-service/src/unstract/prompt_service/services/retrieval.py
@@ -1,3 +1,4 @@
+import datetime
from typing import Any
from unstract.prompt_service.constants import PromptServiceConstants as PSKeys
@@ -5,6 +6,7 @@ from unstract.prompt_service.core.retrievers.simple import SimpleRetriever
from unstract.prompt_service.core.retrievers.subquestion import SubquestionRetriever
from unstract.prompt_service.services.answer_prompt import AnswerPromptService
from unstract.prompt_service.utils.file_utils import FileUtils
+from unstract.prompt_service.utils.metrics import Metrics
from unstract.sdk.llm import LLM
from unstract.sdk.vector_db import VectorDB
@@ -22,11 +24,15 @@ class RetrievalService:
chunk_size: int,
execution_source: str,
file_path: str,
+ context_retrieval_metrics: dict[str, Any],
) -> tuple[str, list[str]]:
context: list[str]
if chunk_size == 0:
context = RetrievalService.retrieve_complete_context(
- execution_source=execution_source, file_path=file_path
+ execution_source=execution_source,
+ file_path=file_path,
+ context_retrieval_metrics=context_retrieval_metrics,
+ prompt_key=output[PSKeys.PROMPTX],
)
else:
context = RetrievalService.run_retrieval(
@@ -35,6 +41,7 @@ class RetrievalService:
llm=llm,
vector_db=vector_db,
retrieval_type=retrieval_type,
+ context_retrieval_metrics=context_retrieval_metrics,
)
answer = AnswerPromptService.construct_and_run_prompt( # type:ignore
tool_settings=tool_settings,
@@ -55,10 +62,13 @@ class RetrievalService:
llm: LLM,
vector_db: VectorDB,
retrieval_type: str,
+ context_retrieval_metrics: dict[str, Any],
) -> list[str]:
context: set[str]
prompt = output[PSKeys.PROMPTX]
top_k = output[PSKeys.SIMILARITY_TOP_K]
+ prompt_key = output[PSKeys.NAME]
+ retrieval_start_time = datetime.datetime.now()
if retrieval_type == PSKeys.SUBQUESTION:
context = SubquestionRetriever(
vector_db=vector_db,
@@ -75,11 +85,18 @@ class RetrievalService:
top_k=top_k,
llm=llm,
).retrieve()
-
+ context_retrieval_metrics[prompt_key] = {
+ "time_taken(s)": Metrics.elapsed_time(start_time=retrieval_start_time)
+ }
return list(context)
@staticmethod
- def retrieve_complete_context(execution_source: str, file_path: str) -> list[str]:
+ def retrieve_complete_context(
+ execution_source: str,
+ file_path: str,
+ context_retrieval_metrics: dict[str, Any],
+ prompt_key: str,
+ ) -> list[str]:
"""
Loads full context from raw file for zero chunk size retrieval
Args:
@@ -90,5 +107,9 @@ class RetrievalService:
list[str]: context from extracted file.
"""
fs_instance = FileUtils.get_fs_instance(execution_source=execution_source)
+ retrieval_start_time = datetime.datetime.now()
context = fs_instance.read(path=file_path, mode="r")
+ context_retrieval_metrics[prompt_key] = {
+ "time_taken(s)": Metrics.elapsed_time(start_time=retrieval_start_time)
+ }
return [context]
diff --git a/prompt-service/src/unstract/prompt_service/utils/metrics.py b/prompt-service/src/unstract/prompt_service/utils/metrics.py
new file mode 100644
index 000000000..02a5783f4
--- /dev/null
+++ b/prompt-service/src/unstract/prompt_service/utils/metrics.py
@@ -0,0 +1,9 @@
+import datetime
+
+
+class Metrics:
+
+ @staticmethod
+ def elapsed_time(start_time) -> float:
+ """Returns the elapsed time since the process was started."""
+ return (datetime.datetime.now() - start_time).total_seconds()
diff --git a/tools/structure/src/helpers.py b/tools/structure/src/helpers.py
index e29b5f31c..aac6a4293 100644
--- a/tools/structure/src/helpers.py
+++ b/tools/structure/src/helpers.py
@@ -1,3 +1,4 @@
+import datetime
from typing import Any, Optional
from constants import IndexingConstants as IKeys
@@ -91,3 +92,8 @@ class StructureToolHelper:
)
doc_id = responder.index(payload=payload)
return doc_id
+
+ @staticmethod
+ def elapsed_time(start_time) -> float:
+ """Returns the elapsed time since the process was started."""
+ return (datetime.datetime.now() - start_time).total_seconds()
diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py
index 34e52b9f1..0f0d79492 100644
--- a/tools/structure/src/main.py
+++ b/tools/structure/src/main.py
@@ -1,3 +1,4 @@
+import datetime
import json
import logging
import os
@@ -131,7 +132,6 @@ class StructureTool(BaseTool):
if tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION]:
if summarize_as_source:
summarize_file_hash = self._summarize_and_index(
- tool_id=tool_id,
tool_settings=tool_settings,
tool_data_dir=tool_data_dir,
responder=responder,
@@ -139,25 +139,9 @@ class StructureTool(BaseTool):
usage_kwargs=usage_kwargs,
)
payload[SettingsKeys.FILE_HASH] = summarize_file_hash
- # STHelper.dynamic_indexing(
- # tool_settings=tool_settings,
- # run_id=self.file_execution_id,
- # file_path=tool_data_dir / SettingsKeys.EXTRACT,
- # tool=self,
- # execution_run_data_folder=execution_run_data_folder,
- # chunk_overlap=tool_settings[SettingsKeys.CHUNK_OVERLAP],
- # chunk_size=tool_settings[SettingsKeys.CHUNK_SIZE],
- # reIndex=True,
- # usage_kwargs=usage_kwargs,
- # enable_highlight=enable_highlight,
- # extracted_text=extracted_text,
- # tool_id=tool_metadata[SettingsKeys.TOOL_ID],
- # file_hash=file_hash,
- # )
- # Handle metrics for single pass extraction
- # index_metrics = {SettingsKeys.INDEXING: index.get_metrics()}
-
self.stream_log("Fetching response for single pass extraction")
+ # Since indexing is not involved for single pass
+ index_metrics = {"time_taken(s)": 0}
prompt_service_resp = responder.single_pass_extraction(
payload=payload,
)
@@ -169,7 +153,6 @@ class StructureTool(BaseTool):
for output in outputs:
if summarize_as_source:
summarize_file_hash = self._summarize_and_index(
- tool_id=tool_id,
tool_settings=tool_settings,
tool_data_dir=tool_data_dir,
responder=responder,
@@ -178,6 +161,8 @@ class StructureTool(BaseTool):
)
payload[SettingsKeys.OUTPUTS] = outputs
payload[SettingsKeys.FILE_HASH] = summarize_file_hash
+ # Since indexing is not involved for summary
+ index_metrics[output[SettingsKeys.NAME]] = {"time_taken(s)": 0}
break
self.stream_log(
f"Chunk size '{output[SettingsKeys.CHUNK_SIZE]}', "
@@ -186,6 +171,7 @@ class StructureTool(BaseTool):
if (reindex or not summarize_as_source) and output[
SettingsKeys.CHUNK_SIZE
] != 0:
+ indexing_start_time = datetime.datetime.now()
self.stream_log(
f"Sucessfully extracted text, "
f"indexing {tool_data_dir / SettingsKeys.EXTRACT}.."
@@ -205,11 +191,13 @@ class StructureTool(BaseTool):
file_hash=file_hash,
extracted_text=extracted_text,
)
- # Handle metrics for single pass extraction
- # index_metrics[output[SettingsKeys.NAME]] = {
- # SettingsKeys.INDEXING: index.get_metrics()
- # }
- # index.clear_metrics()
+ index_metrics[output[SettingsKeys.NAME]] = {
+ SettingsKeys.INDEXING: {
+ "time_taken(s)": STHelper.elapsed_time(
+ start_time=indexing_start_time
+ )
+ }
+ }
reindex = False
except Exception as e:
@@ -289,14 +277,11 @@ class StructureTool(BaseTool):
def _summarize_and_index(
self,
- tool_id: str,
tool_settings: dict[str, Any],
tool_data_dir: Path,
responder: PromptTool,
outputs: dict[str, Any],
usage_kwargs: dict[Any, Any] = {},
- enable_highlight: bool = False,
- execution_run_data_folder: str = None,
) -> str:
"""Summarizes the context of the file and indexes the summarized
content.
@@ -364,21 +349,6 @@ class StructureTool(BaseTool):
summarize_file_hash: str = self.workflow_filestorage.get_hash_from_file(
path=summarize_file_path
)
- STHelper.dynamic_indexing(
- file_path=summarize_file_path,
- tool_settings=tool_settings,
- run_id=self.file_execution_id,
- execution_run_data_folder=execution_run_data_folder,
- reindex=True,
- usage_kwargs=usage_kwargs,
- enable_highlight=enable_highlight,
- file_hash=summarize_file_hash,
- tool_id=tool_id,
- tool=self,
- chunk_overlap=0,
- chunk_size=0,
- extracted_text=summarized_context,
- )
return summarize_file_hash