Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Feat/sperate highlighting (#1137)

vishnuszipstackeaa1e26bfb2c
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py
index 656a33ae7..65556020a 100644
--- a/backend/api_v2/api_deployment_views.py
+++ b/backend/api_v2/api_deployment_views.py
@@ -87,11 +87,11 @@ class DeploymentExecution(views.APIView):
response: ExecutionResponse = DeploymentHelper.get_execution_status(
execution_id
)
-
# Determine response status
response_status = status.HTTP_422_UNPROCESSABLE_ENTITY
if response.execution_status == CeleryTaskState.COMPLETED.value:
response_status = status.HTTP_200_OK
+ response.remove_result_metadata_keys(["highlight_data"])
if not include_metadata:
response.remove_result_metadata_keys()
if not include_metrics:
diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.py
index b04f4744b..fb2b9f42e 100644
--- a/backend/api_v2/deployment_helper.py
+++ b/backend/api_v2/deployment_helper.py
@@ -185,6 +185,7 @@ class DeploymentHelper(BaseAPIKeyValidator):
result.status_api = DeploymentHelper.construct_status_endpoint(
api_endpoint=api.api_endpoint, execution_id=execution_id
)
+ result.remove_result_metadata_keys(["highlight_data"])
if not include_metadata:
result.remove_result_metadata_keys()
if not include_metrics:
diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
index 06a2fadbf..3495e9d8b 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
@@ -385,6 +385,7 @@ class PromptStudioHelper:
user_id=user_id,
process_text=process_text,
fs=fs_instance,
+ enable_highlight=tool.enable_highlight,
)
elapsed_time = time.time() - start_time
@@ -761,6 +762,7 @@ class PromptStudioHelper:
user_id=user_id,
process_text=process_text,
fs=fs_instance,
+ enable_highlight=tool.enable_highlight,
)
if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:
return {
@@ -915,6 +917,7 @@ class PromptStudioHelper:
run_id: str = None,
process_text: Optional[Callable[[str], str]] = None,
fs: FileStorage = FileStorage(provider=FileStorageProvider.LOCAL),
+ enable_highlight: bool = False,
) -> Any:
"""Used to index a file based on the passed arguments.
@@ -998,6 +1001,7 @@ class PromptStudioHelper:
usage_kwargs=usage_kwargs.copy(),
process_text=process_text,
fs=fs,
+ enable_highlight=enable_highlight,
)
PromptStudioIndexHelper.handle_index_manager(
@@ -1075,6 +1079,7 @@ class PromptStudioHelper:
user_id=user_id,
process_text=process_text,
fs=fs_instance,
+ enable_highlight=tool.enable_highlight,
)
if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:
return {
diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.py
index 6aed9e689..1eadb430d 100644
--- a/prompt-service/src/unstract/prompt_service/constants.py
+++ b/prompt-service/src/unstract/prompt_service/constants.py
@@ -77,6 +77,8 @@ class PromptServiceContants:
EXECUTION_SOURCE = "execution_source"
METRICS = "metrics"
LINE_ITEM = "line-item"
+ LINE_NUMBERS = "line_numbers"
+ WHISPER_HASH = "whisper_hash"
class RunLevel(Enum):
diff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.py
index cb70f9f50..00559efb9 100644
--- a/prompt-service/src/unstract/prompt_service/helper.py
+++ b/prompt-service/src/unstract/prompt_service/helper.py
@@ -327,9 +327,12 @@ def run_completion(
answer: str = completion[PSKeys.RESPONSE].text
highlight_data = completion.get(PSKeys.HIGHLIGHT_DATA, [])
confidence_data = completion.get(PSKeys.CONFIDENCE_DATA)
+ line_numbers = completion.get(PSKeys.LINE_NUMBERS, [])
+ whisper_hash = completion.get(PSKeys.WHISPER_HASH, "")
if metadata is not None and prompt_key:
metadata.setdefault(PSKeys.HIGHLIGHT_DATA, {})[prompt_key] = highlight_data
-
+ metadata.setdefault(PSKeys.LINE_NUMBERS, {})[prompt_key] = line_numbers
+ metadata[PSKeys.WHISPER_HASH] = whisper_hash
if confidence_data:
metadata.setdefault(PSKeys.CONFIDENCE_DATA, {})[
prompt_key
diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py
index d1351ed93..d748ca7c0 100644
--- a/tools/structure/src/main.py
+++ b/tools/structure/src/main.py
@@ -146,6 +146,7 @@ class StructureTool(BaseTool):
usage_kwargs=usage_kwargs,
process_text=process_text,
tags=self.tags,
+ enable_highlight=enable_highlight,
**({"fs": self.workflow_filestorage}),
)
index_metrics = {SettingsKeys.INDEXING: index.get_metrics()}
@@ -185,6 +186,7 @@ class StructureTool(BaseTool):
usage_kwargs=usage_kwargs,
process_text=process_text,
tags=self.tags,
+ enable_highlight=enable_highlight,
**({"fs": self.workflow_filestorage}),
)
index_metrics[output[SettingsKeys.NAME]] = {