LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Feat/sperate highlighting (#1137)
vishnuszipstackeaa1e26bfb2c
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.pyindex 656a33ae7..65556020a 100644--- a/backend/api_v2/api_deployment_views.py+++ b/backend/api_v2/api_deployment_views.py@@ -87,11 +87,11 @@ class DeploymentExecution(views.APIView):response: ExecutionResponse = DeploymentHelper.get_execution_status(execution_id)-# Determine response statusresponse_status = status.HTTP_422_UNPROCESSABLE_ENTITYif response.execution_status == CeleryTaskState.COMPLETED.value:response_status = status.HTTP_200_OK+ response.remove_result_metadata_keys(["highlight_data"])if not include_metadata:response.remove_result_metadata_keys()if not include_metrics:diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.pyindex b04f4744b..fb2b9f42e 100644--- a/backend/api_v2/deployment_helper.py+++ b/backend/api_v2/deployment_helper.py@@ -185,6 +185,7 @@ class DeploymentHelper(BaseAPIKeyValidator):result.status_api = DeploymentHelper.construct_status_endpoint(api_endpoint=api.api_endpoint, execution_id=execution_id)+ result.remove_result_metadata_keys(["highlight_data"])if not include_metadata:result.remove_result_metadata_keys()if not include_metrics:diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.pyindex 06a2fadbf..3495e9d8b 100644--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py@@ -385,6 +385,7 @@ class PromptStudioHelper:user_id=user_id,process_text=process_text,fs=fs_instance,+ enable_highlight=tool.enable_highlight,)elapsed_time = time.time() - start_time@@ -761,6 +762,7 @@ class PromptStudioHelper:user_id=user_id,process_text=process_text,fs=fs_instance,+ enable_highlight=tool.enable_highlight,)if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:return {@@ -915,6 +917,7 @@ class PromptStudioHelper:run_id: str = None,process_text: Optional[Callable[[str], str]] = None,fs: FileStorage = FileStorage(provider=FileStorageProvider.LOCAL),+ enable_highlight: bool = False,) -> Any:"""Used to index a file based on the passed arguments.@@ -998,6 +1001,7 @@ class PromptStudioHelper:usage_kwargs=usage_kwargs.copy(),process_text=process_text,fs=fs,+ enable_highlight=enable_highlight,)PromptStudioIndexHelper.handle_index_manager(@@ -1075,6 +1079,7 @@ class PromptStudioHelper:user_id=user_id,process_text=process_text,fs=fs_instance,+ enable_highlight=tool.enable_highlight,)if index_result.get("status") == IndexingStatus.PENDING_STATUS.value:return {diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.pyindex 6aed9e689..1eadb430d 100644--- a/prompt-service/src/unstract/prompt_service/constants.py+++ b/prompt-service/src/unstract/prompt_service/constants.py@@ -77,6 +77,8 @@ class PromptServiceContants:EXECUTION_SOURCE = "execution_source"METRICS = "metrics"LINE_ITEM = "line-item"+ LINE_NUMBERS = "line_numbers"+ WHISPER_HASH = "whisper_hash"class RunLevel(Enum):diff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.pyindex cb70f9f50..00559efb9 100644--- a/prompt-service/src/unstract/prompt_service/helper.py+++ b/prompt-service/src/unstract/prompt_service/helper.py@@ -327,9 +327,12 @@ def run_completion(answer: str = completion[PSKeys.RESPONSE].texthighlight_data = completion.get(PSKeys.HIGHLIGHT_DATA, [])confidence_data = completion.get(PSKeys.CONFIDENCE_DATA)+ line_numbers = completion.get(PSKeys.LINE_NUMBERS, [])+ whisper_hash = completion.get(PSKeys.WHISPER_HASH, "")if metadata is not None and prompt_key:metadata.setdefault(PSKeys.HIGHLIGHT_DATA, {})[prompt_key] = highlight_data-+ metadata.setdefault(PSKeys.LINE_NUMBERS, {})[prompt_key] = line_numbers+ metadata[PSKeys.WHISPER_HASH] = whisper_hashif confidence_data:metadata.setdefault(PSKeys.CONFIDENCE_DATA, {})[prompt_keydiff --git a/tools/structure/src/main.py b/tools/structure/src/main.pyindex d1351ed93..d748ca7c0 100644--- a/tools/structure/src/main.py+++ b/tools/structure/src/main.py@@ -146,6 +146,7 @@ class StructureTool(BaseTool):usage_kwargs=usage_kwargs,process_text=process_text,tags=self.tags,+ enable_highlight=enable_highlight,**({"fs": self.workflow_filestorage}),)index_metrics = {SettingsKeys.INDEXING: index.get_metrics()}@@ -185,6 +186,7 @@ class StructureTool(BaseTool):usage_kwargs=usage_kwargs,process_text=process_text,tags=self.tags,+ enable_highlight=enable_highlight,**({"fs": self.workflow_filestorage}),)index_metrics[output[SettingsKeys.NAME]] = {