LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Fix/adding whisper hash in metadata (#418)
Rahul Johnyd200c8ed9597
diff --git a/backend/prompt_studio/prompt_studio_core/migrations/0013_customtool_enable_highlight.py b/backend/prompt_studio/prompt_studio_core/migrations/0013_customtool_enable_highlight.pynew file mode 100644index 000000000..eb7ee01f7--- /dev/null+++ b/backend/prompt_studio/prompt_studio_core/migrations/0013_customtool_enable_highlight.py@@ -0,0 +1,21 @@+# Generated by Django 4.2.1 on 2024-06-13 06:45++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("prompt_studio_core", "0012_customtool_shared_users"),+ ]++ operations = [+ migrations.AddField(+ model_name="customtool",+ name="enable_highlight",+ field=models.BooleanField(+ db_comment="Flag to enable or disable document highlighting",+ default=False,+ ),+ ),+ ]diff --git a/backend/prompt_studio/prompt_studio_core/models.py b/backend/prompt_studio/prompt_studio_core/models.pyindex ca49db6d8..5075caceb 100644--- a/backend/prompt_studio/prompt_studio_core/models.py+++ b/backend/prompt_studio/prompt_studio_core/models.py@@ -120,6 +120,10 @@ class CustomTool(BaseModel):db_comment="Flag to enable or disable challenge", default=False)+ enable_highlight = models.BooleanField(+ db_comment="Flag to enable or disable document highlighting", default=False+ )+# Introduced field to establish M2M relation between users and custom_tool.# This will introduce intermediary table which relates both the models.shared_users = models.ManyToManyField(User, related_name="shared_custom_tool")diff --git a/backend/prompt_studio/prompt_studio_registry/constants.py b/backend/prompt_studio/prompt_studio_registry/constants.pyindex 0b7ab8835..6dbde1c09 100644--- a/backend/prompt_studio/prompt_studio_registry/constants.py+++ b/backend/prompt_studio/prompt_studio_registry/constants.py@@ -94,6 +94,7 @@ class JsonSchemaKey:IMAGE_TAG = "image_tag"SUMMARIZE_PROMPT = "summarize_prompt"SUMMARIZE_AS_SOURCE = "summarize_as_source"+ ENABLE_HIGHLIGHT = "enable_highlight"class SpecKey:diff --git a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.pyindex e7bd232cd..67f84b450 100644--- a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py+++ b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py@@ -280,6 +280,7 @@ class PromptStudioRegistryHelper:tool_settings[JsonSchemaKey.ENABLE_SINGLE_PASS_EXTRACTION] = (tool.single_pass_extraction_mode)+ tool_settings[JsonSchemaKey.ENABLE_HIGHLIGHT] = tool.enable_highlightfor prompt in prompts:diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.pyindex df1db367d..464652d7f 100644--- a/backend/workflow_manager/endpoint/destination.py+++ b/backend/workflow_manager/endpoint/destination.py@@ -102,6 +102,7 @@ class DestinationConnector(BaseConnector):"""Handle the output based on the connection type."""connection_type = self.endpoint.connection_typeresult: Optional[str] = None+ meta_data: Optional[str] = Noneif error:if connection_type == WorkflowEndpoint.ConnectionType.API:self._handle_api_result(file_name=file_name, error=error, result=result)@@ -112,13 +113,18 @@ class DestinationConnector(BaseConnector):self.insert_into_db(file_history)elif connection_type == WorkflowEndpoint.ConnectionType.API:result = self.get_result(file_history)- self._handle_api_result(file_name=file_name, error=error, result=result)+ meta_data = self.get_metadata(file_history)++ self._handle_api_result(+ file_name=file_name, error=error, result=result, meta_data=meta_data+ )if not file_history:FileHistoryHelper.create_file_history(cache_key=file_hash,workflow=workflow,status=ExecutionStatus.COMPLETED,result=result,+ metadata=meta_data,file_name=file_name,)@@ -235,6 +241,7 @@ class DestinationConnector(BaseConnector):file_name: str,error: Optional[str] = None,result: Optional[str] = None,+ meta_data: Optional[dict[str, Any]] = None,) -> None:"""Handle the API result.@@ -249,7 +256,7 @@ class DestinationConnector(BaseConnector):Returns:None"""- api_result = {"file": file_name}+ api_result: dict[str, Any] = {"file": file_name}if error:api_result.update({"status": ApiDeploymentResultStatus.FAILED, "error": error}@@ -260,6 +267,7 @@ class DestinationConnector(BaseConnector):{"status": ApiDeploymentResultStatus.SUCCESS,"result": result,+ "metadata": meta_data,})else:@@ -311,7 +319,7 @@ class DestinationConnector(BaseConnector):if file_history and file_history.result:return self.parse_string(file_history.result)output_file = os.path.join(self.execution_dir, WorkflowFileType.INFILE)- metadata = self.get_workflow_metadata()+ metadata: dict[str, Any] = self.get_workflow_metadata()output_type = self.get_output_type(metadata)result: Optional[Any] = Nonetry:@@ -337,6 +345,20 @@ class DestinationConnector(BaseConnector):logger.error(f"Error while getting result {err}")return result+ def get_metadata(+ self, file_history: Optional[FileHistory]+ ) -> Optional[dict[str, Any]]:+ """Get meta_data from the output file.++ Returns:+ Union[dict[str, Any], str]: Meta data.+ """+ if file_history and file_history.meta_data:+ return self.parse_string(file_history.meta_data)+ metadata: dict[str, Any] = self.get_workflow_metadata()++ return metadata+def delete_execution_directory(self) -> None:"""Delete the execution directory.diff --git a/backend/workflow_manager/workflow/file_history_helper.py b/backend/workflow_manager/workflow/file_history_helper.pyindex 193c96b33..8317f106e 100644--- a/backend/workflow_manager/workflow/file_history_helper.py+++ b/backend/workflow_manager/workflow/file_history_helper.py@@ -40,6 +40,7 @@ class FileHistoryHelper:workflow: Workflow,status: ExecutionStatus,result: Any,+ metadata: Any,error: Optional[str] = None,file_name: Optional[str] = None,) -> FileHistory:@@ -60,6 +61,7 @@ class FileHistoryHelper:cache_key=cache_key,status=status.value,result=str(result),+ meta_data=str(metadata),error=str(error) if error else "",)except IntegrityError:diff --git a/backend/workflow_manager/workflow/migrations/0006_filehistory_meta_data.py b/backend/workflow_manager/workflow/migrations/0006_filehistory_meta_data.pynew file mode 100644index 000000000..46b43dc79--- /dev/null+++ b/backend/workflow_manager/workflow/migrations/0006_filehistory_meta_data.py@@ -0,0 +1,18 @@+# Generated by Django 4.2.1 on 2024-06-21 18:00++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("workflow", "0005_executionlog"),+ ]++ operations = [+ migrations.AddField(+ model_name="filehistory",+ name="meta_data",+ field=models.TextField(blank=True, db_comment="MetaData from execution"),+ ),+ ]diff --git a/backend/workflow_manager/workflow/models/file_history.py b/backend/workflow_manager/workflow/models/file_history.pyindex 03c8bbeb3..a31064803 100644--- a/backend/workflow_manager/workflow/models/file_history.py+++ b/backend/workflow_manager/workflow/models/file_history.py@@ -39,6 +39,7 @@ class FileHistory(BaseModel):db_comment="Error message",)result = models.TextField(blank=True, db_comment="Result from execution")+ meta_data = models.TextField(blank=True, db_comment="MetaData from execution")class Meta:constraints = [diff --git a/frontend/src/components/custom-tools/highlight-metadata/HighLightMetaData.css b/frontend/src/components/custom-tools/highlight-metadata/HighLightMetaData.cssnew file mode 100644index 000000000..8fcec2dba--- /dev/null+++ b/frontend/src/components/custom-tools/highlight-metadata/HighLightMetaData.css@@ -0,0 +1,3 @@+.highlight{+ margin-top: 16px;+}diff --git a/frontend/src/components/custom-tools/highlight-metadata/HighLightMetaData.jsx b/frontend/src/components/custom-tools/highlight-metadata/HighLightMetaData.jsxnew file mode 100644index 000000000..60295d5e7--- /dev/null+++ b/frontend/src/components/custom-tools/highlight-metadata/HighLightMetaData.jsx@@ -0,0 +1,106 @@+import { Checkbox, Form, Space, Typography, Alert } from "antd";+import PropTypes from "prop-types";+import { useEffect, useState } from "react";++import SpaceWrapper from "../../widgets/space-wrapper/SpaceWrapper";+import { CustomButton } from "../../../components/widgets/custom-button/CustomButton";+import { useCustomToolStore } from "../../../store/custom-tool-store";+import { useAlertStore } from "../../../store/alert-store";+import { useExceptionHandler } from "../../../hooks/useExceptionHandler";+import "./HighLightMetaData.css";++function HighLightMetaData({ handleUpdateTool }) {+ const [enableHighlighting, setEnableHighlighting] = useState(false);+ const [isLoading, setIsLoading] = useState(false);++ const { details, updateCustomTool } = useCustomToolStore();++ const { setAlertDetails } = useAlertStore();+ const handleException = useExceptionHandler();++ useEffect(() => {+ setEnableHighlighting(details?.enable_challenge || false);+ }, []);++ const handleSubmit = () => {+ const body = {+ enable_highlight: enableHighlighting,+ };+ setIsLoading(true);+ handleUpdateTool(body)+ .then((res) => {+ const data = res?.data;+ const updatedDetails = { ...details };++ updatedDetails["enable_highlight"] = data?.enable_challenge;+ updateCustomTool({ details: updatedDetails });++ setAlertDetails({+ type: "success",+ content: "Saved highlight settings successfully",+ });+ })+ .catch((err) => {+ setAlertDetails(handleException(err, "Failed to update"));+ })+ .finally(() => {+ setIsLoading(false);+ });+ };++ return (+ <SpaceWrapper>+ <div className="settings-body-pad-top">+ <Typography.Text strong className="pre-post-amble-title">+ Enable highlighting ( Will only work with LLM whisperer as text+ extractor)+ </Typography.Text>+ </div>+ <Form layout="vertical" onFinish={handleSubmit}>+ <Form.Item>+ <Space>+ <Checkbox+ checked={enableHighlighting}+ onChange={(e) => setEnableHighlighting(e.target.checked)}+ />+ <Typography.Text>Enable Highlighting</Typography.Text>+ </Space>+ <Space>+ <Alert+ className="highlight"+ message={+ <>+ Turn this option on if downstream use cases need you to+ highlight extracted portions in the source document. e.g: for+ manual review. This option instructs LLMWhisperer to store+ highlighting information, which can be extracted using the+ returned Whisper Hash. Please see the{" "}+ <a+ href="https://docs.unstract.com/llm_whisperer/apis/llm_whisperer_text_extraction_highlight_api"+ target="_blank"+ rel="noopener noreferrer"+ >+ Highlighting API documentation+ </a>{" "}+ for more details.+ </>+ }+ type="info"+ />+ </Space>+ </Form.Item>+ <Form.Item className="display-flex-right">+ <CustomButton type="primary" htmlType="submit" loading={isLoading}>+ Save+ </CustomButton>+ </Form.Item>+ </Form>+ </SpaceWrapper>+ );+}++HighLightMetaData.propTypes = {+ handleUpdateTool: PropTypes.func.isRequired,+};++export { HighLightMetaData };diff --git a/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsx b/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsxindex c46f2163a..9f0b4c25e 100644--- a/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsx+++ b/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsx@@ -14,6 +14,7 @@ import { ManageLlmProfiles } from "../manage-llm-profiles/ManageLlmProfiles";import { useCustomToolStore } from "../../../store/custom-tool-store";import { CustomSynonyms } from "../custom-synonyms/CustomSynonyms";import { PreAndPostAmbleModal } from "../pre-and-post-amble-modal/PreAndPostAmbleModal";+import { HighLightMetaData } from "../highlight-metadata/HighLightMetaData.jsx";import "./SettingsModal.css";@@ -41,6 +42,7 @@ function SettingsModal({ open, setOpen, handleUpdateTool }) {getMenuItem("Manage Grammar", 5, <MessageOutlined />),getMenuItem("Preamble", 6, <DiffOutlined />),getMenuItem("Postamble", 7, <DiffOutlined />),+ getMenuItem("Manage Higlighting", 8, <CodeOutlined />),];const listOfComponents = {@@ -58,6 +60,7 @@ function SettingsModal({ open, setOpen, handleUpdateTool }) {handleUpdateTool={handleUpdateTool}/>),+ 8: <HighLightMetaData handleUpdateTool={handleUpdateTool} />,};let position = 1;diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txtindex 7447148a0..24ee83f5a 100644--- a/tools/structure/requirements.txt+++ b/tools/structure/requirements.txt@@ -1,4 +1,4 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=0.33.1+unstract-sdk~=0.35.0diff --git a/tools/structure/src/constants.py b/tools/structure/src/constants.pyindex fe9df3a16..86bd21cd1 100644--- a/tools/structure/src/constants.py+++ b/tools/structure/src/constants.py@@ -65,4 +65,5 @@ class SettingsKeys:OK = "OK"FILE_NAME = "file_name"FILE_HASH = "file_hash"+ ENABLE_HIGHLIGHT = "enable_highlight"NAME = "name"diff --git a/tools/structure/src/main.py b/tools/structure/src/main.pyindex b52c6f275..8453351ab 100644--- a/tools/structure/src/main.py+++ b/tools/structure/src/main.py@@ -27,6 +27,7 @@ class StructureTool(BaseTool):prompt_registry_id: str = settings[SettingsKeys.PROMPT_REGISTRY_ID]challenge_llm: str = settings[SettingsKeys.CHALLENGE_LLM_ADAPTER_ID]enable_challenge: bool = settings[SettingsKeys.ENABLE_CHALLENGE]+summarize_as_source: bool = settings[SettingsKeys.SUMMARIZE_AS_SOURCE]single_pass_extraction_mode: bool = settings[SettingsKeys.SINGLE_PASS_EXTRACTION_MODE@@ -57,6 +58,7 @@ class StructureTool(BaseTool):tool_id = tool_metadata[SettingsKeys.TOOL_ID]tool_settings = tool_metadata[SettingsKeys.TOOL_SETTINGS]outputs = tool_metadata[SettingsKeys.OUTPUTS]+ enable_highlight: bool = tool_settings.get(SettingsKeys.ENABLE_HIGHLIGHT, False)tool_settings[SettingsKeys.CHALLENGE_LLM] = challenge_llmtool_settings[SettingsKeys.ENABLE_CHALLENGE] = enable_challengetool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION] = (@@ -97,6 +99,7 @@ class StructureTool(BaseTool):output_file_path=tool_data_dir / SettingsKeys.EXTRACT,reindex=True,usage_kwargs=usage_kwargs,+ enable_highlight=enable_highlight,)if summarize_as_source:summarize_file_hash = self._summarize_and_index(@@ -107,6 +110,7 @@ class StructureTool(BaseTool):outputs=outputs,index=index,usage_kwargs=usage_kwargs,+ enable_highlight=enable_highlight,)payload[SettingsKeys.FILE_HASH] = summarize_file_hashself.stream_log("Fetching response for single pass extraction")@@ -130,7 +134,9 @@ class StructureTool(BaseTool):output_file_path=tool_data_dir / SettingsKeys.EXTRACT,reindex=reindex,usage_kwargs=usage_kwargs,+ enable_highlight=enable_highlight,)+if summarize_as_source:summarize_file_hash = self._summarize_and_index(tool_id=tool_id,@@ -190,6 +196,7 @@ class StructureTool(BaseTool):outputs: dict[str, Any],index: Index,usage_kwargs: dict[Any, Any] = {},+ enable_highlight: bool = False,) -> str:"""Summarizes the context of the file and indexes the summarizedcontent.@@ -263,6 +270,7 @@ class StructureTool(BaseTool):chunk_size=0,chunk_overlap=0,usage_kwargs=usage_kwargs,+ enable_highlight=enable_highlight,)return summarize_file_hash