Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Fix/adding whisper hash in metadata (#418)

Rahul Johnyd200c8ed9597
diff --git a/backend/prompt_studio/prompt_studio_core/migrations/0013_customtool_enable_highlight.py b/backend/prompt_studio/prompt_studio_core/migrations/0013_customtool_enable_highlight.py
new file mode 100644
index 000000000..eb7ee01f7
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_core/migrations/0013_customtool_enable_highlight.py
@@ -0,0 +1,21 @@
+# Generated by Django 4.2.1 on 2024-06-13 06:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("prompt_studio_core", "0012_customtool_shared_users"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="customtool",
+ name="enable_highlight",
+ field=models.BooleanField(
+ db_comment="Flag to enable or disable document highlighting",
+ default=False,
+ ),
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_studio_core/models.py b/backend/prompt_studio/prompt_studio_core/models.py
index ca49db6d8..5075caceb 100644
--- a/backend/prompt_studio/prompt_studio_core/models.py
+++ b/backend/prompt_studio/prompt_studio_core/models.py
@@ -120,6 +120,10 @@ class CustomTool(BaseModel):
db_comment="Flag to enable or disable challenge", default=False
)
+ enable_highlight = models.BooleanField(
+ db_comment="Flag to enable or disable document highlighting", default=False
+ )
+
# Introduced field to establish M2M relation between users and custom_tool.
# This will introduce intermediary table which relates both the models.
shared_users = models.ManyToManyField(User, related_name="shared_custom_tool")
diff --git a/backend/prompt_studio/prompt_studio_registry/constants.py b/backend/prompt_studio/prompt_studio_registry/constants.py
index 0b7ab8835..6dbde1c09 100644
--- a/backend/prompt_studio/prompt_studio_registry/constants.py
+++ b/backend/prompt_studio/prompt_studio_registry/constants.py
@@ -94,6 +94,7 @@ class JsonSchemaKey:
IMAGE_TAG = "image_tag"
SUMMARIZE_PROMPT = "summarize_prompt"
SUMMARIZE_AS_SOURCE = "summarize_as_source"
+ ENABLE_HIGHLIGHT = "enable_highlight"
class SpecKey:
diff --git a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py
index e7bd232cd..67f84b450 100644
--- a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py
+++ b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py
@@ -280,6 +280,7 @@ class PromptStudioRegistryHelper:
tool_settings[JsonSchemaKey.ENABLE_SINGLE_PASS_EXTRACTION] = (
tool.single_pass_extraction_mode
)
+ tool_settings[JsonSchemaKey.ENABLE_HIGHLIGHT] = tool.enable_highlight
for prompt in prompts:
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.py
index df1db367d..464652d7f 100644
--- a/backend/workflow_manager/endpoint/destination.py
+++ b/backend/workflow_manager/endpoint/destination.py
@@ -102,6 +102,7 @@ class DestinationConnector(BaseConnector):
"""Handle the output based on the connection type."""
connection_type = self.endpoint.connection_type
result: Optional[str] = None
+ meta_data: Optional[str] = None
if error:
if connection_type == WorkflowEndpoint.ConnectionType.API:
self._handle_api_result(file_name=file_name, error=error, result=result)
@@ -112,13 +113,18 @@ class DestinationConnector(BaseConnector):
self.insert_into_db(file_history)
elif connection_type == WorkflowEndpoint.ConnectionType.API:
result = self.get_result(file_history)
- self._handle_api_result(file_name=file_name, error=error, result=result)
+ meta_data = self.get_metadata(file_history)
+
+ self._handle_api_result(
+ file_name=file_name, error=error, result=result, meta_data=meta_data
+ )
if not file_history:
FileHistoryHelper.create_file_history(
cache_key=file_hash,
workflow=workflow,
status=ExecutionStatus.COMPLETED,
result=result,
+ metadata=meta_data,
file_name=file_name,
)
@@ -235,6 +241,7 @@ class DestinationConnector(BaseConnector):
file_name: str,
error: Optional[str] = None,
result: Optional[str] = None,
+ meta_data: Optional[dict[str, Any]] = None,
) -> None:
"""Handle the API result.
@@ -249,7 +256,7 @@ class DestinationConnector(BaseConnector):
Returns:
None
"""
- api_result = {"file": file_name}
+ api_result: dict[str, Any] = {"file": file_name}
if error:
api_result.update(
{"status": ApiDeploymentResultStatus.FAILED, "error": error}
@@ -260,6 +267,7 @@ class DestinationConnector(BaseConnector):
{
"status": ApiDeploymentResultStatus.SUCCESS,
"result": result,
+ "metadata": meta_data,
}
)
else:
@@ -311,7 +319,7 @@ class DestinationConnector(BaseConnector):
if file_history and file_history.result:
return self.parse_string(file_history.result)
output_file = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
- metadata = self.get_workflow_metadata()
+ metadata: dict[str, Any] = self.get_workflow_metadata()
output_type = self.get_output_type(metadata)
result: Optional[Any] = None
try:
@@ -337,6 +345,20 @@ class DestinationConnector(BaseConnector):
logger.error(f"Error while getting result {err}")
return result
+ def get_metadata(
+ self, file_history: Optional[FileHistory]
+ ) -> Optional[dict[str, Any]]:
+ """Get meta_data from the output file.
+
+ Returns:
+ Union[dict[str, Any], str]: Meta data.
+ """
+ if file_history and file_history.meta_data:
+ return self.parse_string(file_history.meta_data)
+ metadata: dict[str, Any] = self.get_workflow_metadata()
+
+ return metadata
+
def delete_execution_directory(self) -> None:
"""Delete the execution directory.
diff --git a/backend/workflow_manager/workflow/file_history_helper.py b/backend/workflow_manager/workflow/file_history_helper.py
index 193c96b33..8317f106e 100644
--- a/backend/workflow_manager/workflow/file_history_helper.py
+++ b/backend/workflow_manager/workflow/file_history_helper.py
@@ -40,6 +40,7 @@ class FileHistoryHelper:
workflow: Workflow,
status: ExecutionStatus,
result: Any,
+ metadata: Any,
error: Optional[str] = None,
file_name: Optional[str] = None,
) -> FileHistory:
@@ -60,6 +61,7 @@ class FileHistoryHelper:
cache_key=cache_key,
status=status.value,
result=str(result),
+ meta_data=str(metadata),
error=str(error) if error else "",
)
except IntegrityError:
diff --git a/backend/workflow_manager/workflow/migrations/0006_filehistory_meta_data.py b/backend/workflow_manager/workflow/migrations/0006_filehistory_meta_data.py
new file mode 100644
index 000000000..46b43dc79
--- /dev/null
+++ b/backend/workflow_manager/workflow/migrations/0006_filehistory_meta_data.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.1 on 2024-06-21 18:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("workflow", "0005_executionlog"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="filehistory",
+ name="meta_data",
+ field=models.TextField(blank=True, db_comment="MetaData from execution"),
+ ),
+ ]
diff --git a/backend/workflow_manager/workflow/models/file_history.py b/backend/workflow_manager/workflow/models/file_history.py
index 03c8bbeb3..a31064803 100644
--- a/backend/workflow_manager/workflow/models/file_history.py
+++ b/backend/workflow_manager/workflow/models/file_history.py
@@ -39,6 +39,7 @@ class FileHistory(BaseModel):
db_comment="Error message",
)
result = models.TextField(blank=True, db_comment="Result from execution")
+ meta_data = models.TextField(blank=True, db_comment="MetaData from execution")
class Meta:
constraints = [
diff --git a/frontend/src/components/custom-tools/highlight-metadata/HighLightMetaData.css b/frontend/src/components/custom-tools/highlight-metadata/HighLightMetaData.css
new file mode 100644
index 000000000..8fcec2dba
--- /dev/null
+++ b/frontend/src/components/custom-tools/highlight-metadata/HighLightMetaData.css
@@ -0,0 +1,3 @@
+.highlight{
+ margin-top: 16px;
+}
diff --git a/frontend/src/components/custom-tools/highlight-metadata/HighLightMetaData.jsx b/frontend/src/components/custom-tools/highlight-metadata/HighLightMetaData.jsx
new file mode 100644
index 000000000..60295d5e7
--- /dev/null
+++ b/frontend/src/components/custom-tools/highlight-metadata/HighLightMetaData.jsx
@@ -0,0 +1,106 @@
+import { Checkbox, Form, Space, Typography, Alert } from "antd";
+import PropTypes from "prop-types";
+import { useEffect, useState } from "react";
+
+import SpaceWrapper from "../../widgets/space-wrapper/SpaceWrapper";
+import { CustomButton } from "../../../components/widgets/custom-button/CustomButton";
+import { useCustomToolStore } from "../../../store/custom-tool-store";
+import { useAlertStore } from "../../../store/alert-store";
+import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
+import "./HighLightMetaData.css";
+
+function HighLightMetaData({ handleUpdateTool }) {
+ const [enableHighlighting, setEnableHighlighting] = useState(false);
+ const [isLoading, setIsLoading] = useState(false);
+
+ const { details, updateCustomTool } = useCustomToolStore();
+
+ const { setAlertDetails } = useAlertStore();
+ const handleException = useExceptionHandler();
+
+ useEffect(() => {
+ setEnableHighlighting(details?.enable_challenge || false);
+ }, []);
+
+ const handleSubmit = () => {
+ const body = {
+ enable_highlight: enableHighlighting,
+ };
+ setIsLoading(true);
+ handleUpdateTool(body)
+ .then((res) => {
+ const data = res?.data;
+ const updatedDetails = { ...details };
+
+ updatedDetails["enable_highlight"] = data?.enable_challenge;
+ updateCustomTool({ details: updatedDetails });
+
+ setAlertDetails({
+ type: "success",
+ content: "Saved highlight settings successfully",
+ });
+ })
+ .catch((err) => {
+ setAlertDetails(handleException(err, "Failed to update"));
+ })
+ .finally(() => {
+ setIsLoading(false);
+ });
+ };
+
+ return (
+ <SpaceWrapper>
+ <div className="settings-body-pad-top">
+ <Typography.Text strong className="pre-post-amble-title">
+ Enable highlighting ( Will only work with LLM whisperer as text
+ extractor)
+ </Typography.Text>
+ </div>
+ <Form layout="vertical" onFinish={handleSubmit}>
+ <Form.Item>
+ <Space>
+ <Checkbox
+ checked={enableHighlighting}
+ onChange={(e) => setEnableHighlighting(e.target.checked)}
+ />
+ <Typography.Text>Enable Highlighting</Typography.Text>
+ </Space>
+ <Space>
+ <Alert
+ className="highlight"
+ message={
+ <>
+ Turn this option on if downstream use cases need you to
+ highlight extracted portions in the source document. e.g: for
+ manual review. This option instructs LLMWhisperer to store
+ highlighting information, which can be extracted using the
+ returned Whisper Hash. Please see the{" "}
+ <a
+ href="https://docs.unstract.com/llm_whisperer/apis/llm_whisperer_text_extraction_highlight_api"
+ target="_blank"
+ rel="noopener noreferrer"
+ >
+ Highlighting API documentation
+ </a>{" "}
+ for more details.
+ </>
+ }
+ type="info"
+ />
+ </Space>
+ </Form.Item>
+ <Form.Item className="display-flex-right">
+ <CustomButton type="primary" htmlType="submit" loading={isLoading}>
+ Save
+ </CustomButton>
+ </Form.Item>
+ </Form>
+ </SpaceWrapper>
+ );
+}
+
+HighLightMetaData.propTypes = {
+ handleUpdateTool: PropTypes.func.isRequired,
+};
+
+export { HighLightMetaData };
diff --git a/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsx b/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsx
index c46f2163a..9f0b4c25e 100644
--- a/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsx
+++ b/frontend/src/components/custom-tools/settings-modal/SettingsModal.jsx
@@ -14,6 +14,7 @@ import { ManageLlmProfiles } from "../manage-llm-profiles/ManageLlmProfiles";
import { useCustomToolStore } from "../../../store/custom-tool-store";
import { CustomSynonyms } from "../custom-synonyms/CustomSynonyms";
import { PreAndPostAmbleModal } from "../pre-and-post-amble-modal/PreAndPostAmbleModal";
+import { HighLightMetaData } from "../highlight-metadata/HighLightMetaData.jsx";
import "./SettingsModal.css";
@@ -41,6 +42,7 @@ function SettingsModal({ open, setOpen, handleUpdateTool }) {
getMenuItem("Manage Grammar", 5, <MessageOutlined />),
getMenuItem("Preamble", 6, <DiffOutlined />),
getMenuItem("Postamble", 7, <DiffOutlined />),
+ getMenuItem("Manage Higlighting", 8, <CodeOutlined />),
];
const listOfComponents = {
@@ -58,6 +60,7 @@ function SettingsModal({ open, setOpen, handleUpdateTool }) {
handleUpdateTool={handleUpdateTool}
/>
),
+ 8: <HighLightMetaData handleUpdateTool={handleUpdateTool} />,
};
let position = 1;
diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txt
index 7447148a0..24ee83f5a 100644
--- a/tools/structure/requirements.txt
+++ b/tools/structure/requirements.txt
@@ -1,4 +1,4 @@
# Add your dependencies here
# Required for all unstract tools
-unstract-sdk~=0.33.1
+unstract-sdk~=0.35.0
diff --git a/tools/structure/src/constants.py b/tools/structure/src/constants.py
index fe9df3a16..86bd21cd1 100644
--- a/tools/structure/src/constants.py
+++ b/tools/structure/src/constants.py
@@ -65,4 +65,5 @@ class SettingsKeys:
OK = "OK"
FILE_NAME = "file_name"
FILE_HASH = "file_hash"
+ ENABLE_HIGHLIGHT = "enable_highlight"
NAME = "name"
diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py
index b52c6f275..8453351ab 100644
--- a/tools/structure/src/main.py
+++ b/tools/structure/src/main.py
@@ -27,6 +27,7 @@ class StructureTool(BaseTool):
prompt_registry_id: str = settings[SettingsKeys.PROMPT_REGISTRY_ID]
challenge_llm: str = settings[SettingsKeys.CHALLENGE_LLM_ADAPTER_ID]
enable_challenge: bool = settings[SettingsKeys.ENABLE_CHALLENGE]
+
summarize_as_source: bool = settings[SettingsKeys.SUMMARIZE_AS_SOURCE]
single_pass_extraction_mode: bool = settings[
SettingsKeys.SINGLE_PASS_EXTRACTION_MODE
@@ -57,6 +58,7 @@ class StructureTool(BaseTool):
tool_id = tool_metadata[SettingsKeys.TOOL_ID]
tool_settings = tool_metadata[SettingsKeys.TOOL_SETTINGS]
outputs = tool_metadata[SettingsKeys.OUTPUTS]
+ enable_highlight: bool = tool_settings.get(SettingsKeys.ENABLE_HIGHLIGHT, False)
tool_settings[SettingsKeys.CHALLENGE_LLM] = challenge_llm
tool_settings[SettingsKeys.ENABLE_CHALLENGE] = enable_challenge
tool_settings[SettingsKeys.ENABLE_SINGLE_PASS_EXTRACTION] = (
@@ -97,6 +99,7 @@ class StructureTool(BaseTool):
output_file_path=tool_data_dir / SettingsKeys.EXTRACT,
reindex=True,
usage_kwargs=usage_kwargs,
+ enable_highlight=enable_highlight,
)
if summarize_as_source:
summarize_file_hash = self._summarize_and_index(
@@ -107,6 +110,7 @@ class StructureTool(BaseTool):
outputs=outputs,
index=index,
usage_kwargs=usage_kwargs,
+ enable_highlight=enable_highlight,
)
payload[SettingsKeys.FILE_HASH] = summarize_file_hash
self.stream_log("Fetching response for single pass extraction")
@@ -130,7 +134,9 @@ class StructureTool(BaseTool):
output_file_path=tool_data_dir / SettingsKeys.EXTRACT,
reindex=reindex,
usage_kwargs=usage_kwargs,
+ enable_highlight=enable_highlight,
)
+
if summarize_as_source:
summarize_file_hash = self._summarize_and_index(
tool_id=tool_id,
@@ -190,6 +196,7 @@ class StructureTool(BaseTool):
outputs: dict[str, Any],
index: Index,
usage_kwargs: dict[Any, Any] = {},
+ enable_highlight: bool = False,
) -> str:
"""Summarizes the context of the file and indexes the summarized
content.
@@ -263,6 +270,7 @@ class StructureTool(BaseTool):
chunk_size=0,
chunk_overlap=0,
usage_kwargs=usage_kwargs,
+ enable_highlight=enable_highlight,
)
return summarize_file_hash