Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[Feat] Added highlight support for Single pass and JSON enforce type in prompt studio (#1159)

jagadeeswaran-zipstack0b2cd0d7fba8
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0003_promptstudiooutputmanager_confidence_data.py b/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0003_promptstudiooutputmanager_confidence_data.py
new file mode 100644
index 000000000..693d5a0e6
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0003_promptstudiooutputmanager_confidence_data.py
@@ -0,0 +1,23 @@
+# Generated by Django 4.2.1 on 2025-03-07 07:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ (
+ "prompt_studio_output_manager_v2",
+ "0002_promptstudiooutputmanager_highlight_data",
+ ),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="promptstudiooutputmanager",
+ name="confidence_data",
+ field=models.JSONField(
+ blank=True, db_comment="Field to store confidence data", null=True
+ ),
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/models.py b/backend/prompt_studio/prompt_studio_output_manager_v2/models.py
index 70efd1746..ab61a4b23 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/models.py
@@ -30,6 +30,12 @@ class PromptStudioOutputManager(BaseModel):
highlight_data = models.JSONField(
db_comment="Field to store highlight data", editable=True, null=True, blank=True
)
+ confidence_data = models.JSONField(
+ db_comment="Field to store confidence data",
+ editable=True,
+ null=True,
+ blank=True,
+ )
eval_metrics = models.JSONField(
db_column="eval_metrics",
null=False,
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
index f6ec9bf9d..35ac876b1 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py
@@ -61,6 +61,7 @@ class OutputManagerHelper:
context: str,
challenge_data: Optional[dict[str, Any]],
highlight_data: Optional[dict[str, Any]],
+ confidence_data: Optional[dict[str, Any]],
) -> PromptStudioOutputManager:
"""Handles creating or updating a single prompt output and returns
the instance."""
@@ -78,6 +79,7 @@ class OutputManagerHelper:
"context": context,
"challenge_data": challenge_data,
"highlight_data": highlight_data,
+ "confidence_data": confidence_data,
},
)
)
@@ -100,6 +102,7 @@ class OutputManagerHelper:
"context": context,
"challenge_data": challenge_data,
"highlight_data": highlight_data,
+ "confidence_data": confidence_data,
}
PromptStudioOutputManager.objects.filter(
document_manager=document_manager,
@@ -122,6 +125,7 @@ class OutputManagerHelper:
context = metadata.get("context")
challenge_data = metadata.get("challenge_data")
highlight_data = metadata.get("highlight_data")
+ confidence_data = metadata.get("confidence_data")
if not prompts:
return serialized_data
@@ -140,6 +144,8 @@ class OutputManagerHelper:
context = context.get(prompt.prompt_key)
if highlight_data:
highlight_data = highlight_data.get(prompt.prompt_key)
+ if confidence_data:
+ confidence_data = confidence_data.get(prompt.prompt_key)
if challenge_data:
challenge_data = challenge_data.get(prompt.prompt_key)
@@ -163,6 +169,7 @@ class OutputManagerHelper:
context=json.dumps(context),
challenge_data=challenge_data,
highlight_data=highlight_data,
+ confidence_data=confidence_data,
)
# Serialize the instance
diff --git a/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx b/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx
index 7ec85eb5e..a19bec8ca 100644
--- a/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx
+++ b/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx
@@ -7,7 +7,7 @@ import {
import "@react-pdf-viewer/core/lib/styles/index.css";
import "@react-pdf-viewer/default-layout/lib/styles/index.css";
import "@react-pdf-viewer/page-navigation/lib/styles/index.css";
-import { Button, Space, Tabs, Tooltip, Typography } from "antd";
+import { Button, Space, Tabs, Tag, Tooltip, Typography } from "antd";
import PropTypes from "prop-types";
import { useEffect, useState } from "react";
import { useParams } from "react-router-dom";
@@ -370,6 +370,18 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) {
</div>
{!isSimplePromptStudio && (
<Space>
+ {details?.enable_highlight && (
+ <div>
+ <Tag color="rgb(45, 183, 245)">
+ Confidence Score:{" "}
+ {selectedHighlight?.confidence?.[0]?.[0]?.confidence ||
+ (Array.isArray(selectedHighlight?.confidence?.[0]) &&
+ selectedHighlight?.confidence?.[0]?.length === 0 &&
+ "1") ||
+ "NA"}
+ </Tag>
+ </div>
+ )}
<div className="doc-main-title-div">
{selectedDoc ? (
<Tooltip title={selectedDoc?.document_name}>
diff --git a/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx b/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx
index f5bba6460..465f34ae3 100644
--- a/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx
@@ -1,21 +1,37 @@
-import { Spin, Typography } from "antd";
+import { Space, Spin, Typography } from "antd";
import PropTypes from "prop-types";
import { InfoCircleFilled } from "@ant-design/icons";
+import { useEffect, useState } from "react";
import {
displayPromptResult,
generateApiRunStatusId,
+ generateUUID,
PROMPT_RUN_API_STATUSES,
} from "../../../helpers/GetStaticData";
import "./PromptCard.css";
import { useCustomToolStore } from "../../../store/custom-tool-store";
-import { useEffect, useState } from "react";
import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader";
-function DisplayPromptResult({ output, profileId, docId, promptRunStatus }) {
+function DisplayPromptResult({
+ output,
+ profileId,
+ docId,
+ promptRunStatus,
+ handleSelectHighlight,
+ highlightData,
+ promptDetails,
+ confidenceData,
+}) {
const [isLoading, setIsLoading] = useState(false);
- const { singlePassExtractMode, isSinglePassExtractLoading } =
- useCustomToolStore();
+ const [parsedOutput, setParsedOutput] = useState(null);
+ const [selectedKey, setSelectedKey] = useState(null);
+ const {
+ singlePassExtractMode,
+ isSinglePassExtractLoading,
+ details,
+ selectedHighlight,
+ } = useCustomToolStore();
useEffect(() => {
if (singlePassExtractMode && isSinglePassExtractLoading) {
@@ -30,7 +46,15 @@ function DisplayPromptResult({ output, profileId, docId, promptRunStatus }) {
}
setIsLoading(false);
- }, [promptRunStatus, isSinglePassExtractLoading]);
+ setParsedOutput(
+ displayPromptResult(output, true, details?.enable_highlight)
+ );
+ }, [
+ promptRunStatus,
+ isSinglePassExtractLoading,
+ details?.enable_highlight,
+ output,
+ ]);
if (isLoading) {
return <Spin indicator={<SpinnerLoader size="small" />} />;
@@ -47,18 +71,180 @@ function DisplayPromptResult({ output, profileId, docId, promptRunStatus }) {
);
}
+ const handleClick = (highlightData, confidenceData, key, keyPath) => {
+ if (highlightData?.[key]) {
+ handleSelectHighlight(
+ highlightData[key],
+ promptDetails?.prompt_id,
+ profileId,
+ confidenceData?.[key]
+ );
+ setSelectedKey(keyPath);
+ }
+ };
+
+ const isObject = (value) =>
+ typeof value === "object" && value !== null && !Array.isArray(value);
+
+ const renderJson = (
+ data,
+ highlightData,
+ confidenceData,
+ indent = 0,
+ path = ""
+ ) => {
+ if (typeof data === "object" && !details?.enable_highlight) {
+ return JSON.stringify(data, null, 4);
+ }
+
+ if (typeof data === "string") {
+ return `"${data}"`;
+ }
+
+ if (typeof data === "number" || typeof data === "boolean") {
+ return data.toString();
+ }
+
+ if (Array.isArray(data)) {
+ return (
+ <>
+ {"["}
+ <div style={{ paddingLeft: "20px" }}>
+ {data.map((item, index) => (
+ <div key={generateUUID()}>
+ {renderJson(
+ item,
+ highlightData[index],
+ confidenceData?.[index],
+ indent + 1,
+ `${path}[${index}]`
+ )}
+ {index < data.length - 1 ? "," : ""}
+ </div>
+ ))}
+ </div>
+ {"]"}
+ </>
+ );
+ }
+
+ if (isObject(data)) {
+ return (
+ <>
+ {"{"}
+ <div style={{ paddingLeft: "20px" }}>
+ {Object.entries(data).map(([key, value], index, array) => {
+ const isClickable = !isObject(value) && !Array.isArray(value); // Only primitive values should be clickable
+ const newPath = path ? `${path}.${key}` : key;
+ const isSelected = selectedKey === newPath;
+ return (
+ <div key={key}>
+ <Space wrap className="json-key">
+ {key}
+ </Space>
+ {": "}
+ <Typography.Text
+ className={`prompt-output-result json-value ${
+ isClickable && highlightData?.[key] ? "clickable" : ""
+ } ${isSelected ? "selected" : ""}`}
+ onClick={() => {
+ if (isClickable && highlightData?.[key]) {
+ handleClick(
+ highlightData,
+ confidenceData,
+ key,
+ newPath
+ );
+ }
+ }}
+ >
+ {renderJson(
+ value,
+ highlightData?.[key],
+ confidenceData?.[key],
+ indent + 1,
+ newPath
+ )}
+ </Typography.Text>
+ {index < array.length - 1 ? "," : ""}
+ </div>
+ );
+ })}
+ </div>
+ {"}"}
+ </>
+ );
+ }
+
+ return String(data);
+ };
+
return (
<Typography.Paragraph className="prompt-card-display-output font-size-12">
- <div>{displayPromptResult(output, true)}</div>
+ {parsedOutput && typeof parsedOutput === "object" ? (
+ renderJson(parsedOutput, highlightData, confidenceData, 0)
+ ) : (
+ <TextResult
+ enableHighlight={details?.enable_highlight}
+ highlightData={highlightData}
+ promptId={promptDetails?.prompt_id}
+ profileId={profileId}
+ confidenceData={confidenceData}
+ selectedHighlight={selectedHighlight}
+ parsedOutput={parsedOutput}
+ onSelectHighlight={handleSelectHighlight}
+ />
+ )}
</Typography.Paragraph>
);
}
+const TextResult = ({
+ enableHighlight,
+ highlightData,
+ promptId,
+ profileId,
+ confidenceData,
+ selectedHighlight,
+ parsedOutput,
+ onSelectHighlight,
+}) => {
+ return enableHighlight ? (
+ <Typography.Text
+ wrap
+ onClick={() =>
+ onSelectHighlight(highlightData, promptId, profileId, confidenceData)
+ }
+ className={`prompt-output-result json-value ${
+ highlightData ? "clickable" : ""
+ } ${selectedHighlight?.highlightedPrompt === promptId ? "selected" : ""}`}
+ >
+ {parsedOutput}
+ </Typography.Text>
+ ) : (
+ <div>{parsedOutput}</div>
+ );
+};
+
+TextResult.propTypes = {
+ enableHighlight: PropTypes.bool,
+ highlightData: PropTypes.any,
+ promptId: PropTypes.string,
+ profileId: PropTypes.string,
+ confidenceData: PropTypes.any,
+ selectedHighlight: PropTypes.object,
+ parsedOutput: PropTypes.any,
+ onSelectHighlight: PropTypes.func.isRequired,
+};
DisplayPromptResult.propTypes = {
output: PropTypes.any,
profileId: PropTypes.string,
docId: PropTypes.string,
promptRunStatus: PropTypes.object,
+ handleSelectHighlight: PropTypes.func,
+ highlightData: PropTypes.object,
+ promptDetails: PropTypes.object,
+ confidenceData: PropTypes.object,
};
export { DisplayPromptResult };
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.css b/frontend/src/components/custom-tools/prompt-card/PromptCard.css
index 31926462a..3a331819f 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.css
+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.css
@@ -312,3 +312,26 @@
border-style: solid;
border-color: #ffb400;
}
+
+.json-value {
+ color: black;
+ cursor: default;
+ text-decoration: none;
+}
+
+.json-value.clickable {
+ color: #0097D8;
+ cursor: pointer;
+}
+
+.json-value.clickable:hover {
+ text-decoration: underline;
+}
+
+.json-value.selected {
+ color: #5A8300;
+}
+
+.prompt-output-result{
+ font-size: 12px;
+}
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
index 8bccb5607..ce0d1c0a8 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
@@ -169,17 +169,16 @@ const PromptCard = memo(
const handleSelectHighlight = (
highlightData,
highlightedPrompt,
- highlightedProfile
+ highlightedProfile,
+ confidenceData
) => {
- if (
- details?.enable_highlight &&
- promptDetailsState?.enforce_type !== "json"
- ) {
+ if (details?.enable_highlight) {
updateCustomTool({
selectedHighlight: {
highlight: highlightData,
highlightedPrompt: highlightedPrompt,
highlightedProfile: highlightedProfile,
+ confidence: confidenceData,
},
});
}
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx
index 2a62d4600..3f3a25ba9 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx
@@ -254,7 +254,7 @@ function PromptCardItems({
</Space>
<Space>
{details?.enable_highlight &&
- ["json", "table", "record"].includes(enforceType) && (
+ ["table", "record"].includes(enforceType) && (
<Tag
color="red"
style={{
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
index deebd36d0..b4c2f7257 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
@@ -84,9 +84,7 @@ function PromptOutput({
const { generatePromptOutputKey } = usePromptOutput();
const isTableExtraction =
enforceType === TABLE_ENFORCE_TYPE || enforceType === RECORD_ENFORCE_TYPE;
- const noHighlightEnforceType = !["json", "table", "record"].includes(
- enforceType
- );
+ const noHighlightEnforceType = !["table", "record"].includes(enforceType);
const tooltipContent = (adapterConf) => (
<div>
{Object.entries(adapterConf)?.map(([key, value]) => (
@@ -144,16 +142,54 @@ function PromptOutput({
const promptOutput = promptOutputs[promptOutputKey]?.output;
+ let promptOutputData = {};
+ if (promptOutputs && Object.keys(promptOutputs)) {
+ const promptOutputKey = generatePromptOutputKey(
+ promptId,
+ docId,
+ defaultLlmProfile,
+ singlePassExtractMode,
+ true
+ );
+ if (promptOutputs[promptOutputKey] !== undefined) {
+ promptOutputData = promptOutputs[promptOutputKey];
+ }
+ }
+
return (
<>
<Divider className="prompt-card-divider" />
- <div className="prompt-card-result prompt-card-div">
- <DisplayPromptResult output={promptOutput} />
+ <Space
+ wrap
+ className={`prompt-card-result prompt-card-div ${
+ details?.enable_highlight &&
+ noHighlightEnforceType &&
+ selectedHighlight?.highlightedPrompt === promptId &&
+ selectedHighlight?.highlightedProfile === defaultLlmProfile &&
+ "highlighted-prompt-cell"
+ }`}
+ >
+ <DisplayPromptResult
+ output={promptOutput}
+ highlightData={
+ promptOutputData?.highlightData?.[promptDetails.prompt_key]
+ }
+ handleSelectHighlight={handleSelectHighlight}
+ confidenceData={
+ promptOutputData?.confidenceData?.[promptDetails.prompt_key]
+ }
+ />
<div className="prompt-profile-run">
<CopyPromptOutputBtn
isDisabled={isTableExtraction}
copyToClipboard={() =>
- copyOutputToClipboard(displayPromptResult(promptOutput, true))
+ copyOutputToClipboard(
+ displayPromptResult(
+ promptOutput,
+ true,
+ promptDetails?.enable_highlight
+ )
+ )
}
/>
<PromptOutputExpandBtn
@@ -165,7 +201,7 @@ function PromptOutput({
promptRunStatus={promptRunStatus}
/>
</div>
- </div>
+ </Space>
</>
);
}
@@ -230,11 +266,13 @@ function PromptOutput({
direction="vertical"
className="prompt-card-llm-layout"
onClick={() => {
- handleSelectHighlight(
- promptOutputData?.highlightData,
- promptId,
- profileId
- );
+ enforceType !== "json" &&
+ handleSelectHighlight(
+ promptOutputData?.highlightData,
+ promptId,
+ profileId,
+ promptOutputData?.confidenceData
+ );
}}
>
<div className="llm-info">
@@ -396,6 +434,10 @@ function PromptOutput({
profileId={profileId}
docId={selectedDoc?.document_id}
promptRunStatus={promptRunStatus}
+ handleSelectHighlight={handleSelectHighlight}
+ highlightData={promptOutputData?.highlightData}
+ confidenceData={promptOutputData?.confidenceData}
+ promptDetails={promptDetails}
/>
<div className="prompt-profile-run">
<CopyPromptOutputBtn
diff --git a/frontend/src/helpers/GetStaticData.js b/frontend/src/helpers/GetStaticData.js
index a0af83971..092da7e58 100644
--- a/frontend/src/helpers/GetStaticData.js
+++ b/frontend/src/helpers/GetStaticData.js
@@ -338,12 +338,15 @@ const isJson = (text) => {
}
};
-const displayPromptResult = (output, isFormat = false) => {
+const displayPromptResult = (
+ output,
+ isFormat = false,
+ isHighlightEnabled = false
+) => {
/*
output: The data to be displayed or parsed
isFormat: A flag indicating whether the output should be formatted
*/
-
let i = 0;
let parsedData = output;
@@ -365,9 +368,11 @@ const displayPromptResult = (output, isFormat = false) => {
// Check if the parsed data is an array or object and formatting is requested
if (Array.isArray(parsedData) || typeof parsedData === "object") {
// If formatting is requested, return the JSON string with indentation
+ if (isHighlightEnabled) {
+ return parsedData;
+ }
return JSON.stringify(parsedData, null, 4);
}
-
return String(parsedData);
};
diff --git a/frontend/src/hooks/usePromptOutput.js b/frontend/src/hooks/usePromptOutput.js
index b96230da7..42de48453 100644
--- a/frontend/src/hooks/usePromptOutput.js
+++ b/frontend/src/hooks/usePromptOutput.js
@@ -120,6 +120,7 @@ const usePromptOutput = () => {
timer,
coverage: item?.coverage,
highlightData: item?.highlight_data,
+ confidenceData: item?.confidence_data,
};
if (item?.is_single_pass_extract && isTokenUsageForSinglePassAdded)