LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
[Feat] Added highlight support for Single pass and JSON enforce type in prompt studio (#1159)
jagadeeswaran-zipstack0b2cd0d7fba8
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0003_promptstudiooutputmanager_confidence_data.py b/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0003_promptstudiooutputmanager_confidence_data.pynew file mode 100644index 000000000..693d5a0e6--- /dev/null+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0003_promptstudiooutputmanager_confidence_data.py@@ -0,0 +1,23 @@+# Generated by Django 4.2.1 on 2025-03-07 07:35++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ (+ "prompt_studio_output_manager_v2",+ "0002_promptstudiooutputmanager_highlight_data",+ ),+ ]++ operations = [+ migrations.AddField(+ model_name="promptstudiooutputmanager",+ name="confidence_data",+ field=models.JSONField(+ blank=True, db_comment="Field to store confidence data", null=True+ ),+ ),+ ]diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/models.py b/backend/prompt_studio/prompt_studio_output_manager_v2/models.pyindex 70efd1746..ab61a4b23 100644--- a/backend/prompt_studio/prompt_studio_output_manager_v2/models.py+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/models.py@@ -30,6 +30,12 @@ class PromptStudioOutputManager(BaseModel):highlight_data = models.JSONField(db_comment="Field to store highlight data", editable=True, null=True, blank=True)+ confidence_data = models.JSONField(+ db_comment="Field to store confidence data",+ editable=True,+ null=True,+ blank=True,+ )eval_metrics = models.JSONField(db_column="eval_metrics",null=False,diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.pyindex f6ec9bf9d..35ac876b1 100644--- a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_helper.py@@ -61,6 +61,7 @@ class OutputManagerHelper:context: str,challenge_data: Optional[dict[str, Any]],highlight_data: Optional[dict[str, Any]],+ confidence_data: Optional[dict[str, Any]],) -> PromptStudioOutputManager:"""Handles creating or updating a single prompt output and returnsthe instance."""@@ -78,6 +79,7 @@ class OutputManagerHelper:"context": context,"challenge_data": challenge_data,"highlight_data": highlight_data,+ "confidence_data": confidence_data,},))@@ -100,6 +102,7 @@ class OutputManagerHelper:"context": context,"challenge_data": challenge_data,"highlight_data": highlight_data,+ "confidence_data": confidence_data,}PromptStudioOutputManager.objects.filter(document_manager=document_manager,@@ -122,6 +125,7 @@ class OutputManagerHelper:context = metadata.get("context")challenge_data = metadata.get("challenge_data")highlight_data = metadata.get("highlight_data")+ confidence_data = metadata.get("confidence_data")if not prompts:return serialized_data@@ -140,6 +144,8 @@ class OutputManagerHelper:context = context.get(prompt.prompt_key)if highlight_data:highlight_data = highlight_data.get(prompt.prompt_key)+ if confidence_data:+ confidence_data = confidence_data.get(prompt.prompt_key)if challenge_data:challenge_data = challenge_data.get(prompt.prompt_key)@@ -163,6 +169,7 @@ class OutputManagerHelper:context=json.dumps(context),challenge_data=challenge_data,highlight_data=highlight_data,+ confidence_data=confidence_data,)# Serialize the instancediff --git a/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx b/frontend/src/components/custom-tools/document-manager/DocumentManager.jsxindex 7ec85eb5e..a19bec8ca 100644--- a/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx+++ b/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx@@ -7,7 +7,7 @@ import {import "@react-pdf-viewer/core/lib/styles/index.css";import "@react-pdf-viewer/default-layout/lib/styles/index.css";import "@react-pdf-viewer/page-navigation/lib/styles/index.css";-import { Button, Space, Tabs, Tooltip, Typography } from "antd";+import { Button, Space, Tabs, Tag, Tooltip, Typography } from "antd";import PropTypes from "prop-types";import { useEffect, useState } from "react";import { useParams } from "react-router-dom";@@ -370,6 +370,18 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) {</div>{!isSimplePromptStudio && (<Space>+ {details?.enable_highlight && (+ <div>+ <Tag color="rgb(45, 183, 245)">+ Confidence Score:{" "}+ {selectedHighlight?.confidence?.[0]?.[0]?.confidence ||+ (Array.isArray(selectedHighlight?.confidence?.[0]) &&+ selectedHighlight?.confidence?.[0]?.length === 0 &&+ "1") ||+ "NA"}+ </Tag>+ </div>+ )}<div className="doc-main-title-div">{selectedDoc ? (<Tooltip title={selectedDoc?.document_name}>diff --git a/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx b/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsxindex f5bba6460..465f34ae3 100644--- a/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx+++ b/frontend/src/components/custom-tools/prompt-card/DisplayPromptResult.jsx@@ -1,21 +1,37 @@-import { Spin, Typography } from "antd";+import { Space, Spin, Typography } from "antd";import PropTypes from "prop-types";import { InfoCircleFilled } from "@ant-design/icons";+import { useEffect, useState } from "react";import {displayPromptResult,generateApiRunStatusId,+ generateUUID,PROMPT_RUN_API_STATUSES,} from "../../../helpers/GetStaticData";import "./PromptCard.css";import { useCustomToolStore } from "../../../store/custom-tool-store";-import { useEffect, useState } from "react";import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader";-function DisplayPromptResult({ output, profileId, docId, promptRunStatus }) {+function DisplayPromptResult({+ output,+ profileId,+ docId,+ promptRunStatus,+ handleSelectHighlight,+ highlightData,+ promptDetails,+ confidenceData,+}) {const [isLoading, setIsLoading] = useState(false);- const { singlePassExtractMode, isSinglePassExtractLoading } =- useCustomToolStore();+ const [parsedOutput, setParsedOutput] = useState(null);+ const [selectedKey, setSelectedKey] = useState(null);+ const {+ singlePassExtractMode,+ isSinglePassExtractLoading,+ details,+ selectedHighlight,+ } = useCustomToolStore();useEffect(() => {if (singlePassExtractMode && isSinglePassExtractLoading) {@@ -30,7 +46,15 @@ function DisplayPromptResult({ output, profileId, docId, promptRunStatus }) {}setIsLoading(false);- }, [promptRunStatus, isSinglePassExtractLoading]);+ setParsedOutput(+ displayPromptResult(output, true, details?.enable_highlight)+ );+ }, [+ promptRunStatus,+ isSinglePassExtractLoading,+ details?.enable_highlight,+ output,+ ]);if (isLoading) {return <Spin indicator={<SpinnerLoader size="small" />} />;@@ -47,18 +71,180 @@ function DisplayPromptResult({ output, profileId, docId, promptRunStatus }) {);}+ const handleClick = (highlightData, confidenceData, key, keyPath) => {+ if (highlightData?.[key]) {+ handleSelectHighlight(+ highlightData[key],+ promptDetails?.prompt_id,+ profileId,+ confidenceData?.[key]+ );+ setSelectedKey(keyPath);+ }+ };++ const isObject = (value) =>+ typeof value === "object" && value !== null && !Array.isArray(value);++ const renderJson = (+ data,+ highlightData,+ confidenceData,+ indent = 0,+ path = ""+ ) => {+ if (typeof data === "object" && !details?.enable_highlight) {+ return JSON.stringify(data, null, 4);+ }++ if (typeof data === "string") {+ return `"${data}"`;+ }++ if (typeof data === "number" || typeof data === "boolean") {+ return data.toString();+ }++ if (Array.isArray(data)) {+ return (+ <>+ {"["}+ <div style={{ paddingLeft: "20px" }}>+ {data.map((item, index) => (+ <div key={generateUUID()}>+ {renderJson(+ item,+ highlightData[index],+ confidenceData?.[index],+ indent + 1,+ `${path}[${index}]`+ )}+ {index < data.length - 1 ? "," : ""}+ </div>+ ))}+ </div>+ {"]"}+ </>+ );+ }++ if (isObject(data)) {+ return (+ <>+ {"{"}+ <div style={{ paddingLeft: "20px" }}>+ {Object.entries(data).map(([key, value], index, array) => {+ const isClickable = !isObject(value) && !Array.isArray(value); // Only primitive values should be clickable+ const newPath = path ? `${path}.${key}` : key;+ const isSelected = selectedKey === newPath;+ return (+ <div key={key}>+ <Space wrap className="json-key">+ {key}+ </Space>+ {": "}+ <Typography.Text+ className={`prompt-output-result json-value ${+ isClickable && highlightData?.[key] ? "clickable" : ""+ } ${isSelected ? "selected" : ""}`}+ onClick={() => {+ if (isClickable && highlightData?.[key]) {+ handleClick(+ highlightData,+ confidenceData,+ key,+ newPath+ );+ }+ }}+ >+ {renderJson(+ value,+ highlightData?.[key],+ confidenceData?.[key],+ indent + 1,+ newPath+ )}+ </Typography.Text>+ {index < array.length - 1 ? "," : ""}+ </div>+ );+ })}+ </div>+ {"}"}+ </>+ );+ }++ return String(data);+ };+return (<Typography.Paragraph className="prompt-card-display-output font-size-12">- <div>{displayPromptResult(output, true)}</div>+ {parsedOutput && typeof parsedOutput === "object" ? (+ renderJson(parsedOutput, highlightData, confidenceData, 0)+ ) : (+ <TextResult+ enableHighlight={details?.enable_highlight}+ highlightData={highlightData}+ promptId={promptDetails?.prompt_id}+ profileId={profileId}+ confidenceData={confidenceData}+ selectedHighlight={selectedHighlight}+ parsedOutput={parsedOutput}+ onSelectHighlight={handleSelectHighlight}+ />+ )}</Typography.Paragraph>);}+const TextResult = ({+ enableHighlight,+ highlightData,+ promptId,+ profileId,+ confidenceData,+ selectedHighlight,+ parsedOutput,+ onSelectHighlight,+}) => {+ return enableHighlight ? (+ <Typography.Text+ wrap+ onClick={() =>+ onSelectHighlight(highlightData, promptId, profileId, confidenceData)+ }+ className={`prompt-output-result json-value ${+ highlightData ? "clickable" : ""+ } ${selectedHighlight?.highlightedPrompt === promptId ? "selected" : ""}`}+ >+ {parsedOutput}+ </Typography.Text>+ ) : (+ <div>{parsedOutput}</div>+ );+};++TextResult.propTypes = {+ enableHighlight: PropTypes.bool,+ highlightData: PropTypes.any,+ promptId: PropTypes.string,+ profileId: PropTypes.string,+ confidenceData: PropTypes.any,+ selectedHighlight: PropTypes.object,+ parsedOutput: PropTypes.any,+ onSelectHighlight: PropTypes.func.isRequired,+};DisplayPromptResult.propTypes = {output: PropTypes.any,profileId: PropTypes.string,docId: PropTypes.string,promptRunStatus: PropTypes.object,+ handleSelectHighlight: PropTypes.func,+ highlightData: PropTypes.object,+ promptDetails: PropTypes.object,+ confidenceData: PropTypes.object,};export { DisplayPromptResult };diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.css b/frontend/src/components/custom-tools/prompt-card/PromptCard.cssindex 31926462a..3a331819f 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.css+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.css@@ -312,3 +312,26 @@border-style: solid;border-color: #ffb400;}++.json-value {+ color: black;+ cursor: default;+ text-decoration: none;+}++.json-value.clickable {+ color: #0097D8;+ cursor: pointer;+}++.json-value.clickable:hover {+ text-decoration: underline;+}++.json-value.selected {+ color: #5A8300;+}++.prompt-output-result{+ font-size: 12px;+}diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsxindex 8bccb5607..ce0d1c0a8 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx@@ -169,17 +169,16 @@ const PromptCard = memo(const handleSelectHighlight = (highlightData,highlightedPrompt,- highlightedProfile+ highlightedProfile,+ confidenceData) => {- if (- details?.enable_highlight &&- promptDetailsState?.enforce_type !== "json"- ) {+ if (details?.enable_highlight) {updateCustomTool({selectedHighlight: {highlight: highlightData,highlightedPrompt: highlightedPrompt,highlightedProfile: highlightedProfile,+ confidence: confidenceData,},});}diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsxindex 2a62d4600..3f3a25ba9 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx@@ -254,7 +254,7 @@ function PromptCardItems({</Space><Space>{details?.enable_highlight &&- ["json", "table", "record"].includes(enforceType) && (+ ["table", "record"].includes(enforceType) && (<Tagcolor="red"style={{diff --git a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsxindex deebd36d0..b4c2f7257 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx@@ -84,9 +84,7 @@ function PromptOutput({const { generatePromptOutputKey } = usePromptOutput();const isTableExtraction =enforceType === TABLE_ENFORCE_TYPE || enforceType === RECORD_ENFORCE_TYPE;- const noHighlightEnforceType = !["json", "table", "record"].includes(- enforceType- );+ const noHighlightEnforceType = !["table", "record"].includes(enforceType);const tooltipContent = (adapterConf) => (<div>{Object.entries(adapterConf)?.map(([key, value]) => (@@ -144,16 +142,54 @@ function PromptOutput({const promptOutput = promptOutputs[promptOutputKey]?.output;+ let promptOutputData = {};+ if (promptOutputs && Object.keys(promptOutputs)) {+ const promptOutputKey = generatePromptOutputKey(+ promptId,+ docId,+ defaultLlmProfile,+ singlePassExtractMode,+ true+ );+ if (promptOutputs[promptOutputKey] !== undefined) {+ promptOutputData = promptOutputs[promptOutputKey];+ }+ }+return (<><Divider className="prompt-card-divider" />- <div className="prompt-card-result prompt-card-div">- <DisplayPromptResult output={promptOutput} />+ <Space+ wrap+ className={`prompt-card-result prompt-card-div ${+ details?.enable_highlight &&+ noHighlightEnforceType &&+ selectedHighlight?.highlightedPrompt === promptId &&+ selectedHighlight?.highlightedProfile === defaultLlmProfile &&+ "highlighted-prompt-cell"+ }`}+ >+ <DisplayPromptResult+ output={promptOutput}+ highlightData={+ promptOutputData?.highlightData?.[promptDetails.prompt_key]+ }+ handleSelectHighlight={handleSelectHighlight}+ confidenceData={+ promptOutputData?.confidenceData?.[promptDetails.prompt_key]+ }+ /><div className="prompt-profile-run"><CopyPromptOutputBtnisDisabled={isTableExtraction}copyToClipboard={() =>- copyOutputToClipboard(displayPromptResult(promptOutput, true))+ copyOutputToClipboard(+ displayPromptResult(+ promptOutput,+ true,+ promptDetails?.enable_highlight+ )+ )}/><PromptOutputExpandBtn@@ -165,7 +201,7 @@ function PromptOutput({promptRunStatus={promptRunStatus}/></div>- </div>+ </Space></>);}@@ -230,11 +266,13 @@ function PromptOutput({direction="vertical"className="prompt-card-llm-layout"onClick={() => {- handleSelectHighlight(- promptOutputData?.highlightData,- promptId,- profileId- );+ enforceType !== "json" &&+ handleSelectHighlight(+ promptOutputData?.highlightData,+ promptId,+ profileId,+ promptOutputData?.confidenceData+ );}}><div className="llm-info">@@ -396,6 +434,10 @@ function PromptOutput({profileId={profileId}docId={selectedDoc?.document_id}promptRunStatus={promptRunStatus}+ handleSelectHighlight={handleSelectHighlight}+ highlightData={promptOutputData?.highlightData}+ confidenceData={promptOutputData?.confidenceData}+ promptDetails={promptDetails}/><div className="prompt-profile-run"><CopyPromptOutputBtndiff --git a/frontend/src/helpers/GetStaticData.js b/frontend/src/helpers/GetStaticData.jsindex a0af83971..092da7e58 100644--- a/frontend/src/helpers/GetStaticData.js+++ b/frontend/src/helpers/GetStaticData.js@@ -338,12 +338,15 @@ const isJson = (text) => {}};-const displayPromptResult = (output, isFormat = false) => {+const displayPromptResult = (+ output,+ isFormat = false,+ isHighlightEnabled = false+) => {/*output: The data to be displayed or parsedisFormat: A flag indicating whether the output should be formatted*/-let i = 0;let parsedData = output;@@ -365,9 +368,11 @@ const displayPromptResult = (output, isFormat = false) => {// Check if the parsed data is an array or object and formatting is requestedif (Array.isArray(parsedData) || typeof parsedData === "object") {// If formatting is requested, return the JSON string with indentation+ if (isHighlightEnabled) {+ return parsedData;+ }return JSON.stringify(parsedData, null, 4);}-return String(parsedData);};diff --git a/frontend/src/hooks/usePromptOutput.js b/frontend/src/hooks/usePromptOutput.jsindex b96230da7..42de48453 100644--- a/frontend/src/hooks/usePromptOutput.js+++ b/frontend/src/hooks/usePromptOutput.js@@ -120,6 +120,7 @@ const usePromptOutput = () => {timer,coverage: item?.coverage,highlightData: item?.highlight_data,+ confidenceData: item?.confidence_data,};if (item?.is_single_pass_extract && isTokenUsageForSinglePassAdded)