LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Fix: Coverage count in Prompt studio (#727)
jagadeeswaran-zipstack4109251f280f
diff --git a/backend/prompt_studio/prompt_studio_core_v2/serializers.py b/backend/prompt_studio/prompt_studio_core_v2/serializers.pyindex c8371967c..d013acb37 100644--- a/backend/prompt_studio/prompt_studio_core_v2/serializers.py+++ b/backend/prompt_studio/prompt_studio_core_v2/serializers.py@@ -8,6 +8,9 @@ from file_management.constants import FileInformationKeyfrom prompt_studio.prompt_profile_manager_v2.models import ProfileManagerfrom prompt_studio.prompt_studio_core_v2.constants import ToolStudioKeys as TSKeysfrom prompt_studio.prompt_studio_core_v2.exceptions import DefaultProfileError+from prompt_studio.prompt_studio_output_manager_v2.output_manager_util import (+ OutputManagerUtils,+)from prompt_studio.prompt_studio_v2.models import ToolStudioPromptfrom prompt_studio.prompt_studio_v2.serializers import ToolStudioPromptSerializerfrom rest_framework import serializers@@ -55,11 +58,10 @@ class CustomToolSerializer(IntegrityErrorMixin, AuditSerializer):profile_manager = ProfileManager.get_default_llm_profile(instance)data[TSKeys.DEFAULT_PROFILE] = profile_manager.profile_idexcept DefaultProfileError:- logger.info(+ logger.warning("Default LLM profile doesnt exist for prompt tool %s",str(instance.tool_id),)- try:prompt_instance: ToolStudioPrompt = ToolStudioPrompt.objects.filter(tool_id=data.get(TSKeys.TOOL_ID)).order_by("sequence_number")@@ -69,11 +71,15 @@ class CustomToolSerializer(IntegrityErrorMixin, AuditSerializer):if prompt_instance.count() != 0:for prompt in prompt_instance:prompt_serializer = ToolStudioPromptSerializer(prompt)- output.append(prompt_serializer.data)+ coverage = OutputManagerUtils.get_coverage(+ data.get(TSKeys.TOOL_ID),+ profile_manager.profile_id,+ prompt.prompt_id,+ )+ serialized_data = prompt_serializer.data+ serialized_data["coverage"] = coverage+ output.append(serialized_data)data[TSKeys.PROMPTS] = output- except Exception as e:- logger.error(f"Error occured while appending prompts {e}")- return datadata["created_by_email"] = instance.created_by.emaildiff --git a/backend/prompt_studio/prompt_studio_output_manager/serializers.py b/backend/prompt_studio/prompt_studio_output_manager/serializers.pyindex 14840745a..237a337ec 100644--- a/backend/prompt_studio/prompt_studio_output_manager/serializers.py+++ b/backend/prompt_studio/prompt_studio_output_manager/serializers.py@@ -1,6 +1,7 @@import jsonimport logging+from django.db.models import Countfrom usage.helper import UsageHelperfrom backend.serializers import AuditSerializer@@ -26,6 +27,35 @@ class PromptStudioOutputSerializer(AuditSerializer):)token_usage = {}data["token_usage"] = token_usage+ # Get the coverage for the current tool_id and profile_manager_id+ try:+ # Fetch all relevant outputs for the current tool and profile+ related_outputs = (+ PromptStudioOutputManager.objects.filter(+ tool_id=instance.tool_id,+ profile_manager_id=instance.profile_manager_id,+ prompt_id=instance.prompt_id,+ )+ .values("prompt_id", "profile_manager_id")+ .annotate(document_count=Count("document_manager_id"))+ )++ coverage = {}+ for output in related_outputs:+ prompt_key = str(output["prompt_id"])+ profile_key = str(output["profile_manager_id"])+ coverage[f"coverage_{profile_key}_{prompt_key}"] = output[+ "document_count"+ ]++ data["coverage"] = coverage+ except Exception as e:+ logger.error(+ "Error occurred while fetching "+ f"coverage for tool_id {instance.tool_id} "+ f"and profile_manager_id {instance.profile_manager_id}: {e}"+ )+ data["coverage"] = {}# Convert string to listtry:data["context"] = json.loads(data["context"])diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.pynew file mode 100644index 000000000..152a75bb6--- /dev/null+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py@@ -0,0 +1,46 @@+from django.db.models import Count+from prompt_studio.prompt_studio_output_manager_v2.models import (+ PromptStudioOutputManager,+)+++class OutputManagerUtils:+ @staticmethod+ def get_coverage(+ tool_id: str, profile_manager_id: str, prompt_id: str = None+ ) -> dict[str, int]:+ """+ Method to fetch coverage data for given tool and profile manager.++ Args:+ tool (CustomTool): The tool instance or ID for which coverage is fetched.+ profile_manager_id (str): The ID of the profile manager+ for which coverage is calculated.+ prompt_id (Optional[str]): The ID of the prompt (optional).+ If provided, coverage is fetched for the specific prompt.++ Returns:+ dict[str, int]: A dictionary containing coverage information.+ Keys are formatted as "coverage_<prompt_id>_<profile_manager_id>".+ Values are the count of documents associated with each prompt+ and profile combination.+ """++ prompt_outputs = (+ PromptStudioOutputManager.objects.filter(+ tool_id=tool_id,+ profile_manager_id=profile_manager_id,+ **({"prompt_id": prompt_id} if prompt_id else {}),+ )+ .values("prompt_id", "profile_manager_id")+ .annotate(document_count=Count("document_manager_id"))+ )++ coverage = {}+ for prompt_output in prompt_outputs:+ prompt_key = str(prompt_output["prompt_id"])+ profile_key = str(prompt_output["profile_manager_id"])+ coverage[f"coverage_{prompt_key}_{profile_key}"] = prompt_output[+ "document_count"+ ]+ return coveragediff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.pyindex 0d40826ec..346c582f7 100644--- a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py@@ -6,6 +6,7 @@ from usage_v2.helper import UsageHelperfrom backend.serializers import AuditSerializerfrom .models import PromptStudioOutputManager+from .output_manager_util import OutputManagerUtilslogger = logging.getLogger(__name__)@@ -16,16 +17,34 @@ class PromptStudioOutputSerializer(AuditSerializer):fields = "__all__"def to_representation(self, instance):+data = super().to_representation(instance)try:token_usage = UsageHelper.get_aggregated_token_count(instance.run_id)except Exception as e:- logger.error(+ logger.warning("Error occured while fetching token usage for run_id"f"{instance.run_id}: {e}"+ " | Process continued")token_usage = {}data["token_usage"] = token_usage+ # Get the coverage for the current tool_id and profile_manager_id+ try:+ # Fetch all relevant outputs for the current tool and profile+ coverage = OutputManagerUtils.get_coverage(+ instance.tool_id, instance.profile_manager_id, instance.prompt_id+ )+ data["coverage"] = coverage++ except Exception as e:+ logger.error(+ "Error occurred while fetching "+ f"coverage for tool_id {instance.tool_id} "+ f"and profile_manager_id {instance.profile_manager_id}: {e}"+ " | Process continued"+ )+ data["coverage"] = {}# Convert string to listtry:data["context"] = json.loads(data["context"])diff --git a/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx b/frontend/src/components/custom-tools/document-parser/DocumentParser.jsxindex 0156dc806..b075c7e4e 100644--- a/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx+++ b/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx@@ -30,9 +30,15 @@ function DocumentParser({}) {const [enforceTypeList, setEnforceTypeList] = useState([]);const [updatedPromptsCopy, setUpdatedPromptsCopy] = useState({});+ const [isChallenge, setIsChallenge] = useState(false);const bottomRef = useRef(null);- const { details, isSimplePromptStudio, updateCustomTool, getDropdownItems } =- useCustomToolStore();+ const {+ details,+ isSimplePromptStudio,+ updateCustomTool,+ getDropdownItems,+ isChallengeEnabled,+ } = useCustomToolStore();const { sessionDetails } = useSessionStore();const { setAlertDetails } = useAlertStore();const axiosPrivate = useAxiosPrivate();@@ -45,6 +51,7 @@ function DocumentParser({return { value: outputTypeData[item] };});setEnforceTypeList(dropdownList1);+ setIsChallenge(isChallengeEnabled);return () => {// Set the prompts with updated changes when the component is unmounted@@ -63,6 +70,10 @@ function DocumentParser({};}, []);+ useEffect(() => {+ setIsChallenge(details.enable_challenge);+ }, [details.enable_challenge]);+useEffect(() => {if (scrollToBottom) {// Scroll down to the lastest chat.@@ -169,6 +180,29 @@ function DocumentParser({return outputs;};+ const getPromptCoverageCount = (promptId) => {+ const keys = Object.keys(promptOutputs || {});+ const coverageKey = `coverage_${promptId}`;+ const outputs = {};+ if (!keys?.length) {+ details?.prompts?.forEach((prompt) => {+ if (prompt?.coverage) {+ const key = Object.keys(prompt?.coverage)[0];+ if (key?.startsWith(coverageKey)) {+ outputs[key] = prompt?.coverage[key];+ }+ }+ });+ return outputs;+ }+ keys?.forEach((key) => {+ if (key?.startsWith(coverageKey)) {+ outputs[key] = promptOutputs[key];+ }+ });+ return outputs;+ };+if (!details?.prompts?.length) {if (isSimplePromptStudio && SpsPromptsEmptyState) {return <SpsPromptsEmptyState />;@@ -196,6 +230,8 @@ function DocumentParser({outputs={getPromptOutputs(item?.prompt_id)}enforceTypeList={enforceTypeList}setUpdatedPromptsCopy={setUpdatedPromptsCopy}+ coverageCountData={getPromptCoverageCount(item?.prompt_id)}+ isChallenge={isChallenge}/><div ref={bottomRef} className="doc-parser-pad-bottom" /></div>diff --git a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsxindex b1743a5d6..acc9d700d 100644--- a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx+++ b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx@@ -21,7 +21,6 @@ import {import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader";import { useAlertStore } from "../../../store/alert-store";import { useExceptionHandler } from "../../../hooks/useExceptionHandler";-import { TokenUsage } from "../token-usage/TokenUsage";import { useTokenUsageStore } from "../../../store/token-usage-store";import { ProfileInfoBar } from "../profile-info-bar/ProfileInfoBar";@@ -42,7 +41,7 @@ const outputStatus = {fail: "FAIL",};-const errorTypes = ["null", "undefined", "false"];+const errorTypes = [null, undefined, false];function OutputForDocModal({open,@@ -243,6 +242,7 @@ function OutputForDocModal({status = outputStatus.success;message = displayPromptResult(output?.output, true);}+ const promptTokenUsage = output?.token_usage?.total_tokens;if (output?.output === undefined) {status = outputStatus.yet_to_process;@@ -253,17 +253,7 @@ function OutputForDocModal({const result = {key: item?.document_id,document: item?.document_name,- token_count: !singlePassExtractMode && (- <TokenUsage- tokenUsageId={- promptId +- "__" +- item?.document_id +- "__" +- (selectedProfile || profileManagerId)- }- />- ),+ token_count: !singlePassExtractMode && (promptTokenUsage || "NA"),value: (<>{isLoading ? (diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.css b/frontend/src/components/custom-tools/prompt-card/PromptCard.cssindex 44fa1ab8c..f217c417f 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.css+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.css@@ -77,8 +77,16 @@padding-top: 12px;position: relative;background-color: #fff8e6;+ max-height: 300px;+ overflow-y: scroll;+ scrollbar-color: inherit #fff8e6 !important;+ display: flex;+ flex-grow: 1;}+.prompt-card-result::-webkit-scrollbar-track {+ background-color: #fff8e6 !important;+}.prompt-card-result .ant-typography {margin-bottom: 0px;}diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsxindex 62b53147f..1cda03555 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx@@ -13,7 +13,6 @@ import usePostHogEvents from "../../../hooks/usePostHogEvents";import { PromptCardItems } from "./PromptCardItems";import "./PromptCard.css";import { handleUpdateStatus } from "./constants";-// import { usePromptRunStatusStore } from "../../../store/prompt-run-status-store";const PromptCard = memo(({@@ -26,6 +25,8 @@ const PromptCard = memo(setUpdatedPromptsCopy,handlePromptRunRequest,promptRunStatus,+ coverageCountData,+ isChallenge,}) => {const [promptDetailsState, setPromptDetailsState] = useState({});const [isPromptDetailsStateUpdated, setIsPromptDetailsStateUpdated] =@@ -258,6 +259,8 @@ const PromptCard = memo(handleSpsLoading={handleSpsLoading}promptOutputs={promptOutputs}promptRunStatus={promptRunStatus}+ coverageCountData={coverageCountData}+ isChallenge={isChallenge}/><OutputForDocModalopen={openOutputForDoc}@@ -283,6 +286,8 @@ PromptCard.propTypes = {setUpdatedPromptsCopy: PropTypes.func.isRequired,handlePromptRunRequest: PropTypes.func.isRequired,promptRunStatus: PropTypes.object.isRequired,+ coverageCountData: PropTypes.object.isRequired,+ isChallenge: PropTypes.bool.isRequired,};export { PromptCard };diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsxindex e279f76bd..0e50ff9d0 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx@@ -19,6 +19,7 @@ import { Header } from "./Header";import { OutputForIndex } from "./OutputForIndex";import { PromptOutput } from "./PromptOutput";import { TABLE_ENFORCE_TYPE, RECORD_ENFORCE_TYPE } from "./constants";+import { generateCoverageKey } from "../../../helpers/GetStaticData";let TableExtractionSettingsBtn;try {@@ -50,6 +51,8 @@ function PromptCardItems({handleSpsLoading,promptOutputs,promptRunStatus,+ coverageCountData,+ isChallenge,}) {const {llmProfiles,@@ -60,13 +63,13 @@ function PromptCardItems({isSimplePromptStudio,isPublicSource,adapters,+ defaultLlmProfile,} = useCustomToolStore();const [isEditingPrompt, setIsEditingPrompt] = useState(false);const [isEditingTitle, setIsEditingTitle] = useState(false);const [expandCard, setExpandCard] = useState(true);const [llmProfileDetails, setLlmProfileDetails] = useState([]);const [openIndexProfile, setOpenIndexProfile] = useState(null);- const [coverageCount] = useState(0);const [enabledProfiles, setEnabledProfiles] = useState(llmProfiles.map((profile) => profile.profile_id));@@ -74,6 +77,10 @@ function PromptCardItems({const isNotSingleLlmProfile = llmProfiles.length > 1;const divRef = useRef(null);const [enforceType, setEnforceType] = useState("");+ const coverageKey = generateCoverageKey(+ promptDetails?.prompt_id,+ selectedLlmProfileId || defaultLlmProfile+ );useEffect(() => {if (enforceType !== promptDetails?.enforce_type) {@@ -205,7 +212,7 @@ function PromptCardItems({<SearchOutlined className="font-size-12" />)}<Typography.Link className="font-size-12">- Coverage: {coverageCount} of{" "}+ Coverage: {coverageCountData[coverageKey] || 0} of{" "}{listOfDocs?.length || 0} docs</Typography.Link></Space>@@ -257,6 +264,7 @@ function PromptCardItems({enforceType={enforceType}promptOutputs={promptOutputs}promptRunStatus={promptRunStatus}+ isChallenge={isChallenge}/></Row></Collapse.Panel>@@ -292,6 +300,8 @@ PromptCardItems.propTypes = {handleSpsLoading: PropTypes.func.isRequired,promptOutputs: PropTypes.object.isRequired,promptRunStatus: PropTypes.object.isRequired,+ coverageCountData: PropTypes.object.isRequired,+ isChallenge: PropTypes.bool.isRequired,};export { PromptCardItems };diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCardWrapper.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCardWrapper.jsxindex 2fd4c65b4..5226a2ddf 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptCardWrapper.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptCardWrapper.jsx@@ -15,6 +15,8 @@ const PromptCardWrapper = memo(function PromptCardWrapper({outputs,enforceTypeList,setUpdatedPromptsCopy,+ coverageCountData,+ isChallenge,}) {const { isSimplePromptStudio } = useCustomToolStore();const { handlePromptRunRequest } = usePromptRun();@@ -35,6 +37,8 @@ const PromptCardWrapper = memo(function PromptCardWrapper({setUpdatedPromptsCopy={setUpdatedPromptsCopy}handlePromptRunRequest={handlePromptRunRequest}promptRunStatus={promptRunStatus}+ coverageCountData={coverageCountData}+ isChallenge={isChallenge}/>)}{item.prompt_type === promptType.notes && (@@ -59,6 +63,8 @@ PromptCardWrapper.propTypes = {outputs: PropTypes.object.isRequired,enforceTypeList: PropTypes.array.isRequired,setUpdatedPromptsCopy: PropTypes.func.isRequired,+ coverageCountData: PropTypes.object.isRequired,+ isChallenge: PropTypes.bool.isRequired,};export { PromptCardWrapper };diff --git a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsxindex 7f766d800..e5a181635 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx@@ -66,6 +66,7 @@ function PromptOutput({enforceType,promptOutputs,promptRunStatus,+ isChallenge,}) {const { width: windowWidth } = useWindowDimensions();const componentWidth = windowWidth * 0.4;@@ -238,7 +239,7 @@ function PromptOutput({className="prompt-card-actions-head"/></Tooltip>- {ChallengeModal && (+ {ChallengeModal && isChallenge && (<ChallengeModalchallengeData={promptOutputData?.challengeData || {}@@ -405,6 +406,7 @@ PromptOutput.propTypes = {enforceType: PropTypes.string,promptOutputs: PropTypes.object.isRequired,promptRunStatus: PropTypes.object.isRequired,+ isChallenge: PropTypes.bool,};export { PromptOutput };diff --git a/frontend/src/components/custom-tools/token-usage/TokenUsage.jsx b/frontend/src/components/custom-tools/token-usage/TokenUsage.jsxindex 54b39fb2b..5439724bd 100644--- a/frontend/src/components/custom-tools/token-usage/TokenUsage.jsx+++ b/frontend/src/components/custom-tools/token-usage/TokenUsage.jsx@@ -1,5 +1,6 @@import PropTypes from "prop-types";import { useEffect, useState } from "react";+import { useTokenUsageStore } from "../../../store/token-usage-store";import { formatNumberWithCommas } from "../../../helpers/GetStaticData";diff --git a/frontend/src/helpers/GetStaticData.js b/frontend/src/helpers/GetStaticData.jsindex 85b8b6548..6da20d91b 100644--- a/frontend/src/helpers/GetStaticData.js+++ b/frontend/src/helpers/GetStaticData.js@@ -544,6 +544,10 @@ const generateApiRunStatusId = (docId, profileId) => {return `${docId}__${profileId}`;};+const generateCoverageKey = (promptId, profileId) => {+ return `coverage_${promptId}_${profileId}`;+};+export {CONNECTOR_TYPE_MAP,O_AUTH_PROVIDERS,@@ -593,4 +597,5 @@ export {PROMPT_RUN_TYPES,PROMPT_RUN_API_STATUSES,generateApiRunStatusId,+ generateCoverageKey,};diff --git a/frontend/src/hooks/usePromptOutput.js b/frontend/src/hooks/usePromptOutput.jsindex 28c3a6fba..ad32591e3 100644--- a/frontend/src/hooks/usePromptOutput.js+++ b/frontend/src/hooks/usePromptOutput.js@@ -1,4 +1,5 @@import { useParams } from "react-router-dom";+import { useCustomToolStore } from "../store/custom-tool-store";import { usePromptOutputStore } from "../store/prompt-output-store";import { useSessionStore } from "../store/session-store";@@ -108,7 +109,7 @@ const usePromptOutput = () => {isSinglePass,true);-+ const coverageKey = `coverage_${item?.prompt_id}_${llmProfile}`;outputs[key] = {runId: item?.run_id,promptOutputId: item?.prompt_output_id,@@ -119,6 +120,7 @@ const usePromptOutput = () => {output: item?.output,timer,};+ outputs[coverageKey] = item?.coverage[coverageKey] || 0;if (item?.is_single_pass_extract && isTokenUsageForSinglePassAdded)return;