Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Fix: Coverage count in Prompt studio (#727)

jagadeeswaran-zipstack4109251f280f
diff --git a/backend/prompt_studio/prompt_studio_core_v2/serializers.py b/backend/prompt_studio/prompt_studio_core_v2/serializers.py
index c8371967c..d013acb37 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/serializers.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/serializers.py
@@ -8,6 +8,9 @@ from file_management.constants import FileInformationKey
from prompt_studio.prompt_profile_manager_v2.models import ProfileManager
from prompt_studio.prompt_studio_core_v2.constants import ToolStudioKeys as TSKeys
from prompt_studio.prompt_studio_core_v2.exceptions import DefaultProfileError
+from prompt_studio.prompt_studio_output_manager_v2.output_manager_util import (
+ OutputManagerUtils,
+)
from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt
from prompt_studio.prompt_studio_v2.serializers import ToolStudioPromptSerializer
from rest_framework import serializers
@@ -55,11 +58,10 @@ class CustomToolSerializer(IntegrityErrorMixin, AuditSerializer):
profile_manager = ProfileManager.get_default_llm_profile(instance)
data[TSKeys.DEFAULT_PROFILE] = profile_manager.profile_id
except DefaultProfileError:
- logger.info(
+ logger.warning(
"Default LLM profile doesnt exist for prompt tool %s",
str(instance.tool_id),
)
- try:
prompt_instance: ToolStudioPrompt = ToolStudioPrompt.objects.filter(
tool_id=data.get(TSKeys.TOOL_ID)
).order_by("sequence_number")
@@ -69,11 +71,15 @@ class CustomToolSerializer(IntegrityErrorMixin, AuditSerializer):
if prompt_instance.count() != 0:
for prompt in prompt_instance:
prompt_serializer = ToolStudioPromptSerializer(prompt)
- output.append(prompt_serializer.data)
+ coverage = OutputManagerUtils.get_coverage(
+ data.get(TSKeys.TOOL_ID),
+ profile_manager.profile_id,
+ prompt.prompt_id,
+ )
+ serialized_data = prompt_serializer.data
+ serialized_data["coverage"] = coverage
+ output.append(serialized_data)
data[TSKeys.PROMPTS] = output
- except Exception as e:
- logger.error(f"Error occured while appending prompts {e}")
- return data
data["created_by_email"] = instance.created_by.email
diff --git a/backend/prompt_studio/prompt_studio_output_manager/serializers.py b/backend/prompt_studio/prompt_studio_output_manager/serializers.py
index 14840745a..237a337ec 100644
--- a/backend/prompt_studio/prompt_studio_output_manager/serializers.py
+++ b/backend/prompt_studio/prompt_studio_output_manager/serializers.py
@@ -1,6 +1,7 @@
import json
import logging
+from django.db.models import Count
from usage.helper import UsageHelper
from backend.serializers import AuditSerializer
@@ -26,6 +27,35 @@ class PromptStudioOutputSerializer(AuditSerializer):
)
token_usage = {}
data["token_usage"] = token_usage
+ # Get the coverage for the current tool_id and profile_manager_id
+ try:
+ # Fetch all relevant outputs for the current tool and profile
+ related_outputs = (
+ PromptStudioOutputManager.objects.filter(
+ tool_id=instance.tool_id,
+ profile_manager_id=instance.profile_manager_id,
+ prompt_id=instance.prompt_id,
+ )
+ .values("prompt_id", "profile_manager_id")
+ .annotate(document_count=Count("document_manager_id"))
+ )
+
+ coverage = {}
+ for output in related_outputs:
+ prompt_key = str(output["prompt_id"])
+ profile_key = str(output["profile_manager_id"])
+ coverage[f"coverage_{profile_key}_{prompt_key}"] = output[
+ "document_count"
+ ]
+
+ data["coverage"] = coverage
+ except Exception as e:
+ logger.error(
+ "Error occurred while fetching "
+ f"coverage for tool_id {instance.tool_id} "
+ f"and profile_manager_id {instance.profile_manager_id}: {e}"
+ )
+ data["coverage"] = {}
# Convert string to list
try:
data["context"] = json.loads(data["context"])
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py
new file mode 100644
index 000000000..152a75bb6
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/output_manager_util.py
@@ -0,0 +1,46 @@
+from django.db.models import Count
+from prompt_studio.prompt_studio_output_manager_v2.models import (
+ PromptStudioOutputManager,
+)
+
+
+class OutputManagerUtils:
+ @staticmethod
+ def get_coverage(
+ tool_id: str, profile_manager_id: str, prompt_id: str = None
+ ) -> dict[str, int]:
+ """
+ Method to fetch coverage data for given tool and profile manager.
+
+ Args:
+ tool (CustomTool): The tool instance or ID for which coverage is fetched.
+ profile_manager_id (str): The ID of the profile manager
+ for which coverage is calculated.
+ prompt_id (Optional[str]): The ID of the prompt (optional).
+ If provided, coverage is fetched for the specific prompt.
+
+ Returns:
+ dict[str, int]: A dictionary containing coverage information.
+ Keys are formatted as "coverage_<prompt_id>_<profile_manager_id>".
+ Values are the count of documents associated with each prompt
+ and profile combination.
+ """
+
+ prompt_outputs = (
+ PromptStudioOutputManager.objects.filter(
+ tool_id=tool_id,
+ profile_manager_id=profile_manager_id,
+ **({"prompt_id": prompt_id} if prompt_id else {}),
+ )
+ .values("prompt_id", "profile_manager_id")
+ .annotate(document_count=Count("document_manager_id"))
+ )
+
+ coverage = {}
+ for prompt_output in prompt_outputs:
+ prompt_key = str(prompt_output["prompt_id"])
+ profile_key = str(prompt_output["profile_manager_id"])
+ coverage[f"coverage_{prompt_key}_{profile_key}"] = prompt_output[
+ "document_count"
+ ]
+ return coverage
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py
index 0d40826ec..346c582f7 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/serializers.py
@@ -6,6 +6,7 @@ from usage_v2.helper import UsageHelper
from backend.serializers import AuditSerializer
from .models import PromptStudioOutputManager
+from .output_manager_util import OutputManagerUtils
logger = logging.getLogger(__name__)
@@ -16,16 +17,34 @@ class PromptStudioOutputSerializer(AuditSerializer):
fields = "__all__"
def to_representation(self, instance):
+
data = super().to_representation(instance)
try:
token_usage = UsageHelper.get_aggregated_token_count(instance.run_id)
except Exception as e:
- logger.error(
+ logger.warning(
"Error occured while fetching token usage for run_id"
f"{instance.run_id}: {e}"
+ " | Process continued"
)
token_usage = {}
data["token_usage"] = token_usage
+ # Get the coverage for the current tool_id and profile_manager_id
+ try:
+ # Fetch all relevant outputs for the current tool and profile
+ coverage = OutputManagerUtils.get_coverage(
+ instance.tool_id, instance.profile_manager_id, instance.prompt_id
+ )
+ data["coverage"] = coverage
+
+ except Exception as e:
+ logger.error(
+ "Error occurred while fetching "
+ f"coverage for tool_id {instance.tool_id} "
+ f"and profile_manager_id {instance.profile_manager_id}: {e}"
+ " | Process continued"
+ )
+ data["coverage"] = {}
# Convert string to list
try:
data["context"] = json.loads(data["context"])
diff --git a/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx b/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx
index 0156dc806..b075c7e4e 100644
--- a/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx
+++ b/frontend/src/components/custom-tools/document-parser/DocumentParser.jsx
@@ -30,9 +30,15 @@ function DocumentParser({
}) {
const [enforceTypeList, setEnforceTypeList] = useState([]);
const [updatedPromptsCopy, setUpdatedPromptsCopy] = useState({});
+ const [isChallenge, setIsChallenge] = useState(false);
const bottomRef = useRef(null);
- const { details, isSimplePromptStudio, updateCustomTool, getDropdownItems } =
- useCustomToolStore();
+ const {
+ details,
+ isSimplePromptStudio,
+ updateCustomTool,
+ getDropdownItems,
+ isChallengeEnabled,
+ } = useCustomToolStore();
const { sessionDetails } = useSessionStore();
const { setAlertDetails } = useAlertStore();
const axiosPrivate = useAxiosPrivate();
@@ -45,6 +51,7 @@ function DocumentParser({
return { value: outputTypeData[item] };
});
setEnforceTypeList(dropdownList1);
+ setIsChallenge(isChallengeEnabled);
return () => {
// Set the prompts with updated changes when the component is unmounted
@@ -63,6 +70,10 @@ function DocumentParser({
};
}, []);
+ useEffect(() => {
+ setIsChallenge(details.enable_challenge);
+ }, [details.enable_challenge]);
+
useEffect(() => {
if (scrollToBottom) {
// Scroll down to the lastest chat.
@@ -169,6 +180,29 @@ function DocumentParser({
return outputs;
};
+ const getPromptCoverageCount = (promptId) => {
+ const keys = Object.keys(promptOutputs || {});
+ const coverageKey = `coverage_${promptId}`;
+ const outputs = {};
+ if (!keys?.length) {
+ details?.prompts?.forEach((prompt) => {
+ if (prompt?.coverage) {
+ const key = Object.keys(prompt?.coverage)[0];
+ if (key?.startsWith(coverageKey)) {
+ outputs[key] = prompt?.coverage[key];
+ }
+ }
+ });
+ return outputs;
+ }
+ keys?.forEach((key) => {
+ if (key?.startsWith(coverageKey)) {
+ outputs[key] = promptOutputs[key];
+ }
+ });
+ return outputs;
+ };
+
if (!details?.prompts?.length) {
if (isSimplePromptStudio && SpsPromptsEmptyState) {
return <SpsPromptsEmptyState />;
@@ -196,6 +230,8 @@ function DocumentParser({
outputs={getPromptOutputs(item?.prompt_id)}
enforceTypeList={enforceTypeList}
setUpdatedPromptsCopy={setUpdatedPromptsCopy}
+ coverageCountData={getPromptCoverageCount(item?.prompt_id)}
+ isChallenge={isChallenge}
/>
<div ref={bottomRef} className="doc-parser-pad-bottom" />
</div>
diff --git a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx
index b1743a5d6..acc9d700d 100644
--- a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx
+++ b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx
@@ -21,7 +21,6 @@ import {
import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader";
import { useAlertStore } from "../../../store/alert-store";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
-import { TokenUsage } from "../token-usage/TokenUsage";
import { useTokenUsageStore } from "../../../store/token-usage-store";
import { ProfileInfoBar } from "../profile-info-bar/ProfileInfoBar";
@@ -42,7 +41,7 @@ const outputStatus = {
fail: "FAIL",
};
-const errorTypes = ["null", "undefined", "false"];
+const errorTypes = [null, undefined, false];
function OutputForDocModal({
open,
@@ -243,6 +242,7 @@ function OutputForDocModal({
status = outputStatus.success;
message = displayPromptResult(output?.output, true);
}
+ const promptTokenUsage = output?.token_usage?.total_tokens;
if (output?.output === undefined) {
status = outputStatus.yet_to_process;
@@ -253,17 +253,7 @@ function OutputForDocModal({
const result = {
key: item?.document_id,
document: item?.document_name,
- token_count: !singlePassExtractMode && (
- <TokenUsage
- tokenUsageId={
- promptId +
- "__" +
- item?.document_id +
- "__" +
- (selectedProfile || profileManagerId)
- }
- />
- ),
+ token_count: !singlePassExtractMode && (promptTokenUsage || "NA"),
value: (
<>
{isLoading ? (
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.css b/frontend/src/components/custom-tools/prompt-card/PromptCard.css
index 44fa1ab8c..f217c417f 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.css
+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.css
@@ -77,8 +77,16 @@
padding-top: 12px;
position: relative;
background-color: #fff8e6;
+ max-height: 300px;
+ overflow-y: scroll;
+ scrollbar-color: inherit #fff8e6 !important;
+ display: flex;
+ flex-grow: 1;
}
+.prompt-card-result::-webkit-scrollbar-track {
+ background-color: #fff8e6 !important;
+}
.prompt-card-result .ant-typography {
margin-bottom: 0px;
}
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
index 62b53147f..1cda03555 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
@@ -13,7 +13,6 @@ import usePostHogEvents from "../../../hooks/usePostHogEvents";
import { PromptCardItems } from "./PromptCardItems";
import "./PromptCard.css";
import { handleUpdateStatus } from "./constants";
-// import { usePromptRunStatusStore } from "../../../store/prompt-run-status-store";
const PromptCard = memo(
({
@@ -26,6 +25,8 @@ const PromptCard = memo(
setUpdatedPromptsCopy,
handlePromptRunRequest,
promptRunStatus,
+ coverageCountData,
+ isChallenge,
}) => {
const [promptDetailsState, setPromptDetailsState] = useState({});
const [isPromptDetailsStateUpdated, setIsPromptDetailsStateUpdated] =
@@ -258,6 +259,8 @@ const PromptCard = memo(
handleSpsLoading={handleSpsLoading}
promptOutputs={promptOutputs}
promptRunStatus={promptRunStatus}
+ coverageCountData={coverageCountData}
+ isChallenge={isChallenge}
/>
<OutputForDocModal
open={openOutputForDoc}
@@ -283,6 +286,8 @@ PromptCard.propTypes = {
setUpdatedPromptsCopy: PropTypes.func.isRequired,
handlePromptRunRequest: PropTypes.func.isRequired,
promptRunStatus: PropTypes.object.isRequired,
+ coverageCountData: PropTypes.object.isRequired,
+ isChallenge: PropTypes.bool.isRequired,
};
export { PromptCard };
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx
index e279f76bd..0e50ff9d0 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptCardItems.jsx
@@ -19,6 +19,7 @@ import { Header } from "./Header";
import { OutputForIndex } from "./OutputForIndex";
import { PromptOutput } from "./PromptOutput";
import { TABLE_ENFORCE_TYPE, RECORD_ENFORCE_TYPE } from "./constants";
+import { generateCoverageKey } from "../../../helpers/GetStaticData";
let TableExtractionSettingsBtn;
try {
@@ -50,6 +51,8 @@ function PromptCardItems({
handleSpsLoading,
promptOutputs,
promptRunStatus,
+ coverageCountData,
+ isChallenge,
}) {
const {
llmProfiles,
@@ -60,13 +63,13 @@ function PromptCardItems({
isSimplePromptStudio,
isPublicSource,
adapters,
+ defaultLlmProfile,
} = useCustomToolStore();
const [isEditingPrompt, setIsEditingPrompt] = useState(false);
const [isEditingTitle, setIsEditingTitle] = useState(false);
const [expandCard, setExpandCard] = useState(true);
const [llmProfileDetails, setLlmProfileDetails] = useState([]);
const [openIndexProfile, setOpenIndexProfile] = useState(null);
- const [coverageCount] = useState(0);
const [enabledProfiles, setEnabledProfiles] = useState(
llmProfiles.map((profile) => profile.profile_id)
);
@@ -74,6 +77,10 @@ function PromptCardItems({
const isNotSingleLlmProfile = llmProfiles.length > 1;
const divRef = useRef(null);
const [enforceType, setEnforceType] = useState("");
+ const coverageKey = generateCoverageKey(
+ promptDetails?.prompt_id,
+ selectedLlmProfileId || defaultLlmProfile
+ );
useEffect(() => {
if (enforceType !== promptDetails?.enforce_type) {
@@ -205,7 +212,7 @@ function PromptCardItems({
<SearchOutlined className="font-size-12" />
)}
<Typography.Link className="font-size-12">
- Coverage: {coverageCount} of{" "}
+ Coverage: {coverageCountData[coverageKey] || 0} of{" "}
{listOfDocs?.length || 0} docs
</Typography.Link>
</Space>
@@ -257,6 +264,7 @@ function PromptCardItems({
enforceType={enforceType}
promptOutputs={promptOutputs}
promptRunStatus={promptRunStatus}
+ isChallenge={isChallenge}
/>
</Row>
</Collapse.Panel>
@@ -292,6 +300,8 @@ PromptCardItems.propTypes = {
handleSpsLoading: PropTypes.func.isRequired,
promptOutputs: PropTypes.object.isRequired,
promptRunStatus: PropTypes.object.isRequired,
+ coverageCountData: PropTypes.object.isRequired,
+ isChallenge: PropTypes.bool.isRequired,
};
export { PromptCardItems };
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCardWrapper.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCardWrapper.jsx
index 2fd4c65b4..5226a2ddf 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptCardWrapper.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptCardWrapper.jsx
@@ -15,6 +15,8 @@ const PromptCardWrapper = memo(function PromptCardWrapper({
outputs,
enforceTypeList,
setUpdatedPromptsCopy,
+ coverageCountData,
+ isChallenge,
}) {
const { isSimplePromptStudio } = useCustomToolStore();
const { handlePromptRunRequest } = usePromptRun();
@@ -35,6 +37,8 @@ const PromptCardWrapper = memo(function PromptCardWrapper({
setUpdatedPromptsCopy={setUpdatedPromptsCopy}
handlePromptRunRequest={handlePromptRunRequest}
promptRunStatus={promptRunStatus}
+ coverageCountData={coverageCountData}
+ isChallenge={isChallenge}
/>
)}
{item.prompt_type === promptType.notes && (
@@ -59,6 +63,8 @@ PromptCardWrapper.propTypes = {
outputs: PropTypes.object.isRequired,
enforceTypeList: PropTypes.array.isRequired,
setUpdatedPromptsCopy: PropTypes.func.isRequired,
+ coverageCountData: PropTypes.object.isRequired,
+ isChallenge: PropTypes.bool.isRequired,
};
export { PromptCardWrapper };
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
index 7f766d800..e5a181635 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
@@ -66,6 +66,7 @@ function PromptOutput({
enforceType,
promptOutputs,
promptRunStatus,
+ isChallenge,
}) {
const { width: windowWidth } = useWindowDimensions();
const componentWidth = windowWidth * 0.4;
@@ -238,7 +239,7 @@ function PromptOutput({
className="prompt-card-actions-head"
/>
</Tooltip>
- {ChallengeModal && (
+ {ChallengeModal && isChallenge && (
<ChallengeModal
challengeData={
promptOutputData?.challengeData || {}
@@ -405,6 +406,7 @@ PromptOutput.propTypes = {
enforceType: PropTypes.string,
promptOutputs: PropTypes.object.isRequired,
promptRunStatus: PropTypes.object.isRequired,
+ isChallenge: PropTypes.bool,
};
export { PromptOutput };
diff --git a/frontend/src/components/custom-tools/token-usage/TokenUsage.jsx b/frontend/src/components/custom-tools/token-usage/TokenUsage.jsx
index 54b39fb2b..5439724bd 100644
--- a/frontend/src/components/custom-tools/token-usage/TokenUsage.jsx
+++ b/frontend/src/components/custom-tools/token-usage/TokenUsage.jsx
@@ -1,5 +1,6 @@
import PropTypes from "prop-types";
import { useEffect, useState } from "react";
+
import { useTokenUsageStore } from "../../../store/token-usage-store";
import { formatNumberWithCommas } from "../../../helpers/GetStaticData";
diff --git a/frontend/src/helpers/GetStaticData.js b/frontend/src/helpers/GetStaticData.js
index 85b8b6548..6da20d91b 100644
--- a/frontend/src/helpers/GetStaticData.js
+++ b/frontend/src/helpers/GetStaticData.js
@@ -544,6 +544,10 @@ const generateApiRunStatusId = (docId, profileId) => {
return `${docId}__${profileId}`;
};
+const generateCoverageKey = (promptId, profileId) => {
+ return `coverage_${promptId}_${profileId}`;
+};
+
export {
CONNECTOR_TYPE_MAP,
O_AUTH_PROVIDERS,
@@ -593,4 +597,5 @@ export {
PROMPT_RUN_TYPES,
PROMPT_RUN_API_STATUSES,
generateApiRunStatusId,
+ generateCoverageKey,
};
diff --git a/frontend/src/hooks/usePromptOutput.js b/frontend/src/hooks/usePromptOutput.js
index 28c3a6fba..ad32591e3 100644
--- a/frontend/src/hooks/usePromptOutput.js
+++ b/frontend/src/hooks/usePromptOutput.js
@@ -1,4 +1,5 @@
import { useParams } from "react-router-dom";
+
import { useCustomToolStore } from "../store/custom-tool-store";
import { usePromptOutputStore } from "../store/prompt-output-store";
import { useSessionStore } from "../store/session-store";
@@ -108,7 +109,7 @@ const usePromptOutput = () => {
isSinglePass,
true
);
-
+ const coverageKey = `coverage_${item?.prompt_id}_${llmProfile}`;
outputs[key] = {
runId: item?.run_id,
promptOutputId: item?.prompt_output_id,
@@ -119,6 +120,7 @@ const usePromptOutput = () => {
output: item?.output,
timer,
};
+ outputs[coverageKey] = item?.coverage[coverageKey] || 0;
if (item?.is_single_pass_extract && isTokenUsageForSinglePassAdded)
return;