Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[FEATURE] Changes required to display challenge data in UI (#629)

Deepak Kce89545a26c8
diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
index d0bf9ce33..594063d36 100644
--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
@@ -420,6 +420,7 @@ class PromptStudioHelper:
else:
return PromptStudioHelper._execute_prompts_in_single_pass(
doc_path=doc_path,
+ doc_name=doc_name,
tool_id=tool_id,
org_id=org_id,
user_id=user_id,
@@ -524,6 +525,7 @@ class PromptStudioHelper:
@staticmethod
def _execute_prompts_in_single_pass(
doc_path,
+ doc_name,
tool_id,
org_id,
user_id,
@@ -556,6 +558,7 @@ class PromptStudioHelper:
tool = prompts[0].tool_id
response = PromptStudioHelper._fetch_single_pass_response(
file_path=doc_path,
+ doc_name=doc_name,
tool=tool,
prompts=prompts,
org_id=org_id,
@@ -632,7 +635,7 @@ class PromptStudioHelper:
document_id=document_id,
is_single_pass_extract=is_single_pass,
profile_manager_id=profile_manager_id,
- context=response["metadata"].get("context"),
+ metadata=response["metadata"],
)
return response
@@ -974,6 +977,7 @@ class PromptStudioHelper:
def _fetch_single_pass_response(
tool: CustomTool,
file_path: str,
+ doc_name: str,
prompts: list[ToolStudioPrompt],
org_id: str,
user_id: str,
@@ -1061,6 +1065,7 @@ class PromptStudioHelper:
TSPKeys.TOOL_ID: tool_id,
TSPKeys.RUN_ID: run_id,
TSPKeys.FILE_HASH: file_hash,
+ TSPKeys.FILE_NAME: doc_name,
Common.LOG_EVENTS_ID: StateStore.get(Common.LOG_EVENTS_ID),
}
diff --git a/backend/prompt_studio/prompt_studio_output_manager/migrations/0015_promptstudiooutputmanager_challenge_data.py b/backend/prompt_studio/prompt_studio_output_manager/migrations/0015_promptstudiooutputmanager_challenge_data.py
new file mode 100644
index 000000000..0f3987c13
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_output_manager/migrations/0015_promptstudiooutputmanager_challenge_data.py
@@ -0,0 +1,23 @@
+# Generated by Django 4.2.1 on 2024-08-21 00:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ (
+ "prompt_studio_output_manager",
+ "0014_alter_promptstudiooutputmanager_context",
+ ),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="promptstudiooutputmanager",
+ name="challenge_data",
+ field=models.JSONField(
+ blank=True, db_comment="Field to store challenge data", null=True
+ ),
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_studio_output_manager/models.py b/backend/prompt_studio/prompt_studio_output_manager/models.py
index e1f7f5b86..ac7e17178 100644
--- a/backend/prompt_studio/prompt_studio_output_manager/models.py
+++ b/backend/prompt_studio/prompt_studio_output_manager/models.py
@@ -24,6 +24,9 @@ class PromptStudioOutputManager(BaseModel):
context = models.TextField(
db_comment="Field to store chunks used", editable=True, null=True, blank=True
)
+ challenge_data = models.JSONField(
+ db_comment="Field to store challenge data", editable=True, null=True, blank=True
+ )
eval_metrics = models.JSONField(
db_column="eval_metrics",
null=False,
diff --git a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
index f72406f43..bec2f535b 100644
--- a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
+++ b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
@@ -25,9 +25,9 @@ class OutputManagerHelper:
run_id: str,
prompts: list[ToolStudioPrompt],
outputs: Any,
- context: Any,
document_id: str,
is_single_pass_extract: bool,
+ metadata: dict[str, Any],
profile_manager_id: Optional[str] = None,
) -> None:
"""Handles updating prompt outputs in the database.
@@ -49,6 +49,7 @@ class OutputManagerHelper:
eval_metrics: list[Any],
tool: CustomTool,
context: str,
+ challenge_data: Optional[dict[str, Any]],
):
try:
_, success = PromptStudioOutputManager.objects.get_or_create(
@@ -61,6 +62,7 @@ class OutputManagerHelper:
"output": output,
"eval_metrics": eval_metrics,
"context": context,
+ "challenge_data": challenge_data,
},
)
@@ -75,11 +77,12 @@ class OutputManagerHelper:
f"profile {profile_manager.profile_id}"
)
- args: dict[str, str] = {
+ args: dict[str, Any] = {
"run_id": run_id,
"output": output,
"eval_metrics": eval_metrics,
"context": context,
+ "challenge_data": challenge_data,
}
PromptStudioOutputManager.objects.filter(
document_manager=document_manager,
@@ -92,6 +95,9 @@ class OutputManagerHelper:
except Exception as e:
raise AnswerFetchError(f"Error updating prompt output {e}") from e
+ context = metadata.get("context")
+ challenge_data = metadata.get("challenge_data")
+
if not prompts:
return # Return early if prompts list is empty
@@ -107,6 +113,11 @@ class OutputManagerHelper:
if not is_single_pass_extract:
context = context.get(prompt.prompt_key)
+ if challenge_data:
+ challenge_data = challenge_data.get(prompt.prompt_key)
+
+ if challenge_data:
+ challenge_data["file_name"] = metadata.get("file_name")
output = outputs.get(prompt.prompt_key)
if prompt.enforce_type in {"json", "table"}:
@@ -121,6 +132,7 @@ class OutputManagerHelper:
eval_metrics=eval_metrics,
tool=tool,
context=context,
+ challenge_data=challenge_data,
)
@staticmethod
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
index 425b09301..8cceaa993 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
@@ -654,8 +654,9 @@ function PromptCard({
promptOutputId: outputResult?.prompt_output_id,
profileManager: outputResult?.profile_manager,
context: outputResult?.context,
+ challengeData: outputResult?.challenge_data,
+ tokenUsage: outputResult?.token_usage,
output: outputResult?.output,
- totalCost: outputResult?.token_usage?.cost_in_dollars,
evalMetrics: getEvalMetrics(
promptDetails?.evaluate,
outputResult?.eval_metrics || []
@@ -742,7 +743,11 @@ function PromptCard({
item?.profile_manager === defaultLlmProfile &&
item?.document_manager === selectedDoc?.document_id
);
-
+ setTokenUsage(`${tokenUsageId}__context`, usage?.context);
+ setTokenUsage(
+ `${tokenUsageId}__challenge_data`,
+ usage?.challenge_data
+ );
if (usage) {
setTokenUsage(tokenUsageId, usage?.token_usage);
}
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
index 6f314a640..936f10509 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx
@@ -43,6 +43,13 @@ try {
} catch {
// The component will remain null of it is not available
}
+let ChallengeModal;
+try {
+ ChallengeModal =
+ require("../../../plugins/challenge-modal/ChallengeModal").ChallengeModal;
+} catch {
+ // The component will remain null of it is not available
+}
function PromptOutput({
promptDetails,
@@ -164,6 +171,8 @@ function PromptOutput({
selectedDoc?.document_id +
"__" +
profileId;
+ const profileData =
+ result.find((r) => r?.profileManager === profileId) || {};
return (
<motion.div
key={profileId}
@@ -244,21 +253,24 @@ function PromptOutput({
</CheckableTag>
<div className="llm-info-container">
<Tooltip title={tooltipContent(profile?.conf)}>
- <InfoCircleOutlined />
+ <InfoCircleOutlined className="prompt-card-actions-head" />
</Tooltip>
<Tooltip title="Chunk used">
<DatabaseOutlined
onClick={() => {
setIsIndexOpen(true);
- setOpenIndexProfile(
- result.find(
- (r) => r?.profileManager === profileId
- )?.context
- );
+ setOpenIndexProfile(profileData?.context);
}}
className="prompt-card-actions-head"
/>
</Tooltip>
+ {ChallengeModal && (
+ <ChallengeModal
+ challengeData={profileData?.challengeData || {}}
+ context={profileData?.context || ""}
+ tokenUsage={profileData?.tokenUsage || {}}
+ />
+ )}
{isNotSingleLlmProfile && (
<Radio
checked={profileId === selectedLlmProfileId}
@@ -298,11 +310,7 @@ function PromptOutput({
</Typography.Text>
) : (
<DisplayPromptResult
- output={
- result.find(
- (r) => r?.profileManager === profileId
- )?.output
- }
+ output={profileData?.output}
/>
)}
</div>
@@ -345,12 +353,7 @@ function PromptOutput({
isDisabled={enforceType === TABLE_ENFORCE_TYPE}
copyToClipboard={() =>
copyOutputToClipboard(
- displayPromptResult(
- result.find(
- (r) => r?.profileManager === profileId
- )?.output,
- true
- )
+ displayPromptResult(profileData?.output, true)
)
}
/>
@@ -363,12 +366,7 @@ function PromptOutput({
</div>
</div>
{enforceType === TABLE_ENFORCE_TYPE && TableOutput && (
- <TableOutput
- output={
- result.find((r) => r?.profileManager === profileId)
- ?.output
- }
- />
+ <TableOutput output={profileData?.output} />
)}
</>
</Col>
diff --git a/frontend/src/components/custom-tools/tools-main/ToolsMainActionBtns.jsx b/frontend/src/components/custom-tools/tools-main/ToolsMainActionBtns.jsx
index 3f0b61229..3868e1c0c 100644
--- a/frontend/src/components/custom-tools/tools-main/ToolsMainActionBtns.jsx
+++ b/frontend/src/components/custom-tools/tools-main/ToolsMainActionBtns.jsx
@@ -4,6 +4,7 @@ import { useNavigate } from "react-router-dom";
import { useCustomToolStore } from "../../../store/custom-tool-store";
import usePostHogEvents from "../../../hooks/usePostHogEvents";
+import { useTokenUsageStore } from "../../../store/token-usage-store";
// Import single pass related components
let RunSinglePassBtn;
@@ -23,15 +24,27 @@ try {
// The variable will remain undefined if the component is not available
}
+let ChallengeModal;
+try {
+ ChallengeModal =
+ require("../../../plugins/challenge-modal/ChallengeModal").ChallengeModal;
+} catch {
+ // The component will remain null of it is not available
+}
+
function ToolsMainActionBtns() {
const {
disableLlmOrDocChange,
singlePassExtractMode,
isSinglePassExtractLoading,
isSimplePromptStudio,
+ defaultLlmProfile,
+ selectedDoc,
} = useCustomToolStore();
const navigate = useNavigate();
const { setPostHogCustomEvent } = usePostHogEvents();
+ const { tokenUsage } = useTokenUsageStore();
+ const tokenUsageId = `single_pass__${defaultLlmProfile}__${selectedDoc?.document_id}`;
const handleOutputAnalyzerBtnClick = () => {
navigate("outputAnalyzer");
@@ -52,6 +65,13 @@ function ToolsMainActionBtns() {
return (
<Space>
{singlePassExtractMode && RunSinglePassBtn && <RunSinglePassBtn />}
+ {singlePassExtractMode && ChallengeModal && (
+ <ChallengeModal
+ challengeData={tokenUsage?.[`${tokenUsageId}__challenge_data`]}
+ context={tokenUsage?.[`${tokenUsageId}__context`]}
+ tokenUsage={tokenUsage?.[tokenUsageId]}
+ />
+ )}
<Tooltip title="Output Analyzer">
<Button
icon={<BarChartOutlined />}
diff --git a/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js b/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js
index 529c0f48c..fe3a5f5e6 100644
--- a/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js
+++ b/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js
@@ -55,7 +55,7 @@ function CustomToolsHelper() {
data?.single_pass_extraction_mode;
selectedDocId = data?.output;
updatedCusTool["isSimplePromptStudio"] = false;
-
+ updatedCusTool["isChallengeEnabled"] = data?.enable_challenge;
const reqOpsDocs = {
method: "GET",
url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-document?tool_id=${data?.tool_id}`,
diff --git a/frontend/src/helpers/GetStaticData.js b/frontend/src/helpers/GetStaticData.js
index 659cbbcd0..f0a0fe0a0 100644
--- a/frontend/src/helpers/GetStaticData.js
+++ b/frontend/src/helpers/GetStaticData.js
@@ -445,8 +445,8 @@ function getLLMModelNamesForProfiles(profiles, adapters) {
function getFormattedTotalCost(result, profile) {
// Find the relevant object in the result array
const value =
- result.find((r) => r?.profileManager === profile?.profile_id)?.totalCost ??
- 0;
+ result.find((r) => r?.profileManager === profile?.profile_id)?.tokenUsage
+ ?.cost_in_dollars ?? 0;
// Format the value to 5 decimal places or return "0" if the value is zero
return value === 0 ? 0 : value.toFixed(5);
diff --git a/frontend/src/store/custom-tool-store.js b/frontend/src/store/custom-tool-store.js
index 988fa111b..b7b8980b9 100644
--- a/frontend/src/store/custom-tool-store.js
+++ b/frontend/src/store/custom-tool-store.js
@@ -19,6 +19,7 @@ const defaultState = {
isSimplePromptStudio: false,
shareId: null,
isPublicSource: false,
+ isChallengeEnabled: false,
adapters: [],
};
diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.py
index c9880c444..241c83b59 100644
--- a/prompt-service/src/unstract/prompt_service/main.py
+++ b/prompt-service/src/unstract/prompt_service/main.py
@@ -94,6 +94,7 @@ def prompt_processor() -> Any:
structured_output: dict[str, Any] = {}
metadata: dict[str, Any] = {
PSKeys.RUN_ID: run_id,
+ PSKeys.FILE_NAME: doc_name,
PSKeys.CONTEXT: {},
}
variable_names: list[str] = []
@@ -556,6 +557,7 @@ def prompt_processor() -> Any:
structured_output=structured_output,
logger=app.logger,
platform_key=platform_key,
+ metadata=metadata,
)
# Will inline replace the structured output passed.
challenge.run()