LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
[FEATURE] Changes required to display challenge data in UI (#629)
Deepak Kce89545a26c8
diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.pyindex d0bf9ce33..594063d36 100644--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py@@ -420,6 +420,7 @@ class PromptStudioHelper:else:return PromptStudioHelper._execute_prompts_in_single_pass(doc_path=doc_path,+ doc_name=doc_name,tool_id=tool_id,org_id=org_id,user_id=user_id,@@ -524,6 +525,7 @@ class PromptStudioHelper:@staticmethoddef _execute_prompts_in_single_pass(doc_path,+ doc_name,tool_id,org_id,user_id,@@ -556,6 +558,7 @@ class PromptStudioHelper:tool = prompts[0].tool_idresponse = PromptStudioHelper._fetch_single_pass_response(file_path=doc_path,+ doc_name=doc_name,tool=tool,prompts=prompts,org_id=org_id,@@ -632,7 +635,7 @@ class PromptStudioHelper:document_id=document_id,is_single_pass_extract=is_single_pass,profile_manager_id=profile_manager_id,- context=response["metadata"].get("context"),+ metadata=response["metadata"],)return response@@ -974,6 +977,7 @@ class PromptStudioHelper:def _fetch_single_pass_response(tool: CustomTool,file_path: str,+ doc_name: str,prompts: list[ToolStudioPrompt],org_id: str,user_id: str,@@ -1061,6 +1065,7 @@ class PromptStudioHelper:TSPKeys.TOOL_ID: tool_id,TSPKeys.RUN_ID: run_id,TSPKeys.FILE_HASH: file_hash,+ TSPKeys.FILE_NAME: doc_name,Common.LOG_EVENTS_ID: StateStore.get(Common.LOG_EVENTS_ID),}diff --git a/backend/prompt_studio/prompt_studio_output_manager/migrations/0015_promptstudiooutputmanager_challenge_data.py b/backend/prompt_studio/prompt_studio_output_manager/migrations/0015_promptstudiooutputmanager_challenge_data.pynew file mode 100644index 000000000..0f3987c13--- /dev/null+++ b/backend/prompt_studio/prompt_studio_output_manager/migrations/0015_promptstudiooutputmanager_challenge_data.py@@ -0,0 +1,23 @@+# Generated by Django 4.2.1 on 2024-08-21 00:59++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ (+ "prompt_studio_output_manager",+ "0014_alter_promptstudiooutputmanager_context",+ ),+ ]++ operations = [+ migrations.AddField(+ model_name="promptstudiooutputmanager",+ name="challenge_data",+ field=models.JSONField(+ blank=True, db_comment="Field to store challenge data", null=True+ ),+ ),+ ]diff --git a/backend/prompt_studio/prompt_studio_output_manager/models.py b/backend/prompt_studio/prompt_studio_output_manager/models.pyindex e1f7f5b86..ac7e17178 100644--- a/backend/prompt_studio/prompt_studio_output_manager/models.py+++ b/backend/prompt_studio/prompt_studio_output_manager/models.py@@ -24,6 +24,9 @@ class PromptStudioOutputManager(BaseModel):context = models.TextField(db_comment="Field to store chunks used", editable=True, null=True, blank=True)+ challenge_data = models.JSONField(+ db_comment="Field to store challenge data", editable=True, null=True, blank=True+ )eval_metrics = models.JSONField(db_column="eval_metrics",null=False,diff --git a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.pyindex f72406f43..bec2f535b 100644--- a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py+++ b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py@@ -25,9 +25,9 @@ class OutputManagerHelper:run_id: str,prompts: list[ToolStudioPrompt],outputs: Any,- context: Any,document_id: str,is_single_pass_extract: bool,+ metadata: dict[str, Any],profile_manager_id: Optional[str] = None,) -> None:"""Handles updating prompt outputs in the database.@@ -49,6 +49,7 @@ class OutputManagerHelper:eval_metrics: list[Any],tool: CustomTool,context: str,+ challenge_data: Optional[dict[str, Any]],):try:_, success = PromptStudioOutputManager.objects.get_or_create(@@ -61,6 +62,7 @@ class OutputManagerHelper:"output": output,"eval_metrics": eval_metrics,"context": context,+ "challenge_data": challenge_data,},)@@ -75,11 +77,12 @@ class OutputManagerHelper:f"profile {profile_manager.profile_id}")- args: dict[str, str] = {+ args: dict[str, Any] = {"run_id": run_id,"output": output,"eval_metrics": eval_metrics,"context": context,+ "challenge_data": challenge_data,}PromptStudioOutputManager.objects.filter(document_manager=document_manager,@@ -92,6 +95,9 @@ class OutputManagerHelper:except Exception as e:raise AnswerFetchError(f"Error updating prompt output {e}") from e+ context = metadata.get("context")+ challenge_data = metadata.get("challenge_data")+if not prompts:return # Return early if prompts list is empty@@ -107,6 +113,11 @@ class OutputManagerHelper:if not is_single_pass_extract:context = context.get(prompt.prompt_key)+ if challenge_data:+ challenge_data = challenge_data.get(prompt.prompt_key)++ if challenge_data:+ challenge_data["file_name"] = metadata.get("file_name")output = outputs.get(prompt.prompt_key)if prompt.enforce_type in {"json", "table"}:@@ -121,6 +132,7 @@ class OutputManagerHelper:eval_metrics=eval_metrics,tool=tool,context=context,+ challenge_data=challenge_data,)@staticmethoddiff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsxindex 425b09301..8cceaa993 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx@@ -654,8 +654,9 @@ function PromptCard({promptOutputId: outputResult?.prompt_output_id,profileManager: outputResult?.profile_manager,context: outputResult?.context,+ challengeData: outputResult?.challenge_data,+ tokenUsage: outputResult?.token_usage,output: outputResult?.output,- totalCost: outputResult?.token_usage?.cost_in_dollars,evalMetrics: getEvalMetrics(promptDetails?.evaluate,outputResult?.eval_metrics || []@@ -742,7 +743,11 @@ function PromptCard({item?.profile_manager === defaultLlmProfile &&item?.document_manager === selectedDoc?.document_id);-+ setTokenUsage(`${tokenUsageId}__context`, usage?.context);+ setTokenUsage(+ `${tokenUsageId}__challenge_data`,+ usage?.challenge_data+ );if (usage) {setTokenUsage(tokenUsageId, usage?.token_usage);}diff --git a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsxindex 6f314a640..936f10509 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptOutput.jsx@@ -43,6 +43,13 @@ try {} catch {// The component will remain null of it is not available}+let ChallengeModal;+try {+ ChallengeModal =+ require("../../../plugins/challenge-modal/ChallengeModal").ChallengeModal;+} catch {+ // The component will remain null of it is not available+}function PromptOutput({promptDetails,@@ -164,6 +171,8 @@ function PromptOutput({selectedDoc?.document_id +"__" +profileId;+ const profileData =+ result.find((r) => r?.profileManager === profileId) || {};return (<motion.divkey={profileId}@@ -244,21 +253,24 @@ function PromptOutput({</CheckableTag><div className="llm-info-container"><Tooltip title={tooltipContent(profile?.conf)}>- <InfoCircleOutlined />+ <InfoCircleOutlined className="prompt-card-actions-head" /></Tooltip><Tooltip title="Chunk used"><DatabaseOutlinedonClick={() => {setIsIndexOpen(true);- setOpenIndexProfile(- result.find(- (r) => r?.profileManager === profileId- )?.context- );+ setOpenIndexProfile(profileData?.context);}}className="prompt-card-actions-head"/></Tooltip>+ {ChallengeModal && (+ <ChallengeModal+ challengeData={profileData?.challengeData || {}}+ context={profileData?.context || ""}+ tokenUsage={profileData?.tokenUsage || {}}+ />+ )}{isNotSingleLlmProfile && (<Radiochecked={profileId === selectedLlmProfileId}@@ -298,11 +310,7 @@ function PromptOutput({</Typography.Text>) : (<DisplayPromptResult- output={- result.find(- (r) => r?.profileManager === profileId- )?.output- }+ output={profileData?.output}/>)}</div>@@ -345,12 +353,7 @@ function PromptOutput({isDisabled={enforceType === TABLE_ENFORCE_TYPE}copyToClipboard={() =>copyOutputToClipboard(- displayPromptResult(- result.find(- (r) => r?.profileManager === profileId- )?.output,- true- )+ displayPromptResult(profileData?.output, true))}/>@@ -363,12 +366,7 @@ function PromptOutput({</div></div>{enforceType === TABLE_ENFORCE_TYPE && TableOutput && (- <TableOutput- output={- result.find((r) => r?.profileManager === profileId)- ?.output- }- />+ <TableOutput output={profileData?.output} />)}</></Col>diff --git a/frontend/src/components/custom-tools/tools-main/ToolsMainActionBtns.jsx b/frontend/src/components/custom-tools/tools-main/ToolsMainActionBtns.jsxindex 3f0b61229..3868e1c0c 100644--- a/frontend/src/components/custom-tools/tools-main/ToolsMainActionBtns.jsx+++ b/frontend/src/components/custom-tools/tools-main/ToolsMainActionBtns.jsx@@ -4,6 +4,7 @@ import { useNavigate } from "react-router-dom";import { useCustomToolStore } from "../../../store/custom-tool-store";import usePostHogEvents from "../../../hooks/usePostHogEvents";+import { useTokenUsageStore } from "../../../store/token-usage-store";// Import single pass related componentslet RunSinglePassBtn;@@ -23,15 +24,27 @@ try {// The variable will remain undefined if the component is not available}+let ChallengeModal;+try {+ ChallengeModal =+ require("../../../plugins/challenge-modal/ChallengeModal").ChallengeModal;+} catch {+ // The component will remain null of it is not available+}+function ToolsMainActionBtns() {const {disableLlmOrDocChange,singlePassExtractMode,isSinglePassExtractLoading,isSimplePromptStudio,+ defaultLlmProfile,+ selectedDoc,} = useCustomToolStore();const navigate = useNavigate();const { setPostHogCustomEvent } = usePostHogEvents();+ const { tokenUsage } = useTokenUsageStore();+ const tokenUsageId = `single_pass__${defaultLlmProfile}__${selectedDoc?.document_id}`;const handleOutputAnalyzerBtnClick = () => {navigate("outputAnalyzer");@@ -52,6 +65,13 @@ function ToolsMainActionBtns() {return (<Space>{singlePassExtractMode && RunSinglePassBtn && <RunSinglePassBtn />}+ {singlePassExtractMode && ChallengeModal && (+ <ChallengeModal+ challengeData={tokenUsage?.[`${tokenUsageId}__challenge_data`]}+ context={tokenUsage?.[`${tokenUsageId}__context`]}+ tokenUsage={tokenUsage?.[tokenUsageId]}+ />+ )}<Tooltip title="Output Analyzer"><Buttonicon={<BarChartOutlined />}diff --git a/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js b/frontend/src/components/helpers/custom-tools/CustomToolsHelper.jsindex 529c0f48c..fe3a5f5e6 100644--- a/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js+++ b/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js@@ -55,7 +55,7 @@ function CustomToolsHelper() {data?.single_pass_extraction_mode;selectedDocId = data?.output;updatedCusTool["isSimplePromptStudio"] = false;-+ updatedCusTool["isChallengeEnabled"] = data?.enable_challenge;const reqOpsDocs = {method: "GET",url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-document?tool_id=${data?.tool_id}`,diff --git a/frontend/src/helpers/GetStaticData.js b/frontend/src/helpers/GetStaticData.jsindex 659cbbcd0..f0a0fe0a0 100644--- a/frontend/src/helpers/GetStaticData.js+++ b/frontend/src/helpers/GetStaticData.js@@ -445,8 +445,8 @@ function getLLMModelNamesForProfiles(profiles, adapters) {function getFormattedTotalCost(result, profile) {// Find the relevant object in the result arrayconst value =- result.find((r) => r?.profileManager === profile?.profile_id)?.totalCost ??- 0;+ result.find((r) => r?.profileManager === profile?.profile_id)?.tokenUsage+ ?.cost_in_dollars ?? 0;// Format the value to 5 decimal places or return "0" if the value is zeroreturn value === 0 ? 0 : value.toFixed(5);diff --git a/frontend/src/store/custom-tool-store.js b/frontend/src/store/custom-tool-store.jsindex 988fa111b..b7b8980b9 100644--- a/frontend/src/store/custom-tool-store.js+++ b/frontend/src/store/custom-tool-store.js@@ -19,6 +19,7 @@ const defaultState = {isSimplePromptStudio: false,shareId: null,isPublicSource: false,+ isChallengeEnabled: false,adapters: [],};diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.pyindex c9880c444..241c83b59 100644--- a/prompt-service/src/unstract/prompt_service/main.py+++ b/prompt-service/src/unstract/prompt_service/main.py@@ -94,6 +94,7 @@ def prompt_processor() -> Any:structured_output: dict[str, Any] = {}metadata: dict[str, Any] = {PSKeys.RUN_ID: run_id,+ PSKeys.FILE_NAME: doc_name,PSKeys.CONTEXT: {},}variable_names: list[str] = []@@ -556,6 +557,7 @@ def prompt_processor() -> Any:structured_output=structured_output,logger=app.logger,platform_key=platform_key,+ metadata=metadata,)# Will inline replace the structured output passed.challenge.run()