LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Fixed processing of notes in prompt studio (#172)
Deepak K87999330bc67
diff --git a/backend/prompt_studio/prompt_studio_core/constants.py b/backend/prompt_studio/prompt_studio_core/constants.pyindex 7c2b5db52..ac998aa92 100644--- a/backend/prompt_studio/prompt_studio_core/constants.py+++ b/backend/prompt_studio/prompt_studio_core/constants.py@@ -82,6 +82,7 @@ class ToolStudioPromptKeys:CHALLENGE_LLM = "challenge_llm"SINGLE_PASS_EXTRACTION_MODE = "single_pass_extraction_mode"SINGLE_PASS_EXTRACTION = "single_pass_extraction"+ NOTES = "NOTES"class LogLevels:diff --git a/backend/prompt_studio/prompt_studio_core/exceptions.py b/backend/prompt_studio/prompt_studio_core/exceptions.pyindex f7210f309..7004b42b4 100644--- a/backend/prompt_studio/prompt_studio_core/exceptions.py+++ b/backend/prompt_studio/prompt_studio_core/exceptions.py@@ -59,3 +59,8 @@ class NoPromptsFound(APIException):class PermissionError(APIException):status_code = 403default_detail = "You do not have permission to perform this action."+++class EmptyPromptError(APIException):+ status_code = 422+ default_detail = "Prompt(s) cannot be empty"diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.pyindex 16a930d41..958590b29 100644--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py@@ -17,6 +17,7 @@ from prompt_studio.prompt_studio_core.constants import (from prompt_studio.prompt_studio_core.exceptions import (AnswerFetchError,DefaultProfileError,+ EmptyPromptError,IndexingError,NoPromptsFound,PermissionError,@@ -373,8 +374,15 @@ class PromptStudioHelper:return responseelse:prompts = PromptStudioHelper.fetch_prompt_from_tool(tool_id)+ prompts = [+ prompt+ for prompt in prompts+ if prompt.prompt_type != TSPKeys.NOTES+ ]if not prompts:- logger.error(f"[{tool_id or 'NA'}] No prompts found id: {id}")+ logger.error(+ f"[{tool_id or 'NA'}] No prompts found for id: {id}"+ )raise NoPromptsFound()logger.info(f"[{tool_id}] Executing prompts in single pass")@@ -705,6 +713,8 @@ class PromptStudioHelper:tool_settings[TSPKeys.CHALLENGE_LLM] = challenge_llmfor prompt in prompts:+ if not prompt.prompt:+ raise EmptyPromptError()output: dict[str, Any] = {}output[TSPKeys.PROMPT] = prompt.promptoutput[TSPKeys.ACTIVE] = prompt.activediff --git a/backend/prompt_studio/prompt_studio_output_manager/constants.py b/backend/prompt_studio/prompt_studio_output_manager/constants.pyindex 1caf8a66f..a9c046aae 100644--- a/backend/prompt_studio/prompt_studio_output_manager/constants.py+++ b/backend/prompt_studio/prompt_studio_output_manager/constants.py@@ -4,3 +4,4 @@ class PromptStudioOutputManagerKeys:PROFILE_MANAGER = "profile_manager"DOCUMENT_MANAGER = "document_manager"IS_SINGLE_PASS_EXTRACT = "is_single_pass_extract"+ NOTES = "NOTES"diff --git a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.pyindex 170a7eecc..24e1f022e 100644--- a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py+++ b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py@@ -1,8 +1,12 @@import jsonfrom typing import Any+from prompt_studio.prompt_profile_manager.models import ProfileManagerfrom prompt_studio.prompt_studio.models import ToolStudioPromptfrom prompt_studio.prompt_studio_document_manager.models import DocumentManager+from prompt_studio.prompt_studio_output_manager.constants import (+ PromptStudioOutputManagerKeys as PSOMKeys,+)from prompt_studio.prompt_studio_output_manager.models import (PromptStudioOutputManager,)@@ -31,10 +35,15 @@ class OutputManagerHelper:tool = prompts[0].tool_iddocument_manager = DocumentManager.objects.get(pk=document_id)-+ default_profile = ProfileManager.get_default_llm_profile(tool=tool)# Iterate through each prompt in the listfor prompt in prompts:- profile_manager = prompt.profile_manager+ if prompt.prompt_type == PSOMKeys.NOTES:+ continue+ if is_single_pass_extract:+ profile_manager = default_profile+ else:+ profile_manager = prompt.profile_manageroutput = json.dumps(outputs.get(prompt.prompt_key))eval_metrics = outputs.get(f"{prompt.prompt_key}__evaluation", [])diff --git a/backend/prompt_studio/prompt_studio_registry/constants.py b/backend/prompt_studio/prompt_studio_registry/constants.pyindex 73eee1e9f..745c59975 100644--- a/backend/prompt_studio/prompt_studio_registry/constants.py+++ b/backend/prompt_studio/prompt_studio_registry/constants.py@@ -89,6 +89,7 @@ class JsonSchemaKey:EMBEDDING_SUFFIX = "embedding_suffix"FUNCTION_NAME = "function_name"PROMPT_REGISTRY_ID = "prompt_registry_id"+ NOTES = "NOTES"class SpecKey:diff --git a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.pyindex fd33d8da9..c9f328f70 100644--- a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py+++ b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py@@ -115,10 +115,10 @@ class PromptStudioRegistryHelper:PromptStudioRegistryHelper.frame_properties(tool=custom_tool))spec: Spec = PromptStudioRegistryHelper.frame_spec(tool=custom_tool)- prompts: list[ToolStudioPrompt] = (- PromptStudioHelper.fetch_prompt_from_tool(- tool_id=custom_tool.tool_id- )+ prompts: list[+ ToolStudioPrompt+ ] = PromptStudioHelper.fetch_prompt_from_tool(+ tool_id=custom_tool.tool_id)metadata = PromptStudioRegistryHelper.frame_export_json(tool=custom_tool, prompts=prompts@@ -183,6 +183,8 @@ class PromptStudioRegistryHelper:default_llm_profile = ProfileManager.get_default_llm_profile(tool)for prompt in prompts:+ if prompt.prompt_type == JsonSchemaKey.NOTES:+ continueif not prompt.profile_manager:prompt.profile_manager = default_llm_profile@@ -193,9 +195,9 @@ class PromptStudioRegistryHelper:adapter_id = str(prompt.profile_manager.embedding_model.adapter_id)embedding_suffix = adapter_id.split("|")[0]- output[JsonSchemaKey.ASSERTION_FAILURE_PROMPT] = (- prompt.assertion_failure_prompt- )+ output[+ JsonSchemaKey.ASSERTION_FAILURE_PROMPT+ ] = prompt.assertion_failure_promptoutput[JsonSchemaKey.ASSERT_PROMPT] = prompt.assert_promptoutput[JsonSchemaKey.IS_ASSERT] = prompt.is_assertoutput[JsonSchemaKey.PROMPT] = prompt.prompt@@ -204,21 +206,21 @@ class PromptStudioRegistryHelper:output[JsonSchemaKey.VECTOR_DB] = vector_dboutput[JsonSchemaKey.EMBEDDING] = embedding_modeloutput[JsonSchemaKey.X2TEXT_ADAPTER] = x2text- output[JsonSchemaKey.CHUNK_OVERLAP] = (- prompt.profile_manager.chunk_overlap- )+ output[+ JsonSchemaKey.CHUNK_OVERLAP+ ] = prompt.profile_manager.chunk_overlapoutput[JsonSchemaKey.LLM] = llmoutput[JsonSchemaKey.PREAMBLE] = tool.preambleoutput[JsonSchemaKey.POSTAMBLE] = tool.postambleoutput[JsonSchemaKey.GRAMMAR] = grammar_listoutput[JsonSchemaKey.TYPE] = prompt.enforce_typeoutput[JsonSchemaKey.NAME] = prompt.prompt_key- output[JsonSchemaKey.RETRIEVAL_STRATEGY] = (- prompt.profile_manager.retrieval_strategy- )- output[JsonSchemaKey.SIMILARITY_TOP_K] = (- prompt.profile_manager.similarity_top_k- )+ output[+ JsonSchemaKey.RETRIEVAL_STRATEGY+ ] = prompt.profile_manager.retrieval_strategy+ output[+ JsonSchemaKey.SIMILARITY_TOP_K+ ] = prompt.profile_manager.similarity_top_koutput[JsonSchemaKey.SECTION] = prompt.profile_manager.sectionoutput[JsonSchemaKey.REINDEX] = prompt.profile_manager.reindexoutput[JsonSchemaKey.EMBEDDING_SUFFIX] = embedding_suffixdiff --git a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsxindex c43ba4da9..507bf6704 100644--- a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx+++ b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx@@ -21,8 +21,12 @@ import { useExceptionHandler } from "../../../hooks/useExceptionHandler";function CombinedOutput({ docId, setFilledFields }) {const [combinedOutput, setCombinedOutput] = useState({});const [isOutputLoading, setIsOutputLoading] = useState(false);- const { details, singlePassExtractMode, isSinglePassExtractLoading } =- useCustomToolStore();+ const {+ details,+ defaultLlmProfile,+ singlePassExtractMode,+ isSinglePassExtractLoading,+ } = useCustomToolStore();const { sessionDetails } = useSessionStore();const { setAlertDetails } = useAlertStore();const axiosPrivate = useAxiosPrivate();@@ -47,10 +51,14 @@ function CombinedOutput({ docId, setFilledFields }) {}output[item?.prompt_key] = "";+ let profileManager = item?.profile_manager;+ if (singlePassExtractMode) {+ profileManager = defaultLlmProfile;+ }const outputDetails = data.find((outputValue) =>outputValue?.prompt_id === item?.prompt_id &&- outputValue?.profile_manager === item?.profile_manager+ outputValue?.profile_manager === profileManager);if (!outputDetails) {diff --git a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsxindex 257fcf0a0..962de7c7d 100644--- a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx+++ b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx@@ -52,6 +52,8 @@ function OutputForDocModal({details,listOfDocs,selectedDoc,+ defaultLlmProfile,+ disableLlmOrDocChange,singlePassExtractMode,isSinglePassExtractLoading,} = useCustomToolStore();@@ -62,7 +64,7 @@ function OutputForDocModal({const { handleException } = useExceptionHandler();useEffect(() => {- if (!open || isSinglePassExtractLoading) {+ if (!open) {return;}handleGetOutputForDocs();@@ -135,13 +137,18 @@ function OutputForDocModal({};const handleGetOutputForDocs = () => {- if (!profileManagerId) {+ let profile = profileManagerId;+ if (singlePassExtractMode) {+ profile = defaultLlmProfile;+ }++ if (!profile) {setRows([]);return;}const requestOptions = {method: "GET",- url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&prompt_id=${promptId}&profile_manager=${profileManagerId}&is_single_pass_extract=${singlePassExtractMode}`,+ url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&prompt_id=${promptId}&profile_manager=${profile}&is_single_pass_extract=${singlePassExtractMode}`,headers: {"X-CSRFToken": sessionDetails?.csrfToken,},@@ -232,7 +239,13 @@ function OutputForDocModal({</div><div className="output-doc-gap" /><div className="display-flex-right">- <Button size="small" onClick={() => navigate("outputAnalyzer")}>+ <Button+ size="small"+ onClick={() => navigate("outputAnalyzer")}+ disabled={+ disableLlmOrDocChange?.length > 0 || isSinglePassExtractLoading+ }+ >View in Output Analyzer</Button></div>diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsxindex ac209eb99..dced14415 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx@@ -101,6 +101,7 @@ function PromptCard({listOfDocs,updateCustomTool,details,+ defaultLlmProfile,disableLlmOrDocChange,indexDocs,summarizeIndexStatus,@@ -333,6 +334,7 @@ function PromptCard({details?.summarize_llm_profile) {// Summary needs to be indexed before running the prompt+ setIsRunLoading(false);handleStepsAfterRunCompletion();setAlertDetails({type: "error",@@ -456,7 +458,7 @@ function PromptCard({};const handleGetOutput = () => {- if (!selectedDoc || !selectedLlmProfileId) {+ if (!selectedDoc || (!singlePassExtractMode && !selectedLlmProfileId)) {setResult({promptOutputId: null,output: "",@@ -495,11 +497,14 @@ function PromptCard({};const handleGetCoverage = () => {- if (!selectedLlmProfileId) {+ if (+ (singlePassExtractMode && !defaultLlmProfile) ||+ (!singlePassExtractMode && !selectedLlmProfileId)+ ) {+ setCoverage(0);return;}- setCoverage(0);handleOutputApiRequest(false).then((res) => {const data = res?.data;@@ -511,7 +516,11 @@ function PromptCard({};const handleOutputApiRequest = async (isOutput) => {- let url = `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&prompt_id=${promptDetails?.prompt_id}&profile_manager=${selectedLlmProfileId}&is_single_pass_extract=${singlePassExtractMode}`;+ let profileManager = selectedLlmProfileId;+ if (singlePassExtractMode) {+ profileManager = defaultLlmProfile;+ }+ let url = `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&prompt_id=${promptDetails?.prompt_id}&profile_manager=${profileManager}&is_single_pass_extract=${singlePassExtractMode}`;if (isOutput) {url += `&document_manager=${selectedDoc?.document_id}`;@@ -815,59 +824,70 @@ function PromptCard({</Space></div><div className="prompt-card-llm-profiles">- {llmProfiles?.length > 0 &&- promptDetails?.profile_manager?.length > 0 &&- selectedLlmProfileId ? (- <div>- {llmProfiles- .filter(- (profile) => profile.profile_id === selectedLlmProfileId- )- .map((profile, index) => (- <div key={index}>- <Tag>{profile.llm}</Tag>- <Tag>{profile.vector_store}</Tag>- <Tag>{profile.embedding_model}</Tag>- <Tag>{profile.x2text}</Tag>- <Tag>{`${profile.chunk_size}/${profile.chunk_overlap}/${profile.retrieval_strategy}/${profile.similarity_top_k}/${profile.section}`}</Tag>- </div>- ))}- </div>- ) : (- <div>- <Typography.Text className="font-size-12">- No LLM Profile Selected- </Typography.Text>+ {!singlePassExtractMode && (+ <>+ {llmProfiles?.length > 0 &&+ promptDetails?.profile_manager?.length > 0 &&+ selectedLlmProfileId ? (+ <div>+ {llmProfiles+ .filter(+ (profile) =>+ profile.profile_id === selectedLlmProfileId+ )+ .map((profile, index) => (+ <div key={index}>+ <Tag>{profile.llm}</Tag>+ <Tag>{profile.vector_store}</Tag>+ <Tag>{profile.embedding_model}</Tag>+ <Tag>{profile.x2text}</Tag>+ <Tag>{`${profile.chunk_size}/${profile.chunk_overlap}/${profile.retrieval_strategy}/${profile.similarity_top_k}/${profile.section}`}</Tag>+ </div>+ ))}+ </div>+ ) : (+ <div>+ <Typography.Text className="font-size-12">+ No LLM Profile Selected+ </Typography.Text>+ </div>+ )}+ </>+ )}+ {!singlePassExtractMode && (+ <div className="display-flex-right prompt-card-paginate-div">+ <Button+ type="text"+ size="small"+ disabled={+ page <= 1 ||+ disableLlmOrDocChange.includes(+ promptDetails?.prompt_id+ ) ||+ isSinglePassExtractLoading ||+ indexDocs.includes(selectedDoc?.document_id)+ }+ onClick={handlePageLeft}+ >+ <LeftOutlined className="prompt-card-paginate" />+ </Button>+ <Button+ type="text"+ size="small"+ disabled={+ page >= llmProfiles?.length ||+ disableLlmOrDocChange.includes(+ promptDetails?.prompt_id+ ) ||+ isSinglePassExtractLoading ||+ indexDocs.includes(selectedDoc?.document_id)+ }+ onClick={handlePageRight}+ >+ <RightOutlined className="prompt-card-paginate" />+ </Button></div>)}- <div className="display-flex-right prompt-card-paginate-div">- <Button- type="text"- size="small"- disabled={- page <= 1 ||- disableLlmOrDocChange.includes(promptDetails?.prompt_id) ||- isSinglePassExtractLoading ||- indexDocs.includes(selectedDoc?.document_id)- }- onClick={handlePageLeft}- >- <LeftOutlined className="prompt-card-paginate" />- </Button>- <Button- type="text"- size="small"- disabled={- page >= llmProfiles?.length ||- disableLlmOrDocChange.includes(promptDetails?.prompt_id) ||- isSinglePassExtractLoading ||- indexDocs.includes(selectedDoc?.document_id)- }- onClick={handlePageRight}- >- <RightOutlined className="prompt-card-paginate" />- </Button>- </div></div>{EvalMetrics && <EvalMetrics result={result} />}</Space>diff --git a/frontend/src/components/custom-tools/tools-main/ToolsMain.jsx b/frontend/src/components/custom-tools/tools-main/ToolsMain.jsxindex ab0816c27..f9377f13a 100644--- a/frontend/src/components/custom-tools/tools-main/ToolsMain.jsx+++ b/frontend/src/components/custom-tools/tools-main/ToolsMain.jsx@@ -34,6 +34,7 @@ function ToolsMain() {updateCustomTool,disableLlmOrDocChange,singlePassExtractMode,+ isSinglePassExtractLoading,} = useCustomToolStore();const { setAlertDetails } = useAlertStore();const axiosPrivate = useAxiosPrivate();@@ -147,6 +148,10 @@ function ToolsMain() {<Buttonicon={<BarChartOutlined />}onClick={() => navigate("outputAnalyzer")}+ disabled={+ disableLlmOrDocChange?.length > 0 ||+ isSinglePassExtractLoading+ }/></Tooltip>{singlePassExtractMode && RunSinglePassBtn && <RunSinglePassBtn />}diff --git a/frontend/src/index.js b/frontend/src/index.jsindex 1646cbf99..1be39951e 100644--- a/frontend/src/index.js+++ b/frontend/src/index.js@@ -1,11 +1,11 @@import React from "react";import ReactDOM from "react-dom/client";+import posthog from "posthog-js";+import { PostHogProvider } from "posthog-js/react";import { Logo64 } from "./assets";import { LazyLoader } from "./components/widgets/lazy-loader/LazyLoader.jsx";import { SocketProvider } from "./helpers/SocketContext.js";-import posthog from "posthog-js";-import { PostHogProvider } from "posthog-js/react";import "./index.css";const API_KEY = "phc_f1kLKkipCK3kBtA9bT0SfAChvFrlUxYSMD91GBNbwr1";