Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Fixed processing of notes in prompt studio (#172)

Deepak K87999330bc67
diff --git a/backend/prompt_studio/prompt_studio_core/constants.py b/backend/prompt_studio/prompt_studio_core/constants.py
index 7c2b5db52..ac998aa92 100644
--- a/backend/prompt_studio/prompt_studio_core/constants.py
+++ b/backend/prompt_studio/prompt_studio_core/constants.py
@@ -82,6 +82,7 @@ class ToolStudioPromptKeys:
CHALLENGE_LLM = "challenge_llm"
SINGLE_PASS_EXTRACTION_MODE = "single_pass_extraction_mode"
SINGLE_PASS_EXTRACTION = "single_pass_extraction"
+ NOTES = "NOTES"
class LogLevels:
diff --git a/backend/prompt_studio/prompt_studio_core/exceptions.py b/backend/prompt_studio/prompt_studio_core/exceptions.py
index f7210f309..7004b42b4 100644
--- a/backend/prompt_studio/prompt_studio_core/exceptions.py
+++ b/backend/prompt_studio/prompt_studio_core/exceptions.py
@@ -59,3 +59,8 @@ class NoPromptsFound(APIException):
class PermissionError(APIException):
status_code = 403
default_detail = "You do not have permission to perform this action."
+
+
+class EmptyPromptError(APIException):
+ status_code = 422
+ default_detail = "Prompt(s) cannot be empty"
diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
index 16a930d41..958590b29 100644
--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
@@ -17,6 +17,7 @@ from prompt_studio.prompt_studio_core.constants import (
from prompt_studio.prompt_studio_core.exceptions import (
AnswerFetchError,
DefaultProfileError,
+ EmptyPromptError,
IndexingError,
NoPromptsFound,
PermissionError,
@@ -373,8 +374,15 @@ class PromptStudioHelper:
return response
else:
prompts = PromptStudioHelper.fetch_prompt_from_tool(tool_id)
+ prompts = [
+ prompt
+ for prompt in prompts
+ if prompt.prompt_type != TSPKeys.NOTES
+ ]
if not prompts:
- logger.error(f"[{tool_id or 'NA'}] No prompts found id: {id}")
+ logger.error(
+ f"[{tool_id or 'NA'}] No prompts found for id: {id}"
+ )
raise NoPromptsFound()
logger.info(f"[{tool_id}] Executing prompts in single pass")
@@ -705,6 +713,8 @@ class PromptStudioHelper:
tool_settings[TSPKeys.CHALLENGE_LLM] = challenge_llm
for prompt in prompts:
+ if not prompt.prompt:
+ raise EmptyPromptError()
output: dict[str, Any] = {}
output[TSPKeys.PROMPT] = prompt.prompt
output[TSPKeys.ACTIVE] = prompt.active
diff --git a/backend/prompt_studio/prompt_studio_output_manager/constants.py b/backend/prompt_studio/prompt_studio_output_manager/constants.py
index 1caf8a66f..a9c046aae 100644
--- a/backend/prompt_studio/prompt_studio_output_manager/constants.py
+++ b/backend/prompt_studio/prompt_studio_output_manager/constants.py
@@ -4,3 +4,4 @@ class PromptStudioOutputManagerKeys:
PROFILE_MANAGER = "profile_manager"
DOCUMENT_MANAGER = "document_manager"
IS_SINGLE_PASS_EXTRACT = "is_single_pass_extract"
+ NOTES = "NOTES"
diff --git a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
index 170a7eecc..24e1f022e 100644
--- a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
+++ b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
@@ -1,8 +1,12 @@
import json
from typing import Any
+from prompt_studio.prompt_profile_manager.models import ProfileManager
from prompt_studio.prompt_studio.models import ToolStudioPrompt
from prompt_studio.prompt_studio_document_manager.models import DocumentManager
+from prompt_studio.prompt_studio_output_manager.constants import (
+ PromptStudioOutputManagerKeys as PSOMKeys,
+)
from prompt_studio.prompt_studio_output_manager.models import (
PromptStudioOutputManager,
)
@@ -31,10 +35,15 @@ class OutputManagerHelper:
tool = prompts[0].tool_id
document_manager = DocumentManager.objects.get(pk=document_id)
-
+ default_profile = ProfileManager.get_default_llm_profile(tool=tool)
# Iterate through each prompt in the list
for prompt in prompts:
- profile_manager = prompt.profile_manager
+ if prompt.prompt_type == PSOMKeys.NOTES:
+ continue
+ if is_single_pass_extract:
+ profile_manager = default_profile
+ else:
+ profile_manager = prompt.profile_manager
output = json.dumps(outputs.get(prompt.prompt_key))
eval_metrics = outputs.get(f"{prompt.prompt_key}__evaluation", [])
diff --git a/backend/prompt_studio/prompt_studio_registry/constants.py b/backend/prompt_studio/prompt_studio_registry/constants.py
index 73eee1e9f..745c59975 100644
--- a/backend/prompt_studio/prompt_studio_registry/constants.py
+++ b/backend/prompt_studio/prompt_studio_registry/constants.py
@@ -89,6 +89,7 @@ class JsonSchemaKey:
EMBEDDING_SUFFIX = "embedding_suffix"
FUNCTION_NAME = "function_name"
PROMPT_REGISTRY_ID = "prompt_registry_id"
+ NOTES = "NOTES"
class SpecKey:
diff --git a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py
index fd33d8da9..c9f328f70 100644
--- a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py
+++ b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py
@@ -115,10 +115,10 @@ class PromptStudioRegistryHelper:
PromptStudioRegistryHelper.frame_properties(tool=custom_tool)
)
spec: Spec = PromptStudioRegistryHelper.frame_spec(tool=custom_tool)
- prompts: list[ToolStudioPrompt] = (
- PromptStudioHelper.fetch_prompt_from_tool(
- tool_id=custom_tool.tool_id
- )
+ prompts: list[
+ ToolStudioPrompt
+ ] = PromptStudioHelper.fetch_prompt_from_tool(
+ tool_id=custom_tool.tool_id
)
metadata = PromptStudioRegistryHelper.frame_export_json(
tool=custom_tool, prompts=prompts
@@ -183,6 +183,8 @@ class PromptStudioRegistryHelper:
default_llm_profile = ProfileManager.get_default_llm_profile(tool)
for prompt in prompts:
+ if prompt.prompt_type == JsonSchemaKey.NOTES:
+ continue
if not prompt.profile_manager:
prompt.profile_manager = default_llm_profile
@@ -193,9 +195,9 @@ class PromptStudioRegistryHelper:
adapter_id = str(prompt.profile_manager.embedding_model.adapter_id)
embedding_suffix = adapter_id.split("|")[0]
- output[JsonSchemaKey.ASSERTION_FAILURE_PROMPT] = (
- prompt.assertion_failure_prompt
- )
+ output[
+ JsonSchemaKey.ASSERTION_FAILURE_PROMPT
+ ] = prompt.assertion_failure_prompt
output[JsonSchemaKey.ASSERT_PROMPT] = prompt.assert_prompt
output[JsonSchemaKey.IS_ASSERT] = prompt.is_assert
output[JsonSchemaKey.PROMPT] = prompt.prompt
@@ -204,21 +206,21 @@ class PromptStudioRegistryHelper:
output[JsonSchemaKey.VECTOR_DB] = vector_db
output[JsonSchemaKey.EMBEDDING] = embedding_model
output[JsonSchemaKey.X2TEXT_ADAPTER] = x2text
- output[JsonSchemaKey.CHUNK_OVERLAP] = (
- prompt.profile_manager.chunk_overlap
- )
+ output[
+ JsonSchemaKey.CHUNK_OVERLAP
+ ] = prompt.profile_manager.chunk_overlap
output[JsonSchemaKey.LLM] = llm
output[JsonSchemaKey.PREAMBLE] = tool.preamble
output[JsonSchemaKey.POSTAMBLE] = tool.postamble
output[JsonSchemaKey.GRAMMAR] = grammar_list
output[JsonSchemaKey.TYPE] = prompt.enforce_type
output[JsonSchemaKey.NAME] = prompt.prompt_key
- output[JsonSchemaKey.RETRIEVAL_STRATEGY] = (
- prompt.profile_manager.retrieval_strategy
- )
- output[JsonSchemaKey.SIMILARITY_TOP_K] = (
- prompt.profile_manager.similarity_top_k
- )
+ output[
+ JsonSchemaKey.RETRIEVAL_STRATEGY
+ ] = prompt.profile_manager.retrieval_strategy
+ output[
+ JsonSchemaKey.SIMILARITY_TOP_K
+ ] = prompt.profile_manager.similarity_top_k
output[JsonSchemaKey.SECTION] = prompt.profile_manager.section
output[JsonSchemaKey.REINDEX] = prompt.profile_manager.reindex
output[JsonSchemaKey.EMBEDDING_SUFFIX] = embedding_suffix
diff --git a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx
index c43ba4da9..507bf6704 100644
--- a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx
+++ b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx
@@ -21,8 +21,12 @@ import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
function CombinedOutput({ docId, setFilledFields }) {
const [combinedOutput, setCombinedOutput] = useState({});
const [isOutputLoading, setIsOutputLoading] = useState(false);
- const { details, singlePassExtractMode, isSinglePassExtractLoading } =
- useCustomToolStore();
+ const {
+ details,
+ defaultLlmProfile,
+ singlePassExtractMode,
+ isSinglePassExtractLoading,
+ } = useCustomToolStore();
const { sessionDetails } = useSessionStore();
const { setAlertDetails } = useAlertStore();
const axiosPrivate = useAxiosPrivate();
@@ -47,10 +51,14 @@ function CombinedOutput({ docId, setFilledFields }) {
}
output[item?.prompt_key] = "";
+ let profileManager = item?.profile_manager;
+ if (singlePassExtractMode) {
+ profileManager = defaultLlmProfile;
+ }
const outputDetails = data.find(
(outputValue) =>
outputValue?.prompt_id === item?.prompt_id &&
- outputValue?.profile_manager === item?.profile_manager
+ outputValue?.profile_manager === profileManager
);
if (!outputDetails) {
diff --git a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx
index 257fcf0a0..962de7c7d 100644
--- a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx
+++ b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx
@@ -52,6 +52,8 @@ function OutputForDocModal({
details,
listOfDocs,
selectedDoc,
+ defaultLlmProfile,
+ disableLlmOrDocChange,
singlePassExtractMode,
isSinglePassExtractLoading,
} = useCustomToolStore();
@@ -62,7 +64,7 @@ function OutputForDocModal({
const { handleException } = useExceptionHandler();
useEffect(() => {
- if (!open || isSinglePassExtractLoading) {
+ if (!open) {
return;
}
handleGetOutputForDocs();
@@ -135,13 +137,18 @@ function OutputForDocModal({
};
const handleGetOutputForDocs = () => {
- if (!profileManagerId) {
+ let profile = profileManagerId;
+ if (singlePassExtractMode) {
+ profile = defaultLlmProfile;
+ }
+
+ if (!profile) {
setRows([]);
return;
}
const requestOptions = {
method: "GET",
- url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&prompt_id=${promptId}&profile_manager=${profileManagerId}&is_single_pass_extract=${singlePassExtractMode}`,
+ url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&prompt_id=${promptId}&profile_manager=${profile}&is_single_pass_extract=${singlePassExtractMode}`,
headers: {
"X-CSRFToken": sessionDetails?.csrfToken,
},
@@ -232,7 +239,13 @@ function OutputForDocModal({
</div>
<div className="output-doc-gap" />
<div className="display-flex-right">
- <Button size="small" onClick={() => navigate("outputAnalyzer")}>
+ <Button
+ size="small"
+ onClick={() => navigate("outputAnalyzer")}
+ disabled={
+ disableLlmOrDocChange?.length > 0 || isSinglePassExtractLoading
+ }
+ >
View in Output Analyzer
</Button>
</div>
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
index ac209eb99..dced14415 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
@@ -101,6 +101,7 @@ function PromptCard({
listOfDocs,
updateCustomTool,
details,
+ defaultLlmProfile,
disableLlmOrDocChange,
indexDocs,
summarizeIndexStatus,
@@ -333,6 +334,7 @@ function PromptCard({
details?.summarize_llm_profile
) {
// Summary needs to be indexed before running the prompt
+ setIsRunLoading(false);
handleStepsAfterRunCompletion();
setAlertDetails({
type: "error",
@@ -456,7 +458,7 @@ function PromptCard({
};
const handleGetOutput = () => {
- if (!selectedDoc || !selectedLlmProfileId) {
+ if (!selectedDoc || (!singlePassExtractMode && !selectedLlmProfileId)) {
setResult({
promptOutputId: null,
output: "",
@@ -495,11 +497,14 @@ function PromptCard({
};
const handleGetCoverage = () => {
- if (!selectedLlmProfileId) {
+ if (
+ (singlePassExtractMode && !defaultLlmProfile) ||
+ (!singlePassExtractMode && !selectedLlmProfileId)
+ ) {
+ setCoverage(0);
return;
}
- setCoverage(0);
handleOutputApiRequest(false)
.then((res) => {
const data = res?.data;
@@ -511,7 +516,11 @@ function PromptCard({
};
const handleOutputApiRequest = async (isOutput) => {
- let url = `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&prompt_id=${promptDetails?.prompt_id}&profile_manager=${selectedLlmProfileId}&is_single_pass_extract=${singlePassExtractMode}`;
+ let profileManager = selectedLlmProfileId;
+ if (singlePassExtractMode) {
+ profileManager = defaultLlmProfile;
+ }
+ let url = `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&prompt_id=${promptDetails?.prompt_id}&profile_manager=${profileManager}&is_single_pass_extract=${singlePassExtractMode}`;
if (isOutput) {
url += `&document_manager=${selectedDoc?.document_id}`;
@@ -815,59 +824,70 @@ function PromptCard({
</Space>
</div>
<div className="prompt-card-llm-profiles">
- {llmProfiles?.length > 0 &&
- promptDetails?.profile_manager?.length > 0 &&
- selectedLlmProfileId ? (
- <div>
- {llmProfiles
- .filter(
- (profile) => profile.profile_id === selectedLlmProfileId
- )
- .map((profile, index) => (
- <div key={index}>
- <Tag>{profile.llm}</Tag>
- <Tag>{profile.vector_store}</Tag>
- <Tag>{profile.embedding_model}</Tag>
- <Tag>{profile.x2text}</Tag>
- <Tag>{`${profile.chunk_size}/${profile.chunk_overlap}/${profile.retrieval_strategy}/${profile.similarity_top_k}/${profile.section}`}</Tag>
- </div>
- ))}
- </div>
- ) : (
- <div>
- <Typography.Text className="font-size-12">
- No LLM Profile Selected
- </Typography.Text>
+ {!singlePassExtractMode && (
+ <>
+ {llmProfiles?.length > 0 &&
+ promptDetails?.profile_manager?.length > 0 &&
+ selectedLlmProfileId ? (
+ <div>
+ {llmProfiles
+ .filter(
+ (profile) =>
+ profile.profile_id === selectedLlmProfileId
+ )
+ .map((profile, index) => (
+ <div key={index}>
+ <Tag>{profile.llm}</Tag>
+ <Tag>{profile.vector_store}</Tag>
+ <Tag>{profile.embedding_model}</Tag>
+ <Tag>{profile.x2text}</Tag>
+ <Tag>{`${profile.chunk_size}/${profile.chunk_overlap}/${profile.retrieval_strategy}/${profile.similarity_top_k}/${profile.section}`}</Tag>
+ </div>
+ ))}
+ </div>
+ ) : (
+ <div>
+ <Typography.Text className="font-size-12">
+ No LLM Profile Selected
+ </Typography.Text>
+ </div>
+ )}
+ </>
+ )}
+ {!singlePassExtractMode && (
+ <div className="display-flex-right prompt-card-paginate-div">
+ <Button
+ type="text"
+ size="small"
+ disabled={
+ page <= 1 ||
+ disableLlmOrDocChange.includes(
+ promptDetails?.prompt_id
+ ) ||
+ isSinglePassExtractLoading ||
+ indexDocs.includes(selectedDoc?.document_id)
+ }
+ onClick={handlePageLeft}
+ >
+ <LeftOutlined className="prompt-card-paginate" />
+ </Button>
+ <Button
+ type="text"
+ size="small"
+ disabled={
+ page >= llmProfiles?.length ||
+ disableLlmOrDocChange.includes(
+ promptDetails?.prompt_id
+ ) ||
+ isSinglePassExtractLoading ||
+ indexDocs.includes(selectedDoc?.document_id)
+ }
+ onClick={handlePageRight}
+ >
+ <RightOutlined className="prompt-card-paginate" />
+ </Button>
</div>
)}
- <div className="display-flex-right prompt-card-paginate-div">
- <Button
- type="text"
- size="small"
- disabled={
- page <= 1 ||
- disableLlmOrDocChange.includes(promptDetails?.prompt_id) ||
- isSinglePassExtractLoading ||
- indexDocs.includes(selectedDoc?.document_id)
- }
- onClick={handlePageLeft}
- >
- <LeftOutlined className="prompt-card-paginate" />
- </Button>
- <Button
- type="text"
- size="small"
- disabled={
- page >= llmProfiles?.length ||
- disableLlmOrDocChange.includes(promptDetails?.prompt_id) ||
- isSinglePassExtractLoading ||
- indexDocs.includes(selectedDoc?.document_id)
- }
- onClick={handlePageRight}
- >
- <RightOutlined className="prompt-card-paginate" />
- </Button>
- </div>
</div>
{EvalMetrics && <EvalMetrics result={result} />}
</Space>
diff --git a/frontend/src/components/custom-tools/tools-main/ToolsMain.jsx b/frontend/src/components/custom-tools/tools-main/ToolsMain.jsx
index ab0816c27..f9377f13a 100644
--- a/frontend/src/components/custom-tools/tools-main/ToolsMain.jsx
+++ b/frontend/src/components/custom-tools/tools-main/ToolsMain.jsx
@@ -34,6 +34,7 @@ function ToolsMain() {
updateCustomTool,
disableLlmOrDocChange,
singlePassExtractMode,
+ isSinglePassExtractLoading,
} = useCustomToolStore();
const { setAlertDetails } = useAlertStore();
const axiosPrivate = useAxiosPrivate();
@@ -147,6 +148,10 @@ function ToolsMain() {
<Button
icon={<BarChartOutlined />}
onClick={() => navigate("outputAnalyzer")}
+ disabled={
+ disableLlmOrDocChange?.length > 0 ||
+ isSinglePassExtractLoading
+ }
/>
</Tooltip>
{singlePassExtractMode && RunSinglePassBtn && <RunSinglePassBtn />}
diff --git a/frontend/src/index.js b/frontend/src/index.js
index 1646cbf99..1be39951e 100644
--- a/frontend/src/index.js
+++ b/frontend/src/index.js
@@ -1,11 +1,11 @@
import React from "react";
import ReactDOM from "react-dom/client";
+import posthog from "posthog-js";
+import { PostHogProvider } from "posthog-js/react";
import { Logo64 } from "./assets";
import { LazyLoader } from "./components/widgets/lazy-loader/LazyLoader.jsx";
import { SocketProvider } from "./helpers/SocketContext.js";
-import posthog from "posthog-js";
-import { PostHogProvider } from "posthog-js/react";
import "./index.css";
const API_KEY = "phc_f1kLKkipCK3kBtA9bT0SfAChvFrlUxYSMD91GBNbwr1";