Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Single pass extraction enhancement (#165)

Deepak K086116386f36
diff --git a/backend/prompt_studio/prompt_studio_core/constants.py b/backend/prompt_studio/prompt_studio_core/constants.py
index 0bb31987f..52173f93a 100644
--- a/backend/prompt_studio/prompt_studio_core/constants.py
+++ b/backend/prompt_studio/prompt_studio_core/constants.py
@@ -78,6 +78,7 @@ class ToolStudioPromptKeys:
DOCUMENT_ID = "document_id"
EXTRACT = "extract"
LLM_PROFILE_MANAGER = "llm_profile_manager"
+ SINGLE_PASS_EXTRACTION_MODE = "single_pass_extraction_mode"
class LogLevels:
diff --git a/backend/prompt_studio/prompt_studio_core/migrations/0009_customtool_single_pass_extraction_mode_and_more.py b/backend/prompt_studio/prompt_studio_core/migrations/0009_customtool_single_pass_extraction_mode_and_more.py
new file mode 100644
index 000000000..040497c0f
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_core/migrations/0009_customtool_single_pass_extraction_mode_and_more.py
@@ -0,0 +1,31 @@
+# Generated by Django 4.2.1 on 2024-03-18 09:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ (
+ "prompt_studio_core",
+ "0008_customtool_exclude_failed_customtool_monitor_llm",
+ ),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="customtool",
+ name="single_pass_extraction_mode",
+ field=models.BooleanField(
+ db_comment="Flag to enable or disable single pass extraction mode",
+ default=False,
+ ),
+ ),
+ migrations.AlterField(
+ model_name="customtool",
+ name="exclude_failed",
+ field=models.BooleanField(
+ db_comment="Flag to make the answer null if it is incorrect",
+ default=True,
+ ),
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_studio_core/models.py b/backend/prompt_studio/prompt_studio_core/models.py
index 291ff9c73..bed468549 100644
--- a/backend/prompt_studio/prompt_studio_core/models.py
+++ b/backend/prompt_studio/prompt_studio_core/models.py
@@ -76,4 +76,11 @@ class CustomTool(BaseModel):
blank=True,
editable=False,
)
- exclude_failed = models.BooleanField(default=True)
+ exclude_failed = models.BooleanField(
+ db_comment="Flag to make the answer null if it is incorrect",
+ default=True,
+ )
+ single_pass_extraction_mode = models.BooleanField(
+ db_comment="Flag to enable or disable single pass extraction mode",
+ default=False,
+ )
diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
index 71926139d..1df3f6dc6 100644
--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
@@ -183,9 +183,9 @@ class PromptStudioHelper:
Returns:
List[ToolStudioPrompt]: List of instance of the model
"""
- prompt_instances: list[ToolStudioPrompt] = (
- ToolStudioPrompt.objects.filter(tool_id=tool_id)
- )
+ prompt_instances: list[
+ ToolStudioPrompt
+ ] = ToolStudioPrompt.objects.filter(tool_id=tool_id)
return prompt_instances
@staticmethod
@@ -344,7 +344,7 @@ class PromptStudioHelper:
prompts=prompts,
outputs=response,
document_id=document_id,
- is_single_pass_extract=False
+ is_single_pass_extract=False,
)
except PermissionError as e:
raise e
@@ -397,9 +397,9 @@ class PromptStudioHelper:
OutputManagerHelper.handle_prompt_output_update(
prompts=prompts,
- outputs=response,
+ outputs=response["single_pass_extraction"],
document_id=document_id,
- is_single_pass_extract=True
+ is_single_pass_extract=True,
)
except PermissionError as e:
raise e
@@ -447,17 +447,15 @@ class PromptStudioHelper:
"""
monitor_llm_instance: Optional[AdapterInstance] = tool.monitor_llm
monitor_llm: Optional[str] = None
- if monitor_llm_instance:
- monitor_llm = str(monitor_llm_instance.id)
prompt_grammer = tool.prompt_grammer
outputs: list[dict[str, Any]] = []
grammer_dict = {}
grammar_list = []
- # Using default profile manager llm if monitor_llm is None
if monitor_llm_instance:
monitor_llm = str(monitor_llm_instance.id)
else:
+ # Using default profile manager llm if monitor_llm is None
default_profile = ProfileManager.get_default_llm_profile(tool)
monitor_llm = str(default_profile.llm.id)
@@ -494,9 +492,9 @@ class PromptStudioHelper:
)
output: dict[str, Any] = {}
- output[TSPKeys.ASSERTION_FAILURE_PROMPT] = (
- prompt.assertion_failure_prompt
- )
+ output[
+ TSPKeys.ASSERTION_FAILURE_PROMPT
+ ] = prompt.assertion_failure_prompt
output[TSPKeys.ASSERT_PROMPT] = prompt.assert_prompt
output[TSPKeys.IS_ASSERT] = prompt.is_assert
output[TSPKeys.PROMPT] = prompt.prompt
@@ -511,15 +509,14 @@ class PromptStudioHelper:
output[TSPKeys.GRAMMAR] = grammar_list
output[TSPKeys.TYPE] = prompt.enforce_type
output[TSPKeys.NAME] = prompt.prompt_key
- output[TSPKeys.RETRIEVAL_STRATEGY] = (
- prompt.profile_manager.retrieval_strategy
- )
- output[TSPKeys.SIMILARITY_TOP_K] = (
- prompt.profile_manager.similarity_top_k
- )
+ output[
+ TSPKeys.RETRIEVAL_STRATEGY
+ ] = prompt.profile_manager.retrieval_strategy
+ output[
+ TSPKeys.SIMILARITY_TOP_K
+ ] = prompt.profile_manager.similarity_top_k
output[TSPKeys.SECTION] = prompt.profile_manager.section
output[TSPKeys.X2TEXT_ADAPTER] = x2text
-
# Eval settings for the prompt
output[TSPKeys.EVAL_SETTINGS] = {}
output[TSPKeys.EVAL_SETTINGS][
@@ -531,6 +528,9 @@ class PromptStudioHelper:
output[TSPKeys.EVAL_SETTINGS][
TSPKeys.EVAL_SETTINGS_EXCLUDE_FAILED
] = tool.exclude_failed
+ output[
+ TSPKeys.SINGLE_PASS_EXTRACTION_MODE
+ ] = tool.single_pass_extraction_mode
for attr in dir(prompt):
if attr.startswith(TSPKeys.EVAL_METRIC_PREFIX):
attr_val = getattr(prompt, attr)
@@ -655,8 +655,7 @@ class PromptStudioHelper:
if prompt_grammar:
for word, synonyms in prompt_grammar.items():
- grammar.append(
- {TSPKeys.WORD: word, TSPKeys.SYNONYMS: synonyms})
+ grammar.append({TSPKeys.WORD: word, TSPKeys.SYNONYMS: synonyms})
if not default_profile:
raise DefaultProfileError()
@@ -683,9 +682,9 @@ class PromptStudioHelper:
llm_profile_manager[TSPKeys.VECTOR_DB] = vector_db
llm_profile_manager[TSPKeys.EMBEDDING] = embedding_model
llm_profile_manager[TSPKeys.CHUNK_SIZE] = default_profile.chunk_size
- llm_profile_manager[TSPKeys.CHUNK_OVERLAP] = (
- default_profile.chunk_overlap
- )
+ llm_profile_manager[
+ TSPKeys.CHUNK_OVERLAP
+ ] = default_profile.chunk_overlap
for prompt in prompts:
output: dict[str, Any] = {}
diff --git a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
index 9a0b608a4..170a7eecc 100644
--- a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
+++ b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
@@ -14,7 +14,7 @@ class OutputManagerHelper:
prompts: list[ToolStudioPrompt],
outputs: Any,
document_id: str,
- is_single_pass_extract: bool
+ is_single_pass_extract: bool,
) -> None:
"""Handles updating prompt outputs in the database.
@@ -36,6 +36,7 @@ class OutputManagerHelper:
for prompt in prompts:
profile_manager = prompt.profile_manager
output = json.dumps(outputs.get(prompt.prompt_key))
+ eval_metrics = outputs.get(f"{prompt.prompt_key}__evaluation", [])
# Attempt to update an existing output manager,
# for the given criteria,
@@ -46,5 +47,5 @@ class OutputManagerHelper:
profile_manager=profile_manager,
prompt_id=prompt,
is_single_pass_extract=is_single_pass_extract,
- defaults={'output': output}
+ defaults={"output": output, "eval_metrics": eval_metrics},
)
diff --git a/backend/prompt_studio/prompt_studio_output_manager/views.py b/backend/prompt_studio/prompt_studio_output_manager/views.py
index 7c5dc3264..6167f2b15 100644
--- a/backend/prompt_studio/prompt_studio_output_manager/views.py
+++ b/backend/prompt_studio/prompt_studio_output_manager/views.py
@@ -34,20 +34,18 @@ class PromptStudioOutputView(viewsets.ModelViewSet):
)
# Get the query parameter for "is_single_pass_extract"
- is_single_pass_extract_param = (
- self.request.GET.get(
- PromptStudioOutputManagerKeys.IS_SINGLE_PASS_EXTRACT, "false")
+ is_single_pass_extract_param = self.request.GET.get(
+ PromptStudioOutputManagerKeys.IS_SINGLE_PASS_EXTRACT, "false"
)
# Convert the string representation to a boolean value
is_single_pass_extract = CommonUtils.str_to_bool(
- is_single_pass_extract_param)
+ is_single_pass_extract_param
+ )
filter_args[
PromptStudioOutputManagerKeys.IS_SINGLE_PASS_EXTRACT
- ] = (
- is_single_pass_extract
- )
+ ] = is_single_pass_extract
if filter_args:
queryset = PromptStudioOutputManager.objects.filter(**filter_args)
diff --git a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx
index 9cc8313f0..c43ba4da9 100644
--- a/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx
+++ b/frontend/src/components/custom-tools/combined-output/CombinedOutput.jsx
@@ -18,15 +18,10 @@ import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader";
import "./CombinedOutput.css";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
-function CombinedOutput({
- docId,
- setFilledFields,
- triggerRunSinglePass,
- setTriggerRunSinglePass,
-}) {
+function CombinedOutput({ docId, setFilledFields }) {
const [combinedOutput, setCombinedOutput] = useState({});
const [isOutputLoading, setIsOutputLoading] = useState(false);
- const { details, isSinglePassExtract, updateCustomTool } =
+ const { details, singlePassExtractMode, isSinglePassExtractLoading } =
useCustomToolStore();
const { sessionDetails } = useSessionStore();
const { setAlertDetails } = useAlertStore();
@@ -34,22 +29,16 @@ function CombinedOutput({
const handleException = useExceptionHandler();
useEffect(() => {
- if (!docId) {
- return;
- }
-
- if (isSinglePassExtract) {
+ if (!docId || isSinglePassExtractLoading) {
return;
}
let filledFields = 0;
setIsOutputLoading(true);
+ setCombinedOutput({});
handleOutputApiRequest()
.then((res) => {
const data = res?.data || [];
- data.sort((a, b) => {
- return new Date(b.modified_at) - new Date(a.modified_at);
- });
const prompts = details?.prompts;
const output = {};
prompts.forEach((item) => {
@@ -77,6 +66,7 @@ function CombinedOutput({
filledFields++;
}
});
+
setCombinedOutput(output);
if (setFilledFields) {
@@ -91,24 +81,16 @@ function CombinedOutput({
.finally(() => {
setIsOutputLoading(false);
});
- }, [docId]);
+ }, [docId, singlePassExtractMode, isSinglePassExtractLoading]);
useEffect(() => {
Prism.highlightAll();
}, [combinedOutput]);
- useEffect(() => {
- if (!triggerRunSinglePass) {
- return;
- }
- setTriggerRunSinglePass(false);
- runSinglePassExtraction();
- }, [triggerRunSinglePass]);
-
const handleOutputApiRequest = async () => {
const requestOptions = {
method: "GET",
- url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&document_manager=${docId}`,
+ url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&document_manager=${docId}&is_single_pass_extract=${singlePassExtractMode}`,
headers: {
"X-CSRFToken": sessionDetails?.csrfToken,
},
@@ -121,37 +103,6 @@ function CombinedOutput({
});
};
- const runSinglePassExtraction = () => {
- const body = {
- document_id: docId,
- tool_id: details?.tool_id,
- };
-
- const requestOptions = {
- method: "POST",
- url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/single-pass-extraction`,
- headers: {
- "X-CSRFToken": sessionDetails?.csrfToken,
- "Content-Type": "application/json",
- },
- data: body,
- };
-
- axiosPrivate(requestOptions)
- .then((res) => {
- const data = res?.data || {};
- setCombinedOutput(data);
- })
- .catch((err) => {
- setAlertDetails(
- handleException(err, "Failed to generate single pass extraction")
- );
- })
- .finally(() => {
- updateCustomTool({ isSinglePassExtract: false });
- });
- };
-
if (isOutputLoading) {
return <SpinnerLoader />;
}
@@ -179,8 +130,6 @@ function CombinedOutput({
CombinedOutput.propTypes = {
docId: PropTypes.string,
setFilledFields: PropTypes.func,
- triggerRunSinglePass: PropTypes.bool.isRequired,
- setTriggerRunSinglePass: PropTypes.func.isRequired,
};
export { CombinedOutput };
diff --git a/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx b/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx
index 71d0319ab..7474be81c 100644
--- a/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx
+++ b/frontend/src/components/custom-tools/document-manager/DocumentManager.jsx
@@ -65,7 +65,7 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) {
disableLlmOrDocChange,
details,
indexDocs,
- isSinglePassExtract,
+ isSinglePassExtractLoading,
} = useCustomToolStore();
const { sessionDetails } = useSessionStore();
const axiosPrivate = useAxiosPrivate();
@@ -227,8 +227,8 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) {
disabled={
!selectedDoc ||
disableLlmOrDocChange?.length > 0 ||
- page <= 1 ||
- isSinglePassExtract
+ isSinglePassExtractLoading ||
+ page <= 1
}
onClick={handlePageLeft}
>
@@ -240,8 +240,8 @@ function DocumentManager({ generateIndex, handleUpdateTool, handleDocChange }) {
disabled={
!selectedDoc ||
disableLlmOrDocChange?.length > 0 ||
- page >= listOfDocs?.length ||
- isSinglePassExtract
+ isSinglePassExtractLoading ||
+ page >= listOfDocs?.length
}
onClick={handlePageRight}
>
diff --git a/frontend/src/components/custom-tools/editable-text/EditableText.jsx b/frontend/src/components/custom-tools/editable-text/EditableText.jsx
index c53ef82ea..2344429e5 100644
--- a/frontend/src/components/custom-tools/editable-text/EditableText.jsx
+++ b/frontend/src/components/custom-tools/editable-text/EditableText.jsx
@@ -20,8 +20,12 @@ function EditableText({
const [triggerHandleChange, setTriggerHandleChange] = useState(false);
const [isHovered, setIsHovered] = useState(false);
const divRef = useRef(null);
- const { disableLlmOrDocChange, indexDocs, selectedDoc } =
- useCustomToolStore();
+ const {
+ disableLlmOrDocChange,
+ indexDocs,
+ selectedDoc,
+ isSinglePassExtractLoading,
+ } = useCustomToolStore();
useEffect(() => {
setText(defaultText);
@@ -85,7 +89,9 @@ function EditableText({
onMouseOut={() => setIsHovered(false)}
onBlur={handleBlur}
onClick={() => setIsEditing(true)}
- disabled={disableLlmOrDocChange.includes(promptId)}
+ disabled={
+ disableLlmOrDocChange.includes(promptId) || isSinglePassExtractLoading
+ }
/>
);
}
@@ -107,7 +113,8 @@ function EditableText({
onClick={() => setIsEditing(true)}
disabled={
disableLlmOrDocChange.includes(promptId) ||
- indexDocs.includes(selectedDoc?.document_id)
+ indexDocs.includes(selectedDoc?.document_id) ||
+ isSinglePassExtractLoading
}
/>
);
diff --git a/frontend/src/components/custom-tools/header/Header.jsx b/frontend/src/components/custom-tools/header/Header.jsx
index 2007a4cac..1e7e1c6bc 100644
--- a/frontend/src/components/custom-tools/header/Header.jsx
+++ b/frontend/src/components/custom-tools/header/Header.jsx
@@ -17,7 +17,14 @@ import { useSessionStore } from "../../../store/session-store";
import { CustomButton } from "../../widgets/custom-button/CustomButton";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
-function Header({ setOpenSettings }) {
+let SinglePassToggleSwitch;
+try {
+ SinglePassToggleSwitch =
+ require("../../../plugins/single-pass-toggle-switch/SinglePassToggleSwitch").SinglePassToggleSwitch;
+} catch {
+ // The variable will remain undefined if the component is not available.
+}
+function Header({ setOpenSettings, handleUpdateTool }) {
const [isExportLoading, setIsExportLoading] = useState(false);
const { details } = useCustomToolStore();
const { sessionDetails } = useSessionStore();
@@ -68,6 +75,9 @@ function Header({ setOpenSettings }) {
</Button>
</div>
<div className="custom-tools-header-btns">
+ {SinglePassToggleSwitch && (
+ <SinglePassToggleSwitch handleUpdateTool={handleUpdateTool} />
+ )}
<div>
<Tooltip title="Settings">
<Button
@@ -95,6 +105,7 @@ function Header({ setOpenSettings }) {
Header.propTypes = {
setOpenSettings: PropTypes.func.isRequired,
+ handleUpdateTool: PropTypes.func.isRequired,
};
export { Header };
diff --git a/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx b/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx
index 9147c7607..3c5bc9c85 100644
--- a/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx
+++ b/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx
@@ -73,7 +73,7 @@ function ManageDocsModal({
indexDocs,
rawIndexStatus,
summarizeIndexStatus,
- isSinglePassExtract,
+ isSinglePassExtractLoading,
} = useCustomToolStore();
const { messages } = useSocketCustomToolStore();
const axiosPrivate = useAxiosPrivate();
@@ -352,10 +352,10 @@ function ManageDocsModal({
onClick={() => generateIndex(item)}
disabled={
disableLlmOrDocChange?.length > 0 ||
+ isSinglePassExtractLoading ||
indexDocs.includes(item?.document_id) ||
isUploading ||
- !defaultLlmProfile ||
- isSinglePassExtract
+ !defaultLlmProfile
}
/>
</Tooltip>
@@ -375,9 +375,9 @@ function ManageDocsModal({
className="display-flex-align-center"
disabled={
disableLlmOrDocChange?.length > 0 ||
+ isSinglePassExtractLoading ||
indexDocs.includes(item?.document_id) ||
- isUploading ||
- isSinglePassExtract
+ isUploading
}
>
<DeleteOutlined className="manage-llm-pro-icon" />
@@ -391,8 +391,8 @@ function ManageDocsModal({
onClick={() => handleDocChange(item?.document_id)}
disabled={
disableLlmOrDocChange?.length > 0 ||
- indexDocs.includes(item?.document_id) ||
- isSinglePassExtract
+ isSinglePassExtractLoading ||
+ indexDocs.includes(item?.document_id)
}
/>
),
@@ -406,8 +406,8 @@ function ManageDocsModal({
rawIndexStatus,
summarizeIndexStatus,
indexDocs,
- isSinglePassExtract,
messages,
+ isSinglePassExtractLoading,
]);
const beforeUpload = (file) => {
@@ -530,7 +530,7 @@ function ManageDocsModal({
disabled={
!defaultLlmProfile ||
disableLlmOrDocChange?.length > 0 ||
- isSinglePassExtract
+ isSinglePassExtractLoading
}
>
Upload New File
diff --git a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx
index 63218961c..ee71e30b5 100644
--- a/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx
+++ b/frontend/src/components/custom-tools/output-for-doc-modal/OutputForDocModal.jsx
@@ -38,7 +38,13 @@ function OutputForDocModal({
const [promptOutputs, setPromptOutputs] = useState([]);
const [rows, setRows] = useState([]);
const [isLoading, setIsLoading] = useState(false);
- const { details, listOfDocs, selectedDoc } = useCustomToolStore();
+ const {
+ details,
+ listOfDocs,
+ selectedDoc,
+ singlePassExtractMode,
+ isSinglePassExtractLoading,
+ } = useCustomToolStore();
const { sessionDetails } = useSessionStore();
const axiosPrivate = useAxiosPrivate();
const navigate = useNavigate();
@@ -46,11 +52,11 @@ function OutputForDocModal({
const { handleException } = useExceptionHandler();
useEffect(() => {
- if (!open) {
+ if (!open || isSinglePassExtractLoading) {
return;
}
handleGetOutputForDocs();
- }, [open]);
+ }, [open, singlePassExtractMode, isSinglePassExtractLoading]);
useEffect(() => {
updatePromptOutput();
@@ -122,7 +128,7 @@ function OutputForDocModal({
}
const requestOptions = {
method: "GET",
- url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&prompt_id=${promptId}&profile_manager=${profileManagerId}`,
+ url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&prompt_id=${promptId}&profile_manager=${profileManagerId}&is_single_pass_extract=${singlePassExtractMode}`,
headers: {
"X-CSRFToken": sessionDetails?.csrfToken,
},
diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
index ab3a40024..6b51d30df 100644
--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx
@@ -46,11 +46,12 @@ import { OutputForDocModal } from "../output-for-doc-modal/OutputForDocModal";
import "./PromptCard.css";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
import { useSocketCustomToolStore } from "../../../store/socket-custom-tool";
+import { TokenCount } from "../token-count/TokenCount";
let EvalBtn = null;
let EvalMetrics = null;
let EvalModal = null;
-let getEvalMetrics = (param1, param2, param3) => {
+let getEvalMetrics = (param1, param2) => {
return [];
};
try {
@@ -91,6 +92,7 @@ function PromptCard({
const [openOutputForDoc, setOpenOutputForDoc] = useState(false);
const [progressMsg, setProgressMsg] = useState({});
const [docOutputs, setDocOutputs] = useState({});
+ const [tokenCount, setTokenCount] = useState({});
const divRef = useRef(null);
const {
getDropdownItems,
@@ -102,6 +104,8 @@ function PromptCard({
disableLlmOrDocChange,
indexDocs,
summarizeIndexStatus,
+ singlePassExtractMode,
+ isSinglePassExtractLoading,
} = useCustomToolStore();
const { messages } = useSocketCustomToolStore();
const { sessionDetails } = useSessionStore();
@@ -148,9 +152,19 @@ function PromptCard({
}, [promptDetails]);
useEffect(() => {
+ if (isSinglePassExtractLoading) {
+ return;
+ }
+
handleGetOutput();
handleGetCoverage();
- }, [selectedLlmProfileId, selectedDoc, listOfDocs]);
+ }, [
+ selectedLlmProfileId,
+ selectedDoc,
+ listOfDocs,
+ singlePassExtractMode,
+ isSinglePassExtractLoading,
+ ]);
useEffect(() => {
let listOfIds = [...disableLlmOrDocChange];
@@ -253,11 +267,7 @@ function PromptCard({
};
const handleTypeChange = (value) => {
- handleChange(value, promptDetails?.prompt_id, "enforce_type", true).then(
- () => {
- handleRun();
- }
- );
+ handleChange(value, promptDetails?.prompt_id, "enforce_type", true);
};
const handleDocOutputs = (docId, isLoading, output) => {
@@ -340,6 +350,9 @@ function PromptCard({
}
handleDocOutputs(docId, false, value);
handleGetOutput();
+
+ const usage = data[`${promptDetails?.prompt_key}__usage`] || {};
+ setTokenCount(usage);
})
.catch((err) => {
setIsRunLoading(false);
@@ -468,7 +481,6 @@ function PromptCard({
output: outputResult?.output,
evalMetrics: getEvalMetrics(
promptDetails?.evaluate,
- promptDetails?.prompt_key,
outputResult?.eval_metrics || []
),
});
@@ -498,11 +510,7 @@ function PromptCard({
};
const handleOutputApiRequest = async (isOutput) => {
- let url = `/api/v1/unstract/${
- sessionDetails?.orgId
- }/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&prompt_id=${
- promptDetails?.prompt_id
- }&profile_manager=${selectedLlmProfileId}&is_single_pass_extract=${false}`;
+ let url = `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/?tool_id=${details?.tool_id}&prompt_id=${promptDetails?.prompt_id}&profile_manager=${selectedLlmProfileId}&is_single_pass_extract=${singlePassExtractMode}`;
if (isOutput) {
url += `&document_manager=${selectedDoc?.document_id}`;
@@ -562,6 +570,7 @@ function PromptCard({
onChange={onSearchDebounce}
disabled={
disableLlmOrDocChange.includes(promptDetails?.prompt_id) ||
+ isSinglePassExtractLoading ||
indexDocs.includes(selectedDoc?.document_id)
}
/>
@@ -578,6 +587,7 @@ function PromptCard({
onChange={onSearchDebounce}
disabled={
disableLlmOrDocChange.includes(promptDetails?.prompt_id) ||
+ isSinglePassExtractLoading ||
indexDocs.includes(selectedDoc?.document_id)
}
/>
@@ -673,7 +683,9 @@ function PromptCard({
disabled={
disableLlmOrDocChange.includes(
promptDetails?.prompt_id
- ) || indexDocs.includes(selectedDoc?.document_id)
+ ) ||
+ isSinglePassExtractLoading ||
+ indexDocs.includes(selectedDoc?.document_id)
}
>
<EditOutlined className="prompt-card-actions-head" />
@@ -690,24 +702,27 @@ function PromptCard({
<AssertionIcon className="prompt-card-actions-head" />
</Button>
</Tooltip>
- <Tooltip title="Run">
- <Button
- size="small"
- type="text"
- onClick={handleRun}
- disabled={
- (updateStatus?.promptId === promptDetails?.prompt_id &&
- updateStatus?.status ===
- promptStudioUpdateStatus.isUpdating) ||
- disableLlmOrDocChange.includes(
- promptDetails?.prompt_id
- ) ||
- indexDocs.includes(selectedDoc?.document_id)
- }
- >
- <PlayCircleOutlined className="prompt-card-actions-head" />
- </Button>
- </Tooltip>
+ {!singlePassExtractMode && (
+ <Tooltip title="Run">
+ <Button
+ size="small"
+ type="text"
+ onClick={handleRun}
+ disabled={
+ (updateStatus?.promptId ===
+ promptDetails?.prompt_id &&
+ updateStatus?.status ===
+ promptStudioUpdateStatus.isUpdating) ||
+ disableLlmOrDocChange.includes(
+ promptDetails?.prompt_id
+ ) ||
+ indexDocs.includes(selectedDoc?.document_id)
+ }
+ >
+ <PlayCircleOutlined className="prompt-card-actions-head" />
+ </Button>
+ </Tooltip>
+ )}
<ConfirmModal
handleConfirm={() => handleDelete(promptDetails?.prompt_id)}
content="The prompt will be permanently deleted."
@@ -719,7 +734,9 @@ function PromptCard({
disabled={
disableLlmOrDocChange.includes(
promptDetails?.prompt_id
- ) || indexDocs.includes(selectedDoc?.document_id)
+ ) ||
+ isSinglePassExtractLoading ||
+ indexDocs.includes(selectedDoc?.document_id)
}
>
<DeleteOutlined className="prompt-card-actions-head" />
@@ -753,7 +770,7 @@ function PromptCard({
>
<div className="prompt-card-llm-profiles">
<Space direction="horizontal">
- {EvalBtn && (
+ {EvalBtn && !singlePassExtractMode && (
<EvalBtn
btnText={promptDetails?.evaluate ? "On" : "Off"}
promptId={promptDetails.prompt_id}
@@ -778,7 +795,8 @@ function PromptCard({
</Space>
</Button>
</Space>
- <div>
+ <Space>
+ <TokenCount tokenCount={tokenCount} />
<Select
className="prompt-card-select-type"
size="small"
@@ -788,11 +806,12 @@ function PromptCard({
value={promptDetails?.enforce_type || null}
disabled={
disableLlmOrDocChange.includes(promptDetails?.prompt_id) ||
+ isSinglePassExtractLoading ||
indexDocs.includes(selectedDoc?.document_id)
}
onChange={(value) => handleTypeChange(value)}
/>
- </div>
+ </Space>
</div>
<div className="prompt-card-llm-profiles">
{llmProfiles?.length > 0 &&
@@ -827,6 +846,7 @@ function PromptCard({
disabled={
page <= 1 ||
disableLlmOrDocChange.includes(promptDetails?.prompt_id) ||
+ isSinglePassExtractLoading ||
indexDocs.includes(selectedDoc?.document_id)
}
onClick={handlePageLeft}
@@ -839,6 +859,7 @@ function PromptCard({
disabled={
page >= llmProfiles?.length ||
disableLlmOrDocChange.includes(promptDetails?.prompt_id) ||
+ isSinglePassExtractLoading ||
indexDocs.includes(selectedDoc?.document_id)
}
onClick={handlePageRight}
@@ -865,7 +886,7 @@ function PromptCard({
</>
)}
</Card>
- {EvalModal && (
+ {EvalModal && !singlePassExtractMode && (
<EvalModal
open={openEval}
setOpen={setOpenEval}
diff --git a/frontend/src/components/custom-tools/token-count/TokenCount.jsx b/frontend/src/components/custom-tools/token-count/TokenCount.jsx
new file mode 100644
index 000000000..1fc434fdd
--- /dev/null
+++ b/frontend/src/components/custom-tools/token-count/TokenCount.jsx
@@ -0,0 +1,20 @@
+import { Tag, Tooltip } from "antd";
+import PropTypes from "prop-types";
+
+function TokenCount({ tokenCount }) {
+ if (!tokenCount || !Object.keys(tokenCount)?.length) {
+ return <></>;
+ }
+
+ return (
+ <Tooltip title="Prompt Token Count | Completion Token Count | Total Token Count">
+ <Tag color="#2db7f5">{`${tokenCount?.prompt_token_count} | ${tokenCount?.completion_token_count} | ${tokenCount?.total_token_count}`}</Tag>
+ </Tooltip>
+ );
+}
+
+TokenCount.propTypes = {
+ tokenCount: PropTypes.object,
+};
+
+export { TokenCount };
diff --git a/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx b/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx
index ea5428413..0e543d400 100644
--- a/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx
+++ b/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx
@@ -168,7 +168,6 @@ function ToolIde() {
<div className="tool-ide-layout">
<div>
<Header
- setOpenCusSynonymsModal={setOpenCusSynonymsModal}
handleUpdateTool={handleUpdateTool}
setOpenSettings={setOpenSettings}
/>
diff --git a/frontend/src/components/custom-tools/tools-main/ToolsMain.jsx b/frontend/src/components/custom-tools/tools-main/ToolsMain.jsx
index 53fb59bcc..ab0816c27 100644
--- a/frontend/src/components/custom-tools/tools-main/ToolsMain.jsx
+++ b/frontend/src/components/custom-tools/tools-main/ToolsMain.jsx
@@ -1,6 +1,6 @@
import { Button, Space, Tabs, Tooltip } from "antd";
import { useEffect, useState } from "react";
-import { BarChartOutlined, PlayCircleOutlined } from "@ant-design/icons";
+import { BarChartOutlined } from "@ant-design/icons";
import { promptType } from "../../../helpers/GetStaticData";
import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";
@@ -14,10 +14,17 @@ import "./ToolsMain.css";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
import { useNavigate } from "react-router-dom";
+let RunSinglePassBtn;
+try {
+ RunSinglePassBtn =
+ require("../../../plugins/run-single-pass-btn/RunSinglePassBtn").RunSinglePassBtn;
+} catch {
+ // The variable is remain undefined if the component is not available
+}
+
function ToolsMain() {
const [activeKey, setActiveKey] = useState("1");
const [prompts, setPrompts] = useState([]);
- const [triggerRunSinglePass, setTriggerRunSinglePass] = useState(false);
const [scrollToBottom, setScrollToBottom] = useState(false);
const { sessionDetails } = useSessionStore();
const {
@@ -26,7 +33,7 @@ function ToolsMain() {
selectedDoc,
updateCustomTool,
disableLlmOrDocChange,
- isSinglePassExtract,
+ singlePassExtractMode,
} = useCustomToolStore();
const { setAlertDetails } = useAlertStore();
const axiosPrivate = useAxiosPrivate();
@@ -37,7 +44,6 @@ function ToolsMain() {
{
key: "1",
label: "Document Parser",
- disabled: isSinglePassExtract,
},
{
key: "2",
@@ -93,14 +99,6 @@ function ToolsMain() {
setPrompts(details?.prompts || []);
}, [details]);
- useEffect(() => {
- if (!isSinglePassExtract) {
- return;
- }
- setActiveKey("2");
- setTriggerRunSinglePass(true);
- }, [isSinglePassExtract]);
-
const onChange = (key) => {
setActiveKey(key);
};
@@ -137,10 +135,6 @@ function ToolsMain() {
});
};
- const handleSinglePassExtraction = () => {
- updateCustomTool({ isSinglePassExtract: true });
- };
-
return (
<div className="tools-main-layout">
<div className="doc-manager-header">
@@ -155,14 +149,7 @@ function ToolsMain() {
onClick={() => navigate("outputAnalyzer")}
/>
</Tooltip>
- <Tooltip title="Single Pass Extraction">
- <Button
- onClick={handleSinglePassExtraction}
- loading={isSinglePassExtract}
- disabled={disableLlmOrDocChange?.length > 0}
- icon={<PlayCircleOutlined />}
- />
- </Tooltip>
+ {singlePassExtractMode && RunSinglePassBtn && <RunSinglePassBtn />}
</Space>
</div>
</div>
@@ -175,11 +162,7 @@ function ToolsMain() {
/>
)}
{activeKey === "2" && (
- <CombinedOutput
- docId={selectedDoc?.document_id}
- triggerRunSinglePass={triggerRunSinglePass}
- setTriggerRunSinglePass={setTriggerRunSinglePass}
- />
+ <CombinedOutput docId={selectedDoc?.document_id} />
)}
</div>
<div className="tools-main-footer">
diff --git a/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js b/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js
index 483d5fd0f..23cae6abf 100644
--- a/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js
+++ b/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js
@@ -43,6 +43,8 @@ function CustomToolsHelper() {
const data = res?.data;
updatedCusTool["defaultLlmProfile"] = data?.default_profile;
updatedCusTool["details"] = data;
+ updatedCusTool["singlePassExtractMode"] =
+ data?.single_pass_extraction_mode;
selectedDocId = data?.output;
setLogId(sessionDetails?.logEventsId);
diff --git a/frontend/src/helpers/GetStaticData.js b/frontend/src/helpers/GetStaticData.js
index 40f7082b5..53be3f7b9 100644
--- a/frontend/src/helpers/GetStaticData.js
+++ b/frontend/src/helpers/GetStaticData.js
@@ -300,15 +300,27 @@ const isJson = (text) => {
};
const displayPromptResult = (output, isFormat = false) => {
- try {
- const parsedData = JSON.parse(output.replace(/'/g, '"'));
- if (Array.isArray(parsedData) || typeof parsedData === "object") {
- return isFormat ? JSON.stringify(parsedData, null, 4) : parsedData;
+ let i = 0;
+ let parsedData = output;
+
+ while (i < 3) {
+ i++;
+ try {
+ parsedData = JSON.parse(parsedData);
+ } catch {
+ break;
}
- return output;
- } catch (err) {
- return output;
}
+
+ if (
+ (Array.isArray(parsedData) || typeof parsedData === "object") &&
+ isFormat
+ ) {
+ // If isFormat is true, return the formatted JSON string
+ return JSON.stringify(parsedData, null, 4);
+ }
+
+ return parsedData;
};
const onboardCompleted = (adaptersList) => {
diff --git a/frontend/src/store/custom-tool-store.js b/frontend/src/store/custom-tool-store.js
index 2bac346dc..952c9ceb6 100644
--- a/frontend/src/store/custom-tool-store.js
+++ b/frontend/src/store/custom-tool-store.js
@@ -13,7 +13,8 @@ const defaultState = {
indexDocs: [],
rawIndexStatus: [],
summarizeIndexStatus: [],
- isSinglePassExtract: false,
+ singlePassExtractMode: false,
+ isSinglePassExtractLoading: false,
};
const defaultPromptInstance = {