LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
FIX - UX Improvements and Bug Fixes in Document Manager (#98)
Tahier Hussainb767b20381cf
diff --git a/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx b/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsxindex 14122ae4d..4d29ec40f 100644--- a/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx+++ b/frontend/src/components/custom-tools/manage-docs-modal/ManageDocsModal.jsx@@ -2,7 +2,6 @@ import {CheckCircleFilled,CloseCircleFilled,DeleteOutlined,- InfoCircleFilled,PlusOutlined,ReloadOutlined,} from "@ant-design/icons";@@ -11,7 +10,6 @@ import {Divider,Modal,Radio,- Select,Space,Table,Tooltip,@@ -32,6 +30,14 @@ import "./ManageDocsModal.css";import { useExceptionHandler } from "../../../hooks/useExceptionHandler";import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader";+let SummarizeStatusTitle = null;+try {+ SummarizeStatusTitle =+ require("../../../plugins/summarize-status-title/SummarizeStatusTitle111").SummarizeStatusTitle;+} catch {+ // The component will remain null if it is not available+}+const indexTypes = {raw: "RAW",summarize: "Summarize",@@ -46,13 +52,10 @@ function ManageDocsModal({}) {const [isUploading, setIsUploading] = useState(false);const [rows, setRows] = useState([]);- const [rawLlmProfile, setRawLlmProfile] = useState("");+ const [rawLlmProfile, setRawLlmProfile] = useState(null);const [isRawDataLoading, setIsRawDataLoading] = useState(false);- const [rawIndexStatus, setRawIndexStatus] = useState([]);- const [summarizeLlmProfile, setSummarizeLlmProfile] = useState("");+ const [summarizeLlmProfile, setSummarizeLlmProfile] = useState(null);const [isSummarizeDataLoading, setIsSummarizeDataLoading] = useState(false);- const [summarizeIndexStatus, setSummarizeIndexStatus] = useState([]);- const [llmItems, setLlmItems] = useState([]);const { sessionDetails } = useSessionStore();const { setAlertDetails } = useAlertStore();const {@@ -64,6 +67,8 @@ function ManageDocsModal({defaultLlmProfile,disableLlmOrDocChange,indexDocs,+ rawIndexStatus,+ summarizeIndexStatus,} = useCustomToolStore();const axiosPrivate = useAxiosPrivate();const handleException = useExceptionHandler();@@ -86,31 +91,27 @@ function ManageDocsModal({</Typography.Text>);- const infoIndex = (+ const failedSummary = (<Typography.Text><span style={{ marginRight: "8px" }}>- <InfoCircleFilled style={{ color: "#009AF0" }} />+ <CloseCircleFilled style={{ color: "#FF4D4F" }} /></span>{" "}- LLM Profile not selected+ Not Summarized</Typography.Text>);useEffect(() => {setRawLlmProfile(defaultLlmProfile);setSummarizeLlmProfile(details?.summarize_llm_profile);- }, [llmItems]);-- useEffect(() => {- getLlmProfilesDropdown();- }, [llmProfiles]);+ }, [defaultLlmProfile, details]);useEffect(() => {handleGetIndexStatus(rawLlmProfile, indexTypes.raw);- }, [rawLlmProfile, indexDocs]);+ }, [indexDocs, rawLlmProfile]);useEffect(() => {handleGetIndexStatus(summarizeLlmProfile, indexTypes.summarize);- }, [summarizeLlmProfile, indexDocs]);+ }, [indexDocs, summarizeLlmProfile]);const handleLoading = (indexType, value) => {if (indexType === indexTypes.raw) {@@ -124,11 +125,11 @@ function ManageDocsModal({const handleIndexStatus = (indexType, data) => {if (indexType === indexTypes.raw) {- setRawIndexStatus(data);+ updateCustomTool({ rawIndexStatus: data });}if (indexType === indexTypes.summarize) {- setSummarizeIndexStatus(data);+ updateCustomTool({ summarizeIndexStatus: data });}};@@ -147,6 +148,7 @@ function ManageDocsModal({const handleGetIndexStatus = (llmProfileId, indexType) => {if (!llmProfileId) {+ handleIndexStatus(indexType, []);return;}@@ -176,14 +178,12 @@ function ManageDocsModal({});};- const getLlmProfilesDropdown = () => {- const items = [...llmProfiles].map((item) => {- return {- value: item?.profile_id,- label: item?.profile_name,- };- });- setLlmItems(items);+ const getLlmProfileName = (llmProfile) => {+ const llmProfileName = llmProfiles.find(+ (item) => item?.profile_id === llmProfile+ );++ return llmProfileName?.profile_name || "No LLM Profile Selected";};const columns = [@@ -195,42 +195,16 @@ function ManageDocsModal({{title: (<Space className="w-100">- <Typography.Text>Raw</Typography.Text>- <Select- size="small"- style={{- width: 100,- }}- value={rawLlmProfile}- options={llmItems}- onChange={(value) => setRawLlmProfile(value)}- />+ <Typography.Text>Index</Typography.Text>+ <Typography.Text type="secondary">+ {"(" + getLlmProfileName(rawLlmProfile) + ")"}+ </Typography.Text>{isRawDataLoading && <SpinnerLoader />}</Space>),- dataIndex: "rawIndex",- key: "rawIndex",- width: 250,- },- {- title: (- <Space className="w-100">- <Typography.Text>Summarize</Typography.Text>- <Select- size="small"- style={{- width: 100,- }}- value={summarizeLlmProfile}- options={llmItems}- onChange={(value) => setSummarizeLlmProfile(value)}- />- {isSummarizeDataLoading && <SpinnerLoader />}- </Space>- ),- dataIndex: "summarizeIndex",- key: "summarizeIndex",- width: 250,+ dataIndex: "index",+ key: "index",+ width: 260,},{title: "",@@ -252,21 +226,32 @@ function ManageDocsModal({},];+ if (SummarizeStatusTitle) {+ columns.splice(2, 0, {+ title: (+ <SummarizeStatusTitle+ profileName={"(" + getLlmProfileName(summarizeLlmProfile) + ")"}+ isLoading={isSummarizeDataLoading}+ />+ ),+ dataIndex: "summary",+ key: "summary",+ width: 260,+ });+ }+const getIndexStatusMessage = (docId, indexType) => {let instance = null;+ let failed = null;if (indexType === indexTypes.raw) {- if (!rawLlmProfile) {- return infoIndex;- }instance = rawIndexStatus.find((item) => item?.docId === docId);+ failed = failedIndex;} else {- if (!summarizeLlmProfile) {- return infoIndex;- }instance = summarizeIndexStatus.find((item) => item?.docId === docId);+ failed = failedSummary;}- return instance?.isIndexed ? successIndex : failedIndex;+ return instance?.isIndexed ? successIndex : failed;};useEffect(() => {@@ -274,39 +259,46 @@ function ManageDocsModal({return {key: item?.document_id,document: item?.document_name || "",- rawIndex: getIndexStatusMessage(item?.document_id, indexTypes.raw),- summarizeIndex: getIndexStatusMessage(- item?.document_id,- indexTypes.summarize- ),+ index: getIndexStatusMessage(item?.document_id, indexTypes.raw),+ summary:+ SummarizeStatusTitle &&+ getIndexStatusMessage(item?.document_id, indexTypes.summarize),reindex: indexDocs.includes(item?.document_id) ? (<SpinnerLoader />) : (- <Button- size="small"- icon={<ReloadOutlined />}- onClick={() => generateIndex(item)}- disabled={- disableLlmOrDocChange?.length > 0 ||- indexDocs.includes(item?.document_id)- }- />+ <Tooltip title="Index">+ <Button+ size="small"+ icon={<ReloadOutlined />}+ onClick={() => generateIndex(item)}+ disabled={+ disableLlmOrDocChange?.length > 0 ||+ indexDocs.includes(item?.document_id) ||+ isUploading ||+ !defaultLlmProfile+ }+ />+ </Tooltip>),delete: (<ConfirmModalhandleConfirm={() => handleDelete(item?.document_id)}content="The document will be permanently deleted.">- <Button- size="small"- className="display-flex-align-center"- disabled={- disableLlmOrDocChange?.length > 0 ||- indexDocs.includes(item?.document_id)- }- >- <DeleteOutlined className="manage-llm-pro-icon" />- </Button>+ <Tooltip title="Delete">+ <Button+ size="small"+ className="display-flex-align-center"+ disabled={+ disableLlmOrDocChange?.length > 0 ||+ indexDocs.includes(item?.document_id) ||+ isUploading ||+ !defaultLlmProfile+ }+ >+ <DeleteOutlined className="manage-llm-pro-icon" />+ </Button>+ </Tooltip></ConfirmModal>),select: (diff --git a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsxindex d9a280ad3..f9bee09f6 100644--- a/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx+++ b/frontend/src/components/custom-tools/prompt-card/PromptCard.jsx@@ -104,6 +104,7 @@ function PromptCard({details,disableLlmOrDocChange,indexDocs,+ summarizeIndexStatus,} = useCustomToolStore();const { sessionDetails } = useSessionStore();const { setAlertDetails } = useAlertStore();@@ -180,6 +181,7 @@ function PromptCard({useEffect(() => {if (isCoverageLoading && coverageTotal === listOfDocs?.length) {setIsCoverageLoading(false);+ setCoverageTotal(0);}}, [coverageTotal]);@@ -282,6 +284,26 @@ function PromptCard({method = "PATCH";url += `${result?.promptOutputId}/`;}++ const isSummaryIndexed = [...summarizeIndexStatus].find(+ (item) =>+ item?.docId === selectedDoc?.document_id && item?.isIndexed === true+ );++ if (+ !isSummaryIndexed &&+ details?.summarize_as_source &&+ details?.summarize_llm_profile+ ) {+ // Summary needs to be indexed before running the prompt+ handleUpdateOutput(null, selectedDoc?.document_id, [], method, url);+ handleStepsAfterRunCompletion();+ setAlertDetails({+ type: "error",+ content: `Summary needs to be indexed before running the prompt - ${selectedDoc?.document_name}.`,+ });+ return;+ }handleRunApiRequest(selectedDoc?.document_id).then((res) => {const data = res?.data;@@ -314,12 +336,16 @@ function PromptCard({);}).finally(() => {- setIsRunLoading(false);- setCoverageTotal((prev) => prev + 1);- handleCoverage();+ handleStepsAfterRunCompletion();});};+ const handleStepsAfterRunCompletion = () => {+ setIsRunLoading(false);+ setCoverageTotal(1);+ handleCoverage();+ };+// Get the coverage for all the documents except the one that's currently selectedconst handleCoverage = () => {const listOfDocsToProcess = [...listOfDocs].filter(@@ -331,6 +357,7 @@ function PromptCard({return;}+ let totalCoverageValue = 1;listOfDocsToProcess.forEach((item) => {let method = "POST";let url = `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/prompt-output/`;@@ -341,6 +368,28 @@ function PromptCard({method = "PATCH";url += `${outputId?.promptOutputId}/`;}++ const isSummaryIndexed = [...summarizeIndexStatus].find(+ (indexStatus) =>+ indexStatus?.docId === item?.document_id &&+ indexStatus?.isIndexed === true+ );++ if (+ !isSummaryIndexed &&+ details?.summarize_as_source &&+ details?.summarize_llm_profile+ ) {+ // Summary needs to be indexed before running the prompt+ handleUpdateOutput(null, item?.document_id, [], method, url);+ totalCoverageValue++;+ setCoverageTotal(totalCoverageValue);+ setAlertDetails({+ type: "error",+ content: `Summary needs to be indexed before running the prompt - ${item?.document_name}.`,+ });+ return;+ }handleRunApiRequest(item?.document_id).then((res) => {const data = res?.data;@@ -373,7 +422,8 @@ function PromptCard({);}).finally(() => {- setCoverageTotal((prev) => prev + 1);+ totalCoverageValue++;+ setCoverageTotal(totalCoverageValue);});});};diff --git a/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx b/frontend/src/components/custom-tools/tool-ide/ToolIde.jsxindex ba2a20917..54ca7c2f1 100644--- a/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx+++ b/frontend/src/components/custom-tools/tool-ide/ToolIde.jsx@@ -28,6 +28,8 @@ function ToolIde() {selectedDoc,listOfDocs,indexDocs,+ pushIndexDoc,+ deleteIndexDoc,} = useCustomToolStore();const { sessionDetails } = useSessionStore();const { setAlertDetails } = useAlertStore();@@ -72,9 +74,8 @@ function ToolIde() {const generateIndex = async (doc) => {const docId = doc?.document_id;- const listOfIndexDocs = [...indexDocs];- if (listOfIndexDocs.includes(docId)) {+ if (indexDocs.includes(docId)) {setAlertDetails({type: "error",content: "This document is already getting indexed",@@ -86,6 +87,7 @@ function ToolIde() {tool_id: details?.tool_id,document_id: docId,};+const requestOptions = {method: "POST",url: `/api/v1/unstract/${sessionDetails?.orgId}/prompt-studio/index-document/`,@@ -96,8 +98,7 @@ function ToolIde() {data: body,};- listOfIndexDocs.push(docId);- updateCustomTool({ indexDocs: listOfIndexDocs });+ pushIndexDoc(docId);return axiosPrivate(requestOptions).then(() => {setAlertDetails({@@ -111,10 +112,7 @@ function ToolIde() {);}).finally(() => {- const newListOfIndexDocs = [...indexDocs].filter(- (item) => item !== docId- );- updateCustomTool({ indexDocs: newListOfIndexDocs });+ deleteIndexDoc(docId);});};diff --git a/frontend/src/store/custom-tool-store.js b/frontend/src/store/custom-tool-store.jsindex 6e457ca35..9fb8689a0 100644--- a/frontend/src/store/custom-tool-store.js+++ b/frontend/src/store/custom-tool-store.js@@ -11,6 +11,8 @@ const defaultState = {details: {},disableLlmOrDocChange: [],indexDocs: [],+ rawIndexStatus: [],+ summarizeIndexStatus: [],};const defaultPromptInstance = {@@ -73,6 +75,22 @@ const useCustomToolStore = create((setState, getState) => ({const dropdownItems = existingState?.dropdownItems || {};return dropdownItems[propertyName];},+ pushIndexDoc: (docId) => {+ const existingState = { ...getState() };+ const docs = [...(existingState?.indexDocs || [])];+ docs.push(docId);++ existingState.indexDocs = docs;+ setState(existingState);+ },+ deleteIndexDoc: (docId) => {+ const existingState = { ...getState() };+ const docs = [...(existingState?.indexDocs || [])].filter(+ (item) => item !== docId+ );+ existingState.indexDocs = docs;+ setState(existingState);+ },}));export { useCustomToolStore };