LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
import { ArrowLeft } from "lucide-react";
import PropTypes from "prop-types";
import { useEffect, useState } from "react";
import { Button } from "@/components/ui/shims/antd-button";
import { Modal } from "@/components/ui/shims/antd-overlays";
import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
import useRequestUrl from "../../../hooks/useRequestUrl";
import { useAlertStore } from "../../../store/alert-store";
import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader";
import { AddSource } from "../add-source/AddSource";
import { ListOfSources } from "../list-of-sources/ListOfSources";
import "./AddSourceModal.css";
function AddSourceModal({
open,
setOpen,
type,
isConnector = false,
connectorMode,
addNewItem,
editItemId,
setEditItemId,
}) {
const [selectedSourceId, setSelectedSourceId] = useState(null);
const [metadata, setMetadata] = useState({});
const [titles, setTitles] = useState({});
const [selectedSourceName, setSelectedSourceName] = useState("");
const [selectedDocUrl, setSelectedDocUrl] = useState("");
const { setAlertDetails } = useAlertStore();
const axiosPrivate = useAxiosPrivate();
const handleException = useExceptionHandler();
const { getUrl } = useRequestUrl();
const [isLoading, setIsLoading] = useState(false);
const [sourcesList, setSourcesList] = useState([]);
const disabledIdsByType = {
EMBEDDING: ["huggingface|90ec9ec2-1768-4d69-8fb1-c88b95de5e5a"],
LLM: ["replicate|2715ce84-05af-4ab4-b8e9-67ac3211b81e"],
VECTOR_DB: [],
};
useEffect(() => {
const addOrEdit = editItemId?.length ? "Edit" : "Add";
setTitles({
llm: addOrEdit + " LLM",
vector_db: addOrEdit + " Vector DB",
embedding: addOrEdit + " Embedding",
x2text: addOrEdit + " Text Extractor",
connectors: addOrEdit + " Connector",
});
if (editItemId?.length) {
getSourceDetails();
}
}, [editItemId]);
useEffect(() => {
if (!open) {
setTimeout(() => {
// A delay added in order to avoid glitch in the UI when the modal is closed.
setSelectedSourceId(null);
setEditItemId(null);
// Clear metadata to prevent stale data when adding a new connector
setMetadata({});
}, 500);
}
getListOfSources();
}, [open]);
useEffect(() => {
const selectedSource = sourcesList.find(
(item) => item?.id === selectedSourceId,
);
setSelectedSourceName(selectedSource?.name);
setSelectedDocUrl(selectedSource?.doc_url || "");
}, [selectedSourceId]);
const getSourceDetails = () => {
let url;
if (isConnector) {
url = getUrl(`connector/${editItemId}/`);
} else {
url = getUrl(`adapter/${editItemId}/`);
}
const requestOptions = {
method: "GET",
url,
};
axiosPrivate(requestOptions)
.then((res) => {
setOpen(true);
const data = res?.data;
if (isConnector) {
setSelectedSourceId(data?.connector_id);
const connectorMetadata = data?.connector_metadata;
connectorMetadata["connectorName"] = data?.connector_name;
setMetadata(connectorMetadata);
} else {
setSelectedSourceId(data?.adapter_id);
const adapterMetadata = data?.adapter_metadata;
adapterMetadata["adapter_name"] = data?.adapter_name;
setMetadata(adapterMetadata);
}
})
.catch((err) => {
setAlertDetails(handleException(err));
setOpen(false);
setEditItemId(null);
});
};
const getListOfSources = () => {
let url;
if (isConnector) {
if (connectorMode) {
url = getUrl(`supported_connectors/?connector_mode=${connectorMode}`);
} else {
url = getUrl(`supported_connectors/`);
}
} else {
if (!type) {
return;
}
url = getUrl(`supported_adapters/?adapter_type=${type.toUpperCase()}`);
}
const requestOptions = {
method: "GET",
url,
};
setIsLoading(true);
setSourcesList([]);
axiosPrivate(requestOptions)
.then((res) => {
const sources = res?.data || [];
const updatedSources = sources?.map((source) => ({
...source,
isDisabled: disabledIdsByType[source?.adapter_type]?.includes(
source?.id,
),
}));
setSourcesList(updatedSources || []);
})
.catch((err) => {
setAlertDetails(handleException(err));
})
.finally(() => {
setIsLoading(false);
});
};
const handleBack = () => {
setSelectedSourceId(null);
setMetadata(null);
};
const showBackButton = selectedSourceId && !editItemId;
const modalTitle = showBackButton ? (
<div style={{ display: "flex", alignItems: "center", gap: "8px" }}>
<Button
type="text"
shape="circle"
size="small"
icon={<ArrowLeft />}
onClick={handleBack}
aria-label="Go back to source selection"
/>
<span>{isConnector ? titles["connectors"] : titles[type]}</span>
</div>
) : isConnector ? (
titles["connectors"]
) : (
titles[type]
);
return (
<Modal
open={open}
onCancel={() => {
setOpen(false);
setMetadata(null);
}}
maskClosable={false}
title={modalTitle}
width={selectedSourceId?.length ? 500 : 1100}
centered
footer={null}
closable={true}
data-testid="add-source-modal"
className="add-source-modal"
>
{selectedSourceId ? (
<AddSource
selectedSourceId={selectedSourceId}
selectedSourceName={selectedSourceName}
setOpen={setOpen}
type={type}
isConnector={isConnector}
addNewItem={addNewItem}
editItemId={editItemId}
metadata={metadata}
selectedDocUrl={selectedDocUrl}
/>
) : isLoading ? (
<SpinnerLoader />
) : (
<ListOfSources
setSelectedSourceId={setSelectedSourceId}
sourcesList={sourcesList}
type={type}
isConnector={isConnector}
connectorMode={connectorMode}
/>
)}
</Modal>
);
}
AddSourceModal.propTypes = {
open: PropTypes.bool.isRequired,
setOpen: PropTypes.func.isRequired,
type: PropTypes.any,
isConnector: PropTypes.bool,
connectorMode: PropTypes.string,
addNewItem: PropTypes.func.isRequired,
editItemId: PropTypes.string,
setEditItemId: PropTypes.func.isRequired,
};
export { AddSourceModal };