LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Fix/etl task pipeline bugs (#45)
Siddhiqbb9775b4965e
diff --git a/backend/connector/connector_instance_helper.py b/backend/connector/connector_instance_helper.pyindex 098f5a691..2589d3c17 100644--- a/backend/connector/connector_instance_helper.py+++ b/backend/connector/connector_instance_helper.py@@ -7,9 +7,10 @@ from connector.models import ConnectorInstancefrom connector.unstract_account import UnstractAccountfrom django.conf import settingsfrom django.db import connection+from workflow_manager.workflow.models.workflow import Workflow+from unstract.connectors.filesystems.ucs import UnstractCloudStoragefrom unstract.connectors.filesystems.ucs.constants import UCSKey-from workflow_manager.workflow.models.workflow import Workflowlogger = logging.getLogger(__name__)@@ -314,3 +315,24 @@ class ConnectorInstanceHelper:ConnectorInstance.ConnectorMode.DATABASE,values,)++ @staticmethod+ def get_input_output_connector_instances_by_workflow(+ workflow_id: str,+ ) -> list[ConnectorInstance]:+ """Method to fetch input and output connectors by workflow.++ Args:+ workflow_id (str)++ Returns:+ list[ConnectorInstance]+ """+ filter_params: dict[str, Any] = {+ "workflow": workflow_id,+ }+ connector_instances = ConnectorInstance.objects.filter(+ **filter_params+ ).all()+ logger.info(f"Retrived connector instance values {connector_instances}")+ return list(connector_instances)diff --git a/backend/pipeline/serializers/crud.py b/backend/pipeline/serializers/crud.pyindex a42dd63a3..07c2f781a 100644--- a/backend/pipeline/serializers/crud.py+++ b/backend/pipeline/serializers/crud.py@@ -2,7 +2,7 @@ import loggingfrom collections import OrderedDictfrom typing import Any-from backend.serializers import AuditSerializer+from connector.connector_instance_helper import ConnectorInstanceHelperfrom connector.models import ConnectorInstancefrom connector_processor.connector_processor import ConnectorProcessorfrom cron_expression_generator.constants import CronKeys@@ -14,11 +14,12 @@ from pipeline.models import Pipelinefrom rest_framework.exceptions import ValidationErrorfrom rest_framework.serializers import UUIDFieldfrom scheduler.helper import SchedulerHelper-from tool_instance.serializers import ToolInstanceSerializer-from tool_instance.tool_instance_helper import ToolInstanceHelperfrom utils.serializer_utils import SerializerUtilsfrom workflow_manager.workflow.constants import WorkflowKey+from backend.serializers import AuditSerializer+from unstract.connectors.connectorkit import Connectorkit+logger = logging.getLogger(__name__)@@ -81,30 +82,68 @@ class PipelineSerializer(AuditSerializer):}return repr+ def _get_name_and_icon(+ self, connectors: list[Any], connector_id: Any+ ) -> Any:+ for obj in connectors:+ if obj["id"] == connector_id:+ return obj["name"], obj["icon"]+ return PC.NOT_CONFIGURED, None++ def _add_connector_data(+ self,+ repr: OrderedDict[str, Any],+ connector_instance_list: list[Any],+ connectors: list[Any],+ ) -> OrderedDict[str, Any]:+ """Adds connector Input/Output data.++ Args:+ sef (_type_): _description_+ repr (OrderedDict[str, Any]): _description_++ Returns:+ OrderedDict[str, Any]: _description_+ """+ repr[PC.SOURCE_NAME] = PC.NOT_CONFIGURED+ repr[PC.DESTINATION_NAME] = PC.NOT_CONFIGURED+ for instance in connector_instance_list:+ if instance.connector_type == "INPUT":+ repr[PC.SOURCE_NAME], repr[PC.SOURCE_ICON] = (+ self._get_name_and_icon(+ connectors=connectors,+ connector_id=instance.connector_id,+ )+ )+ if instance.connector_type == "OUTPUT":+ repr[PC.DESTINATION_NAME], repr[PC.DESTINATION_ICON] = (+ self._get_name_and_icon(+ connectors=connectors,+ connector_id=instance.connector_id,+ )+ )+ return repr+def to_representation(self, instance: Pipeline) -> OrderedDict[str, Any]:"""To set Source, Destination & Agency for Pipelines."""repr: OrderedDict[str, Any] = super().to_representation(instance)+ connector_kit = Connectorkit()+ connectors = connector_kit.get_connectors_list()+if SerializerUtils.check_context_for_GET_or_POST(context=self.context):workflow = instance.workflow- tool_instances = ToolInstanceSerializer(- instance=ToolInstanceHelper.get_tool_instances_by_workflow(- workflow_id=instance.workflow_id, order_by="step"- ),- many=True,- context=self.context,- ).data+ connector_instance_list = ConnectorInstanceHelper.get_input_output_connector_instances_by_workflow( # noqa+ workflow.id+ )repr[PK.WORKFLOW_ID] = workflow.idrepr[PK.WORKFLOW_NAME] = workflow.workflow_namerepr = self._add_cron_summary(repr=repr)-- if not tool_instances:- repr[PC.SOURCE_NAME] = PC.SOURCE_NOT_CONFIGURED- repr[PC.DESTINATION_NAME] = PC.DESTINATION_NOT_CONFIGURED- return repr-- # TODO: Change here to handle pipeline src/dest based on WF- repr[PC.DESTINATION_NAME] = PC.NOT_CONFIGURED+ repr = self._add_connector_data(+ repr=repr,+ connector_instance_list=connector_instance_list,+ connectors=connectors,+ )return reprdiff --git a/frontend/package-lock.json b/frontend/package-lock.jsonindex 76884e3f5..91399eb6c 100644--- a/frontend/package-lock.json+++ b/frontend/package-lock.json@@ -27,6 +27,7 @@"js-yaml": "^4.1.0","markdown-to-jsx": "^7.2.1","moment": "^2.29.4",+ "moment-timezone": "^0.5.45","pdfjs-dist": "^3.4.120","prismjs": "^1.29.0","react": "^18.2.0",@@ -13958,6 +13959,17 @@"node": "*"}},+ "node_modules/moment-timezone": {+ "version": "0.5.45",+ "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.45.tgz",+ "integrity": "sha512-HIWmqA86KcmCAhnMAN0wuDOARV/525R2+lOLotuGFzn4HO+FH+/645z2wx0Dt3iDv6/p61SIvKnDstISainhLQ==",+ "dependencies": {+ "moment": "^2.29.4"+ },+ "engines": {+ "node": "*"+ }+ },"node_modules/mri": {"version": "1.2.0","resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz",@@ -30457,6 +30469,14 @@"resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz","integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w=="},+ "moment-timezone": {+ "version": "0.5.45",+ "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.45.tgz",+ "integrity": "sha512-HIWmqA86KcmCAhnMAN0wuDOARV/525R2+lOLotuGFzn4HO+FH+/645z2wx0Dt3iDv6/p61SIvKnDstISainhLQ==",+ "requires": {+ "moment": "^2.29.4"+ }+ },"mri": {"version": "1.2.0","resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz",diff --git a/frontend/package.json b/frontend/package.jsonindex 755de543c..939df4ff2 100644--- a/frontend/package.json+++ b/frontend/package.json@@ -22,6 +22,7 @@"js-yaml": "^4.1.0","markdown-to-jsx": "^7.2.1","moment": "^2.29.4",+ "moment-timezone": "^0.5.45","pdfjs-dist": "^3.4.120","prismjs": "^1.29.0","react": "^18.2.0",diff --git a/frontend/src/components/agency/actions/Actions.jsx b/frontend/src/components/agency/actions/Actions.jsxindex 98f89141f..1f8f22920 100644--- a/frontend/src/components/agency/actions/Actions.jsx+++ b/frontend/src/components/agency/actions/Actions.jsx@@ -19,8 +19,10 @@ import { useAlertStore } from "../../../store/alert-store";import { useSessionStore } from "../../../store/session-store";import { useWorkflowStore } from "../../../store/workflow-store";import { CreateApiDeploymentModal } from "../../deployments/create-api-deployment-modal/CreateApiDeploymentModal.jsx";+import { EtlTaskDeploy } from "../../pipelines-or-deployments/etl-task-deploy/EtlTaskDeploy.jsx";import { SocketMessages } from "../../helpers/socket-messages/SocketMessages";import FileUpload from "../file-upload/FileUpload.jsx";+import { deploymentsStaticContent } from "../../../helpers/GetStaticData";import "./Actions.css";function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {@@ -33,9 +35,19 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {const [wfExecutionParams, setWfExecutionParams] = useState([]);const [openAddApiModal, setOpenAddApiModal] = useState(false);const [apiOpsPresent, setApiOpsPresent] = useState(false);-- const { details, isLoading, loadingType, updateWorkflow, source } =- useWorkflowStore();+ const [canAddTaskPipeline, setCanAddTaskPipeline] = useState(false);+ const [canAddETLPipeline, setCanAddETAPipeline] = useState(false);+ const [openAddTaskModal, setOpenAddTaskModal] = useState(false);+ const [openAddETLModal, setOpenAddETLModal] = useState(false);++ const {+ details,+ isLoading,+ loadingType,+ updateWorkflow,+ source,+ destination,+ } = useWorkflowStore();const { setAlertDetails } = useAlertStore();const { sessionDetails } = useSessionStore();@@ -45,6 +57,11 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {setApiOpsPresent(source?.connection_type === "API");}, [source]);+ useEffect(() => {+ setCanAddTaskPipeline(destination?.connection_type === "FILESYSTEM");+ setCanAddETAPipeline(destination?.connection_type === "DATABASE");+ }, [destination]);+useEffect(() => {if (stepExecType === wfExecutionTypes[1]) {setStepExecType("");@@ -233,7 +250,7 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {setExecutionId("");};- const createAPIDeployment = () => {+ const createDeployment = (type) => {const workflowId = details?.id;if (!workflowId) {setAlertDetails({@@ -242,7 +259,15 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {});return;}- setOpenAddApiModal(true);+ if (type === "API") {+ setOpenAddApiModal(true);+ }+ if (type === "TASK") {+ setOpenAddTaskModal(true);+ }+ if (type === "ETL") {+ setOpenAddETLModal(true);+ }};const handleClearCache = () => {@@ -381,13 +406,27 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {</Button></Tooltip><Divider type="vertical" />+ <Tooltip title="Deploy as API">+ <Button+ disabled={!apiOpsPresent}+ onClick={() => createDeployment("API")}+ >+ <ApiOutlined />+ </Button>+ </Tooltip><Tooltip title="Deploy as ETL Pipeline">- <Button disabled={true}>+ <Button+ disabled={!canAddETLPipeline}+ onClick={() => createDeployment("ETL")}+ ><DeploymentUnitOutlined /></Button></Tooltip>- <Tooltip title="Deploy API">- <Button disabled={!apiOpsPresent} onClick={createAPIDeployment}>+ <Tooltip title="Deploy as Task Pipeline">+ <Button+ disabled={!canAddTaskPipeline}+ onClick={() => createDeployment("TASK")}+ ><ApiOutlined /></Button></Tooltip>@@ -414,6 +453,24 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {workflowId={details?.id}/>)}+ {openAddTaskModal && (+ <EtlTaskDeploy+ open={openAddTaskModal}+ setOpen={setOpenAddTaskModal}+ type="task"+ title={deploymentsStaticContent["task"].modalTitle}+ workflowId={details?.id}+ />+ )}+ {openAddETLModal && (+ <EtlTaskDeploy+ open={openAddETLModal}+ setOpen={setOpenAddETLModal}+ type="etl"+ title={deploymentsStaticContent["etl"].modalTitle}+ workflowId={details?.id}+ />+ )}<SocketMessages logId={logId} /></>);diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsxindex 1af2e6815..7f5f55bec 100644--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx@@ -3,6 +3,7 @@ import {ExportOutlined,ImportOutlined,SettingOutlined,+ CheckCircleTwoTone,} from "@ant-design/icons";import {Button,@@ -17,7 +18,7 @@ import {import PropTypes from "prop-types";import { useEffect, useState } from "react";-import { handleException } from "../../../helpers/GetStaticData";+import { handleException, titleCase } from "../../../helpers/GetStaticData";import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";import { useAlertStore } from "../../../store/alert-store";import { useSessionStore } from "../../../store/session-store";@@ -41,7 +42,7 @@ const inputOptions = [},];-function DsSettingsCard({ type, endpointDetails, message }) {+function DsSettingsCard({ type, endpointDetails, message, dependent }) {const [options, setOptions] = useState([...inputOptions]);const [openModal, setOpenModal] = useState(false);@@ -260,7 +261,9 @@ function DsSettingsCard({ type, endpointDetails, message }) {type="text"size="small"onClick={() => setOpenModal(true)}- disabled={!endpointDetails?.connection_type}+ disabled={+ !endpointDetails?.connection_type || connType === "API"+ }><SettingOutlined /></Button>@@ -281,7 +284,18 @@ function DsSettingsCard({ type, endpointDetails, message }) {</Space>) : (<>- {connType !== "API" && (+ {connType === "API" ? (+ <Typography.Text+ className="font-size-12 display-flex-align-center"+ ellipsis={{ rows: 1, expandable: false }}+ type="secondary"+ >+ <CheckCircleTwoTone twoToneColor="#52c41a" />+ <span style={{ marginLeft: "5px" }}>+ {titleCase(type)} set to API successfully+ </span>+ </Typography.Text>+ ) : (<Typography.TextclassName="font-size-12 display-flex-align-center"ellipsis={{ rows: 1, expandable: false }}@@ -331,6 +345,7 @@ DsSettingsCard.propTypes = {type: PropTypes.string.isRequired,endpointDetails: PropTypes.object.isRequired,message: PropTypes.string,+ dependent: PropTypes.object.isRequired,};export { DsSettingsCard };diff --git a/frontend/src/components/agency/steps/Steps.jsx b/frontend/src/components/agency/steps/Steps.jsxindex 1c75435c7..3c166108d 100644--- a/frontend/src/components/agency/steps/Steps.jsx+++ b/frontend/src/components/agency/steps/Steps.jsx@@ -158,6 +158,7 @@ function Steps({ steps, setSteps, activeToolId, sourceMsg, destinationMsg }) {type={sourceTypes.connectors[0]}endpointDetails={source}message={sourceMsg}+ dependent={destination}/><Divider className="wf-steps-div" /></div>@@ -184,6 +185,7 @@ function Steps({ steps, setSteps, activeToolId, sourceMsg, destinationMsg }) {type={sourceTypes.connectors[1]}endpointDetails={destination}message={destinationMsg}+ dependent={source}/></div></div>diff --git a/frontend/src/components/pipelines-or-deployments/etl-task-deploy/EtlTaskDeploy.jsx b/frontend/src/components/pipelines-or-deployments/etl-task-deploy/EtlTaskDeploy.jsxindex 000430a03..49bf2c470 100644--- a/frontend/src/components/pipelines-or-deployments/etl-task-deploy/EtlTaskDeploy.jsx+++ b/frontend/src/components/pipelines-or-deployments/etl-task-deploy/EtlTaskDeploy.jsx@@ -37,7 +37,7 @@ const EtlTaskDeploy = ({type,title,setTableData,- workflowData = {},+ workflowId,}) => {const { sessionDetails } = useSessionStore();const { setAlertDetails } = useAlertStore();@@ -54,7 +54,13 @@ const EtlTaskDeploy = ({const [isCronStringValid, setCronStringValid] = useState(true);const [isLoading, setLoading] = useState(false);- const getWorkflows = () => {+ useEffect(() => {+ if (workflowId) {+ setFormDetails({ ...formDetails, workflow_id: workflowId });+ }+ }, [workflowId]);++ const getWorkflowList = () => {workflowApiService.getWorkflowList().then((res) => {@@ -64,10 +70,29 @@ const EtlTaskDeploy = ({console.error("Unable to get workflow list");});};+ const getWorkflows = () => {+ const connectorType = type === "task" ? "FILESYSTEM" : "DATABASE";+ workflowApiService+ .getWorkflowEndpointList("DESTINATION", connectorType)+ .then((res) => {+ const updatedData = res?.data.map((record) => ({+ ...record,+ id: record.workflow,+ }));+ setWorkflowList(updatedData);+ })+ .catch(() => {+ console.error("Unable to get workflow list");+ });+ };useEffect(() => {- getWorkflows();- }, []);+ if (type === "app") {+ getWorkflowList();+ } else {+ getWorkflows();+ }+ }, [type]);const onChangeHandler = (propertyName, value) => {const body = {@@ -75,9 +100,6 @@ const EtlTaskDeploy = ({};setFormDetails({ ...formDetails, ...body });};- useEffect(() => {- getWorkflows();- }, []);useEffect(() => {if (open) {@@ -86,12 +108,6 @@ const EtlTaskDeploy = ({}}, [open]);- useEffect(() => {- if (Object.keys(workflowData).length) {- setFormDetails({ ...formDetails, workflow_id: workflowData.workflow_id });- }- }, [workflowData]);-useEffect(() => {const cronString = formDetails?.cron_string || "";if (cronString?.length === 0 || isValidCron(cronString)) {@@ -247,7 +263,7 @@ const EtlTaskDeploy = ({setLoading(true);axiosPrivate(requestOptions).then((res) => {- if (!Object.keys(workflowData)?.length) {+ if (!workflowId) {addPipeline(res?.data);}setOpen(false);@@ -291,7 +307,7 @@ const EtlTaskDeploy = ({value={formDetails.pipeline_name || ""}></Input></SpaceWrapper>- {Object.keys(workflowData).length === 0 && (+ {!workflowId && (<SpaceWrapper><Typography>Workflow</Typography><Select@@ -382,6 +398,6 @@ EtlTaskDeploy.propTypes = {type: PropTypes.string.isRequired,title: PropTypes.string.isRequired,setTableData: PropTypes.func,- workflowData: PropTypes.object,+ workflowId: PropTypes.string,};export { EtlTaskDeploy };diff --git a/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx b/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsxindex cb9743481..89360ecc2 100644--- a/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx+++ b/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx@@ -1,13 +1,11 @@import {- ArrowRightOutlined,ClearOutlined,DeleteOutlined,EditOutlined,EllipsisOutlined,EyeOutlined,- HighlightOutlined,- InfoCircleOutlined,SyncOutlined,+ HighlightOutlined,} from "@ant-design/icons";import { Dropdown, Image, Space, Switch, Typography } from "antd";import PropTypes from "prop-types";@@ -26,6 +24,7 @@ import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader.jsx";import { DeleteModal } from "../delete-modal/DeleteModal.jsx";import { EtlTaskDeploy } from "../etl-task-deploy/EtlTaskDeploy.jsx";import "./Pipelines.css";+import { formattedDateTime } from "../../../helpers/GetStaticData";function Pipelines({ type }) {const [tableData, setTableData] = useState([]);@@ -265,19 +264,6 @@ function Pipelines({ type }) {},{key: "4",- label: (- <Space direction="horizontal" className="action-items">- <div>- <InfoCircleOutlined />- </div>- <div>- <Typography.Text>View Information</Typography.Text>- </div>- </Space>- ),- },- {- key: "5",label: (<Spacedirection="horizontal"@@ -294,7 +280,7 @@ function Pipelines({ type }) {),},{- key: "6",+ key: "5",label: (<Spacedirection="horizontal"@@ -311,7 +297,7 @@ function Pipelines({ type }) {),},{- key: "7",+ key: "6",label: (<Spacedirection="horizontal"@@ -331,26 +317,6 @@ function Pipelines({ type }) {</Space>),},- {- key: "8",- label: (- <Space- direction="horizontal"- className="action-items"- onClick={() => window.open(selectedPorD?.goto, "_blank")}- >- <div>- <ArrowRightOutlined />- </div>- <div>- <Typography.Text disabled={type !== "app"}>- Go to App- </Typography.Text>- </div>- </Space>- ),- disabled: type !== "app",- },];const columns = [@@ -399,7 +365,7 @@ function Pipelines({ type }) {align: "center",},{- title: "Status",+ title: "Status of Previous Run",dataIndex: "last_run_status",key: "last_run_status",align: "center",@@ -416,14 +382,27 @@ function Pipelines({ type }) {),},{- title: "Last Run",+ title: "Previous Run At",+ key: "last_run_time",+ dataIndex: "last_run_time",+ align: "center",+ render: (_, record) => (+ <div>+ <Typography.Text className="p-or-d-typography" strong>+ {formattedDateTime(record?.last_run_time)}+ </Typography.Text>+ </div>+ ),+ },+ {+ title: "Frequency",key: "last_run_time",dataIndex: "last_run_time",align: "center",render: (_, record) => (<div><Typography.Text className="p-or-d-typography" strong>- {record?.last_run_time}+ {record?.cron_data?.cron_summary}</Typography.Text></div>),diff --git a/frontend/src/helpers/GetStaticData.js b/frontend/src/helpers/GetStaticData.jsindex 7e72776e2..72c4b0353 100644--- a/frontend/src/helpers/GetStaticData.js+++ b/frontend/src/helpers/GetStaticData.js@@ -1,3 +1,6 @@+import moment from "moment";+import momentTz from "moment-timezone";+const THEME = {DARK: "dark",LIGHT: "light",@@ -327,6 +330,19 @@ const onboardCompleted = (adaptersList) => {return MANDATORY_ADAPTERS.every((value) => adaptersList.includes(value));};+// Input: ISOdateTime format+// Output: Mar 10, 2023 7:33 PM IST+const formattedDateTime = (ISOdateTime) => {+ if (ISOdateTime) {+ const validIsoDate = moment.utc(ISOdateTime).toISOString();+ // eslint-disable-next-line new-cap+ const zone = Intl.DateTimeFormat().resolvedOptions().timeZone;+ return momentTz.tz(validIsoDate, zone).format("lll z");+ } else {+ return "";+ }+};+const getBackendErrorDetail = (attr, backendErrors) => {if (backendErrors) {const error = backendErrors?.errors.find((error) => error?.attr === attr);@@ -335,6 +351,17 @@ const getBackendErrorDetail = (attr, backendErrors) => {return null;};+const titleCase = (str) => {+ if (str === null || str.length === 0) {+ return "";+ }+ const words = str.toLowerCase().split(" ");+ for (let i = 0; i < words.length; i++) {+ words[i] = words[i][0].toUpperCase() + words[i].slice(1);+ }+ return words.join(" ");+};+export {CONNECTOR_TYPE_MAP,O_AUTH_PROVIDERS,@@ -344,6 +371,7 @@ export {deploymentsStaticContent,endpointType,formatBytes,+ formattedDateTime,getBaseUrl,getOrgNameFromPathname,getReadableDateAndTime,@@ -365,4 +393,5 @@ export {isJson,displayPromptResult,getBackendErrorDetail,+ titleCase,};