LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Feat/app deployment with multidoc chat (#392)
Rahul Johnybd07b59a9a3e
diff --git a/backend/apps/__init__.py b/backend/apps/__init__.pydeleted file mode 100644index e69de29bb..000000000diff --git a/backend/apps/constants.py b/backend/apps/constants.pydeleted file mode 100644index 15297aefc..000000000--- a/backend/apps/constants.py+++ /dev/null@@ -1,2 +0,0 @@-class AppConstants:- """Constants for Apps."""diff --git a/backend/apps/exceptions.py b/backend/apps/exceptions.pydeleted file mode 100644index fb1980ae9..000000000--- a/backend/apps/exceptions.py+++ /dev/null@@ -1,6 +0,0 @@-from rest_framework.exceptions import APIException---class FetchAppListFailed(APIException):- status_code = 400- default_detail = "Failed to fetch App list."diff --git a/backend/apps/urls.py b/backend/apps/urls.pydeleted file mode 100644index d60c9b2bb..000000000--- a/backend/apps/urls.py+++ /dev/null@@ -1,9 +0,0 @@-from apps import views-from django.urls import path-from rest_framework.urlpatterns import format_suffix_patterns--urlpatterns = format_suffix_patterns(- [- path("app/", views.get_app_list, name="app-list"),- ]-)diff --git a/backend/apps/views.py b/backend/apps/views.pydeleted file mode 100644index 944987319..000000000--- a/backend/apps/views.py+++ /dev/null@@ -1,22 +0,0 @@-import logging--from apps.exceptions import FetchAppListFailed-from rest_framework import status-from rest_framework.decorators import api_view-from rest_framework.request import Request-from rest_framework.response import Response--logger = logging.getLogger(__name__)---@api_view(("GET",))-def get_app_list(request: Request) -> Response:- """API to fetch List of Apps."""- if request.method == "GET":- try:- return Response(data=[], status=status.HTTP_200_OK)- # Refactored dated: 19/12/2023- # ( Removed -> backend/apps/app_processor.py )- except Exception as exe:- logger.error(f"Error occured while fetching app list {exe}")- raise FetchAppListFailed()diff --git a/backend/backend/urls.py b/backend/backend/urls.pyindex 9a4f70bc5..e48790d89 100644--- a/backend/backend/urls.py+++ b/backend/backend/urls.py@@ -29,7 +29,6 @@ urlpatterns = [path("", include("file_management.urls")),path("", include("tool_instance.urls")),path("", include("pipeline.urls")),- path("", include("apps.urls")),path("", include("feature_flag.urls")),path("workflow/", include("workflow_manager.urls")),path("platform/", include("platform_settings.urls")),@@ -63,10 +62,21 @@ urlpatterns = []try:- import pluggable_apps.subscription.urls # noqa: F401+ import pluggable_apps.apps.app_deployment.urls # noqa # pylint: disable=unused-import+ import pluggable_apps.apps.canned_question.urls # noqa # pylint: disable=unused-import+ import pluggable_apps.apps.chat_history.urls # noqa # pylint: disable=unused-import+ import pluggable_apps.apps.chat_transcript.urls # noqa # pylint: disable=unused-import+ import pluggable_apps.subscription.urls # noqa # pylint: disable=unused-importurlpatterns += [path("", include("pluggable_apps.subscription.urls")),+ path(+ "canned_question/",+ include("pluggable_apps.apps.canned_question.urls"),+ ),+ path("app/", include("pluggable_apps.apps.app_deployment.urls")),+ path("chat_history/", include("pluggable_apps.apps.chat_history.urls")),+ path("chat/", include("pluggable_apps.apps.chat_transcript.urls")),]except ImportError:passdiff --git a/backend/connector/migrations/0003_alter_connectorinstance_connector_mode.py b/backend/connector/migrations/0003_alter_connectorinstance_connector_mode.pynew file mode 100644index 000000000..5e5f1d50e--- /dev/null+++ b/backend/connector/migrations/0003_alter_connectorinstance_connector_mode.py@@ -0,0 +1,27 @@+# Generated by Django 4.2.1 on 2024-06-24 12:51++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("connector", "0002_connectorinstance_connector_metadata_b"),+ ]++ operations = [+ migrations.AlterField(+ model_name="connectorinstance",+ name="connector_mode",+ field=models.CharField(+ choices=[+ (0, "UNKNOWN"),+ (1, "FILE_SYSTEM"),+ (2, "DATABASE"),+ (3, "APPDEPLOYMENT"),+ ],+ db_comment="Choices of connectors",+ default=0,+ ),+ ),+ ]diff --git a/backend/connector/models.py b/backend/connector/models.pyindex 352ee3694..19b845341 100644--- a/backend/connector/models.py+++ b/backend/connector/models.py@@ -29,6 +29,7 @@ class ConnectorInstance(BaseModel):UNKNOWN = 0, "UNKNOWN"FILE_SYSTEM = 1, "FILE_SYSTEM"DATABASE = 2, "DATABASE"+ APPDEPLOYMENT = 3, "APPDEPLOYMENT"id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)connector_name = models.TextField(@@ -62,7 +63,7 @@ class ConnectorInstance(BaseModel):connector_mode = models.CharField(choices=ConnectorMode.choices,default=ConnectorMode.UNKNOWN,- db_comment="0: UNKNOWN, 1: FILE_SYSTEM, 2: DATABASE",+ db_comment="Choices of connectors",)created_by = models.ForeignKey(diff --git a/backend/workflow_manager/endpoint/migrations/0002_alter_workflowendpoint_connection_type.py b/backend/workflow_manager/endpoint/migrations/0002_alter_workflowendpoint_connection_type.pynew file mode 100644index 000000000..d0e56e146--- /dev/null+++ b/backend/workflow_manager/endpoint/migrations/0002_alter_workflowendpoint_connection_type.py@@ -0,0 +1,27 @@+# Generated by Django 4.2.1 on 2024-06-06 06:26++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("endpoint", "0001_initial"),+ ]++ operations = [+ migrations.AlterField(+ model_name="workflowendpoint",+ name="connection_type",+ field=models.CharField(+ blank=True,+ choices=[+ ("FILESYSTEM", "FileSystem connector"),+ ("DATABASE", "Database Connector"),+ ("API", "API Connector"),+ ("APPDEPLOYMENT", "App Deployment"),+ ],+ db_comment="Connection type (Filesystem, Database or API)",+ ),+ ),+ ]diff --git a/backend/workflow_manager/endpoint/models.py b/backend/workflow_manager/endpoint/models.pyindex a90c583ac..96f64b58c 100644--- a/backend/workflow_manager/endpoint/models.py+++ b/backend/workflow_manager/endpoint/models.py@@ -15,6 +15,7 @@ class WorkflowEndpoint(BaseModel):FILESYSTEM = "FILESYSTEM", "FileSystem connector"DATABASE = "DATABASE", "Database Connector"API = "API", "API Connector"+ APPDEPLOYMENT = "APPDEPLOYMENT", "App Deployment"id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)workflow = models.ForeignKey(diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsxindex 1963dbe2c..bf0d78e4d 100644--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx@@ -33,21 +33,6 @@ const tooltip = {output: "Data Destination Settings",};-const inputOptions = [- {- value: "API",- label: "API",- },- {- value: "FILESYSTEM",- label: "File System",- },- {- value: "DATABASE",- label: "Database",- },-];-const disabledIdsByType = {FILE_SYSTEM: ["box|4d94d237-ce4b-45d8-8f34-ddeefc37c0bf",@@ -79,17 +64,48 @@ function DsSettingsCard({ type, endpointDetails, message }) {const [selectedId, setSelectedId] = useState("");const [selectedItemName, setSelectedItemName] = useState("");+ const [inputOptions, setInputOptions] = useState([+ {+ value: "API",+ label: "API",+ },+ {+ value: "FILESYSTEM",+ label: "File System",+ },+ {+ value: "DATABASE",+ label: "Database",+ },+ ]);+const { sessionDetails } = useSessionStore();const { updateWorkflow } = useWorkflowStore();const { setAlertDetails } = useAlertStore();const axiosPrivate = useAxiosPrivate();const handleException = useExceptionHandler();+ const { flags } = sessionDetails;+const icons = {input: <ImportOutlined className="ds-set-icon-size" />,output: <ExportOutlined className="ds-set-icon-size" />,};+ useEffect(() => {+ try {+ const inputOption =+ require("../../../plugins/dscard-input-options/AppDeploymentCardInputOptions").appDeploymentInputOption;+ if (flags.app_deployment && inputOption) {+ const updatedInputOptions = inputOptions;+ updatedInputOptions.push(inputOption);+ setInputOptions(updatedInputOptions);+ }+ } catch {+ // The component will remain null of it is not available+ }+ }, []);+useEffect(() => {if (type === "output") {if (source?.connection_type === "") {@@ -108,7 +124,8 @@ function DsSettingsCard({ type, endpointDetails, message }) {if (type === "input") {// Remove Database from Source Dropdownconst filteredOptions = inputOptions.filter(- (option) => option.value !== "DATABASE"+ (option) =>+ option.value !== "DATABASE" && option.value !== "APPDEPLOYMENT");setOptions(filteredOptions);}@@ -357,7 +374,9 @@ function DsSettingsCard({ type, endpointDetails, message }) {size="small"onClick={() => setOpenModal(true)}disabled={- !endpointDetails?.connection_type || connType === "API"+ !endpointDetails?.connection_type ||+ connType === "API" ||+ connType === "APPDEPLOYMENT"}><SettingOutlined />diff --git a/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx b/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsxindex 2e881a32b..60f54256a 100644--- a/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx+++ b/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx@@ -19,7 +19,7 @@ import { useSessionStore } from "../../../store/session-store";let getMenuItem;try {- getMenuItem = require("../../../plugins/app-deployments/app-deployment-components/helpers/getMenuItem");+ getMenuItem = require("../../../plugins/app-deployment/getMenuItem");} catch (err) {// Plugin unavailable.}