Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2682 [FEAT] Centralize connector management (#1476)

Chandrasekharan Mf90b141de739
diff --git a/backend/adapter_processor_v2/views.py b/backend/adapter_processor_v2/views.py
index 66da2b8cf..36a2db011 100644
--- a/backend/adapter_processor_v2/views.py
+++ b/backend/adapter_processor_v2/views.py
@@ -346,12 +346,6 @@ class AdapterInstanceViewSet(ModelViewSet):
def update(
self, request: Request, *args: tuple[Any], **kwargs: dict[str, Any]
) -> Response:
- import logging
-
- logger = logging.getLogger(__name__)
-
- logger.info("ADAPTER UPDATE CALLED")
-
# Check if adapter metadata is being updated and contains the platform key flag
use_platform_unstract_key = False
adapter_metadata = request.data.get(AdapterKeys.ADAPTER_METADATA)
diff --git a/backend/connector_processor/connector_processor.py b/backend/connector_processor/connector_processor.py
index c621956f7..b1b3dc8a5 100644
--- a/backend/connector_processor/connector_processor.py
+++ b/backend/connector_processor/connector_processor.py
@@ -10,8 +10,8 @@ from connector_v2.constants import ConnectorInstanceKey as CIKey
from backend.exceptions import UnstractFSException
from connector_processor.constants import ConnectorKeys
from connector_processor.exceptions import (
- InValidConnectorId,
- InValidConnectorMode,
+ InvalidConnectorID,
+ InvalidConnectorMode,
OAuthTimeOut,
TestConnectorInputError,
)
@@ -20,6 +20,7 @@ from unstract.connectors.connectorkit import Connectorkit
from unstract.connectors.enums import ConnectorMode
from unstract.connectors.exceptions import ConnectorError, FSAccessDeniedError
from unstract.connectors.filesystems.ucs import UnstractCloudStorage
+from unstract.connectors.queues.redis import RedisQueue
logger = logging.getLogger(__name__)
@@ -45,12 +46,9 @@ class ConnectorProcessor:
return schema_details
updated_connectors = fetch_connectors_by_key_value(ConnectorKeys.ID, connector_id)
if len(updated_connectors) == 0:
- logger.error(
- f"Invalid connector Id : {connector_id} "
- f"while fetching "
- f"JSON Schema"
+ raise InvalidConnectorID(
+ f"Invalid connector ID '{connector_id}' while fetching JSON Schema"
)
- raise InValidConnectorId()
connector = updated_connectors[0]
schema_details[ConnectorKeys.OAUTH] = connector.get(ConnectorKeys.OAUTH)
@@ -69,30 +67,40 @@ class ConnectorProcessor:
@staticmethod
def get_all_supported_connectors(
- type: str, connector_mode: ConnectorMode | None = None
+ type: str | None = None, connector_mode: ConnectorMode | None = None
) -> list[dict]:
"""Function to return list of all supported connectors except PCS."""
supported_connectors = []
updated_connectors = []
+ # Connectors that are marked active but not supported explicitly
+ unsupported_connectors = [RedisQueue.get_id()]
+
if type == ConnectorKeys.INPUT:
updated_connectors = fetch_connectors_by_key_value(
ConnectorKeys.CAN_READ, True, connector_mode=connector_mode
)
- if type == ConnectorKeys.OUTPUT:
+ elif type == ConnectorKeys.OUTPUT:
updated_connectors = fetch_connectors_by_key_value(
ConnectorKeys.CAN_WRITE, True, connector_mode=connector_mode
)
-
- for each_connector in updated_connectors:
+ else:
+ # When type is None, get all connectors directly
+ connector_kit = Connectorkit()
+ updated_connectors = connector_kit.get_connectors_list(mode=connector_mode)
+
+ for connector in updated_connectors:
+ if connector.get(ConnectorKeys.ID) in unsupported_connectors:
+ continue
+ mode = connector.get(CIKey.CONNECTOR_MODE)
supported_connectors.append(
{
- ConnectorKeys.ID: each_connector.get(ConnectorKeys.ID),
- ConnectorKeys.NAME: each_connector.get(ConnectorKeys.NAME),
- ConnectorKeys.DESCRIPTION: each_connector.get(
- ConnectorKeys.DESCRIPTION
- ),
- ConnectorKeys.ICON: each_connector.get(ConnectorKeys.ICON),
- CIKey.CONNECTOR_MODE: each_connector.get(CIKey.CONNECTOR_MODE).name,
+ ConnectorKeys.ID: connector.get(ConnectorKeys.ID),
+ ConnectorKeys.NAME: connector.get(ConnectorKeys.NAME),
+ ConnectorKeys.DESCRIPTION: connector.get(ConnectorKeys.DESCRIPTION),
+ ConnectorKeys.ICON: connector.get(ConnectorKeys.ICON),
+ ConnectorKeys.CAN_READ: connector.get(ConnectorKeys.CAN_READ),
+ ConnectorKeys.CAN_WRITE: connector.get(ConnectorKeys.CAN_WRITE),
+ CIKey.CONNECTOR_MODE: getattr(mode, "value", None),
}
)
@@ -112,8 +120,7 @@ class ConnectorProcessor:
)
except Exception as exc:
logger.error(
- f"Error while testing file based OAuth supported "
- f"connectors: {exc}"
+ f"Error while testing file based OAuth supported connectors: {exc}"
)
raise OAuthTimeOut()
@@ -131,8 +138,9 @@ class ConnectorProcessor:
"""Generic Function to get connector data with provided key."""
updated_connectors = fetch_connectors_by_key_value("id", connector_id)
if len(updated_connectors) == 0:
- logger.error(f"Invalid connector ID {connector_id} while invoking utility")
- raise InValidConnectorId()
+ raise InvalidConnectorID(
+ f"Invalid connector ID '{connector_id}' while invoking utility"
+ )
return fetch_connectors_by_key_value("id", connector_id)[0].get(key_value)
@staticmethod
@@ -151,5 +159,5 @@ class ConnectorProcessor:
try:
connector_mode = ConnectorMode(connector_mode)
except ValueError:
- raise InValidConnectorMode
+ raise InvalidConnectorMode
return connector_mode
diff --git a/backend/connector_processor/exceptions.py b/backend/connector_processor/exceptions.py
index 22a1e2966..36508038c 100644
--- a/backend/connector_processor/exceptions.py
+++ b/backend/connector_processor/exceptions.py
@@ -4,41 +4,21 @@ from backend.exceptions import UnstractBaseException
from unstract.connectors.exceptions import ConnectorError
-class IdIsMandatory(APIException):
- status_code = 400
- default_detail = "ID is Mandatory."
-
-
-class InValidType(APIException):
- status_code = 400
- default_detail = "Type is not Valid."
-
-
-class InValidConnectorMode(APIException):
+class InvalidConnectorMode(APIException):
status_code = 400
default_detail = "Connector mode is not Valid."
-class InValidConnectorId(APIException):
+class InvalidConnectorID(APIException):
status_code = 400
default_detail = "Connector ID is not Valid."
-class JSONParseException(APIException):
- status_code = 500
- default_detail = "Exception occured while Parsing JSON Schema."
-
-
class OAuthTimeOut(APIException):
status_code = 408
default_detail = "Timed out. Please re-authenticate."
-class InternalServiceError(APIException):
- status_code = 500
- default_detail = "Internal Service error"
-
-
class TestConnectorException(APIException):
status_code = 500
default_detail = "Error while testing connector."
diff --git a/backend/connector_processor/serializers.py b/backend/connector_processor/serializers.py
index 81fcf5669..bd0bcf0b8 100644
--- a/backend/connector_processor/serializers.py
+++ b/backend/connector_processor/serializers.py
@@ -1,8 +1,32 @@
from rest_framework import serializers
from backend.constants import FieldLengthConstants as FLC
+from connector_processor.constants import ConnectorKeys
class TestConnectorSerializer(serializers.Serializer):
connector_id = serializers.CharField(max_length=FLC.CONNECTOR_ID_LENGTH)
connector_metadata = serializers.JSONField()
+
+
+class ConnectorSchemaQuerySerializer(serializers.Serializer):
+ """Serializer for validating connector schema query parameters."""
+
+ id = serializers.CharField(
+ required=True,
+ allow_blank=False,
+ error_messages={"required": "ID is mandatory.", "blank": "ID cannot be empty."},
+ )
+
+
+class SupportedConnectorsQuerySerializer(serializers.Serializer):
+ """Serializer for validating supported connectors query parameters."""
+
+ type = serializers.ChoiceField(
+ choices=[ConnectorKeys.INPUT, ConnectorKeys.OUTPUT],
+ required=False,
+ help_text="Optional filter by connector type. If not provided, returns all connectors.",
+ )
+ connector_mode = serializers.CharField(
+ required=False, allow_blank=True, help_text="Optional connector mode filter"
+ )
diff --git a/backend/connector_processor/urls.py b/backend/connector_processor/urls.py
index a2c67f54a..70a6dd8ed 100644
--- a/backend/connector_processor/urls.py
+++ b/backend/connector_processor/urls.py
@@ -2,19 +2,19 @@ from django.urls import path
from connector_processor.views import ConnectorViewSet
-from . import views
-
connector_test = ConnectorViewSet.as_view({"post": "test"})
+connector_schema = ConnectorViewSet.as_view({"get": "connector_schema"})
+supported_connectors = ConnectorViewSet.as_view({"get": "supported_connectors"})
urlpatterns = [
path(
"connector_schema/",
- views.get_connector_schema,
+ connector_schema,
name="get_connector_schema",
),
path(
"supported_connectors/",
- views.get_supported_connectors,
+ supported_connectors,
name="get_supported_connectors",
),
path("test_connectors/", connector_test, name="connector-test"),
diff --git a/backend/connector_processor/views.py b/backend/connector_processor/views.py
index 4a9509db5..498398f4f 100644
--- a/backend/connector_processor/views.py
+++ b/backend/connector_processor/views.py
@@ -1,8 +1,6 @@
from connector_v2.constants import ConnectorInstanceKey as CIKey
-from django.http.request import HttpRequest
-from django.http.response import HttpResponse
from rest_framework import status
-from rest_framework.decorators import api_view
+from rest_framework.decorators import action
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.serializers import Serializer
@@ -11,48 +9,11 @@ from rest_framework.viewsets import GenericViewSet
from connector_processor.connector_processor import ConnectorProcessor
from connector_processor.constants import ConnectorKeys
-from connector_processor.exceptions import IdIsMandatory, InValidType
-from connector_processor.serializers import TestConnectorSerializer
-
-
-@api_view(("GET",))
-def get_connector_schema(request: HttpRequest) -> HttpResponse:
- if request.method == "GET":
- connector_name = request.GET.get(ConnectorKeys.ID)
- if connector_name is None or connector_name == "":
- raise IdIsMandatory()
- json_schema = ConnectorProcessor.get_json_schema(connector_id=connector_name)
- return Response(data=json_schema, status=status.HTTP_200_OK)
-
-
-@api_view(("GET",))
-def get_supported_connectors(request: HttpRequest) -> HttpResponse:
- """Retrieves a list of supported connectors based on the provided connector
- type and mode.
-
- Args:
- request (HttpRequest): The HTTP request object.
-
- Returns:
- HttpResponse: The HTTP response containing the list of supported
- connectors in JSON format.
- """
- if request.method == "GET":
- connector_type = request.GET.get(ConnectorKeys.TYPE)
- connector_mode = request.GET.get(ConnectorKeys.CONNECTOR_MODE)
- if connector_mode:
- connector_mode = ConnectorProcessor.validate_connector_mode(connector_mode)
-
- if (
- connector_type == ConnectorKeys.INPUT
- or connector_type == ConnectorKeys.OUTPUT
- ):
- json_schema = ConnectorProcessor.get_all_supported_connectors(
- type=connector_type, connector_mode=connector_mode
- )
- return Response(json_schema, status=status.HTTP_200_OK)
- else:
- raise InValidType
+from connector_processor.serializers import (
+ ConnectorSchemaQuerySerializer,
+ SupportedConnectorsQuerySerializer,
+ TestConnectorSerializer,
+)
class ConnectorViewSet(GenericViewSet):
@@ -62,6 +23,10 @@ class ConnectorViewSet(GenericViewSet):
def get_serializer_class(self) -> Serializer:
if self.action == "test":
return TestConnectorSerializer
+ elif self.action == "connector_schema":
+ return ConnectorSchemaQuerySerializer
+ elif self.action == "supported_connectors":
+ return SupportedConnectorsQuerySerializer
return super().get_serializer_class()
def test(self, request: Request) -> Response:
@@ -77,3 +42,40 @@ class ConnectorViewSet(GenericViewSet):
{ConnectorKeys.IS_VALID: test_result},
status=status.HTTP_200_OK,
)
+
+ @action(detail=False, methods=["get"])
+ def connector_schema(self, request: Request) -> Response:
+ """Returns the JSON schema for a specific connector.
+
+ Replaces the get_connector_schema function-based view.
+ """
+ serializer = self.get_serializer(data=request.query_params)
+ serializer.is_valid(raise_exception=True)
+
+ connector_id = serializer.validated_data.get("id")
+ json_schema = ConnectorProcessor.get_json_schema(connector_id=connector_id)
+
+ return Response(data=json_schema, status=status.HTTP_200_OK)
+
+ @action(detail=False, methods=["get"])
+ def supported_connectors(self, request: Request) -> Response:
+ """Retrieves a list of supported connectors based on the provided
+ connector type and mode.
+
+ Replaces the get_supported_connectors function-based view.
+ """
+ serializer = self.get_serializer(data=request.query_params)
+ serializer.is_valid(raise_exception=True)
+
+ connector_type = serializer.validated_data.get("type")
+ connector_mode = serializer.validated_data.get("connector_mode")
+
+ # Convert connector_mode to the appropriate enum if provided
+ if connector_mode:
+ connector_mode = ConnectorProcessor.validate_connector_mode(connector_mode)
+
+ json_schema = ConnectorProcessor.get_all_supported_connectors(
+ type=connector_type, connector_mode=connector_mode
+ )
+
+ return Response(json_schema, status=status.HTTP_200_OK)
diff --git a/backend/connector_v2/exceptions.py b/backend/connector_v2/exceptions.py
new file mode 100644
index 000000000..2c371f5f0
--- /dev/null
+++ b/backend/connector_v2/exceptions.py
@@ -0,0 +1,20 @@
+from rest_framework.exceptions import APIException
+
+
+class DeleteConnectorInUseError(APIException):
+ status_code = 409
+
+ def __init__(
+ self,
+ detail: str | None = None,
+ code: str | None = None,
+ connector_name: str = "connector",
+ ):
+ if detail is None:
+ if connector_name != "connector":
+ connector_name = f"'{connector_name}'"
+ detail = (
+ f"Cannot delete {connector_name}. "
+ "It is currently being used in one or more workflows"
+ )
+ super().__init__(detail, code)
diff --git a/backend/connector_v2/migrations/0002_alter_connectorinstance_connector_metadata_and_more.py b/backend/connector_v2/migrations/0002_alter_connectorinstance_connector_metadata_and_more.py
index 267bcb87d..b22b506f5 100644
--- a/backend/connector_v2/migrations/0002_alter_connectorinstance_connector_metadata_and_more.py
+++ b/backend/connector_v2/migrations/0002_alter_connectorinstance_connector_metadata_and_more.py
@@ -28,6 +28,7 @@ class Migration(migrations.Migration):
model_name="connectorinstance",
name="workflow",
field=models.ForeignKey(
+ blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="connector_workflow",
diff --git a/backend/connector_v2/migrations/0003_migrate_to_centralized_connectors.py b/backend/connector_v2/migrations/0003_migrate_to_centralized_connectors.py
new file mode 100644
index 000000000..af11c120d
--- /dev/null
+++ b/backend/connector_v2/migrations/0003_migrate_to_centralized_connectors.py
@@ -0,0 +1,428 @@
+"""Migration to convert workflow-specific connectors to centralized connectors."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import uuid
+from typing import Any
+
+from django.db import migrations
+
+logger = logging.getLogger(__name__)
+
+
+def _compute_metadata_hash(connector_metadata: dict[str, Any] | None) -> str | None:
+ """Compute SHA256 hash of connector metadata."""
+ if not connector_metadata:
+ return None
+ metadata_json = json.dumps(connector_metadata, sort_keys=True)
+ return hashlib.sha256(metadata_json.encode("utf-8")).hexdigest()
+
+
+def _extract_connector_sys_name(connector_id: str) -> str:
+ """Extract platform's connector name from 'name|uuid' format."""
+ if "|" in connector_id:
+ return connector_id.split("|")[0]
+ return connector_id
+
+
+def _get_short_group_key(group_key: tuple[Any, str, str | None]) -> tuple[Any, str, str]:
+ """Generate a shortened group key for logging."""
+ return (
+ group_key[0],
+ group_key[1],
+ group_key[2][:8] if group_key[2] else "None",
+ )
+
+
+def _group_connectors(
+ connector_instances: Any,
+) -> dict[tuple[Any, str, str | None], list[Any]]:
+ """Group connectors by organization, connector type, and metadata hash."""
+ connector_groups = {}
+
+ for connector in connector_instances:
+ try:
+ metadata_hash = _compute_metadata_hash(connector.connector_metadata)
+ connector_sys_name = _extract_connector_sys_name(connector.connector_id)
+
+ group_key = (
+ connector.organization_id,
+ connector_sys_name,
+ metadata_hash,
+ )
+
+ if group_key not in connector_groups:
+ connector_groups[group_key] = []
+ connector_groups[group_key].append(connector)
+
+ except Exception as e:
+ logger.error(f"Error processing connector {connector.id}: {str(e)}")
+ raise
+
+ return connector_groups
+
+
+def _process_single_connector(
+ connector: Any,
+ processed_groups: int,
+ total_groups: int,
+ short_group_key: tuple[Any, str, str],
+) -> None:
+ """Process a group with only one connector."""
+ connector.connector_name = f"{connector.connector_name}-{uuid.uuid4().hex[:8]}"
+ logger.info(
+ f"[Group {processed_groups}/{total_groups}] {short_group_key}: "
+ f"Only 1 connector present, renaming to '{connector.connector_name}'"
+ )
+ connector.save()
+
+
+def _centralize_connector_group(
+ connectors: list[Any],
+ processed_groups: int,
+ total_groups: int,
+ short_group_key: tuple[Any, str, str],
+) -> tuple[Any, dict[Any, Any], set[Any]]:
+ """Centralize a group of multiple connectors."""
+ logger.info(
+ f"[Group {processed_groups}/{total_groups}] {short_group_key}: "
+ f"Found {len(connectors)} similar connectors to unify"
+ )
+
+ # First connector becomes the centralized one
+ centralized_connector = connectors[0]
+ original_name = centralized_connector.connector_name
+ centralized_connector.connector_name = f"{original_name}-{uuid.uuid4().hex[:8]}"
+
+ logger.info(
+ f"[Group {processed_groups}/{total_groups}] {short_group_key}: "
+ f"Centralizing '{original_name}' -> '{centralized_connector.connector_name}'"
+ )
+ centralized_connector.save()
+
+ # Build mapping for connectors to be replaced
+ connector_mapping = {}
+ connectors_to_delete = set()
+
+ for connector_to_delete in connectors[1:]:
+ connector_mapping[connector_to_delete.id] = centralized_connector
+ connectors_to_delete.add(connector_to_delete.id)
+
+ return centralized_connector, connector_mapping, connectors_to_delete
+
+
+def _update_workflow_endpoints(
+ connector_mapping: dict[Any, Any], workflow_endpoint_model: Any
+) -> tuple[int, set[Any]]:
+ """Update WorkflowEndpoint references to use centralized connectors."""
+ endpoint_updates = 0
+ connectors_to_delete = set()
+
+ for old_connector_id, new_connector in connector_mapping.items():
+ try:
+ updated_count = workflow_endpoint_model.objects.filter(
+ connector_instance_id=old_connector_id
+ ).update(connector_instance=new_connector)
+
+ if updated_count == 0:
+ logger.debug(f"No WorkflowEndpoint uses connector '{old_connector_id}'")
+ connectors_to_delete.add(old_connector_id)
+ else:
+ endpoint_updates += updated_count
+ logger.debug(
+ f"Updated {updated_count} endpoints for connector '{old_connector_id}'"
+ )
+
+ except Exception as e:
+ logger.error(
+ f"Error updating endpoints for connector {old_connector_id}: {str(e)}"
+ )
+ raise
+
+ logger.info(f"Updated {endpoint_updates} WorkflowEndpoint references")
+ return endpoint_updates, connectors_to_delete
+
+
+def _delete_redundant_connectors(
+ connectors_to_delete_ids: set[Any], connector_instance_model: Any
+) -> int:
+ """Delete redundant connectors in bulk."""
+ if not connectors_to_delete_ids:
+ return 0
+
+ try:
+ delete_count = connector_instance_model.objects.filter(
+ id__in=connectors_to_delete_ids
+ ).delete()[0]
+ logger.info(f"Deleted {delete_count} redundant connectors")
+ return delete_count
+ except Exception as e:
+ logger.error(f"Error deleting redundant connectors: {str(e)}")
+ raise
+
+
+def migrate_to_centralized_connectors(apps, schema_editor): # noqa: ARG001
+ """Migrate existing workflow-specific connectors to centralized connectors.
+
+ This migration:
+ 1. Finds all workflow-specific connectors
+ 2. Groups them by organization, connector ID, and connector metadata
+ 3. Marks first connector of each group as centralized
+ 4. Updates WorkflowEndpoint references to use the new centralized connectors
+ """
+ ConnectorInstance = apps.get_model("connector_v2", "ConnectorInstance") # NOSONAR
+ WorkflowEndpoint = apps.get_model("endpoint_v2", "WorkflowEndpoint") # NOSONAR
+
+ # Get all connector instances with select_related for performance
+ connector_instances = ConnectorInstance.objects.select_related(
+ "organization", "created_by", "modified_by"
+ ).all()
+
+ total_connectors = connector_instances.count()
+ logger.info(f"Processing {total_connectors} connector instances for centralization")
+
+ # Group connectors by organization and unique credential fingerprint
+ connector_groups = _group_connectors(connector_instances)
+
+ # Process each group and centralize connectors
+ processed_groups = 0
+ centralized_count = 0
+ total_groups = len(connector_groups)
+ all_connector_mappings = {}
+ all_connectors_to_delete = set()
+
+ for group_key, connectors in connector_groups.items():
+ processed_groups += 1
+ short_group_key = _get_short_group_key(group_key)
+
+ try:
+ # Process single connector groups differently
+ if len(connectors) == 1:
+ _process_single_connector(
+ connectors[0], processed_groups, total_groups, short_group_key
+ )
+ continue
+
+ # Centralize multiple connectors
+ _, connector_mapping, connectors_to_delete = _centralize_connector_group(
+ connectors, processed_groups, total_groups, short_group_key
+ )
+
+ centralized_count += 1
+ all_connector_mappings.update(connector_mapping)
+ all_connectors_to_delete.update(connectors_to_delete)
+
+ except Exception as e:
+ logger.error(f"Error processing group {short_group_key}: {str(e)}")
+ raise
+
+ logger.info(
+ f"Processed {processed_groups} connector groups, centralized {centralized_count}"
+ )
+
+ # Update WorkflowEndpoint references
+ _, additional_deletes = _update_workflow_endpoints(
+ all_connector_mappings, WorkflowEndpoint
+ )
+ all_connectors_to_delete.update(additional_deletes)
+
+ # Delete redundant connectors
+ _delete_redundant_connectors(all_connectors_to_delete, ConnectorInstance)
+
+ logger.info(
+ f"Migration completed: {centralized_count} centralized connectors created"
+ )
+
+
+def _get_connector_type_from_endpoint(endpoint_type: str) -> str:
+ """Map endpoint type to connector type."""
+ return "INPUT" if endpoint_type == "SOURCE" else "OUTPUT"
+
+
+def _find_unused_connectors(
+ centralized_connectors: Any, workflow_endpoint_model: Any, total_connectors: int
+) -> list[Any]:
+ """Find centralized connectors that have no endpoint references."""
+ unused_connectors = []
+ processed_connectors = 0
+
+ for centralized_connector in centralized_connectors:
+ processed_connectors += 1
+
+ endpoints = workflow_endpoint_model.objects.filter(
+ connector_instance=centralized_connector
+ )
+
+ if not endpoints.exists():
+ logger.info(
+ f"[{processed_connectors}/{total_connectors}] Centralized connector "
+ f"'{centralized_connector}' has no endpoints, marking for deletion"
+ )
+ unused_connectors.append(centralized_connector.id)
+
+ return unused_connectors
+
+
+def _create_workflow_specific_connector(
+ centralized_connector: Any,
+ workflow: Any,
+ connector_type: str,
+ connector_instance_model: Any,
+) -> Any:
+ """Create a new workflow-specific connector from a centralized one."""
+ return connector_instance_model.objects.create(
+ connector_name=centralized_connector.connector_name,
+ connector_id=centralized_connector.connector_id,
+ connector_metadata=centralized_connector.connector_metadata,
+ connector_version=centralized_connector.connector_version,
+ connector_type=connector_type,
+ connector_auth=centralized_connector.connector_auth,
+ connector_mode=centralized_connector.connector_mode,
+ workflow=workflow,
+ organization=centralized_connector.organization,
+ created_by=centralized_connector.created_by,
+ modified_by=centralized_connector.modified_by,
+ )
+
+
+def _process_connector_endpoints(
+ centralized_connector: Any,
+ endpoints: Any,
+ connector_instance_model: Any,
+ processed_count: int,
+ total_count: int,
+) -> int:
+ """Process endpoints for a centralized connector, creating workflow-specific copies."""
+ endpoint_ref_count = endpoints.count()
+ added_connector_count = 0
+
+ logger.info(
+ f"[{processed_count}/{total_count}] Centralized connector "
+ f"'{centralized_connector}' has {endpoint_ref_count} endpoint(s) to process"
+ )
+
+ # Process each endpoint, reusing the last connector to avoid unnecessary creation
+ for index, endpoint in enumerate(endpoints):
+ workflow = endpoint.workflow
+ endpoint_type = endpoint.endpoint_type
+ connector_type = _get_connector_type_from_endpoint(endpoint_type)
+
+ if index == endpoint_ref_count - 1:
+ # Last endpoint reuses the existing centralized connector
+ centralized_connector.workflow = workflow
+ centralized_connector.connector_type = connector_type
+ centralized_connector.save()
+ logger.debug(f"Reused centralized connector for endpoint {endpoint.id}")
+ else:
+ # Create new workflow-specific connector for other endpoints
+ endpoint_connector = _create_workflow_specific_connector(
+ centralized_connector, workflow, connector_type, connector_instance_model
+ )
+ added_connector_count += 1
+ endpoint.connector_instance = endpoint_connector
+ endpoint.save()
+ logger.debug(f"Created new connector for endpoint {endpoint.id}")
+
+ return added_connector_count
+
+
+def _delete_unused_centralized_connectors(
+ unused_connector_ids: list[Any], connector_instance_model: Any
+) -> int:
+ """Delete unused centralized connectors in bulk."""
+ if not unused_connector_ids:
+ logger.info("No unused centralized connectors to delete")
+ return 0
+
+ try:
+ delete_count = connector_instance_model.objects.filter(
+ id__in=unused_connector_ids
+ ).delete()[0]
+ logger.info(f"Deleted {delete_count} unused centralized connectors")
+ return delete_count
+ except Exception as e:
+ logger.error(f"Error deleting unused connectors: {str(e)}")
+ raise
+
+
+def reverse_centralized_connectors(apps, schema_editor): # noqa: ARG001
+ """Reverse the migration by converting centralized connectors back to workflow-specific ones.
+
+ This is a best-effort reversal that:
+ 1. Finds all centralized connectors
+ 2. For each WorkflowEndpoint using a centralized connector, creates a workflow-specific copy
+ 3. Updates the WorkflowEndpoint to use the workflow-specific copy
+ """
+ ConnectorInstance = apps.get_model("connector_v2", "ConnectorInstance") # NOSONAR
+ WorkflowEndpoint = apps.get_model("endpoint_v2", "WorkflowEndpoint") # NOSONAR
+
+ # Get all centralized connectors with prefetch for better performance
+ centralized_connectors = ConnectorInstance.objects.prefetch_related(
+ "workflow_endpoints"
+ ).all()
+
+ total_connectors = centralized_connectors.count()
+ logger.info(f"Processing {total_connectors} centralized connectors for reversal")
+
+ # Find unused connectors that can be deleted
+ unused_connectors = _find_unused_connectors(
+ centralized_connectors, WorkflowEndpoint, total_connectors
+ )
+
+ # Process connectors with endpoints to create workflow-specific copies
+ added_connector_count = 0
+ processed_connectors = 0
+
+ for centralized_connector in centralized_connectors:
+ processed_connectors += 1
+
+ # Skip connectors already marked for deletion
+ if centralized_connector.id in unused_connectors:
+ continue
+
+ try:
+ endpoints = WorkflowEndpoint.objects.filter(
+ connector_instance=centralized_connector
+ )
+
+ if endpoints.exists():
+ connector_count = _process_connector_endpoints(
+ centralized_connector,
+ endpoints,
+ ConnectorInstance,
+ processed_connectors,
+ total_connectors,
+ )
+ added_connector_count += connector_count
+
+ except Exception as e:
+ logger.error(
+ f"Error processing connector {centralized_connector.id}: {str(e)}"
+ )
+ raise
+
+ # Delete unused centralized connectors
+ _delete_unused_centralized_connectors(unused_connectors, ConnectorInstance)
+
+ logger.info(
+ f"Reverse migration completed: added {added_connector_count} workflow-specific connectors"
+ )
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ (
+ "connector_v2",
+ "0002_alter_connectorinstance_connector_metadata_and_more",
+ ),
+ ("endpoint_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.RunPython(
+ migrate_to_centralized_connectors, reverse_centralized_connectors
+ ),
+ ]
diff --git a/backend/connector_v2/migrations/0004_remove_connectorinstance_unique_workflow_connector_and_more.py b/backend/connector_v2/migrations/0004_remove_connectorinstance_unique_workflow_connector_and_more.py
new file mode 100644
index 000000000..18ac0f52f
--- /dev/null
+++ b/backend/connector_v2/migrations/0004_remove_connectorinstance_unique_workflow_connector_and_more.py
@@ -0,0 +1,64 @@
+# Generated by Django 4.2.1 on 2025-07-22 07:45
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ("connector_v2", "0003_migrate_to_centralized_connectors"),
+ ]
+
+ operations = [
+ migrations.RemoveConstraint(
+ model_name="connectorinstance",
+ name="unique_workflow_connector",
+ ),
+ migrations.RemoveField(
+ model_name="connectorinstance",
+ name="connector_type",
+ ),
+ migrations.RemoveField(
+ model_name="connectorinstance",
+ name="workflow",
+ ),
+ migrations.AddConstraint(
+ model_name="connectorinstance",
+ constraint=models.UniqueConstraint(
+ fields=("connector_name", "organization"),
+ name="unique_organization_connector",
+ ),
+ ),
+ migrations.AddField(
+ model_name="connectorinstance",
+ name="shared_to_org",
+ field=models.BooleanField(
+ db_comment="Is the connector shared to entire org", default=False
+ ),
+ ),
+ migrations.AddField(
+ model_name="connectorinstance",
+ name="shared_users",
+ field=models.ManyToManyField(
+ related_name="shared_connectors",
+ to=settings.AUTH_USER_MODEL,
+ blank=True,
+ ),
+ ),
+ migrations.AlterField(
+ model_name="connectorinstance",
+ name="connector_mode",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("UNKNOWN", "UNKNOWN"),
+ ("FILE_SYSTEM", "FILE_SYSTEM"),
+ ("DATABASE", "DATABASE"),
+ ],
+ db_comment="0: UNKNOWN, 1: FILE_SYSTEM, 2: DATABASE",
+ default="UNKNOWN",
+ null=True,
+ ),
+ ),
+ ]
diff --git a/backend/connector_v2/models.py b/backend/connector_v2/models.py
index d7276aa19..bcd7bb597 100644
--- a/backend/connector_v2/models.py
+++ b/backend/connector_v2/models.py
@@ -23,35 +23,38 @@ logger = logging.getLogger(__name__)
class ConnectorInstanceModelManager(DefaultOrganizationManagerMixin, models.Manager):
- pass
+ def get_queryset(self) -> models.QuerySet:
+ return super().get_queryset()
+
+ def for_user(self, user: User) -> models.QuerySet:
+ return (
+ self.get_queryset()
+ .filter(
+ models.Q(created_by=user)
+ | models.Q(shared_users=user)
+ | models.Q(shared_to_org=True)
+ )
+ .distinct("id")
+ )
class ConnectorInstance(DefaultOrganizationMixin, BaseModel):
- # TODO: handle all cascade deletions
class ConnectorType(models.TextChoices):
INPUT = "INPUT", "Input"
OUTPUT = "OUTPUT", "Output"
- class ConnectorMode(models.IntegerChoices):
- UNKNOWN = 0, "UNKNOWN"
- FILE_SYSTEM = 1, "FILE_SYSTEM"
- DATABASE = 2, "DATABASE"
+ class ConnectorMode(models.TextChoices):
+ UNKNOWN = "UNKNOWN", "UNKNOWN"
+ FILE_SYSTEM = "FILE_SYSTEM", "FILE_SYSTEM"
+ DATABASE = "DATABASE", "DATABASE"
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
connector_name = models.TextField(
max_length=CONNECTOR_NAME_SIZE, null=False, blank=False
)
- workflow = models.ForeignKey(
- "workflow_v2.Workflow",
- on_delete=models.CASCADE,
- related_name="connector_workflow",
- null=True,
- blank=False,
- )
connector_id = models.CharField(max_length=FLC.CONNECTOR_ID_LENGTH, default="")
connector_metadata = EncryptedBinaryField(null=True)
connector_version = models.CharField(max_length=VERSION_NAME_SIZE, default="")
- connector_type = models.CharField(choices=ConnectorType.choices, null=True)
# TODO: handle connector_auth cascade deletion
connector_auth = models.ForeignKey(
ConnectorAuth,
@@ -60,10 +63,13 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):
blank=True,
related_name="connector_instances",
)
+ # TODO: Remove unused connector_mode field, resolved from the ConnectorKit instead
connector_mode = models.CharField(
choices=ConnectorMode.choices,
default=ConnectorMode.UNKNOWN,
db_comment="0: UNKNOWN, 1: FILE_SYSTEM, 2: DATABASE",
+ null=True,
+ blank=True,
)
created_by = models.ForeignKey(
@@ -81,7 +87,17 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):
blank=True,
)
- # Manager
+ shared_to_org = models.BooleanField(
+ default=False,
+ db_comment="Is the connector shared to entire org",
+ )
+
+ # Introduced field to establish M2M relation between users and connectors.
+ # This will introduce intermediary table which relates both the models.
+ shared_users = models.ManyToManyField(
+ User, related_name="shared_connectors", blank=True
+ )
+
objects = ConnectorInstanceModelManager()
@staticmethod
@@ -94,7 +110,7 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):
def __str__(self) -> str:
return (
- f"Connector({self.id}, ID={self.connector_id}, mode: {self.connector_mode})"
+ f"Connector(pk={self.id}, name={self.connector_name}, ID={self.connector_id})"
)
@property
@@ -128,7 +144,7 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):
verbose_name_plural = "Connector Instances"
constraints = [
models.UniqueConstraint(
- fields=["connector_name", "workflow", "connector_type"],
- name="unique_workflow_connector",
+ fields=["connector_name", "organization"],
+ name="unique_organization_connector",
),
]
diff --git a/backend/connector_v2/serializers.py b/backend/connector_v2/serializers.py
index c88598e34..c0546fa54 100644
--- a/backend/connector_v2/serializers.py
+++ b/backend/connector_v2/serializers.py
@@ -7,9 +7,8 @@ from connector_auth_v2.pipeline.common import ConnectorAuthHelper
from connector_processor.connector_processor import ConnectorProcessor
from connector_processor.constants import ConnectorKeys
from connector_processor.exceptions import OAuthTimeOut
-from rest_framework.serializers import SerializerMethodField
+from rest_framework.serializers import CharField, SerializerMethodField
from utils.fields import EncryptedBinaryFieldSerializer
-from utils.serializer_utils import SerializerUtils
from backend.serializers import AuditSerializer
from connector_v2.constants import ConnectorInstanceKey as CIKey
@@ -23,6 +22,7 @@ logger = logging.getLogger(__name__)
class ConnectorInstanceSerializer(AuditSerializer):
connector_metadata = EncryptedBinaryFieldSerializer(required=False, allow_null=True)
icon = SerializerMethodField()
+ created_by_email = CharField(source="created_by.email", read_only=True)
class Meta:
model = ConnectorInstance
@@ -53,11 +53,6 @@ class ConnectorInstanceSerializer(AuditSerializer):
)
raise OAuthTimeOut
- connector_mode = ConnectorProcessor.get_connector_data_with_key(
- connector_id, CIKey.CONNECTOR_MODE
- )
- kwargs[CIKey.CONNECTOR_MODE] = connector_mode.value
-
instance = super().save(**kwargs)
return instance
@@ -76,8 +71,13 @@ class ConnectorInstanceSerializer(AuditSerializer):
rep: OrderedDict[str, Any] = super().to_representation(instance)
if instance.connector_id == UnstractCloudStorage.get_id():
rep[CIKey.CONNECTOR_METADATA] = {}
- if SerializerUtils.check_context_for_GET_or_POST(context=self.context):
- rep.pop(CIKey.CONNECTOR_AUTH)
- rep[ConnectorKeys.ICON] = self.get_icon(instance)
+
+ connector_mode = ConnectorProcessor.get_connector_data_with_key(
+ instance.connector_id, CIKey.CONNECTOR_MODE
+ )
+ rep[CIKey.CONNECTOR_MODE] = connector_mode.value
+
+ # Remove sensitive connector auth from the response
+ rep.pop(CIKey.CONNECTOR_AUTH)
return rep
diff --git a/backend/connector_v2/views.py b/backend/connector_v2/views.py
index 5f5a543b7..e9be939fb 100644
--- a/backend/connector_v2/views.py
+++ b/backend/connector_v2/views.py
@@ -7,7 +7,8 @@ from connector_auth_v2.exceptions import CacheMissException, MissingParamExcepti
from connector_auth_v2.pipeline.common import ConnectorAuthHelper
from connector_processor.exceptions import OAuthTimeOut
from django.db import IntegrityError
-from django.db.models import QuerySet
+from django.db.models import ProtectedError, QuerySet
+from permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg
from rest_framework import status, viewsets
from rest_framework.response import Response
from rest_framework.versioning import URLPathVersioning
@@ -15,7 +16,10 @@ from utils.filtering import FilterHelper
from backend.constants import RequestKey
from connector_v2.constants import ConnectorInstanceKey as CIKey
+from unstract.connectors.connectorkit import Connectorkit
+from unstract.connectors.enums import ConnectorMode
+from .exceptions import DeleteConnectorInUseError
from .models import ConnectorInstance
from .serializers import ConnectorInstanceSerializer
@@ -26,18 +30,42 @@ class ConnectorInstanceViewSet(viewsets.ModelViewSet):
versioning_class = URLPathVersioning
serializer_class = ConnectorInstanceSerializer
+ def get_permissions(self) -> list[Any]:
+ if self.action == "destroy":
+ return [IsOwner()]
+
+ return [IsOwnerOrSharedUserOrSharedToOrg()]
+
def get_queryset(self) -> QuerySet | None:
+ queryset = ConnectorInstance.objects.for_user(self.request.user)
+
filter_args = FilterHelper.build_filter_args(
self.request,
RequestKey.WORKFLOW,
RequestKey.CREATED_BY,
CIKey.CONNECTOR_TYPE,
- CIKey.CONNECTOR_MODE,
)
if filter_args:
- queryset = ConnectorInstance.objects.filter(**filter_args)
- else:
- queryset = ConnectorInstance.objects.all()
+ queryset = queryset.filter(**filter_args)
+
+ # Filter by connector_mode
+ connector_mode_param = self.request.query_params.get("connector_mode")
+ if connector_mode_param:
+ try:
+ connector_mode = ConnectorMode(connector_mode_param)
+ connectors = Connectorkit().get_connectors_list(mode=connector_mode)
+ connector_ids = [conn.get("id") for conn in connectors if conn.get("id")]
+
+ if connector_ids:
+ queryset = queryset.filter(connector_id__in=connector_ids)
+ else:
+ queryset = queryset.none()
+ except ValueError:
+ logger.warning(
+ f"Invalid connector_mode parameter: {connector_mode_param}"
+ )
+ queryset = queryset.none()
+
return queryset
def _get_connector_metadata(self, connector_id: str) -> dict[str, str] | None:
@@ -123,3 +151,21 @@ class ConnectorInstanceViewSet(viewsets.ModelViewSet):
)
headers = self.get_success_headers(serializer.data)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
+
+ def perform_destroy(self, instance: ConnectorInstance) -> None:
+ """Override perform_destroy to handle ProtectedError gracefully.
+
+ Args:
+ instance: The ConnectorInstance to be deleted
+
+ Raises:
+ DeleteConnectorInUseError: If the connector is being used in workflows
+ """
+ try:
+ super().perform_destroy(instance)
+ except ProtectedError:
+ logger.error(
+ f"Failed to delete connector: {instance.connector_id}"
+ f" named {instance.connector_name}"
+ )
+ raise DeleteConnectorInUseError(connector_name=instance.connector_name)
diff --git a/backend/pipeline_v2/serializers/crud.py b/backend/pipeline_v2/serializers/crud.py
index ab4f9705b..2fefe9bc6 100644
--- a/backend/pipeline_v2/serializers/crud.py
+++ b/backend/pipeline_v2/serializers/crud.py
@@ -2,8 +2,6 @@ import logging
from collections import OrderedDict
from typing import Any
-from connector_processor.connector_processor import ConnectorProcessor
-from connector_v2.models import ConnectorInstance
from croniter import croniter
from django.conf import settings
from pipeline_v2.constants import PipelineConstants as PC
@@ -286,6 +284,3 @@ class PipelineSerializer(IntegrityErrorMixin, AuditSerializer):
)
return repr
-
- def get_connector_data(self, connector: ConnectorInstance, key: str) -> Any:
- return ConnectorProcessor.get_connector_data_with_key(connector.connector_id, key)
diff --git a/backend/workflow_manager/endpoint_v2/migrations/0002_alter_workflowendpoint_connector_instance.py b/backend/workflow_manager/endpoint_v2/migrations/0002_alter_workflowendpoint_connector_instance.py
new file mode 100644
index 000000000..a7e0fcbc6
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/migrations/0002_alter_workflowendpoint_connector_instance.py
@@ -0,0 +1,28 @@
+# Generated by Django 4.2.1 on 2025-08-12 07:50
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ (
+ "connector_v2",
+ "0004_remove_connectorinstance_unique_workflow_connector_and_more",
+ ),
+ ("endpoint_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="workflowendpoint",
+ name="connector_instance",
+ field=models.ForeignKey(
+ db_comment="Foreign key from ConnectorInstance model",
+ null=True,
+ on_delete=django.db.models.deletion.PROTECT,
+ related_name="workflow_endpoints",
+ to="connector_v2.connectorinstance",
+ ),
+ ),
+ ]
diff --git a/backend/workflow_manager/endpoint_v2/models.py b/backend/workflow_manager/endpoint_v2/models.py
index f884de384..e07737ba7 100644
--- a/backend/workflow_manager/endpoint_v2/models.py
+++ b/backend/workflow_manager/endpoint_v2/models.py
@@ -53,7 +53,7 @@ class WorkflowEndpoint(BaseModel):
connector_instance = models.ForeignKey(
ConnectorInstance,
- on_delete=models.CASCADE,
+ on_delete=models.PROTECT,
db_index=True,
null=True,
db_comment="Foreign key from ConnectorInstance model",
diff --git a/frontend/src/assets/connectors.svg b/frontend/src/assets/connectors.svg
new file mode 100644
index 000000000..d7cc156f5
--- /dev/null
+++ b/frontend/src/assets/connectors.svg
@@ -0,0 +1,3 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+ <path d="M14.1667 7.5H17.5C18.4204 7.5 19.1667 8.24629 19.1667 9.16667V10.8333C19.1667 11.7537 18.4204 12.5 17.5 12.5H14.1667V15.8333C14.1667 16.7537 13.4204 17.5 12.5 17.5H10.8333C9.91292 17.5 9.16663 16.7537 9.16663 15.8333V12.5H5.83329C4.91292 12.5 4.16663 11.7537 4.16663 10.8333V9.16667C4.16663 8.24629 4.91292 7.5 5.83329 7.5H9.16663V4.16667C9.16663 3.24629 9.91292 2.5 10.8333 2.5H12.5C13.4204 2.5 14.1667 3.24629 14.1667 4.16667V7.5ZM12.5 4.16667H10.8333V7.91667C10.8333 8.14767 10.6477 8.33333 10.4167 8.33333H5.83329C5.60229 8.33333 5.41663 8.51899 5.41663 8.75V11.25C5.41663 11.481 5.60229 11.6667 5.83329 11.6667H10.4167C10.6477 11.6667 10.8333 11.8523 10.8333 12.0833V15.8333H12.5V12.0833C12.5 11.8523 12.6856 11.6667 12.9166 11.6667H17.5C17.731 11.6667 17.9166 11.481 17.9166 11.25V8.75C17.9166 8.51899 17.731 8.33333 17.5 8.33333H12.9166C12.6856 8.33333 12.5 8.14767 12.5 7.91667V4.16667Z" fill="#90A4B7"/>
+</svg>
diff --git a/frontend/src/components/agency/agency/Agency.jsx b/frontend/src/components/agency/agency/Agency.jsx
index 025f0fa08..98040134f 100644
--- a/frontend/src/components/agency/agency/Agency.jsx
+++ b/frontend/src/components/agency/agency/Agency.jsx
@@ -185,8 +185,8 @@ function Agency() {
setCanAddETLPipeline(
source?.connector_instance &&
((destination?.connection_type === "DATABASE" &&
- destination.connector_instance) ||
- destination.connection_type === "MANUALREVIEW")
+ destination?.connector_instance) ||
+ destination?.connection_type === "MANUALREVIEW")
);
}, [source, destination]);
@@ -280,11 +280,44 @@ function Agency() {
}
try {
- // Fetch API deployments and pipelines in parallel
- const [apiDeployments, pipelines] = await Promise.all([
- apiDeploymentService.getDeploymentsByWorkflowId(projectId),
- pipelineServiceInstance.getPipelinesByWorkflowId(projectId),
- ]);
+ // Determine which APIs to call based on connector configuration
+ const shouldFetchApiDeployments =
+ source?.connection_type === "API" &&
+ destination?.connection_type === "API";
+
+ const shouldFetchPipelines =
+ source?.connection_type === "FILESYSTEM" &&
+ (destination?.connection_type === "DATABASE" ||
+ destination?.connection_type === "FILESYSTEM" ||
+ destination?.connection_type === "MANUALREVIEW");
+
+ // If no valid deployment configuration, set null and return
+ if (!shouldFetchApiDeployments && !shouldFetchPipelines) {
+ if (!signal?.aborted) {
+ setDeploymentInfo(null);
+ }
+ return;
+ }
+
+ // Fetch only the relevant deployment types
+ const promises = [];
+ if (shouldFetchApiDeployments) {
+ promises.push(
+ apiDeploymentService.getDeploymentsByWorkflowId(projectId)
+ );
+ } else {
+ promises.push(Promise.resolve({ data: [] }));
+ }
+
+ if (shouldFetchPipelines) {
+ promises.push(
+ pipelineServiceInstance.getPipelinesByWorkflowId(projectId)
+ );
+ } else {
+ promises.push(Promise.resolve({ data: [] }));
+ }
+
+ const [apiDeployments, pipelines] = await Promise.all(promises);
// Check if request was aborted before setting state
if (signal?.aborted) {
@@ -595,16 +628,16 @@ function Agency() {
}
// For API connections, just having an endpoint is sufficient
- if (endpoint.connection_type === "API") {
+ if (endpoint?.connection_type === "API") {
return {
configured: true,
type: "API",
- name: endpoint.connector_name || "API Endpoint",
+ name: endpoint?.connector_name || "API Endpoint",
};
}
// For filesystem connectors, they are automatically configured
- if (endpoint.connection_type === "FILESYSTEM") {
+ if (endpoint?.connection_type === "FILESYSTEM") {
return {
configured: true,
type: "File System",
@@ -619,8 +652,8 @@ function Agency() {
return {
configured: true,
- type: endpoint.connection_type,
- name: endpoint.connector_name || "Configured",
+ type: endpoint?.connection_type,
+ name: endpoint?.connector_name || "Configured",
};
};
@@ -663,6 +696,7 @@ function Agency() {
}, [details?.tool_instances, details?.id]);
// Refresh deployment info when allowChangeEndpoint changes (indicates deployment status change)
+ // Also refresh when source/destination connector types change since API calls are now conditional
useEffect(() => {
if (projectId) {
const abortController = new AbortController();
@@ -672,7 +706,12 @@ function Agency() {
abortController.abort();
};
}
- }, [allowChangeEndpoint, projectId]);
+ }, [
+ allowChangeEndpoint,
+ projectId,
+ source?.connection_type,
+ destination?.connection_type,
+ ]);
// Update progress whenever relevant state changes
useEffect(() => {
@@ -1047,22 +1086,13 @@ function Agency() {
source,
!allowChangeEndpoint
);
- // Show connector type for non-API configured connectors
- if (
- status?.configured &&
- source?.connection_type &&
- source?.connection_type !== "API"
- ) {
- return source?.connection_type;
- }
return status?.configured ? "✓" : "1";
})()}
title="Configure Source Connector"
description="Select and configure your data input connector"
- type={sourceTypes.connectors[0]}
+ connType={sourceTypes.connectors[0]}
endpointDetails={source}
message={sourceMsg}
- connectorIcon={source?.connector_instance?.icon}
/>
</Col>
@@ -1074,22 +1104,13 @@ function Agency() {
destination,
!allowChangeEndpoint
);
- // Show connector type for non-API configured connectors
- if (
- status?.configured &&
- destination?.connection_type &&
- destination?.connection_type !== "API"
- ) {
- return destination?.connection_type;
- }
return status?.configured ? "✓" : "2";
})()}
- title="Configure Output Destination"
+ title="Configure Destination Connector"
description="Select and configure your data output connector"
- type={sourceTypes.connectors[1]}
+ connType={sourceTypes.connectors[1]}
endpointDetails={destination}
message={destinationMsg}
- connectorIcon={destination?.connector_instance?.icon}
/>
</Col>
<Col span={12}>
diff --git a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.css b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.css
index 6c92c0579..6e42fd6e4 100644
--- a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.css
+++ b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.css
@@ -1,10 +1,13 @@
/* Styles for ConfigureConnectorModal */
.conn-modal-body {
- height: 800px;
+ height: 80vh; /* 80% of viewport height */
+ max-height: 800px; /* Cap at 800px for large screens */
+ min-height: 400px; /* Minimum height for usability */
display: flex;
flex-direction: column;
overflow-y: hidden;
+ overflow-x: hidden;
}
.conn-modal-row {
@@ -19,19 +22,15 @@
display: flex;
flex-direction: column;
overflow-y: auto;
+ border-right: 1px solid #f0f0f0;
}
.conn-modal-col-left {
background-color: var(--page-bg-3);
}
-.conn-modal-tab-body {
- flex: 1;
- overflow-y: hidden;
-}
-
-.conn-modal-gap {
- margin-bottom: 8px;
+.conn-modal-col .ant-tabs-nav {
+ margin: 0px;
}
.conn-modal-menu {
@@ -49,48 +48,146 @@
padding: 4px 20px 0px 20px;
}
-.conn-modal-form-right {
- padding: 12px 12px 0px 0px;
- padding-right: 20px;
- border-right: 1px solid #D4D4D4;
- height: 100%;
+.form-right {
+ padding: 8px 8px 0px 0px;
}
-.conn-modal-form-left {
- padding: 12px 0px 0px 12px;
- padding-left: 20px;
- height: 100%;
+/* Styles for connector selection dropdown */
+.connector-selection-section {
+ padding: 16px 8px;
+}
+
+.connector-selection-label {
+ display: block;
+ margin-bottom: 8px;
}
-.conn-modal-flex {
+.connector-selection-dropdown {
+ width: 100%;
+}
+
+.connector-selection-dropdown .ant-select-selector {
+ height: 40px;
+ padding: 4px 11px 4px 8px;
display: flex;
- flex-direction: column;
- height: 100%;
+ align-items: center;
+}
+
+.connector-selection-dropdown .ant-select-arrow {
+ right: 8px;
+}
+
+.connector-dropdown-add-new {
+ border-top: 1px solid #f0f0f0;
+ padding: 8px 12px;
+ cursor: pointer;
+ background-color: #fafafa;
+ transition: background-color 0.2s;
+
+ /* Button reset styles */
+ border-left: none;
+ border-right: none;
+ border-bottom: none;
+ width: 100%;
+ text-align: left;
+ font-family: inherit;
+ font-size: inherit;
+}
+
+.connector-dropdown-add-new:hover {
+ background-color: #f0f0f0;
+}
+
+.connector-dropdown-add-new:focus {
+ outline: 2px solid #1890ff;
+ outline-offset: -2px;
+ background-color: #f0f0f0;
+}
+
+.connector-dropdown-menu-container {
+ max-height: 200px;
+ overflow-y: auto;
+}
+
+.conn-modal-fs-config {
overflow-y: hidden;
}
-.conn-modal-flex-1 {
+.file-browser-section {
+ padding-top: 16px;
+ height: 100%;
+}
+
+.file-browser-header {
+ display: flex;
+ justify-content: space-between;
+ align-items: center;
+}
+
+.file-browser-header button {
+ margin-right: 16px;
+}
+
+.file-browser-content {
flex: 1;
- overflow-y: auto;
}
-.form-right {
- padding: 8px 8px 0px 0px;
+.field-description {
+ display: block;
+ font-size: 12px;
}
-.sidebar-menu {
- margin-top: 10px;
- background-color: transparent;
+
+/* Placeholder styles for empty state */
+.connector-placeholder {
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ justify-content: center;
+ height: 400px;
+ color: #8c8c8c;
}
-.sidebar-menu .ant-menu-item {
- padding: 5px !important;
+.connector-placeholder-icon {
+ font-size: 48px;
+ margin-bottom: 16px;
+ color: #bfbfbf;
}
-.sidebar-menu .ant-menu-title-content {
- margin-left: 5px;
+.connector-placeholder-text {
+ font-size: 16px;
+ color: #8c8c8c;
+ text-align: center;
}
-.conn-modal-col .ant-tabs-nav {
- margin: 0px;
+
+/* Responsive adjustments for different screen sizes */
+@media (max-height: 900px) {
+ .conn-modal-body {
+ height: 75vh; /* Reduce height on smaller screens */
+ }
+}
+
+@media (max-height: 768px) {
+ .conn-modal-body {
+ height: 60vh; /* Further reduce for very small screens */
+ min-height: 350px; /* Adjust minimum height */
+ }
+}
+
+@media (max-width: 1200px) {
+ .file-browser-section {
+ padding: 12px;
+ }
+
+ .file-browser-header {
+ flex-direction: column;
+ align-items: flex-start;
+ gap: 12px;
+ }
+
+ .file-browser-content {
+ width: 100%;
+ }
+
}
diff --git a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx
index 7eaeacee7..9f3dafb88 100644
--- a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx
+++ b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx
@@ -1,11 +1,18 @@
-import { Col, Modal, Row, Tabs, Typography } from "antd";
+import { Col, Modal, Row, Typography, Select, Space, Image, Tabs } from "antd";
+import { CloudDownloadOutlined, CloudUploadOutlined } from "@ant-design/icons";
import PropTypes from "prop-types";
-import { useEffect, useState } from "react";
+import { useEffect, useState, useRef, useCallback } from "react";
+import { isEqual, cloneDeep } from "lodash";
-import usePostHogEvents from "../../../hooks/usePostHogEvents";
+import { useAlertStore } from "../../../store/alert-store";
+import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";
+import useRequestUrl from "../../../hooks/useRequestUrl";
+import { AddSourceModal } from "../../input-output/add-source-modal/AddSourceModal";
import { ManageFiles } from "../../input-output/manage-files/ManageFiles";
import { ConfigureFormsLayout } from "../configure-forms-layout/ConfigureFormsLayout";
-import { ListOfConnectors } from "../list-of-connectors/ListOfConnectors";
+import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
+import usePostHogEvents from "../../../hooks/usePostHogEvents";
+import { CustomButton } from "../../widgets/custom-button/CustomButton";
import "./ConfigureConnectorModal.css";
let DBRules;
@@ -19,148 +26,376 @@ try {
function ConfigureConnectorModal({
open,
setOpen,
- type,
- selectedId,
- setSelectedId,
- handleUpdate,
- filteredList,
- connectorId,
- connectorMetadata,
- specConfig,
- formDataConfig,
- setFormDataConfig,
- isSpecConfigLoading,
connDetails,
+ setConnDetails,
connType,
- selectedItemName,
- setSelectedItemName,
+ connMode,
workflowDetails,
+ handleEndpointUpdate,
+ endpointDetails,
+ formDataConfig,
+ setFormDataConfig,
}) {
- const [activeKey, setActiveKey] = useState("1");
- useEffect(() => {
- if (connectorMetadata && connType === "FILESYSTEM") {
- setActiveKey("2"); // If connector is already configured
- } else {
- setActiveKey("1"); // default value
- }
- }, [open, connectorMetadata]);
-
- // Auto-select the configured connector when modal opens
- useEffect(() => {
- if (open && connDetails?.connector_id && filteredList?.length > 0) {
- const configuredConnector = filteredList.find(
- (item) => item?.key === connDetails?.connector_id
- );
-
- if (configuredConnector) {
- setSelectedId(connDetails?.connector_id);
- setSelectedItemName(configuredConnector?.label);
- }
- }
- }, [open, connDetails, filteredList]);
- const { setPostHogCustomEvent, posthogConnectorEventText } =
- usePostHogEvents();
-
+ const [availableConnectors, setAvailableConnectors] = useState([]);
+ const [addNewOption, setAddNewOption] = useState(null);
+ const [isLoadingConnectors, setIsLoadingConnectors] = useState(false);
+ const [showAddSourceModal, setShowAddSourceModal] = useState(false);
+ const [isSavingEndpoint, setIsSavingEndpoint] = useState(false);
+ const [activeTabKey, setActiveTabKey] = useState("1");
const [tabItems, setTabItems] = useState([
{
key: "1",
label: "Settings",
visible: true,
},
- {
- key: "2",
- label: "File System",
- disabled:
- !connectorId ||
- connDetails?.connector_id !== selectedId ||
- connType === "DATABASE",
- visible: false,
- },
]);
+ const [specConfig, setSpecConfig] = useState({});
+ const [isSpecConfigLoading, setIsSpecConfigLoading] = useState(false);
+ const [selectedFolderPath, setSelectedFolderPath] = useState("");
+ const [isFolderSelected, setIsFolderSelected] = useState(false);
+ const [initialFormDataConfig, setInitialFormDataConfig] = useState({});
+ const [initialConnectorId, setInitialConnectorId] = useState(null);
+
+ const fileExplorerRef = useRef(null);
+
+ const { setAlertDetails } = useAlertStore();
+ const axiosPrivate = useAxiosPrivate();
+ const { getUrl } = useRequestUrl();
+ const handleException = useExceptionHandler();
+ const { setPostHogCustomEvent, posthogConnectorEventText } =
+ usePostHogEvents();
+
const setUpdatedTabOptions = (tabOption) => {
setTabItems((prevTabOptions) => {
- // Check if inputOption already exists in prevTabOptions
+ // Check if tabOption already exists in prevTabOptions
+ // Return previous state unchanged if it does or create new array
if (prevTabOptions.some((opt) => opt?.key === tabOption?.key)) {
- return prevTabOptions; // Return previous state unchanged
+ return prevTabOptions;
} else {
- // Create a new array with the existing options and the new option
const updatedTabOptions = [...prevTabOptions, tabOption];
return updatedTabOptions;
}
});
};
- useEffect(() => {
- const updatedTabItems = tabItems.map((item) => {
- if (item?.key === "2") {
- item.visible = connType === "FILESYSTEM";
- item.disabled =
- !connectorId ||
- connDetails?.connector_id !== selectedId ||
- connType === "DATABASE";
- } else if (item.key === "MANUALREVIEW") {
- item.disabled =
- !connectorId || connDetails?.connector_id !== selectedId;
- item.visible = connType === "DATABASE" || connType === "MANUALREVIEW";
- } else {
- item.visible = true;
+ const onDBConfigTabChange = (key) => {
+ setActiveTabKey(key);
+ };
+
+ const fetchEndpointConfigSchema = () => {
+ if (!endpointDetails?.id) {
+ return;
+ }
+
+ const requestOptions = {
+ method: "GET",
+ url: getUrl(`workflow/endpoint/${endpointDetails.id}/settings/`),
+ };
+
+ setIsSpecConfigLoading(true);
+ axiosPrivate(requestOptions)
+ .then((res) => {
+ const data = res?.data;
+ setSpecConfig(data?.schema || {});
+ })
+ .catch((err) => {
+ setAlertDetails(handleException(err, "Failed to load the spec"));
+ })
+ .finally(() => {
+ setIsSpecConfigLoading(false);
+ });
+ };
+
+ const fetchAvailableConnectors = (connectionType) => {
+ if (!connectionType || connectionType === "API") {
+ setAvailableConnectors([]);
+ return;
+ }
+
+ setIsLoadingConnectors(true);
+
+ const requestOptions = {
+ method: "GET",
+ url: getUrl(`connector/?connector_mode=${connectionType}`),
+ };
+
+ axiosPrivate(requestOptions)
+ .then((response) => {
+ const connectors = response?.data || [];
+
+ // Separate regular connectors from "Add new connector" option
+ const regularConnectors = connectors.map((conn) => ({
+ value: conn?.id,
+ label: conn?.connector_name,
+ icon: conn?.icon,
+ connector: conn,
+ }));
+
+ // Keep "Add new connector" separate for sticky positioning
+ const addNewOption = {
+ value: "add_new",
+ label: "+ Add new connector",
+ isAddNew: true,
+ };
+
+ // For the Select options, only include regular connectors
+ const options = regularConnectors;
+
+ setAvailableConnectors(options);
+ setAddNewOption(addNewOption);
+ })
+ .catch((error) => {
+ setAvailableConnectors([]);
+ setAlertDetails(handleException(error, "Failed to load connectors"));
+ })
+ .finally(() => {
+ setIsLoadingConnectors(false);
+ });
+ };
+
+ const handleConnectorSelect = (value) => {
+ if (value === "add_new") {
+ setShowAddSourceModal(true);
+ } else {
+ const selectedConnector = availableConnectors.find(
+ (conn) => conn.value === value
+ );
+ if (selectedConnector?.connector) {
+ setConnDetails(selectedConnector.connector);
+
+ // Track connector selection with PostHog
+ try {
+ setPostHogCustomEvent(
+ posthogConnectorEventText[`${connMode}:${connType}`],
+ {
+ info: `Selected a connector`,
+ connector_name: selectedConnector.connector.connector_name,
+ }
+ );
+ } catch (err) {
+ // If an error occurs while setting custom posthog event, ignore it and continue
+ }
}
- return item;
- });
- setTabItems(updatedTabItems);
- }, [open, selectedId, connType, connectorId, connDetails]);
+ }
+ };
- useEffect(() => {
- const updatedTabItems = tabItems.map((item) => {
- if (item?.key === "MANUALREVIEW") {
- item.disabled =
- !connectorId || connDetails?.connector_id !== selectedId;
+ const handleConnectorCreated = (newConnectorData) => {
+ setShowAddSourceModal(false);
+
+ if (newConnectorData) {
+ const newConnectorOption = {
+ value: newConnectorData.id,
+ label: newConnectorData.connector_name,
+ icon: newConnectorData.icon,
+ connector: newConnectorData,
+ };
+
+ // Simply add the new connector to the existing list
+ setAvailableConnectors((prev) => [...prev, newConnectorOption]);
+
+ // Select the new connector
+ handleConnectorSelect(newConnectorData.id);
+ }
+ };
+
+ const handleFolderSelect = (folderPath, itemType) => {
+ setSelectedFolderPath(folderPath);
+ setIsFolderSelected(itemType === "folder");
+ };
+
+ const handleAddFolder = () => {
+ if (!selectedFolderPath) return;
+
+ if (connType === "input") {
+ // SOURCE mode: Add to folders array (existing behavior)
+ const currentFolders = formDataConfig?.folders || [];
+ if (!currentFolders.includes(selectedFolderPath)) {
+ setFormDataConfig((prev) => ({
+ ...prev,
+ folders: [...currentFolders, selectedFolderPath],
+ }));
}
- return item;
- });
- setTabItems(updatedTabItems);
- }, [connectorMetadata]);
+ } else if (connType === "output") {
+ // DESTINATION mode: Set outputFolder as single value
+ setFormDataConfig((prev) => ({
+ ...prev,
+ outputFolder: selectedFolderPath,
+ }));
+ }
+
+ setSelectedFolderPath("");
+ setIsFolderSelected(false);
+ };
+ // Configuration for UI text based on connector type
+ const folderSectionConfig = {
+ input: {
+ title: "Select Folder to Process",
+ description: "Browse and select a folder to add to processing",
+ buttonText: "Add Folder",
+ },
+ output: {
+ title: "Select Output Folder",
+ description: "Browse and select a folder for output files",
+ buttonText: "Select Folder",
+ },
+ };
+
+ const currentConfig =
+ folderSectionConfig[connType] || folderSectionConfig.input;
+
+ const handleModalClose = async () => {
+ const hasConfigChanges = !isEqual(formDataConfig, initialFormDataConfig);
+ const hasConnectorChanged = connDetails?.id !== initialConnectorId;
+ const hasChanges = hasConfigChanges || hasConnectorChanged;
+
+ // Auto-save if there are actual changes
+ if (hasChanges && connDetails?.id) {
+ setIsSavingEndpoint(true);
+ try {
+ const updatePayload = {};
+ if (hasConnectorChanged) {
+ updatePayload.connector_instance_id = connDetails.id;
+ }
+ if (hasConfigChanges && Object.keys(formDataConfig).length > 0) {
+ updatePayload.configuration = formDataConfig;
+ }
+ if (Object.keys(updatePayload).length > 0) {
+ await handleEndpointUpdate(updatePayload);
+ }
+ setOpen(false);
+ } catch (error) {
+ setAlertDetails({
+ type: "error",
+ content:
+ error?.message || "Failed to save changes. Please try again.",
+ });
+ } finally {
+ setIsSavingEndpoint(false);
+ }
+ } else {
+ setOpen(false);
+ }
+ };
+
+ // Load plugin tab for Human In The Loop (DATABASE connectors only)
useEffect(() => {
+ if (connMode !== "DATABASE") {
+ return;
+ }
+
try {
const tabOption =
require("../../../plugins/manual-review/connector-config-tab-mrq/ConnectorConfigTabMRQ").mrqTabs;
if (tabOption) {
- tabOption["disabled"] =
- !connectorId || connDetails?.connector_id !== selectedId;
- tabOption["visible"] = false;
+ tabOption["disabled"] = !connDetails?.id;
+ tabOption["visible"] = true;
setUpdatedTabOptions(tabOption);
}
} catch {
- // The component will remain null of it is not available
+ // The component will remain null if it is not available
}
- }, []);
- const handleSelectItem = (e) => {
- const id = e?.key;
- setSelectedId(id?.toString());
- setActiveKey("1");
+ }, [connMode, connDetails?.id]);
- const connectorData = [...filteredList].find((item) => item?.key === id);
- setSelectedItemName(connectorData?.label);
+ // Handle click-outside to clear file selection
+ useEffect(() => {
+ const handleClickOutside = (event) => {
+ if (
+ fileExplorerRef.current &&
+ !fileExplorerRef.current.contains(event.target)
+ ) {
+ setSelectedFolderPath("");
+ setIsFolderSelected(false);
+ }
+ };
- try {
- setPostHogCustomEvent(posthogConnectorEventText[`${connType}:${type}`], {
- info: `Selected a connector`,
- connector_name: connectorData?.label,
- });
- } catch (err) {
- // If an error occurs while setting custom posthog event, ignore it and continue
+ if (open && connMode === "FILESYSTEM") {
+ document.addEventListener("mousedown", handleClickOutside);
}
- };
- const onTabChange = (key) => {
- setActiveKey(key);
+ return () => {
+ document.removeEventListener("mousedown", handleClickOutside);
+ };
+ }, [open, connMode]);
+
+ // Fetch available connectors when modal opens and connection type is available
+ useEffect(() => {
+ if (open) {
+ fetchAvailableConnectors(connMode);
+ fetchEndpointConfigSchema();
+ }
+ }, [open, connMode]);
+
+ // Capture initial configuration and connector when modal opens
+ useEffect(() => {
+ if (open) {
+ // Store a deep copy of the initial configuration
+ if (endpointDetails?.configuration) {
+ setInitialFormDataConfig(cloneDeep(endpointDetails.configuration));
+ }
+ setInitialConnectorId(endpointDetails?.connector_instance?.id || null);
+ } else {
+ // Reset when modal closes
+ setInitialFormDataConfig({});
+ setInitialConnectorId(null);
+ }
+ }, [
+ open,
+ endpointDetails?.configuration,
+ endpointDetails?.connector_instance,
+ ]);
+
+ // Helper function to render connector label
+ const renderConnectorLabel = (connDetails, availableConnectors) => {
+ if (!connDetails?.id) return undefined;
+
+ const selectedConnector = availableConnectors.find(
+ (conn) => conn.value === connDetails.id
+ );
+
+ return (
+ <Space>
+ {selectedConnector?.icon && !selectedConnector?.isAddNew && (
+ <Image
+ src={selectedConnector.icon}
+ width={20}
+ height={20}
+ preview={false}
+ />
+ )}
+ <span>{selectedConnector?.label}</span>
+ </Space>
+ );
};
+ // Memoized dropdown render function
+ const renderDropdown = useCallback(
+ (menu) => (
+ <>
+ <div className="connector-dropdown-menu-container">{menu}</div>
+ {addNewOption && (
+ <button
+ className="connector-dropdown-add-new"
+ onClick={() => handleConnectorSelect("add_new")}
+ type="button"
+ aria-label="Add new connector"
+ >
+ <Space>
+ <span>{addNewOption.label}</span>
+ </Space>
+ </button>
+ )}
+ </>
+ ),
+ [addNewOption, handleConnectorSelect]
+ );
+
return (
<Modal
open={open}
- onCancel={() => setOpen(false)}
+ onCancel={handleModalClose}
+ confirmLoading={isSavingEndpoint}
+ closable={!isSavingEndpoint}
centered
footer={null}
width={1200}
@@ -170,54 +405,169 @@ function ConfigureConnectorModal({
<Typography.Text className="modal-header" strong>
Configure Connector
</Typography.Text>
- <div className="conn-modal-gap" />
- <Row className="conn-modal-row">
- <Col
- span={4}
- className="conn-modal-col conn-modal-col-left conn-modal-form-pad-right"
- >
- <ListOfConnectors
- listOfConnectors={filteredList}
- selectedId={selectedId}
- handleSelectItem={handleSelectItem}
- />
- </Col>
- <Col span={20} className="conn-modal-col conn-modal-form-pad-left">
- <Tabs
- activeKey={activeKey}
- items={tabItems.filter((item) => item?.visible !== false)}
- onChange={onTabChange}
- moreIcon={<></>}
- />
- {activeKey === "1" && (
- <ConfigureFormsLayout
- selectedId={selectedId}
- type={type}
- handleUpdate={handleUpdate}
- editItemId={connectorId}
- connectorMetadata={connectorMetadata}
- isConnAvailable={!!filteredList?.length}
- specConfig={specConfig}
- formDataConfig={formDataConfig}
- setFormDataConfig={setFormDataConfig}
- isSpecConfigLoading={isSpecConfigLoading}
- connDetails={connDetails}
- connType={connType}
- selectedItemName={selectedItemName}
- />
- )}
- {activeKey === "2" && connType === "FILESYSTEM" && (
- <ManageFiles selectedItem={connectorId} />
+
+ {/* Connector Selection Dropdown */}
+ <div className="connector-selection-section">
+ <Typography.Text strong className="connector-selection-label">
+ Select Connector
+ </Typography.Text>
+ <Select
+ className="connector-selection-dropdown"
+ placeholder="Select a connector"
+ showSearch
+ filterOption={(input, option) => {
+ return option?.data?.label
+ ?.toLowerCase()
+ .includes(input.toLowerCase());
+ }}
+ value={
+ connDetails?.id
+ ? {
+ value: connDetails.id,
+ label: renderConnectorLabel(
+ connDetails,
+ availableConnectors
+ ),
+ }
+ : undefined
+ }
+ labelInValue
+ onChange={(option) => handleConnectorSelect(option?.value)}
+ loading={isLoadingConnectors}
+ options={availableConnectors.map((conn) => ({
+ value: conn.value,
+ label: (
+ <Space>
+ {conn.icon && !conn.isAddNew && (
+ <Image
+ src={conn.icon}
+ width={20}
+ height={20}
+ preview={false}
+ />
+ )}
+ <span>{conn.label}</span>
+ </Space>
+ ),
+ data: conn,
+ }))}
+ optionRender={(option) => option.label}
+ dropdownRender={renderDropdown}
+ />
+ </div>
+
+ {/* Show placeholder when no connector is selected */}
+ {!connDetails?.id && (
+ <div className="connector-placeholder">
+ {connType === "input" ? (
+ <CloudDownloadOutlined className="connector-placeholder-icon" />
+ ) : (
+ <CloudUploadOutlined className="connector-placeholder-icon" />
)}
- {activeKey === "MANUALREVIEW" && DBRules && (
- <DBRules
- connDetails={connDetails}
- workflowDetails={workflowDetails}
+ <Typography.Text className="connector-placeholder-text">
+ Select an existing connector or create a new connector to continue
+ </Typography.Text>
+ </div>
+ )}
+
+ {/* Only show configuration form and file browser after a connector is selected */}
+ {connDetails?.id && (
+ <>
+ {/* DATABASE connectors: Show tabs with Settings and Human In The Loop */}
+ {connMode === "DATABASE" ? (
+ <Tabs
+ activeKey={activeTabKey}
+ onChange={onDBConfigTabChange}
+ className="conn-modal-col"
+ items={tabItems
+ .filter((item) => item.visible !== false)
+ .map((item) => ({
+ key: item.key,
+ label: item.label,
+ disabled: item.disabled,
+ children: (
+ <>
+ {item.key === "1" && (
+ <ConfigureFormsLayout
+ specConfig={specConfig}
+ formDataConfig={formDataConfig}
+ setFormDataConfig={setFormDataConfig}
+ isSpecConfigLoading={isSpecConfigLoading}
+ />
+ )}
+ {item.key === "MANUALREVIEW" && DBRules && (
+ <DBRules workflowDetails={workflowDetails} />
+ )}
+ </>
+ ),
+ }))}
/>
+ ) : (
+ /* Other connector types: Show existing layout */
+ <Row className="conn-modal-row" gutter={24}>
+ {/* Left side - Configuration Form */}
+ <Col span={12} className="conn-modal-col">
+ <div className="conn-modal-fs-config">
+ <ConfigureFormsLayout
+ specConfig={specConfig}
+ formDataConfig={formDataConfig}
+ setFormDataConfig={setFormDataConfig}
+ isSpecConfigLoading={isSpecConfigLoading}
+ />
+ </div>
+ </Col>
+
+ {/* Right side - File System Browser (only for FILESYSTEM connectors) */}
+ {connMode === "FILESYSTEM" && connDetails?.id && (
+ <Col span={12} className="conn-modal-col form-right">
+ <div className="file-browser-section" ref={fileExplorerRef}>
+ <div className="file-browser-header">
+ <div className="file-browser-content">
+ <Typography.Text strong style={{ display: "block" }}>
+ {currentConfig.title}
+ </Typography.Text>
+ <Typography.Text
+ type="secondary"
+ className="field-description"
+ >
+ {currentConfig.description}
+ </Typography.Text>
+ </div>
+ <CustomButton
+ type="primary"
+ size="small"
+ disabled={!isFolderSelected}
+ onClick={handleAddFolder}
+ >
+ {currentConfig.buttonText}
+ </CustomButton>
+ </div>
+ <ManageFiles
+ selectedConnector={connDetails?.id}
+ onFolderSelect={handleFolderSelect}
+ selectedFolderPath={selectedFolderPath}
+ />
+ </div>
+ </Col>
+ )}
+ </Row>
)}
- </Col>
- </Row>
+ </>
+ )}
</div>
+
+ {/* Add Source Modal for creating new connectors */}
+ {showAddSourceModal && (
+ <AddSourceModal
+ open={showAddSourceModal}
+ setOpen={setShowAddSourceModal}
+ isConnector={true}
+ type={connType}
+ addNewItem={handleConnectorCreated}
+ editItemId={null}
+ setEditItemId={() => {}}
+ />
+ )}
</Modal>
);
}
@@ -225,22 +575,15 @@ function ConfigureConnectorModal({
ConfigureConnectorModal.propTypes = {
open: PropTypes.bool.isRequired,
setOpen: PropTypes.func.isRequired,
- type: PropTypes.string.isRequired,
- connectorId: PropTypes.string,
- selectedId: PropTypes.string,
- setSelectedId: PropTypes.func.isRequired,
- handleUpdate: PropTypes.func.isRequired,
- filteredList: PropTypes.array,
- connectorMetadata: PropTypes.object,
- specConfig: PropTypes.object,
+ connDetails: PropTypes.object,
+ setConnDetails: PropTypes.func.isRequired,
+ connType: PropTypes.string.isRequired,
+ connMode: PropTypes.string.isRequired,
+ workflowDetails: PropTypes.object.isRequired,
+ handleEndpointUpdate: PropTypes.func.isRequired,
+ endpointDetails: PropTypes.object,
formDataConfig: PropTypes.object,
setFormDataConfig: PropTypes.func.isRequired,
- isSpecConfigLoading: PropTypes.bool.isRequired,
- connDetails: PropTypes.object,
- connType: PropTypes.string,
- selectedItemName: PropTypes.string,
- setSelectedItemName: PropTypes.func.isRequired,
- workflowDetails: PropTypes.object,
};
export { ConfigureConnectorModal };
diff --git a/frontend/src/components/agency/configure-forms-layout/ConfigureFormsLayout.css b/frontend/src/components/agency/configure-forms-layout/ConfigureFormsLayout.css
new file mode 100644
index 000000000..d6865e0e1
--- /dev/null
+++ b/frontend/src/components/agency/configure-forms-layout/ConfigureFormsLayout.css
@@ -0,0 +1,6 @@
+.config-content-area {
+ padding: 16px;
+ height: 100%;
+ overflow-y: auto;
+ overflow-x: hidden;
+}
diff --git a/frontend/src/components/agency/configure-forms-layout/ConfigureFormsLayout.jsx b/frontend/src/components/agency/configure-forms-layout/ConfigureFormsLayout.jsx
index d213fe89e..a270d4852 100644
--- a/frontend/src/components/agency/configure-forms-layout/ConfigureFormsLayout.jsx
+++ b/frontend/src/components/agency/configure-forms-layout/ConfigureFormsLayout.jsx
@@ -1,96 +1,52 @@
-import { Col, Row, Typography } from "antd";
import PropTypes from "prop-types";
import { SettingsForm } from "../settings-form/SettingsForm";
-import { AddSource } from "../../input-output/add-source/AddSource";
import { EmptyState } from "../../widgets/empty-state/EmptyState";
+import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader";
+
+import "./ConfigureFormsLayout.css";
function ConfigureFormsLayout({
- selectedId,
- type,
- handleUpdate,
- editItemId,
- connectorMetadata,
- isConnAvailable,
specConfig,
formDataConfig,
setFormDataConfig,
isSpecConfigLoading,
- connDetails,
- connType,
- selectedItemName,
}) {
+ // First check: Still loading
+ if (isSpecConfigLoading) {
+ return (
+ <div className="config-content-area">
+ <SpinnerLoader text="Loading configuration..." />
+ </div>
+ );
+ }
+
+ // Second check: Loaded but empty/failed
+ if (!specConfig || Object.keys(specConfig || {})?.length === 0) {
+ return (
+ <div className="config-content-area">
+ <EmptyState text="Failed to load the configuration form" />
+ </div>
+ );
+ }
+
return (
- <Row className="conn-modal-tab-body">
- <Col span={12} className="conn-modal-form-right">
- <div className="conn-modal-flex">
- <Typography.Text strong>Connection Settings</Typography.Text>
- <div className="conn-modal-gap" />
- <div className="conn-modal-flex-1">
- {!selectedId ? (
- <EmptyState
- text={
- isConnAvailable
- ? "Select the connector"
- : "No Connector available"
- }
- />
- ) : (
- <AddSource
- selectedSourceId={selectedId}
- metadata={connectorMetadata}
- type={type}
- editItemId={editItemId}
- handleUpdate={handleUpdate}
- connDetails={connDetails}
- connType={connType}
- selectedSourceName={selectedItemName}
- formDataConfig={formDataConfig}
- />
- )}
- </div>
- </div>
- </Col>
- {connType !== "MANUALREVIEW" && (
- <Col span={12} className="conn-modal-form-left">
- <div className="conn-modal-flex">
- <Typography.Text strong>Configuration</Typography.Text>
- <div className="conn-modal-gap" />
- <div className="conn-modal-flex-1">
- {!specConfig || Object.keys(specConfig)?.length === 0 ? (
- <EmptyState text="Failed to load the configuration form" />
- ) : (
- <SettingsForm
- selectedId={selectedId}
- handleUpdate={handleUpdate}
- spec={specConfig}
- formData={formDataConfig}
- setFormData={setFormDataConfig}
- isLoading={isSpecConfigLoading}
- />
- )}
- </div>
- </div>
- </Col>
- )}
- </Row>
+ <div className="config-content-area">
+ <SettingsForm
+ spec={specConfig}
+ formData={formDataConfig}
+ setFormData={setFormDataConfig}
+ isLoading={isSpecConfigLoading}
+ />
+ </div>
);
}
ConfigureFormsLayout.propTypes = {
- selectedId: PropTypes.string.isRequired,
- type: PropTypes.string.isRequired,
- handleUpdate: PropTypes.func.isRequired,
- editItemId: PropTypes.string,
- connectorMetadata: PropTypes.object,
- isConnAvailable: PropTypes.bool.isRequired,
specConfig: PropTypes.object,
formDataConfig: PropTypes.object,
setFormDataConfig: PropTypes.func.isRequired,
isSpecConfigLoading: PropTypes.bool.isRequired,
- connDetails: PropTypes.object,
- connType: PropTypes.string,
- selectedItemName: PropTypes.string,
};
export { ConfigureFormsLayout };
diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.css b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.css
index bbfca93af..4804fd85d 100644
--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.css
+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.css
@@ -14,6 +14,21 @@
align-self: flex-end;
}
+.ds-connector-info-wrapper {
+ margin-top: 16px;
+ max-width: 400px;
+}
+
+.ds-connector-info {
+ display: flex;
+ align-items: center;
+ gap: 8px;
+}
+
+.ds-connector-name {
+ max-width: 100%;
+}
+
.ds-set-card-message {
margin-bottom: 0px !important;
}
diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
index 7299fa16f..a70654a48 100644
--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
@@ -1,17 +1,16 @@
import {
Button,
Col,
- Image,
Row,
Select,
Space,
Tooltip,
Typography,
+ Image,
} from "antd";
import PropTypes from "prop-types";
import { useEffect, useState } from "react";
-import { getMenuItem } from "../../../helpers/GetStaticData";
import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";
import { useAlertStore } from "../../../store/alert-store";
import { useSessionStore } from "../../../store/session-store";
@@ -22,34 +21,17 @@ import { ConfigureConnectorModal } from "../configure-connector-modal/ConfigureC
import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
import "./DsSettingsCard.css";
-const disabledIdsByType = {
- FILE_SYSTEM: [
- "box|4d94d237-ce4b-45d8-8f34-ddeefc37c0bf",
- "http|6fdea346-86e4-4383-9a21-132db7c9a576",
- ],
-};
-
-const needToRemove = {
- FILE_SYSTEM: ["pcs|b8cd25cd-4452-4d54-bd5e-e7d71459b702"],
-};
-
-function DsSettingsCard({ type, endpointDetails, message }) {
+function DsSettingsCard({ connType, endpointDetails, message }) {
const workflowStore = useWorkflowStore();
const { source, destination, allowChangeEndpoint, details } = workflowStore;
- const [options, setOptions] = useState({});
+ const [options, setOptions] = useState([]);
const [openModal, setOpenModal] = useState(false);
- const [listOfConnectors, setListOfConnectors] = useState([]);
- const [filteredList, setFilteredList] = useState([]);
-
- const [connType, setConnType] = useState(null);
+ const [connMode, setConnMode] = useState(null);
const [connDetails, setConnDetails] = useState({});
- const [specConfig, setSpecConfig] = useState({});
- const [isSpecConfigLoading, setIsSpecConfigLoading] = useState(false);
const [formDataConfig, setFormDataConfig] = useState({});
- const [selectedId, setSelectedId] = useState("");
- const [selectedItemName, setSelectedItemName] = useState("");
+
const [inputOptions, setInputOptions] = useState([
{
value: "API",
@@ -76,10 +58,10 @@ function DsSettingsCard({ type, endpointDetails, message }) {
const setUpdatedInputoptions = (inputOption) => {
setInputOptions((prevInputOptions) => {
// Check if inputOption already exists in prevInputOptions
+ // Return previous state unchanged if it does or create new array
if (prevInputOptions.some((opt) => opt.value === inputOption.value)) {
- return prevInputOptions; // Return previous state unchanged
+ return prevInputOptions;
} else {
- // Create a new array with the existing options and the new option
const updatedInputOptions = [...prevInputOptions, inputOption];
return updatedInputOptions;
}
@@ -99,10 +81,10 @@ function DsSettingsCard({ type, endpointDetails, message }) {
}, []);
useEffect(() => {
- if (type === "output") {
+ if (connType === "output") {
if (source?.connection_type === "") {
// Clear options when source is blank
- setOptions({});
+ setOptions([]);
} else {
// Filter options based on source connection type
const isAPI = source?.connection_type === "API";
@@ -114,7 +96,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {
}
}
- if (type === "input") {
+ if (connType === "input") {
// Remove Database from Source Dropdown
const filteredOptions = inputOptions.filter(
(option) =>
@@ -124,9 +106,16 @@ function DsSettingsCard({ type, endpointDetails, message }) {
}
}, [source, destination]);
+ // Set formDataConfig from endpointDetails configuration
useEffect(() => {
- if (endpointDetails?.connection_type !== connType) {
- setConnType(endpointDetails?.connection_type);
+ if (endpointDetails?.configuration) {
+ setFormDataConfig(endpointDetails.configuration);
+ }
+ }, [endpointDetails?.configuration]);
+
+ useEffect(() => {
+ if (endpointDetails?.connection_type !== connMode) {
+ setConnMode(endpointDetails?.connection_type);
}
if (!endpointDetails?.connector_instance) {
@@ -136,117 +125,16 @@ function DsSettingsCard({ type, endpointDetails, message }) {
// Use connector_instance data directly from endpointDetails if it's an object
if (typeof endpointDetails?.connector_instance === "object") {
- const connectorData = endpointDetails.connector_instance;
- connectorData.connector_metadata = connectorData.connector_metadata || {};
- connectorData.connector_metadata.connectorName =
- connectorData?.connector_name || "";
- setConnDetails(connectorData);
- setSelectedId(connectorData?.connector_id);
- return;
- }
-
- // Fallback for legacy connector_instance ID format (string)
- if (typeof endpointDetails?.connector_instance === "string") {
- // Only call getSourceDetails if we haven't already loaded this connector
- if (connDetails?.id !== endpointDetails?.connector_instance) {
- getSourceDetails();
- }
+ setConnDetails(endpointDetails.connector_instance);
return;
}
}, [endpointDetails]);
- useEffect(() => {
- const menuItems = [];
- [...listOfConnectors].forEach((item) => {
- if (
- endpointDetails?.connection_type &&
- item?.connector_mode.split("_").join("") !==
- endpointDetails?.connection_type
- ) {
- return;
- }
- menuItems.push(
- getMenuItem(
- item?.name,
- item?.id,
- sourceIcon(item?.icon),
- undefined,
- undefined,
- item?.isDisabled
- )
- );
- });
- setSelectedId("");
- setFilteredList(menuItems);
-
- if (!endpointDetails?.id) {
- return;
- }
-
- setFormDataConfig(endpointDetails.configuration || {});
- const requestOptions = {
- method: "GET",
- url: getUrl(`workflow/endpoint/${endpointDetails?.id}/settings/`),
- };
-
- setIsSpecConfigLoading(true);
- axiosPrivate(requestOptions)
- .then((res) => {
- const data = res?.data;
- setSpecConfig(data?.schema || {});
- })
- .catch((err) => {
- setAlertDetails(handleException(err, "Failed to load the spec"));
- })
- .finally(() => {
- setIsSpecConfigLoading(false);
- });
- }, [connType, listOfConnectors]);
-
- useEffect(() => {
- if (!type) {
+ const clearDestination = (updatedData) => {
+ if (!destination) {
return;
}
- const requestOptions = {
- method: "GET",
- url: getUrl(`supported_connectors/?type=${type.toUpperCase()}`),
- };
-
- axiosPrivate(requestOptions)
- .then((res) => {
- let data = res?.data;
- // Remove items specified in needToRemove from data
- Object.keys(needToRemove).forEach((mode) => {
- const idsToRemove = needToRemove[mode];
- data = data.filter(
- (source) =>
- !(
- source.connector_mode === mode &&
- idsToRemove.includes(source.id)
- )
- );
- });
-
- const updatedSources = data?.map((source) => ({
- ...source,
- isDisabled: disabledIdsByType[source?.connector_mode]?.includes(
- source?.id
- ),
- }));
- setListOfConnectors(updatedSources || []);
- })
- .catch((err) => {
- setAlertDetails(handleException(err));
- })
- .finally(() => {});
- }, [type]);
-
- const sourceIcon = (src) => {
- return <Image src={src} height={25} width={25} preview={false} />;
- };
-
- const clearDestination = (updatedData) => {
const requestOptions = {
method: "PATCH",
url: getUrl(`workflow/endpoint/${destination?.id}/`),
@@ -271,7 +159,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {
const updateDestination = () => {
// Clear destination dropdown & data when input is switched
- if (type === "input") {
+ if (connType === "input") {
clearDestination({
connection_type: "",
connector_instance_id: null,
@@ -279,7 +167,11 @@ function DsSettingsCard({ type, endpointDetails, message }) {
}
};
- const handleUpdate = (updatedData, showSuccess) => {
+ const handleEndpointUpdate = (updatedData, showSuccess) => {
+ if (!endpointDetails?.id) {
+ return Promise.resolve();
+ }
+
const requestOptions = {
method: "PATCH",
url: getUrl(`workflow/endpoint/${endpointDetails?.id}/`),
@@ -289,11 +181,12 @@ function DsSettingsCard({ type, endpointDetails, message }) {
},
data: updatedData,
};
- axiosPrivate(requestOptions)
+
+ return axiosPrivate(requestOptions)
.then((res) => {
const data = res?.data || {};
const updatedData = {};
- if (type === "input") {
+ if (connType === "input") {
updatedData["source"] = data;
} else {
updatedData["destination"] = data;
@@ -305,28 +198,11 @@ function DsSettingsCard({ type, endpointDetails, message }) {
content: "Successfully updated",
});
}
+ return res;
})
.catch((err) => {
setAlertDetails(handleException(err, "Failed to update"));
- });
- };
-
- const getSourceDetails = () => {
- const requestOptions = {
- method: "GET",
- url: getUrl(`connector/${endpointDetails?.connector_instance}/`),
- };
-
- axiosPrivate(requestOptions)
- .then((res) => {
- const data = res?.data;
- data["connector_metadata"]["connectorName"] =
- data?.connector_name || "";
- setConnDetails(data);
- setSelectedId(data?.connector_id);
- })
- .catch((err) => {
- setAlertDetails(handleException(err));
+ throw err;
});
};
@@ -338,27 +214,6 @@ function DsSettingsCard({ type, endpointDetails, message }) {
return "";
};
- // Check if connector is configured
- const isConnectorConfigured = () => {
- // For API connections, connector_instance is not required
- if (endpointDetails?.connection_type === "API") {
- return true;
- }
-
- // For other connection types (Database, etc.), check if connector instance is configured
- // connector_instance represents the actual configured connector with credentials
- return !!endpointDetails?.connector_instance;
- };
-
- // Get configure button text based on configuration status
- const getConfigureButtonText = () => {
- if (!endpointDetails?.connection_type) {
- return "Configure";
- }
-
- return isConnectorConfigured() ? "Configured" : "Configure";
- };
-
return (
<>
<Row className="ds-set-card-row">
@@ -387,7 +242,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {
value={endpointDetails?.connection_type || undefined}
disabled={!allowChangeEndpoint}
onChange={(value) => {
- handleUpdate({
+ handleEndpointUpdate({
connection_type: value,
connector_instance_id: null,
});
@@ -402,35 +257,49 @@ function DsSettingsCard({ type, endpointDetails, message }) {
onClick={() => setOpenModal(true)}
disabled={
!endpointDetails?.connection_type ||
- connType === "API" ||
- connType === "APPDEPLOYMENT"
+ connMode === "API" ||
+ connMode === "APPDEPLOYMENT"
}
>
- {getConfigureButtonText()}
+ Configure
</Button>
</Tooltip>
</Space>
</SpaceWrapper>
+ {connDetails?.id && (
+ <div className="ds-connector-info-wrapper">
+ <Space className="ds-connector-info">
+ {connDetails?.icon && (
+ <Image
+ src={connDetails.icon}
+ width={20}
+ height={20}
+ preview={false}
+ alt="connector-icon"
+ />
+ )}
+ <Typography.Text
+ className="ds-connector-name"
+ ellipsis={{ tooltip: true }}
+ >
+ {connDetails?.connector_name || "Unnamed Connector"}
+ </Typography.Text>
+ </Space>
+ </div>
+ )}
</Col>
</Row>
<ConfigureConnectorModal
open={openModal}
setOpen={setOpenModal}
- type={type}
- selectedId={selectedId}
- setSelectedId={setSelectedId}
- handleUpdate={handleUpdate}
- filteredList={filteredList}
- connectorMetadata={connDetails?.connector_metadata}
- connectorId={connDetails?.id}
- specConfig={specConfig}
+ connType={connType}
+ handleEndpointUpdate={handleEndpointUpdate}
+ endpointDetails={endpointDetails}
formDataConfig={formDataConfig}
setFormDataConfig={setFormDataConfig}
- isSpecConfigLoading={isSpecConfigLoading}
connDetails={connDetails}
- connType={connType}
- selectedItemName={selectedItemName}
- setSelectedItemName={setSelectedItemName}
+ setConnDetails={setConnDetails}
+ connMode={connMode}
workflowDetails={details}
/>
</>
@@ -438,7 +307,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {
}
DsSettingsCard.propTypes = {
- type: PropTypes.string.isRequired,
+ connType: PropTypes.string.isRequired,
endpointDetails: PropTypes.object.isRequired,
message: PropTypes.string,
};
diff --git a/frontend/src/components/agency/list-of-connectors/ListOfConnectors.css b/frontend/src/components/agency/list-of-connectors/ListOfConnectors.css
deleted file mode 100644
index c506b8df0..000000000
--- a/frontend/src/components/agency/list-of-connectors/ListOfConnectors.css
+++ /dev/null
@@ -1,13 +0,0 @@
-.coming-soon-img {
- position: absolute;
- top: 0;
- right: 0;
- width: 40px;
- height: 40px;
-}
-
-.coming-soon-container {
- position: relative;
- display: inline-block;
- width: 100%;
-}
diff --git a/frontend/src/components/agency/list-of-connectors/ListOfConnectors.jsx b/frontend/src/components/agency/list-of-connectors/ListOfConnectors.jsx
deleted file mode 100644
index 1357eb656..000000000
--- a/frontend/src/components/agency/list-of-connectors/ListOfConnectors.jsx
+++ /dev/null
@@ -1,49 +0,0 @@
-import { Menu, Tooltip } from "antd";
-import PropTypes from "prop-types";
-import comingSoon from "../../../assets/coming-soon.png";
-import "./ListOfConnectors.css";
-
-function ListOfConnectors({ listOfConnectors, selectedId, handleSelectItem }) {
- return (
- <div className="conn-modal-menu">
- <Menu
- className="sidebar-menu"
- style={{ border: 0 }}
- selectedKeys={[selectedId]}
- mode="inline"
- items={listOfConnectors.map((item) => ({
- key: item?.key,
- label: (
- <>
- {item?.isDisabled ? (
- <Tooltip title="Coming Soon">
- <div className="coming-soon-container">
- <img
- src={comingSoon}
- alt="Coming Soon"
- className="coming-soon-img"
- />
- {item?.label}
- </div>
- </Tooltip>
- ) : (
- item?.label
- )}
- </>
- ),
- disabled: item?.isDisabled,
- icon: item?.icon,
- }))}
- onClick={handleSelectItem}
- />
- </div>
- );
-}
-
-ListOfConnectors.propTypes = {
- listOfConnectors: PropTypes.array.isRequired,
- selectedId: PropTypes.string,
- handleSelectItem: PropTypes.func.isRequired,
-};
-
-export { ListOfConnectors };
diff --git a/frontend/src/components/agency/settings-form/SettingsForm.jsx b/frontend/src/components/agency/settings-form/SettingsForm.jsx
index 6da9347e1..f7e4c162f 100644
--- a/frontend/src/components/agency/settings-form/SettingsForm.jsx
+++ b/frontend/src/components/agency/settings-form/SettingsForm.jsx
@@ -1,34 +1,16 @@
import PropTypes from "prop-types";
import { RjsfFormLayout } from "../../../layouts/rjsf-form-layout/RjsfFormLayout";
-import { CustomButton } from "../../widgets/custom-button/CustomButton";
-
-function SettingsForm({
- spec,
- formData,
- setFormData,
- isLoading,
- handleUpdate,
-}) {
- const handleSave = () => {
- handleUpdate({ configuration: formData }, true);
- };
+function SettingsForm({ spec, formData, setFormData, isLoading }) {
return (
<RjsfFormLayout
schema={spec}
formData={formData}
setFormData={setFormData}
isLoading={isLoading}
- validateAndSubmit={handleSave}
isStateUpdateRequired={true}
- >
- <div className="display-flex-right tool-settings-submit-btn">
- <CustomButton type="primary" htmlType="submit">
- Save
- </CustomButton>
- </div>
- </RjsfFormLayout>
+ />
);
}
@@ -37,7 +19,6 @@ SettingsForm.propTypes = {
formData: PropTypes.object,
setFormData: PropTypes.func.isRequired,
isLoading: PropTypes.bool.isRequired,
- handleUpdate: PropTypes.func.isRequired,
};
export { SettingsForm };
diff --git a/frontend/src/components/agency/workflow-card/WorkflowCard.jsx b/frontend/src/components/agency/workflow-card/WorkflowCard.jsx
index 783f0a63d..56e232307 100644
--- a/frontend/src/components/agency/workflow-card/WorkflowCard.jsx
+++ b/frontend/src/components/agency/workflow-card/WorkflowCard.jsx
@@ -68,7 +68,7 @@ function WorkflowCard({
number,
title,
description,
- type,
+ connType,
endpointDetails,
message,
customContent,
@@ -97,7 +97,7 @@ function WorkflowCard({
<div className="workflow-card-content">
{customContent || (
<DsSettingsCard
- type={type}
+ connType={connType}
endpointDetails={endpointDetails}
message={message}
/>
@@ -113,7 +113,7 @@ WorkflowCard.propTypes = {
number: PropTypes.oneOfType([PropTypes.string, PropTypes.number]).isRequired,
title: PropTypes.string.isRequired,
description: PropTypes.string.isRequired,
- type: PropTypes.string,
+ connType: PropTypes.string,
endpointDetails: PropTypes.object,
message: PropTypes.string,
customContent: PropTypes.node,
diff --git a/frontend/src/components/connectors/connector-card/ConnectorCard.css b/frontend/src/components/connectors/connector-card/ConnectorCard.css
new file mode 100644
index 000000000..fc5b0802d
--- /dev/null
+++ b/frontend/src/components/connectors/connector-card/ConnectorCard.css
@@ -0,0 +1,16 @@
+/* Additional styles for ConnectorCard that extend DataSourceCard */
+.connector-card:hover {
+ transform: translateY(-2px);
+ box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15);
+}
+
+.connector-card.selected {
+ border-color: #1890ff;
+ box-shadow: 0 0 0 2px rgba(24, 144, 255, 0.2);
+}
+
+.display-flex-center {
+ display: flex;
+ align-items: center;
+ justify-content: center;
+}
diff --git a/frontend/src/components/connectors/connector-card/ConnectorCard.jsx b/frontend/src/components/connectors/connector-card/ConnectorCard.jsx
new file mode 100644
index 000000000..183aa6214
--- /dev/null
+++ b/frontend/src/components/connectors/connector-card/ConnectorCard.jsx
@@ -0,0 +1,63 @@
+import { Card, Image, Typography } from "antd";
+import PropTypes from "prop-types";
+
+import "../../input-output/data-source-card/DataSourceCard.css";
+import "./ConnectorCard.css";
+
+function ConnectorCard({ connector, onSelect, isSelected = false }) {
+ const handleSelect = () => {
+ if (connector?.isDisabled) {
+ return;
+ }
+ onSelect(connector);
+ };
+
+ return (
+ <Card
+ hoverable={!connector?.isDisabled}
+ size="small"
+ type="inner"
+ bordered={true}
+ className={`ds-card connector-card ${
+ connector?.isDisabled ? "disabled" : ""
+ } ${isSelected ? "selected" : ""}`}
+ onClick={handleSelect}
+ >
+ <div className="cover-container">
+ {connector?.isDisabled && (
+ <div className="disabled-overlay">
+ <Typography.Text strong>Coming Soon</Typography.Text>
+ </div>
+ )}
+ <div className="cover-img">
+ <Image
+ src={connector?.icon}
+ width="80%"
+ height="auto"
+ preview={false}
+ fallback="/api/static/default-connector-icon.svg" // Fallback icon
+ />
+ </div>
+ <div className="ds-card-name display-flex-center">
+ <Typography.Text ellipsis={{ tooltip: connector?.name }}>
+ {connector?.name}
+ </Typography.Text>
+ </div>
+ </div>
+ </Card>
+ );
+}
+
+ConnectorCard.propTypes = {
+ connector: PropTypes.shape({
+ id: PropTypes.string.isRequired,
+ name: PropTypes.string.isRequired,
+ icon: PropTypes.string,
+ isDisabled: PropTypes.bool,
+ connector_mode: PropTypes.string,
+ }).isRequired,
+ onSelect: PropTypes.func.isRequired,
+ isSelected: PropTypes.bool,
+};
+
+export { ConnectorCard };
diff --git a/frontend/src/components/connectors/connector-list-modal/ConnectorListModal.css b/frontend/src/components/connectors/connector-list-modal/ConnectorListModal.css
new file mode 100644
index 000000000..71e2b15f5
--- /dev/null
+++ b/frontend/src/components/connectors/connector-list-modal/ConnectorListModal.css
@@ -0,0 +1,49 @@
+.connector-list-modal {
+ padding: 0;
+}
+
+.connector-search-section {
+ margin-bottom: 20px;
+}
+
+.connector-search-section .search-icon {
+ color: #bfbfbf;
+}
+
+.connector-tabs {
+ margin-bottom: 20px;
+}
+
+.connector-tabs .ant-tabs-nav {
+ margin-bottom: 16px;
+}
+
+.connector-grid-container {
+ min-height: 300px;
+ max-height: 400px;
+ overflow-y: auto;
+ padding: 16px 0;
+}
+
+.no-connectors {
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ min-height: 200px;
+ text-align: center;
+}
+
+.connector-list-loading {
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ justify-content: center;
+ min-height: 200px;
+ padding: 40px;
+}
+
+.connector-list-loading-text {
+ margin-top: 16px;
+ display: block;
+ text-align: center;
+}
diff --git a/frontend/src/components/connectors/connector-list-modal/ConnectorListModal.jsx b/frontend/src/components/connectors/connector-list-modal/ConnectorListModal.jsx
new file mode 100644
index 000000000..57a63449c
--- /dev/null
+++ b/frontend/src/components/connectors/connector-list-modal/ConnectorListModal.jsx
@@ -0,0 +1,147 @@
+import { SearchOutlined } from "@ant-design/icons";
+import { Input, Row, Col, Tabs, Typography, Spin } from "antd";
+import PropTypes from "prop-types";
+import { useEffect, useState } from "react";
+import debounce from "lodash/debounce";
+
+import { ConnectorCard } from "../connector-card/ConnectorCard";
+import "./ConnectorListModal.css";
+
+function ConnectorListModal({
+ connectors,
+ onSelectConnector,
+ selectedConnectorId,
+ loading = false,
+}) {
+ const [filteredConnectors, setFilteredConnectors] = useState([]);
+ const [searchText, setSearchText] = useState("");
+ const [activeTab, setActiveTab] = useState("all");
+
+ useEffect(() => {
+ filterConnectors(searchText, activeTab);
+ }, [connectors, searchText, activeTab]);
+
+ const filterConnectors = debounce((searchText, mode) => {
+ let filtered = [...connectors];
+
+ // Filter by search text
+ if (searchText?.trim()) {
+ const searchUpper = searchText.toUpperCase().trim();
+ filtered = filtered.filter((connector) =>
+ connector?.name?.toUpperCase().includes(searchUpper)
+ );
+ }
+
+ // Filter by connector mode
+ if (mode !== "all") {
+ filtered = filtered.filter((connector) => {
+ if (mode === "FILESYSTEM") {
+ return (
+ connector?.connector_mode === "FILESYSTEM" ||
+ connector?.can_write ||
+ connector?.can_read
+ ); // fallback logic
+ }
+ if (mode === "DATABASE") {
+ return connector?.connector_mode === "DATABASE";
+ }
+ return true;
+ });
+ }
+
+ setFilteredConnectors(filtered);
+ }, 300);
+
+ const handleSearchChange = (event) => {
+ const { value } = event.target;
+ setSearchText(value);
+ };
+
+ const handleTabChange = (key) => {
+ setActiveTab(key);
+ };
+
+ const handleConnectorSelect = (connector) => {
+ onSelectConnector(connector);
+ };
+
+ const tabItems = [
+ {
+ key: "all",
+ label: "All Connectors",
+ },
+ {
+ key: "FILESYSTEM",
+ label: "File Systems",
+ },
+ {
+ key: "DATABASE",
+ label: "Databases",
+ },
+ ];
+
+ if (loading) {
+ return (
+ <div className="connector-list-loading">
+ <Spin size="large" />
+ <Typography.Text className="connector-list-loading-text">
+ Loading connectors...
+ </Typography.Text>
+ </div>
+ );
+ }
+
+ return (
+ <div className="connector-list-modal">
+ <div className="connector-search-section">
+ <Input
+ placeholder="Search for connectors..."
+ prefix={<SearchOutlined className="search-icon" />}
+ onChange={handleSearchChange}
+ value={searchText}
+ size="large"
+ />
+ </div>
+
+ <Tabs
+ activeKey={activeTab}
+ onChange={handleTabChange}
+ items={tabItems}
+ className="connector-tabs"
+ />
+
+ <div className="connector-grid-container">
+ {filteredConnectors.length === 0 ? (
+ <div className="no-connectors">
+ <Typography.Text type="secondary">
+ {searchText
+ ? "No connectors found matching your search."
+ : "No connectors available."}
+ </Typography.Text>
+ </div>
+ ) : (
+ <Row gutter={[24, 24]} justify="start">
+ {filteredConnectors.map((connector) => (
+ <Col key={connector.id} xs={24} sm={12} md={8} lg={6} xl={6}>
+ <ConnectorCard
+ connector={connector}
+ onSelect={handleConnectorSelect}
+ isSelected={selectedConnectorId === connector.id}
+ />
+ </Col>
+ ))}
+ </Row>
+ )}
+ </div>
+ </div>
+ );
+}
+
+ConnectorListModal.propTypes = {
+ connectors: PropTypes.array.isRequired,
+ onSelectConnector: PropTypes.func.isRequired,
+ selectedConnectorId: PropTypes.string,
+ loading: PropTypes.bool,
+};
+
+export { ConnectorListModal };
diff --git a/frontend/src/components/input-output/add-source-modal/AddSourceModal.jsx b/frontend/src/components/input-output/add-source-modal/AddSourceModal.jsx
index 399fd1e47..bd803a816 100644
--- a/frontend/src/components/input-output/add-source-modal/AddSourceModal.jsx
+++ b/frontend/src/components/input-output/add-source-modal/AddSourceModal.jsx
@@ -2,19 +2,19 @@ import { Modal } from "antd";
import PropTypes from "prop-types";
import { useEffect, useState } from "react";
-import { sourceTypes } from "../../../helpers/GetStaticData";
import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";
import { useAlertStore } from "../../../store/alert-store";
-import { useSessionStore } from "../../../store/session-store";
import { AddSource } from "../add-source/AddSource";
import { ListOfSources } from "../list-of-sources/ListOfSources";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader";
+import useRequestUrl from "../../../hooks/useRequestUrl";
function AddSourceModal({
open,
setOpen,
type,
+ isConnector,
addNewItem,
editItemId,
setEditItemId,
@@ -23,10 +23,10 @@ function AddSourceModal({
const [metadata, setMetadata] = useState({});
const [titles, setTitles] = useState({});
const [selectedSourceName, setSelectedSourceName] = useState("");
- const { sessionDetails } = useSessionStore();
const { setAlertDetails } = useAlertStore();
const axiosPrivate = useAxiosPrivate();
const handleException = useExceptionHandler();
+ const { getUrl } = useRequestUrl();
const [isLoading, setIsLoading] = useState(false);
const [sourcesList, setSourcesList] = useState([]);
@@ -39,12 +39,11 @@ function AddSourceModal({
useEffect(() => {
const addOrEdit = editItemId?.length ? "Edit" : "Add";
setTitles({
- input: addOrEdit + " Data Source",
- output: addOrEdit + " Data Destination",
llm: addOrEdit + " LLM",
vector_db: addOrEdit + " Vector DB",
embedding: addOrEdit + " Embedding",
x2text: addOrEdit + " Text Extractor",
+ connectors: addOrEdit + " Connector",
});
if (editItemId?.length) {
@@ -61,9 +60,7 @@ function AddSourceModal({
}, 500);
}
- if (type && type !== null) {
- getListOfSources();
- }
+ getListOfSources();
}, [open]);
useEffect(() => {
@@ -74,12 +71,11 @@ function AddSourceModal({
}, [selectedSourceId]);
const getSourceDetails = () => {
- const isConnector = sourceTypes.connectors.includes(type);
- let url = `/api/v1/unstract/${sessionDetails?.orgId}`;
+ let url;
if (isConnector) {
- url += `/connector/${editItemId}/`;
+ url = getUrl(`connector/${editItemId}/`);
} else {
- url += `/adapter/${editItemId}/`;
+ url = getUrl(`adapter/${editItemId}/`);
}
const requestOptions = {
@@ -111,11 +107,13 @@ function AddSourceModal({
};
const getListOfSources = () => {
- let url = `/api/v1/unstract/${sessionDetails?.orgId}`;
- if (type && sourceTypes.connectors.includes(type)) {
- url += `/supported_connectors/?type=${type?.toUpperCase()}`;
+ let url;
+ if (isConnector) {
+ // For centralized connectors, get all connectors
+ url = getUrl(`supported_connectors/`);
} else {
- url += `/supported_adapters/?adapter_type=${type?.toUpperCase()}`;
+ if (!type) return;
+ url = getUrl(`supported_adapters/?adapter_type=${type.toUpperCase()}`);
}
// API to get the list of adapters.
const requestOptions = {
@@ -152,7 +150,7 @@ function AddSourceModal({
setMetadata(null);
}}
maskClosable={false}
- title={titles[type]}
+ title={isConnector ? titles["connectors"] : titles[type]}
width={selectedSourceId?.length ? 500 : 1100}
centered
footer={null}
@@ -164,6 +162,7 @@ function AddSourceModal({
selectedSourceName={selectedSourceName}
setOpen={setOpen}
type={type}
+ isConnector={isConnector}
addNewItem={addNewItem}
editItemId={editItemId}
metadata={metadata}
@@ -186,9 +185,14 @@ AddSourceModal.propTypes = {
open: PropTypes.bool.isRequired,
setOpen: PropTypes.func.isRequired,
type: PropTypes.any,
+ isConnector: PropTypes.bool,
addNewItem: PropTypes.func.isRequired,
editItemId: PropTypes.string,
setEditItemId: PropTypes.func.isRequired,
};
+AddSourceModal.defaultProps = {
+ isConnector: false,
+};
+
export { AddSourceModal };
diff --git a/frontend/src/components/input-output/add-source/AddSource.jsx b/frontend/src/components/input-output/add-source/AddSource.jsx
index 96fb5d60a..30f5abd3d 100644
--- a/frontend/src/components/input-output/add-source/AddSource.jsx
+++ b/frontend/src/components/input-output/add-source/AddSource.jsx
@@ -2,13 +2,12 @@ import { Typography } from "antd";
import PropTypes from "prop-types";
import { useEffect, useMemo, useState } from "react";
-import { sourceTypes } from "../../../helpers/GetStaticData";
import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";
import { useAlertStore } from "../../../store/alert-store";
-import { useSessionStore } from "../../../store/session-store";
import { EmptyState } from "../../widgets/empty-state/EmptyState";
import { ConfigureDs } from "../configure-ds/ConfigureDs";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
+import useRequestUrl from "../../../hooks/useRequestUrl";
let transformLlmWhispererJsonSchema;
let LLMW_V2_ID;
@@ -33,22 +32,19 @@ function AddSource({
selectedSourceName,
setOpen,
type,
+ isConnector,
addNewItem,
editItemId,
metadata,
- handleUpdate,
- connDetails,
- connType,
- formDataConfig,
}) {
const [spec, setSpec] = useState({});
const [formData, setFormData] = useState({});
const [isLoading, setIsLoading] = useState(false);
const [oAuthProvider, setOAuthProvider] = useState("");
- const { sessionDetails } = useSessionStore();
const { setAlertDetails } = useAlertStore();
const axiosPrivate = useAxiosPrivate();
const handleException = useExceptionHandler();
+ const { getUrl } = useRequestUrl();
let transformLlmWhispererFormData;
try {
@@ -99,11 +95,11 @@ function AddSource({
return;
}
- let url = `/api/v1/unstract/${sessionDetails?.orgId}`;
- if (sourceTypes.connectors.includes(type)) {
- url += `/connector_schema/?id=${selectedSourceId}`;
+ let url;
+ if (isConnector) {
+ url = getUrl(`connector_schema/?id=${selectedSourceId}`);
} else {
- url += `/adapter_schema/?id=${selectedSourceId}`;
+ url = getUrl(`adapter_schema/?id=${selectedSourceId}`);
}
const requestOptions = {
@@ -168,17 +164,9 @@ function AddSource({
addNewItem={addNewItem}
type={type}
editItemId={editItemId}
- sourceType={
- sourceTypes.connectors.includes(type)
- ? Object.keys(sourceTypes)[0]
- : Object.keys(sourceTypes)[1]
- }
- handleUpdate={handleUpdate}
- connDetails={connDetails}
+ isConnector={isConnector}
metadata={metadata}
selectedSourceName={selectedSourceName}
- connType={connType}
- formDataConfig={formDataConfig}
/>
);
}
@@ -187,14 +175,11 @@ AddSource.propTypes = {
selectedSourceId: PropTypes.string.isRequired,
selectedSourceName: PropTypes.string,
setOpen: PropTypes.func,
- type: PropTypes.string.isRequired,
+ type: PropTypes.string,
+ isConnector: PropTypes.bool.isRequired,
addNewItem: PropTypes.func,
editItemId: PropTypes.string,
metadata: PropTypes.object,
- handleUpdate: PropTypes.func,
- connDetails: PropTypes.object,
- connType: PropTypes.string,
- formDataConfig: PropTypes.object,
};
export { AddSource };
diff --git a/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx b/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx
index bfbd27e34..12c81062a 100644
--- a/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx
+++ b/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx
@@ -1,9 +1,7 @@
import { Col, Row } from "antd";
import PropTypes from "prop-types";
import { createRef, useEffect, useState } from "react";
-import { useParams } from "react-router-dom";
-import { sourceTypes } from "../../../helpers/GetStaticData.js";
import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx";
import { RjsfFormLayout } from "../../../layouts/rjsf-form-layout/RjsfFormLayout.jsx";
@@ -13,6 +11,7 @@ import { OAuthDs } from "../../oauth-ds/oauth-ds/OAuthDs.jsx";
import { CustomButton } from "../../widgets/custom-button/CustomButton.jsx";
import "./ConfigureDs.css";
import usePostHogEvents from "../../../hooks/usePostHogEvents.js";
+import useRequestUrl from "../../../hooks/useRequestUrl";
function ConfigureDs({
spec,
@@ -25,13 +24,9 @@ function ConfigureDs({
addNewItem,
type,
editItemId,
- sourceType,
- handleUpdate,
- connDetails,
+ isConnector,
metadata,
selectedSourceName,
- connType,
- formDataConfig,
}) {
const formRef = createRef(null);
const axiosPrivate = useAxiosPrivate();
@@ -51,14 +46,10 @@ function ConfigureDs({
posthogConnectorAddedEventText,
setPostHogCustomEvent,
} = usePostHogEvents();
+ const { getUrl } = useRequestUrl();
- const { id } = useParams();
-
- // Map connector type to proper role for OAuth isolation
- const connectorRole = type === "input" ? "SOURCE" : "DESTINATION";
-
- const oauthCacheKey = `oauth-cachekey-${id}-${connectorRole}-${selectedSourceId}`;
- const oauthStatusKey = `oauth-status-${id}-${connectorRole}-${selectedSourceId}`;
+ const oauthCacheKey = `oauth-cachekey-${selectedSourceId}`;
+ const oauthStatusKey = `oauth-status-${selectedSourceId}`;
// Initialize OAuth state from localStorage after keys are available
useEffect(() => {
@@ -106,15 +97,9 @@ function ConfigureDs({
}, [formData]);
useEffect(() => {
- const { connector_id: connectorId } = connDetails || {};
-
- // Check if connectorId matches selectedSourceId and metadata is available
- const shouldSetMetadata = connectorId === selectedSourceId && metadata;
- if (!shouldSetMetadata) return;
-
- // Set formData based on the condition
+ if (!metadata) return;
setFormData(metadata);
- }, [selectedSourceId]);
+ }, [selectedSourceId, metadata, setFormData]);
// Clear OAuth state when switching to a different connector
useEffect(() => {
@@ -129,7 +114,7 @@ function ConfigureDs({
setStatus("");
setCacheKey("");
}
- }, [selectedSourceId, id, connectorRole, oauthStatusKey, oauthCacheKey]);
+ }, [selectedSourceId, oauthStatusKey, oauthCacheKey]);
// Restore OAuth state when returning to a connector with stored credentials
useEffect(() => {
@@ -145,14 +130,7 @@ function ConfigureDs({
setCacheKey(storedCacheKey);
}
}
- }, [
- selectedSourceId,
- id,
- connectorRole,
- oAuthProvider,
- oauthStatusKey,
- oauthCacheKey,
- ]);
+ }, [selectedSourceId, oAuthProvider, oauthStatusKey, oauthCacheKey]);
const isFormValid = () => {
if (formRef) {
@@ -171,34 +149,35 @@ function ConfigureDs({
return;
}
if (oAuthProvider?.length && (status !== "success" || !cacheKey?.length)) {
+ const providerName =
+ oAuthProvider === "google-oauth2" ? "Google" : "OAuth provider";
setAlertDetails({
type: "error",
- content:
- "OAuth authentication required. Please sign in with Google first.",
+ content: `OAuth authentication required. Please sign in with ${providerName} first.`,
});
return;
}
let body = {};
- let url = `/api/v1/unstract/${sessionDetails?.orgId}/`;
+ let url;
- if (sourceType === Object.keys(sourceTypes)[0]) {
+ if (isConnector) {
const connectorMetadata = { ...updatedFormData };
delete connectorMetadata.connectorName;
body = {
connector_id: selectedSourceId,
connector_metadata: connectorMetadata,
};
- url += "test_connectors/";
+ url = getUrl("test_connectors/");
} else {
const adapterMetadata = { ...updatedFormData };
delete adapterMetadata.adapterName;
body = {
adapter_id: selectedSourceId,
adapter_metadata: adapterMetadata,
- adapter_type: type.toUpperCase(),
+ adapter_type: type?.toUpperCase(),
};
- url += "test_adapters/";
+ url = getUrl("test_adapters/");
try {
setPostHogCustomEvent(posthogTcEventText[type], {
@@ -263,9 +242,9 @@ function ConfigureDs({
}
let body = {};
- let url = `/api/v1/unstract/${sessionDetails?.orgId}/`;
+ let url;
- if (sourceType === Object.keys(sourceTypes)[0]) {
+ if (isConnector) {
const connectorMetadata = { ...formData };
const connectorName = connectorMetadata?.connectorName;
delete connectorMetadata.connectorName;
@@ -275,20 +254,18 @@ function ConfigureDs({
connector_metadata: connectorMetadata,
connector_name: connectorName,
created_by: sessionDetails?.id,
- workflow: id,
- connector_type: type.toUpperCase(),
};
- url += "connector/";
+ url = getUrl("connector/");
try {
- setPostHogCustomEvent(
- posthogConnectorAddedEventText[`${connType}:${type}`],
- {
+ const eventKey = `${type.toUpperCase()}`;
+ if (posthogConnectorAddedEventText[eventKey]) {
+ setPostHogCustomEvent(posthogConnectorAddedEventText[eventKey], {
info: `Clicked on 'Submit' button`,
connector_name: selectedSourceName,
- }
- );
+ });
+ }
} catch (err) {
// If an error occurs while setting custom posthog event, ignore it and continue
}
@@ -299,10 +276,10 @@ function ConfigureDs({
body = {
adapter_id: selectedSourceId,
adapter_metadata: adapterMetadata,
- adapter_type: type.toUpperCase(),
+ adapter_type: type?.toUpperCase(),
adapter_name: adapterName,
};
- url += "adapter/";
+ url = getUrl("adapter/");
try {
setPostHogCustomEvent(posthogSubmitEventText[type], {
@@ -317,7 +294,7 @@ function ConfigureDs({
let method = "POST";
if (editItemId?.length) {
method = "PUT";
- url += `${editItemId}/`;
+ url = `${url}${editItemId}/`;
}
if (oAuthProvider?.length > 0) {
@@ -339,17 +316,8 @@ function ConfigureDs({
axiosPrivate(requestOptions)
.then((res) => {
const data = res?.data;
- if (sourceTypes.connectors.includes(type)) {
- handleUpdate(
- {
- connector_instance_id: data?.id,
- configuration: formDataConfig,
- },
- true
- );
- setIsTcSuccessful(false);
- return;
- }
+
+ // For centralized connectors or adapters
if (data) {
addNewItem(data, !!editItemId);
}
@@ -359,7 +327,7 @@ function ConfigureDs({
method === "POST" ? "added" : "updated"
} connector`,
});
- if (sourceType === Object.keys(sourceTypes)[1] && method === "POST") {
+ if (!isConnector && method === "POST") {
updateSession(type);
}
@@ -394,8 +362,6 @@ function ConfigureDs({
setCacheKey={handleSetCacheKey}
setStatus={handleSetStatus}
selectedSourceId={selectedSourceId}
- workflowId={id}
- connType={connectorRole}
/>
)}
<RjsfFormLayout
@@ -447,13 +413,9 @@ ConfigureDs.propTypes = {
addNewItem: PropTypes.func,
type: PropTypes.string,
editItemId: PropTypes.string,
- sourceType: PropTypes.string.isRequired,
- handleUpdate: PropTypes.func,
- connDetails: PropTypes.object,
+ isConnector: PropTypes.bool.isRequired,
metadata: PropTypes.object,
selectedSourceName: PropTypes.string.isRequired,
- connType: PropTypes.string,
- formDataConfig: PropTypes.object,
};
export { ConfigureDs };
diff --git a/frontend/src/components/input-output/edit-ds-modal/EditDsModal.css b/frontend/src/components/input-output/edit-ds-modal/EditDsModal.css
deleted file mode 100644
index ba721b4b5..000000000
--- a/frontend/src/components/input-output/edit-ds-modal/EditDsModal.css
+++ /dev/null
@@ -1,11 +0,0 @@
-/* Styles for EditDsModal */
-
-.edit-ds-layout {
- max-height: 800px;
- overflow-y: scroll;
- margin-top: 10px;
-}
-
-.edit-ds-modal {
- width: 25% !important;
-}
diff --git a/frontend/src/components/input-output/edit-ds-modal/EditDsModal.jsx b/frontend/src/components/input-output/edit-ds-modal/EditDsModal.jsx
deleted file mode 100644
index 0d31c6ee6..000000000
--- a/frontend/src/components/input-output/edit-ds-modal/EditDsModal.jsx
+++ /dev/null
@@ -1,117 +0,0 @@
-import { Modal } from "antd";
-import PropTypes from "prop-types";
-import { useEffect, useState } from "react";
-import axios from "axios";
-
-import "./EditDsModal.css";
-import { ConfigureDs } from "../configure-ds/ConfigureDs.jsx";
-import { useSessionStore } from "../../../store/session-store";
-import { CONNECTOR_TYPE_MAP } from "../../../helpers/GetStaticData";
-import { useAlertStore } from "../../../store/alert-store";
-import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate.js";
-
-function EditDsModal({
- isOpen,
- setOpen,
- currentConnectorId,
- connectorType,
- setReloadList,
-}) {
- const [dsSelected, setDsSelected] = useState("");
- const [spec, setSpec] = useState({});
- const [isSpecLoading, setIsSpecLoading] = useState(false);
- const [oAuthProvider, setOAuthProvider] = useState("");
- const { sessionDetails } = useSessionStore();
- const { setAlertDetails } = useAlertStore();
- const axiosPrivate = useAxiosPrivate();
- const [connectorData, setConnectorData] = useState({});
-
- const orgId = sessionDetails?.orgId;
-
- useEffect(() => {
- const requestOptions = {
- method: "GET",
- url: `/api/v1/unstract/${orgId}/connector/${currentConnectorId}/`,
- };
-
- axiosPrivate(requestOptions)
- .then((res) => {
- setDsSelected(res?.data?.connector_id);
- setConnectorData(res?.data);
- })
- .catch((err) => {
- setAlertDetails({
- type: "error",
- content: "Failed to fetch the connector data.",
- });
- });
- }, [currentConnectorId]);
-
- useEffect(() => {
- if (dsSelected.length === 0) return;
- const requestOptions = {
- method: "GET",
- url: `/api/v1/unstract/${orgId}/connector_schema/?id=${dsSelected}`,
- };
-
- setIsSpecLoading(true);
- axios(requestOptions)
- .then((res) => {
- const data = res?.data;
- setSpec(data?.json_schema);
- if (data?.oauth) {
- setOAuthProvider(data?.python_social_auth_backend);
- }
- })
- .catch((err) => {
- setAlertDetails({
- type: "error",
- content: "Failed to fetch the data source details.",
- });
- })
- .finally(() => {
- setIsSpecLoading(false);
- });
- }, [dsSelected]);
-
- const handleCancel = () => {
- setReloadList(true);
- setOpen(false);
- };
-
- return (
- <Modal
- className="edit-ds-modal"
- title={`Edit Data ${CONNECTOR_TYPE_MAP[connectorType]}`}
- open={isOpen}
- centered
- maskClosable={false}
- footer={null}
- onCancel={handleCancel}
- >
- <div className="edit-ds-layout">
- {connectorData && (
- <ConfigureDs
- spec={spec}
- oAuthProvider={oAuthProvider}
- dsSelected={dsSelected}
- isLoading={isSpecLoading}
- handleCancel={handleCancel}
- connectorType={connectorType}
- connectorData={connectorData}
- />
- )}
- </div>
- </Modal>
- );
-}
-
-EditDsModal.propTypes = {
- isOpen: PropTypes.bool.isRequired,
- setOpen: PropTypes.func.isRequired,
- currentConnectorId: PropTypes.string,
- connectorType: PropTypes.string,
- setReloadList: PropTypes.func.isRequired,
-};
-
-export { EditDsModal };
diff --git a/frontend/src/components/input-output/file-system/FileSystem.css b/frontend/src/components/input-output/file-system/FileSystem.css
index 462593c4b..0e6cb52b3 100644
--- a/frontend/src/components/input-output/file-system/FileSystem.css
+++ b/frontend/src/components/input-output/file-system/FileSystem.css
@@ -1,12 +1,11 @@
.storageExplorer {
height: 100%;
- overflow: hidden;
+ overflow: auto;
}
.explorerTree {
height: calc(100% - 66px);
overflow: auto;
- padding-inline: 8px;
padding-block: 8px;
margin-block-start: 4px;
}
@@ -21,7 +20,6 @@
display: inline-grid;
grid-template-columns: 1fr auto auto;
column-gap: 20px;
- margin-inline-start: 2px;
}
.toolBar {
diff --git a/frontend/src/components/input-output/file-system/FileSystem.jsx b/frontend/src/components/input-output/file-system/FileSystem.jsx
index c70a5769a..3bf74f325 100644
--- a/frontend/src/components/input-output/file-system/FileSystem.jsx
+++ b/frontend/src/components/input-output/file-system/FileSystem.jsx
@@ -7,14 +7,24 @@ import { Document, Folder } from "../../../assets";
import { formatBytes } from "../../../helpers/GetStaticData";
import { inputService } from "../../input-output/input-output/input-service.js";
import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader.jsx";
+import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
import "./FileSystem.css";
const { DirectoryTree } = Tree;
const { Text } = Typography;
-function FileExplorer({ selectedItem = "", data = [], loadingData, error }) {
+function FileExplorer({
+ selectedConnector = "",
+ data = [],
+ loadingData,
+ error,
+ setError,
+ onFolderSelect,
+ selectedFolderPath,
+}) {
const inpService = inputService();
+ const handleException = useExceptionHandler();
const [tree, setTree] = useState([]);
const [expandedKeys, setExpandedKeys] = useState([]);
@@ -32,7 +42,17 @@ function FileExplorer({ selectedItem = "", data = [], loadingData, error }) {
// Clear selected items/keys when the data source is changed.
setExpandedKeys([]);
setSelectedKeys([]);
- }, [selectedItem]);
+ if (setError) {
+ setError("");
+ }
+ }, [selectedConnector]);
+
+ useEffect(() => {
+ // Clear tree selection when selectedFolderPath is cleared
+ if (!selectedFolderPath) {
+ setSelectedKeys([]);
+ }
+ }, [selectedFolderPath]);
function onLoadData({ key, children }) {
return new Promise((resolve) => {
@@ -49,8 +69,11 @@ function FileExplorer({ selectedItem = "", data = [], loadingData, error }) {
}
function getAndUpdateFiles(path) {
+ if (setError) {
+ setError("");
+ }
return inpService
- .getFileList(selectedItem, path)
+ .getFileList(selectedConnector, path)
.then((res) => {
let newTree = tree;
if (path) {
@@ -61,10 +84,15 @@ function FileExplorer({ selectedItem = "", data = [], loadingData, error }) {
}
updateTree(newTree);
})
- .catch(() => {
- console.error(
- `Unable to get files on "${selectedItem}" for the folder "${path}"`
+ .catch((err) => {
+ const errorDetails = handleException(
+ err,
+ `Error loading files from "${path || "root"}"`
);
+ if (setError) {
+ setError(errorDetails.content);
+ }
+ throw err;
});
}
@@ -72,8 +100,16 @@ function FileExplorer({ selectedItem = "", data = [], loadingData, error }) {
setSelectedKeys(selectedKeys);
if (event.node.isLeaf) {
uploadPathRef.current = "";
+ // Call the folder selection callback for files (disabled state)
+ if (onFolderSelect) {
+ onFolderSelect(event.node.key, "file");
+ }
} else {
uploadPathRef.current = event.node.key;
+ // Call the folder selection callback for folders (enabled state)
+ if (onFolderSelect) {
+ onFolderSelect(event.node.key, "folder");
+ }
}
}
@@ -126,7 +162,7 @@ function FileExplorer({ selectedItem = "", data = [], loadingData, error }) {
)}
{error && (
<div className="center">
- <Text>Error loading the data</Text>
+ <Text>{error}</Text>
</div>
)}
</div>
@@ -186,10 +222,13 @@ function transformTree(tree) {
}
FileExplorer.propTypes = {
- selectedItem: PropTypes.string,
+ selectedConnector: PropTypes.string,
data: PropTypes.array,
loadingData: PropTypes.bool,
- error: PropTypes.bool,
+ error: PropTypes.string,
+ setError: PropTypes.func,
+ onFolderSelect: PropTypes.func,
+ selectedFolderPath: PropTypes.string,
};
export { FileExplorer };
diff --git a/frontend/src/components/input-output/input-output/InputOutput.css b/frontend/src/components/input-output/input-output/InputOutput.css
deleted file mode 100644
index b66d489d3..000000000
--- a/frontend/src/components/input-output/input-output/InputOutput.css
+++ /dev/null
@@ -1,14 +0,0 @@
-.input-layout {
- height: 100%;
-}
-
-.input-sidebar {
- background-color: var(--page-bg-3);
- padding: 12px;
-}
-
-.input-main {
- padding: 12px;
- height: 100%;
- background-color: var(--white);
-}
diff --git a/frontend/src/components/input-output/input-output/InputOutput.jsx b/frontend/src/components/input-output/input-output/InputOutput.jsx
deleted file mode 100644
index 31a7c6198..000000000
--- a/frontend/src/components/input-output/input-output/InputOutput.jsx
+++ /dev/null
@@ -1,216 +0,0 @@
-import { Col, Image, Row } from "antd";
-import { useEffect, useState } from "react";
-import { useLocation, useParams } from "react-router-dom";
-
-import { CONNECTOR_TYPE_MAP } from "../../../helpers/GetStaticData.js";
-import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";
-import { useAlertStore } from "../../../store/alert-store";
-import { useSessionStore } from "../../../store/session-store";
-import useRequestUrl from "../../../hooks/useRequestUrl";
-import { AddSourceModal } from "../add-source-modal/AddSourceModal.jsx";
-import { ManageFiles } from "../manage-files/ManageFiles.jsx";
-import { Sidebar } from "../sidebar/Sidebar.jsx";
-import "./InputOutput.css";
-import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx";
-const INPUT = "input";
-const OUTPUT = "output";
-
-function InputOutput() {
- const [selectedItem, setSelectedItem] = useState();
- const [listOfItems, setListOfItems] = useState([]);
- const [openModal, setOpenModal] = useState(false);
- const [connectorType, setConnectorType] = useState("");
- const [reloadList, setReloadList] = useState(false);
- const [editItemId, setEditItemId] = useState(null);
- const { sessionDetails } = useSessionStore();
- const { setAlertDetails } = useAlertStore();
- const { id } = useParams();
- const axiosPrivate = useAxiosPrivate();
- const handleException = useExceptionHandler();
- const { getUrl } = useRequestUrl();
- const location = useLocation();
- const currentPath = location.pathname;
-
- const sourceIcon = (src) => {
- return <Image src={src} height={25} width={25} preview={false} />;
- };
-
- useEffect(() => {
- const currentPathSplit = currentPath.split("/");
- const currentPathLastIndex = currentPathSplit?.length - 1;
-
- const type = currentPathSplit[currentPathLastIndex];
-
- if (type !== INPUT && type !== OUTPUT) {
- return;
- }
-
- setConnectorType(type);
-
- const endpointType = CONNECTOR_TYPE_MAP[type]?.toUpperCase();
-
- const requestOptions = {
- method: "GET",
- url: getUrl(
- `workflow/endpoint/?workflow=${id}&endpoint_type=${endpointType}`
- ),
- };
-
- axiosPrivate(requestOptions)
- .then((res) => {
- const endpoints = res?.data;
- if (endpoints?.length === 0) {
- setSelectedItem("");
- setListOfItems([]);
- return;
- }
- const menuItems = endpoints.map((item) =>
- getItem(
- item?.connector_instance?.connector_name,
- item?.connector_instance?.id,
- sourceIcon(item?.connector_instance?.connector_icon)
- )
- );
-
- // Try to restore previously selected item from localStorage
- const persistenceKey = `file-system-selected-${id}-${type}`;
- const storedSelectedItem = localStorage.getItem(persistenceKey);
- const availableIds = endpoints.map((item) =>
- item?.connector_instance?.id?.toString()
- );
-
- let itemToSelect;
- if (storedSelectedItem && availableIds.includes(storedSelectedItem)) {
- // Restore previously selected item if it still exists
- itemToSelect = storedSelectedItem;
- } else {
- // Fall back to first item
- itemToSelect = endpoints[0]?.connector_instance?.id?.toString();
- }
-
- setSelectedItem(itemToSelect);
- setListOfItems(menuItems);
-
- // Persist the selected item
- if (itemToSelect) {
- localStorage.setItem(persistenceKey, itemToSelect);
- }
- })
- .catch((err) => {
- setAlertDetails(handleException(err));
- })
- .finally(() => {
- setReloadList(false);
- });
- }, [currentPath, reloadList]);
-
- const getItem = (label, key, icon, children, type) => {
- return {
- key,
- icon,
- children,
- label,
- type,
- };
- };
-
- const handleOpenModal = () => {
- setOpenModal(true);
- };
-
- const handleDelete = () => {
- const requestOptions = {
- method: "DELETE",
- url: getUrl(`connector/${selectedItem}/`),
- headers: {
- "X-CSRFToken": sessionDetails?.csrfToken,
- },
- };
-
- axiosPrivate(requestOptions)
- .then(() => {
- // Clear localStorage if the deleted item was the selected one
- if (selectedItem && connectorType) {
- const persistenceKey = `file-system-selected-${id}-${connectorType}`;
- const storedItem = localStorage.getItem(persistenceKey);
- if (storedItem === selectedItem) {
- localStorage.removeItem(persistenceKey);
- }
- }
- setReloadList(true);
- })
- .catch((err) => {
- setAlertDetails(handleException(err));
- });
- };
-
- const handleSetSelectedItem = (itemId) => {
- setSelectedItem(itemId);
- // Persist selected item to localStorage
- if (itemId && connectorType) {
- const persistenceKey = `file-system-selected-${id}-${connectorType}`;
- localStorage.setItem(persistenceKey, itemId);
- }
- };
-
- const addNewItem = (newItem, isEdit) => {
- let items = [...listOfItems];
- if (isEdit) {
- items = [...listOfItems].map((item) => {
- if (item?.key !== newItem?.id) {
- return item;
- }
- item["label"] = newItem?.connector_name;
- return item;
- });
- } else {
- const itemToAdd = getItem(
- newItem?.connector_name,
- newItem?.id,
- sourceIcon(newItem?.icon)
- );
- items.push(itemToAdd);
- }
- setListOfItems(items);
-
- handleSetSelectedItem(newItem?.id);
- };
-
- return (
- <>
- {connectorType?.length > 0 && (
- <>
- <Row className="input-layout">
- <Col className="input-sidebar" span={4}>
- <Sidebar
- selectedItem={selectedItem}
- setSelectedItem={handleSetSelectedItem}
- listOfItems={listOfItems}
- handleOpenModal={handleOpenModal}
- btnText={`Data ${CONNECTOR_TYPE_MAP[connectorType]}`}
- />
- </Col>
- <Col className="input-main" span={20}>
- <ManageFiles
- selectedItem={selectedItem}
- listOfitems={listOfItems}
- setEditItemId={setEditItemId}
- handleDelete={handleDelete}
- />
- </Col>
- </Row>
- <AddSourceModal
- open={openModal}
- setOpen={setOpenModal}
- type={connectorType}
- addNewItem={addNewItem}
- editItemId={editItemId}
- setEditItemId={setEditItemId}
- />
- </>
- )}
- </>
- );
-}
-
-export { InputOutput };
diff --git a/frontend/src/components/input-output/manage-files/ManageFiles.jsx b/frontend/src/components/input-output/manage-files/ManageFiles.jsx
index 75b2aa041..a926f9e93 100644
--- a/frontend/src/components/input-output/manage-files/ManageFiles.jsx
+++ b/frontend/src/components/input-output/manage-files/ManageFiles.jsx
@@ -3,44 +3,64 @@ import PropTypes from "prop-types";
import { inputService } from "../../input-output/input-output/input-service.js";
import { FileExplorer } from "../file-system/FileSystem.jsx";
+import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
-function ManageFiles({ selectedItem }) {
+function ManageFiles({
+ selectedConnector,
+ onFolderSelect,
+ selectedFolderPath,
+}) {
const inpService = inputService();
+ const handleException = useExceptionHandler();
const [files, setFiles] = useState([]);
const [loadingData, setLoadingData] = useState(false);
- const [error, setError] = useState(false);
+ const [error, setError] = useState("");
useEffect(() => {
setFiles([]);
- if (!selectedItem) return;
+ setError("");
+ if (!selectedConnector) return;
setLoadingData(true);
+ let cancelled = false;
inpService
- .getFileList(selectedItem)
+ .getFileList(selectedConnector)
.then((res) => {
+ if (cancelled) return;
setFiles(res.data);
- setError(false);
+ setError("");
})
- .catch(() => {
- setError(true);
+ .catch((err) => {
+ if (cancelled) return;
+ const errorDetails = handleException(err, "Error loading files");
+ setError(errorDetails.content);
})
.finally(() => {
+ if (cancelled) return;
setLoadingData(false);
});
- }, [selectedItem]);
+ return () => {
+ cancelled = true;
+ };
+ }, [selectedConnector]);
return (
<FileExplorer
- selectedItem={selectedItem}
+ selectedConnector={selectedConnector}
data={files}
loadingData={loadingData}
error={error}
+ setError={setError}
+ onFolderSelect={onFolderSelect}
+ selectedFolderPath={selectedFolderPath}
/>
);
}
ManageFiles.propTypes = {
- selectedItem: PropTypes.string,
+ selectedConnector: PropTypes.string,
+ onFolderSelect: PropTypes.func,
+ selectedFolderPath: PropTypes.string,
};
export { ManageFiles };
diff --git a/frontend/src/components/input-output/sidebar/Sidebar.css b/frontend/src/components/input-output/sidebar/Sidebar.css
deleted file mode 100644
index 9663abe5c..000000000
--- a/frontend/src/components/input-output/sidebar/Sidebar.css
+++ /dev/null
@@ -1,16 +0,0 @@
-.sidebar-layout {
- margin: 5px;
-}
-
-.sidebar-menu {
- margin-top: 10px;
- background-color: transparent;
-}
-
-.sidebar-menu .ant-menu-item {
- padding: 5px !important;
-}
-
-.sidebar-menu .ant-menu-title-content {
- margin-left: 5px;
-}
diff --git a/frontend/src/components/input-output/sidebar/Sidebar.jsx b/frontend/src/components/input-output/sidebar/Sidebar.jsx
deleted file mode 100644
index d59968cdb..000000000
--- a/frontend/src/components/input-output/sidebar/Sidebar.jsx
+++ /dev/null
@@ -1,46 +0,0 @@
-import { Button, Menu } from "antd";
-import { PlusOutlined } from "@ant-design/icons";
-import PropTypes from "prop-types";
-
-import "./Sidebar.css";
-
-function Sidebar({
- selectedItem,
- setSelectedItem,
- listOfItems,
- handleOpenModal,
- btnText,
-}) {
- const handleSelectItem = (e) => {
- const id = e.key;
- setSelectedItem(id?.toString());
- };
-
- return (
- <div className="sidebar-layout">
- <Button block icon={<PlusOutlined />} onClick={handleOpenModal}>
- {btnText}
- </Button>
- {selectedItem && (
- <Menu
- className="sidebar-menu"
- style={{ border: 0 }}
- selectedKeys={[selectedItem]}
- mode="inline"
- items={listOfItems}
- onClick={handleSelectItem}
- />
- )}
- </div>
- );
-}
-
-Sidebar.propTypes = {
- selectedItem: PropTypes.string,
- setSelectedItem: PropTypes.func,
- listOfItems: PropTypes.array,
- handleOpenModal: PropTypes.func.isRequired,
- btnText: PropTypes.string.isRequired,
-};
-
-export { Sidebar };
diff --git a/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx b/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx
index 1b45d6625..bc300d369 100644
--- a/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx
+++ b/frontend/src/components/navigations/side-nav-bar/SideNavBar.jsx
@@ -16,6 +16,7 @@ import task from "../../../assets/task.svg";
import VectorDbIcon from "../../../assets/vector-db.svg";
import TextExtractorIcon from "../../../assets/text-extractor.svg";
import TerminalIcon from "../../../assets/terminal.svg";
+import ConnectorsIcon from "../../../assets/connectors.svg";
import "./SideNavBar.css";
@@ -193,6 +194,16 @@ const SideNavBar = ({ collapsed }) => {
},
{
id: 3.5,
+ title: "Connectors",
+ description: "Manage connectors for data sources and destinations",
+ image: ConnectorsIcon,
+ path: `/${orgName}/settings/connectors`,
+ active: window.location.pathname.startsWith(
+ `/${orgName}/settings/connectors`
+ ),
+ },
+ {
+ id: 3.6,
title: "Platform",
description: "Settings for the platform",
image: PlatformSettingsIcon,
diff --git a/frontend/src/components/oauth-ds/oauth-ds/OAuthDs.jsx b/frontend/src/components/oauth-ds/oauth-ds/OAuthDs.jsx
index 7c8556468..17021502f 100644
--- a/frontend/src/components/oauth-ds/oauth-ds/OAuthDs.jsx
+++ b/frontend/src/components/oauth-ds/oauth-ds/OAuthDs.jsx
@@ -7,29 +7,20 @@ import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate.js";
import { useAlertStore } from "../../../store/alert-store";
import GoogleOAuthButton from "../google/GoogleOAuthButton.jsx";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx";
-function OAuthDs({
- oAuthProvider,
- setCacheKey,
- setStatus,
- selectedSourceId,
- workflowId,
- connType,
-}) {
+function OAuthDs({ oAuthProvider, setCacheKey, setStatus, selectedSourceId }) {
const axiosPrivate = useAxiosPrivate();
const { setAlertDetails } = useAlertStore();
const handleException = useExceptionHandler();
- const oauthCacheKey = `oauth-cachekey-${workflowId}-${connType}-${selectedSourceId}`;
+ const oauthCacheKey = `oauth-cachekey-${selectedSourceId}`;
const [oauthStatus, setOAuthStatus] = useState(() => {
// Initialize from connector-specific status only to prevent contamination
- const currentConnectorId = `${workflowId}-${connType}-${selectedSourceId}`;
- return localStorage.getItem(`oauth-status-${currentConnectorId}`);
+ return localStorage.getItem(`oauth-status-${selectedSourceId}`);
});
useEffect(() => {
- const currentConnectorId = `${workflowId}-${connType}-${selectedSourceId}`;
- const connectorStatusKey = `oauth-status-${currentConnectorId}`;
+ const connectorStatusKey = `oauth-status-${selectedSourceId}`;
const handleStorageChange = () => {
// Listen for changes to our specific connector's status only
@@ -59,13 +50,12 @@ function OAuthDs({
window.removeEventListener("storage", handleStorageChange);
// Don't clear localStorage on unmount to persist across tab switches
};
- }, [selectedSourceId, workflowId, connType]);
+ }, [selectedSourceId]);
const handleOAuth = async () => {
try {
// Store connector ID in sessionStorage for OAuth callback (survives window.open)
- const currentConnectorId = `${workflowId}-${connType}-${selectedSourceId}`;
- sessionStorage.setItem("oauth-current-connector", currentConnectorId);
+ sessionStorage.setItem("oauth-current-connector", selectedSourceId);
const requestOptions = {
method: "GET",
@@ -111,8 +101,6 @@ OAuthDs.propTypes = {
setCacheKey: PropTypes.func,
setStatus: PropTypes.func,
selectedSourceId: PropTypes.string.isRequired,
- workflowId: PropTypes.string.isRequired,
- connType: PropTypes.string.isRequired,
};
export { OAuthDs };
diff --git a/frontend/src/components/pipelines-or-deployments/etl-task-deploy/EtlTaskDeploy.jsx b/frontend/src/components/pipelines-or-deployments/etl-task-deploy/EtlTaskDeploy.jsx
index 22978d786..bc39e0534 100644
--- a/frontend/src/components/pipelines-or-deployments/etl-task-deploy/EtlTaskDeploy.jsx
+++ b/frontend/src/components/pipelines-or-deployments/etl-task-deploy/EtlTaskDeploy.jsx
@@ -137,13 +137,17 @@ const EtlTaskDeploy = ({
};
useEffect(() => {
- if (type === "app") {
- getWorkflowList();
- } else {
- getWorkflows();
+ // Only fetch workflow list when workflowId is not provided
+ // If workflowId exists, the workflow dropdown is hidden and list fetching is unnecessary
+ if (!workflowId) {
+ if (type === "app") {
+ getWorkflowList();
+ } else {
+ getWorkflows();
+ }
}
fetchCount();
- }, [type, fetchCount]);
+ }, [type, fetchCount, workflowId]);
const clearFormDetails = () => {
setFormDetails({ ...defaultFromDetails });
diff --git a/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx b/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx
index 8b22d6a8b..765fcc131 100644
--- a/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx
+++ b/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx
@@ -620,7 +620,7 @@ function Pipelines({ type }) {
setOpen={setOpenEtlOrTaskModal}
setTableData={setTableData}
type={type}
- title={deploymentsStaticContent[type].modalTitle}
+ title={deploymentsStaticContent[type].addBtn}
isEdit={isEdit}
selectedRow={selectedPorD}
setSelectedRow={setSelectedPorD}
diff --git a/frontend/src/components/widgets/list-view/ListView.css b/frontend/src/components/widgets/list-view/ListView.css
index 5d586fec9..0deda612b 100644
--- a/frontend/src/components/widgets/list-view/ListView.css
+++ b/frontend/src/components/widgets/list-view/ListView.css
@@ -6,7 +6,7 @@
width: 70%;
min-width: 800px;
max-width: 1400px;
- padding: 0px 10px;
+ padding: 0px 10px 40px 10px;
}
.list-view-item {
@@ -22,12 +22,19 @@
overflow: hidden;
}
+.action-button-container {
+ display: flex;
+ align-items: center;
+ gap: 24px;
+ padding-right: 12px;
+}
+
.action-icon-buttons {
font-size: 18px;
- margin-left: 20px;
z-index: 20;
color: #092c4c;
- flex: content;
+ cursor: pointer;
+ transition: color 0.2s ease;
}
.cur-pointer .ant-list-item-meta {
@@ -92,9 +99,6 @@
padding-bottom: 10px;
}
-.edit-icon {
- margin-right: 10px;
-}
.edit-icon:hover {
color: #1890ff;
@@ -103,3 +107,7 @@
.delete-icon:hover {
color: #ff4d4f;
}
+
+.share-icon:hover {
+ color: #1890ff;
+}
diff --git a/frontend/src/components/widgets/list-view/ListView.jsx b/frontend/src/components/widgets/list-view/ListView.jsx
index e34011539..3a6851779 100644
--- a/frontend/src/components/widgets/list-view/ListView.jsx
+++ b/frontend/src/components/widgets/list-view/ListView.jsx
@@ -115,7 +115,7 @@ function ListView({
{handleShare && (
<ShareAltOutlined
key={`${item.id}-share`}
- className="action-icon-buttons"
+ className="action-icon-buttons share-icon"
onClick={(event) => handleShareClick(event, item, true)}
/>
)}
diff --git a/frontend/src/components/widgets/share-permission/SharePermission.jsx b/frontend/src/components/widgets/share-permission/SharePermission.jsx
index 31c052391..65637acb8 100644
--- a/frontend/src/components/widgets/share-permission/SharePermission.jsx
+++ b/frontend/src/components/widgets/share-permission/SharePermission.jsx
@@ -87,9 +87,13 @@ function SharePermission({
sharedWithContent = (
<List
dataSource={selectedUsers.map((userId) => {
- const user = allUsers.find(
- (u) => u?.id.toString() === userId.toString()
- );
+ const user = allUsers.find((u) => {
+ if (u?.id !== undefined) {
+ return u?.id.toString() === userId.toString();
+ } else {
+ return u?.toString() === userId.toString();
+ }
+ });
return {
id: user?.id,
email: user?.email,
@@ -102,8 +106,8 @@ function SharePermission({
<div onClick={(event) => event.stopPropagation()} role="none">
<Popconfirm
key={`${item.id}-delete`}
- title="Delete the User"
- description={`Are you sure to remove ${item?.email}?`}
+ title="Revoke Access"
+ description={`Are you sure you want to revoke access to '${item?.email}'?`}
okText="Yes"
cancelText="No"
icon={<QuestionCircleOutlined />}
diff --git a/frontend/src/hooks/usePostHogEvents.js b/frontend/src/hooks/usePostHogEvents.js
index e890ef29b..d53bfed8b 100644
--- a/frontend/src/hooks/usePostHogEvents.js
+++ b/frontend/src/hooks/usePostHogEvents.js
@@ -37,9 +37,8 @@ const usePostHogEvents = () => {
};
const posthogConnectorAddedEventText = {
- "FILESYSTEM:input": "intent_success_wf_fs_source",
- "FILESYSTEM:output": "intent_success_wf_fs_dest",
- "DATABASE:output": "intent_success_wf_db_dest",
+ FILESYSTEM: "intent_success_fs_connector",
+ DATABASE: "intent_success_db_connector",
};
const posthogWfDeploymentEventText = {
diff --git a/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.jsx b/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.jsx
index e56e9ea07..3799a7790 100644
--- a/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.jsx
+++ b/frontend/src/layouts/rjsf-form-layout/RjsfFormLayout.jsx
@@ -2,6 +2,7 @@ import { useCallback, useMemo } from "react";
import Form from "@rjsf/antd";
import validator from "@rjsf/validator-ajv8";
import PropTypes from "prop-types";
+import { Alert, Space } from "antd";
import { AltDateTimeWidget } from "../../components/rjsf-custom-widgets/alt-date-time-widget/AltDateTimeWidget.jsx";
import { AltDateWidget } from "../../components/rjsf-custom-widgets/alt-date-widget/AltDateWidget.jsx";
@@ -22,7 +23,6 @@ import { TextWidget } from "../../components/rjsf-custom-widgets/text-widget/Tex
import { PasswordWidget } from "../../components/rjsf-custom-widgets/password-widget/PasswordWidget.jsx";
import { UpDownWidget } from "../../components/rjsf-custom-widgets/up-down-widget/UpDownWidget.jsx";
import { CustomFieldTemplate } from "./CustomFieldTemplate.jsx";
-import { Alert, Space } from "antd";
import CustomMarkdown from "../../components/helpers/custom-markdown/CustomMarkdown.jsx";
import "./RjsfFormLayout.css";
@@ -142,14 +142,14 @@ function RjsfFormLayout({
formData={formData}
transformErrors={transformErrors}
onError={() => {}}
- onSubmit={(e) => validateAndSubmit(e.formData)}
+ onSubmit={(e) => validateAndSubmit?.(e.formData)}
showErrorList={false}
onChange={handleChange}
templates={{
FieldTemplate: CustomFieldTemplate,
}}
>
- {children}
+ {children || <></>}
</Form>
</Space>
)}
diff --git a/frontend/src/pages/ConnectorsPage.css b/frontend/src/pages/ConnectorsPage.css
new file mode 100644
index 000000000..6963363ab
--- /dev/null
+++ b/frontend/src/pages/ConnectorsPage.css
@@ -0,0 +1,22 @@
+.connectors-layout {
+ height: 100%;
+ display: flex;
+ flex-direction: column;
+}
+
+.connectors-pg-layout {
+ background-color: var(--page-bg-2);
+ padding: 12px;
+ height: 100%;
+ overflow-y: auto;
+ flex: 1;
+}
+
+.connectors-pg-body {
+ background-color: var(--white);
+ height: 100%;
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ padding: 20px;
+}
diff --git a/frontend/src/pages/ConnectorsPage.jsx b/frontend/src/pages/ConnectorsPage.jsx
new file mode 100644
index 000000000..44f8c1776
--- /dev/null
+++ b/frontend/src/pages/ConnectorsPage.jsx
@@ -0,0 +1,200 @@
+import { useState, useEffect, useCallback } from "react";
+import { Button } from "antd";
+import { PlusOutlined } from "@ant-design/icons";
+
+import { useAxiosPrivate } from "../hooks/useAxiosPrivate";
+import { useSessionStore } from "../store/session-store";
+import { useAlertStore } from "../store/alert-store";
+import { useExceptionHandler } from "../hooks/useExceptionHandler";
+import useRequestUrl from "../hooks/useRequestUrl";
+import "./ConnectorsPage.css";
+import { ToolNavBar } from "../components/navigations/tool-nav-bar/ToolNavBar";
+import { ViewTools } from "../components/custom-tools/view-tools/ViewTools";
+import { SharePermission } from "../components/widgets/share-permission/SharePermission";
+import { AddSourceModal } from "../components/input-output/add-source-modal/AddSourceModal";
+
+function ConnectorsPage() {
+ const [connectors, setConnectors] = useState([]);
+ const [loading, setLoading] = useState(false);
+ const [modalVisible, setModalVisible] = useState(false);
+ const [editingConnector, setEditingConnector] = useState(null);
+ const [shareModalVisible, setShareModalVisible] = useState(false);
+ const [sharingConnector, setSharingConnector] = useState(null);
+ const [userList, setUserList] = useState([]);
+ const [isPermissionEdit, setIsPermissionEdit] = useState(false);
+ const [isShareLoading, setIsShareLoading] = useState(false);
+
+ const axiosPrivate = useAxiosPrivate();
+ const { sessionDetails } = useSessionStore();
+ const { setAlertDetails } = useAlertStore();
+ const handleException = useExceptionHandler();
+ const { getUrl } = useRequestUrl();
+
+ useEffect(() => {
+ fetchConnectors();
+ fetchUsers();
+ }, []);
+
+ const fetchConnectors = async () => {
+ setLoading(true);
+ try {
+ const response = await axiosPrivate.get(getUrl("connector/"));
+ setConnectors(response.data || []);
+ } catch (error) {
+ setAlertDetails(handleException(error, "Failed to load connectors"));
+ } finally {
+ setLoading(false);
+ }
+ };
+
+ const fetchUsers = async () => {
+ try {
+ const response = await axiosPrivate.get(getUrl("users/"));
+ const users = response?.data?.members || [];
+ setUserList(
+ users.map((user) => ({
+ id: user?.id,
+ email: user?.email,
+ }))
+ );
+ } catch (error) {
+ setAlertDetails(handleException(error, "Failed to load users"));
+ }
+ };
+
+ const handleCreateConnector = () => {
+ setEditingConnector(null);
+ setModalVisible(true);
+ };
+
+ const handleEditConnector = (_event, connector) => {
+ setEditingConnector(connector);
+ setModalVisible(true);
+ };
+
+ const handleDeleteConnector = async (_event, connector) => {
+ try {
+ await axiosPrivate.delete(getUrl(`connector/${connector.id}/`), {
+ headers: {
+ "X-CSRFToken": sessionDetails?.csrfToken,
+ },
+ });
+ setAlertDetails({
+ type: "success",
+ content: "Connector deleted successfully",
+ });
+ fetchConnectors();
+ } catch (error) {
+ setAlertDetails(handleException(error, "Failed to delete connector"));
+ }
+ };
+
+ const handleShareConnector = (_event, connector, isEdit) => {
+ setSharingConnector(connector);
+ setIsPermissionEdit(isEdit);
+ setShareModalVisible(true);
+ };
+
+ const handleShareSave = async (userIds, connector, shareWithEveryone) => {
+ setIsShareLoading(true);
+ try {
+ const updateData = {
+ shared_users: userIds,
+ shared_to_org: shareWithEveryone || false,
+ };
+
+ await axiosPrivate.patch(
+ getUrl(`connector/${connector.id}/`),
+ updateData,
+ {
+ headers: {
+ "X-CSRFToken": sessionDetails?.csrfToken,
+ },
+ }
+ );
+ setShareModalVisible(false);
+ setAlertDetails({
+ type: "success",
+ content: "Connector sharing updated successfully",
+ });
+ } catch (error) {
+ setAlertDetails(handleException(error, "Failed to update sharing"));
+ } finally {
+ setIsShareLoading(false);
+ }
+ };
+
+ const handleConnectorSaved = () => {
+ setModalVisible(false);
+ setEditingConnector(null);
+ fetchConnectors();
+ setAlertDetails({
+ type: "success",
+ content: editingConnector
+ ? "Connector updated successfully"
+ : "Connector created successfully",
+ });
+ };
+
+ const renderCreateConnectorButtons = useCallback(
+ () => (
+ <Button
+ type="primary"
+ icon={<PlusOutlined />}
+ onClick={handleCreateConnector}
+ >
+ New Connector
+ </Button>
+ ),
+ []
+ );
+
+ return (
+ <div className="connectors-layout">
+ <ToolNavBar
+ title="Connectors"
+ CustomButtons={renderCreateConnectorButtons}
+ />
+ <div className="connectors-pg-layout">
+ <div className="connectors-pg-body">
+ <ViewTools
+ listOfTools={connectors}
+ isLoading={loading}
+ handleDelete={handleDeleteConnector}
+ handleEdit={handleEditConnector}
+ handleShare={handleShareConnector}
+ idProp="id"
+ titleProp="connector_name"
+ descriptionProp="connector_type"
+ iconProp="icon"
+ showOwner={true}
+ type="Connector"
+ isEmpty={!connectors.length}
+ centered
+ isClickable={false}
+ />
+ </div>
+ </div>
+ <AddSourceModal
+ open={modalVisible}
+ setOpen={setModalVisible}
+ isConnector={true}
+ addNewItem={handleConnectorSaved}
+ editItemId={editingConnector?.id}
+ setEditItemId={setEditingConnector}
+ />
+ <SharePermission
+ open={shareModalVisible}
+ setOpen={setShareModalVisible}
+ adapter={sharingConnector}
+ allUsers={userList}
+ onApply={handleShareSave}
+ permissionEdit={isPermissionEdit}
+ loading={isShareLoading}
+ isSharableToOrg={true}
+ />
+ </div>
+ );
+}
+
+export default ConnectorsPage;
diff --git a/frontend/src/pages/InputOutputPage.jsx b/frontend/src/pages/InputOutputPage.jsx
deleted file mode 100644
index f1336278d..000000000
--- a/frontend/src/pages/InputOutputPage.jsx
+++ /dev/null
@@ -1,7 +0,0 @@
-import { InputOutput } from "../components/input-output/input-output/InputOutput.jsx";
-
-function InputOutputPage() {
- return <InputOutput />;
-}
-
-export { InputOutputPage };
diff --git a/frontend/src/routes/useMainAppRoutes.js b/frontend/src/routes/useMainAppRoutes.js
index 8f68269af..b2c6a6919 100644
--- a/frontend/src/routes/useMainAppRoutes.js
+++ b/frontend/src/routes/useMainAppRoutes.js
@@ -21,6 +21,7 @@ import { ToolIdePage } from "../pages/ToolIdePage.jsx";
import { OutputAnalyzerPage } from "../pages/OutputAnalyzerPage.jsx";
import { LogsPage } from "../pages/LogsPage.jsx";
import { deploymentTypes } from "../helpers/GetStaticData.js";
+import ConnectorsPage from "../pages/ConnectorsPage.jsx";
let RequirePlatformAdmin;
let PlatformAdminPage;
@@ -163,6 +164,7 @@ function useMainAppRoutes() {
element={<ToolsSettingsPage type="x2text" />}
/>
<Route path="settings/ocr" element={<ToolsSettingsPage type="ocr" />} />
+ <Route path="settings/connectors" element={<ConnectorsPage />} />
<Route path="settings" element={<SettingsPage />} />
{ManualReviewSettings && (
<Route path="settings/review" element={<ManualReviewSettings />} />
diff --git a/unstract/connectors/src/unstract/connectors/enums.py b/unstract/connectors/src/unstract/connectors/enums.py
index e68b4b063..c0417810e 100644
--- a/unstract/connectors/src/unstract/connectors/enums.py
+++ b/unstract/connectors/src/unstract/connectors/enums.py
@@ -3,7 +3,7 @@ from enum import Enum
class ConnectorMode(Enum):
UNKNOWN = "UNKNOWN"
- FILE_SYSTEM = "FILE_SYSTEM"
+ FILE_SYSTEM = "FILESYSTEM"
DATABASE = "DATABASE"
API = "API"
MANUAL_REVIEW = "MANUAL_REVIEW"
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/box/__init__.py b/unstract/connectors/src/unstract/connectors/filesystems/box/__init__.py
index e9486087b..f29ac47cf 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/box/__init__.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/box/__init__.py
@@ -7,5 +7,5 @@ metadata = {
"version": "1.0.0",
"connector": BoxFS,
"description": "BoxFS connector",
- "is_active": True,
+ "is_active": False,
}
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/http/__init__.py b/unstract/connectors/src/unstract/connectors/filesystems/http/__init__.py
index 38370f22e..3d7105991 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/http/__init__.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/http/__init__.py
@@ -7,5 +7,5 @@ metadata = {
"version": "1.0.0",
"connector": HttpFS,
"description": "HttpFS connector",
- "is_active": True,
+ "is_active": False,
}
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/ucs/__init__.py b/unstract/connectors/src/unstract/connectors/filesystems/ucs/__init__.py
index 8bf61eec7..d22f49ed0 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/ucs/__init__.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/ucs/__init__.py
@@ -7,5 +7,5 @@ metadata = {
"version": "1.0.0",
"connector": UnstractCloudStorage,
"description": "UnstractCloudStorage connector",
- "is_active": True,
+ "is_active": False,
}