Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Revert "UN-2585 [MISC] Deprecate connector instance fields and refactor endpoint handling" (#1449)

Chandrasekharan M5357aebff562
diff --git a/.gitignore b/.gitignore
index 53b1a840e..b197d1623 100644
--- a/.gitignore
+++ b/.gitignore
@@ -680,15 +680,7 @@ backend/backend/*_urls.py
backend/backend/*_urls_v2.py
!backend/backend/public_urls_v2.py
-## Vibe coding / Prompting-related files ##
-prompting/
-.prompting/
-
-# Claude
+# Adding claude prompting directories
CLAUDE.md
-.claude/*
-
-# Windsurf
-.qodo
-.windsurfrules
-.windsurf/rules
+CONTRIBUTION_GUIDE.md
+prompting/*
diff --git a/backend/connector_v2/migrations/0002_alter_connectorinstance_connector_metadata_and_more.py b/backend/connector_v2/migrations/0002_alter_connectorinstance_connector_metadata_and_more.py
deleted file mode 100644
index 267bcb87d..000000000
--- a/backend/connector_v2/migrations/0002_alter_connectorinstance_connector_metadata_and_more.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# Generated by Django 4.2.1 on 2025-07-16 10:26
-
-import django.db.models.deletion
-import utils.fields
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
- dependencies = [
- ("workflow_v2", "0015_executionlog_idx_wf_execution_event_time_and_more"),
- ("connector_v2", "0001_initial"),
- ]
-
- operations = [
- migrations.AlterField(
- model_name="connectorinstance",
- name="connector_metadata",
- field=utils.fields.EncryptedBinaryField(null=True),
- ),
- migrations.AlterField(
- model_name="connectorinstance",
- name="connector_type",
- field=models.CharField(
- choices=[("INPUT", "Input"), ("OUTPUT", "Output")], null=True
- ),
- ),
- migrations.AlterField(
- model_name="connectorinstance",
- name="workflow",
- field=models.ForeignKey(
- null=True,
- on_delete=django.db.models.deletion.CASCADE,
- related_name="connector_workflow",
- to="workflow_v2.workflow",
- ),
- ),
- ]
diff --git a/backend/connector_v2/models.py b/backend/connector_v2/models.py
index d7276aa19..dbeaeb019 100644
--- a/backend/connector_v2/models.py
+++ b/backend/connector_v2/models.py
@@ -1,4 +1,4 @@
-import logging
+import json
import uuid
from typing import Any
@@ -6,8 +6,10 @@ from account_v2.models import User
from connector_auth_v2.models import ConnectorAuth
from connector_processor.connector_processor import ConnectorProcessor
from connector_processor.constants import ConnectorKeys
+from cryptography.fernet import Fernet, InvalidToken
+from django.conf import settings
from django.db import models
-from utils.fields import EncryptedBinaryField
+from utils.exceptions import InvalidEncryptionKey
from utils.models.base_model import BaseModel
from utils.models.organization_mixin import (
DefaultOrganizationManagerMixin,
@@ -19,8 +21,6 @@ from backend.constants import FieldLengthConstants as FLC
CONNECTOR_NAME_SIZE = 128
VERSION_NAME_SIZE = 64
-logger = logging.getLogger(__name__)
-
class ConnectorInstanceModelManager(DefaultOrganizationManagerMixin, models.Manager):
pass
@@ -45,13 +45,13 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):
"workflow_v2.Workflow",
on_delete=models.CASCADE,
related_name="connector_workflow",
- null=True,
+ null=False,
blank=False,
)
connector_id = models.CharField(max_length=FLC.CONNECTOR_ID_LENGTH, default="")
- connector_metadata = EncryptedBinaryField(null=True)
+ connector_metadata = models.BinaryField(null=True)
connector_version = models.CharField(max_length=VERSION_NAME_SIZE, default="")
- connector_type = models.CharField(choices=ConnectorType.choices, null=True)
+ connector_type = models.CharField(choices=ConnectorType.choices)
# TODO: handle connector_auth cascade deletion
connector_auth = models.ForeignKey(
ConnectorAuth,
@@ -84,6 +84,21 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):
# Manager
objects = ConnectorInstanceModelManager()
+ # TODO: Remove if unused
+ def get_connector_metadata(self) -> dict[str, str]:
+ """Gets connector metadata and refreshes the tokens if needed in case
+ of OAuth.
+ """
+ tokens_refreshed = False
+ if self.connector_auth:
+ (
+ self.connector_metadata,
+ tokens_refreshed,
+ ) = self.connector_auth.get_and_refresh_tokens()
+ if tokens_refreshed:
+ self.save()
+ return self.connector_metadata
+
@staticmethod
def supportsOAuth(connector_id: str) -> bool:
return bool(
@@ -94,9 +109,19 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):
def __str__(self) -> str:
return (
- f"Connector({self.id}, ID={self.connector_id}, mode: {self.connector_mode})"
+ f"Connector({self.id}, type{self.connector_type}, workflow: {self.workflow})"
)
+ def get_connector_metadata_bytes(self):
+ """Convert connector_metadata to bytes if it is a memoryview.
+
+ Returns:
+ bytes: The connector_metadata as bytes.
+ """
+ if isinstance(self.connector_metadata, memoryview):
+ return self.connector_metadata.tobytes()
+ return self.connector_metadata
+
@property
def metadata(self) -> Any:
"""Decrypt and return the connector metadata as a dictionary.
@@ -107,20 +132,21 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):
Returns:
dict: The decrypted connector metadata.
-
- .. deprecated::
- This property is deprecated. Use `connector_metadata` field directly instead.
- This property will be removed in a future version.
"""
- import warnings
-
- warnings.warn(
- "The 'metadata' property is deprecated. Use 'connector_metadata' field directly instead. "
- "This property will be removed in a future version.",
- DeprecationWarning,
- stacklevel=2,
- )
- return self.connector_metadata
+ try:
+ connector_metadata_bytes = self.get_connector_metadata_bytes()
+
+ if connector_metadata_bytes is None:
+ return None
+
+ if isinstance(connector_metadata_bytes, (dict)):
+ return connector_metadata_bytes
+ encryption_secret: str = settings.ENCRYPTION_KEY
+ cipher_suite: Fernet = Fernet(encryption_secret.encode("utf-8"))
+ decrypted_value = cipher_suite.decrypt(connector_metadata_bytes)
+ except InvalidToken:
+ raise InvalidEncryptionKey(entity=InvalidEncryptionKey.Entity.CONNECTOR)
+ return json.loads(decrypted_value.decode("utf-8"))
class Meta:
db_table = "connector_instance"
diff --git a/backend/connector_v2/serializers.py b/backend/connector_v2/serializers.py
index c88598e34..2d274c190 100644
--- a/backend/connector_v2/serializers.py
+++ b/backend/connector_v2/serializers.py
@@ -1,3 +1,4 @@
+import json
import logging
from collections import OrderedDict
from typing import Any
@@ -7,8 +8,8 @@ from connector_auth_v2.pipeline.common import ConnectorAuthHelper
from connector_processor.connector_processor import ConnectorProcessor
from connector_processor.constants import ConnectorKeys
from connector_processor.exceptions import OAuthTimeOut
-from rest_framework.serializers import SerializerMethodField
-from utils.fields import EncryptedBinaryFieldSerializer
+from cryptography.fernet import Fernet
+from django.conf import settings
from utils.serializer_utils import SerializerUtils
from backend.serializers import AuditSerializer
@@ -21,13 +22,22 @@ logger = logging.getLogger(__name__)
class ConnectorInstanceSerializer(AuditSerializer):
- connector_metadata = EncryptedBinaryFieldSerializer(required=False, allow_null=True)
- icon = SerializerMethodField()
-
class Meta:
model = ConnectorInstance
fields = "__all__"
+ def validate_connector_metadata(self, value: dict[Any]) -> dict[Any]:
+ """Validating Json metadata This custom validation is to avoid conflict
+ with user input and db binary data.
+
+ Args:
+ value (Any): dict of metadata
+
+ Returns:
+ dict[Any]: dict of metadata
+ """
+ return value
+
def save(self, **kwargs): # type: ignore
user = self.context.get("request").user or None
connector_id: str = kwargs[CIKey.CONNECTOR_ID]
@@ -58,19 +68,15 @@ class ConnectorInstanceSerializer(AuditSerializer):
)
kwargs[CIKey.CONNECTOR_MODE] = connector_mode.value
+ encryption_secret: str = settings.ENCRYPTION_KEY
+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))
+ json_string: str = json.dumps(kwargs.get(CIKey.CONNECTOR_METADATA))
+
+ kwargs[CIKey.CONNECTOR_METADATA] = f.encrypt(json_string.encode("utf-8"))
+
instance = super().save(**kwargs)
return instance
- def get_icon(self, obj: ConnectorInstance) -> str:
- """Get connector icon from ConnectorProcessor."""
- icon_path = ConnectorProcessor.get_connector_data_with_key(
- obj.connector_id, ConnectorKeys.ICON
- )
- # Ensure icon path is properly formatted for frontend
- if icon_path and not icon_path.startswith("/"):
- return f"/{icon_path}"
- return icon_path
-
def to_representation(self, instance: ConnectorInstance) -> dict[str, str]:
# to remove the sensitive fields being returned
rep: OrderedDict[str, Any] = super().to_representation(instance)
@@ -78,6 +84,11 @@ class ConnectorInstanceSerializer(AuditSerializer):
rep[CIKey.CONNECTOR_METADATA] = {}
if SerializerUtils.check_context_for_GET_or_POST(context=self.context):
rep.pop(CIKey.CONNECTOR_AUTH)
- rep[ConnectorKeys.ICON] = self.get_icon(instance)
+ # set icon fields for UI
+ rep[ConnectorKeys.ICON] = ConnectorProcessor.get_connector_data_with_key(
+ instance.connector_id, ConnectorKeys.ICON
+ )
+ if instance.connector_metadata:
+ rep[CIKey.CONNECTOR_METADATA] = instance.metadata
return rep
diff --git a/backend/file_management/file_management_helper.py b/backend/file_management/file_management_helper.py
index ab56c4dba..cd6ab2d61 100644
--- a/backend/file_management/file_management_helper.py
+++ b/backend/file_management/file_management_helper.py
@@ -35,7 +35,7 @@ class FileManagerHelper:
@staticmethod
def get_file_system(connector: ConnectorInstance) -> UnstractFileSystem:
"""Creates the `UnstractFileSystem` for the corresponding connector."""
- metadata = connector.connector_metadata
+ metadata = connector.metadata
if connector.connector_id in fs_connectors:
connector = fs_connectors[connector.connector_id]["metadata"]["connector"]
connector_class: UnstractFileSystem = connector(metadata)
diff --git a/backend/pipeline_v2/serializers/crud.py b/backend/pipeline_v2/serializers/crud.py
index ab4f9705b..d69f616fd 100644
--- a/backend/pipeline_v2/serializers/crud.py
+++ b/backend/pipeline_v2/serializers/crud.py
@@ -224,42 +224,44 @@ class PipelineSerializer(IntegrityErrorMixin, AuditSerializer):
def _add_connector_data(
self,
repr: OrderedDict[str, Any],
- workflow_endpoints: list[Any],
+ connector_instance_list: list[Any],
connectors: list[Any],
) -> OrderedDict[str, Any]:
"""Adds connector Input/Output data.
Args:
- repr (OrderedDict[str, Any]): The representation dictionary
- workflow_endpoints (list[Any]): List of WorkflowEndpoint objects
- connectors (list[Any]): List of available connectors
+ sef (_type_): _description_
+ repr (OrderedDict[str, Any]): _description_
Returns:
- OrderedDict[str, Any]: Updated representation with connector data
+ OrderedDict[str, Any]: _description_
"""
repr[PC.SOURCE_NAME] = PC.NOT_CONFIGURED
repr[PC.DESTINATION_NAME] = PC.NOT_CONFIGURED
-
- for endpoint in workflow_endpoints:
- if endpoint.endpoint_type == WorkflowEndpoint.EndpointType.SOURCE:
- if endpoint.connector_instance:
- repr[PC.SOURCE_NAME], repr[PC.SOURCE_ICON] = self._get_name_and_icon(
+ for instance in connector_instance_list:
+ if instance.connector_type == "INPUT":
+ repr[PC.SOURCE_NAME], repr[PC.SOURCE_ICON] = self._get_name_and_icon(
+ connectors=connectors,
+ connector_id=instance.connector_id,
+ )
+ if instance.connector_type == "OUTPUT":
+ repr[PC.DESTINATION_NAME], repr[PC.DESTINATION_ICON] = (
+ self._get_name_and_icon(
connectors=connectors,
- connector_id=endpoint.connector_instance.connector_id,
+ connector_id=instance.connector_id,
)
- elif endpoint.endpoint_type == WorkflowEndpoint.EndpointType.DESTINATION:
- if (
- endpoint.connection_type
- == WorkflowEndpoint.ConnectionType.MANUALREVIEW
- ):
- repr[PC.DESTINATION_NAME] = "Manual Review"
- elif endpoint.connector_instance:
- repr[PC.DESTINATION_NAME], repr[PC.DESTINATION_ICON] = (
- self._get_name_and_icon(
- connectors=connectors,
- connector_id=endpoint.connector_instance.connector_id,
- )
+ )
+ if repr[PC.DESTINATION_NAME] == PC.NOT_CONFIGURED:
+ try:
+ check_manual_review = WorkflowEndpoint.objects.get(
+ workflow=instance.workflow,
+ endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,
+ connection_type=WorkflowEndpoint.ConnectionType.MANUALREVIEW,
)
+ if check_manual_review:
+ repr[PC.DESTINATION_NAME] = "Manual Review"
+ except Exception as ex:
+ logger.debug(f"Not a Manual review destination: {ex}")
return repr
def to_representation(self, instance: Pipeline) -> OrderedDict[str, Any]:
@@ -271,17 +273,15 @@ class PipelineSerializer(IntegrityErrorMixin, AuditSerializer):
if SerializerUtils.check_context_for_GET_or_POST(context=self.context):
workflow = instance.workflow
- workflow_endpoints = (
- WorkflowEndpoint.objects.filter(workflow=workflow.id)
- .select_related("connector_instance")
- .all()
- )
+ connector_instance_list = ConnectorInstance.objects.filter(
+ workflow=workflow.id
+ ).all()
repr[PK.WORKFLOW_ID] = workflow.id
repr[PK.WORKFLOW_NAME] = workflow.workflow_name
repr[PK.CRON_STRING] = repr.pop(PK.CRON_STRING)
repr = self._add_connector_data(
repr=repr,
- workflow_endpoints=workflow_endpoints,
+ connector_instance_list=connector_instance_list,
connectors=connectors,
)
diff --git a/backend/utils/fields.py b/backend/utils/fields.py
deleted file mode 100644
index 70b257fce..000000000
--- a/backend/utils/fields.py
+++ /dev/null
@@ -1,195 +0,0 @@
-"""Custom Django model fields and serializer fields for common use cases."""
-
-import json
-import logging
-from typing import Any
-
-from cryptography.fernet import Fernet, InvalidToken
-from django.conf import settings
-from django.db import models
-from rest_framework import serializers
-
-from utils.exceptions import InvalidEncryptionKey
-
-logger = logging.getLogger(__name__)
-
-
-class EncryptedBinaryField(models.BinaryField):
- """A BinaryField that automatically encrypts/decrypts JSON data.
-
- This field transparently handles encryption when storing data to the database
- and decryption when retrieving data from the database. It expects dictionary
- data as input and returns dictionary data as output.
-
- Features:
- - Automatic encryption/decryption using Fernet symmetric encryption
- - JSON serialization/deserialization
- - Proper error handling for encryption failures
- - Null value support
- - Compatible with Django ORM and DRF serializers
-
- Usage:
- class MyModel(models.Model):
- encrypted_data = EncryptedBinaryField(null=True)
-
- # Usage is transparent
- instance = MyModel.objects.create(encrypted_data={'key': 'value'})
- print(instance.encrypted_data) # {'key': 'value'}
- """
-
- description = "A binary field that automatically encrypts/decrypts JSON data"
-
- def __init__(self, *args, **kwargs):
- # Remove any JSON-specific kwargs that might be passed
- kwargs.pop("encoder", None)
- kwargs.pop("decoder", None)
- super().__init__(*args, **kwargs)
-
- def _get_encryption_key(self) -> bytes:
- """Get the encryption key from Django settings."""
- encryption_key = getattr(settings, "ENCRYPTION_KEY", None)
- if not encryption_key:
- raise ValueError("ENCRYPTION_KEY not found in Django settings")
- return encryption_key.encode("utf-8")
-
- def _encrypt_value(self, value: Any) -> bytes:
- """Encrypt a Python value (typically dict) to bytes."""
- if value is None:
- return None
-
- try:
- # Serialize to JSON string
- json_string = json.dumps(value)
-
- # Encrypt the JSON string
- cipher_suite = Fernet(self._get_encryption_key())
- encrypted_bytes = cipher_suite.encrypt(json_string.encode("utf-8"))
-
- return encrypted_bytes
- except Exception as e:
- logger.error(f"Failed to encrypt value: {e}")
- raise
-
- def _decrypt_value(self, value: bytes) -> dict | None:
- """Decrypt bytes to a Python value (typically dict)."""
- if value is None:
- return None
-
- try:
- # Handle memoryview objects from database
- if isinstance(value, memoryview):
- value = value.tobytes()
-
- # If it's already a dict, return as-is (for backward compatibility)
- if isinstance(value, dict):
- return value
-
- # Decrypt the bytes
- cipher_suite = Fernet(self._get_encryption_key())
- decrypted_bytes = cipher_suite.decrypt(value)
-
- # Parse JSON string back to Python object
- json_string = decrypted_bytes.decode("utf-8")
- return json.loads(json_string)
-
- except InvalidToken:
- logger.error("Invalid encryption token - possibly wrong encryption key")
- raise InvalidEncryptionKey(entity=InvalidEncryptionKey.Entity.CONNECTOR)
- except Exception as e:
- logger.error(f"Failed to decrypt value: {e}")
- raise
-
- def from_db_value(self, value, expression, connection):
- """Convert database value to Python value.
- Called when data is loaded from the database.
- """
- return self._decrypt_value(value)
-
- def to_python(self, value):
- """Convert input value to Python value.
- Called during model validation and form processing.
- """
- if value is None:
- return value
-
- # If it's already a Python object (dict), return as-is
- if isinstance(value, dict):
- return value
-
- # If it's bytes (encrypted), decrypt it
- if isinstance(value, (bytes, memoryview)):
- return self._decrypt_value(value)
-
- # If it's a string, try to parse as JSON
- if isinstance(value, str):
- try:
- return json.loads(value)
- except json.JSONDecodeError:
- pass
-
- return value
-
- def get_prep_value(self, value):
- """Convert Python value to database value.
- Called before saving to the database.
- """
- if value is None:
- return None
-
- # If it's already bytes (encrypted), return as-is
- if isinstance(value, (bytes, memoryview)):
- return value
-
- # Encrypt the value
- return self._encrypt_value(value)
-
- def get_db_prep_value(self, value, connection, prepared=False):
- """Convert Python value to database-specific value."""
- if not prepared:
- value = self.get_prep_value(value)
- return super().get_db_prep_value(value, connection, prepared)
-
- def value_to_string(self, obj):
- """Convert field value to string for serialization."""
- value = self.value_from_object(obj)
- if value is None:
- return None
- return json.dumps(value)
-
- def formfield(self, **kwargs):
- """Return a form field for this model field."""
- # Use a CharField for forms since we want JSON input
- from django import forms
-
- defaults = {"widget": forms.Textarea}
- defaults.update(kwargs)
- return forms.CharField(**defaults)
-
-
-class EncryptedBinaryFieldSerializer(serializers.Field):
- """Custom serializer field for EncryptedBinaryField.
-
- This field handles serialization/deserialization of encrypted binary data
- as JSON dictionaries, making it compatible with DRF's serialization process.
- """
-
- def to_representation(self, value):
- """Convert encrypted binary data to JSON dictionary for API responses."""
- if value is None:
- return None
-
- # The EncryptedBinaryField already decrypts the value when accessed
- # so we just need to return it as-is
- return value
-
- def to_internal_value(self, data):
- """Convert JSON dictionary to internal value for database storage."""
- if data is None:
- return None
-
- # Validate that the data is a dictionary
- if not isinstance(data, dict):
- raise serializers.ValidationError("Expected a dictionary.")
-
- # Return the data as-is; the EncryptedBinaryField will handle encryption
- return data
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index 2e68f4682..0d22f0a40 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -100,6 +100,10 @@ class DestinationConnector(BaseConnector):
workflow=workflow,
endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,
)
+ if endpoint.connector_instance:
+ endpoint.connector_instance.connector_metadata = (
+ endpoint.connector_instance.metadata
+ )
return endpoint
def _get_source_endpoint_for_workflow(
@@ -119,6 +123,10 @@ class DestinationConnector(BaseConnector):
workflow=workflow,
endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
)
+ if endpoint.connector_instance:
+ endpoint.connector_instance.connector_metadata = (
+ endpoint.connector_instance.metadata
+ )
return endpoint
def validate(self) -> None:
@@ -141,7 +149,7 @@ class DestinationConnector(BaseConnector):
# Get database class and test connection
db_class = DatabaseUtils.get_db_class(
connector_id=connector.connector_id,
- connector_settings=connector.connector_metadata,
+ connector_settings=connector.metadata,
)
engine = db_class.get_engine()
if hasattr(engine, "close"):
@@ -310,7 +318,7 @@ class DestinationConnector(BaseConnector):
def insert_into_db(self, input_file_path: str) -> None:
"""Insert data into the database."""
connector_instance: ConnectorInstance = self.endpoint.connector_instance
- connector_settings: dict[str, Any] = connector_instance.connector_metadata
+ connector_settings: dict[str, Any] = connector_instance.metadata
destination_configurations: dict[str, Any] = self.endpoint.configuration
table_name: str = str(destination_configurations.get(DestinationKey.TABLE))
include_agent: bool = bool(
diff --git a/backend/workflow_manager/endpoint_v2/serializers.py b/backend/workflow_manager/endpoint_v2/serializers.py
index 7f01c4bc3..2ce3d4a8f 100644
--- a/backend/workflow_manager/endpoint_v2/serializers.py
+++ b/backend/workflow_manager/endpoint_v2/serializers.py
@@ -1,25 +1,12 @@
import logging
-from connector_v2.models import ConnectorInstance
-from connector_v2.serializers import ConnectorInstanceSerializer
-from rest_framework import serializers
-from rest_framework.serializers import (
- ModelSerializer,
-)
+from rest_framework.serializers import ModelSerializer
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
logger = logging.getLogger(__name__)
class WorkflowEndpointSerializer(ModelSerializer):
- connector_instance = ConnectorInstanceSerializer(read_only=True)
- connector_instance_id = serializers.PrimaryKeyRelatedField(
- queryset=ConnectorInstance.objects.all(),
- source="connector_instance",
- write_only=True,
- allow_null=True,
- )
-
class Meta:
model = WorkflowEndpoint
fields = "__all__"
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index f1cc4d72b..4098963da 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -118,6 +118,10 @@ class SourceConnector(BaseConnector):
workflow=workflow,
endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
)
+ if endpoint.connector_instance:
+ endpoint.connector_instance.connector_metadata = (
+ endpoint.connector_instance.metadata
+ )
return endpoint
def validate(self) -> None:
diff --git a/backend/workflow_manager/endpoint_v2/urls.py b/backend/workflow_manager/endpoint_v2/urls.py
index d4dec7c3b..522859e23 100644
--- a/backend/workflow_manager/endpoint_v2/urls.py
+++ b/backend/workflow_manager/endpoint_v2/urls.py
@@ -6,7 +6,7 @@ workflow_endpoint_list = WorkflowEndpointViewSet.as_view(
)
endpoint_list = WorkflowEndpointViewSet.as_view({"get": "list"})
workflow_endpoint_detail = WorkflowEndpointViewSet.as_view(
- {"get": "retrieve", "put": "update", "patch": "partial_update"}
+ {"get": "retrieve", "put": "update"}
)
endpoint_settings_detail = WorkflowEndpointViewSet.as_view(
{"get": WorkflowEndpointViewSet.get_settings.__name__}
diff --git a/backend/workflow_manager/endpoint_v2/views.py b/backend/workflow_manager/endpoint_v2/views.py
index 986803e86..1766cf450 100644
--- a/backend/workflow_manager/endpoint_v2/views.py
+++ b/backend/workflow_manager/endpoint_v2/views.py
@@ -6,8 +6,8 @@ from rest_framework.response import Response
from workflow_manager.endpoint_v2.destination import DestinationConnector
from workflow_manager.endpoint_v2.endpoint_utils import WorkflowEndpointUtils
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
-from workflow_manager.endpoint_v2.serializers import WorkflowEndpointSerializer
from workflow_manager.endpoint_v2.source import SourceConnector
+from workflow_manager.workflow_v2.serializers import WorkflowEndpointSerializer
class WorkflowEndpointViewSet(viewsets.ModelViewSet):
@@ -19,15 +19,10 @@ class WorkflowEndpointViewSet(viewsets.ModelViewSet):
.select_related("workflow")
.filter(workflow__created_by=self.request.user)
)
- workflow_filter = self.request.query_params.get("workflow", None)
- if workflow_filter:
- queryset = queryset.filter(workflow_id=workflow_filter)
-
endpoint_type_filter = self.request.query_params.get("endpoint_type", None)
+ connection_type_filter = self.request.query_params.get("connection_type", None)
if endpoint_type_filter:
queryset = queryset.filter(endpoint_type=endpoint_type_filter)
-
- connection_type_filter = self.request.query_params.get("connection_type", None)
if connection_type_filter:
queryset = queryset.filter(connection_type=connection_type_filter)
return queryset
diff --git a/backend/workflow_manager/workflow_v2/serializers.py b/backend/workflow_manager/workflow_v2/serializers.py
index 1ec9a559a..b076b8895 100644
--- a/backend/workflow_manager/workflow_v2/serializers.py
+++ b/backend/workflow_manager/workflow_v2/serializers.py
@@ -16,6 +16,7 @@ from utils.serializer.integrity_error_mixin import IntegrityErrorMixin
from backend.constants import RequestKey
from backend.serializers import AuditSerializer
+from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.workflow_v2.constants import WorkflowExecutionKey, WorkflowKey
from workflow_manager.workflow_v2.models.execution import WorkflowExecution
from workflow_manager.workflow_v2.models.execution_log import ExecutionLog
@@ -103,6 +104,14 @@ class ExecuteWorkflowResponseSerializer(Serializer):
result = JSONField()
+class WorkflowEndpointSerializer(ModelSerializer):
+ workflow_name = CharField(source="workflow.workflow_name", read_only=True)
+
+ class Meta:
+ model = WorkflowEndpoint
+ fields = "__all__"
+
+
class WorkflowExecutionSerializer(ModelSerializer):
class Meta:
model = WorkflowExecution
diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.py
index 01f750345..a0d60772c 100644
--- a/backend/workflow_manager/workflow_v2/views.py
+++ b/backend/workflow_manager/workflow_v2/views.py
@@ -100,11 +100,10 @@ class WorkflowViewSet(viewsets.ModelViewSet):
is_active=True,
)
try:
- # Create empty WorkflowEndpoints for UI compatibility
- # ConnectorInstances will be created when users actually configure connectors
WorkflowEndpointUtils.create_endpoints_for_workflow(workflow)
+ # NOTE: Add default connector here if needed
except Exception as e:
- logger.error(f"Error creating workflow endpoints: {e}")
+ logger.error(f"Error saving workflow to DB: {e}")
raise WorkflowGenerationError
return workflow
diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
index 046caeb9f..d41064a6c 100644
--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
@@ -23,7 +23,6 @@ import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";
import { useAlertStore } from "../../../store/alert-store";
import { useSessionStore } from "../../../store/session-store";
import { useWorkflowStore } from "../../../store/workflow-store";
-import useRequestUrl from "../../../hooks/useRequestUrl";
import SpaceWrapper from "../../widgets/space-wrapper/SpaceWrapper";
import { ConfigureConnectorModal } from "../configure-connector-modal/ConfigureConnectorModal";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
@@ -83,7 +82,6 @@ function DsSettingsCard({ type, endpointDetails, message }) {
const axiosPrivate = useAxiosPrivate();
const handleException = useExceptionHandler();
const { flags } = sessionDetails;
- const { getUrl } = useRequestUrl();
const icons = {
input: <ImportOutlined className="ds-set-icon-size" />,
@@ -146,30 +144,16 @@ function DsSettingsCard({ type, endpointDetails, message }) {
setConnType(endpointDetails?.connection_type);
}
- if (!endpointDetails?.connector_instance) {
+ if (!endpointDetails?.connector_instance?.length) {
setConnDetails({});
return;
}
- // Use connector_instance data directly from endpointDetails if it's an object
- if (typeof endpointDetails?.connector_instance === "object") {
- const connectorData = endpointDetails.connector_instance;
- connectorData.connector_metadata = connectorData.connector_metadata || {};
- connectorData.connector_metadata.connectorName =
- connectorData?.connector_name || "";
- setConnDetails(connectorData);
- setSelectedId(connectorData?.connector_id);
+ if (connDetails?.id === endpointDetails?.connector_instance) {
return;
}
- // Fallback for legacy connector_instance ID format (string)
- if (typeof endpointDetails?.connector_instance === "string") {
- // Only call getSourceDetails if we haven't already loaded this connector
- if (connDetails?.id !== endpointDetails?.connector_instance) {
- getSourceDetails();
- }
- return;
- }
+ getSourceDetails();
}, [endpointDetails]);
useEffect(() => {
@@ -203,7 +187,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {
setFormDataConfig(endpointDetails.configuration || {});
const requestOptions = {
method: "GET",
- url: getUrl(`workflow/endpoint/${endpointDetails?.id}/settings/`),
+ url: `/api/v1/unstract/${sessionDetails?.orgId}/workflow/endpoint/${endpointDetails?.id}/settings/`,
};
setIsSpecConfigLoading(true);
@@ -227,7 +211,9 @@ function DsSettingsCard({ type, endpointDetails, message }) {
const requestOptions = {
method: "GET",
- url: getUrl(`supported_connectors/?type=${type.toUpperCase()}`),
+ url: `/api/v1/unstract/${
+ sessionDetails?.orgId
+ }/supported_connectors/?type=${type.toUpperCase()}`,
};
axiosPrivate(requestOptions)
@@ -268,7 +254,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {
const requestOptions = {
method: "PUT",
- url: getUrl(`workflow/endpoint/${destination?.id}/`),
+ url: `/api/v1/unstract/${sessionDetails?.orgId}/workflow/endpoint/${destination?.id}/`,
headers: {
"X-CSRFToken": sessionDetails?.csrfToken,
"Content-Type": "application/json",
@@ -293,20 +279,22 @@ function DsSettingsCard({ type, endpointDetails, message }) {
if (type === "input") {
clearDestination({
connection_type: "",
- connector_instance_id: null,
+ connector_instance: null,
});
}
};
const handleUpdate = (updatedData, showSuccess) => {
+ const body = { ...endpointDetails, ...updatedData };
+
const requestOptions = {
- method: "PATCH",
- url: getUrl(`workflow/endpoint/${endpointDetails?.id}/`),
+ method: "PUT",
+ url: `/api/v1/unstract/${sessionDetails?.orgId}/workflow/endpoint/${endpointDetails?.id}/`,
headers: {
"X-CSRFToken": sessionDetails?.csrfToken,
"Content-Type": "application/json",
},
- data: updatedData,
+ data: body,
};
axiosPrivate(requestOptions)
.then((res) => {
@@ -333,7 +321,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {
const getSourceDetails = () => {
const requestOptions = {
method: "GET",
- url: getUrl(`connector/${endpointDetails?.connector_instance}/`),
+ url: `/api/v1/unstract/${sessionDetails?.orgId}/connector/${endpointDetails?.connector_instance}/`,
};
axiosPrivate(requestOptions)
@@ -374,7 +362,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {
onChange={(value) => {
handleUpdate({
connection_type: value,
- connector_instance_id: null,
+ connector_instance: null,
});
updateDestination();
}}
diff --git a/frontend/src/components/api/prompt-studio-service.js b/frontend/src/components/api/prompt-studio-service.js
index 8a2a7c17c..407726fab 100644
--- a/frontend/src/components/api/prompt-studio-service.js
+++ b/frontend/src/components/api/prompt-studio-service.js
@@ -1,5 +1,4 @@
import { useCallback } from "react";
-
import useRequestUrl from "../../hooks/useRequestUrl";
import { useAxiosPrivate } from "../../hooks/useAxiosPrivate";
diff --git a/frontend/src/components/deployments/create-api-deployment-from-prompt-studio/CreateApiDeploymentFromPromptStudio.jsx b/frontend/src/components/deployments/create-api-deployment-from-prompt-studio/CreateApiDeploymentFromPromptStudio.jsx
index b65630f64..dcb5bddf7 100644
--- a/frontend/src/components/deployments/create-api-deployment-from-prompt-studio/CreateApiDeploymentFromPromptStudio.jsx
+++ b/frontend/src/components/deployments/create-api-deployment-from-prompt-studio/CreateApiDeploymentFromPromptStudio.jsx
@@ -460,13 +460,14 @@ const CreateApiDeploymentFromPromptStudio = ({
// Update each endpoint to set connection_type to API
for (const endpoint of endpoints) {
await axiosPrivate({
- method: "PATCH",
+ method: "PUT",
url: getUrl(`workflow/endpoint/${endpoint.id}/`),
headers: {
"X-CSRFToken": sessionDetails?.csrfToken,
"Content-Type": "application/json",
},
data: {
+ ...endpoint,
connection_type: "API",
configuration: endpoint.configuration || {},
},
diff --git a/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx b/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx
index e4a527b4b..75ecb2e11 100644
--- a/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx
+++ b/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx
@@ -184,16 +184,14 @@ function ConfigureDs({
const connectorMetadata = { ...formData };
const connectorName = connectorMetadata?.connectorName;
delete connectorMetadata.connectorName;
-
body = {
+ workflow: id,
+ created_by: sessionDetails?.id,
connector_id: selectedSourceId,
connector_metadata: connectorMetadata,
- connector_name: connectorName,
- created_by: sessionDetails?.id,
- workflow: id,
connector_type: type.toUpperCase(),
+ connector_name: connectorName,
};
-
url += "connector/";
try {
@@ -256,10 +254,7 @@ function ConfigureDs({
const data = res?.data;
if (sourceTypes.connectors.includes(type)) {
handleUpdate(
- {
- connector_instance_id: data?.id,
- configuration: formDataConfig,
- },
+ { connector_instance: data?.id, configuration: formDataConfig },
true
);
setIsTcSuccessful(false);
diff --git a/frontend/src/components/input-output/input-output/InputOutput.jsx b/frontend/src/components/input-output/input-output/InputOutput.jsx
index 95e37b97b..a8059ce0f 100644
--- a/frontend/src/components/input-output/input-output/InputOutput.jsx
+++ b/frontend/src/components/input-output/input-output/InputOutput.jsx
@@ -6,7 +6,6 @@ import { CONNECTOR_TYPE_MAP } from "../../../helpers/GetStaticData.js";
import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";
import { useAlertStore } from "../../../store/alert-store";
import { useSessionStore } from "../../../store/session-store";
-import useRequestUrl from "../../../hooks/useRequestUrl";
import { AddSourceModal } from "../add-source-modal/AddSourceModal.jsx";
import { ManageFiles } from "../manage-files/ManageFiles.jsx";
import { Sidebar } from "../sidebar/Sidebar.jsx";
@@ -27,7 +26,6 @@ function InputOutput() {
const { id } = useParams();
const axiosPrivate = useAxiosPrivate();
const handleException = useExceptionHandler();
- const { getUrl } = useRequestUrl();
const location = useLocation();
const currentPath = location.pathname;
@@ -47,31 +45,25 @@ function InputOutput() {
setConnectorType(type);
- const endpointType = CONNECTOR_TYPE_MAP[type]?.toUpperCase();
-
const requestOptions = {
method: "GET",
- url: getUrl(
- `workflow/endpoint/?workflow=${id}&endpoint_type=${endpointType}`
- ),
+ url: `/api/v1/unstract/${
+ sessionDetails?.orgId
+ }/connector/?workflow=${id}&connector_type=${type.toUpperCase()}`,
};
axiosPrivate(requestOptions)
.then((res) => {
- const endpoints = res?.data;
- if (endpoints?.length === 0) {
+ const sources = res?.data;
+ if (sources?.length === 0) {
setSelectedItem("");
setListOfItems([]);
return;
}
- const menuItems = endpoints.map((item) =>
- getItem(
- item?.connector_instance?.connector_name,
- item?.connector_instance?.id,
- sourceIcon(item?.connector_instance?.connector_icon)
- )
+ const menuItems = sources.map((item) =>
+ getItem(item?.connector_name, item?.id, sourceIcon(item?.icon))
);
- const firstId = endpoints[0]?.connector_instance?.id?.toString();
+ const firstId = sources[0].id.toString();
setSelectedItem(firstId);
setListOfItems(menuItems);
})
@@ -100,7 +92,7 @@ function InputOutput() {
const handleDelete = () => {
const requestOptions = {
method: "DELETE",
- url: getUrl(`connector/${selectedItem}/`),
+ url: `/api/v1/unstract/${sessionDetails?.orgId}/connector/${selectedItem}/`,
headers: {
"X-CSRFToken": sessionDetails?.csrfToken,
},
diff --git a/frontend/src/hooks/usePostHogEvents.js b/frontend/src/hooks/usePostHogEvents.js
index e890ef29b..33a5ada0e 100644
--- a/frontend/src/hooks/usePostHogEvents.js
+++ b/frontend/src/hooks/usePostHogEvents.js
@@ -1,5 +1,4 @@
import { usePostHog } from "posthog-js/react";
-
import { useSessionStore } from "../store/session-store";
const usePostHogEvents = () => {
diff --git a/frontend/src/store/alert-store.js b/frontend/src/store/alert-store.js
index 95dec9b79..0e42186b5 100644
--- a/frontend/src/store/alert-store.js
+++ b/frontend/src/store/alert-store.js
@@ -1,7 +1,6 @@
import { create } from "zustand";
-import { uniqueId } from "lodash";
-
import { isNonNegativeNumber } from "../helpers/GetStaticData";
+import { uniqueId } from "lodash";
const DEFAULT_DURATION = 6;