LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2585 [MISC] Deprecate connector instance fields and refactor endpoint handling (#1454)
Chandrasekharan Mff5dd19b09c3
diff --git a/.gitignore b/.gitignoreindex b197d1623..53b1a840e 100644--- a/.gitignore+++ b/.gitignore@@ -680,7 +680,15 @@ backend/backend/*_urls.pybackend/backend/*_urls_v2.py!backend/backend/public_urls_v2.py-# Adding claude prompting directories+## Vibe coding / Prompting-related files ##+prompting/+.prompting/++# ClaudeCLAUDE.md-CONTRIBUTION_GUIDE.md-prompting/*+.claude/*++# Windsurf+.qodo+.windsurfrules+.windsurf/rulesdiff --git a/backend/connector_v2/migrations/0002_alter_connectorinstance_connector_metadata_and_more.py b/backend/connector_v2/migrations/0002_alter_connectorinstance_connector_metadata_and_more.pynew file mode 100644index 000000000..267bcb87d--- /dev/null+++ b/backend/connector_v2/migrations/0002_alter_connectorinstance_connector_metadata_and_more.py@@ -0,0 +1,37 @@+# Generated by Django 4.2.1 on 2025-07-16 10:26++import django.db.models.deletion+import utils.fields+from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("workflow_v2", "0015_executionlog_idx_wf_execution_event_time_and_more"),+ ("connector_v2", "0001_initial"),+ ]++ operations = [+ migrations.AlterField(+ model_name="connectorinstance",+ name="connector_metadata",+ field=utils.fields.EncryptedBinaryField(null=True),+ ),+ migrations.AlterField(+ model_name="connectorinstance",+ name="connector_type",+ field=models.CharField(+ choices=[("INPUT", "Input"), ("OUTPUT", "Output")], null=True+ ),+ ),+ migrations.AlterField(+ model_name="connectorinstance",+ name="workflow",+ field=models.ForeignKey(+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="connector_workflow",+ to="workflow_v2.workflow",+ ),+ ),+ ]diff --git a/backend/connector_v2/models.py b/backend/connector_v2/models.pyindex dbeaeb019..d7276aa19 100644--- a/backend/connector_v2/models.py+++ b/backend/connector_v2/models.py@@ -1,4 +1,4 @@-import json+import loggingimport uuidfrom typing import Any@@ -6,10 +6,8 @@ from account_v2.models import Userfrom connector_auth_v2.models import ConnectorAuthfrom connector_processor.connector_processor import ConnectorProcessorfrom connector_processor.constants import ConnectorKeys-from cryptography.fernet import Fernet, InvalidToken-from django.conf import settingsfrom django.db import models-from utils.exceptions import InvalidEncryptionKey+from utils.fields import EncryptedBinaryFieldfrom utils.models.base_model import BaseModelfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,@@ -21,6 +19,8 @@ from backend.constants import FieldLengthConstants as FLCCONNECTOR_NAME_SIZE = 128VERSION_NAME_SIZE = 64+logger = logging.getLogger(__name__)+class ConnectorInstanceModelManager(DefaultOrganizationManagerMixin, models.Manager):pass@@ -45,13 +45,13 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):"workflow_v2.Workflow",on_delete=models.CASCADE,related_name="connector_workflow",- null=False,+ null=True,blank=False,)connector_id = models.CharField(max_length=FLC.CONNECTOR_ID_LENGTH, default="")- connector_metadata = models.BinaryField(null=True)+ connector_metadata = EncryptedBinaryField(null=True)connector_version = models.CharField(max_length=VERSION_NAME_SIZE, default="")- connector_type = models.CharField(choices=ConnectorType.choices)+ connector_type = models.CharField(choices=ConnectorType.choices, null=True)# TODO: handle connector_auth cascade deletionconnector_auth = models.ForeignKey(ConnectorAuth,@@ -84,21 +84,6 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):# Managerobjects = ConnectorInstanceModelManager()- # TODO: Remove if unused- def get_connector_metadata(self) -> dict[str, str]:- """Gets connector metadata and refreshes the tokens if needed in case- of OAuth.- """- tokens_refreshed = False- if self.connector_auth:- (- self.connector_metadata,- tokens_refreshed,- ) = self.connector_auth.get_and_refresh_tokens()- if tokens_refreshed:- self.save()- return self.connector_metadata-@staticmethoddef supportsOAuth(connector_id: str) -> bool:return bool(@@ -109,19 +94,9 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):def __str__(self) -> str:return (- f"Connector({self.id}, type{self.connector_type}, workflow: {self.workflow})"+ f"Connector({self.id}, ID={self.connector_id}, mode: {self.connector_mode})")- def get_connector_metadata_bytes(self):- """Convert connector_metadata to bytes if it is a memoryview.-- Returns:- bytes: The connector_metadata as bytes.- """- if isinstance(self.connector_metadata, memoryview):- return self.connector_metadata.tobytes()- return self.connector_metadata-@propertydef metadata(self) -> Any:"""Decrypt and return the connector metadata as a dictionary.@@ -132,21 +107,20 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):Returns:dict: The decrypted connector metadata.++ .. deprecated::+ This property is deprecated. Use `connector_metadata` field directly instead.+ This property will be removed in a future version."""- try:- connector_metadata_bytes = self.get_connector_metadata_bytes()-- if connector_metadata_bytes is None:- return None-- if isinstance(connector_metadata_bytes, (dict)):- return connector_metadata_bytes- encryption_secret: str = settings.ENCRYPTION_KEY- cipher_suite: Fernet = Fernet(encryption_secret.encode("utf-8"))- decrypted_value = cipher_suite.decrypt(connector_metadata_bytes)- except InvalidToken:- raise InvalidEncryptionKey(entity=InvalidEncryptionKey.Entity.CONNECTOR)- return json.loads(decrypted_value.decode("utf-8"))+ import warnings++ warnings.warn(+ "The 'metadata' property is deprecated. Use 'connector_metadata' field directly instead. "+ "This property will be removed in a future version.",+ DeprecationWarning,+ stacklevel=2,+ )+ return self.connector_metadataclass Meta:db_table = "connector_instance"diff --git a/backend/connector_v2/serializers.py b/backend/connector_v2/serializers.pyindex 2d274c190..c88598e34 100644--- a/backend/connector_v2/serializers.py+++ b/backend/connector_v2/serializers.py@@ -1,4 +1,3 @@-import jsonimport loggingfrom collections import OrderedDictfrom typing import Any@@ -8,8 +7,8 @@ from connector_auth_v2.pipeline.common import ConnectorAuthHelperfrom connector_processor.connector_processor import ConnectorProcessorfrom connector_processor.constants import ConnectorKeysfrom connector_processor.exceptions import OAuthTimeOut-from cryptography.fernet import Fernet-from django.conf import settings+from rest_framework.serializers import SerializerMethodField+from utils.fields import EncryptedBinaryFieldSerializerfrom utils.serializer_utils import SerializerUtilsfrom backend.serializers import AuditSerializer@@ -22,22 +21,13 @@ logger = logging.getLogger(__name__)class ConnectorInstanceSerializer(AuditSerializer):+ connector_metadata = EncryptedBinaryFieldSerializer(required=False, allow_null=True)+ icon = SerializerMethodField()+class Meta:model = ConnectorInstancefields = "__all__"- def validate_connector_metadata(self, value: dict[Any]) -> dict[Any]:- """Validating Json metadata This custom validation is to avoid conflict- with user input and db binary data.-- Args:- value (Any): dict of metadata-- Returns:- dict[Any]: dict of metadata- """- return value-def save(self, **kwargs): # type: ignoreuser = self.context.get("request").user or Noneconnector_id: str = kwargs[CIKey.CONNECTOR_ID]@@ -68,15 +58,19 @@ class ConnectorInstanceSerializer(AuditSerializer):)kwargs[CIKey.CONNECTOR_MODE] = connector_mode.value- encryption_secret: str = settings.ENCRYPTION_KEY- f: Fernet = Fernet(encryption_secret.encode("utf-8"))- json_string: str = json.dumps(kwargs.get(CIKey.CONNECTOR_METADATA))-- kwargs[CIKey.CONNECTOR_METADATA] = f.encrypt(json_string.encode("utf-8"))-instance = super().save(**kwargs)return instance+ def get_icon(self, obj: ConnectorInstance) -> str:+ """Get connector icon from ConnectorProcessor."""+ icon_path = ConnectorProcessor.get_connector_data_with_key(+ obj.connector_id, ConnectorKeys.ICON+ )+ # Ensure icon path is properly formatted for frontend+ if icon_path and not icon_path.startswith("/"):+ return f"/{icon_path}"+ return icon_path+def to_representation(self, instance: ConnectorInstance) -> dict[str, str]:# to remove the sensitive fields being returnedrep: OrderedDict[str, Any] = super().to_representation(instance)@@ -84,11 +78,6 @@ class ConnectorInstanceSerializer(AuditSerializer):rep[CIKey.CONNECTOR_METADATA] = {}if SerializerUtils.check_context_for_GET_or_POST(context=self.context):rep.pop(CIKey.CONNECTOR_AUTH)- # set icon fields for UI- rep[ConnectorKeys.ICON] = ConnectorProcessor.get_connector_data_with_key(- instance.connector_id, ConnectorKeys.ICON- )+ rep[ConnectorKeys.ICON] = self.get_icon(instance)- if instance.connector_metadata:- rep[CIKey.CONNECTOR_METADATA] = instance.metadatareturn repdiff --git a/backend/file_management/file_management_helper.py b/backend/file_management/file_management_helper.pyindex cd6ab2d61..ab56c4dba 100644--- a/backend/file_management/file_management_helper.py+++ b/backend/file_management/file_management_helper.py@@ -35,7 +35,7 @@ class FileManagerHelper:@staticmethoddef get_file_system(connector: ConnectorInstance) -> UnstractFileSystem:"""Creates the `UnstractFileSystem` for the corresponding connector."""- metadata = connector.metadata+ metadata = connector.connector_metadataif connector.connector_id in fs_connectors:connector = fs_connectors[connector.connector_id]["metadata"]["connector"]connector_class: UnstractFileSystem = connector(metadata)diff --git a/backend/pipeline_v2/serializers/crud.py b/backend/pipeline_v2/serializers/crud.pyindex d69f616fd..ab4f9705b 100644--- a/backend/pipeline_v2/serializers/crud.py+++ b/backend/pipeline_v2/serializers/crud.py@@ -224,44 +224,42 @@ class PipelineSerializer(IntegrityErrorMixin, AuditSerializer):def _add_connector_data(self,repr: OrderedDict[str, Any],- connector_instance_list: list[Any],+ workflow_endpoints: list[Any],connectors: list[Any],) -> OrderedDict[str, Any]:"""Adds connector Input/Output data.Args:- sef (_type_): _description_- repr (OrderedDict[str, Any]): _description_+ repr (OrderedDict[str, Any]): The representation dictionary+ workflow_endpoints (list[Any]): List of WorkflowEndpoint objects+ connectors (list[Any]): List of available connectorsReturns:- OrderedDict[str, Any]: _description_+ OrderedDict[str, Any]: Updated representation with connector data"""repr[PC.SOURCE_NAME] = PC.NOT_CONFIGUREDrepr[PC.DESTINATION_NAME] = PC.NOT_CONFIGURED- for instance in connector_instance_list:- if instance.connector_type == "INPUT":- repr[PC.SOURCE_NAME], repr[PC.SOURCE_ICON] = self._get_name_and_icon(- connectors=connectors,- connector_id=instance.connector_id,- )- if instance.connector_type == "OUTPUT":- repr[PC.DESTINATION_NAME], repr[PC.DESTINATION_ICON] = (- self._get_name_and_icon(++ for endpoint in workflow_endpoints:+ if endpoint.endpoint_type == WorkflowEndpoint.EndpointType.SOURCE:+ if endpoint.connector_instance:+ repr[PC.SOURCE_NAME], repr[PC.SOURCE_ICON] = self._get_name_and_icon(connectors=connectors,- connector_id=instance.connector_id,+ connector_id=endpoint.connector_instance.connector_id,)- )- if repr[PC.DESTINATION_NAME] == PC.NOT_CONFIGURED:- try:- check_manual_review = WorkflowEndpoint.objects.get(- workflow=instance.workflow,- endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,- connection_type=WorkflowEndpoint.ConnectionType.MANUALREVIEW,+ elif endpoint.endpoint_type == WorkflowEndpoint.EndpointType.DESTINATION:+ if (+ endpoint.connection_type+ == WorkflowEndpoint.ConnectionType.MANUALREVIEW+ ):+ repr[PC.DESTINATION_NAME] = "Manual Review"+ elif endpoint.connector_instance:+ repr[PC.DESTINATION_NAME], repr[PC.DESTINATION_ICON] = (+ self._get_name_and_icon(+ connectors=connectors,+ connector_id=endpoint.connector_instance.connector_id,+ ))- if check_manual_review:- repr[PC.DESTINATION_NAME] = "Manual Review"- except Exception as ex:- logger.debug(f"Not a Manual review destination: {ex}")return reprdef to_representation(self, instance: Pipeline) -> OrderedDict[str, Any]:@@ -273,15 +271,17 @@ class PipelineSerializer(IntegrityErrorMixin, AuditSerializer):if SerializerUtils.check_context_for_GET_or_POST(context=self.context):workflow = instance.workflow- connector_instance_list = ConnectorInstance.objects.filter(- workflow=workflow.id- ).all()+ workflow_endpoints = (+ WorkflowEndpoint.objects.filter(workflow=workflow.id)+ .select_related("connector_instance")+ .all()+ )repr[PK.WORKFLOW_ID] = workflow.idrepr[PK.WORKFLOW_NAME] = workflow.workflow_namerepr[PK.CRON_STRING] = repr.pop(PK.CRON_STRING)repr = self._add_connector_data(repr=repr,- connector_instance_list=connector_instance_list,+ workflow_endpoints=workflow_endpoints,connectors=connectors,)diff --git a/backend/sample.env b/backend/sample.envindex 9c55b2034..1e0076cd4 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -169,25 +169,27 @@ TOOL_REGISTRY_STORAGE_CREDENTIALS='{"provider":"local"}'ENABLE_HIGHLIGHT_API_DEPLOYMENT=False# Execution result and cache expire time-# For API results cached per workflow execution-EXECUTION_RESULT_TTL_SECONDS=86400 # 24 hours-# For execution metadata cached per workflow execution-EXECUTION_CACHE_TTL_SECONDS=86400 # 24 hours-# Instant workflow polling timeout in seconds-INSTANT_WF_POLLING_TIMEOUT=300 # 5 minutes--# Maximum number of batches (i.e., parallel tasks) created for a single workflow execution-MAX_PARALLEL_FILE_BATCHES=1 # 1 file at a time+# For API results cached per workflow execution (24 hours)+EXECUTION_RESULT_TTL_SECONDS=86400+# For execution metadata cached per workflow execution (24 hours)+EXECUTION_CACHE_TTL_SECONDS=86400+# Instant workflow polling timeout in seconds (5 minutes)+INSTANT_WF_POLLING_TIMEOUT=300++# Maximum number of batches (i.e., parallel tasks) created for a single workflow execution (1 file at a time)+MAX_PARALLEL_FILE_BATCHES=1# Maximum allowed value for MAX_PARALLEL_FILE_BATCHES (upper limit for validation)MAX_PARALLEL_FILE_BATCHES_MAX_VALUE=100-# File execution tracker ttl in seconds-FILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000 # 5 hours-FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND=600 # 10 minutes+# File execution tracker TTL in seconds (5 hours)+FILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000+# File execution tracker completed TTL in seconds (10 minutes)+FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND=600-# Runner polling timeout-MAX_RUNNER_POLLING_WAIT_SECONDS=10800 # 3 hours-RUNNER_POLLING_INTERVAL_SECONDS=2 # 2 seconds+# Runner polling timeout (3 hours)+MAX_RUNNER_POLLING_WAIT_SECONDS=10800+# Runner polling interval (2 seconds)+RUNNER_POLLING_INTERVAL_SECONDS=2# ETL Pipeline minimum schedule interval (in seconds)# Default: 1800 seconds (30 minutes)diff --git a/backend/utils/fields.py b/backend/utils/fields.pynew file mode 100644index 000000000..e64f04121--- /dev/null+++ b/backend/utils/fields.py@@ -0,0 +1,199 @@+"""Custom Django model fields and serializer fields for common use cases."""++import json+import logging+from typing import Any++from cryptography.fernet import Fernet, InvalidToken+from django.conf import settings+from django.db import models+from rest_framework import serializers++from utils.exceptions import InvalidEncryptionKey++logger = logging.getLogger(__name__)+++class EncryptedBinaryField(models.BinaryField):+ """A BinaryField that automatically encrypts/decrypts JSON data.++ This field transparently handles encryption when storing data to the database+ and decryption when retrieving data from the database. It expects dictionary+ data as input and returns dictionary data as output.++ Features:+ - Automatic encryption/decryption using Fernet symmetric encryption+ - JSON serialization/deserialization+ - Proper error handling for encryption failures+ - Null value support+ - Compatible with Django ORM and DRF serializers++ Usage:+ class MyModel(models.Model):+ encrypted_data = EncryptedBinaryField(null=True)++ # Usage is transparent+ instance = MyModel.objects.create(encrypted_data={'key': 'value'})+ print(instance.encrypted_data) # {'key': 'value'}+ """++ description = "A binary field that automatically encrypts/decrypts JSON data"++ def __init__(self, *args, **kwargs):+ # Remove any JSON-specific kwargs that might be passed+ kwargs.pop("encoder", None)+ kwargs.pop("decoder", None)+ super().__init__(*args, **kwargs)++ def _get_encryption_key(self) -> bytes:+ """Get the encryption key from Django settings."""+ encryption_key = getattr(settings, "ENCRYPTION_KEY", None)+ if not encryption_key:+ raise ValueError("ENCRYPTION_KEY not found in Django settings")+ return encryption_key.encode("utf-8")++ def _encrypt_value(self, value: Any) -> bytes | None:+ """Encrypt a Python value (typically dict) to bytes."""+ if value is None:+ return None++ try:+ # Serialize to JSON string+ json_string = json.dumps(value)++ # Encrypt the JSON string+ cipher_suite = Fernet(self._get_encryption_key())+ encrypted_bytes = cipher_suite.encrypt(json_string.encode("utf-8"))++ return encrypted_bytes+ except Exception as e:+ logger.error(f"Failed to encrypt value: {e}")+ raise++ def _decrypt_value(self, value: bytes) -> dict | None:+ """Decrypt bytes to a Python value (typically dict)."""+ if value is None:+ return None++ try:+ # Handle memoryview objects from database+ if isinstance(value, memoryview):+ value = value.tobytes()++ # If it's already a dict, return as-is (for backward compatibility)+ if isinstance(value, dict):+ logger.warning(+ "Detected unencrypted dict in EncryptedBinaryField – "+ "legacy data will remain unencrypted. "+ "Run a migration to secure all records."+ )+ return value+ # Decrypt the bytes+ cipher_suite = Fernet(self._get_encryption_key())+ decrypted_bytes = cipher_suite.decrypt(value)++ # Parse JSON string back to Python object+ json_string = decrypted_bytes.decode("utf-8")+ return json.loads(json_string)++ except InvalidToken:+ logger.error("Invalid encryption token - possibly wrong encryption key")+ raise InvalidEncryptionKey(entity=InvalidEncryptionKey.Entity.CONNECTOR)+ except Exception as e:+ logger.error(f"Failed to decrypt value: {e}")+ raise++ def from_db_value(self, value, expression, connection):+ """Convert database value to Python value.+ Called when data is loaded from the database.+ """+ return self._decrypt_value(value)++ def to_python(self, value):+ """Convert input value to Python value.+ Called during model validation and form processing.+ """+ if value is None:+ return value++ # If it's already a Python object (dict), return as-is+ if isinstance(value, dict):+ return value++ # If it's bytes (encrypted), decrypt it+ if isinstance(value, (bytes, memoryview)):+ return self._decrypt_value(value)++ # If it's a string, try to parse as JSON+ if isinstance(value, str):+ try:+ return json.loads(value)+ except json.JSONDecodeError:+ pass++ return value++ def get_prep_value(self, value):+ """Convert Python value to database value.+ Called before saving to the database.+ """+ if value is None:+ return None++ # If it's already bytes (encrypted), return as-is+ if isinstance(value, (bytes, memoryview)):+ return value++ # Encrypt the value+ return self._encrypt_value(value)++ def get_db_prep_value(self, value, connection, prepared=False):+ """Convert Python value to database-specific value."""+ if not prepared:+ value = self.get_prep_value(value)+ return super().get_db_prep_value(value, connection, prepared)++ def value_to_string(self, obj):+ """Convert field value to string for serialization."""+ value = self.value_from_object(obj)+ if value is None:+ return None+ return json.dumps(value)++ def formfield(self, **kwargs):+ """Return a form field for this model field."""+ # Use a CharField for forms since we want JSON input+ from django import forms++ defaults = {"widget": forms.Textarea}+ defaults.update(kwargs)+ return forms.CharField(**defaults)+++class EncryptedBinaryFieldSerializer(serializers.Field):+ """Custom serializer field for EncryptedBinaryField.++ This field handles serialization/deserialization of encrypted binary data+ as JSON dictionaries, making it compatible with DRF's serialization process.+ """++ def to_representation(self, value):+ """Convert encrypted binary data to JSON dictionary for API responses."""+ if value is None:+ return None++ # The EncryptedBinaryField already decrypts the value when accessed+ # so we just need to return it as-is+ return value++ def to_internal_value(self, data):+ """Convert JSON dictionary to internal value for database storage."""+ if data is None:+ return None++ # Validate that the data is a dictionary+ if not isinstance(data, dict):+ raise serializers.ValidationError("Expected a dictionary.")++ # Return the data as-is; the EncryptedBinaryField will handle encryption+ return datadiff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex b83d86214..ab3c7fd4f 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -100,10 +100,6 @@ class DestinationConnector(BaseConnector):workflow=workflow,endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,)- if endpoint.connector_instance:- endpoint.connector_instance.connector_metadata = (- endpoint.connector_instance.metadata- )return endpointdef _get_source_endpoint_for_workflow(@@ -123,10 +119,6 @@ class DestinationConnector(BaseConnector):workflow=workflow,endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,)- if endpoint.connector_instance:- endpoint.connector_instance.connector_metadata = (- endpoint.connector_instance.metadata- )return endpointdef validate(self) -> None:@@ -149,7 +141,7 @@ class DestinationConnector(BaseConnector):# Get database class and test connectiondb_class = DatabaseUtils.get_db_class(connector_id=connector.connector_id,- connector_settings=connector.metadata,+ connector_settings=connector.connector_metadata,)engine = db_class.get_engine()if hasattr(engine, "close"):@@ -319,7 +311,7 @@ class DestinationConnector(BaseConnector):def insert_into_db(self, input_file_path: str) -> None:"""Insert data into the database."""connector_instance: ConnectorInstance = self.endpoint.connector_instance- connector_settings: dict[str, Any] = connector_instance.metadata+ connector_settings: dict[str, Any] = connector_instance.connector_metadatadestination_configurations: dict[str, Any] = self.endpoint.configurationtable_name: str = str(destination_configurations.get(DestinationKey.TABLE))include_agent: bool = bool(diff --git a/backend/workflow_manager/endpoint_v2/serializers.py b/backend/workflow_manager/endpoint_v2/serializers.pyindex 2ce3d4a8f..27990f408 100644--- a/backend/workflow_manager/endpoint_v2/serializers.py+++ b/backend/workflow_manager/endpoint_v2/serializers.py@@ -1,12 +1,36 @@import logging-from rest_framework.serializers import ModelSerializer+from connector_v2.models import ConnectorInstance+from connector_v2.serializers import ConnectorInstanceSerializer+from rest_framework import serializers+from rest_framework.serializers import (+ ModelSerializer,+)from workflow_manager.endpoint_v2.models import WorkflowEndpointlogger = logging.getLogger(__name__)class WorkflowEndpointSerializer(ModelSerializer):+ connector_instance = ConnectorInstanceSerializer(read_only=True)+ connector_instance_id = serializers.PrimaryKeyRelatedField(+ queryset=ConnectorInstance.objects.all(), # To not make DRF shout+ source="connector_instance",+ write_only=True,+ allow_null=True,+ )+ workflow_name = serializers.CharField(source="workflow.workflow_name", read_only=True)+class Meta:model = WorkflowEndpointfields = "__all__"++ def get_fields(self):+ """Override get_fields to dynamically set the connector_instance_id queryset.++ This is needed to ensure that the queryset is set after the organization+ context is available.+ """+ fields = super().get_fields()+ fields["connector_instance_id"].queryset = ConnectorInstance.objects.all()+ return fieldsdiff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex 4098963da..f1cc4d72b 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -118,10 +118,6 @@ class SourceConnector(BaseConnector):workflow=workflow,endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,)- if endpoint.connector_instance:- endpoint.connector_instance.connector_metadata = (- endpoint.connector_instance.metadata- )return endpointdef validate(self) -> None:diff --git a/backend/workflow_manager/endpoint_v2/urls.py b/backend/workflow_manager/endpoint_v2/urls.pyindex 522859e23..d4dec7c3b 100644--- a/backend/workflow_manager/endpoint_v2/urls.py+++ b/backend/workflow_manager/endpoint_v2/urls.py@@ -6,7 +6,7 @@ workflow_endpoint_list = WorkflowEndpointViewSet.as_view()endpoint_list = WorkflowEndpointViewSet.as_view({"get": "list"})workflow_endpoint_detail = WorkflowEndpointViewSet.as_view(- {"get": "retrieve", "put": "update"}+ {"get": "retrieve", "put": "update", "patch": "partial_update"})endpoint_settings_detail = WorkflowEndpointViewSet.as_view({"get": WorkflowEndpointViewSet.get_settings.__name__}diff --git a/backend/workflow_manager/endpoint_v2/views.py b/backend/workflow_manager/endpoint_v2/views.pyindex 1766cf450..986803e86 100644--- a/backend/workflow_manager/endpoint_v2/views.py+++ b/backend/workflow_manager/endpoint_v2/views.py@@ -6,8 +6,8 @@ from rest_framework.response import Responsefrom workflow_manager.endpoint_v2.destination import DestinationConnectorfrom workflow_manager.endpoint_v2.endpoint_utils import WorkflowEndpointUtilsfrom workflow_manager.endpoint_v2.models import WorkflowEndpoint+from workflow_manager.endpoint_v2.serializers import WorkflowEndpointSerializerfrom workflow_manager.endpoint_v2.source import SourceConnector-from workflow_manager.workflow_v2.serializers import WorkflowEndpointSerializerclass WorkflowEndpointViewSet(viewsets.ModelViewSet):@@ -19,10 +19,15 @@ class WorkflowEndpointViewSet(viewsets.ModelViewSet):.select_related("workflow").filter(workflow__created_by=self.request.user))+ workflow_filter = self.request.query_params.get("workflow", None)+ if workflow_filter:+ queryset = queryset.filter(workflow_id=workflow_filter)+endpoint_type_filter = self.request.query_params.get("endpoint_type", None)- connection_type_filter = self.request.query_params.get("connection_type", None)if endpoint_type_filter:queryset = queryset.filter(endpoint_type=endpoint_type_filter)++ connection_type_filter = self.request.query_params.get("connection_type", None)if connection_type_filter:queryset = queryset.filter(connection_type=connection_type_filter)return querysetdiff --git a/backend/workflow_manager/workflow_v2/serializers.py b/backend/workflow_manager/workflow_v2/serializers.pyindex b076b8895..1ec9a559a 100644--- a/backend/workflow_manager/workflow_v2/serializers.py+++ b/backend/workflow_manager/workflow_v2/serializers.py@@ -16,7 +16,6 @@ from utils.serializer.integrity_error_mixin import IntegrityErrorMixinfrom backend.constants import RequestKeyfrom backend.serializers import AuditSerializer-from workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.workflow_v2.constants import WorkflowExecutionKey, WorkflowKeyfrom workflow_manager.workflow_v2.models.execution import WorkflowExecutionfrom workflow_manager.workflow_v2.models.execution_log import ExecutionLog@@ -104,14 +103,6 @@ class ExecuteWorkflowResponseSerializer(Serializer):result = JSONField()-class WorkflowEndpointSerializer(ModelSerializer):- workflow_name = CharField(source="workflow.workflow_name", read_only=True)-- class Meta:- model = WorkflowEndpoint- fields = "__all__"--class WorkflowExecutionSerializer(ModelSerializer):class Meta:model = WorkflowExecutiondiff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.pyindex a0d60772c..01f750345 100644--- a/backend/workflow_manager/workflow_v2/views.py+++ b/backend/workflow_manager/workflow_v2/views.py@@ -100,10 +100,11 @@ class WorkflowViewSet(viewsets.ModelViewSet):is_active=True,)try:+ # Create empty WorkflowEndpoints for UI compatibility+ # ConnectorInstances will be created when users actually configure connectorsWorkflowEndpointUtils.create_endpoints_for_workflow(workflow)- # NOTE: Add default connector here if neededexcept Exception as e:- logger.error(f"Error saving workflow to DB: {e}")+ logger.error(f"Error creating workflow endpoints: {e}")raise WorkflowGenerationErrorreturn workflowdiff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsxindex d41064a6c..8c1e34b82 100644--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx@@ -23,6 +23,7 @@ import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";import { useAlertStore } from "../../../store/alert-store";import { useSessionStore } from "../../../store/session-store";import { useWorkflowStore } from "../../../store/workflow-store";+import useRequestUrl from "../../../hooks/useRequestUrl";import SpaceWrapper from "../../widgets/space-wrapper/SpaceWrapper";import { ConfigureConnectorModal } from "../configure-connector-modal/ConfigureConnectorModal";import { useExceptionHandler } from "../../../hooks/useExceptionHandler";@@ -82,6 +83,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {const axiosPrivate = useAxiosPrivate();const handleException = useExceptionHandler();const { flags } = sessionDetails;+ const { getUrl } = useRequestUrl();const icons = {input: <ImportOutlined className="ds-set-icon-size" />,@@ -144,16 +146,30 @@ function DsSettingsCard({ type, endpointDetails, message }) {setConnType(endpointDetails?.connection_type);}- if (!endpointDetails?.connector_instance?.length) {+ if (!endpointDetails?.connector_instance) {setConnDetails({});return;}- if (connDetails?.id === endpointDetails?.connector_instance) {+ // Use connector_instance data directly from endpointDetails if it's an object+ if (typeof endpointDetails?.connector_instance === "object") {+ const connectorData = endpointDetails.connector_instance;+ connectorData.connector_metadata = connectorData.connector_metadata || {};+ connectorData.connector_metadata.connectorName =+ connectorData?.connector_name || "";+ setConnDetails(connectorData);+ setSelectedId(connectorData?.connector_id);return;}- getSourceDetails();+ // Fallback for legacy connector_instance ID format (string)+ if (typeof endpointDetails?.connector_instance === "string") {+ // Only call getSourceDetails if we haven't already loaded this connector+ if (connDetails?.id !== endpointDetails?.connector_instance) {+ getSourceDetails();+ }+ return;+ }}, [endpointDetails]);useEffect(() => {@@ -187,7 +203,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {setFormDataConfig(endpointDetails.configuration || {});const requestOptions = {method: "GET",- url: `/api/v1/unstract/${sessionDetails?.orgId}/workflow/endpoint/${endpointDetails?.id}/settings/`,+ url: getUrl(`workflow/endpoint/${endpointDetails?.id}/settings/`),};setIsSpecConfigLoading(true);@@ -211,9 +227,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {const requestOptions = {method: "GET",- url: `/api/v1/unstract/${- sessionDetails?.orgId- }/supported_connectors/?type=${type.toUpperCase()}`,+ url: getUrl(`supported_connectors/?type=${type.toUpperCase()}`),};axiosPrivate(requestOptions)@@ -250,16 +264,14 @@ function DsSettingsCard({ type, endpointDetails, message }) {};const clearDestination = (updatedData) => {- const body = { ...destination, ...updatedData };-const requestOptions = {- method: "PUT",- url: `/api/v1/unstract/${sessionDetails?.orgId}/workflow/endpoint/${destination?.id}/`,+ method: "PATCH",+ url: getUrl(`workflow/endpoint/${destination?.id}/`),headers: {"X-CSRFToken": sessionDetails?.csrfToken,"Content-Type": "application/json",},- data: body,+ data: updatedData,};axiosPrivate(requestOptions)@@ -279,22 +291,20 @@ function DsSettingsCard({ type, endpointDetails, message }) {if (type === "input") {clearDestination({connection_type: "",- connector_instance: null,+ connector_instance_id: null,});}};const handleUpdate = (updatedData, showSuccess) => {- const body = { ...endpointDetails, ...updatedData };-const requestOptions = {- method: "PUT",- url: `/api/v1/unstract/${sessionDetails?.orgId}/workflow/endpoint/${endpointDetails?.id}/`,+ method: "PATCH",+ url: getUrl(`workflow/endpoint/${endpointDetails?.id}/`),headers: {"X-CSRFToken": sessionDetails?.csrfToken,"Content-Type": "application/json",},- data: body,+ data: updatedData,};axiosPrivate(requestOptions).then((res) => {@@ -321,7 +331,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {const getSourceDetails = () => {const requestOptions = {method: "GET",- url: `/api/v1/unstract/${sessionDetails?.orgId}/connector/${endpointDetails?.connector_instance}/`,+ url: getUrl(`connector/${endpointDetails?.connector_instance}/`),};axiosPrivate(requestOptions)@@ -362,7 +372,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {onChange={(value) => {handleUpdate({connection_type: value,- connector_instance: null,+ connector_instance_id: null,});updateDestination();}}diff --git a/frontend/src/components/api/prompt-studio-service.js b/frontend/src/components/api/prompt-studio-service.jsindex 407726fab..8a2a7c17c 100644--- a/frontend/src/components/api/prompt-studio-service.js+++ b/frontend/src/components/api/prompt-studio-service.js@@ -1,4 +1,5 @@import { useCallback } from "react";+import useRequestUrl from "../../hooks/useRequestUrl";import { useAxiosPrivate } from "../../hooks/useAxiosPrivate";diff --git a/frontend/src/components/deployments/create-api-deployment-from-prompt-studio/CreateApiDeploymentFromPromptStudio.jsx b/frontend/src/components/deployments/create-api-deployment-from-prompt-studio/CreateApiDeploymentFromPromptStudio.jsxindex dcb5bddf7..b65630f64 100644--- a/frontend/src/components/deployments/create-api-deployment-from-prompt-studio/CreateApiDeploymentFromPromptStudio.jsx+++ b/frontend/src/components/deployments/create-api-deployment-from-prompt-studio/CreateApiDeploymentFromPromptStudio.jsx@@ -460,14 +460,13 @@ const CreateApiDeploymentFromPromptStudio = ({// Update each endpoint to set connection_type to APIfor (const endpoint of endpoints) {await axiosPrivate({- method: "PUT",+ method: "PATCH",url: getUrl(`workflow/endpoint/${endpoint.id}/`),headers: {"X-CSRFToken": sessionDetails?.csrfToken,"Content-Type": "application/json",},data: {- ...endpoint,connection_type: "API",configuration: endpoint.configuration || {},},diff --git a/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx b/frontend/src/components/input-output/configure-ds/ConfigureDs.jsxindex 75ecb2e11..e4a527b4b 100644--- a/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx+++ b/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx@@ -184,14 +184,16 @@ function ConfigureDs({const connectorMetadata = { ...formData };const connectorName = connectorMetadata?.connectorName;delete connectorMetadata.connectorName;+body = {- workflow: id,- created_by: sessionDetails?.id,connector_id: selectedSourceId,connector_metadata: connectorMetadata,- connector_type: type.toUpperCase(),connector_name: connectorName,+ created_by: sessionDetails?.id,+ workflow: id,+ connector_type: type.toUpperCase(),};+url += "connector/";try {@@ -254,7 +256,10 @@ function ConfigureDs({const data = res?.data;if (sourceTypes.connectors.includes(type)) {handleUpdate(- { connector_instance: data?.id, configuration: formDataConfig },+ {+ connector_instance_id: data?.id,+ configuration: formDataConfig,+ },true);setIsTcSuccessful(false);diff --git a/frontend/src/components/input-output/input-output/InputOutput.jsx b/frontend/src/components/input-output/input-output/InputOutput.jsxindex a8059ce0f..95e37b97b 100644--- a/frontend/src/components/input-output/input-output/InputOutput.jsx+++ b/frontend/src/components/input-output/input-output/InputOutput.jsx@@ -6,6 +6,7 @@ import { CONNECTOR_TYPE_MAP } from "../../../helpers/GetStaticData.js";import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";import { useAlertStore } from "../../../store/alert-store";import { useSessionStore } from "../../../store/session-store";+import useRequestUrl from "../../../hooks/useRequestUrl";import { AddSourceModal } from "../add-source-modal/AddSourceModal.jsx";import { ManageFiles } from "../manage-files/ManageFiles.jsx";import { Sidebar } from "../sidebar/Sidebar.jsx";@@ -26,6 +27,7 @@ function InputOutput() {const { id } = useParams();const axiosPrivate = useAxiosPrivate();const handleException = useExceptionHandler();+ const { getUrl } = useRequestUrl();const location = useLocation();const currentPath = location.pathname;@@ -45,25 +47,31 @@ function InputOutput() {setConnectorType(type);+ const endpointType = CONNECTOR_TYPE_MAP[type]?.toUpperCase();+const requestOptions = {method: "GET",- url: `/api/v1/unstract/${- sessionDetails?.orgId- }/connector/?workflow=${id}&connector_type=${type.toUpperCase()}`,+ url: getUrl(+ `workflow/endpoint/?workflow=${id}&endpoint_type=${endpointType}`+ ),};axiosPrivate(requestOptions).then((res) => {- const sources = res?.data;- if (sources?.length === 0) {+ const endpoints = res?.data;+ if (endpoints?.length === 0) {setSelectedItem("");setListOfItems([]);return;}- const menuItems = sources.map((item) =>- getItem(item?.connector_name, item?.id, sourceIcon(item?.icon))+ const menuItems = endpoints.map((item) =>+ getItem(+ item?.connector_instance?.connector_name,+ item?.connector_instance?.id,+ sourceIcon(item?.connector_instance?.connector_icon)+ ));- const firstId = sources[0].id.toString();+ const firstId = endpoints[0]?.connector_instance?.id?.toString();setSelectedItem(firstId);setListOfItems(menuItems);})@@ -92,7 +100,7 @@ function InputOutput() {const handleDelete = () => {const requestOptions = {method: "DELETE",- url: `/api/v1/unstract/${sessionDetails?.orgId}/connector/${selectedItem}/`,+ url: getUrl(`connector/${selectedItem}/`),headers: {"X-CSRFToken": sessionDetails?.csrfToken,},diff --git a/frontend/src/hooks/usePostHogEvents.js b/frontend/src/hooks/usePostHogEvents.jsindex 33a5ada0e..e890ef29b 100644--- a/frontend/src/hooks/usePostHogEvents.js+++ b/frontend/src/hooks/usePostHogEvents.js@@ -1,4 +1,5 @@import { usePostHog } from "posthog-js/react";+import { useSessionStore } from "../store/session-store";const usePostHogEvents = () => {diff --git a/frontend/src/store/alert-store.js b/frontend/src/store/alert-store.jsindex 0e42186b5..95dec9b79 100644--- a/frontend/src/store/alert-store.js+++ b/frontend/src/store/alert-store.js@@ -1,7 +1,8 @@import { create } from "zustand";-import { isNonNegativeNumber } from "../helpers/GetStaticData";import { uniqueId } from "lodash";+import { isNonNegativeNumber } from "../helpers/GetStaticData";+const DEFAULT_DURATION = 6;const STORE_VARIABLES = {