LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Revert "UN-2585 [MISC] Deprecate connector instance fields and refactor endpoint handling" (#1449)
Chandrasekharan M5357aebff562
diff --git a/.gitignore b/.gitignoreindex 53b1a840e..b197d1623 100644--- a/.gitignore+++ b/.gitignore@@ -680,15 +680,7 @@ backend/backend/*_urls.pybackend/backend/*_urls_v2.py!backend/backend/public_urls_v2.py-## Vibe coding / Prompting-related files ##-prompting/-.prompting/--# Claude+# Adding claude prompting directoriesCLAUDE.md-.claude/*--# Windsurf-.qodo-.windsurfrules-.windsurf/rules+CONTRIBUTION_GUIDE.md+prompting/*diff --git a/backend/connector_v2/migrations/0002_alter_connectorinstance_connector_metadata_and_more.py b/backend/connector_v2/migrations/0002_alter_connectorinstance_connector_metadata_and_more.pydeleted file mode 100644index 267bcb87d..000000000--- a/backend/connector_v2/migrations/0002_alter_connectorinstance_connector_metadata_and_more.py+++ /dev/null@@ -1,37 +0,0 @@-# Generated by Django 4.2.1 on 2025-07-16 10:26--import django.db.models.deletion-import utils.fields-from django.db import migrations, models---class Migration(migrations.Migration):- dependencies = [- ("workflow_v2", "0015_executionlog_idx_wf_execution_event_time_and_more"),- ("connector_v2", "0001_initial"),- ]-- operations = [- migrations.AlterField(- model_name="connectorinstance",- name="connector_metadata",- field=utils.fields.EncryptedBinaryField(null=True),- ),- migrations.AlterField(- model_name="connectorinstance",- name="connector_type",- field=models.CharField(- choices=[("INPUT", "Input"), ("OUTPUT", "Output")], null=True- ),- ),- migrations.AlterField(- model_name="connectorinstance",- name="workflow",- field=models.ForeignKey(- null=True,- on_delete=django.db.models.deletion.CASCADE,- related_name="connector_workflow",- to="workflow_v2.workflow",- ),- ),- ]diff --git a/backend/connector_v2/models.py b/backend/connector_v2/models.pyindex d7276aa19..dbeaeb019 100644--- a/backend/connector_v2/models.py+++ b/backend/connector_v2/models.py@@ -1,4 +1,4 @@-import logging+import jsonimport uuidfrom typing import Any@@ -6,8 +6,10 @@ from account_v2.models import Userfrom connector_auth_v2.models import ConnectorAuthfrom connector_processor.connector_processor import ConnectorProcessorfrom connector_processor.constants import ConnectorKeys+from cryptography.fernet import Fernet, InvalidToken+from django.conf import settingsfrom django.db import models-from utils.fields import EncryptedBinaryField+from utils.exceptions import InvalidEncryptionKeyfrom utils.models.base_model import BaseModelfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,@@ -19,8 +21,6 @@ from backend.constants import FieldLengthConstants as FLCCONNECTOR_NAME_SIZE = 128VERSION_NAME_SIZE = 64-logger = logging.getLogger(__name__)-class ConnectorInstanceModelManager(DefaultOrganizationManagerMixin, models.Manager):pass@@ -45,13 +45,13 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):"workflow_v2.Workflow",on_delete=models.CASCADE,related_name="connector_workflow",- null=True,+ null=False,blank=False,)connector_id = models.CharField(max_length=FLC.CONNECTOR_ID_LENGTH, default="")- connector_metadata = EncryptedBinaryField(null=True)+ connector_metadata = models.BinaryField(null=True)connector_version = models.CharField(max_length=VERSION_NAME_SIZE, default="")- connector_type = models.CharField(choices=ConnectorType.choices, null=True)+ connector_type = models.CharField(choices=ConnectorType.choices)# TODO: handle connector_auth cascade deletionconnector_auth = models.ForeignKey(ConnectorAuth,@@ -84,6 +84,21 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):# Managerobjects = ConnectorInstanceModelManager()+ # TODO: Remove if unused+ def get_connector_metadata(self) -> dict[str, str]:+ """Gets connector metadata and refreshes the tokens if needed in case+ of OAuth.+ """+ tokens_refreshed = False+ if self.connector_auth:+ (+ self.connector_metadata,+ tokens_refreshed,+ ) = self.connector_auth.get_and_refresh_tokens()+ if tokens_refreshed:+ self.save()+ return self.connector_metadata+@staticmethoddef supportsOAuth(connector_id: str) -> bool:return bool(@@ -94,9 +109,19 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):def __str__(self) -> str:return (- f"Connector({self.id}, ID={self.connector_id}, mode: {self.connector_mode})"+ f"Connector({self.id}, type{self.connector_type}, workflow: {self.workflow})")+ def get_connector_metadata_bytes(self):+ """Convert connector_metadata to bytes if it is a memoryview.++ Returns:+ bytes: The connector_metadata as bytes.+ """+ if isinstance(self.connector_metadata, memoryview):+ return self.connector_metadata.tobytes()+ return self.connector_metadata+@propertydef metadata(self) -> Any:"""Decrypt and return the connector metadata as a dictionary.@@ -107,20 +132,21 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):Returns:dict: The decrypted connector metadata.-- .. deprecated::- This property is deprecated. Use `connector_metadata` field directly instead.- This property will be removed in a future version."""- import warnings-- warnings.warn(- "The 'metadata' property is deprecated. Use 'connector_metadata' field directly instead. "- "This property will be removed in a future version.",- DeprecationWarning,- stacklevel=2,- )- return self.connector_metadata+ try:+ connector_metadata_bytes = self.get_connector_metadata_bytes()++ if connector_metadata_bytes is None:+ return None++ if isinstance(connector_metadata_bytes, (dict)):+ return connector_metadata_bytes+ encryption_secret: str = settings.ENCRYPTION_KEY+ cipher_suite: Fernet = Fernet(encryption_secret.encode("utf-8"))+ decrypted_value = cipher_suite.decrypt(connector_metadata_bytes)+ except InvalidToken:+ raise InvalidEncryptionKey(entity=InvalidEncryptionKey.Entity.CONNECTOR)+ return json.loads(decrypted_value.decode("utf-8"))class Meta:db_table = "connector_instance"diff --git a/backend/connector_v2/serializers.py b/backend/connector_v2/serializers.pyindex c88598e34..2d274c190 100644--- a/backend/connector_v2/serializers.py+++ b/backend/connector_v2/serializers.py@@ -1,3 +1,4 @@+import jsonimport loggingfrom collections import OrderedDictfrom typing import Any@@ -7,8 +8,8 @@ from connector_auth_v2.pipeline.common import ConnectorAuthHelperfrom connector_processor.connector_processor import ConnectorProcessorfrom connector_processor.constants import ConnectorKeysfrom connector_processor.exceptions import OAuthTimeOut-from rest_framework.serializers import SerializerMethodField-from utils.fields import EncryptedBinaryFieldSerializer+from cryptography.fernet import Fernet+from django.conf import settingsfrom utils.serializer_utils import SerializerUtilsfrom backend.serializers import AuditSerializer@@ -21,13 +22,22 @@ logger = logging.getLogger(__name__)class ConnectorInstanceSerializer(AuditSerializer):- connector_metadata = EncryptedBinaryFieldSerializer(required=False, allow_null=True)- icon = SerializerMethodField()-class Meta:model = ConnectorInstancefields = "__all__"+ def validate_connector_metadata(self, value: dict[Any]) -> dict[Any]:+ """Validating Json metadata This custom validation is to avoid conflict+ with user input and db binary data.++ Args:+ value (Any): dict of metadata++ Returns:+ dict[Any]: dict of metadata+ """+ return value+def save(self, **kwargs): # type: ignoreuser = self.context.get("request").user or Noneconnector_id: str = kwargs[CIKey.CONNECTOR_ID]@@ -58,19 +68,15 @@ class ConnectorInstanceSerializer(AuditSerializer):)kwargs[CIKey.CONNECTOR_MODE] = connector_mode.value+ encryption_secret: str = settings.ENCRYPTION_KEY+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))+ json_string: str = json.dumps(kwargs.get(CIKey.CONNECTOR_METADATA))++ kwargs[CIKey.CONNECTOR_METADATA] = f.encrypt(json_string.encode("utf-8"))+instance = super().save(**kwargs)return instance- def get_icon(self, obj: ConnectorInstance) -> str:- """Get connector icon from ConnectorProcessor."""- icon_path = ConnectorProcessor.get_connector_data_with_key(- obj.connector_id, ConnectorKeys.ICON- )- # Ensure icon path is properly formatted for frontend- if icon_path and not icon_path.startswith("/"):- return f"/{icon_path}"- return icon_path-def to_representation(self, instance: ConnectorInstance) -> dict[str, str]:# to remove the sensitive fields being returnedrep: OrderedDict[str, Any] = super().to_representation(instance)@@ -78,6 +84,11 @@ class ConnectorInstanceSerializer(AuditSerializer):rep[CIKey.CONNECTOR_METADATA] = {}if SerializerUtils.check_context_for_GET_or_POST(context=self.context):rep.pop(CIKey.CONNECTOR_AUTH)- rep[ConnectorKeys.ICON] = self.get_icon(instance)+ # set icon fields for UI+ rep[ConnectorKeys.ICON] = ConnectorProcessor.get_connector_data_with_key(+ instance.connector_id, ConnectorKeys.ICON+ )+ if instance.connector_metadata:+ rep[CIKey.CONNECTOR_METADATA] = instance.metadatareturn repdiff --git a/backend/file_management/file_management_helper.py b/backend/file_management/file_management_helper.pyindex ab56c4dba..cd6ab2d61 100644--- a/backend/file_management/file_management_helper.py+++ b/backend/file_management/file_management_helper.py@@ -35,7 +35,7 @@ class FileManagerHelper:@staticmethoddef get_file_system(connector: ConnectorInstance) -> UnstractFileSystem:"""Creates the `UnstractFileSystem` for the corresponding connector."""- metadata = connector.connector_metadata+ metadata = connector.metadataif connector.connector_id in fs_connectors:connector = fs_connectors[connector.connector_id]["metadata"]["connector"]connector_class: UnstractFileSystem = connector(metadata)diff --git a/backend/pipeline_v2/serializers/crud.py b/backend/pipeline_v2/serializers/crud.pyindex ab4f9705b..d69f616fd 100644--- a/backend/pipeline_v2/serializers/crud.py+++ b/backend/pipeline_v2/serializers/crud.py@@ -224,42 +224,44 @@ class PipelineSerializer(IntegrityErrorMixin, AuditSerializer):def _add_connector_data(self,repr: OrderedDict[str, Any],- workflow_endpoints: list[Any],+ connector_instance_list: list[Any],connectors: list[Any],) -> OrderedDict[str, Any]:"""Adds connector Input/Output data.Args:- repr (OrderedDict[str, Any]): The representation dictionary- workflow_endpoints (list[Any]): List of WorkflowEndpoint objects- connectors (list[Any]): List of available connectors+ sef (_type_): _description_+ repr (OrderedDict[str, Any]): _description_Returns:- OrderedDict[str, Any]: Updated representation with connector data+ OrderedDict[str, Any]: _description_"""repr[PC.SOURCE_NAME] = PC.NOT_CONFIGUREDrepr[PC.DESTINATION_NAME] = PC.NOT_CONFIGURED-- for endpoint in workflow_endpoints:- if endpoint.endpoint_type == WorkflowEndpoint.EndpointType.SOURCE:- if endpoint.connector_instance:- repr[PC.SOURCE_NAME], repr[PC.SOURCE_ICON] = self._get_name_and_icon(+ for instance in connector_instance_list:+ if instance.connector_type == "INPUT":+ repr[PC.SOURCE_NAME], repr[PC.SOURCE_ICON] = self._get_name_and_icon(+ connectors=connectors,+ connector_id=instance.connector_id,+ )+ if instance.connector_type == "OUTPUT":+ repr[PC.DESTINATION_NAME], repr[PC.DESTINATION_ICON] = (+ self._get_name_and_icon(connectors=connectors,- connector_id=endpoint.connector_instance.connector_id,+ connector_id=instance.connector_id,)- elif endpoint.endpoint_type == WorkflowEndpoint.EndpointType.DESTINATION:- if (- endpoint.connection_type- == WorkflowEndpoint.ConnectionType.MANUALREVIEW- ):- repr[PC.DESTINATION_NAME] = "Manual Review"- elif endpoint.connector_instance:- repr[PC.DESTINATION_NAME], repr[PC.DESTINATION_ICON] = (- self._get_name_and_icon(- connectors=connectors,- connector_id=endpoint.connector_instance.connector_id,- )+ )+ if repr[PC.DESTINATION_NAME] == PC.NOT_CONFIGURED:+ try:+ check_manual_review = WorkflowEndpoint.objects.get(+ workflow=instance.workflow,+ endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,+ connection_type=WorkflowEndpoint.ConnectionType.MANUALREVIEW,)+ if check_manual_review:+ repr[PC.DESTINATION_NAME] = "Manual Review"+ except Exception as ex:+ logger.debug(f"Not a Manual review destination: {ex}")return reprdef to_representation(self, instance: Pipeline) -> OrderedDict[str, Any]:@@ -271,17 +273,15 @@ class PipelineSerializer(IntegrityErrorMixin, AuditSerializer):if SerializerUtils.check_context_for_GET_or_POST(context=self.context):workflow = instance.workflow- workflow_endpoints = (- WorkflowEndpoint.objects.filter(workflow=workflow.id)- .select_related("connector_instance")- .all()- )+ connector_instance_list = ConnectorInstance.objects.filter(+ workflow=workflow.id+ ).all()repr[PK.WORKFLOW_ID] = workflow.idrepr[PK.WORKFLOW_NAME] = workflow.workflow_namerepr[PK.CRON_STRING] = repr.pop(PK.CRON_STRING)repr = self._add_connector_data(repr=repr,- workflow_endpoints=workflow_endpoints,+ connector_instance_list=connector_instance_list,connectors=connectors,)diff --git a/backend/utils/fields.py b/backend/utils/fields.pydeleted file mode 100644index 70b257fce..000000000--- a/backend/utils/fields.py+++ /dev/null@@ -1,195 +0,0 @@-"""Custom Django model fields and serializer fields for common use cases."""--import json-import logging-from typing import Any--from cryptography.fernet import Fernet, InvalidToken-from django.conf import settings-from django.db import models-from rest_framework import serializers--from utils.exceptions import InvalidEncryptionKey--logger = logging.getLogger(__name__)---class EncryptedBinaryField(models.BinaryField):- """A BinaryField that automatically encrypts/decrypts JSON data.-- This field transparently handles encryption when storing data to the database- and decryption when retrieving data from the database. It expects dictionary- data as input and returns dictionary data as output.-- Features:- - Automatic encryption/decryption using Fernet symmetric encryption- - JSON serialization/deserialization- - Proper error handling for encryption failures- - Null value support- - Compatible with Django ORM and DRF serializers-- Usage:- class MyModel(models.Model):- encrypted_data = EncryptedBinaryField(null=True)-- # Usage is transparent- instance = MyModel.objects.create(encrypted_data={'key': 'value'})- print(instance.encrypted_data) # {'key': 'value'}- """-- description = "A binary field that automatically encrypts/decrypts JSON data"-- def __init__(self, *args, **kwargs):- # Remove any JSON-specific kwargs that might be passed- kwargs.pop("encoder", None)- kwargs.pop("decoder", None)- super().__init__(*args, **kwargs)-- def _get_encryption_key(self) -> bytes:- """Get the encryption key from Django settings."""- encryption_key = getattr(settings, "ENCRYPTION_KEY", None)- if not encryption_key:- raise ValueError("ENCRYPTION_KEY not found in Django settings")- return encryption_key.encode("utf-8")-- def _encrypt_value(self, value: Any) -> bytes:- """Encrypt a Python value (typically dict) to bytes."""- if value is None:- return None-- try:- # Serialize to JSON string- json_string = json.dumps(value)-- # Encrypt the JSON string- cipher_suite = Fernet(self._get_encryption_key())- encrypted_bytes = cipher_suite.encrypt(json_string.encode("utf-8"))-- return encrypted_bytes- except Exception as e:- logger.error(f"Failed to encrypt value: {e}")- raise-- def _decrypt_value(self, value: bytes) -> dict | None:- """Decrypt bytes to a Python value (typically dict)."""- if value is None:- return None-- try:- # Handle memoryview objects from database- if isinstance(value, memoryview):- value = value.tobytes()-- # If it's already a dict, return as-is (for backward compatibility)- if isinstance(value, dict):- return value-- # Decrypt the bytes- cipher_suite = Fernet(self._get_encryption_key())- decrypted_bytes = cipher_suite.decrypt(value)-- # Parse JSON string back to Python object- json_string = decrypted_bytes.decode("utf-8")- return json.loads(json_string)-- except InvalidToken:- logger.error("Invalid encryption token - possibly wrong encryption key")- raise InvalidEncryptionKey(entity=InvalidEncryptionKey.Entity.CONNECTOR)- except Exception as e:- logger.error(f"Failed to decrypt value: {e}")- raise-- def from_db_value(self, value, expression, connection):- """Convert database value to Python value.- Called when data is loaded from the database.- """- return self._decrypt_value(value)-- def to_python(self, value):- """Convert input value to Python value.- Called during model validation and form processing.- """- if value is None:- return value-- # If it's already a Python object (dict), return as-is- if isinstance(value, dict):- return value-- # If it's bytes (encrypted), decrypt it- if isinstance(value, (bytes, memoryview)):- return self._decrypt_value(value)-- # If it's a string, try to parse as JSON- if isinstance(value, str):- try:- return json.loads(value)- except json.JSONDecodeError:- pass-- return value-- def get_prep_value(self, value):- """Convert Python value to database value.- Called before saving to the database.- """- if value is None:- return None-- # If it's already bytes (encrypted), return as-is- if isinstance(value, (bytes, memoryview)):- return value-- # Encrypt the value- return self._encrypt_value(value)-- def get_db_prep_value(self, value, connection, prepared=False):- """Convert Python value to database-specific value."""- if not prepared:- value = self.get_prep_value(value)- return super().get_db_prep_value(value, connection, prepared)-- def value_to_string(self, obj):- """Convert field value to string for serialization."""- value = self.value_from_object(obj)- if value is None:- return None- return json.dumps(value)-- def formfield(self, **kwargs):- """Return a form field for this model field."""- # Use a CharField for forms since we want JSON input- from django import forms-- defaults = {"widget": forms.Textarea}- defaults.update(kwargs)- return forms.CharField(**defaults)---class EncryptedBinaryFieldSerializer(serializers.Field):- """Custom serializer field for EncryptedBinaryField.-- This field handles serialization/deserialization of encrypted binary data- as JSON dictionaries, making it compatible with DRF's serialization process.- """-- def to_representation(self, value):- """Convert encrypted binary data to JSON dictionary for API responses."""- if value is None:- return None-- # The EncryptedBinaryField already decrypts the value when accessed- # so we just need to return it as-is- return value-- def to_internal_value(self, data):- """Convert JSON dictionary to internal value for database storage."""- if data is None:- return None-- # Validate that the data is a dictionary- if not isinstance(data, dict):- raise serializers.ValidationError("Expected a dictionary.")-- # Return the data as-is; the EncryptedBinaryField will handle encryption- return datadiff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex 2e68f4682..0d22f0a40 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -100,6 +100,10 @@ class DestinationConnector(BaseConnector):workflow=workflow,endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,)+ if endpoint.connector_instance:+ endpoint.connector_instance.connector_metadata = (+ endpoint.connector_instance.metadata+ )return endpointdef _get_source_endpoint_for_workflow(@@ -119,6 +123,10 @@ class DestinationConnector(BaseConnector):workflow=workflow,endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,)+ if endpoint.connector_instance:+ endpoint.connector_instance.connector_metadata = (+ endpoint.connector_instance.metadata+ )return endpointdef validate(self) -> None:@@ -141,7 +149,7 @@ class DestinationConnector(BaseConnector):# Get database class and test connectiondb_class = DatabaseUtils.get_db_class(connector_id=connector.connector_id,- connector_settings=connector.connector_metadata,+ connector_settings=connector.metadata,)engine = db_class.get_engine()if hasattr(engine, "close"):@@ -310,7 +318,7 @@ class DestinationConnector(BaseConnector):def insert_into_db(self, input_file_path: str) -> None:"""Insert data into the database."""connector_instance: ConnectorInstance = self.endpoint.connector_instance- connector_settings: dict[str, Any] = connector_instance.connector_metadata+ connector_settings: dict[str, Any] = connector_instance.metadatadestination_configurations: dict[str, Any] = self.endpoint.configurationtable_name: str = str(destination_configurations.get(DestinationKey.TABLE))include_agent: bool = bool(diff --git a/backend/workflow_manager/endpoint_v2/serializers.py b/backend/workflow_manager/endpoint_v2/serializers.pyindex 7f01c4bc3..2ce3d4a8f 100644--- a/backend/workflow_manager/endpoint_v2/serializers.py+++ b/backend/workflow_manager/endpoint_v2/serializers.py@@ -1,25 +1,12 @@import logging-from connector_v2.models import ConnectorInstance-from connector_v2.serializers import ConnectorInstanceSerializer-from rest_framework import serializers-from rest_framework.serializers import (- ModelSerializer,-)+from rest_framework.serializers import ModelSerializerfrom workflow_manager.endpoint_v2.models import WorkflowEndpointlogger = logging.getLogger(__name__)class WorkflowEndpointSerializer(ModelSerializer):- connector_instance = ConnectorInstanceSerializer(read_only=True)- connector_instance_id = serializers.PrimaryKeyRelatedField(- queryset=ConnectorInstance.objects.all(),- source="connector_instance",- write_only=True,- allow_null=True,- )-class Meta:model = WorkflowEndpointfields = "__all__"diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex f1cc4d72b..4098963da 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -118,6 +118,10 @@ class SourceConnector(BaseConnector):workflow=workflow,endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,)+ if endpoint.connector_instance:+ endpoint.connector_instance.connector_metadata = (+ endpoint.connector_instance.metadata+ )return endpointdef validate(self) -> None:diff --git a/backend/workflow_manager/endpoint_v2/urls.py b/backend/workflow_manager/endpoint_v2/urls.pyindex d4dec7c3b..522859e23 100644--- a/backend/workflow_manager/endpoint_v2/urls.py+++ b/backend/workflow_manager/endpoint_v2/urls.py@@ -6,7 +6,7 @@ workflow_endpoint_list = WorkflowEndpointViewSet.as_view()endpoint_list = WorkflowEndpointViewSet.as_view({"get": "list"})workflow_endpoint_detail = WorkflowEndpointViewSet.as_view(- {"get": "retrieve", "put": "update", "patch": "partial_update"}+ {"get": "retrieve", "put": "update"})endpoint_settings_detail = WorkflowEndpointViewSet.as_view({"get": WorkflowEndpointViewSet.get_settings.__name__}diff --git a/backend/workflow_manager/endpoint_v2/views.py b/backend/workflow_manager/endpoint_v2/views.pyindex 986803e86..1766cf450 100644--- a/backend/workflow_manager/endpoint_v2/views.py+++ b/backend/workflow_manager/endpoint_v2/views.py@@ -6,8 +6,8 @@ from rest_framework.response import Responsefrom workflow_manager.endpoint_v2.destination import DestinationConnectorfrom workflow_manager.endpoint_v2.endpoint_utils import WorkflowEndpointUtilsfrom workflow_manager.endpoint_v2.models import WorkflowEndpoint-from workflow_manager.endpoint_v2.serializers import WorkflowEndpointSerializerfrom workflow_manager.endpoint_v2.source import SourceConnector+from workflow_manager.workflow_v2.serializers import WorkflowEndpointSerializerclass WorkflowEndpointViewSet(viewsets.ModelViewSet):@@ -19,15 +19,10 @@ class WorkflowEndpointViewSet(viewsets.ModelViewSet):.select_related("workflow").filter(workflow__created_by=self.request.user))- workflow_filter = self.request.query_params.get("workflow", None)- if workflow_filter:- queryset = queryset.filter(workflow_id=workflow_filter)-endpoint_type_filter = self.request.query_params.get("endpoint_type", None)+ connection_type_filter = self.request.query_params.get("connection_type", None)if endpoint_type_filter:queryset = queryset.filter(endpoint_type=endpoint_type_filter)-- connection_type_filter = self.request.query_params.get("connection_type", None)if connection_type_filter:queryset = queryset.filter(connection_type=connection_type_filter)return querysetdiff --git a/backend/workflow_manager/workflow_v2/serializers.py b/backend/workflow_manager/workflow_v2/serializers.pyindex 1ec9a559a..b076b8895 100644--- a/backend/workflow_manager/workflow_v2/serializers.py+++ b/backend/workflow_manager/workflow_v2/serializers.py@@ -16,6 +16,7 @@ from utils.serializer.integrity_error_mixin import IntegrityErrorMixinfrom backend.constants import RequestKeyfrom backend.serializers import AuditSerializer+from workflow_manager.endpoint_v2.models import WorkflowEndpointfrom workflow_manager.workflow_v2.constants import WorkflowExecutionKey, WorkflowKeyfrom workflow_manager.workflow_v2.models.execution import WorkflowExecutionfrom workflow_manager.workflow_v2.models.execution_log import ExecutionLog@@ -103,6 +104,14 @@ class ExecuteWorkflowResponseSerializer(Serializer):result = JSONField()+class WorkflowEndpointSerializer(ModelSerializer):+ workflow_name = CharField(source="workflow.workflow_name", read_only=True)++ class Meta:+ model = WorkflowEndpoint+ fields = "__all__"++class WorkflowExecutionSerializer(ModelSerializer):class Meta:model = WorkflowExecutiondiff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.pyindex 01f750345..a0d60772c 100644--- a/backend/workflow_manager/workflow_v2/views.py+++ b/backend/workflow_manager/workflow_v2/views.py@@ -100,11 +100,10 @@ class WorkflowViewSet(viewsets.ModelViewSet):is_active=True,)try:- # Create empty WorkflowEndpoints for UI compatibility- # ConnectorInstances will be created when users actually configure connectorsWorkflowEndpointUtils.create_endpoints_for_workflow(workflow)+ # NOTE: Add default connector here if neededexcept Exception as e:- logger.error(f"Error creating workflow endpoints: {e}")+ logger.error(f"Error saving workflow to DB: {e}")raise WorkflowGenerationErrorreturn workflowdiff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsxindex 046caeb9f..d41064a6c 100644--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx@@ -23,7 +23,6 @@ import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";import { useAlertStore } from "../../../store/alert-store";import { useSessionStore } from "../../../store/session-store";import { useWorkflowStore } from "../../../store/workflow-store";-import useRequestUrl from "../../../hooks/useRequestUrl";import SpaceWrapper from "../../widgets/space-wrapper/SpaceWrapper";import { ConfigureConnectorModal } from "../configure-connector-modal/ConfigureConnectorModal";import { useExceptionHandler } from "../../../hooks/useExceptionHandler";@@ -83,7 +82,6 @@ function DsSettingsCard({ type, endpointDetails, message }) {const axiosPrivate = useAxiosPrivate();const handleException = useExceptionHandler();const { flags } = sessionDetails;- const { getUrl } = useRequestUrl();const icons = {input: <ImportOutlined className="ds-set-icon-size" />,@@ -146,30 +144,16 @@ function DsSettingsCard({ type, endpointDetails, message }) {setConnType(endpointDetails?.connection_type);}- if (!endpointDetails?.connector_instance) {+ if (!endpointDetails?.connector_instance?.length) {setConnDetails({});return;}- // Use connector_instance data directly from endpointDetails if it's an object- if (typeof endpointDetails?.connector_instance === "object") {- const connectorData = endpointDetails.connector_instance;- connectorData.connector_metadata = connectorData.connector_metadata || {};- connectorData.connector_metadata.connectorName =- connectorData?.connector_name || "";- setConnDetails(connectorData);- setSelectedId(connectorData?.connector_id);+ if (connDetails?.id === endpointDetails?.connector_instance) {return;}- // Fallback for legacy connector_instance ID format (string)- if (typeof endpointDetails?.connector_instance === "string") {- // Only call getSourceDetails if we haven't already loaded this connector- if (connDetails?.id !== endpointDetails?.connector_instance) {- getSourceDetails();- }- return;- }+ getSourceDetails();}, [endpointDetails]);useEffect(() => {@@ -203,7 +187,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {setFormDataConfig(endpointDetails.configuration || {});const requestOptions = {method: "GET",- url: getUrl(`workflow/endpoint/${endpointDetails?.id}/settings/`),+ url: `/api/v1/unstract/${sessionDetails?.orgId}/workflow/endpoint/${endpointDetails?.id}/settings/`,};setIsSpecConfigLoading(true);@@ -227,7 +211,9 @@ function DsSettingsCard({ type, endpointDetails, message }) {const requestOptions = {method: "GET",- url: getUrl(`supported_connectors/?type=${type.toUpperCase()}`),+ url: `/api/v1/unstract/${+ sessionDetails?.orgId+ }/supported_connectors/?type=${type.toUpperCase()}`,};axiosPrivate(requestOptions)@@ -268,7 +254,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {const requestOptions = {method: "PUT",- url: getUrl(`workflow/endpoint/${destination?.id}/`),+ url: `/api/v1/unstract/${sessionDetails?.orgId}/workflow/endpoint/${destination?.id}/`,headers: {"X-CSRFToken": sessionDetails?.csrfToken,"Content-Type": "application/json",@@ -293,20 +279,22 @@ function DsSettingsCard({ type, endpointDetails, message }) {if (type === "input") {clearDestination({connection_type: "",- connector_instance_id: null,+ connector_instance: null,});}};const handleUpdate = (updatedData, showSuccess) => {+ const body = { ...endpointDetails, ...updatedData };+const requestOptions = {- method: "PATCH",- url: getUrl(`workflow/endpoint/${endpointDetails?.id}/`),+ method: "PUT",+ url: `/api/v1/unstract/${sessionDetails?.orgId}/workflow/endpoint/${endpointDetails?.id}/`,headers: {"X-CSRFToken": sessionDetails?.csrfToken,"Content-Type": "application/json",},- data: updatedData,+ data: body,};axiosPrivate(requestOptions).then((res) => {@@ -333,7 +321,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {const getSourceDetails = () => {const requestOptions = {method: "GET",- url: getUrl(`connector/${endpointDetails?.connector_instance}/`),+ url: `/api/v1/unstract/${sessionDetails?.orgId}/connector/${endpointDetails?.connector_instance}/`,};axiosPrivate(requestOptions)@@ -374,7 +362,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {onChange={(value) => {handleUpdate({connection_type: value,- connector_instance_id: null,+ connector_instance: null,});updateDestination();}}diff --git a/frontend/src/components/api/prompt-studio-service.js b/frontend/src/components/api/prompt-studio-service.jsindex 8a2a7c17c..407726fab 100644--- a/frontend/src/components/api/prompt-studio-service.js+++ b/frontend/src/components/api/prompt-studio-service.js@@ -1,5 +1,4 @@import { useCallback } from "react";-import useRequestUrl from "../../hooks/useRequestUrl";import { useAxiosPrivate } from "../../hooks/useAxiosPrivate";diff --git a/frontend/src/components/deployments/create-api-deployment-from-prompt-studio/CreateApiDeploymentFromPromptStudio.jsx b/frontend/src/components/deployments/create-api-deployment-from-prompt-studio/CreateApiDeploymentFromPromptStudio.jsxindex b65630f64..dcb5bddf7 100644--- a/frontend/src/components/deployments/create-api-deployment-from-prompt-studio/CreateApiDeploymentFromPromptStudio.jsx+++ b/frontend/src/components/deployments/create-api-deployment-from-prompt-studio/CreateApiDeploymentFromPromptStudio.jsx@@ -460,13 +460,14 @@ const CreateApiDeploymentFromPromptStudio = ({// Update each endpoint to set connection_type to APIfor (const endpoint of endpoints) {await axiosPrivate({- method: "PATCH",+ method: "PUT",url: getUrl(`workflow/endpoint/${endpoint.id}/`),headers: {"X-CSRFToken": sessionDetails?.csrfToken,"Content-Type": "application/json",},data: {+ ...endpoint,connection_type: "API",configuration: endpoint.configuration || {},},diff --git a/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx b/frontend/src/components/input-output/configure-ds/ConfigureDs.jsxindex e4a527b4b..75ecb2e11 100644--- a/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx+++ b/frontend/src/components/input-output/configure-ds/ConfigureDs.jsx@@ -184,16 +184,14 @@ function ConfigureDs({const connectorMetadata = { ...formData };const connectorName = connectorMetadata?.connectorName;delete connectorMetadata.connectorName;-body = {+ workflow: id,+ created_by: sessionDetails?.id,connector_id: selectedSourceId,connector_metadata: connectorMetadata,- connector_name: connectorName,- created_by: sessionDetails?.id,- workflow: id,connector_type: type.toUpperCase(),+ connector_name: connectorName,};-url += "connector/";try {@@ -256,10 +254,7 @@ function ConfigureDs({const data = res?.data;if (sourceTypes.connectors.includes(type)) {handleUpdate(- {- connector_instance_id: data?.id,- configuration: formDataConfig,- },+ { connector_instance: data?.id, configuration: formDataConfig },true);setIsTcSuccessful(false);diff --git a/frontend/src/components/input-output/input-output/InputOutput.jsx b/frontend/src/components/input-output/input-output/InputOutput.jsxindex 95e37b97b..a8059ce0f 100644--- a/frontend/src/components/input-output/input-output/InputOutput.jsx+++ b/frontend/src/components/input-output/input-output/InputOutput.jsx@@ -6,7 +6,6 @@ import { CONNECTOR_TYPE_MAP } from "../../../helpers/GetStaticData.js";import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";import { useAlertStore } from "../../../store/alert-store";import { useSessionStore } from "../../../store/session-store";-import useRequestUrl from "../../../hooks/useRequestUrl";import { AddSourceModal } from "../add-source-modal/AddSourceModal.jsx";import { ManageFiles } from "../manage-files/ManageFiles.jsx";import { Sidebar } from "../sidebar/Sidebar.jsx";@@ -27,7 +26,6 @@ function InputOutput() {const { id } = useParams();const axiosPrivate = useAxiosPrivate();const handleException = useExceptionHandler();- const { getUrl } = useRequestUrl();const location = useLocation();const currentPath = location.pathname;@@ -47,31 +45,25 @@ function InputOutput() {setConnectorType(type);- const endpointType = CONNECTOR_TYPE_MAP[type]?.toUpperCase();-const requestOptions = {method: "GET",- url: getUrl(- `workflow/endpoint/?workflow=${id}&endpoint_type=${endpointType}`- ),+ url: `/api/v1/unstract/${+ sessionDetails?.orgId+ }/connector/?workflow=${id}&connector_type=${type.toUpperCase()}`,};axiosPrivate(requestOptions).then((res) => {- const endpoints = res?.data;- if (endpoints?.length === 0) {+ const sources = res?.data;+ if (sources?.length === 0) {setSelectedItem("");setListOfItems([]);return;}- const menuItems = endpoints.map((item) =>- getItem(- item?.connector_instance?.connector_name,- item?.connector_instance?.id,- sourceIcon(item?.connector_instance?.connector_icon)- )+ const menuItems = sources.map((item) =>+ getItem(item?.connector_name, item?.id, sourceIcon(item?.icon)));- const firstId = endpoints[0]?.connector_instance?.id?.toString();+ const firstId = sources[0].id.toString();setSelectedItem(firstId);setListOfItems(menuItems);})@@ -100,7 +92,7 @@ function InputOutput() {const handleDelete = () => {const requestOptions = {method: "DELETE",- url: getUrl(`connector/${selectedItem}/`),+ url: `/api/v1/unstract/${sessionDetails?.orgId}/connector/${selectedItem}/`,headers: {"X-CSRFToken": sessionDetails?.csrfToken,},diff --git a/frontend/src/hooks/usePostHogEvents.js b/frontend/src/hooks/usePostHogEvents.jsindex e890ef29b..33a5ada0e 100644--- a/frontend/src/hooks/usePostHogEvents.js+++ b/frontend/src/hooks/usePostHogEvents.js@@ -1,5 +1,4 @@import { usePostHog } from "posthog-js/react";-import { useSessionStore } from "../store/session-store";const usePostHogEvents = () => {diff --git a/frontend/src/store/alert-store.js b/frontend/src/store/alert-store.jsindex 95dec9b79..0e42186b5 100644--- a/frontend/src/store/alert-store.js+++ b/frontend/src/store/alert-store.js@@ -1,7 +1,6 @@import { create } from "zustand";-import { uniqueId } from "lodash";-import { isNonNegativeNumber } from "../helpers/GetStaticData";+import { uniqueId } from "lodash";const DEFAULT_DURATION = 6;