Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

v2 changes of pipeline and api (#467)

alif057d3b60c99
diff --git a/backend/api_v2/__init__.py b/backend/api_v2/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/api_v2/admin.py b/backend/api_v2/admin.py
new file mode 100644
index 000000000..37f0837a7
--- /dev/null
+++ b/backend/api_v2/admin.py
@@ -0,0 +1,5 @@
+from django.contrib import admin
+
+from .models import APIDeployment, APIKey
+
+admin.site.register([APIDeployment, APIKey])
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py
new file mode 100644
index 000000000..802d8f8bc
--- /dev/null
+++ b/backend/api_v2/api_deployment_views.py
@@ -0,0 +1,152 @@
+import json
+import logging
+from typing import Any, Optional
+
+from api_v2.constants import ApiExecution
+from api_v2.deployment_helper import DeploymentHelper
+from api_v2.exceptions import InvalidAPIRequest, NoActiveAPIKeyError
+from api_v2.models import APIDeployment
+from api_v2.postman_collection.dto import PostmanCollection
+from api_v2.serializers import (
+ APIDeploymentListSerializer,
+ APIDeploymentSerializer,
+ DeploymentResponseSerializer,
+ ExecutionRequestSerializer,
+)
+from django.db.models import QuerySet
+from django.http import HttpResponse
+from permissions.permission import IsOwner
+from rest_framework import serializers, status, views, viewsets
+from rest_framework.decorators import action
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.serializers import Serializer
+from utils.enums import CeleryTaskState
+from workflow_manager.workflow_v2.dto import ExecutionResponse
+
+logger = logging.getLogger(__name__)
+
+
+class DeploymentExecution(views.APIView):
+ def initialize_request(
+ self, request: Request, *args: Any, **kwargs: Any
+ ) -> Request:
+ """To remove csrf request for public API.
+
+ Args:
+ request (Request): _description_
+
+ Returns:
+ Request: _description_
+ """
+ setattr(request, "csrf_processing_done", True)
+ return super().initialize_request(request, *args, **kwargs)
+
+ @DeploymentHelper.validate_api_key
+ def post(
+ self, request: Request, org_name: str, api_name: str, api: APIDeployment
+ ) -> Response:
+ file_objs = request.FILES.getlist(ApiExecution.FILES_FORM_DATA)
+ serializer = ExecutionRequestSerializer(data=request.data)
+ serializer.is_valid(raise_exception=True)
+ timeout = serializer.get_timeout(serializer.validated_data)
+ include_metadata = (
+ request.data.get(ApiExecution.INCLUDE_METADATA, "false").lower() == "true"
+ )
+ if not file_objs or len(file_objs) == 0:
+ raise InvalidAPIRequest("File shouldn't be empty")
+ response = DeploymentHelper.execute_workflow(
+ organization_name=org_name,
+ api=api,
+ file_objs=file_objs,
+ timeout=timeout,
+ include_metadata=include_metadata,
+ )
+ if "error" in response and response["error"]:
+ return Response(
+ {"message": response},
+ status=status.HTTP_422_UNPROCESSABLE_ENTITY,
+ )
+ return Response({"message": response}, status=status.HTTP_200_OK)
+
+ @DeploymentHelper.validate_api_key
+ def get(
+ self, request: Request, org_name: str, api_name: str, api: APIDeployment
+ ) -> Response:
+ execution_id = request.query_params.get("execution_id")
+ if not execution_id:
+ raise InvalidAPIRequest("execution_id shouldn't be empty")
+ response: ExecutionResponse = DeploymentHelper.get_execution_status(
+ execution_id=execution_id
+ )
+ if response.execution_status != CeleryTaskState.SUCCESS.value:
+ return Response(
+ {
+ "status": response.execution_status,
+ "message": response.result,
+ },
+ status=status.HTTP_422_UNPROCESSABLE_ENTITY,
+ )
+ return Response(
+ {"status": response.execution_status, "message": response.result},
+ status=status.HTTP_200_OK,
+ )
+
+
+class APIDeploymentViewSet(viewsets.ModelViewSet):
+ permission_classes = [IsOwner]
+
+ def get_queryset(self) -> Optional[QuerySet]:
+ return APIDeployment.objects.filter(created_by=self.request.user)
+
+ def get_serializer_class(self) -> serializers.Serializer:
+ if self.action in ["list"]:
+ return APIDeploymentListSerializer
+ return APIDeploymentSerializer
+
+ @action(detail=True, methods=["get"])
+ def fetch_one(self, request: Request, pk: Optional[str] = None) -> Response:
+ """Custom action to fetch a single instance."""
+ instance = self.get_object()
+ serializer = self.get_serializer(instance)
+ return Response(serializer.data)
+
+ def create(
+ self, request: Request, *args: tuple[Any], **kwargs: dict[str, Any]
+ ) -> Response:
+ serializer: Serializer = self.get_serializer(data=request.data)
+ serializer.is_valid(raise_exception=True)
+ self.perform_create(serializer)
+ api_key = DeploymentHelper.create_api_key(serializer=serializer)
+ response_serializer = DeploymentResponseSerializer(
+ {"api_key": api_key.api_key, **serializer.data}
+ )
+
+ headers = self.get_success_headers(serializer.data)
+ return Response(
+ response_serializer.data,
+ status=status.HTTP_201_CREATED,
+ headers=headers,
+ )
+
+ @action(detail=True, methods=["get"])
+ def download_postman_collection(
+ self, request: Request, pk: Optional[str] = None
+ ) -> Response:
+ """Downloads a Postman Collection of the API deployment instance."""
+ instance = self.get_object()
+ api_key_inst = instance.apikey_set.filter(is_active=True).first()
+ if not api_key_inst:
+ logger.error(f"No active API key set for deployment {instance.pk}")
+ raise NoActiveAPIKeyError(deployment_name=instance.display_name)
+
+ postman_collection = PostmanCollection.create(
+ instance=instance, api_key=api_key_inst.api_key
+ )
+ response = HttpResponse(
+ json.dumps(postman_collection.to_dict()), content_type="application/json"
+ )
+ response["Content-Disposition"] = (
+ f'attachment; filename="{instance.display_name}.json"'
+ )
+ return response
diff --git a/backend/api_v2/api_key_views.py b/backend/api_v2/api_key_views.py
new file mode 100644
index 000000000..906127c40
--- /dev/null
+++ b/backend/api_v2/api_key_views.py
@@ -0,0 +1,28 @@
+from api_v2.deployment_helper import DeploymentHelper
+from api_v2.exceptions import APINotFound
+from api_v2.key_helper import KeyHelper
+from api_v2.models import APIKey
+from api_v2.serializers import APIKeyListSerializer, APIKeySerializer
+from rest_framework import serializers, viewsets
+from rest_framework.decorators import action
+from rest_framework.request import Request
+from rest_framework.response import Response
+
+
+class APIKeyViewSet(viewsets.ModelViewSet):
+ queryset = APIKey.objects.all()
+
+ def get_serializer_class(self) -> serializers.Serializer:
+ if self.action in ["api_keys"]:
+ return APIKeyListSerializer
+ return APIKeySerializer
+
+ @action(detail=True, methods=["get"])
+ def api_keys(self, request: Request, api_id: str) -> Response:
+ """Custom action to fetch api keys of an api deployment."""
+ api = DeploymentHelper.get_api_by_id(api_id=api_id)
+ if not api:
+ raise APINotFound()
+ keys = KeyHelper.list_api_keys_of_api(api_instance=api)
+ serializer = self.get_serializer(keys, many=True)
+ return Response(serializer.data)
diff --git a/backend/api_v2/apps.py b/backend/api_v2/apps.py
new file mode 100644
index 000000000..16cbd4ad5
--- /dev/null
+++ b/backend/api_v2/apps.py
@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class ApiConfig(AppConfig):
+ name = "api_v2"
diff --git a/backend/api_v2/constants.py b/backend/api_v2/constants.py
new file mode 100644
index 000000000..0ec324cc9
--- /dev/null
+++ b/backend/api_v2/constants.py
@@ -0,0 +1,6 @@
+class ApiExecution:
+ PATH: str = "deployment/api"
+ MAXIMUM_TIMEOUT_IN_SEC: int = 300 # 5 minutes
+ FILES_FORM_DATA: str = "files"
+ TIMEOUT_FORM_DATA: str = "timeout"
+ INCLUDE_METADATA: str = "include_metadata"
diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.py
new file mode 100644
index 000000000..e94c84e46
--- /dev/null
+++ b/backend/api_v2/deployment_helper.py
@@ -0,0 +1,244 @@
+import logging
+import uuid
+from functools import wraps
+from typing import Any, Optional
+from urllib.parse import urlencode
+
+from api_v2.constants import ApiExecution
+from api_v2.exceptions import (
+ ApiKeyCreateException,
+ APINotFound,
+ Forbidden,
+ InactiveAPI,
+ UnauthorizedKey,
+)
+from api_v2.key_helper import KeyHelper
+from api_v2.models import APIDeployment, APIKey
+from api_v2.serializers import APIExecutionResponseSerializer
+from django.core.files.uploadedfile import UploadedFile
+from django.db import connection
+from rest_framework import status
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.serializers import Serializer
+from rest_framework.utils.serializer_helpers import ReturnDict
+from utils.constants import Account
+from utils.local_context import StateStore
+from workflow_manager.endpoint_v2.destination import DestinationConnector
+from workflow_manager.endpoint_v2.source import SourceConnector
+from workflow_manager.workflow_v2.dto import ExecutionResponse
+from workflow_manager.workflow_v2.models.workflow import Workflow
+from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper
+
+logger = logging.getLogger(__name__)
+
+
+class DeploymentHelper:
+ @staticmethod
+ def validate_api_key(func: Any) -> Any:
+ """Decorator that validates the API key.
+
+ Sample header:
+ Authorization: Bearer 123e4567-e89b-12d3-a456-426614174001
+ Args:
+ func (Any): Function to wrap for validation
+ """
+
+ @wraps(func)
+ def wrapper(self: Any, request: Request, *args: Any, **kwargs: Any) -> Any:
+ """Wrapper to validate the inputs and key.
+
+ Args:
+ request (Request): Request context
+
+ Raises:
+ Forbidden: _description_
+ APINotFound: _description_
+
+ Returns:
+ Any: _description_
+ """
+ try:
+ authorization_header = request.headers.get("Authorization")
+ api_key = None
+ if authorization_header and authorization_header.startswith("Bearer "):
+ api_key = authorization_header.split(" ")[1]
+ if not api_key:
+ raise Forbidden("Missing api key")
+ org_name = kwargs.get("org_name") or request.data.get("org_name")
+ api_name = kwargs.get("api_name") or request.data.get("api_name")
+ if not api_name:
+ raise Forbidden("Missing api_name")
+ # Set organization in state store for API
+ StateStore.set(Account.ORGANIZATION_ID, org_name)
+
+ api_deployment = DeploymentHelper.get_deployment_by_api_name(
+ api_name=api_name
+ )
+ DeploymentHelper.validate_api(
+ api_deployment=api_deployment, api_key=api_key
+ )
+ kwargs["api"] = api_deployment
+ return func(self, request, *args, **kwargs)
+
+ except (UnauthorizedKey, InactiveAPI, APINotFound):
+ raise
+ except Exception as exception:
+ logger.error(f"Exception: {exception}")
+ return Response(
+ {"error": str(exception)}, status=status.HTTP_403_FORBIDDEN
+ )
+
+ return wrapper
+
+ @staticmethod
+ def validate_api(api_deployment: Optional[APIDeployment], api_key: str) -> None:
+ """Validating API and API key.
+
+ Args:
+ api_deployment (Optional[APIDeployment]): _description_
+ api_key (str): _description_
+
+ Raises:
+ APINotFound: _description_
+ InactiveAPI: _description_
+ """
+ if not api_deployment:
+ raise APINotFound()
+ if not api_deployment.is_active:
+ raise InactiveAPI()
+ KeyHelper.validate_api_key(api_key=api_key, api_instance=api_deployment)
+
+ @staticmethod
+ def validate_and_get_workflow(workflow_id: str) -> Workflow:
+ """Validate that the specified workflow_id exists in the Workflow
+ model."""
+ return WorkflowHelper.get_workflow_by_id(workflow_id)
+
+ @staticmethod
+ def get_api_by_id(api_id: str) -> Optional[APIDeployment]:
+ try:
+ api_deployment: APIDeployment = APIDeployment.objects.get(pk=api_id)
+ return api_deployment
+ except APIDeployment.DoesNotExist:
+ return None
+
+ @staticmethod
+ def construct_complete_endpoint(api_name: str) -> str:
+ """Constructs the complete API endpoint by appending organization
+ schema, endpoint path, and Django app backend URL.
+
+ Parameters:
+ - endpoint (str): The endpoint path to be appended to the complete URL.
+
+ Returns:
+ - str: The complete API endpoint URL.
+ """
+ org_schema = connection.get_tenant().schema_name
+ return f"{ApiExecution.PATH}/{org_schema}/{api_name}/"
+
+ @staticmethod
+ def construct_status_endpoint(api_endpoint: str, execution_id: str) -> str:
+ """Construct a complete status endpoint URL by appending the
+ execution_id as a query parameter.
+
+ Args:
+ api_endpoint (str): The base API endpoint.
+ execution_id (str): The execution ID to be included as
+ a query parameter.
+
+ Returns:
+ str: The complete status endpoint URL.
+ """
+ query_parameters = urlencode({"execution_id": execution_id})
+ complete_endpoint = f"/{api_endpoint}?{query_parameters}"
+ return complete_endpoint
+
+ @staticmethod
+ def get_deployment_by_api_name(
+ api_name: str,
+ ) -> Optional[APIDeployment]:
+ """Get and return the APIDeployment object by api_name."""
+ try:
+ api: APIDeployment = APIDeployment.objects.get(api_name=api_name)
+ return api
+ except APIDeployment.DoesNotExist:
+ return None
+
+ @staticmethod
+ def create_api_key(serializer: Serializer) -> APIKey:
+ """To make API key for an API.
+
+ Args:
+ serializer (Serializer): Request serializer
+
+ Raises:
+ ApiKeyCreateException: Exception
+ """
+ api_deployment: APIDeployment = serializer.instance
+ try:
+ api_key: APIKey = KeyHelper.create_api_key(api_deployment)
+ return api_key
+ except Exception as error:
+ logger.error(f"Error while creating API key error: {str(error)}")
+ api_deployment.delete()
+ logger.info("Deleted the deployment instance")
+ raise ApiKeyCreateException()
+
+ @staticmethod
+ def execute_workflow(
+ organization_name: str,
+ api: APIDeployment,
+ file_objs: list[UploadedFile],
+ timeout: int,
+ ) -> ReturnDict:
+ """Execute workflow by api.
+
+ Args:
+ organization_name (str): organization name
+ api (APIDeployment): api model object
+ file_obj (UploadedFile): input file
+
+ Returns:
+ ReturnDict: execution status/ result
+ """
+ workflow_id = api.workflow.id
+ pipeline_id = api.id
+ execution_id = str(uuid.uuid4())
+ hash_values_of_files = SourceConnector.add_input_file_to_api_storage(
+ workflow_id=workflow_id,
+ execution_id=execution_id,
+ file_objs=file_objs,
+ )
+ try:
+ result = WorkflowHelper.execute_workflow_async(
+ workflow_id=workflow_id,
+ pipeline_id=pipeline_id,
+ hash_values_of_files=hash_values_of_files,
+ timeout=timeout,
+ execution_id=execution_id,
+ )
+ result.status_api = DeploymentHelper.construct_status_endpoint(
+ api_endpoint=api.api_endpoint, execution_id=execution_id
+ )
+ except Exception:
+ DestinationConnector.delete_api_storage_dir(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ raise
+ return APIExecutionResponseSerializer(result).data
+
+ @staticmethod
+ def get_execution_status(execution_id: str) -> ExecutionResponse:
+ """Current status of api execution.
+
+ Args:
+ execution_id (str): execution id
+
+ Returns:
+ ReturnDict: status/result of execution
+ """
+ execution_response: ExecutionResponse = WorkflowHelper.get_status_of_async_task(
+ execution_id=execution_id
+ )
+ return execution_response
diff --git a/backend/api_v2/exceptions.py b/backend/api_v2/exceptions.py
new file mode 100644
index 000000000..c3a58ff05
--- /dev/null
+++ b/backend/api_v2/exceptions.py
@@ -0,0 +1,55 @@
+from typing import Optional
+
+from rest_framework.exceptions import APIException
+
+
+class MandatoryWorkflowId(APIException):
+ status_code = 400
+ default_detail = "Workflow ID is mandatory"
+
+
+class ApiKeyCreateException(APIException):
+ status_code = 500
+ default_detail = "Exception while create API key"
+
+
+class Forbidden(APIException):
+ status_code = 403
+ default_detail = (
+ "User is forbidden from performing this action. Please contact admin"
+ )
+
+
+class APINotFound(APIException):
+ status_code = 404
+ default_detail = "API not found"
+
+
+class InvalidAPIRequest(APIException):
+ status_code = 400
+ default_detail = "Bad request"
+
+
+class InactiveAPI(APIException):
+ status_code = 404
+ default_detail = "API not found or Inactive"
+
+
+class UnauthorizedKey(APIException):
+ status_code = 401
+ default_detail = "Unauthorized"
+
+
+class NoActiveAPIKeyError(APIException):
+ status_code = 409
+ default_detail = "No active API keys configured for this deployment"
+
+ def __init__(
+ self,
+ detail: Optional[str] = None,
+ code: Optional[str] = None,
+ deployment_name: str = "this deployment",
+ ):
+ if detail is None:
+ detail = f"No active API keys configured for {deployment_name}"
+ super().__init__(detail, code)
diff --git a/backend/api_v2/key_helper.py b/backend/api_v2/key_helper.py
new file mode 100644
index 000000000..2a1a430a9
--- /dev/null
+++ b/backend/api_v2/key_helper.py
@@ -0,0 +1,72 @@
+import logging
+
+from api_v2.exceptions import Forbidden, UnauthorizedKey
+from api_v2.models import APIDeployment, APIKey
+from api_v2.serializers import APIKeySerializer
+from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper
+
+logger = logging.getLogger(__name__)
+
+
+class KeyHelper:
+ @staticmethod
+ def validate_api_key(api_key: str, api_instance: APIDeployment) -> None:
+ """Validate api key.
+
+ Args:
+ api_key (str): api key from request
+ api_instance (APIDeployment): api deployment instance
+
+ Raises:
+ Forbidden: _description_
+ """
+ try:
+ api_key_instance: APIKey = APIKey.objects.get(api_key=api_key)
+ if not KeyHelper.has_access(api_key_instance, api_instance):
+ raise UnauthorizedKey()
+ except APIKey.DoesNotExist:
+ raise UnauthorizedKey()
+ except APIDeployment.DoesNotExist:
+ raise Forbidden("API not found.")
+
+ @staticmethod
+ def list_api_keys_of_api(api_instance: APIDeployment) -> list[APIKey]:
+ api_keys: list[APIKey] = APIKey.objects.filter(api=api_instance).all()
+ return api_keys
+
+ @staticmethod
+ def has_access(api_key: APIKey, api_instance: APIDeployment) -> bool:
+ """Check if the provided API key has access to the specified API
+ instance.
+
+ Args:
+ api_key (APIKey): api key associated with the api
+ api_instance (APIDeployment): api model
+
+ Returns:
+ bool: True if allowed to execute, False otherwise
+ """
+ if not api_key.is_active:
+ return False
+ if isinstance(api_key.api, APIDeployment):
+ return api_key.api == api_instance
+ return False
+
+ @staticmethod
+ def validate_workflow_exists(workflow_id: str) -> None:
+ """Validate that the specified workflow_id exists in the Workflow
+ model."""
+ WorkflowHelper.get_workflow_by_id(workflow_id)
+
+ @staticmethod
+ def create_api_key(deployment: APIDeployment) -> APIKey:
+ """Create an APIKey entity with the data from the provided
+ APIDeployment instance."""
+ # Create an instance of the APIKey model
+ api_key_serializer = APIKeySerializer(
+ data={"api": deployment.id, "description": "Initial Access Key"},
+ context={"deployment": deployment},
+ )
+ api_key_serializer.is_valid(raise_exception=True)
+ api_key: APIKey = api_key_serializer.save()
+ return api_key
diff --git a/backend/api_v2/models.py b/backend/api_v2/models.py
new file mode 100644
index 000000000..545426537
--- /dev/null
+++ b/backend/api_v2/models.py
@@ -0,0 +1,167 @@
+import uuid
+from typing import Any
+
+from account_v2.models import User
+from api_v2.constants import ApiExecution
+from django.db import models
+from utils.models.base_model import BaseModel
+from utils.models.organization_mixin import (
+ DefaultOrganizationManagerMixin,
+ DefaultOrganizationMixin,
+)
+from utils.user_context import UserContext
+from workflow_manager.workflow_v2.models.workflow import Workflow
+
+API_NAME_MAX_LENGTH = 30
+DESCRIPTION_MAX_LENGTH = 255
+API_ENDPOINT_MAX_LENGTH = 255
+
+
+class APIDeploymentModelManager(DefaultOrganizationManagerMixin, models.Manager):
+ pass
+
+
+class APIDeployment(DefaultOrganizationMixin, BaseModel):
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ display_name = models.CharField(
+ max_length=API_NAME_MAX_LENGTH,
+ default="default api",
+ db_comment="User-given display name for the API.",
+ )
+ description = models.CharField(
+ max_length=DESCRIPTION_MAX_LENGTH,
+ blank=True,
+ default="",
+ db_comment="User-given description for the API.",
+ )
+ workflow = models.ForeignKey(
+ Workflow,
+ on_delete=models.CASCADE,
+ db_comment="Foreign key reference to the Workflow model.",
+ related_name="apis",
+ )
+ is_active = models.BooleanField(
+ default=True,
+ db_comment="Flag indicating whether the API is active or not.",
+ )
+ api_endpoint = models.CharField(
+ max_length=API_ENDPOINT_MAX_LENGTH,
+ unique=True,
+ editable=False,
+ db_comment="URL endpoint for the API deployment.",
+ )
+ api_name = models.CharField(
+ max_length=API_NAME_MAX_LENGTH,
+ default=uuid.uuid4,
+ db_comment="Short name for the API deployment.",
+ )
+ created_by = models.ForeignKey(
+ User,
+ on_delete=models.SET_NULL,
+ related_name="apis_created",
+ null=True,
+ blank=True,
+ editable=False,
+ )
+ modified_by = models.ForeignKey(
+ User,
+ on_delete=models.SET_NULL,
+ related_name="apis_modified",
+ null=True,
+ blank=True,
+ editable=False,
+ )
+
+ # Manager
+ objects = APIDeploymentModelManager()
+
+ def __str__(self) -> str:
+ return f"{self.id} - {self.display_name}"
+
+ def save(self, *args: Any, **kwargs: Any) -> None:
+ """Save hook to update api_endpoint.
+
+ Custom save hook for updating the 'api_endpoint' based on
+ 'api_name'. If the instance is being updated, it checks for
+ changes in 'api_name' and adjusts 'api_endpoint'
+ accordingly. If the instance is new, 'api_endpoint' is set
+ based on 'api_name' and the current database schema.
+ """
+ if self.pk is not None:
+ organization_id = UserContext.get_organization_identifier()
+ try:
+ original = APIDeployment.objects.get(pk=self.pk)
+ if original.api_name != self.api_name:
+ self.api_endpoint = (
+ f"{ApiExecution.PATH}/{organization_id}/{self.api_name}/"
+ )
+ except APIDeployment.DoesNotExist:
+ self.api_endpoint = (
+ f"{ApiExecution.PATH}/{organization_id}/{self.api_name}/"
+ )
+ super().save(*args, **kwargs)
+
+ class Meta:
+ verbose_name = "Api Deployment"
+ verbose_name_plural = "Api Deployments"
+ db_table = "api_deployment_v2"
+ constraints = [
+ models.UniqueConstraint(
+ fields=["api_name", "organization"],
+ name="unique_api_name",
+ ),
+ ]
+
+
+class APIKey(BaseModel):
+ id = models.UUIDField(
+ primary_key=True,
+ editable=False,
+ default=uuid.uuid4,
+ db_comment="Unique identifier for the API key.",
+ )
+ api_key = models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ unique=True,
+ db_comment="Actual key UUID.",
+ )
+ api = models.ForeignKey(
+ APIDeployment,
+ on_delete=models.CASCADE,
+ db_comment="Foreign key reference to the APIDeployment model.",
+ related_name="api_keys",
+ )
+ description = models.CharField(
+ max_length=DESCRIPTION_MAX_LENGTH,
+ null=True,
+ db_comment="Description of the API key.",
+ )
+ is_active = models.BooleanField(
+ default=True,
+ db_comment="Flag indicating whether the API key is active or not.",
+ )
+ created_by = models.ForeignKey(
+ User,
+ on_delete=models.SET_NULL,
+ related_name="api_keys_created",
+ null=True,
+ blank=True,
+ editable=False,
+ )
+ modified_by = models.ForeignKey(
+ User,
+ on_delete=models.SET_NULL,
+ related_name="api_keys_modified",
+ null=True,
+ blank=True,
+ editable=False,
+ )
+
+ def __str__(self) -> str:
+ return f"{self.api.api_name} - {self.id} - {self.api_key}"
+
+ class Meta:
+ verbose_name = "Api Deployment key"
+ verbose_name_plural = "Api Deployment keys"
+ db_table = "api_deployment_key_v2"
diff --git a/backend/api_v2/postman_collection/__init__.py b/backend/api_v2/postman_collection/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/api_v2/postman_collection/constants.py b/backend/api_v2/postman_collection/constants.py
new file mode 100644
index 000000000..ffb3ef729
--- /dev/null
+++ b/backend/api_v2/postman_collection/constants.py
@@ -0,0 +1,6 @@
+class CollectionKey:
+ POSTMAN_COLLECTION_V210 = "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" # noqa: E501
+ EXECUTE_API_KEY = "Process document"
+ STATUS_API_KEY = "Execution status"
+ STATUS_EXEC_ID_DEFAULT = "REPLACE_WITH_EXECUTION_ID"
+ AUTH_QUERY_PARAM_DEFAULT = "REPLACE_WITH_API_KEY"
diff --git a/backend/api_v2/postman_collection/dto.py b/backend/api_v2/postman_collection/dto.py
new file mode 100644
index 000000000..08f2294f1
--- /dev/null
+++ b/backend/api_v2/postman_collection/dto.py
@@ -0,0 +1,139 @@
+from dataclasses import asdict, dataclass, field
+from typing import Any, Optional
+from urllib.parse import urlencode, urljoin
+
+from api_v2.constants import ApiExecution
+from api_v2.models import APIDeployment
+from api_v2.postman_collection.constants import CollectionKey
+from django.conf import settings
+from utils.request import HTTPMethod
+
+
+@dataclass
+class HeaderItem:
+ key: str
+ value: str
+
+
+@dataclass
+class FormDataItem:
+ key: str
+ type: str
+ src: Optional[str] = None
+ value: Optional[str] = None
+
+ def __post_init__(self) -> None:
+ if self.type == "file":
+ if self.src is None:
+ raise ValueError("src must be provided for type 'file'")
+ elif self.type == "text":
+ if self.value is None:
+ raise ValueError("value must be provided for type 'text'")
+ else:
+ raise ValueError(f"Unsupported type for form data: {self.type}")
+
+
+@dataclass
+class BodyItem:
+ formdata: list[FormDataItem]
+ mode: str = "formdata"
+
+
+@dataclass
+class RequestItem:
+ method: HTTPMethod
+ url: str
+ header: list[HeaderItem]
+ body: Optional[BodyItem] = None
+
+
+@dataclass
+class PostmanItem:
+ name: str
+ request: RequestItem
+
+
+@dataclass
+class PostmanInfo:
+ name: str = "Unstract's API deployment"
+ schema: str = CollectionKey.POSTMAN_COLLECTION_V210
+ description: str = "Contains APIs meant for using the deployed Unstract API"
+
+
+@dataclass
+class PostmanCollection:
+ info: PostmanInfo
+ item: list[PostmanItem] = field(default_factory=list)
+
+ @classmethod
+ def create(
+ cls,
+ instance: APIDeployment,
+ api_key: str = CollectionKey.AUTH_QUERY_PARAM_DEFAULT,
+ ) -> "PostmanCollection":
+ """Creates a PostmanCollection instance.
+
+ This instance can help represent Postman collections (v2 format) that
+ can be used to easily invoke workflows deployed as APIs
+
+ Args:
+ instance (APIDeployment): API deployment to generate collection for
+ api_key (str, optional): Active API key used to authenticate requests for
+ deployed APIs. Defaults to CollectionKey.AUTH_QUERY_PARAM_DEFAULT.
+
+ Returns:
+ PostmanCollection: Instance representing PostmanCollection
+ """
+ postman_info = PostmanInfo(
+ name=instance.display_name, description=instance.description
+ )
+ header_list = [HeaderItem(key="Authorization", value=f"Bearer {api_key}")]
+ abs_api_endpoint = urljoin(settings.WEB_APP_ORIGIN_URL, instance.api_endpoint)
+
+ # API execution API
+ execute_body = BodyItem(
+ formdata=[
+ FormDataItem(
+ key=ApiExecution.FILES_FORM_DATA, type="file", src="/path_to_file"
+ ),
+ FormDataItem(
+ key=ApiExecution.TIMEOUT_FORM_DATA,
+ type="text",
+ value=ApiExecution.MAXIMUM_TIMEOUT_IN_SEC,
+ ),
+ FormDataItem(
+ key=ApiExecution.INCLUDE_METADATA,
+ type="text",
+ value=False,
+ ),
+ ]
+ )
+ execute_request = RequestItem(
+ method=HTTPMethod.POST,
+ header=header_list,
+ body=execute_body,
+ url=abs_api_endpoint,
+ )
+
+ # Status API
+ status_query_param = {"execution_id": CollectionKey.STATUS_EXEC_ID_DEFAULT}
+ status_query_str = urlencode(status_query_param)
+ status_url = urljoin(abs_api_endpoint, "?" + status_query_str)
+ status_request = RequestItem(
+ method=HTTPMethod.GET, header=header_list, url=status_url
+ )
+
+ postman_item_list = [
+ PostmanItem(name=CollectionKey.EXECUTE_API_KEY, request=execute_request),
+ PostmanItem(name=CollectionKey.STATUS_API_KEY, request=status_request),
+ ]
+ return cls(info=postman_info, item=postman_item_list)
+
+ def to_dict(self) -> dict[str, Any]:
+ """Convert PostmanCollection instance to a dict.
+
+ Returns:
+ dict[str, Any]: PostmanCollection as a dict
+ """
+ collection_dict = asdict(self)
+ return collection_dict
diff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.py
new file mode 100644
index 000000000..522e5a23e
--- /dev/null
+++ b/backend/api_v2/serializers.py
@@ -0,0 +1,123 @@
+from collections import OrderedDict
+from typing import Any, Union
+
+from api_v2.constants import ApiExecution
+from api_v2.models import APIDeployment, APIKey
+from django.core.validators import RegexValidator
+from rest_framework.serializers import (
+ CharField,
+ IntegerField,
+ JSONField,
+ ModelSerializer,
+ Serializer,
+ ValidationError,
+)
+
+from backend.serializers import AuditSerializer
+
+
+class APIDeploymentSerializer(AuditSerializer):
+ class Meta:
+ model = APIDeployment
+ fields = "__all__"
+
+ def validate_api_name(self, value: str) -> str:
+ api_name_validator = RegexValidator(
+ regex=r"^[a-zA-Z0-9_-]+$",
+ message="Only letters, numbers, hyphen and \
+ underscores are allowed.",
+ code="invalid_api_name",
+ )
+ api_name_validator(value)
+ return value
+
+
+class APIKeySerializer(AuditSerializer):
+ class Meta:
+ model = APIKey
+ fields = "__all__"
+
+ def to_representation(self, instance: APIKey) -> OrderedDict[str, Any]:
+ """Override the to_representation method to include additional
+ context."""
+ context = self.context.get("context", {})
+ deployment: APIDeployment = context.get("deployment")
+ representation: OrderedDict[str, Any] = super().to_representation(instance)
+ if deployment:
+ representation["api"] = deployment.id
+ representation["description"] = f"API Key for {deployment.name}"
+ representation["is_active"] = True
+
+ return representation
+
+
+class ExecutionRequestSerializer(Serializer):
+ """Execution request serializer
+ timeout: 0: maximum value of timeout, -1: async execution
+ """
+
+ timeout = IntegerField(
+ min_value=-1, max_value=ApiExecution.MAXIMUM_TIMEOUT_IN_SEC, default=-1
+ )
+
+ def validate_timeout(self, value: Any) -> int:
+ if not isinstance(value, int):
+ raise ValidationError("timeout must be a integer.")
+ if value == 0:
+ value = ApiExecution.MAXIMUM_TIMEOUT_IN_SEC
+ return value
+
+ def get_timeout(self, validated_data: dict[str, Union[int, None]]) -> int:
+ value = validated_data.get(ApiExecution.TIMEOUT_FORM_DATA, -1)
+ if not isinstance(value, int):
+ raise ValidationError("timeout must be a integer.")
+ return value
+
+
+class APIDeploymentListSerializer(ModelSerializer):
+ workflow_name = CharField(source="workflow.workflow_name", read_only=True)
+
+ class Meta:
+ model = APIDeployment
+ fields = [
+ "id",
+ "workflow",
+ "workflow_name",
+ "display_name",
+ "description",
+ "is_active",
+ "api_endpoint",
+ "api_name",
+ "created_by",
+ ]
+
+
+class APIKeyListSerializer(ModelSerializer):
+ class Meta:
+ model = APIKey
+ fields = [
+ "id",
+ "created_at",
+ "modified_at",
+ "api_key",
+ "is_active",
+ "description",
+ "api",
+ ]
+
+
+class DeploymentResponseSerializer(Serializer):
+ is_active = CharField()
+ id = CharField()
+ api_key = CharField()
+ api_endpoint = CharField()
+ display_name = CharField()
+ description = CharField()
+ api_name = CharField()
+
+
+class APIExecutionResponseSerializer(Serializer):
+ execution_status = CharField()
+ status_api = CharField()
+ error = CharField()
+ result = JSONField()
diff --git a/backend/api_v2/urls.py b/backend/api_v2/urls.py
new file mode 100644
index 000000000..065617d66
--- /dev/null
+++ b/backend/api_v2/urls.py
@@ -0,0 +1,63 @@
+from api_v2.api_deployment_views import APIDeploymentViewSet, DeploymentExecution
+from api_v2.api_key_views import APIKeyViewSet
+from django.urls import path, re_path
+from rest_framework.urlpatterns import format_suffix_patterns
+
+deployment = APIDeploymentViewSet.as_view(
+ {
+ "get": APIDeploymentViewSet.list.__name__,
+ "post": APIDeploymentViewSet.create.__name__,
+ }
+)
+deployment_details = APIDeploymentViewSet.as_view(
+ {
+ "get": APIDeploymentViewSet.retrieve.__name__,
+ "put": APIDeploymentViewSet.update.__name__,
+ "patch": APIDeploymentViewSet.partial_update.__name__,
+ "delete": APIDeploymentViewSet.destroy.__name__,
+ }
+)
+download_postman_collection = APIDeploymentViewSet.as_view(
+ {
+ "get": APIDeploymentViewSet.download_postman_collection.__name__,
+ }
+)
+
+execute = DeploymentExecution.as_view()
+
+key_details = APIKeyViewSet.as_view(
+ {
+ "get": APIKeyViewSet.retrieve.__name__,
+ "put": APIKeyViewSet.update.__name__,
+ "delete": APIKeyViewSet.destroy.__name__,
+ }
+)
+api_key = APIKeyViewSet.as_view(
+ {
+ "get": APIKeyViewSet.api_keys.__name__,
+ "post": APIKeyViewSet.create.__name__,
+ }
+)
+
+urlpatterns = format_suffix_patterns(
+ [
+ path("deployment/", deployment, name="api_deployment"),
+ path(
+ "deployment/<uuid:pk>/",
+ deployment_details,
+ name="api_deployment_details",
+ ),
+ path(
+ "postman_collection/<uuid:pk>/",
+ download_postman_collection,
+ name="download_postman_collection",
+ ),
+ re_path(
+ r"^api/(?P<org_name>[\w-]+)/(?P<api_name>[\w-]+)/?$",
+ execute,
+ name="api_deployment_execution",
+ ),
+ path("keys/<uuid:pk>/", key_details, name="key_details"),
+ path("keys/api/<str:api_id>/", api_key, name="api_key"),
+ ]
+)
diff --git a/backend/pipeline_v2/__init__.py b/backend/pipeline_v2/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/pipeline_v2/constants.py b/backend/pipeline_v2/constants.py
new file mode 100644
index 000000000..2475ad79f
--- /dev/null
+++ b/backend/pipeline_v2/constants.py
@@ -0,0 +1,66 @@
+class PipelineConstants:
+ """Constants for Pipelines."""
+
+ TYPE = "type"
+ ETL_PIPELINE = "ETL"
+ TASK_PIPELINE = "TASK"
+ ETL = "etl"
+ TASK = "task"
+ CREATE_ACTION = "create"
+ UPDATE_ACTION = "update"
+ PIPELINE_GUID = "id"
+ ACTION = "action"
+ NOT_CONFIGURED = "Connector not configured."
+ SOURCE_NOT_CONFIGURED = "Source not configured."
+ DESTINATION_NOT_CONFIGURED = "Destination not configured."
+ SOURCE_ICON = "source_icon"
+ DESTINATION_ICON = "destination_icon"
+ SOURCE_NAME = "source_name"
+ DESTINATION_NAME = "destination_name"
+ INPUT_FILE = "input_file_connector"
+ INPUT_DB = "input_db_connector"
+ OUTPUT_FILE = "output_file_connector"
+ OUTPUT_DB = "output_db_connector"
+ SOURCE = "source"
+ DEST = "dest"
+
+
+class PipelineExecutionKey:
+ PIPELINE = "pipeline"
+ EXECUTION = "execution"
+
+
+class PipelineKey:
+ """Constants for the Pipeline model."""
+
+ PIPELINE_GUID = "id"
+ PIPELINE_NAME = "pipeline_name"
+ WORKFLOW = "workflow"
+ APP_ID = "app_id"
+ ACTIVE = "active"
+ SCHEDULED = "scheduled"
+ PIPELINE_TYPE = "pipeline_type"
+ RUN_COUNT = "run_count"
+ LAST_RUN_TIME = "last_run_time"
+ LAST_RUN_STATUS = "last_run_status"
+ # Used by serializer
+ CRON_DATA = "cron_data"
+ WORKFLOW_NAME = "workflow_name"
+ WORKFLOW_ID = "workflow_id"
+ CRON_STRING = "cron_string"
+ PIPELINE_ID = "pipeline_id"
+
+
+class PipelineErrors:
+ PIPELINE_EXISTS = "Pipeline with this configuration might already exist or some mandatory field is missing." # noqa: E501
+ DUPLICATE_API = "It appears that a duplicate call may have been made."
+ INVALID_WF = "The provided workflow does not exist"
+
+
+class PipelineURL:
+ """Constants for URL names."""
+
+ DETAIL = "pipeline-detail"
+ EXECUTIONS = "pipeline-executions"
+ LIST = "pipeline-list"
+ EXECUTE = "tenant:pipeline-execute"
diff --git a/backend/pipeline_v2/exceptions.py b/backend/pipeline_v2/exceptions.py
new file mode 100644
index 000000000..8da26c481
--- /dev/null
+++ b/backend/pipeline_v2/exceptions.py
@@ -0,0 +1,46 @@
+from typing import Optional
+
+from rest_framework.exceptions import APIException
+
+
+class WorkflowTriggerError(APIException):
+ status_code = 400
+ default_detail = "Error triggering workflow. Pipeline created"
+
+
+class PipelineExecuteError(APIException):
+ status_code = 500
+ default_detail = "Error executing pipline"
+
+
+class InactivePipelineError(APIException):
+ status_code = 422
+ default_detail = "Pipeline is inactive, please activate the pipeline"
+
+ def __init__(
+ self,
+ pipeline_name: Optional[str] = None,
+ detail: Optional[str] = None,
+ code: Optional[str] = None,
+ ):
+ if pipeline_name:
+ self.default_detail = (
+ f"Pipeline '{pipeline_name}' is inactive, "
+ "please activate the pipeline"
+ )
+ super().__init__(detail, code)
+
+
+class MandatoryPipelineType(APIException):
+ status_code = 400
+ default_detail = "Pipeline type is mandatory"
+
+
+class MandatoryWorkflowId(APIException):
+ status_code = 400
+ default_detail = "Workflow ID is mandatory"
+
+
+class MandatoryCronSchedule(APIException):
+ status_code = 400
+ default_detail = "Cron schedule is mandatory"
diff --git a/backend/pipeline_v2/execution_view.py b/backend/pipeline_v2/execution_view.py
new file mode 100644
index 000000000..fa514615a
--- /dev/null
+++ b/backend/pipeline_v2/execution_view.py
@@ -0,0 +1,35 @@
+from permissions.permission import IsOwner
+from pipeline_v2.serializers.execute import DateRangeSerializer
+from rest_framework import viewsets
+from rest_framework.versioning import URLPathVersioning
+from utils.pagination import CustomPagination
+from workflow_manager.workflow_v2.models.execution import WorkflowExecution
+from workflow_manager.workflow_v2.serializers import WorkflowExecutionSerializer
+
+
+class PipelineExecutionViewSet(viewsets.ModelViewSet):
+ versioning_class = URLPathVersioning
+ permission_classes = [IsOwner]
+ serializer_class = WorkflowExecutionSerializer
+ pagination_class = CustomPagination
+
+ CREATED_AT_FIELD_DESC = "-created_at"
+ START_DATE_FIELD = "start_date"
+ END_DATE_FIELD = "end_date"
+
+ def get_queryset(self):
+ # Get the pipeline_id from the URL path
+ pipeline_id = self.kwargs.get("pk")
+ queryset = WorkflowExecution.objects.filter(pipeline_id=pipeline_id)
+
+ # Validate start_date and end_date parameters using DateRangeSerializer
+ date_range_serializer = DateRangeSerializer(data=self.request.query_params)
+ date_range_serializer.is_valid(raise_exception=True)
+ start_date = date_range_serializer.validated_data.get(self.START_DATE_FIELD)
+ end_date = date_range_serializer.validated_data.get(self.END_DATE_FIELD)
+
+ if start_date and end_date:
+ queryset = queryset.filter(created_at__range=(start_date, end_date))
+
+ queryset = queryset.order_by(self.CREATED_AT_FIELD_DESC)
+ return queryset
diff --git a/backend/pipeline_v2/manager.py b/backend/pipeline_v2/manager.py
new file mode 100644
index 000000000..d5297107d
--- /dev/null
+++ b/backend/pipeline_v2/manager.py
@@ -0,0 +1,59 @@
+import logging
+from typing import Any, Optional
+
+from django.conf import settings
+from django.urls import reverse
+from pipeline_v2.constants import PipelineKey, PipelineURL
+from pipeline_v2.models import Pipeline
+from pipeline_v2.pipeline_processor import PipelineProcessor
+from rest_framework.request import Request
+from rest_framework.response import Response
+from utils.request.constants import RequestConstants
+from workflow_manager.workflow_v2.constants import WorkflowExecutionKey, WorkflowKey
+from workflow_manager.workflow_v2.views import WorkflowViewSet
+
+from backend.constants import RequestHeader
+
+logger = logging.getLogger(__name__)
+
+
+class PipelineManager:
+ """Helps manage the execution and scheduling of pipelines."""
+
+ @staticmethod
+ def execute_pipeline(
+ request: Request,
+ pipeline_id: str,
+ execution_id: Optional[str] = None,
+ ) -> Response:
+ """Used to execute a pipeline.
+
+ Args:
+ pipeline_id (str): UUID of the pipeline to execute
+ execution_id (Optional[str], optional):
+ Uniquely identifies an execution. Defaults to None.
+ """
+ logger.info(f"Executing pipeline {pipeline_id}, execution: {execution_id}")
+ pipeline: Pipeline = PipelineProcessor.initialize_pipeline_sync(pipeline_id)
+ # TODO: Use DRF's request and as_view() instead
+ request.data[WorkflowKey.WF_ID] = pipeline.workflow.id
+ if execution_id is not None:
+ request.data[WorkflowExecutionKey.EXECUTION_ID] = execution_id
+ wf_viewset = WorkflowViewSet()
+ return wf_viewset.execute(request=request, pipeline_guid=str(pipeline.pk))
+
+ @staticmethod
+ def get_pipeline_execution_data_for_scheduled_run(
+ pipeline_id: str,
+ ) -> Optional[dict[str, Any]]:
+ """Gets the required data to be passed while executing a pipeline Any
+ changes to pipeline execution needs to be propagated here."""
+ callback_url = settings.DJANGO_APP_BACKEND_URL + reverse(PipelineURL.EXECUTE)
+ job_headers = {RequestHeader.X_API_KEY: settings.INTERNAL_SERVICE_API_KEY}
+ job_kwargs = {
+ RequestConstants.VERB: "POST",
+ RequestConstants.URL: callback_url,
+ RequestConstants.HEADERS: job_headers,
+ RequestConstants.DATA: {PipelineKey.PIPELINE_ID: pipeline_id},
+ }
+ return job_kwargs
diff --git a/backend/pipeline_v2/models.py b/backend/pipeline_v2/models.py
new file mode 100644
index 000000000..35ef1914c
--- /dev/null
+++ b/backend/pipeline_v2/models.py
@@ -0,0 +1,111 @@
+import uuid
+
+from account_v2.models import User
+from django.db import models
+from utils.models.base_model import BaseModel
+from utils.models.organization_mixin import (
+ DefaultOrganizationManagerMixin,
+ DefaultOrganizationMixin,
+)
+from workflow_manager.workflow_v2.models.workflow import Workflow
+
+from backend.constants import FieldLengthConstants as FieldLength
+
+APP_ID_LENGTH = 32
+PIPELINE_NAME_LENGTH = 32
+
+
+class PipelineModelManager(DefaultOrganizationManagerMixin, models.Manager):
+ pass
+
+
+class Pipeline(DefaultOrganizationMixin, BaseModel):
+ """Model to hold data related to Pipelines."""
+
+ class PipelineType(models.TextChoices):
+ ETL = "ETL", "ETL"
+ TASK = "TASK", "TASK"
+ DEFAULT = "DEFAULT", "Default"
+ APP = "APP", "App"
+
+ class PipelineStatus(models.TextChoices):
+ SUCCESS = "SUCCESS", "Success"
+ FAILURE = "FAILURE", "Failure"
+ INPROGRESS = "INPROGRESS", "Inprogress"
+ YET_TO_START = "YET_TO_START", "Yet to start"
+ RESTARTING = "RESTARTING", "Restarting"
+ PAUSED = "PAUSED", "Paused"
+
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ pipeline_name = models.CharField(max_length=PIPELINE_NAME_LENGTH, default="")
+ workflow = models.ForeignKey(
+ Workflow,
+ on_delete=models.CASCADE,
+ related_name="pipelines",
+ null=False,
+ blank=False,
+ )
+ # Added as text field until a model for App is included.
+ app_id = models.TextField(null=True, blank=True, max_length=APP_ID_LENGTH)
+ active = models.BooleanField(default=False) # TODO: Add dbcomment
+ scheduled = models.BooleanField(default=False) # TODO: Add dbcomment
+ cron_string = models.TextField(
+ db_comment="UNIX cron string",
+ null=False,
+ blank=False,
+ max_length=FieldLength.CRON_LENGTH,
+ )
+ pipeline_type = models.CharField(
+ choices=PipelineType.choices, default=PipelineType.DEFAULT
+ )
+ run_count = models.IntegerField(default=0)
+ last_run_time = models.DateTimeField(null=True, blank=True)
+ last_run_status = models.CharField(
+ choices=PipelineStatus.choices, default=PipelineStatus.YET_TO_START
+ )
+ app_icon = models.URLField(
+ null=True, blank=True, db_comment="Field to store icon url for Apps"
+ )
+ app_url = models.URLField(
+ null=True, blank=True, db_comment="Stores deployed URL for App"
+ )
+ # TODO: Change this to a Forgein key once the bundle is created.
+ access_control_bundle_id = models.TextField(null=True, blank=True)
+ created_by = models.ForeignKey(
+ User,
+ on_delete=models.SET_NULL,
+ related_name="pipelines_created",
+ null=True,
+ blank=True,
+ )
+ modified_by = models.ForeignKey(
+ User,
+ on_delete=models.SET_NULL,
+ related_name="pipelines_modified",
+ null=True,
+ blank=True,
+ )
+
+ # Manager
+ objects = PipelineModelManager()
+
+ def __str__(self) -> str:
+ return f"Pipeline({self.id})"
+
+ class Meta:
+ verbose_name = "Pipeline"
+ verbose_name_plural = "Pipelines"
+ db_table = "pipeline_v2"
+ constraints = [
+ models.UniqueConstraint(
+ fields=["id", "pipeline_type"],
+ name="unique_pipeline_entity",
+ ),
+ models.UniqueConstraint(
+ fields=["pipeline_name", "organization"],
+ name="unique_pipeline_name",
+ ),
+ ]
+
+ def is_active(self) -> bool:
+ return bool(self.active)
diff --git a/backend/pipeline_v2/pipeline_processor.py b/backend/pipeline_v2/pipeline_processor.py
new file mode 100644
index 000000000..fa3328db2
--- /dev/null
+++ b/backend/pipeline_v2/pipeline_processor.py
@@ -0,0 +1,81 @@
+import logging
+from typing import Optional
+
+from django.utils import timezone
+from pipeline_v2.exceptions import InactivePipelineError
+from pipeline_v2.models import Pipeline
+
+logger = logging.getLogger(__name__)
+
+
+class PipelineProcessor:
+ @staticmethod
+ def initialize_pipeline_sync(pipeline_id: str) -> Pipeline:
+ """Fetches and initializes the sync for a pipeline.
+
+ Args:
+ pipeline_id (str): UUID of the pipeline to sync
+ """
+ pipeline: Pipeline = PipelineProcessor.fetch_pipeline(pipeline_id)
+ pipeline.run_count = pipeline.run_count + 1
+ return PipelineProcessor._update_pipeline_status(
+ pipeline=pipeline,
+ status=Pipeline.PipelineStatus.RESTARTING,
+ is_end=False,
+ )
+
+ @staticmethod
+ def fetch_pipeline(pipeline_id: str, check_active: bool = True) -> Pipeline:
+ """Retrieves and checks for an active pipeline.
+
+ Raises:
+ InactivePipelineError: If an active pipeline is not found
+ """
+ pipeline: Pipeline = Pipeline.objects.get(pk=pipeline_id)
+ if check_active and not pipeline.is_active():
+ logger.error(f"Inactive pipeline fetched: {pipeline_id}")
+ raise InactivePipelineError(pipeline_name=pipeline.pipeline_name)
+ return pipeline
+
+ @staticmethod
+ def _update_pipeline_status(
+ pipeline: Pipeline,
+ status: tuple[str, str],
+ is_end: bool,
+ is_active: Optional[bool] = None,
+ ) -> Pipeline:
+ """Updates pipeline status during execution.
+
+ Raises:
+ PipelineSaveError: Exception while saving a pipeline
+
+ Returns:
+ Pipeline: Updated pipeline
+ """
+ if is_end:
+ pipeline.last_run_time = timezone.now()
+ if status:
+ pipeline.last_run_status = status
+ if is_active is not None:
+ pipeline.active = is_active
+
+ pipeline.save()
+ return pipeline
+
+ @staticmethod
+ def update_pipeline(
+ pipeline_guid: Optional[str],
+ status: tuple[str, str],
+ is_active: Optional[bool] = None,
+ ) -> None:
+ if not pipeline_guid:
+ return
+ # Skip check if we are enabling an inactive pipeline
+ check_active = not is_active
+ pipeline: Pipeline = PipelineProcessor.fetch_pipeline(
+ pipeline_id=pipeline_guid, check_active=check_active
+ )
+ PipelineProcessor._update_pipeline_status(
+ pipeline=pipeline, is_end=True, status=status, is_active=is_active
+ )
+ logger.info(f"Updated pipeline {pipeline_guid} status: {status}")
diff --git a/backend/pipeline_v2/serializers/crud.py b/backend/pipeline_v2/serializers/crud.py
new file mode 100644
index 000000000..bb051abef
--- /dev/null
+++ b/backend/pipeline_v2/serializers/crud.py
@@ -0,0 +1,104 @@
+import logging
+from collections import OrderedDict
+from typing import Any
+
+from connector_processor.connector_processor import ConnectorProcessor
+from connector_v2.connector_instance_helper import ConnectorInstanceHelper
+from connector_v2.models import ConnectorInstance
+from pipeline_v2.constants import PipelineConstants as PC
+from pipeline_v2.constants import PipelineKey as PK
+from pipeline_v2.models import Pipeline
+from scheduler.helper import SchedulerHelper
+from utils.serializer_utils import SerializerUtils
+
+from backend.serializers import AuditSerializer
+from unstract.connectors.connectorkit import Connectorkit
+
+logger = logging.getLogger(__name__)
+
+
+class PipelineSerializer(AuditSerializer):
+
+ class Meta:
+ model = Pipeline
+ fields = "__all__"
+
+ def create(self, validated_data: dict[str, Any]) -> Any:
+ # TODO: Deduce pipeline type based on WF?
+ validated_data[PK.ACTIVE] = True # Add this as default instead?
+ validated_data[PK.SCHEDULED] = True
+ return super().create(validated_data)
+
+ def save(self, **kwargs: Any) -> Pipeline:
+ pipeline: Pipeline = super().save(**kwargs)
+ SchedulerHelper.add_job(
+ str(pipeline.pk),
+ cron_string=pipeline.cron_string,
+ )
+ return pipeline
+
+ def _get_name_and_icon(self, connectors: list[Any], connector_id: Any) -> Any:
+ for obj in connectors:
+ if obj["id"] == connector_id:
+ return obj["name"], obj["icon"]
+ return PC.NOT_CONFIGURED, None
+
+ def _add_connector_data(
+ self,
+ repr: OrderedDict[str, Any],
+ connector_instance_list: list[Any],
+ connectors: list[Any],
+ ) -> OrderedDict[str, Any]:
+ """Adds connector Input/Output data.
+
+ Args:
+ sef (_type_): _description_
+ repr (OrderedDict[str, Any]): _description_
+
+ Returns:
+ OrderedDict[str, Any]: _description_
+ """
+ repr[PC.SOURCE_NAME] = PC.NOT_CONFIGURED
+ repr[PC.DESTINATION_NAME] = PC.NOT_CONFIGURED
+ for instance in connector_instance_list:
+ if instance.connector_type == "INPUT":
+ repr[PC.SOURCE_NAME], repr[PC.SOURCE_ICON] = self._get_name_and_icon(
+ connectors=connectors,
+ connector_id=instance.connector_id,
+ )
+ if instance.connector_type == "OUTPUT":
+ repr[PC.DESTINATION_NAME], repr[PC.DESTINATION_ICON] = (
+ self._get_name_and_icon(
+ connectors=connectors,
+ connector_id=instance.connector_id,
+ )
+ )
+ return repr
+
+ def to_representation(self, instance: Pipeline) -> OrderedDict[str, Any]:
+ """To set Source, Destination & Agency for Pipelines."""
+ repr: OrderedDict[str, Any] = super().to_representation(instance)
+
+ connector_kit = Connectorkit()
+ connectors = connector_kit.get_connectors_list()
+
+ if SerializerUtils.check_context_for_GET_or_POST(context=self.context):
+ workflow = instance.workflow
+ connector_instance_list = ConnectorInstanceHelper.get_input_output_connector_instances_by_workflow( # noqa
+ workflow.id
+ )
+ repr[PK.WORKFLOW_ID] = workflow.id
+ repr[PK.WORKFLOW_NAME] = workflow.workflow_name
+ repr[PK.CRON_STRING] = repr.pop(PK.CRON_STRING)
+ repr = self._add_connector_data(
+ repr=repr,
+ connector_instance_list=connector_instance_list,
+ connectors=connectors,
+ )
+
+ return repr
+
+ def get_connector_data(self, connector: ConnectorInstance, key: str) -> Any:
+ return ConnectorProcessor.get_connector_data_with_key(
+ connector.connector_id, key
+ )
diff --git a/backend/pipeline_v2/serializers/execute.py b/backend/pipeline_v2/serializers/execute.py
new file mode 100644
index 000000000..4e5a7fd58
--- /dev/null
+++ b/backend/pipeline_v2/serializers/execute.py
@@ -0,0 +1,24 @@
+import logging
+
+from pipeline_v2.models import Pipeline
+from rest_framework import serializers
+
+logger = logging.getLogger(__name__)
+
+
+class PipelineExecuteSerializer(serializers.Serializer):
+ # TODO: Add pipeline as a read_only related field
+ pipeline_id = serializers.UUIDField()
+ execution_id = serializers.UUIDField(required=False)
+
+ def validate_pipeline_id(self, value: str) -> str:
+ try:
+ Pipeline.objects.get(pk=value)
+ except Pipeline.DoesNotExist:
+ raise serializers.ValidationError("Invalid pipeline ID")
+ return value
+
+
+class DateRangeSerializer(serializers.Serializer):
+ start_date = serializers.DateTimeField(required=False)
+ end_date = serializers.DateTimeField(required=False)
diff --git a/backend/pipeline_v2/serializers/update.py b/backend/pipeline_v2/serializers/update.py
new file mode 100644
index 000000000..6fc9f66dc
--- /dev/null
+++ b/backend/pipeline_v2/serializers/update.py
@@ -0,0 +1,14 @@
+from pipeline_v2.models import Pipeline
+from rest_framework import serializers
+
+
+class PipelineUpdateSerializer(serializers.Serializer):
+ pipeline_id = serializers.UUIDField(required=True)
+ active = serializers.BooleanField(required=True)
+
+ def validate_pipeline_id(self, value: str) -> str:
+ try:
+ Pipeline.objects.get(pk=value)
+ except Pipeline.DoesNotExist:
+ raise serializers.ValidationError("Invalid pipeline ID")
+ return value
diff --git a/backend/pipeline_v2/urls.py b/backend/pipeline_v2/urls.py
new file mode 100644
index 000000000..d6950633d
--- /dev/null
+++ b/backend/pipeline_v2/urls.py
@@ -0,0 +1,41 @@
+from django.urls import path
+from pipeline_v2.constants import PipelineURL
+from pipeline_v2.execution_view import PipelineExecutionViewSet
+from pipeline_v2.views import PipelineViewSet
+from rest_framework.urlpatterns import format_suffix_patterns
+
+pipeline_list = PipelineViewSet.as_view(
+ {
+ "get": "list",
+ "post": "create",
+ }
+)
+execution_list = PipelineExecutionViewSet.as_view(
+ {
+ "get": "list",
+ }
+)
+pipeline_detail = PipelineViewSet.as_view(
+ {
+ "get": "retrieve",
+ "put": "update",
+ "patch": "partial_update",
+ "delete": "destroy",
+ }
+)
+
+pipeline_execute = PipelineViewSet.as_view({"post": "execute"})
+
+
+urlpatterns = format_suffix_patterns(
+ [
+ path("pipeline/", pipeline_list, name=PipelineURL.LIST),
+ path("pipeline/<uuid:pk>/", pipeline_detail, name=PipelineURL.DETAIL),
+ path(
+ "pipeline/<uuid:pk>/executions/",
+ execution_list,
+ name=PipelineURL.EXECUTIONS,
+ ),
+ path("pipeline/execute/", pipeline_execute, name=PipelineURL.EXECUTE),
+ ]
+)
diff --git a/backend/pipeline_v2/views.py b/backend/pipeline_v2/views.py
new file mode 100644
index 000000000..553b0179a
--- /dev/null
+++ b/backend/pipeline_v2/views.py
@@ -0,0 +1,118 @@
+import logging
+from typing import Any, Optional
+
+from account_v2.custom_exceptions import DuplicateData
+from django.db import IntegrityError
+from django.db.models import QuerySet
+from permissions.permission import IsOwner
+from pipeline_v2.constants import (
+ PipelineConstants,
+ PipelineErrors,
+ PipelineExecutionKey,
+)
+from pipeline_v2.constants import PipelineKey as PK
+from pipeline_v2.manager import PipelineManager
+from pipeline_v2.models import Pipeline
+from pipeline_v2.pipeline_processor import PipelineProcessor
+from pipeline_v2.serializers.crud import PipelineSerializer
+from pipeline_v2.serializers.execute import (
+ PipelineExecuteSerializer as ExecuteSerializer,
+)
+from pipeline_v2.serializers.update import PipelineUpdateSerializer
+from rest_framework import serializers, status, viewsets
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.versioning import URLPathVersioning
+from scheduler.helper import SchedulerHelper
+
+logger = logging.getLogger(__name__)
+
+
+class PipelineViewSet(viewsets.ModelViewSet):
+ versioning_class = URLPathVersioning
+ queryset = Pipeline.objects.all()
+ permission_classes = [IsOwner]
+ serializer_class = PipelineSerializer
+
+ def get_queryset(self) -> Optional[QuerySet]:
+ type = self.request.query_params.get(PipelineConstants.TYPE)
+ if type is not None:
+ queryset = Pipeline.objects.filter(
+ created_by=self.request.user, pipeline_type=type
+ )
+ return queryset
+ elif type is None:
+ queryset = Pipeline.objects.filter(created_by=self.request.user)
+ return queryset
+
+ def get_serializer_class(self) -> serializers.Serializer:
+ if self.action == "execute":
+ return ExecuteSerializer
+ else:
+ return PipelineSerializer
+
+ # TODO: Refactor to perform an action with explicit arguments
+ # For eg, passing pipeline ID and with_log=False -> executes pipeline
+ # For FE however we call the same API twice
+ # (first call generates execution ID)
+ def execute(self, request: Request) -> Response:
+ serializer: ExecuteSerializer = self.get_serializer(data=request.data)
+ serializer.is_valid(raise_exception=True)
+ execution_id = serializer.validated_data.get("execution_id", None)
+ pipeline_id = serializer.validated_data[PK.PIPELINE_ID]
+
+ execution = PipelineManager.execute_pipeline(
+ request=request,
+ pipeline_id=pipeline_id,
+ execution_id=execution_id,
+ )
+ pipeline: Pipeline = PipelineProcessor.fetch_pipeline(pipeline_id)
+ serializer = PipelineSerializer(pipeline)
+ response_data = {
+ PipelineExecutionKey.PIPELINE: serializer.data,
+ PipelineExecutionKey.EXECUTION: execution.data,
+ }
+ return Response(data=response_data, status=status.HTTP_200_OK)
+
+ def create(self, request: Request) -> Response:
+ serializer = self.get_serializer(data=request.data)
+ serializer.is_valid(raise_exception=True)
+ try:
+ serializer.save()
+ except IntegrityError:
+ raise DuplicateData(
+ f"{PipelineErrors.PIPELINE_EXISTS}, " f"{PipelineErrors.DUPLICATE_API}"
+ )
+ return Response(data=serializer.data, status=status.HTTP_201_CREATED)
+
+ def perform_destroy(self, instance: Pipeline) -> None:
+ pipeline_to_remove = str(instance.pk)
+ super().perform_destroy(instance)
+ return SchedulerHelper.remove_job(pipeline_to_remove)
+
+ def partial_update(self, request: Request, pk: Any = None) -> Response:
+ serializer = PipelineUpdateSerializer(data=request.data)
+ if serializer.is_valid():
+ pipeline_id = serializer.validated_data.get("pipeline_id")
+ active = serializer.validated_data.get("active")
+ try:
+ if active:
+ SchedulerHelper.resume_job(pipeline_id)
+ else:
+ SchedulerHelper.pause_job(pipeline_id)
+ except Exception as e:
+ logger.error(f"Failed to update pipeline status: {e}")
+ return Response(
+ {"error": "Failed to update pipeline status"},
+ status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+ )
+
+ return Response(
+ {
+ "status": "success",
+ "message": f"Pipeline {pipeline_id} status updated",
+ },
+ status=status.HTTP_200_OK,
+ )
+ else:
+ return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)