LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Delta Changes for Multi-Tenancy V2 API Account Adapter applications (#630)
ali4e37fe84a02d
diff --git a/backend/account_v2/authentication_controller.py b/backend/account_v2/authentication_controller.pyindex 0aec22294..4cdac81b7 100644--- a/backend/account_v2/authentication_controller.py+++ b/backend/account_v2/authentication_controller.py@@ -186,7 +186,9 @@ class AuthenticationController:f"{ErrorMessage.ORGANIZATION_EXIST}, \{ErrorMessage.DUPLICATE_API}")- self.create_tenant_user(organization=organization, user=user)+ organization_member = self.create_tenant_user(+ organization=organization, user=user+ )if new_organization:try:@@ -214,6 +216,7 @@ class AuthenticationController:response: Response = Response(status=status.HTTP_200_OK,data={+ "is_new_org": new_organization,"user": serialized_user_info,"organization": organization_info,f"{Common.LOG_EVENTS_ID}": StateStore.get(Common.LOG_EVENTS_ID),@@ -226,6 +229,7 @@ class AuthenticationController:organization_id=current_organization_id,)UserSessionUtils.set_organization_id(request, organization_id)+ UserSessionUtils.set_organization_member_role(request, organization_member)OrganizationMemberService.set_user_membership_in_organization_cache(user_id=user.user_id, organization_id=organization_id)@@ -435,11 +439,13 @@ class AuthenticationController:organization_user.role = roleorganization_user.save()- def create_tenant_user(self, organization: Organization, user: User) -> None:+ def create_tenant_user(+ self, organization: Organization, user: User+ ) -> OrganizationMember:existing_tenant_user = OrganizationMemberService.get_user_by_id(id=user.id)if existing_tenant_user:- return None+ return existing_tenant_useraccount_user = self.get_or_create_user(user=user)if not account_user:@@ -463,6 +469,7 @@ class AuthenticationController:f"{tenant_user.user.email} added in to the organization "f"{organization.organization_id}")+ return tenant_userexcept IntegrityError:logger.warning(f"Account already exists for {user.email}")diff --git a/backend/account_v2/dto.py b/backend/account_v2/dto.pyindex 1554901fb..66cf6c1aa 100644--- a/backend/account_v2/dto.py+++ b/backend/account_v2/dto.py@@ -59,6 +59,7 @@ class UserSessionInfo:email: strorganization_id: struser: UserInfo+ role: str@staticmethoddef from_dict(data: dict[str, Any]) -> "UserSessionInfo":@@ -67,6 +68,7 @@ class UserSessionInfo:user_id=data["user_id"],email=data["email"],organization_id=data["organization_id"],+ role=data["role"],)def to_dict(self) -> Any:@@ -75,6 +77,7 @@ class UserSessionInfo:"user_id": self.user_id,"email": self.email,"organization_id": self.organization_id,+ "role": self.role,}diff --git a/backend/account_v2/serializer.py b/backend/account_v2/serializer.pyindex 87c531a4a..e31d08d95 100644--- a/backend/account_v2/serializer.py+++ b/backend/account_v2/serializer.py@@ -115,3 +115,4 @@ class UserSessionResponseSerializer(serializers.Serializer):user_id = serializers.CharField()email = serializers.CharField()organization_id = serializers.CharField()+ role = serializers.CharField()diff --git a/backend/account_v2/views.py b/backend/account_v2/views.pyindex b73add92a..731134901 100644--- a/backend/account_v2/views.py+++ b/backend/account_v2/views.py@@ -142,6 +142,7 @@ def make_session_response(email=request.user.email,user=auth_controller.get_user_info(request),organization_id=UserSessionUtils.get_organization_id(request),+ role=UserSessionUtils.get_organization_member_role(request),)).datadiff --git a/backend/adapter_processor_v2/adapter_processor.py b/backend/adapter_processor_v2/adapter_processor.pyindex 16c3b9914..0b95443d7 100644--- a/backend/adapter_processor_v2/adapter_processor.py+++ b/backend/adapter_processor_v2/adapter_processor.py@@ -13,11 +13,11 @@ from django.conf import settingsfrom django.core.exceptions import ObjectDoesNotExistfrom platform_settings_v2.platform_auth_service import PlatformAuthenticationServicefrom tenant_account_v2.organization_member_service import OrganizationMemberService-from unstract.adapters.adapterkit import Adapterkit-from unstract.adapters.base import Adapter-from unstract.adapters.enums import AdapterTypes-from unstract.adapters.exceptions import AdapterError-from unstract.adapters.x2text.constants import X2TextConstants+from unstract.sdk.adapters.adapterkit import Adapterkit+from unstract.sdk.adapters.base import Adapter+from unstract.sdk.adapters.enums import AdapterTypes+from unstract.sdk.adapters.exceptions import AdapterError+from unstract.sdk.adapters.x2text.constants import X2TextConstantsfrom .models import AdapterInstance, UserDefaultAdapter@@ -33,13 +33,9 @@ class AdapterProcessor:AdapterKeys.ID, adapter_id)if len(updated_adapters) != 0:- try:- schema_details[AdapterKeys.JSON_SCHEMA] = json.loads(- updated_adapters[0].get(AdapterKeys.JSON_SCHEMA)- )- except Exception as exc:- logger.error(f"Error occured while parsing JSON Schema : {exc}")- raise InternalServiceError()+ schema_details[AdapterKeys.JSON_SCHEMA] = json.loads(+ updated_adapters[0].get(AdapterKeys.JSON_SCHEMA)+ )else:logger.error(f"Invalid adapter Id : {adapter_id} while fetching JSON Schema"@@ -258,8 +254,5 @@ class AdapterProcessor:except ObjectDoesNotExist as e:logger.error(f"No default adapters found: {e}")raise InternalServiceError(- "No default adapters found, " "configure them through Platform Settings"+ "No default adapters found, configure them through Platform Settings")- except Exception as e:- logger.error(f"Error occurred while fetching default adapters: {e}")- raise InternalServiceError("Error fetching default adapters")diff --git a/backend/adapter_processor_v2/constants.py b/backend/adapter_processor_v2/constants.pyindex 91d555c77..6557491b9 100644--- a/backend/adapter_processor_v2/constants.py+++ b/backend/adapter_processor_v2/constants.py@@ -22,7 +22,8 @@ class AdapterKeys:SHARED_USERS = "shared_users"ADAPTER_NAME_EXISTS = ("Configuration with this name already exists within your organisation. "- "Please try with a different name"+ "Please try with a different name.")+ ADAPTER_NAME = "adapter_name"ADAPTER_CREATED_BY = "created_by_email"ADAPTER_CONTEXT_WINDOW_SIZE = "context_window_size"diff --git a/backend/adapter_processor_v2/exceptions.py b/backend/adapter_processor_v2/exceptions.pyindex 5bddbb3c2..009f1e5c1 100644--- a/backend/adapter_processor_v2/exceptions.py+++ b/backend/adapter_processor_v2/exceptions.py@@ -1,9 +1,8 @@from typing import Optional+from adapter_processor_v2.constants import AdapterKeysfrom rest_framework.exceptions import APIException-from backend.exceptions import UnstractBaseException-class IdIsMandatory(APIException):status_code = 400@@ -20,14 +19,6 @@ class InValidAdapterId(APIException):default_detail = "Adapter ID is not Valid."-class InvalidEncryptionKey(APIException):- status_code = 403- default_detail = (- "Platform encryption key for storing adapter credentials has changed! "- "Please inform the organization admin to contact support."- )--class InternalServiceError(APIException):status_code = 500default_detail = "Internal Service error"@@ -41,16 +32,31 @@ class CannotDeleteDefaultAdapter(APIException):)-class UniqueConstraintViolation(APIException):+class DuplicateAdapterNameError(APIException):status_code = 400- default_detail = "Unique constraint violated"+ default_detail: str = AdapterKeys.ADAPTER_NAME_EXISTS++ def __init__(+ self,+ name: Optional[str] = None,+ detail: Optional[str] = None,+ code: Optional[str] = None,+ ) -> None:+ if name:+ detail = self.default_detail.replace("this name", f"name '{name}'")+ super().__init__(detail, code)-class TestAdapterError(UnstractBaseException):+class TestAdapterError(APIException):status_code = 500default_detail = "Error while testing adapter"+class TestAdapterInputError(APIException):+ status_code = 400+ default_detail = "Error while testing adapter, please check the configuration."++class DeleteAdapterInUseError(APIException):status_code = 409@@ -61,6 +67,8 @@ class DeleteAdapterInUseError(APIException):adapter_name: str = "adapter",):if detail is None:+ if adapter_name != "adapter":+ adapter_name = f"'{adapter_name}'"detail = (f"Cannot delete {adapter_name}. ""It is used in a workflow or a prompt studio project"diff --git a/backend/adapter_processor_v2/models.py b/backend/adapter_processor_v2/models.pyindex 03c257de7..297aa7220 100644--- a/backend/adapter_processor_v2/models.py+++ b/backend/adapter_processor_v2/models.py@@ -4,14 +4,15 @@ import uuidfrom typing import Anyfrom account_v2.models import User-from cryptography.fernet import Fernet+from cryptography.fernet import Fernet, InvalidTokenfrom django.conf import settingsfrom django.db import modelsfrom django.db.models import QuerySetfrom tenant_account_v2.models import OrganizationMember-from unstract.adapters.adapterkit import Adapterkit-from unstract.adapters.enums import AdapterTypes-from unstract.adapters.exceptions import AdapterError+from unstract.sdk.adapters.adapterkit import Adapterkit+from unstract.sdk.adapters.enums import AdapterTypes+from unstract.sdk.adapters.exceptions import AdapterError+from utils.exceptions import InvalidEncryptionKeyfrom utils.models.base_model import BaseModelfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,@@ -142,22 +143,24 @@ class AdapterInstance(DefaultOrganizationMixin, BaseModel):self.save()- def get_adapter_meta_data(self) -> Any:- encryption_secret: str = settings.ENCRYPTION_KEY- f: Fernet = Fernet(encryption_secret.encode("utf-8"))+ @property+ def metadata(self) -> Any:+ try:+ encryption_secret: str = settings.ENCRYPTION_KEY+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))- adapter_metadata = json.loads(- f.decrypt(bytes(self.adapter_metadata_b).decode("utf-8"))- )+ adapter_metadata = json.loads(+ f.decrypt(bytes(self.adapter_metadata_b).decode("utf-8"))+ )+ except InvalidToken:+ raise InvalidEncryptionKey(entity=InvalidEncryptionKey.Entity.ADAPTER)return adapter_metadatadef get_context_window_size(self) -> int:-- adapter_metadata = self.get_adapter_meta_data()# Get the adapter_instanceadapter_class = Adapterkit().get_adapter_class_by_adapter_id(self.adapter_id)try:- adapter_instance = adapter_class(adapter_metadata)+ adapter_instance = adapter_class(self.metadata)return adapter_instance.get_context_window_size()except AdapterError as e:logger.warning(f"Unable to retrieve context window size - {e}")diff --git a/backend/adapter_processor_v2/serializers.py b/backend/adapter_processor_v2/serializers.pyindex 927fe7e91..69a0544a3 100644--- a/backend/adapter_processor_v2/serializers.py+++ b/backend/adapter_processor_v2/serializers.py@@ -4,13 +4,12 @@ from typing import Anyfrom account_v2.serializer import UserSerializerfrom adapter_processor_v2.adapter_processor import AdapterProcessorfrom adapter_processor_v2.constants import AdapterKeys-from adapter_processor_v2.exceptions import InvalidEncryptionKey-from cryptography.fernet import Fernet, InvalidToken+from cryptography.fernet import Fernetfrom django.conf import settingsfrom rest_framework import serializersfrom rest_framework.serializers import ModelSerializer-from unstract.adapters.constants import Common as common-from unstract.adapters.enums import AdapterTypes+from unstract.sdk.adapters.constants import Common as common+from unstract.sdk.adapters.enums import AdapterTypesfrom backend.constants import FieldLengthConstants as FLCfrom backend.serializers import AuditSerializer@@ -62,11 +61,7 @@ class AdapterInstanceSerializer(BaseAdapterSerializer):rep: dict[str, str] = super().to_representation(instance)rep.pop(AdapterKeys.ADAPTER_METADATA_B)-- try:- adapter_metadata = instance.get_adapter_meta_data()- except InvalidToken:- raise InvalidEncryptionKey+ adapter_metadata = instance.metadatarep[AdapterKeys.ADAPTER_METADATA] = adapter_metadata# Retrieve context window if adapter is a LLM# For other adapter types, context_window is not relevant.@@ -124,8 +119,7 @@ class AdapterListSerializer(BaseAdapterSerializer):rep[common.ICON] = AdapterProcessor.get_adapter_data_with_key(instance.adapter_id, common.ICON)- adapter_metadata = instance.get_adapter_meta_data()- model = adapter_metadata.get("model")+ model = instance.metadata.get("model")if model:rep["model"] = modeldiff --git a/backend/adapter_processor_v2/views.py b/backend/adapter_processor_v2/views.pyindex b5a65179f..92eb5f2e7 100644--- a/backend/adapter_processor_v2/views.py+++ b/backend/adapter_processor_v2/views.py@@ -7,9 +7,9 @@ from adapter_processor_v2.constants import AdapterKeysfrom adapter_processor_v2.exceptions import (CannotDeleteDefaultAdapter,DeleteAdapterInUseError,+ DuplicateAdapterNameError,IdIsMandatory,InValidType,- UniqueConstraintViolation,)from adapter_processor_v2.serializers import (AdapterInfoSerializer,@@ -41,7 +41,6 @@ from tenant_account_v2.organization_member_service import OrganizationMemberServfrom utils.filtering import FilterHelperfrom .constants import AdapterKeys as constant-from .exceptions import InternalServiceErrorfrom .models import AdapterInstance, UserDefaultAdapterlogger = logging.getLogger(__name__)@@ -121,17 +120,13 @@ class AdapterViewSet(GenericViewSet):adapter_metadata[AdapterKeys.ADAPTER_TYPE] = serializer.validated_data.get(AdapterKeys.ADAPTER_TYPE)- try:- test_result = AdapterProcessor.test_adapter(- adapter_id=adapter_id, adapter_metadata=adapter_metadata- )- return Response(- {AdapterKeys.IS_VALID: test_result},- status=status.HTTP_200_OK,- )- except Exception as e:- logger.error(f"Error testing adapter : {str(e)}")- raise e+ test_result = AdapterProcessor.test_adapter(+ adapter_id=adapter_id, adapter_metadata=adapter_metadata+ )+ return Response(+ {AdapterKeys.IS_VALID: test_result},+ status=status.HTTP_200_OK,+ )class AdapterInstanceViewSet(ModelViewSet):@@ -217,10 +212,9 @@ class AdapterInstanceViewSet(ModelViewSet):user_default_adapter.save()except IntegrityError:- raise UniqueConstraintViolation(f"{AdapterKeys.ADAPTER_NAME_EXISTS}")- except Exception as e:- logger.error(f"Error saving adapter to DB: {e}")- raise InternalServiceError+ raise DuplicateAdapterNameError(+ name=serializer.validated_data.get(AdapterKeys.ADAPTER_NAME)+ )headers = self.get_success_headers(serializer.data)return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headersdiff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.pyindex 486425cae..5750f6b7f 100644--- a/backend/api_v2/api_deployment_views.py+++ b/backend/api_v2/api_deployment_views.py@@ -74,22 +74,28 @@ class DeploymentExecution(views.APIView):self, request: Request, org_name: str, api_name: str, api: APIDeployment) -> Response:execution_id = request.query_params.get("execution_id")+ include_metadata = (+ request.query_params.get(ApiExecution.INCLUDE_METADATA, "false").lower()+ == "true"+ )if not execution_id:raise InvalidAPIRequest("execution_id shouldn't be empty")response: ExecutionResponse = DeploymentHelper.get_execution_status(execution_id=execution_id)- if response.execution_status != CeleryTaskState.SUCCESS.value:- return Response(- {- "status": response.execution_status,- "message": response.result,- },- status=status.HTTP_422_UNPROCESSABLE_ENTITY,- )+ response_status = status.HTTP_422_UNPROCESSABLE_ENTITY+ if response.execution_status == CeleryTaskState.COMPLETED.value:+ response_status = status.HTTP_200_OK+ if include_metadata:+ response.remove_result_metadata_keys(keys_to_remove=["highlight_data"])+ else:+ response.remove_result_metadata_keys()return Response(- {"status": response.execution_status, "message": response.result},- status=status.HTTP_200_OK,+ data={+ "status": response.execution_status,+ "message": response.result,+ },+ status=response_status,)diff --git a/backend/api_v2/api_key_validator.py b/backend/api_v2/api_key_validator.pynew file mode 100644index 000000000..9a59bb8cb--- /dev/null+++ b/backend/api_v2/api_key_validator.py@@ -0,0 +1,60 @@+import logging+from functools import wraps+from typing import Any++from api_v2.exceptions import Forbidden+from rest_framework.request import Request++logger = logging.getLogger(__name__)+++class BaseAPIKeyValidator:+ @classmethod+ def validate_api_key(cls, func: Any) -> Any:+ """Decorator that validates the API key.++ Sample header:+ Authorization: Bearer 123e4567-e89b-12d3-a456-426614174001+ Args:+ func (Any): Function to wrap for validation+ """++ @wraps(func)+ def wrapper(self: Any, request: Request, *args: Any, **kwargs: Any) -> Any:+ """Wrapper to validate the inputs and key.++ Args:+ request (Request): Request context++ Raises:+ Forbidden: _description_+ APINotFound: _description_++ Returns:+ Any: _description_+ """+ authorization_header = request.headers.get("Authorization")+ api_key = None+ if authorization_header and authorization_header.startswith("Bearer "):+ api_key = authorization_header.split(" ")[1]+ if not api_key:+ raise Forbidden("Missing api key")+ cls.validate_parameters(request, **kwargs)+ return cls.validate_and_process(+ self, request, func, *args, **kwargs, api_key=api_key+ )++ return wrapper++ @staticmethod+ def validate_parameters(request: Request, **kwargs: Any) -> None:+ """Validate specific parameters required by subclasses."""+ pass++ @staticmethod+ def validate_and_process(+ self: Any, request: Request, func: Any, api_key: str, *args: Any, **kwargs: Any+ ) -> Any:+ """Process and validate API key with specific logic required by+ subclasses."""+ passdiff --git a/backend/api_v2/api_key_views.py b/backend/api_v2/api_key_views.pyindex 906127c40..35236ca29 100644--- a/backend/api_v2/api_key_views.py+++ b/backend/api_v2/api_key_views.py@@ -1,8 +1,13 @@+from typing import Optional+from api_v2.deployment_helper import DeploymentHelper-from api_v2.exceptions import APINotFound+from api_v2.exceptions import APINotFound, PathVariablesNotFoundfrom api_v2.key_helper import KeyHelperfrom api_v2.models import APIKeyfrom api_v2.serializers import APIKeyListSerializer, APIKeySerializer+from permissions.permission import IsOwner+from pipeline_v2.exceptions import PipelineNotFound+from pipeline_v2.pipeline_processor import PipelineProcessorfrom rest_framework import serializers, viewsetsfrom rest_framework.decorators import actionfrom rest_framework.request import Request@@ -11,6 +16,7 @@ from rest_framework.response import Responseclass APIKeyViewSet(viewsets.ModelViewSet):queryset = APIKey.objects.all()+ permission_classes = [IsOwner]def get_serializer_class(self) -> serializers.Serializer:if self.action in ["api_keys"]:@@ -18,11 +24,28 @@ class APIKeyViewSet(viewsets.ModelViewSet):return APIKeySerializer@action(detail=True, methods=["get"])- def api_keys(self, request: Request, api_id: str) -> Response:+ def api_keys(+ self,+ request: Request,+ api_id: Optional[str] = None,+ pipeline_id: Optional[str] = None,+ ) -> Response:"""Custom action to fetch api keys of an api deployment."""- api = DeploymentHelper.get_api_by_id(api_id=api_id)- if not api:- raise APINotFound()- keys = KeyHelper.list_api_keys_of_api(api_instance=api)+ if api_id:+ api = DeploymentHelper.get_api_by_id(api_id=api_id)+ if not api:+ raise APINotFound()+ self.check_object_permissions(request, api)+ keys = KeyHelper.list_api_keys_of_api(api_instance=api)+ elif pipeline_id:+ pipeline = PipelineProcessor.get_active_pipeline(pipeline_id=pipeline_id)+ if not pipeline:+ raise PipelineNotFound()+ self.check_object_permissions(request, pipeline)+ keys = KeyHelper.list_api_keys_of_pipeline(pipeline_instance=pipeline)+ else:+ raise PathVariablesNotFound(+ "Either `api_id` or `pipeline_id` path variable must be provided."+ )serializer = self.get_serializer(keys, many=True)return Response(serializer.data)diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.pyindex be9dc0b10..47ec0615e 100644--- a/backend/api_v2/deployment_helper.py+++ b/backend/api_v2/deployment_helper.py@@ -1,95 +1,51 @@import loggingimport uuid-from functools import wrapsfrom typing import Any, Optionalfrom urllib.parse import urlencode-from api_v2.constants import ApiExecution+from api_v2.api_key_validator import BaseAPIKeyValidatorfrom api_v2.exceptions import (ApiKeyCreateException,APINotFound,- Forbidden,InactiveAPI,- UnauthorizedKey,+ InvalidAPIRequest,)from api_v2.key_helper import KeyHelperfrom api_v2.models import APIDeployment, APIKeyfrom api_v2.serializers import APIExecutionResponseSerializer+from api_v2.utils import APIDeploymentUtilsfrom django.core.files.uploadedfile import UploadedFile-from django.db import connection-from rest_framework import statusfrom rest_framework.request import Request-from rest_framework.response import Responsefrom rest_framework.serializers import Serializerfrom rest_framework.utils.serializer_helpers import ReturnDict-from utils.constants import Account-from utils.local_context import StateStorefrom workflow_manager.endpoint_v2.destination import DestinationConnectorfrom workflow_manager.endpoint_v2.source import SourceConnectorfrom workflow_manager.workflow_v2.dto import ExecutionResponse+from workflow_manager.workflow_v2.enums import ExecutionStatusfrom workflow_manager.workflow_v2.models.workflow import Workflowfrom workflow_manager.workflow_v2.workflow_helper import WorkflowHelperlogger = logging.getLogger(__name__)-class DeploymentHelper:+class DeploymentHelper(BaseAPIKeyValidator):@staticmethod- def validate_api_key(func: Any) -> Any:- """Decorator that validates the API key.+ def validate_parameters(request: Request, **kwargs: Any) -> None:+ """Validate api_name for API deployments."""+ api_name = kwargs.get("api_name") or request.data.get("api_name")+ if not api_name:+ raise InvalidAPIRequest("Missing params api_name")- Sample header:- Authorization: Bearer 123e4567-e89b-12d3-a456-426614174001- Args:- func (Any): Function to wrap for validation- """-- @wraps(func)- def wrapper(self: Any, request: Request, *args: Any, **kwargs: Any) -> Any:- """Wrapper to validate the inputs and key.-- Args:- request (Request): Request context-- Raises:- Forbidden: _description_- APINotFound: _description_-- Returns:- Any: _description_- """- try:- authorization_header = request.headers.get("Authorization")- api_key = None- if authorization_header and authorization_header.startswith("Bearer "):- api_key = authorization_header.split(" ")[1]- if not api_key:- raise Forbidden("Missing api key")- org_name = kwargs.get("org_name") or request.data.get("org_name")- api_name = kwargs.get("api_name") or request.data.get("api_name")- if not api_name:- raise Forbidden("Missing api_name")- # Set organization in state store for API- StateStore.set(Account.ORGANIZATION_ID, org_name)-- api_deployment = DeploymentHelper.get_deployment_by_api_name(- api_name=api_name- )- DeploymentHelper.validate_api(- api_deployment=api_deployment, api_key=api_key- )- kwargs["api"] = api_deployment- return func(self, request, *args, **kwargs)-- except (UnauthorizedKey, InactiveAPI, APINotFound):- raise- except Exception as exception:- logger.error(f"Exception: {exception}")- return Response(- {"error": str(exception)}, status=status.HTTP_403_FORBIDDEN- )-- return wrapper+ @staticmethod+ def validate_and_process(+ self: Any, request: Request, func: Any, api_key: str, *args: Any, **kwargs: Any+ ) -> Any:+ """Fetch API deployment and validate API key."""+ api_name = kwargs.get("api_name") or request.data.get("api_name")+ api_deployment = DeploymentHelper.get_deployment_by_api_name(api_name=api_name)+ DeploymentHelper.validate_api(api_deployment=api_deployment, api_key=api_key)+ kwargs["api"] = api_deployment+ return func(self, request, *args, **kwargs)@staticmethoddef validate_api(api_deployment: Optional[APIDeployment], api_key: str) -> None:@@ -107,7 +63,7 @@ class DeploymentHelper:raise APINotFound()if not api_deployment.is_active:raise InactiveAPI()- KeyHelper.validate_api_key(api_key=api_key, api_instance=api_deployment)+ KeyHelper.validate_api_key(api_key=api_key, instance=api_deployment)@staticmethoddef validate_and_get_workflow(workflow_id: str) -> Workflow:@@ -117,25 +73,7 @@ class DeploymentHelper:@staticmethoddef get_api_by_id(api_id: str) -> Optional[APIDeployment]:- try:- api_deployment: APIDeployment = APIDeployment.objects.get(pk=api_id)- return api_deployment- except APIDeployment.DoesNotExist:- return None-- @staticmethod- def construct_complete_endpoint(api_name: str) -> str:- """Constructs the complete API endpoint by appending organization- schema, endpoint path, and Django app backend URL.-- Parameters:- - endpoint (str): The endpoint path to be appended to the complete URL.-- Returns:- - str: The complete API endpoint URL.- """- org_schema = connection.tenant.schema_name- return f"{ApiExecution.PATH}/{org_schema}/{api_name}/"+ return APIDeploymentUtils.get_api_by_id(api_id=api_id)@staticmethoddef construct_status_endpoint(api_endpoint: str, execution_id: str) -> str:@@ -185,8 +123,9 @@ class DeploymentHelper:logger.info("Deleted the deployment instance")raise ApiKeyCreateException()- @staticmethod+ @classmethoddef execute_workflow(+ cls,organization_name: str,api: APIDeployment,file_objs: list[UploadedFile],@@ -206,6 +145,7 @@ class DeploymentHelper:workflow_id = api.workflow.idpipeline_id = api.idexecution_id = str(uuid.uuid4())+hash_values_of_files = SourceConnector.add_input_file_to_api_storage(workflow_id=workflow_id,execution_id=execution_id,@@ -223,11 +163,20 @@ class DeploymentHelper:result.status_api = DeploymentHelper.construct_status_endpoint(api_endpoint=api.api_endpoint, execution_id=execution_id)- except Exception:+ if include_metadata:+ result.remove_result_metadata_keys(keys_to_remove=["highlight_data"])+ else:+ result.remove_result_metadata_keys()+ except Exception as error:DestinationConnector.delete_api_storage_dir(workflow_id=workflow_id, execution_id=execution_id)- raise+ result = ExecutionResponse(+ workflow_id=workflow_id,+ execution_id=execution_id,+ execution_status=ExecutionStatus.ERROR.value,+ error=str(error),+ )return APIExecutionResponseSerializer(result).data@staticmethoddiff --git a/backend/api_v2/exceptions.py b/backend/api_v2/exceptions.pyindex c3a58ff05..f2e4c415e 100644--- a/backend/api_v2/exceptions.py+++ b/backend/api_v2/exceptions.py@@ -3,6 +3,15 @@ from typing import Optionalfrom rest_framework.exceptions import APIException+class NotFoundException(APIException):+ status_code = 404+ default_detail = "The requested resource was not found."+++class PathVariablesNotFound(NotFoundException):+ default_detail = "Path variable must be provided."++class MandatoryWorkflowId(APIException):status_code = 400default_detail = "Workflow ID is mandatory"@@ -20,8 +29,7 @@ class Forbidden(APIException):)-class APINotFound(APIException):- status_code = 404+class APINotFound(NotFoundException):default_detail = "API not found"@@ -30,8 +38,7 @@ class InvalidAPIRequest(APIException):default_detail = "Bad request"-class InactiveAPI(APIException):- status_code = 404+class InactiveAPI(NotFoundException):default_detail = "API not found or Inactive"diff --git a/backend/api_v2/key_helper.py b/backend/api_v2/key_helper.pyindex 2a1a430a9..15d285ac3 100644--- a/backend/api_v2/key_helper.py+++ b/backend/api_v2/key_helper.py@@ -1,8 +1,10 @@import logging+from typing import Union-from api_v2.exceptions import Forbidden, UnauthorizedKey+from api_v2.exceptions import UnauthorizedKeyfrom api_v2.models import APIDeployment, APIKeyfrom api_v2.serializers import APIKeySerializer+from pipeline_v2.models import Pipelinefrom workflow_manager.workflow_v2.workflow_helper import WorkflowHelperlogger = logging.getLogger(__name__)@@ -10,24 +12,24 @@ logger = logging.getLogger(__name__)class KeyHelper:@staticmethod- def validate_api_key(api_key: str, api_instance: APIDeployment) -> None:+ def validate_api_key(+ api_key: str, instance: Union[APIDeployment, Pipeline]+ ) -> None:"""Validate api key.Args:api_key (str): api key from request- api_instance (APIDeployment): api deployment instance+ instance (Union[APIDeployment, Pipeline]): api or pipeline instanceRaises:- Forbidden: _description_+ UnauthorizedKey: if not valid"""try:api_key_instance: APIKey = APIKey.objects.get(api_key=api_key)- if not KeyHelper.has_access(api_key_instance, api_instance):+ if not KeyHelper.has_access(api_key_instance, instance):raise UnauthorizedKey()except APIKey.DoesNotExist:raise UnauthorizedKey()- except APIDeployment.DoesNotExist:- raise Forbidden("API not found.")@staticmethoddef list_api_keys_of_api(api_instance: APIDeployment) -> list[APIKey]:@@ -35,21 +37,28 @@ class KeyHelper:return api_keys@staticmethod- def has_access(api_key: APIKey, api_instance: APIDeployment) -> bool:+ def list_api_keys_of_pipeline(pipeline_instance: Pipeline) -> list[APIKey]:+ api_keys: list[APIKey] = APIKey.objects.filter(pipeline=pipeline_instance).all()+ return api_keys++ @staticmethod+ def has_access(api_key: APIKey, instance: Union[APIDeployment, Pipeline]) -> bool:"""Check if the provided API key has access to the specified APIinstance.Args:- api_key (APIKey): api key associated with the api- api_instance (APIDeployment): api model+ api_key (APIKey): api key associated with the instance+ instance (Union[APIDeployment, Pipeline]): api or pipeline instanceReturns:bool: True if allowed to execute, False otherwise"""if not api_key.is_active:return False- if isinstance(api_key.api, APIDeployment):- return api_key.api == api_instance+ if isinstance(instance, APIDeployment):+ return api_key.api == instance+ if isinstance(instance, Pipeline):+ return api_key.pipeline == instancereturn False@staticmethod@@ -59,12 +68,11 @@ class KeyHelper:WorkflowHelper.get_workflow_by_id(workflow_id)@staticmethod- def create_api_key(deployment: APIDeployment) -> APIKey:- """Create an APIKey entity with the data from the provided- APIDeployment instance."""- # Create an instance of the APIKey model+ def create_api_key(deployment: Union[APIDeployment, Pipeline]) -> APIKey:+ """Create an APIKey entity using the data from the provided+ APIDeployment or Pipeline instance."""api_key_serializer = APIKeySerializer(- data={"api": deployment.id, "description": "Initial Access Key"},+ data=deployment.api_key_data,context={"deployment": deployment},)api_key_serializer.is_valid(raise_exception=True)diff --git a/backend/api_v2/models.py b/backend/api_v2/models.pyindex 6cb99a9f5..28aef3ff5 100644--- a/backend/api_v2/models.py+++ b/backend/api_v2/models.py@@ -4,6 +4,7 @@ from typing import Anyfrom account_v2.models import Userfrom api_v2.constants import ApiExecutionfrom django.db import models+from pipeline_v2.models import Pipelinefrom utils.models.base_model import BaseModelfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,@@ -44,6 +45,8 @@ class APIDeployment(DefaultOrganizationMixin, BaseModel):default=True,db_comment="Flag indicating whether the API is active or not.",)+ # TODO: Implement dynamic generation of API endpoints for API deployments+ # instead of persisting them in the database.api_endpoint = models.CharField(max_length=API_ENDPOINT_MAX_LENGTH,unique=True,@@ -75,6 +78,10 @@ class APIDeployment(DefaultOrganizationMixin, BaseModel):# Managerobjects = APIDeploymentModelManager()+ @property+ def api_key_data(self):+ return {"api": self.id, "description": f"API Key for {self.api_name}"}+def __str__(self) -> str:return f"{self.id} - {self.display_name}"@@ -129,9 +136,18 @@ class APIKey(BaseModel):api = models.ForeignKey(APIDeployment,on_delete=models.CASCADE,+ null=True,+ blank=True,db_comment="Foreign key reference to the APIDeployment model.",related_name="api_keys",)+ pipeline = models.ForeignKey(+ Pipeline,+ on_delete=models.CASCADE,+ null=True,+ blank=True,+ db_comment="Foreign key reference to the Pipeline model.",+ )description = models.CharField(max_length=DESCRIPTION_MAX_LENGTH,null=True,diff --git a/backend/api_v2/notification.py b/backend/api_v2/notification.pynew file mode 100644index 000000000..0267d098d--- /dev/null+++ b/backend/api_v2/notification.py@@ -0,0 +1,37 @@+import logging++from api_v2.models import APIDeployment+from notification_v2.helper import NotificationHelper+from notification_v2.models import Notification+from pipeline_v2.dto import PipelineStatusPayload+from workflow_manager.workflow_v2.models.execution import WorkflowExecution++logger = logging.getLogger(__name__)+++class APINotification:+ def __init__(+ self, api: APIDeployment, workflow_execution: WorkflowExecution+ ) -> None:+ self.notifications = Notification.objects.filter(api=api, is_active=True)+ self.api = api+ self.workflow_execution = workflow_execution++ def send(self):+ if not self.notifications.count():+ logger.info(f"No notifications found for api {self.api}")+ return+ logger.info(f"Sending api status notification for api {self.api}")++ payload_dto = PipelineStatusPayload(+ type="API",+ pipeline_id=self.api.id,+ pipeline_name=self.api.api_name,+ status=self.workflow_execution.status,+ execution_id=self.workflow_execution.id,+ error_message=self.workflow_execution.error_message,+ )++ NotificationHelper.send_notification(+ notifications=self.notifications, payload=payload_dto.to_dict()+ )diff --git a/backend/api_v2/postman_collection/constants.py b/backend/api_v2/postman_collection/constants.pyindex ffb3ef729..2374c478b 100644--- a/backend/api_v2/postman_collection/constants.py+++ b/backend/api_v2/postman_collection/constants.py@@ -1,6 +1,7 @@class CollectionKey:POSTMAN_COLLECTION_V210 = "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" # noqa: E501EXECUTE_API_KEY = "Process document"+ EXECUTE_PIPELINE_API_KEY = "Process pipeline"STATUS_API_KEY = "Execution status"STATUS_EXEC_ID_DEFAULT = "REPLACE_WITH_EXECUTION_ID"AUTH_QUERY_PARAM_DEFAULT = "REPLACE_WITH_API_KEY"diff --git a/backend/api_v2/postman_collection/dto.py b/backend/api_v2/postman_collection/dto.pyindex 08f2294f1..a9ffd01f3 100644--- a/backend/api_v2/postman_collection/dto.py+++ b/backend/api_v2/postman_collection/dto.py@@ -1,11 +1,13 @@+from abc import ABC, abstractmethodfrom dataclasses import asdict, dataclass, field-from typing import Any, Optional+from typing import Any, Optional, Unionfrom urllib.parse import urlencode, urljoinfrom api_v2.constants import ApiExecutionfrom api_v2.models import APIDeploymentfrom api_v2.postman_collection.constants import CollectionKeyfrom django.conf import settings+from pipeline_v2.models import Pipelinefrom utils.request import HTTPMethod@@ -60,6 +62,123 @@ class PostmanInfo:description: str = "Contains APIs meant for using the deployed Unstract API"+class APIBase(ABC):++ @abstractmethod+ def get_form_data_items(self) -> list[FormDataItem]:+ pass++ @abstractmethod+ def get_api_endpoint(self) -> str:+ pass++ @abstractmethod+ def get_postman_items(self) -> list[PostmanItem]:+ pass++ @abstractmethod+ def get_api_key(self) -> str:+ pass++ def get_execute_body(self) -> BodyItem:+ form_data_items = self.get_form_data_items()+ return BodyItem(formdata=form_data_items)++ def get_create_api_request(self) -> RequestItem:+ header_list = [+ HeaderItem(key="Authorization", value=f"Bearer {self.get_api_key()}")+ ]+ abs_api_endpoint = urljoin(settings.WEB_APP_ORIGIN_URL, self.get_api_endpoint())+ return RequestItem(+ method=HTTPMethod.POST,+ header=header_list,+ body=self.get_execute_body(),+ url=abs_api_endpoint,+ )+++@dataclass+class APIDeploymentDto(APIBase):+ display_name: str+ description: str+ api_endpoint: str+ api_key: str++ def get_postman_info(self) -> PostmanInfo:+ return PostmanInfo(name=self.display_name, description=self.description)++ def get_form_data_items(self) -> list[FormDataItem]:+ return [+ FormDataItem(+ key=ApiExecution.FILES_FORM_DATA, type="file", src="/path_to_file"+ ),+ FormDataItem(+ key=ApiExecution.TIMEOUT_FORM_DATA,+ type="text",+ value=ApiExecution.MAXIMUM_TIMEOUT_IN_SEC,+ ),+ FormDataItem(key=ApiExecution.INCLUDE_METADATA, type="text", value="False"),+ ]++ def get_api_key(self) -> str:+ return self.api_key++ def get_api_endpoint(self) -> str:+ return self.api_endpoint++ def _get_status_api_request(self) -> RequestItem:+ header_list = [HeaderItem(key="Authorization", value=f"Bearer {self.api_key}")]+ status_query_param = {+ "execution_id": CollectionKey.STATUS_EXEC_ID_DEFAULT,+ ApiExecution.INCLUDE_METADATA: "False",+ }+ status_query_str = urlencode(status_query_param)+ abs_api_endpoint = urljoin(settings.WEB_APP_ORIGIN_URL, self.api_endpoint)+ status_url = urljoin(abs_api_endpoint, "?" + status_query_str)+ return RequestItem(method=HTTPMethod.GET, header=header_list, url=status_url)++ def get_postman_items(self) -> list[PostmanItem]:+ postman_item_list = [+ PostmanItem(+ name=CollectionKey.EXECUTE_API_KEY,+ request=self.get_create_api_request(),+ ),+ PostmanItem(+ name=CollectionKey.STATUS_API_KEY,+ request=self._get_status_api_request(),+ ),+ ]+ return postman_item_list+++@dataclass+class PipelineDto(APIBase):+ pipeline_name: str+ api_endpoint: str+ api_key: str++ def get_postman_info(self) -> PostmanInfo:+ return PostmanInfo(name=self.pipeline_name, description="")++ def get_form_data_items(self) -> list[FormDataItem]:+ return []++ def get_api_endpoint(self) -> str:+ return self.api_endpoint++ def get_api_key(self) -> str:+ return self.api_key++ def get_postman_items(self) -> list[PostmanItem]:+ postman_item_list = [+ PostmanItem(+ name=CollectionKey.EXECUTE_PIPELINE_API_KEY,+ request=self.get_create_api_request(),+ )+ ]+ return postman_item_list++@dataclassclass PostmanCollection:info: PostmanInfo@@ -68,7 +187,7 @@ class PostmanCollection:@classmethoddef create(cls,- instance: APIDeployment,+ instance: Union[APIDeployment, Pipeline],api_key: str = CollectionKey.AUTH_QUERY_PARAM_DEFAULT,) -> "PostmanCollection":"""Creates a PostmanCollection instance.@@ -84,49 +203,22 @@ class PostmanCollection:Returns:PostmanCollection: Instance representing PostmanCollection"""- postman_info = PostmanInfo(- name=instance.display_name, description=instance.description- )- header_list = [HeaderItem(key="Authorization", value=f"Bearer {api_key}")]- abs_api_endpoint = urljoin(settings.WEB_APP_ORIGIN_URL, instance.api_endpoint)-- # API execution API- execute_body = BodyItem(- formdata=[- FormDataItem(- key=ApiExecution.FILES_FORM_DATA, type="file", src="/path_to_file"- ),- FormDataItem(- key=ApiExecution.TIMEOUT_FORM_DATA,- type="text",- value=ApiExecution.MAXIMUM_TIMEOUT_IN_SEC,- ),- FormDataItem(- key=ApiExecution.INCLUDE_METADATA,- type="text",- value=False,- ),- ]- )- execute_request = RequestItem(- method=HTTPMethod.POST,- header=header_list,- body=execute_body,- url=abs_api_endpoint,- )-- # Status API- status_query_param = {"execution_id": CollectionKey.STATUS_EXEC_ID_DEFAULT}- status_query_str = urlencode(status_query_param)- status_url = urljoin(abs_api_endpoint, "?" + status_query_str)- status_request = RequestItem(- method=HTTPMethod.GET, header=header_list, url=status_url- )-- postman_item_list = [- PostmanItem(name=CollectionKey.EXECUTE_API_KEY, request=execute_request),- PostmanItem(name=CollectionKey.STATUS_API_KEY, request=status_request),- ]+ data_object: APIBase+ if isinstance(instance, APIDeployment):+ data_object = APIDeploymentDto(+ display_name=instance.display_name,+ description=instance.description,+ api_endpoint=instance.api_endpoint,+ api_key=api_key,+ )+ elif isinstance(instance, Pipeline):+ data_object = PipelineDto(+ pipeline_name=instance.pipeline_name,+ api_endpoint=instance.api_endpoint,+ api_key=api_key,+ )+ postman_info: PostmanInfo = data_object.get_postman_info()+ postman_item_list = data_object.get_postman_items()return cls(info=postman_info, item=postman_item_list)def to_dict(self) -> dict[str, Any]:diff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.pyindex 522e5a23e..4273b6093 100644--- a/backend/api_v2/serializers.py+++ b/backend/api_v2/serializers.py@@ -4,6 +4,7 @@ from typing import Any, Unionfrom api_v2.constants import ApiExecutionfrom api_v2.models import APIDeployment, APIKeyfrom django.core.validators import RegexValidator+from pipeline_v2.models import Pipelinefrom rest_framework.serializers import (CharField,IntegerField,@@ -37,15 +38,42 @@ class APIKeySerializer(AuditSerializer):model = APIKeyfields = "__all__"+ def validate(self, data):+ api = data.get("api")+ pipeline = data.get("pipeline")++ if api and pipeline:+ raise ValidationError(+ "Only one of `api` or `pipeline` should be set, not both."+ )+ elif not api and not pipeline:+ raise ValidationError("At least one of `api` or `pipeline` must be set.")++ return data+def to_representation(self, instance: APIKey) -> OrderedDict[str, Any]:"""Override the to_representation method to include additionalcontext."""- context = self.context.get("context", {})- deployment: APIDeployment = context.get("deployment")+ deployment: Union[APIDeployment, Pipeline] = self.context.get("deployment")representation: OrderedDict[str, Any] = super().to_representation(instance)+if deployment:- representation["api"] = deployment.id- representation["description"] = f"API Key for {deployment.name}"+ # Handle APIDeployment and Pipeline separately+ if isinstance(deployment, APIDeployment):+ representation["api"] = deployment.id+ representation["pipeline"] = None+ representation["description"] = f"API Key for {deployment.api_name}"+ elif isinstance(deployment, Pipeline):+ representation["api"] = None+ representation["pipeline"] = deployment.id+ representation["description"] = (+ f"API Key for {deployment.pipeline_name}"+ )+ else:+ raise ValueError(+ "Context must be an instance of APIDeployment or Pipeline"+ )+representation["is_active"] = Truereturn representationdiff --git a/backend/api_v2/urls.py b/backend/api_v2/urls.pyindex 504b13a7f..b987b0754 100644--- a/backend/api_v2/urls.py+++ b/backend/api_v2/urls.py@@ -53,6 +53,9 @@ urlpatterns = format_suffix_patterns(name="download_postman_collection",),path("keys/<uuid:pk>/", key_details, name="key_details"),- path("keys/api/<str:api_id>/", api_key, name="api_key"),+ path("keys/api/<str:api_id>/", api_key, name="api_key_api"),+ path("keys/api/", api_key, name="api_keys_api"),+ path("keys/pipeline/<str:pipeline_id>/", api_key, name="api_key_pipeline"),+ path("keys/pipeline/", api_key, name="api_keys_pipeline"),])diff --git a/backend/api_v2/utils.py b/backend/api_v2/utils.pynew file mode 100644index 000000000..396a81a72--- /dev/null+++ b/backend/api_v2/utils.py@@ -0,0 +1,45 @@+from typing import Optional++from api_v2.models import APIDeployment+from api_v2.notification import APINotification+from workflow_manager.workflow_v2.models.execution import WorkflowExecution+++class APIDeploymentUtils:+ @staticmethod+ def get_api_by_id(api_id: str) -> Optional[APIDeployment]:+ """Retrieves an APIDeployment instance by its unique ID.++ Args:+ api_id (str): The unique identifier of the APIDeployment to retrieve.++ Returns:+ Optional[APIDeployment]: The APIDeployment instance if found,+ otherwise None.+ """+ try:+ api_deployment: APIDeployment = APIDeployment.objects.get(pk=api_id)+ return api_deployment+ except APIDeployment.DoesNotExist:+ return None++ @staticmethod+ def send_notification(+ api: APIDeployment, workflow_execution: WorkflowExecution+ ) -> None:+ """Sends a notification for the specified API deployment and workflow+ execution.++ Args:+ api (APIDeployment): The APIDeployment instance for which the+ notification is being sent.+ workflow_execution (WorkflowExecution): The WorkflowExecution instance+ related to the notification.++ Returns:+ None+ """+ api_notification = APINotification(+ api=api, workflow_execution=workflow_execution+ )+ api_notification.send()diff --git a/backend/notification_v2/__init__.py b/backend/notification_v2/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/notification_v2/apps.py b/backend/notification_v2/apps.pynew file mode 100644index 000000000..7353e3c61--- /dev/null+++ b/backend/notification_v2/apps.py@@ -0,0 +1,6 @@+from django.apps import AppConfig+++class NotificationConfig(AppConfig):+ default_auto_field = "django.db.models.BigAutoField"+ name = "notification_v2"diff --git a/backend/notification_v2/constants.py b/backend/notification_v2/constants.pynew file mode 100644index 000000000..c106997d3--- /dev/null+++ b/backend/notification_v2/constants.py@@ -0,0 +1,5 @@+class NotificationUrlConstant:+ """Constants for Notification Urls."""++ PIPELINE_UID = "pipeline_uuid"+ API_UID = "api_uuid"diff --git a/backend/notification_v2/enums.py b/backend/notification_v2/enums.pynew file mode 100644index 000000000..991b08cac--- /dev/null+++ b/backend/notification_v2/enums.py@@ -0,0 +1,38 @@+from enum import Enum+++class NotificationType(Enum):+ WEBHOOK = "WEBHOOK"+ # Add other notification types as needed+ # Example EMAIL = 'EMAIL'++ def get_valid_platforms(self):+ if self == NotificationType.WEBHOOK:+ return [PlatformType.SLACK.value, PlatformType.API.value]+ return []++ @classmethod+ def choices(cls):+ return [(e.value, e.name.replace("_", " ").capitalize()) for e in cls]+++class AuthorizationType(Enum):+ BEARER = "BEARER"+ API_KEY = "API_KEY"+ CUSTOM_HEADER = "CUSTOM_HEADER"+ NONE = "NONE"++ @classmethod+ def choices(cls):+ return [(e.value, e.name.replace("_", " ").capitalize()) for e in cls]+++class PlatformType(Enum):+ SLACK = "SLACK"+ API = "API"+ # Add other platforms as needed+ # Example TEAMS = 'TEAMS'++ @classmethod+ def choices(cls):+ return [(e.value, e.name.replace("_", " ").capitalize()) for e in cls]diff --git a/backend/notification_v2/helper.py b/backend/notification_v2/helper.pynew file mode 100644index 000000000..3074143c8--- /dev/null+++ b/backend/notification_v2/helper.py@@ -0,0 +1,48 @@+import logging+from typing import Any++from notification_v2.enums import NotificationType, PlatformType+from notification_v2.models import Notification+from notification_v2.provider.notification_provider import NotificationProvider+from notification_v2.provider.registry import get_notification_provider++logger = logging.getLogger(__name__)+++class NotificationHelper:+ @classmethod+ def send_notification(cls, notifications: list[Notification], payload: Any) -> None:+ """Send notification Sends notifications using the appropriate provider+ based on the notification type and platform.++ This method iterates through a list of `Notification` objects, determines the+ appropriate notification provider based on the notification's type and+ platform, and sends the notification with the provided payload. If an error+ occurs due to an invalid notification type or platform, it logs the error.++ Args:+ notifications (list[Notification]): A list of `Notification` instances to+ be processed and sent.+ payload (Any): The data to be sent with the notification. This can be any+ format expected by the provider++ Returns:+ None+ """+ for notification in notifications:+ notification_type = NotificationType(notification.notification_type)+ platform_type = PlatformType(notification.platform)+ try:+ notification_provider = get_notification_provider(+ notification_type, platform_type+ )+ notifier: NotificationProvider = notification_provider(+ notification=notification, payload=payload+ )+ notifier.send()+ logger.info(f"Sending notification to {notification}")+ except ValueError as e:+ logger.error(+ f"Error in notification type {notification_type} and platform "+ f"{platform_type} for notification {notification}: {e}"+ )diff --git a/backend/notification_v2/migrations/__init__.py b/backend/notification_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/notification_v2/models.py b/backend/notification_v2/models.pynew file mode 100644index 000000000..489a8c827--- /dev/null+++ b/backend/notification_v2/models.py@@ -0,0 +1,94 @@+import uuid++from api_v2.models import APIDeployment+from django.db import models+from pipeline_v2.models import Pipeline+from utils.models.base_model import BaseModel++from .enums import AuthorizationType, NotificationType, PlatformType++NOTIFICATION_NAME_MAX_LENGTH = 255+++class Notification(BaseModel):+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)+ name = models.CharField(+ max_length=NOTIFICATION_NAME_MAX_LENGTH,+ db_comment="Name of the notification.",+ default="Notification",+ )+ url = models.URLField(null=True) # URL for webhook or other endpoints+ authorization_key = models.CharField(+ max_length=255, blank=True, null=True+ ) # Authorization Key or API Key+ authorization_header = models.CharField(+ max_length=255, blank=True, null=True+ ) # Header Name for custom headers+ authorization_type = models.CharField(+ max_length=50,+ choices=AuthorizationType.choices(),+ default=AuthorizationType.NONE.value,+ )+ max_retries = models.IntegerField(+ default=0+ ) # Maximum number of times to retry webhook+ platform = models.CharField(+ max_length=50,+ choices=PlatformType.choices(),+ blank=True,+ null=True,+ )+ notification_type = models.CharField(+ max_length=50,+ choices=NotificationType.choices(),+ default=NotificationType.WEBHOOK.value,+ )+ is_active = models.BooleanField(+ default=True,+ db_comment="Flag indicating whether the notification is active or not.",+ )+ # Foreign keys to specific models+ pipeline = models.ForeignKey(+ Pipeline,+ on_delete=models.CASCADE,+ related_name="notifications",+ null=True,+ blank=True,+ )+ api = models.ForeignKey(+ APIDeployment,+ on_delete=models.CASCADE,+ related_name="notifications",+ null=True,+ blank=True,+ )++ class Meta:+ verbose_name = "Notification"+ verbose_name_plural = "Notifications"+ db_table = "notification"+ constraints = [+ models.UniqueConstraint(+ fields=["name", "pipeline"], name="unique_name_pipeline"+ ),+ models.UniqueConstraint(fields=["name", "api"], name="unique_name_api"),+ ]++ def save(self, *args, **kwargs):+ # Validation for platforms+ valid_platforms = NotificationType(self.notification_type).get_valid_platforms()+ if self.platform and self.platform not in valid_platforms:+ raise ValueError(+ f"Invalid platform '{self.platform}' for notification type "+ f"'{self.notification_type}'. "+ f"Valid options are: {', '.join(valid_platforms)}."+ )++ # Allow saving only if the platform is valid or not required+ super().save(*args, **kwargs)++ def __str__(self):+ return (+ f"Notification {self.id}: (Type: {self.notification_type}, "+ f"Platform: {self.platform}, Url: {self.url}))"+ )diff --git a/backend/notification_v2/provider/__init__.py b/backend/notification_v2/provider/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/notification_v2/provider/notification_provider.py b/backend/notification_v2/provider/notification_provider.pynew file mode 100644index 000000000..39ef2d959--- /dev/null+++ b/backend/notification_v2/provider/notification_provider.py@@ -0,0 +1,28 @@+from abc import ABC, abstractmethod++from django.conf import settings+from notification_v2.models import Notification+++class NotificationProvider(ABC):+ NOTIFICATION_TIMEOUT = settings.NOTIFICATION_TIMEOUT+ RETRY_DELAY = 10 # Seconds++ def __init__(self, notification: Notification, payload):+ self.payload = payload+ self.notification = notification++ @abstractmethod+ def send(self):+ """Method to be overridden in child classes for sending the+ notification."""+ raise NotImplementedError("Subclasses should implement this method.")++ def validate(self):+ """Method to validate the notification data."""+ pass++ @abstractmethod+ def get_headers(self):+ """Method to get the headers for the notification."""+ raise NotImplementedError("Subclasses should implement this method.")diff --git a/backend/notification_v2/provider/registry.py b/backend/notification_v2/provider/registry.pynew file mode 100644index 000000000..909ad58b4--- /dev/null+++ b/backend/notification_v2/provider/registry.py@@ -0,0 +1,54 @@+from notification_v2.enums import NotificationType, PlatformType+from notification_v2.provider.notification_provider import NotificationProvider+from notification_v2.provider.webhook.api_webhook import APIWebhook+from notification_v2.provider.webhook.slack_webhook import SlackWebhook++REGISTRY = {+ NotificationType.WEBHOOK: {+ PlatformType.SLACK: SlackWebhook,+ PlatformType.API: APIWebhook,+ # Add other platform-specific classes here+ },+ # Add other notification types and classes here+}+++def get_notification_provider(+ notification_type: NotificationType, platform_type: PlatformType+) -> NotificationProvider:+ """Get Notification provider based on notification type and platform type+ It uses the REGISTRY to map the combination of notification type and+ platform type to the corresponding NotificationProvider class.++ If the provided combination is not found in the REGISTRY, a ValueError is raised.++ Note:+ This function assumes that the REGISTRY dictionary is correctly populated+ with the appropriate NotificationProvider classes for each combination of+ notification type and platform type.++ See Also:+ - NotificationType+ - PlatformType+ - NotificationProvider+ - REGISTRY++ Parameters:+ notification_type (NotificationType): The type of notification.+ platform_type (PlatformType): The platform/provider type for the notification.++ Returns:+ NotificationProvider: The appropriate NotificationProvider class for+ the given combination.++ Raises:+ ValueError: If the provided combination is not found in the REGISTRY.+ """+ if notification_type not in REGISTRY:+ raise ValueError(f"Unsupported notification type: {notification_type}")++ platform_registry = REGISTRY[notification_type]+ if platform_type not in platform_registry:+ raise ValueError(f"Unsupported platform type: {platform_type}")++ return platform_registry[platform_type]diff --git a/backend/notification_v2/provider/webhook/__init__.py b/backend/notification_v2/provider/webhook/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/notification_v2/provider/webhook/api_webhook.py b/backend/notification_v2/provider/webhook/api_webhook.pynew file mode 100644index 000000000..9f1264b9e--- /dev/null+++ b/backend/notification_v2/provider/webhook/api_webhook.py@@ -0,0 +1,13 @@+from notification_v2.provider.webhook.webhook import Webhook+++class APIWebhook(Webhook):+ def send(self):+ """Send the API webhook notification."""+ super().send()++ def get_headers(self):+ """API-specific headers."""+ headers = super().get_headers()+ headers["Content-Type"] = "application/json"+ return headersdiff --git a/backend/notification_v2/provider/webhook/slack_webhook.py b/backend/notification_v2/provider/webhook/slack_webhook.pynew file mode 100644index 000000000..45e260b68--- /dev/null+++ b/backend/notification_v2/provider/webhook/slack_webhook.py@@ -0,0 +1,60 @@+import logging++from notification_v2.provider.webhook.webhook import Webhook++logger = logging.getLogger(__name__)+++class SlackWebhook(Webhook):+ def send(self):+ """Send the Slack webhook notification."""+ formatted_payload = self.format_payload()+ self.payload = formatted_payload+ super().send()++ def get_headers(self):+ """Slack-specific headers."""+ headers = super().get_headers()+ headers["Content-Type"] = "application/json"+ return headers++ def format_payload(self) -> dict:+ """Format the payload to match Slack's expected structure."""+ if "text" not in self.payload:+ # Construct a basic Slack message with 'text' field+ formatted_payload = {+ "text": "Notification",+ "blocks": self.create_blocks_from_payload(),+ }+ else:+ # If 'text' is already present, format accordingly+ formatted_payload = {+ "text": self.payload.pop("text"),+ "blocks": self.create_blocks_from_payload(),+ }+ return formatted_payload++ def create_blocks_from_payload(self) -> list:+ """Create Slack blocks from the given payload."""+ blocks = []+ # Header+ blocks.append(+ {+ "type": "section",+ "text": {"type": "mrkdwn", "text": "*Unstract Update:*"},+ }+ )+ # Add a divider for separation+ blocks.append({"type": "divider"})+ # Add each key-value pair to the blocks+ for key, value in self.payload.items():+ formatted_key = key.replace("_", " ").title()+ blocks.append(+ {+ "type": "section",+ "text": {"type": "mrkdwn", "text": f"*{formatted_key}:* {value}"},+ }+ )+ # Footer+ blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": "*---*"}})+ return blocksdiff --git a/backend/notification_v2/provider/webhook/webhook.py b/backend/notification_v2/provider/webhook/webhook.pynew file mode 100644index 000000000..5bcec1964--- /dev/null+++ b/backend/notification_v2/provider/webhook/webhook.py@@ -0,0 +1,154 @@+import logging+from typing import Any, Optional++import requests+from celery import shared_task+from notification_v2.enums import AuthorizationType+from notification_v2.provider.notification_provider import NotificationProvider++logger = logging.getLogger(__name__)+++class WebhookNotificationArg:+ MAX_RETRIES = "max_retries"+ RETRY_DELAY = "retry_delay"+++class HeaderConstants:+ APPLICATION_JSON = "application/json"+++class Webhook(NotificationProvider):+ def send(self):+ """Send the webhook notification."""+ try:+ headers = self.get_headers()+ self.validate()+ except ValueError as e:+ logger.error(f"Error validating notification {self.notification} :: {e}")+ return+ send_webhook_notification.apply_async(+ (self.notification.url, self.payload, headers, self.NOTIFICATION_TIMEOUT),+ kwargs={+ WebhookNotificationArg.MAX_RETRIES: self.notification.max_retries,+ WebhookNotificationArg.RETRY_DELAY: self.RETRY_DELAY,+ },+ )++ def validate(self):+ """Validate notification.++ Returns:+ _type_: None+ """+ if not self.notification.url:+ raise ValueError("Webhook URL is required.")+ if not self.payload:+ raise ValueError("Payload is required.")+ return super().validate()++ def get_headers(self):+ """+ Get the headers for the notification based on the authorization type and key.+ Raises:+ ValueError: _description_++ Returns:+ dict[str, str]: A dictionary containing the headers.+ """+ headers = {}+ try:+ authorization_type = AuthorizationType(+ self.notification.authorization_type.upper()+ )+ except ValueError:+ raise ValueError(+ "Unsupported authorization type: "+ f"{self.notification.authorization_type}"+ )+ authorization_key = self.notification.authorization_key+ authorization_header = self.notification.authorization_header++ header_formats = {+ AuthorizationType.BEARER: lambda key: {+ "Authorization": f"Bearer {key}",+ "Content-Type": HeaderConstants.APPLICATION_JSON,+ },+ AuthorizationType.API_KEY: lambda key: {+ "Authorization": key,+ "Content-Type": HeaderConstants.APPLICATION_JSON,+ },+ AuthorizationType.CUSTOM_HEADER: lambda key: {+ authorization_header: key,+ "Content-Type": HeaderConstants.APPLICATION_JSON,+ },+ AuthorizationType.NONE: lambda _: {+ "Content-Type": HeaderConstants.APPLICATION_JSON,+ },+ }++ if authorization_type not in header_formats:+ raise ValueError(f"Unsupported authorization type: {authorization_type}")++ headers = header_formats[authorization_type](authorization_key)++ # Check if custom header type has required details+ if authorization_type == AuthorizationType.CUSTOM_HEADER:+ if not authorization_header or not authorization_key:+ raise ValueError(+ "Custom header or key missing for custom authorization."+ )+ return headers+++@shared_task(bind=True, name="send_webhook_notification")+def send_webhook_notification(+ self,+ url: str,+ payload: Any,+ headers: Any = None,+ timeout: int = 10,+ max_retries: Optional[int] = None,+ retry_delay: int = 10,+):+ """Celery task to send a webhook with retries and error handling.++ Args:+ url (str): The URL to which the webhook should be sent.+ payload (dict): The payload to be sent in the webhook request.+ headers (dict, optional): Optional headers to include in the request.+ Defaults to None.+ timeout (int, optional): The request timeout in seconds. Defaults to 10.+ max_retries (int, optional): The maximum number of retries allowed.+ Defaults to None.+ retry_delay (int, optional): The delay between retries in seconds.+ Defaults to 10.++ Returns:+ None+ """+ try:+ response = requests.post(url, json=payload, headers=headers, timeout=timeout)+ response.raise_for_status()+ if not (200 <= response.status_code < 300):+ logger.error(+ f"Request to {url} failed with status code {response.status_code}. "+ f"Response: {response.text}"+ )+ except requests.exceptions.RequestException as exc:+ if max_retries is not None:+ if self.request.retries < max_retries:+ logger.warning(+ f"Request to {url} failed. Retrying in {retry_delay} seconds. "+ f"Attempt {self.request.retries + 1}/{max_retries}. Error: {exc}"+ )+ raise self.retry(exc=exc, countdown=retry_delay)+ else:+ logger.error(+ f"Failed to send webhook to {url} after {max_retries} attempts. "+ f"Error: {exc}"+ )+ return None+ else:+ logger.error(f"Webhook request to {url} failed with error: {exc}")+ return Nonediff --git a/backend/notification_v2/serializers.py b/backend/notification_v2/serializers.pynew file mode 100644index 000000000..a5e405a52--- /dev/null+++ b/backend/notification_v2/serializers.py@@ -0,0 +1,130 @@+from rest_framework import serializers++from .enums import AuthorizationType, NotificationType, PlatformType+from .models import Notification+++class NotificationSerializer(serializers.ModelSerializer):+ notification_type = serializers.ChoiceField(choices=NotificationType.choices())+ authorization_type = serializers.ChoiceField(choices=AuthorizationType.choices())+ platform = serializers.ChoiceField(choices=PlatformType.choices(), required=False)+ max_retries = serializers.IntegerField(+ max_value=4, min_value=0, default=0, required=False+ )++ class Meta:+ model = Notification+ fields = "__all__"++ def validate(self, data):+ """Validate the data for the NotificationSerializer."""+ # General validation for the relationship between api and pipeline+ self._validate_api_or_pipeline(data)+ self._validate_authorization(data)+ return data++ def _validate_api_or_pipeline(self, data):+ """Ensure either 'api' or 'pipeline' is provided, but not both."""+ api = data.get("api", getattr(self.instance, "api", None))+ pipeline = data.get("pipeline", getattr(self.instance, "pipeline", None))+ if api and pipeline:+ raise serializers.ValidationError(+ "Only one of 'api' or 'pipeline' can be provided."+ )++ if not api and not pipeline:+ raise serializers.ValidationError(+ "Either 'api' or 'pipeline' must be provided."+ )++ def _validate_authorization(self, data):+ """Ensure required authorization fields are provided based on the+ authorization type.++ Getting existing data in the case of PATCH request+ """+ authorization_type = data.get(+ "authorization_type", getattr(self.instance, "authorization_type", None)+ )+ authorization_key = data.get(+ "authorization_key", getattr(self.instance, "authorization_key", None)+ )+ authorization_header = data.get(+ "authorization_header", getattr(self.instance, "authorization_header", None)+ )++ try:+ authorization_type_enum = AuthorizationType(authorization_type)+ except ValueError:+ raise serializers.ValidationError(+ f"Invalid authorization type '{authorization_type}'."+ )++ if authorization_type_enum in [+ AuthorizationType.BEARER,+ AuthorizationType.API_KEY,+ AuthorizationType.CUSTOM_HEADER,+ ]:+ if not authorization_key:+ raise serializers.ValidationError(+ {+ "authorization_key": (+ "Authorization key is required for authorization "+ f"type '{authorization_type_enum.value}'."+ )+ }+ )++ if (+ authorization_type_enum == AuthorizationType.CUSTOM_HEADER+ and not authorization_header+ ):+ raise serializers.ValidationError(+ {+ "authorization_header": (+ "Authorization header is required when using "+ "CUSTOM_HEADER authorization type."+ )+ }+ )++ def validate_platform(self, value):+ """Validate the platform field based on the notification_type."""+ notification_type = self.initial_data.get(+ "notification_type", getattr(self.instance, "notification_type", None)+ )+ if not notification_type:+ raise serializers.ValidationError("Notification type must be provided.")++ valid_platforms = NotificationType(notification_type).get_valid_platforms()+ if value and value not in valid_platforms:+ raise serializers.ValidationError(+ f"Invalid platform '{value}' for notification type "+ f"'{notification_type}'. "+ f"Valid options are: {', '.join(valid_platforms)}."+ )+ return value++ def validate_name(self, value):+ """Check uniqueness of the name with respect to either 'api' or+ 'pipeline'."""+ api = self.initial_data.get("api", getattr(self.instance, "api", None))+ pipeline = self.initial_data.get(+ "pipeline", getattr(self.instance, "pipeline", None)+ )++ queryset = Notification.objects.filter(name=value)+ if self.instance:+ queryset = queryset.exclude(id=self.instance.id)++ if api and queryset.filter(api=api).exists():+ raise serializers.ValidationError(+ "A notification with this name and API already exists.",+ code="unique_api",+ )+ elif pipeline and queryset.filter(pipeline=pipeline).exists():+ raise serializers.ValidationError(+ "A notification with this name and pipeline already exists.",+ code="unique_pipeline",+ )+ return valuediff --git a/backend/notification_v2/urls.py b/backend/notification_v2/urls.pynew file mode 100644index 000000000..2e356b400--- /dev/null+++ b/backend/notification_v2/urls.py@@ -0,0 +1,27 @@+from django.urls import path+from rest_framework.urlpatterns import format_suffix_patterns++from .views import NotificationViewSet++notification_list = NotificationViewSet.as_view({"get": "list", "post": "create"})+notification_detail = NotificationViewSet.as_view(+ {+ "get": "retrieve",+ "put": "update",+ "patch": "partial_update",+ "delete": "destroy",+ }+)++urlpatterns = format_suffix_patterns(+ [+ path("", notification_list, name="notification-list"),+ path("<uuid:pk>/", notification_detail, name="notification-detail"),+ path(+ "pipeline/<uuid:pipeline_uuid>/",+ notification_list,+ name="pipeline-notification-list",+ ),+ path("api/<uuid:api_uuid>/", notification_list, name="api-notification-list"),+ ]+)diff --git a/backend/notification_v2/views.py b/backend/notification_v2/views.pynew file mode 100644index 000000000..f83f9437c--- /dev/null+++ b/backend/notification_v2/views.py@@ -0,0 +1,36 @@+from api_v2.deployment_helper import DeploymentHelper+from api_v2.exceptions import APINotFound+from notification_v2.constants import NotificationUrlConstant+from pipeline_v2.exceptions import PipelineNotFound+from pipeline_v2.models import Pipeline+from pipeline_v2.pipeline_processor import PipelineProcessor+from rest_framework import viewsets++from .models import Notification+from .serializers import NotificationSerializer+++class NotificationViewSet(viewsets.ModelViewSet):+ serializer_class = NotificationSerializer++ def get_queryset(self):+ queryset = Notification.objects.all()+ pipeline_uuid = self.kwargs.get(NotificationUrlConstant.PIPELINE_UID)+ api_uuid = self.kwargs.get(NotificationUrlConstant.API_UID)++ if pipeline_uuid:+ try:+ pipeline = PipelineProcessor.fetch_pipeline(+ pipeline_id=pipeline_uuid, check_active=False+ )+ queryset = queryset.filter(pipeline=pipeline)+ except Pipeline.DoesNotExist:+ raise PipelineNotFound()++ elif api_uuid:+ api = DeploymentHelper.get_api_by_id(api_id=api_uuid)+ if not api:+ raise APINotFound()+ queryset = queryset.filter(api=api)++ return queryset