Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[MISC] Removed stale code (#1419)

Chandrasekharan M20e162ee5e89
diff --git a/backend/connector_v2/connector_instance_helper.py b/backend/connector_v2/connector_instance_helper.py
deleted file mode 100644
index 2c77e9aae..000000000
--- a/backend/connector_v2/connector_instance_helper.py
+++ /dev/null
@@ -1,329 +0,0 @@
-import logging
-from typing import Any
-
-from account_v2.models import User
-from django.conf import settings
-from utils.user_context import UserContext
-from workflow_manager.workflow_v2.models.workflow import Workflow
-
-from connector_v2.constants import ConnectorInstanceConstant
-from connector_v2.models import ConnectorInstance
-from connector_v2.unstract_account import UnstractAccount
-from unstract.connectors.filesystems.ucs import UnstractCloudStorage
-from unstract.connectors.filesystems.ucs.constants import UCSKey
-
-logger = logging.getLogger(__name__)
-
-
-class ConnectorInstanceHelper:
- @staticmethod
- def create_default_gcs_connector(workflow: Workflow, user: User) -> None:
- """Method to create default storage connector.
-
- Args:
- org_id (str)
- workflow (Workflow)
- user (User)
- """
- organization_id = UserContext.get_organization_identifier()
- if not user.project_storage_created:
- logger.info("Creating default storage")
- account = UnstractAccount(organization_id, user.email)
- account.provision_s3_storage()
- account.upload_sample_files()
- user.project_storage_created = True
- user.save()
- logger.info("default storage created successfully.")
-
- logger.info("Adding connectors to Unstract")
- connector_name = ConnectorInstanceConstant.USER_STORAGE
- gcs_id = UnstractCloudStorage.get_id()
- bucket_name = settings.UNSTRACT_FREE_STORAGE_BUCKET_NAME
- base_path = f"{bucket_name}/{organization_id}/{user.email}"
-
- connector_metadata = {
- UCSKey.KEY: settings.GOOGLE_STORAGE_ACCESS_KEY_ID,
- UCSKey.SECRET: settings.GOOGLE_STORAGE_SECRET_ACCESS_KEY,
- UCSKey.ENDPOINT_URL: settings.GOOGLE_STORAGE_BASE_URL,
- }
- connector_metadata__input = {
- **connector_metadata,
- UCSKey.PATH: base_path + "/input",
- }
- connector_metadata__output = {
- **connector_metadata,
- UCSKey.PATH: base_path + "/output",
- }
- ConnectorInstance.objects.create(
- connector_name=connector_name,
- workflow=workflow,
- created_by=user,
- connector_id=gcs_id,
- connector_metadata=connector_metadata__input,
- connector_type=ConnectorInstance.ConnectorType.INPUT,
- connector_mode=ConnectorInstance.ConnectorMode.FILE_SYSTEM,
- )
- ConnectorInstance.objects.create(
- connector_name=connector_name,
- workflow=workflow,
- created_by=user,
- connector_id=gcs_id,
- connector_metadata=connector_metadata__output,
- connector_type=ConnectorInstance.ConnectorType.OUTPUT,
- connector_mode=ConnectorInstance.ConnectorMode.FILE_SYSTEM,
- )
- logger.info("Connectors added successfully.")
-
- @staticmethod
- def get_connector_instances_by_workflow(
- workflow_id: str,
- connector_type: tuple[str, str],
- connector_mode: tuple[int, str] | None = None,
- values: list[str] | None = None,
- connector_name: str | None = None,
- ) -> list[ConnectorInstance]:
- """Method to get connector instances by workflow.
-
- Args:
- workflow_id (str)
- connector_type (tuple[str, str]): Specifies input/output
- connector_mode (Optional[tuple[int, str]], optional):
- Specifies database/file
- values (Optional[list[str]], optional): Defaults to None.
- connector_name (Optional[str], optional): Defaults to None.
-
- Returns:
- list[ConnectorInstance]
- """
- logger.info(f"Setting connector mode to {connector_mode}")
- filter_params: dict[str, Any] = {
- "workflow": workflow_id,
- "connector_type": connector_type,
- }
- if connector_mode is not None:
- filter_params["connector_mode"] = connector_mode
- if connector_name is not None:
- filter_params["connector_name"] = connector_name
-
- connector_instances = ConnectorInstance.objects.filter(**filter_params).all()
- logger.debug(f"Retrieved connector instance values {connector_instances}")
- if values is not None:
- filtered_connector_instances = connector_instances.values(*values)
- logger.info(
- f"Returning filtered \
- connector instance value {filtered_connector_instances}"
- )
- return list(filtered_connector_instances)
- logger.info(f"Returning connector instances {connector_instances}")
- return list(connector_instances)
-
- @staticmethod
- def get_connector_instance_by_workflow(
- workflow_id: str,
- connector_type: tuple[str, str],
- connector_mode: tuple[int, str] | None = None,
- connector_name: str | None = None,
- ) -> ConnectorInstance | None:
- """Get one connector instance.
-
- Use this method if the connector instance is unique for \
- filter_params
- Args:
- workflow_id (str): _description_
- connector_type (tuple[str, str]): Specifies input/output
- connector_mode (Optional[tuple[int, str]], optional).
- Specifies database/filesystem.
- connector_name (Optional[str], optional).
-
- Returns:
- list[ConnectorInstance]: _description_
- """
- logger.info("Fetching connector instance by workflow")
- filter_params: dict[str, Any] = {
- "workflow": workflow_id,
- "connector_type": connector_type,
- }
- if connector_mode is not None:
- filter_params["connector_mode"] = connector_mode
- if connector_name is not None:
- filter_params["connector_name"] = connector_name
-
- try:
- connector_instance: ConnectorInstance = ConnectorInstance.objects.filter(
- **filter_params
- ).first()
- except Exception as exc:
- logger.error(f"Error occured while fetching connector instances {exc}")
- raise exc
-
- return connector_instance
-
- @staticmethod
- def get_input_connector_instance_by_name_for_workflow(
- workflow_id: str,
- connector_name: str,
- ) -> ConnectorInstance | None:
- """Method to get Input connector instance name from the workflow.
-
- Args:
- workflow_id (str)
- connector_name (str)
-
- Returns:
- Optional[ConnectorInstance]
- """
- return ConnectorInstanceHelper.get_connector_instance_by_workflow(
- workflow_id=workflow_id,
- connector_type=ConnectorInstance.ConnectorType.INPUT,
- connector_name=connector_name,
- )
-
- @staticmethod
- def get_output_connector_instance_by_name_for_workflow(
- workflow_id: str,
- connector_name: str,
- ) -> ConnectorInstance | None:
- """Method to get output connector name by Workflow.
-
- Args:
- workflow_id (str)
- connector_name (str)
-
- Returns:
- Optional[ConnectorInstance]
- """
- return ConnectorInstanceHelper.get_connector_instance_by_workflow(
- workflow_id=workflow_id,
- connector_type=ConnectorInstance.ConnectorType.OUTPUT,
- connector_name=connector_name,
- )
-
- @staticmethod
- def get_input_connector_instances_by_workflow(
- workflow_id: str,
- ) -> list[ConnectorInstance]:
- """Method to get connector instances by workflow.
-
- Args:
- workflow_id (str)
-
- Returns:
- list[ConnectorInstance]
- """
- return ConnectorInstanceHelper.get_connector_instances_by_workflow(
- workflow_id, ConnectorInstance.ConnectorType.INPUT
- )
-
- @staticmethod
- def get_output_connector_instances_by_workflow(
- workflow_id: str,
- ) -> list[ConnectorInstance]:
- """Method to get output connector instances by workflow.
-
- Args:
- workflow_id (str): _description_
-
- Returns:
- list[ConnectorInstance]: _description_
- """
- return ConnectorInstanceHelper.get_connector_instances_by_workflow(
- workflow_id, ConnectorInstance.ConnectorType.OUTPUT
- )
-
- @staticmethod
- def get_file_system_input_connector_instances_by_workflow(
- workflow_id: str, values: list[str] | None = None
- ) -> list[ConnectorInstance]:
- """Method to fetch file system connector by workflow.
-
- Args:
- workflow_id (str):
- values (Optional[list[str]], optional)
-
- Returns:
- list[ConnectorInstance]
- """
- return ConnectorInstanceHelper.get_connector_instances_by_workflow(
- workflow_id,
- ConnectorInstance.ConnectorType.INPUT,
- ConnectorInstance.ConnectorMode.FILE_SYSTEM,
- values,
- )
-
- @staticmethod
- def get_file_system_output_connector_instances_by_workflow(
- workflow_id: str, values: list[str] | None = None
- ) -> list[ConnectorInstance]:
- """Method to get file system output connector by workflow.
-
- Args:
- workflow_id (str)
- values (Optional[list[str]], optional)
-
- Returns:
- list[ConnectorInstance]
- """
- return ConnectorInstanceHelper.get_connector_instances_by_workflow(
- workflow_id,
- ConnectorInstance.ConnectorType.OUTPUT,
- ConnectorInstance.ConnectorMode.FILE_SYSTEM,
- values,
- )
-
- @staticmethod
- def get_database_input_connector_instances_by_workflow(
- workflow_id: str, values: list[str] | None = None
- ) -> list[ConnectorInstance]:
- """Method to fetch input database connectors by workflow.
-
- Args:
- workflow_id (str)
- values (Optional[list[str]], optional)
-
- Returns:
- list[ConnectorInstance]
- """
- return ConnectorInstanceHelper.get_connector_instances_by_workflow(
- workflow_id,
- ConnectorInstance.ConnectorType.INPUT,
- ConnectorInstance.ConnectorMode.DATABASE,
- values,
- )
-
- @staticmethod
- def get_database_output_connector_instances_by_workflow(
- workflow_id: str, values: list[str] | None = None
- ) -> list[ConnectorInstance]:
- """Method to fetch output database connectors by workflow.
-
- Args:
- workflow_id (str)
- values (Optional[list[str]], optional)
-
- Returns:
- list[ConnectorInstance]
- """
- return ConnectorInstanceHelper.get_connector_instances_by_workflow(
- workflow_id,
- ConnectorInstance.ConnectorType.OUTPUT,
- ConnectorInstance.ConnectorMode.DATABASE,
- values,
- )
-
- @staticmethod
- def get_input_output_connector_instances_by_workflow(
- workflow_id: str,
- ) -> list[ConnectorInstance]:
- """Method to fetch input and output connectors by workflow.
-
- Args:
- workflow_id (str)
-
- Returns:
- list[ConnectorInstance]
- """
- filter_params: dict[str, Any] = {
- "workflow": workflow_id,
- }
- connector_instances = ConnectorInstance.objects.filter(**filter_params).all()
- return list(connector_instances)
diff --git a/backend/connector_v2/models.py b/backend/connector_v2/models.py
index bca57c34e..dbeaeb019 100644
--- a/backend/connector_v2/models.py
+++ b/backend/connector_v2/models.py
@@ -109,8 +109,7 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):
def __str__(self) -> str:
return (
- f"Connector({self.id}, type{self.connector_type},"
- f" workflow: {self.workflow})"
+ f"Connector({self.id}, type{self.connector_type}, workflow: {self.workflow})"
)
def get_connector_metadata_bytes(self):
diff --git a/backend/pipeline_v2/constants.py b/backend/pipeline_v2/constants.py
index 438982577..a234983b6 100644
--- a/backend/pipeline_v2/constants.py
+++ b/backend/pipeline_v2/constants.py
@@ -17,10 +17,6 @@ class PipelineConstants:
DESTINATION_ICON = "destination_icon"
SOURCE_NAME = "source_name"
DESTINATION_NAME = "destination_name"
- INPUT_FILE = "input_file_connector"
- INPUT_DB = "input_db_connector"
- OUTPUT_FILE = "output_file_connector"
- OUTPUT_DB = "output_db_connector"
SOURCE = "source"
DEST = "dest"
diff --git a/backend/pipeline_v2/serializers/crud.py b/backend/pipeline_v2/serializers/crud.py
index fb04dae15..a2cba45c3 100644
--- a/backend/pipeline_v2/serializers/crud.py
+++ b/backend/pipeline_v2/serializers/crud.py
@@ -3,7 +3,6 @@ from collections import OrderedDict
from typing import Any
from connector_processor.connector_processor import ConnectorProcessor
-from connector_v2.connector_instance_helper import ConnectorInstanceHelper
from connector_v2.models import ConnectorInstance
from croniter import croniter
from django.conf import settings
@@ -181,11 +180,9 @@ class PipelineSerializer(IntegrityErrorMixin, AuditSerializer):
if SerializerUtils.check_context_for_GET_or_POST(context=self.context):
workflow = instance.workflow
- connector_instance_list = (
- ConnectorInstanceHelper.get_input_output_connector_instances_by_workflow( # noqa
- workflow.id
- )
- )
+ connector_instance_list = ConnectorInstance.objects.filter(
+ workflow=workflow.id
+ ).all()
repr[PK.WORKFLOW_ID] = workflow.id
repr[PK.WORKFLOW_NAME] = workflow.workflow_name
repr[PK.CRON_STRING] = repr.pop(PK.CRON_STRING)
diff --git a/backend/project/__init__.py b/backend/project/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/backend/project/admin.py b/backend/project/admin.py
deleted file mode 100644
index badc1bba2..000000000
--- a/backend/project/admin.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from django.contrib import admin
-
-from .models import Project
-
-admin.site.register(Project)
diff --git a/backend/project/apps.py b/backend/project/apps.py
deleted file mode 100644
index 4836c68a9..000000000
--- a/backend/project/apps.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from django.apps import AppConfig
-
-
-class ProjectConfig(AppConfig):
- name = "project"
diff --git a/backend/project/constants.py b/backend/project/constants.py
deleted file mode 100644
index a80602e55..000000000
--- a/backend/project/constants.py
+++ /dev/null
@@ -1,8 +0,0 @@
-class ProjectErrors:
- SERIALIZATION_FAILED = "Data Serialization Failed."
- PROJECT_NAME_EXISTS = "Project already exists"
- DUPLICATE_API = "It appears that a duplicate call may have been made."
-
-
-class ProjectKey:
- PROJECT_ID = "project_id"
diff --git a/backend/project/exceptions.py b/backend/project/exceptions.py
deleted file mode 100644
index 5955482a3..000000000
--- a/backend/project/exceptions.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from rest_framework.exceptions import APIException
-
-
-class BadRequestException(APIException):
- status_code = 400
- default_detail = "Bad Request."
diff --git a/backend/project/migrations/0001_initial.py b/backend/project/migrations/0001_initial.py
deleted file mode 100644
index 084af1737..000000000
--- a/backend/project/migrations/0001_initial.py
+++ /dev/null
@@ -1,85 +0,0 @@
-# Generated by Django 4.2.1 on 2024-01-23 11:18
-
-import uuid
-
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
- initial = True
-
- dependencies = [
- migrations.swappable_dependency(settings.AUTH_USER_MODEL),
- ]
-
- operations = [
- migrations.CreateModel(
- name="Project",
- fields=[
- ("created_at", models.DateTimeField(auto_now_add=True)),
- ("modified_at", models.DateTimeField(auto_now=True)),
- ("project_name", models.CharField(max_length=128, unique=True)),
- (
- "settings",
- models.JSONField(db_comment="Project settings", null=True),
- ),
- (
- "id",
- models.UUIDField(
- default=uuid.uuid4,
- editable=False,
- primary_key=True,
- serialize=False,
- ),
- ),
- (
- "description",
- models.TextField(blank=True, max_length=490, null=True),
- ),
- (
- "project_status",
- models.TextField(blank=True, max_length=128, null=True),
- ),
- (
- "project_identifier",
- models.CharField(
- choices=[
- ("ETL", "ETL"),
- ("TASK", "TASK"),
- ("DEFAULT", "Default"),
- ("APP", "App"),
- ],
- default="DEFAULT",
- ),
- ),
- (
- "created_by",
- models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name="created_projects",
- to=settings.AUTH_USER_MODEL,
- ),
- ),
- (
- "modified_by",
- models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name="modified_projects",
- to=settings.AUTH_USER_MODEL,
- ),
- ),
- ],
- ),
- migrations.AddConstraint(
- model_name="project",
- constraint=models.UniqueConstraint(
- fields=("id", "project_name"), name="unique_project_name"
- ),
- ),
- ]
diff --git a/backend/project/migrations/__init__.py b/backend/project/migrations/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/backend/project/models.py b/backend/project/models.py
deleted file mode 100644
index e936a43a1..000000000
--- a/backend/project/models.py
+++ /dev/null
@@ -1,57 +0,0 @@
-import uuid
-
-from account.models import User
-from django.db import models
-from utils.models.base_model import BaseModel
-
-PROJECT_NAME_SIZE = 128
-DESCRIPTION_FIELD_LENGTH = 490
-
-
-class Project(BaseModel):
- """Stores data related to a project created by a :model:`account.User` in
- an :model:`account.Org`.
-
- Every org, project is assumed to be unique
- """
-
- class ProjectIdentifier(models.TextChoices):
- ETL = "ETL", "ETL"
- TASK = "TASK", "TASK"
- DEFAULT = "DEFAULT", "Default"
- APP = "APP", "App"
-
- project_name = models.CharField(max_length=PROJECT_NAME_SIZE, unique=True)
- created_by = models.ForeignKey(
- User,
- on_delete=models.SET_NULL,
- related_name="created_projects",
- null=True,
- blank=True,
- )
- modified_by = models.ForeignKey(
- User,
- on_delete=models.SET_NULL,
- related_name="modified_projects",
- null=True,
- blank=True,
- )
- settings = models.JSONField(null=True, db_comment="Project settings")
- id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
- description = models.TextField(
- null=True, blank=True, max_length=DESCRIPTION_FIELD_LENGTH
- )
- project_status = models.TextField(null=True, blank=True, max_length=PROJECT_NAME_SIZE)
- project_identifier = models.CharField(
- choices=ProjectIdentifier.choices, default=ProjectIdentifier.DEFAULT
- )
-
- def __str__(self) -> str:
- return f"Projects({self.id}, name: {self.project_name})"
-
- class Meta:
- constraints = [
- models.UniqueConstraint(
- fields=["id", "project_name"], name="unique_project_name"
- ),
- ]
diff --git a/backend/project/serializers.py b/backend/project/serializers.py
deleted file mode 100644
index a24ea45a2..000000000
--- a/backend/project/serializers.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from typing import Any
-
-from workflow_manager.workflow.constants import WorkflowKey
-from workflow_manager.workflow.serializers import WorkflowSerializer
-
-from backend.serializers import AuditSerializer
-from project.models import Project
-
-
-class ProjectSerializer(AuditSerializer):
- class Meta:
- model = Project
- fields = "__all__"
-
- def to_representation(self, instance: Project) -> dict[str, Any]:
- representation: dict[str, str] = super().to_representation(instance)
- wf = instance.project_workflow.first()
- representation[WorkflowKey.WF_ID] = wf.id if wf else ""
- return representation
-
- def create(self, validated_data: dict[str, Any]) -> Any:
- project: Project = super().create(validated_data)
- workflow_data = {"project": project.id}
- workflow_serializer = WorkflowSerializer(data=workflow_data, context=self.context)
- workflow_serializer.is_valid(raise_exception=True)
- workflow_serializer.save()
- return project
diff --git a/backend/project/tests/conftest.py b/backend/project/tests/conftest.py
deleted file mode 100644
index 0f6c3b970..000000000
--- a/backend/project/tests/conftest.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import pytest
-from django.core.management import call_command
-
-
[email protected](scope="session")
-def django_db_setup(django_db_blocker): # type: ignore
- fixtures = ["./project/tests/fixtures/fixtures_0001.json"]
- with django_db_blocker.unblock():
- call_command("loaddata", *fixtures)
diff --git a/backend/project/tests/fixtures/fixtures_0001.json b/backend/project/tests/fixtures/fixtures_0001.json
deleted file mode 100644
index e11768655..000000000
--- a/backend/project/tests/fixtures/fixtures_0001.json
+++ /dev/null
@@ -1,51 +0,0 @@
-[
- {
- "model": "account.org",
- "pk": 1,
- "fields": {
- "org_name": "Zipstack",
- "created_by": 1,
- "modified_by": 1,
- "modified_at": "2023-06-14T05:28:47.739Z"
- }
- },
- {
- "model": "account.user",
- "pk": 1,
- "fields": {
- "org": 1,
- "email": "[email protected]",
- "first_name": "John",
- "last_name": "Doe",
- "is_admin": true,
- "created_by": null,
- "modified_by": null,
- "modified_at": "2023-06-14T05:28:47.744Z"
- }
- },
- {
- "model": "account.user",
- "pk": 2,
- "fields": {
- "org": 1,
- "email": "[email protected]",
- "first_name": "Ron",
- "last_name": "Stone",
- "is_admin": false,
- "created_by": 1,
- "modified_by": 1,
- "modified_at": "2023-06-14T05:28:47.750Z"
- }
- },
- {
- "model": "project.project",
- "pk": 1,
- "fields": {
- "org": 1,
- "project_name": "Unstract Test",
- "created_by": 2,
- "modified_by": 2,
- "modified_at": "2023-06-14T05:28:47.759Z"
- }
- }
-]
diff --git a/backend/project/tests/project_tests.py b/backend/project/tests/project_tests.py
deleted file mode 100644
index ce2ef52bb..000000000
--- a/backend/project/tests/project_tests.py
+++ /dev/null
@@ -1,115 +0,0 @@
-import pytest
-from django.urls import reverse
-from project.models import Project
-from rest_framework import status
-from rest_framework.test import APITestCase
-
-pytestmark = pytest.mark.django_db
-
-
-class TestProjects(APITestCase):
- def test_projects_list(self) -> None:
- """Tests to List the projects."""
- url = reverse("projects_v1-list")
- response = self.client.get(url)
-
- self.assertEqual(response.status_code, status.HTTP_200_OK)
-
- def test_projects_detail(self) -> None:
- """Tests to fetch a project with given pk."""
- url = reverse("projects_v1-detail", kwargs={"pk": 1})
- response = self.client.get(url)
-
- self.assertEqual(response.status_code, status.HTTP_200_OK)
-
- def test_projects_detail_not_found(self) -> None:
- """Tests for negative case to fetch non exiting key."""
- url = reverse("projects_v1-detail", kwargs={"pk": 768})
- response = self.client.get(url)
-
- self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
-
- def test_projects_create(self) -> None:
- """Tests to create a new project."""
- url = reverse("projects_v1-list")
- data = {
- "org": 1,
- "project_name": "Unstract Test",
- "created_by": 2,
- "modified_by": 2,
- "modified_at": "2023-06-14T05:28:47.759Z",
- }
- response = self.client.post(url, data, format="json")
-
- self.assertEqual(response.status_code, status.HTTP_201_CREATED)
- self.assertEqual(Project.objects.count(), 2)
-
- def test_projects_create_bad_request(self) -> None:
- """Tests for negative case to throw error on a wrong access."""
- url = reverse("projects_v1-list")
- data = {
- "project_name": "Unstract Test",
- "created_by": 2,
- "modified_by": 2,
- "modified_at": "2023-06-14T05:28:47.759Z",
- }
- response = self.client.post(url, data, format="json")
-
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
- def test_projects_update(self) -> None:
- """Tests to update project."""
- url = reverse("projects_v1-detail", kwargs={"pk": 1})
- data = {
- "org": 1,
- "project_name": "Unstract Test",
- "created_by": 2,
- "modified_by": 2,
- "modified_at": "2023-06-14T05:28:47.759Z",
- }
- response = self.client.put(url, data, format="json")
- project_name = response.data["project_name"]
-
- self.assertEqual(response.status_code, status.HTTP_200_OK)
- self.assertEqual(
- project_name,
- Project.objects.get(project_name=project_name).project_name,
- )
-
- def test_projects_update_pk(self) -> None:
- """Tests the PUT method for 400 error."""
- url = reverse("projects_v1-detail", kwargs={"pk": 1})
- data = {
- "org": 2,
- "project_name": "Unstract Test",
- "created_by": 2,
- "modified_by": 2,
- "modified_at": "2023-06-14T05:28:47.759Z",
- }
- response = self.client.put(url, data, format="json")
-
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
- def test_projects_update_field(self) -> None:
- """Tests the PATCH method."""
- url = reverse("projects_v1-detail", kwargs={"pk": 1})
- data = {"project_name": "Unstract Test"}
- response = self.client.patch(url, data, format="json")
- self.assertEqual(response.status_code, status.HTTP_200_OK)
- project_name = response.data["project_name"]
-
- self.assertEqual(
- project_name,
- Project.objects.get(project_name=project_name).project_name,
- )
-
- def test_projects_delete(self) -> None:
- """Tests the DELETE method."""
- url = reverse("projects_v1-detail", kwargs={"pk": 1})
- response = self.client.delete(url, format="json")
- self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
-
- url = reverse("projects_v1-detail", kwargs={"pk": 1})
- response = self.client.get(url)
- self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
diff --git a/backend/project/urls.py b/backend/project/urls.py
deleted file mode 100644
index f015268e8..000000000
--- a/backend/project/urls.py
+++ /dev/null
@@ -1,36 +0,0 @@
-from django.urls import path
-from rest_framework.urlpatterns import format_suffix_patterns
-
-from .views import ProjectViewSet
-
-project_list = ProjectViewSet.as_view({"get": "list", "post": "create"})
-project_detail = ProjectViewSet.as_view(
- {
- "get": "retrieve",
- "put": "update",
- "patch": "partial_update",
- "delete": "destroy",
- }
-)
-
-project_settings = ProjectViewSet.as_view(
- {"get": "project_settings", "put": "project_settings"}
-)
-project_settings_schema = ProjectViewSet.as_view({"get": "project_settings_schema"})
-
-urlpatterns = format_suffix_patterns(
- [
- path("projects/", project_list, name="projects-list"),
- path("projects/<uuid:pk>/", project_detail, name="projects-detail"),
- path(
- "projects/<uuid:pk>/settings/",
- project_settings,
- name="project-settings",
- ),
- path(
- "projects/settings/",
- project_settings_schema,
- name="project-settings-schema",
- ),
- ]
-)
diff --git a/backend/project/views.py b/backend/project/views.py
deleted file mode 100644
index dfa7fd0d4..000000000
--- a/backend/project/views.py
+++ /dev/null
@@ -1,59 +0,0 @@
-import logging
-from typing import Any
-
-from account.custom_exceptions import DuplicateData
-from connector.connector_instance_helper import ConnectorInstanceHelper
-from django.conf import settings
-from django.db import IntegrityError
-from django.db.models import QuerySet
-from django.http import HttpRequest
-from rest_framework import status, viewsets
-from rest_framework.response import Response
-from rest_framework.versioning import URLPathVersioning
-
-from project.constants import ProjectErrors
-
-from .models import Project
-from .serializers import ProjectSerializer
-
-logger = logging.getLogger(__name__)
-
-
-class ProjectViewSet(viewsets.ModelViewSet):
- versioning_class = URLPathVersioning
- queryset = Project.objects.all()
- serializer_class = ProjectSerializer
-
- def get_queryset(self) -> QuerySet | None:
- created_by = self.request.query_params.get("created_by")
- if created_by is not None:
- queryset = Project.objects.filter(created_by=created_by)
- return queryset
- elif created_by is None:
- queryset = Project.objects.all()
- return queryset
-
- def create(
- self, request: HttpRequest, *args: tuple[Any], **kwargs: dict[str, Any]
- ) -> Response:
- serializer = self.get_serializer(data=request.data)
- # Overriding default exception behavior
- serializer.is_valid(raise_exception=True)
- try:
- self.perform_create(serializer)
- except IntegrityError:
- raise DuplicateData(
- f"{ProjectErrors.PROJECT_NAME_EXISTS}, {ProjectErrors.DUPLICATE_API}"
- )
- # Access the created instance
- created_instance: Project = serializer.instance
-
- # Enable GCS configurations to create GCS while creating a workflow
- if (
- settings.GOOGLE_STORAGE_ACCESS_KEY_ID
- and settings.UNSTRACT_FREE_STORAGE_BUCKET_NAME
- ):
- ConnectorInstanceHelper.create_default_gcs_connector(
- created_instance, request.user
- )
- return Response(serializer.data, status=status.HTTP_201_CREATED)
diff --git a/backend/prompt/__init__.py b/backend/prompt/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/backend/prompt/admin.py b/backend/prompt/admin.py
deleted file mode 100644
index 9de0b12ce..000000000
--- a/backend/prompt/admin.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from django.contrib import admin
-
-from .models import Prompt
-
-admin.site.register(Prompt)
diff --git a/backend/prompt/apps.py b/backend/prompt/apps.py
deleted file mode 100644
index 1c08e89f1..000000000
--- a/backend/prompt/apps.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from django.apps import AppConfig
-
-
-class PromptConfig(AppConfig):
- name = "prompt"
diff --git a/backend/prompt/constants.py b/backend/prompt/constants.py
deleted file mode 100644
index 4357ffc8d..000000000
--- a/backend/prompt/constants.py
+++ /dev/null
@@ -1,3 +0,0 @@
-class PromptErrors:
- PROMPT_EXISTS = "Prompt with this configuration already exists"
- DUPLICATE_API = "It appears that a duplicate call may have been made."
diff --git a/backend/prompt/migrations/0001_initial.py b/backend/prompt/migrations/0001_initial.py
deleted file mode 100644
index 7b00ca293..000000000
--- a/backend/prompt/migrations/0001_initial.py
+++ /dev/null
@@ -1,73 +0,0 @@
-# Generated by Django 4.2.1 on 2024-01-23 11:18
-
-import uuid
-
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
- initial = True
-
- dependencies = [
- migrations.swappable_dependency(settings.AUTH_USER_MODEL),
- ("project", "0001_initial"),
- ]
-
- operations = [
- migrations.CreateModel(
- name="Prompt",
- fields=[
- ("created_at", models.DateTimeField(auto_now_add=True)),
- ("modified_at", models.DateTimeField(auto_now=True)),
- (
- "id",
- models.UUIDField(
- default=uuid.uuid4,
- editable=False,
- primary_key=True,
- serialize=False,
- ),
- ),
- ("version_name", models.CharField(max_length=64)),
- ("prompt_input", models.CharField(max_length=1024)),
- ("promoted", models.BooleanField(default=False)),
- ("associated_workflow", models.IntegerField(null=True)),
- (
- "created_by",
- models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name="created_prompts",
- to=settings.AUTH_USER_MODEL,
- ),
- ),
- (
- "modified_by",
- models.ForeignKey(
- blank=True,
- null=True,
- on_delete=django.db.models.deletion.SET_NULL,
- related_name="modified_prompts",
- to=settings.AUTH_USER_MODEL,
- ),
- ),
- (
- "project",
- models.ForeignKey(
- on_delete=django.db.models.deletion.CASCADE,
- to="project.project",
- ),
- ),
- ],
- ),
- migrations.AddConstraint(
- model_name="prompt",
- constraint=models.UniqueConstraint(
- fields=("project", "version_name"),
- name="unique_project_version",
- ),
- ),
- ]
diff --git a/backend/prompt/migrations/__init__.py b/backend/prompt/migrations/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/backend/prompt/models.py b/backend/prompt/models.py
deleted file mode 100644
index 65af29c35..000000000
--- a/backend/prompt/models.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import uuid
-
-from account.models import User
-from django.db import models
-from project.models import Project
-from utils.models.base_model import BaseModel
-
-VERSION_NAME_SIZE = 64
-PROMPT_INPUT_SIZE = 1024
-
-
-class Prompt(BaseModel):
- """Stores data related to a prompt created by a :model:`account.User` in an
- :model:`account.Org` for a :model:`project.Project`.
-
- Every org, project, version_name is assumed to be unique
- """
-
- id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
- project = models.ForeignKey(Project, on_delete=models.CASCADE)
- version_name = models.CharField(max_length=VERSION_NAME_SIZE)
- created_by = models.ForeignKey(
- User,
- on_delete=models.SET_NULL,
- related_name="created_prompts",
- null=True,
- blank=True,
- )
- modified_by = models.ForeignKey(
- User,
- on_delete=models.SET_NULL,
- related_name="modified_prompts",
- null=True,
- blank=True,
- )
- prompt_input = models.CharField(max_length=PROMPT_INPUT_SIZE)
- promoted = models.BooleanField(default=False)
- # TODO: Replace once Workflow model is added
- # associated_workflow = models.ForeignKey(
- # "Workflow", on_delete=models.CASCADE)
- associated_workflow = models.IntegerField(null=True)
-
- def __str__(self) -> str:
- return f"Prompt({self.id}, input: {self.prompt_input}, \
- version: {self.version_name}, project: {self.project})"
-
- class Meta:
- constraints = [
- models.UniqueConstraint(
- fields=["project", "version_name"],
- name="unique_project_version",
- ),
- ]
diff --git a/backend/prompt/serializers.py b/backend/prompt/serializers.py
deleted file mode 100644
index add5662b7..000000000
--- a/backend/prompt/serializers.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from typing import Any
-
-from rest_framework import serializers
-
-from .models import Prompt
-
-
-class PromptSerializer(serializers.ModelSerializer):
- def create(self, validated_data: dict[str, Any]) -> Any:
- validated_data["created_by"] = self.context.get("request")._user
- return super().create(validated_data)
-
- def update(self, instance: Any, validated_data: dict[str, Any]) -> Any:
- validated_data["modified_by"] = self.context.get("request")._user
- return super().update(instance, validated_data)
-
- class Meta:
- model = Prompt
- fields = "__all__"
diff --git a/backend/prompt/tests/conftest.py b/backend/prompt/tests/conftest.py
deleted file mode 100644
index 5b00d7887..000000000
--- a/backend/prompt/tests/conftest.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import pytest
-from django.core.management import call_command
-
-
[email protected](scope="session")
-def django_db_setup(django_db_blocker):
- fixtures = ["./prompt/tests/fixtures/prompts_001.json"]
- with django_db_blocker.unblock():
- call_command("loaddata", *fixtures)
diff --git a/backend/prompt/tests/fixtures/prompts_001.json b/backend/prompt/tests/fixtures/prompts_001.json
deleted file mode 100644
index 792e35066..000000000
--- a/backend/prompt/tests/fixtures/prompts_001.json
+++ /dev/null
@@ -1,81 +0,0 @@
-[
-{
- "model": "account.org",
- "pk": 1,
- "fields": {
- "org_name": "Zipstack",
- "created_by": 1,
- "modified_by": 1,
- "modified_at": "2023-06-14T05:28:47.739Z"
- }
-},
-{
- "model": "account.user",
- "pk": 1,
- "fields": {
- "org": 1,
- "email": "[email protected]",
- "first_name": "John",
- "last_name": "Doe",
- "is_admin": true,
- "created_by": null,
- "modified_by": null,
- "modified_at": "2023-06-14T05:28:47.744Z"
- }
-},
-{
- "model": "account.user",
- "pk": 2,
- "fields": {
- "org": 1,
- "email": "[email protected]",
- "first_name": "Ron",
- "last_name": "Stone",
- "is_admin": false,
- "created_by": 1,
- "modified_by": 1,
- "modified_at": "2023-06-14T05:28:47.750Z"
- }
-},
-{
- "model": "project.project",
- "pk": 1,
- "fields": {
- "org": 1,
- "project_name": "Unstract Test",
- "created_by": 2,
- "modified_by": 2,
- "modified_at": "2023-06-14T05:28:47.759Z"
- }
-},
-{
- "model": "prompt.prompt",
- "pk": 1,
- "fields": {
- "org": 1,
- "project": 1,
- "version_name": "v0.1.0",
- "created_by": 2,
- "modified_by": 2,
- "modified_at": "2023-06-14T05:28:47.761Z",
- "prompt_input": "You are a Django programmer, write a REST API to support CRUD on a simple model",
- "promoted": false,
- "associated_workflow": null
- }
-},
-{
- "model": "prompt.prompt",
- "pk": 2,
- "fields": {
- "org": 1,
- "project": 1,
- "version_name": "v0.1.1",
- "created_by": 2,
- "modified_by": 2,
- "modified_at": "2023-06-14T05:28:47.764Z",
- "prompt_input": "You are a poet (William Wordsworth), write a poem on generative AI",
- "promoted": false,
- "associated_workflow": null
- }
-}
-]
diff --git a/backend/prompt/tests/test_urls.py b/backend/prompt/tests/test_urls.py
deleted file mode 100644
index d2f81388d..000000000
--- a/backend/prompt/tests/test_urls.py
+++ /dev/null
@@ -1,119 +0,0 @@
-import pytest
-from django.urls import reverse
-from prompt.models import Prompt
-from rest_framework import status
-from rest_framework.test import APITestCase
-
-pytestmark = pytest.mark.django_db
-
-
-class TestPrompts(APITestCase):
- def test_prompts_list(self):
- """Ensure we can list all prompts."""
- url = reverse("prompts_v1-list")
- response = self.client.get(url)
- self.assertEqual(response.status_code, status.HTTP_200_OK)
- self.assertEqual(len(response.data), 2)
-
- def test_prompts_detail(self):
- """Ensure we can retrieve a single prompt."""
- url = reverse("prompts_v1-detail", kwargs={"pk": 2})
- response = self.client.get(url)
- self.assertEqual(response.status_code, status.HTTP_200_OK)
- self.assertEqual(response.data["version_name"], "v0.1.1")
-
- def test_prompts_detail_throw_404(self):
- """Tests whether a 404 error is thrown on retrieving a prompt."""
- url = reverse("prompts-detail", kwargs={"pk": 200}) # Prompt doesn't exist
- response = self.client.get(url)
- self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
-
- def test_prompts_create(self):
- """Ensure we can create a new prompt."""
- url = reverse("prompts_v1-list")
- data = {
- "org": 1,
- "project": 1,
- "version_name": "v0.1.2",
- "created_by": 2,
- "modified_by": 2,
- "prompt_input": "You're a CS undergrad looking to receive an admit from a \
- university for a masters in AI, write a convincing SOP",
- "promoted": False,
- }
- response = self.client.post(url, data, format="json")
- pk = response.data["id"]
- self.assertEqual(response.status_code, status.HTTP_201_CREATED)
- self.assertEqual(Prompt.objects.count(), 3)
- self.assertEqual(Prompt.objects.get(pk=pk).version_name, "v0.1.2")
-
- def test_prompts_create_throw_bad_request(self):
- """Ensure we throw an error in case of a bad request."""
- url = reverse("prompts_v1-list")
- data = {
- "org": 200, # This org does not exist
- "project": 1,
- "version_name": "v0.1.2",
- "created_by": 2,
- "modified_by": 2,
- "prompt_input": "You're a CS undergrad looking to receive an admit from a \
- university for a masters in AI, write a convincing SOP",
- "promoted": False,
- }
- response = self.client.post(url, data, format="json")
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
- def test_prompts_update(self):
- """Tests the PUT method."""
- url = reverse("prompts_v1-detail", kwargs={"pk": 2})
- data = {
- "org": 1,
- "project": 1,
- "version_name": "v0.1.3",
- "created_by": 2,
- "modified_by": 2,
- "prompt_input": "You're a CS undergrad looking to receive an admit from a \
- university for a masters in AI, write a convincing SOP",
- "promoted": False,
- }
- response = self.client.put(url, data, format="json")
- self.assertEqual(response.status_code, status.HTTP_200_OK)
- pk = response.data["id"]
- self.assertEqual(Prompt.objects.get(pk=pk).version_name, "v0.1.3")
-
- def test_prompts_update_bad_request(self):
- """Tests the PUT method for 400 error."""
- url = reverse("prompts_v1-detail", kwargs={"pk": 2})
- data = {
- "org": 200, # This org does not exist
- "project": 1,
- "version_name": "v0.1.3",
- "created_by": 2,
- "modified_by": 2,
- "prompt_input": "You're a CS undergrad looking to receive an admit from a \
- university for a masters in AI, write a convincing SOP",
- "promoted": False,
- }
- response = self.client.put(url, data, format="json")
- self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
- def test_prompts_partial_update(self):
- """Tests the PATCH method."""
- url = reverse("prompts_v1-detail", kwargs={"pk": 2})
- data = {"promoted": True}
- response = self.client.patch(url, data, format="json")
- self.assertEqual(response.status_code, status.HTTP_200_OK)
- pk = response.data["id"]
- self.assertEqual(Prompt.objects.get(pk=pk).promoted, True)
-
- def test_prompts_delete(self):
- """Tests the DELETE method."""
- url = reverse("prompts_v1-detail", kwargs={"pk": 2})
- response = self.client.delete(url, format="json")
- self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
-
- # Check if data was deleted as well
- url = reverse("prompts_v1-detail", kwargs={"pk": 2})
- response = self.client.get(url)
- self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
diff --git a/backend/prompt/urls.py b/backend/prompt/urls.py
deleted file mode 100644
index 6d04bca19..000000000
--- a/backend/prompt/urls.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from django.urls import path
-from rest_framework.urlpatterns import format_suffix_patterns
-
-from .views import PromptViewSet
-
-prompt_list = PromptViewSet.as_view({"get": "list", "post": "create"})
-prompt_detail = PromptViewSet.as_view(
- {
- "get": "retrieve",
- "put": "update",
- "patch": "partial_update",
- "delete": "destroy",
- }
-)
-
-urlpatterns = format_suffix_patterns(
- [
- path("prompt/", prompt_list, name="prompt-list"),
- path("prompt/<uuid:pk>/", prompt_detail, name="prompt-detail"),
- ]
-)
diff --git a/backend/prompt/views.py b/backend/prompt/views.py
deleted file mode 100644
index 443b76e8d..000000000
--- a/backend/prompt/views.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from typing import Any
-
-from account.custom_exceptions import DuplicateData
-from django.db import IntegrityError
-from rest_framework import status, viewsets
-from rest_framework.decorators import action
-from rest_framework.request import Request
-from rest_framework.response import Response
-from rest_framework.versioning import URLPathVersioning
-
-from prompt.constants import PromptErrors
-
-from .models import Prompt
-from .serializers import PromptSerializer
-
-
-class PromptViewSet(viewsets.ModelViewSet):
- """Used to view and edit prompts.
-
- Handles GET,POST,PUT,PATCH and DELETE
- """
-
- versioning_class = URLPathVersioning
- queryset = Prompt.objects.all()
- serializer_class = PromptSerializer
-
- @action(detail=False)
- def get_all_prompt(self, request: Request) -> Response:
- # has not specific funstions. Added for testing purpose
- serializer = self.get_serializer(many=True)
- return Response(serializer.data)
-
- def create(self, request: Any) -> Response:
- # Overriding default exception behaviour
- serializer = self.get_serializer(data=request.data)
- serializer.is_valid(raise_exception=True)
- try:
- self.perform_create(serializer)
- except IntegrityError:
- raise DuplicateData(
- f"{PromptErrors.PROMPT_EXISTS}, {PromptErrors.DUPLICATE_API}"
- )
-
- headers = self.get_success_headers(serializer.data)
- return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
diff --git a/backend/tool_instance_v2/constants.py b/backend/tool_instance_v2/constants.py
index 17a3891c5..1bc23d78e 100644
--- a/backend/tool_instance_v2/constants.py
+++ b/backend/tool_instance_v2/constants.py
@@ -16,17 +16,7 @@ class ToolInstanceKey:
class JsonSchemaKey:
"""Dict Keys for Tool's Json schema."""
- PROPERTIES = "properties"
- THEN = "then"
- INPUT_FILE_CONNECTOR = "inputFileConnector"
- OUTPUT_FILE_CONNECTOR = "outputFileConnector"
- OUTPUT_FOLDER = "outputFolder"
- ROOT_FOLDER = "rootFolder"
TENANT_ID = "tenant_id"
- INPUT_DB_CONNECTOR = "inputDBConnector"
- OUTPUT_DB_CONNECTOR = "outputDBConnector"
- ENUM = "enum"
- PROJECT_DEFAULT = "Project Default"
class ToolInstanceErrors:
diff --git a/backend/tool_instance_v2/migrations/0002_remove_toolinstance_input_db_connector_and_more.py b/backend/tool_instance_v2/migrations/0002_remove_toolinstance_input_db_connector_and_more.py
new file mode 100644
index 000000000..501f5561d
--- /dev/null
+++ b/backend/tool_instance_v2/migrations/0002_remove_toolinstance_input_db_connector_and_more.py
@@ -0,0 +1,28 @@
+# Generated by Django 4.2.1 on 2025-07-09 18:35
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("tool_instance_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name="toolinstance",
+ name="input_db_connector",
+ ),
+ migrations.RemoveField(
+ model_name="toolinstance",
+ name="input_file_connector",
+ ),
+ migrations.RemoveField(
+ model_name="toolinstance",
+ name="output_db_connector",
+ ),
+ migrations.RemoveField(
+ model_name="toolinstance",
+ name="output_file_connector",
+ ),
+ ]
diff --git a/backend/tool_instance_v2/models.py b/backend/tool_instance_v2/models.py
index 831e16276..971f1249b 100644
--- a/backend/tool_instance_v2/models.py
+++ b/backend/tool_instance_v2/models.py
@@ -1,7 +1,6 @@
import uuid
from account_v2.models import User
-from connector_v2.models import ConnectorInstance
from django.db import models
from django.db.models import QuerySet
from utils.models.base_model import BaseModel
@@ -58,37 +57,6 @@ class ToolInstance(BaseModel):
null=True,
blank=True,
)
- # Added these connectors separately
- # for file and db for scalability
- input_file_connector = models.ForeignKey(
- ConnectorInstance,
- on_delete=models.SET_NULL,
- related_name="input_file_connectors",
- null=True,
- blank=True,
- )
- output_file_connector = models.ForeignKey(
- ConnectorInstance,
- on_delete=models.SET_NULL,
- related_name="output_file_connectors",
- null=True,
- blank=True,
- )
- input_db_connector = models.ForeignKey(
- ConnectorInstance,
- on_delete=models.SET_NULL,
- related_name="input_db_connectors",
- null=True,
- blank=True,
- )
- output_db_connector = models.ForeignKey(
- ConnectorInstance,
- on_delete=models.SET_NULL,
- related_name="output_db_connectors",
- null=True,
- blank=True,
- )
-
objects = ToolInstanceManager()
class Meta:
diff --git a/backend/tool_instance_v2/serializers.py b/backend/tool_instance_v2/serializers.py
index 5fe18047d..9d7a80669 100644
--- a/backend/tool_instance_v2/serializers.py
+++ b/backend/tool_instance_v2/serializers.py
@@ -7,13 +7,11 @@ from rest_framework.serializers import ListField, Serializer, UUIDField, Validat
from workflow_manager.workflow_v2.constants import WorkflowKey
from workflow_manager.workflow_v2.models.workflow import Workflow
-from backend.constants import RequestKey
from backend.serializers import AuditSerializer
from tool_instance_v2.constants import ToolInstanceKey as TIKey
from tool_instance_v2.constants import ToolKey
from tool_instance_v2.exceptions import ToolDoesNotExist
from tool_instance_v2.models import ToolInstance
-from tool_instance_v2.tool_instance_helper import ToolInstanceHelper
from tool_instance_v2.tool_processor import ToolProcessor
from unstract.tool_registry.dto import Tool
@@ -53,11 +51,6 @@ class ToolInstanceSerializer(AuditSerializer):
return rep
rep[ToolKey.ICON] = tool.icon
rep[ToolKey.NAME] = tool.properties.display_name
- # Need to Change it into better method
- if self.context.get(RequestKey.REQUEST):
- metadata = ToolInstanceHelper.get_altered_metadata(instance)
- if metadata:
- rep[TIKey.METADATA] = metadata
return rep
def create(self, validated_data: dict[str, Any]) -> Any:
diff --git a/backend/tool_instance_v2/tool_instance_helper.py b/backend/tool_instance_v2/tool_instance_helper.py
index 8b9330bcb..f8ffc4a93 100644
--- a/backend/tool_instance_v2/tool_instance_helper.py
+++ b/backend/tool_instance_v2/tool_instance_helper.py
@@ -6,7 +6,6 @@ from typing import Any
from account_v2.models import User
from adapter_processor_v2.adapter_processor import AdapterProcessor
from adapter_processor_v2.models import AdapterInstance
-from connector_v2.connector_instance_helper import ConnectorInstanceHelper
from django.core.exceptions import PermissionDenied
from django.core.exceptions import ValidationError as DjangoValidationError
from jsonschema.exceptions import ValidationError as JSONValidationError
@@ -54,35 +53,6 @@ class ToolInstanceHelper:
def update_instance_metadata(
org_id: str, tool_instance: ToolInstance, metadata: dict[str, Any]
) -> None:
- if (
- JsonSchemaKey.OUTPUT_FILE_CONNECTOR in metadata
- and JsonSchemaKey.OUTPUT_FOLDER in metadata
- ):
- output_connector_name = metadata[JsonSchemaKey.OUTPUT_FILE_CONNECTOR]
- output_connector = ConnectorInstanceHelper.get_output_connector_instance_by_name_for_workflow( # noqa
- tool_instance.workflow_id, output_connector_name
- )
- if output_connector and "path" in output_connector.metadata:
- metadata[JsonSchemaKey.OUTPUT_FOLDER] = os.path.join(
- output_connector.metadata["path"],
- *(metadata[JsonSchemaKey.OUTPUT_FOLDER].split("/")),
- )
- if (
- JsonSchemaKey.INPUT_FILE_CONNECTOR in metadata
- and JsonSchemaKey.ROOT_FOLDER in metadata
- ):
- input_connector_name = metadata[JsonSchemaKey.INPUT_FILE_CONNECTOR]
- input_connector = (
- ConnectorInstanceHelper.get_input_connector_instance_by_name_for_workflow( # noqa
- tool_instance.workflow_id, input_connector_name
- )
- )
-
- if input_connector and "path" in input_connector.metadata:
- metadata[JsonSchemaKey.ROOT_FOLDER] = os.path.join(
- input_connector.metadata["path"],
- *(metadata[JsonSchemaKey.ROOT_FOLDER].split("/")),
- )
ToolInstanceHelper.update_metadata_with_adapter_instances(
metadata, tool_instance.tool_id
)
@@ -183,58 +153,6 @@ class ToolInstanceHelper:
adapter_type=AdapterTypes.OCR,
)
- # TODO: Review if adding this metadata is still required
- @staticmethod
- def get_altered_metadata(
- tool_instance: ToolInstance,
- ) -> dict[str, Any] | None:
- """Get altered metadata by resolving relative paths.
-
- This method retrieves the metadata from the given tool instance
- and checks if there are output and input file connectors.
- If output and input file connectors exist in the metadata,
- it resolves the relative paths using connector instances.
-
- Args:
- tool_instance (ToolInstance).
-
- Returns:
- Optional[dict[str, Any]]: Altered metadata with resolved relative \
- paths.
- """
- metadata: dict[str, Any] = tool_instance.metadata
- if (
- JsonSchemaKey.OUTPUT_FILE_CONNECTOR in metadata
- and JsonSchemaKey.OUTPUT_FOLDER in metadata
- ):
- output_connector_name = metadata[JsonSchemaKey.OUTPUT_FILE_CONNECTOR]
- output_connector = ConnectorInstanceHelper.get_output_connector_instance_by_name_for_workflow( # noqa
- tool_instance.workflow_id, output_connector_name
- )
- if output_connector and "path" in output_connector.metadata:
- relative_path = ToolInstanceHelper.get_relative_path(
- metadata[JsonSchemaKey.OUTPUT_FOLDER],
- output_connector.metadata["path"],
- )
- metadata[JsonSchemaKey.OUTPUT_FOLDER] = relative_path
- if (
- JsonSchemaKey.INPUT_FILE_CONNECTOR in metadata
- and JsonSchemaKey.ROOT_FOLDER in metadata
- ):
- input_connector_name = metadata[JsonSchemaKey.INPUT_FILE_CONNECTOR]
- input_connector = (
- ConnectorInstanceHelper.get_input_connector_instance_by_name_for_workflow( # noqa
- tool_instance.workflow_id, input_connector_name
- )
- )
- if input_connector and "path" in input_connector.metadata:
- relative_path = ToolInstanceHelper.get_relative_path(
- metadata[JsonSchemaKey.ROOT_FOLDER],
- input_connector.metadata["path"],
- )
- metadata[JsonSchemaKey.ROOT_FOLDER] = relative_path
- return metadata
-
@staticmethod
def update_metadata_with_default_adapter(
adapter_type: AdapterTypes,
diff --git a/backend/workflow_manager/workflow_v2/serializers.py b/backend/workflow_manager/workflow_v2/serializers.py
index b40e2fa77..b076b8895 100644
--- a/backend/workflow_manager/workflow_v2/serializers.py
+++ b/backend/workflow_manager/workflow_v2/serializers.py
@@ -1,7 +1,6 @@
import logging
from typing import Any
-from project.constants import ProjectKey
from rest_framework.serializers import (
CharField,
ChoiceField,
@@ -68,7 +67,6 @@ class WorkflowSerializer(IntegrityErrorMixin, AuditSerializer):
class ExecuteWorkflowSerializer(Serializer):
workflow_id = UUIDField(required=False)
- project_id = UUIDField(required=False)
execution_action = ChoiceField(
choices=Workflow.ExecutionAction.choices, required=False
)
@@ -79,9 +77,6 @@ class ExecuteWorkflowSerializer(Serializer):
def get_workflow_id(self, validated_data: dict[str, str | None]) -> str | None:
return validated_data.get(WorkflowKey.WF_ID)
- def get_project_id(self, validated_data: dict[str, str | None]) -> str | None:
- return validated_data.get(ProjectKey.PROJECT_ID)
-
def get_execution_id(self, validated_data: dict[str, str | None]) -> str | None:
return validated_data.get(WorkflowExecutionKey.EXECUTION_ID)
@@ -93,12 +88,9 @@ class ExecuteWorkflowSerializer(Serializer):
def validate(self, data: dict[str, str | None]) -> dict[str, str | None]:
workflow_id = data.get(WorkflowKey.WF_ID)
- project_id = data.get(ProjectKey.PROJECT_ID)
- if not workflow_id and not project_id:
- raise ValidationError(
- "At least one of 'workflow_id' or 'project_id' is required."
- )
+ if not workflow_id:
+ raise ValidationError("'workflow_id' is required.")
return data
diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.py
index c8bcafb26..a0d60772c 100644
--- a/backend/workflow_manager/workflow_v2/views.py
+++ b/backend/workflow_manager/workflow_v2/views.py
@@ -111,27 +111,23 @@ class WorkflowViewSet(viewsets.ModelViewSet):
execution = WorkflowHelper.get_current_execution(pk)
return Response(make_execution_response(execution), status=status.HTTP_200_OK)
- def get_workflow_by_id_or_project_id(
+ def get_workflow_by_id(
self,
workflow_id: str | None = None,
- project_id: str | None = None,
) -> Workflow:
- """Retrieve workflow by workflow id or project Id.
+ """Retrieve workflow by workflow id.
Args:
workflow_id (Optional[str], optional): workflow Id.
- project_id (Optional[str], optional): Project Id.
Raises:
- WorkflowDoesNotExistError: _description_
+ WorkflowDoesNotExistError: Raised when workflow_id is not provided or workflow doesn't exist.
Returns:
Workflow: workflow
"""
if workflow_id:
workflow = WorkflowHelper.get_workflow_by_id(workflow_id)
- elif project_id:
- workflow = WorkflowHelper.get_active_workflow_by_project_id(project_id)
else:
raise WorkflowDoesNotExistError()
return workflow
@@ -145,7 +141,6 @@ class WorkflowViewSet(viewsets.ModelViewSet):
serializer = ExecuteWorkflowSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
workflow_id = serializer.get_workflow_id(serializer.validated_data)
- project_id = serializer.get_project_id(serializer.validated_data)
execution_id = serializer.get_execution_id(serializer.validated_data)
execution_action = serializer.get_execution_action(serializer.validated_data)
file_objs = request.FILES.getlist("files")
@@ -163,9 +158,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
)
try:
- workflow = self.get_workflow_by_id_or_project_id(
- workflow_id=workflow_id, project_id=project_id
- )
+ workflow = self.get_workflow_by_id(workflow_id=workflow_id)
execution_response = self.execute_workflow(
workflow=workflow,
execution_action=execution_action,
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index a2ac15a20..4594d2f19 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -78,18 +78,6 @@ class WorkflowHelper:
logger.error(f"Error getting workflow: {id}")
raise WorkflowDoesNotExistError()
- @staticmethod
- def get_active_workflow_by_project_id(project_id: str) -> Workflow:
- try:
- workflow: Workflow = Workflow.objects.filter(
- project_id=project_id, is_active=True
- ).first()
- if not workflow or workflow is None:
- raise WorkflowDoesNotExistError()
- return workflow
- except Workflow.DoesNotExist:
- raise WorkflowDoesNotExistError()
-
@staticmethod
def active_project_workflow(workflow_id: str) -> Workflow:
workflow: Workflow = WorkflowHelper.get_workflow_by_id(workflow_id)