LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
schema migrations and settings for v2 applications (#536)
alif99fc60f0cbf
diff --git a/backend/account_v2/migrations/0001_initial.py b/backend/account_v2/migrations/0001_initial.pynew file mode 100644index 000000000..2394c9f59--- /dev/null+++ b/backend/account_v2/migrations/0001_initial.py@@ -0,0 +1,265 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.contrib.auth.models+import django.contrib.auth.validators+import django.db.models.deletion+import django.utils.timezone+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ ("auth", "0012_alter_user_first_name_max_length"),+ ]++ operations = [+ migrations.CreateModel(+ name="User",+ fields=[+ (+ "id",+ models.BigAutoField(+ auto_created=True,+ primary_key=True,+ serialize=False,+ verbose_name="ID",+ ),+ ),+ ("password", models.CharField(max_length=128, verbose_name="password")),+ (+ "last_login",+ models.DateTimeField(+ blank=True, null=True, verbose_name="last login"+ ),+ ),+ (+ "is_superuser",+ models.BooleanField(+ default=False,+ help_text="Designates that this user has all permissions without explicitly assigning them.",+ verbose_name="superuser status",+ ),+ ),+ (+ "username",+ models.CharField(+ error_messages={+ "unique": "A user with that username already exists."+ },+ help_text="Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.",+ max_length=150,+ unique=True,+ validators=[+ django.contrib.auth.validators.UnicodeUsernameValidator()+ ],+ verbose_name="username",+ ),+ ),+ (+ "first_name",+ models.CharField(+ blank=True, max_length=150, verbose_name="first name"+ ),+ ),+ (+ "last_name",+ models.CharField(+ blank=True, max_length=150, verbose_name="last name"+ ),+ ),+ (+ "email",+ models.EmailField(+ blank=True, max_length=254, verbose_name="email address"+ ),+ ),+ (+ "is_staff",+ models.BooleanField(+ default=False,+ help_text="Designates whether the user can log into this admin site.",+ verbose_name="staff status",+ ),+ ),+ (+ "is_active",+ models.BooleanField(+ default=True,+ help_text="Designates whether this user should be treated as active. Unselect this instead of deleting accounts.",+ verbose_name="active",+ ),+ ),+ (+ "date_joined",+ models.DateTimeField(+ default=django.utils.timezone.now, verbose_name="date joined"+ ),+ ),+ ("user_id", models.CharField()),+ ("project_storage_created", models.BooleanField(default=False)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ ("created_at", models.DateTimeField(auto_now_add=True)),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="users_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "groups",+ models.ManyToManyField(+ blank=True,+ related_name="users",+ related_query_name="user",+ to="auth.group",+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="users_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "user_permissions",+ models.ManyToManyField(+ blank=True,+ related_name="users",+ related_query_name="user",+ to="auth.permission",+ ),+ ),+ ],+ options={+ "verbose_name": "User",+ "verbose_name_plural": "Users",+ "db_table": "user",+ },+ managers=[+ ("objects", django.contrib.auth.models.UserManager()),+ ],+ ),+ migrations.CreateModel(+ name="Organization",+ fields=[+ (+ "id",+ models.BigAutoField(+ auto_created=True,+ primary_key=True,+ serialize=False,+ verbose_name="ID",+ ),+ ),+ ("name", models.CharField(max_length=64)),+ ("display_name", models.CharField(max_length=64)),+ ("organization_id", models.CharField(max_length=64, unique=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ ("created_at", models.DateTimeField(auto_now=True)),+ (+ "allowed_token_limit",+ models.IntegerField(+ db_comment="token limit set in case of frition less onbaoarded org",+ default=-1,+ ),+ ),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="orgs_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="orgs_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ ],+ options={+ "verbose_name": "Organization",+ "verbose_name_plural": "Organizations",+ "db_table": "organization",+ },+ ),+ migrations.CreateModel(+ name="PlatformKey",+ fields=[+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("key", models.UUIDField(default=uuid.uuid4)),+ ("key_name", models.CharField(blank=True, default="", max_length=64)),+ ("is_active", models.BooleanField(default=False)),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="platform_keys_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="platform_keys_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "organization",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="platform_keys",+ to="account_v2.organization",+ ),+ ),+ ],+ options={+ "verbose_name": "Platform Key",+ "verbose_name_plural": "Platform Keys",+ "db_table": "platform_key",+ },+ ),+ migrations.AddConstraint(+ model_name="platformkey",+ constraint=models.UniqueConstraint(+ fields=("key_name", "organization"), name="unique_key_name_organization"+ ),+ ),+ ]diff --git a/backend/account_v2/models.py b/backend/account_v2/models.pyindex 61428bda0..495d2913f 100644--- a/backend/account_v2/models.py+++ b/backend/account_v2/models.py@@ -45,7 +45,7 @@ class Organization(models.Model):class Meta:verbose_name = "Organization"verbose_name_plural = "Organizations"- db_table = "organization_v2"+ db_table = "organization"class User(AbstractUser):@@ -96,7 +96,7 @@ class User(AbstractUser):class Meta:verbose_name = "User"verbose_name_plural = "Users"- db_table = "user_v2"+ db_table = "user"class PlatformKey(models.Model):@@ -135,7 +135,7 @@ class PlatformKey(models.Model):class Meta:verbose_name = "Platform Key"verbose_name_plural = "Platform Keys"- db_table = "platform_key_v2"+ db_table = "platform_key"constraints = [models.UniqueConstraint(fields=["key_name", "organization"],diff --git a/backend/account_v2/subscription_loader.py b/backend/account_v2/subscription_loader.pyindex d380ed19d..9502f135a 100644--- a/backend/account_v2/subscription_loader.py+++ b/backend/account_v2/subscription_loader.py@@ -1,5 +1,6 @@import loggingimport os+from datetime import timezonefrom importlib import import_modulefrom typing import Any@@ -75,3 +76,32 @@ def load_plugins() -> list[Any]:logger.info("No subscription plugins found.")return subscription_plugins+++def validate_etl_run(org_id: str) -> bool:+ """Method to check subscription status before ETL runs.++ Args:+ org_id: The ID of the organization.++ Returns:+ A boolean indicating whether the pre-run check passed or not.+ """+ try:+ from pluggable_apps.subscription.subscription_helper import SubscriptionHelper+ except ModuleNotFoundError:+ logger.error("Subscription plugin not found.")+ return False++ org_plans = SubscriptionHelper.get_subscription(org_id)+ if not org_plans or not org_plans.is_active:+ return False++ if org_plans.is_paid:+ return True++ if timezone.now() >= org_plans.end_date:+ logger.debug(f"Trial expired for org {org_id}")+ return False++ return Truediff --git a/backend/account_v2/urls.py b/backend/account_v2/urls.pyindex 767be8aba..605ab2621 100644--- a/backend/account_v2/urls.py+++ b/backend/account_v2/urls.py@@ -16,7 +16,7 @@ urlpatterns = [path("logout", logout, name="logout"),path("callback", callback, name="callback"),path("session", get_session_data, name="session"),- path("organizations", get_organizations, name="get_organizations"),+ path("organization", get_organizations, name="get_organizations"),path("organization/<str:id>/set", set_organization, name="set_organization"),path("organization/create", create_organization, name="create_organization"),]diff --git a/backend/adapter_processor_v2/adapter_processor.py b/backend/adapter_processor_v2/adapter_processor.pyindex dd9d44ff3..16c3b9914 100644--- a/backend/adapter_processor_v2/adapter_processor.py+++ b/backend/adapter_processor_v2/adapter_processor.py@@ -12,6 +12,7 @@ from adapter_processor_v2.exceptions import (from django.conf import settingsfrom django.core.exceptions import ObjectDoesNotExistfrom platform_settings_v2.platform_auth_service import PlatformAuthenticationService+from tenant_account_v2.organization_member_service import OrganizationMemberServicefrom unstract.adapters.adapterkit import Adapterkitfrom unstract.adapters.base import Adapterfrom unstract.adapters.enums import AdapterTypes@@ -112,10 +113,13 @@ class AdapterProcessor:@staticmethoddef set_default_triad(default_triad: dict[str, str], user: User) -> None:try:+ organization_member = OrganizationMemberService.get_user_by_id(user.id)(user_default_adapter,created,- ) = UserDefaultAdapter.objects.get_or_create(user=user)+ ) = UserDefaultAdapter.objects.get_or_create(+ organization_member=organization_member+ )if default_triad.get(AdapterKeys.LLM_DEFAULT, None):user_default_adapter.default_llm_adapter = AdapterInstance.objects.get(@@ -235,7 +239,11 @@ class AdapterProcessor:"""try:adapters: list[AdapterInstance] = []- default_adapter = UserDefaultAdapter.objects.get(user=user)++ organization_member = OrganizationMemberService.get_user_by_id(id=user.id)+ default_adapter: UserDefaultAdapter = UserDefaultAdapter.objects.get(+ organization_member=organization_member+ )if default_adapter.default_embedding_adapter:adapters.append(default_adapter.default_embedding_adapter)diff --git a/backend/adapter_processor_v2/migrations/0001_initial.py b/backend/adapter_processor_v2/migrations/0001_initial.pynew file mode 100644index 000000000..9835825db--- /dev/null+++ b/backend/adapter_processor_v2/migrations/0001_initial.py@@ -0,0 +1,220 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("account_v2", "0001_initial"),+ ("tenant_account_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="AdapterInstance",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ db_comment="Unique identifier for the Adapter Instance",+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "adapter_name",+ models.TextField(+ db_comment="Name of the Adapter Instance", max_length=128+ ),+ ),+ (+ "adapter_id",+ models.CharField(+ db_comment="Unique identifier of the Adapter",+ default="",+ max_length=128,+ ),+ ),+ (+ "adapter_metadata",+ models.JSONField(+ db_column="adapter_metadata",+ db_comment="JSON adapter metadata submitted by the user",+ default=dict,+ ),+ ),+ ("adapter_metadata_b", models.BinaryField(null=True)),+ (+ "adapter_type",+ models.CharField(+ choices=[+ ("UNKNOWN", "UNKNOWN"),+ ("LLM", "LLM"),+ ("EMBEDDING", "EMBEDDING"),+ ("VECTOR_DB", "VECTOR_DB"),+ ("OCR", "OCR"),+ ("X2TEXT", "X2TEXT"),+ ],+ db_comment="Type of adapter LLM/EMBEDDING/VECTOR_DB",+ ),+ ),+ (+ "is_active",+ models.BooleanField(+ db_comment="Is the adapter instance currently being used",+ default=False,+ ),+ ),+ (+ "shared_to_org",+ models.BooleanField(+ db_comment="Is the adapter shared to entire org", default=False+ ),+ ),+ (+ "is_friction_less",+ models.BooleanField(+ db_comment="Was the adapter created through frictionless onboarding",+ default=False,+ ),+ ),+ (+ "is_usable",+ models.BooleanField(+ db_comment="Is the Adpater Usable", default=True+ ),+ ),+ ("description", models.TextField(blank=True, default=None, null=True)),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="adapters_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="adapters_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "organization",+ models.ForeignKey(+ blank=True,+ db_comment="Foreign key reference to the Organization model.",+ default=None,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ to="account_v2.organization",+ ),+ ),+ (+ "shared_users",+ models.ManyToManyField(+ related_name="shared_adapters_instance",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ ],+ options={+ "verbose_name": "adapter instance",+ "verbose_name_plural": "adapter instances",+ "db_table": "adapter_instance",+ },+ ),+ migrations.CreateModel(+ name="UserDefaultAdapter",+ fields=[+ (+ "id",+ models.BigAutoField(+ auto_created=True,+ primary_key=True,+ serialize=False,+ verbose_name="ID",+ ),+ ),+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "default_embedding_adapter",+ models.ForeignKey(+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="user_default_embedding_adapter",+ to="adapter_processor_v2.adapterinstance",+ ),+ ),+ (+ "default_llm_adapter",+ models.ForeignKey(+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="user_default_llm_adapter",+ to="adapter_processor_v2.adapterinstance",+ ),+ ),+ (+ "default_vector_db_adapter",+ models.ForeignKey(+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="user_default_vector_db_adapter",+ to="adapter_processor_v2.adapterinstance",+ ),+ ),+ (+ "default_x2text_adapter",+ models.ForeignKey(+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="user_default_x2text_adapter",+ to="adapter_processor_v2.adapterinstance",+ ),+ ),+ (+ "organization_member",+ models.OneToOneField(+ db_comment="Foreign key reference to the OrganizationMember model.",+ default=None,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="default_adapters",+ to="tenant_account_v2.organizationmember",+ ),+ ),+ ],+ options={+ "verbose_name": "Default Adapter for Organization User",+ "verbose_name_plural": "Default Adapters for Organization Users",+ "db_table": "default_organization_user_adapter",+ },+ ),+ migrations.AddConstraint(+ model_name="adapterinstance",+ constraint=models.UniqueConstraint(+ fields=("adapter_name", "adapter_type", "organization"),+ name="unique_organization_adapter",+ ),+ ),+ ]diff --git a/backend/platform_settings_v2/migrations/__init__.py b/backend/adapter_processor_v2/migrations/__init__.pysimilarity index 100%rename from backend/platform_settings_v2/migrations/__init__.pyrename to backend/adapter_processor_v2/migrations/__init__.pydiff --git a/backend/adapter_processor_v2/models.py b/backend/adapter_processor_v2/models.pyindex 5e63056c5..03c257de7 100644--- a/backend/adapter_processor_v2/models.py+++ b/backend/adapter_processor_v2/models.py@@ -122,7 +122,7 @@ class AdapterInstance(DefaultOrganizationMixin, BaseModel):class Meta:verbose_name = "adapter instance"verbose_name_plural = "adapter instances"- db_table = "adapter_instance_v2"+ db_table = "adapter_instance"constraints = [models.UniqueConstraint(fields=["adapter_name", "adapter_type", "organization"],@@ -165,9 +165,6 @@ class AdapterInstance(DefaultOrganizationMixin, BaseModel):class UserDefaultAdapter(BaseModel):- user = models.OneToOneField(- User, on_delete=models.CASCADE, related_name="organization_default_adapters"- )organization_member = models.OneToOneField(OrganizationMember,on_delete=models.CASCADE,@@ -205,4 +202,4 @@ class UserDefaultAdapter(BaseModel):class Meta:verbose_name = "Default Adapter for Organization User"verbose_name_plural = "Default Adapters for Organization Users"- db_table = "default_organization_user_adapter_v2"+ db_table = "default_organization_user_adapter"diff --git a/backend/adapter_processor_v2/views.py b/backend/adapter_processor_v2/views.pyindex 93cddf5aa..b5a65179f 100644--- a/backend/adapter_processor_v2/views.py+++ b/backend/adapter_processor_v2/views.py@@ -65,7 +65,12 @@ class DefaultAdapterViewSet(ModelViewSet):self, request: Request, *args: tuple[Any], **kwargs: dict[str, Any]) -> HttpResponse:try:- user_default_adapter = UserDefaultAdapter.objects.get(user=request.user)+ organization_member = OrganizationMemberService.get_user_by_id(+ request.user.id+ )+ user_default_adapter = UserDefaultAdapter.objects.get(+ organization_member=organization_member+ )serializer = UserDefaultAdapterSerializer(user_default_adapter).datareturn Response(serializer)@@ -172,13 +177,18 @@ class AdapterInstanceViewSet(ModelViewSet):serializer.is_valid(raise_exception=True)try:instance = serializer.save()+ organization_member = OrganizationMemberService.get_user_by_id(+ request.user.id+ )# Check to see if there is a default configured# for this adapter_type and for the current user(user_default_adapter,created,- ) = UserDefaultAdapter.objects.get_or_create(user=request.user)+ ) = UserDefaultAdapter.objects.get_or_create(+ organization_member=organization_member+ )adapter_type = serializer.validated_data.get(AdapterKeys.ADAPTER_TYPE)if (adapter_type == AdapterKeys.LLM) and (@@ -222,8 +232,11 @@ class AdapterInstanceViewSet(ModelViewSet):adapter_instance: AdapterInstance = self.get_object()adapter_type = adapter_instance.adapter_typetry:+ organization_member = OrganizationMemberService.get_user_by_id(+ request.user.id+ )user_default_adapter: UserDefaultAdapter = UserDefaultAdapter.objects.get(- user=request.user+ organization_member=organization_member)if (@@ -279,8 +292,13 @@ class AdapterInstanceViewSet(ModelViewSet):# Remove the same from his defaultfor user_id in removed_users:try:- user_default_adapter = UserDefaultAdapter.objects.get(- user_id=user_id+ organization_member = OrganizationMemberService.get_user_by_id(+ id=user_id+ )+ user_default_adapter: UserDefaultAdapter = (+ UserDefaultAdapter.objects.get(+ organization_member=organization_member+ ))if user_default_adapter.default_llm_adapter == adapter:diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.pyindex 802d8f8bc..486425cae 100644--- a/backend/api_v2/api_deployment_views.py+++ b/backend/api_v2/api_deployment_views.py@@ -135,7 +135,7 @@ class APIDeploymentViewSet(viewsets.ModelViewSet):) -> Response:"""Downloads a Postman Collection of the API deployment instance."""instance = self.get_object()- api_key_inst = instance.apikey_set.filter(is_active=True).first()+ api_key_inst = instance.api_keys.filter(is_active=True).first()if not api_key_inst:logger.error(f"No active API key set for deployment {instance.pk}")raise NoActiveAPIKeyError(deployment_name=instance.display_name)diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.pyindex 4c6d1698b..be9dc0b10 100644--- a/backend/api_v2/deployment_helper.py+++ b/backend/api_v2/deployment_helper.py@@ -191,6 +191,7 @@ class DeploymentHelper:api: APIDeployment,file_objs: list[UploadedFile],timeout: int,+ include_metadata: bool = False,) -> ReturnDict:"""Execute workflow by api.@@ -217,6 +218,7 @@ class DeploymentHelper:hash_values_of_files=hash_values_of_files,timeout=timeout,execution_id=execution_id,+ include_metadata=include_metadata,)result.status_api = DeploymentHelper.construct_status_endpoint(api_endpoint=api.api_endpoint, execution_id=execution_iddiff --git a/backend/api_v2/execution_urls.py b/backend/api_v2/execution_urls.pynew file mode 100644index 000000000..a8ca4feae--- /dev/null+++ b/backend/api_v2/execution_urls.py@@ -0,0 +1,16 @@+from api_v2.api_deployment_views import DeploymentExecution+from django.urls import re_path+from rest_framework.urlpatterns import format_suffix_patterns++execute = DeploymentExecution.as_view()+++urlpatterns = format_suffix_patterns(+ [+ re_path(+ r"^api/(?P<org_name>[\w-]+)/(?P<api_name>[\w-]+)/?$",+ execute,+ name="api_deployment_execution",+ )+ ]+)diff --git a/backend/api_v2/migrations/0001_initial.py b/backend/api_v2/migrations/0001_initial.pynew file mode 100644index 000000000..89653832b--- /dev/null+++ b/backend/api_v2/migrations/0001_initial.py@@ -0,0 +1,208 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("account_v2", "0001_initial"),+ ("workflow_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="APIDeployment",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "display_name",+ models.CharField(+ db_comment="User-given display name for the API.",+ default="default api",+ max_length=30,+ ),+ ),+ (+ "description",+ models.CharField(+ blank=True,+ db_comment="User-given description for the API.",+ default="",+ max_length=255,+ ),+ ),+ (+ "is_active",+ models.BooleanField(+ db_comment="Flag indicating whether the API is active or not.",+ default=True,+ ),+ ),+ (+ "api_endpoint",+ models.CharField(+ db_comment="URL endpoint for the API deployment.",+ editable=False,+ max_length=255,+ unique=True,+ ),+ ),+ (+ "api_name",+ models.CharField(+ db_comment="Short name for the API deployment.",+ default=uuid.uuid4,+ max_length=30,+ ),+ ),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="apis_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="apis_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "organization",+ models.ForeignKey(+ blank=True,+ db_comment="Foreign key reference to the Organization model.",+ default=None,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ to="account_v2.organization",+ ),+ ),+ (+ "workflow",+ models.ForeignKey(+ db_comment="Foreign key reference to the Workflow model.",+ on_delete=django.db.models.deletion.CASCADE,+ related_name="apis",+ to="workflow_v2.workflow",+ ),+ ),+ ],+ options={+ "verbose_name": "Api Deployment",+ "verbose_name_plural": "Api Deployments",+ "db_table": "api_deployment",+ },+ ),+ migrations.CreateModel(+ name="APIKey",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ db_comment="Unique identifier for the API key.",+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "api_key",+ models.UUIDField(+ db_comment="Actual key UUID.",+ default=uuid.uuid4,+ editable=False,+ unique=True,+ ),+ ),+ (+ "description",+ models.CharField(+ db_comment="Description of the API key.",+ max_length=255,+ null=True,+ ),+ ),+ (+ "is_active",+ models.BooleanField(+ db_comment="Flag indicating whether the API key is active or not.",+ default=True,+ ),+ ),+ (+ "api",+ models.ForeignKey(+ db_comment="Foreign key reference to the APIDeployment model.",+ on_delete=django.db.models.deletion.CASCADE,+ related_name="api_keys",+ to="api_v2.apideployment",+ ),+ ),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="api_keys_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="api_keys_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ ],+ options={+ "verbose_name": "Api Deployment key",+ "verbose_name_plural": "Api Deployment keys",+ "db_table": "api_deployment_key",+ },+ ),+ migrations.AddConstraint(+ model_name="apideployment",+ constraint=models.UniqueConstraint(+ fields=("api_name", "organization"), name="unique_api_name"+ ),+ ),+ ]diff --git a/backend/api_v2/migrations/__init__.py b/backend/api_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/api_v2/models.py b/backend/api_v2/models.pyindex 545426537..6cb99a9f5 100644--- a/backend/api_v2/models.py+++ b/backend/api_v2/models.py@@ -104,7 +104,7 @@ class APIDeployment(DefaultOrganizationMixin, BaseModel):class Meta:verbose_name = "Api Deployment"verbose_name_plural = "Api Deployments"- db_table = "api_deployment_v2"+ db_table = "api_deployment"constraints = [models.UniqueConstraint(fields=["api_name", "organization"],@@ -164,4 +164,4 @@ class APIKey(BaseModel):class Meta:verbose_name = "Api Deployment key"verbose_name_plural = "Api Deployment keys"- db_table = "api_deployment_key_v2"+ db_table = "api_deployment_key"diff --git a/backend/api_v2/urls.py b/backend/api_v2/urls.pyindex 065617d66..504b13a7f 100644--- a/backend/api_v2/urls.py+++ b/backend/api_v2/urls.py@@ -1,6 +1,6 @@from api_v2.api_deployment_views import APIDeploymentViewSet, DeploymentExecutionfrom api_v2.api_key_views import APIKeyViewSet-from django.urls import path, re_path+from django.urls import pathfrom rest_framework.urlpatterns import format_suffix_patternsdeployment = APIDeploymentViewSet.as_view(@@ -52,11 +52,6 @@ urlpatterns = format_suffix_patterns(download_postman_collection,name="download_postman_collection",),- re_path(- r"^api/(?P<org_name>[\w-]+)/(?P<api_name>[\w-]+)/?$",- execute,- name="api_deployment_execution",- ),path("keys/<uuid:pk>/", key_details, name="key_details"),path("keys/api/<str:api_id>/", api_key, name="api_key"),]diff --git a/backend/backend/base_urls.py b/backend/backend/base_urls.pynew file mode 100644index 000000000..b0c728867--- /dev/null+++ b/backend/backend/base_urls.py@@ -0,0 +1,21 @@+# base_urls.py+from django.conf import settings+from django.urls import include, path++from .public_urls_v2 import urlpatterns as public_urls++# Import urlpatterns from each file+from .urls_v2 import urlpatterns as tenant_urls++# Combine the URL patterns+urlpatterns = [+ path(+ f"{settings.TENANT_SUBFOLDER_PREFIX}/",+ include((tenant_urls, "tenant"), namespace="tenant"),+ ),+ path(+ f"{settings.PATH_PREFIX}/", include((public_urls, "public"), namespace="public")+ ),+ # API deployment+ path(f"{settings.API_DEPLOYMENT_PATH_PREFIX}/", include("api_v2.execution_urls")),+]diff --git a/backend/backend/public_urls_v2.py b/backend/backend/public_urls_v2.pynew file mode 100644index 000000000..1e6d3550e--- /dev/null+++ b/backend/backend/public_urls_v2.py@@ -0,0 +1,56 @@+"""URL configuration for backend project.++The `urlpatterns` list routes URLs to views. For more information please see:+ https://docs.djangoproject.com/en/4.2/topics/http/urls/+Examples:+Function views+ 1. Add an import: from my_app import views+ 2. Add a URL to urlpatterns: path('', views.home, name='home')+Class-based views+ 1. Add an import: from other_app.views import Home+ 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')+Including another URLconf+ 1. Import the include() function: from django.urls import include, path+ 2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))+"""++from account_v2.admin import admin+from django.conf import settings+from django.conf.urls import * # noqa: F401, F403+from django.conf.urls.static import static+from django.urls import include, path++path_prefix = settings.PATH_PREFIX+api_path_prefix = settings.API_DEPLOYMENT_PATH_PREFIX++urlpatterns = [+ path("", include("account_v2.urls")),+ # Connector OAuth+ path("", include("connector_auth_v2.urls")),+ # # Docs+ path("", include("docs.urls")),+ # # # API deployment+ # path("api/", include("api_v2.execution_urls")),+ # # Feature flags+ path("flags/", include("feature_flag.urls")),+]+if settings.ADMIN_ENABLED:+ # Admin URLs+ urlpatterns += [+ path("admin/", admin.site.urls),+ path(+ "admin/doc/",+ include("django.contrib.admindocs.urls"),+ ),+ ]+urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)+++try:+ import pluggable_apps.platform_admin.urls # noqa: F401++ urlpatterns += [+ path("", include("pluggable_apps.platform_admin.urls")),+ ]+except ImportError:+ passdiff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 260d4561b..cb87a99cb 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -139,6 +139,7 @@ DB_USER = os.environ.get("DB_USER", "unstract_dev")DB_HOST = os.environ.get("DB_HOST", "backend-db-1")DB_PASSWORD = os.environ.get("DB_PASSWORD", "unstract_pass")DB_PORT = os.environ.get("DB_PORT", 5432)+DB_SCHEMA = os.environ.get("DB_SCHEMA", "unstract_v2")DEFAULT_ORGANIZATION = "default_org"FLIPT_BASE_URL = os.environ.get("FLIPT_BASE_URL", "http://localhost:9005")@@ -175,6 +176,9 @@ CELERY_BROKER_URL = get_required_setting(INDEXING_FLAG_TTL = int(get_required_setting("INDEXING_FLAG_TTL"))NOTIFICATION_TIMEOUT = int(get_required_setting("NOTIFICATION_TIMEOUT", "5"))+ATOMIC_REQUESTS = CommonUtils.str_to_bool(+ os.environ.get("DJANGO_ATOMIC_REQUESTS", "False")+)# Flag to Enable django adminADMIN_ENABLED = False@@ -192,68 +196,117 @@ ALLOWED_HOSTS = ["*"]CSRF_TRUSTED_ORIGINS = [WEB_APP_ORIGIN_URL]CORS_ALLOW_ALL_ORIGINS = False--# Application definition-SHARED_APPS = (- # Multitenancy- "corsheaders",- # For the organization model- "account",- "account_usage",- # Django apps should go below this line- "django.contrib.admin",- "django.contrib.auth",- "django.contrib.contenttypes",- "django.contrib.sessions",- "django.contrib.messages",- "django.contrib.staticfiles",- "django.contrib.admindocs",- # Third party apps should go below this line,- "rest_framework",- # Connector OAuth- "connector_auth",- "social_django",- # Doc generator- "drf_yasg",- "docs",- # Plugins- "plugins",- "feature_flag",- "django_celery_beat",-)-if not check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):- SHARED_APPS = ("django_tenants",) + SHARED_APPS--TENANT_APPS = (- # your tenant-specific apps- "django.contrib.admin",- "django.contrib.auth",- "django.contrib.contenttypes",- "django.contrib.messages",- "django.contrib.staticfiles",- "tenant_account",- "project",- "prompt",- "connector",- "adapter_processor",- "file_management",- "workflow_manager.endpoint",- "workflow_manager.workflow",- "tool_instance",- "pipeline",- "platform_settings",- "api",- "prompt_studio.prompt_profile_manager",- "prompt_studio.prompt_studio",- "prompt_studio.prompt_studio_core",- "prompt_studio.prompt_studio_registry",- "prompt_studio.prompt_studio_output_manager",- "prompt_studio.prompt_studio_document_manager",- "prompt_studio.prompt_studio_index_manager",- "usage",- "notification",-)+ SHARED_APPS = (+ # Multitenancy+ "django_tenants",+ "corsheaders",+ # For the organization model+ "account",+ "account_usage",+ # Django apps should go below this line+ "django.contrib.admin",+ "django.contrib.auth",+ "django.contrib.contenttypes",+ "django.contrib.sessions",+ "django.contrib.messages",+ "django.contrib.staticfiles",+ "django.contrib.admindocs",+ # Third party apps should go below this line,+ "rest_framework",+ # Connector OAuth+ "connector_auth",+ "social_django",+ # Doc generator+ "drf_yasg",+ "docs",+ # Plugins+ "plugins",+ "feature_flag",+ "django_celery_beat",+ )+ TENANT_APPS = (+ # your tenant-specific apps+ "django.contrib.admin",+ "django.contrib.auth",+ "django.contrib.contenttypes",+ "django.contrib.messages",+ "django.contrib.staticfiles",+ "tenant_account",+ "project",+ "prompt",+ "connector",+ "adapter_processor",+ "file_management",+ "workflow_manager.endpoint",+ "workflow_manager.workflow",+ "tool_instance",+ "pipeline",+ "platform_settings",+ "api",+ "prompt_studio.prompt_profile_manager",+ "prompt_studio.prompt_studio",+ "prompt_studio.prompt_studio_core",+ "prompt_studio.prompt_studio_registry",+ "prompt_studio.prompt_studio_output_manager",+ "prompt_studio.prompt_studio_document_manager",+ "prompt_studio.prompt_studio_index_manager",+ "usage",+ "notification",+ )+else:+ SHARED_APPS = (+ # Multitenancy+ # "django_tenants",+ "corsheaders",+ # For the organization model+ "account_v2",+ # Django apps should go below this line+ "django.contrib.admin",+ "django.contrib.auth",+ "django.contrib.contenttypes",+ "django.contrib.sessions",+ "django.contrib.messages",+ "django.contrib.staticfiles",+ "django.contrib.admindocs",+ # Third party apps should go below this line,+ "rest_framework",+ # Connector OAuth+ # "connector_auth",+ "social_django",+ # Doc generator+ "drf_yasg",+ "docs",+ # Plugins+ "plugins",+ "feature_flag",+ "django_celery_beat",+ )+ v2_apps = (+ "migration_tools",+ # "account_v2",+ "connector_auth_v2",+ "tenant_account_v2",+ "connector_v2",+ "adapter_processor_v2",+ "file_management",+ "workflow_manager.endpoint_v2",+ "workflow_manager.workflow_v2",+ "tool_instance_v2",+ "pipeline_v2",+ "platform_settings_v2",+ "api_v2",+ "usage_v2",+ "prompt_studio.prompt_profile_manager_v2",+ "prompt_studio.prompt_studio_v2",+ "prompt_studio.prompt_studio_core_v2",+ "prompt_studio.prompt_studio_registry_v2",+ "prompt_studio.prompt_studio_output_manager_v2",+ "prompt_studio.prompt_studio_document_manager_v2",+ "prompt_studio.prompt_studio_index_manager_v2",+ )+ SHARED_APPS += v2_apps+ TENANT_APPS = []INSTALLED_APPS = list(SHARED_APPS) + [app for app in TENANT_APPS if app not in SHARED_APPS@@ -297,6 +350,21 @@ if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):# NamespacesSOCIAL_AUTH_URL_NAMESPACE = "public:social"LOGIN_CALLBACK_URL_NAMESPACE = "public:callback"+ DATABASES = {+ "default": {+ "ENGINE": DB_ENGINE,+ "NAME": f"{DB_NAME}",+ "USER": f"{DB_USER}",+ "HOST": f"{DB_HOST}",+ "PASSWORD": f"{DB_PASSWORD}",+ "PORT": f"{DB_PORT}",+ "ATOMIC_REQUESTS": ATOMIC_REQUESTS,+ "OPTIONS": {+ "options": f"-c search_path={DB_SCHEMA}",+ "application_name": os.environ.get("APPLICATION_NAME", ""),+ },+ }+ }else:# Middleware ConfigurationTENANT_MIDDLEWARE = "django_tenants.middleware.TenantSubfolderMiddleware"@@ -326,6 +394,20 @@ else:# NamespacesSOCIAL_AUTH_URL_NAMESPACE = "social"LOGIN_CALLBACK_URL_NAMESPACE = "callback"+ DATABASES = {+ "default": {+ "ENGINE": DB_ENGINE,+ "NAME": f"{DB_NAME}",+ "USER": f"{DB_USER}",+ "HOST": f"{DB_HOST}",+ "PASSWORD": f"{DB_PASSWORD}",+ "PORT": f"{DB_PORT}",+ "ATOMIC_REQUESTS": ATOMIC_REQUESTS,+ "OPTIONS": {+ "application_name": os.environ.get("APPLICATION_NAME", ""),+ },+ }+ }MIDDLEWARE = ["log_request_id.middleware.RequestIDMiddleware",@@ -366,27 +448,6 @@ TEMPLATES = [WSGI_APPLICATION = "backend.wsgi.application"-ATOMIC_REQUESTS = CommonUtils.str_to_bool(- os.environ.get("DJANGO_ATOMIC_REQUESTS", "False")-)-# Database-# https://docs.djangoproject.com/en/4.2/ref/settings/#databases--DATABASES = {- "default": {- "ENGINE": DB_ENGINE,- "NAME": f"{DB_NAME}",- "USER": f"{DB_USER}",- "HOST": f"{DB_HOST}",- "PASSWORD": f"{DB_PASSWORD}",- "PORT": f"{DB_PORT}",- "ATOMIC_REQUESTS": ATOMIC_REQUESTS,- "OPTIONS": {- "application_name": os.environ.get("APPLICATION_NAME", ""),- },- }-}-# SocketIO connection managerSOCKET_IO_MANAGER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}"diff --git a/backend/backend/urls_v2.py b/backend/backend/urls_v2.pynew file mode 100644index 000000000..0c0b3f40e--- /dev/null+++ b/backend/backend/urls_v2.py@@ -0,0 +1,69 @@+"""URL configuration for backend project.++The `urlpatterns` list routes URLs to views. For more information please see:+ https://docs.djangoproject.com/en/4.2/topics/http/urls/+Examples:+Function views+ 1. Add an import: from my_app import views+ 2. Add a URL to urlpatterns: path('', views.home, name='home')+Class-based views+ 1. Add an import: from other_app.views import Home+ 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')+Including another URLconf+ 1. Import the include() function: from django.urls import include, path+ 2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))+"""++from django.conf.urls import * # noqa: F401, F403+from django.urls import include, path++from backend.constants import UrlPathConstants++urlpatterns = [+ path("", include("tenant_account_v2.urls")),+ path("", include("connector_v2.urls")),+ path("", include("connector_processor.urls")),+ path("", include("adapter_processor_v2.urls")),+ path("", include("file_management.urls")),+ path("", include("tool_instance_v2.urls")),+ path("", include("pipeline_v2.urls")),+ path("", include("feature_flag.urls")),+ path("workflow/", include("workflow_manager.urls")),+ path("platform/", include("platform_settings_v2.urls")),+ path("api/", include("api_v2.urls")),+ path("usage/", include("usage_v2.urls")),+ path(+ UrlPathConstants.PROMPT_STUDIO,+ include("prompt_studio.prompt_profile_manager_v2.urls"),+ ),+ path(+ UrlPathConstants.PROMPT_STUDIO,+ include("prompt_studio.prompt_studio_v2.urls"),+ ),+ path("", include("prompt_studio.prompt_studio_core_v2.urls")),+ path(+ UrlPathConstants.PROMPT_STUDIO,+ include("prompt_studio.prompt_studio_registry_v2.urls"),+ ),+ path(+ UrlPathConstants.PROMPT_STUDIO,+ include("prompt_studio.prompt_studio_output_manager_v2.urls"),+ ),+ path(+ UrlPathConstants.PROMPT_STUDIO,+ include("prompt_studio.prompt_studio_document_manager_v2.urls"),+ ),+ path(+ UrlPathConstants.PROMPT_STUDIO,+ include("prompt_studio.prompt_studio_index_manager_v2.urls"),+ ),+]++try:+ import pluggable_apps.subscription.urls # noqa: F401++ urlpatterns += [+ path("", include("pluggable_apps.subscription.urls")),+ ]+except ImportError:+ passdiff --git a/backend/connector_auth_v2/migrations/0001_initial.py b/backend/connector_auth_v2/migrations/0001_initial.pynew file mode 100644index 000000000..26f9a0969--- /dev/null+++ b/backend/connector_auth_v2/migrations/0001_initial.py@@ -0,0 +1,60 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+import social_django.storage+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ]++ operations = [+ migrations.CreateModel(+ name="ConnectorAuth",+ fields=[+ ("provider", models.CharField(max_length=32)),+ ("uid", models.CharField(db_index=True, max_length=255)),+ ("extra_data", models.JSONField(default=dict)),+ ("created", models.DateTimeField(auto_now_add=True)),+ ("modified", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "user",+ models.ForeignKey(+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="connector_auths",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ ],+ options={+ "verbose_name": "Connector Auth",+ "verbose_name_plural": "Connector Auths",+ "db_table": "connector_auth",+ },+ bases=(models.Model, social_django.storage.DjangoUserMixin),+ ),+ migrations.AddConstraint(+ model_name="connectorauth",+ constraint=models.UniqueConstraint(+ fields=("provider", "uid"), name="unique_provider_uid_index"+ ),+ ),+ ]diff --git a/backend/connector_auth_v2/migrations/__init__.py b/backend/connector_auth_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/connector_auth_v2/models.py b/backend/connector_auth_v2/models.pyindex ca8caaefd..762f6605f 100644--- a/backend/connector_auth_v2/models.py+++ b/backend/connector_auth_v2/models.py@@ -125,7 +125,7 @@ class ConnectorAuth(AbstractUserSocialAuth):app_label = "connector_auth_v2"verbose_name = "Connector Auth"verbose_name_plural = "Connector Auths"- db_table = "connector_auth_v2"+ db_table = "connector_auth"constraints = [models.UniqueConstraint(fields=[diff --git a/backend/connector_v2/migrations/0001_initial.py b/backend/connector_v2/migrations/0001_initial.pynew file mode 100644index 000000000..91d2d043e--- /dev/null+++ b/backend/connector_v2/migrations/0001_initial.py@@ -0,0 +1,117 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ ("connector_auth_v2", "0001_initial"),+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("workflow_v2", "0001_initial"),+ ("account_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="ConnectorInstance",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("connector_name", models.TextField(max_length=128)),+ ("connector_id", models.CharField(default="", max_length=128)),+ ("connector_metadata", models.BinaryField(null=True)),+ ("connector_version", models.CharField(default="", max_length=64)),+ (+ "connector_type",+ models.CharField(+ choices=[("INPUT", "Input"), ("OUTPUT", "Output")]+ ),+ ),+ (+ "connector_mode",+ models.CharField(+ choices=[(0, "UNKNOWN"), (1, "FILE_SYSTEM"), (2, "DATABASE")],+ db_comment="0: UNKNOWN, 1: FILE_SYSTEM, 2: DATABASE",+ default=0,+ ),+ ),+ (+ "connector_auth",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="connector_instances",+ to="connector_auth_v2.connectorauth",+ ),+ ),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="connectors_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="connectors_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "organization",+ models.ForeignKey(+ blank=True,+ db_comment="Foreign key reference to the Organization model.",+ default=None,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ to="account_v2.organization",+ ),+ ),+ (+ "workflow",+ models.ForeignKey(+ on_delete=django.db.models.deletion.CASCADE,+ related_name="connector_workflow",+ to="workflow_v2.workflow",+ ),+ ),+ ],+ options={+ "verbose_name": "Connector Instance",+ "verbose_name_plural": "Connector Instances",+ "db_table": "connector_instance",+ },+ ),+ migrations.AddConstraint(+ model_name="connectorinstance",+ constraint=models.UniqueConstraint(+ fields=("connector_name", "workflow", "connector_type"),+ name="unique_workflow_connector",+ ),+ ),+ ]diff --git a/backend/connector_v2/models.py b/backend/connector_v2/models.pyindex 3fd535b05..f8db9915a 100644--- a/backend/connector_v2/models.py+++ b/backend/connector_v2/models.py@@ -111,17 +111,42 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):f" workflow: {self.workflow})")+ def get_connector_metadata_bytes(self):+ """Convert connector_metadata to bytes if it is a memoryview.++ Returns:+ bytes: The connector_metadata as bytes.+ """+ if isinstance(self.connector_metadata, memoryview):+ return self.connector_metadata.tobytes()+ return self.connector_metadata+@propertydef metadata(self) -> Any:+ """Decrypt and return the connector metadata as a dictionary.++ This property handles the decryption of the connector_metadata,+ converting it to bytes if necessary, and then loading the decrypted+ JSON string into a dictionary.++ Returns:+ dict: The decrypted connector metadata.+ """++ connector_metadata_bytes = self.get_connector_metadata_bytes()++ if connector_metadata_bytes is None:+ return None++ if isinstance(connector_metadata_bytes, (dict)):+ return connector_metadata_bytesencryption_secret: str = settings.ENCRYPTION_KEYcipher_suite: Fernet = Fernet(encryption_secret.encode("utf-8"))- decrypted_value = cipher_suite.decrypt(- bytes(self.connector_metadata).decode("utf-8")- )- return json.loads(decrypted_value)+ decrypted_value = cipher_suite.decrypt(connector_metadata_bytes)+ return json.loads(decrypted_value.decode("utf-8"))class Meta:- db_table = "connector_instance_v2"+ db_table = "connector_instance"verbose_name = "Connector Instance"verbose_name_plural = "Connector Instances"constraints = [diff --git a/backend/connector_v2/serializers.py b/backend/connector_v2/serializers.pyindex cd9727e84..4704635ba 100644--- a/backend/connector_v2/serializers.py+++ b/backend/connector_v2/serializers.py@@ -71,8 +71,6 @@ class ConnectorInstanceSerializer(AuditSerializer):encryption_secret: str = settings.ENCRYPTION_KEYf: Fernet = Fernet(encryption_secret.encode("utf-8"))json_string: str = json.dumps(kwargs.get(CIKey.CONNECTOR_METADATA))- if self.validated_data:- self.validated_data.pop(CIKey.CONNECTOR_METADATA)kwargs[CIKey.CONNECTOR_METADATA] = f.encrypt(json_string.encode("utf-8"))diff --git a/backend/connector_v2/views.py b/backend/connector_v2/views.pyindex 3f098d355..4431bfcfa 100644--- a/backend/connector_v2/views.py+++ b/backend/connector_v2/views.py@@ -24,7 +24,6 @@ logger = logging.getLogger(__name__)class ConnectorInstanceViewSet(viewsets.ModelViewSet):versioning_class = URLPathVersioning- queryset = ConnectorInstance.objects.all()serializer_class = ConnectorInstanceSerializerdef get_queryset(self) -> Optional[QuerySet]:diff --git a/backend/middleware/organization_middleware.py b/backend/middleware/organization_middleware.pynew file mode 100644index 000000000..a2b9450ce--- /dev/null+++ b/backend/middleware/organization_middleware.py@@ -0,0 +1,20 @@+import re++from django.conf import settings+from django.utils.deprecation import MiddlewareMixin+++class OrganizationMiddleware(MiddlewareMixin):+ def process_request(self, request):+ tenant_prefix = settings.TENANT_SUBFOLDER_PREFIX.rstrip("/") + "/"+ pattern = rf"^{tenant_prefix}(?P<org_id>[^/]+)/"++ # Check if the URL matches the pattern with organization ID+ match = re.match(pattern, request.path)+ if match:+ org_id = match.group("org_id")+ request.organization_id = org_id+ new_path = re.sub(pattern, "/" + tenant_prefix, request.path_info)+ request.path_info = new_path+ else:+ request.organization_id = Nonediff --git a/backend/permissions/permission.py b/backend/permissions/permission.pyindex e568ece7d..98bda6a17 100644--- a/backend/permissions/permission.py+++ b/backend/permissions/permission.py@@ -1,10 +1,17 @@from typing import Any-from adapter_processor.models import AdapterInstancefrom rest_framework import permissionsfrom rest_framework.request import Requestfrom rest_framework.views import APIView+from backend.constants import FeatureFlag+from unstract.flags.feature_flag import check_feature_flag_status++if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):+ from adapter_processor_v2.models import AdapterInstance+else:+ from adapter_processor.models import AdapterInstance+class IsOwner(permissions.BasePermission):"""Custom permission to only allow owners of an object."""diff --git a/backend/pipeline_v2/apps.py b/backend/pipeline_v2/apps.pynew file mode 100644index 000000000..dd81e66c7--- /dev/null+++ b/backend/pipeline_v2/apps.py@@ -0,0 +1,5 @@+from django.apps import AppConfig+++class PipelineConfig(AppConfig):+ name = "pipeline_v2"diff --git a/backend/pipeline_v2/constants.py b/backend/pipeline_v2/constants.pyindex 2475ad79f..438982577 100644--- a/backend/pipeline_v2/constants.py+++ b/backend/pipeline_v2/constants.py@@ -63,4 +63,5 @@ class PipelineURL:DETAIL = "pipeline-detail"EXECUTIONS = "pipeline-executions"LIST = "pipeline-list"- EXECUTE = "tenant:pipeline-execute"+ EXECUTE = "pipeline-execute"+ EXECUTE_NAMESPACE = "tenant:pipeline-execute"diff --git a/backend/pipeline_v2/manager.py b/backend/pipeline_v2/manager.pyindex d5297107d..8e17477f7 100644--- a/backend/pipeline_v2/manager.py+++ b/backend/pipeline_v2/manager.py@@ -48,7 +48,9 @@ class PipelineManager:) -> Optional[dict[str, Any]]:"""Gets the required data to be passed while executing a pipeline Anychanges to pipeline execution needs to be propagated here."""- callback_url = settings.DJANGO_APP_BACKEND_URL + reverse(PipelineURL.EXECUTE)+ callback_url = settings.DJANGO_APP_BACKEND_URL + reverse(+ PipelineURL.EXECUTE_NAMESPACE+ )job_headers = {RequestHeader.X_API_KEY: settings.INTERNAL_SERVICE_API_KEY}job_kwargs = {RequestConstants.VERB: "POST",diff --git a/backend/pipeline_v2/migrations/0001_initial.py b/backend/pipeline_v2/migrations/0001_initial.pynew file mode 100644index 000000000..47c8eebbb--- /dev/null+++ b/backend/pipeline_v2/migrations/0001_initial.py@@ -0,0 +1,144 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("account_v2", "0001_initial"),+ ("workflow_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="Pipeline",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("pipeline_name", models.CharField(default="", max_length=32)),+ ("app_id", models.TextField(blank=True, max_length=32, null=True)),+ ("active", models.BooleanField(default=False)),+ ("scheduled", models.BooleanField(default=False)),+ (+ "cron_string",+ models.TextField(db_comment="UNIX cron string", max_length=256),+ ),+ (+ "pipeline_type",+ models.CharField(+ choices=[+ ("ETL", "ETL"),+ ("TASK", "TASK"),+ ("DEFAULT", "Default"),+ ("APP", "App"),+ ],+ default="DEFAULT",+ ),+ ),+ ("run_count", models.IntegerField(default=0)),+ ("last_run_time", models.DateTimeField(blank=True, null=True)),+ (+ "last_run_status",+ models.CharField(+ choices=[+ ("SUCCESS", "Success"),+ ("FAILURE", "Failure"),+ ("INPROGRESS", "Inprogress"),+ ("YET_TO_START", "Yet to start"),+ ("RESTARTING", "Restarting"),+ ("PAUSED", "Paused"),+ ],+ default="YET_TO_START",+ ),+ ),+ (+ "app_icon",+ models.URLField(+ blank=True,+ db_comment="Field to store icon url for Apps",+ null=True,+ ),+ ),+ (+ "app_url",+ models.URLField(+ blank=True, db_comment="Stores deployed URL for App", null=True+ ),+ ),+ ("access_control_bundle_id", models.TextField(blank=True, null=True)),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="pipelines_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="pipelines_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "organization",+ models.ForeignKey(+ blank=True,+ db_comment="Foreign key reference to the Organization model.",+ default=None,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ to="account_v2.organization",+ ),+ ),+ (+ "workflow",+ models.ForeignKey(+ on_delete=django.db.models.deletion.CASCADE,+ related_name="pipelines",+ to="workflow_v2.workflow",+ ),+ ),+ ],+ options={+ "verbose_name": "Pipeline",+ "verbose_name_plural": "Pipelines",+ "db_table": "pipeline",+ },+ ),+ migrations.AddConstraint(+ model_name="pipeline",+ constraint=models.UniqueConstraint(+ fields=("id", "pipeline_type"), name="unique_pipeline_entity"+ ),+ ),+ migrations.AddConstraint(+ model_name="pipeline",+ constraint=models.UniqueConstraint(+ fields=("pipeline_name", "organization"), name="unique_pipeline_name"+ ),+ ),+ ]diff --git a/backend/pipeline_v2/migrations/__init__.py b/backend/pipeline_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/pipeline_v2/models.py b/backend/pipeline_v2/models.pyindex 35ef1914c..ccb90313a 100644--- a/backend/pipeline_v2/models.py+++ b/backend/pipeline_v2/models.py@@ -95,7 +95,7 @@ class Pipeline(DefaultOrganizationMixin, BaseModel):class Meta:verbose_name = "Pipeline"verbose_name_plural = "Pipelines"- db_table = "pipeline_v2"+ db_table = "pipeline"constraints = [models.UniqueConstraint(fields=["id", "pipeline_type"],diff --git a/backend/prompt_studio/prompt_profile_manager_v2/migrations/0001_initial.py b/backend/prompt_studio/prompt_profile_manager_v2/migrations/0001_initial.pynew file mode 100644index 000000000..01a98c45a--- /dev/null+++ b/backend/prompt_studio/prompt_profile_manager_v2/migrations/0001_initial.py@@ -0,0 +1,159 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("prompt_studio_core_v2", "0001_initial"),+ ("adapter_processor_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="ProfileManager",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "profile_id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("profile_name", models.TextField()),+ ("chunk_size", models.IntegerField(blank=True, null=True)),+ ("chunk_overlap", models.IntegerField(blank=True, null=True)),+ ("reindex", models.BooleanField(default=False)),+ (+ "retrieval_strategy",+ models.TextField(+ blank=True,+ choices=[+ ("simple", "Simple retrieval"),+ ("subquestion", "Subquestion retrieval"),+ ],+ db_comment="Field to store the retrieval strategy for prompts",+ ),+ ),+ (+ "similarity_top_k",+ models.IntegerField(+ blank=True,+ db_comment="Field to store number of top embeddings to take into context",+ null=True,+ ),+ ),+ (+ "section",+ models.TextField(+ blank=True,+ db_comment="Field to store limit to section",+ null=True,+ ),+ ),+ (+ "is_default",+ models.BooleanField(+ db_comment="Default LLM Profile used in prompt", default=False+ ),+ ),+ (+ "is_summarize_llm",+ models.BooleanField(+ db_comment="Default LLM Profile used for summarizing",+ default=False,+ ),+ ),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="profile_managers_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "embedding_model",+ models.ForeignKey(+ on_delete=django.db.models.deletion.PROTECT,+ related_name="profiles_embedding_model",+ to="adapter_processor_v2.adapterinstance",+ ),+ ),+ (+ "llm",+ models.ForeignKey(+ db_comment="Field to store the LLM chosen by the user",+ on_delete=django.db.models.deletion.PROTECT,+ related_name="profiles_llm",+ to="adapter_processor_v2.adapterinstance",+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="profile_managers_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "prompt_studio_tool",+ models.ForeignKey(+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="profile_managers",+ to="prompt_studio_core_v2.customtool",+ ),+ ),+ (+ "vector_store",+ models.ForeignKey(+ db_comment="Field to store the chosen vector store.",+ on_delete=django.db.models.deletion.PROTECT,+ related_name="profiles_vector_store",+ to="adapter_processor_v2.adapterinstance",+ ),+ ),+ (+ "x2text",+ models.ForeignKey(+ db_comment="Field to store the X2Text Adapter chosen by the user",+ on_delete=django.db.models.deletion.PROTECT,+ related_name="profiles_x2text",+ to="adapter_processor_v2.adapterinstance",+ ),+ ),+ ],+ options={+ "verbose_name": "Profile Manager",+ "verbose_name_plural": "Profile Managers",+ "db_table": "profile_manager",+ },+ ),+ migrations.AddConstraint(+ model_name="profilemanager",+ constraint=models.UniqueConstraint(+ fields=("prompt_studio_tool", "profile_name"),+ name="unique_prompt_studio_tool_profile_name_index",+ ),+ ),+ ]diff --git a/backend/prompt_studio/prompt_profile_manager_v2/migrations/__init__.py b/backend/prompt_studio/prompt_profile_manager_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/prompt_studio/prompt_profile_manager_v2/models.py b/backend/prompt_studio/prompt_profile_manager_v2/models.pyindex 303e96f0f..be73899c0 100644--- a/backend/prompt_studio/prompt_profile_manager_v2/models.py+++ b/backend/prompt_studio/prompt_profile_manager_v2/models.py@@ -97,7 +97,7 @@ class ProfileManager(BaseModel):class Meta:verbose_name = "Profile Manager"verbose_name_plural = "Profile Managers"- db_table = "profile_manager_v2"+ db_table = "profile_manager"constraints = [models.UniqueConstraint(fields=["prompt_studio_tool", "profile_name"],diff --git a/backend/prompt_studio/prompt_studio_core_v2/constants.py b/backend/prompt_studio/prompt_studio_core_v2/constants.pyindex 55d61e32e..213fc066f 100644--- a/backend/prompt_studio/prompt_studio_core_v2/constants.py+++ b/backend/prompt_studio/prompt_studio_core_v2/constants.py@@ -88,6 +88,7 @@ class ToolStudioPromptKeys:PROFILE_MANAGER_ID = "profile_manager"CONTEXT = "context"METADATA = "metadata"+ INCLUDE_METADATA = "include_metadata"class FileViewTypes:diff --git a/backend/prompt_studio/prompt_studio_core_v2/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_core_v2/migrations/0001_initial.pynew file mode 100644index 000000000..7719320ca--- /dev/null+++ b/backend/prompt_studio/prompt_studio_core_v2/migrations/0001_initial.py@@ -0,0 +1,207 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ ("adapter_processor_v2", "0001_initial"),+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("account_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="CustomTool",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "tool_id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("tool_name", models.TextField()),+ ("description", models.TextField()),+ (+ "author",+ models.TextField(+ db_comment="Specific to the user who created the tool."+ ),+ ),+ (+ "icon",+ models.TextField(+ blank=True,+ db_comment="Field to store icon url for the custom tool.",+ ),+ ),+ (+ "output",+ models.TextField(+ blank=True, db_comment="Field to store the output format type."+ ),+ ),+ (+ "log_id",+ models.UUIDField(+ db_comment="Field to store unique log_id for polling",+ default=uuid.uuid4,+ ),+ ),+ (+ "summarize_context",+ models.BooleanField(+ db_comment="Flag to summarize content", default=False+ ),+ ),+ (+ "summarize_as_source",+ models.BooleanField(+ db_comment="Flag to use summarized content as source",+ default=False,+ ),+ ),+ (+ "summarize_prompt",+ models.TextField(+ blank=True, db_comment="Field to store the summarize prompt"+ ),+ ),+ (+ "preamble",+ models.TextField(+ blank=True,+ db_comment="Preamble to the prompts",+ default="Your ability to extract and summarize this context accurately is essential for effective analysis. Pay close attention to the context's language, structure, and any cross-references to ensure a comprehensive and precise extraction of information. Do not use prior knowledge or information from outside the context to answer the questions. Only use the information provided in the context to answer the questions.",+ ),+ ),+ (+ "postamble",+ models.TextField(+ blank=True,+ db_comment="Appended as postable to prompts.",+ default="Do not include any explanation in the reply. Only include the extracted information in the reply.",+ ),+ ),+ (+ "prompt_grammer",+ models.JSONField(+ blank=True,+ db_comment="Synonymous words used in prompt",+ null=True,+ ),+ ),+ (+ "exclude_failed",+ models.BooleanField(+ db_comment="Flag to make the answer null if it is incorrect",+ default=True,+ ),+ ),+ (+ "single_pass_extraction_mode",+ models.BooleanField(+ db_comment="Flag to enable or disable single pass extraction mode",+ default=False,+ ),+ ),+ (+ "enable_challenge",+ models.BooleanField(+ db_comment="Flag to enable or disable challenge", default=False+ ),+ ),+ (+ "enable_highlight",+ models.BooleanField(+ db_comment="Flag to enable or disable document highlighting",+ default=False,+ ),+ ),+ (+ "challenge_llm",+ models.ForeignKey(+ blank=True,+ db_comment="Field to store challenge llm",+ null=True,+ on_delete=django.db.models.deletion.PROTECT,+ related_name="custom_tools_challenge",+ to="adapter_processor_v2.adapterinstance",+ ),+ ),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="custom_tools_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="custom_tools_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "monitor_llm",+ models.ForeignKey(+ blank=True,+ db_comment="Field to store monitor llm",+ null=True,+ on_delete=django.db.models.deletion.PROTECT,+ related_name="custom_tools_monitor",+ to="adapter_processor_v2.adapterinstance",+ ),+ ),+ (+ "organization",+ models.ForeignKey(+ blank=True,+ db_comment="Foreign key reference to the Organization model.",+ default=None,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ to="account_v2.organization",+ ),+ ),+ (+ "shared_users",+ models.ManyToManyField(+ related_name="shared_custom_tools", to=settings.AUTH_USER_MODEL+ ),+ ),+ ],+ options={+ "verbose_name": "Custom Tool",+ "verbose_name_plural": "Custom Tools",+ "db_table": "custom_tool",+ },+ ),+ migrations.AddConstraint(+ model_name="customtool",+ constraint=models.UniqueConstraint(+ fields=("tool_name", "organization"), name="unique_tool_name"+ ),+ ),+ ]diff --git a/backend/prompt_studio/prompt_studio_core_v2/migrations/__init__.py b/backend/prompt_studio/prompt_studio_core_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/prompt_studio/prompt_studio_core_v2/models.py b/backend/prompt_studio/prompt_studio_core_v2/models.pyindex f4784e47d..87abd4766 100644--- a/backend/prompt_studio/prompt_studio_core_v2/models.py+++ b/backend/prompt_studio/prompt_studio_core_v2/models.py@@ -121,6 +121,9 @@ class CustomTool(DefaultOrganizationMixin, BaseModel):enable_challenge = models.BooleanField(db_comment="Flag to enable or disable challenge", default=False)+ enable_highlight = models.BooleanField(+ db_comment="Flag to enable or disable document highlighting", default=False+ )# Introduced field to establish M2M relation between users and custom_tool.# This will introduce intermediary table which relates both the models.@@ -149,7 +152,7 @@ class CustomTool(DefaultOrganizationMixin, BaseModel):class Meta:verbose_name = "Custom Tool"verbose_name_plural = "Custom Tools"- db_table = "custom_tool_v2"+ db_table = "custom_tool"constraints = [models.UniqueConstraint(fields=["tool_name", "organization"],diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.pyindex 74a157e9b..82ba23fe7 100644--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py@@ -500,7 +500,11 @@ class PromptStudioHelper:doc_path, tool_id, org_id, user_id, document_id, run_id):prompts = PromptStudioHelper.fetch_prompt_from_tool(tool_id)- prompts = [prompt for prompt in prompts if prompt.prompt_type != TSPKeys.NOTES]+ prompts = [+ prompt+ for prompt in prompts+ if prompt.prompt_type != TSPKeys.NOTES and prompt.active+ ]if not prompts:logger.error(f"[{tool_id or 'NA'}] No prompts found for id: {id}")raise NoPromptsFound()@@ -733,8 +737,9 @@ class PromptStudioHelper:prompt_host=settings.PROMPT_HOST,prompt_port=settings.PROMPT_PORT,)+ include_metadata = {TSPKeys.INCLUDE_METADATA: True}- answer = responder.answer_prompt(payload)+ answer = responder.answer_prompt(payload, include_metadata)# TODO: Make use of dataclassesif answer["status"] == "ERROR":# TODO: Publish to FE logs from here@@ -849,6 +854,7 @@ class PromptStudioHelper:)return {"status": IndexingStatus.COMPLETED_STATUS.value, "output": doc_id}except (IndexingError, IndexingAPIError, SdkError) as e:+ logger.error(f"Indexing failed : {e} ", stack_info=True, exc_info=True)doc_name = os.path.split(file_path)[1]PromptStudioHelper._publish_log({"tool_id": tool_id, "run_id": run_id, "doc_name": doc_name},@@ -959,8 +965,9 @@ class PromptStudioHelper:prompt_host=settings.PROMPT_HOST,prompt_port=settings.PROMPT_PORT,)+ include_metadata = {TSPKeys.INCLUDE_METADATA: True}- answer = responder.single_pass_extraction(payload)+ answer = responder.single_pass_extraction(payload, include_metadata)# TODO: Make use of dataclassesif answer["status"] == "ERROR":error_message = answer.get("error", None)@@ -969,3 +976,11 @@ class PromptStudioHelper:)output_response = json.loads(answer["structure_output"])return output_response++ @staticmethod+ def get_tool_from_tool_id(tool_id: str) -> Optional[CustomTool]:+ try:+ tool: CustomTool = CustomTool.objects.get(tool_id=tool_id)+ return tool+ except CustomTool.DoesNotExist:+ return Nonediff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.pyindex e2b1e1d2a..dcdd22a22 100644--- a/backend/prompt_studio/prompt_studio_core_v2/views.py+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py@@ -46,7 +46,7 @@ from rest_framework.decorators import actionfrom rest_framework.request import Requestfrom rest_framework.response import Responsefrom rest_framework.versioning import URLPathVersioning-from tool_instance.models import ToolInstance+from tool_instance_v2.models import ToolInstancefrom unstract.sdk.utils.common_utils import CommonUtilsfrom utils.user_session import UserSessionUtilsdiff --git a/backend/prompt_studio/prompt_studio_document_manager_v2/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_document_manager_v2/migrations/0001_initial.pynew file mode 100644index 000000000..6df0ccc49--- /dev/null+++ b/backend/prompt_studio/prompt_studio_document_manager_v2/migrations/0001_initial.py@@ -0,0 +1,83 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("prompt_studio_core_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="DocumentManager",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "document_id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "document_name",+ models.CharField(+ db_comment="Field to store the document name", editable=False+ ),+ ),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="document_managers_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="document_managers_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "tool",+ models.ForeignKey(+ on_delete=django.db.models.deletion.CASCADE,+ related_name="document_managers",+ to="prompt_studio_core_v2.customtool",+ ),+ ),+ ],+ options={+ "verbose_name": "Document Manager",+ "verbose_name_plural": "Document Managers",+ "db_table": "document_manager",+ },+ ),+ migrations.AddConstraint(+ model_name="documentmanager",+ constraint=models.UniqueConstraint(+ fields=("document_name", "tool"), name="unique_document_name_tool_index"+ ),+ ),+ ]diff --git a/backend/prompt_studio/prompt_studio_document_manager_v2/migrations/__init__.py b/backend/prompt_studio/prompt_studio_document_manager_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/prompt_studio/prompt_studio_document_manager_v2/models.py b/backend/prompt_studio/prompt_studio_document_manager_v2/models.pyindex 139d67e31..d29978c16 100644--- a/backend/prompt_studio/prompt_studio_document_manager_v2/models.py+++ b/backend/prompt_studio/prompt_studio_document_manager_v2/models.py@@ -47,7 +47,7 @@ class DocumentManager(BaseModel):class Meta:verbose_name = "Document Manager"verbose_name_plural = "Document Managers"- db_table = "document_manager_v2"+ db_table = "document_manager"constraints = [models.UniqueConstraint(fields=["document_name", "tool"],diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/0001_initial.pynew file mode 100644index 000000000..f587ef667--- /dev/null+++ b/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/0001_initial.py@@ -0,0 +1,113 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("prompt_studio_document_manager_v2", "0001_initial"),+ ("prompt_profile_manager_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="IndexManager",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "index_manager_id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "raw_index_id",+ models.CharField(+ blank=True,+ db_comment="Field to store the raw index id",+ editable=False,+ null=True,+ ),+ ),+ (+ "summarize_index_id",+ models.CharField(+ blank=True,+ db_comment="Field to store the summarize index id",+ editable=False,+ null=True,+ ),+ ),+ (+ "index_ids_history",+ models.JSONField(db_comment="List of index ids", default=list),+ ),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="index_managers_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "document_manager",+ models.ForeignKey(+ editable=False,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="index_managers",+ to="prompt_studio_document_manager_v2.documentmanager",+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="index_managers_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "profile_manager",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="index_managers",+ to="prompt_profile_manager_v2.profilemanager",+ ),+ ),+ ],+ options={+ "verbose_name": "Index Manager",+ "verbose_name_plural": "Index Managers",+ "db_table": "index_manager",+ },+ ),+ migrations.AddConstraint(+ model_name="indexmanager",+ constraint=models.UniqueConstraint(+ fields=("document_manager", "profile_manager"),+ name="unique_document_manager_profile_manager_index",+ ),+ ),+ ]diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/__init__.py b/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/models.py b/backend/prompt_studio/prompt_studio_index_manager_v2/models.pyindex c160f8561..e58ff87b0 100644--- a/backend/prompt_studio/prompt_studio_index_manager_v2/models.py+++ b/backend/prompt_studio/prompt_studio_index_manager_v2/models.py@@ -84,7 +84,7 @@ class IndexManager(BaseModel):class Meta:verbose_name = "Index Manager"verbose_name_plural = "Index Managers"- db_table = "index_manager_v2"+ db_table = "index_manager"constraints = [models.UniqueConstraint(fields=["document_manager", "profile_manager"],diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py b/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.pyindex 084b2c458..84747d52b 100644--- a/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py+++ b/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py@@ -49,7 +49,10 @@ class PromptStudioIndexHelper:)index_ids = index_manager.index_ids_history- index_ids_list = json.loads(index_ids) if index_ids else []+ if not isinstance(index_ids, list):+ index_ids_list = json.loads(index_ids) if index_ids else []+ else:+ index_ids_list = index_idsif doc_id not in index_ids:index_ids_list.append(doc_id)diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/constants.py b/backend/prompt_studio/prompt_studio_output_manager_v2/constants.pyindex a9c046aae..1cee0b394 100644--- a/backend/prompt_studio/prompt_studio_output_manager_v2/constants.py+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/constants.py@@ -5,3 +5,8 @@ class PromptStudioOutputManagerKeys:DOCUMENT_MANAGER = "document_manager"IS_SINGLE_PASS_EXTRACT = "is_single_pass_extract"NOTES = "NOTES"+++class PromptOutputManagerErrorMessage:+ TOOL_VALIDATION = "tool_id parameter is required"+ TOOL_NOT_FOUND = "Tool not found"diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0001_initial.pynew file mode 100644index 000000000..825ad27c0--- /dev/null+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0001_initial.py@@ -0,0 +1,139 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("prompt_studio_v2", "0001_initial"),+ ("prompt_profile_manager_v2", "0001_initial"),+ ("prompt_studio_core_v2", "0001_initial"),+ ("prompt_studio_document_manager_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="PromptStudioOutputManager",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "prompt_output_id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "output",+ models.CharField(+ blank=True, db_comment="Field to store output", null=True+ ),+ ),+ (+ "context",+ models.TextField(+ blank=True, db_comment="Field to store chunks used", null=True+ ),+ ),+ (+ "eval_metrics",+ models.JSONField(+ db_column="eval_metrics",+ db_comment="Field to store the evaluation metrics",+ default=list,+ ),+ ),+ (+ "is_single_pass_extract",+ models.BooleanField(+ db_comment="Is the single pass extraction mode active",+ default=False,+ ),+ ),+ ("run_id", models.UUIDField(default=uuid.uuid4, editable=False)),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="prompt_studio_outputs_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "document_manager",+ models.ForeignKey(+ on_delete=django.db.models.deletion.CASCADE,+ related_name="prompt_studio_outputs",+ to="prompt_studio_document_manager_v2.documentmanager",+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="prompt_studio_outputs_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "profile_manager",+ models.ForeignKey(+ on_delete=django.db.models.deletion.CASCADE,+ related_name="prompt_studio_outputs",+ to="prompt_profile_manager_v2.profilemanager",+ ),+ ),+ (+ "prompt_id",+ models.ForeignKey(+ on_delete=django.db.models.deletion.CASCADE,+ related_name="prompt_studio_outputs",+ to="prompt_studio_v2.toolstudioprompt",+ ),+ ),+ (+ "tool_id",+ models.ForeignKey(+ on_delete=django.db.models.deletion.CASCADE,+ related_name="prompt_studio_outputs",+ to="prompt_studio_core_v2.customtool",+ ),+ ),+ ],+ options={+ "verbose_name": "Prompt Studio Output Manager",+ "verbose_name_plural": "Prompt Studio Output Managers",+ "db_table": "prompt_studio_output_manager",+ },+ ),+ migrations.AddConstraint(+ model_name="promptstudiooutputmanager",+ constraint=models.UniqueConstraint(+ fields=(+ "prompt_id",+ "document_manager",+ "profile_manager",+ "tool_id",+ "is_single_pass_extract",+ ),+ name="unique_prompt_output_index",+ ),+ ),+ ]diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/__init__.py b/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/models.py b/backend/prompt_studio/prompt_studio_output_manager_v2/models.pyindex afc133f17..0a9cd8bc7 100644--- a/backend/prompt_studio/prompt_studio_output_manager_v2/models.py+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/models.py@@ -76,7 +76,7 @@ class PromptStudioOutputManager(BaseModel):class Meta:verbose_name = "Prompt Studio Output Manager"verbose_name_plural = "Prompt Studio Output Managers"- db_table = "prompt_studio_output_manager_v2"+ db_table = "prompt_studio_output_manager"constraints = [models.UniqueConstraint(fields=[diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/urls.py b/backend/prompt_studio/prompt_studio_output_manager_v2/urls.pyindex 45af0ccee..61ec8540f 100644--- a/backend/prompt_studio/prompt_studio_output_manager_v2/urls.py+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/urls.py@@ -4,9 +4,17 @@ from rest_framework.urlpatterns import format_suffix_patternsfrom .views import PromptStudioOutputViewprompt_doc_list = PromptStudioOutputView.as_view({"get": "list"})+get_output_for_tool_default = PromptStudioOutputView.as_view(+ {"get": "get_output_for_tool_default"}+)urlpatterns = format_suffix_patterns([path("prompt-output/", prompt_doc_list, name="prompt-doc-list"),+ path(+ "prompt-output/prompt-default-profile/",+ get_output_for_tool_default,+ name="prompt-default-profile-outputs",+ ),])diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/views.py b/backend/prompt_studio/prompt_studio_output_manager_v2/views.pyindex e4dd99b9f..441100d90 100644--- a/backend/prompt_studio/prompt_studio_output_manager_v2/views.py+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/views.py@@ -1,14 +1,20 @@import logging-from typing import Optional+from typing import Any, Optional+from django.core.exceptions import ObjectDoesNotExistfrom django.db.models import QuerySet+from django.http import HttpRequestfrom prompt_studio.prompt_studio_output_manager_v2.constants import (+ PromptOutputManagerErrorMessage,PromptStudioOutputManagerKeys,)from prompt_studio.prompt_studio_output_manager_v2.serializers import (PromptStudioOutputSerializer,)-from rest_framework import viewsets+from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt+from rest_framework import status, viewsets+from rest_framework.exceptions import APIException+from rest_framework.response import Responsefrom rest_framework.versioning import URLPathVersioningfrom utils.common_utils import CommonUtilsfrom utils.filtering import FilterHelper@@ -49,3 +55,51 @@ class PromptStudioOutputView(viewsets.ModelViewSet):queryset = PromptStudioOutputManager.objects.filter(**filter_args)return queryset++ def get_output_for_tool_default(self, request: HttpRequest) -> Response:+ # TODO: Setup Serializer here+ # Get the tool_id from request parameters+ tool_id = request.GET.get("tool_id")+ document_manager_id = request.GET.get("document_manager")+ tool_validation_message = PromptOutputManagerErrorMessage.TOOL_VALIDATION+ tool_not_found = PromptOutputManagerErrorMessage.TOOL_NOT_FOUND+ if not tool_id:+ raise APIException(detail=tool_validation_message, code=400)++ try:+ # Fetch ToolStudioPrompt records based on tool_id+ tool_studio_prompts = ToolStudioPrompt.objects.filter(tool_id=tool_id)+ except ObjectDoesNotExist:+ raise APIException(detail=tool_not_found, code=400)++ # Initialize the result dictionary+ result: dict[str, Any] = {}++ # Iterate over ToolStudioPrompt records+ for tool_prompt in tool_studio_prompts:+ prompt_id = str(tool_prompt.prompt_id)+ profile_manager_id = str(tool_prompt.profile_manager.profile_id)++ # If profile_manager is not set, skip this record+ if not profile_manager_id:+ result[tool_prompt.prompt_key] = ""+ continue++ try:+ queryset = PromptStudioOutputManager.objects.filter(+ prompt_id=prompt_id,+ profile_manager=profile_manager_id,+ is_single_pass_extract=False,+ document_manager_id=document_manager_id,+ )++ if not queryset.exists():+ result[tool_prompt.prompt_key] = ""+ continue++ for output in queryset:+ result[tool_prompt.prompt_key] = output.output+ except ObjectDoesNotExist:+ result[tool_prompt.prompt_key] = ""++ return Response(result, status=status.HTTP_200_OK)diff --git a/backend/prompt_studio/prompt_studio_registry_v2/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_registry_v2/migrations/0001_initial.pynew file mode 100644index 000000000..393fbd21f--- /dev/null+++ b/backend/prompt_studio/prompt_studio_registry_v2/migrations/0001_initial.py@@ -0,0 +1,132 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+import prompt_studio.prompt_studio_registry_v2.fields+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("prompt_studio_core_v2", "0001_initial"),+ ("account_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="PromptStudioRegistry",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "prompt_registry_id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("name", models.CharField(default="", editable=False)),+ ("description", models.CharField(default="", editable=False)),+ (+ "tool_property",+ prompt_studio.prompt_studio_registry_v2.fields.ToolPropertyJSONField(+ db_column="tool_property",+ db_comment="PROPERTIES of the tool",+ default=dict,+ ),+ ),+ (+ "tool_spec",+ prompt_studio.prompt_studio_registry_v2.fields.ToolSpecJSONField(+ db_column="tool_spec",+ db_comment="SPEC of the tool",+ default=dict,+ ),+ ),+ (+ "tool_metadata",+ prompt_studio.prompt_studio_registry_v2.fields.ToolMetadataJSONField(+ db_column="tool_metadata",+ db_comment="Metadata from Prompt Studio",+ default=dict,+ ),+ ),+ (+ "icon",+ models.CharField(+ db_comment="Tool icon in svg format", editable=False+ ),+ ),+ ("url", models.CharField(editable=False)),+ (+ "shared_to_org",+ models.BooleanField(+ db_comment="Is the exported tool shared with entire org",+ default=False,+ ),+ ),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="prompt_registries_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "custom_tool",+ models.OneToOneField(+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="prompt_studio_registries",+ to="prompt_studio_core_v2.customtool",+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="prompt_registries_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "organization",+ models.ForeignKey(+ blank=True,+ db_comment="Foreign key reference to the Organization model.",+ default=None,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ to="account_v2.organization",+ ),+ ),+ (+ "shared_users",+ models.ManyToManyField(+ related_name="prompt_registries", to=settings.AUTH_USER_MODEL+ ),+ ),+ ],+ options={+ "verbose_name": "Prompt Studio Registry",+ "verbose_name_plural": "Prompt Studio Registries",+ "db_table": "prompt_studio_registry",+ },+ ),+ ]diff --git a/backend/prompt_studio/prompt_studio_registry_v2/migrations/__init__.py b/backend/prompt_studio/prompt_studio_registry_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/prompt_studio/prompt_studio_registry_v2/models.py b/backend/prompt_studio/prompt_studio_registry_v2/models.pyindex ec768a854..332ca34ea 100644--- a/backend/prompt_studio/prompt_studio_registry_v2/models.py+++ b/backend/prompt_studio/prompt_studio_registry_v2/models.py@@ -107,4 +107,4 @@ class PromptStudioRegistry(DefaultOrganizationMixin, BaseModel):class Meta:verbose_name = "Prompt Studio Registry"verbose_name_plural = "Prompt Studio Registries"- db_table = "prompt_studio_registry_v2"+ db_table = "prompt_studio_registry"diff --git a/backend/prompt_studio/prompt_studio_v2/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_v2/migrations/0001_initial.pynew file mode 100644index 000000000..117bc51c7--- /dev/null+++ b/backend/prompt_studio/prompt_studio_v2/migrations/0001_initial.py@@ -0,0 +1,162 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("prompt_studio_core_v2", "0001_initial"),+ ("prompt_profile_manager_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="ToolStudioPrompt",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "prompt_id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "prompt_key",+ models.TextField(db_comment="Field to store the prompt key"),+ ),+ (+ "enforce_type",+ models.TextField(+ blank=True,+ choices=[+ ("Text", "Response sent as Text"),+ ("number", "Response sent as number"),+ ("email", "Response sent as email"),+ ("date", "Response sent as date"),+ ("boolean", "Response sent as boolean"),+ ("json", "Response sent as json"),+ ],+ db_comment="Field to store the type in which the response to be returned.",+ default="Text",+ ),+ ),+ (+ "prompt",+ models.TextField(+ blank=True, db_comment="Field to store the prompt"+ ),+ ),+ ("sequence_number", models.IntegerField(blank=True, null=True)),+ (+ "prompt_type",+ models.TextField(+ blank=True,+ choices=[+ ("PROMPT", "Response sent as Text"),+ ("NOTES", "Response sent as float"),+ ],+ db_comment="Field to store the type of the input prompt",+ ),+ ),+ ("output", models.TextField(blank=True)),+ (+ "assert_prompt",+ models.TextField(+ blank=True,+ db_comment="Field to store the asserted prompt",+ null=True,+ ),+ ),+ (+ "assertion_failure_prompt",+ models.TextField(+ blank=True,+ db_comment="Field to store the prompt key",+ null=True,+ ),+ ),+ ("is_assert", models.BooleanField(default=False)),+ ("active", models.BooleanField(default=True)),+ (+ "output_metadata",+ models.JSONField(+ db_column="output_metadata",+ db_comment="JSON adapter metadata for the FE to load the pagination",+ default=dict,+ ),+ ),+ ("evaluate", models.BooleanField(default=True)),+ ("eval_quality_faithfulness", models.BooleanField(default=True)),+ ("eval_quality_correctness", models.BooleanField(default=True)),+ ("eval_quality_relevance", models.BooleanField(default=True)),+ ("eval_security_pii", models.BooleanField(default=True)),+ ("eval_guidance_toxicity", models.BooleanField(default=True)),+ ("eval_guidance_completeness", models.BooleanField(default=True)),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="tool_studio_prompts_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ editable=False,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="tool_studio_prompts_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "profile_manager",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="tool_studio_prompts",+ to="prompt_profile_manager_v2.profilemanager",+ ),+ ),+ (+ "tool_id",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="tool_studio_prompts",+ to="prompt_studio_core_v2.customtool",+ ),+ ),+ ],+ options={+ "verbose_name": "Tool Studio Prompt",+ "verbose_name_plural": "Tool Studio Prompts",+ "db_table": "tool_studio_prompt",+ },+ ),+ migrations.AddConstraint(+ model_name="toolstudioprompt",+ constraint=models.UniqueConstraint(+ fields=("prompt_key", "tool_id"), name="unique_prompt_key_tool_id_index"+ ),+ ),+ ]diff --git a/backend/prompt_studio/prompt_studio_v2/migrations/__init__.py b/backend/prompt_studio/prompt_studio_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/prompt_studio/prompt_studio_v2/models.py b/backend/prompt_studio/prompt_studio_v2/models.pyindex ba928f7c5..7e65d3c84 100644--- a/backend/prompt_studio/prompt_studio_v2/models.py+++ b/backend/prompt_studio/prompt_studio_v2/models.py@@ -118,7 +118,7 @@ class ToolStudioPrompt(BaseModel):class Meta:verbose_name = "Tool Studio Prompt"verbose_name_plural = "Tool Studio Prompts"- db_table = "tool_studio_prompt_v2"+ db_table = "tool_studio_prompt"constraints = [models.UniqueConstraint(fields=["prompt_key", "tool_id"],diff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.pyindex 762be6609..9415c3829 100644--- a/backend/scheduler/tasks.py+++ b/backend/scheduler/tasks.py@@ -159,9 +159,7 @@ def execute_pipeline_task_v2(except Exception as e:logger.warning(f"Failed to disable task: {pipeline_id}. Error: {e}")return- workflow = WorkflowHelper.get_workflow_by_id(- id=workflow_id, organization_id=organization_id- )+ workflow = WorkflowHelper.get_workflow_by_id(id=workflow_id)logger.info(f"Executing workflow: {workflow}")PipelineProcessor.update_pipeline(pipeline_id, Pipeline.PipelineStatus.INPROGRESSdiff --git a/backend/tenant_account_v2/migrations/0001_initial.py b/backend/tenant_account_v2/migrations/0001_initial.pynew file mode 100644index 000000000..70c7ba135--- /dev/null+++ b/backend/tenant_account_v2/migrations/0001_initial.py@@ -0,0 +1,70 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("account_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="OrganizationMember",+ fields=[+ ("member_id", models.BigAutoField(primary_key=True, serialize=False)),+ ("role", models.CharField()),+ (+ "is_login_onboarding_msg",+ models.BooleanField(+ db_comment="Flag to indicate whether the onboarding messages are shown",+ default=True,+ ),+ ),+ (+ "is_prompt_studio_onboarding_msg",+ models.BooleanField(+ db_comment="Flag to indicate whether the prompt studio messages are shown",+ default=True,+ ),+ ),+ (+ "organization",+ models.ForeignKey(+ blank=True,+ db_comment="Foreign key reference to the Organization model.",+ default=None,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ to="account_v2.organization",+ ),+ ),+ (+ "user",+ models.ForeignKey(+ default=None,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="organization_user",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ ],+ options={+ "verbose_name": "Organization Member",+ "verbose_name_plural": "Organization Members",+ "db_table": "organization_member",+ },+ ),+ migrations.AddConstraint(+ model_name="organizationmember",+ constraint=models.UniqueConstraint(+ fields=("organization", "user"), name="unique_organization_member"+ ),+ ),+ ]diff --git a/backend/tenant_account_v2/migrations/__init__.py b/backend/tenant_account_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/tenant_account_v2/models.py b/backend/tenant_account_v2/models.pyindex c2c5b3cdc..e5fdacda6 100644--- a/backend/tenant_account_v2/models.py+++ b/backend/tenant_account_v2/models.py@@ -31,13 +31,13 @@ class OrganizationMember(DefaultOrganizationMixin):def __str__(self): # type: ignorereturn (f"OrganizationMember("- f"{self.member_id}, role: {self.role}, userId: {self.user.user_id})"+ f"{self.member_id}, role: {self.role}, user: {self.user})")objects = OrganizationMemberModelManager()class Meta:- db_table = "organization_member_v2"+ db_table = "organization_member"verbose_name = "Organization Member"verbose_name_plural = "Organization Members"constraints = [diff --git a/backend/tool_instance_v2/migrations/0001_initial.py b/backend/tool_instance_v2/migrations/0001_initial.pynew file mode 100644index 000000000..4e75c99e5--- /dev/null+++ b/backend/tool_instance_v2/migrations/0001_initial.py@@ -0,0 +1,132 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("connector_v2", "0001_initial"),+ ("workflow_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="ToolInstance",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "tool_id",+ models.CharField(+ db_comment="Function name of the tool being used", max_length=64+ ),+ ),+ (+ "input",+ models.JSONField(+ db_comment="Provisional WF input to a tool", null=True+ ),+ ),+ (+ "output",+ models.JSONField(+ db_comment="Provisional WF output to a tool", null=True+ ),+ ),+ ("version", models.CharField(max_length=16)),+ ("metadata", models.JSONField(db_comment="Stores config for a tool")),+ ("step", models.IntegerField()),+ ("status", models.CharField(default="Ready to start", max_length=32)),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="tool_instances_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "input_db_connector",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="input_db_connectors",+ to="connector_v2.connectorinstance",+ ),+ ),+ (+ "input_file_connector",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="input_file_connectors",+ to="connector_v2.connectorinstance",+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="tool_instances_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "output_db_connector",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="output_db_connectors",+ to="connector_v2.connectorinstance",+ ),+ ),+ (+ "output_file_connector",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="output_file_connectors",+ to="connector_v2.connectorinstance",+ ),+ ),+ (+ "workflow",+ models.ForeignKey(+ on_delete=django.db.models.deletion.CASCADE,+ related_name="tool_instances",+ to="workflow_v2.workflow",+ ),+ ),+ ],+ options={+ "verbose_name": "Tool Instance",+ "verbose_name_plural": "Tool Instances",+ "db_table": "tool_instance",+ },+ ),+ ]diff --git a/backend/tool_instance_v2/migrations/__init__.py b/backend/tool_instance_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/tool_instance_v2/models.py b/backend/tool_instance_v2/models.pyindex a7609566d..a05e834b1 100644--- a/backend/tool_instance_v2/models.py+++ b/backend/tool_instance_v2/models.py@@ -96,4 +96,4 @@ class ToolInstance(BaseModel):class Meta:verbose_name = "Tool Instance"verbose_name_plural = "Tool Instances"- db_table = "tool_instance_v2"+ db_table = "tool_instance"diff --git a/backend/usage_v2/migrations/0001_initial.py b/backend/usage_v2/migrations/0001_initial.pynew file mode 100644index 000000000..28dfd79c7--- /dev/null+++ b/backend/usage_v2/migrations/0001_initial.py@@ -0,0 +1,144 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ ("account_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="Usage",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ db_comment="Primary key for the usage entry, automatically generated UUID",+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "workflow_id",+ models.CharField(+ blank=True,+ db_comment="Identifier for the workflow",+ max_length=255,+ null=True,+ ),+ ),+ (+ "execution_id",+ models.CharField(+ blank=True,+ db_comment="Identifier for the execution instance",+ max_length=255,+ null=True,+ ),+ ),+ (+ "adapter_instance_id",+ models.CharField(+ db_comment="Identifier for the adapter instance", max_length=255+ ),+ ),+ (+ "run_id",+ models.CharField(+ blank=True,+ db_comment="Identifier for the run",+ max_length=255,+ null=True,+ ),+ ),+ (+ "usage_type",+ models.CharField(+ choices=[+ ("llm", "LLM Usage"),+ ("embedding", "Embedding Usage"),+ ],+ db_comment="Type of usage, either 'llm' or 'embedding'",+ max_length=255,+ ),+ ),+ (+ "llm_usage_reason",+ models.CharField(+ blank=True,+ choices=[+ ("extraction", "Extraction"),+ ("challenge", "Challenge"),+ ("summarize", "Summarize"),+ ],+ db_comment="Reason for LLM usage. Empty if usage_type is 'embedding'. ",+ max_length=255,+ null=True,+ ),+ ),+ (+ "model_name",+ models.CharField(+ db_comment="Name of the model used", max_length=255+ ),+ ),+ (+ "embedding_tokens",+ models.IntegerField(+ db_comment="Number of tokens used for embedding"+ ),+ ),+ (+ "prompt_tokens",+ models.IntegerField(+ db_comment="Number of tokens used for the prompt"+ ),+ ),+ (+ "completion_tokens",+ models.IntegerField(+ db_comment="Number of tokens used for the completion"+ ),+ ),+ (+ "total_tokens",+ models.IntegerField(db_comment="Total number of tokens used"),+ ),+ (+ "cost_in_dollars",+ models.FloatField(db_comment="Total number of tokens used"),+ ),+ (+ "organization",+ models.ForeignKey(+ blank=True,+ db_comment="Foreign key reference to the Organization model.",+ default=None,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ to="account_v2.organization",+ ),+ ),+ ],+ options={+ "db_table": "usage",+ "indexes": [+ models.Index(+ fields=["run_id"], name="token_usage_run_id_cd3578_idx"+ )+ ],+ },+ ),+ ]diff --git a/backend/usage_v2/migrations/__init__.py b/backend/usage_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/usage_v2/models.py b/backend/usage_v2/models.pyindex 8d4a35069..8e836024e 100644--- a/backend/usage_v2/models.py+++ b/backend/usage_v2/models.py@@ -76,7 +76,7 @@ class Usage(DefaultOrganizationMixin, BaseModel):return str(self.id)class Meta:- db_table = "token_usage_v2"+ db_table = "usage"indexes = [models.Index(fields=["run_id"]),]diff --git a/backend/utils/serializer_utils.py b/backend/utils/serializer_utils.pyindex 2534404a4..fdecc7fae 100644--- a/backend/utils/serializer_utils.py+++ b/backend/utils/serializer_utils.py@@ -1,9 +1,16 @@from typing import Any-from account.models import Userfrom rest_framework.request import Requestfrom utils.constants import Account+from backend.constants import FeatureFlag+from unstract.flags.feature_flag import check_feature_flag_status++if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):+ from account_v2.models import User+else:+ from account.models import User+class SerializerUtils:@staticmethoddiff --git a/backend/utils/user_context.py b/backend/utils/user_context.pyindex 87addb52f..1be52f02a 100644--- a/backend/utils/user_context.py+++ b/backend/utils/user_context.py@@ -1,15 +1,14 @@from typing import Optionalfrom django.db import connection+from django.db.utils import ProgrammingErrorfrom utils.constants import Account, FeatureFlagfrom utils.local_context import StateStorefrom unstract.flags.feature_flag import check_feature_flag_statusif check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):- # TODO: uncomment Once v2 implemented- # from account_v2.models import Organization- pass+ from account_v2.models import Organizationelse:from account.models import Organization@@ -37,6 +36,11 @@ class UserContext:)except Organization.DoesNotExist:return None+ except ProgrammingError:+ # Handle cases where the database schema might not be fully set up,+ # especially during the execution of management commands+ # other than runserver+ return Noneelse:organization: Organization = connection.tenantreturn organizationdiff --git a/backend/utils/user_session.py b/backend/utils/user_session.pyindex e3cb36d0d..972f2a28f 100644--- a/backend/utils/user_session.py+++ b/backend/utils/user_session.py@@ -3,11 +3,15 @@ from typing import Optionalfrom django.conf import settingsfrom django.db import connectionfrom django.http import HttpRequest-from tenant_account.models import OrganizationMemberfrom utils.constants import FeatureFlagfrom unstract.flags.feature_flag import check_feature_flag_status+if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):+ from tenant_account_v2.models import OrganizationMember+else:+ from tenant_account.models import OrganizationMember+class UserSessionUtils:@staticmethoddiff --git a/backend/workflow_manager/endpoint_v2/base_connector.py b/backend/workflow_manager/endpoint_v2/base_connector.pyindex 3207537e6..2523523e3 100644--- a/backend/workflow_manager/endpoint_v2/base_connector.py+++ b/backend/workflow_manager/endpoint_v2/base_connector.py@@ -45,11 +45,26 @@ class BaseConnector(ExecutionFileHandler):Raises:KeyError: If the connector_id is not found in the connectors dictionary."""+ return self.get_fs_connector(+ settings=settings, connector_id=connector_id+ ).get_fsspec_fs()++ def get_fs_connector(+ self, settings: dict[str, Any], connector_id: str+ ) -> UnstractFileSystem:+ """Get an fs connector based specified connector settings.++ Parameters:+ - settings (dict): Connector-specific settings.+ - connector_id (str): Identifier for the desired connector.++ Returns:+ UnstractFileSystem: An unstract fs connector instance.+ """if connector_id not in connectors:- raise ValueError(f"Invalid connector_id: {connector_id}")+ raise ValueError(f"Connector '{connector_id}' is not supported.")connector = connectors[connector_id][Common.METADATA][Common.CONNECTOR]- connector_class: UnstractFileSystem = connector(settings)- return connector_class.get_fsspec_fs()+ return connector(settings)@classmethoddef get_json_schema(cls, file_path: str) -> dict[str, Any]:diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex c882c1b0e..7839e1c8c 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -135,7 +135,7 @@ class DestinationConnector(BaseConnector):"""Handle the output based on the connection type."""connection_type = self.endpoint.connection_typeresult: Optional[str] = None- meta_data: Optional[str] = None+ metadata: Optional[str] = Noneif error:if connection_type == WorkflowEndpoint.ConnectionType.API:self._handle_api_result(file_name=file_name, error=error, result=result)@@ -146,19 +146,19 @@ class DestinationConnector(BaseConnector):self.insert_into_db(file_history)elif connection_type == WorkflowEndpoint.ConnectionType.API:result = self.get_result(file_history)- meta_data = self.get_metadata(file_history)+ metadata = self.get_metadata(file_history)self._handle_api_result(- file_name=file_name, error=error, result=result, meta_data=meta_data+ file_name=file_name, error=error, result=result, metadata=metadata)elif connection_type == WorkflowEndpoint.ConnectionType.MANUALREVIEW:result = self.get_result(file_history)- meta_data = self.get_metadata(file_history)+ metadata = self.get_metadata(file_history)self._push_to_queue(file_name=file_name,workflow=workflow,result=result,input_file_path=input_file_path,- meta_data=meta_data,+ metadata=metadata,)if not file_history:FileHistoryHelper.create_file_history(@@ -166,7 +166,7 @@ class DestinationConnector(BaseConnector):workflow=workflow,status=ExecutionStatus.COMPLETED,result=result,- metadata=meta_data,+ metadata=metadata,file_name=file_name,)@@ -175,32 +175,29 @@ class DestinationConnector(BaseConnector):connector: ConnectorInstance = self.endpoint.connector_instanceconnector_settings: dict[str, Any] = connector.connector_metadatadestination_configurations: dict[str, Any] = self.endpoint.configuration- root_path = str(connector_settings.get(DestinationKey.PATH))- output_folder = str(+ root_path = str(connector_settings.get(DestinationKey.PATH, ""))++ output_directory = str(destination_configurations.get(DestinationKey.OUTPUT_FOLDER, "/"))- overwrite = bool(- destination_configurations.get(- DestinationKey.OVERWRITE_OUTPUT_DOCUMENT, False- )+ destination_fs = self.get_fs_connector(+ settings=connector_settings, connector_id=connector.connector_id)- output_directory = os.path.join(root_path, output_folder)-+ output_directory = destination_fs.get_connector_root_dir(+ input_dir=output_directory, root_path=root_path+ )+ logger.debug(f"destination output directory {output_directory}")destination_volume_path = os.path.join(self.execution_dir, ToolExecKey.OUTPUT_DIR)-- connector_fs = self.get_fsspec(- settings=connector_settings, connector_id=connector.connector_id- )- if not connector_fs.isdir(output_directory):- connector_fs.mkdir(output_directory)+ destination_fs.create_dir_if_not_exists(input_dir=output_directory)+ destination_fsspec = destination_fs.get_fsspec_fs()# Traverse local directory and create the same structure in the# output_directoryfor root, dirs, files in os.walk(destination_volume_path):for dir_name in dirs:- connector_fs.mkdir(+ destination_fsspec.mkdir(os.path.join(output_directory,os.path.relpath(root, destination_volume_path),@@ -217,9 +214,7 @@ class DestinationConnector(BaseConnector):)normalized_path = os.path.normpath(destination_path)with open(source_path, "rb") as source_file:- connector_fs.write_bytes(- normalized_path, source_file.read(), overwrite=overwrite- )+ destination_fsspec.write_bytes(normalized_path, source_file.read())def insert_into_db(self, file_history: Optional[FileHistory]) -> None:"""Insert data into the database."""@@ -283,7 +278,7 @@ class DestinationConnector(BaseConnector):file_name: str,error: Optional[str] = None,result: Optional[str] = None,- meta_data: Optional[dict[str, Any]] = None,+ metadata: Optional[dict[str, Any]] = None,) -> None:"""Handle the API result.@@ -309,7 +304,7 @@ class DestinationConnector(BaseConnector):{"status": ApiDeploymentResultStatus.SUCCESS,"result": result,- "metadata": meta_data,+ "metadata": metadata,})else:@@ -390,13 +385,13 @@ class DestinationConnector(BaseConnector):def get_metadata(self, file_history: Optional[FileHistory]) -> Optional[dict[str, Any]]:- """Get meta_data from the output file.+ """Get metadata from the output file.Returns:Union[dict[str, Any], str]: Meta data."""- if file_history and file_history.meta_data:- return self.parse_string(file_history.meta_data)+ if file_history and file_history.metadata:+ return self.parse_string(file_history.metadata)metadata: dict[str, Any] = self.get_workflow_metadata()return metadata@@ -482,7 +477,7 @@ class DestinationConnector(BaseConnector):workflow: Workflow,result: Optional[str] = None,input_file_path: Optional[str] = None,- meta_data: Optional[dict[str, Any]] = None,+ metadata: Optional[dict[str, Any]] = None,) -> None:"""Handle the Manual Review QUEUE result.@@ -497,7 +492,7 @@ class DestinationConnector(BaseConnector):input_file_path (Optional[str], optional):The path to the input file.Defaults to None.- meta_data (Optional[dict[str, Any]], optional):+ metadata (Optional[dict[str, Any]], optional):A dictionary containing additionalmetadata related to the file. Defaults to None.@@ -519,7 +514,7 @@ class DestinationConnector(BaseConnector):q_name = f"review_queue_{self.organization_id}_{workflow.workflow_name}"queue_result = QueueResult(file=file_name,- whisper_hash=meta_data["whisper-hash"],+ whisper_hash=metadata["whisper-hash"],status=QueueResultStatus.SUCCESS,result=result,workflow_id=str(self.workflow_id),diff --git a/backend/workflow_manager/endpoint_v2/migrations/0001_initial.py b/backend/workflow_manager/endpoint_v2/migrations/0001_initial.pynew file mode 100644index 000000000..1cc0d1b1b--- /dev/null+++ b/backend/workflow_manager/endpoint_v2/migrations/0001_initial.py@@ -0,0 +1,90 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ ("connector_v2", "0001_initial"),+ ("workflow_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="WorkflowEndpoint",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "endpoint_type",+ models.CharField(+ choices=[+ ("SOURCE", "Source connector"),+ ("DESTINATION", "Destination Connector"),+ ],+ db_comment="Endpoint type (source or destination)",+ editable=False,+ ),+ ),+ (+ "connection_type",+ models.CharField(+ blank=True,+ choices=[+ ("FILESYSTEM", "FileSystem connector"),+ ("DATABASE", "Database Connector"),+ ("API", "API Connector"),+ ("APPDEPLOYMENT", "App Deployment"),+ ("MANUALREVIEW", "Manual Review Queue Connector"),+ ],+ db_comment="Connection type (Filesystem, Database, API or Manualreview)",+ ),+ ),+ (+ "configuration",+ models.JSONField(+ blank=True, db_comment="Configuration in JSON format", null=True+ ),+ ),+ (+ "connector_instance",+ models.ForeignKey(+ db_comment="Foreign key from ConnectorInstance model",+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="workflow_endpoints",+ to="connector_v2.connectorinstance",+ ),+ ),+ (+ "workflow",+ models.ForeignKey(+ db_comment="Foreign key from Workflow model",+ editable=False,+ on_delete=django.db.models.deletion.CASCADE,+ to="workflow_v2.workflow",+ ),+ ),+ ],+ options={+ "verbose_name": "Workflow Endpoint",+ "verbose_name_plural": "Workflow Endpoints",+ "db_table": "workflow_endpoints",+ },+ ),+ ]diff --git a/backend/workflow_manager/endpoint_v2/migrations/__init__.py b/backend/workflow_manager/endpoint_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/workflow_manager/endpoint_v2/models.py b/backend/workflow_manager/endpoint_v2/models.pyindex 6b067160e..2659a3b6f 100644--- a/backend/workflow_manager/endpoint_v2/models.py+++ b/backend/workflow_manager/endpoint_v2/models.py@@ -51,4 +51,4 @@ class WorkflowEndpoint(BaseModel):class Meta:verbose_name = "Workflow Endpoint"verbose_name_plural = "Workflow Endpoints"- db_table = "workflow_endpoints_v2"+ db_table = "workflow_endpoints"diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex d9283fa7a..dfa5aea5d 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -4,7 +4,6 @@ import osimport shutilfrom hashlib import md5, sha256from io import BytesIO-from pathlib import Pathfrom typing import Any, Optionalimport fsspec@@ -154,21 +153,27 @@ class SourceConnector(BaseConnector):))root_dir_path = connector_settings.get(ConnectorKeys.PATH, "")+input_directory = str(source_configurations.get(SourceKey.ROOT_FOLDER, ""))- if root_dir_path: # user needs to manually type the optional file path- input_directory = str(Path(root_dir_path, input_directory.lstrip("/")))- if not isinstance(required_patterns, list):- required_patterns = [required_patterns]- source_fs = self.get_fsspec(+ source_fs = self.get_fs_connector(settings=connector_settings, connector_id=connector.connector_id)+ input_directory = source_fs.get_connector_root_dir(+ input_dir=input_directory, root_path=root_dir_path+ )+ logger.debug(f"source input directory {input_directory}")+ if not isinstance(required_patterns, list):+ required_patterns = [required_patterns]++ source_fs_fsspec = source_fs.get_fsspec_fs()+patterns = self.valid_file_patterns(required_patterns=required_patterns)- is_directory = source_fs.isdir(input_directory)+ is_directory = source_fs_fsspec.isdir(input_directory)if not is_directory:raise InvalidInputDirectory()matched_files = self._get_matched_files(- source_fs, input_directory, patterns, recursive, limit+ source_fs_fsspec, input_directory, patterns, recursive, limit)self.publish_input_output_list_file_logs(input_directory, matched_files)return matched_files@@ -387,6 +392,17 @@ class SourceConnector(BaseConnector):results.append({"file": file_name, "result": result})def load_file(self, input_file_path: str) -> tuple[str, BytesIO]:+ """Load file contnt and file name based on the file path.++ Args:+ input_file_path (str): source file++ Raises:+ InvalidSource: _description_++ Returns:+ tuple[str, BytesIO]: file_name , file content+ """connector: ConnectorInstance = self.endpoint.connector_instanceconnector_settings: dict[str, Any] = connector.connector_metadatasource_fs: fsspec.AbstractFileSystem = self.get_fsspec(@@ -396,7 +412,7 @@ class SourceConnector(BaseConnector):file_content = remote_file.read()file_stream = BytesIO(file_content)- return remote_file.key, file_stream+ return os.path.basename(input_file_path), file_stream@classmethoddef add_input_file_to_api_storage(diff --git a/backend/workflow_manager/endpoint_v2/static/dest/file.json b/backend/workflow_manager/endpoint_v2/static/dest/file.jsonindex aa3a1263e..95111993f 100644--- a/backend/workflow_manager/endpoint_v2/static/dest/file.json+++ b/backend/workflow_manager/endpoint_v2/static/dest/file.json@@ -14,12 +14,6 @@"minLength": 1,"maxLength": 100,"format": "file-path"- },- "overwriteOutput": {- "type": "boolean",- "title": "Overwrite existing output",- "default": true,- "description": "Used to overwrite output document"}}}diff --git a/backend/workflow_manager/urls.py b/backend/workflow_manager/urls.pyindex aa5aebd36..7140cc174 100644--- a/backend/workflow_manager/urls.py+++ b/backend/workflow_manager/urls.py@@ -1,6 +1,14 @@from django.urls import include, path-from workflow_manager.endpoint import urls as endpoint_urls-from workflow_manager.workflow import urls as workflow_urls++from backend.constants import FeatureFlag+from unstract.flags.feature_flag import check_feature_flag_status++if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):+ from workflow_manager.endpoint_v2 import urls as endpoint_urls+ from workflow_manager.workflow_v2 import urls as workflow_urls+else:+ from workflow_manager.endpoint import urls as endpoint_urls+ from workflow_manager.workflow import urls as workflow_urlsurlpatterns = [path("endpoint/", include(endpoint_urls)),diff --git a/backend/workflow_manager/workflow_v2/file_history_helper.py b/backend/workflow_manager/workflow_v2/file_history_helper.pyindex 7d84cf0da..21f8eda93 100644--- a/backend/workflow_manager/workflow_v2/file_history_helper.py+++ b/backend/workflow_manager/workflow_v2/file_history_helper.py@@ -61,7 +61,7 @@ class FileHistoryHelper:cache_key=cache_key,status=status.value,result=str(result),- meta_data=str(metadata),+ metadata=str(metadata),error=str(error) if error else "",)except IntegrityError:diff --git a/backend/workflow_manager/workflow_v2/migrations/0001_initial.py b/backend/workflow_manager/workflow_v2/migrations/0001_initial.pynew file mode 100644index 000000000..2fff00246--- /dev/null+++ b/backend/workflow_manager/workflow_v2/migrations/0001_initial.py@@ -0,0 +1,329 @@+# Generated by Django 4.2.1 on 2024-07-29 09:16++import uuid++import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = [+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("account_v2", "0001_initial"),+ ]++ operations = [+ migrations.CreateModel(+ name="ExecutionLog",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "execution_id",+ models.UUIDField(db_comment="Execution ID", editable=False),+ ),+ ("data", models.JSONField(db_comment="Execution log data")),+ (+ "event_time",+ models.DateTimeField(db_comment="Execution log event time"),+ ),+ ],+ options={+ "verbose_name": "Execution Log",+ "verbose_name_plural": "Execution Logs",+ "db_table": "execution_log",+ },+ ),+ migrations.CreateModel(+ name="WorkflowExecution",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "pipeline_id",+ models.UUIDField(+ db_comment="ID of the associated pipeline, if applicable",+ editable=False,+ null=True,+ ),+ ),+ (+ "task_id",+ models.UUIDField(+ db_comment="task id of asynchronous execution",+ editable=False,+ null=True,+ ),+ ),+ (+ "workflow_id",+ models.UUIDField(+ db_comment="Id of workflow to be executed", editable=False+ ),+ ),+ (+ "execution_mode",+ models.CharField(+ choices=[+ ("INSTANT", "will be executed immediately"),+ ("QUEUE", "will be placed in a queue"),+ ],+ db_comment="Mode of execution",+ ),+ ),+ (+ "execution_method",+ models.CharField(+ choices=[+ ("DIRECT", " Execution triggered manually"),+ ("SCHEDULED", "Scheduled execution"),+ ],+ db_comment="Method of execution",+ ),+ ),+ (+ "execution_type",+ models.CharField(+ choices=[+ ("COMPLETE", "For complete execution"),+ ("STEP", "For step-by-step execution "),+ ],+ db_comment="Type of execution",+ ),+ ),+ (+ "execution_log_id",+ models.CharField(+ db_comment="Execution log events Id", default="", editable=False+ ),+ ),+ (+ "status",+ models.CharField(+ db_comment="Current status of execution", default=""+ ),+ ),+ (+ "error_message",+ models.CharField(+ blank=True,+ db_comment="Details of encountered errors",+ default="",+ max_length=256,+ ),+ ),+ (+ "attempts",+ models.IntegerField(+ db_comment="number of attempts taken", default=0+ ),+ ),+ (+ "execution_time",+ models.FloatField(+ db_comment="execution time in seconds", default=0+ ),+ ),+ ],+ options={+ "verbose_name": "Workflow Execution",+ "verbose_name_plural": "Workflow Executions",+ "db_table": "workflow_execution",+ },+ ),+ migrations.CreateModel(+ name="Workflow",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("prompt_name", models.CharField(default="", max_length=32)),+ ("description", models.TextField(default="", max_length=490)),+ ("workflow_name", models.CharField(max_length=128)),+ ("prompt_text", models.TextField(default="")),+ ("is_active", models.BooleanField(default=False)),+ ("status", models.CharField(default="", max_length=16)),+ ("llm_response", models.TextField()),+ (+ "deployment_type",+ models.CharField(+ choices=[+ ("DEFAULT", "Not ready yet"),+ ("ETL", "ETL pipeline"),+ ("TASK", "TASK pipeline"),+ ("API", "API deployment"),+ ("APP", "App deployment"),+ ],+ db_comment="Type of workflow deployment",+ default="DEFAULT",+ ),+ ),+ (+ "source_settings",+ models.JSONField(+ db_comment="Settings for the Source module", null=True+ ),+ ),+ (+ "destination_settings",+ models.JSONField(+ db_comment="Settings for the Destination module", null=True+ ),+ ),+ (+ "created_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="workflows_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "modified_by",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="workflows_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ (+ "organization",+ models.ForeignKey(+ blank=True,+ db_comment="Foreign key reference to the Organization model.",+ default=None,+ null=True,+ on_delete=django.db.models.deletion.CASCADE,+ to="account_v2.organization",+ ),+ ),+ (+ "workflow_owner",+ models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="workflows_owned",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ ],+ options={+ "verbose_name": "Workflow",+ "verbose_name_plural": "Workflows",+ "db_table": "workflow",+ },+ ),+ migrations.CreateModel(+ name="FileHistory",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ (+ "cache_key",+ models.CharField(+ db_comment="Hash value of file contents, WF and tool modified times",+ max_length=64,+ ),+ ),+ (+ "status",+ models.TextField(+ choices=[+ ("PENDING", "PENDING"),+ ("INITIATED", "INITIATED"),+ ("QUEUED", "QUEUED"),+ ("READY", "READY"),+ ("EXECUTING", "EXECUTING"),+ ("COMPLETED", "COMPLETED"),+ ("STOPPED", "STOPPED"),+ ("ERROR", "ERROR"),+ ],+ db_comment="Latest status of execution",+ ),+ ),+ (+ "error",+ models.TextField(+ blank=True, db_comment="Error message", default=""+ ),+ ),+ (+ "result",+ models.TextField(blank=True, db_comment="Result from execution"),+ ),+ (+ "metadata",+ models.TextField(blank=True, db_comment="MetaData from execution"),+ ),+ (+ "workflow",+ models.ForeignKey(+ on_delete=django.db.models.deletion.CASCADE,+ related_name="file_histories",+ to="workflow_v2.workflow",+ ),+ ),+ ],+ options={+ "verbose_name": "File History",+ "verbose_name_plural": "File Histories",+ "db_table": "file_history",+ },+ ),+ migrations.AddConstraint(+ model_name="workflow",+ constraint=models.UniqueConstraint(+ fields=("workflow_name", "organization"), name="unique_workflow_name"+ ),+ ),+ migrations.AddConstraint(+ model_name="filehistory",+ constraint=models.UniqueConstraint(+ fields=("workflow", "cache_key"), name="unique_workflow_cacheKey"+ ),+ ),+ ]diff --git a/backend/workflow_manager/workflow_v2/migrations/__init__.py b/backend/workflow_manager/workflow_v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.pyindex 23ceb2bbd..7a41dfc22 100644--- a/backend/workflow_manager/workflow_v2/models/execution.py+++ b/backend/workflow_manager/workflow_v2/models/execution.py@@ -31,15 +31,9 @@ class WorkflowExecution(BaseModel):null=True,db_comment="task id of asynchronous execution",)- # We can remove workflow_id if it not requiredworkflow_id = models.UUIDField(editable=False, db_comment="Id of workflow to be executed")- project_settings_id = models.UUIDField(- editable=False,- default=uuid.uuid4,- db_comment="Id of project settings used while execution",- )execution_mode = models.CharField(choices=Mode.choices, db_comment="Mode of execution")@@ -78,4 +72,4 @@ class WorkflowExecution(BaseModel):class Meta:verbose_name = "Workflow Execution"verbose_name_plural = "Workflow Executions"- db_table = "workflow_execution_v2"+ db_table = "workflow_execution"diff --git a/backend/workflow_manager/workflow_v2/models/execution_log.py b/backend/workflow_manager/workflow_v2/models/execution_log.pyindex 6bec96009..68faa9a4e 100644--- a/backend/workflow_manager/workflow_v2/models/execution_log.py+++ b/backend/workflow_manager/workflow_v2/models/execution_log.py@@ -19,4 +19,4 @@ class ExecutionLog(BaseModel):class Meta:verbose_name = "Execution Log"verbose_name_plural = "Execution Logs"- db_table = "execution_log_v2"+ db_table = "execution_log"diff --git a/backend/workflow_manager/workflow_v2/models/file_history.py b/backend/workflow_manager/workflow_v2/models/file_history.pyindex 5ad521cd0..c6ca20d0a 100644--- a/backend/workflow_manager/workflow_v2/models/file_history.py+++ b/backend/workflow_manager/workflow_v2/models/file_history.py@@ -39,11 +39,12 @@ class FileHistory(BaseModel):db_comment="Error message",)result = models.TextField(blank=True, db_comment="Result from execution")+ metadata = models.TextField(blank=True, db_comment="MetaData from execution")class Meta:verbose_name = "File History"verbose_name_plural = "File Histories"- db_table = "file_history_v2"+ db_table = "file_history"constraints = [models.UniqueConstraint(fields=["workflow", "cache_key"],diff --git a/backend/workflow_manager/workflow_v2/models/workflow.py b/backend/workflow_manager/workflow_v2/models/workflow.pyindex bfde5e86e..18f3015b3 100644--- a/backend/workflow_manager/workflow_v2/models/workflow.py+++ b/backend/workflow_manager/workflow_v2/models/workflow.py@@ -85,7 +85,7 @@ class Workflow(DefaultOrganizationMixin, BaseModel):class Meta:verbose_name = "Workflow"verbose_name_plural = "Workflows"- db_table = "workflow_v2"+ db_table = "workflow"constraints = [models.UniqueConstraint(fields=["workflow_name", "organization"],diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.pyindex f9b9c50f4..55bede227 100644--- a/backend/workflow_manager/workflow_v2/views.py+++ b/backend/workflow_manager/workflow_v2/views.py@@ -1,8 +1,6 @@import loggingfrom typing import Any, Optional-from connector_v2.connector_instance_helper import ConnectorInstanceHelper-from django.conf import settingsfrom django.db.models.query import QuerySetfrom permissions.permission import IsOwnerfrom pipeline_v2.models import Pipeline@@ -116,16 +114,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):is_active=True,)WorkflowEndpointUtils.create_endpoints_for_workflow(workflow)-- # Enable GCS configurations to create GCS while creating a workflow- if (- settings.GOOGLE_STORAGE_ACCESS_KEY_ID- and settings.UNSTRACT_FREE_STORAGE_BUCKET_NAME- ):- ConnectorInstanceHelper.create_default_gcs_connector(- workflow, self.request.user- )-+ # NOTE: Add default connector here if neededexcept Exception as e:logger.error(f"Error saving workflow to DB: {e}")raise WorkflowGenerationError