Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

schema migrations and settings for v2 applications (#536)

alif99fc60f0cbf
diff --git a/backend/account_v2/migrations/0001_initial.py b/backend/account_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..2394c9f59
--- /dev/null
+++ b/backend/account_v2/migrations/0001_initial.py
@@ -0,0 +1,265 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.contrib.auth.models
+import django.contrib.auth.validators
+import django.db.models.deletion
+import django.utils.timezone
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ("auth", "0012_alter_user_first_name_max_length"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="User",
+ fields=[
+ (
+ "id",
+ models.BigAutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("password", models.CharField(max_length=128, verbose_name="password")),
+ (
+ "last_login",
+ models.DateTimeField(
+ blank=True, null=True, verbose_name="last login"
+ ),
+ ),
+ (
+ "is_superuser",
+ models.BooleanField(
+ default=False,
+ help_text="Designates that this user has all permissions without explicitly assigning them.",
+ verbose_name="superuser status",
+ ),
+ ),
+ (
+ "username",
+ models.CharField(
+ error_messages={
+ "unique": "A user with that username already exists."
+ },
+ help_text="Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.",
+ max_length=150,
+ unique=True,
+ validators=[
+ django.contrib.auth.validators.UnicodeUsernameValidator()
+ ],
+ verbose_name="username",
+ ),
+ ),
+ (
+ "first_name",
+ models.CharField(
+ blank=True, max_length=150, verbose_name="first name"
+ ),
+ ),
+ (
+ "last_name",
+ models.CharField(
+ blank=True, max_length=150, verbose_name="last name"
+ ),
+ ),
+ (
+ "email",
+ models.EmailField(
+ blank=True, max_length=254, verbose_name="email address"
+ ),
+ ),
+ (
+ "is_staff",
+ models.BooleanField(
+ default=False,
+ help_text="Designates whether the user can log into this admin site.",
+ verbose_name="staff status",
+ ),
+ ),
+ (
+ "is_active",
+ models.BooleanField(
+ default=True,
+ help_text="Designates whether this user should be treated as active. Unselect this instead of deleting accounts.",
+ verbose_name="active",
+ ),
+ ),
+ (
+ "date_joined",
+ models.DateTimeField(
+ default=django.utils.timezone.now, verbose_name="date joined"
+ ),
+ ),
+ ("user_id", models.CharField()),
+ ("project_storage_created", models.BooleanField(default=False)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="users_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "groups",
+ models.ManyToManyField(
+ blank=True,
+ related_name="users",
+ related_query_name="user",
+ to="auth.group",
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="users_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "user_permissions",
+ models.ManyToManyField(
+ blank=True,
+ related_name="users",
+ related_query_name="user",
+ to="auth.permission",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "User",
+ "verbose_name_plural": "Users",
+ "db_table": "user",
+ },
+ managers=[
+ ("objects", django.contrib.auth.models.UserManager()),
+ ],
+ ),
+ migrations.CreateModel(
+ name="Organization",
+ fields=[
+ (
+ "id",
+ models.BigAutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("name", models.CharField(max_length=64)),
+ ("display_name", models.CharField(max_length=64)),
+ ("organization_id", models.CharField(max_length=64, unique=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ ("created_at", models.DateTimeField(auto_now=True)),
+ (
+ "allowed_token_limit",
+ models.IntegerField(
+ db_comment="token limit set in case of frition less onbaoarded org",
+ default=-1,
+ ),
+ ),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="orgs_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="orgs_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Organization",
+ "verbose_name_plural": "Organizations",
+ "db_table": "organization",
+ },
+ ),
+ migrations.CreateModel(
+ name="PlatformKey",
+ fields=[
+ (
+ "id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ ("key", models.UUIDField(default=uuid.uuid4)),
+ ("key_name", models.CharField(blank=True, default="", max_length=64)),
+ ("is_active", models.BooleanField(default=False)),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="platform_keys_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="platform_keys_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "organization",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="platform_keys",
+ to="account_v2.organization",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Platform Key",
+ "verbose_name_plural": "Platform Keys",
+ "db_table": "platform_key",
+ },
+ ),
+ migrations.AddConstraint(
+ model_name="platformkey",
+ constraint=models.UniqueConstraint(
+ fields=("key_name", "organization"), name="unique_key_name_organization"
+ ),
+ ),
+ ]
diff --git a/backend/account_v2/models.py b/backend/account_v2/models.py
index 61428bda0..495d2913f 100644
--- a/backend/account_v2/models.py
+++ b/backend/account_v2/models.py
@@ -45,7 +45,7 @@ class Organization(models.Model):
class Meta:
verbose_name = "Organization"
verbose_name_plural = "Organizations"
- db_table = "organization_v2"
+ db_table = "organization"
class User(AbstractUser):
@@ -96,7 +96,7 @@ class User(AbstractUser):
class Meta:
verbose_name = "User"
verbose_name_plural = "Users"
- db_table = "user_v2"
+ db_table = "user"
class PlatformKey(models.Model):
@@ -135,7 +135,7 @@ class PlatformKey(models.Model):
class Meta:
verbose_name = "Platform Key"
verbose_name_plural = "Platform Keys"
- db_table = "platform_key_v2"
+ db_table = "platform_key"
constraints = [
models.UniqueConstraint(
fields=["key_name", "organization"],
diff --git a/backend/account_v2/subscription_loader.py b/backend/account_v2/subscription_loader.py
index d380ed19d..9502f135a 100644
--- a/backend/account_v2/subscription_loader.py
+++ b/backend/account_v2/subscription_loader.py
@@ -1,5 +1,6 @@
import logging
import os
+from datetime import timezone
from importlib import import_module
from typing import Any
@@ -75,3 +76,32 @@ def load_plugins() -> list[Any]:
logger.info("No subscription plugins found.")
return subscription_plugins
+
+
+def validate_etl_run(org_id: str) -> bool:
+ """Method to check subscription status before ETL runs.
+
+ Args:
+ org_id: The ID of the organization.
+
+ Returns:
+ A boolean indicating whether the pre-run check passed or not.
+ """
+ try:
+ from pluggable_apps.subscription.subscription_helper import SubscriptionHelper
+ except ModuleNotFoundError:
+ logger.error("Subscription plugin not found.")
+ return False
+
+ org_plans = SubscriptionHelper.get_subscription(org_id)
+ if not org_plans or not org_plans.is_active:
+ return False
+
+ if org_plans.is_paid:
+ return True
+
+ if timezone.now() >= org_plans.end_date:
+ logger.debug(f"Trial expired for org {org_id}")
+ return False
+
+ return True
diff --git a/backend/account_v2/urls.py b/backend/account_v2/urls.py
index 767be8aba..605ab2621 100644
--- a/backend/account_v2/urls.py
+++ b/backend/account_v2/urls.py
@@ -16,7 +16,7 @@ urlpatterns = [
path("logout", logout, name="logout"),
path("callback", callback, name="callback"),
path("session", get_session_data, name="session"),
- path("organizations", get_organizations, name="get_organizations"),
+ path("organization", get_organizations, name="get_organizations"),
path("organization/<str:id>/set", set_organization, name="set_organization"),
path("organization/create", create_organization, name="create_organization"),
]
diff --git a/backend/adapter_processor_v2/adapter_processor.py b/backend/adapter_processor_v2/adapter_processor.py
index dd9d44ff3..16c3b9914 100644
--- a/backend/adapter_processor_v2/adapter_processor.py
+++ b/backend/adapter_processor_v2/adapter_processor.py
@@ -12,6 +12,7 @@ from adapter_processor_v2.exceptions import (
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from platform_settings_v2.platform_auth_service import PlatformAuthenticationService
+from tenant_account_v2.organization_member_service import OrganizationMemberService
from unstract.adapters.adapterkit import Adapterkit
from unstract.adapters.base import Adapter
from unstract.adapters.enums import AdapterTypes
@@ -112,10 +113,13 @@ class AdapterProcessor:
@staticmethod
def set_default_triad(default_triad: dict[str, str], user: User) -> None:
try:
+ organization_member = OrganizationMemberService.get_user_by_id(user.id)
(
user_default_adapter,
created,
- ) = UserDefaultAdapter.objects.get_or_create(user=user)
+ ) = UserDefaultAdapter.objects.get_or_create(
+ organization_member=organization_member
+ )
if default_triad.get(AdapterKeys.LLM_DEFAULT, None):
user_default_adapter.default_llm_adapter = AdapterInstance.objects.get(
@@ -235,7 +239,11 @@ class AdapterProcessor:
"""
try:
adapters: list[AdapterInstance] = []
- default_adapter = UserDefaultAdapter.objects.get(user=user)
+
+ organization_member = OrganizationMemberService.get_user_by_id(id=user.id)
+ default_adapter: UserDefaultAdapter = UserDefaultAdapter.objects.get(
+ organization_member=organization_member
+ )
if default_adapter.default_embedding_adapter:
adapters.append(default_adapter.default_embedding_adapter)
diff --git a/backend/adapter_processor_v2/migrations/0001_initial.py b/backend/adapter_processor_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..9835825db
--- /dev/null
+++ b/backend/adapter_processor_v2/migrations/0001_initial.py
@@ -0,0 +1,220 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ("account_v2", "0001_initial"),
+ ("tenant_account_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="AdapterInstance",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ db_comment="Unique identifier for the Adapter Instance",
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "adapter_name",
+ models.TextField(
+ db_comment="Name of the Adapter Instance", max_length=128
+ ),
+ ),
+ (
+ "adapter_id",
+ models.CharField(
+ db_comment="Unique identifier of the Adapter",
+ default="",
+ max_length=128,
+ ),
+ ),
+ (
+ "adapter_metadata",
+ models.JSONField(
+ db_column="adapter_metadata",
+ db_comment="JSON adapter metadata submitted by the user",
+ default=dict,
+ ),
+ ),
+ ("adapter_metadata_b", models.BinaryField(null=True)),
+ (
+ "adapter_type",
+ models.CharField(
+ choices=[
+ ("UNKNOWN", "UNKNOWN"),
+ ("LLM", "LLM"),
+ ("EMBEDDING", "EMBEDDING"),
+ ("VECTOR_DB", "VECTOR_DB"),
+ ("OCR", "OCR"),
+ ("X2TEXT", "X2TEXT"),
+ ],
+ db_comment="Type of adapter LLM/EMBEDDING/VECTOR_DB",
+ ),
+ ),
+ (
+ "is_active",
+ models.BooleanField(
+ db_comment="Is the adapter instance currently being used",
+ default=False,
+ ),
+ ),
+ (
+ "shared_to_org",
+ models.BooleanField(
+ db_comment="Is the adapter shared to entire org", default=False
+ ),
+ ),
+ (
+ "is_friction_less",
+ models.BooleanField(
+ db_comment="Was the adapter created through frictionless onboarding",
+ default=False,
+ ),
+ ),
+ (
+ "is_usable",
+ models.BooleanField(
+ db_comment="Is the Adpater Usable", default=True
+ ),
+ ),
+ ("description", models.TextField(blank=True, default=None, null=True)),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="adapters_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="adapters_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "organization",
+ models.ForeignKey(
+ blank=True,
+ db_comment="Foreign key reference to the Organization model.",
+ default=None,
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ to="account_v2.organization",
+ ),
+ ),
+ (
+ "shared_users",
+ models.ManyToManyField(
+ related_name="shared_adapters_instance",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "adapter instance",
+ "verbose_name_plural": "adapter instances",
+ "db_table": "adapter_instance",
+ },
+ ),
+ migrations.CreateModel(
+ name="UserDefaultAdapter",
+ fields=[
+ (
+ "id",
+ models.BigAutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "default_embedding_adapter",
+ models.ForeignKey(
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="user_default_embedding_adapter",
+ to="adapter_processor_v2.adapterinstance",
+ ),
+ ),
+ (
+ "default_llm_adapter",
+ models.ForeignKey(
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="user_default_llm_adapter",
+ to="adapter_processor_v2.adapterinstance",
+ ),
+ ),
+ (
+ "default_vector_db_adapter",
+ models.ForeignKey(
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="user_default_vector_db_adapter",
+ to="adapter_processor_v2.adapterinstance",
+ ),
+ ),
+ (
+ "default_x2text_adapter",
+ models.ForeignKey(
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="user_default_x2text_adapter",
+ to="adapter_processor_v2.adapterinstance",
+ ),
+ ),
+ (
+ "organization_member",
+ models.OneToOneField(
+ db_comment="Foreign key reference to the OrganizationMember model.",
+ default=None,
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="default_adapters",
+ to="tenant_account_v2.organizationmember",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Default Adapter for Organization User",
+ "verbose_name_plural": "Default Adapters for Organization Users",
+ "db_table": "default_organization_user_adapter",
+ },
+ ),
+ migrations.AddConstraint(
+ model_name="adapterinstance",
+ constraint=models.UniqueConstraint(
+ fields=("adapter_name", "adapter_type", "organization"),
+ name="unique_organization_adapter",
+ ),
+ ),
+ ]
diff --git a/backend/platform_settings_v2/migrations/__init__.py b/backend/adapter_processor_v2/migrations/__init__.py
similarity index 100%
rename from backend/platform_settings_v2/migrations/__init__.py
rename to backend/adapter_processor_v2/migrations/__init__.py
diff --git a/backend/adapter_processor_v2/models.py b/backend/adapter_processor_v2/models.py
index 5e63056c5..03c257de7 100644
--- a/backend/adapter_processor_v2/models.py
+++ b/backend/adapter_processor_v2/models.py
@@ -122,7 +122,7 @@ class AdapterInstance(DefaultOrganizationMixin, BaseModel):
class Meta:
verbose_name = "adapter instance"
verbose_name_plural = "adapter instances"
- db_table = "adapter_instance_v2"
+ db_table = "adapter_instance"
constraints = [
models.UniqueConstraint(
fields=["adapter_name", "adapter_type", "organization"],
@@ -165,9 +165,6 @@ class AdapterInstance(DefaultOrganizationMixin, BaseModel):
class UserDefaultAdapter(BaseModel):
- user = models.OneToOneField(
- User, on_delete=models.CASCADE, related_name="organization_default_adapters"
- )
organization_member = models.OneToOneField(
OrganizationMember,
on_delete=models.CASCADE,
@@ -205,4 +202,4 @@ class UserDefaultAdapter(BaseModel):
class Meta:
verbose_name = "Default Adapter for Organization User"
verbose_name_plural = "Default Adapters for Organization Users"
- db_table = "default_organization_user_adapter_v2"
+ db_table = "default_organization_user_adapter"
diff --git a/backend/adapter_processor_v2/views.py b/backend/adapter_processor_v2/views.py
index 93cddf5aa..b5a65179f 100644
--- a/backend/adapter_processor_v2/views.py
+++ b/backend/adapter_processor_v2/views.py
@@ -65,7 +65,12 @@ class DefaultAdapterViewSet(ModelViewSet):
self, request: Request, *args: tuple[Any], **kwargs: dict[str, Any]
) -> HttpResponse:
try:
- user_default_adapter = UserDefaultAdapter.objects.get(user=request.user)
+ organization_member = OrganizationMemberService.get_user_by_id(
+ request.user.id
+ )
+ user_default_adapter = UserDefaultAdapter.objects.get(
+ organization_member=organization_member
+ )
serializer = UserDefaultAdapterSerializer(user_default_adapter).data
return Response(serializer)
@@ -172,13 +177,18 @@ class AdapterInstanceViewSet(ModelViewSet):
serializer.is_valid(raise_exception=True)
try:
instance = serializer.save()
+ organization_member = OrganizationMemberService.get_user_by_id(
+ request.user.id
+ )
# Check to see if there is a default configured
# for this adapter_type and for the current user
(
user_default_adapter,
created,
- ) = UserDefaultAdapter.objects.get_or_create(user=request.user)
+ ) = UserDefaultAdapter.objects.get_or_create(
+ organization_member=organization_member
+ )
adapter_type = serializer.validated_data.get(AdapterKeys.ADAPTER_TYPE)
if (adapter_type == AdapterKeys.LLM) and (
@@ -222,8 +232,11 @@ class AdapterInstanceViewSet(ModelViewSet):
adapter_instance: AdapterInstance = self.get_object()
adapter_type = adapter_instance.adapter_type
try:
+ organization_member = OrganizationMemberService.get_user_by_id(
+ request.user.id
+ )
user_default_adapter: UserDefaultAdapter = UserDefaultAdapter.objects.get(
- user=request.user
+ organization_member=organization_member
)
if (
@@ -279,8 +292,13 @@ class AdapterInstanceViewSet(ModelViewSet):
# Remove the same from his default
for user_id in removed_users:
try:
- user_default_adapter = UserDefaultAdapter.objects.get(
- user_id=user_id
+ organization_member = OrganizationMemberService.get_user_by_id(
+ id=user_id
+ )
+ user_default_adapter: UserDefaultAdapter = (
+ UserDefaultAdapter.objects.get(
+ organization_member=organization_member
+ )
)
if user_default_adapter.default_llm_adapter == adapter:
diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py
index 802d8f8bc..486425cae 100644
--- a/backend/api_v2/api_deployment_views.py
+++ b/backend/api_v2/api_deployment_views.py
@@ -135,7 +135,7 @@ class APIDeploymentViewSet(viewsets.ModelViewSet):
) -> Response:
"""Downloads a Postman Collection of the API deployment instance."""
instance = self.get_object()
- api_key_inst = instance.apikey_set.filter(is_active=True).first()
+ api_key_inst = instance.api_keys.filter(is_active=True).first()
if not api_key_inst:
logger.error(f"No active API key set for deployment {instance.pk}")
raise NoActiveAPIKeyError(deployment_name=instance.display_name)
diff --git a/backend/api_v2/deployment_helper.py b/backend/api_v2/deployment_helper.py
index 4c6d1698b..be9dc0b10 100644
--- a/backend/api_v2/deployment_helper.py
+++ b/backend/api_v2/deployment_helper.py
@@ -191,6 +191,7 @@ class DeploymentHelper:
api: APIDeployment,
file_objs: list[UploadedFile],
timeout: int,
+ include_metadata: bool = False,
) -> ReturnDict:
"""Execute workflow by api.
@@ -217,6 +218,7 @@ class DeploymentHelper:
hash_values_of_files=hash_values_of_files,
timeout=timeout,
execution_id=execution_id,
+ include_metadata=include_metadata,
)
result.status_api = DeploymentHelper.construct_status_endpoint(
api_endpoint=api.api_endpoint, execution_id=execution_id
diff --git a/backend/api_v2/execution_urls.py b/backend/api_v2/execution_urls.py
new file mode 100644
index 000000000..a8ca4feae
--- /dev/null
+++ b/backend/api_v2/execution_urls.py
@@ -0,0 +1,16 @@
+from api_v2.api_deployment_views import DeploymentExecution
+from django.urls import re_path
+from rest_framework.urlpatterns import format_suffix_patterns
+
+execute = DeploymentExecution.as_view()
+
+
+urlpatterns = format_suffix_patterns(
+ [
+ re_path(
+ r"^api/(?P<org_name>[\w-]+)/(?P<api_name>[\w-]+)/?$",
+ execute,
+ name="api_deployment_execution",
+ )
+ ]
+)
diff --git a/backend/api_v2/migrations/0001_initial.py b/backend/api_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..89653832b
--- /dev/null
+++ b/backend/api_v2/migrations/0001_initial.py
@@ -0,0 +1,208 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ("account_v2", "0001_initial"),
+ ("workflow_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="APIDeployment",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "display_name",
+ models.CharField(
+ db_comment="User-given display name for the API.",
+ default="default api",
+ max_length=30,
+ ),
+ ),
+ (
+ "description",
+ models.CharField(
+ blank=True,
+ db_comment="User-given description for the API.",
+ default="",
+ max_length=255,
+ ),
+ ),
+ (
+ "is_active",
+ models.BooleanField(
+ db_comment="Flag indicating whether the API is active or not.",
+ default=True,
+ ),
+ ),
+ (
+ "api_endpoint",
+ models.CharField(
+ db_comment="URL endpoint for the API deployment.",
+ editable=False,
+ max_length=255,
+ unique=True,
+ ),
+ ),
+ (
+ "api_name",
+ models.CharField(
+ db_comment="Short name for the API deployment.",
+ default=uuid.uuid4,
+ max_length=30,
+ ),
+ ),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="apis_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="apis_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "organization",
+ models.ForeignKey(
+ blank=True,
+ db_comment="Foreign key reference to the Organization model.",
+ default=None,
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ to="account_v2.organization",
+ ),
+ ),
+ (
+ "workflow",
+ models.ForeignKey(
+ db_comment="Foreign key reference to the Workflow model.",
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="apis",
+ to="workflow_v2.workflow",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Api Deployment",
+ "verbose_name_plural": "Api Deployments",
+ "db_table": "api_deployment",
+ },
+ ),
+ migrations.CreateModel(
+ name="APIKey",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ db_comment="Unique identifier for the API key.",
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "api_key",
+ models.UUIDField(
+ db_comment="Actual key UUID.",
+ default=uuid.uuid4,
+ editable=False,
+ unique=True,
+ ),
+ ),
+ (
+ "description",
+ models.CharField(
+ db_comment="Description of the API key.",
+ max_length=255,
+ null=True,
+ ),
+ ),
+ (
+ "is_active",
+ models.BooleanField(
+ db_comment="Flag indicating whether the API key is active or not.",
+ default=True,
+ ),
+ ),
+ (
+ "api",
+ models.ForeignKey(
+ db_comment="Foreign key reference to the APIDeployment model.",
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="api_keys",
+ to="api_v2.apideployment",
+ ),
+ ),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="api_keys_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="api_keys_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Api Deployment key",
+ "verbose_name_plural": "Api Deployment keys",
+ "db_table": "api_deployment_key",
+ },
+ ),
+ migrations.AddConstraint(
+ model_name="apideployment",
+ constraint=models.UniqueConstraint(
+ fields=("api_name", "organization"), name="unique_api_name"
+ ),
+ ),
+ ]
diff --git a/backend/api_v2/migrations/__init__.py b/backend/api_v2/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/api_v2/models.py b/backend/api_v2/models.py
index 545426537..6cb99a9f5 100644
--- a/backend/api_v2/models.py
+++ b/backend/api_v2/models.py
@@ -104,7 +104,7 @@ class APIDeployment(DefaultOrganizationMixin, BaseModel):
class Meta:
verbose_name = "Api Deployment"
verbose_name_plural = "Api Deployments"
- db_table = "api_deployment_v2"
+ db_table = "api_deployment"
constraints = [
models.UniqueConstraint(
fields=["api_name", "organization"],
@@ -164,4 +164,4 @@ class APIKey(BaseModel):
class Meta:
verbose_name = "Api Deployment key"
verbose_name_plural = "Api Deployment keys"
- db_table = "api_deployment_key_v2"
+ db_table = "api_deployment_key"
diff --git a/backend/api_v2/urls.py b/backend/api_v2/urls.py
index 065617d66..504b13a7f 100644
--- a/backend/api_v2/urls.py
+++ b/backend/api_v2/urls.py
@@ -1,6 +1,6 @@
from api_v2.api_deployment_views import APIDeploymentViewSet, DeploymentExecution
from api_v2.api_key_views import APIKeyViewSet
-from django.urls import path, re_path
+from django.urls import path
from rest_framework.urlpatterns import format_suffix_patterns
deployment = APIDeploymentViewSet.as_view(
@@ -52,11 +52,6 @@ urlpatterns = format_suffix_patterns(
download_postman_collection,
name="download_postman_collection",
),
- re_path(
- r"^api/(?P<org_name>[\w-]+)/(?P<api_name>[\w-]+)/?$",
- execute,
- name="api_deployment_execution",
- ),
path("keys/<uuid:pk>/", key_details, name="key_details"),
path("keys/api/<str:api_id>/", api_key, name="api_key"),
]
diff --git a/backend/backend/base_urls.py b/backend/backend/base_urls.py
new file mode 100644
index 000000000..b0c728867
--- /dev/null
+++ b/backend/backend/base_urls.py
@@ -0,0 +1,21 @@
+# base_urls.py
+from django.conf import settings
+from django.urls import include, path
+
+from .public_urls_v2 import urlpatterns as public_urls
+
+# Import urlpatterns from each file
+from .urls_v2 import urlpatterns as tenant_urls
+
+# Combine the URL patterns
+urlpatterns = [
+ path(
+ f"{settings.TENANT_SUBFOLDER_PREFIX}/",
+ include((tenant_urls, "tenant"), namespace="tenant"),
+ ),
+ path(
+ f"{settings.PATH_PREFIX}/", include((public_urls, "public"), namespace="public")
+ ),
+ # API deployment
+ path(f"{settings.API_DEPLOYMENT_PATH_PREFIX}/", include("api_v2.execution_urls")),
+]
diff --git a/backend/backend/public_urls_v2.py b/backend/backend/public_urls_v2.py
new file mode 100644
index 000000000..1e6d3550e
--- /dev/null
+++ b/backend/backend/public_urls_v2.py
@@ -0,0 +1,56 @@
+"""URL configuration for backend project.
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+ https://docs.djangoproject.com/en/4.2/topics/http/urls/
+Examples:
+Function views
+ 1. Add an import: from my_app import views
+ 2. Add a URL to urlpatterns: path('', views.home, name='home')
+Class-based views
+ 1. Add an import: from other_app.views import Home
+ 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
+Including another URLconf
+ 1. Import the include() function: from django.urls import include, path
+ 2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
+"""
+
+from account_v2.admin import admin
+from django.conf import settings
+from django.conf.urls import * # noqa: F401, F403
+from django.conf.urls.static import static
+from django.urls import include, path
+
+path_prefix = settings.PATH_PREFIX
+api_path_prefix = settings.API_DEPLOYMENT_PATH_PREFIX
+
+urlpatterns = [
+ path("", include("account_v2.urls")),
+ # Connector OAuth
+ path("", include("connector_auth_v2.urls")),
+ # # Docs
+ path("", include("docs.urls")),
+ # # # API deployment
+ # path("api/", include("api_v2.execution_urls")),
+ # # Feature flags
+ path("flags/", include("feature_flag.urls")),
+]
+if settings.ADMIN_ENABLED:
+ # Admin URLs
+ urlpatterns += [
+ path("admin/", admin.site.urls),
+ path(
+ "admin/doc/",
+ include("django.contrib.admindocs.urls"),
+ ),
+ ]
+urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
+
+
+try:
+ import pluggable_apps.platform_admin.urls # noqa: F401
+
+ urlpatterns += [
+ path("", include("pluggable_apps.platform_admin.urls")),
+ ]
+except ImportError:
+ pass
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 260d4561b..cb87a99cb 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -139,6 +139,7 @@ DB_USER = os.environ.get("DB_USER", "unstract_dev")
DB_HOST = os.environ.get("DB_HOST", "backend-db-1")
DB_PASSWORD = os.environ.get("DB_PASSWORD", "unstract_pass")
DB_PORT = os.environ.get("DB_PORT", 5432)
+DB_SCHEMA = os.environ.get("DB_SCHEMA", "unstract_v2")
DEFAULT_ORGANIZATION = "default_org"
FLIPT_BASE_URL = os.environ.get("FLIPT_BASE_URL", "http://localhost:9005")
@@ -175,6 +176,9 @@ CELERY_BROKER_URL = get_required_setting(
INDEXING_FLAG_TTL = int(get_required_setting("INDEXING_FLAG_TTL"))
NOTIFICATION_TIMEOUT = int(get_required_setting("NOTIFICATION_TIMEOUT", "5"))
+ATOMIC_REQUESTS = CommonUtils.str_to_bool(
+ os.environ.get("DJANGO_ATOMIC_REQUESTS", "False")
+)
# Flag to Enable django admin
ADMIN_ENABLED = False
@@ -192,68 +196,117 @@ ALLOWED_HOSTS = ["*"]
CSRF_TRUSTED_ORIGINS = [WEB_APP_ORIGIN_URL]
CORS_ALLOW_ALL_ORIGINS = False
-
-# Application definition
-SHARED_APPS = (
- # Multitenancy
- "corsheaders",
- # For the organization model
- "account",
- "account_usage",
- # Django apps should go below this line
- "django.contrib.admin",
- "django.contrib.auth",
- "django.contrib.contenttypes",
- "django.contrib.sessions",
- "django.contrib.messages",
- "django.contrib.staticfiles",
- "django.contrib.admindocs",
- # Third party apps should go below this line,
- "rest_framework",
- # Connector OAuth
- "connector_auth",
- "social_django",
- # Doc generator
- "drf_yasg",
- "docs",
- # Plugins
- "plugins",
- "feature_flag",
- "django_celery_beat",
-)
-
if not check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
- SHARED_APPS = ("django_tenants",) + SHARED_APPS
-
-TENANT_APPS = (
- # your tenant-specific apps
- "django.contrib.admin",
- "django.contrib.auth",
- "django.contrib.contenttypes",
- "django.contrib.messages",
- "django.contrib.staticfiles",
- "tenant_account",
- "project",
- "prompt",
- "connector",
- "adapter_processor",
- "file_management",
- "workflow_manager.endpoint",
- "workflow_manager.workflow",
- "tool_instance",
- "pipeline",
- "platform_settings",
- "api",
- "prompt_studio.prompt_profile_manager",
- "prompt_studio.prompt_studio",
- "prompt_studio.prompt_studio_core",
- "prompt_studio.prompt_studio_registry",
- "prompt_studio.prompt_studio_output_manager",
- "prompt_studio.prompt_studio_document_manager",
- "prompt_studio.prompt_studio_index_manager",
- "usage",
- "notification",
-)
+ SHARED_APPS = (
+ # Multitenancy
+ "django_tenants",
+ "corsheaders",
+ # For the organization model
+ "account",
+ "account_usage",
+ # Django apps should go below this line
+ "django.contrib.admin",
+ "django.contrib.auth",
+ "django.contrib.contenttypes",
+ "django.contrib.sessions",
+ "django.contrib.messages",
+ "django.contrib.staticfiles",
+ "django.contrib.admindocs",
+ # Third party apps should go below this line,
+ "rest_framework",
+ # Connector OAuth
+ "connector_auth",
+ "social_django",
+ # Doc generator
+ "drf_yasg",
+ "docs",
+ # Plugins
+ "plugins",
+ "feature_flag",
+ "django_celery_beat",
+ )
+ TENANT_APPS = (
+ # your tenant-specific apps
+ "django.contrib.admin",
+ "django.contrib.auth",
+ "django.contrib.contenttypes",
+ "django.contrib.messages",
+ "django.contrib.staticfiles",
+ "tenant_account",
+ "project",
+ "prompt",
+ "connector",
+ "adapter_processor",
+ "file_management",
+ "workflow_manager.endpoint",
+ "workflow_manager.workflow",
+ "tool_instance",
+ "pipeline",
+ "platform_settings",
+ "api",
+ "prompt_studio.prompt_profile_manager",
+ "prompt_studio.prompt_studio",
+ "prompt_studio.prompt_studio_core",
+ "prompt_studio.prompt_studio_registry",
+ "prompt_studio.prompt_studio_output_manager",
+ "prompt_studio.prompt_studio_document_manager",
+ "prompt_studio.prompt_studio_index_manager",
+ "usage",
+ "notification",
+ )
+else:
+ SHARED_APPS = (
+ # Multitenancy
+ # "django_tenants",
+ "corsheaders",
+ # For the organization model
+ "account_v2",
+ # Django apps should go below this line
+ "django.contrib.admin",
+ "django.contrib.auth",
+ "django.contrib.contenttypes",
+ "django.contrib.sessions",
+ "django.contrib.messages",
+ "django.contrib.staticfiles",
+ "django.contrib.admindocs",
+ # Third party apps should go below this line,
+ "rest_framework",
+ # Connector OAuth
+ # "connector_auth",
+ "social_django",
+ # Doc generator
+ "drf_yasg",
+ "docs",
+ # Plugins
+ "plugins",
+ "feature_flag",
+ "django_celery_beat",
+ )
+ v2_apps = (
+ "migration_tools",
+ # "account_v2",
+ "connector_auth_v2",
+ "tenant_account_v2",
+ "connector_v2",
+ "adapter_processor_v2",
+ "file_management",
+ "workflow_manager.endpoint_v2",
+ "workflow_manager.workflow_v2",
+ "tool_instance_v2",
+ "pipeline_v2",
+ "platform_settings_v2",
+ "api_v2",
+ "usage_v2",
+ "prompt_studio.prompt_profile_manager_v2",
+ "prompt_studio.prompt_studio_v2",
+ "prompt_studio.prompt_studio_core_v2",
+ "prompt_studio.prompt_studio_registry_v2",
+ "prompt_studio.prompt_studio_output_manager_v2",
+ "prompt_studio.prompt_studio_document_manager_v2",
+ "prompt_studio.prompt_studio_index_manager_v2",
+ )
+ SHARED_APPS += v2_apps
+ TENANT_APPS = []
INSTALLED_APPS = list(SHARED_APPS) + [
app for app in TENANT_APPS if app not in SHARED_APPS
@@ -297,6 +350,21 @@ if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
# Namespaces
SOCIAL_AUTH_URL_NAMESPACE = "public:social"
LOGIN_CALLBACK_URL_NAMESPACE = "public:callback"
+ DATABASES = {
+ "default": {
+ "ENGINE": DB_ENGINE,
+ "NAME": f"{DB_NAME}",
+ "USER": f"{DB_USER}",
+ "HOST": f"{DB_HOST}",
+ "PASSWORD": f"{DB_PASSWORD}",
+ "PORT": f"{DB_PORT}",
+ "ATOMIC_REQUESTS": ATOMIC_REQUESTS,
+ "OPTIONS": {
+ "options": f"-c search_path={DB_SCHEMA}",
+ "application_name": os.environ.get("APPLICATION_NAME", ""),
+ },
+ }
+ }
else:
# Middleware Configuration
TENANT_MIDDLEWARE = "django_tenants.middleware.TenantSubfolderMiddleware"
@@ -326,6 +394,20 @@ else:
# Namespaces
SOCIAL_AUTH_URL_NAMESPACE = "social"
LOGIN_CALLBACK_URL_NAMESPACE = "callback"
+ DATABASES = {
+ "default": {
+ "ENGINE": DB_ENGINE,
+ "NAME": f"{DB_NAME}",
+ "USER": f"{DB_USER}",
+ "HOST": f"{DB_HOST}",
+ "PASSWORD": f"{DB_PASSWORD}",
+ "PORT": f"{DB_PORT}",
+ "ATOMIC_REQUESTS": ATOMIC_REQUESTS,
+ "OPTIONS": {
+ "application_name": os.environ.get("APPLICATION_NAME", ""),
+ },
+ }
+ }
MIDDLEWARE = [
"log_request_id.middleware.RequestIDMiddleware",
@@ -366,27 +448,6 @@ TEMPLATES = [
WSGI_APPLICATION = "backend.wsgi.application"
-ATOMIC_REQUESTS = CommonUtils.str_to_bool(
- os.environ.get("DJANGO_ATOMIC_REQUESTS", "False")
-)
-# Database
-# https://docs.djangoproject.com/en/4.2/ref/settings/#databases
-
-DATABASES = {
- "default": {
- "ENGINE": DB_ENGINE,
- "NAME": f"{DB_NAME}",
- "USER": f"{DB_USER}",
- "HOST": f"{DB_HOST}",
- "PASSWORD": f"{DB_PASSWORD}",
- "PORT": f"{DB_PORT}",
- "ATOMIC_REQUESTS": ATOMIC_REQUESTS,
- "OPTIONS": {
- "application_name": os.environ.get("APPLICATION_NAME", ""),
- },
- }
-}
-
# SocketIO connection manager
SOCKET_IO_MANAGER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}"
diff --git a/backend/backend/urls_v2.py b/backend/backend/urls_v2.py
new file mode 100644
index 000000000..0c0b3f40e
--- /dev/null
+++ b/backend/backend/urls_v2.py
@@ -0,0 +1,69 @@
+"""URL configuration for backend project.
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+ https://docs.djangoproject.com/en/4.2/topics/http/urls/
+Examples:
+Function views
+ 1. Add an import: from my_app import views
+ 2. Add a URL to urlpatterns: path('', views.home, name='home')
+Class-based views
+ 1. Add an import: from other_app.views import Home
+ 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
+Including another URLconf
+ 1. Import the include() function: from django.urls import include, path
+ 2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
+"""
+
+from django.conf.urls import * # noqa: F401, F403
+from django.urls import include, path
+
+from backend.constants import UrlPathConstants
+
+urlpatterns = [
+ path("", include("tenant_account_v2.urls")),
+ path("", include("connector_v2.urls")),
+ path("", include("connector_processor.urls")),
+ path("", include("adapter_processor_v2.urls")),
+ path("", include("file_management.urls")),
+ path("", include("tool_instance_v2.urls")),
+ path("", include("pipeline_v2.urls")),
+ path("", include("feature_flag.urls")),
+ path("workflow/", include("workflow_manager.urls")),
+ path("platform/", include("platform_settings_v2.urls")),
+ path("api/", include("api_v2.urls")),
+ path("usage/", include("usage_v2.urls")),
+ path(
+ UrlPathConstants.PROMPT_STUDIO,
+ include("prompt_studio.prompt_profile_manager_v2.urls"),
+ ),
+ path(
+ UrlPathConstants.PROMPT_STUDIO,
+ include("prompt_studio.prompt_studio_v2.urls"),
+ ),
+ path("", include("prompt_studio.prompt_studio_core_v2.urls")),
+ path(
+ UrlPathConstants.PROMPT_STUDIO,
+ include("prompt_studio.prompt_studio_registry_v2.urls"),
+ ),
+ path(
+ UrlPathConstants.PROMPT_STUDIO,
+ include("prompt_studio.prompt_studio_output_manager_v2.urls"),
+ ),
+ path(
+ UrlPathConstants.PROMPT_STUDIO,
+ include("prompt_studio.prompt_studio_document_manager_v2.urls"),
+ ),
+ path(
+ UrlPathConstants.PROMPT_STUDIO,
+ include("prompt_studio.prompt_studio_index_manager_v2.urls"),
+ ),
+]
+
+try:
+ import pluggable_apps.subscription.urls # noqa: F401
+
+ urlpatterns += [
+ path("", include("pluggable_apps.subscription.urls")),
+ ]
+except ImportError:
+ pass
diff --git a/backend/connector_auth_v2/migrations/0001_initial.py b/backend/connector_auth_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..26f9a0969
--- /dev/null
+++ b/backend/connector_auth_v2/migrations/0001_initial.py
@@ -0,0 +1,60 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+import social_django.storage
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="ConnectorAuth",
+ fields=[
+ ("provider", models.CharField(max_length=32)),
+ ("uid", models.CharField(db_index=True, max_length=255)),
+ ("extra_data", models.JSONField(default=dict)),
+ ("created", models.DateTimeField(auto_now_add=True)),
+ ("modified", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "user",
+ models.ForeignKey(
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="connector_auths",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Connector Auth",
+ "verbose_name_plural": "Connector Auths",
+ "db_table": "connector_auth",
+ },
+ bases=(models.Model, social_django.storage.DjangoUserMixin),
+ ),
+ migrations.AddConstraint(
+ model_name="connectorauth",
+ constraint=models.UniqueConstraint(
+ fields=("provider", "uid"), name="unique_provider_uid_index"
+ ),
+ ),
+ ]
diff --git a/backend/connector_auth_v2/migrations/__init__.py b/backend/connector_auth_v2/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/connector_auth_v2/models.py b/backend/connector_auth_v2/models.py
index ca8caaefd..762f6605f 100644
--- a/backend/connector_auth_v2/models.py
+++ b/backend/connector_auth_v2/models.py
@@ -125,7 +125,7 @@ class ConnectorAuth(AbstractUserSocialAuth):
app_label = "connector_auth_v2"
verbose_name = "Connector Auth"
verbose_name_plural = "Connector Auths"
- db_table = "connector_auth_v2"
+ db_table = "connector_auth"
constraints = [
models.UniqueConstraint(
fields=[
diff --git a/backend/connector_v2/migrations/0001_initial.py b/backend/connector_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..91d2d043e
--- /dev/null
+++ b/backend/connector_v2/migrations/0001_initial.py
@@ -0,0 +1,117 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ("connector_auth_v2", "0001_initial"),
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ("workflow_v2", "0001_initial"),
+ ("account_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="ConnectorInstance",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ ("connector_name", models.TextField(max_length=128)),
+ ("connector_id", models.CharField(default="", max_length=128)),
+ ("connector_metadata", models.BinaryField(null=True)),
+ ("connector_version", models.CharField(default="", max_length=64)),
+ (
+ "connector_type",
+ models.CharField(
+ choices=[("INPUT", "Input"), ("OUTPUT", "Output")]
+ ),
+ ),
+ (
+ "connector_mode",
+ models.CharField(
+ choices=[(0, "UNKNOWN"), (1, "FILE_SYSTEM"), (2, "DATABASE")],
+ db_comment="0: UNKNOWN, 1: FILE_SYSTEM, 2: DATABASE",
+ default=0,
+ ),
+ ),
+ (
+ "connector_auth",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="connector_instances",
+ to="connector_auth_v2.connectorauth",
+ ),
+ ),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="connectors_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="connectors_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "organization",
+ models.ForeignKey(
+ blank=True,
+ db_comment="Foreign key reference to the Organization model.",
+ default=None,
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ to="account_v2.organization",
+ ),
+ ),
+ (
+ "workflow",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="connector_workflow",
+ to="workflow_v2.workflow",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Connector Instance",
+ "verbose_name_plural": "Connector Instances",
+ "db_table": "connector_instance",
+ },
+ ),
+ migrations.AddConstraint(
+ model_name="connectorinstance",
+ constraint=models.UniqueConstraint(
+ fields=("connector_name", "workflow", "connector_type"),
+ name="unique_workflow_connector",
+ ),
+ ),
+ ]
diff --git a/backend/connector_v2/models.py b/backend/connector_v2/models.py
index 3fd535b05..f8db9915a 100644
--- a/backend/connector_v2/models.py
+++ b/backend/connector_v2/models.py
@@ -111,17 +111,42 @@ class ConnectorInstance(DefaultOrganizationMixin, BaseModel):
f" workflow: {self.workflow})"
)
+ def get_connector_metadata_bytes(self):
+ """Convert connector_metadata to bytes if it is a memoryview.
+
+ Returns:
+ bytes: The connector_metadata as bytes.
+ """
+ if isinstance(self.connector_metadata, memoryview):
+ return self.connector_metadata.tobytes()
+ return self.connector_metadata
+
@property
def metadata(self) -> Any:
+ """Decrypt and return the connector metadata as a dictionary.
+
+ This property handles the decryption of the connector_metadata,
+ converting it to bytes if necessary, and then loading the decrypted
+ JSON string into a dictionary.
+
+ Returns:
+ dict: The decrypted connector metadata.
+ """
+
+ connector_metadata_bytes = self.get_connector_metadata_bytes()
+
+ if connector_metadata_bytes is None:
+ return None
+
+ if isinstance(connector_metadata_bytes, (dict)):
+ return connector_metadata_bytes
encryption_secret: str = settings.ENCRYPTION_KEY
cipher_suite: Fernet = Fernet(encryption_secret.encode("utf-8"))
- decrypted_value = cipher_suite.decrypt(
- bytes(self.connector_metadata).decode("utf-8")
- )
- return json.loads(decrypted_value)
+ decrypted_value = cipher_suite.decrypt(connector_metadata_bytes)
+ return json.loads(decrypted_value.decode("utf-8"))
class Meta:
- db_table = "connector_instance_v2"
+ db_table = "connector_instance"
verbose_name = "Connector Instance"
verbose_name_plural = "Connector Instances"
constraints = [
diff --git a/backend/connector_v2/serializers.py b/backend/connector_v2/serializers.py
index cd9727e84..4704635ba 100644
--- a/backend/connector_v2/serializers.py
+++ b/backend/connector_v2/serializers.py
@@ -71,8 +71,6 @@ class ConnectorInstanceSerializer(AuditSerializer):
encryption_secret: str = settings.ENCRYPTION_KEY
f: Fernet = Fernet(encryption_secret.encode("utf-8"))
json_string: str = json.dumps(kwargs.get(CIKey.CONNECTOR_METADATA))
- if self.validated_data:
- self.validated_data.pop(CIKey.CONNECTOR_METADATA)
kwargs[CIKey.CONNECTOR_METADATA] = f.encrypt(json_string.encode("utf-8"))
diff --git a/backend/connector_v2/views.py b/backend/connector_v2/views.py
index 3f098d355..4431bfcfa 100644
--- a/backend/connector_v2/views.py
+++ b/backend/connector_v2/views.py
@@ -24,7 +24,6 @@ logger = logging.getLogger(__name__)
class ConnectorInstanceViewSet(viewsets.ModelViewSet):
versioning_class = URLPathVersioning
- queryset = ConnectorInstance.objects.all()
serializer_class = ConnectorInstanceSerializer
def get_queryset(self) -> Optional[QuerySet]:
diff --git a/backend/middleware/organization_middleware.py b/backend/middleware/organization_middleware.py
new file mode 100644
index 000000000..a2b9450ce
--- /dev/null
+++ b/backend/middleware/organization_middleware.py
@@ -0,0 +1,20 @@
+import re
+
+from django.conf import settings
+from django.utils.deprecation import MiddlewareMixin
+
+
+class OrganizationMiddleware(MiddlewareMixin):
+ def process_request(self, request):
+ tenant_prefix = settings.TENANT_SUBFOLDER_PREFIX.rstrip("/") + "/"
+ pattern = rf"^{tenant_prefix}(?P<org_id>[^/]+)/"
+
+ # Check if the URL matches the pattern with organization ID
+ match = re.match(pattern, request.path)
+ if match:
+ org_id = match.group("org_id")
+ request.organization_id = org_id
+ new_path = re.sub(pattern, "/" + tenant_prefix, request.path_info)
+ request.path_info = new_path
+ else:
+ request.organization_id = None
diff --git a/backend/permissions/permission.py b/backend/permissions/permission.py
index e568ece7d..98bda6a17 100644
--- a/backend/permissions/permission.py
+++ b/backend/permissions/permission.py
@@ -1,10 +1,17 @@
from typing import Any
-from adapter_processor.models import AdapterInstance
from rest_framework import permissions
from rest_framework.request import Request
from rest_framework.views import APIView
+from backend.constants import FeatureFlag
+from unstract.flags.feature_flag import check_feature_flag_status
+
+if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
+ from adapter_processor_v2.models import AdapterInstance
+else:
+ from adapter_processor.models import AdapterInstance
+
class IsOwner(permissions.BasePermission):
"""Custom permission to only allow owners of an object."""
diff --git a/backend/pipeline_v2/apps.py b/backend/pipeline_v2/apps.py
new file mode 100644
index 000000000..dd81e66c7
--- /dev/null
+++ b/backend/pipeline_v2/apps.py
@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class PipelineConfig(AppConfig):
+ name = "pipeline_v2"
diff --git a/backend/pipeline_v2/constants.py b/backend/pipeline_v2/constants.py
index 2475ad79f..438982577 100644
--- a/backend/pipeline_v2/constants.py
+++ b/backend/pipeline_v2/constants.py
@@ -63,4 +63,5 @@ class PipelineURL:
DETAIL = "pipeline-detail"
EXECUTIONS = "pipeline-executions"
LIST = "pipeline-list"
- EXECUTE = "tenant:pipeline-execute"
+ EXECUTE = "pipeline-execute"
+ EXECUTE_NAMESPACE = "tenant:pipeline-execute"
diff --git a/backend/pipeline_v2/manager.py b/backend/pipeline_v2/manager.py
index d5297107d..8e17477f7 100644
--- a/backend/pipeline_v2/manager.py
+++ b/backend/pipeline_v2/manager.py
@@ -48,7 +48,9 @@ class PipelineManager:
) -> Optional[dict[str, Any]]:
"""Gets the required data to be passed while executing a pipeline Any
changes to pipeline execution needs to be propagated here."""
- callback_url = settings.DJANGO_APP_BACKEND_URL + reverse(PipelineURL.EXECUTE)
+ callback_url = settings.DJANGO_APP_BACKEND_URL + reverse(
+ PipelineURL.EXECUTE_NAMESPACE
+ )
job_headers = {RequestHeader.X_API_KEY: settings.INTERNAL_SERVICE_API_KEY}
job_kwargs = {
RequestConstants.VERB: "POST",
diff --git a/backend/pipeline_v2/migrations/0001_initial.py b/backend/pipeline_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..47c8eebbb
--- /dev/null
+++ b/backend/pipeline_v2/migrations/0001_initial.py
@@ -0,0 +1,144 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ("account_v2", "0001_initial"),
+ ("workflow_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="Pipeline",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ ("pipeline_name", models.CharField(default="", max_length=32)),
+ ("app_id", models.TextField(blank=True, max_length=32, null=True)),
+ ("active", models.BooleanField(default=False)),
+ ("scheduled", models.BooleanField(default=False)),
+ (
+ "cron_string",
+ models.TextField(db_comment="UNIX cron string", max_length=256),
+ ),
+ (
+ "pipeline_type",
+ models.CharField(
+ choices=[
+ ("ETL", "ETL"),
+ ("TASK", "TASK"),
+ ("DEFAULT", "Default"),
+ ("APP", "App"),
+ ],
+ default="DEFAULT",
+ ),
+ ),
+ ("run_count", models.IntegerField(default=0)),
+ ("last_run_time", models.DateTimeField(blank=True, null=True)),
+ (
+ "last_run_status",
+ models.CharField(
+ choices=[
+ ("SUCCESS", "Success"),
+ ("FAILURE", "Failure"),
+ ("INPROGRESS", "Inprogress"),
+ ("YET_TO_START", "Yet to start"),
+ ("RESTARTING", "Restarting"),
+ ("PAUSED", "Paused"),
+ ],
+ default="YET_TO_START",
+ ),
+ ),
+ (
+ "app_icon",
+ models.URLField(
+ blank=True,
+ db_comment="Field to store icon url for Apps",
+ null=True,
+ ),
+ ),
+ (
+ "app_url",
+ models.URLField(
+ blank=True, db_comment="Stores deployed URL for App", null=True
+ ),
+ ),
+ ("access_control_bundle_id", models.TextField(blank=True, null=True)),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="pipelines_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="pipelines_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "organization",
+ models.ForeignKey(
+ blank=True,
+ db_comment="Foreign key reference to the Organization model.",
+ default=None,
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ to="account_v2.organization",
+ ),
+ ),
+ (
+ "workflow",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="pipelines",
+ to="workflow_v2.workflow",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Pipeline",
+ "verbose_name_plural": "Pipelines",
+ "db_table": "pipeline",
+ },
+ ),
+ migrations.AddConstraint(
+ model_name="pipeline",
+ constraint=models.UniqueConstraint(
+ fields=("id", "pipeline_type"), name="unique_pipeline_entity"
+ ),
+ ),
+ migrations.AddConstraint(
+ model_name="pipeline",
+ constraint=models.UniqueConstraint(
+ fields=("pipeline_name", "organization"), name="unique_pipeline_name"
+ ),
+ ),
+ ]
diff --git a/backend/pipeline_v2/migrations/__init__.py b/backend/pipeline_v2/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/pipeline_v2/models.py b/backend/pipeline_v2/models.py
index 35ef1914c..ccb90313a 100644
--- a/backend/pipeline_v2/models.py
+++ b/backend/pipeline_v2/models.py
@@ -95,7 +95,7 @@ class Pipeline(DefaultOrganizationMixin, BaseModel):
class Meta:
verbose_name = "Pipeline"
verbose_name_plural = "Pipelines"
- db_table = "pipeline_v2"
+ db_table = "pipeline"
constraints = [
models.UniqueConstraint(
fields=["id", "pipeline_type"],
diff --git a/backend/prompt_studio/prompt_profile_manager_v2/migrations/0001_initial.py b/backend/prompt_studio/prompt_profile_manager_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..01a98c45a
--- /dev/null
+++ b/backend/prompt_studio/prompt_profile_manager_v2/migrations/0001_initial.py
@@ -0,0 +1,159 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ("prompt_studio_core_v2", "0001_initial"),
+ ("adapter_processor_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="ProfileManager",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "profile_id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ ("profile_name", models.TextField()),
+ ("chunk_size", models.IntegerField(blank=True, null=True)),
+ ("chunk_overlap", models.IntegerField(blank=True, null=True)),
+ ("reindex", models.BooleanField(default=False)),
+ (
+ "retrieval_strategy",
+ models.TextField(
+ blank=True,
+ choices=[
+ ("simple", "Simple retrieval"),
+ ("subquestion", "Subquestion retrieval"),
+ ],
+ db_comment="Field to store the retrieval strategy for prompts",
+ ),
+ ),
+ (
+ "similarity_top_k",
+ models.IntegerField(
+ blank=True,
+ db_comment="Field to store number of top embeddings to take into context",
+ null=True,
+ ),
+ ),
+ (
+ "section",
+ models.TextField(
+ blank=True,
+ db_comment="Field to store limit to section",
+ null=True,
+ ),
+ ),
+ (
+ "is_default",
+ models.BooleanField(
+ db_comment="Default LLM Profile used in prompt", default=False
+ ),
+ ),
+ (
+ "is_summarize_llm",
+ models.BooleanField(
+ db_comment="Default LLM Profile used for summarizing",
+ default=False,
+ ),
+ ),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="profile_managers_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "embedding_model",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.PROTECT,
+ related_name="profiles_embedding_model",
+ to="adapter_processor_v2.adapterinstance",
+ ),
+ ),
+ (
+ "llm",
+ models.ForeignKey(
+ db_comment="Field to store the LLM chosen by the user",
+ on_delete=django.db.models.deletion.PROTECT,
+ related_name="profiles_llm",
+ to="adapter_processor_v2.adapterinstance",
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="profile_managers_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "prompt_studio_tool",
+ models.ForeignKey(
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="profile_managers",
+ to="prompt_studio_core_v2.customtool",
+ ),
+ ),
+ (
+ "vector_store",
+ models.ForeignKey(
+ db_comment="Field to store the chosen vector store.",
+ on_delete=django.db.models.deletion.PROTECT,
+ related_name="profiles_vector_store",
+ to="adapter_processor_v2.adapterinstance",
+ ),
+ ),
+ (
+ "x2text",
+ models.ForeignKey(
+ db_comment="Field to store the X2Text Adapter chosen by the user",
+ on_delete=django.db.models.deletion.PROTECT,
+ related_name="profiles_x2text",
+ to="adapter_processor_v2.adapterinstance",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Profile Manager",
+ "verbose_name_plural": "Profile Managers",
+ "db_table": "profile_manager",
+ },
+ ),
+ migrations.AddConstraint(
+ model_name="profilemanager",
+ constraint=models.UniqueConstraint(
+ fields=("prompt_studio_tool", "profile_name"),
+ name="unique_prompt_studio_tool_profile_name_index",
+ ),
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_profile_manager_v2/migrations/__init__.py b/backend/prompt_studio/prompt_profile_manager_v2/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/prompt_studio/prompt_profile_manager_v2/models.py b/backend/prompt_studio/prompt_profile_manager_v2/models.py
index 303e96f0f..be73899c0 100644
--- a/backend/prompt_studio/prompt_profile_manager_v2/models.py
+++ b/backend/prompt_studio/prompt_profile_manager_v2/models.py
@@ -97,7 +97,7 @@ class ProfileManager(BaseModel):
class Meta:
verbose_name = "Profile Manager"
verbose_name_plural = "Profile Managers"
- db_table = "profile_manager_v2"
+ db_table = "profile_manager"
constraints = [
models.UniqueConstraint(
fields=["prompt_studio_tool", "profile_name"],
diff --git a/backend/prompt_studio/prompt_studio_core_v2/constants.py b/backend/prompt_studio/prompt_studio_core_v2/constants.py
index 55d61e32e..213fc066f 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/constants.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/constants.py
@@ -88,6 +88,7 @@ class ToolStudioPromptKeys:
PROFILE_MANAGER_ID = "profile_manager"
CONTEXT = "context"
METADATA = "metadata"
+ INCLUDE_METADATA = "include_metadata"
class FileViewTypes:
diff --git a/backend/prompt_studio/prompt_studio_core_v2/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_core_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..7719320ca
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_core_v2/migrations/0001_initial.py
@@ -0,0 +1,207 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ("adapter_processor_v2", "0001_initial"),
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ("account_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="CustomTool",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "tool_id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ ("tool_name", models.TextField()),
+ ("description", models.TextField()),
+ (
+ "author",
+ models.TextField(
+ db_comment="Specific to the user who created the tool."
+ ),
+ ),
+ (
+ "icon",
+ models.TextField(
+ blank=True,
+ db_comment="Field to store icon url for the custom tool.",
+ ),
+ ),
+ (
+ "output",
+ models.TextField(
+ blank=True, db_comment="Field to store the output format type."
+ ),
+ ),
+ (
+ "log_id",
+ models.UUIDField(
+ db_comment="Field to store unique log_id for polling",
+ default=uuid.uuid4,
+ ),
+ ),
+ (
+ "summarize_context",
+ models.BooleanField(
+ db_comment="Flag to summarize content", default=False
+ ),
+ ),
+ (
+ "summarize_as_source",
+ models.BooleanField(
+ db_comment="Flag to use summarized content as source",
+ default=False,
+ ),
+ ),
+ (
+ "summarize_prompt",
+ models.TextField(
+ blank=True, db_comment="Field to store the summarize prompt"
+ ),
+ ),
+ (
+ "preamble",
+ models.TextField(
+ blank=True,
+ db_comment="Preamble to the prompts",
+ default="Your ability to extract and summarize this context accurately is essential for effective analysis. Pay close attention to the context's language, structure, and any cross-references to ensure a comprehensive and precise extraction of information. Do not use prior knowledge or information from outside the context to answer the questions. Only use the information provided in the context to answer the questions.",
+ ),
+ ),
+ (
+ "postamble",
+ models.TextField(
+ blank=True,
+ db_comment="Appended as postable to prompts.",
+ default="Do not include any explanation in the reply. Only include the extracted information in the reply.",
+ ),
+ ),
+ (
+ "prompt_grammer",
+ models.JSONField(
+ blank=True,
+ db_comment="Synonymous words used in prompt",
+ null=True,
+ ),
+ ),
+ (
+ "exclude_failed",
+ models.BooleanField(
+ db_comment="Flag to make the answer null if it is incorrect",
+ default=True,
+ ),
+ ),
+ (
+ "single_pass_extraction_mode",
+ models.BooleanField(
+ db_comment="Flag to enable or disable single pass extraction mode",
+ default=False,
+ ),
+ ),
+ (
+ "enable_challenge",
+ models.BooleanField(
+ db_comment="Flag to enable or disable challenge", default=False
+ ),
+ ),
+ (
+ "enable_highlight",
+ models.BooleanField(
+ db_comment="Flag to enable or disable document highlighting",
+ default=False,
+ ),
+ ),
+ (
+ "challenge_llm",
+ models.ForeignKey(
+ blank=True,
+ db_comment="Field to store challenge llm",
+ null=True,
+ on_delete=django.db.models.deletion.PROTECT,
+ related_name="custom_tools_challenge",
+ to="adapter_processor_v2.adapterinstance",
+ ),
+ ),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="custom_tools_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="custom_tools_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "monitor_llm",
+ models.ForeignKey(
+ blank=True,
+ db_comment="Field to store monitor llm",
+ null=True,
+ on_delete=django.db.models.deletion.PROTECT,
+ related_name="custom_tools_monitor",
+ to="adapter_processor_v2.adapterinstance",
+ ),
+ ),
+ (
+ "organization",
+ models.ForeignKey(
+ blank=True,
+ db_comment="Foreign key reference to the Organization model.",
+ default=None,
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ to="account_v2.organization",
+ ),
+ ),
+ (
+ "shared_users",
+ models.ManyToManyField(
+ related_name="shared_custom_tools", to=settings.AUTH_USER_MODEL
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Custom Tool",
+ "verbose_name_plural": "Custom Tools",
+ "db_table": "custom_tool",
+ },
+ ),
+ migrations.AddConstraint(
+ model_name="customtool",
+ constraint=models.UniqueConstraint(
+ fields=("tool_name", "organization"), name="unique_tool_name"
+ ),
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_studio_core_v2/migrations/__init__.py b/backend/prompt_studio/prompt_studio_core_v2/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/prompt_studio/prompt_studio_core_v2/models.py b/backend/prompt_studio/prompt_studio_core_v2/models.py
index f4784e47d..87abd4766 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/models.py
@@ -121,6 +121,9 @@ class CustomTool(DefaultOrganizationMixin, BaseModel):
enable_challenge = models.BooleanField(
db_comment="Flag to enable or disable challenge", default=False
)
+ enable_highlight = models.BooleanField(
+ db_comment="Flag to enable or disable document highlighting", default=False
+ )
# Introduced field to establish M2M relation between users and custom_tool.
# This will introduce intermediary table which relates both the models.
@@ -149,7 +152,7 @@ class CustomTool(DefaultOrganizationMixin, BaseModel):
class Meta:
verbose_name = "Custom Tool"
verbose_name_plural = "Custom Tools"
- db_table = "custom_tool_v2"
+ db_table = "custom_tool"
constraints = [
models.UniqueConstraint(
fields=["tool_name", "organization"],
diff --git a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
index 74a157e9b..82ba23fe7 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
@@ -500,7 +500,11 @@ class PromptStudioHelper:
doc_path, tool_id, org_id, user_id, document_id, run_id
):
prompts = PromptStudioHelper.fetch_prompt_from_tool(tool_id)
- prompts = [prompt for prompt in prompts if prompt.prompt_type != TSPKeys.NOTES]
+ prompts = [
+ prompt
+ for prompt in prompts
+ if prompt.prompt_type != TSPKeys.NOTES and prompt.active
+ ]
if not prompts:
logger.error(f"[{tool_id or 'NA'}] No prompts found for id: {id}")
raise NoPromptsFound()
@@ -733,8 +737,9 @@ class PromptStudioHelper:
prompt_host=settings.PROMPT_HOST,
prompt_port=settings.PROMPT_PORT,
)
+ include_metadata = {TSPKeys.INCLUDE_METADATA: True}
- answer = responder.answer_prompt(payload)
+ answer = responder.answer_prompt(payload, include_metadata)
# TODO: Make use of dataclasses
if answer["status"] == "ERROR":
# TODO: Publish to FE logs from here
@@ -849,6 +854,7 @@ class PromptStudioHelper:
)
return {"status": IndexingStatus.COMPLETED_STATUS.value, "output": doc_id}
except (IndexingError, IndexingAPIError, SdkError) as e:
+ logger.error(f"Indexing failed : {e} ", stack_info=True, exc_info=True)
doc_name = os.path.split(file_path)[1]
PromptStudioHelper._publish_log(
{"tool_id": tool_id, "run_id": run_id, "doc_name": doc_name},
@@ -959,8 +965,9 @@ class PromptStudioHelper:
prompt_host=settings.PROMPT_HOST,
prompt_port=settings.PROMPT_PORT,
)
+ include_metadata = {TSPKeys.INCLUDE_METADATA: True}
- answer = responder.single_pass_extraction(payload)
+ answer = responder.single_pass_extraction(payload, include_metadata)
# TODO: Make use of dataclasses
if answer["status"] == "ERROR":
error_message = answer.get("error", None)
@@ -969,3 +976,11 @@ class PromptStudioHelper:
)
output_response = json.loads(answer["structure_output"])
return output_response
+
+ @staticmethod
+ def get_tool_from_tool_id(tool_id: str) -> Optional[CustomTool]:
+ try:
+ tool: CustomTool = CustomTool.objects.get(tool_id=tool_id)
+ return tool
+ except CustomTool.DoesNotExist:
+ return None
diff --git a/backend/prompt_studio/prompt_studio_core_v2/views.py b/backend/prompt_studio/prompt_studio_core_v2/views.py
index e2b1e1d2a..dcdd22a22 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/views.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/views.py
@@ -46,7 +46,7 @@ from rest_framework.decorators import action
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.versioning import URLPathVersioning
-from tool_instance.models import ToolInstance
+from tool_instance_v2.models import ToolInstance
from unstract.sdk.utils.common_utils import CommonUtils
from utils.user_session import UserSessionUtils
diff --git a/backend/prompt_studio/prompt_studio_document_manager_v2/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_document_manager_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..6df0ccc49
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_document_manager_v2/migrations/0001_initial.py
@@ -0,0 +1,83 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ("prompt_studio_core_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="DocumentManager",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "document_id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "document_name",
+ models.CharField(
+ db_comment="Field to store the document name", editable=False
+ ),
+ ),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="document_managers_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="document_managers_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "tool",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="document_managers",
+ to="prompt_studio_core_v2.customtool",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Document Manager",
+ "verbose_name_plural": "Document Managers",
+ "db_table": "document_manager",
+ },
+ ),
+ migrations.AddConstraint(
+ model_name="documentmanager",
+ constraint=models.UniqueConstraint(
+ fields=("document_name", "tool"), name="unique_document_name_tool_index"
+ ),
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_studio_document_manager_v2/migrations/__init__.py b/backend/prompt_studio/prompt_studio_document_manager_v2/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/prompt_studio/prompt_studio_document_manager_v2/models.py b/backend/prompt_studio/prompt_studio_document_manager_v2/models.py
index 139d67e31..d29978c16 100644
--- a/backend/prompt_studio/prompt_studio_document_manager_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_document_manager_v2/models.py
@@ -47,7 +47,7 @@ class DocumentManager(BaseModel):
class Meta:
verbose_name = "Document Manager"
verbose_name_plural = "Document Managers"
- db_table = "document_manager_v2"
+ db_table = "document_manager"
constraints = [
models.UniqueConstraint(
fields=["document_name", "tool"],
diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..f587ef667
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/0001_initial.py
@@ -0,0 +1,113 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ("prompt_studio_document_manager_v2", "0001_initial"),
+ ("prompt_profile_manager_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="IndexManager",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "index_manager_id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "raw_index_id",
+ models.CharField(
+ blank=True,
+ db_comment="Field to store the raw index id",
+ editable=False,
+ null=True,
+ ),
+ ),
+ (
+ "summarize_index_id",
+ models.CharField(
+ blank=True,
+ db_comment="Field to store the summarize index id",
+ editable=False,
+ null=True,
+ ),
+ ),
+ (
+ "index_ids_history",
+ models.JSONField(db_comment="List of index ids", default=list),
+ ),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="index_managers_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "document_manager",
+ models.ForeignKey(
+ editable=False,
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="index_managers",
+ to="prompt_studio_document_manager_v2.documentmanager",
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="index_managers_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "profile_manager",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="index_managers",
+ to="prompt_profile_manager_v2.profilemanager",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Index Manager",
+ "verbose_name_plural": "Index Managers",
+ "db_table": "index_manager",
+ },
+ ),
+ migrations.AddConstraint(
+ model_name="indexmanager",
+ constraint=models.UniqueConstraint(
+ fields=("document_manager", "profile_manager"),
+ name="unique_document_manager_profile_manager_index",
+ ),
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/__init__.py b/backend/prompt_studio/prompt_studio_index_manager_v2/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/models.py b/backend/prompt_studio/prompt_studio_index_manager_v2/models.py
index c160f8561..e58ff87b0 100644
--- a/backend/prompt_studio/prompt_studio_index_manager_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_index_manager_v2/models.py
@@ -84,7 +84,7 @@ class IndexManager(BaseModel):
class Meta:
verbose_name = "Index Manager"
verbose_name_plural = "Index Managers"
- db_table = "index_manager_v2"
+ db_table = "index_manager"
constraints = [
models.UniqueConstraint(
fields=["document_manager", "profile_manager"],
diff --git a/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py b/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py
index 084b2c458..84747d52b 100644
--- a/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py
+++ b/backend/prompt_studio/prompt_studio_index_manager_v2/prompt_studio_index_helper.py
@@ -49,7 +49,10 @@ class PromptStudioIndexHelper:
)
index_ids = index_manager.index_ids_history
- index_ids_list = json.loads(index_ids) if index_ids else []
+ if not isinstance(index_ids, list):
+ index_ids_list = json.loads(index_ids) if index_ids else []
+ else:
+ index_ids_list = index_ids
if doc_id not in index_ids:
index_ids_list.append(doc_id)
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/constants.py b/backend/prompt_studio/prompt_studio_output_manager_v2/constants.py
index a9c046aae..1cee0b394 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/constants.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/constants.py
@@ -5,3 +5,8 @@ class PromptStudioOutputManagerKeys:
DOCUMENT_MANAGER = "document_manager"
IS_SINGLE_PASS_EXTRACT = "is_single_pass_extract"
NOTES = "NOTES"
+
+
+class PromptOutputManagerErrorMessage:
+ TOOL_VALIDATION = "tool_id parameter is required"
+ TOOL_NOT_FOUND = "Tool not found"
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..825ad27c0
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/0001_initial.py
@@ -0,0 +1,139 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ("prompt_studio_v2", "0001_initial"),
+ ("prompt_profile_manager_v2", "0001_initial"),
+ ("prompt_studio_core_v2", "0001_initial"),
+ ("prompt_studio_document_manager_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="PromptStudioOutputManager",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "prompt_output_id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "output",
+ models.CharField(
+ blank=True, db_comment="Field to store output", null=True
+ ),
+ ),
+ (
+ "context",
+ models.TextField(
+ blank=True, db_comment="Field to store chunks used", null=True
+ ),
+ ),
+ (
+ "eval_metrics",
+ models.JSONField(
+ db_column="eval_metrics",
+ db_comment="Field to store the evaluation metrics",
+ default=list,
+ ),
+ ),
+ (
+ "is_single_pass_extract",
+ models.BooleanField(
+ db_comment="Is the single pass extraction mode active",
+ default=False,
+ ),
+ ),
+ ("run_id", models.UUIDField(default=uuid.uuid4, editable=False)),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="prompt_studio_outputs_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "document_manager",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="prompt_studio_outputs",
+ to="prompt_studio_document_manager_v2.documentmanager",
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="prompt_studio_outputs_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "profile_manager",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="prompt_studio_outputs",
+ to="prompt_profile_manager_v2.profilemanager",
+ ),
+ ),
+ (
+ "prompt_id",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="prompt_studio_outputs",
+ to="prompt_studio_v2.toolstudioprompt",
+ ),
+ ),
+ (
+ "tool_id",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="prompt_studio_outputs",
+ to="prompt_studio_core_v2.customtool",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Prompt Studio Output Manager",
+ "verbose_name_plural": "Prompt Studio Output Managers",
+ "db_table": "prompt_studio_output_manager",
+ },
+ ),
+ migrations.AddConstraint(
+ model_name="promptstudiooutputmanager",
+ constraint=models.UniqueConstraint(
+ fields=(
+ "prompt_id",
+ "document_manager",
+ "profile_manager",
+ "tool_id",
+ "is_single_pass_extract",
+ ),
+ name="unique_prompt_output_index",
+ ),
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/__init__.py b/backend/prompt_studio/prompt_studio_output_manager_v2/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/models.py b/backend/prompt_studio/prompt_studio_output_manager_v2/models.py
index afc133f17..0a9cd8bc7 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/models.py
@@ -76,7 +76,7 @@ class PromptStudioOutputManager(BaseModel):
class Meta:
verbose_name = "Prompt Studio Output Manager"
verbose_name_plural = "Prompt Studio Output Managers"
- db_table = "prompt_studio_output_manager_v2"
+ db_table = "prompt_studio_output_manager"
constraints = [
models.UniqueConstraint(
fields=[
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/urls.py b/backend/prompt_studio/prompt_studio_output_manager_v2/urls.py
index 45af0ccee..61ec8540f 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/urls.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/urls.py
@@ -4,9 +4,17 @@ from rest_framework.urlpatterns import format_suffix_patterns
from .views import PromptStudioOutputView
prompt_doc_list = PromptStudioOutputView.as_view({"get": "list"})
+get_output_for_tool_default = PromptStudioOutputView.as_view(
+ {"get": "get_output_for_tool_default"}
+)
urlpatterns = format_suffix_patterns(
[
path("prompt-output/", prompt_doc_list, name="prompt-doc-list"),
+ path(
+ "prompt-output/prompt-default-profile/",
+ get_output_for_tool_default,
+ name="prompt-default-profile-outputs",
+ ),
]
)
diff --git a/backend/prompt_studio/prompt_studio_output_manager_v2/views.py b/backend/prompt_studio/prompt_studio_output_manager_v2/views.py
index e4dd99b9f..441100d90 100644
--- a/backend/prompt_studio/prompt_studio_output_manager_v2/views.py
+++ b/backend/prompt_studio/prompt_studio_output_manager_v2/views.py
@@ -1,14 +1,20 @@
import logging
-from typing import Optional
+from typing import Any, Optional
+from django.core.exceptions import ObjectDoesNotExist
from django.db.models import QuerySet
+from django.http import HttpRequest
from prompt_studio.prompt_studio_output_manager_v2.constants import (
+ PromptOutputManagerErrorMessage,
PromptStudioOutputManagerKeys,
)
from prompt_studio.prompt_studio_output_manager_v2.serializers import (
PromptStudioOutputSerializer,
)
-from rest_framework import viewsets
+from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt
+from rest_framework import status, viewsets
+from rest_framework.exceptions import APIException
+from rest_framework.response import Response
from rest_framework.versioning import URLPathVersioning
from utils.common_utils import CommonUtils
from utils.filtering import FilterHelper
@@ -49,3 +55,51 @@ class PromptStudioOutputView(viewsets.ModelViewSet):
queryset = PromptStudioOutputManager.objects.filter(**filter_args)
return queryset
+
+ def get_output_for_tool_default(self, request: HttpRequest) -> Response:
+ # TODO: Setup Serializer here
+ # Get the tool_id from request parameters
+ tool_id = request.GET.get("tool_id")
+ document_manager_id = request.GET.get("document_manager")
+ tool_validation_message = PromptOutputManagerErrorMessage.TOOL_VALIDATION
+ tool_not_found = PromptOutputManagerErrorMessage.TOOL_NOT_FOUND
+ if not tool_id:
+ raise APIException(detail=tool_validation_message, code=400)
+
+ try:
+ # Fetch ToolStudioPrompt records based on tool_id
+ tool_studio_prompts = ToolStudioPrompt.objects.filter(tool_id=tool_id)
+ except ObjectDoesNotExist:
+ raise APIException(detail=tool_not_found, code=400)
+
+ # Initialize the result dictionary
+ result: dict[str, Any] = {}
+
+ # Iterate over ToolStudioPrompt records
+ for tool_prompt in tool_studio_prompts:
+ prompt_id = str(tool_prompt.prompt_id)
+ profile_manager_id = str(tool_prompt.profile_manager.profile_id)
+
+ # If profile_manager is not set, skip this record
+ if not profile_manager_id:
+ result[tool_prompt.prompt_key] = ""
+ continue
+
+ try:
+ queryset = PromptStudioOutputManager.objects.filter(
+ prompt_id=prompt_id,
+ profile_manager=profile_manager_id,
+ is_single_pass_extract=False,
+ document_manager_id=document_manager_id,
+ )
+
+ if not queryset.exists():
+ result[tool_prompt.prompt_key] = ""
+ continue
+
+ for output in queryset:
+ result[tool_prompt.prompt_key] = output.output
+ except ObjectDoesNotExist:
+ result[tool_prompt.prompt_key] = ""
+
+ return Response(result, status=status.HTTP_200_OK)
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_registry_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..393fbd21f
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_registry_v2/migrations/0001_initial.py
@@ -0,0 +1,132 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+import prompt_studio.prompt_studio_registry_v2.fields
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ("prompt_studio_core_v2", "0001_initial"),
+ ("account_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="PromptStudioRegistry",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "prompt_registry_id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ ("name", models.CharField(default="", editable=False)),
+ ("description", models.CharField(default="", editable=False)),
+ (
+ "tool_property",
+ prompt_studio.prompt_studio_registry_v2.fields.ToolPropertyJSONField(
+ db_column="tool_property",
+ db_comment="PROPERTIES of the tool",
+ default=dict,
+ ),
+ ),
+ (
+ "tool_spec",
+ prompt_studio.prompt_studio_registry_v2.fields.ToolSpecJSONField(
+ db_column="tool_spec",
+ db_comment="SPEC of the tool",
+ default=dict,
+ ),
+ ),
+ (
+ "tool_metadata",
+ prompt_studio.prompt_studio_registry_v2.fields.ToolMetadataJSONField(
+ db_column="tool_metadata",
+ db_comment="Metadata from Prompt Studio",
+ default=dict,
+ ),
+ ),
+ (
+ "icon",
+ models.CharField(
+ db_comment="Tool icon in svg format", editable=False
+ ),
+ ),
+ ("url", models.CharField(editable=False)),
+ (
+ "shared_to_org",
+ models.BooleanField(
+ db_comment="Is the exported tool shared with entire org",
+ default=False,
+ ),
+ ),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="prompt_registries_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "custom_tool",
+ models.OneToOneField(
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="prompt_studio_registries",
+ to="prompt_studio_core_v2.customtool",
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="prompt_registries_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "organization",
+ models.ForeignKey(
+ blank=True,
+ db_comment="Foreign key reference to the Organization model.",
+ default=None,
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ to="account_v2.organization",
+ ),
+ ),
+ (
+ "shared_users",
+ models.ManyToManyField(
+ related_name="prompt_registries", to=settings.AUTH_USER_MODEL
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Prompt Studio Registry",
+ "verbose_name_plural": "Prompt Studio Registries",
+ "db_table": "prompt_studio_registry",
+ },
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/migrations/__init__.py b/backend/prompt_studio/prompt_studio_registry_v2/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/models.py b/backend/prompt_studio/prompt_studio_registry_v2/models.py
index ec768a854..332ca34ea 100644
--- a/backend/prompt_studio/prompt_studio_registry_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_registry_v2/models.py
@@ -107,4 +107,4 @@ class PromptStudioRegistry(DefaultOrganizationMixin, BaseModel):
class Meta:
verbose_name = "Prompt Studio Registry"
verbose_name_plural = "Prompt Studio Registries"
- db_table = "prompt_studio_registry_v2"
+ db_table = "prompt_studio_registry"
diff --git a/backend/prompt_studio/prompt_studio_v2/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..117bc51c7
--- /dev/null
+++ b/backend/prompt_studio/prompt_studio_v2/migrations/0001_initial.py
@@ -0,0 +1,162 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ("prompt_studio_core_v2", "0001_initial"),
+ ("prompt_profile_manager_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="ToolStudioPrompt",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "prompt_id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "prompt_key",
+ models.TextField(db_comment="Field to store the prompt key"),
+ ),
+ (
+ "enforce_type",
+ models.TextField(
+ blank=True,
+ choices=[
+ ("Text", "Response sent as Text"),
+ ("number", "Response sent as number"),
+ ("email", "Response sent as email"),
+ ("date", "Response sent as date"),
+ ("boolean", "Response sent as boolean"),
+ ("json", "Response sent as json"),
+ ],
+ db_comment="Field to store the type in which the response to be returned.",
+ default="Text",
+ ),
+ ),
+ (
+ "prompt",
+ models.TextField(
+ blank=True, db_comment="Field to store the prompt"
+ ),
+ ),
+ ("sequence_number", models.IntegerField(blank=True, null=True)),
+ (
+ "prompt_type",
+ models.TextField(
+ blank=True,
+ choices=[
+ ("PROMPT", "Response sent as Text"),
+ ("NOTES", "Response sent as float"),
+ ],
+ db_comment="Field to store the type of the input prompt",
+ ),
+ ),
+ ("output", models.TextField(blank=True)),
+ (
+ "assert_prompt",
+ models.TextField(
+ blank=True,
+ db_comment="Field to store the asserted prompt",
+ null=True,
+ ),
+ ),
+ (
+ "assertion_failure_prompt",
+ models.TextField(
+ blank=True,
+ db_comment="Field to store the prompt key",
+ null=True,
+ ),
+ ),
+ ("is_assert", models.BooleanField(default=False)),
+ ("active", models.BooleanField(default=True)),
+ (
+ "output_metadata",
+ models.JSONField(
+ db_column="output_metadata",
+ db_comment="JSON adapter metadata for the FE to load the pagination",
+ default=dict,
+ ),
+ ),
+ ("evaluate", models.BooleanField(default=True)),
+ ("eval_quality_faithfulness", models.BooleanField(default=True)),
+ ("eval_quality_correctness", models.BooleanField(default=True)),
+ ("eval_quality_relevance", models.BooleanField(default=True)),
+ ("eval_security_pii", models.BooleanField(default=True)),
+ ("eval_guidance_toxicity", models.BooleanField(default=True)),
+ ("eval_guidance_completeness", models.BooleanField(default=True)),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="tool_studio_prompts_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ editable=False,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="tool_studio_prompts_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "profile_manager",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="tool_studio_prompts",
+ to="prompt_profile_manager_v2.profilemanager",
+ ),
+ ),
+ (
+ "tool_id",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="tool_studio_prompts",
+ to="prompt_studio_core_v2.customtool",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Tool Studio Prompt",
+ "verbose_name_plural": "Tool Studio Prompts",
+ "db_table": "tool_studio_prompt",
+ },
+ ),
+ migrations.AddConstraint(
+ model_name="toolstudioprompt",
+ constraint=models.UniqueConstraint(
+ fields=("prompt_key", "tool_id"), name="unique_prompt_key_tool_id_index"
+ ),
+ ),
+ ]
diff --git a/backend/prompt_studio/prompt_studio_v2/migrations/__init__.py b/backend/prompt_studio/prompt_studio_v2/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/prompt_studio/prompt_studio_v2/models.py b/backend/prompt_studio/prompt_studio_v2/models.py
index ba928f7c5..7e65d3c84 100644
--- a/backend/prompt_studio/prompt_studio_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_v2/models.py
@@ -118,7 +118,7 @@ class ToolStudioPrompt(BaseModel):
class Meta:
verbose_name = "Tool Studio Prompt"
verbose_name_plural = "Tool Studio Prompts"
- db_table = "tool_studio_prompt_v2"
+ db_table = "tool_studio_prompt"
constraints = [
models.UniqueConstraint(
fields=["prompt_key", "tool_id"],
diff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.py
index 762be6609..9415c3829 100644
--- a/backend/scheduler/tasks.py
+++ b/backend/scheduler/tasks.py
@@ -159,9 +159,7 @@ def execute_pipeline_task_v2(
except Exception as e:
logger.warning(f"Failed to disable task: {pipeline_id}. Error: {e}")
return
- workflow = WorkflowHelper.get_workflow_by_id(
- id=workflow_id, organization_id=organization_id
- )
+ workflow = WorkflowHelper.get_workflow_by_id(id=workflow_id)
logger.info(f"Executing workflow: {workflow}")
PipelineProcessor.update_pipeline(
pipeline_id, Pipeline.PipelineStatus.INPROGRESS
diff --git a/backend/tenant_account_v2/migrations/0001_initial.py b/backend/tenant_account_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..70c7ba135
--- /dev/null
+++ b/backend/tenant_account_v2/migrations/0001_initial.py
@@ -0,0 +1,70 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ("account_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="OrganizationMember",
+ fields=[
+ ("member_id", models.BigAutoField(primary_key=True, serialize=False)),
+ ("role", models.CharField()),
+ (
+ "is_login_onboarding_msg",
+ models.BooleanField(
+ db_comment="Flag to indicate whether the onboarding messages are shown",
+ default=True,
+ ),
+ ),
+ (
+ "is_prompt_studio_onboarding_msg",
+ models.BooleanField(
+ db_comment="Flag to indicate whether the prompt studio messages are shown",
+ default=True,
+ ),
+ ),
+ (
+ "organization",
+ models.ForeignKey(
+ blank=True,
+ db_comment="Foreign key reference to the Organization model.",
+ default=None,
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ to="account_v2.organization",
+ ),
+ ),
+ (
+ "user",
+ models.ForeignKey(
+ default=None,
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="organization_user",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Organization Member",
+ "verbose_name_plural": "Organization Members",
+ "db_table": "organization_member",
+ },
+ ),
+ migrations.AddConstraint(
+ model_name="organizationmember",
+ constraint=models.UniqueConstraint(
+ fields=("organization", "user"), name="unique_organization_member"
+ ),
+ ),
+ ]
diff --git a/backend/tenant_account_v2/migrations/__init__.py b/backend/tenant_account_v2/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/tenant_account_v2/models.py b/backend/tenant_account_v2/models.py
index c2c5b3cdc..e5fdacda6 100644
--- a/backend/tenant_account_v2/models.py
+++ b/backend/tenant_account_v2/models.py
@@ -31,13 +31,13 @@ class OrganizationMember(DefaultOrganizationMixin):
def __str__(self): # type: ignore
return (
f"OrganizationMember("
- f"{self.member_id}, role: {self.role}, userId: {self.user.user_id})"
+ f"{self.member_id}, role: {self.role}, user: {self.user})"
)
objects = OrganizationMemberModelManager()
class Meta:
- db_table = "organization_member_v2"
+ db_table = "organization_member"
verbose_name = "Organization Member"
verbose_name_plural = "Organization Members"
constraints = [
diff --git a/backend/tool_instance_v2/migrations/0001_initial.py b/backend/tool_instance_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..4e75c99e5
--- /dev/null
+++ b/backend/tool_instance_v2/migrations/0001_initial.py
@@ -0,0 +1,132 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ("connector_v2", "0001_initial"),
+ ("workflow_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="ToolInstance",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "tool_id",
+ models.CharField(
+ db_comment="Function name of the tool being used", max_length=64
+ ),
+ ),
+ (
+ "input",
+ models.JSONField(
+ db_comment="Provisional WF input to a tool", null=True
+ ),
+ ),
+ (
+ "output",
+ models.JSONField(
+ db_comment="Provisional WF output to a tool", null=True
+ ),
+ ),
+ ("version", models.CharField(max_length=16)),
+ ("metadata", models.JSONField(db_comment="Stores config for a tool")),
+ ("step", models.IntegerField()),
+ ("status", models.CharField(default="Ready to start", max_length=32)),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="tool_instances_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "input_db_connector",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="input_db_connectors",
+ to="connector_v2.connectorinstance",
+ ),
+ ),
+ (
+ "input_file_connector",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="input_file_connectors",
+ to="connector_v2.connectorinstance",
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="tool_instances_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "output_db_connector",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="output_db_connectors",
+ to="connector_v2.connectorinstance",
+ ),
+ ),
+ (
+ "output_file_connector",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="output_file_connectors",
+ to="connector_v2.connectorinstance",
+ ),
+ ),
+ (
+ "workflow",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="tool_instances",
+ to="workflow_v2.workflow",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Tool Instance",
+ "verbose_name_plural": "Tool Instances",
+ "db_table": "tool_instance",
+ },
+ ),
+ ]
diff --git a/backend/tool_instance_v2/migrations/__init__.py b/backend/tool_instance_v2/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/tool_instance_v2/models.py b/backend/tool_instance_v2/models.py
index a7609566d..a05e834b1 100644
--- a/backend/tool_instance_v2/models.py
+++ b/backend/tool_instance_v2/models.py
@@ -96,4 +96,4 @@ class ToolInstance(BaseModel):
class Meta:
verbose_name = "Tool Instance"
verbose_name_plural = "Tool Instances"
- db_table = "tool_instance_v2"
+ db_table = "tool_instance"
diff --git a/backend/usage_v2/migrations/0001_initial.py b/backend/usage_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..28dfd79c7
--- /dev/null
+++ b/backend/usage_v2/migrations/0001_initial.py
@@ -0,0 +1,144 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ("account_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="Usage",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ db_comment="Primary key for the usage entry, automatically generated UUID",
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "workflow_id",
+ models.CharField(
+ blank=True,
+ db_comment="Identifier for the workflow",
+ max_length=255,
+ null=True,
+ ),
+ ),
+ (
+ "execution_id",
+ models.CharField(
+ blank=True,
+ db_comment="Identifier for the execution instance",
+ max_length=255,
+ null=True,
+ ),
+ ),
+ (
+ "adapter_instance_id",
+ models.CharField(
+ db_comment="Identifier for the adapter instance", max_length=255
+ ),
+ ),
+ (
+ "run_id",
+ models.CharField(
+ blank=True,
+ db_comment="Identifier for the run",
+ max_length=255,
+ null=True,
+ ),
+ ),
+ (
+ "usage_type",
+ models.CharField(
+ choices=[
+ ("llm", "LLM Usage"),
+ ("embedding", "Embedding Usage"),
+ ],
+ db_comment="Type of usage, either 'llm' or 'embedding'",
+ max_length=255,
+ ),
+ ),
+ (
+ "llm_usage_reason",
+ models.CharField(
+ blank=True,
+ choices=[
+ ("extraction", "Extraction"),
+ ("challenge", "Challenge"),
+ ("summarize", "Summarize"),
+ ],
+ db_comment="Reason for LLM usage. Empty if usage_type is 'embedding'. ",
+ max_length=255,
+ null=True,
+ ),
+ ),
+ (
+ "model_name",
+ models.CharField(
+ db_comment="Name of the model used", max_length=255
+ ),
+ ),
+ (
+ "embedding_tokens",
+ models.IntegerField(
+ db_comment="Number of tokens used for embedding"
+ ),
+ ),
+ (
+ "prompt_tokens",
+ models.IntegerField(
+ db_comment="Number of tokens used for the prompt"
+ ),
+ ),
+ (
+ "completion_tokens",
+ models.IntegerField(
+ db_comment="Number of tokens used for the completion"
+ ),
+ ),
+ (
+ "total_tokens",
+ models.IntegerField(db_comment="Total number of tokens used"),
+ ),
+ (
+ "cost_in_dollars",
+ models.FloatField(db_comment="Total number of tokens used"),
+ ),
+ (
+ "organization",
+ models.ForeignKey(
+ blank=True,
+ db_comment="Foreign key reference to the Organization model.",
+ default=None,
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ to="account_v2.organization",
+ ),
+ ),
+ ],
+ options={
+ "db_table": "usage",
+ "indexes": [
+ models.Index(
+ fields=["run_id"], name="token_usage_run_id_cd3578_idx"
+ )
+ ],
+ },
+ ),
+ ]
diff --git a/backend/usage_v2/migrations/__init__.py b/backend/usage_v2/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/usage_v2/models.py b/backend/usage_v2/models.py
index 8d4a35069..8e836024e 100644
--- a/backend/usage_v2/models.py
+++ b/backend/usage_v2/models.py
@@ -76,7 +76,7 @@ class Usage(DefaultOrganizationMixin, BaseModel):
return str(self.id)
class Meta:
- db_table = "token_usage_v2"
+ db_table = "usage"
indexes = [
models.Index(fields=["run_id"]),
]
diff --git a/backend/utils/serializer_utils.py b/backend/utils/serializer_utils.py
index 2534404a4..fdecc7fae 100644
--- a/backend/utils/serializer_utils.py
+++ b/backend/utils/serializer_utils.py
@@ -1,9 +1,16 @@
from typing import Any
-from account.models import User
from rest_framework.request import Request
from utils.constants import Account
+from backend.constants import FeatureFlag
+from unstract.flags.feature_flag import check_feature_flag_status
+
+if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
+ from account_v2.models import User
+else:
+ from account.models import User
+
class SerializerUtils:
@staticmethod
diff --git a/backend/utils/user_context.py b/backend/utils/user_context.py
index 87addb52f..1be52f02a 100644
--- a/backend/utils/user_context.py
+++ b/backend/utils/user_context.py
@@ -1,15 +1,14 @@
from typing import Optional
from django.db import connection
+from django.db.utils import ProgrammingError
from utils.constants import Account, FeatureFlag
from utils.local_context import StateStore
from unstract.flags.feature_flag import check_feature_flag_status
if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
- # TODO: uncomment Once v2 implemented
- # from account_v2.models import Organization
- pass
+ from account_v2.models import Organization
else:
from account.models import Organization
@@ -37,6 +36,11 @@ class UserContext:
)
except Organization.DoesNotExist:
return None
+ except ProgrammingError:
+ # Handle cases where the database schema might not be fully set up,
+ # especially during the execution of management commands
+ # other than runserver
+ return None
else:
organization: Organization = connection.tenant
return organization
diff --git a/backend/utils/user_session.py b/backend/utils/user_session.py
index e3cb36d0d..972f2a28f 100644
--- a/backend/utils/user_session.py
+++ b/backend/utils/user_session.py
@@ -3,11 +3,15 @@ from typing import Optional
from django.conf import settings
from django.db import connection
from django.http import HttpRequest
-from tenant_account.models import OrganizationMember
from utils.constants import FeatureFlag
from unstract.flags.feature_flag import check_feature_flag_status
+if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
+ from tenant_account_v2.models import OrganizationMember
+else:
+ from tenant_account.models import OrganizationMember
+
class UserSessionUtils:
@staticmethod
diff --git a/backend/workflow_manager/endpoint_v2/base_connector.py b/backend/workflow_manager/endpoint_v2/base_connector.py
index 3207537e6..2523523e3 100644
--- a/backend/workflow_manager/endpoint_v2/base_connector.py
+++ b/backend/workflow_manager/endpoint_v2/base_connector.py
@@ -45,11 +45,26 @@ class BaseConnector(ExecutionFileHandler):
Raises:
KeyError: If the connector_id is not found in the connectors dictionary.
"""
+ return self.get_fs_connector(
+ settings=settings, connector_id=connector_id
+ ).get_fsspec_fs()
+
+ def get_fs_connector(
+ self, settings: dict[str, Any], connector_id: str
+ ) -> UnstractFileSystem:
+ """Get an fs connector based specified connector settings.
+
+ Parameters:
+ - settings (dict): Connector-specific settings.
+ - connector_id (str): Identifier for the desired connector.
+
+ Returns:
+ UnstractFileSystem: An unstract fs connector instance.
+ """
if connector_id not in connectors:
- raise ValueError(f"Invalid connector_id: {connector_id}")
+ raise ValueError(f"Connector '{connector_id}' is not supported.")
connector = connectors[connector_id][Common.METADATA][Common.CONNECTOR]
- connector_class: UnstractFileSystem = connector(settings)
- return connector_class.get_fsspec_fs()
+ return connector(settings)
@classmethod
def get_json_schema(cls, file_path: str) -> dict[str, Any]:
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index c882c1b0e..7839e1c8c 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -135,7 +135,7 @@ class DestinationConnector(BaseConnector):
"""Handle the output based on the connection type."""
connection_type = self.endpoint.connection_type
result: Optional[str] = None
- meta_data: Optional[str] = None
+ metadata: Optional[str] = None
if error:
if connection_type == WorkflowEndpoint.ConnectionType.API:
self._handle_api_result(file_name=file_name, error=error, result=result)
@@ -146,19 +146,19 @@ class DestinationConnector(BaseConnector):
self.insert_into_db(file_history)
elif connection_type == WorkflowEndpoint.ConnectionType.API:
result = self.get_result(file_history)
- meta_data = self.get_metadata(file_history)
+ metadata = self.get_metadata(file_history)
self._handle_api_result(
- file_name=file_name, error=error, result=result, meta_data=meta_data
+ file_name=file_name, error=error, result=result, metadata=metadata
)
elif connection_type == WorkflowEndpoint.ConnectionType.MANUALREVIEW:
result = self.get_result(file_history)
- meta_data = self.get_metadata(file_history)
+ metadata = self.get_metadata(file_history)
self._push_to_queue(
file_name=file_name,
workflow=workflow,
result=result,
input_file_path=input_file_path,
- meta_data=meta_data,
+ metadata=metadata,
)
if not file_history:
FileHistoryHelper.create_file_history(
@@ -166,7 +166,7 @@ class DestinationConnector(BaseConnector):
workflow=workflow,
status=ExecutionStatus.COMPLETED,
result=result,
- metadata=meta_data,
+ metadata=metadata,
file_name=file_name,
)
@@ -175,32 +175,29 @@ class DestinationConnector(BaseConnector):
connector: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector.connector_metadata
destination_configurations: dict[str, Any] = self.endpoint.configuration
- root_path = str(connector_settings.get(DestinationKey.PATH))
- output_folder = str(
+ root_path = str(connector_settings.get(DestinationKey.PATH, ""))
+
+ output_directory = str(
destination_configurations.get(DestinationKey.OUTPUT_FOLDER, "/")
)
- overwrite = bool(
- destination_configurations.get(
- DestinationKey.OVERWRITE_OUTPUT_DOCUMENT, False
- )
+ destination_fs = self.get_fs_connector(
+ settings=connector_settings, connector_id=connector.connector_id
)
- output_directory = os.path.join(root_path, output_folder)
-
+ output_directory = destination_fs.get_connector_root_dir(
+ input_dir=output_directory, root_path=root_path
+ )
+ logger.debug(f"destination output directory {output_directory}")
destination_volume_path = os.path.join(
self.execution_dir, ToolExecKey.OUTPUT_DIR
)
-
- connector_fs = self.get_fsspec(
- settings=connector_settings, connector_id=connector.connector_id
- )
- if not connector_fs.isdir(output_directory):
- connector_fs.mkdir(output_directory)
+ destination_fs.create_dir_if_not_exists(input_dir=output_directory)
+ destination_fsspec = destination_fs.get_fsspec_fs()
# Traverse local directory and create the same structure in the
# output_directory
for root, dirs, files in os.walk(destination_volume_path):
for dir_name in dirs:
- connector_fs.mkdir(
+ destination_fsspec.mkdir(
os.path.join(
output_directory,
os.path.relpath(root, destination_volume_path),
@@ -217,9 +214,7 @@ class DestinationConnector(BaseConnector):
)
normalized_path = os.path.normpath(destination_path)
with open(source_path, "rb") as source_file:
- connector_fs.write_bytes(
- normalized_path, source_file.read(), overwrite=overwrite
- )
+ destination_fsspec.write_bytes(normalized_path, source_file.read())
def insert_into_db(self, file_history: Optional[FileHistory]) -> None:
"""Insert data into the database."""
@@ -283,7 +278,7 @@ class DestinationConnector(BaseConnector):
file_name: str,
error: Optional[str] = None,
result: Optional[str] = None,
- meta_data: Optional[dict[str, Any]] = None,
+ metadata: Optional[dict[str, Any]] = None,
) -> None:
"""Handle the API result.
@@ -309,7 +304,7 @@ class DestinationConnector(BaseConnector):
{
"status": ApiDeploymentResultStatus.SUCCESS,
"result": result,
- "metadata": meta_data,
+ "metadata": metadata,
}
)
else:
@@ -390,13 +385,13 @@ class DestinationConnector(BaseConnector):
def get_metadata(
self, file_history: Optional[FileHistory]
) -> Optional[dict[str, Any]]:
- """Get meta_data from the output file.
+ """Get metadata from the output file.
Returns:
Union[dict[str, Any], str]: Meta data.
"""
- if file_history and file_history.meta_data:
- return self.parse_string(file_history.meta_data)
+ if file_history and file_history.metadata:
+ return self.parse_string(file_history.metadata)
metadata: dict[str, Any] = self.get_workflow_metadata()
return metadata
@@ -482,7 +477,7 @@ class DestinationConnector(BaseConnector):
workflow: Workflow,
result: Optional[str] = None,
input_file_path: Optional[str] = None,
- meta_data: Optional[dict[str, Any]] = None,
+ metadata: Optional[dict[str, Any]] = None,
) -> None:
"""Handle the Manual Review QUEUE result.
@@ -497,7 +492,7 @@ class DestinationConnector(BaseConnector):
input_file_path (Optional[str], optional):
The path to the input file.
Defaults to None.
- meta_data (Optional[dict[str, Any]], optional):
+ metadata (Optional[dict[str, Any]], optional):
A dictionary containing additional
metadata related to the file. Defaults to None.
@@ -519,7 +514,7 @@ class DestinationConnector(BaseConnector):
q_name = f"review_queue_{self.organization_id}_{workflow.workflow_name}"
queue_result = QueueResult(
file=file_name,
- whisper_hash=meta_data["whisper-hash"],
+ whisper_hash=metadata["whisper-hash"],
status=QueueResultStatus.SUCCESS,
result=result,
workflow_id=str(self.workflow_id),
diff --git a/backend/workflow_manager/endpoint_v2/migrations/0001_initial.py b/backend/workflow_manager/endpoint_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..1cc0d1b1b
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/migrations/0001_initial.py
@@ -0,0 +1,90 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ("connector_v2", "0001_initial"),
+ ("workflow_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="WorkflowEndpoint",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "endpoint_type",
+ models.CharField(
+ choices=[
+ ("SOURCE", "Source connector"),
+ ("DESTINATION", "Destination Connector"),
+ ],
+ db_comment="Endpoint type (source or destination)",
+ editable=False,
+ ),
+ ),
+ (
+ "connection_type",
+ models.CharField(
+ blank=True,
+ choices=[
+ ("FILESYSTEM", "FileSystem connector"),
+ ("DATABASE", "Database Connector"),
+ ("API", "API Connector"),
+ ("APPDEPLOYMENT", "App Deployment"),
+ ("MANUALREVIEW", "Manual Review Queue Connector"),
+ ],
+ db_comment="Connection type (Filesystem, Database, API or Manualreview)",
+ ),
+ ),
+ (
+ "configuration",
+ models.JSONField(
+ blank=True, db_comment="Configuration in JSON format", null=True
+ ),
+ ),
+ (
+ "connector_instance",
+ models.ForeignKey(
+ db_comment="Foreign key from ConnectorInstance model",
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="workflow_endpoints",
+ to="connector_v2.connectorinstance",
+ ),
+ ),
+ (
+ "workflow",
+ models.ForeignKey(
+ db_comment="Foreign key from Workflow model",
+ editable=False,
+ on_delete=django.db.models.deletion.CASCADE,
+ to="workflow_v2.workflow",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Workflow Endpoint",
+ "verbose_name_plural": "Workflow Endpoints",
+ "db_table": "workflow_endpoints",
+ },
+ ),
+ ]
diff --git a/backend/workflow_manager/endpoint_v2/migrations/__init__.py b/backend/workflow_manager/endpoint_v2/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/workflow_manager/endpoint_v2/models.py b/backend/workflow_manager/endpoint_v2/models.py
index 6b067160e..2659a3b6f 100644
--- a/backend/workflow_manager/endpoint_v2/models.py
+++ b/backend/workflow_manager/endpoint_v2/models.py
@@ -51,4 +51,4 @@ class WorkflowEndpoint(BaseModel):
class Meta:
verbose_name = "Workflow Endpoint"
verbose_name_plural = "Workflow Endpoints"
- db_table = "workflow_endpoints_v2"
+ db_table = "workflow_endpoints"
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index d9283fa7a..dfa5aea5d 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -4,7 +4,6 @@ import os
import shutil
from hashlib import md5, sha256
from io import BytesIO
-from pathlib import Path
from typing import Any, Optional
import fsspec
@@ -154,21 +153,27 @@ class SourceConnector(BaseConnector):
)
)
root_dir_path = connector_settings.get(ConnectorKeys.PATH, "")
+
input_directory = str(source_configurations.get(SourceKey.ROOT_FOLDER, ""))
- if root_dir_path: # user needs to manually type the optional file path
- input_directory = str(Path(root_dir_path, input_directory.lstrip("/")))
- if not isinstance(required_patterns, list):
- required_patterns = [required_patterns]
- source_fs = self.get_fsspec(
+ source_fs = self.get_fs_connector(
settings=connector_settings, connector_id=connector.connector_id
)
+ input_directory = source_fs.get_connector_root_dir(
+ input_dir=input_directory, root_path=root_dir_path
+ )
+ logger.debug(f"source input directory {input_directory}")
+ if not isinstance(required_patterns, list):
+ required_patterns = [required_patterns]
+
+ source_fs_fsspec = source_fs.get_fsspec_fs()
+
patterns = self.valid_file_patterns(required_patterns=required_patterns)
- is_directory = source_fs.isdir(input_directory)
+ is_directory = source_fs_fsspec.isdir(input_directory)
if not is_directory:
raise InvalidInputDirectory()
matched_files = self._get_matched_files(
- source_fs, input_directory, patterns, recursive, limit
+ source_fs_fsspec, input_directory, patterns, recursive, limit
)
self.publish_input_output_list_file_logs(input_directory, matched_files)
return matched_files
@@ -387,6 +392,17 @@ class SourceConnector(BaseConnector):
results.append({"file": file_name, "result": result})
def load_file(self, input_file_path: str) -> tuple[str, BytesIO]:
+ """Load file contnt and file name based on the file path.
+
+ Args:
+ input_file_path (str): source file
+
+ Raises:
+ InvalidSource: _description_
+
+ Returns:
+ tuple[str, BytesIO]: file_name , file content
+ """
connector: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector.connector_metadata
source_fs: fsspec.AbstractFileSystem = self.get_fsspec(
@@ -396,7 +412,7 @@ class SourceConnector(BaseConnector):
file_content = remote_file.read()
file_stream = BytesIO(file_content)
- return remote_file.key, file_stream
+ return os.path.basename(input_file_path), file_stream
@classmethod
def add_input_file_to_api_storage(
diff --git a/backend/workflow_manager/endpoint_v2/static/dest/file.json b/backend/workflow_manager/endpoint_v2/static/dest/file.json
index aa3a1263e..95111993f 100644
--- a/backend/workflow_manager/endpoint_v2/static/dest/file.json
+++ b/backend/workflow_manager/endpoint_v2/static/dest/file.json
@@ -14,12 +14,6 @@
"minLength": 1,
"maxLength": 100,
"format": "file-path"
- },
- "overwriteOutput": {
- "type": "boolean",
- "title": "Overwrite existing output",
- "default": true,
- "description": "Used to overwrite output document"
}
}
}
diff --git a/backend/workflow_manager/urls.py b/backend/workflow_manager/urls.py
index aa5aebd36..7140cc174 100644
--- a/backend/workflow_manager/urls.py
+++ b/backend/workflow_manager/urls.py
@@ -1,6 +1,14 @@
from django.urls import include, path
-from workflow_manager.endpoint import urls as endpoint_urls
-from workflow_manager.workflow import urls as workflow_urls
+
+from backend.constants import FeatureFlag
+from unstract.flags.feature_flag import check_feature_flag_status
+
+if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
+ from workflow_manager.endpoint_v2 import urls as endpoint_urls
+ from workflow_manager.workflow_v2 import urls as workflow_urls
+else:
+ from workflow_manager.endpoint import urls as endpoint_urls
+ from workflow_manager.workflow import urls as workflow_urls
urlpatterns = [
path("endpoint/", include(endpoint_urls)),
diff --git a/backend/workflow_manager/workflow_v2/file_history_helper.py b/backend/workflow_manager/workflow_v2/file_history_helper.py
index 7d84cf0da..21f8eda93 100644
--- a/backend/workflow_manager/workflow_v2/file_history_helper.py
+++ b/backend/workflow_manager/workflow_v2/file_history_helper.py
@@ -61,7 +61,7 @@ class FileHistoryHelper:
cache_key=cache_key,
status=status.value,
result=str(result),
- meta_data=str(metadata),
+ metadata=str(metadata),
error=str(error) if error else "",
)
except IntegrityError:
diff --git a/backend/workflow_manager/workflow_v2/migrations/0001_initial.py b/backend/workflow_manager/workflow_v2/migrations/0001_initial.py
new file mode 100644
index 000000000..2fff00246
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/migrations/0001_initial.py
@@ -0,0 +1,329 @@
+# Generated by Django 4.2.1 on 2024-07-29 09:16
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ("account_v2", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="ExecutionLog",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "execution_id",
+ models.UUIDField(db_comment="Execution ID", editable=False),
+ ),
+ ("data", models.JSONField(db_comment="Execution log data")),
+ (
+ "event_time",
+ models.DateTimeField(db_comment="Execution log event time"),
+ ),
+ ],
+ options={
+ "verbose_name": "Execution Log",
+ "verbose_name_plural": "Execution Logs",
+ "db_table": "execution_log",
+ },
+ ),
+ migrations.CreateModel(
+ name="WorkflowExecution",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "pipeline_id",
+ models.UUIDField(
+ db_comment="ID of the associated pipeline, if applicable",
+ editable=False,
+ null=True,
+ ),
+ ),
+ (
+ "task_id",
+ models.UUIDField(
+ db_comment="task id of asynchronous execution",
+ editable=False,
+ null=True,
+ ),
+ ),
+ (
+ "workflow_id",
+ models.UUIDField(
+ db_comment="Id of workflow to be executed", editable=False
+ ),
+ ),
+ (
+ "execution_mode",
+ models.CharField(
+ choices=[
+ ("INSTANT", "will be executed immediately"),
+ ("QUEUE", "will be placed in a queue"),
+ ],
+ db_comment="Mode of execution",
+ ),
+ ),
+ (
+ "execution_method",
+ models.CharField(
+ choices=[
+ ("DIRECT", " Execution triggered manually"),
+ ("SCHEDULED", "Scheduled execution"),
+ ],
+ db_comment="Method of execution",
+ ),
+ ),
+ (
+ "execution_type",
+ models.CharField(
+ choices=[
+ ("COMPLETE", "For complete execution"),
+ ("STEP", "For step-by-step execution "),
+ ],
+ db_comment="Type of execution",
+ ),
+ ),
+ (
+ "execution_log_id",
+ models.CharField(
+ db_comment="Execution log events Id", default="", editable=False
+ ),
+ ),
+ (
+ "status",
+ models.CharField(
+ db_comment="Current status of execution", default=""
+ ),
+ ),
+ (
+ "error_message",
+ models.CharField(
+ blank=True,
+ db_comment="Details of encountered errors",
+ default="",
+ max_length=256,
+ ),
+ ),
+ (
+ "attempts",
+ models.IntegerField(
+ db_comment="number of attempts taken", default=0
+ ),
+ ),
+ (
+ "execution_time",
+ models.FloatField(
+ db_comment="execution time in seconds", default=0
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Workflow Execution",
+ "verbose_name_plural": "Workflow Executions",
+ "db_table": "workflow_execution",
+ },
+ ),
+ migrations.CreateModel(
+ name="Workflow",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ ("prompt_name", models.CharField(default="", max_length=32)),
+ ("description", models.TextField(default="", max_length=490)),
+ ("workflow_name", models.CharField(max_length=128)),
+ ("prompt_text", models.TextField(default="")),
+ ("is_active", models.BooleanField(default=False)),
+ ("status", models.CharField(default="", max_length=16)),
+ ("llm_response", models.TextField()),
+ (
+ "deployment_type",
+ models.CharField(
+ choices=[
+ ("DEFAULT", "Not ready yet"),
+ ("ETL", "ETL pipeline"),
+ ("TASK", "TASK pipeline"),
+ ("API", "API deployment"),
+ ("APP", "App deployment"),
+ ],
+ db_comment="Type of workflow deployment",
+ default="DEFAULT",
+ ),
+ ),
+ (
+ "source_settings",
+ models.JSONField(
+ db_comment="Settings for the Source module", null=True
+ ),
+ ),
+ (
+ "destination_settings",
+ models.JSONField(
+ db_comment="Settings for the Destination module", null=True
+ ),
+ ),
+ (
+ "created_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="workflows_created",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "modified_by",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="workflows_modified",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "organization",
+ models.ForeignKey(
+ blank=True,
+ db_comment="Foreign key reference to the Organization model.",
+ default=None,
+ null=True,
+ on_delete=django.db.models.deletion.CASCADE,
+ to="account_v2.organization",
+ ),
+ ),
+ (
+ "workflow_owner",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="workflows_owned",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Workflow",
+ "verbose_name_plural": "Workflows",
+ "db_table": "workflow",
+ },
+ ),
+ migrations.CreateModel(
+ name="FileHistory",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ (
+ "cache_key",
+ models.CharField(
+ db_comment="Hash value of file contents, WF and tool modified times",
+ max_length=64,
+ ),
+ ),
+ (
+ "status",
+ models.TextField(
+ choices=[
+ ("PENDING", "PENDING"),
+ ("INITIATED", "INITIATED"),
+ ("QUEUED", "QUEUED"),
+ ("READY", "READY"),
+ ("EXECUTING", "EXECUTING"),
+ ("COMPLETED", "COMPLETED"),
+ ("STOPPED", "STOPPED"),
+ ("ERROR", "ERROR"),
+ ],
+ db_comment="Latest status of execution",
+ ),
+ ),
+ (
+ "error",
+ models.TextField(
+ blank=True, db_comment="Error message", default=""
+ ),
+ ),
+ (
+ "result",
+ models.TextField(blank=True, db_comment="Result from execution"),
+ ),
+ (
+ "metadata",
+ models.TextField(blank=True, db_comment="MetaData from execution"),
+ ),
+ (
+ "workflow",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="file_histories",
+ to="workflow_v2.workflow",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "File History",
+ "verbose_name_plural": "File Histories",
+ "db_table": "file_history",
+ },
+ ),
+ migrations.AddConstraint(
+ model_name="workflow",
+ constraint=models.UniqueConstraint(
+ fields=("workflow_name", "organization"), name="unique_workflow_name"
+ ),
+ ),
+ migrations.AddConstraint(
+ model_name="filehistory",
+ constraint=models.UniqueConstraint(
+ fields=("workflow", "cache_key"), name="unique_workflow_cacheKey"
+ ),
+ ),
+ ]
diff --git a/backend/workflow_manager/workflow_v2/migrations/__init__.py b/backend/workflow_manager/workflow_v2/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.py
index 23ceb2bbd..7a41dfc22 100644
--- a/backend/workflow_manager/workflow_v2/models/execution.py
+++ b/backend/workflow_manager/workflow_v2/models/execution.py
@@ -31,15 +31,9 @@ class WorkflowExecution(BaseModel):
null=True,
db_comment="task id of asynchronous execution",
)
- # We can remove workflow_id if it not required
workflow_id = models.UUIDField(
editable=False, db_comment="Id of workflow to be executed"
)
- project_settings_id = models.UUIDField(
- editable=False,
- default=uuid.uuid4,
- db_comment="Id of project settings used while execution",
- )
execution_mode = models.CharField(
choices=Mode.choices, db_comment="Mode of execution"
)
@@ -78,4 +72,4 @@ class WorkflowExecution(BaseModel):
class Meta:
verbose_name = "Workflow Execution"
verbose_name_plural = "Workflow Executions"
- db_table = "workflow_execution_v2"
+ db_table = "workflow_execution"
diff --git a/backend/workflow_manager/workflow_v2/models/execution_log.py b/backend/workflow_manager/workflow_v2/models/execution_log.py
index 6bec96009..68faa9a4e 100644
--- a/backend/workflow_manager/workflow_v2/models/execution_log.py
+++ b/backend/workflow_manager/workflow_v2/models/execution_log.py
@@ -19,4 +19,4 @@ class ExecutionLog(BaseModel):
class Meta:
verbose_name = "Execution Log"
verbose_name_plural = "Execution Logs"
- db_table = "execution_log_v2"
+ db_table = "execution_log"
diff --git a/backend/workflow_manager/workflow_v2/models/file_history.py b/backend/workflow_manager/workflow_v2/models/file_history.py
index 5ad521cd0..c6ca20d0a 100644
--- a/backend/workflow_manager/workflow_v2/models/file_history.py
+++ b/backend/workflow_manager/workflow_v2/models/file_history.py
@@ -39,11 +39,12 @@ class FileHistory(BaseModel):
db_comment="Error message",
)
result = models.TextField(blank=True, db_comment="Result from execution")
+ metadata = models.TextField(blank=True, db_comment="MetaData from execution")
class Meta:
verbose_name = "File History"
verbose_name_plural = "File Histories"
- db_table = "file_history_v2"
+ db_table = "file_history"
constraints = [
models.UniqueConstraint(
fields=["workflow", "cache_key"],
diff --git a/backend/workflow_manager/workflow_v2/models/workflow.py b/backend/workflow_manager/workflow_v2/models/workflow.py
index bfde5e86e..18f3015b3 100644
--- a/backend/workflow_manager/workflow_v2/models/workflow.py
+++ b/backend/workflow_manager/workflow_v2/models/workflow.py
@@ -85,7 +85,7 @@ class Workflow(DefaultOrganizationMixin, BaseModel):
class Meta:
verbose_name = "Workflow"
verbose_name_plural = "Workflows"
- db_table = "workflow_v2"
+ db_table = "workflow"
constraints = [
models.UniqueConstraint(
fields=["workflow_name", "organization"],
diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.py
index f9b9c50f4..55bede227 100644
--- a/backend/workflow_manager/workflow_v2/views.py
+++ b/backend/workflow_manager/workflow_v2/views.py
@@ -1,8 +1,6 @@
import logging
from typing import Any, Optional
-from connector_v2.connector_instance_helper import ConnectorInstanceHelper
-from django.conf import settings
from django.db.models.query import QuerySet
from permissions.permission import IsOwner
from pipeline_v2.models import Pipeline
@@ -116,16 +114,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
is_active=True,
)
WorkflowEndpointUtils.create_endpoints_for_workflow(workflow)
-
- # Enable GCS configurations to create GCS while creating a workflow
- if (
- settings.GOOGLE_STORAGE_ACCESS_KEY_ID
- and settings.UNSTRACT_FREE_STORAGE_BUCKET_NAME
- ):
- ConnectorInstanceHelper.create_default_gcs_connector(
- workflow, self.request.user
- )
-
+ # NOTE: Add default connector here if needed
except Exception as e:
logger.error(f"Error saving workflow to DB: {e}")
raise WorkflowGenerationError