LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Feat/friction less onboarding and usage reporting (#256)
Rahul Johny9d49cdb0883d
diff --git a/backend/account/authentication_controller.py b/backend/account/authentication_controller.pyindex 9160f8d13..e9098444c 100644--- a/backend/account/authentication_controller.py+++ b/backend/account/authentication_controller.py@@ -11,7 +11,12 @@ from account.constants import (ErrorMessage,OrganizationMemberModel,)-from account.custom_exceptions import DuplicateData, Forbidden, UserNotExistError+from account.custom_exceptions import (+ DuplicateData,+ Forbidden,+ MethodNotImplemented,+ UserNotExistError,+)from account.dto import (MemberInvitation,OrganizationData,@@ -173,6 +178,15 @@ class AuthenticationController:{ErrorMessage.DUPLICATE_API}")self.create_tenant_user(organization=organization, user=user)++ if new_organization:+ try:+ self.auth_service.frictionless_onboarding(+ organization=organization, user=user+ )+ except MethodNotImplemented:+ Logger.info("frictionless_onboarding not implemented")+if new_organization:self.authentication_helper.create_initial_platform_key(user=user, organization=organization@@ -191,7 +205,8 @@ class AuthenticationController:current_organization_id = UserSessionUtils.get_organization_id(request)if current_organization_id:OrganizationMemberService.remove_user_membership_in_organization_cache(- user_id=user.user_id, organization_id=current_organization_id+ user_id=user.user_id,+ organization_id=current_organization_id,)UserSessionUtils.set_organization_id(request, organization_id)OrganizationMemberService.set_user_membership_in_organization_cache(diff --git a/backend/account/authentication_helper.py b/backend/account/authentication_helper.pyindex fcf69150e..928624e07 100644--- a/backend/account/authentication_helper.py+++ b/backend/account/authentication_helper.py@@ -43,6 +43,8 @@ class AuthenticationHelper:"""user_service = UserService()user = user_service.get_user_by_email(email)+ if user and not user.user_id:+ user = user_service.update_user(user, user_id)if not user:user = user_service.create_user(email, user_id)return userdiff --git a/backend/account/authentication_service.py b/backend/account/authentication_service.pyindex fe4ed2bac..debc9d58e 100644--- a/backend/account/authentication_service.py+++ b/backend/account/authentication_service.py@@ -256,6 +256,9 @@ class AuthenticationService:def get_invitations(self, organization_id: str) -> list[MemberInvitation]:raise MethodNotImplemented()+ def frictionless_onboarding(self, organization: Organization, user: User) -> None:+ raise MethodNotImplemented()+def delete_invitation(self, organization_id: str, invitation_id: str) -> bool:raise MethodNotImplemented()diff --git a/backend/account/migrations/0007_organization_allowed_token_limit.py b/backend/account/migrations/0007_organization_allowed_token_limit.pynew file mode 100644index 000000000..af6baa8a1--- /dev/null+++ b/backend/account/migrations/0007_organization_allowed_token_limit.py@@ -0,0 +1,20 @@+# Generated by Django 4.2.1 on 2024-04-25 07:55++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("account", "0006_delete_encryptionsecret"),+ ]++ operations = [+ migrations.AddField(+ model_name="organization",+ name="allowed_token_limit",+ field=models.IntegerField(+ db_comment="token limit set in case of frition less onbaoarded org",+ default=-1,+ ),+ ),+ ]diff --git a/backend/account/models.py b/backend/account/models.pyindex 2eae51073..063a7f7cd 100644--- a/backend/account/models.py+++ b/backend/account/models.py@@ -36,6 +36,10 @@ class Organization(TenantMixin):)modified_at = models.DateTimeField(auto_now=True)created_at = models.DateTimeField(auto_now=True)+ allowed_token_limit = models.IntegerField(+ default=-1,+ db_comment="token limit set in case of frition less onbaoarded org",+ )auto_create_schema = Truediff --git a/backend/account/user.py b/backend/account/user.pyindex f875d4572..c1499e69c 100644--- a/backend/account/user.py+++ b/backend/account/user.py@@ -22,6 +22,11 @@ class UserService:raise errorreturn user+ def update_user(self, user: User, user_id: str) -> User:+ user.user_id = user_id+ user.save()+ return user+def get_user_by_email(self, email: str) -> Optional[User]:try:user: User = User.objects.get(email=email)diff --git a/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py b/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.pyindex d5a9bb4cf..75abf4ae1 100644--- a/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py+++ b/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py@@ -3,7 +3,6 @@import jsonfrom typing import Any-from adapter_processor.models import AdapterInstancefrom cryptography.fernet import Fernetfrom django.conf import settingsfrom django.db import migrations, models@@ -17,6 +16,7 @@ class Migration(migrations.Migration):def EncryptCredentials(apps: Any, schema_editor: Any) -> None:encryption_secret: str = settings.ENCRYPTION_KEYf: Fernet = Fernet(encryption_secret.encode("utf-8"))+ AdapterInstance = apps.get_model("adapter_processor", "AdapterInstance")queryset = AdapterInstance.objects.all()for obj in queryset: # type: ignorediff --git a/backend/adapter_processor/migrations/0008_adapterinstance_description_and_more.py b/backend/adapter_processor/migrations/0008_adapterinstance_description_and_more.pynew file mode 100644index 000000000..ec2c607c6--- /dev/null+++ b/backend/adapter_processor/migrations/0008_adapterinstance_description_and_more.py@@ -0,0 +1,41 @@+# Generated by Django 4.2.1 on 2024-04-29 05:16++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ (+ "adapter_processor",+ "0007_remove_adapterinstance_is_default_userdefaultadapter",+ ),+ ]++ operations = [+ migrations.AddField(+ model_name="adapterinstance",+ name="description",+ field=models.TextField(blank=True, default=None, null=True),+ ),+ migrations.AddField(+ model_name="adapterinstance",+ name="is_friction_less",+ field=models.BooleanField(+ db_comment="Was the adapter created through frictionless onboarding",+ default=False,+ ),+ ),+ migrations.AddField(+ model_name="adapterinstance",+ name="is_usable",+ field=models.BooleanField(db_comment="Is the Adpater Usable", default=True),+ ),+ migrations.AddField(+ model_name="adapterinstance",+ name="shared_to_org",+ field=models.BooleanField(+ db_comment="Is the adapter shared to entire org", default=False+ ),+ ),+ ]diff --git a/backend/adapter_processor/models.py b/backend/adapter_processor/models.pyindex fd84db17c..5a2699f9f 100644--- a/backend/adapter_processor/models.py+++ b/backend/adapter_processor/models.py@@ -24,7 +24,12 @@ class AdapterInstanceModelManager(models.Manager):def for_user(self, user: User) -> QuerySet[Any]:return (self.get_queryset()- .filter(models.Q(created_by=user) | models.Q(shared_users=user))+ .filter(+ models.Q(created_by=user)+ | models.Q(shared_users=user)+ | models.Q(shared_to_org=True)+ | models.Q(is_friction_less=True)+ ).distinct("id"))@@ -80,10 +85,27 @@ class AdapterInstance(BaseModel):default=False,db_comment="Is the adapter instance currently being used",)+ shared_to_org = models.BooleanField(+ default=False,+ db_comment="Is the adapter shared to entire org",+ )++ is_friction_less = models.BooleanField(+ default=False,+ db_comment="Was the adapter created through frictionless onboarding",+ )++ # Can be used if the adapter usage gets exhausted+ # Can also be used in other possible scenarios in feature+ is_usable = models.BooleanField(+ default=True,+ db_comment="Is the Adpater Usable",+ )# Introduced field to establish M2M relation between users and adapters.# This will introduce intermediary table which relates both the models.shared_users = models.ManyToManyField(User, related_name="shared_adapters")+ description = models.TextField(blank=True, null=True, default=None)objects = AdapterInstanceModelManager()@@ -98,6 +120,18 @@ class AdapterInstance(BaseModel):),]+ def create_adapter(self) -> None:++ encryption_secret: str = settings.ENCRYPTION_KEY+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))++ self.adapter_metadata_b = f.encrypt(+ json.dumps(self.adapter_metadata).encode("utf-8")+ )+ self.adapter_metadata = {}++ self.save()+def get_adapter_meta_data(self) -> Any:encryption_secret: str = settings.ENCRYPTION_KEYf: Fernet = Fernet(encryption_secret.encode("utf-8"))diff --git a/backend/adapter_processor/serializers.py b/backend/adapter_processor/serializers.pyindex bb570c1bc..087cfe221 100644--- a/backend/adapter_processor/serializers.py+++ b/backend/adapter_processor/serializers.py@@ -108,6 +108,7 @@ class AdapterListSerializer(BaseAdapterSerializer):"adapter_name","adapter_type","created_by",+ "description",) # type: ignoredef to_representation(self, instance: AdapterInstance) -> dict[str, str]:@@ -115,7 +116,11 @@ class AdapterListSerializer(BaseAdapterSerializer):rep[common.ICON] = AdapterProcessor.get_adapter_data_with_key(instance.adapter_id, common.ICON)- rep["created_by_email"] = instance.created_by.email++ if instance.is_friction_less:+ rep["created_by_email"] = "Unstract"+ else:+ rep["created_by_email"] = instance.created_by.emailreturn repdiff --git a/backend/adapter_processor/views.py b/backend/adapter_processor/views.pyindex 37ca36886..492d593f6 100644--- a/backend/adapter_processor/views.py+++ b/backend/adapter_processor/views.py@@ -24,7 +24,12 @@ from django.db import IntegrityErrorfrom django.db.models import ProtectedError, QuerySetfrom django.http import HttpRequestfrom django.http.response import HttpResponse-from permissions.permission import IsOwner, IsOwnerOrSharedUser+from permissions.permission import (+ IsFrictionLessAdapter,+ IsFrictionLessAdapterDelete,+ IsOwner,+ IsOwnerOrSharedUser,+)from rest_framework import statusfrom rest_framework.decorators import actionfrom rest_framework.request import Request@@ -124,9 +129,21 @@ class AdapterViewSet(GenericViewSet):class AdapterInstanceViewSet(ModelViewSet):- permission_classes: list[type[IsOwner]] = [IsOwner]+serializer_class = AdapterInstanceSerializer+ def get_permissions(self) -> list[Any]:++ if self.action in ["update", "retrieve"]:+ return [IsFrictionLessAdapter()]++ elif self.action == "destroy":+ return [IsFrictionLessAdapterDelete()]++ # Hack for friction-less onboarding+ # User cant view/update metadata but can delete/share etc+ return [IsOwner()]+def get_queryset(self) -> Optional[QuerySet]:if filter_args := FilterHelper.build_filter_args(self.request,@@ -195,27 +212,36 @@ class AdapterInstanceViewSet(ModelViewSet):) -> Response:adapter_instance: AdapterInstance = self.get_object()adapter_type = adapter_instance.adapter_type- user_default_adapter = UserDefaultAdapter.objects.get(user=request.user)- if (- (- adapter_type == AdapterKeys.LLM- and adapter_instance == user_default_adapter.default_llm_adapter- )- or (- adapter_type == AdapterKeys.EMBEDDING- and adapter_instance == user_default_adapter.default_embedding_adapter- )- or (- adapter_type == AdapterKeys.VECTOR_DB- and adapter_instance == user_default_adapter.default_vector_db_adapter- )- or (- adapter_type == AdapterKeys.X2TEXT- and adapter_instance == user_default_adapter.default_x2text_adapter+ try:+ user_default_adapter: UserDefaultAdapter = UserDefaultAdapter.objects.get(+ user=request.user)- ):- logger.error("Cannot delete a default adapter")- raise CannotDeleteDefaultAdapter()++ if (+ (+ adapter_type == AdapterKeys.LLM+ and adapter_instance == user_default_adapter.default_llm_adapter+ )+ or (+ adapter_type == AdapterKeys.EMBEDDING+ and adapter_instance+ == user_default_adapter.default_embedding_adapter+ )+ or (+ adapter_type == AdapterKeys.VECTOR_DB+ and adapter_instance+ == user_default_adapter.default_vector_db_adapter+ )+ or (+ adapter_type == AdapterKeys.X2TEXT+ and adapter_instance == user_default_adapter.default_x2text_adapter+ )+ ):+ logger.error("Cannot delete a default adapter")+ raise CannotDeleteDefaultAdapter()+ except UserDefaultAdapter.DoesNotExist:+ # We can go head and remove adapter here+ logger.info("User default adpater doesnt not exist")try:super().perform_destroy(adapter_instance)diff --git a/backend/backend/public_urls.py b/backend/backend/public_urls.pyindex e81a265f4..27ece8786 100644--- a/backend/backend/public_urls.py+++ b/backend/backend/public_urls.py@@ -44,3 +44,13 @@ if settings.ADMIN_ENABLED:),]urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)+++try:+ import pluggable_apps.platform_admin.urls # noqa: F401++ urlpatterns += [+ path(f"{path_prefix}/", include("pluggable_apps.platform_admin.urls")),+ ]+except ImportError:+ passdiff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 5575730a5..82ce2d481 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -221,6 +221,7 @@ TENANT_APPS = ("prompt_studio.prompt_studio_output_manager","prompt_studio.prompt_studio_document_manager","prompt_studio.prompt_studio_index_manager",+ "usage",)INSTALLED_APPS = list(SHARED_APPS) + [diff --git a/backend/pdm.lock b/backend/pdm.lockindex 343ddca48..0492a11f6 100644--- a/backend/pdm.lock+++ b/backend/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "deploy", "test"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:d32c9f74e40f02d7c0f0af6a18e11038de233982b899a716c4571c50543ba2d7"+content_hash = "sha256:217a1cf800d393c22fe30815724239a07c6578d6b75799296ba8e4df6cf0ad3d"[[package]]name = "adlfs"@@ -366,6 +366,16 @@ files = [{file = "Authlib-1.2.1.tar.gz", hash = "sha256:421f7c6b468d907ca2d9afede256f068f87e34d23dd221c07d13d4c234726afb"},]+[[package]]+name = "azure-common"+version = "1.1.28"+summary = "Microsoft Azure Client Library for Python (Common)"+groups = ["default"]+files = [+ {file = "azure-common-1.1.28.zip", hash = "sha256:4ac0cd3214e36b6a1b6a442686722a5d8cc449603aa833f3f0f40bda836704a3"},+ {file = "azure_common-1.1.28-py2.py3-none-any.whl", hash = "sha256:5c12d3dcf4ec20599ca6b0d3e09e86e146353d443e7fcc050c9a19c1f9df20ad"},+]+[[package]]name = "azure-core"version = "1.30.1"@@ -414,6 +424,36 @@ files = [{file = "azure_identity-1.16.0-py3-none-any.whl", hash = "sha256:722fdb60b8fdd55fa44dc378b8072f4b419b56a5e54c0de391f644949f3a826f"},]+[[package]]+name = "azure-mgmt-apimanagement"+version = "3.0.0"+requires_python = ">=3.6"+summary = "Microsoft Azure API Management Client Library for Python"+groups = ["default"]+dependencies = [+ "azure-common~=1.1",+ "azure-mgmt-core<2.0.0,>=1.3.0",+ "msrest>=0.6.21",+]+files = [+ {file = "azure-mgmt-apimanagement-3.0.0.zip", hash = "sha256:9262f54ed387eb083d8dae66d32a8df35647319b902bd498cdc376f50a12d154"},+ {file = "azure_mgmt_apimanagement-3.0.0-py3-none-any.whl", hash = "sha256:58a96261266c51c63a64113dbb0ad96cabbf3b6c32fb9b79c395462d01470964"},+]++[[package]]+name = "azure-mgmt-core"+version = "1.4.0"+requires_python = ">=3.7"+summary = "Microsoft Azure Management Core Library for Python"+groups = ["default"]+dependencies = [+ "azure-core<2.0.0,>=1.26.2",+]+files = [+ {file = "azure-mgmt-core-1.4.0.zip", hash = "sha256:d195208340094f98e5a6661b781cde6f6a051e79ce317caabd8ff97030a9b3ae"},+ {file = "azure_mgmt_core-1.4.0-py3-none-any.whl", hash = "sha256:81071675f186a585555ef01816f2774d49c1c9024cb76e5720c3c0f6b337bb7d"},+]+[[package]]name = "azure-storage-blob"version = "12.19.1"@@ -2069,7 +2109,7 @@ files = [[[package]]name = "llama-index-core"-version = "0.10.31"+version = "0.10.32"requires_python = "<4.0,>=3.8.1"summary = "Interface between LLMs and your data"groups = ["default"]@@ -2099,8 +2139,8 @@ dependencies = ["wrapt",]files = [- {file = "llama_index_core-0.10.31-py3-none-any.whl", hash = "sha256:b894680fa320a94de56d9a933ac7edb646cabf15fe67ae1cf8fa53ac52ab4542"},- {file = "llama_index_core-0.10.31.tar.gz", hash = "sha256:66d39d6f253e20311a21e0b98ea386089f099be12f2d23dbe11379a6d908ddf1"},+ {file = "llama_index_core-0.10.32-py3-none-any.whl", hash = "sha256:215f7389dadb78f2df13c20312a3e1e03c41f23e3063907469c4bae67bfd458c"},+ {file = "llama_index_core-0.10.32.tar.gz", hash = "sha256:0078c06d9143390e14c86a40e69716c88c7828533341559edd15e52249ede65a"},][[package]]@@ -2136,7 +2176,7 @@ files = [[[package]]name = "llama-index-embeddings-openai"-version = "0.1.8"+version = "0.1.9"requires_python = "<4.0,>=3.8.1"summary = "llama-index embeddings openai integration"groups = ["default"]@@ -2144,8 +2184,8 @@ dependencies = ["llama-index-core<0.11.0,>=0.10.1",]files = [- {file = "llama_index_embeddings_openai-0.1.8-py3-none-any.whl", hash = "sha256:186be52c29a8c82a39ff6e033e3a61b9c5a17cf6177a5507abe4f5c79fea3a53"},- {file = "llama_index_embeddings_openai-0.1.8.tar.gz", hash = "sha256:aae4ce3ec2fe2cb604d944646982c71663a8b99ed5f8c370823ee82eef4ddd3b"},+ {file = "llama_index_embeddings_openai-0.1.9-py3-none-any.whl", hash = "sha256:fbd16d6197b91f4dbdc6d0707e573cc224ac2b0a48d5b370c6232dd8a2282473"},+ {file = "llama_index_embeddings_openai-0.1.9.tar.gz", hash = "sha256:0fd292b2f9a0ad4534a790d6374726bc885853188087eb018167dcf239643924"},][[package]]@@ -2517,7 +2557,7 @@ files = [[[package]]name = "minio"-version = "7.2.5"+version = "7.2.6"summary = "MinIO Python SDK for Amazon S3 Compatible Cloud Storage"groups = ["default"]dependencies = [@@ -2528,8 +2568,8 @@ dependencies = ["urllib3",]files = [- {file = "minio-7.2.5-py3-none-any.whl", hash = "sha256:ed9176c96d4271cb1022b9ecb8a538b1e55b32ae06add6de16425cab99ef2304"},- {file = "minio-7.2.5.tar.gz", hash = "sha256:59d8906e2da248a9caac34d4958a859cc3a44abbe6447910c82b5abfa9d6a2e1"},+ {file = "minio-7.2.6-py3-none-any.whl", hash = "sha256:4972273a924f274e2d71f38f6d2afdf841a034801e60ba758e5c5aff4234b768"},+ {file = "minio-7.2.6.tar.gz", hash = "sha256:c545d0dda1ff26cefcfc754242be3d27a4e620e37ef3e51ecbe7212cf7ecc274"},][[package]]@@ -2581,6 +2621,24 @@ files = [{file = "msal_extensions-1.1.0-py3-none-any.whl", hash = "sha256:01be9711b4c0b1a151450068eeb2c4f0997df3bba085ac299de3a66f585e382f"},]+[[package]]+name = "msrest"+version = "0.7.1"+requires_python = ">=3.6"+summary = "AutoRest swagger generator Python client runtime."+groups = ["default"]+dependencies = [+ "azure-core>=1.24.0",+ "certifi>=2017.4.17",+ "isodate>=0.6.0",+ "requests-oauthlib>=0.5.0",+ "requests~=2.16",+]+files = [+ {file = "msrest-0.7.1-py3-none-any.whl", hash = "sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32"},+ {file = "msrest-0.7.1.zip", hash = "sha256:6e7661f46f3afd88b75667b7187a92829924446c7ea1d169be8c4bb7eeb788b9"},+]+[[package]]name = "multidict"version = "6.0.5"@@ -4721,7 +4779,7 @@ dependencies = [[[package]]name = "unstract-sdk"-version = "0.22.1"+version = "0.23.0"requires_python = "<3.11.1,>=3.9"summary = "A framework for writing Unstract Tools/Apps"groups = ["default"]@@ -4735,8 +4793,8 @@ dependencies = ["unstract-adapters~=0.12.1",]files = [- {file = "unstract_sdk-0.22.1-py3-none-any.whl", hash = "sha256:af29fec4c56c3c1916b7af639d331d9c7b115ebff6f1183e4be50bef01bd1792"},- {file = "unstract_sdk-0.22.1.tar.gz", hash = "sha256:f49db6c62bca6baa2474912a63f305df01ec12ada34af9a2fb93b8ce91e7fa16"},+ {file = "unstract_sdk-0.23.0-py3-none-any.whl", hash = "sha256:769bfa6048261241b0c2f76c64520d42992bcbba46403948d7474eb7e53740aa"},+ {file = "unstract_sdk-0.23.0.tar.gz", hash = "sha256:f327d1d4631947d611da7221667bdd82c40c836ae8d7fbef81eb8ad25818f165"},][[package]]diff --git a/backend/permissions/permission.py b/backend/permissions/permission.pyindex bf2f85906..5668c9f66 100644--- a/backend/permissions/permission.py+++ b/backend/permissions/permission.py@@ -1,5 +1,6 @@from typing import Any+from adapter_processor.models import AdapterInstancefrom rest_framework import permissionsfrom rest_framework.request import Requestfrom rest_framework.views import APIView@@ -9,6 +10,7 @@ class IsOwner(permissions.BasePermission):"""Custom permission to only allow owners of an object."""def has_object_permission(self, request: Request, view: APIView, obj: Any) -> bool:+return True if obj.created_by == request.user else False@@ -24,3 +26,31 @@ class IsOwnerOrSharedUser(permissions.BasePermission):)else False)+++class IsFrictionLessAdapter(permissions.BasePermission):+ """Hack for friction-less onboarding not allowing user to view or updating+ friction less adapter."""++ def has_object_permission(+ self, request: Request, view: APIView, obj: AdapterInstance+ ) -> bool:++ if obj.is_friction_less:+ return False++ return True if obj.created_by == request.user else False+++class IsFrictionLessAdapterDelete(permissions.BasePermission):+ """Hack for friction-less onboarding Allows frticon less adapter to rmoved+ by an org member."""++ def has_object_permission(+ self, request: Request, view: APIView, obj: AdapterInstance+ ) -> bool:++ if obj.is_friction_less:+ return True++ return True if obj.created_by == request.user else Falsediff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.pyindex 21e1e3eb3..8f8849ce4 100644--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py@@ -49,6 +49,33 @@ logger = logging.getLogger(__name__)class PromptStudioHelper:"""Helper class for Custom tool operations."""+ @staticmethod+ def validate_adapter_status(+ profile_manager: ProfileManager,+ ) -> None:+ """Helper method to validate the status of adapters in profile manager.++ Args:+ profile_manager (ProfileManager): The profile manager instance to+ validate.++ Raises:+ PermissionError: If the owner does not have permission to perform+ the action.+ """++ error_msg = "Permission Error: Free usage for the configured trial adapter exhausted.Please connect your own service accounts to continue.Please see our documentation for more details:https://docs.unstract.com/unstract_platform/setup_accounts/whats_needed" # noqa: E501+ adapters = [+ profile_manager.llm,+ profile_manager.vector_store,+ profile_manager.embedding_model,+ profile_manager.x2text,+ ]++ for adapter in adapters:+ if not adapter.is_usable:+ raise PermissionError(error_msg)+@staticmethoddef validate_profile_manager_owner_access(profile_manager: ProfileManager,@@ -237,6 +264,9 @@ class PromptStudioHelper:LogLevels.RUN,"Indexing started",)++ # Validate the status of adapter in profile manager+ PromptStudioHelper.validate_adapter_status(default_profile)# Need to check the user who created profile manager# has access to adapters configured in profile managerPromptStudioHelper.validate_profile_manager_owner_access(default_profile)@@ -492,6 +522,8 @@ class PromptStudioHelper:grammar_list.append(grammer_dict)grammer_dict = {}for prompt in prompts:+ # Need to check the user who created profile manager+ PromptStudioHelper.validate_adapter_status(prompt.profile_manager)# Need to check the user who created profile manager# has access to adaptersPromptStudioHelper.validate_profile_manager_owner_access(@@ -691,6 +723,7 @@ class PromptStudioHelper:else:challenge_llm = str(default_profile.llm.id)# Need to check the user who created profile manager+ PromptStudioHelper.validate_adapter_status(default_profile)# has access to adapters configured in profile managerPromptStudioHelper.validate_profile_manager_owner_access(default_profile)default_profile.chunk_size = 0 # To retrive full contextdiff --git a/backend/prompt_studio/prompt_studio_core/views.py b/backend/prompt_studio/prompt_studio_core/views.pyindex b3d01dd4a..9b7ffee85 100644--- a/backend/prompt_studio/prompt_studio_core/views.py+++ b/backend/prompt_studio/prompt_studio_core/views.py@@ -6,6 +6,7 @@ from account.custom_exceptions import DuplicateDatafrom django.db import IntegrityErrorfrom django.db.models import QuerySetfrom django.http import HttpRequest+from file_management.exceptions import FileNotFoundfrom file_management.file_management_helper import FileManagerHelperfrom permissions.permission import IsOwner, IsOwnerOrSharedUserfrom prompt_studio.processor_loader import ProcessorConfig, load_plugins@@ -371,7 +372,21 @@ class PromptStudioCoreView(viewsets.ModelViewSet):if not file_path.endswith("/"):file_path += "/"file_path += file_name- contents = FileManagerHelper.fetch_file_contents(file_system, file_path)+ # Temporary Hack for frictionless onboarding as the user id will be empty+ try:+ contents = FileManagerHelper.fetch_file_contents(file_system, file_path)+ except FileNotFound:+ file_path = file_path = FileManagerHelper.handle_sub_directory_for_tenants(+ UserSessionUtils.get_organization_id(request),+ is_create=True,+ user_id="",+ tool_id=str(custom_tool.tool_id),+ )+ if not file_path.endswith("/"):+ file_path += "/"+ file_path += file_name+ contents = FileManagerHelper.fetch_file_contents(file_system, file_path)+return Response({"data": contents}, status=status.HTTP_200_OK)@action(detail=True, methods=["post"])diff --git a/backend/pyproject.toml b/backend/pyproject.tomlindex cb36faca2..405a6e154 100644--- a/backend/pyproject.toml+++ b/backend/pyproject.toml@@ -30,7 +30,7 @@ dependencies = ["python-socketio==5.9.0", # For log_events"social-auth-app-django==5.3.0", # For OAuth"social-auth-core==4.4.2", # For OAuth- "unstract-sdk~=0.22.1",+ "unstract-sdk~=0.23.0","unstract-adapters~=0.12.1",# ! IMPORTANT!# Indirect local dependencies usually need to be added in their own projects@@ -47,6 +47,8 @@ dependencies = ["unstract-tool-registry @ file:///${PROJECT_ROOT}/../unstract/tool-registry","unstract-tool-sandbox @ file:///${PROJECT_ROOT}/../unstract/tool-sandbox","unstract-workflow-execution @ file:///${PROJECT_ROOT}/../unstract/workflow-execution",+ "azure-identity==1.16.0",+ "azure-mgmt-apimanagement==3.0.0",]# <3.11.1 due to resolution error from Unstract SDKrequires-python = ">=3.9,<3.11.1"diff --git a/backend/tool_instance/tool_instance_helper.py b/backend/tool_instance/tool_instance_helper.pyindex 87ef6052c..a261fcc29 100644--- a/backend/tool_instance/tool_instance_helper.py+++ b/backend/tool_instance/tool_instance_helper.py@@ -406,6 +406,15 @@ class ToolInstanceHelper:adapter_instances = AdapterInstance.objects.filter(id__in=adapter_ids).all()for adapter_instance in adapter_instances:+ if not adapter_instance.is_usable:+ logger.error(+ "Free usage for the configured sample adapter %s exhausted",+ adapter_instance.id,+ )+ error_msg = "Permission Error: Free usage for the configured trial adapter exhausted.Please connect your own service accounts to continue.Please see our documentation for more details:https://docs.unstract.com/unstract_platform/setup_accounts/whats_needed" # noqa: E501++ raise PermissionDenied(error_msg)+if not (adapter_instance.created_by == useror adapter_instance.shared_users.filter(pk=user.pk).exists()diff --git a/backend/usage/__init__.py b/backend/usage/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/usage/admin.py b/backend/usage/admin.pynew file mode 100644index 000000000..c7469a4fb--- /dev/null+++ b/backend/usage/admin.py@@ -0,0 +1,5 @@+from django.contrib import admin++from .models import Usage++admin.site.register(Usage)diff --git a/backend/usage/apps.py b/backend/usage/apps.pynew file mode 100644index 000000000..abe8f39c4--- /dev/null+++ b/backend/usage/apps.py@@ -0,0 +1,5 @@+from django.apps import AppConfig+++class UsageConfig(AppConfig):+ name = "usage"diff --git a/backend/usage/migrations/0001_initial.py b/backend/usage/migrations/0001_initial.pynew file mode 100644index 000000000..3bad77c1f--- /dev/null+++ b/backend/usage/migrations/0001_initial.py@@ -0,0 +1,44 @@+# Generated by Django 4.2.1 on 2024-04-22 12:55++import uuid++from django.db import migrations, models+++class Migration(migrations.Migration):++ initial = True++ dependencies = []++ operations = [+ migrations.CreateModel(+ name="Usage",+ fields=[+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("modified_at", models.DateTimeField(auto_now=True)),+ (+ "id",+ models.UUIDField(+ default=uuid.uuid4,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("workflow_id", models.CharField(max_length=255)),+ ("execution_id", models.CharField(max_length=255)),+ ("adapter_instance_id", models.CharField(max_length=255)),+ ("run_id", models.CharField(max_length=255)),+ ("usage_type", models.CharField(max_length=255)),+ ("model_name", models.CharField(max_length=255)),+ ("embedding_tokens", models.IntegerField()),+ ("prompt_tokens", models.IntegerField()),+ ("completion_tokens", models.IntegerField()),+ ("total_tokens", models.IntegerField()),+ ],+ options={+ "db_table": "token_usage",+ },+ ),+ ]diff --git a/backend/usage/migrations/__init__.py b/backend/usage/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/usage/models.py b/backend/usage/models.pynew file mode 100644index 000000000..4c8205987--- /dev/null+++ b/backend/usage/models.py@@ -0,0 +1,24 @@+import uuid++from django.db import models+from utils.models.base_model import BaseModel+++class Usage(BaseModel):+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)+ workflow_id = models.CharField(max_length=255)+ execution_id = models.CharField(max_length=255)+ adapter_instance_id = models.CharField(max_length=255)+ run_id = models.CharField(max_length=255)+ usage_type = models.CharField(max_length=255)+ model_name = models.CharField(max_length=255)+ embedding_tokens = models.IntegerField()+ prompt_tokens = models.IntegerField()+ completion_tokens = models.IntegerField()+ total_tokens = models.IntegerField()++ def __str__(self):+ return str(self.id)++ class Meta:+ db_table = "token_usage"diff --git a/backend/usage/tests.py b/backend/usage/tests.pynew file mode 100644index 000000000..a39b155ac--- /dev/null+++ b/backend/usage/tests.py@@ -0,0 +1 @@+# Create your tests here.diff --git a/backend/usage/views.py b/backend/usage/views.pynew file mode 100644index 000000000..60f00ef0e--- /dev/null+++ b/backend/usage/views.py@@ -0,0 +1 @@+# Create your views here.diff --git a/frontend/src/components/agency/actions/Actions.jsx b/frontend/src/components/agency/actions/Actions.jsxindex 843f5caa6..55c3ee3af 100644--- a/frontend/src/components/agency/actions/Actions.jsx+++ b/frontend/src/components/agency/actions/Actions.jsx@@ -23,13 +23,11 @@ import { useSessionStore } from "../../../store/session-store";import { useWorkflowStore } from "../../../store/workflow-store";import { CreateApiDeploymentModal } from "../../deployments/create-api-deployment-modal/CreateApiDeploymentModal.jsx";import { EtlTaskDeploy } from "../../pipelines-or-deployments/etl-task-deploy/EtlTaskDeploy.jsx";-import { SocketMessages } from "../../helpers/socket-messages/SocketMessages";import FileUpload from "../file-upload/FileUpload.jsx";import "./Actions.css";import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx";function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {- const [logId, setLogId] = useState("");const [executionId, setExecutionId] = useState("");const [execType, setExecType] = useState("");const [stepExecType, setStepExecType] = useState("");@@ -122,10 +120,8 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {try {const initialRes = await handleWfExecutionApi(body);const execIdValue = initialRes?.data?.execution_id;- const logIdValue = initialRes?.data?.log_id;setExecutionId(execIdValue);- setLogId(logIdValue);body["execution_id"] = execIdValue;if (isStepExecution) {body["execution_action"] = wfExecutionTypes[executionAction];@@ -535,7 +531,6 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {setDeploymentName={setDeploymentName}/>)}- <SocketMessages logId={logId} /></>);}diff --git a/frontend/src/components/helpers/auth/PersistentLogin.js b/frontend/src/components/helpers/auth/PersistentLogin.jsindex 3d89bc4df..4ea218d72 100644--- a/frontend/src/components/helpers/auth/PersistentLogin.js+++ b/frontend/src/components/helpers/auth/PersistentLogin.js@@ -4,6 +4,7 @@ import { Outlet } from "react-router-dom";import { Logo64 } from "../../../assets";import useSessionValid from "../../../hooks/useSessionValid";import { useSessionStore } from "../../../store/session-store";+import { SocketMessages } from "../socket-messages/SocketMessages";function PersistentLogin() {const [isLoading, setIsLoading] = useState(true);@@ -33,7 +34,12 @@ function PersistentLogin() {</div>);}- return <Outlet />;+ return (+ <>+ <Outlet />+ <SocketMessages />+ </>+ );}export { PersistentLogin };diff --git a/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js b/frontend/src/components/helpers/custom-tools/CustomToolsHelper.jsindex 85b5a9d09..b9c60781b 100644--- a/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js+++ b/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js@@ -8,11 +8,9 @@ import { useCustomToolStore } from "../../../store/custom-tool-store";import { useSessionStore } from "../../../store/session-store";import { useSocketCustomToolStore } from "../../../store/socket-custom-tool";import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader";-import { SocketMessages } from "../socket-messages/SocketMessages";function CustomToolsHelper() {const [isLoading, setIsLoading] = useState(true);- const [logId, setLogId] = useState(null);const { id } = useParams();const { sessionDetails } = useSessionStore();const { updateCustomTool, setDefaultCustomTool } = useCustomToolStore();@@ -46,7 +44,6 @@ function CustomToolsHelper() {updatedCusTool["singlePassExtractMode"] =data?.single_pass_extraction_mode;selectedDocId = data?.output;- setLogId(sessionDetails?.logEventsId);const reqOpsDocs = {method: "GET",@@ -110,12 +107,7 @@ function CustomToolsHelper() {if (isLoading) {return <SpinnerLoader />;}- return (- <>- <Outlet />- <SocketMessages logId={logId} />- </>- );+ return <Outlet />;}export { CustomToolsHelper };diff --git a/frontend/src/components/helpers/socket-messages/SocketMessages.js b/frontend/src/components/helpers/socket-messages/SocketMessages.jsindex 9a7e5ec2b..207120ef8 100644--- a/frontend/src/components/helpers/socket-messages/SocketMessages.js+++ b/frontend/src/components/helpers/socket-messages/SocketMessages.js@@ -1,5 +1,4 @@-import PropTypes from "prop-types";-import { useContext, useEffect } from "react";+import { useContext, useEffect, useState } from "react";import { SocketContext } from "../../../helpers/SocketContext";import { useExceptionHandler } from "../../../hooks/useExceptionHandler";@@ -7,7 +6,11 @@ import { useAlertStore } from "../../../store/alert-store";import { useSocketLogsStore } from "../../../store/socket-logs-store";import { useSocketMessagesStore } from "../../../store/socket-messages-store";import { useSocketCustomToolStore } from "../../../store/socket-custom-tool";-function SocketMessages({ logId }) {+import { useSessionStore } from "../../../store/session-store";+import { useUsageStore } from "../../../store/usage-store";++function SocketMessages() {+ const [logId, setLogId] = useState("");const {pushStagedMessage,updateMessage,@@ -17,9 +20,15 @@ function SocketMessages({ logId }) {} = useSocketMessagesStore();const { pushLogMessages } = useSocketLogsStore();const { updateCusToolMessages } = useSocketCustomToolStore();+ const { sessionDetails } = useSessionStore();const socket = useContext(SocketContext);const { setAlertDetails } = useAlertStore();const handleException = useExceptionHandler();+ const { setLLMTokenUsage } = useUsageStore();++ useEffect(() => {+ setLogId(sessionDetails?.logEventsId || "");+ }, [sessionDetails]);const onMessage = (data) => {try {@@ -37,6 +46,11 @@ function SocketMessages({ logId }) {if (msg?.type === "LOG" && msg?.service === "prompt") {updateCusToolMessages(msg);}+ if (msg?.type === "LOG" && msg?.service === "usage") {+ const remainingTokens =+ msg?.max_token_count_set - msg?.added_token_count;+ setLLMTokenUsage(Math.max(remainingTokens, 0));+ }} catch (err) {setAlertDetails(handleException(err, "Failed to process socket message"));}@@ -68,8 +82,4 @@ function SocketMessages({ logId }) {}, [stagedMessages, pointer]);}-SocketMessages.propTypes = {- logId: PropTypes.string,-};-export { SocketMessages };diff --git a/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsx b/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsxindex c4fd833f7..5d5339baf 100644--- a/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsx+++ b/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsx@@ -1,4 +1,13 @@-import { Alert, Button, Col, Dropdown, Image, Row, Typography } from "antd";+import {+ Alert,+ Button,+ Col,+ Dropdown,+ Image,+ Row,+ Space,+ Typography,+} from "antd";import { useEffect, useState } from "react";import { useNavigate } from "react-router-dom";import axios from "axios";@@ -163,26 +172,28 @@ function TopNavBar() {</Col><Col span={4}><Row justify="end" align="middle">- {TrialDaysInfo && (- <TrialDaysInfo remainingTrialDays={remainingTrialDays} />- )}- <Dropdown menu={{ items }} placement="bottomLeft" arrow>- <div className="top-navbar-dp">- {sessionDetails?.picture ? (- <Image- className="navbar-img"- height="100%"- width="100%"- preview={false}- src={sessionDetails?.picture}- />- ) : (- <Typography.Text className="initials">- {getInitials(sessionDetails?.name)}- </Typography.Text>- )}- </div>- </Dropdown>+ <Space>+ {TrialDaysInfo && (+ <TrialDaysInfo remainingTrialDays={remainingTrialDays} />+ )}+ <Dropdown menu={{ items }} placement="bottomLeft" arrow>+ <div className="top-navbar-dp">+ {sessionDetails?.picture ? (+ <Image+ className="navbar-img"+ height="100%"+ width="100%"+ preview={false}+ src={sessionDetails?.picture}+ />+ ) : (+ <Typography.Text className="initials">+ {getInitials(sessionDetails?.name)}+ </Typography.Text>+ )}+ </div>+ </Dropdown>+ </Space></Row></Col></Row>diff --git a/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx b/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsxindex 9ca1532a2..8258f6fa2 100644--- a/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx+++ b/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx@@ -16,7 +16,6 @@ import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate.js";import { useAlertStore } from "../../../store/alert-store.js";import { useSessionStore } from "../../../store/session-store.js";import { Layout } from "../../deployments/layout/Layout.jsx";-import { SocketMessages } from "../../helpers/socket-messages/SocketMessages.js";import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader.jsx";import { DeleteModal } from "../delete-modal/DeleteModal.jsx";import { EtlTaskDeploy } from "../etl-task-deploy/EtlTaskDeploy.jsx";@@ -29,7 +28,6 @@ function Pipelines({ type }) {const [openDeleteModal, setOpenDeleteModal] = useState(false);const [selectedPorD, setSelectedPorD] = useState({});const [tableLoading, setTableLoading] = useState(true);- const [logId, setLogId] = useState("");const { sessionDetails } = useSessionStore();const { setAlertDetails } = useAlertStore();const axiosPrivate = useAxiosPrivate();@@ -77,8 +75,6 @@ function Pipelines({ type }) {handleSyncApiReq(body).then((res) => {- const logIdValue = res?.data?.execution?.log_id;- setLogId(logIdValue);const executionId = res?.data?.execution?.execution_id;body["execution_id"] = executionId;return handleSyncApiReq(body);@@ -460,7 +456,6 @@ function Pipelines({ type }) {deleteRecord={deletePipeline}/></div>- <SocketMessages logId={logId} /></>);}diff --git a/frontend/src/components/settings/settings/Settings.jsx b/frontend/src/components/settings/settings/Settings.jsxindex f843c559a..490d19a8f 100644--- a/frontend/src/components/settings/settings/Settings.jsx+++ b/frontend/src/components/settings/settings/Settings.jsx@@ -45,6 +45,18 @@ function Settings() {Default Triad</Typography.Link></div>++ {sessionDetails?.isPlatformAdmin && (+ <div className="settings-plt">+ <Typography.Link+ className="settings-plt-typo"+ strong+ onClick={() => navigate("admin")}+ >+ Admin settings+ </Typography.Link>+ </div>+ )}</div></IslandLayout></div>diff --git a/frontend/src/components/tool-settings/tool-settings/ToolSettings.jsx b/frontend/src/components/tool-settings/tool-settings/ToolSettings.jsxindex 659539f3f..61fac5924 100644--- a/frontend/src/components/tool-settings/tool-settings/ToolSettings.jsx+++ b/frontend/src/components/tool-settings/tool-settings/ToolSettings.jsx@@ -213,6 +213,7 @@ function ToolSettings({ type }) {handleEdit={(_event, item) => setEditItemId(item?.id)}idProp="id"titleProp="adapter_name"+ descriptionProp="description"iconProp="icon"isEmpty={!tableRows?.length}centereddiff --git a/frontend/src/components/widgets/list-view/ListView.jsx b/frontend/src/components/widgets/list-view/ListView.jsxindex 441f72697..e34011539 100644--- a/frontend/src/components/widgets/list-view/ListView.jsx+++ b/frontend/src/components/widgets/list-view/ListView.jsx@@ -1,4 +1,12 @@-import { Avatar, Flex, Image, List, Popconfirm, Typography } from "antd";+import {+ Avatar,+ Flex,+ Image,+ List,+ Popconfirm,+ Tooltip,+ Typography,+} from "antd";import PropTypes from "prop-types";import "./ListView.css";import {@@ -9,6 +17,7 @@ import {UserOutlined,} from "@ant-design/icons";import { useNavigate } from "react-router-dom";+import { useSessionStore } from "../../../store/session-store";function ListView({@@ -157,7 +166,9 @@ function ListView({title={renderTitle(item)}description={<Typography.Text type="secondary" ellipsis>- {item[descriptionProp]}+ <Tooltip title={item[descriptionProp]}>+ {item[descriptionProp]}+ </Tooltip></Typography.Text>}></List.Item.Meta>diff --git a/frontend/src/helpers/GetSessionData.js b/frontend/src/helpers/GetSessionData.jsindex 8eb04f263..ef69ca57f 100644--- a/frontend/src/helpers/GetSessionData.js+++ b/frontend/src/helpers/GetSessionData.js@@ -22,6 +22,7 @@ function getSessionData(sessionData) {logEventsId: sessionData?.logEventsId,remainingTrialDays: sessionData?.remainingTrialDays,allOrganization: sessionData?.allOrganization,+ isPlatformAdmin: sessionData?.isPlatformAdmin,};}diff --git a/frontend/src/hooks/useSessionValid.js b/frontend/src/hooks/useSessionValid.jsindex 3f2026261..e81ae943e 100644--- a/frontend/src/hooks/useSessionValid.js+++ b/frontend/src/hooks/useSessionValid.js@@ -1,15 +1,19 @@import axios from "axios";+import Cookies from "js-cookie";+import { useNavigate } from "react-router-dom";import { getSessionData } from "../helpers/GetSessionData";-import Cookies from "js-cookie";-import { useSessionStore } from "../store/session-store";+import { useExceptionHandler } from "../hooks/useExceptionHandler.jsx";-import { useNavigate } from "react-router-dom";+import { useSessionStore } from "../store/session-store";import { useUserSession } from "./useUserSession.js";let getTrialDetails;+let isPlatformAdmin;try {getTrialDetails = require("../plugins/subscription/trial-helper/fetchTrialDetails.jsx");+ isPlatformAdmin =+ require("../plugins/hooks/usePlatformAdmin.js").usePlatformAdmin();} catch (err) {// Plugin not available}@@ -98,7 +102,9 @@ function useSessionValid() {userAndOrgDetails["remainingTrialDays"] = remainingTrialDays;}userAndOrgDetails["allOrganization"] = orgs;-+ if (isPlatformAdmin) {+ userAndOrgDetails["isPlatformAdmin"] = await isPlatformAdmin();+ }// Set the session detailssetSessionDetails(getSessionData(userAndOrgDetails));} catch (err) {diff --git a/frontend/src/routes/Router.jsx b/frontend/src/routes/Router.jsxindex 38c29edad..2def3ab18 100644--- a/frontend/src/routes/Router.jsx+++ b/frontend/src/routes/Router.jsx@@ -20,20 +20,26 @@ import { DeploymentsPage } from "../pages/DeploymentsPage.jsx";import { InviteEditUserPage } from "../pages/InviteEditUserPage.jsx";import { LandingPage } from "../pages/LandingPage.jsx";import { OnBoardPage } from "../pages/OnBoardPage.jsx";+import { OutputAnalyzerPage } from "../pages/OutputAnalyzerPage.jsx";import { PipelinesOrDeploymentsPage } from "../pages/PipelinesOrDeploymentsPage.jsx";import { ProfilePage } from "../pages/ProfilePage.jsx";+import { SetOrgPage } from "../pages/SetOrgPage.jsx";import { SettingsPage } from "../pages/SettingsPage.jsx";import { ToolIdePage } from "../pages/ToolIdePage.jsx";import { ToolsSettingsPage } from "../pages/ToolsSettingsPage.jsx";import { UsersPage } from "../pages/UsersPage.jsx";import { WorkflowsPage } from "../pages/WorkflowsPage.jsx";-import { OutputAnalyzerPage } from "../pages/OutputAnalyzerPage.jsx";-import { SetOrgPage } from "../pages/SetOrgPage.jsx";let TrialRoutes;+let RequirePlatformAdmin;+let PlatformAdminPage;try {TrialRoutes =require("../plugins/subscription/trial-page/TrialEndPage.jsx").TrialEndPage;+ RequirePlatformAdmin =+ require("../plugins/frictionless-onboard/RequirePlatformAdmin.jsx").RequirePlatformAdmin;+ PlatformAdminPage =+ require("../plugins/frictionless-onboard/platform-admin-page/PlatformAdminPage.jsx").PlatformAdminPage;} catch (err) {TrialRoutes = NotFound;}@@ -111,6 +117,11 @@ function Router() {<Route path="users/edit" element={<InviteEditUserPage />} /></Route><Route path="settings/triad" element={<DefaultTriad />} />+ {RequirePlatformAdmin && PlatformAdminPage && (+ <Route element={<RequirePlatformAdmin />}>+ <Route path="settings/admin" element={<PlatformAdminPage />} />+ </Route>+ )}</Route></Route>{TrialRoutes && (diff --git a/frontend/src/store/usage-store.js b/frontend/src/store/usage-store.jsnew file mode 100644index 000000000..1b97b3d8d--- /dev/null+++ b/frontend/src/store/usage-store.js@@ -0,0 +1,10 @@+import { create } from "zustand";++const useUsageStore = create((setState) => ({+ llmTokenUsage: null,+ setLLMTokenUsage: (data) => {+ setState({ llmTokenUsage: data });+ },+}));++export { useUsageStore };diff --git a/pdm.lock b/pdm.lockindex a299064e2..2218c836a 100644--- a/pdm.lock+++ b/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "hook-check-django-migrations", "lint"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:a426842385fd1039ecfd7e2b5c37a717241c377b8eb4e17f91f91dd28341653f"+content_hash = "sha256:af031601a5f06a763f954431a4943aa82931a379a79df6c68e59ea37e04dce9c"[[package]]name = "adlfs"@@ -4905,7 +4905,7 @@ dependencies = [[[package]]name = "unstract-sdk"-version = "0.22.1"+version = "0.23.0"requires_python = "<3.11.1,>=3.9"summary = "A framework for writing Unstract Tools/Apps"groups = ["hook-check-django-migrations"]@@ -4919,8 +4919,8 @@ dependencies = ["unstract-adapters~=0.12.1",]files = [- {file = "unstract_sdk-0.22.1-py3-none-any.whl", hash = "sha256:af29fec4c56c3c1916b7af639d331d9c7b115ebff6f1183e4be50bef01bd1792"},- {file = "unstract_sdk-0.22.1.tar.gz", hash = "sha256:f49db6c62bca6baa2474912a63f305df01ec12ada34af9a2fb93b8ce91e7fa16"},+ {file = "unstract_sdk-0.23.0-py3-none-any.whl", hash = "sha256:769bfa6048261241b0c2f76c64520d42992bcbba46403948d7474eb7e53740aa"},+ {file = "unstract_sdk-0.23.0.tar.gz", hash = "sha256:f327d1d4631947d611da7221667bdd82c40c836ae8d7fbef81eb8ad25818f165"},][[package]]diff --git a/platform-service/src/unstract/platform_service/main.py b/platform-service/src/unstract/platform_service/main.pyindex 702ca4238..de9591d00 100644--- a/platform-service/src/unstract/platform_service/main.py+++ b/platform-service/src/unstract/platform_service/main.py@@ -1,15 +1,14 @@-import datetime-import jsonimport loggingimport osimport uuid+from datetime import datetimefrom typing import Any, Literal, Optionalimport peeweeimport redisfrom cryptography.fernet import Fernetfrom dotenv import load_dotenv-from flask import Flask, Request, Response, jsonify, request+from flask import Flask, Request, Response, json, jsonify, make_response, requestfrom unstract.platform_service.exceptions import CustomExceptionfrom unstract.platform_service.helper import (AdapterInstanceRequestHelper,@@ -55,30 +54,15 @@ be_db.init(PG_BE_DATABASE)be_db.connect()-class UnstractUsage(peewee.Model):- id = peewee.UUIDField(primary_key=True, default=uuid.uuid4)- created_at = peewee.DateTimeField(default=datetime.datetime.now)- organization_id = peewee.CharField()- workflow_id = peewee.CharField()- execution_id = peewee.CharField()- status = peewee.CharField()- usage_type = peewee.CharField()- external_service = peewee.CharField()- tokens_valid = peewee.BooleanField(default=False)- embedding_tokens = peewee.IntegerField()- prompt_tokens = peewee.IntegerField()- completion_tokens = peewee.IntegerField()- total_tokens = peewee.IntegerField()- usage_value_valid = peewee.BooleanField(peewee.BooleanField(default=False))- usage_value = peewee.FloatField()- usage_units = peewee.CharField()-- class Meta:- database = be_db # This model uses the "BE_DB" database.- table_name = "unstract_usage"+def drop_unstract_usage_table() -> None:+ query = "DROP TABLE IF EXISTS unstract_usage"+ try:+ be_db.execute_sql(query)+ except Exception as e:+ app.logger.error(f"Error dropping 'unstract_usage' table: {e}")-UnstractUsage.create_table()+drop_unstract_usage_table()def get_token_from_auth_header(request: Request) -> Any:@@ -173,8 +157,6 @@ def usage() -> Any:....}'"""- bearer_token = get_token_from_auth_header(request)- organization_id = get_account_from_bearer_token(bearer_token)result: dict[str, Any] = {"status": "ERROR","error": "",@@ -183,43 +165,54 @@ def usage() -> Any:payload: Optional[dict[Any, Any]] = request.jsonif not payload:result["error"] = INVALID_PAYLOAD- return result, 400-- org_id = organization_id+ return make_response(result, 400)+ bearer_token = get_token_from_auth_header(request)+ org_id = get_account_from_bearer_token(bearer_token)workflow_id = payload.get("workflow_id")execution_id = payload.get("execution_id", "")- status = payload.get("status", "")+ adapter_instance_id = payload.get("adapter_instance_id", "")+ run_id = payload.get("run_id", "")usage_type = payload.get("usage_type", "")- external_service = payload.get("external_service", "")- tokens_valid = payload.get("tokens_valid", False)+ model_name = payload.get("model_name", "")embedding_tokens = payload.get("embedding_tokens", 0)prompt_tokens = payload.get("prompt_tokens", 0)completion_tokens = payload.get("completion_tokens", 0)total_tokens = payload.get("total_tokens", 0)- usage_value_valid = payload.get("usage_value_valid", False)- usage_value = payload.get("usage_value", 0.0)- usage_units = payload.get("usage_units", "")-- usage: UnstractUsage = UnstractUsage.create(- organization_id=org_id,- workflow_id=workflow_id,- execution_id=execution_id,- status=status,- usage_type=usage_type,- external_service=external_service,- tokens_valid=tokens_valid,- embedding_tokens=embedding_tokens,- prompt_tokens=prompt_tokens,- completion_tokens=completion_tokens,- total_tokens=total_tokens,- usage_value_valid=usage_value_valid,- usage_value=usage_value,- usage_units=usage_units,+ query = f"""+ INSERT INTO "{org_id}"."token_usage" (id, workflow_id, execution_id,+ adapter_instance_id, run_id, usage_type, model_name, embedding_tokens,+ prompt_tokens, completion_tokens, total_tokens, created_at, modified_at)+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)+ """+ usage_id = uuid.uuid4()+ current_time = datetime.now()+ params = (+ usage_id,+ workflow_id,+ execution_id,+ adapter_instance_id,+ run_id,+ usage_type,+ model_name,+ embedding_tokens,+ prompt_tokens,+ completion_tokens,+ total_tokens,+ current_time,+ current_time,)- app.logger.info("Entry created with id %s for %s", usage.id, organization_id)- result["status"] = "OK"- result["unique_id"] = usage.id- return result+ try:+ with be_db.atomic() as transaction:+ be_db.execute_sql(query, params)+ transaction.commit()+ app.logger.info("Entry created with id %s for %s", usage_id, org_id)+ result["status"] = "OK"+ result["unique_id"] = usage_id+ return make_response(result, 200)+ except Exception as e:+ app.logger.error(f"Error while creating usage entry: {e}")+ result["error"] = "Internal Server Error"+ return make_response(result, 500)@app.route(diff --git a/prompt-service/pdm.lock b/prompt-service/pdm.lockindex 6a3279630..47620bcc4 100644--- a/prompt-service/pdm.lock+++ b/prompt-service/pdm.lock@@ -5,7 +5,7 @@groups = ["default", "deploy"]strategy = ["cross_platform", "inherit_metadata"]lock_version = "4.4.1"-content_hash = "sha256:01e5530f1b1b786278ad93198c75ad40ec70926cd0a971cd2feeca1692586d4e"+content_hash = "sha256:08f2cd847abb3fbc248303f8f2c7bfbac22ef48e3605e8c12990bb6e7ec45d1d"[[package]]name = "aiohttp"@@ -3295,8 +3295,8 @@ requires_python = ">=3.7"summary = "Database Abstraction Library"groups = ["default"]dependencies = [+ "SQLAlchemy==2.0.29","greenlet!=0.4.17",- "sqlalchemy==2.0.29",]files = [{file = "SQLAlchemy-2.0.29-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4c142852ae192e9fe5aad5c350ea6befe9db14370b34047e1f0f7cf99e63c63b"},@@ -3637,7 +3637,7 @@ dependencies = [[[package]]name = "unstract-sdk"-version = "0.22.1"+version = "0.23.0"requires_python = "<3.11.1,>=3.9"summary = "A framework for writing Unstract Tools/Apps"groups = ["default"]@@ -3651,8 +3651,8 @@ dependencies = ["unstract-adapters~=0.12.1",]files = [- {file = "unstract_sdk-0.22.1-py3-none-any.whl", hash = "sha256:af29fec4c56c3c1916b7af639d331d9c7b115ebff6f1183e4be50bef01bd1792"},- {file = "unstract_sdk-0.22.1.tar.gz", hash = "sha256:f49db6c62bca6baa2474912a63f305df01ec12ada34af9a2fb93b8ce91e7fa16"},+ {file = "unstract_sdk-0.23.0-py3-none-any.whl", hash = "sha256:769bfa6048261241b0c2f76c64520d42992bcbba46403948d7474eb7e53740aa"},+ {file = "unstract_sdk-0.23.0.tar.gz", hash = "sha256:f327d1d4631947d611da7221667bdd82c40c836ae8d7fbef81eb8ad25818f165"},][[package]]diff --git a/prompt-service/pyproject.toml b/prompt-service/pyproject.tomlindex 769098f6a..983d04ec9 100644--- a/prompt-service/pyproject.toml+++ b/prompt-service/pyproject.toml@@ -15,7 +15,7 @@ dependencies = ["flask~=3.0","llama-index==0.10.28","python-dotenv==1.0.0",- "unstract-sdk~=0.22.1",+ "unstract-sdk~=0.23.0","redis>=5.0.3","unstract-core @ file:///${PROJECT_ROOT}/../unstract/core",]diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.pyindex 205667f9a..2e0471a10 100644--- a/prompt-service/src/unstract/prompt_service/constants.py+++ b/prompt-service/src/unstract/prompt_service/constants.py@@ -45,7 +45,6 @@ class PromptServiceContants:TOOL_SETTINGS = "tool_settings"LOG_EVENTS_ID = "log_events_id"CHALLENGE_LLM = "challenge_llm"- USAGE = "usage"class LogLevel(Enum):diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.pyindex 217ef8e48..ea11608ee 100644--- a/prompt-service/src/unstract/prompt_service/main.py+++ b/prompt-service/src/unstract/prompt_service/main.py@@ -161,7 +161,6 @@ def authentication_middleware(func: Any) -> Any:@authentication_middlewaredef prompt_processor() -> Any:result: dict[str, Any] = {}- usage = {}platform_key = AuthenticationMiddleware.get_token_from_auth_header(request)if request.method == "POST":payload: dict[Any, Any] = request.json@@ -192,6 +191,7 @@ def prompt_processor() -> Any:chunk_size = output[PSKeys.CHUNK_SIZE]util = PromptServiceBaseTool(log_level=LogLevel.INFO, platform_key=platform_key)tool_index = ToolIndex(tool=util)+ adapter_instance_id = output[PSKeys.LLM]if active is False:app.logger.info(f"[{tool_id}] Skipping inactive prompt: {name}")@@ -239,10 +239,10 @@ def prompt_processor() -> Any:llm_helper = ToolLLM(tool=util)llm_li: Optional[LLM] = llm_helper.get_llm(- adapter_instance_id=output[PSKeys.LLM]+ adapter_instance_id=adapter_instance_id)if llm_li is None:- msg = f"Couldn't fetch LLM {output[PSKeys.LLM]}"+ msg = f"Couldn't fetch LLM {adapter_instance_id}"app.logger.error(msg)_publish_log(log_events_id,@@ -321,7 +321,7 @@ def prompt_processor() -> Any:is_assert = output[PSKeys.IS_ASSERT]if is_assert:app.logger.debug(f'Asserting prompt: {output["assert_prompt"]}')- answer, usage = construct_and_run_prompt(+ answer = construct_and_run_prompt(output,llm_helper,llm_li,@@ -349,7 +349,7 @@ def prompt_processor() -> Any:]app.logger.info(f"[Assigning] {answer} to the output")else:- answer, usage = construct_and_run_prompt(+ answer = construct_and_run_prompt(output,llm_helper,llm_li,@@ -358,7 +358,7 @@ def prompt_processor() -> Any:)else:if chunk_size == 0:- answer, usage = construct_and_run_prompt(+ answer = construct_and_run_prompt(output,llm_helper,llm_li,@@ -376,7 +376,7 @@ def prompt_processor() -> Any:)if output[PSKeys.RETRIEVAL_STRATEGY] == PSKeys.SIMPLE:- answer, context, usage = simple_retriver(+ answer, context = simple_retriver(output,doc_id,llm_helper,@@ -420,10 +420,11 @@ def prompt_processor() -> Any:percentages or other grouping \characters. No explanation is required.\If you cannot extract the number, output 0."- answer, usage = run_completion(- llm_helper,- llm_li,- prompt,+ answer = run_completion(+ llm_helper=llm_helper,+ llm_li=llm_li,+ prompt=prompt,+ adapter_instance_id=adapter_instance_id,)try:structured_output[output[PSKeys.NAME]] = float(answer)@@ -440,10 +441,11 @@ def prompt_processor() -> Any:prompt = f'Extract the email from the following text:\n{answer}\n\nOutput just the email. \The email should be directly assignable to a string variable. \No explanation is required. If you cannot extract the email, output "NA".' # noqa- answer, usage = run_completion(- llm_helper,- llm_li,- prompt,+ answer = run_completion(+ llm_helper=llm_helper,+ llm_li=llm_li,+ prompt=prompt,+ adapter_instance_id=adapter_instance_id,)structured_output[output[PSKeys.NAME]] = answerelif output[PSKeys.TYPE] == PSKeys.DATE:@@ -454,10 +456,11 @@ def prompt_processor() -> Any:The date should be in ISO date time format. No explanation is required. \The date should be directly assignable to a date variable. \If you cannot convert the string into a date, output "NA".' # noqa- answer, usage = run_completion(- llm_helper,- llm_li,- prompt,+ answer = run_completion(+ llm_helper=llm_helper,+ llm_li=llm_li,+ prompt=prompt,+ adapter_instance_id=adapter_instance_id,)structured_output[output[PSKeys.NAME]] = answer@@ -469,10 +472,11 @@ def prompt_processor() -> Any:Output in single word.\If the context is trying to convey that the answer is true, \then return "yes", else return "no".'- answer, usage = run_completion(- llm_helper,- llm_li,- prompt,+ answer = run_completion(+ llm_helper=llm_helper,+ llm_li=llm_li,+ prompt=prompt,+ adapter_instance_id=adapter_instance_id,)if answer.lower() == "yes":structured_output[output[PSKeys.NAME]] = True@@ -487,11 +491,11 @@ def prompt_processor() -> Any:into a Python dictionary. \Output just the JSON string. No explanation is required. \If you cannot extract the JSON string, output {{}}"-- answer, usage = run_completion(- llm_helper,- llm_li,- prompt,+ answer = run_completion(+ llm_helper=llm_helper,+ llm_li=llm_li,+ prompt=prompt,+ adapter_instance_id=adapter_instance_id,)try:structured_output[output[PSKeys.NAME]] = json.loads(answer)@@ -519,7 +523,7 @@ def prompt_processor() -> Any:PSKeys.PREAMBLE: output[PSKeys.PREAMBLE],PSKeys.POSTAMBLE: output[PSKeys.POSTAMBLE],PSKeys.GRAMMAR: output[PSKeys.GRAMMAR],- PSKeys.LLM: output[PSKeys.LLM],+ PSKeys.LLM: adapter_instance_id,PSKeys.CHALLENGE_LLM: output[PSKeys.CHALLENGE_LLM],}challenge = challenge_plugin["entrypoint_cls"](@@ -626,7 +630,6 @@ def prompt_processor() -> Any:RunLevel.RUN,"Execution complete",)- app.logger.info("Usage details : %s", str(usage))return structured_output@@ -647,10 +650,11 @@ def simple_retriver( # type:ignoref"Generate a sub-question from the following verbose prompt that will"f" help extract relevant documents from a vector store:\n\n{prompt}")- answer, usage = run_completion(- llm_helper,- llm_li,- subq_prompt,+ answer = run_completion(+ llm_helper=llm_helper,+ llm_li=llm_li,+ prompt=subq_prompt,+ adapter_instance_id=output[PSKeys.LLM],)retriever = vector_index.as_retriever(@@ -672,14 +676,14 @@ def simple_retriver( # type:ignoreelse:app.logger.info("Node score is less than 0.6. " f"Ignored: {node.score}")- answer, usage = construct_and_run_prompt( # type:ignore+ answer = construct_and_run_prompt( # type:ignoreoutput,llm_helper,llm_li,text,"promptx",)- return (answer, text, usage)+ return (answer, text)def construct_and_run_prompt(@@ -697,9 +701,10 @@ def construct_and_run_prompt(context=context,)return run_completion(- llm_helper,- llm_li,- prompt,+ llm_helper=llm_helper,+ llm_li=llm_li,+ prompt=prompt,+ adapter_instance_id=output[PSKeys.LLM],)@@ -707,18 +712,21 @@ def run_completion(llm_helper: ToolLLM,llm_li: Optional[LLM],prompt: str,+ adapter_instance_id: str,) -> tuple[str, dict[str, Any]]:try:platform_api_key = llm_helper.tool.get_env_or_die(PSKeys.PLATFORM_SERVICE_API_KEY)- completion = llm_helper.run_completion(llm_li, platform_api_key, prompt, 3)+ completion = llm_helper.run_completion(+ llm=llm_li,+ platform_api_key=platform_api_key,+ prompt=prompt,+ adapter_instance_id=adapter_instance_id,+ )answer: str = completion[PSKeys.RESPONSE].text- usage = {}- if PSKeys.USAGE in completion:- usage = completion[PSKeys.USAGE]- return answer, usage+ return answerexcept Exception as e:app.logger.info(f"Error completing prompt: {e}.")raise ediff --git a/pyproject.toml b/pyproject.tomlindex aa4772dc4..ed27f11ed 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -40,7 +40,7 @@ hook-check-django-migrations = ["psycopg2-binary==2.9.9","python-dotenv==1.0.0","python-magic==0.4.27",- "unstract-sdk~=0.22.1",+ "unstract-sdk~=0.23.0","unstract-adapters~=0.12.1","-e unstract-connectors @ file:///${PROJECT_ROOT}/unstract/connectors","-e unstract-core @ file:///${PROJECT_ROOT}/unstract/core",diff --git a/tools/classifier/requirements.txt b/tools/classifier/requirements.txtindex f9fd56248..a4baabc2b 100644--- a/tools/classifier/requirements.txt+++ b/tools/classifier/requirements.txt@@ -1,4 +1,4 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=0.22.1+unstract-sdk~=0.23.0diff --git a/tools/classifier/src/config/properties.json b/tools/classifier/src/config/properties.jsonindex 688acbacc..4f820745c 100644--- a/tools/classifier/src/config/properties.json+++ b/tools/classifier/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "File Classifier","functionName": "classify",- "toolVersion": "0.0.13",+ "toolVersion": "0.0.14","description": "Classifies a file into a bin based on its contents","input": {"description": "File to be classified"diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txtindex f9fd56248..a4baabc2b 100644--- a/tools/structure/requirements.txt+++ b/tools/structure/requirements.txt@@ -1,4 +1,4 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=0.22.1+unstract-sdk~=0.23.0diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.jsonindex ba112333d..424bdd35e 100644--- a/tools/structure/src/config/properties.json+++ b/tools/structure/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Structure Tool","functionName": "structure_tool",- "toolVersion": "0.0.16",+ "toolVersion": "0.0.17","description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio","input": {"description": "File that needs to be indexed and parsed for answers"diff --git a/tools/text_extractor/requirements.txt b/tools/text_extractor/requirements.txtindex f9fd56248..a4baabc2b 100644--- a/tools/text_extractor/requirements.txt+++ b/tools/text_extractor/requirements.txt@@ -1,4 +1,4 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=0.22.1+unstract-sdk~=0.23.0diff --git a/tools/text_extractor/src/config/properties.json b/tools/text_extractor/src/config/properties.jsonindex fb1fd2cda..e20a737fa 100644--- a/tools/text_extractor/src/config/properties.json+++ b/tools/text_extractor/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Text Extractor","functionName": "text_extractor",- "toolVersion": "0.0.12",+ "toolVersion": "0.0.13","description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents","input": {"description": "Document"diff --git a/unstract/core/src/unstract/core/pubsub_helper.py b/unstract/core/src/unstract/core/pubsub_helper.pyindex f51d2a8b8..855a4d3f6 100644--- a/unstract/core/src/unstract/core/pubsub_helper.py+++ b/unstract/core/src/unstract/core/pubsub_helper.py@@ -15,6 +15,23 @@ class LogPublisher:password=os.environ.get("REDIS_PASSWORD", ""),)+ @staticmethod+ def log_usage(+ level: str = "INFO",+ added_token_count: Optional[int] = None,+ max_token_count_set: Optional[int] = None,+ enabled: bool = False,+ ) -> dict[str, Any]:+ return {+ "timestamp": datetime.now(timezone.utc).timestamp(),+ "type": "LOG",+ "service": "usage",+ "level": level,+ "added_token_count": added_token_count,+ "max_token_count_set": max_token_count_set,+ "enabled": enabled,+ }+@staticmethoddef log_workflow(stage: str,