Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Feat/friction less onboarding and usage reporting (#256)

Rahul Johny9d49cdb0883d
diff --git a/backend/account/authentication_controller.py b/backend/account/authentication_controller.py
index 9160f8d13..e9098444c 100644
--- a/backend/account/authentication_controller.py
+++ b/backend/account/authentication_controller.py
@@ -11,7 +11,12 @@ from account.constants import (
ErrorMessage,
OrganizationMemberModel,
)
-from account.custom_exceptions import DuplicateData, Forbidden, UserNotExistError
+from account.custom_exceptions import (
+ DuplicateData,
+ Forbidden,
+ MethodNotImplemented,
+ UserNotExistError,
+)
from account.dto import (
MemberInvitation,
OrganizationData,
@@ -173,6 +178,15 @@ class AuthenticationController:
{ErrorMessage.DUPLICATE_API}"
)
self.create_tenant_user(organization=organization, user=user)
+
+ if new_organization:
+ try:
+ self.auth_service.frictionless_onboarding(
+ organization=organization, user=user
+ )
+ except MethodNotImplemented:
+ Logger.info("frictionless_onboarding not implemented")
+
if new_organization:
self.authentication_helper.create_initial_platform_key(
user=user, organization=organization
@@ -191,7 +205,8 @@ class AuthenticationController:
current_organization_id = UserSessionUtils.get_organization_id(request)
if current_organization_id:
OrganizationMemberService.remove_user_membership_in_organization_cache(
- user_id=user.user_id, organization_id=current_organization_id
+ user_id=user.user_id,
+ organization_id=current_organization_id,
)
UserSessionUtils.set_organization_id(request, organization_id)
OrganizationMemberService.set_user_membership_in_organization_cache(
diff --git a/backend/account/authentication_helper.py b/backend/account/authentication_helper.py
index fcf69150e..928624e07 100644
--- a/backend/account/authentication_helper.py
+++ b/backend/account/authentication_helper.py
@@ -43,6 +43,8 @@ class AuthenticationHelper:
"""
user_service = UserService()
user = user_service.get_user_by_email(email)
+ if user and not user.user_id:
+ user = user_service.update_user(user, user_id)
if not user:
user = user_service.create_user(email, user_id)
return user
diff --git a/backend/account/authentication_service.py b/backend/account/authentication_service.py
index fe4ed2bac..debc9d58e 100644
--- a/backend/account/authentication_service.py
+++ b/backend/account/authentication_service.py
@@ -256,6 +256,9 @@ class AuthenticationService:
def get_invitations(self, organization_id: str) -> list[MemberInvitation]:
raise MethodNotImplemented()
+ def frictionless_onboarding(self, organization: Organization, user: User) -> None:
+ raise MethodNotImplemented()
+
def delete_invitation(self, organization_id: str, invitation_id: str) -> bool:
raise MethodNotImplemented()
diff --git a/backend/account/migrations/0007_organization_allowed_token_limit.py b/backend/account/migrations/0007_organization_allowed_token_limit.py
new file mode 100644
index 000000000..af6baa8a1
--- /dev/null
+++ b/backend/account/migrations/0007_organization_allowed_token_limit.py
@@ -0,0 +1,20 @@
+# Generated by Django 4.2.1 on 2024-04-25 07:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("account", "0006_delete_encryptionsecret"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="organization",
+ name="allowed_token_limit",
+ field=models.IntegerField(
+ db_comment="token limit set in case of frition less onbaoarded org",
+ default=-1,
+ ),
+ ),
+ ]
diff --git a/backend/account/models.py b/backend/account/models.py
index 2eae51073..063a7f7cd 100644
--- a/backend/account/models.py
+++ b/backend/account/models.py
@@ -36,6 +36,10 @@ class Organization(TenantMixin):
)
modified_at = models.DateTimeField(auto_now=True)
created_at = models.DateTimeField(auto_now=True)
+ allowed_token_limit = models.IntegerField(
+ default=-1,
+ db_comment="token limit set in case of frition less onbaoarded org",
+ )
auto_create_schema = True
diff --git a/backend/account/user.py b/backend/account/user.py
index f875d4572..c1499e69c 100644
--- a/backend/account/user.py
+++ b/backend/account/user.py
@@ -22,6 +22,11 @@ class UserService:
raise error
return user
+ def update_user(self, user: User, user_id: str) -> User:
+ user.user_id = user_id
+ user.save()
+ return user
+
def get_user_by_email(self, email: str) -> Optional[User]:
try:
user: User = User.objects.get(email=email)
diff --git a/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py b/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py
index d5a9bb4cf..75abf4ae1 100644
--- a/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py
+++ b/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py
@@ -3,7 +3,6 @@
import json
from typing import Any
-from adapter_processor.models import AdapterInstance
from cryptography.fernet import Fernet
from django.conf import settings
from django.db import migrations, models
@@ -17,6 +16,7 @@ class Migration(migrations.Migration):
def EncryptCredentials(apps: Any, schema_editor: Any) -> None:
encryption_secret: str = settings.ENCRYPTION_KEY
f: Fernet = Fernet(encryption_secret.encode("utf-8"))
+ AdapterInstance = apps.get_model("adapter_processor", "AdapterInstance")
queryset = AdapterInstance.objects.all()
for obj in queryset: # type: ignore
diff --git a/backend/adapter_processor/migrations/0008_adapterinstance_description_and_more.py b/backend/adapter_processor/migrations/0008_adapterinstance_description_and_more.py
new file mode 100644
index 000000000..ec2c607c6
--- /dev/null
+++ b/backend/adapter_processor/migrations/0008_adapterinstance_description_and_more.py
@@ -0,0 +1,41 @@
+# Generated by Django 4.2.1 on 2024-04-29 05:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ (
+ "adapter_processor",
+ "0007_remove_adapterinstance_is_default_userdefaultadapter",
+ ),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="adapterinstance",
+ name="description",
+ field=models.TextField(blank=True, default=None, null=True),
+ ),
+ migrations.AddField(
+ model_name="adapterinstance",
+ name="is_friction_less",
+ field=models.BooleanField(
+ db_comment="Was the adapter created through frictionless onboarding",
+ default=False,
+ ),
+ ),
+ migrations.AddField(
+ model_name="adapterinstance",
+ name="is_usable",
+ field=models.BooleanField(db_comment="Is the Adpater Usable", default=True),
+ ),
+ migrations.AddField(
+ model_name="adapterinstance",
+ name="shared_to_org",
+ field=models.BooleanField(
+ db_comment="Is the adapter shared to entire org", default=False
+ ),
+ ),
+ ]
diff --git a/backend/adapter_processor/models.py b/backend/adapter_processor/models.py
index fd84db17c..5a2699f9f 100644
--- a/backend/adapter_processor/models.py
+++ b/backend/adapter_processor/models.py
@@ -24,7 +24,12 @@ class AdapterInstanceModelManager(models.Manager):
def for_user(self, user: User) -> QuerySet[Any]:
return (
self.get_queryset()
- .filter(models.Q(created_by=user) | models.Q(shared_users=user))
+ .filter(
+ models.Q(created_by=user)
+ | models.Q(shared_users=user)
+ | models.Q(shared_to_org=True)
+ | models.Q(is_friction_less=True)
+ )
.distinct("id")
)
@@ -80,10 +85,27 @@ class AdapterInstance(BaseModel):
default=False,
db_comment="Is the adapter instance currently being used",
)
+ shared_to_org = models.BooleanField(
+ default=False,
+ db_comment="Is the adapter shared to entire org",
+ )
+
+ is_friction_less = models.BooleanField(
+ default=False,
+ db_comment="Was the adapter created through frictionless onboarding",
+ )
+
+ # Can be used if the adapter usage gets exhausted
+ # Can also be used in other possible scenarios in feature
+ is_usable = models.BooleanField(
+ default=True,
+ db_comment="Is the Adpater Usable",
+ )
# Introduced field to establish M2M relation between users and adapters.
# This will introduce intermediary table which relates both the models.
shared_users = models.ManyToManyField(User, related_name="shared_adapters")
+ description = models.TextField(blank=True, null=True, default=None)
objects = AdapterInstanceModelManager()
@@ -98,6 +120,18 @@ class AdapterInstance(BaseModel):
),
]
+ def create_adapter(self) -> None:
+
+ encryption_secret: str = settings.ENCRYPTION_KEY
+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))
+
+ self.adapter_metadata_b = f.encrypt(
+ json.dumps(self.adapter_metadata).encode("utf-8")
+ )
+ self.adapter_metadata = {}
+
+ self.save()
+
def get_adapter_meta_data(self) -> Any:
encryption_secret: str = settings.ENCRYPTION_KEY
f: Fernet = Fernet(encryption_secret.encode("utf-8"))
diff --git a/backend/adapter_processor/serializers.py b/backend/adapter_processor/serializers.py
index bb570c1bc..087cfe221 100644
--- a/backend/adapter_processor/serializers.py
+++ b/backend/adapter_processor/serializers.py
@@ -108,6 +108,7 @@ class AdapterListSerializer(BaseAdapterSerializer):
"adapter_name",
"adapter_type",
"created_by",
+ "description",
) # type: ignore
def to_representation(self, instance: AdapterInstance) -> dict[str, str]:
@@ -115,7 +116,11 @@ class AdapterListSerializer(BaseAdapterSerializer):
rep[common.ICON] = AdapterProcessor.get_adapter_data_with_key(
instance.adapter_id, common.ICON
)
- rep["created_by_email"] = instance.created_by.email
+
+ if instance.is_friction_less:
+ rep["created_by_email"] = "Unstract"
+ else:
+ rep["created_by_email"] = instance.created_by.email
return rep
diff --git a/backend/adapter_processor/views.py b/backend/adapter_processor/views.py
index 37ca36886..492d593f6 100644
--- a/backend/adapter_processor/views.py
+++ b/backend/adapter_processor/views.py
@@ -24,7 +24,12 @@ from django.db import IntegrityError
from django.db.models import ProtectedError, QuerySet
from django.http import HttpRequest
from django.http.response import HttpResponse
-from permissions.permission import IsOwner, IsOwnerOrSharedUser
+from permissions.permission import (
+ IsFrictionLessAdapter,
+ IsFrictionLessAdapterDelete,
+ IsOwner,
+ IsOwnerOrSharedUser,
+)
from rest_framework import status
from rest_framework.decorators import action
from rest_framework.request import Request
@@ -124,9 +129,21 @@ class AdapterViewSet(GenericViewSet):
class AdapterInstanceViewSet(ModelViewSet):
- permission_classes: list[type[IsOwner]] = [IsOwner]
+
serializer_class = AdapterInstanceSerializer
+ def get_permissions(self) -> list[Any]:
+
+ if self.action in ["update", "retrieve"]:
+ return [IsFrictionLessAdapter()]
+
+ elif self.action == "destroy":
+ return [IsFrictionLessAdapterDelete()]
+
+ # Hack for friction-less onboarding
+ # User cant view/update metadata but can delete/share etc
+ return [IsOwner()]
+
def get_queryset(self) -> Optional[QuerySet]:
if filter_args := FilterHelper.build_filter_args(
self.request,
@@ -195,27 +212,36 @@ class AdapterInstanceViewSet(ModelViewSet):
) -> Response:
adapter_instance: AdapterInstance = self.get_object()
adapter_type = adapter_instance.adapter_type
- user_default_adapter = UserDefaultAdapter.objects.get(user=request.user)
- if (
- (
- adapter_type == AdapterKeys.LLM
- and adapter_instance == user_default_adapter.default_llm_adapter
- )
- or (
- adapter_type == AdapterKeys.EMBEDDING
- and adapter_instance == user_default_adapter.default_embedding_adapter
- )
- or (
- adapter_type == AdapterKeys.VECTOR_DB
- and adapter_instance == user_default_adapter.default_vector_db_adapter
- )
- or (
- adapter_type == AdapterKeys.X2TEXT
- and adapter_instance == user_default_adapter.default_x2text_adapter
+ try:
+ user_default_adapter: UserDefaultAdapter = UserDefaultAdapter.objects.get(
+ user=request.user
)
- ):
- logger.error("Cannot delete a default adapter")
- raise CannotDeleteDefaultAdapter()
+
+ if (
+ (
+ adapter_type == AdapterKeys.LLM
+ and adapter_instance == user_default_adapter.default_llm_adapter
+ )
+ or (
+ adapter_type == AdapterKeys.EMBEDDING
+ and adapter_instance
+ == user_default_adapter.default_embedding_adapter
+ )
+ or (
+ adapter_type == AdapterKeys.VECTOR_DB
+ and adapter_instance
+ == user_default_adapter.default_vector_db_adapter
+ )
+ or (
+ adapter_type == AdapterKeys.X2TEXT
+ and adapter_instance == user_default_adapter.default_x2text_adapter
+ )
+ ):
+ logger.error("Cannot delete a default adapter")
+ raise CannotDeleteDefaultAdapter()
+ except UserDefaultAdapter.DoesNotExist:
+ # We can go head and remove adapter here
+ logger.info("User default adpater doesnt not exist")
try:
super().perform_destroy(adapter_instance)
diff --git a/backend/backend/public_urls.py b/backend/backend/public_urls.py
index e81a265f4..27ece8786 100644
--- a/backend/backend/public_urls.py
+++ b/backend/backend/public_urls.py
@@ -44,3 +44,13 @@ if settings.ADMIN_ENABLED:
),
]
urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
+
+
+try:
+ import pluggable_apps.platform_admin.urls # noqa: F401
+
+ urlpatterns += [
+ path(f"{path_prefix}/", include("pluggable_apps.platform_admin.urls")),
+ ]
+except ImportError:
+ pass
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 5575730a5..82ce2d481 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -221,6 +221,7 @@ TENANT_APPS = (
"prompt_studio.prompt_studio_output_manager",
"prompt_studio.prompt_studio_document_manager",
"prompt_studio.prompt_studio_index_manager",
+ "usage",
)
INSTALLED_APPS = list(SHARED_APPS) + [
diff --git a/backend/pdm.lock b/backend/pdm.lock
index 343ddca48..0492a11f6 100644
--- a/backend/pdm.lock
+++ b/backend/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "deploy", "test"]
strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.1"
-content_hash = "sha256:d32c9f74e40f02d7c0f0af6a18e11038de233982b899a716c4571c50543ba2d7"
+content_hash = "sha256:217a1cf800d393c22fe30815724239a07c6578d6b75799296ba8e4df6cf0ad3d"
[[package]]
name = "adlfs"
@@ -366,6 +366,16 @@ files = [
{file = "Authlib-1.2.1.tar.gz", hash = "sha256:421f7c6b468d907ca2d9afede256f068f87e34d23dd221c07d13d4c234726afb"},
]
+[[package]]
+name = "azure-common"
+version = "1.1.28"
+summary = "Microsoft Azure Client Library for Python (Common)"
+groups = ["default"]
+files = [
+ {file = "azure-common-1.1.28.zip", hash = "sha256:4ac0cd3214e36b6a1b6a442686722a5d8cc449603aa833f3f0f40bda836704a3"},
+ {file = "azure_common-1.1.28-py2.py3-none-any.whl", hash = "sha256:5c12d3dcf4ec20599ca6b0d3e09e86e146353d443e7fcc050c9a19c1f9df20ad"},
+]
+
[[package]]
name = "azure-core"
version = "1.30.1"
@@ -414,6 +424,36 @@ files = [
{file = "azure_identity-1.16.0-py3-none-any.whl", hash = "sha256:722fdb60b8fdd55fa44dc378b8072f4b419b56a5e54c0de391f644949f3a826f"},
]
+[[package]]
+name = "azure-mgmt-apimanagement"
+version = "3.0.0"
+requires_python = ">=3.6"
+summary = "Microsoft Azure API Management Client Library for Python"
+groups = ["default"]
+dependencies = [
+ "azure-common~=1.1",
+ "azure-mgmt-core<2.0.0,>=1.3.0",
+ "msrest>=0.6.21",
+]
+files = [
+ {file = "azure-mgmt-apimanagement-3.0.0.zip", hash = "sha256:9262f54ed387eb083d8dae66d32a8df35647319b902bd498cdc376f50a12d154"},
+ {file = "azure_mgmt_apimanagement-3.0.0-py3-none-any.whl", hash = "sha256:58a96261266c51c63a64113dbb0ad96cabbf3b6c32fb9b79c395462d01470964"},
+]
+
+[[package]]
+name = "azure-mgmt-core"
+version = "1.4.0"
+requires_python = ">=3.7"
+summary = "Microsoft Azure Management Core Library for Python"
+groups = ["default"]
+dependencies = [
+ "azure-core<2.0.0,>=1.26.2",
+]
+files = [
+ {file = "azure-mgmt-core-1.4.0.zip", hash = "sha256:d195208340094f98e5a6661b781cde6f6a051e79ce317caabd8ff97030a9b3ae"},
+ {file = "azure_mgmt_core-1.4.0-py3-none-any.whl", hash = "sha256:81071675f186a585555ef01816f2774d49c1c9024cb76e5720c3c0f6b337bb7d"},
+]
+
[[package]]
name = "azure-storage-blob"
version = "12.19.1"
@@ -2069,7 +2109,7 @@ files = [
[[package]]
name = "llama-index-core"
-version = "0.10.31"
+version = "0.10.32"
requires_python = "<4.0,>=3.8.1"
summary = "Interface between LLMs and your data"
groups = ["default"]
@@ -2099,8 +2139,8 @@ dependencies = [
"wrapt",
]
files = [
- {file = "llama_index_core-0.10.31-py3-none-any.whl", hash = "sha256:b894680fa320a94de56d9a933ac7edb646cabf15fe67ae1cf8fa53ac52ab4542"},
- {file = "llama_index_core-0.10.31.tar.gz", hash = "sha256:66d39d6f253e20311a21e0b98ea386089f099be12f2d23dbe11379a6d908ddf1"},
+ {file = "llama_index_core-0.10.32-py3-none-any.whl", hash = "sha256:215f7389dadb78f2df13c20312a3e1e03c41f23e3063907469c4bae67bfd458c"},
+ {file = "llama_index_core-0.10.32.tar.gz", hash = "sha256:0078c06d9143390e14c86a40e69716c88c7828533341559edd15e52249ede65a"},
]
[[package]]
@@ -2136,7 +2176,7 @@ files = [
[[package]]
name = "llama-index-embeddings-openai"
-version = "0.1.8"
+version = "0.1.9"
requires_python = "<4.0,>=3.8.1"
summary = "llama-index embeddings openai integration"
groups = ["default"]
@@ -2144,8 +2184,8 @@ dependencies = [
"llama-index-core<0.11.0,>=0.10.1",
]
files = [
- {file = "llama_index_embeddings_openai-0.1.8-py3-none-any.whl", hash = "sha256:186be52c29a8c82a39ff6e033e3a61b9c5a17cf6177a5507abe4f5c79fea3a53"},
- {file = "llama_index_embeddings_openai-0.1.8.tar.gz", hash = "sha256:aae4ce3ec2fe2cb604d944646982c71663a8b99ed5f8c370823ee82eef4ddd3b"},
+ {file = "llama_index_embeddings_openai-0.1.9-py3-none-any.whl", hash = "sha256:fbd16d6197b91f4dbdc6d0707e573cc224ac2b0a48d5b370c6232dd8a2282473"},
+ {file = "llama_index_embeddings_openai-0.1.9.tar.gz", hash = "sha256:0fd292b2f9a0ad4534a790d6374726bc885853188087eb018167dcf239643924"},
]
[[package]]
@@ -2517,7 +2557,7 @@ files = [
[[package]]
name = "minio"
-version = "7.2.5"
+version = "7.2.6"
summary = "MinIO Python SDK for Amazon S3 Compatible Cloud Storage"
groups = ["default"]
dependencies = [
@@ -2528,8 +2568,8 @@ dependencies = [
"urllib3",
]
files = [
- {file = "minio-7.2.5-py3-none-any.whl", hash = "sha256:ed9176c96d4271cb1022b9ecb8a538b1e55b32ae06add6de16425cab99ef2304"},
- {file = "minio-7.2.5.tar.gz", hash = "sha256:59d8906e2da248a9caac34d4958a859cc3a44abbe6447910c82b5abfa9d6a2e1"},
+ {file = "minio-7.2.6-py3-none-any.whl", hash = "sha256:4972273a924f274e2d71f38f6d2afdf841a034801e60ba758e5c5aff4234b768"},
+ {file = "minio-7.2.6.tar.gz", hash = "sha256:c545d0dda1ff26cefcfc754242be3d27a4e620e37ef3e51ecbe7212cf7ecc274"},
]
[[package]]
@@ -2581,6 +2621,24 @@ files = [
{file = "msal_extensions-1.1.0-py3-none-any.whl", hash = "sha256:01be9711b4c0b1a151450068eeb2c4f0997df3bba085ac299de3a66f585e382f"},
]
+[[package]]
+name = "msrest"
+version = "0.7.1"
+requires_python = ">=3.6"
+summary = "AutoRest swagger generator Python client runtime."
+groups = ["default"]
+dependencies = [
+ "azure-core>=1.24.0",
+ "certifi>=2017.4.17",
+ "isodate>=0.6.0",
+ "requests-oauthlib>=0.5.0",
+ "requests~=2.16",
+]
+files = [
+ {file = "msrest-0.7.1-py3-none-any.whl", hash = "sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32"},
+ {file = "msrest-0.7.1.zip", hash = "sha256:6e7661f46f3afd88b75667b7187a92829924446c7ea1d169be8c4bb7eeb788b9"},
+]
+
[[package]]
name = "multidict"
version = "6.0.5"
@@ -4721,7 +4779,7 @@ dependencies = [
[[package]]
name = "unstract-sdk"
-version = "0.22.1"
+version = "0.23.0"
requires_python = "<3.11.1,>=3.9"
summary = "A framework for writing Unstract Tools/Apps"
groups = ["default"]
@@ -4735,8 +4793,8 @@ dependencies = [
"unstract-adapters~=0.12.1",
]
files = [
- {file = "unstract_sdk-0.22.1-py3-none-any.whl", hash = "sha256:af29fec4c56c3c1916b7af639d331d9c7b115ebff6f1183e4be50bef01bd1792"},
- {file = "unstract_sdk-0.22.1.tar.gz", hash = "sha256:f49db6c62bca6baa2474912a63f305df01ec12ada34af9a2fb93b8ce91e7fa16"},
+ {file = "unstract_sdk-0.23.0-py3-none-any.whl", hash = "sha256:769bfa6048261241b0c2f76c64520d42992bcbba46403948d7474eb7e53740aa"},
+ {file = "unstract_sdk-0.23.0.tar.gz", hash = "sha256:f327d1d4631947d611da7221667bdd82c40c836ae8d7fbef81eb8ad25818f165"},
]
[[package]]
diff --git a/backend/permissions/permission.py b/backend/permissions/permission.py
index bf2f85906..5668c9f66 100644
--- a/backend/permissions/permission.py
+++ b/backend/permissions/permission.py
@@ -1,5 +1,6 @@
from typing import Any
+from adapter_processor.models import AdapterInstance
from rest_framework import permissions
from rest_framework.request import Request
from rest_framework.views import APIView
@@ -9,6 +10,7 @@ class IsOwner(permissions.BasePermission):
"""Custom permission to only allow owners of an object."""
def has_object_permission(self, request: Request, view: APIView, obj: Any) -> bool:
+
return True if obj.created_by == request.user else False
@@ -24,3 +26,31 @@ class IsOwnerOrSharedUser(permissions.BasePermission):
)
else False
)
+
+
+class IsFrictionLessAdapter(permissions.BasePermission):
+ """Hack for friction-less onboarding not allowing user to view or updating
+ friction less adapter."""
+
+ def has_object_permission(
+ self, request: Request, view: APIView, obj: AdapterInstance
+ ) -> bool:
+
+ if obj.is_friction_less:
+ return False
+
+ return True if obj.created_by == request.user else False
+
+
+class IsFrictionLessAdapterDelete(permissions.BasePermission):
+ """Hack for friction-less onboarding Allows frticon less adapter to rmoved
+ by an org member."""
+
+ def has_object_permission(
+ self, request: Request, view: APIView, obj: AdapterInstance
+ ) -> bool:
+
+ if obj.is_friction_less:
+ return True
+
+ return True if obj.created_by == request.user else False
diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
index 21e1e3eb3..8f8849ce4 100644
--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
@@ -49,6 +49,33 @@ logger = logging.getLogger(__name__)
class PromptStudioHelper:
"""Helper class for Custom tool operations."""
+ @staticmethod
+ def validate_adapter_status(
+ profile_manager: ProfileManager,
+ ) -> None:
+ """Helper method to validate the status of adapters in profile manager.
+
+ Args:
+ profile_manager (ProfileManager): The profile manager instance to
+ validate.
+
+ Raises:
+ PermissionError: If the owner does not have permission to perform
+ the action.
+ """
+
+ error_msg = "Permission Error: Free usage for the configured trial adapter exhausted.Please connect your own service accounts to continue.Please see our documentation for more details:https://docs.unstract.com/unstract_platform/setup_accounts/whats_needed" # noqa: E501
+ adapters = [
+ profile_manager.llm,
+ profile_manager.vector_store,
+ profile_manager.embedding_model,
+ profile_manager.x2text,
+ ]
+
+ for adapter in adapters:
+ if not adapter.is_usable:
+ raise PermissionError(error_msg)
+
@staticmethod
def validate_profile_manager_owner_access(
profile_manager: ProfileManager,
@@ -237,6 +264,9 @@ class PromptStudioHelper:
LogLevels.RUN,
"Indexing started",
)
+
+ # Validate the status of adapter in profile manager
+ PromptStudioHelper.validate_adapter_status(default_profile)
# Need to check the user who created profile manager
# has access to adapters configured in profile manager
PromptStudioHelper.validate_profile_manager_owner_access(default_profile)
@@ -492,6 +522,8 @@ class PromptStudioHelper:
grammar_list.append(grammer_dict)
grammer_dict = {}
for prompt in prompts:
+ # Need to check the user who created profile manager
+ PromptStudioHelper.validate_adapter_status(prompt.profile_manager)
# Need to check the user who created profile manager
# has access to adapters
PromptStudioHelper.validate_profile_manager_owner_access(
@@ -691,6 +723,7 @@ class PromptStudioHelper:
else:
challenge_llm = str(default_profile.llm.id)
# Need to check the user who created profile manager
+ PromptStudioHelper.validate_adapter_status(default_profile)
# has access to adapters configured in profile manager
PromptStudioHelper.validate_profile_manager_owner_access(default_profile)
default_profile.chunk_size = 0 # To retrive full context
diff --git a/backend/prompt_studio/prompt_studio_core/views.py b/backend/prompt_studio/prompt_studio_core/views.py
index b3d01dd4a..9b7ffee85 100644
--- a/backend/prompt_studio/prompt_studio_core/views.py
+++ b/backend/prompt_studio/prompt_studio_core/views.py
@@ -6,6 +6,7 @@ from account.custom_exceptions import DuplicateData
from django.db import IntegrityError
from django.db.models import QuerySet
from django.http import HttpRequest
+from file_management.exceptions import FileNotFound
from file_management.file_management_helper import FileManagerHelper
from permissions.permission import IsOwner, IsOwnerOrSharedUser
from prompt_studio.processor_loader import ProcessorConfig, load_plugins
@@ -371,7 +372,21 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
if not file_path.endswith("/"):
file_path += "/"
file_path += file_name
- contents = FileManagerHelper.fetch_file_contents(file_system, file_path)
+ # Temporary Hack for frictionless onboarding as the user id will be empty
+ try:
+ contents = FileManagerHelper.fetch_file_contents(file_system, file_path)
+ except FileNotFound:
+ file_path = file_path = FileManagerHelper.handle_sub_directory_for_tenants(
+ UserSessionUtils.get_organization_id(request),
+ is_create=True,
+ user_id="",
+ tool_id=str(custom_tool.tool_id),
+ )
+ if not file_path.endswith("/"):
+ file_path += "/"
+ file_path += file_name
+ contents = FileManagerHelper.fetch_file_contents(file_system, file_path)
+
return Response({"data": contents}, status=status.HTTP_200_OK)
@action(detail=True, methods=["post"])
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index cb36faca2..405a6e154 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -30,7 +30,7 @@ dependencies = [
"python-socketio==5.9.0", # For log_events
"social-auth-app-django==5.3.0", # For OAuth
"social-auth-core==4.4.2", # For OAuth
- "unstract-sdk~=0.22.1",
+ "unstract-sdk~=0.23.0",
"unstract-adapters~=0.12.1",
# ! IMPORTANT!
# Indirect local dependencies usually need to be added in their own projects
@@ -47,6 +47,8 @@ dependencies = [
"unstract-tool-registry @ file:///${PROJECT_ROOT}/../unstract/tool-registry",
"unstract-tool-sandbox @ file:///${PROJECT_ROOT}/../unstract/tool-sandbox",
"unstract-workflow-execution @ file:///${PROJECT_ROOT}/../unstract/workflow-execution",
+ "azure-identity==1.16.0",
+ "azure-mgmt-apimanagement==3.0.0",
]
# <3.11.1 due to resolution error from Unstract SDK
requires-python = ">=3.9,<3.11.1"
diff --git a/backend/tool_instance/tool_instance_helper.py b/backend/tool_instance/tool_instance_helper.py
index 87ef6052c..a261fcc29 100644
--- a/backend/tool_instance/tool_instance_helper.py
+++ b/backend/tool_instance/tool_instance_helper.py
@@ -406,6 +406,15 @@ class ToolInstanceHelper:
adapter_instances = AdapterInstance.objects.filter(id__in=adapter_ids).all()
for adapter_instance in adapter_instances:
+ if not adapter_instance.is_usable:
+ logger.error(
+ "Free usage for the configured sample adapter %s exhausted",
+ adapter_instance.id,
+ )
+ error_msg = "Permission Error: Free usage for the configured trial adapter exhausted.Please connect your own service accounts to continue.Please see our documentation for more details:https://docs.unstract.com/unstract_platform/setup_accounts/whats_needed" # noqa: E501
+
+ raise PermissionDenied(error_msg)
+
if not (
adapter_instance.created_by == user
or adapter_instance.shared_users.filter(pk=user.pk).exists()
diff --git a/backend/usage/__init__.py b/backend/usage/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/usage/admin.py b/backend/usage/admin.py
new file mode 100644
index 000000000..c7469a4fb
--- /dev/null
+++ b/backend/usage/admin.py
@@ -0,0 +1,5 @@
+from django.contrib import admin
+
+from .models import Usage
+
+admin.site.register(Usage)
diff --git a/backend/usage/apps.py b/backend/usage/apps.py
new file mode 100644
index 000000000..abe8f39c4
--- /dev/null
+++ b/backend/usage/apps.py
@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class UsageConfig(AppConfig):
+ name = "usage"
diff --git a/backend/usage/migrations/0001_initial.py b/backend/usage/migrations/0001_initial.py
new file mode 100644
index 000000000..3bad77c1f
--- /dev/null
+++ b/backend/usage/migrations/0001_initial.py
@@ -0,0 +1,44 @@
+# Generated by Django 4.2.1 on 2024-04-22 12:55
+
+import uuid
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = []
+
+ operations = [
+ migrations.CreateModel(
+ name="Usage",
+ fields=[
+ ("created_at", models.DateTimeField(auto_now_add=True)),
+ ("modified_at", models.DateTimeField(auto_now=True)),
+ (
+ "id",
+ models.UUIDField(
+ default=uuid.uuid4,
+ editable=False,
+ primary_key=True,
+ serialize=False,
+ ),
+ ),
+ ("workflow_id", models.CharField(max_length=255)),
+ ("execution_id", models.CharField(max_length=255)),
+ ("adapter_instance_id", models.CharField(max_length=255)),
+ ("run_id", models.CharField(max_length=255)),
+ ("usage_type", models.CharField(max_length=255)),
+ ("model_name", models.CharField(max_length=255)),
+ ("embedding_tokens", models.IntegerField()),
+ ("prompt_tokens", models.IntegerField()),
+ ("completion_tokens", models.IntegerField()),
+ ("total_tokens", models.IntegerField()),
+ ],
+ options={
+ "db_table": "token_usage",
+ },
+ ),
+ ]
diff --git a/backend/usage/migrations/__init__.py b/backend/usage/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/usage/models.py b/backend/usage/models.py
new file mode 100644
index 000000000..4c8205987
--- /dev/null
+++ b/backend/usage/models.py
@@ -0,0 +1,24 @@
+import uuid
+
+from django.db import models
+from utils.models.base_model import BaseModel
+
+
+class Usage(BaseModel):
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ workflow_id = models.CharField(max_length=255)
+ execution_id = models.CharField(max_length=255)
+ adapter_instance_id = models.CharField(max_length=255)
+ run_id = models.CharField(max_length=255)
+ usage_type = models.CharField(max_length=255)
+ model_name = models.CharField(max_length=255)
+ embedding_tokens = models.IntegerField()
+ prompt_tokens = models.IntegerField()
+ completion_tokens = models.IntegerField()
+ total_tokens = models.IntegerField()
+
+ def __str__(self):
+ return str(self.id)
+
+ class Meta:
+ db_table = "token_usage"
diff --git a/backend/usage/tests.py b/backend/usage/tests.py
new file mode 100644
index 000000000..a39b155ac
--- /dev/null
+++ b/backend/usage/tests.py
@@ -0,0 +1 @@
+# Create your tests here.
diff --git a/backend/usage/views.py b/backend/usage/views.py
new file mode 100644
index 000000000..60f00ef0e
--- /dev/null
+++ b/backend/usage/views.py
@@ -0,0 +1 @@
+# Create your views here.
diff --git a/frontend/src/components/agency/actions/Actions.jsx b/frontend/src/components/agency/actions/Actions.jsx
index 843f5caa6..55c3ee3af 100644
--- a/frontend/src/components/agency/actions/Actions.jsx
+++ b/frontend/src/components/agency/actions/Actions.jsx
@@ -23,13 +23,11 @@ import { useSessionStore } from "../../../store/session-store";
import { useWorkflowStore } from "../../../store/workflow-store";
import { CreateApiDeploymentModal } from "../../deployments/create-api-deployment-modal/CreateApiDeploymentModal.jsx";
import { EtlTaskDeploy } from "../../pipelines-or-deployments/etl-task-deploy/EtlTaskDeploy.jsx";
-import { SocketMessages } from "../../helpers/socket-messages/SocketMessages";
import FileUpload from "../file-upload/FileUpload.jsx";
import "./Actions.css";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler.jsx";
function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {
- const [logId, setLogId] = useState("");
const [executionId, setExecutionId] = useState("");
const [execType, setExecType] = useState("");
const [stepExecType, setStepExecType] = useState("");
@@ -122,10 +120,8 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {
try {
const initialRes = await handleWfExecutionApi(body);
const execIdValue = initialRes?.data?.execution_id;
- const logIdValue = initialRes?.data?.log_id;
setExecutionId(execIdValue);
- setLogId(logIdValue);
body["execution_id"] = execIdValue;
if (isStepExecution) {
body["execution_action"] = wfExecutionTypes[executionAction];
@@ -535,7 +531,6 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {
setDeploymentName={setDeploymentName}
/>
)}
- <SocketMessages logId={logId} />
</>
);
}
diff --git a/frontend/src/components/helpers/auth/PersistentLogin.js b/frontend/src/components/helpers/auth/PersistentLogin.js
index 3d89bc4df..4ea218d72 100644
--- a/frontend/src/components/helpers/auth/PersistentLogin.js
+++ b/frontend/src/components/helpers/auth/PersistentLogin.js
@@ -4,6 +4,7 @@ import { Outlet } from "react-router-dom";
import { Logo64 } from "../../../assets";
import useSessionValid from "../../../hooks/useSessionValid";
import { useSessionStore } from "../../../store/session-store";
+import { SocketMessages } from "../socket-messages/SocketMessages";
function PersistentLogin() {
const [isLoading, setIsLoading] = useState(true);
@@ -33,7 +34,12 @@ function PersistentLogin() {
</div>
);
}
- return <Outlet />;
+ return (
+ <>
+ <Outlet />
+ <SocketMessages />
+ </>
+ );
}
export { PersistentLogin };
diff --git a/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js b/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js
index 85b5a9d09..b9c60781b 100644
--- a/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js
+++ b/frontend/src/components/helpers/custom-tools/CustomToolsHelper.js
@@ -8,11 +8,9 @@ import { useCustomToolStore } from "../../../store/custom-tool-store";
import { useSessionStore } from "../../../store/session-store";
import { useSocketCustomToolStore } from "../../../store/socket-custom-tool";
import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader";
-import { SocketMessages } from "../socket-messages/SocketMessages";
function CustomToolsHelper() {
const [isLoading, setIsLoading] = useState(true);
- const [logId, setLogId] = useState(null);
const { id } = useParams();
const { sessionDetails } = useSessionStore();
const { updateCustomTool, setDefaultCustomTool } = useCustomToolStore();
@@ -46,7 +44,6 @@ function CustomToolsHelper() {
updatedCusTool["singlePassExtractMode"] =
data?.single_pass_extraction_mode;
selectedDocId = data?.output;
- setLogId(sessionDetails?.logEventsId);
const reqOpsDocs = {
method: "GET",
@@ -110,12 +107,7 @@ function CustomToolsHelper() {
if (isLoading) {
return <SpinnerLoader />;
}
- return (
- <>
- <Outlet />
- <SocketMessages logId={logId} />
- </>
- );
+ return <Outlet />;
}
export { CustomToolsHelper };
diff --git a/frontend/src/components/helpers/socket-messages/SocketMessages.js b/frontend/src/components/helpers/socket-messages/SocketMessages.js
index 9a7e5ec2b..207120ef8 100644
--- a/frontend/src/components/helpers/socket-messages/SocketMessages.js
+++ b/frontend/src/components/helpers/socket-messages/SocketMessages.js
@@ -1,5 +1,4 @@
-import PropTypes from "prop-types";
-import { useContext, useEffect } from "react";
+import { useContext, useEffect, useState } from "react";
import { SocketContext } from "../../../helpers/SocketContext";
import { useExceptionHandler } from "../../../hooks/useExceptionHandler";
@@ -7,7 +6,11 @@ import { useAlertStore } from "../../../store/alert-store";
import { useSocketLogsStore } from "../../../store/socket-logs-store";
import { useSocketMessagesStore } from "../../../store/socket-messages-store";
import { useSocketCustomToolStore } from "../../../store/socket-custom-tool";
-function SocketMessages({ logId }) {
+import { useSessionStore } from "../../../store/session-store";
+import { useUsageStore } from "../../../store/usage-store";
+
+function SocketMessages() {
+ const [logId, setLogId] = useState("");
const {
pushStagedMessage,
updateMessage,
@@ -17,9 +20,15 @@ function SocketMessages({ logId }) {
} = useSocketMessagesStore();
const { pushLogMessages } = useSocketLogsStore();
const { updateCusToolMessages } = useSocketCustomToolStore();
+ const { sessionDetails } = useSessionStore();
const socket = useContext(SocketContext);
const { setAlertDetails } = useAlertStore();
const handleException = useExceptionHandler();
+ const { setLLMTokenUsage } = useUsageStore();
+
+ useEffect(() => {
+ setLogId(sessionDetails?.logEventsId || "");
+ }, [sessionDetails]);
const onMessage = (data) => {
try {
@@ -37,6 +46,11 @@ function SocketMessages({ logId }) {
if (msg?.type === "LOG" && msg?.service === "prompt") {
updateCusToolMessages(msg);
}
+ if (msg?.type === "LOG" && msg?.service === "usage") {
+ const remainingTokens =
+ msg?.max_token_count_set - msg?.added_token_count;
+ setLLMTokenUsage(Math.max(remainingTokens, 0));
+ }
} catch (err) {
setAlertDetails(handleException(err, "Failed to process socket message"));
}
@@ -68,8 +82,4 @@ function SocketMessages({ logId }) {
}, [stagedMessages, pointer]);
}
-SocketMessages.propTypes = {
- logId: PropTypes.string,
-};
-
export { SocketMessages };
diff --git a/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsx b/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsx
index c4fd833f7..5d5339baf 100644
--- a/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsx
+++ b/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsx
@@ -1,4 +1,13 @@
-import { Alert, Button, Col, Dropdown, Image, Row, Typography } from "antd";
+import {
+ Alert,
+ Button,
+ Col,
+ Dropdown,
+ Image,
+ Row,
+ Space,
+ Typography,
+} from "antd";
import { useEffect, useState } from "react";
import { useNavigate } from "react-router-dom";
import axios from "axios";
@@ -163,26 +172,28 @@ function TopNavBar() {
</Col>
<Col span={4}>
<Row justify="end" align="middle">
- {TrialDaysInfo && (
- <TrialDaysInfo remainingTrialDays={remainingTrialDays} />
- )}
- <Dropdown menu={{ items }} placement="bottomLeft" arrow>
- <div className="top-navbar-dp">
- {sessionDetails?.picture ? (
- <Image
- className="navbar-img"
- height="100%"
- width="100%"
- preview={false}
- src={sessionDetails?.picture}
- />
- ) : (
- <Typography.Text className="initials">
- {getInitials(sessionDetails?.name)}
- </Typography.Text>
- )}
- </div>
- </Dropdown>
+ <Space>
+ {TrialDaysInfo && (
+ <TrialDaysInfo remainingTrialDays={remainingTrialDays} />
+ )}
+ <Dropdown menu={{ items }} placement="bottomLeft" arrow>
+ <div className="top-navbar-dp">
+ {sessionDetails?.picture ? (
+ <Image
+ className="navbar-img"
+ height="100%"
+ width="100%"
+ preview={false}
+ src={sessionDetails?.picture}
+ />
+ ) : (
+ <Typography.Text className="initials">
+ {getInitials(sessionDetails?.name)}
+ </Typography.Text>
+ )}
+ </div>
+ </Dropdown>
+ </Space>
</Row>
</Col>
</Row>
diff --git a/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx b/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx
index 9ca1532a2..8258f6fa2 100644
--- a/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx
+++ b/frontend/src/components/pipelines-or-deployments/pipelines/Pipelines.jsx
@@ -16,7 +16,6 @@ import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate.js";
import { useAlertStore } from "../../../store/alert-store.js";
import { useSessionStore } from "../../../store/session-store.js";
import { Layout } from "../../deployments/layout/Layout.jsx";
-import { SocketMessages } from "../../helpers/socket-messages/SocketMessages.js";
import { SpinnerLoader } from "../../widgets/spinner-loader/SpinnerLoader.jsx";
import { DeleteModal } from "../delete-modal/DeleteModal.jsx";
import { EtlTaskDeploy } from "../etl-task-deploy/EtlTaskDeploy.jsx";
@@ -29,7 +28,6 @@ function Pipelines({ type }) {
const [openDeleteModal, setOpenDeleteModal] = useState(false);
const [selectedPorD, setSelectedPorD] = useState({});
const [tableLoading, setTableLoading] = useState(true);
- const [logId, setLogId] = useState("");
const { sessionDetails } = useSessionStore();
const { setAlertDetails } = useAlertStore();
const axiosPrivate = useAxiosPrivate();
@@ -77,8 +75,6 @@ function Pipelines({ type }) {
handleSyncApiReq(body)
.then((res) => {
- const logIdValue = res?.data?.execution?.log_id;
- setLogId(logIdValue);
const executionId = res?.data?.execution?.execution_id;
body["execution_id"] = executionId;
return handleSyncApiReq(body);
@@ -460,7 +456,6 @@ function Pipelines({ type }) {
deleteRecord={deletePipeline}
/>
</div>
- <SocketMessages logId={logId} />
</>
);
}
diff --git a/frontend/src/components/settings/settings/Settings.jsx b/frontend/src/components/settings/settings/Settings.jsx
index f843c559a..490d19a8f 100644
--- a/frontend/src/components/settings/settings/Settings.jsx
+++ b/frontend/src/components/settings/settings/Settings.jsx
@@ -45,6 +45,18 @@ function Settings() {
Default Triad
</Typography.Link>
</div>
+
+ {sessionDetails?.isPlatformAdmin && (
+ <div className="settings-plt">
+ <Typography.Link
+ className="settings-plt-typo"
+ strong
+ onClick={() => navigate("admin")}
+ >
+ Admin settings
+ </Typography.Link>
+ </div>
+ )}
</div>
</IslandLayout>
</div>
diff --git a/frontend/src/components/tool-settings/tool-settings/ToolSettings.jsx b/frontend/src/components/tool-settings/tool-settings/ToolSettings.jsx
index 659539f3f..61fac5924 100644
--- a/frontend/src/components/tool-settings/tool-settings/ToolSettings.jsx
+++ b/frontend/src/components/tool-settings/tool-settings/ToolSettings.jsx
@@ -213,6 +213,7 @@ function ToolSettings({ type }) {
handleEdit={(_event, item) => setEditItemId(item?.id)}
idProp="id"
titleProp="adapter_name"
+ descriptionProp="description"
iconProp="icon"
isEmpty={!tableRows?.length}
centered
diff --git a/frontend/src/components/widgets/list-view/ListView.jsx b/frontend/src/components/widgets/list-view/ListView.jsx
index 441f72697..e34011539 100644
--- a/frontend/src/components/widgets/list-view/ListView.jsx
+++ b/frontend/src/components/widgets/list-view/ListView.jsx
@@ -1,4 +1,12 @@
-import { Avatar, Flex, Image, List, Popconfirm, Typography } from "antd";
+import {
+ Avatar,
+ Flex,
+ Image,
+ List,
+ Popconfirm,
+ Tooltip,
+ Typography,
+} from "antd";
import PropTypes from "prop-types";
import "./ListView.css";
import {
@@ -9,6 +17,7 @@ import {
UserOutlined,
} from "@ant-design/icons";
import { useNavigate } from "react-router-dom";
+
import { useSessionStore } from "../../../store/session-store";
function ListView({
@@ -157,7 +166,9 @@ function ListView({
title={renderTitle(item)}
description={
<Typography.Text type="secondary" ellipsis>
- {item[descriptionProp]}
+ <Tooltip title={item[descriptionProp]}>
+ {item[descriptionProp]}
+ </Tooltip>
</Typography.Text>
}
></List.Item.Meta>
diff --git a/frontend/src/helpers/GetSessionData.js b/frontend/src/helpers/GetSessionData.js
index 8eb04f263..ef69ca57f 100644
--- a/frontend/src/helpers/GetSessionData.js
+++ b/frontend/src/helpers/GetSessionData.js
@@ -22,6 +22,7 @@ function getSessionData(sessionData) {
logEventsId: sessionData?.logEventsId,
remainingTrialDays: sessionData?.remainingTrialDays,
allOrganization: sessionData?.allOrganization,
+ isPlatformAdmin: sessionData?.isPlatformAdmin,
};
}
diff --git a/frontend/src/hooks/useSessionValid.js b/frontend/src/hooks/useSessionValid.js
index 3f2026261..e81ae943e 100644
--- a/frontend/src/hooks/useSessionValid.js
+++ b/frontend/src/hooks/useSessionValid.js
@@ -1,15 +1,19 @@
import axios from "axios";
+import Cookies from "js-cookie";
+import { useNavigate } from "react-router-dom";
import { getSessionData } from "../helpers/GetSessionData";
-import Cookies from "js-cookie";
-import { useSessionStore } from "../store/session-store";
+
import { useExceptionHandler } from "../hooks/useExceptionHandler.jsx";
-import { useNavigate } from "react-router-dom";
+import { useSessionStore } from "../store/session-store";
import { useUserSession } from "./useUserSession.js";
let getTrialDetails;
+let isPlatformAdmin;
try {
getTrialDetails = require("../plugins/subscription/trial-helper/fetchTrialDetails.jsx");
+ isPlatformAdmin =
+ require("../plugins/hooks/usePlatformAdmin.js").usePlatformAdmin();
} catch (err) {
// Plugin not available
}
@@ -98,7 +102,9 @@ function useSessionValid() {
userAndOrgDetails["remainingTrialDays"] = remainingTrialDays;
}
userAndOrgDetails["allOrganization"] = orgs;
-
+ if (isPlatformAdmin) {
+ userAndOrgDetails["isPlatformAdmin"] = await isPlatformAdmin();
+ }
// Set the session details
setSessionDetails(getSessionData(userAndOrgDetails));
} catch (err) {
diff --git a/frontend/src/routes/Router.jsx b/frontend/src/routes/Router.jsx
index 38c29edad..2def3ab18 100644
--- a/frontend/src/routes/Router.jsx
+++ b/frontend/src/routes/Router.jsx
@@ -20,20 +20,26 @@ import { DeploymentsPage } from "../pages/DeploymentsPage.jsx";
import { InviteEditUserPage } from "../pages/InviteEditUserPage.jsx";
import { LandingPage } from "../pages/LandingPage.jsx";
import { OnBoardPage } from "../pages/OnBoardPage.jsx";
+import { OutputAnalyzerPage } from "../pages/OutputAnalyzerPage.jsx";
import { PipelinesOrDeploymentsPage } from "../pages/PipelinesOrDeploymentsPage.jsx";
import { ProfilePage } from "../pages/ProfilePage.jsx";
+import { SetOrgPage } from "../pages/SetOrgPage.jsx";
import { SettingsPage } from "../pages/SettingsPage.jsx";
import { ToolIdePage } from "../pages/ToolIdePage.jsx";
import { ToolsSettingsPage } from "../pages/ToolsSettingsPage.jsx";
import { UsersPage } from "../pages/UsersPage.jsx";
import { WorkflowsPage } from "../pages/WorkflowsPage.jsx";
-import { OutputAnalyzerPage } from "../pages/OutputAnalyzerPage.jsx";
-import { SetOrgPage } from "../pages/SetOrgPage.jsx";
let TrialRoutes;
+let RequirePlatformAdmin;
+let PlatformAdminPage;
try {
TrialRoutes =
require("../plugins/subscription/trial-page/TrialEndPage.jsx").TrialEndPage;
+ RequirePlatformAdmin =
+ require("../plugins/frictionless-onboard/RequirePlatformAdmin.jsx").RequirePlatformAdmin;
+ PlatformAdminPage =
+ require("../plugins/frictionless-onboard/platform-admin-page/PlatformAdminPage.jsx").PlatformAdminPage;
} catch (err) {
TrialRoutes = NotFound;
}
@@ -111,6 +117,11 @@ function Router() {
<Route path="users/edit" element={<InviteEditUserPage />} />
</Route>
<Route path="settings/triad" element={<DefaultTriad />} />
+ {RequirePlatformAdmin && PlatformAdminPage && (
+ <Route element={<RequirePlatformAdmin />}>
+ <Route path="settings/admin" element={<PlatformAdminPage />} />
+ </Route>
+ )}
</Route>
</Route>
{TrialRoutes && (
diff --git a/frontend/src/store/usage-store.js b/frontend/src/store/usage-store.js
new file mode 100644
index 000000000..1b97b3d8d
--- /dev/null
+++ b/frontend/src/store/usage-store.js
@@ -0,0 +1,10 @@
+import { create } from "zustand";
+
+const useUsageStore = create((setState) => ({
+ llmTokenUsage: null,
+ setLLMTokenUsage: (data) => {
+ setState({ llmTokenUsage: data });
+ },
+}));
+
+export { useUsageStore };
diff --git a/pdm.lock b/pdm.lock
index a299064e2..2218c836a 100644
--- a/pdm.lock
+++ b/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "hook-check-django-migrations", "lint"]
strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.1"
-content_hash = "sha256:a426842385fd1039ecfd7e2b5c37a717241c377b8eb4e17f91f91dd28341653f"
+content_hash = "sha256:af031601a5f06a763f954431a4943aa82931a379a79df6c68e59ea37e04dce9c"
[[package]]
name = "adlfs"
@@ -4905,7 +4905,7 @@ dependencies = [
[[package]]
name = "unstract-sdk"
-version = "0.22.1"
+version = "0.23.0"
requires_python = "<3.11.1,>=3.9"
summary = "A framework for writing Unstract Tools/Apps"
groups = ["hook-check-django-migrations"]
@@ -4919,8 +4919,8 @@ dependencies = [
"unstract-adapters~=0.12.1",
]
files = [
- {file = "unstract_sdk-0.22.1-py3-none-any.whl", hash = "sha256:af29fec4c56c3c1916b7af639d331d9c7b115ebff6f1183e4be50bef01bd1792"},
- {file = "unstract_sdk-0.22.1.tar.gz", hash = "sha256:f49db6c62bca6baa2474912a63f305df01ec12ada34af9a2fb93b8ce91e7fa16"},
+ {file = "unstract_sdk-0.23.0-py3-none-any.whl", hash = "sha256:769bfa6048261241b0c2f76c64520d42992bcbba46403948d7474eb7e53740aa"},
+ {file = "unstract_sdk-0.23.0.tar.gz", hash = "sha256:f327d1d4631947d611da7221667bdd82c40c836ae8d7fbef81eb8ad25818f165"},
]
[[package]]
diff --git a/platform-service/src/unstract/platform_service/main.py b/platform-service/src/unstract/platform_service/main.py
index 702ca4238..de9591d00 100644
--- a/platform-service/src/unstract/platform_service/main.py
+++ b/platform-service/src/unstract/platform_service/main.py
@@ -1,15 +1,14 @@
-import datetime
-import json
import logging
import os
import uuid
+from datetime import datetime
from typing import Any, Literal, Optional
import peewee
import redis
from cryptography.fernet import Fernet
from dotenv import load_dotenv
-from flask import Flask, Request, Response, jsonify, request
+from flask import Flask, Request, Response, json, jsonify, make_response, request
from unstract.platform_service.exceptions import CustomException
from unstract.platform_service.helper import (
AdapterInstanceRequestHelper,
@@ -55,30 +54,15 @@ be_db.init(PG_BE_DATABASE)
be_db.connect()
-class UnstractUsage(peewee.Model):
- id = peewee.UUIDField(primary_key=True, default=uuid.uuid4)
- created_at = peewee.DateTimeField(default=datetime.datetime.now)
- organization_id = peewee.CharField()
- workflow_id = peewee.CharField()
- execution_id = peewee.CharField()
- status = peewee.CharField()
- usage_type = peewee.CharField()
- external_service = peewee.CharField()
- tokens_valid = peewee.BooleanField(default=False)
- embedding_tokens = peewee.IntegerField()
- prompt_tokens = peewee.IntegerField()
- completion_tokens = peewee.IntegerField()
- total_tokens = peewee.IntegerField()
- usage_value_valid = peewee.BooleanField(peewee.BooleanField(default=False))
- usage_value = peewee.FloatField()
- usage_units = peewee.CharField()
-
- class Meta:
- database = be_db # This model uses the "BE_DB" database.
- table_name = "unstract_usage"
+def drop_unstract_usage_table() -> None:
+ query = "DROP TABLE IF EXISTS unstract_usage"
+ try:
+ be_db.execute_sql(query)
+ except Exception as e:
+ app.logger.error(f"Error dropping 'unstract_usage' table: {e}")
-UnstractUsage.create_table()
+drop_unstract_usage_table()
def get_token_from_auth_header(request: Request) -> Any:
@@ -173,8 +157,6 @@ def usage() -> Any:
....
}'
"""
- bearer_token = get_token_from_auth_header(request)
- organization_id = get_account_from_bearer_token(bearer_token)
result: dict[str, Any] = {
"status": "ERROR",
"error": "",
@@ -183,43 +165,54 @@ def usage() -> Any:
payload: Optional[dict[Any, Any]] = request.json
if not payload:
result["error"] = INVALID_PAYLOAD
- return result, 400
-
- org_id = organization_id
+ return make_response(result, 400)
+ bearer_token = get_token_from_auth_header(request)
+ org_id = get_account_from_bearer_token(bearer_token)
workflow_id = payload.get("workflow_id")
execution_id = payload.get("execution_id", "")
- status = payload.get("status", "")
+ adapter_instance_id = payload.get("adapter_instance_id", "")
+ run_id = payload.get("run_id", "")
usage_type = payload.get("usage_type", "")
- external_service = payload.get("external_service", "")
- tokens_valid = payload.get("tokens_valid", False)
+ model_name = payload.get("model_name", "")
embedding_tokens = payload.get("embedding_tokens", 0)
prompt_tokens = payload.get("prompt_tokens", 0)
completion_tokens = payload.get("completion_tokens", 0)
total_tokens = payload.get("total_tokens", 0)
- usage_value_valid = payload.get("usage_value_valid", False)
- usage_value = payload.get("usage_value", 0.0)
- usage_units = payload.get("usage_units", "")
-
- usage: UnstractUsage = UnstractUsage.create(
- organization_id=org_id,
- workflow_id=workflow_id,
- execution_id=execution_id,
- status=status,
- usage_type=usage_type,
- external_service=external_service,
- tokens_valid=tokens_valid,
- embedding_tokens=embedding_tokens,
- prompt_tokens=prompt_tokens,
- completion_tokens=completion_tokens,
- total_tokens=total_tokens,
- usage_value_valid=usage_value_valid,
- usage_value=usage_value,
- usage_units=usage_units,
+ query = f"""
+ INSERT INTO "{org_id}"."token_usage" (id, workflow_id, execution_id,
+ adapter_instance_id, run_id, usage_type, model_name, embedding_tokens,
+ prompt_tokens, completion_tokens, total_tokens, created_at, modified_at)
+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+ """
+ usage_id = uuid.uuid4()
+ current_time = datetime.now()
+ params = (
+ usage_id,
+ workflow_id,
+ execution_id,
+ adapter_instance_id,
+ run_id,
+ usage_type,
+ model_name,
+ embedding_tokens,
+ prompt_tokens,
+ completion_tokens,
+ total_tokens,
+ current_time,
+ current_time,
)
- app.logger.info("Entry created with id %s for %s", usage.id, organization_id)
- result["status"] = "OK"
- result["unique_id"] = usage.id
- return result
+ try:
+ with be_db.atomic() as transaction:
+ be_db.execute_sql(query, params)
+ transaction.commit()
+ app.logger.info("Entry created with id %s for %s", usage_id, org_id)
+ result["status"] = "OK"
+ result["unique_id"] = usage_id
+ return make_response(result, 200)
+ except Exception as e:
+ app.logger.error(f"Error while creating usage entry: {e}")
+ result["error"] = "Internal Server Error"
+ return make_response(result, 500)
@app.route(
diff --git a/prompt-service/pdm.lock b/prompt-service/pdm.lock
index 6a3279630..47620bcc4 100644
--- a/prompt-service/pdm.lock
+++ b/prompt-service/pdm.lock
@@ -5,7 +5,7 @@
groups = ["default", "deploy"]
strategy = ["cross_platform", "inherit_metadata"]
lock_version = "4.4.1"
-content_hash = "sha256:01e5530f1b1b786278ad93198c75ad40ec70926cd0a971cd2feeca1692586d4e"
+content_hash = "sha256:08f2cd847abb3fbc248303f8f2c7bfbac22ef48e3605e8c12990bb6e7ec45d1d"
[[package]]
name = "aiohttp"
@@ -3295,8 +3295,8 @@ requires_python = ">=3.7"
summary = "Database Abstraction Library"
groups = ["default"]
dependencies = [
+ "SQLAlchemy==2.0.29",
"greenlet!=0.4.17",
- "sqlalchemy==2.0.29",
]
files = [
{file = "SQLAlchemy-2.0.29-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4c142852ae192e9fe5aad5c350ea6befe9db14370b34047e1f0f7cf99e63c63b"},
@@ -3637,7 +3637,7 @@ dependencies = [
[[package]]
name = "unstract-sdk"
-version = "0.22.1"
+version = "0.23.0"
requires_python = "<3.11.1,>=3.9"
summary = "A framework for writing Unstract Tools/Apps"
groups = ["default"]
@@ -3651,8 +3651,8 @@ dependencies = [
"unstract-adapters~=0.12.1",
]
files = [
- {file = "unstract_sdk-0.22.1-py3-none-any.whl", hash = "sha256:af29fec4c56c3c1916b7af639d331d9c7b115ebff6f1183e4be50bef01bd1792"},
- {file = "unstract_sdk-0.22.1.tar.gz", hash = "sha256:f49db6c62bca6baa2474912a63f305df01ec12ada34af9a2fb93b8ce91e7fa16"},
+ {file = "unstract_sdk-0.23.0-py3-none-any.whl", hash = "sha256:769bfa6048261241b0c2f76c64520d42992bcbba46403948d7474eb7e53740aa"},
+ {file = "unstract_sdk-0.23.0.tar.gz", hash = "sha256:f327d1d4631947d611da7221667bdd82c40c836ae8d7fbef81eb8ad25818f165"},
]
[[package]]
diff --git a/prompt-service/pyproject.toml b/prompt-service/pyproject.toml
index 769098f6a..983d04ec9 100644
--- a/prompt-service/pyproject.toml
+++ b/prompt-service/pyproject.toml
@@ -15,7 +15,7 @@ dependencies = [
"flask~=3.0",
"llama-index==0.10.28",
"python-dotenv==1.0.0",
- "unstract-sdk~=0.22.1",
+ "unstract-sdk~=0.23.0",
"redis>=5.0.3",
"unstract-core @ file:///${PROJECT_ROOT}/../unstract/core",
]
diff --git a/prompt-service/src/unstract/prompt_service/constants.py b/prompt-service/src/unstract/prompt_service/constants.py
index 205667f9a..2e0471a10 100644
--- a/prompt-service/src/unstract/prompt_service/constants.py
+++ b/prompt-service/src/unstract/prompt_service/constants.py
@@ -45,7 +45,6 @@ class PromptServiceContants:
TOOL_SETTINGS = "tool_settings"
LOG_EVENTS_ID = "log_events_id"
CHALLENGE_LLM = "challenge_llm"
- USAGE = "usage"
class LogLevel(Enum):
diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.py
index 217ef8e48..ea11608ee 100644
--- a/prompt-service/src/unstract/prompt_service/main.py
+++ b/prompt-service/src/unstract/prompt_service/main.py
@@ -161,7 +161,6 @@ def authentication_middleware(func: Any) -> Any:
@authentication_middleware
def prompt_processor() -> Any:
result: dict[str, Any] = {}
- usage = {}
platform_key = AuthenticationMiddleware.get_token_from_auth_header(request)
if request.method == "POST":
payload: dict[Any, Any] = request.json
@@ -192,6 +191,7 @@ def prompt_processor() -> Any:
chunk_size = output[PSKeys.CHUNK_SIZE]
util = PromptServiceBaseTool(log_level=LogLevel.INFO, platform_key=platform_key)
tool_index = ToolIndex(tool=util)
+ adapter_instance_id = output[PSKeys.LLM]
if active is False:
app.logger.info(f"[{tool_id}] Skipping inactive prompt: {name}")
@@ -239,10 +239,10 @@ def prompt_processor() -> Any:
llm_helper = ToolLLM(tool=util)
llm_li: Optional[LLM] = llm_helper.get_llm(
- adapter_instance_id=output[PSKeys.LLM]
+ adapter_instance_id=adapter_instance_id
)
if llm_li is None:
- msg = f"Couldn't fetch LLM {output[PSKeys.LLM]}"
+ msg = f"Couldn't fetch LLM {adapter_instance_id}"
app.logger.error(msg)
_publish_log(
log_events_id,
@@ -321,7 +321,7 @@ def prompt_processor() -> Any:
is_assert = output[PSKeys.IS_ASSERT]
if is_assert:
app.logger.debug(f'Asserting prompt: {output["assert_prompt"]}')
- answer, usage = construct_and_run_prompt(
+ answer = construct_and_run_prompt(
output,
llm_helper,
llm_li,
@@ -349,7 +349,7 @@ def prompt_processor() -> Any:
]
app.logger.info(f"[Assigning] {answer} to the output")
else:
- answer, usage = construct_and_run_prompt(
+ answer = construct_and_run_prompt(
output,
llm_helper,
llm_li,
@@ -358,7 +358,7 @@ def prompt_processor() -> Any:
)
else:
if chunk_size == 0:
- answer, usage = construct_and_run_prompt(
+ answer = construct_and_run_prompt(
output,
llm_helper,
llm_li,
@@ -376,7 +376,7 @@ def prompt_processor() -> Any:
)
if output[PSKeys.RETRIEVAL_STRATEGY] == PSKeys.SIMPLE:
- answer, context, usage = simple_retriver(
+ answer, context = simple_retriver(
output,
doc_id,
llm_helper,
@@ -420,10 +420,11 @@ def prompt_processor() -> Any:
percentages or other grouping \
characters. No explanation is required.\
If you cannot extract the number, output 0."
- answer, usage = run_completion(
- llm_helper,
- llm_li,
- prompt,
+ answer = run_completion(
+ llm_helper=llm_helper,
+ llm_li=llm_li,
+ prompt=prompt,
+ adapter_instance_id=adapter_instance_id,
)
try:
structured_output[output[PSKeys.NAME]] = float(answer)
@@ -440,10 +441,11 @@ def prompt_processor() -> Any:
prompt = f'Extract the email from the following text:\n{answer}\n\nOutput just the email. \
The email should be directly assignable to a string variable. \
No explanation is required. If you cannot extract the email, output "NA".' # noqa
- answer, usage = run_completion(
- llm_helper,
- llm_li,
- prompt,
+ answer = run_completion(
+ llm_helper=llm_helper,
+ llm_li=llm_li,
+ prompt=prompt,
+ adapter_instance_id=adapter_instance_id,
)
structured_output[output[PSKeys.NAME]] = answer
elif output[PSKeys.TYPE] == PSKeys.DATE:
@@ -454,10 +456,11 @@ def prompt_processor() -> Any:
The date should be in ISO date time format. No explanation is required. \
The date should be directly assignable to a date variable. \
If you cannot convert the string into a date, output "NA".' # noqa
- answer, usage = run_completion(
- llm_helper,
- llm_li,
- prompt,
+ answer = run_completion(
+ llm_helper=llm_helper,
+ llm_li=llm_li,
+ prompt=prompt,
+ adapter_instance_id=adapter_instance_id,
)
structured_output[output[PSKeys.NAME]] = answer
@@ -469,10 +472,11 @@ def prompt_processor() -> Any:
Output in single word.\
If the context is trying to convey that the answer is true, \
then return "yes", else return "no".'
- answer, usage = run_completion(
- llm_helper,
- llm_li,
- prompt,
+ answer = run_completion(
+ llm_helper=llm_helper,
+ llm_li=llm_li,
+ prompt=prompt,
+ adapter_instance_id=adapter_instance_id,
)
if answer.lower() == "yes":
structured_output[output[PSKeys.NAME]] = True
@@ -487,11 +491,11 @@ def prompt_processor() -> Any:
into a Python dictionary. \
Output just the JSON string. No explanation is required. \
If you cannot extract the JSON string, output {{}}"
-
- answer, usage = run_completion(
- llm_helper,
- llm_li,
- prompt,
+ answer = run_completion(
+ llm_helper=llm_helper,
+ llm_li=llm_li,
+ prompt=prompt,
+ adapter_instance_id=adapter_instance_id,
)
try:
structured_output[output[PSKeys.NAME]] = json.loads(answer)
@@ -519,7 +523,7 @@ def prompt_processor() -> Any:
PSKeys.PREAMBLE: output[PSKeys.PREAMBLE],
PSKeys.POSTAMBLE: output[PSKeys.POSTAMBLE],
PSKeys.GRAMMAR: output[PSKeys.GRAMMAR],
- PSKeys.LLM: output[PSKeys.LLM],
+ PSKeys.LLM: adapter_instance_id,
PSKeys.CHALLENGE_LLM: output[PSKeys.CHALLENGE_LLM],
}
challenge = challenge_plugin["entrypoint_cls"](
@@ -626,7 +630,6 @@ def prompt_processor() -> Any:
RunLevel.RUN,
"Execution complete",
)
- app.logger.info("Usage details : %s", str(usage))
return structured_output
@@ -647,10 +650,11 @@ def simple_retriver( # type:ignore
f"Generate a sub-question from the following verbose prompt that will"
f" help extract relevant documents from a vector store:\n\n{prompt}"
)
- answer, usage = run_completion(
- llm_helper,
- llm_li,
- subq_prompt,
+ answer = run_completion(
+ llm_helper=llm_helper,
+ llm_li=llm_li,
+ prompt=subq_prompt,
+ adapter_instance_id=output[PSKeys.LLM],
)
retriever = vector_index.as_retriever(
@@ -672,14 +676,14 @@ def simple_retriver( # type:ignore
else:
app.logger.info("Node score is less than 0.6. " f"Ignored: {node.score}")
- answer, usage = construct_and_run_prompt( # type:ignore
+ answer = construct_and_run_prompt( # type:ignore
output,
llm_helper,
llm_li,
text,
"promptx",
)
- return (answer, text, usage)
+ return (answer, text)
def construct_and_run_prompt(
@@ -697,9 +701,10 @@ def construct_and_run_prompt(
context=context,
)
return run_completion(
- llm_helper,
- llm_li,
- prompt,
+ llm_helper=llm_helper,
+ llm_li=llm_li,
+ prompt=prompt,
+ adapter_instance_id=output[PSKeys.LLM],
)
@@ -707,18 +712,21 @@ def run_completion(
llm_helper: ToolLLM,
llm_li: Optional[LLM],
prompt: str,
+ adapter_instance_id: str,
) -> tuple[str, dict[str, Any]]:
try:
platform_api_key = llm_helper.tool.get_env_or_die(
PSKeys.PLATFORM_SERVICE_API_KEY
)
- completion = llm_helper.run_completion(llm_li, platform_api_key, prompt, 3)
+ completion = llm_helper.run_completion(
+ llm=llm_li,
+ platform_api_key=platform_api_key,
+ prompt=prompt,
+ adapter_instance_id=adapter_instance_id,
+ )
answer: str = completion[PSKeys.RESPONSE].text
- usage = {}
- if PSKeys.USAGE in completion:
- usage = completion[PSKeys.USAGE]
- return answer, usage
+ return answer
except Exception as e:
app.logger.info(f"Error completing prompt: {e}.")
raise e
diff --git a/pyproject.toml b/pyproject.toml
index aa4772dc4..ed27f11ed 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,7 +40,7 @@ hook-check-django-migrations = [
"psycopg2-binary==2.9.9",
"python-dotenv==1.0.0",
"python-magic==0.4.27",
- "unstract-sdk~=0.22.1",
+ "unstract-sdk~=0.23.0",
"unstract-adapters~=0.12.1",
"-e unstract-connectors @ file:///${PROJECT_ROOT}/unstract/connectors",
"-e unstract-core @ file:///${PROJECT_ROOT}/unstract/core",
diff --git a/tools/classifier/requirements.txt b/tools/classifier/requirements.txt
index f9fd56248..a4baabc2b 100644
--- a/tools/classifier/requirements.txt
+++ b/tools/classifier/requirements.txt
@@ -1,4 +1,4 @@
# Add your dependencies here
# Required for all unstract tools
-unstract-sdk~=0.22.1
+unstract-sdk~=0.23.0
diff --git a/tools/classifier/src/config/properties.json b/tools/classifier/src/config/properties.json
index 688acbacc..4f820745c 100644
--- a/tools/classifier/src/config/properties.json
+++ b/tools/classifier/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "File Classifier",
"functionName": "classify",
- "toolVersion": "0.0.13",
+ "toolVersion": "0.0.14",
"description": "Classifies a file into a bin based on its contents",
"input": {
"description": "File to be classified"
diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txt
index f9fd56248..a4baabc2b 100644
--- a/tools/structure/requirements.txt
+++ b/tools/structure/requirements.txt
@@ -1,4 +1,4 @@
# Add your dependencies here
# Required for all unstract tools
-unstract-sdk~=0.22.1
+unstract-sdk~=0.23.0
diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.json
index ba112333d..424bdd35e 100644
--- a/tools/structure/src/config/properties.json
+++ b/tools/structure/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "Structure Tool",
"functionName": "structure_tool",
- "toolVersion": "0.0.16",
+ "toolVersion": "0.0.17",
"description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio",
"input": {
"description": "File that needs to be indexed and parsed for answers"
diff --git a/tools/text_extractor/requirements.txt b/tools/text_extractor/requirements.txt
index f9fd56248..a4baabc2b 100644
--- a/tools/text_extractor/requirements.txt
+++ b/tools/text_extractor/requirements.txt
@@ -1,4 +1,4 @@
# Add your dependencies here
# Required for all unstract tools
-unstract-sdk~=0.22.1
+unstract-sdk~=0.23.0
diff --git a/tools/text_extractor/src/config/properties.json b/tools/text_extractor/src/config/properties.json
index fb1fd2cda..e20a737fa 100644
--- a/tools/text_extractor/src/config/properties.json
+++ b/tools/text_extractor/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "Text Extractor",
"functionName": "text_extractor",
- "toolVersion": "0.0.12",
+ "toolVersion": "0.0.13",
"description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents",
"input": {
"description": "Document"
diff --git a/unstract/core/src/unstract/core/pubsub_helper.py b/unstract/core/src/unstract/core/pubsub_helper.py
index f51d2a8b8..855a4d3f6 100644
--- a/unstract/core/src/unstract/core/pubsub_helper.py
+++ b/unstract/core/src/unstract/core/pubsub_helper.py
@@ -15,6 +15,23 @@ class LogPublisher:
password=os.environ.get("REDIS_PASSWORD", ""),
)
+ @staticmethod
+ def log_usage(
+ level: str = "INFO",
+ added_token_count: Optional[int] = None,
+ max_token_count_set: Optional[int] = None,
+ enabled: bool = False,
+ ) -> dict[str, Any]:
+ return {
+ "timestamp": datetime.now(timezone.utc).timestamp(),
+ "type": "LOG",
+ "service": "usage",
+ "level": level,
+ "added_token_count": added_token_count,
+ "max_token_count_set": max_token_count_set,
+ "enabled": enabled,
+ }
+
@staticmethod
def log_workflow(
stage: str,