Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

fix: flake8 black file changes (#259)

Kirtiman Mishra6871fe1d8dd5
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 181d316ee..13835970d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -50,23 +50,23 @@ repos:
rev: 24.3.0
hooks:
- id: black
- args: [--config=pyproject.toml, -l 80]
+ args: [--config=pyproject.toml, -l 88]
# language: system
exclude: |
(?x)^(
unstract/flags/src/unstract/flags/evaluation_.*\.py|
)$
- # - repo: https://github.com/pycqa/flake8
- # rev: 7.0.0
- # hooks:
- # - id: flake8
- # args: [--max-line-length=80]
- # exclude: |
- # (?x)^(
- # .*migrations/.*\.py|
- # core/tests/.*|
- # unstract/flags/src/unstract/flags/evaluation_.*\.py|
- # )$
+ - repo: https://github.com/pycqa/flake8
+ rev: 7.0.0
+ hooks:
+ - id: flake8
+ args: [--max-line-length=88]
+ exclude: |
+ (?x)^(
+ .*migrations/.*\.py|
+ core/tests/.*|
+ unstract/flags/src/unstract/flags/evaluation_.*\.py|
+ )$
- repo: https://github.com/pycqa/isort
rev: 5.13.2
hooks:
diff --git a/backend/account/authentication_controller.py b/backend/account/authentication_controller.py
index ede02b04a..48fa8afdb 100644
--- a/backend/account/authentication_controller.py
+++ b/backend/account/authentication_controller.py
@@ -12,11 +12,7 @@ from account.constants import (
ErrorMessage,
OrganizationMemberModel,
)
-from account.custom_exceptions import (
- DuplicateData,
- Forbidden,
- UserNotExistError,
-)
+from account.custom_exceptions import DuplicateData, Forbidden, UserNotExistError
from account.dto import (
CallbackData,
MemberInvitation,
@@ -109,28 +105,20 @@ class AuthenticationController:
frontend/src/components/error/GenericError/GenericError.jsx."""
if ex.code == AuthorizationErrorCode.IDM: # type: ignore
query_params = {"code": AuthorizationErrorCode.IDM}
- return redirect(
- f"{settings.ERROR_URL}?{urlencode(query_params)}"
- )
+ return redirect(f"{settings.ERROR_URL}?{urlencode(query_params)}")
elif ex.code == AuthorizationErrorCode.UMM: # type: ignore
query_params = {"code": AuthorizationErrorCode.UMM}
- return redirect(
- f"{settings.ERROR_URL}?{urlencode(query_params)}"
- )
+ return redirect(f"{settings.ERROR_URL}?{urlencode(query_params)}")
return redirect(f"{settings.ERROR_URL}")
if member.organization_id and member.role and len(member.role) > 0:
organization: Optional[Organization] = (
- OrganizationService.get_organization_by_org_id(
- member.organization_id
- )
+ OrganizationService.get_organization_by_org_id(member.organization_id)
)
if organization:
try:
- self.create_tenant_user(
- organization=organization, user=user
- )
+ self.create_tenant_user(organization=organization, user=user)
except UndefinedTable:
pass
@@ -159,7 +147,10 @@ class AuthenticationController:
except Exception as ex:
#
self.user_logout(request)
- if hasattr(ex, "code") and ex.code == AuthorizationErrorCode.USF: # type: ignore
+ if (
+ hasattr(ex, "code")
+ and ex.code == AuthorizationErrorCode.USF # type: ignore
+ ):
response = Response(
status=status.HTTP_412_PRECONDITION_FAILED,
data={"domain": ex.data.get("domain")}, # type: ignore
@@ -185,9 +176,7 @@ class AuthenticationController:
return response
- def set_user_organization(
- self, request: Request, organization_id: str
- ) -> Response:
+ def set_user_organization(self, request: Request, organization_id: str) -> Response:
user: User = request.user
new_organization = False
organization_ids = CacheService.get_user_organizations(user.user_id)
@@ -203,9 +192,7 @@ class AuthenticationController:
if not organization:
try:
organization_data: OrganizationData = (
- self.auth_service.get_organization_by_org_id(
- organization_id
- )
+ self.auth_service.get_organization_by_org_id(organization_id)
)
except ValueError:
raise OrganizationNotExist()
@@ -227,23 +214,19 @@ class AuthenticationController:
user=user, organization=organization
)
user_info: Optional[UserInfo] = self.get_user_info(request)
- serialized_user_info = SetOrganizationsResponseSerializer(
- user_info
- ).data
+ serialized_user_info = SetOrganizationsResponseSerializer(user_info).data
organization_info = OrganizationSerializer(organization).data
response: Response = Response(
status=status.HTTP_200_OK,
data={
"user": serialized_user_info,
"organization": organization_info,
- f"{Common.LOG_EVENTS_ID}": StateStore.get(
- Common.LOG_EVENTS_ID
- ),
+ f"{Common.LOG_EVENTS_ID}": StateStore.get(Common.LOG_EVENTS_ID),
},
)
# Update user session data in redis
- user_session_info: dict[str, Any] = (
- CacheService.get_user_session_info(user.email)
+ user_session_info: dict[str, Any] = CacheService.get_user_session_info(
+ user.email
)
user_session_info["current_org"] = organization_id
CacheService.set_user_session_info(user_session_info)
@@ -269,9 +252,7 @@ class AuthenticationController:
def get_organization_info(self, org_id: str) -> Optional[Organization]:
organization = self.auth_service.get_organization_info(org_id)
if not organization:
- organization = OrganizationService.get_organization_by_org_id(
- org_id=org_id
- )
+ organization = OrganizationService.get_organization_by_org_id(org_id=org_id)
return organization
def make_organization_and_add_member(
@@ -302,9 +283,7 @@ class AuthenticationController:
members: list[OrganizationMember] = OrganizationMember.objects.all()
return members
- def get_organization_members_by_user(
- self, user: User
- ) -> OrganizationMember:
+ def get_organization_members_by_user(self, user: User) -> OrganizationMember:
member: OrganizationMember = OrganizationMember.objects.filter(
user=user
).first()
@@ -313,16 +292,10 @@ class AuthenticationController:
def get_user_roles(self) -> list[UserRoleData]:
return self.auth_service.get_roles()
- def get_user_invitations(
- self, organization_id: str
- ) -> list[MemberInvitation]:
- return self.auth_service.get_invitations(
- organization_id=organization_id
- )
+ def get_user_invitations(self, organization_id: str) -> list[MemberInvitation]:
+ return self.auth_service.get_invitations(organization_id=organization_id)
- def delete_user_invitation(
- self, organization_id: str, invitation_id: str
- ) -> bool:
+ def delete_user_invitation(self, organization_id: str, invitation_id: str) -> bool:
return self.auth_service.delete_invitation(
organization_id=organization_id, invitation_id=invitation_id
)
@@ -355,9 +328,7 @@ class AuthenticationController:
email = user_item.get("email")
role = user_item.get("role")
if email:
- user = self.organization_member_service.get_user_by_email(
- email=email
- )
+ user = self.organization_member_service.get_user_by_email(email=email)
user_response = {}
user_response["email"] = email
status = False
@@ -384,9 +355,7 @@ class AuthenticationController:
admin_user = OrganizationMember.objects.get(user=admin.id)
user_ids = OrganizationMember.objects.filter(
user__email__in=user_emails
- ).values_list(
- OrganizationMemberModel.USER_ID, OrganizationMemberModel.ID
- )
+ ).values_list(OrganizationMemberModel.USER_ID, OrganizationMemberModel.ID)
user_ids_list: list[str] = []
ids_list: list[str] = []
for user in user_ids:
@@ -430,8 +399,8 @@ class AuthenticationController:
self, admin: User, org_id: str, email: str, role: str
) -> Optional[str]:
admin_user = OrganizationMember.objects.get(user=admin.id)
- organization_member = (
- self.organization_member_service.get_user_by_email(email=email)
+ organization_member = self.organization_member_service.get_user_by_email(
+ email=email
)
if organization_member:
current_roles = self.auth_service.remove_organization_user_role(
@@ -447,33 +416,27 @@ class AuthenticationController:
return None
def save_orgnanization_user_role(self, user_id: str, role: str) -> None:
- organization_user = (
- self.organization_member_service.get_user_by_user_id(
- user_id=user_id
- )
+ organization_user = self.organization_member_service.get_user_by_user_id(
+ user_id=user_id
)
if organization_user:
# consider single role
organization_user.role = role
organization_user.save()
- def create_tenant_user(
- self, organization: Organization, user: User
- ) -> None:
+ def create_tenant_user(self, organization: Organization, user: User) -> None:
with tenant_context(organization):
- existing_tenant_user = (
- self.organization_member_service.get_user_by_id(id=user.id)
+ existing_tenant_user = self.organization_member_service.get_user_by_id(
+ id=user.id
)
if existing_tenant_user:
Logger.info(f"{existing_tenant_user.user.email} Already exist")
else:
account_user = self.get_or_create_user(user=user)
if account_user:
- user_roles = (
- self.auth_service.get_organization_role_of_user(
- user_id=account_user.user_id,
- organization_id=organization.organization_id,
- )
+ user_roles = self.auth_service.get_organization_role_of_user(
+ user_id=account_user.user_id,
+ organization_id=organization.organization_id,
)
user_role = user_roles[0]
tenant_user: OrganizationMember = OrganizationMember(
diff --git a/backend/account/authentication_helper.py b/backend/account/authentication_helper.py
index 2aa6f341b..5336b672b 100644
--- a/backend/account/authentication_helper.py
+++ b/backend/account/authentication_helper.py
@@ -3,9 +3,7 @@ from typing import Any
from account.dto import MemberData
from account.models import Organization, User
-from platform_settings.platform_auth_service import (
- PlatformAuthenticationService,
-)
+from platform_settings.platform_auth_service import PlatformAuthenticationService
logger = logging.getLogger(__name__)
diff --git a/backend/account/authentication_plugin_registry.py b/backend/account/authentication_plugin_registry.py
index 9503b108d..cd630fdaf 100644
--- a/backend/account/authentication_plugin_registry.py
+++ b/backend/account/authentication_plugin_registry.py
@@ -92,7 +92,5 @@ class AuthenticationPluginRegistry:
"""
chosen_auth_module = next(iter(cls.auth_modules.values()))
chosen_metadata = chosen_auth_module[PluginConfig.AUTH_METADATA]
- service_class_name = chosen_metadata[
- PluginConfig.METADATA_SERVICE_CLASS
- ]
+ service_class_name = chosen_metadata[PluginConfig.METADATA_SERVICE_CLASS]
return service_class_name()
diff --git a/backend/account/authentication_service.py b/backend/account/authentication_service.py
index d37ed473a..fe3c92aca 100644
--- a/backend/account/authentication_service.py
+++ b/backend/account/authentication_service.py
@@ -62,9 +62,7 @@ class AuthenticationService:
if self.authenticate_and_login(request, username, password):
return redirect(settings.WEB_APP_ORIGIN_URL)
- return self.render_login_page_with_error(
- request, ErrorMessage.USER_LOGIN_ERROR
- )
+ return self.render_login_page_with_error(request, ErrorMessage.USER_LOGIN_ERROR)
def authenticate_and_login(
self, request: Request, username: str, password: str
@@ -98,9 +96,7 @@ class AuthenticationService:
def render_login_page(self, request: Request) -> Any:
return render(request, UserLoginTemplate.TEMPLATE)
- def render_login_page_with_error(
- self, request: Request, error_message: str
- ) -> Any:
+ def render_login_page_with_error(self, request: Request, error_message: str) -> Any:
return render(
request,
UserLoginTemplate.TEMPLATE,
@@ -251,9 +247,7 @@ class AuthenticationService:
def get_invitations(self, organization_id: str) -> list[MemberInvitation]:
raise MethodNotImplemented()
- def delete_invitation(
- self, organization_id: str, invitation_id: str
- ) -> bool:
+ def delete_invitation(self, organization_id: str, invitation_id: str) -> bool:
raise MethodNotImplemented()
def add_organization_user_role(
@@ -307,9 +301,7 @@ class AuthenticationService:
):
return False
- user, created = User.objects.get_or_create(
- username=DefaultOrg.MOCK_USER
- )
+ user, created = User.objects.get_or_create(username=DefaultOrg.MOCK_USER)
if created:
user.password = make_password(DefaultOrg.MOCK_USER_PASSWORD)
else:
diff --git a/backend/account/custom_auth_middleware.py b/backend/account/custom_auth_middleware.py
index 13533b60d..4812df842 100644
--- a/backend/account/custom_auth_middleware.py
+++ b/backend/account/custom_auth_middleware.py
@@ -20,9 +20,7 @@ class CustomAuthMiddleware:
def __call__(self, request: HttpRequest) -> HttpResponse:
# Returns result without authenticated if added in whitelisted paths
- if any(
- request.path.startswith(path) for path in settings.WHITELISTED_PATHS
- ):
+ if any(request.path.startswith(path) for path in settings.WHITELISTED_PATHS):
return self.get_response(request)
tenantAccessiblePublicPath = False
@@ -56,9 +54,7 @@ class CustomAuthMiddleware:
org_id = DefaultOrg.MOCK_ORG
if not request.user.is_authenticated:
return
- user_session_info = CacheService.get_user_session_info(
- request.user.email
- )
+ user_session_info = CacheService.get_user_session_info(request.user.email)
if not user_session_info:
user_info: UserSessionInfo = UserSessionInfo(
id=request.user.id,
@@ -67,9 +63,7 @@ class CustomAuthMiddleware:
current_org=org_id,
)
CacheService.set_user_session_info(user_info)
- user_session_info = CacheService.get_user_session_info(
- request.user.email
- )
+ user_session_info = CacheService.get_user_session_info(request.user.email)
request.org_id = org_id
request.session["user"] = user_session_info
request.session.save()
@@ -93,16 +87,10 @@ class CustomAuthMiddleware:
if not current_org:
return
- if (
- current_org != connection.get_schema()
- and not tenantAccessiblePublicPath
- ):
+ if current_org != connection.get_schema() and not tenantAccessiblePublicPath:
return
- if (
- current_org == Common.PUBLIC_SCHEMA_NAME
- or tenantAccessiblePublicPath
- ):
+ if current_org == Common.PUBLIC_SCHEMA_NAME or tenantAccessiblePublicPath:
user_service = UserService()
else:
organization_member_service = OrganizationMemberService()
diff --git a/backend/account/custom_exceptions.py b/backend/account/custom_exceptions.py
index 66e3671df..bec24e16c 100644
--- a/backend/account/custom_exceptions.py
+++ b/backend/account/custom_exceptions.py
@@ -18,9 +18,7 @@ class DuplicateData(APIException):
status_code = 400
default_detail = "Duplicate Data"
- def __init__(
- self, detail: Optional[str] = None, code: Optional[int] = None
- ):
+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):
if detail is not None:
self.detail = detail
if code is not None:
@@ -32,9 +30,7 @@ class TableNotExistError(APIException):
status_code = 400
default_detail = "Unknown Table"
- def __init__(
- self, detail: Optional[str] = None, code: Optional[int] = None
- ):
+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):
if detail is not None:
self.detail = detail
if code is not None:
@@ -46,9 +42,7 @@ class UserNotExistError(APIException):
status_code = 400
default_detail = "Unknown User"
- def __init__(
- self, detail: Optional[str] = None, code: Optional[int] = None
- ):
+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):
if detail is not None:
self.detail = detail
if code is not None:
diff --git a/backend/account/migrations/0001_initial.py b/backend/account/migrations/0001_initial.py
index 0ff85cfe8..e8622577b 100644
--- a/backend/account/migrations/0001_initial.py
+++ b/backend/account/migrations/0001_initial.py
@@ -222,9 +222,7 @@ class Migration(migrations.Migration):
),
(
"domain",
- models.CharField(
- db_index=True, max_length=253, unique=True
- ),
+ models.CharField(db_index=True, max_length=253, unique=True),
),
(
"is_primary",
diff --git a/backend/account/migrations/0003_platformkey.py b/backend/account/migrations/0003_platformkey.py
index 80cdc227d..f01cc5295 100644
--- a/backend/account/migrations/0003_platformkey.py
+++ b/backend/account/migrations/0003_platformkey.py
@@ -28,9 +28,7 @@ class Migration(migrations.Migration):
("key", models.UUIDField(default=uuid.uuid4)),
(
"key_name",
- models.CharField(
- blank=True, max_length=64, null=True, unique=True
- ),
+ models.CharField(blank=True, max_length=64, null=True, unique=True),
),
("is_active", models.BooleanField(default=False)),
(
diff --git a/backend/account/models.py b/backend/account/models.py
index 63dade30d..2eae51073 100644
--- a/backend/account/models.py
+++ b/backend/account/models.py
@@ -99,9 +99,7 @@ class PlatformKey(models.Model):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
key = models.UUIDField(default=uuid.uuid4)
- key_name = models.CharField(
- max_length=KEY_SIZE, null=False, blank=True, default=""
- )
+ key_name = models.CharField(max_length=KEY_SIZE, null=False, blank=True, default="")
is_active = models.BooleanField(default=False)
organization = models.ForeignKey(
"Organization",
diff --git a/backend/account/organization.py b/backend/account/organization.py
index a584bf3a8..5cd547b3f 100644
--- a/backend/account/organization.py
+++ b/backend/account/organization.py
@@ -41,9 +41,7 @@ class OrganizationService:
cls.add(organization_id=organization_id)
except IntegrityError as error:
- Logger.info(
- f"[Duplicate Id] Failed to create Organization Error: {error}"
- )
+ Logger.info(f"[Duplicate Id] Failed to create Organization Error: {error}")
raise error
# Add one or more domains for the tenant
domain = Domain()
diff --git a/backend/account/subscription_loader.py b/backend/account/subscription_loader.py
index c516dc3bf..d380ed19d 100644
--- a/backend/account/subscription_loader.py
+++ b/backend/account/subscription_loader.py
@@ -24,12 +24,8 @@ def load_plugins() -> list[Any]:
"""Iterate through the subscription plugins and register them."""
plugins_app = apps.get_app_config(SubscriptionConfig.PLUGINS_APP)
package_path = plugins_app.module.__package__
- subscription_dir = os.path.join(
- plugins_app.path, SubscriptionConfig.PLUGIN_DIR
- )
- subscription_package_path = (
- f"{package_path}.{SubscriptionConfig.PLUGIN_DIR}"
- )
+ subscription_dir = os.path.join(plugins_app.path, SubscriptionConfig.PLUGIN_DIR)
+ subscription_package_path = f"{package_path}.{SubscriptionConfig.PLUGIN_DIR}"
subscription_plugins: list[Any] = []
if not os.path.exists(subscription_dir):
@@ -48,9 +44,7 @@ def load_plugins() -> list[Any]:
else:
continue
try:
- full_module_path = (
- f"{subscription_package_path}.{subscription_module_name}"
- )
+ full_module_path = f"{subscription_package_path}.{subscription_module_name}"
module = import_module(full_module_path)
metadata = getattr(module, SubscriptionConfig.METADATA, {})
diff --git a/backend/account/urls.py b/backend/account/urls.py
index 1987336a6..8cc60e65c 100644
--- a/backend/account/urls.py
+++ b/backend/account/urls.py
@@ -15,10 +15,6 @@ urlpatterns = [
path("logout", logout, name="logout"),
path("callback", callback, name="callback"),
path("organization", get_organizations, name="get_organizations"),
- path(
- "organization/<str:id>/set", set_organization, name="set_organization"
- ),
- path(
- "organization/create", create_organization, name="create_organization"
- ),
+ path("organization/<str:id>/set", set_organization, name="set_organization"),
+ path("organization/create", create_organization, name="create_organization"),
]
diff --git a/backend/account/views.py b/backend/account/views.py
index 3fdea3772..22a9b927a 100644
--- a/backend/account/views.py
+++ b/backend/account/views.py
@@ -2,10 +2,7 @@ import logging
from typing import Any
from account.authentication_controller import AuthenticationController
-from account.dto import (
- OrganizationSignupRequestBody,
- OrganizationSignupResponse,
-)
+from account.dto import OrganizationSignupRequestBody, OrganizationSignupResponse
from account.models import Organization
from account.organization import OrganizationService
from account.serializer import (
@@ -25,9 +22,7 @@ def create_organization(request: Request) -> Response:
serializer = OrganizationSignupSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
try:
- requestBody: OrganizationSignupRequestBody = makeSignupRequestParams(
- serializer
- )
+ requestBody: OrganizationSignupRequestBody = makeSignupRequestParams(serializer)
organization: Organization = OrganizationService.create_organization(
requestBody.name,
diff --git a/backend/adapter_processor/adapter_processor.py b/backend/adapter_processor/adapter_processor.py
index 38585488f..f8c4bb1ef 100644
--- a/backend/adapter_processor/adapter_processor.py
+++ b/backend/adapter_processor/adapter_processor.py
@@ -11,9 +11,7 @@ from adapter_processor.exceptions import (
)
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
-from platform_settings.platform_auth_service import (
- PlatformAuthenticationService,
-)
+from platform_settings.platform_auth_service import PlatformAuthenticationService
from unstract.adapters.adapterkit import Adapterkit
from unstract.adapters.base import Adapter
from unstract.adapters.enums import AdapterTypes
@@ -61,9 +59,7 @@ class AdapterProcessor:
{
AdapterKeys.ID: each_adapter.get(AdapterKeys.ID),
AdapterKeys.NAME: each_adapter.get(AdapterKeys.NAME),
- AdapterKeys.DESCRIPTION: each_adapter.get(
- AdapterKeys.DESCRIPTION
- ),
+ AdapterKeys.DESCRIPTION: each_adapter.get(AdapterKeys.DESCRIPTION),
AdapterKeys.ICON: each_adapter.get(AdapterKeys.ICON),
AdapterKeys.ADAPTER_TYPE: each_adapter.get(
AdapterKeys.ADAPTER_TYPE
@@ -79,9 +75,7 @@ class AdapterProcessor:
"id", adapter_id
)
if len(updated_adapters) == 0:
- logger.error(
- f"Invalid adapter ID {adapter_id} while invoking utility"
- )
+ logger.error(f"Invalid adapter ID {adapter_id} while invoking utility")
raise InValidAdapterId()
return updated_adapters[0].get(key_value)
@@ -89,25 +83,14 @@ class AdapterProcessor:
def test_adapter(adapter_id: str, adapter_metadata: dict[str, Any]) -> bool:
logger.info(f"Testing adapter: {adapter_id}")
try:
- adapter_class = Adapterkit().get_adapter_class_by_adapter_id(
- adapter_id
- )
-
- if (
- adapter_metadata.pop(AdapterKeys.ADAPTER_TYPE)
- == AdapterKeys.X2TEXT
- ):
- adapter_metadata[X2TextConstants.X2TEXT_HOST] = (
- settings.X2TEXT_HOST
- )
- adapter_metadata[X2TextConstants.X2TEXT_PORT] = (
- settings.X2TEXT_PORT
- )
- platform_key = (
- PlatformAuthenticationService.get_active_platform_key()
- )
- adapter_metadata[X2TextConstants.PLATFORM_SERVICE_API_KEY] = (
- str(platform_key.key)
+ adapter_class = Adapterkit().get_adapter_class_by_adapter_id(adapter_id)
+
+ if adapter_metadata.pop(AdapterKeys.ADAPTER_TYPE) == AdapterKeys.X2TEXT:
+ adapter_metadata[X2TextConstants.X2TEXT_HOST] = settings.X2TEXT_HOST
+ adapter_metadata[X2TextConstants.X2TEXT_PORT] = settings.X2TEXT_PORT
+ platform_key = PlatformAuthenticationService.get_active_platform_key()
+ adapter_metadata[X2TextConstants.PLATFORM_SERVICE_API_KEY] = str(
+ platform_key.key
)
adapter_instance = adapter_class(adapter_metadata)
@@ -135,10 +118,8 @@ class AdapterProcessor:
) = UserDefaultAdapter.objects.get_or_create(user=user)
if default_triad.get(AdapterKeys.LLM_DEFAULT, None):
- user_default_adapter.default_llm_adapter = (
- AdapterInstance.objects.get(
- pk=default_triad[AdapterKeys.LLM_DEFAULT]
- )
+ user_default_adapter.default_llm_adapter = AdapterInstance.objects.get(
+ pk=default_triad[AdapterKeys.LLM_DEFAULT]
)
if default_triad.get(AdapterKeys.EMBEDDING_DEFAULT, None):
user_default_adapter.default_embedding_adapter = (
@@ -207,9 +188,7 @@ class AdapterProcessor:
the specified adapter type.
"""
- adapters: list[AdapterInstance] = AdapterInstance.objects.for_user(
- user
- ).filter(
+ adapters: list[AdapterInstance] = AdapterInstance.objects.for_user(user).filter(
adapter_type=adapter_type.value,
)
return adapters
@@ -271,8 +250,7 @@ class AdapterProcessor:
except ObjectDoesNotExist as e:
logger.error(f"No default adapters found: {e}")
raise InternalServiceError(
- "No default adapters found, "
- "configure them through Platform Settings"
+ "No default adapters found, " "configure them through Platform Settings"
)
except Exception as e:
logger.error(f"Error occurred while fetching default adapters: {e}")
diff --git a/backend/adapter_processor/constants.py b/backend/adapter_processor/constants.py
index 196166066..8c9326066 100644
--- a/backend/adapter_processor/constants.py
+++ b/backend/adapter_processor/constants.py
@@ -21,8 +21,7 @@ class AdapterKeys:
X2TEXT_DEFAULT = "x2text_default"
SHARED_USERS = "shared_users"
ADAPTER_NAME_EXISTS = (
- "Configuration with this ID already exists. "
- "Please try with a different ID"
+ "Configuration with this ID already exists. " "Please try with a different ID"
)
ADAPTER_CREATED_BY = "created_by_email"
ADAPTER_CONTEXT_WINDOW_SIZE = "context_window_size"
diff --git a/backend/adapter_processor/serializers.py b/backend/adapter_processor/serializers.py
index 7d1f75022..ef42ae773 100644
--- a/backend/adapter_processor/serializers.py
+++ b/backend/adapter_processor/serializers.py
@@ -31,9 +31,7 @@ class BaseAdapterSerializer(AuditSerializer):
class DefaultAdapterSerializer(serializers.Serializer):
- llm_default = serializers.CharField(
- max_length=FLC.UUID_LENGTH, required=False
- )
+ llm_default = serializers.CharField(max_length=FLC.UUID_LENGTH, required=False)
embedding_default = serializers.CharField(
max_length=FLC.UUID_LENGTH, required=False
)
@@ -52,9 +50,7 @@ class AdapterInstanceSerializer(BaseAdapterSerializer):
if data.get(AdapterKeys.ADAPTER_METADATA, None):
encryption_secret: str = settings.ENCRYPTION_KEY
f: Fernet = Fernet(encryption_secret.encode("utf-8"))
- json_string: str = json.dumps(
- data.pop(AdapterKeys.ADAPTER_METADATA)
- )
+ json_string: str = json.dumps(data.pop(AdapterKeys.ADAPTER_METADATA))
data[AdapterKeys.ADAPTER_METADATA_B] = f.encrypt(
json_string.encode("utf-8")
diff --git a/backend/adapter_processor/views.py b/backend/adapter_processor/views.py
index 8025f10d5..650bb5a9d 100644
--- a/backend/adapter_processor/views.py
+++ b/backend/adapter_processor/views.py
@@ -57,9 +57,7 @@ class DefaultAdapterViewSet(ModelViewSet):
self, request: Request, *args: tuple[Any], **kwargs: dict[str, Any]
) -> HttpResponse:
try:
- user_default_adapter = UserDefaultAdapter.objects.get(
- user=request.user
- )
+ user_default_adapter = UserDefaultAdapter.objects.get(user=request.user)
serializer = UserDefaultAdapterSerializer(user_default_adapter).data
return Response(serializer)
@@ -98,23 +96,17 @@ class AdapterViewSet(GenericViewSet):
adapter_name = request.GET.get(AdapterKeys.ID)
if adapter_name is None or adapter_name == "":
raise IdIsMandatory()
- json_schema = AdapterProcessor.get_json_schema(
- adapter_id=adapter_name
- )
+ json_schema = AdapterProcessor.get_json_schema(adapter_id=adapter_name)
return Response(data=json_schema, status=status.HTTP_200_OK)
def test(self, request: Request) -> Response:
"""Tests the connector against the credentials passed."""
- serializer: AdapterInstanceSerializer = self.get_serializer(
- data=request.data
- )
+ serializer: AdapterInstanceSerializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
adapter_id = serializer.validated_data.get(AdapterKeys.ADAPTER_ID)
- adapter_metadata = serializer.validated_data.get(
- AdapterKeys.ADAPTER_METADATA
- )
- adapter_metadata[AdapterKeys.ADAPTER_TYPE] = (
- serializer.validated_data.get(AdapterKeys.ADAPTER_TYPE)
+ adapter_metadata = serializer.validated_data.get(AdapterKeys.ADAPTER_METADATA)
+ adapter_metadata[AdapterKeys.ADAPTER_TYPE] = serializer.validated_data.get(
+ AdapterKeys.ADAPTER_TYPE
)
try:
test_result = AdapterProcessor.test_adapter(
@@ -138,9 +130,9 @@ class AdapterInstanceViewSet(ModelViewSet):
self.request,
constant.ADAPTER_TYPE,
):
- queryset = AdapterInstance.objects.for_user(
- self.request.user
- ).filter(**filter_args)
+ queryset = AdapterInstance.objects.for_user(self.request.user).filter(
+ **filter_args
+ )
else:
queryset = AdapterInstance.objects.for_user(self.request.user)
return queryset
@@ -165,9 +157,7 @@ class AdapterInstanceViewSet(ModelViewSet):
created,
) = UserDefaultAdapter.objects.get_or_create(user=request.user)
- adapter_type = serializer.validated_data.get(
- AdapterKeys.ADAPTER_TYPE
- )
+ adapter_type = serializer.validated_data.get(AdapterKeys.ADAPTER_TYPE)
if (adapter_type == AdapterKeys.LLM) and (
not user_default_adapter.default_llm_adapter
):
@@ -189,9 +179,7 @@ class AdapterInstanceViewSet(ModelViewSet):
user_default_adapter.save()
except IntegrityError:
- raise UniqueConstraintViolation(
- f"{AdapterKeys.ADAPTER_NAME_EXISTS}"
- )
+ raise UniqueConstraintViolation(f"{AdapterKeys.ADAPTER_NAME_EXISTS}")
except Exception as e:
logger.error(f"Error saving adapter to DB: {e}")
raise InternalServiceError
@@ -213,18 +201,15 @@ class AdapterInstanceViewSet(ModelViewSet):
)
or (
adapter_type == AdapterKeys.EMBEDDING
- and adapter_instance
- == user_default_adapter.default_embedding_adapter
+ and adapter_instance == user_default_adapter.default_embedding_adapter
)
or (
adapter_type == AdapterKeys.VECTOR_DB
- and adapter_instance
- == user_default_adapter.default_vector_db_adapter
+ and adapter_instance == user_default_adapter.default_vector_db_adapter
)
or (
adapter_type == AdapterKeys.X2TEXT
- and adapter_instance
- == user_default_adapter.default_x2text_adapter
+ and adapter_instance == user_default_adapter.default_x2text_adapter
)
):
logger.error("Cannot delete a default adapter")
@@ -238,9 +223,7 @@ class AdapterInstanceViewSet(ModelViewSet):
f" named {adapter_instance.adapter_name}"
)
# TODO: Provide details of adpter usage with exception object
- raise DeleteAdapterInUseError(
- adapter_name=adapter_instance.adapter_name
- )
+ raise DeleteAdapterInUseError(adapter_name=adapter_instance.adapter_name)
return Response(status=status.HTTP_204_NO_CONTENT)
def partial_update(
@@ -265,15 +248,9 @@ class AdapterInstanceViewSet(ModelViewSet):
if user_default_adapter.default_llm_adapter == adapter:
user_default_adapter.default_llm_adapter = None
- elif (
- user_default_adapter.default_embedding_adapter
- == adapter
- ):
+ elif user_default_adapter.default_embedding_adapter == adapter:
user_default_adapter.default_embedding_adapter = None
- elif (
- user_default_adapter.default_vector_db_adapter
- == adapter
- ):
+ elif user_default_adapter.default_vector_db_adapter == adapter:
user_default_adapter.default_vector_db_adapter = None
elif user_default_adapter.default_x2text_adapter == adapter:
user_default_adapter.default_x2text_adapter = None
@@ -289,9 +266,7 @@ class AdapterInstanceViewSet(ModelViewSet):
return super().partial_update(request, *args, **kwargs)
@action(detail=True, methods=["get"])
- def list_of_shared_users(
- self, request: HttpRequest, pk: Any = None
- ) -> Response:
+ def list_of_shared_users(self, request: HttpRequest, pk: Any = None) -> Response:
self.permission_classes = [IsOwnerOrSharedUser]
adapter = (
self.get_object()
diff --git a/backend/api/deployment_helper.py b/backend/api/deployment_helper.py
index d2136990b..8969bdb99 100644
--- a/backend/api/deployment_helper.py
+++ b/backend/api/deployment_helper.py
@@ -44,9 +44,7 @@ class DeploymentHelper:
"""
@wraps(func)
- def wrapper(
- self: Any, request: Request, *args: Any, **kwargs: Any
- ) -> Any:
+ def wrapper(self: Any, request: Request, *args: Any, **kwargs: Any) -> Any:
"""Wrapper to validate the inputs and key.
Args:
@@ -62,26 +60,18 @@ class DeploymentHelper:
try:
authorization_header = request.headers.get("Authorization")
api_key = None
- if authorization_header and authorization_header.startswith(
- "Bearer "
- ):
+ if authorization_header and authorization_header.startswith("Bearer "):
api_key = authorization_header.split(" ")[1]
if not api_key:
raise Forbidden("Missing api key")
- org_name = kwargs.get("org_name") or request.data.get(
- "org_name"
- )
- api_name = kwargs.get("api_name") or request.data.get(
- "api_name"
- )
+ org_name = kwargs.get("org_name") or request.data.get("org_name")
+ api_name = kwargs.get("api_name") or request.data.get("api_name")
if not api_name:
raise Forbidden("Missing api_name")
tenant = get_tenant_model().objects.get(schema_name=org_name)
with tenant_context(tenant):
- api_deployment = (
- DeploymentHelper.get_deployment_by_api_name(
- api_name=api_name
- )
+ api_deployment = DeploymentHelper.get_deployment_by_api_name(
+ api_name=api_name
)
DeploymentHelper.validate_api(
api_deployment=api_deployment, api_key=api_key
@@ -99,9 +89,7 @@ class DeploymentHelper:
return wrapper
@staticmethod
- def validate_api(
- api_deployment: Optional[APIDeployment], api_key: str
- ) -> None:
+ def validate_api(api_deployment: Optional[APIDeployment], api_key: str) -> None:
"""Validating API and API key.
Args:
@@ -248,7 +236,7 @@ class DeploymentHelper:
Returns:
ReturnDict: status/result of execution
"""
- execution_response: ExecutionResponse = (
- WorkflowHelper.get_status_of_async_task(execution_id=execution_id)
+ execution_response: ExecutionResponse = WorkflowHelper.get_status_of_async_task(
+ execution_id=execution_id
)
return execution_response
diff --git a/backend/api/exceptions.py b/backend/api/exceptions.py
index 6d1b65ac0..f8bb97c5b 100644
--- a/backend/api/exceptions.py
+++ b/backend/api/exceptions.py
@@ -12,9 +12,7 @@ class ApiKeyCreateException(APIException):
status_code = 500
default_detail = "Exception while create API key"
- def __init__(
- self, detail: Optional[str] = None, code: Optional[int] = None
- ):
+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):
if detail is not None:
self.detail = detail
if code is not None:
@@ -28,9 +26,7 @@ class Forbidden(APIException):
"User is forbidden from performing this action. Please contact admin"
)
- def __init__(
- self, detail: Optional[str] = None, code: Optional[int] = None
- ):
+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):
if detail is not None:
self.detail = detail
if code is not None:
@@ -42,9 +38,7 @@ class APINotFound(APIException):
status_code = 404
default_detail = "Api not found"
- def __init__(
- self, detail: Optional[str] = None, code: Optional[int] = None
- ):
+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):
if detail is not None:
self.detail = detail
if code is not None:
@@ -56,9 +50,7 @@ class InvalidAPIRequest(APIException):
status_code = 400
default_detail = "Bad request"
- def __init__(
- self, detail: Optional[str] = None, code: Optional[int] = None
- ):
+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):
if detail is not None:
self.detail = detail
if code is not None:
@@ -70,9 +62,7 @@ class InactiveAPI(APIException):
status_code = 404
default_detail = "API not found or Inactive"
- def __init__(
- self, detail: Optional[str] = None, code: Optional[int] = None
- ):
+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):
if detail is not None:
self.detail = detail
if code is not None:
@@ -84,9 +74,7 @@ class UnauthorizedKey(APIException):
status_code = 401
default_detail = "Unauthorized"
- def __init__(
- self, detail: Optional[str] = None, code: Optional[int] = None
- ):
+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):
if detail is not None:
self.detail = detail
if code is not None:
diff --git a/backend/api/models.py b/backend/api/models.py
index ac07ec74f..a2c81f4fc 100644
--- a/backend/api/models.py
+++ b/backend/api/models.py
@@ -87,9 +87,7 @@ class APIDeployment(BaseModel):
except APIDeployment.DoesNotExist:
org_schema = connection.get_tenant().schema_name
- self.api_endpoint = (
- f"{ApiExecution.PATH}/{org_schema}/{self.api_name}/"
- )
+ self.api_endpoint = f"{ApiExecution.PATH}/{org_schema}/{self.api_name}/"
super().save(*args, **kwargs)
diff --git a/backend/api/serializers.py b/backend/api/serializers.py
index b1622b431..1b56e0d55 100644
--- a/backend/api/serializers.py
+++ b/backend/api/serializers.py
@@ -42,9 +42,7 @@ class APIKeySerializer(AuditSerializer):
context."""
context = self.context.get("context", {})
deployment: APIDeployment = context.get("deployment")
- representation: OrderedDict[str, Any] = super().to_representation(
- instance
- )
+ representation: OrderedDict[str, Any] = super().to_representation(instance)
if deployment:
representation["api"] = deployment.id
representation["description"] = f"API Key for {deployment.name}"
diff --git a/backend/backend/exceptions.py b/backend/backend/exceptions.py
index ae98cdd05..01c59ad70 100644
--- a/backend/backend/exceptions.py
+++ b/backend/backend/exceptions.py
@@ -22,8 +22,7 @@ class UnstractBaseException(APIException):
detail = core_err.user_message
if detail and "Name or service not known" in str(detail):
detail = (
- "Failed to establish a new connection: "
- "Name or service not known"
+ "Failed to establish a new connection: " "Name or service not known"
)
super().__init__(detail=detail, **kwargs)
self._core_err = core_err
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 8284e1d9e..0f14e0b25 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -84,9 +84,7 @@ if ENV_FILE:
WORKFLOW_ACTION_EXPIRATION_TIME_IN_SECOND = os.environ.get(
"WORKFLOW_ACTION_EXPIRATION_TIME_IN_SECOND", 10800
)
-WEB_APP_ORIGIN_URL = os.environ.get(
- "WEB_APP_ORIGIN_URL", "http://localhost:3000"
-)
+WEB_APP_ORIGIN_URL = os.environ.get("WEB_APP_ORIGIN_URL", "http://localhost:3000")
LOGIN_NEXT_URL = os.environ.get("LOGIN_NEXT_URL", "http://localhost:3000/org")
LANDING_URL = os.environ.get("LANDING_URL", "http://localhost:3000/landing")
@@ -98,9 +96,7 @@ DJANGO_APP_BACKEND_URL = os.environ.get(
INTERNAL_SERVICE_API_KEY = os.environ.get("INTERNAL_SERVICE_API_KEY")
GOOGLE_STORAGE_ACCESS_KEY_ID = os.environ.get("GOOGLE_STORAGE_ACCESS_KEY_ID")
-GOOGLE_STORAGE_SECRET_ACCESS_KEY = os.environ.get(
- "GOOGLE_STORAGE_SECRET_ACCESS_KEY"
-)
+GOOGLE_STORAGE_SECRET_ACCESS_KEY = os.environ.get("GOOGLE_STORAGE_SECRET_ACCESS_KEY")
UNSTRACT_FREE_STORAGE_BUCKET_NAME = os.environ.get(
"UNSTRACT_FREE_STORAGE_BUCKET_NAME", "pandora-user-storage"
)
@@ -337,9 +333,7 @@ CELERY_TASK_MAX_RETRIES = 3
CELERY_TASK_RETRY_BACKOFF = 60 # Time in seconds before retrying the task
# Feature Flag
-FEATURE_FLAG_SERVICE_URL = {
- "evaluate": f"{FLIPT_BASE_URL}/api/v1/flags/evaluate/"
-}
+FEATURE_FLAG_SERVICE_URL = {"evaluate": f"{FLIPT_BASE_URL}/api/v1/flags/evaluate/"}
SCHEDULER_KWARGS = {
"coalesce": True,
@@ -357,16 +351,13 @@ AUTH_PASSWORD_VALIDATORS = [
"UserAttributeSimilarityValidator",
},
{
- "NAME": "django.contrib.auth.password_validation."
- "MinimumLengthValidator",
+ "NAME": "django.contrib.auth.password_validation." "MinimumLengthValidator",
},
{
- "NAME": "django.contrib.auth.password_validation."
- "CommonPasswordValidator",
+ "NAME": "django.contrib.auth.password_validation." "CommonPasswordValidator",
},
{
- "NAME": "django.contrib.auth.password_validation."
- "NumericPasswordValidator",
+ "NAME": "django.contrib.auth.password_validation." "NumericPasswordValidator",
},
]
@@ -427,9 +418,7 @@ REDOC_SETTINGS = {
}
# Social Auth Settings
-SOCIAL_AUTH_LOGIN_REDIRECT_URL = (
- f"{WEB_APP_ORIGIN_URL}/oauth-status/?status=success"
-)
+SOCIAL_AUTH_LOGIN_REDIRECT_URL = f"{WEB_APP_ORIGIN_URL}/oauth-status/?status=success"
SOCIAL_AUTH_LOGIN_ERROR_URL = f"{WEB_APP_ORIGIN_URL}/oauth-status/?status=error"
SOCIAL_AUTH_EXTRA_DATA_EXPIRATION_TIME_IN_SECOND = os.environ.get(
"SOCIAL_AUTH_EXTRA_DATA_EXPIRATION_TIME_IN_SECOND", 3600
diff --git a/backend/connector/connector_instance_helper.py b/backend/connector/connector_instance_helper.py
index 2589d3c17..65a378f26 100644
--- a/backend/connector/connector_instance_helper.py
+++ b/backend/connector/connector_instance_helper.py
@@ -106,9 +106,7 @@ class ConnectorInstanceHelper:
if connector_name is not None:
filter_params["connector_name"] = connector_name
- connector_instances = ConnectorInstance.objects.filter(
- **filter_params
- ).all()
+ connector_instances = ConnectorInstance.objects.filter(**filter_params).all()
logger.info(f"Retrived connector instance values {connector_instances}")
if values is not None:
filtered_connector_instances = connector_instances.values(*values)
@@ -153,13 +151,11 @@ class ConnectorInstanceHelper:
filter_params["connector_name"] = connector_name
try:
- connector_instance: ConnectorInstance = (
- ConnectorInstance.objects.filter(**filter_params).first()
- )
+ connector_instance: ConnectorInstance = ConnectorInstance.objects.filter(
+ **filter_params
+ ).first()
except Exception as exc:
- logger.error(
- f"Error occured while fetching connector instances {exc}"
- )
+ logger.error(f"Error occured while fetching connector instances {exc}")
raise exc
return connector_instance
@@ -331,8 +327,6 @@ class ConnectorInstanceHelper:
filter_params: dict[str, Any] = {
"workflow": workflow_id,
}
- connector_instances = ConnectorInstance.objects.filter(
- **filter_params
- ).all()
+ connector_instances = ConnectorInstance.objects.filter(**filter_params).all()
logger.info(f"Retrived connector instance values {connector_instances}")
return list(connector_instances)
diff --git a/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py b/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py
index 1e4a342c9..7c075439d 100644
--- a/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py
+++ b/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py
@@ -24,9 +24,7 @@ class Migration(migrations.Migration):
if hasattr(obj, "connector_metadata"):
json_string: str = json.dumps(obj.connector_metadata)
- obj.connector_metadata_b = f.encrypt(
- json_string.encode("utf-8")
- )
+ obj.connector_metadata_b = f.encrypt(json_string.encode("utf-8"))
obj.save()
operations = [
diff --git a/backend/connector/models.py b/backend/connector/models.py
index eb0a3f7da..352ee3694 100644
--- a/backend/connector/models.py
+++ b/backend/connector/models.py
@@ -48,17 +48,13 @@ class ConnectorInstance(BaseModel):
null=False,
blank=False,
)
- connector_id = models.CharField(
- max_length=FLC.CONNECTOR_ID_LENGTH, default=""
- )
+ connector_id = models.CharField(max_length=FLC.CONNECTOR_ID_LENGTH, default="")
# TODO Required to be removed
connector_metadata = ConnectorAuthJSONField(
db_column="connector_metadata", null=False, blank=False, default=dict
)
connector_metadata_b = models.BinaryField(null=True)
- connector_version = models.CharField(
- max_length=VERSION_NAME_SIZE, default=""
- )
+ connector_version = models.CharField(max_length=VERSION_NAME_SIZE, default="")
connector_type = models.CharField(choices=ConnectorType.choices)
connector_auth = models.ForeignKey(
ConnectorAuth, on_delete=models.SET_NULL, null=True, blank=True
diff --git a/backend/connector/serializers.py b/backend/connector/serializers.py
index 992e2cfbf..13fea7ae8 100644
--- a/backend/connector/serializers.py
+++ b/backend/connector/serializers.py
@@ -35,11 +35,9 @@ class ConnectorInstanceSerializer(AuditSerializer):
and CIKey.CONNECTOR_METADATA in kwargs
):
try:
- connector_oauth = (
- ConnectorAuthHelper.get_or_create_connector_auth(
- user=user, # type: ignore
- oauth_credentials=kwargs[CIKey.CONNECTOR_METADATA],
- )
+ connector_oauth = ConnectorAuthHelper.get_or_create_connector_auth(
+ user=user, # type: ignore
+ oauth_credentials=kwargs[CIKey.CONNECTOR_METADATA],
)
kwargs[CIKey.CONNECTOR_AUTH] = connector_oauth
(
@@ -61,9 +59,7 @@ class ConnectorInstanceSerializer(AuditSerializer):
if self.validated_data:
self.validated_data.pop(CIKey.CONNECTOR_METADATA)
- kwargs[CIKey.CONNECTOR_METADATA_B] = f.encrypt(
- json_string.encode("utf-8")
- )
+ kwargs[CIKey.CONNECTOR_METADATA_B] = f.encrypt(json_string.encode("utf-8"))
instance = super().save(**kwargs)
return instance
@@ -76,10 +72,8 @@ class ConnectorInstanceSerializer(AuditSerializer):
if SerializerUtils.check_context_for_GET_or_POST(context=self.context):
rep.pop(CIKey.CONNECTOR_AUTH)
# set icon fields for UI
- rep[ConnectorKeys.ICON] = (
- ConnectorProcessor.get_connector_data_with_key(
- instance.connector_id, ConnectorKeys.ICON
- )
+ rep[ConnectorKeys.ICON] = ConnectorProcessor.get_connector_data_with_key(
+ instance.connector_id, ConnectorKeys.ICON
)
encryption_secret: str = settings.ENCRYPTION_KEY
f: Fernet = Fernet(encryption_secret.encode("utf-8"))
diff --git a/backend/connector/tests/connector_tests.py b/backend/connector/tests/connector_tests.py
index 4e909f68b..cad6512f2 100644
--- a/backend/connector/tests/connector_tests.py
+++ b/backend/connector/tests/connector_tests.py
@@ -203,9 +203,9 @@ class TestConnector(APITestCase):
},
}
response = self.client.put(url, data, format="json")
- nested_value = response.data["connector_metadata"][
- "sample_metadata_json"
- ]["key1"]
+ nested_value = response.data["connector_metadata"]["sample_metadata_json"][
+ "key1"
+ ]
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(nested_value, "value1")
@@ -229,9 +229,9 @@ class TestConnector(APITestCase):
},
}
response = self.client.put(url, data, format="json")
- nested_value = response.data["connector_metadata"][
- "sample_metadata_json"
- ]["key1"]
+ nested_value = response.data["connector_metadata"]["sample_metadata_json"][
+ "key1"
+ ]
nested_list = response.data["connector_metadata"]["file_list"]
last_val = nested_list.pop()
@@ -296,9 +296,7 @@ class TestConnector(APITestCase):
self.assertEqual(
connector_id,
- ConnectorInstance.objects.get(
- connector_id=connector_id
- ).connector_id,
+ ConnectorInstance.objects.get(connector_id=connector_id).connector_id,
)
def test_connectors_update_json_field_patch(self) -> None:
diff --git a/backend/connector/unstract_account.py b/backend/connector/unstract_account.py
index 96d998a22..e8f496089 100644
--- a/backend/connector/unstract_account.py
+++ b/backend/connector/unstract_account.py
@@ -35,9 +35,7 @@ class UnstractAccount:
except ClientError as e:
logger.info(f"{bucket_name} Folder {account_folder} does not exist")
if e.response["Error"]["Code"] == "404":
- logger.info(
- f"Folder {account_folder} does not exist. Creating it..."
- )
+ logger.info(f"Folder {account_folder} does not exist. Creating it...")
s3.put_object(Bucket=bucket_name, Key=account_folder)
account_folder_output = f"{self.tenant}/{self.username}/output/"
s3.put_object(Bucket=bucket_name, Key=account_folder_output)
@@ -67,8 +65,7 @@ class UnstractAccount:
folder, os.path.relpath(local_file_path, local_path)
)
logger.info(
- f"Uploading: {local_file_path} => "
- f"s3://{bucket_name}/{s3_key}"
+ f"Uploading: {local_file_path} => " f"s3://{bucket_name}/{s3_key}"
)
try:
s3.upload_file(local_file_path, bucket_name, s3_key)
diff --git a/backend/connector/views.py b/backend/connector/views.py
index 428a3678b..2443d6bd2 100644
--- a/backend/connector/views.py
+++ b/backend/connector/views.py
@@ -41,9 +41,7 @@ class ConnectorInstanceViewSet(viewsets.ModelViewSet):
queryset = ConnectorInstance.objects.all()
return queryset
- def _get_connector_metadata(
- self, connector_id: str
- ) -> Optional[dict[str, str]]:
+ def _get_connector_metadata(self, connector_id: str) -> Optional[dict[str, str]]:
"""Gets connector metadata for the ConnectorInstance.
For non oauth based - obtains from request
@@ -58,9 +56,7 @@ class ConnectorInstanceViewSet(viewsets.ModelViewSet):
connector_metadata = None
if ConnectorInstance.supportsOAuth(connector_id=connector_id):
logger.info(f"Fetching oauth data for {connector_id}")
- oauth_key = self.request.query_params.get(
- ConnectorAuthKey.OAUTH_KEY
- )
+ oauth_key = self.request.query_params.get(ConnectorAuthKey.OAUTH_KEY)
if oauth_key is None:
logger.error("OAuth key missing")
raise MissingParamException(param=ConnectorAuthKey.OAUTH_KEY)
@@ -99,9 +95,7 @@ class ConnectorInstanceViewSet(viewsets.ModelViewSet):
connector_metadata = None
connector_id = self.request.data.get(CIKey.CONNECTOR_ID)
try:
- connector_metadata = self._get_connector_metadata(
- connector_id=connector_id
- )
+ connector_metadata = self._get_connector_metadata(connector_id=connector_id)
except Exception as exc:
logger.error(f"Error while obtaining ConnectorAuth: {exc}")
raise OAuthTimeOut
diff --git a/backend/connector_auth/models.py b/backend/connector_auth/models.py
index a9331e5b2..653400055 100644
--- a/backend/connector_auth/models.py
+++ b/backend/connector_auth/models.py
@@ -57,10 +57,7 @@ class ConnectorAuth(AbstractUserSocialAuth):
def set_extra_data(self, extra_data=None): # type: ignore
ConnectorAuth.check_credential_format(extra_data)
- if (
- extra_data[SocialAuthConstants.PROVIDER]
- == SocialAuthConstants.GOOGLE_OAUTH
- ):
+ if extra_data[SocialAuthConstants.PROVIDER] == SocialAuthConstants.GOOGLE_OAUTH:
extra_data = GoogleAuthHelper.enrich_connector_metadata(extra_data)
return super().set_extra_data(extra_data)
@@ -73,17 +70,13 @@ class ConnectorAuth(AbstractUserSocialAuth):
backend = self.get_backend_instance(strategy)
if token and backend and hasattr(backend, "refresh_token"):
response = backend.refresh_token(token, *args, **kwargs)
- extra_data = backend.extra_data(
- self, self.uid, response, self.extra_data
- )
+ extra_data = backend.extra_data(self, self.uid, response, self.extra_data)
extra_data[SocialAuthConstants.PROVIDER] = backend.name
extra_data[SocialAuthConstants.UID] = self.uid
if self.set_extra_data(extra_data): # type: ignore
self.save()
- def get_and_refresh_tokens(
- self, request: Request = None
- ) -> tuple[JSONField, bool]:
+ def get_and_refresh_tokens(self, request: Request = None) -> tuple[JSONField, bool]:
"""Uses Social Auth's ability to refresh tokens if necessary.
Returns:
diff --git a/backend/connector_auth/pipeline/common.py b/backend/connector_auth/pipeline/common.py
index 96d8e365c..3d82cbbf5 100644
--- a/backend/connector_auth/pipeline/common.py
+++ b/backend/connector_auth/pipeline/common.py
@@ -13,9 +13,7 @@ from social_core.backends.oauth import BaseOAuth2
logger = logging.getLogger(__name__)
-def check_user_exists(
- backend: BaseOAuth2, user: User, **kwargs: Any
-) -> dict[str, str]:
+def check_user_exists(backend: BaseOAuth2, user: User, **kwargs: Any) -> dict[str, str]:
"""Checks if user is authenticated (will be handled in auth middleware,
present as a fail safe)
@@ -51,9 +49,7 @@ def cache_oauth_creds(
settings.SOCIAL_AUTH_FIELDS_STORED_IN_SESSION[0],
ConnectorAuthKey.OAUTH_KEY,
)
- extra_data = backend.extra_data(
- user, uid, response, details, *args, **kwargs
- )
+ extra_data = backend.extra_data(user, uid, response, details, *args, **kwargs)
extra_data[SocialAuthConstants.PROVIDER] = backend.name
extra_data[SocialAuthConstants.UID] = uid
diff --git a/backend/connector_auth/pipeline/google.py b/backend/connector_auth/pipeline/google.py
index 6c505a31b..71a56fb33 100644
--- a/backend/connector_auth/pipeline/google.py
+++ b/backend/connector_auth/pipeline/google.py
@@ -4,9 +4,7 @@ from connector_auth.constants import SocialAuthConstants as AuthConstants
from connector_auth.exceptions import EnrichConnectorMetadataException
from connector_processor.constants import ConnectorKeys
-from unstract.connectors.filesystems.google_drive.constants import (
- GDriveConstants,
-)
+from unstract.connectors.filesystems.google_drive.constants import GDriveConstants
class GoogleAuthHelper:
diff --git a/backend/connector_processor/connector_processor.py b/backend/connector_processor/connector_processor.py
index 3edd053eb..dcababce3 100644
--- a/backend/connector_processor/connector_processor.py
+++ b/backend/connector_processor/connector_processor.py
@@ -48,9 +48,7 @@ class ConnectorProcessor:
)
if len(updated_connectors) != 0:
connector = updated_connectors[0]
- schema_details[ConnectorKeys.OAUTH] = connector.get(
- ConnectorKeys.OAUTH
- )
+ schema_details[ConnectorKeys.OAUTH] = connector.get(ConnectorKeys.OAUTH)
schema_details[ConnectorKeys.SOCIAL_AUTH_URL] = connector.get(
ConnectorKeys.SOCIAL_AUTH_URL
)
@@ -95,9 +93,7 @@ class ConnectorProcessor:
ConnectorKeys.DESCRIPTION
),
ConnectorKeys.ICON: each_connector.get(ConnectorKeys.ICON),
- CIKey.CONNECTOR_MODE: each_connector.get(
- CIKey.CONNECTOR_MODE
- ).name,
+ CIKey.CONNECTOR_MODE: each_connector.get(CIKey.CONNECTOR_MODE).name,
}
)
@@ -140,13 +136,9 @@ class ConnectorProcessor:
"""Generic Function to get connector data with provided key."""
updated_connectors = fetch_connectors_by_key_value("id", connector_id)
if len(updated_connectors) == 0:
- logger.error(
- f"Invalid connector ID {connector_id} while invoking utility"
- )
+ logger.error(f"Invalid connector ID {connector_id} while invoking utility")
raise InValidConnectorId()
- return fetch_connectors_by_key_value("id", connector_id)[0].get(
- key_value
- )
+ return fetch_connectors_by_key_value("id", connector_id)[0].get(key_value)
@staticmethod
def validate_connector_mode(connector_mode: str) -> ConnectorMode:
diff --git a/backend/connector_processor/views.py b/backend/connector_processor/views.py
index ada158542..55367c40d 100644
--- a/backend/connector_processor/views.py
+++ b/backend/connector_processor/views.py
@@ -20,9 +20,7 @@ def get_connector_schema(request: HttpRequest) -> HttpResponse:
connector_name = request.GET.get(ConnectorKeys.ID)
if connector_name is None or connector_name == "":
raise IdIsMandatory()
- json_schema = ConnectorProcessor.get_json_schema(
- connector_id=connector_name
- )
+ json_schema = ConnectorProcessor.get_json_schema(connector_id=connector_name)
return Response(data=json_schema, status=status.HTTP_200_OK)
@@ -42,9 +40,7 @@ def get_supported_connectors(request: HttpRequest) -> HttpResponse:
connector_type = request.GET.get(ConnectorKeys.TYPE)
connector_mode = request.GET.get(ConnectorKeys.CONNECTOR_MODE)
if connector_mode:
- connector_mode = ConnectorProcessor.validate_connector_mode(
- connector_mode
- )
+ connector_mode = ConnectorProcessor.validate_connector_mode(connector_mode)
if (
connector_type == ConnectorKeys.INPUT
@@ -69,9 +65,7 @@ class ConnectorViewSet(GenericViewSet):
def test(self, request: Request) -> Response:
"""Tests the connector against the credentials passed."""
- serializer: TestConnectorSerializer = self.get_serializer(
- data=request.data
- )
+ serializer: TestConnectorSerializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
connector_id = serializer.validated_data.get(ConnectorKeys.CONNECTOR_ID)
cred_string = serializer.validated_data.get(CIKey.CONNECTOR_METADATA)
diff --git a/backend/feature_flag/urls.py b/backend/feature_flag/urls.py
index 0bc4c274d..c90426d1e 100644
--- a/backend/feature_flag/urls.py
+++ b/backend/feature_flag/urls.py
@@ -7,7 +7,5 @@ import feature_flag.views as views
from django.urls import path
urlpatterns = [
- path(
- "evaluate/", views.evaluate_feature_flag, name="evaluate_feature_flag"
- ),
+ path("evaluate/", views.evaluate_feature_flag, name="evaluate_feature_flag"),
]
diff --git a/backend/file_management/exceptions.py b/backend/file_management/exceptions.py
index b7bf42d22..b8cfb4700 100644
--- a/backend/file_management/exceptions.py
+++ b/backend/file_management/exceptions.py
@@ -69,9 +69,7 @@ class ValidationError(APIException):
status_code = 400
default_detail = "Validation Error"
- def __init__(
- self, detail: Optional[str] = None, code: Optional[int] = None
- ):
+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):
if detail is not None:
self.detail = detail
if code is not None:
@@ -83,9 +81,7 @@ class FileDeletionFailed(APIException):
status_code = 400
default_detail = "Unable to delete file."
- def __init__(
- self, detail: Optional[str] = None, code: Optional[int] = None
- ):
+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):
if detail is not None:
self.detail = detail
if code is not None:
diff --git a/backend/file_management/file_management_dto.py b/backend/file_management/file_management_dto.py
index 100efe1a4..a5cf585b7 100644
--- a/backend/file_management/file_management_dto.py
+++ b/backend/file_management/file_management_dto.py
@@ -21,17 +21,13 @@ class FileInformation:
self.type = file_info[FileInformationKey.FILE_TYPE]
modified_at = file_info.get(FileInformationKey.FILE_LAST_MODIFIED)
- self.modified_at = (
- self.parse_datetime(modified_at) if modified_at else None
- )
+ self.modified_at = self.parse_datetime(modified_at) if modified_at else None
self.content_type = file_content_type
self.size = file_info[FileInformationKey.FILE_SIZE]
@staticmethod
- def parse_datetime(
- dt_string: Optional[Union[str, datetime]]
- ) -> Optional[datetime]:
+ def parse_datetime(dt_string: Optional[Union[str, datetime]]) -> Optional[datetime]:
if isinstance(dt_string, str):
return datetime.strptime(dt_string, "%Y-%m-%dT%H:%M:%S.%f%z")
return dt_string
diff --git a/backend/file_management/file_management_helper.py b/backend/file_management/file_management_helper.py
index 2e904f942..0004176b2 100644
--- a/backend/file_management/file_management_helper.py
+++ b/backend/file_management/file_management_helper.py
@@ -25,9 +25,7 @@ from fsspec import AbstractFileSystem
from pydrive2.files import ApiRequestError
from unstract.connectors.filesystems import connectors as fs_connectors
-from unstract.connectors.filesystems.unstract_file_system import (
- UnstractFileSystem,
-)
+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
class FileManagerHelper:
@@ -38,9 +36,7 @@ class FileManagerHelper:
"""Creates the `UnstractFileSystem` for the corresponding connector."""
metadata = connector.metadata
if connector.connector_id in fs_connectors:
- connector = fs_connectors[connector.connector_id]["metadata"][
- "connector"
- ]
+ connector = fs_connectors[connector.connector_id]["metadata"]["connector"]
connector_class: UnstractFileSystem = connector(metadata)
return connector_class
else:
@@ -50,9 +46,7 @@ class FileManagerHelper:
raise ConnectorClassNotFound
@staticmethod
- def list_files(
- file_system: UnstractFileSystem, path: str
- ) -> list[FileInformation]:
+ def list_files(file_system: UnstractFileSystem, path: str) -> list[FileInformation]:
fs = file_system.get_fsspec_fs()
file_path = f"{path}"
# TODO: Add below logic by checking each connector?
@@ -73,9 +67,7 @@ class FileManagerHelper:
raise FileListError()
@staticmethod
- def get_files(
- fs: AbstractFileSystem, file_path: str
- ) -> list[FileInformation]:
+ def get_files(fs: AbstractFileSystem, file_path: str) -> list[FileInformation]:
"""Iterate through the directories and make a list of
FileInformation."""
if not file_path.endswith("/"):
@@ -89,10 +81,7 @@ class FileManagerHelper:
continue
# Call fs.info() to get file size if its missing
- if (
- file_info.get("type") == "file"
- and file_info.get("size") is None
- ):
+ if file_info.get("type") == "file" and file_info.get("size") is None:
file_info = fs.info(file_name)
file_content_type = file_info.get("ContentType")
@@ -123,12 +112,8 @@ class FileManagerHelper:
file.close()
file = fs.open(file_path, "rb")
- response = StreamingHttpResponse(
- file, content_type=file_content_type
- )
- response["Content-Disposition"] = (
- f"attachment; filename={base_name}"
- )
+ response = StreamingHttpResponse(file, content_type=file_content_type)
+ response["Content-Disposition"] = f"attachment; filename={base_name}"
return response
except ApiRequestError as exception:
FileManagerHelper.logger.error(
@@ -150,9 +135,7 @@ class FileManagerHelper:
if fs.path and (not path or path == "/"):
file_path = f"{fs.path}/"
- file_path = (
- file_path + "/" if not file_path.endswith("/") else file_path
- )
+ file_path = file_path + "/" if not file_path.endswith("/") else file_path
# adding filename with path
file_path += file_name
@@ -160,9 +143,7 @@ class FileManagerHelper:
remote_file.write(file.read())
@staticmethod
- def fetch_file_contents(
- file_system: UnstractFileSystem, file_path: str
- ) -> Any:
+ def fetch_file_contents(file_system: UnstractFileSystem, file_path: str) -> Any:
fs = file_system.get_fsspec_fs()
try:
file_info = fs.info(file_path)
@@ -201,9 +182,7 @@ class FileManagerHelper:
raise InvalidFileType
@staticmethod
- def delete_file(
- file_system: UnstractFileSystem, path: str, file_name: str
- ) -> bool:
+ def delete_file(file_system: UnstractFileSystem, path: str, file_name: str) -> bool:
fs = file_system.get_fsspec_fs()
file_path = f"{path}"
@@ -214,9 +193,7 @@ class FileManagerHelper:
if fs.path and (not path or path == "/"):
file_path = f"{fs.path}/"
- file_path = (
- file_path + "/" if not file_path.endswith("/") else file_path
- )
+ file_path = file_path + "/" if not file_path.endswith("/") else file_path
# adding filename with path
file_path += file_name
@@ -254,8 +231,6 @@ class FileManagerHelper:
os.makedirs(extract_file_path, exist_ok=True)
os.makedirs(summarize_file_path, exist_ok=True)
except OSError as e:
- FileManagerHelper.logger.error(
- f"Error while creating {file_path}: {e}"
- )
+ FileManagerHelper.logger.error(f"Error while creating {file_path}: {e}")
raise TenantDirCreationError
return str(file_path)
diff --git a/backend/file_management/views.py b/backend/file_management/views.py
index 879ead194..6cf34e456 100644
--- a/backend/file_management/views.py
+++ b/backend/file_management/views.py
@@ -24,9 +24,7 @@ from rest_framework.response import Response
from rest_framework.versioning import URLPathVersioning
from unstract.connectors.exceptions import ConnectorError
-from unstract.connectors.filesystems.local_storage.local_storage import (
- LocalStorageFS,
-)
+from unstract.connectors.filesystems.local_storage.local_storage import LocalStorageFS
logger = logging.getLogger(__name__)
@@ -57,9 +55,7 @@ class FileManagementViewSet(viewsets.ModelViewSet):
id: str = serializer.validated_data.get("connector_id")
path: str = serializer.validated_data.get("path")
try:
- connector_instance: ConnectorInstance = (
- ConnectorInstance.objects.get(pk=id)
- )
+ connector_instance: ConnectorInstance = ConnectorInstance.objects.get(pk=id)
file_system = FileManagerHelper.get_file_system(connector_instance)
files = FileManagerHelper.list_files(file_system, path)
serializer = FileInfoSerializer(files, many=True)
@@ -73,9 +69,7 @@ class FileManagementViewSet(viewsets.ModelViewSet):
)
raise ConnectorOAuthError()
except ConnectorError as error:
- logger.error(
- f"ConnectorError thrown during file list, error {error}"
- )
+ logger.error(f"ConnectorError thrown during file list, error {error}")
raise FileListError(core_err=error)
except Exception as error:
logger.error(f"Exception thrown from file list, error {error}")
@@ -87,9 +81,7 @@ class FileManagementViewSet(viewsets.ModelViewSet):
serializer.is_valid(raise_exception=True)
id: str = serializer.validated_data.get("connector_id")
path: str = serializer.validated_data.get("path")
- connector_instance: ConnectorInstance = ConnectorInstance.objects.get(
- pk=id
- )
+ connector_instance: ConnectorInstance = ConnectorInstance.objects.get(pk=id)
file_system = FileManagerHelper.get_file_system(connector_instance)
return FileManagerHelper.download_file(file_system, path)
@@ -101,21 +93,15 @@ class FileManagementViewSet(viewsets.ModelViewSet):
path: str = serializer.validated_data.get("path")
uploaded_files: Any = serializer.validated_data.get("file")
- connector_instance: ConnectorInstance = ConnectorInstance.objects.get(
- pk=id
- )
+ connector_instance: ConnectorInstance = ConnectorInstance.objects.get(pk=id)
file_system = FileManagerHelper.get_file_system(connector_instance)
for uploaded_file in uploaded_files:
file_name = uploaded_file.name
logger.info(
- f"Uploading file: {file_name}"
- if file_name
- else "Uploading file"
- )
- FileManagerHelper.upload_file(
- file_system, path, uploaded_file, file_name
+ f"Uploading file: {file_name}" if file_name else "Uploading file"
)
+ FileManagerHelper.upload_file(file_system, path, uploaded_file, file_name)
return Response({"message": "Files are uploaded successfully!"})
@action(detail=True, methods=["get"])
diff --git a/backend/middleware/exception.py b/backend/middleware/exception.py
index 6f4ba637f..7d4be00a6 100644
--- a/backend/middleware/exception.py
+++ b/backend/middleware/exception.py
@@ -82,14 +82,12 @@ class ExceptionLoggingMiddleware:
exception (Exception): Exception that was raised to be logged
"""
if hasattr(response, "status_code") and response.status_code >= 500:
- message = (
- "{method} {url} {status}\n\n{error}\n\n````{tb}````".format(
- method=request.method,
- url=request.build_absolute_uri(),
- status=response.status_code,
- error=repr(exception),
- tb=traceback.format_exc(),
- )
+ message = "{method} {url} {status}\n\n{error}\n\n````{tb}````".format(
+ method=request.method,
+ url=request.build_absolute_uri(),
+ status=response.status_code,
+ error=repr(exception),
+ tb=traceback.format_exc(),
)
else:
message = "{method} {url} {status} {error}".format(
diff --git a/backend/permissions/permission.py b/backend/permissions/permission.py
index 978d2e03b..bf2f85906 100644
--- a/backend/permissions/permission.py
+++ b/backend/permissions/permission.py
@@ -8,18 +8,14 @@ from rest_framework.views import APIView
class IsOwner(permissions.BasePermission):
"""Custom permission to only allow owners of an object."""
- def has_object_permission(
- self, request: Request, view: APIView, obj: Any
- ) -> bool:
+ def has_object_permission(self, request: Request, view: APIView, obj: Any) -> bool:
return True if obj.created_by == request.user else False
class IsOwnerOrSharedUser(permissions.BasePermission):
"""Custom permission to only allow owners and shared users of an object."""
- def has_object_permission(
- self, request: Request, view: APIView, obj: Any
- ) -> bool:
+ def has_object_permission(self, request: Request, view: APIView, obj: Any) -> bool:
return (
True
if (
diff --git a/backend/pipeline/manager.py b/backend/pipeline/manager.py
index 3c6bfbc8e..bee1b0c93 100644
--- a/backend/pipeline/manager.py
+++ b/backend/pipeline/manager.py
@@ -10,10 +10,7 @@ from pipeline.pipeline_processor import PipelineProcessor
from rest_framework.request import Request
from rest_framework.response import Response
from utils.request.constants import RequestConstants
-from workflow_manager.workflow.constants import (
- WorkflowExecutionKey,
- WorkflowKey,
-)
+from workflow_manager.workflow.constants import WorkflowExecutionKey, WorkflowKey
from workflow_manager.workflow.views import WorkflowViewSet
from backend.constants import RequestHeader
@@ -45,9 +42,7 @@ class PipelineManager:
f"with_log = {with_log}"
)
try:
- pipeline: Pipeline = PipelineProcessor.initialize_pipeline_sync(
- pipeline_id
- )
+ pipeline: Pipeline = PipelineProcessor.initialize_pipeline_sync(pipeline_id)
# TODO: Use DRF's request and as_view() instead
request.data[WorkflowKey.WF_ID] = pipeline.workflow.id
if execution_id is not None:
@@ -69,12 +64,8 @@ class PipelineManager:
) -> Optional[dict[str, Any]]:
"""Gets the required data to be passed while executing a pipeline Any
changes to pipeline execution needs to be propagated here."""
- callback_url = settings.DJANGO_APP_BACKEND_URL + reverse(
- PipelineURL.EXECUTE
- )
- job_headers = {
- RequestHeader.X_API_KEY: settings.INTERNAL_SERVICE_API_KEY
- }
+ callback_url = settings.DJANGO_APP_BACKEND_URL + reverse(PipelineURL.EXECUTE)
+ job_headers = {RequestHeader.X_API_KEY: settings.INTERNAL_SERVICE_API_KEY}
job_params = {WorkflowExecutionKey.WITH_LOG: "False"}
job_kwargs = {
RequestConstants.VERB: "POST",
diff --git a/backend/pipeline/migrations/0001_initial.py b/backend/pipeline/migrations/0001_initial.py
index 828bae01b..67c4d4ca7 100644
--- a/backend/pipeline/migrations/0001_initial.py
+++ b/backend/pipeline/migrations/0001_initial.py
@@ -42,9 +42,7 @@ class Migration(migrations.Migration):
("scheduled", models.BooleanField(default=False)),
(
"cron_string",
- models.TextField(
- db_comment="UNIX cron string", max_length=256
- ),
+ models.TextField(db_comment="UNIX cron string", max_length=256),
),
(
"pipeline_type",
diff --git a/backend/pipeline/serializers/crud.py b/backend/pipeline/serializers/crud.py
index 3a3fdf721..56e5d1142 100644
--- a/backend/pipeline/serializers/crud.py
+++ b/backend/pipeline/serializers/crud.py
@@ -37,9 +37,7 @@ class PipelineSerializer(AuditSerializer):
)
return pipeline
- def _get_name_and_icon(
- self, connectors: list[Any], connector_id: Any
- ) -> Any:
+ def _get_name_and_icon(self, connectors: list[Any], connector_id: Any) -> Any:
for obj in connectors:
if obj["id"] == connector_id:
return obj["name"], obj["icon"]
@@ -64,11 +62,9 @@ class PipelineSerializer(AuditSerializer):
repr[PC.DESTINATION_NAME] = PC.NOT_CONFIGURED
for instance in connector_instance_list:
if instance.connector_type == "INPUT":
- repr[PC.SOURCE_NAME], repr[PC.SOURCE_ICON] = (
- self._get_name_and_icon(
- connectors=connectors,
- connector_id=instance.connector_id,
- )
+ repr[PC.SOURCE_NAME], repr[PC.SOURCE_ICON] = self._get_name_and_icon(
+ connectors=connectors,
+ connector_id=instance.connector_id,
)
if instance.connector_type == "OUTPUT":
repr[PC.DESTINATION_NAME], repr[PC.DESTINATION_ICON] = (
diff --git a/backend/pipeline/views.py b/backend/pipeline/views.py
index 34c1fb05d..96cadd74f 100644
--- a/backend/pipeline/views.py
+++ b/backend/pipeline/views.py
@@ -7,28 +7,20 @@ from django.db.models import QuerySet
from drf_yasg import openapi
from drf_yasg.utils import swagger_auto_schema
from permissions.permission import IsOwner
-from pipeline.constants import (
- PipelineConstants,
- PipelineErrors,
- PipelineExecutionKey,
-)
+from pipeline.constants import PipelineConstants, PipelineErrors, PipelineExecutionKey
from pipeline.constants import PipelineKey as PK
from pipeline.manager import PipelineManager
from pipeline.models import Pipeline
from pipeline.pipeline_processor import PipelineProcessor
from pipeline.serializers.crud import PipelineSerializer
-from pipeline.serializers.execute import (
- PipelineExecuteSerializer as ExecuteSerializer,
-)
+from pipeline.serializers.execute import PipelineExecuteSerializer as ExecuteSerializer
from pipeline.serializers.update import PipelineUpdateSerializer
from rest_framework import serializers, status, viewsets
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.versioning import URLPathVersioning
from scheduler.helper import SchedulerHelper
-from workflow_manager.workflow.constants import (
- WorkflowExecutionKey as WFExecKey,
-)
+from workflow_manager.workflow.constants import WorkflowExecutionKey as WFExecKey
logger = logging.getLogger(__name__)
@@ -99,8 +91,7 @@ class PipelineViewSet(viewsets.ModelViewSet):
serializer.save()
except IntegrityError:
raise DuplicateData(
- f"{PipelineErrors.PIPELINE_EXISTS}, "
- f"{PipelineErrors.DUPLICATE_API}"
+ f"{PipelineErrors.PIPELINE_EXISTS}, " f"{PipelineErrors.DUPLICATE_API}"
)
return Response(data=serializer.data, status=status.HTTP_201_CREATED)
@@ -134,6 +125,4 @@ class PipelineViewSet(viewsets.ModelViewSet):
status=status.HTTP_200_OK,
)
else:
- return Response(
- serializer.errors, status=status.HTTP_400_BAD_REQUEST
- )
+ return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
diff --git a/backend/platform_settings/exceptions.py b/backend/platform_settings/exceptions.py
index 4e54b58ef..538b270fd 100644
--- a/backend/platform_settings/exceptions.py
+++ b/backend/platform_settings/exceptions.py
@@ -41,9 +41,7 @@ class DuplicateData(APIException):
status_code = 400
default_detail = "Duplicate Data"
- def __init__(
- self, detail: Optional[str] = None, code: Optional[int] = None
- ):
+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):
if detail is not None:
self.detail = detail
if code is not None:
diff --git a/backend/platform_settings/platform_auth_helper.py b/backend/platform_settings/platform_auth_helper.py
index 167db67da..c3d7d2d4b 100644
--- a/backend/platform_settings/platform_auth_helper.py
+++ b/backend/platform_settings/platform_auth_helper.py
@@ -32,9 +32,7 @@ class PlatformAuthHelper:
)
raise error
if not auth_controller.is_admin_by_role(member.role):
- logger.error(
- "User is not having right access to perform this operation."
- )
+ logger.error("User is not having right access to perform this operation.")
raise UserForbidden()
else:
pass
diff --git a/backend/platform_settings/platform_auth_service.py b/backend/platform_settings/platform_auth_service.py
index 7258b982f..054291ba8 100644
--- a/backend/platform_settings/platform_auth_service.py
+++ b/backend/platform_settings/platform_auth_service.py
@@ -73,9 +73,7 @@ class PlatformAuthenticationService:
logger.info(f"platform_key is generated for {organization.id}")
return result
except IntegrityError as error:
- logger.error(
- f"Integrity error - failed to generate platform key : {error}"
- )
+ logger.error(f"Integrity error - failed to generate platform key : {error}")
raise DuplicateData(
f"{ErrorMessage.KEY_EXIST}, \
{ErrorMessage.DUPLICATE_API}"
@@ -133,9 +131,7 @@ class PlatformAuthenticationService:
logger.info(f"platform_key {id} is updated by user {user.id}")
return result
except IntegrityError as error:
- logger.error(
- f"Integrity error - failed to refresh platform key : {error}"
- )
+ logger.error(f"Integrity error - failed to refresh platform key : {error}")
raise DuplicateData(
f"{ErrorMessage.KEY_EXIST}, \
{ErrorMessage.DUPLICATE_API}"
@@ -186,9 +182,7 @@ class PlatformAuthenticationService:
{ErrorMessage.DUPLICATE_API}"
)
except Exception as error:
- logger.error(
- f"Failed to activate/deactivate platform key : {error}"
- )
+ logger.error(f"Failed to activate/deactivate platform key : {error}")
raise InternalServiceError()
@staticmethod
diff --git a/backend/platform_settings/views.py b/backend/platform_settings/views.py
index ace584262..6b4c918fa 100644
--- a/backend/platform_settings/views.py
+++ b/backend/platform_settings/views.py
@@ -7,9 +7,7 @@ from account.models import Organization, PlatformKey
from django.db import connection
from platform_settings.constants import PlatformServiceConstants
from platform_settings.platform_auth_helper import PlatformAuthHelper
-from platform_settings.platform_auth_service import (
- PlatformAuthenticationService,
-)
+from platform_settings.platform_auth_service import PlatformAuthenticationService
from platform_settings.serializers import (
PlatformKeyGenerateSerializer,
PlatformKeyIDSerializer,
diff --git a/backend/plugins/authentication/auth_sample/auth_service.py b/backend/plugins/authentication/auth_sample/auth_service.py
index 3ec4e7888..499d11c5f 100644
--- a/backend/plugins/authentication/auth_sample/auth_service.py
+++ b/backend/plugins/authentication/auth_sample/auth_service.py
@@ -7,13 +7,7 @@ from rest_framework.request import Request
from rest_framework.response import Response
from .auth_helper import AuthHelper
-from .dto import (
- AuthOrganization,
- ResetUserPasswordDto,
- TokenData,
- User,
- UserInfo,
-)
+from .dto import AuthOrganization, ResetUserPasswordDto, TokenData, User, UserInfo
from .enums import Region
from .exceptions import MethodNotImplemented
@@ -73,9 +67,7 @@ class AuthService(ABC):
def get_user_id_from_token(self, token: dict[str, Any]) -> Response:
return token["userinfo"]["sub"]
- def get_organization_members_by_org_id(
- self, organization_id: str
- ) -> Response:
+ def get_organization_members_by_org_id(self, organization_id: str) -> Response:
raise MethodNotImplemented()
def reset_user_password(self, user: User) -> ResetUserPasswordDto:
diff --git a/backend/project/urls.py b/backend/project/urls.py
index 4af97d0d8..f015268e8 100644
--- a/backend/project/urls.py
+++ b/backend/project/urls.py
@@ -16,9 +16,7 @@ project_detail = ProjectViewSet.as_view(
project_settings = ProjectViewSet.as_view(
{"get": "project_settings", "put": "project_settings"}
)
-project_settings_schema = ProjectViewSet.as_view(
- {"get": "project_settings_schema"}
-)
+project_settings_schema = ProjectViewSet.as_view({"get": "project_settings_schema"})
urlpatterns = format_suffix_patterns(
[
diff --git a/backend/prompt/tests/test_urls.py b/backend/prompt/tests/test_urls.py
index 94469f5c2..0dffabd36 100644
--- a/backend/prompt/tests/test_urls.py
+++ b/backend/prompt/tests/test_urls.py
@@ -26,9 +26,7 @@ class TestPrompts(APITestCase):
def test_prompts_detail_throw_404(self):
"""Tests whether a 404 error is thrown on retrieving a prompt."""
- url = reverse(
- "prompts-detail", kwargs={"pk": 200}
- ) # Prompt doesn't exist
+ url = reverse("prompts-detail", kwargs={"pk": 200}) # Prompt doesn't exist
response = self.client.get(url)
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
diff --git a/backend/prompt_studio/permission.py b/backend/prompt_studio/permission.py
index ba036172a..e1b84e86f 100644
--- a/backend/prompt_studio/permission.py
+++ b/backend/prompt_studio/permission.py
@@ -8,9 +8,7 @@ from rest_framework.views import APIView
class PromptAcesssToUser(permissions.BasePermission):
"""Is the crud to Prompt/Notes allowed to user."""
- def has_object_permission(
- self, request: Request, view: APIView, obj: Any
- ) -> bool:
+ def has_object_permission(self, request: Request, view: APIView, obj: Any) -> bool:
return (
True
if (
diff --git a/backend/prompt_studio/processor_loader.py b/backend/prompt_studio/processor_loader.py
index 28d12f8d9..2a371a458 100644
--- a/backend/prompt_studio/processor_loader.py
+++ b/backend/prompt_studio/processor_loader.py
@@ -41,9 +41,7 @@ def load_plugins() -> list[Any]:
else:
continue
try:
- full_module_path = (
- f"{processor_package_path}.{processor_module_name}"
- )
+ full_module_path = f"{processor_package_path}.{processor_module_name}"
module = import_module(full_module_path)
metadata = getattr(module, ProcessorConfig.METADATA, {})
diff --git a/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py b/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py
index d7be28309..482ad5f80 100644
--- a/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py
+++ b/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py
@@ -14,9 +14,7 @@ def fill_with_default_x2text(apps, schema):
encryption_secret: str = settings.ENCRYPTION_KEY
f: Fernet = Fernet(encryption_secret.encode("utf-8"))
- metadata = {
- "url": "http://unstract-unstructured-io:8000/general/v0/general"
- }
+ metadata = {"url": "http://unstract-unstructured-io:8000/general/v0/general"}
json_string = json.dumps(metadata)
metadata_b = f.encrypt(json_string.encode("utf-8"))
@@ -27,9 +25,7 @@ def fill_with_default_x2text(apps, schema):
adapter_metadata_b=metadata_b,
)
adapter_instance.save()
- ProfileManager.objects.filter(x2text__isnull=True).update(
- x2text=adapter_instance
- )
+ ProfileManager.objects.filter(x2text__isnull=True).update(x2text=adapter_instance)
def reversal_x2text(*args):
@@ -38,9 +34,7 @@ def reversal_x2text(*args):
def disable_triggers(apps, schema_editor):
with connection.cursor() as cursor:
- cursor.execute(
- "ALTER TABLE adapter_adapterinstance DISABLE TRIGGER ALL;"
- )
+ cursor.execute("ALTER TABLE adapter_adapterinstance DISABLE TRIGGER ALL;")
class Migration(migrations.Migration):
diff --git a/backend/prompt_studio/prompt_profile_manager/migrations/0006_alter_profilemanager_x2text.py b/backend/prompt_studio/prompt_profile_manager/migrations/0006_alter_profilemanager_x2text.py
index b2088ef12..d36581058 100644
--- a/backend/prompt_studio/prompt_profile_manager/migrations/0006_alter_profilemanager_x2text.py
+++ b/backend/prompt_studio/prompt_profile_manager/migrations/0006_alter_profilemanager_x2text.py
@@ -6,9 +6,7 @@ from django.db import connection, migrations, models
def enable_triggers(apps, schema_editor):
with connection.cursor() as cursor:
- cursor.execute(
- "ALTER TABLE adapter_adapterinstance ENABLE TRIGGER ALL;"
- )
+ cursor.execute("ALTER TABLE adapter_adapterinstance ENABLE TRIGGER ALL;")
class Migration(migrations.Migration):
diff --git a/backend/prompt_studio/prompt_profile_manager/migrations/0008_profilemanager_migration.py b/backend/prompt_studio/prompt_profile_manager/migrations/0008_profilemanager_migration.py
index 33f3cca4e..e80273f38 100644
--- a/backend/prompt_studio/prompt_profile_manager/migrations/0008_profilemanager_migration.py
+++ b/backend/prompt_studio/prompt_profile_manager/migrations/0008_profilemanager_migration.py
@@ -23,9 +23,7 @@ class Migration(migrations.Migration):
def MigrateProfileManager(apps: Any, schema_editor: Any) -> None:
CustomTool = apps.get_model("prompt_studio_core", "CustomTool")
- ProfileManager = apps.get_model(
- "prompt_profile_manager", "ProfileManager"
- )
+ ProfileManager = apps.get_model("prompt_profile_manager", "ProfileManager")
ToolStudioPrompt = apps.get_model("prompt_studio", "ToolStudioPrompt")
# Iterate over prompt studio tools
@@ -55,9 +53,7 @@ class Migration(migrations.Migration):
# Replicate those profile manager
replicated_profile_manager = profile_manager
replicated_profile_manager.profile_id = None
- replicated_profile_manager.profile_name = (
- f"Profile_{index+1}"
- )
+ replicated_profile_manager.profile_name = f"Profile_{index+1}"
"""Attach the replicated profile manager to the
corresponding prompt tool."""
replicated_profile_manager.prompt_studio_tool = custom_tool
diff --git a/backend/prompt_studio/prompt_profile_manager/models.py b/backend/prompt_studio/prompt_profile_manager/models.py
index adfffe2f1..65bdbcffd 100644
--- a/backend/prompt_studio/prompt_profile_manager/models.py
+++ b/backend/prompt_studio/prompt_profile_manager/models.py
@@ -14,9 +14,7 @@ class ProfileManager(BaseModel):
class RetrievalStrategy(models.TextChoices):
SIMPLE = "simple", "Simple retrieval"
- profile_id = models.UUIDField(
- primary_key=True, default=uuid.uuid4, editable=False
- )
+ profile_id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
profile_name = models.TextField(blank=False)
vector_store = models.ForeignKey(
AdapterInstance,
diff --git a/backend/prompt_studio/prompt_profile_manager/serializers.py b/backend/prompt_studio/prompt_profile_manager/serializers.py
index 1f9ef62dd..4d4753561 100644
--- a/backend/prompt_studio/prompt_profile_manager/serializers.py
+++ b/backend/prompt_studio/prompt_profile_manager/serializers.py
@@ -22,8 +22,8 @@ class ProfileManagerSerializer(AuditSerializer):
vector_db = rep[ProfileManagerKeys.VECTOR_STORE]
x2text = rep[ProfileManagerKeys.X2TEXT]
if llm:
- rep[ProfileManagerKeys.LLM] = (
- AdapterProcessor.get_adapter_instance_by_id(llm)
+ rep[ProfileManagerKeys.LLM] = AdapterProcessor.get_adapter_instance_by_id(
+ llm
)
if embedding:
rep[ProfileManagerKeys.EMBEDDING_MODEL] = (
diff --git a/backend/prompt_studio/prompt_profile_manager/views.py b/backend/prompt_studio/prompt_profile_manager/views.py
index 16d66ef1b..e42ed100b 100644
--- a/backend/prompt_studio/prompt_profile_manager/views.py
+++ b/backend/prompt_studio/prompt_profile_manager/views.py
@@ -9,9 +9,7 @@ from prompt_studio.prompt_profile_manager.constants import (
ProfileManagerErrors,
ProfileManagerKeys,
)
-from prompt_studio.prompt_profile_manager.serializers import (
- ProfileManagerSerializer,
-)
+from prompt_studio.prompt_profile_manager.serializers import ProfileManagerSerializer
from rest_framework import status, viewsets
from rest_framework.response import Response
from rest_framework.versioning import URLPathVersioning
@@ -41,9 +39,7 @@ class ProfileManagerView(viewsets.ModelViewSet):
def create(
self, request: HttpRequest, *args: tuple[Any], **kwargs: dict[str, Any]
) -> Response:
- serializer: ProfileManagerSerializer = self.get_serializer(
- data=request.data
- )
+ serializer: ProfileManagerSerializer = self.get_serializer(data=request.data)
# Overriding default exception behaviour
# TO DO : Handle model related exceptions.
serializer.is_valid(raise_exception=True)
diff --git a/backend/prompt_studio/prompt_studio/migrations/0004_alter_toolstudioprompt_prompt.py b/backend/prompt_studio/prompt_studio/migrations/0004_alter_toolstudioprompt_prompt.py
index 72be66dd4..5fe3253fe 100644
--- a/backend/prompt_studio/prompt_studio/migrations/0004_alter_toolstudioprompt_prompt.py
+++ b/backend/prompt_studio/prompt_studio/migrations/0004_alter_toolstudioprompt_prompt.py
@@ -12,8 +12,6 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name="toolstudioprompt",
name="prompt",
- field=models.TextField(
- blank=True, db_comment="Field to store the prompt"
- ),
+ field=models.TextField(blank=True, db_comment="Field to store the prompt"),
),
]
diff --git a/backend/prompt_studio/prompt_studio/models.py b/backend/prompt_studio/prompt_studio/models.py
index d59f5f8f9..245277278 100644
--- a/backend/prompt_studio/prompt_studio/models.py
+++ b/backend/prompt_studio/prompt_studio/models.py
@@ -28,9 +28,7 @@ class ToolStudioPrompt(BaseModel):
class Mode(models.TextChoices):
DEFAULT = "Default", "Default choice for output"
- prompt_id = models.UUIDField(
- primary_key=True, default=uuid.uuid4, editable=False
- )
+ prompt_id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
prompt_key = models.TextField(
blank=False,
db_comment="Field to store the prompt key",
diff --git a/backend/prompt_studio/prompt_studio_core/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_core/migrations/0001_initial.py
index a22e616ce..14ae4c61a 100644
--- a/backend/prompt_studio/prompt_studio_core/migrations/0001_initial.py
+++ b/backend/prompt_studio/prompt_studio_core/migrations/0001_initial.py
@@ -65,9 +65,7 @@ class Migration(migrations.Migration):
),
(
"preamble",
- models.TextField(
- blank=True, db_comment="Preamble to the prompts"
- ),
+ models.TextField(blank=True, db_comment="Preamble to the prompts"),
),
(
"postamble",
diff --git a/backend/prompt_studio/prompt_studio_core/models.py b/backend/prompt_studio/prompt_studio_core/models.py
index cac0a0607..e9bbff42d 100644
--- a/backend/prompt_studio/prompt_studio_core/models.py
+++ b/backend/prompt_studio/prompt_studio_core/models.py
@@ -24,9 +24,7 @@ class CustomToolModelManager(models.Manager):
class CustomTool(BaseModel):
"""Model to store the custom tools designed in the tool studio."""
- tool_id = models.UUIDField(
- primary_key=True, default=uuid.uuid4, editable=False
- )
+ tool_id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
tool_name = models.TextField(unique=True, blank=False, null=False)
description = models.TextField(blank=False, null=False)
author = models.TextField(
@@ -119,8 +117,6 @@ class CustomTool(BaseModel):
# Introduced field to establish M2M relation between users and custom_tool.
# This will introduce intermediary table which relates both the models.
- shared_users = models.ManyToManyField(
- User, related_name="shared_custom_tool"
- )
+ shared_users = models.ManyToManyField(User, related_name="shared_custom_tool")
objects = CustomToolModelManager()
diff --git a/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.py b/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.py
index 111a891d7..0c460a048 100644
--- a/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.py
+++ b/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.py
@@ -1,16 +1,12 @@
import os
-from platform_settings.platform_auth_service import (
- PlatformAuthenticationService,
-)
+from platform_settings.platform_auth_service import PlatformAuthenticationService
from prompt_studio.prompt_studio_core.constants import LogLevel, ToolStudioKeys
from unstract.sdk.tool.stream import StreamMixin
class PromptIdeBaseTool(StreamMixin):
- def __init__(
- self, log_level: LogLevel = LogLevel.INFO, org_id: str = ""
- ) -> None:
+ def __init__(self, log_level: LogLevel = LogLevel.INFO, org_id: str = "") -> None:
"""
Args:
tool (UnstractAbstractTool): Instance of UnstractAbstractTool
@@ -34,17 +30,13 @@ class PromptIdeBaseTool(StreamMixin):
"""
# HACK: Adding platform key for multitenancy
if env_key == ToolStudioKeys.PLATFORM_SERVICE_API_KEY:
- platform_key = (
- PlatformAuthenticationService.get_active_platform_key(
- self.org_id
- )
+ platform_key = PlatformAuthenticationService.get_active_platform_key(
+ self.org_id
)
key: str = str(platform_key.key)
return key
else:
env_value = os.environ.get(env_key)
if env_value is None or env_value == "":
- self.stream_error_and_exit(
- f"Env variable {env_key} is required"
- )
+ self.stream_error_and_exit(f"Env variable {env_key} is required")
return env_value # type:ignore
diff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
index 0f137abec..154600369 100644
--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py
@@ -11,9 +11,7 @@ from file_management.file_management_helper import FileManagerHelper
from prompt_studio.prompt_profile_manager.models import ProfileManager
from prompt_studio.prompt_studio.models import ToolStudioPrompt
from prompt_studio.prompt_studio_core.constants import LogLevels
-from prompt_studio.prompt_studio_core.constants import (
- ToolStudioPromptKeys as TSPKeys,
-)
+from prompt_studio.prompt_studio_core.constants import ToolStudioPromptKeys as TSPKeys
from prompt_studio.prompt_studio_core.exceptions import (
AnswerFetchError,
DefaultProfileError,
@@ -25,9 +23,7 @@ from prompt_studio.prompt_studio_core.exceptions import (
ToolNotValid,
)
from prompt_studio.prompt_studio_core.models import CustomTool
-from prompt_studio.prompt_studio_core.prompt_ide_base_tool import (
- PromptIdeBaseTool,
-)
+from prompt_studio.prompt_studio_core.prompt_ide_base_tool import PromptIdeBaseTool
from prompt_studio.prompt_studio_index_manager.prompt_studio_index_helper import ( # noqa: E501
PromptStudioIndexHelper,
)
@@ -185,8 +181,8 @@ class PromptStudioHelper:
Returns:
List[ToolStudioPrompt]: List of instance of the model
"""
- prompt_instances: list[ToolStudioPrompt] = (
- ToolStudioPrompt.objects.filter(tool_id=tool_id)
+ prompt_instances: list[ToolStudioPrompt] = ToolStudioPrompt.objects.filter(
+ tool_id=tool_id
)
return prompt_instances
@@ -243,9 +239,7 @@ class PromptStudioHelper:
)
# Need to check the user who created profile manager
# has access to adapters configured in profile manager
- PromptStudioHelper.validate_profile_manager_owner_access(
- default_profile
- )
+ PromptStudioHelper.validate_profile_manager_owner_access(default_profile)
doc_id = PromptStudioHelper.dynamic_indexer(
profile_manager=default_profile,
@@ -326,9 +320,7 @@ class PromptStudioHelper:
os.path.splitext(filename)[0] + ".txt",
)
- logger.info(
- f"[{tool.tool_id}] Invoking prompt service for prompt {id}"
- )
+ logger.info(f"[{tool.tool_id}] Invoking prompt service for prompt {id}")
PromptStudioHelper._publish_log(
{"tool_id": tool_id, "prompt_id": id},
LogLevels.DEBUG,
@@ -382,14 +374,10 @@ class PromptStudioHelper:
else:
prompts = PromptStudioHelper.fetch_prompt_from_tool(tool_id)
prompts = [
- prompt
- for prompt in prompts
- if prompt.prompt_type != TSPKeys.NOTES
+ prompt for prompt in prompts if prompt.prompt_type != TSPKeys.NOTES
]
if not prompts:
- logger.error(
- f"[{tool_id or 'NA'}] No prompts found for id: {id}"
- )
+ logger.error(f"[{tool_id or 'NA'}] No prompts found for id: {id}")
raise NoPromptsFound()
logger.info(f"[{tool_id}] Executing prompts in single pass")
@@ -444,9 +432,7 @@ class PromptStudioHelper:
)
raise AnswerFetchError()
- logger.info(
- f"[{tool.tool_id}] Single pass response fetched successfully"
- )
+ logger.info(f"[{tool.tool_id}] Single pass response fetched successfully")
PromptStudioHelper._publish_log(
{"tool_id": tool_id, "prompt_id": str(id)},
LogLevels.INFO,
@@ -530,9 +516,7 @@ class PromptStudioHelper:
)
output: dict[str, Any] = {}
- output[TSPKeys.ASSERTION_FAILURE_PROMPT] = (
- prompt.assertion_failure_prompt
- )
+ output[TSPKeys.ASSERTION_FAILURE_PROMPT] = prompt.assertion_failure_prompt
output[TSPKeys.ASSERT_PROMPT] = prompt.assert_prompt
output[TSPKeys.IS_ASSERT] = prompt.is_assert
output[TSPKeys.PROMPT] = prompt.prompt
@@ -550,9 +534,7 @@ class PromptStudioHelper:
output[TSPKeys.RETRIEVAL_STRATEGY] = (
prompt.profile_manager.retrieval_strategy
)
- output[TSPKeys.SIMILARITY_TOP_K] = (
- prompt.profile_manager.similarity_top_k
- )
+ output[TSPKeys.SIMILARITY_TOP_K] = prompt.profile_manager.similarity_top_k
output[TSPKeys.SECTION] = prompt.profile_manager.section
output[TSPKeys.X2TEXT_ADAPTER] = x2text
# Eval settings for the prompt
@@ -709,9 +691,7 @@ class PromptStudioHelper:
challenge_llm = str(default_profile.llm.id)
# Need to check the user who created profile manager
# has access to adapters configured in profile manager
- PromptStudioHelper.validate_profile_manager_owner_access(
- default_profile
- )
+ PromptStudioHelper.validate_profile_manager_owner_access(default_profile)
default_profile.chunk_size = 0 # To retrive full context
if prompt_grammar:
@@ -759,9 +739,7 @@ class PromptStudioHelper:
if tool.summarize_as_source:
path = Path(file_path)
- file_path = str(
- path.parent / TSPKeys.SUMMARIZE / (path.stem + ".txt")
- )
+ file_path = str(path.parent / TSPKeys.SUMMARIZE / (path.stem + ".txt"))
file_hash = ToolUtils.get_hash_from_file(file_path=file_path)
payload = {
diff --git a/backend/prompt_studio/prompt_studio_core/urls.py b/backend/prompt_studio/prompt_studio_core/urls.py
index becc267c0..3a32f63c4 100644
--- a/backend/prompt_studio/prompt_studio_core/urls.py
+++ b/backend/prompt_studio/prompt_studio_core/urls.py
@@ -3,9 +3,7 @@ from rest_framework.urlpatterns import format_suffix_patterns
from .views import PromptStudioCoreView
-prompt_studio_list = PromptStudioCoreView.as_view(
- {"get": "list", "post": "create"}
-)
+prompt_studio_list = PromptStudioCoreView.as_view({"get": "list", "post": "create"})
prompt_studio_detail = PromptStudioCoreView.as_view(
{
"get": "retrieve",
@@ -14,9 +12,7 @@ prompt_studio_detail = PromptStudioCoreView.as_view(
"delete": "destroy",
}
)
-prompt_studio_choices = PromptStudioCoreView.as_view(
- {"get": "get_select_choices"}
-)
+prompt_studio_choices = PromptStudioCoreView.as_view({"get": "get_select_choices"})
prompt_studio_profiles = PromptStudioCoreView.as_view(
{"get": "list_profiles", "patch": "make_profile_default"}
)
@@ -27,21 +23,15 @@ prompt_studio_profilemanager = PromptStudioCoreView.as_view(
{"post": "create_profile_manager"}
)
-prompt_studio_prompt_index = PromptStudioCoreView.as_view(
- {"post": "index_document"}
-)
-prompt_studio_prompt_response = PromptStudioCoreView.as_view(
- {"post": "fetch_response"}
-)
+prompt_studio_prompt_index = PromptStudioCoreView.as_view({"post": "index_document"})
+prompt_studio_prompt_response = PromptStudioCoreView.as_view({"post": "fetch_response"})
prompt_studio_adapter_choices = PromptStudioCoreView.as_view(
{"get": "get_adapter_choices"}
)
prompt_studio_single_pass_extraction = PromptStudioCoreView.as_view(
{"post": "single_pass_extraction"}
)
-prompt_studio_users = PromptStudioCoreView.as_view(
- {"get": "list_of_shared_users"}
-)
+prompt_studio_users = PromptStudioCoreView.as_view({"get": "list_of_shared_users"})
prompt_studio_file = PromptStudioCoreView.as_view(
diff --git a/backend/prompt_studio/prompt_studio_core/views.py b/backend/prompt_studio/prompt_studio_core/views.py
index 0ce3619be..412fdb3fa 100644
--- a/backend/prompt_studio/prompt_studio_core/views.py
+++ b/backend/prompt_studio/prompt_studio_core/views.py
@@ -11,9 +11,7 @@ from permissions.permission import IsOwner, IsOwnerOrSharedUser
from prompt_studio.processor_loader import ProcessorConfig, load_plugins
from prompt_studio.prompt_profile_manager.constants import ProfileManagerErrors
from prompt_studio.prompt_profile_manager.models import ProfileManager
-from prompt_studio.prompt_profile_manager.serializers import (
- ProfileManagerSerializer,
-)
+from prompt_studio.prompt_profile_manager.serializers import ProfileManagerSerializer
from prompt_studio.prompt_studio.constants import ToolStudioPromptErrors
from prompt_studio.prompt_studio.exceptions import FilenameMissingError
from prompt_studio.prompt_studio.serializers import ToolStudioPromptSerializer
@@ -27,9 +25,7 @@ from prompt_studio.prompt_studio_core.exceptions import (
IndexingAPIError,
ToolDeleteError,
)
-from prompt_studio.prompt_studio_core.prompt_studio_helper import (
- PromptStudioHelper,
-)
+from prompt_studio.prompt_studio_core.prompt_studio_helper import PromptStudioHelper
from prompt_studio.prompt_studio_document_manager.models import DocumentManager
from prompt_studio.prompt_studio_document_manager.prompt_studio_document_helper import ( # noqa: E501
PromptStudioDocumentHelper,
@@ -48,9 +44,7 @@ from rest_framework.response import Response
from rest_framework.versioning import URLPathVersioning
from tool_instance.models import ToolInstance
-from unstract.connectors.filesystems.local_storage.local_storage import (
- LocalStorageFS,
-)
+from unstract.connectors.filesystems.local_storage.local_storage import LocalStorageFS
from .models import CustomTool
from .serializers import (
@@ -126,12 +120,10 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
if summarize_llm_profile_id:
prompt_tool = self.get_object()
- ProfileManager.objects.filter(
- prompt_studio_tool=prompt_tool
- ).update(is_summarize_llm=False)
- profile_manager = ProfileManager.objects.get(
- pk=summarize_llm_profile_id
+ ProfileManager.objects.filter(prompt_studio_tool=prompt_tool).update(
+ is_summarize_llm=False
)
+ profile_manager = ProfileManager.objects.get(pk=summarize_llm_profile_id)
profile_manager.is_summarize_llm = True
profile_manager.save()
@@ -147,9 +139,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
Response: Reponse of dropdown dict
"""
try:
- select_choices: dict[str, Any] = (
- PromptStudioHelper.get_select_fields()
- )
+ select_choices: dict[str, Any] = PromptStudioHelper.get_select_fields()
return Response(select_choices, status=status.HTTP_200_OK)
except Exception as e:
logger.error(f"Error occured while fetching select fields {e}")
@@ -172,9 +162,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
return Response(serialized_instances)
@action(detail=True, methods=["patch"])
- def make_profile_default(
- self, request: HttpRequest, pk: Any = None
- ) -> Response:
+ def make_profile_default(self, request: HttpRequest, pk: Any = None) -> Response:
prompt_tool = (
self.get_object()
) # Assuming you have a get_object method in your viewset
@@ -183,9 +171,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
is_default=False
)
- profile_manager = ProfileManager.objects.get(
- pk=request.data["default_profile"]
- )
+ profile_manager = ProfileManager.objects.get(pk=request.data["default_profile"])
profile_manager.is_default = True
profile_manager.save()
@@ -242,9 +228,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
status=status.HTTP_200_OK,
)
else:
- logger.error(
- "Error occured while indexing. Unique ID is not valid."
- )
+ logger.error("Error occured while indexing. Unique ID is not valid.")
raise IndexingAPIError()
@action(detail=True, methods=["post"])
@@ -281,9 +265,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
return Response(response, status=status.HTTP_200_OK)
@action(detail=True, methods=["post"])
- def single_pass_extraction(
- self, request: HttpRequest, pk: uuid
- ) -> Response:
+ def single_pass_extraction(self, request: HttpRequest, pk: uuid) -> Response:
"""API Entry point method to fetch response to prompt.
Args:
@@ -317,9 +299,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
return Response(response, status=status.HTTP_200_OK)
@action(detail=True, methods=["get"])
- def list_of_shared_users(
- self, request: HttpRequest, pk: Any = None
- ) -> Response:
+ def list_of_shared_users(self, request: HttpRequest, pk: Any = None) -> Response:
custom_tool = (
self.get_object()
@@ -332,9 +312,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
@action(detail=True, methods=["post"])
def create_prompt(self, request: HttpRequest, pk: Any = None) -> Response:
context = super().get_serializer_context()
- serializer = ToolStudioPromptSerializer(
- data=request.data, context=context
- )
+ serializer = ToolStudioPromptSerializer(data=request.data, context=context)
serializer.is_valid(raise_exception=True)
try:
# serializer.save()
@@ -347,13 +325,9 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
return Response(serializer.data, status=status.HTTP_201_CREATED)
@action(detail=True, methods=["post"])
- def create_profile_manager(
- self, request: HttpRequest, pk: Any = None
- ) -> Response:
+ def create_profile_manager(self, request: HttpRequest, pk: Any = None) -> Response:
context = super().get_serializer_context()
- serializer = ProfileManagerSerializer(
- data=request.data, context=context
- )
+ serializer = ProfileManagerSerializer(data=request.data, context=context)
serializer.is_valid(raise_exception=True)
try:
@@ -366,9 +340,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
return Response(serializer.data, status=status.HTTP_201_CREATED)
@action(detail=True, methods=["get"])
- def fetch_contents_ide(
- self, request: HttpRequest, pk: Any = None
- ) -> Response:
+ def fetch_contents_ide(self, request: HttpRequest, pk: Any = None) -> Response:
custom_tool = self.get_object()
serializer = FileInfoIdeSerializer(data=request.GET)
serializer.is_valid(raise_exception=True)
@@ -380,8 +352,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
filename_without_extension = file_name.rsplit(".", 1)[0]
if view_type == FileViewTypes.EXTRACT:
file_name = (
- f"{FileViewTypes.EXTRACT.lower()}/"
- f"{filename_without_extension}.txt"
+ f"{FileViewTypes.EXTRACT.lower()}/" f"{filename_without_extension}.txt"
)
if view_type == FileViewTypes.SUMMARIZE:
file_name = (
@@ -389,13 +360,11 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
f"{filename_without_extension}.txt"
)
- file_path = file_path = (
- FileManagerHelper.handle_sub_directory_for_tenants(
- request.org_id,
- is_create=True,
- user_id=custom_tool.created_by.user_id,
- tool_id=str(custom_tool.tool_id),
- )
+ file_path = file_path = FileManagerHelper.handle_sub_directory_for_tenants(
+ request.org_id,
+ is_create=True,
+ user_id=custom_tool.created_by.user_id,
+ tool_id=str(custom_tool.tool_id),
)
file_system = LocalStorageFS(settings={"path": file_path})
if not file_path.endswith("/"):
@@ -435,9 +404,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
}
# Store file
logger.info(
- f"Uploading file: {file_name}"
- if file_name
- else "Uploading file"
+ f"Uploading file: {file_name}" if file_name else "Uploading file"
)
FileManagerHelper.upload_file(
file_system,
@@ -489,9 +456,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):
custom_tool = self.get_object()
serializer = ExportToolRequestSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
- is_shared_with_org: bool = serializer.validated_data.get(
- "is_shared_with_org"
- )
+ is_shared_with_org: bool = serializer.validated_data.get("is_shared_with_org")
user_ids = set(serializer.validated_data.get("user_id"))
PromptStudioRegistryHelper.update_or_create_psr_tool(
diff --git a/backend/prompt_studio/prompt_studio_document_manager/models.py b/backend/prompt_studio/prompt_studio_document_manager/models.py
index f45ed3830..256d60e45 100644
--- a/backend/prompt_studio/prompt_studio_document_manager/models.py
+++ b/backend/prompt_studio/prompt_studio_document_manager/models.py
@@ -9,9 +9,7 @@ from utils.models.base_model import BaseModel
class DocumentManager(BaseModel):
"""Model to store the document details."""
- document_id = models.UUIDField(
- primary_key=True, default=uuid.uuid4, editable=False
- )
+ document_id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
document_name = models.CharField(
db_comment="Field to store the document name",
diff --git a/backend/prompt_studio/prompt_studio_index_manager/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_index_manager/migrations/0001_initial.py
index 48841d074..e2a5bb3d6 100644
--- a/backend/prompt_studio/prompt_studio_index_manager/migrations/0001_initial.py
+++ b/backend/prompt_studio/prompt_studio_index_manager/migrations/0001_initial.py
@@ -51,9 +51,7 @@ class Migration(migrations.Migration):
),
(
"index_ids_history",
- models.JSONField(
- db_comment="List of index ids", default=list
- ),
+ models.JSONField(db_comment="List of index ids", default=list),
),
(
"created_by",
diff --git a/backend/prompt_studio/prompt_studio_index_manager/urls.py b/backend/prompt_studio/prompt_studio_index_manager/urls.py
index cc7d45448..b082bc1d8 100644
--- a/backend/prompt_studio/prompt_studio_index_manager/urls.py
+++ b/backend/prompt_studio/prompt_studio_index_manager/urls.py
@@ -3,9 +3,7 @@ from rest_framework.urlpatterns import format_suffix_patterns
from .views import IndexManagerView
-prompt_studio_index_list = IndexManagerView.as_view(
- {"get": "list", "post": "create"}
-)
+prompt_studio_index_list = IndexManagerView.as_view({"get": "list", "post": "create"})
prompt_studio_index_detail = IndexManagerView.as_view(
{
diff --git a/backend/prompt_studio/prompt_studio_index_manager/views.py b/backend/prompt_studio/prompt_studio_index_manager/views.py
index a07e6f6c8..414db28cd 100644
--- a/backend/prompt_studio/prompt_studio_index_manager/views.py
+++ b/backend/prompt_studio/prompt_studio_index_manager/views.py
@@ -2,9 +2,7 @@ from typing import Optional
from django.db.models import QuerySet
from prompt_studio.prompt_studio_index_manager.constants import IndexManagerKeys
-from prompt_studio.prompt_studio_index_manager.serializers import (
- IndexManagerSerializer,
-)
+from prompt_studio.prompt_studio_index_manager.serializers import IndexManagerSerializer
from rest_framework import viewsets
from rest_framework.versioning import URLPathVersioning
from utils.filtering import FilterHelper
diff --git a/backend/prompt_studio/prompt_studio_output_manager/migrations/0004_alter_promptstudiooutputmanager_doc_name.py b/backend/prompt_studio/prompt_studio_output_manager/migrations/0004_alter_promptstudiooutputmanager_doc_name.py
index 76f6081b8..b51b80233 100644
--- a/backend/prompt_studio/prompt_studio_output_manager/migrations/0004_alter_promptstudiooutputmanager_doc_name.py
+++ b/backend/prompt_studio/prompt_studio_output_manager/migrations/0004_alter_promptstudiooutputmanager_doc_name.py
@@ -15,8 +15,6 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name="promptstudiooutputmanager",
name="doc_name",
- field=models.CharField(
- db_comment="Field to store the document name"
- ),
+ field=models.CharField(db_comment="Field to store the document name"),
),
]
diff --git a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
index 24e1f022e..4efbbcb27 100644
--- a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
+++ b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py
@@ -7,9 +7,7 @@ from prompt_studio.prompt_studio_document_manager.models import DocumentManager
from prompt_studio.prompt_studio_output_manager.constants import (
PromptStudioOutputManagerKeys as PSOMKeys,
)
-from prompt_studio.prompt_studio_output_manager.models import (
- PromptStudioOutputManager,
-)
+from prompt_studio.prompt_studio_output_manager.models import PromptStudioOutputManager
class OutputManagerHelper:
diff --git a/backend/prompt_studio/prompt_studio_output_manager/views.py b/backend/prompt_studio/prompt_studio_output_manager/views.py
index affcbd0d4..3be3c5888 100644
--- a/backend/prompt_studio/prompt_studio_output_manager/views.py
+++ b/backend/prompt_studio/prompt_studio_output_manager/views.py
@@ -39,9 +39,7 @@ class PromptStudioOutputView(viewsets.ModelViewSet):
)
# Convert the string representation to a boolean value
- is_single_pass_extract = CommonUtils.str_to_bool(
- is_single_pass_extract_param
- )
+ is_single_pass_extract = CommonUtils.str_to_bool(is_single_pass_extract_param)
filter_args[PromptStudioOutputManagerKeys.IS_SINGLE_PASS_EXTRACT] = (
is_single_pass_extract
diff --git a/backend/prompt_studio/prompt_studio_registry/constants.py b/backend/prompt_studio/prompt_studio_registry/constants.py
index 745c59975..4c8713636 100644
--- a/backend/prompt_studio/prompt_studio_registry/constants.py
+++ b/backend/prompt_studio/prompt_studio_registry/constants.py
@@ -14,9 +14,7 @@ class PromptStudioRegistryKeys:
class PromptStudioRegistryErrors:
SERIALIZATION_FAILED = "Data Serialization Failed."
DUPLICATE_API = "It appears that a duplicate call may have been made."
- CUSTOM_TOOL_EXISTS = (
- "Custom tool with similiar configuration already exists"
- )
+ CUSTOM_TOOL_EXISTS = "Custom tool with similiar configuration already exists"
class LogLevels:
diff --git a/backend/prompt_studio/prompt_studio_registry/models.py b/backend/prompt_studio/prompt_studio_registry/models.py
index bfff30d36..2ce06af3e 100644
--- a/backend/prompt_studio/prompt_studio_registry/models.py
+++ b/backend/prompt_studio/prompt_studio_registry/models.py
@@ -7,11 +7,7 @@ from django.db.models import QuerySet
from prompt_studio.prompt_studio.models import CustomTool
from utils.models.base_model import BaseModel
-from .fields import (
- ToolMetadataJSONField,
- ToolPropertyJSONField,
- ToolSpecJSONField,
-)
+from .fields import ToolMetadataJSONField, ToolPropertyJSONField, ToolSpecJSONField
class PromptStudioRegistryModelManager(models.Manager):
@@ -59,9 +55,7 @@ class PromptStudioRegistry(BaseModel):
blank=False,
default=dict,
)
- icon = models.CharField(
- db_comment="Tool icon in svg format", editable=False
- )
+ icon = models.CharField(db_comment="Tool icon in svg format", editable=False)
url = models.CharField(editable=False)
custom_tool = models.OneToOneField(
CustomTool,
@@ -92,8 +86,6 @@ class PromptStudioRegistry(BaseModel):
)
# Introduced field to establish M2M relation between users and tools.
# This will introduce intermediary table which relates both the models.
- shared_users = models.ManyToManyField(
- User, related_name="shared_exported_tools"
- )
+ shared_users = models.ManyToManyField(User, related_name="shared_exported_tools")
objects = PromptStudioRegistryModelManager()
diff --git a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py
index ee0b3b028..bb6a85c4a 100644
--- a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py
+++ b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py
@@ -7,9 +7,7 @@ from django.db import IntegrityError
from prompt_studio.prompt_profile_manager.models import ProfileManager
from prompt_studio.prompt_studio.models import ToolStudioPrompt
from prompt_studio.prompt_studio_core.models import CustomTool
-from prompt_studio.prompt_studio_core.prompt_studio_helper import (
- PromptStudioHelper,
-)
+from prompt_studio.prompt_studio_core.prompt_studio_helper import PromptStudioHelper
from unstract.tool_registry.dto import Properties, Spec, Tool
from .constants import JsonSchemaKey
@@ -111,14 +109,12 @@ class PromptStudioRegistryHelper:
obj: PromptStudioRegistry instance that was updated or created
"""
try:
- properties: Properties = (
- PromptStudioRegistryHelper.frame_properties(tool=custom_tool)
+ properties: Properties = PromptStudioRegistryHelper.frame_properties(
+ tool=custom_tool
)
spec: Spec = PromptStudioRegistryHelper.frame_spec(tool=custom_tool)
- prompts: list[ToolStudioPrompt] = (
- PromptStudioHelper.fetch_prompt_from_tool(
- tool_id=custom_tool.tool_id
- )
+ prompts: list[ToolStudioPrompt] = PromptStudioHelper.fetch_prompt_from_tool(
+ tool_id=custom_tool.tool_id
)
metadata = PromptStudioRegistryHelper.frame_export_json(
tool=custom_tool, prompts=prompts
@@ -210,9 +206,7 @@ class PromptStudioRegistryHelper:
output[JsonSchemaKey.VECTOR_DB] = vector_db
output[JsonSchemaKey.EMBEDDING] = embedding_model
output[JsonSchemaKey.X2TEXT_ADAPTER] = x2text
- output[JsonSchemaKey.CHUNK_OVERLAP] = (
- prompt.profile_manager.chunk_overlap
- )
+ output[JsonSchemaKey.CHUNK_OVERLAP] = prompt.profile_manager.chunk_overlap
output[JsonSchemaKey.LLM] = llm
output[JsonSchemaKey.PREAMBLE] = tool.preamble
output[JsonSchemaKey.POSTAMBLE] = tool.postamble
@@ -248,9 +242,7 @@ class PromptStudioRegistryHelper:
instance=prompt_studio_tools, many=True
)
except Exception as error:
- logger.error(
- f"Error occured while fetching tool for tool_id: {error}"
- )
+ logger.error(f"Error occured while fetching tool for tool_id: {error}")
raise InternalError()
tool_metadata: dict[str, Any] = {}
tool_list = []
diff --git a/backend/prompt_studio/prompt_studio_registry/serializers.py b/backend/prompt_studio/prompt_studio_registry/serializers.py
index dd1d6ea20..d0ab1bdb8 100644
--- a/backend/prompt_studio/prompt_studio_registry/serializers.py
+++ b/backend/prompt_studio/prompt_studio_registry/serializers.py
@@ -22,6 +22,4 @@ class PromptStudioRegistryInfoSerializer(AuditSerializer):
class ExportToolRequestSerializer(serializers.Serializer):
is_shared_with_org = serializers.BooleanField(default=False)
- user_id = serializers.ListField(
- child=serializers.IntegerField(), required=False
- )
+ user_id = serializers.ListField(child=serializers.IntegerField(), required=False)
diff --git a/backend/prompt_studio/prompt_studio_registry/views.py b/backend/prompt_studio/prompt_studio_registry/views.py
index 5b0bd74cc..d050088a2 100644
--- a/backend/prompt_studio/prompt_studio_registry/views.py
+++ b/backend/prompt_studio/prompt_studio_registry/views.py
@@ -2,9 +2,7 @@ import logging
from typing import Optional
from django.db.models import QuerySet
-from prompt_studio.prompt_studio_registry.constants import (
- PromptStudioRegistryKeys,
-)
+from prompt_studio.prompt_studio_registry.constants import PromptStudioRegistryKeys
from prompt_studio.prompt_studio_registry.serializers import (
PromptStudioRegistrySerializer,
)
diff --git a/backend/scheduler/helper.py b/backend/scheduler/helper.py
index 110b5398e..20a2d7d29 100644
--- a/backend/scheduler/helper.py
+++ b/backend/scheduler/helper.py
@@ -12,9 +12,7 @@ logger = logging.getLogger(__name__)
class SchedulerHelper:
@staticmethod
- def add_job(
- pipeline_id: str, cron_string: str = SC.DEFAULT_CRON_STRING
- ) -> None:
+ def add_job(pipeline_id: str, cron_string: str = SC.DEFAULT_CRON_STRING) -> None:
logger.info(f"Scheduling job for {pipeline_id} with {cron_string}")
name = f"Pipeline job-{pipeline_id}"
job_serialize_data = {
diff --git a/backend/scheduler/serializer.py b/backend/scheduler/serializer.py
index 585cba98f..2ea3c3b53 100644
--- a/backend/scheduler/serializer.py
+++ b/backend/scheduler/serializer.py
@@ -25,9 +25,7 @@ class JobKwargsSerializer(serializers.Serializer):
class SchedulerKwargsSerializer(serializers.Serializer):
coalesce = serializers.BooleanField()
- misfire_grace_time = serializers.IntegerField(
- allow_null=True, required=False
- )
+ misfire_grace_time = serializers.IntegerField(allow_null=True, required=False)
max_instances = serializers.IntegerField()
replace_existing = serializers.BooleanField()
diff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.py
index 2f6479645..b42244576 100644
--- a/backend/scheduler/tasks.py
+++ b/backend/scheduler/tasks.py
@@ -51,9 +51,7 @@ def create_periodic_task(
task=task_path,
args=task_args_json,
)
- logger.info(
- f"Completed running task addition to beat table: {task_name}"
- )
+ logger.info(f"Completed running task addition to beat table: {task_name}")
except Exception as e:
logger.error(f"Failed to create periodic task: {e}")
@@ -97,18 +95,14 @@ def execute_pipeline_task(
workflow = Workflow.objects.get(id=workflow_id)
try:
logger.info(f"Executing workflow: {workflow}")
- update_pipeline(
- pipepline_id, Pipeline.PipelineStatus.INPROGRESS
- )
+ update_pipeline(pipepline_id, Pipeline.PipelineStatus.INPROGRESS)
execution_response = WorkflowHelper.complete_execution(
workflow, execution_id, pipepline_id, with_logs
)
logger.info(f"Execution response: {execution_response}")
update_pipeline(pipepline_id, Pipeline.PipelineStatus.SUCCESS)
except IntegrityError:
- logger.error(
- f"Failed to complete execution for workflow: {workflow}"
- )
+ logger.error(f"Failed to complete execution for workflow: {workflow}")
update_pipeline(pipepline_id, Pipeline.PipelineStatus.FAILURE)
logger.info(f"Execution completed for pipeline: {name}")
except Exception as e:
diff --git a/backend/scheduler/views.py b/backend/scheduler/views.py
index 33a09d610..1e02a93d1 100644
--- a/backend/scheduler/views.py
+++ b/backend/scheduler/views.py
@@ -7,13 +7,8 @@ from pipeline.models import Pipeline
from pipeline.pipeline_processor import PipelineProcessor
from rest_framework.response import Response
from scheduler.tasks import create_periodic_task
-from workflow_manager.workflow.constants import (
- WorkflowExecutionKey,
- WorkflowKey,
-)
-from workflow_manager.workflow.exceptions import (
- WorkflowExecutionBadRequestException,
-)
+from workflow_manager.workflow.constants import WorkflowExecutionKey, WorkflowKey
+from workflow_manager.workflow.exceptions import WorkflowExecutionBadRequestException
from workflow_manager.workflow.serializers import ExecuteWorkflowSerializer
from workflow_manager.workflow.views import WorkflowViewSet
@@ -27,9 +22,7 @@ def schedule_task_job(pipeline_id: str, job_data: Any) -> Response:
return Response({"error": "cron_string is required"}, status=400)
if "id" not in job_data:
return Response({"error": "id is required"}, status=400)
- if "job_kwargs" in job_data and not isinstance(
- job_data["job_kwargs"], dict
- ):
+ if "job_kwargs" in job_data and not isinstance(job_data["job_kwargs"], dict):
return Response(
{"error": "job_kwargs is required to be a dict"}, status=400
)
@@ -57,9 +50,7 @@ def schedule_task_job(pipeline_id: str, job_data: Any) -> Response:
workflow_id = serializer.get_workflow_id(serializer.validated_data)
- execution_action = serializer.get_execution_action(
- serializer.validated_data
- )
+ execution_action = serializer.get_execution_action(serializer.validated_data)
org_schema = connection.tenant.schema_name
create_periodic_task(
diff --git a/backend/tenant_account/invitation_views.py b/backend/tenant_account/invitation_views.py
index de49bd84e..8d727eb74 100644
--- a/backend/tenant_account/invitation_views.py
+++ b/backend/tenant_account/invitation_views.py
@@ -15,10 +15,8 @@ class InvitationViewSet(viewsets.ViewSet):
@action(detail=False, methods=["GET"])
def list_invitations(self, request: Request) -> Response:
auth_controller = AuthenticationController()
- invitations: list[MemberInvitation] = (
- auth_controller.get_user_invitations(
- organization_id=request.org_id,
- )
+ invitations: list[MemberInvitation] = auth_controller.get_user_invitations(
+ organization_id=request.org_id,
)
serialized_members = ListInvitationsResponseSerializer(
invitations, many=True
diff --git a/backend/tenant_account/serializer.py b/backend/tenant_account/serializer.py
index 305efa749..250bbdb18 100644
--- a/backend/tenant_account/serializer.py
+++ b/backend/tenant_account/serializer.py
@@ -35,9 +35,7 @@ class OrganizationMemberSerializer(serializers.ModelSerializer):
class LimitedUserEmailListSerializer(serializers.ListSerializer):
def __init__(self, *args: Any, **kwargs: Any) -> None:
- self.max_elements: int = kwargs.pop(
- "max_elements", Common.MAX_EMAIL_IN_REQUEST
- )
+ self.max_elements: int = kwargs.pop("max_elements", Common.MAX_EMAIL_IN_REQUEST)
super().__init__(*args, **kwargs)
def validate(self, data: list[str]) -> Any:
@@ -50,9 +48,7 @@ class LimitedUserEmailListSerializer(serializers.ListSerializer):
class LimitedUserListSerializer(serializers.ListSerializer):
def __init__(self, *args: Any, **kwargs: Any) -> None:
- self.max_elements: int = kwargs.pop(
- "max_elements", Common.MAX_EMAIL_IN_REQUEST
- )
+ self.max_elements: int = kwargs.pop("max_elements", Common.MAX_EMAIL_IN_REQUEST)
super().__init__(*args, **kwargs)
def validate(
@@ -65,13 +61,9 @@ class LimitedUserListSerializer(serializers.ListSerializer):
for item in data:
if not isinstance(item, dict):
- raise ValidationError(
- "Each item in the list must be a dictionary."
- )
+ raise ValidationError("Each item in the list must be a dictionary.")
if "email" not in item:
- raise ValidationError(
- "Each item in the list must have 'email' key."
- )
+ raise ValidationError("Each item in the list must have 'email' key.")
if "role" not in item:
item["role"] = None
@@ -125,9 +117,7 @@ class ChangeUserRoleRequestSerializer(serializers.Serializer):
class DeleteInvitationRequestSerializer(serializers.Serializer):
id = serializers.EmailField(required=True)
- def get_id(
- self, validated_data: dict[str, Union[str, None]]
- ) -> Optional[str]:
+ def get_id(self, validated_data: dict[str, Union[str, None]]) -> Optional[str]:
return validated_data.get(Common.ID)
diff --git a/backend/tenant_account/users_view.py b/backend/tenant_account/users_view.py
index 2f8852107..3e244332b 100644
--- a/backend/tenant_account/users_view.py
+++ b/backend/tenant_account/users_view.py
@@ -78,9 +78,7 @@ class OrganizationUserViewSet(viewsets.ViewSet):
try:
# z_code = request.COOKIES.get(Cookie.Z_CODE)
user_info = auth_controller.get_user_info(request)
- role = auth_controller.get_organization_members_by_user(
- request.user
- )
+ role = auth_controller.get_organization_members_by_user(request.user)
if not user_info:
return Response(
status=status.HTTP_404_NOT_FOUND,
@@ -112,9 +110,7 @@ class OrganizationUserViewSet(viewsets.ViewSet):
admin=request.user, org_id=request.org_id, user_list=user_list
)
- response_serializer = UserInviteResponseSerializer(
- invite_response, many=True
- )
+ response_serializer = UserInviteResponseSerializer(invite_response, many=True)
if invite_response and len(invite_response) != 0:
response = Response(
@@ -160,9 +156,7 @@ class OrganizationUserViewSet(viewsets.ViewSet):
members: list[OrganizationMember] = (
auth_controller.get_organization_members_by_org_id()
)
- serialized_members = OrganizationMemberSerializer(
- members, many=True
- ).data
+ serialized_members = OrganizationMemberSerializer(members, many=True).data
return Response(
status=status.HTTP_200_OK,
data={"message": "success", "members": serialized_members},
diff --git a/backend/tenant_account/views.py b/backend/tenant_account/views.py
index 598a72ce7..1f95d07ae 100644
--- a/backend/tenant_account/views.py
+++ b/backend/tenant_account/views.py
@@ -37,9 +37,7 @@ def get_roles(request: Request) -> Response:
@api_view(["POST"])
def reset_password(request: Request) -> Response:
auth_controller = AuthenticationController()
- data: ResetUserPasswordDto = auth_controller.reset_user_password(
- request.user
- )
+ data: ResetUserPasswordDto = auth_controller.reset_user_password(request.user)
if data.status:
return Response(
status=status.HTTP_200_OK,
diff --git a/backend/tool_instance/models.py b/backend/tool_instance/models.py
index 7c9bb5122..7ddc73cff 100644
--- a/backend/tool_instance/models.py
+++ b/backend/tool_instance/models.py
@@ -39,19 +39,13 @@ class ToolInstance(BaseModel):
max_length=TOOL_ID_LENGTH,
db_comment="Function name of the tool being used",
)
- input = models.JSONField(
- null=True, db_comment="Provisional WF input to a tool"
- )
- output = models.JSONField(
- null=True, db_comment="Provisional WF output to a tool"
- )
+ input = models.JSONField(null=True, db_comment="Provisional WF input to a tool")
+ output = models.JSONField(null=True, db_comment="Provisional WF output to a tool")
version = models.CharField(max_length=TOOL_VERSION_LENGTH)
metadata = models.JSONField(db_comment="Stores config for a tool")
step = models.IntegerField()
# TODO: Make as an enum supporting fixed values once we have clarity
- status = models.CharField(
- max_length=TOOL_STATUS_LENGTH, default="Ready to start"
- )
+ status = models.CharField(max_length=TOOL_STATUS_LENGTH, default="Ready to start")
created_by = models.ForeignKey(
User,
on_delete=models.SET_NULL,
diff --git a/backend/tool_instance/serializers.py b/backend/tool_instance/serializers.py
index 944447b3f..4ce323b49 100644
--- a/backend/tool_instance/serializers.py
+++ b/backend/tool_instance/serializers.py
@@ -2,15 +2,8 @@ import logging
import uuid
from typing import Any
-from prompt_studio.prompt_studio_registry.constants import (
- PromptStudioRegistryKeys,
-)
-from rest_framework.serializers import (
- ListField,
- Serializer,
- UUIDField,
- ValidationError,
-)
+from prompt_studio.prompt_studio_registry.constants import PromptStudioRegistryKeys
+from rest_framework.serializers import ListField, Serializer, UUIDField, ValidationError
from tool_instance.constants import ToolInstanceKey as TIKey
from tool_instance.constants import ToolKey
from tool_instance.exceptions import ToolDoesNotExist
@@ -72,9 +65,7 @@ class ToolInstanceSerializer(AuditSerializer):
try:
workflow = Workflow.objects.get(pk=workflow_id)
except Workflow.DoesNotExist:
- raise ValidationError(
- f"Workflow with ID {workflow_id} does not exist."
- )
+ raise ValidationError(f"Workflow with ID {workflow_id} does not exist.")
validated_data[TIKey.WORKFLOW] = workflow
tool_uid = validated_data.get(TIKey.TOOL_ID)
@@ -113,9 +104,7 @@ class ToolInstanceReorderSerializer(Serializer):
try:
workflow = Workflow.objects.get(pk=workflow_id)
except Workflow.DoesNotExist:
- raise ValidationError(
- f"Workflow with ID {workflow_id} does not exist."
- )
+ raise ValidationError(f"Workflow with ID {workflow_id} does not exist.")
# Check if the number of tool instances matches the actual count
tool_instance_count = workflow.workflow_tool.count()
@@ -128,9 +117,7 @@ class ToolInstanceReorderSerializer(Serializer):
raise ValidationError(detail=msg)
# Check if each tool instance exists in the workflow
- existing_tool_instance_ids = workflow.workflow_tool.values_list(
- "id", flat=True
- )
+ existing_tool_instance_ids = workflow.workflow_tool.values_list("id", flat=True)
for tool_instance_id in tool_instances:
if tool_instance_id not in existing_tool_instance_ids:
raise ValidationError(
diff --git a/backend/tool_instance/tool_instance_helper.py b/backend/tool_instance/tool_instance_helper.py
index 167093ddb..87ef6052c 100644
--- a/backend/tool_instance/tool_instance_helper.py
+++ b/backend/tool_instance/tool_instance_helper.py
@@ -44,13 +44,11 @@ class ToolInstanceHelper:
offset_value = 0 if not offset else offset
to = offset_value + limit
return list(
- ToolInstance.objects.filter(**wf_filter)[
- offset_value:to
- ].order_by(order_by)
+ ToolInstance.objects.filter(**wf_filter)[offset_value:to].order_by(
+ order_by
+ )
)
- return list(
- ToolInstance.objects.filter(**wf_filter).all().order_by(order_by)
- )
+ return list(ToolInstance.objects.filter(**wf_filter).all().order_by(order_by))
@staticmethod
def update_instance_metadata(
@@ -60,9 +58,7 @@ class ToolInstanceHelper:
JsonSchemaKey.OUTPUT_FILE_CONNECTOR in metadata
and JsonSchemaKey.OUTPUT_FOLDER in metadata
):
- output_connector_name = metadata[
- JsonSchemaKey.OUTPUT_FILE_CONNECTOR
- ]
+ output_connector_name = metadata[JsonSchemaKey.OUTPUT_FILE_CONNECTOR]
output_connector = ConnectorInstanceHelper.get_output_connector_instance_by_name_for_workflow( # noqa
tool_instance.workflow_id, output_connector_name
)
@@ -210,9 +206,7 @@ class ToolInstanceHelper:
JsonSchemaKey.OUTPUT_FILE_CONNECTOR in metadata
and JsonSchemaKey.OUTPUT_FOLDER in metadata
):
- output_connector_name = metadata[
- JsonSchemaKey.OUTPUT_FILE_CONNECTOR
- ]
+ output_connector_name = metadata[JsonSchemaKey.OUTPUT_FILE_CONNECTOR]
output_connector = ConnectorInstanceHelper.get_output_connector_instance_by_name_for_workflow( # noqa
tool_instance.workflow_id, output_connector_name
)
@@ -350,9 +344,7 @@ class ToolInstanceHelper:
tool: Tool = ToolProcessor.get_tool_by_uid(tool_uid=tool_uid)
tool_name: str = (
- tool.properties.display_name
- if tool.properties.display_name
- else tool_uid
+ tool.properties.display_name if tool.properties.display_name else tool_uid
)
schema_json: dict[str, Any] = ToolProcessor.get_json_schema_for_tool(
tool_uid=tool_uid, user=user
@@ -379,51 +371,39 @@ class ToolInstanceHelper:
if llm.is_enabled and llm.adapter_id:
adapter_id = tool_meta[llm.adapter_id]
elif llm.is_enabled:
- adapter_id = tool_meta[
- AdapterPropertyKey.DEFAULT_LLM_ADAPTER_ID
- ]
+ adapter_id = tool_meta[AdapterPropertyKey.DEFAULT_LLM_ADAPTER_ID]
adapter_ids.add(adapter_id)
for vdb in tool.properties.adapter.vector_stores:
if vdb.is_enabled and vdb.adapter_id:
adapter_id = tool_meta[vdb.adapter_id]
elif vdb.is_enabled:
- adapter_id = tool_meta[
- AdapterPropertyKey.DEFAULT_VECTOR_DB_ADAPTER_ID
- ]
+ adapter_id = tool_meta[AdapterPropertyKey.DEFAULT_VECTOR_DB_ADAPTER_ID]
adapter_ids.add(adapter_id)
for embedding in tool.properties.adapter.embedding_services:
if embedding.is_enabled and embedding.adapter_id:
adapter_id = tool_meta[embedding.adapter_id]
elif embedding.is_enabled:
- adapter_id = tool_meta[
- AdapterPropertyKey.DEFAULT_EMBEDDING_ADAPTER_ID
- ]
+ adapter_id = tool_meta[AdapterPropertyKey.DEFAULT_EMBEDDING_ADAPTER_ID]
adapter_ids.add(adapter_id)
for text_extractor in tool.properties.adapter.text_extractors:
if text_extractor.is_enabled and text_extractor.adapter_id:
adapter_id = tool_meta[text_extractor.adapter_id]
elif text_extractor.is_enabled:
- adapter_id = tool_meta[
- AdapterPropertyKey.DEFAULT_X2TEXT_ADAPTER_ID
- ]
+ adapter_id = tool_meta[AdapterPropertyKey.DEFAULT_X2TEXT_ADAPTER_ID]
adapter_ids.add(adapter_id)
- ToolInstanceHelper.validate_adapter_access(
- user=user, adapter_ids=adapter_ids
- )
+ ToolInstanceHelper.validate_adapter_access(user=user, adapter_ids=adapter_ids)
@staticmethod
def validate_adapter_access(
user: User,
adapter_ids: set[str],
) -> None:
- adapter_instances = AdapterInstance.objects.filter(
- id__in=adapter_ids
- ).all()
+ adapter_instances = AdapterInstance.objects.filter(id__in=adapter_ids).all()
for adapter_instance in adapter_instances:
if not (
@@ -458,6 +438,4 @@ class ToolInstanceHelper:
):
return
else:
- raise PermissionDenied(
- "You don't have permission to perform this action."
- )
+ raise PermissionDenied("You don't have permission to perform this action.")
diff --git a/backend/tool_instance/tool_processor.py b/backend/tool_instance/tool_processor.py
index acc0ab06f..bd0a01cf7 100644
--- a/backend/tool_instance/tool_processor.py
+++ b/backend/tool_instance/tool_processor.py
@@ -62,9 +62,7 @@ class ToolProcessor:
return schema_json
@staticmethod
- def update_schema_with_adapter_configurations(
- schema: Spec, user: User
- ) -> None:
+ def update_schema_with_adapter_configurations(schema: Spec, user: User) -> None:
"""Updates the JSON schema with the available adapter configurations
for the LLM, embedding, and vector DB adapters.
@@ -85,9 +83,7 @@ class ToolProcessor:
AdapterTypes.LLM, user=user
)
for key in llm_keys:
- adapter_names = map(
- lambda adapter: str(adapter.adapter_name), adapters
- )
+ adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)
schema.properties[key]["enum"] = list(adapter_names)
if embedding_keys:
@@ -95,9 +91,7 @@ class ToolProcessor:
AdapterTypes.EMBEDDING, user=user
)
for key in embedding_keys:
- adapter_names = map(
- lambda adapter: str(adapter.adapter_name), adapters
- )
+ adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)
schema.properties[key]["enum"] = list(adapter_names)
if vector_db_keys:
@@ -105,9 +99,7 @@ class ToolProcessor:
AdapterTypes.VECTOR_DB, user=user
)
for key in vector_db_keys:
- adapter_names = map(
- lambda adapter: str(adapter.adapter_name), adapters
- )
+ adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)
schema.properties[key]["enum"] = list(adapter_names)
if x2text_keys:
@@ -115,9 +107,7 @@ class ToolProcessor:
AdapterTypes.X2TEXT, user=user
)
for key in x2text_keys:
- adapter_names = map(
- lambda adapter: str(adapter.adapter_name), adapters
- )
+ adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)
schema.properties[key]["enum"] = list(adapter_names)
if ocr_keys:
@@ -125,9 +115,7 @@ class ToolProcessor:
AdapterTypes.OCR, user=user
)
for key in ocr_keys:
- adapter_names = map(
- lambda adapter: str(adapter.adapter_name), adapters
- )
+ adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)
schema.properties[key]["enum"] = list(adapter_names)
@staticmethod
@@ -137,9 +125,7 @@ class ToolProcessor:
prompt_studio_tools: list[dict[str, Any]] = (
PromptStudioRegistryHelper.fetch_json_for_registry(user)
)
- tool_list: list[dict[str, Any]] = (
- tool_registry.fetch_tools_descriptions()
- )
+ tool_list: list[dict[str, Any]] = tool_registry.fetch_tools_descriptions()
tool_list = tool_list + prompt_studio_tools
return tool_list
diff --git a/backend/tool_instance/views.py b/backend/tool_instance/views.py
index 98bf6c8a8..e6b537003 100644
--- a/backend/tool_instance/views.py
+++ b/backend/tool_instance/views.py
@@ -13,10 +13,7 @@ from rest_framework.versioning import URLPathVersioning
from tool_instance.constants import ToolInstanceErrors
from tool_instance.constants import ToolInstanceKey as TIKey
from tool_instance.constants import ToolKey
-from tool_instance.exceptions import (
- FetchToolListFailed,
- ToolFunctionIsMandatory,
-)
+from tool_instance.exceptions import FetchToolListFailed, ToolFunctionIsMandatory
from tool_instance.models import ToolInstance
from tool_instance.serializers import (
ToolInstanceReorderSerializer as TIReorderSerializer,
@@ -134,19 +131,13 @@ class ToolInstanceViewSet(viewsets.ModelViewSet):
instance.save()
return
- def partial_update(
- self, request: Request, *args: Any, **kwargs: Any
- ) -> Response:
+ def partial_update(self, request: Request, *args: Any, **kwargs: Any) -> Response:
"""Allows partial updates on a tool instance."""
instance: ToolInstance = self.get_object()
- serializer = self.get_serializer(
- instance, data=request.data, partial=True
- )
+ serializer = self.get_serializer(instance, data=request.data, partial=True)
serializer.is_valid(raise_exception=True)
if serializer.validated_data.get(TIKey.METADATA):
- metadata: dict[str, Any] = serializer.validated_data.get(
- TIKey.METADATA
- )
+ metadata: dict[str, Any] = serializer.validated_data.get(TIKey.METADATA)
# TODO: Move update logic into serializer
ToolInstanceHelper.update_instance_metadata(
@@ -168,10 +159,6 @@ class ToolInstanceViewSet(viewsets.ModelViewSet):
]
ToolInstanceHelper.reorder_tool_instances(instances_to_reorder)
- tool_instances = ToolInstance.objects.get_instances_for_workflow(
- workflow=wf_id
- )
- ti_serializer = ToolInstanceSerializer(
- instance=tool_instances, many=True
- )
+ tool_instances = ToolInstance.objects.get_instances_for_workflow(workflow=wf_id)
+ ti_serializer = ToolInstanceSerializer(instance=tool_instances, many=True)
return Response(ti_serializer.data, status=status.HTTP_200_OK)
diff --git a/backend/utils/log_events.py b/backend/utils/log_events.py
index 0e510db53..d83cc03ee 100644
--- a/backend/utils/log_events.py
+++ b/backend/utils/log_events.py
@@ -59,9 +59,7 @@ def _pubsub_listen_forever() -> None:
message = pubsub.get_message()
if message:
- logger.debug(
- f"[{os.getpid()}] Pub sub message received: {message}"
- )
+ logger.debug(f"[{os.getpid()}] Pub sub message received: {message}")
if message["type"] == "pmessage":
_handle_pubsub_messages(message)
@@ -75,9 +73,7 @@ def _pubsub_listen_forever() -> None:
def start_server(django_app: WSGIHandler, namespace: str) -> WSGIHandler:
django_app = socketio.WSGIApp(sio, django_app, socketio_path=namespace)
- pubsub_listener = threading.Thread(
- target=_pubsub_listen_forever, daemon=True
- )
+ pubsub_listener = threading.Thread(target=_pubsub_listen_forever, daemon=True)
pubsub_listener.start()
return django_app
diff --git a/backend/utils/request/request.py b/backend/utils/request/request.py
index e976e8274..9c05df155 100644
--- a/backend/utils/request/request.py
+++ b/backend/utils/request/request.py
@@ -28,15 +28,11 @@ def make_http_request(
if verb == HTTPMethod.GET:
response = pyrequests.get(url, params=params, headers=headers)
elif verb == HTTPMethod.POST:
- response = pyrequests.post(
- url, json=data, params=params, headers=headers
- )
+ response = pyrequests.post(url, json=data, params=params, headers=headers)
elif verb == HTTPMethod.DELETE:
response = pyrequests.delete(url, params=params, headers=headers)
else:
- raise ValueError(
- "Invalid HTTP verb. Supported verbs: GET, POST, DELETE"
- )
+ raise ValueError("Invalid HTTP verb. Supported verbs: GET, POST, DELETE")
response.raise_for_status()
return_val: str = (
diff --git a/backend/workflow_manager/endpoint/base_connector.py b/backend/workflow_manager/endpoint/base_connector.py
index 7d4d7e6ae..e90ae03c2 100644
--- a/backend/workflow_manager/endpoint/base_connector.py
+++ b/backend/workflow_manager/endpoint/base_connector.py
@@ -4,15 +4,11 @@ from typing import Any
from django.conf import settings
from django.db import connection
from fsspec import AbstractFileSystem
-from unstract.workflow_execution.execution_file_handler import (
- ExecutionFileHandler,
-)
+from unstract.workflow_execution.execution_file_handler import ExecutionFileHandler
from utils.constants import Common
from unstract.connectors.filesystems import connectors
-from unstract.connectors.filesystems.unstract_file_system import (
- UnstractFileSystem,
-)
+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
class BaseConnector(ExecutionFileHandler):
@@ -76,9 +72,7 @@ class BaseConnector(ExecutionFileHandler):
return schema
@classmethod
- def get_api_storage_dir_path(
- cls, workflow_id: str, execution_id: str
- ) -> str:
+ def get_api_storage_dir_path(cls, workflow_id: str, execution_id: str) -> str:
"""Get the directory path for storing api files.
Parameters:
diff --git a/backend/workflow_manager/endpoint/database_utils.py b/backend/workflow_manager/endpoint/database_utils.py
index 6402b0697..f852e2002 100644
--- a/backend/workflow_manager/endpoint/database_utils.py
+++ b/backend/workflow_manager/endpoint/database_utils.py
@@ -159,9 +159,7 @@ class DatabaseUtils:
)
column_types = DatabaseUtils.get_column_types_util(results)
except Exception as e:
- logger.error(
- f"Error getting column types for {table_name}: {str(e)}"
- )
+ logger.error(f"Error getting column types for {table_name}: {str(e)}")
raise e
return column_types
@@ -307,9 +305,7 @@ class DatabaseUtils:
def get_db_class(
connector_id: str, connector_settings: dict[str, Any]
) -> UnstractDB:
- connector = db_connectors[connector_id][Common.METADATA][
- Common.CONNECTOR
- ]
+ connector = db_connectors[connector_id][Common.METADATA][Common.CONNECTOR]
connector_class: UnstractDB = connector(connector_settings)
return connector_class
@@ -317,9 +313,7 @@ class DatabaseUtils:
def execute_and_fetch_data(
connector_id: str, connector_settings: dict[str, Any], query: str
) -> Any:
- connector = db_connectors[connector_id][Common.METADATA][
- Common.CONNECTOR
- ]
+ connector = db_connectors[connector_id][Common.METADATA][Common.CONNECTOR]
connector_class: UnstractDB = connector(connector_settings)
return connector_class.execute(query=query)
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.py
index f76bd6eda..a293eaf2e 100644
--- a/backend/workflow_manager/endpoint/destination.py
+++ b/backend/workflow_manager/endpoint/destination.py
@@ -88,10 +88,7 @@ class DestinationConnector(BaseConnector):
raise MissingDestinationConnectionType()
if connection_type not in WorkflowEndpoint.ConnectionType.values:
raise InvalidDestinationConnectionType()
- if (
- connection_type != WorkflowEndpoint.ConnectionType.API
- and connector is None
- ):
+ if connection_type != WorkflowEndpoint.ConnectionType.API and connector is None:
raise DestinationConnectorNotConfigured()
def handle_output(
@@ -107,9 +104,7 @@ class DestinationConnector(BaseConnector):
result: Optional[str] = None
if error:
if connection_type == WorkflowEndpoint.ConnectionType.API:
- self._handle_api_result(
- file_name=file_name, error=error, result=result
- )
+ self._handle_api_result(file_name=file_name, error=error, result=result)
return
if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
self.copy_output_to_output_directory()
@@ -117,9 +112,7 @@ class DestinationConnector(BaseConnector):
self.insert_into_db(file_history)
elif connection_type == WorkflowEndpoint.ConnectionType.API:
result = self.get_result(file_history)
- self._handle_api_result(
- file_name=file_name, error=error, result=result
- )
+ self._handle_api_result(file_name=file_name, error=error, result=result)
if not file_history:
FileHistoryHelper.create_file_history(
cache_key=file_hash,
@@ -184,27 +177,17 @@ class DestinationConnector(BaseConnector):
connector_instance: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector_instance.metadata
destination_configurations: dict[str, Any] = self.endpoint.configuration
- table_name: str = str(
- destination_configurations.get(DestinationKey.TABLE)
- )
+ table_name: str = str(destination_configurations.get(DestinationKey.TABLE))
include_agent: bool = bool(
destination_configurations.get(DestinationKey.INCLUDE_AGENT, False)
)
include_timestamp = bool(
- destination_configurations.get(
- DestinationKey.INCLUDE_TIMESTAMP, False
- )
- )
- agent_name = str(
- destination_configurations.get(DestinationKey.AGENT_NAME)
- )
- column_mode = str(
- destination_configurations.get(DestinationKey.COLUMN_MODE)
+ destination_configurations.get(DestinationKey.INCLUDE_TIMESTAMP, False)
)
+ agent_name = str(destination_configurations.get(DestinationKey.AGENT_NAME))
+ column_mode = str(destination_configurations.get(DestinationKey.COLUMN_MODE))
single_column_name = str(
- destination_configurations.get(
- DestinationKey.SINGLE_COLUMN_NAME, "data"
- )
+ destination_configurations.get(DestinationKey.SINGLE_COLUMN_NAME, "data")
)
data = self.get_result(file_history)
@@ -230,14 +213,12 @@ class DestinationConnector(BaseConnector):
table_name=table_name,
database_entry=values,
)
- sql_columns_and_values = (
- DatabaseUtils.get_sql_columns_and_values_for_query(
- engine=engine,
- connector_id=connector_instance.connector_id,
- connector_settings=connector_settings,
- table_name=table_name,
- values=values,
- )
+ sql_columns_and_values = DatabaseUtils.get_sql_columns_and_values_for_query(
+ engine=engine,
+ connector_id=connector_instance.connector_id,
+ connector_settings=connector_settings,
+ table_name=table_name,
+ values=values,
)
DatabaseUtils.execute_write_query(
engine=engine,
@@ -364,9 +345,7 @@ class DestinationConnector(BaseConnector):
self.delete_api_storage_dir(self.workflow_id, self.execution_id)
@classmethod
- def delete_api_storage_dir(
- cls, workflow_id: str, execution_id: str
- ) -> None:
+ def delete_api_storage_dir(cls, workflow_id: str, execution_id: str) -> None:
"""Delete the api storage path.
Returns:
diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.py
index 973655457..c97ea149f 100644
--- a/backend/workflow_manager/endpoint/source.py
+++ b/backend/workflow_manager/endpoint/source.py
@@ -102,10 +102,7 @@ class SourceConnector(BaseConnector):
raise MissingSourceConnectionType()
if connection_type not in WorkflowEndpoint.ConnectionType.values:
raise InvalidSourceConnectionType()
- if (
- connection_type != WorkflowEndpoint.ConnectionType.API
- and connector is None
- ):
+ if connection_type != WorkflowEndpoint.ConnectionType.API and connector is None:
raise SourceConnectorNotConfigured()
def valid_file_patterns(self, required_patterns: list[Any]) -> list[str]:
@@ -147,9 +144,7 @@ class SourceConnector(BaseConnector):
connector: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector.connector_metadata
source_configurations: dict[str, Any] = self.endpoint.configuration
- required_patterns = source_configurations.get(
- SourceKey.FILE_EXTENSIONS, []
- )
+ required_patterns = source_configurations.get(SourceKey.FILE_EXTENSIONS, [])
recursive = bool(
source_configurations.get(SourceKey.PROCESS_SUB_DIRECTORIES, False)
)
@@ -159,13 +154,9 @@ class SourceConnector(BaseConnector):
)
)
root_dir_path = connector_settings.get(ConnectorKeys.PATH, "")
- input_directory = str(
- source_configurations.get(SourceKey.ROOT_FOLDER, "")
- )
+ input_directory = str(source_configurations.get(SourceKey.ROOT_FOLDER, ""))
if root_dir_path: # user needs to manually type the optional file path
- input_directory = str(
- Path(root_dir_path, input_directory.lstrip("/"))
- )
+ input_directory = str(Path(root_dir_path, input_directory.lstrip("/")))
if not isinstance(required_patterns, list):
required_patterns = [required_patterns]
@@ -187,9 +178,7 @@ class SourceConnector(BaseConnector):
) -> None:
if not self.execution_service:
return None
- input_log = (
- f"##Input folder:\n\n `{os.path.basename(input_directory)}`\n\n"
- )
+ input_log = f"##Input folder:\n\n `{os.path.basename(input_directory)}`\n\n"
self.execution_service.publish_update_log(
state=LogState.INPUT_UPDATE, message=input_log
)
@@ -198,13 +187,9 @@ class SourceConnector(BaseConnector):
state=LogState.OUTPUT_UPDATE, message=output_log
)
- def publish_input_file_content(
- self, input_file_path: str, input_text: str
- ) -> None:
+ def publish_input_file_content(self, input_file_path: str, input_text: str) -> None:
if self.execution_service:
- output_log_message = (
- f"##Input text:\n\n```text\n{input_text}\n```\n\n"
- )
+ output_log_message = f"##Input text:\n\n```text\n{input_text}\n```\n\n"
input_log_message = (
"##Input file:\n\n```text\n"
f"{os.path.basename(input_file_path)}\n```\n\n"
@@ -254,9 +239,7 @@ class SourceConnector(BaseConnector):
count = 0
max_depth = int(SourceConstant.MAX_RECURSIVE_DEPTH) if recursive else 1
- for root, dirs, files in source_fs.walk(
- input_directory, maxdepth=max_depth
- ):
+ for root, dirs, files in source_fs.walk(input_directory, maxdepth=max_depth):
if count >= limit:
break
for file in files:
@@ -326,9 +309,7 @@ class SourceConnector(BaseConnector):
"""
connector: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector.connector_metadata
- source_file_path = os.path.join(
- self.execution_dir, WorkflowFileType.SOURCE
- )
+ source_file_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)
infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
source_file = f"file://{source_file_path}"
@@ -346,8 +327,7 @@ class SourceConnector(BaseConnector):
f": {hash_value_of_file_content}"
)
input_log = (
- file_content[:500].decode("utf-8", errors="replace")
- + "...(truncated)"
+ file_content[:500].decode("utf-8", errors="replace") + "...(truncated)"
)
self.publish_input_file_content(input_file_path, input_log)
@@ -356,9 +336,7 @@ class SourceConnector(BaseConnector):
logger.info(f"{input_file_path} is added in to execution directory")
return hash_value_of_file_content
- def add_input_from_api_storage_to_volume(
- self, input_file_path: str
- ) -> None:
+ def add_input_from_api_storage_to_volume(self, input_file_path: str) -> None:
"""Add input file to execution directory from api storage."""
infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
source_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)
@@ -386,9 +364,7 @@ class SourceConnector(BaseConnector):
input_file_path=input_file_path,
)
elif connection_type == WorkflowEndpoint.ConnectionType.API:
- self.add_input_from_api_storage_to_volume(
- input_file_path=input_file_path
- )
+ self.add_input_from_api_storage_to_volume(input_file_path=input_file_path)
if file_name not in hash_values_of_files:
raise FileHashNotFound()
file_content_hash = hash_values_of_files[file_name]
diff --git a/backend/workflow_manager/endpoint/urls.py b/backend/workflow_manager/endpoint/urls.py
index 9efd62f80..1b658b27c 100644
--- a/backend/workflow_manager/endpoint/urls.py
+++ b/backend/workflow_manager/endpoint/urls.py
@@ -14,9 +14,7 @@ endpoint_settings_detail = WorkflowEndpointViewSet.as_view(
urlpatterns = [
path("", endpoint_list, name="endpoint-list"),
- path(
- "<str:pk>/", workflow_endpoint_detail, name="workflow-endpoint-detail"
- ),
+ path("<str:pk>/", workflow_endpoint_detail, name="workflow-endpoint-detail"),
path(
"<str:pk>/settings/",
endpoint_settings_detail,
diff --git a/backend/workflow_manager/endpoint/views.py b/backend/workflow_manager/endpoint/views.py
index b1cc1c35c..50375d97f 100644
--- a/backend/workflow_manager/endpoint/views.py
+++ b/backend/workflow_manager/endpoint/views.py
@@ -21,12 +21,8 @@ class WorkflowEndpointViewSet(viewsets.ModelViewSet):
.select_related("workflow")
.filter(workflow__created_by=self.request.user)
)
- endpoint_type_filter = self.request.query_params.get(
- "endpoint_type", None
- )
- connection_type_filter = self.request.query_params.get(
- "connection_type", None
- )
+ endpoint_type_filter = self.request.query_params.get("endpoint_type", None)
+ connection_type_filter = self.request.query_params.get("connection_type", None)
if endpoint_type_filter:
queryset = queryset.filter(endpoint_type=endpoint_type_filter)
if connection_type_filter:
diff --git a/backend/workflow_manager/workflow/exceptions.py b/backend/workflow_manager/workflow/exceptions.py
index 4715a9719..56f111600 100644
--- a/backend/workflow_manager/workflow/exceptions.py
+++ b/backend/workflow_manager/workflow/exceptions.py
@@ -62,9 +62,7 @@ class WorkflowExecutionBadRequestException(APIException):
status_code = 400
default_detail = "Bad request"
- def __init__(
- self, detail: Optional[str] = None, code: Optional[int] = None
- ):
+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):
if detail is not None:
self.detail = detail
if code is not None:
diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.py
index 035c21e31..acfff7fee 100644
--- a/backend/workflow_manager/workflow/execution.py
+++ b/backend/workflow_manager/workflow/execution.py
@@ -5,23 +5,15 @@ from typing import Optional
from account.constants import Common
from api.exceptions import InvalidAPIRequest
from django.db import connection
-from platform_settings.platform_auth_service import (
- PlatformAuthenticationService,
-)
+from platform_settings.platform_auth_service import PlatformAuthenticationService
from tool_instance.constants import JsonSchemaKey
from tool_instance.models import ToolInstance
from tool_instance.tool_processor import ToolProcessor
from unstract.tool_registry.dto import Tool
from unstract.workflow_execution import WorkflowExecutionService
-from unstract.workflow_execution.dto import (
- ToolInstance as ToolInstanceDataClass,
-)
+from unstract.workflow_execution.dto import ToolInstance as ToolInstanceDataClass
from unstract.workflow_execution.dto import WorkflowDto
-from unstract.workflow_execution.enums import (
- ExecutionType,
- LogComponent,
- LogState,
-)
+from unstract.workflow_execution.enums import ExecutionType, LogComponent, LogState
from unstract.workflow_execution.exceptions import StopExecution
from utils.local_context import StateStore
from workflow_manager.workflow.constants import WorkflowKey
@@ -51,8 +43,8 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
tool_instances_as_dto.append(
self.convert_tool_instance_model_to_data_class(tool_instance)
)
- workflow_as_dto: WorkflowDto = (
- self.convert_workflow_model_to_data_class(workflow=workflow)
+ workflow_as_dto: WorkflowDto = self.convert_workflow_model_to_data_class(
+ workflow=workflow
)
organization_id = organization_id or connection.tenant.schema_name
if not organization_id:
@@ -71,9 +63,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
# Use pipline_id for pipelines / API deployment
# since session might not be present.
log_events_id = StateStore.get(Common.LOG_EVENTS_ID)
- self.execution_log_id = (
- log_events_id if log_events_id else pipeline_id
- )
+ self.execution_log_id = log_events_id if log_events_id else pipeline_id
self.execution_mode = mode
self.execution_method: tuple[str, str] = (
WorkflowExecution.Method.SCHEDULED
@@ -103,17 +93,13 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
self.set_messaging_channel(str(self.execution_log_id))
project_settings = {}
- project_settings[WorkflowKey.WF_PROJECT_GUID] = str(
- self.execution_log_id
- )
+ project_settings[WorkflowKey.WF_PROJECT_GUID] = str(self.execution_log_id)
self.workflow_id = workflow.id
self.project_settings = project_settings
self.pipeline_id = pipeline_id
self.execution_id = str(workflow_execution.id)
- self.compilation_result = self.compile_workflow(
- execution_id=self.execution_id
- )
+ self.compilation_result = self.compile_workflow(execution_id=self.execution_id)
def _initiate_api_execution(
self, tool_instance: ToolInstance, execution_path: Optional[str]
@@ -250,18 +236,14 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
end_time = time.time()
execution_time = end_time - start_time
message = str(exception)[:EXECUTION_ERROR_LENGTH]
- logger.info(
- f"Execution {self.execution_id} Error {exception}"
- )
+ logger.info(f"Execution {self.execution_id} Error {exception}")
raise self.__execution_error(
error_message=message,
detailed_message=message,
execution_time=execution_time,
)
else:
- error_message = (
- f"Unknown Execution Method {self.execution_mode}"
- )
+ error_message = f"Unknown Execution Method {self.execution_mode}"
raise self.__execution_error(
error_message=error_message, detailed_message=error_message
)
@@ -282,9 +264,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
None
"""
self.publish_log(f"Total matched files: {total_files}")
- self.publish_update_log(
- LogState.BEGIN_WORKFLOW, "1", LogComponent.STATUS_BAR
- )
+ self.publish_update_log(LogState.BEGIN_WORKFLOW, "1", LogComponent.STATUS_BAR)
self.publish_update_log(
LogState.RUNNING, "Ready for execution", LogComponent.WORKFLOW
)
@@ -295,9 +275,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
Returns:
None
"""
- self.publish_update_log(
- LogState.END_WORKFLOW, "1", LogComponent.STATUS_BAR
- )
+ self.publish_update_log(LogState.END_WORKFLOW, "1", LogComponent.STATUS_BAR)
self.publish_update_log(
LogState.SUCCESS, "Executed successfully", LogComponent.WORKFLOW
)
@@ -344,9 +322,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
if single_step:
execution_type = ExecutionType.STEP
if not (file_history and file_history.is_completed()):
- self.execute_uncached_input(
- file_name=file_name, single_step=single_step
- )
+ self.execute_uncached_input(file_name=file_name, single_step=single_step)
else:
self.publish_log(
f"Skipping file {file_name} as it is already processed."
@@ -400,9 +376,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
execution_type = ExecutionType.COMPLETE
if single_step:
execution_type = ExecutionType.STEP
- self.publish_initial_tool_execution_logs(
- current_step, total_step, file_name
- )
+ self.publish_initial_tool_execution_logs(current_step, total_step, file_name)
self._handle_execution_type(execution_type)
source_status_message = (
@@ -416,9 +390,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
self.publish_log("Trying to fetch results from cache")
@staticmethod
- def update_execution_status(
- execution_id: str, status: ExecutionStatus
- ) -> None:
+ def update_execution_status(execution_id: str, status: ExecutionStatus) -> None:
try:
execution = WorkflowExecution.objects.get(pk=execution_id)
execution.status = status.value
diff --git a/backend/workflow_manager/workflow/generator.py b/backend/workflow_manager/workflow/generator.py
index f52a0a9ce..7c901b4bf 100644
--- a/backend/workflow_manager/workflow/generator.py
+++ b/backend/workflow_manager/workflow/generator.py
@@ -35,9 +35,7 @@ class WorkflowGenerator:
def provisional_wf(self) -> ProvisionalWorkflow:
return self._provisional_wf
- def _get_provisional_workflow(
- self, tools: list[Tool]
- ) -> ProvisionalWorkflow:
+ def _get_provisional_workflow(self, tools: list[Tool]) -> ProvisionalWorkflow:
"""Helper to generate the provisional workflow Gets stored as
`workflow.Workflow.llm_response` eventually."""
provisional_wf: ProvisionalWorkflow
@@ -48,13 +46,11 @@ class WorkflowGenerator:
)
llm_interface = LLMInterface()
- provisional_wf_dict = (
- llm_interface.get_provisional_workflow_from_llm(
- workflow_id=self._workflow_id,
- tools=tools,
- user_prompt=self._request.data.get(WorkflowKey.PROMPT_TEXT),
- use_cache=True,
- )
+ provisional_wf_dict = llm_interface.get_provisional_workflow_from_llm(
+ workflow_id=self._workflow_id,
+ tools=tools,
+ user_prompt=self._request.data.get(WorkflowKey.PROMPT_TEXT),
+ use_cache=True,
)
provisional_wf = ProvisionalWorkflow(provisional_wf_dict)
if provisional_wf.result != "OK":
@@ -112,8 +108,6 @@ class WorkflowGenerator:
}
tool_instance_data_list.append(tool_instance_data)
except Exception as e:
- logger.error(
- f"Error while getting data for {tool_function}: {e}"
- )
+ logger.error(f"Error while getting data for {tool_function}: {e}")
raise ToolInstantiationError(tool_name=tool_function)
return tool_instance_data_list
diff --git a/backend/workflow_manager/workflow/migrations/0001_initial.py b/backend/workflow_manager/workflow/migrations/0001_initial.py
index 9b5e2a4d2..50483fe13 100644
--- a/backend/workflow_manager/workflow/migrations/0001_initial.py
+++ b/backend/workflow_manager/workflow/migrations/0001_initial.py
@@ -267,9 +267,7 @@ class Migration(migrations.Migration):
),
(
"result",
- models.TextField(
- blank=True, db_comment="Result from execution"
- ),
+ models.TextField(blank=True, db_comment="Result from execution"),
),
(
"workflow",
diff --git a/backend/workflow_manager/workflow/models/execution.py b/backend/workflow_manager/workflow/models/execution.py
index ddba5b475..1f70df3b1 100644
--- a/backend/workflow_manager/workflow/models/execution.py
+++ b/backend/workflow_manager/workflow/models/execution.py
@@ -52,18 +52,14 @@ class WorkflowExecution(BaseModel):
execution_log_id = models.CharField(
default="", editable=False, db_comment="Execution log events Id"
)
- status = models.CharField(
- default="", db_comment="Current status of execution"
- )
+ status = models.CharField(default="", db_comment="Current status of execution")
error_message = models.CharField(
max_length=EXECUTION_ERROR_LENGTH,
blank=True,
default="",
db_comment="Details of encountered errors",
)
- attempts = models.IntegerField(
- default=0, db_comment="number of attempts taken"
- )
+ attempts = models.IntegerField(default=0, db_comment="number of attempts taken")
execution_time = models.FloatField(
default=0, db_comment="execution time in seconds"
)
diff --git a/backend/workflow_manager/workflow/models/file_history.py b/backend/workflow_manager/workflow/models/file_history.py
index f518d2ba9..7bc61f480 100644
--- a/backend/workflow_manager/workflow/models/file_history.py
+++ b/backend/workflow_manager/workflow/models/file_history.py
@@ -16,8 +16,7 @@ class FileHistory(BaseModel):
bool: True if the execution status is completed, False otherwise.
"""
return (
- self.status is not None
- and self.status == ExecutionStatus.COMPLETED.value
+ self.status is not None and self.status == ExecutionStatus.COMPLETED.value
)
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
diff --git a/backend/workflow_manager/workflow/models/workflow.py b/backend/workflow_manager/workflow/models/workflow.py
index 3da08344b..94505da55 100644
--- a/backend/workflow_manager/workflow/models/workflow.py
+++ b/backend/workflow_manager/workflow/models/workflow.py
@@ -37,9 +37,7 @@ class Workflow(BaseModel):
)
# TODO: Move prompt fields as a One-One relationship/into Prompt instead
prompt_name = models.CharField(max_length=PROMPT_NAME_LENGTH, default="")
- description = models.TextField(
- max_length=DESCRIPTION_FIELD_LENGTH, default=""
- )
+ description = models.TextField(max_length=DESCRIPTION_FIELD_LENGTH, default="")
workflow_name = models.CharField(max_length=WORKFLOW_NAME_SIZE, unique=True)
prompt_text = models.TextField(default="")
is_active = models.BooleanField(default=False)
diff --git a/backend/workflow_manager/workflow/serializers.py b/backend/workflow_manager/workflow/serializers.py
index b9ba92db7..180884887 100644
--- a/backend/workflow_manager/workflow/serializers.py
+++ b/backend/workflow_manager/workflow/serializers.py
@@ -14,10 +14,7 @@ from rest_framework.serializers import (
from tool_instance.serializers import ToolInstanceSerializer
from tool_instance.tool_instance_helper import ToolInstanceHelper
from workflow_manager.endpoint.models import WorkflowEndpoint
-from workflow_manager.workflow.constants import (
- WorkflowExecutionKey,
- WorkflowKey,
-)
+from workflow_manager.workflow.constants import WorkflowExecutionKey, WorkflowKey
from workflow_manager.workflow.exceptions import WorkflowGenerationError
from workflow_manager.workflow.generator import WorkflowGenerator
from workflow_manager.workflow.models.workflow import Workflow
diff --git a/backend/workflow_manager/workflow/urls.py b/backend/workflow_manager/workflow/urls.py
index 648fe6513..144ad09b7 100644
--- a/backend/workflow_manager/workflow/urls.py
+++ b/backend/workflow_manager/workflow/urls.py
@@ -18,14 +18,10 @@ workflow_detail = WorkflowViewSet.as_view(
}
# fmt: on
)
-workflow_execute = WorkflowViewSet.as_view(
- {"post": "execute", "put": "activate"}
-)
+workflow_execute = WorkflowViewSet.as_view({"post": "execute", "put": "activate"})
execution_entity = WorkflowViewSet.as_view({"get": "get_execution"})
workflow_clear_cache = WorkflowViewSet.as_view({"get": "clear_cache"})
-workflow_clear_file_marker = WorkflowViewSet.as_view(
- {"get": "clear_file_marker"}
-)
+workflow_clear_file_marker = WorkflowViewSet.as_view({"get": "clear_file_marker"})
workflow_schema = WorkflowViewSet.as_view({"get": "get_schema"})
can_update = WorkflowViewSet.as_view({"get": "can_update"})
urlpatterns = format_suffix_patterns(
diff --git a/backend/workflow_manager/workflow/views.py b/backend/workflow_manager/workflow/views.py
index 40ae2eb88..7ce7b2cdc 100644
--- a/backend/workflow_manager/workflow/views.py
+++ b/backend/workflow_manager/workflow/views.py
@@ -46,13 +46,9 @@ from backend.constants import RequestKey
logger = logging.getLogger(__name__)
-def update_pipeline(
- pipeline_guid: Optional[str], status: tuple[str, str]
-) -> Any:
+def update_pipeline(pipeline_guid: Optional[str], status: tuple[str, str]) -> Any:
if pipeline_guid:
- pipeline: Pipeline = PipelineProcessor.fetch_pipeline(
- pipeline_id=pipeline_guid
- )
+ pipeline: Pipeline = PipelineProcessor.fetch_pipeline(pipeline_id=pipeline_guid)
PipelineProcessor.update_pipeline_status(
pipeline=pipeline, is_end=True, status=status
)
@@ -155,13 +151,9 @@ class WorkflowViewSet(viewsets.ModelViewSet):
def get_execution(self, request: Request, pk: str) -> Response:
execution = WorkflowHelper.get_current_execution(pk)
- return Response(
- make_execution_response(execution), status=status.HTTP_200_OK
- )
+ return Response(make_execution_response(execution), status=status.HTTP_200_OK)
- def get_error_from_serializer(
- self, error_details: dict[str, Any]
- ) -> Optional[str]:
+ def get_error_from_serializer(self, error_details: dict[str, Any]) -> Optional[str]:
"""Validation error."""
error_key = next(iter(error_details))
# Get the first error message
@@ -188,9 +180,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
if workflow_id:
workflow = WorkflowHelper.get_workflow_by_id(workflow_id)
elif project_id:
- workflow = WorkflowHelper.get_active_workflow_by_project_id(
- project_id
- )
+ workflow = WorkflowHelper.get_active_workflow_by_project_id(project_id)
else:
raise WorkflowDoesNotExistError()
return workflow
@@ -211,9 +201,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
workflow_id = serializer.get_workflow_id(serializer.validated_data)
project_id = serializer.get_project_id(serializer.validated_data)
execution_id = serializer.get_execution_id(serializer.validated_data)
- execution_action = serializer.get_execution_action(
- serializer.validated_data
- )
+ execution_action = serializer.get_execution_action(serializer.validated_data)
file_objs = request.FILES.getlist("files")
hashes_of_files = {}
if file_objs and execution_id and workflow_id:
@@ -247,10 +235,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
update_pipeline(pipeline_guid, Pipeline.PipelineStatus.FAILURE)
logger.error(f"Error while executing workflow: {exception}")
return Response(
- {
- "error": "Please check the logs for more details: "
- + str(exception)
- },
+ {"error": "Please check the logs for more details: " + str(exception)},
status=status.HTTP_400_BAD_REQUEST,
)
except Exception as exception:
@@ -305,13 +290,9 @@ class WorkflowViewSet(viewsets.ModelViewSet):
return Response(serializer.data, status=status.HTTP_200_OK)
@action(detail=True, methods=["get"])
- def clear_cache(
- self, request: Request, *args: Any, **kwargs: Any
- ) -> Response:
+ def clear_cache(self, request: Request, *args: Any, **kwargs: Any) -> Response:
workflow = self.get_object()
- response: dict[str, Any] = WorkflowHelper.clear_cache(
- workflow_id=workflow.id
- )
+ response: dict[str, Any] = WorkflowHelper.clear_cache(workflow_id=workflow.id)
return Response(response.get("message"), status=response.get("status"))
@action(detail=True, methods=["get"])
@@ -330,9 +311,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
return Response(response.get("message"), status=response.get("status"))
@action(detail=False, methods=["get"])
- def get_schema(
- self, request: Request, *args: Any, **kwargs: Any
- ) -> Response:
+ def get_schema(self, request: Request, *args: Any, **kwargs: Any) -> Response:
"""Retrieves the JSON schema for source/destination type modules for
entities file/API/DB.
@@ -343,9 +322,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):
Response: JSON schema for the request made
"""
schema_type = request.query_params.get("type", SchemaType.SRC.value)
- schema_entity = request.query_params.get(
- "entity", SchemaEntity.FILE.value
- )
+ schema_entity = request.query_params.get("entity", SchemaEntity.FILE.value)
WorkflowSchemaHelper.validate_request(
schema_type=SchemaType(schema_type),
diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.py
index d84aafb29..db8a73d35 100644
--- a/backend/workflow_manager/workflow/workflow_helper.py
+++ b/backend/workflow_manager/workflow/workflow_helper.py
@@ -30,11 +30,7 @@ from workflow_manager.workflow.constants import (
WorkflowMessages,
)
from workflow_manager.workflow.dto import AsyncResultData, ExecutionResponse
-from workflow_manager.workflow.enums import (
- ExecutionStatus,
- SchemaEntity,
- SchemaType,
-)
+from workflow_manager.workflow.enums import ExecutionStatus, SchemaEntity, SchemaType
from workflow_manager.workflow.exceptions import (
InvalidRequest,
TaskDoesNotExistError,
@@ -221,9 +217,7 @@ class WorkflowHelper:
)
)
- WorkflowHelper.validate_tool_instances_meta(
- tool_instances=tool_instances
- )
+ WorkflowHelper.validate_tool_instances_meta(tool_instances=tool_instances)
execution_mode = execution_mode or WorkflowExecution.Mode.INSTANT
execution_service = WorkflowHelper.build_workflow_execution_service(
organization_id=organization_id,
@@ -242,9 +236,7 @@ class WorkflowHelper:
execution_id=execution_id,
execution_service=execution_service,
)
- destination = DestinationConnector(
- workflow=workflow, execution_id=execution_id
- )
+ destination = DestinationConnector(workflow=workflow, execution_id=execution_id)
# Validating endpoints
source.validate()
destination.validate()
@@ -422,9 +414,7 @@ class WorkflowHelper:
)
except IntegrityError:
# Use existing instance on retry attempt
- workflow_execution = WorkflowExecution.objects.get(
- pk=execution_id
- )
+ workflow_execution = WorkflowExecution.objects.get(pk=execution_id)
WorkflowExecutionServiceHelper.update_execution_task(
execution_id=execution_id, task_id=task_id
)
@@ -462,13 +452,11 @@ class WorkflowHelper:
if pipeline_id:
# pipeline scheduled execution
execution_id = str(uuid.uuid4())
- response: ExecutionResponse = (
- WorkflowHelper.execute_workflow_async(
- workflow_id=workflow.id,
- pipeline_id=pipeline_id,
- execution_id=execution_id,
- hash_values_of_files=hash_values_of_files,
- )
+ response: ExecutionResponse = WorkflowHelper.execute_workflow_async(
+ workflow_id=workflow.id,
+ pipeline_id=pipeline_id,
+ execution_id=execution_id,
+ hash_values_of_files=hash_values_of_files,
)
return response
return WorkflowHelper.run_workflow(workflow=workflow)
@@ -482,8 +470,7 @@ class WorkflowHelper:
workflow_execution = WorkflowExecution.objects.get(pk=execution_id)
if (
workflow_execution.status != ExecutionStatus.PENDING.value
- or workflow_execution.execution_type
- != WorkflowExecution.Type.COMPLETE
+ or workflow_execution.execution_type != WorkflowExecution.Type.COMPLETE
):
raise InvalidRequest(WorkflowErrors.INVALID_EXECUTION_ID)
return WorkflowHelper.run_workflow(
@@ -518,18 +505,13 @@ class WorkflowHelper:
execution_id: Optional[str] = None,
hash_values_of_files: dict[str, str] = {},
) -> ExecutionResponse:
- if (
- execution_action
- is Workflow.ExecutionAction.START.value # type: ignore
- ):
+ if execution_action is Workflow.ExecutionAction.START.value: # type: ignore
if execution_id is None:
return WorkflowHelper.create_and_make_execution_response(
workflow_id=workflow.id, single_step=True
)
try:
- workflow_execution = WorkflowExecution.objects.get(
- pk=execution_id
- )
+ workflow_execution = WorkflowExecution.objects.get(pk=execution_id)
return WorkflowHelper.run_workflow(
workflow=workflow,
single_step=True,
@@ -545,17 +527,12 @@ class WorkflowHelper:
if execution_id is None:
raise InvalidRequest("execution_id is missed")
try:
- workflow_execution = WorkflowExecution.objects.get(
- pk=execution_id
- )
+ workflow_execution = WorkflowExecution.objects.get(pk=execution_id)
except WorkflowExecution.DoesNotExist:
- raise WorkflowExecutionNotExist(
- WorkflowErrors.INVALID_EXECUTION_ID
- )
+ raise WorkflowExecutionNotExist(WorkflowErrors.INVALID_EXECUTION_ID)
if (
workflow_execution.status != ExecutionStatus.PENDING.value
- or workflow_execution.execution_type
- != WorkflowExecution.Type.STEP
+ or workflow_execution.execution_type != WorkflowExecution.Type.STEP
):
raise InvalidRequest(WorkflowErrors.INVALID_EXECUTION_ID)
current_action: Optional[str] = CacheService.get_key(execution_id)
@@ -581,13 +558,11 @@ class WorkflowHelper:
single_step: bool = False,
mode: tuple[str, str] = WorkflowExecution.Mode.INSTANT,
) -> ExecutionResponse:
- workflow_execution = (
- WorkflowExecutionServiceHelper.create_workflow_execution(
- workflow_id=workflow_id,
- single_step=single_step,
- pipeline_id=pipeline_id,
- mode=mode,
- )
+ workflow_execution = WorkflowExecutionServiceHelper.create_workflow_execution(
+ workflow_id=workflow_id,
+ single_step=single_step,
+ pipeline_id=pipeline_id,
+ mode=mode,
)
return ExecutionResponse(
workflow_execution.workflow_id,
@@ -667,9 +642,7 @@ class WorkflowHelper:
raise WorkflowDoesNotExistError()
used_count = Pipeline.objects.filter(workflow=workflow).count()
if used_count == 0:
- used_count = APIDeployment.objects.filter(
- workflow=workflow
- ).count()
+ used_count = APIDeployment.objects.filter(workflow=workflow).count()
return {"can_update": used_count == 0}
except Workflow.DoesNotExist:
logger.error(f"Error getting workflow: {id}")
@@ -680,9 +653,7 @@ class WorkflowSchemaHelper:
"""Helper class for workflow schema related methods."""
@staticmethod
- def validate_request(
- schema_type: SchemaType, schema_entity: SchemaEntity
- ) -> bool:
+ def validate_request(schema_type: SchemaType, schema_entity: SchemaEntity) -> bool:
"""Validates the given args for reading the JSON schema.
Schema type of `src`, allows entities `file` and `api`
@@ -714,9 +685,7 @@ class WorkflowSchemaHelper:
f"should be one of {possible_entities}"
)
- if (
- schema_type == SchemaType.SRC and schema_entity == SchemaEntity.DB
- ) or (
+ if (schema_type == SchemaType.SRC and schema_entity == SchemaEntity.DB) or (
schema_type == SchemaType.DEST and schema_entity != SchemaEntity.DB
):
raise serializers.ValidationError(
@@ -743,8 +712,7 @@ class WorkflowSchemaHelper:
dict[str, Any]: JSON schema for the requested entity
"""
schema_path = (
- f"{os.path.dirname(__file__)}/static/"
- f"{schema_type}/{schema_entity}.json"
+ f"{os.path.dirname(__file__)}/static/" f"{schema_type}/{schema_entity}.json"
)
with open(schema_path, encoding="utf-8") as file:
schema = json.load(file)
diff --git a/docker/scripts/merge_env.py b/docker/scripts/merge_env.py
index 9d736ae49..3099da501 100644
--- a/docker/scripts/merge_env.py
+++ b/docker/scripts/merge_env.py
@@ -49,9 +49,7 @@ def _extract_from_env_file(file_path: str) -> dict[str, str]:
return env
-def _merge_to_env_file(
- base_env_file_path: str, target_env: dict[str, str] = {}
-) -> str:
+def _merge_to_env_file(base_env_file_path: str, target_env: dict[str, str] = {}) -> str:
"""Generates file contents after merging input base env file path with
target env.
@@ -118,12 +116,8 @@ def merge_env(
base_env_file_path: str, target_env_file_path: str, dry_run: bool = False
) -> None:
target_env = _extract_from_env_file(target_env_file_path)
- merged_contents = _merge_to_env_file(
- base_env_file_path, target_env=target_env
- )
- _save_merged_contents(
- target_env_file_path, merged_contents, dry_run=dry_run
- )
+ merged_contents = _merge_to_env_file(base_env_file_path, target_env=target_env)
+ _save_merged_contents(target_env_file_path, merged_contents, dry_run=dry_run)
if __name__ == "__main__":
diff --git a/document-service/src/unstract/document_service/main.py b/document-service/src/unstract/document_service/main.py
index fe88c6093..731a419b9 100644
--- a/document-service/src/unstract/document_service/main.py
+++ b/document-service/src/unstract/document_service/main.py
@@ -16,13 +16,9 @@ logging.basicConfig(
)
UPLOAD_FOLDER = os.environ.get("UPLOAD_FOLDER", "/tmp/document_service/upload")
-PROCESS_FOLDER = os.environ.get(
- "PROCESS_FOLDER", "/tmp/document_service/process"
-)
+PROCESS_FOLDER = os.environ.get("PROCESS_FOLDER", "/tmp/document_service/process")
LIBREOFFICE_PYTHON = os.environ.get("LIBREOFFICE_PYTHON", "/usr/bin/python3")
-MAX_FILE_SIZE = int(
- os.environ.get("MAX_FILE_SIZE", 10485760)
-) # 10 * 1024 * 1024
+MAX_FILE_SIZE = int(os.environ.get("MAX_FILE_SIZE", 10485760)) # 10 * 1024 * 1024
SERVICE_API_TOKEN = os.environ.get("SERVICE_API_TOKEN", "")
app = Flask("document_service")
@@ -104,9 +100,7 @@ def upload_file():
redis_host = os.environ.get("REDIS_HOST")
redis_port = int(os.environ.get("REDIS_PORT"))
redis_password = os.environ.get("REDIS_PASSWORD")
- r = redis.Redis(
- host=redis_host, port=redis_port, password=redis_password
- )
+ r = redis.Redis(host=redis_host, port=redis_port, password=redis_password)
# TODO: Create a file reaper process to look at uploaded time and delete
redis_key = f"upload_time:{account_id}_{file_name}"
current_timestamp = int(time.time())
@@ -130,9 +124,7 @@ def find_and_replace():
output_format = request.args.get("output_format").lower()
find_and_replace_text = request.json
- app.logger.info(
- f"Find and replace for file {file_name} for account {account_id}"
- )
+ app.logger.info(f"Find and replace for file {file_name} for account {account_id}")
app.logger.info(f"Output format: {output_format}")
if output_format not in ["pdf"]:
@@ -152,9 +144,7 @@ def find_and_replace():
try:
command = f"{LIBREOFFICE_PYTHON} -m unoserver.converter --convert-to odt \
--filter writer8 {file_namex} {file_name_odt}"
- result = subprocess.run(
- command, shell=True, capture_output=True, text=True
- )
+ result = subprocess.run(command, shell=True, capture_output=True, text=True)
app.logger.info(result)
if result.returncode != 0:
app.logger.error(
@@ -166,9 +156,7 @@ def find_and_replace():
app.logger.info(
f"File converted to ODT format successfully! {file_name_odt}"
)
- app.logger.info(
- f"ODT convertion result: {result.stdout} | {result.stderr}"
- )
+ app.logger.info(f"ODT convertion result: {result.stdout} | {result.stderr}")
except Exception as e:
app.logger.error(f"Error while converting file to ODT format: {e}")
return "Error while converting file to ODT format!", 500
@@ -182,13 +170,9 @@ def find_and_replace():
replace_str = find_and_replace_text[find_str]
for element in doc.getElementsByType(text.Span):
if find_str in teletype.extractText(element):
- app.logger.info(
- f"Found {find_str} in {teletype.extractText(element)}"
- )
+ app.logger.info(f"Found {find_str} in {teletype.extractText(element)}")
new_element = text.Span()
- new_element.setAttribute(
- "stylename", element.getAttribute("stylename")
- )
+ new_element.setAttribute("stylename", element.getAttribute("stylename"))
t = teletype.extractText(element)
t = t.replace(find_str, replace_str)
new_element.addText(t)
@@ -205,9 +189,7 @@ def find_and_replace():
f"{LIBREOFFICE_PYTHON} -m unoserver.converter --convert-to pdf "
f"--filter writer_pdf_Export {file_name_odt} {file_name_output}"
)
- result = subprocess.run(
- command, shell=True, capture_output=True, text=True
- )
+ result = subprocess.run(command, shell=True, capture_output=True, text=True)
if result.returncode != 0:
app.logger.error(
f"Failed to convert file to {output_format} format: "
@@ -219,13 +201,9 @@ def find_and_replace():
f"File converted to {output_format} format successfully! "
f"{file_name_output}"
)
- app.logger.info(
- f"ODT convertion result: {result.stdout} | {result.stderr}"
- )
+ app.logger.info(f"ODT convertion result: {result.stdout} | {result.stderr}")
except Exception as e:
- app.logger.error(
- f"Error while converting file to {output_format} format: {e}"
- )
+ app.logger.error(f"Error while converting file to {output_format} format: {e}")
return f"Error while converting file to {output_format} format!", 500
return send_file(file_name_output, as_attachment=True)
diff --git a/platform-service/src/unstract/platform_service/exceptions.py b/platform-service/src/unstract/platform_service/exceptions.py
index fd310e0a9..c9a0c89c7 100644
--- a/platform-service/src/unstract/platform_service/exceptions.py
+++ b/platform-service/src/unstract/platform_service/exceptions.py
@@ -1,7 +1,5 @@
class CustomException(Exception):
- def __init__(
- self, message: str = "An error occurred", code: int = 500
- ) -> None:
+ def __init__(self, message: str = "An error occurred", code: int = 500) -> None:
self.message = message
self.code = code
super().__init__(self.message)
diff --git a/platform-service/src/unstract/platform_service/main.py b/platform-service/src/unstract/platform_service/main.py
index fb3d2c5fd..702ca4238 100644
--- a/platform-service/src/unstract/platform_service/main.py
+++ b/platform-service/src/unstract/platform_service/main.py
@@ -106,14 +106,10 @@ def authentication_middleware(func: Any) -> Any:
def get_account_from_bearer_token(token: Optional[str]) -> str:
- query = (
- "SELECT organization_id FROM account_platformkey "
- f"WHERE key='{token}'"
- )
+ query = "SELECT organization_id FROM account_platformkey " f"WHERE key='{token}'"
organization = execute_query(query)
query_org = (
- "SELECT schema_name FROM account_organization "
- f"WHERE id='{organization}'"
+ "SELECT schema_name FROM account_organization " f"WHERE id='{organization}'"
)
schema_name: str = execute_query(query_org)
return schema_name
@@ -139,9 +135,7 @@ def validate_bearer_token(token: Optional[str]) -> bool:
result_row = cursor.fetchone()
cursor.close()
if not result_row or len(result_row) == 0:
- app.logger.error(
- f"Authentication failed. bearer token not found {token}"
- )
+ app.logger.error(f"Authentication failed. bearer token not found {token}")
return False
platform_key = str(result_row[1])
is_active = bool(result_row[2])
@@ -151,9 +145,7 @@ def validate_bearer_token(token: Optional[str]) -> bool:
)
return False
if platform_key != token:
- app.logger.error(
- f"Authentication failed. Invalid bearer token: {token}"
- )
+ app.logger.error(f"Authentication failed. Invalid bearer token: {token}")
return False
except Exception as e:
@@ -224,9 +216,7 @@ def usage() -> Any:
usage_value=usage_value,
usage_units=usage_units,
)
- app.logger.info(
- "Entry created with id %s for %s", usage.id, organization_id
- )
+ app.logger.info("Entry created with id %s for %s", usage.id, organization_id)
result["status"] = "OK"
result["unique_id"] = usage.id
return result
@@ -368,20 +358,16 @@ def adapter_instance() -> Any:
adapter_instance_id = request.args.get("adapter_instance_id")
try:
- data_dict = (
- AdapterInstanceRequestHelper.get_adapter_instance_from_db(
- db_instance=be_db,
- organization_id=organization_id,
- adapter_instance_id=adapter_instance_id,
- )
+ data_dict = AdapterInstanceRequestHelper.get_adapter_instance_from_db(
+ db_instance=be_db,
+ organization_id=organization_id,
+ adapter_instance_id=adapter_instance_id,
)
f: Fernet = Fernet(ENCRYPTION_KEY.encode("utf-8"))
data_dict["adapter_metadata"] = json.loads(
- f.decrypt(
- bytes(data_dict.pop("adapter_metadata_b")).decode("utf-8")
- )
+ f.decrypt(bytes(data_dict.pop("adapter_metadata_b")).decode("utf-8"))
)
return jsonify(data_dict)
diff --git a/platform-service/src/unstract/platform_service/utils.py b/platform-service/src/unstract/platform_service/utils.py
index 04fddd4b9..aa57b46bd 100644
--- a/platform-service/src/unstract/platform_service/utils.py
+++ b/platform-service/src/unstract/platform_service/utils.py
@@ -35,8 +35,7 @@ class EnvManager:
ValueError: Error mentioning envs which are not configured.
"""
if cls.missing_settings:
- ERROR_MESSAGE = (
- "Below required settings are missing.\n"
- + ",\n".join(cls.missing_settings)
+ ERROR_MESSAGE = "Below required settings are missing.\n" + ",\n".join(
+ cls.missing_settings
)
raise ValueError(ERROR_MESSAGE)
diff --git a/prompt-service/src/unstract/prompt_service/authentication_middleware.py b/prompt-service/src/unstract/prompt_service/authentication_middleware.py
index 3c91c4c16..0ad5c6aaa 100644
--- a/prompt-service/src/unstract/prompt_service/authentication_middleware.py
+++ b/prompt-service/src/unstract/prompt_service/authentication_middleware.py
@@ -11,9 +11,7 @@ class AuthenticationMiddleware:
def validate_bearer_token(cls, token: Optional[str]) -> bool:
try:
if token is None:
- current_app.logger.error(
- "Authentication failed. Empty bearer token"
- )
+ current_app.logger.error("Authentication failed. Empty bearer token")
return False
query = f"SELECT * FROM account_platformkey WHERE key = '{token}'"
@@ -40,9 +38,7 @@ class AuthenticationMiddleware:
return False
except Exception as e:
- current_app.logger.error(
- f"Error while validating bearer token: {e}"
- )
+ current_app.logger.error(f"Error while validating bearer token: {e}")
return False
return True
@@ -61,13 +57,11 @@ class AuthenticationMiddleware:
@classmethod
def get_account_from_bearer_token(cls, token: Optional[str]) -> str:
query = (
- "SELECT organization_id FROM account_platformkey "
- f"WHERE key='{token}'"
+ "SELECT organization_id FROM account_platformkey " f"WHERE key='{token}'"
)
organization = AuthenticationMiddleware.execute_query(query)
query_org = (
- "SELECT schema_name FROM account_organization "
- f"WHERE id='{organization}'"
+ "SELECT schema_name FROM account_organization " f"WHERE id='{organization}'"
)
schema_name: str = AuthenticationMiddleware.execute_query(query_org)
return schema_name
diff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.py
index ab96860f8..0ce9da214 100644
--- a/prompt-service/src/unstract/prompt_service/helper.py
+++ b/prompt-service/src/unstract/prompt_service/helper.py
@@ -63,9 +63,7 @@ def plugin_loader(app: Flask) -> dict[str, dict[str, Any]]:
metadata: dict[str, Any] = module.metadata
if metadata.get("disable", False):
- app.logger.info(
- f"Ignore disabled plugin: {pkg} v{metadata['version']}"
- )
+ app.logger.info(f"Ignore disabled plugin: {pkg} v{metadata['version']}")
continue
try:
diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.py
index 9c6329128..130a0b5f8 100644
--- a/prompt-service/src/unstract/prompt_service/main.py
+++ b/prompt-service/src/unstract/prompt_service/main.py
@@ -7,9 +7,7 @@ from flask import Flask, json, request
from llama_index.core import VectorStoreIndex
from llama_index.core.llms import LLM
from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters
-from unstract.prompt_service.authentication_middleware import (
- AuthenticationMiddleware,
-)
+from unstract.prompt_service.authentication_middleware import AuthenticationMiddleware
from unstract.prompt_service.constants import PromptServiceContants as PSKeys
from unstract.prompt_service.constants import RunLevel
from unstract.prompt_service.helper import EnvLoader, plugin_loader
@@ -18,9 +16,7 @@ from unstract.sdk.constants import LogLevel
from unstract.sdk.embedding import ToolEmbedding
from unstract.sdk.index import ToolIndex
from unstract.sdk.llm import ToolLLM
-from unstract.sdk.utils.callback_manager import (
- CallbackManager as UNCallbackManager,
-)
+from unstract.sdk.utils.callback_manager import CallbackManager as UNCallbackManager
from unstract.sdk.vector_db import ToolVectorDB
from werkzeug.exceptions import HTTPException
@@ -148,9 +144,7 @@ def authentication_middleware(func: Any) -> Any:
def wrapper(*args: Any, **kwargs: Any) -> Any:
token = AuthenticationMiddleware.get_token_from_auth_header(request)
# Check if bearer token exists and validate it
- if not token or not AuthenticationMiddleware.validate_bearer_token(
- token
- ):
+ if not token or not AuthenticationMiddleware.validate_bearer_token(token):
return "Unauthorized", 401
return func(*args, **kwargs)
@@ -195,9 +189,7 @@ def prompt_processor() -> Any:
name = output[PSKeys.NAME]
promptx = output[PSKeys.PROMPT]
chunk_size = output[PSKeys.CHUNK_SIZE]
- util = PromptServiceBaseTool(
- log_level=LogLevel.INFO, platform_key=platform_key
- )
+ util = PromptServiceBaseTool(log_level=LogLevel.INFO, platform_key=platform_key)
tool_index = ToolIndex(tool=util)
if active is False:
@@ -347,15 +339,9 @@ def prompt_processor() -> Any:
)
assertion_failed = True
answer = ""
- if (
- output[PSKeys.ASSERTION_FAILURE_PROMPT]
- .lower()
- .startswith("@assign")
- ):
+ if output[PSKeys.ASSERTION_FAILURE_PROMPT].lower().startswith("@assign"):
answer = "NA"
- first_space_index = output[
- PSKeys.ASSERTION_FAILURE_PROMPT
- ].find(" ")
+ first_space_index = output[PSKeys.ASSERTION_FAILURE_PROMPT].find(" ")
if first_space_index > 0:
answer = output[PSKeys.ASSERTION_FAILURE_PROMPT][
first_space_index + 1
@@ -483,11 +469,7 @@ def prompt_processor() -> Any:
else:
structured_output[output[PSKeys.NAME]] = False
elif output[PSKeys.TYPE] == PSKeys.JSON:
- if (
- assertion_failed
- or answer.lower() == "[]"
- or answer.lower() == "na"
- ):
+ if assertion_failed or answer.lower() == "[]" or answer.lower() == "na":
structured_output[output[PSKeys.NAME]] = None
else:
# Remove any markdown code blocks
@@ -500,9 +482,7 @@ def prompt_processor() -> Any:
try:
structured_output[output[PSKeys.NAME]] = json.loads(answer)
except Exception as e:
- app.logger.info(
- f"JSON format error : {answer}", LogLevel.ERROR
- )
+ app.logger.info(f"JSON format error : {answer}", LogLevel.ERROR)
app.logger.info(
f"Error parsing response (to json): {e}", LogLevel.ERROR
)
@@ -676,9 +656,7 @@ def simple_retriver( # type:ignore
if node.score > 0.6:
text += node.get_content() + "\n"
else:
- app.logger.info(
- "Node score is less than 0.6. " f"Ignored: {node.score}"
- )
+ app.logger.info("Node score is less than 0.6. " f"Ignored: {node.score}")
answer, usage = construct_and_run_prompt( # type:ignore
output,
@@ -720,9 +698,7 @@ def run_completion(
platform_api_key = llm_helper.tool.get_env_or_die(
PSKeys.PLATFORM_SERVICE_API_KEY
)
- completion = llm_helper.run_completion(
- llm_li, platform_api_key, prompt, 3
- )
+ completion = llm_helper.run_completion(llm_li, platform_api_key, prompt, 3)
answer: str = completion[PSKeys.RESPONSE].text
usage = {}
@@ -749,8 +725,7 @@ def extract_variable(
)
else:
raise ValueError(
- f"Variable {variable_name} not found "
- "in structured output"
+ f"Variable {variable_name} not found " "in structured output"
)
if promptx != output[PSKeys.PROMPT]:
diff --git a/pyproject.toml b/pyproject.toml
index e693e6115..4241a031e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -51,14 +51,14 @@ hook-check-django-migrations = [
]
[tool.black]
-line-length = 80
+line-length = 88
target-version = ['py39']
[tool.pycln]
all = true
[tool.isort]
-line_length = 80
+line_length = 88
multi_line_output = 3
include_trailing_comma = true
force_grid_wrap = 0
diff --git a/tools/classifier/src/helper.py b/tools/classifier/src/helper.py
index 8870c5e25..a57e4c2a5 100644
--- a/tools/classifier/src/helper.py
+++ b/tools/classifier/src/helper.py
@@ -30,9 +30,7 @@ class ClassifierHelper:
return self._extract_from_adapter(file, text_extraction_adapter_id)
- def _extract_from_adapter(
- self, file: str, adapter_id: str
- ) -> Optional[str]:
+ def _extract_from_adapter(self, file: str, adapter_id: str) -> Optional[str]:
"""Extract text from adapter.
Args:
@@ -42,15 +40,12 @@ class ClassifierHelper:
str: page content
"""
self.tool.stream_log(
- "Creating text extraction adapter "
- f"using adapter_id: {adapter_id}"
+ "Creating text extraction adapter " f"using adapter_id: {adapter_id}"
)
tool_extraction = X2Text(tool=self.tool)
adapter = tool_extraction.get_x2text(adapter_id)
- self.tool.stream_log(
- "Text extraction adapter has been created successfully."
- )
+ self.tool.stream_log("Text extraction adapter has been created successfully.")
self.tool.stream_log("Adapter created")
try:
@@ -112,9 +107,7 @@ class ClassifierHelper:
classification = self.get_result_from_cache(cache_key=cache_key)
if classification is None:
- self.tool.stream_log(
- "No classification found in cache, calling LLM."
- )
+ self.tool.stream_log("No classification found in cache, calling LLM.")
classification = self.call_llm(prompt=prompt, llm=llm)
if not classification:
classification = "unknown"
diff --git a/tools/classifier/src/main.py b/tools/classifier/src/main.py
index 6b1e3df2c..17d13d4fb 100644
--- a/tools/classifier/src/main.py
+++ b/tools/classifier/src/main.py
@@ -14,9 +14,7 @@ from unstract.sdk.constants import (
from unstract.sdk.llm import ToolLLM
from unstract.sdk.tool.base import BaseTool
from unstract.sdk.tool.entrypoint import ToolEntrypoint
-from unstract.sdk.utils.callback_manager import (
- CallbackManager as UNCallbackManager,
-)
+from unstract.sdk.utils.callback_manager import CallbackManager as UNCallbackManager
class UnstractClassifier(BaseTool):
@@ -33,9 +31,7 @@ class UnstractClassifier(BaseTool):
elif len(bins) < 2:
self.stream_error_and_exit("At least two bins are required")
if not llm_adapter_instance_id:
- self.stream_error_and_exit(
- "Choose an LLM to process the classifier"
- )
+ self.stream_error_and_exit("Choose an LLM to process the classifier")
if not text_extraction_adapter_id:
self.stream_error_and_exit("Choose an LLM to extract the documents")
@@ -95,12 +91,8 @@ class UnstractClassifier(BaseTool):
)
max_tokens = tool_llm.get_max_tokens(reserved_for_output=50 + 1000)
max_bytes = int(max_tokens * 1.3)
- self.stream_log(
- f"LLM Max tokens: {max_tokens} ==> Max bytes: {max_bytes}"
- )
- self.stream_log(
- f"LLM Max tokens: {max_tokens} ==> Max bytes: {max_bytes}"
- )
+ self.stream_log(f"LLM Max tokens: {max_tokens} ==> Max bytes: {max_bytes}")
+ self.stream_log(f"LLM Max tokens: {max_tokens} ==> Max bytes: {max_bytes}")
limited_text = ""
for byte in text.encode():
if len(limited_text.encode()) < max_bytes:
diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py
index 4eabd063c..770eca2c1 100644
--- a/tools/structure/src/main.py
+++ b/tools/structure/src/main.py
@@ -34,18 +34,12 @@ class StructureTool(BaseTool):
tool=self, prompt_registry_id=prompt_registry_id
)
tool_metadata = exported_tool[SettingsKeys.TOOL_METADATA]
- self.stream_log(
- f"Tool Metadata retrived succesfully: {tool_metadata}"
- )
+ self.stream_log(f"Tool Metadata retrived succesfully: {tool_metadata}")
except Exception as e:
- self.stream_error_and_exit(
- f"Error loading structure definition: {e}"
- )
+ self.stream_error_and_exit(f"Error loading structure definition: {e}")
# Update GUI
- input_log = (
- f"### Structure Definition:\n```json\n{tool_metadata}\n```\n\n"
- )
+ input_log = f"### Structure Definition:\n```json\n{tool_metadata}\n```\n\n"
output_log = "### Indexing..."
self.stream_update(input_log, state=LogState.INPUT_UPDATE)
self.stream_update(output_log, state=LogState.OUTPUT_UPDATE)
@@ -94,9 +88,7 @@ class StructureTool(BaseTool):
f"### Structure Definition:\n"
f"```json\n{json.dumps(tool_metadata, indent=2)}\n```\n\n"
)
- output_log = (
- f"### Parsed output:\n```json\n{structured_output}\n```\n\n"
- )
+ output_log = f"### Parsed output:\n```json\n{structured_output}\n```\n\n"
self.stream_update(input_log, state=LogState.INPUT_UPDATE)
self.stream_update(output_log, state=LogState.OUTPUT_UPDATE)
diff --git a/tools/text_extractor/src/main.py b/tools/text_extractor/src/main.py
index 390da2a04..41ec9cfc8 100644
--- a/tools/text_extractor/src/main.py
+++ b/tools/text_extractor/src/main.py
@@ -58,20 +58,14 @@ class TextExtractor(BaseTool):
text_extraction_adapter = tool_extraction.get_x2text(
adapter_instance_id=text_extraction_adapter_id
)
- self.stream_log(
- "Text extraction adapter has been created successfully."
- )
- extracted_text = text_extraction_adapter.process(
- input_file_path=input_file
- )
+ self.stream_log("Text extraction adapter has been created successfully.")
+ extracted_text = text_extraction_adapter.process(input_file_path=input_file)
extracted_text = self.convert_to_actual_string(extracted_text)
self.stream_log("Text has been extracted successfully.")
first_5_lines = "\n\n".join(extracted_text.split("\n")[:5])
- output_log = (
- f"### Text\n\n```text\n{first_5_lines}\n```\n\n...(truncated)"
- )
+ output_log = f"### Text\n\n```text\n{first_5_lines}\n```\n\n...(truncated)"
self.stream_update(output_log, state=LogState.OUTPUT_UPDATE)
try:
@@ -87,9 +81,7 @@ class TextExtractor(BaseTool):
"Error creating/writing output file: source_name not found"
)
except Exception as e:
- self.stream_error_and_exit(
- f"Error creating/writing output file: {e}"
- )
+ self.stream_error_and_exit(f"Error creating/writing output file: {e}")
self.write_tool_result(data=extracted_text)
def convert_to_actual_string(self, text: Any) -> str:
diff --git a/unstract/connectors/src/unstract/connectors/connectorkit.py b/unstract/connectors/src/unstract/connectors/connectorkit.py
index e53ad43e8..1ef216491 100644
--- a/unstract/connectors/src/unstract/connectors/connectorkit.py
+++ b/unstract/connectors/src/unstract/connectors/connectorkit.py
@@ -40,23 +40,15 @@ class Connectorkit:
connector_registry_metadata,
) in self._connectors.items():
if (
- connector_registry_metadata[Common.METADATA][
- Common.CONNECTOR
- ].__name__
+ connector_registry_metadata[Common.METADATA][Common.CONNECTOR].__name__
is connector_name
):
- connector_class: UnstractConnector = (
- connector_registry_metadata[Common.METADATA][
- Common.CONNECTOR
- ]
- )
+ connector_class: UnstractConnector = connector_registry_metadata[
+ Common.METADATA
+ ][Common.CONNECTOR]
return connector_class
- logging.error(
- f">> Connector '{connector_name}' not found in connectorkit"
- )
- logging.error(
- f">> Connectors in connectorkit : {self._connectors.keys()}"
- )
+ logging.error(f">> Connector '{connector_name}' not found in connectorkit")
+ logging.error(f">> Connectors in connectorkit : {self._connectors.keys()}")
return None
def get_connector_by_id(
@@ -73,8 +65,8 @@ class Connectorkit:
Returns:
UnstractConnector: Concrete impl of the `UnstractConnector` base
"""
- connector_class: UnstractConnector = (
- self.get_connector_class_by_connector_id(connector_id)
+ connector_class: UnstractConnector = self.get_connector_class_by_connector_id(
+ connector_id
)
return connector_class(*args, **kwargs)
diff --git a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py
index 5661f32ff..2cd8b75f6 100644
--- a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py
+++ b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py
@@ -13,9 +13,7 @@ from unstract.connectors.exceptions import ConnectorError
class BigQuery(UnstractDB):
def __init__(self, settings: dict[str, Any]):
super().__init__("BigQuery")
- self.json_credentials = json.loads(
- settings.get("json_credentials", "{}")
- )
+ self.json_credentials = json.loads(settings.get("json_credentials", "{}"))
@staticmethod
def get_id() -> str:
diff --git a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py
index 7b2ddf051..6025f5ee5 100644
--- a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py
+++ b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py
@@ -21,11 +21,7 @@ class PostgreSQL(UnstractDB):
if not self.schema:
self.schema = "public"
if not self.connection_url and not (
- self.user
- and self.password
- and self.host
- and self.port
- and self.database
+ self.user and self.password and self.host and self.port and self.database
):
raise ValueError(
"Either ConnectionURL or connection parameters must be provided."
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py
index 91389f34f..d83426e2c 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py
@@ -5,9 +5,7 @@ from typing import Any
from adlfs import AzureBlobFileSystem
from unstract.connectors.exceptions import ConnectorError
-from unstract.connectors.filesystems.unstract_file_system import (
- UnstractFileSystem,
-)
+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
logger = logging.getLogger(__name__)
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/box/box.py b/unstract/connectors/src/unstract/connectors/filesystems/box/box.py
index 7229413c1..e30cd264d 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/box/box.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/box/box.py
@@ -9,9 +9,7 @@ from boxsdk import JWTAuth
from boxsdk.exception import BoxOAuthException
from unstract.connectors.exceptions import ConnectorError
-from unstract.connectors.filesystems.unstract_file_system import (
- UnstractFileSystem,
-)
+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
logger = logging.getLogger(__name__)
logging.getLogger("boxsdk").setLevel(logging.ERROR)
@@ -80,9 +78,7 @@ class BoxFS(UnstractFileSystem):
@staticmethod
def get_description() -> str:
- return (
- "Fetch and store data to and from the Box content management system"
- )
+ return "Fetch and store data to and from the Box content management system"
@staticmethod
def get_icon() -> str:
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py
index ba1f32fbb..915cb2b2c 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py
@@ -6,9 +6,7 @@ from typing import Any
from gcsfs import GCSFileSystem
from unstract.connectors.exceptions import ConnectorError
-from unstract.connectors.filesystems.unstract_file_system import (
- UnstractFileSystem,
-)
+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
logger = logging.getLogger(__name__)
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py
index 7fe9f1e62..1a2241b78 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py
@@ -8,12 +8,8 @@ from pydrive2.auth import GoogleAuth
from pydrive2.fs import GDriveFileSystem
from unstract.connectors.exceptions import ConnectorError
-from unstract.connectors.filesystems.google_drive.constants import (
- GDriveConstants,
-)
-from unstract.connectors.filesystems.unstract_file_system import (
- UnstractFileSystem,
-)
+from unstract.connectors.filesystems.google_drive.constants import GDriveConstants
+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
from unstract.connectors.gcs_helper import GCSHelper
logger = logging.getLogger(__name__)
@@ -39,9 +35,7 @@ class GoogleDriveFS(UnstractFileSystem):
"invalid": False,
"access_token": settings["access_token"],
"refresh_token": settings["refresh_token"],
- GDriveConstants.TOKEN_EXPIRY: settings[
- GDriveConstants.TOKEN_EXPIRY
- ],
+ GDriveConstants.TOKEN_EXPIRY: settings[GDriveConstants.TOKEN_EXPIRY],
}
gauth = GoogleAuth(
settings_file=f"{os.path.dirname(__file__)}/static/settings.yaml",
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py b/unstract/connectors/src/unstract/connectors/filesystems/http/http.py
index 9169086a3..c12236e15 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/http/http.py
@@ -6,9 +6,7 @@ import aiohttp
from fsspec.implementations.http import HTTPFileSystem
from unstract.connectors.exceptions import ConnectorError
-from unstract.connectors.filesystems.unstract_file_system import (
- UnstractFileSystem,
-)
+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
logger = logging.getLogger(__name__)
@@ -25,9 +23,7 @@ class HttpFS(UnstractFileSystem):
"base_url": settings["base_url"],
}
if all(settings.get(key) for key in ("username", "password")):
- basic_auth = aiohttp.BasicAuth(
- settings["username"], settings["password"]
- )
+ basic_auth = aiohttp.BasicAuth(settings["username"], settings["password"])
client_kwargs.update({"auth": basic_auth})
self.http_fs = HTTPFileSystem(client_kwargs=client_kwargs)
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py
index b51ab2f95..ec179e990 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py
@@ -4,9 +4,7 @@ from typing import Any, Optional
from fsspec.implementations.local import LocalFileSystem
-from unstract.connectors.filesystems.unstract_file_system import (
- UnstractFileSystem,
-)
+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
logger = logging.getLogger(__name__)
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
index f22bf4c24..f0cf798b1 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
@@ -5,9 +5,7 @@ from typing import Any
from s3fs.core import S3FileSystem
from unstract.connectors.exceptions import ConnectorError
-from unstract.connectors.filesystems.unstract_file_system import (
- UnstractFileSystem,
-)
+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
logger = logging.getLogger(__name__)
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/register.py b/unstract/connectors/src/unstract/connectors/filesystems/register.py
index 5bcc637cd..d820e5f62 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/register.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/register.py
@@ -4,9 +4,7 @@ from importlib import import_module
from typing import Any
from unstract.connectors.constants import Common
-from unstract.connectors.filesystems.unstract_file_system import (
- UnstractFileSystem,
-)
+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
logger = logging.getLogger(__name__)
@@ -24,9 +22,7 @@ def register_connectors(connectors: dict[str, Any]) -> None:
module = import_module(full_module_path)
metadata = getattr(module, Common.METADATA, {})
if metadata.get("is_active", False):
- connector_class: UnstractFileSystem = metadata[
- Common.CONNECTOR
- ]
+ connector_class: UnstractFileSystem = metadata[Common.CONNECTOR]
connector_id = connector_class.get_id()
if not connector_id or (connector_id in connectors):
logger.warning(f"Duplicate Id : {connector_id}")
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py
index 2ac4bb0eb..31b743cbb 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py
@@ -6,9 +6,7 @@ from dropbox.exceptions import DropboxException
from dropboxdrivefs import DropboxDriveFileSystem
from unstract.connectors.exceptions import ConnectorError
-from unstract.connectors.filesystems.unstract_file_system import (
- UnstractFileSystem,
-)
+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
from .exceptions import handle_dropbox_exception
diff --git a/unstract/connectors/src/unstract/connectors/gcs_helper.py b/unstract/connectors/src/unstract/connectors/gcs_helper.py
index 82c351c13..2e606adb2 100644
--- a/unstract/connectors/src/unstract/connectors/gcs_helper.py
+++ b/unstract/connectors/src/unstract/connectors/gcs_helper.py
@@ -31,10 +31,8 @@ class GCSHelper:
"GOOGLE_PROJECT_ID environment variable not set"
)
- self.google_credentials = (
- service_account.Credentials.from_service_account_info(
- json.loads(self.google_service_json)
- )
+ self.google_credentials = service_account.Credentials.from_service_account_info(
+ json.loads(self.google_service_json)
)
def get_google_credentials(self) -> Credentials:
@@ -60,30 +58,22 @@ class GCSHelper:
md5_hash_bytes = base64.b64decode(blob.md5_hash)
md5_hash_hex = md5_hash_bytes.hex()
except Exception:
- logger.error(
- f"Could not get blob {object_name} from bucket {bucket_name}"
- )
+ logger.error(f"Could not get blob {object_name} from bucket {bucket_name}")
return md5_hash_hex
- def upload_file(
- self, bucket_name: str, object_name: str, file_path: str
- ) -> None:
+ def upload_file(self, bucket_name: str, object_name: str, file_path: str) -> None:
client = Client(credentials=self.google_credentials)
bucket = client.bucket(bucket_name)
blob = bucket.blob(object_name)
blob.upload_from_filename(file_path)
- def upload_text(
- self, bucket_name: str, object_name: str, text: str
- ) -> None:
+ def upload_text(self, bucket_name: str, object_name: str, text: str) -> None:
client = Client(credentials=self.google_credentials)
bucket = client.bucket(bucket_name)
blob = bucket.blob(object_name)
blob.upload_from_string(text)
- def upload_object(
- self, bucket_name: str, object_name: str, object: Any
- ) -> None:
+ def upload_object(self, bucket_name: str, object_name: str, object: Any) -> None:
client = Client(credentials=self.google_credentials)
bucket = client.bucket(bucket_name)
blob = bucket.blob(object_name)
@@ -101,6 +91,4 @@ class GCSHelper:
logger.info(f"Reading file {object_name} from bucket {bucket_name}")
return obj
except Exception:
- logger.error(
- f"Could not get blob {object_name} from bucket {bucket_name}"
- )
+ logger.error(f"Could not get blob {object_name} from bucket {bucket_name}")
diff --git a/unstract/connectors/tests/filesystems/test_google_drive_fs.py b/unstract/connectors/tests/filesystems/test_google_drive_fs.py
index bfeccbac8..26b1ac59a 100644
--- a/unstract/connectors/tests/filesystems/test_google_drive_fs.py
+++ b/unstract/connectors/tests/filesystems/test_google_drive_fs.py
@@ -1,8 +1,6 @@
import unittest
-from unstract.connectors.filesystems.google_drive.google_drive import (
- GoogleDriveFS,
-)
+from unstract.connectors.filesystems.google_drive.google_drive import GoogleDriveFS
class TestGoogleDriveFS(unittest.TestCase):
diff --git a/unstract/connectors/tests/filesystems/test_miniofs.py b/unstract/connectors/tests/filesystems/test_miniofs.py
index 1ab09fcc5..0443bbde1 100644
--- a/unstract/connectors/tests/filesystems/test_miniofs.py
+++ b/unstract/connectors/tests/filesystems/test_miniofs.py
@@ -31,9 +31,7 @@ class TestMinoFS(unittest.TestCase):
access_key = os.environ.get("MINIO_ACCESS_KEY_ID")
secret_key = os.environ.get("MINIO_SECRET_ACCESS_KEY")
print(access_key, secret_key)
- bucket_name = os.environ.get(
- "FREE_STORAGE_AWS_BUCKET_NAME", "minio-test"
- )
+ bucket_name = os.environ.get("FREE_STORAGE_AWS_BUCKET_NAME", "minio-test")
s3 = MinioFS(
{
"key": access_key,
diff --git a/unstract/core/src/unstract/core/llm_helper/config.py b/unstract/core/src/unstract/core/llm_helper/config.py
index 2e711300f..1a09cabcd 100644
--- a/unstract/core/src/unstract/core/llm_helper/config.py
+++ b/unstract/core/src/unstract/core/llm_helper/config.py
@@ -33,18 +33,14 @@ class AzureOpenAIConfig:
@classmethod
def from_env(cls) -> "AzureOpenAIConfig":
kwargs = {
- "model": UnstractUtils.get_env(
- OpenAIKeys.OPENAI_API_MODEL, raise_err=True
- ),
+ "model": UnstractUtils.get_env(OpenAIKeys.OPENAI_API_MODEL, raise_err=True),
"deployment_name": UnstractUtils.get_env(
OpenAIKeys.OPENAI_API_ENGINE, raise_err=True
),
"engine": UnstractUtils.get_env(
OpenAIKeys.OPENAI_API_ENGINE, raise_err=True
),
- "api_key": UnstractUtils.get_env(
- OpenAIKeys.OPENAI_API_KEY, raise_err=True
- ),
+ "api_key": UnstractUtils.get_env(OpenAIKeys.OPENAI_API_KEY, raise_err=True),
"api_version": UnstractUtils.get_env(
OpenAIKeys.OPENAI_API_VERSION,
default=OpenAIDefaults.OPENAI_API_VERSION,
diff --git a/unstract/core/src/unstract/core/llm_helper/llm_cache.py b/unstract/core/src/unstract/core/llm_helper/llm_cache.py
index 580c51e5e..5af5b67d1 100644
--- a/unstract/core/src/unstract/core/llm_helper/llm_cache.py
+++ b/unstract/core/src/unstract/core/llm_helper/llm_cache.py
@@ -12,9 +12,7 @@ class LLMCache:
redis_host = os.environ.get("REDIS_HOST")
redis_port = os.environ.get("REDIS_PORT")
if redis_host is None or redis_port is None:
- raise RuntimeError(
- "REDIS_HOST or REDIS_PORT environment variable not set"
- )
+ raise RuntimeError("REDIS_HOST or REDIS_PORT environment variable not set")
redis_password = os.environ.get("REDIS_PASSWORD", None)
if redis_password and (
redis_password == "" or redis_password.lower() == "none"
@@ -107,10 +105,7 @@ class LLMCache:
"""
logger.info(f"Clearing cache with prefix: {self.cache_key_prefix}")
keys_to_delete = [
- key
- for key in self.llm_cache.scan_iter(
- match=self.cache_key_prefix + "*"
- )
+ key for key in self.llm_cache.scan_iter(match=self.cache_key_prefix + "*")
]
if keys_to_delete:
return self.delete(*keys_to_delete)
diff --git a/unstract/core/src/unstract/core/llm_helper/llm_helper.py b/unstract/core/src/unstract/core/llm_helper/llm_helper.py
index 0912a783c..12d947cc6 100644
--- a/unstract/core/src/unstract/core/llm_helper/llm_helper.py
+++ b/unstract/core/src/unstract/core/llm_helper/llm_helper.py
@@ -47,9 +47,7 @@ class LLMHelper:
prompt_for_model = self.prompt
if self.prompt_context == PromptContext.GENERATE_CRON_STRING:
- prompt_for_model = prompt_for_model.replace(
- "{$user_prompt}", user_prompt
- )
+ prompt_for_model = prompt_for_model.replace("{$user_prompt}", user_prompt)
return prompt_for_model
@@ -71,9 +69,7 @@ class LLMHelper:
if ai_service == "azure-open-ai":
logger.info("Using Azure OpenAI")
if use_cache:
- response = self.llm_cache.get_for_prompt(
- prompt=prompt_for_model
- )
+ response = self.llm_cache.get_for_prompt(prompt=prompt_for_model)
if response:
return LLMResponse(
result=LLMResult.OK, output=response, cost_type="cache"
@@ -108,12 +104,8 @@ class LLMHelper:
logger.info(f"OpenAI Response: {resp}")
time_taken = end_time - start_time
- self.llm_cache.set_for_prompt(
- prompt=prompt_for_model, response=resp
- )
- return LLMResponse(
- output=resp, cost_type=ai_service, time_taken=time_taken
- )
+ self.llm_cache.set_for_prompt(prompt=prompt_for_model, response=resp)
+ return LLMResponse(output=resp, cost_type=ai_service, time_taken=time_taken)
else:
logger.error(f"AI service '{ai_service}' not found")
return LLMResponse(
diff --git a/unstract/core/src/unstract/core/llm_workflow_generator/llm_interface.py b/unstract/core/src/unstract/core/llm_workflow_generator/llm_interface.py
index 0a5874318..fb66a8780 100644
--- a/unstract/core/src/unstract/core/llm_workflow_generator/llm_interface.py
+++ b/unstract/core/src/unstract/core/llm_workflow_generator/llm_interface.py
@@ -70,9 +70,7 @@ class LLMInterface:
port=int(redis_port),
password=redis_password,
)
- redis_key = (
- f"cache:{workflow_id}:workflow_prompt:{hash_hex}"
- )
+ redis_key = f"cache:{workflow_id}:workflow_prompt:{hash_hex}"
workflow_bin = r.get(redis_key)
if workflow_bin is not None:
logging.info("Cache hit")
diff --git a/unstract/core/src/unstract/core/utilities.py b/unstract/core/src/unstract/core/utilities.py
index 8f45dfed5..0364d28d9 100644
--- a/unstract/core/src/unstract/core/utilities.py
+++ b/unstract/core/src/unstract/core/utilities.py
@@ -4,9 +4,7 @@ from typing import Optional
class UnstractUtils:
@staticmethod
- def get_env(
- env_key: str, default: Optional[str] = None, raise_err=False
- ) -> str:
+ def get_env(env_key: str, default: Optional[str] = None, raise_err=False) -> str:
"""Returns the value of an env variable.
If its empty or None, raises an error
diff --git a/unstract/core/tests/llm_helper/test_llm_cache.py b/unstract/core/tests/llm_helper/test_llm_cache.py
index d94f74401..3f6389242 100644
--- a/unstract/core/tests/llm_helper/test_llm_cache.py
+++ b/unstract/core/tests/llm_helper/test_llm_cache.py
@@ -18,9 +18,7 @@ class LLMCacheTests(unittest.TestCase):
cache.set_for_prompt("prompt1", "response1")
cache.set_for_prompt("prompt2", "response2")
cache.clear_by_prefix()
- self.assertEqual(
- cache.get_for_prompt("prompt1"), "", "Cache is not cleared"
- )
+ self.assertEqual(cache.get_for_prompt("prompt1"), "", "Cache is not cleared")
if __name__ == "__main__":
diff --git a/unstract/core/tests/test_pubsub_helper.py b/unstract/core/tests/test_pubsub_helper.py
index 33a70cd53..ae3e10269 100644
--- a/unstract/core/tests/test_pubsub_helper.py
+++ b/unstract/core/tests/test_pubsub_helper.py
@@ -11,9 +11,7 @@ class PubSubHelperTestCase(unittest.TestCase):
)
ps2 = Log.publish(
project_guid="test",
- message=Log.log(
- level="ERROR", stage="COMPILE", message="Compile failed"
- ),
+ message=Log.log(level="ERROR", stage="COMPILE", message="Compile failed"),
)
self.assertEqual(ps1, True)
self.assertEqual(ps2, True)
diff --git a/unstract/flags/src/unstract/flags/evaluation_pb2.py b/unstract/flags/src/unstract/flags/evaluation_pb2.py
index e5883270c..bc6c6c575 100644
--- a/unstract/flags/src/unstract/flags/evaluation_pb2.py
+++ b/unstract/flags/src/unstract/flags/evaluation_pb2.py
@@ -12,9 +12,7 @@ from google.protobuf.internal import enum_type_wrapper
_sym_db = _symbol_database.Default()
-from google.protobuf import (
- timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2,
-)
+from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
name="protos/evaluation.proto",
diff --git a/unstract/tool-registry/src/unstract/tool_registry/dto.py b/unstract/tool-registry/src/unstract/tool_registry/dto.py
index 084ba0091..116d02e22 100644
--- a/unstract/tool-registry/src/unstract/tool_registry/dto.py
+++ b/unstract/tool-registry/src/unstract/tool_registry/dto.py
@@ -67,9 +67,7 @@ class Spec:
default_factory=dict[str, dict[str, Any]]
)
- def get_adapter_properties_keys(
- self, adapter_type: AdapterTypes
- ) -> set[str]:
+ def get_adapter_properties_keys(self, adapter_type: AdapterTypes) -> set[str]:
properties = set()
if self.properties is not None:
properties = {
@@ -255,9 +253,7 @@ class Adapter:
text_extractors: list[AdapterProperties] = field(
default_factory=list[AdapterProperties]
)
- ocrs: list[AdapterProperties] = field(
- default_factory=list[AdapterProperties]
- )
+ ocrs: list[AdapterProperties] = field(default_factory=list[AdapterProperties])
@classmethod
def from_dict(cls, data_dict: dict[str, Any]) -> "Adapter":
diff --git a/unstract/tool-registry/src/unstract/tool_registry/helper.py b/unstract/tool-registry/src/unstract/tool_registry/helper.py
index 1dff798c7..a6fcf64a9 100644
--- a/unstract/tool-registry/src/unstract/tool_registry/helper.py
+++ b/unstract/tool-registry/src/unstract/tool_registry/helper.py
@@ -104,9 +104,7 @@ class ToolRegistryHelper:
image_tag=tool_meta.tag,
)
- tool_properties: Optional[dict[str, Any]] = (
- tool_sandbox.get_properties()
- )
+ tool_properties: Optional[dict[str, Any]] = tool_sandbox.get_properties()
if not tool_properties:
return {}
return tool_properties
@@ -165,9 +163,7 @@ class ToolRegistryHelper:
Returns:
ToolData: _description_
"""
- tool_metadata = ToolUtils.get_tool_meta_from_tool_url(
- registry_tool=image_url
- )
+ tool_metadata = ToolUtils.get_tool_meta_from_tool_url(registry_tool=image_url)
spec = self.get_tool_spec(tool_meta=tool_metadata)
properties = self.get_tool_properties(tool_meta=tool_metadata)
icon = self.get_tool_icon(tool_meta=tool_metadata)
@@ -202,9 +198,7 @@ class ToolRegistryHelper:
dict[str, Any]: _description_
"""
try:
- yml_data: dict[str, Any] = ToolUtils.get_registry(
- self.registry_file
- )
+ yml_data: dict[str, Any] = ToolUtils.get_registry(self.registry_file)
return yml_data
except FileNotFoundError:
logger.error(f"File not found: {self.registry_file}")
@@ -325,20 +319,14 @@ class ToolRegistryHelper:
tool_data: dict[str, Any] = tools.pop(tool_uid, {})
return tool_data
- def add_new_tool_to_disk_by_uid(
- self, uuid: str, data: dict[str, Any]
- ) -> None:
+ def add_new_tool_to_disk_by_uid(self, uuid: str, data: dict[str, Any]) -> None:
tools = self.get_all_tools_from_disk()
tools[uuid] = data
- ToolUtils.save_tools_in_to_disk(
- file_path=self.private_tools_file, data=tools
- )
+ ToolUtils.save_tools_in_to_disk(file_path=self.private_tools_file, data=tools)
def add_new_tool_to_disk_by_image_url(self, image_url: str) -> None:
tool_data = self.get_tool_data_by_image_url(image_url=image_url)
- self.add_new_tool_to_disk_by_uid(
- uuid=tool_data.uid, data=tool_data.data
- )
+ self.add_new_tool_to_disk_by_uid(uuid=tool_data.uid, data=tool_data.data)
def remove_tools_from_disk(
self, tools: dict[str, Any], tool_ids: list[str]
diff --git a/unstract/tool-registry/src/unstract/tool_registry/schema_validator.py b/unstract/tool-registry/src/unstract/tool_registry/schema_validator.py
index ef37a8911..9ededb2f7 100644
--- a/unstract/tool-registry/src/unstract/tool_registry/schema_validator.py
+++ b/unstract/tool-registry/src/unstract/tool_registry/schema_validator.py
@@ -27,9 +27,7 @@ class JsonSchemaValidator:
logger.error(f"Validation error: {e}")
raise InvalidSchemaInput
- def validate_and_filter(
- self, data: dict[str, Any]
- ) -> Optional[dict[str, Any]]:
+ def validate_and_filter(self, data: dict[str, Any]) -> Optional[dict[str, Any]]:
"""Validates the input data against the schema and filters the data
based on the schema's properties.
diff --git a/unstract/tool-registry/src/unstract/tool_registry/tool_registry.py b/unstract/tool-registry/src/unstract/tool_registry/tool_registry.py
index 73d54e3c5..ce2cdd8e3 100644
--- a/unstract/tool-registry/src/unstract/tool_registry/tool_registry.py
+++ b/unstract/tool-registry/src/unstract/tool_registry/tool_registry.py
@@ -81,9 +81,7 @@ class ToolRegistry:
self.helper.remove_tool_from_registry(image_url=image_url)
logger.info(f"Tool '{tool_id}' removed from registry successfully.")
self.helper.save_tools(data=tools)
- logger.info(
- f"Tool '{tool_id}' removed from tools file successfully."
- )
+ logger.info(f"Tool '{tool_id}' removed from tools file successfully.")
except Exception as e:
logger.error(f"An error occurred: {str(e)}")
@@ -154,14 +152,10 @@ class ToolRegistry:
tools_list = []
if load_from_source:
for tool in self.helper.get_all_registry_tools():
- tool_meta = ToolUtils.get_tool_meta_from_tool_url(
- registry_tool=tool
- )
+ tool_meta = ToolUtils.get_tool_meta_from_tool_url(registry_tool=tool)
if not tool_meta:
continue
- properties = self.helper.get_tool_properties(
- tool_meta=tool_meta
- )
+ properties = self.helper.get_tool_properties(tool_meta=tool_meta)
icon = self.helper.get_tool_icon(tool_meta=tool_meta)
if not properties:
continue
@@ -169,9 +163,7 @@ class ToolRegistry:
ToolKey.NAME: properties.get(PropKey.DISPLAY_NAME),
ToolKey.DESCRIPTION: properties.get(PropKey.DESCRIPTION),
ToolKey.ICON: icon,
- ToolKey.FUNCTION_NAME: properties.get(
- PropKey.FUNCTION_NAME
- ),
+ ToolKey.FUNCTION_NAME: properties.get(PropKey.FUNCTION_NAME),
}
tools_list.append(tool_data)
else:
@@ -206,13 +198,11 @@ class ToolRegistry:
tools_list = {}
if load_from_source:
for tool in self.helper.get_all_registry_tools():
- tool_meta = ToolUtils.get_tool_meta_from_tool_url(
- registry_tool=tool
- )
+ tool_meta = ToolUtils.get_tool_meta_from_tool_url(registry_tool=tool)
if not tool_meta:
continue
- properties: Optional[dict[str, Any]] = (
- self.helper.get_tool_properties(tool_meta=tool_meta)
+ properties: Optional[dict[str, Any]] = self.helper.get_tool_properties(
+ tool_meta=tool_meta
)
spec: Optional[dict[str, Any]] = self.helper.get_tool_spec(
tool_meta=tool_meta
@@ -235,9 +225,7 @@ class ToolRegistry:
"icon": icon,
}
else:
- tools: dict[str, dict[str, Any]] = (
- self.helper.get_all_tools_from_disk()
- )
+ tools: dict[str, dict[str, Any]] = self.helper.get_all_tools_from_disk()
for tool, configuration in tools.items():
properties = configuration.get("properties")
spec = configuration.get("spec")
@@ -287,9 +275,7 @@ class ToolRegistry:
dict[str, Any]: _description_
"""
tools = self.helper.get_all_tools_from_disk()
- properties: dict[str, Any] = tools.get(tool_id, {}).get(
- "properties", {}
- )
+ properties: dict[str, Any] = tools.get(tool_id, {}).get("properties", {})
return properties
def get_tool_icon_by_tool_id(self, tool_id: str) -> dict[str, Any]:
diff --git a/unstract/tool-registry/src/unstract/tool_registry/tool_utils.py b/unstract/tool-registry/src/unstract/tool_registry/tool_utils.py
index 3dcb33f04..af9ca040f 100644
--- a/unstract/tool-registry/src/unstract/tool_registry/tool_utils.py
+++ b/unstract/tool-registry/src/unstract/tool_registry/tool_utils.py
@@ -18,17 +18,13 @@ class ToolUtils:
@staticmethod
def is_valid_tool_url(image_url: str) -> bool:
# Define a regular expression pattern for Docker image URLs
- docker_pattern = (
- r"^docker:[a-zA-Z0-9_.-]+(/[a-zA-Z0-9_.-]+)*/[a-zA-Z0-9_.-]+$"
- )
+ docker_pattern = r"^docker:[a-zA-Z0-9_.-]+(/[a-zA-Z0-9_.-]+)*/[a-zA-Z0-9_.-]+$"
# Define a regular expression pattern for local tool URLs
local_pattern = r"^local:[a-zA-Z0-9_.-:]+$"
# Check if the URL matches either the Docker image or local tool pattern
- if re.match(docker_pattern, image_url) or re.match(
- local_pattern, image_url
- ):
+ if re.match(docker_pattern, image_url) or re.match(local_pattern, image_url):
return True
else:
return False
@@ -172,9 +168,7 @@ class ToolUtils:
list[AdapterProperties]:
A list of enabled AdapterProperties objects.
"""
- enabled_adapters = [
- adapter for adapter in adapters if adapter.is_enabled
- ]
+ enabled_adapters = [adapter for adapter in adapters if adapter.is_enabled]
return enabled_adapters
@staticmethod
@@ -240,14 +234,10 @@ class ToolUtils:
schema = tool.spec
adapter = tool.properties.adapter
- language_models = ToolUtils.get_enabled_adapters(
- adapter.language_models
- )
+ language_models = ToolUtils.get_enabled_adapters(adapter.language_models)
embeddings = ToolUtils.get_enabled_adapters(adapter.embedding_services)
vector_stores = ToolUtils.get_enabled_adapters(adapter.vector_stores)
- text_extractors = ToolUtils.get_enabled_adapters(
- adapter.text_extractors
- )
+ text_extractors = ToolUtils.get_enabled_adapters(adapter.text_extractors)
ocrs = ToolUtils.get_enabled_adapters(adapter.ocrs)
ToolUtils.process_adapter_models(
diff --git a/unstract/tool-registry/tests/test_tool_registry.py b/unstract/tool-registry/tests/test_tool_registry.py
index 4a09a459a..7fff83b06 100644
--- a/unstract/tool-registry/tests/test_tool_registry.py
+++ b/unstract/tool-registry/tests/test_tool_registry.py
@@ -25,13 +25,9 @@ class TestToolRegistry(unittest.TestCase):
"tools": [self.TEST_IMAGE_URL],
}
directory = os.path.dirname(os.path.abspath(__file__))
- registry_file_path = os.path.join(
- directory, TestToolRegistry.REGISTRY_FILE
- )
+ registry_file_path = os.path.join(directory, TestToolRegistry.REGISTRY_FILE)
with open(registry_file_path, "w") as yaml_file:
- yaml.dump(
- test_registry_content, yaml_file, default_flow_style=False
- )
+ yaml.dump(test_registry_content, yaml_file, default_flow_style=False)
# Apply the patch to 'run_tool_and_get_logs' before each test method
self.mock_run_tool_and_get_logs = patch.object(
@@ -91,9 +87,7 @@ class TestToolRegistry(unittest.TestCase):
def test_get_tool_properties_by_tool_id(self) -> None:
tool_id = "document_indexer"
- properties = self.registry.get_tool_properties_by_tool_id(
- tool_id=tool_id
- )
+ properties = self.registry.get_tool_properties_by_tool_id(tool_id=tool_id)
self.assertIsNotNone(properties)
self.assertEqual(properties.get("function_name"), tool_id)
diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py
index 9e745949a..5967f1306 100644
--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py
+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py
@@ -20,18 +20,14 @@ class ToolSandboxHelper:
) -> None:
worker_host = os.environ.get("UNSTRACT_WORKER_HOST")
worker_port = os.environ.get("UNSTRACT_WORKER_PORT")
- self.base_url = (
- f"{worker_host}:{worker_port}{UnstractWorker.BASE_API_ENDPOINT}"
- )
+ self.base_url = f"{worker_host}:{worker_port}{UnstractWorker.BASE_API_ENDPOINT}"
self.organization_id = str(organization_id)
self.workflow_id = str(workflow_id)
self.execution_id = str(execution_id)
self.envs = environment_variables
self.messaging_channel = str(messaging_channel)
- def convert_str_to_dict(
- self, data: Union[str, dict[str, Any]]
- ) -> dict[str, Any]:
+ def convert_str_to_dict(self, data: Union[str, dict[str, Any]]) -> dict[str, Any]:
if isinstance(data, str):
output: dict[str, Any] = {}
try:
@@ -67,8 +63,7 @@ class ToolSandboxHelper:
)
else:
logger.error(
- f"Error while calling tool {image_name} "
- f" reason: {response.reason}"
+ f"Error while calling tool {image_name} " f" reason: {response.reason}"
)
return result
@@ -107,8 +102,7 @@ class ToolSandboxHelper:
)
else:
logger.error(
- f"Error while calling tool {image_name} "
- f" reason: {response.reason}"
+ f"Error while calling tool {image_name} " f" reason: {response.reason}"
)
return result
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py
index e60a42ebf..48dc77a70 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py
@@ -26,9 +26,7 @@ class ExecutionFileHandler:
self.execution_dir = self.create_execution_dir_path(
workflow_id, execution_id, organization_id
)
- self.source_file = os.path.join(
- self.execution_dir, WorkflowFileType.SOURCE
- )
+ self.source_file = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)
self.infile = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
self.metadata_file = os.path.join(
self.execution_dir, WorkflowFileType.METADATA_JSON
@@ -80,17 +78,13 @@ class ExecutionFileHandler:
)
return output_type
- def get_last_tool_metadata(
- self, metadata: dict[str, Any]
- ) -> dict[str, Any]:
+ def get_last_tool_metadata(self, metadata: dict[str, Any]) -> dict[str, Any]:
tool_metadata = self.get_list_of_tool_metadata(metadata)
if not tool_metadata:
raise ToolMetadataNotFound()
return tool_metadata[-1]
- def add_metadata_to_volume(
- self, input_file_path: str, source_hash: str
- ) -> None:
+ def add_metadata_to_volume(self, input_file_path: str, source_hash: str) -> None:
"""Creating metadata for workflow. This method is responsible for
creating metadata for the workflow. It takes the input file path and
the source hash as parameters. The metadata is stored in a JSON file in
@@ -120,8 +114,7 @@ class ExecutionFileHandler:
with fsspec.open(f"file://{destination_path}", "w") as local_file:
json.dump(content, local_file)
logger.info(
- f"metadata for {input_file_path} is "
- "added in to execution directory"
+ f"metadata for {input_file_path} is " "added in to execution directory"
)
@classmethod
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
index b0f8b6f65..c2f440179 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
@@ -48,18 +48,10 @@ class ToolsUtils:
self.doc_processor_api_key = ToolsUtils.get_env(
ToolRV.DOCUMENT_PROCESSOR_API_KEY, raise_exception=True
)
- self.prompt_host = ToolsUtils.get_env(
- ToolRV.PROMPT_HOST, raise_exception=True
- )
- self.prompt_port = ToolsUtils.get_env(
- ToolRV.PROMPT_PORT, raise_exception=True
- )
- self.x2text_host = ToolsUtils.get_env(
- ToolRV.X2TEXT_HOST, raise_exception=True
- )
- self.x2text_port = ToolsUtils.get_env(
- ToolRV.X2TEXT_PORT, raise_exception=True
- )
+ self.prompt_host = ToolsUtils.get_env(ToolRV.PROMPT_HOST, raise_exception=True)
+ self.prompt_port = ToolsUtils.get_env(ToolRV.PROMPT_PORT, raise_exception=True)
+ self.x2text_host = ToolsUtils.get_env(ToolRV.X2TEXT_HOST, raise_exception=True)
+ self.x2text_port = ToolsUtils.get_env(ToolRV.X2TEXT_PORT, raise_exception=True)
def set_messaging_channel(self, messaging_channel: str) -> None:
self.messaging_channel = messaging_channel
@@ -79,8 +71,8 @@ class ToolsUtils:
dict[str, dict[str, Any]]: tools
"""
tool_uids = [tool_instance.tool_id for tool_instance in tool_instances]
- tools: dict[str, dict[str, Any]] = (
- self.tool_registry.get_available_tools(tool_uids)
+ tools: dict[str, dict[str, Any]] = self.tool_registry.get_available_tools(
+ tool_uids
)
if not (
all(tool_uid in tools for tool_uid in tool_uids)
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
index 2b3f1daa9..584f70243 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
@@ -20,9 +20,7 @@ from unstract.workflow_execution.exceptions import (
StopExecution,
ToolOutputNotFoundException,
)
-from unstract.workflow_execution.execution_file_handler import (
- ExecutionFileHandler,
-)
+from unstract.workflow_execution.execution_file_handler import ExecutionFileHandler
from unstract.workflow_execution.tools_utils import ToolsUtils
from unstract.core.pubsub_helper import LogPublisher
@@ -242,8 +240,7 @@ class WorkflowExecutionService:
"""
with self.redis_con as red:
logger.info(
- f"Setting single stepping flag to "
- f"{ExecutionAction.START.value}"
+ f"Setting single stepping flag to " f"{ExecutionAction.START.value}"
)
red.setex(
self.execution_id,
@@ -272,8 +269,7 @@ class WorkflowExecutionService:
execution_action = ExecutionAction(execution_value)
if execution_action == ExecutionAction.NEXT:
log_message = (
- f"Execution '{self.execution_id}' Executing "
- "NEXT step"
+ f"Execution '{self.execution_id}' Executing " "NEXT step"
)
self.publish_log(log_message)
break
@@ -336,17 +332,12 @@ class WorkflowExecutionService:
Args:
execution_type (ExecutionType): ExecutionType
"""
- if (
- execution_type == ExecutionType.STEP
- and not self.override_single_step
- ):
+ if execution_type == ExecutionType.STEP and not self.override_single_step:
self._handling_step_execution()
def validate_execution_result(self, step: int) -> bool:
workflow_metadata = self.file_handler.get_workflow_metadata()
- metadata_list = self.file_handler.get_list_of_tool_metadata(
- workflow_metadata
- )
+ metadata_list = self.file_handler.get_list_of_tool_metadata(workflow_metadata)
if len(metadata_list) == step:
return True
return False
@@ -406,7 +397,5 @@ class WorkflowExecutionService:
if isinstance(component, LogComponent):
component = component.value
- log_details = LogPublisher.log_workflow_update(
- state.value, message, component
- )
+ log_details = LogPublisher.log_workflow_update(state.value, message, component)
LogPublisher.publish(self.messaging_channel, log_details)
diff --git a/unstract/workflow-execution/tests/workflow_test.py b/unstract/workflow-execution/tests/workflow_test.py
index a10922a62..906d64853 100644
--- a/unstract/workflow-execution/tests/workflow_test.py
+++ b/unstract/workflow-execution/tests/workflow_test.py
@@ -18,16 +18,10 @@ def get_mock_tool_instances() -> list[ToolInstance]:
input=item["input"],
output=item["output"],
metadata=item["metadata"],
- input_file_connector=ConnectorInstance(
- **item["input_file_connector"]
- ),
- output_file_connector=ConnectorInstance(
- **item["output_file_connector"]
- ),
+ input_file_connector=ConnectorInstance(**item["input_file_connector"]),
+ output_file_connector=ConnectorInstance(**item["output_file_connector"]),
input_db_connector=ConnectorInstance(**item["input_db_connector"]),
- output_db_connector=ConnectorInstance(
- **item["output_db_connector"]
- ),
+ output_db_connector=ConnectorInstance(**item["output_db_connector"]),
tool_settings=ToolSettings(**item["tool_settings"]),
)
for item in tool_instance_data
@@ -91,8 +85,8 @@ class TestWorkflow(unittest.TestCase):
mock_tool_instances = get_mock_tool_instances()
mock_tool_utils = Mock()
- mock_tool_utils.validate_tool_instance_with_tools.side_effect = (
- Exception("Test error message")
+ mock_tool_utils.validate_tool_instance_with_tools.side_effect = Exception(
+ "Test error message"
)
workflow = Workflow(
diff --git a/worker/src/unstract/worker/worker.py b/worker/src/unstract/worker/worker.py
index acd8a1d05..6454dd9b0 100644
--- a/worker/src/unstract/worker/worker.py
+++ b/worker/src/unstract/worker/worker.py
@@ -76,14 +76,10 @@ class UnstractWorker:
try:
# Attempt to get the image information
self.client.images.get(image_name_with_tag)
- logger.info(
- f"Image '{image_name_with_tag}' found in the local system."
- )
+ logger.info(f"Image '{image_name_with_tag}' found in the local system.")
return True
except docker.errors.ImageNotFound: # type: ignore[attr-defined]
- logger.info(
- f"Image '{image_name_with_tag}' not found in the local system."
- )
+ logger.info(f"Image '{image_name_with_tag}' not found in the local system.")
return False
except docker.errors.APIError as e: # type: ignore[attr-defined]
logger.error(f"An API error occurred: {e}")
@@ -119,8 +115,7 @@ class UnstractWorker:
return log_dict
except json.JSONDecodeError as e:
logger.warn(
- f"Received invalid JSON log message: {log_message} \n "
- f"error: {e}"
+ f"Received invalid JSON log message: {log_message} \n " f"error: {e}"
)
return None
@@ -136,8 +131,7 @@ class UnstractWorker:
log_type = self.get_log_type(log_dict)
if not self.is_valid_log_type(log_type):
logger.warning(
- f"Received invalid logType: {log_type} with log message: "
- f"{log_dict}"
+ f"Received invalid logType: {log_type} with log message: " f"{log_dict}"
)
return None
if log_type == LogType.RESULT:
@@ -301,17 +295,13 @@ class UnstractWorker:
self.envs = envs
self.messaging_channel = messaging_channel
self.container_name = (
- self.normalize_container_name(self.image_name)
- + "-"
- + self.workflow_id
+ self.normalize_container_name(self.image_name) + "-" + self.workflow_id
)
container_config = self.get_container_run_config()
# Add labels to container for logging with Loki.
# This only required for observability.
try:
- labels = ast.literal_eval(
- os.environ.get(Env.TOOL_CONTAINER_LABELS, "")
- )
+ labels = ast.literal_eval(os.environ.get(Env.TOOL_CONTAINER_LABELS, ""))
container_config["labels"] = labels
except Exception as e:
logger.info(f"Invalid labels for logging: {e}")
diff --git a/x2text-service/app/authentication_middleware.py b/x2text-service/app/authentication_middleware.py
index 5889fc115..0f03ce0d1 100644
--- a/x2text-service/app/authentication_middleware.py
+++ b/x2text-service/app/authentication_middleware.py
@@ -8,9 +8,7 @@ def authentication_middleware(func: Any) -> Any:
def wrapper(*args: Any, **kwargs: Any) -> Any:
token = AuthenticationMiddleware.get_token_from_auth_header(request)
# Check if bearer token exists and validate it
- if not token or not AuthenticationMiddleware.validate_bearer_token(
- token
- ):
+ if not token or not AuthenticationMiddleware.validate_bearer_token(token):
return "Unauthorized", 401
return func(*args, **kwargs)
@@ -24,9 +22,7 @@ class AuthenticationMiddleware:
def validate_bearer_token(cls, token: Optional[str]) -> bool:
try:
if token is None:
- current_app.logger.error(
- "Authentication failed. Empty bearer token"
- )
+ current_app.logger.error("Authentication failed. Empty bearer token")
return False
query = f"SELECT * FROM account_platformkey WHERE key = '{token}'"
@@ -53,9 +49,7 @@ class AuthenticationMiddleware:
return False
except Exception as e:
- current_app.logger.error(
- f"Error while validating bearer token: {e}"
- )
+ current_app.logger.error(f"Error while validating bearer token: {e}")
return False
return True
@@ -74,13 +68,11 @@ class AuthenticationMiddleware:
@classmethod
def get_account_from_bearer_token(cls, token: Optional[str]) -> str:
query = (
- "SELECT organization_id FROM account_platformkey "
- f"WHERE key='{token}'"
+ "SELECT organization_id FROM account_platformkey " f"WHERE key='{token}'"
)
organization = AuthenticationMiddleware.execute_query(query)
query_org = (
- "SELECT schema_name FROM account_organization "
- f"WHERE id='{organization}'"
+ "SELECT schema_name FROM account_organization " f"WHERE id='{organization}'"
)
schema_name: str = AuthenticationMiddleware.execute_query(query_org)
return schema_name
diff --git a/x2text-service/app/controllers/controller.py b/x2text-service/app/controllers/controller.py
index 82f6a2dc8..16dab861e 100644
--- a/x2text-service/app/controllers/controller.py
+++ b/x2text-service/app/controllers/controller.py
@@ -32,13 +32,9 @@ def health() -> str:
@basic.route("/test-connection", methods=["POST"])
@authentication_middleware
def test_connection() -> Any:
- logging.info(
- "Received a test connection request from %s", request.remote_addr
- )
+ logging.info("Received a test connection request from %s", request.remote_addr)
form_data = dict(request.form)
- unstructured_api_key = X2TextUtil.get_value_for_key(
- UNSTRUCTURED_API_KEY, form_data
- )
+ unstructured_api_key = X2TextUtil.get_value_for_key(UNSTRUCTURED_API_KEY, form_data)
url = X2TextUtil.get_value_for_key(UNSTRUCTURED_URL, form_data)
if not url:
return {"message": "Missing or empty url in form data"}, 400
@@ -79,9 +75,7 @@ def test_connection() -> Any:
@basic.route("/process", methods=["POST"])
@authentication_middleware
def process() -> Any:
- logging.info(
- "Received a doc processing request from %s", request.remote_addr
- )
+ logging.info("Received a doc processing request from %s", request.remote_addr)
form_data = dict(request.form)
url = X2TextUtil.get_value_for_key(UNSTRUCTURED_URL, form_data)
if not url:
@@ -97,9 +91,7 @@ def process() -> Any:
file_size_in_kb = int(request.headers["Content-Length"]) / 1024
bearer_token = AuthenticationMiddleware.get_token_from_auth_header(request)
- org_id = AuthenticationMiddleware.get_account_from_bearer_token(
- bearer_token
- )
+ org_id = AuthenticationMiddleware.get_account_from_bearer_token(bearer_token)
x2_text_audit: X2TextAudit = X2TextAudit.create(
org_id=org_id,
@@ -116,9 +108,7 @@ def process() -> Any:
)
}
- unstructured_api_key = X2TextUtil.get_value_for_key(
- UNSTRUCTURED_API_KEY, form_data
- )
+ unstructured_api_key = X2TextUtil.get_value_for_key(UNSTRUCTURED_API_KEY, form_data)
headers = {
"accept": "application/json",
"unstructured-api-key": unstructured_api_key,
@@ -139,13 +129,9 @@ def process() -> Any:
file_stream = BytesIO(response_text.encode("utf-8"))
x2_text_audit.status = "Success"
x2_text_audit.save()
- return send_file(
- file_stream, download_name="infile.txt", as_attachment=True
- )
+ return send_file(file_stream, download_name="infile.txt", as_attachment=True)
x2_text_audit.status = "Failed"
x2_text_audit.save()
return_val = X2TextUtil.read_response(response=response)
- logging.error(
- "Text extraction failed: [%s] %s", response.status_code, return_val
- )
+ logging.error("Text extraction failed: [%s] %s", response.status_code, return_val)
return return_val, response.status_code