LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
fix: flake8 black file changes (#259)
Kirtiman Mishra6871fe1d8dd5
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex 181d316ee..13835970d 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -50,23 +50,23 @@ repos:rev: 24.3.0hooks:- id: black- args: [--config=pyproject.toml, -l 80]+ args: [--config=pyproject.toml, -l 88]# language: systemexclude: |(?x)^(unstract/flags/src/unstract/flags/evaluation_.*\.py|)$- # - repo: https://github.com/pycqa/flake8- # rev: 7.0.0- # hooks:- # - id: flake8- # args: [--max-line-length=80]- # exclude: |- # (?x)^(- # .*migrations/.*\.py|- # core/tests/.*|- # unstract/flags/src/unstract/flags/evaluation_.*\.py|- # )$+ - repo: https://github.com/pycqa/flake8+ rev: 7.0.0+ hooks:+ - id: flake8+ args: [--max-line-length=88]+ exclude: |+ (?x)^(+ .*migrations/.*\.py|+ core/tests/.*|+ unstract/flags/src/unstract/flags/evaluation_.*\.py|+ )$- repo: https://github.com/pycqa/isortrev: 5.13.2hooks:diff --git a/backend/account/authentication_controller.py b/backend/account/authentication_controller.pyindex ede02b04a..48fa8afdb 100644--- a/backend/account/authentication_controller.py+++ b/backend/account/authentication_controller.py@@ -12,11 +12,7 @@ from account.constants import (ErrorMessage,OrganizationMemberModel,)-from account.custom_exceptions import (- DuplicateData,- Forbidden,- UserNotExistError,-)+from account.custom_exceptions import DuplicateData, Forbidden, UserNotExistErrorfrom account.dto import (CallbackData,MemberInvitation,@@ -109,28 +105,20 @@ class AuthenticationController:frontend/src/components/error/GenericError/GenericError.jsx."""if ex.code == AuthorizationErrorCode.IDM: # type: ignorequery_params = {"code": AuthorizationErrorCode.IDM}- return redirect(- f"{settings.ERROR_URL}?{urlencode(query_params)}"- )+ return redirect(f"{settings.ERROR_URL}?{urlencode(query_params)}")elif ex.code == AuthorizationErrorCode.UMM: # type: ignorequery_params = {"code": AuthorizationErrorCode.UMM}- return redirect(- f"{settings.ERROR_URL}?{urlencode(query_params)}"- )+ return redirect(f"{settings.ERROR_URL}?{urlencode(query_params)}")return redirect(f"{settings.ERROR_URL}")if member.organization_id and member.role and len(member.role) > 0:organization: Optional[Organization] = (- OrganizationService.get_organization_by_org_id(- member.organization_id- )+ OrganizationService.get_organization_by_org_id(member.organization_id))if organization:try:- self.create_tenant_user(- organization=organization, user=user- )+ self.create_tenant_user(organization=organization, user=user)except UndefinedTable:pass@@ -159,7 +147,10 @@ class AuthenticationController:except Exception as ex:#self.user_logout(request)- if hasattr(ex, "code") and ex.code == AuthorizationErrorCode.USF: # type: ignore+ if (+ hasattr(ex, "code")+ and ex.code == AuthorizationErrorCode.USF # type: ignore+ ):response = Response(status=status.HTTP_412_PRECONDITION_FAILED,data={"domain": ex.data.get("domain")}, # type: ignore@@ -185,9 +176,7 @@ class AuthenticationController:return response- def set_user_organization(- self, request: Request, organization_id: str- ) -> Response:+ def set_user_organization(self, request: Request, organization_id: str) -> Response:user: User = request.usernew_organization = Falseorganization_ids = CacheService.get_user_organizations(user.user_id)@@ -203,9 +192,7 @@ class AuthenticationController:if not organization:try:organization_data: OrganizationData = (- self.auth_service.get_organization_by_org_id(- organization_id- )+ self.auth_service.get_organization_by_org_id(organization_id))except ValueError:raise OrganizationNotExist()@@ -227,23 +214,19 @@ class AuthenticationController:user=user, organization=organization)user_info: Optional[UserInfo] = self.get_user_info(request)- serialized_user_info = SetOrganizationsResponseSerializer(- user_info- ).data+ serialized_user_info = SetOrganizationsResponseSerializer(user_info).dataorganization_info = OrganizationSerializer(organization).dataresponse: Response = Response(status=status.HTTP_200_OK,data={"user": serialized_user_info,"organization": organization_info,- f"{Common.LOG_EVENTS_ID}": StateStore.get(- Common.LOG_EVENTS_ID- ),+ f"{Common.LOG_EVENTS_ID}": StateStore.get(Common.LOG_EVENTS_ID),},)# Update user session data in redis- user_session_info: dict[str, Any] = (- CacheService.get_user_session_info(user.email)+ user_session_info: dict[str, Any] = CacheService.get_user_session_info(+ user.email)user_session_info["current_org"] = organization_idCacheService.set_user_session_info(user_session_info)@@ -269,9 +252,7 @@ class AuthenticationController:def get_organization_info(self, org_id: str) -> Optional[Organization]:organization = self.auth_service.get_organization_info(org_id)if not organization:- organization = OrganizationService.get_organization_by_org_id(- org_id=org_id- )+ organization = OrganizationService.get_organization_by_org_id(org_id=org_id)return organizationdef make_organization_and_add_member(@@ -302,9 +283,7 @@ class AuthenticationController:members: list[OrganizationMember] = OrganizationMember.objects.all()return members- def get_organization_members_by_user(- self, user: User- ) -> OrganizationMember:+ def get_organization_members_by_user(self, user: User) -> OrganizationMember:member: OrganizationMember = OrganizationMember.objects.filter(user=user).first()@@ -313,16 +292,10 @@ class AuthenticationController:def get_user_roles(self) -> list[UserRoleData]:return self.auth_service.get_roles()- def get_user_invitations(- self, organization_id: str- ) -> list[MemberInvitation]:- return self.auth_service.get_invitations(- organization_id=organization_id- )+ def get_user_invitations(self, organization_id: str) -> list[MemberInvitation]:+ return self.auth_service.get_invitations(organization_id=organization_id)- def delete_user_invitation(- self, organization_id: str, invitation_id: str- ) -> bool:+ def delete_user_invitation(self, organization_id: str, invitation_id: str) -> bool:return self.auth_service.delete_invitation(organization_id=organization_id, invitation_id=invitation_id)@@ -355,9 +328,7 @@ class AuthenticationController:email = user_item.get("email")role = user_item.get("role")if email:- user = self.organization_member_service.get_user_by_email(- email=email- )+ user = self.organization_member_service.get_user_by_email(email=email)user_response = {}user_response["email"] = emailstatus = False@@ -384,9 +355,7 @@ class AuthenticationController:admin_user = OrganizationMember.objects.get(user=admin.id)user_ids = OrganizationMember.objects.filter(user__email__in=user_emails- ).values_list(- OrganizationMemberModel.USER_ID, OrganizationMemberModel.ID- )+ ).values_list(OrganizationMemberModel.USER_ID, OrganizationMemberModel.ID)user_ids_list: list[str] = []ids_list: list[str] = []for user in user_ids:@@ -430,8 +399,8 @@ class AuthenticationController:self, admin: User, org_id: str, email: str, role: str) -> Optional[str]:admin_user = OrganizationMember.objects.get(user=admin.id)- organization_member = (- self.organization_member_service.get_user_by_email(email=email)+ organization_member = self.organization_member_service.get_user_by_email(+ email=email)if organization_member:current_roles = self.auth_service.remove_organization_user_role(@@ -447,33 +416,27 @@ class AuthenticationController:return Nonedef save_orgnanization_user_role(self, user_id: str, role: str) -> None:- organization_user = (- self.organization_member_service.get_user_by_user_id(- user_id=user_id- )+ organization_user = self.organization_member_service.get_user_by_user_id(+ user_id=user_id)if organization_user:# consider single roleorganization_user.role = roleorganization_user.save()- def create_tenant_user(- self, organization: Organization, user: User- ) -> None:+ def create_tenant_user(self, organization: Organization, user: User) -> None:with tenant_context(organization):- existing_tenant_user = (- self.organization_member_service.get_user_by_id(id=user.id)+ existing_tenant_user = self.organization_member_service.get_user_by_id(+ id=user.id)if existing_tenant_user:Logger.info(f"{existing_tenant_user.user.email} Already exist")else:account_user = self.get_or_create_user(user=user)if account_user:- user_roles = (- self.auth_service.get_organization_role_of_user(- user_id=account_user.user_id,- organization_id=organization.organization_id,- )+ user_roles = self.auth_service.get_organization_role_of_user(+ user_id=account_user.user_id,+ organization_id=organization.organization_id,)user_role = user_roles[0]tenant_user: OrganizationMember = OrganizationMember(diff --git a/backend/account/authentication_helper.py b/backend/account/authentication_helper.pyindex 2aa6f341b..5336b672b 100644--- a/backend/account/authentication_helper.py+++ b/backend/account/authentication_helper.py@@ -3,9 +3,7 @@ from typing import Anyfrom account.dto import MemberDatafrom account.models import Organization, User-from platform_settings.platform_auth_service import (- PlatformAuthenticationService,-)+from platform_settings.platform_auth_service import PlatformAuthenticationServicelogger = logging.getLogger(__name__)diff --git a/backend/account/authentication_plugin_registry.py b/backend/account/authentication_plugin_registry.pyindex 9503b108d..cd630fdaf 100644--- a/backend/account/authentication_plugin_registry.py+++ b/backend/account/authentication_plugin_registry.py@@ -92,7 +92,5 @@ class AuthenticationPluginRegistry:"""chosen_auth_module = next(iter(cls.auth_modules.values()))chosen_metadata = chosen_auth_module[PluginConfig.AUTH_METADATA]- service_class_name = chosen_metadata[- PluginConfig.METADATA_SERVICE_CLASS- ]+ service_class_name = chosen_metadata[PluginConfig.METADATA_SERVICE_CLASS]return service_class_name()diff --git a/backend/account/authentication_service.py b/backend/account/authentication_service.pyindex d37ed473a..fe3c92aca 100644--- a/backend/account/authentication_service.py+++ b/backend/account/authentication_service.py@@ -62,9 +62,7 @@ class AuthenticationService:if self.authenticate_and_login(request, username, password):return redirect(settings.WEB_APP_ORIGIN_URL)- return self.render_login_page_with_error(- request, ErrorMessage.USER_LOGIN_ERROR- )+ return self.render_login_page_with_error(request, ErrorMessage.USER_LOGIN_ERROR)def authenticate_and_login(self, request: Request, username: str, password: str@@ -98,9 +96,7 @@ class AuthenticationService:def render_login_page(self, request: Request) -> Any:return render(request, UserLoginTemplate.TEMPLATE)- def render_login_page_with_error(- self, request: Request, error_message: str- ) -> Any:+ def render_login_page_with_error(self, request: Request, error_message: str) -> Any:return render(request,UserLoginTemplate.TEMPLATE,@@ -251,9 +247,7 @@ class AuthenticationService:def get_invitations(self, organization_id: str) -> list[MemberInvitation]:raise MethodNotImplemented()- def delete_invitation(- self, organization_id: str, invitation_id: str- ) -> bool:+ def delete_invitation(self, organization_id: str, invitation_id: str) -> bool:raise MethodNotImplemented()def add_organization_user_role(@@ -307,9 +301,7 @@ class AuthenticationService:):return False- user, created = User.objects.get_or_create(- username=DefaultOrg.MOCK_USER- )+ user, created = User.objects.get_or_create(username=DefaultOrg.MOCK_USER)if created:user.password = make_password(DefaultOrg.MOCK_USER_PASSWORD)else:diff --git a/backend/account/custom_auth_middleware.py b/backend/account/custom_auth_middleware.pyindex 13533b60d..4812df842 100644--- a/backend/account/custom_auth_middleware.py+++ b/backend/account/custom_auth_middleware.py@@ -20,9 +20,7 @@ class CustomAuthMiddleware:def __call__(self, request: HttpRequest) -> HttpResponse:# Returns result without authenticated if added in whitelisted paths- if any(- request.path.startswith(path) for path in settings.WHITELISTED_PATHS- ):+ if any(request.path.startswith(path) for path in settings.WHITELISTED_PATHS):return self.get_response(request)tenantAccessiblePublicPath = False@@ -56,9 +54,7 @@ class CustomAuthMiddleware:org_id = DefaultOrg.MOCK_ORGif not request.user.is_authenticated:return- user_session_info = CacheService.get_user_session_info(- request.user.email- )+ user_session_info = CacheService.get_user_session_info(request.user.email)if not user_session_info:user_info: UserSessionInfo = UserSessionInfo(id=request.user.id,@@ -67,9 +63,7 @@ class CustomAuthMiddleware:current_org=org_id,)CacheService.set_user_session_info(user_info)- user_session_info = CacheService.get_user_session_info(- request.user.email- )+ user_session_info = CacheService.get_user_session_info(request.user.email)request.org_id = org_idrequest.session["user"] = user_session_inforequest.session.save()@@ -93,16 +87,10 @@ class CustomAuthMiddleware:if not current_org:return- if (- current_org != connection.get_schema()- and not tenantAccessiblePublicPath- ):+ if current_org != connection.get_schema() and not tenantAccessiblePublicPath:return- if (- current_org == Common.PUBLIC_SCHEMA_NAME- or tenantAccessiblePublicPath- ):+ if current_org == Common.PUBLIC_SCHEMA_NAME or tenantAccessiblePublicPath:user_service = UserService()else:organization_member_service = OrganizationMemberService()diff --git a/backend/account/custom_exceptions.py b/backend/account/custom_exceptions.pyindex 66e3671df..bec24e16c 100644--- a/backend/account/custom_exceptions.py+++ b/backend/account/custom_exceptions.py@@ -18,9 +18,7 @@ class DuplicateData(APIException):status_code = 400default_detail = "Duplicate Data"- def __init__(- self, detail: Optional[str] = None, code: Optional[int] = None- ):+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):if detail is not None:self.detail = detailif code is not None:@@ -32,9 +30,7 @@ class TableNotExistError(APIException):status_code = 400default_detail = "Unknown Table"- def __init__(- self, detail: Optional[str] = None, code: Optional[int] = None- ):+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):if detail is not None:self.detail = detailif code is not None:@@ -46,9 +42,7 @@ class UserNotExistError(APIException):status_code = 400default_detail = "Unknown User"- def __init__(- self, detail: Optional[str] = None, code: Optional[int] = None- ):+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):if detail is not None:self.detail = detailif code is not None:diff --git a/backend/account/migrations/0001_initial.py b/backend/account/migrations/0001_initial.pyindex 0ff85cfe8..e8622577b 100644--- a/backend/account/migrations/0001_initial.py+++ b/backend/account/migrations/0001_initial.py@@ -222,9 +222,7 @@ class Migration(migrations.Migration):),("domain",- models.CharField(- db_index=True, max_length=253, unique=True- ),+ models.CharField(db_index=True, max_length=253, unique=True),),("is_primary",diff --git a/backend/account/migrations/0003_platformkey.py b/backend/account/migrations/0003_platformkey.pyindex 80cdc227d..f01cc5295 100644--- a/backend/account/migrations/0003_platformkey.py+++ b/backend/account/migrations/0003_platformkey.py@@ -28,9 +28,7 @@ class Migration(migrations.Migration):("key", models.UUIDField(default=uuid.uuid4)),("key_name",- models.CharField(- blank=True, max_length=64, null=True, unique=True- ),+ models.CharField(blank=True, max_length=64, null=True, unique=True),),("is_active", models.BooleanField(default=False)),(diff --git a/backend/account/models.py b/backend/account/models.pyindex 63dade30d..2eae51073 100644--- a/backend/account/models.py+++ b/backend/account/models.py@@ -99,9 +99,7 @@ class PlatformKey(models.Model):id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)key = models.UUIDField(default=uuid.uuid4)- key_name = models.CharField(- max_length=KEY_SIZE, null=False, blank=True, default=""- )+ key_name = models.CharField(max_length=KEY_SIZE, null=False, blank=True, default="")is_active = models.BooleanField(default=False)organization = models.ForeignKey("Organization",diff --git a/backend/account/organization.py b/backend/account/organization.pyindex a584bf3a8..5cd547b3f 100644--- a/backend/account/organization.py+++ b/backend/account/organization.py@@ -41,9 +41,7 @@ class OrganizationService:cls.add(organization_id=organization_id)except IntegrityError as error:- Logger.info(- f"[Duplicate Id] Failed to create Organization Error: {error}"- )+ Logger.info(f"[Duplicate Id] Failed to create Organization Error: {error}")raise error# Add one or more domains for the tenantdomain = Domain()diff --git a/backend/account/subscription_loader.py b/backend/account/subscription_loader.pyindex c516dc3bf..d380ed19d 100644--- a/backend/account/subscription_loader.py+++ b/backend/account/subscription_loader.py@@ -24,12 +24,8 @@ def load_plugins() -> list[Any]:"""Iterate through the subscription plugins and register them."""plugins_app = apps.get_app_config(SubscriptionConfig.PLUGINS_APP)package_path = plugins_app.module.__package__- subscription_dir = os.path.join(- plugins_app.path, SubscriptionConfig.PLUGIN_DIR- )- subscription_package_path = (- f"{package_path}.{SubscriptionConfig.PLUGIN_DIR}"- )+ subscription_dir = os.path.join(plugins_app.path, SubscriptionConfig.PLUGIN_DIR)+ subscription_package_path = f"{package_path}.{SubscriptionConfig.PLUGIN_DIR}"subscription_plugins: list[Any] = []if not os.path.exists(subscription_dir):@@ -48,9 +44,7 @@ def load_plugins() -> list[Any]:else:continuetry:- full_module_path = (- f"{subscription_package_path}.{subscription_module_name}"- )+ full_module_path = f"{subscription_package_path}.{subscription_module_name}"module = import_module(full_module_path)metadata = getattr(module, SubscriptionConfig.METADATA, {})diff --git a/backend/account/urls.py b/backend/account/urls.pyindex 1987336a6..8cc60e65c 100644--- a/backend/account/urls.py+++ b/backend/account/urls.py@@ -15,10 +15,6 @@ urlpatterns = [path("logout", logout, name="logout"),path("callback", callback, name="callback"),path("organization", get_organizations, name="get_organizations"),- path(- "organization/<str:id>/set", set_organization, name="set_organization"- ),- path(- "organization/create", create_organization, name="create_organization"- ),+ path("organization/<str:id>/set", set_organization, name="set_organization"),+ path("organization/create", create_organization, name="create_organization"),]diff --git a/backend/account/views.py b/backend/account/views.pyindex 3fdea3772..22a9b927a 100644--- a/backend/account/views.py+++ b/backend/account/views.py@@ -2,10 +2,7 @@ import loggingfrom typing import Anyfrom account.authentication_controller import AuthenticationController-from account.dto import (- OrganizationSignupRequestBody,- OrganizationSignupResponse,-)+from account.dto import OrganizationSignupRequestBody, OrganizationSignupResponsefrom account.models import Organizationfrom account.organization import OrganizationServicefrom account.serializer import (@@ -25,9 +22,7 @@ def create_organization(request: Request) -> Response:serializer = OrganizationSignupSerializer(data=request.data)serializer.is_valid(raise_exception=True)try:- requestBody: OrganizationSignupRequestBody = makeSignupRequestParams(- serializer- )+ requestBody: OrganizationSignupRequestBody = makeSignupRequestParams(serializer)organization: Organization = OrganizationService.create_organization(requestBody.name,diff --git a/backend/adapter_processor/adapter_processor.py b/backend/adapter_processor/adapter_processor.pyindex 38585488f..f8c4bb1ef 100644--- a/backend/adapter_processor/adapter_processor.py+++ b/backend/adapter_processor/adapter_processor.py@@ -11,9 +11,7 @@ from adapter_processor.exceptions import ()from django.conf import settingsfrom django.core.exceptions import ObjectDoesNotExist-from platform_settings.platform_auth_service import (- PlatformAuthenticationService,-)+from platform_settings.platform_auth_service import PlatformAuthenticationServicefrom unstract.adapters.adapterkit import Adapterkitfrom unstract.adapters.base import Adapterfrom unstract.adapters.enums import AdapterTypes@@ -61,9 +59,7 @@ class AdapterProcessor:{AdapterKeys.ID: each_adapter.get(AdapterKeys.ID),AdapterKeys.NAME: each_adapter.get(AdapterKeys.NAME),- AdapterKeys.DESCRIPTION: each_adapter.get(- AdapterKeys.DESCRIPTION- ),+ AdapterKeys.DESCRIPTION: each_adapter.get(AdapterKeys.DESCRIPTION),AdapterKeys.ICON: each_adapter.get(AdapterKeys.ICON),AdapterKeys.ADAPTER_TYPE: each_adapter.get(AdapterKeys.ADAPTER_TYPE@@ -79,9 +75,7 @@ class AdapterProcessor:"id", adapter_id)if len(updated_adapters) == 0:- logger.error(- f"Invalid adapter ID {adapter_id} while invoking utility"- )+ logger.error(f"Invalid adapter ID {adapter_id} while invoking utility")raise InValidAdapterId()return updated_adapters[0].get(key_value)@@ -89,25 +83,14 @@ class AdapterProcessor:def test_adapter(adapter_id: str, adapter_metadata: dict[str, Any]) -> bool:logger.info(f"Testing adapter: {adapter_id}")try:- adapter_class = Adapterkit().get_adapter_class_by_adapter_id(- adapter_id- )-- if (- adapter_metadata.pop(AdapterKeys.ADAPTER_TYPE)- == AdapterKeys.X2TEXT- ):- adapter_metadata[X2TextConstants.X2TEXT_HOST] = (- settings.X2TEXT_HOST- )- adapter_metadata[X2TextConstants.X2TEXT_PORT] = (- settings.X2TEXT_PORT- )- platform_key = (- PlatformAuthenticationService.get_active_platform_key()- )- adapter_metadata[X2TextConstants.PLATFORM_SERVICE_API_KEY] = (- str(platform_key.key)+ adapter_class = Adapterkit().get_adapter_class_by_adapter_id(adapter_id)++ if adapter_metadata.pop(AdapterKeys.ADAPTER_TYPE) == AdapterKeys.X2TEXT:+ adapter_metadata[X2TextConstants.X2TEXT_HOST] = settings.X2TEXT_HOST+ adapter_metadata[X2TextConstants.X2TEXT_PORT] = settings.X2TEXT_PORT+ platform_key = PlatformAuthenticationService.get_active_platform_key()+ adapter_metadata[X2TextConstants.PLATFORM_SERVICE_API_KEY] = str(+ platform_key.key)adapter_instance = adapter_class(adapter_metadata)@@ -135,10 +118,8 @@ class AdapterProcessor:) = UserDefaultAdapter.objects.get_or_create(user=user)if default_triad.get(AdapterKeys.LLM_DEFAULT, None):- user_default_adapter.default_llm_adapter = (- AdapterInstance.objects.get(- pk=default_triad[AdapterKeys.LLM_DEFAULT]- )+ user_default_adapter.default_llm_adapter = AdapterInstance.objects.get(+ pk=default_triad[AdapterKeys.LLM_DEFAULT])if default_triad.get(AdapterKeys.EMBEDDING_DEFAULT, None):user_default_adapter.default_embedding_adapter = (@@ -207,9 +188,7 @@ class AdapterProcessor:the specified adapter type."""- adapters: list[AdapterInstance] = AdapterInstance.objects.for_user(- user- ).filter(+ adapters: list[AdapterInstance] = AdapterInstance.objects.for_user(user).filter(adapter_type=adapter_type.value,)return adapters@@ -271,8 +250,7 @@ class AdapterProcessor:except ObjectDoesNotExist as e:logger.error(f"No default adapters found: {e}")raise InternalServiceError(- "No default adapters found, "- "configure them through Platform Settings"+ "No default adapters found, " "configure them through Platform Settings")except Exception as e:logger.error(f"Error occurred while fetching default adapters: {e}")diff --git a/backend/adapter_processor/constants.py b/backend/adapter_processor/constants.pyindex 196166066..8c9326066 100644--- a/backend/adapter_processor/constants.py+++ b/backend/adapter_processor/constants.py@@ -21,8 +21,7 @@ class AdapterKeys:X2TEXT_DEFAULT = "x2text_default"SHARED_USERS = "shared_users"ADAPTER_NAME_EXISTS = (- "Configuration with this ID already exists. "- "Please try with a different ID"+ "Configuration with this ID already exists. " "Please try with a different ID")ADAPTER_CREATED_BY = "created_by_email"ADAPTER_CONTEXT_WINDOW_SIZE = "context_window_size"diff --git a/backend/adapter_processor/serializers.py b/backend/adapter_processor/serializers.pyindex 7d1f75022..ef42ae773 100644--- a/backend/adapter_processor/serializers.py+++ b/backend/adapter_processor/serializers.py@@ -31,9 +31,7 @@ class BaseAdapterSerializer(AuditSerializer):class DefaultAdapterSerializer(serializers.Serializer):- llm_default = serializers.CharField(- max_length=FLC.UUID_LENGTH, required=False- )+ llm_default = serializers.CharField(max_length=FLC.UUID_LENGTH, required=False)embedding_default = serializers.CharField(max_length=FLC.UUID_LENGTH, required=False)@@ -52,9 +50,7 @@ class AdapterInstanceSerializer(BaseAdapterSerializer):if data.get(AdapterKeys.ADAPTER_METADATA, None):encryption_secret: str = settings.ENCRYPTION_KEYf: Fernet = Fernet(encryption_secret.encode("utf-8"))- json_string: str = json.dumps(- data.pop(AdapterKeys.ADAPTER_METADATA)- )+ json_string: str = json.dumps(data.pop(AdapterKeys.ADAPTER_METADATA))data[AdapterKeys.ADAPTER_METADATA_B] = f.encrypt(json_string.encode("utf-8")diff --git a/backend/adapter_processor/views.py b/backend/adapter_processor/views.pyindex 8025f10d5..650bb5a9d 100644--- a/backend/adapter_processor/views.py+++ b/backend/adapter_processor/views.py@@ -57,9 +57,7 @@ class DefaultAdapterViewSet(ModelViewSet):self, request: Request, *args: tuple[Any], **kwargs: dict[str, Any]) -> HttpResponse:try:- user_default_adapter = UserDefaultAdapter.objects.get(- user=request.user- )+ user_default_adapter = UserDefaultAdapter.objects.get(user=request.user)serializer = UserDefaultAdapterSerializer(user_default_adapter).datareturn Response(serializer)@@ -98,23 +96,17 @@ class AdapterViewSet(GenericViewSet):adapter_name = request.GET.get(AdapterKeys.ID)if adapter_name is None or adapter_name == "":raise IdIsMandatory()- json_schema = AdapterProcessor.get_json_schema(- adapter_id=adapter_name- )+ json_schema = AdapterProcessor.get_json_schema(adapter_id=adapter_name)return Response(data=json_schema, status=status.HTTP_200_OK)def test(self, request: Request) -> Response:"""Tests the connector against the credentials passed."""- serializer: AdapterInstanceSerializer = self.get_serializer(- data=request.data- )+ serializer: AdapterInstanceSerializer = self.get_serializer(data=request.data)serializer.is_valid(raise_exception=True)adapter_id = serializer.validated_data.get(AdapterKeys.ADAPTER_ID)- adapter_metadata = serializer.validated_data.get(- AdapterKeys.ADAPTER_METADATA- )- adapter_metadata[AdapterKeys.ADAPTER_TYPE] = (- serializer.validated_data.get(AdapterKeys.ADAPTER_TYPE)+ adapter_metadata = serializer.validated_data.get(AdapterKeys.ADAPTER_METADATA)+ adapter_metadata[AdapterKeys.ADAPTER_TYPE] = serializer.validated_data.get(+ AdapterKeys.ADAPTER_TYPE)try:test_result = AdapterProcessor.test_adapter(@@ -138,9 +130,9 @@ class AdapterInstanceViewSet(ModelViewSet):self.request,constant.ADAPTER_TYPE,):- queryset = AdapterInstance.objects.for_user(- self.request.user- ).filter(**filter_args)+ queryset = AdapterInstance.objects.for_user(self.request.user).filter(+ **filter_args+ )else:queryset = AdapterInstance.objects.for_user(self.request.user)return queryset@@ -165,9 +157,7 @@ class AdapterInstanceViewSet(ModelViewSet):created,) = UserDefaultAdapter.objects.get_or_create(user=request.user)- adapter_type = serializer.validated_data.get(- AdapterKeys.ADAPTER_TYPE- )+ adapter_type = serializer.validated_data.get(AdapterKeys.ADAPTER_TYPE)if (adapter_type == AdapterKeys.LLM) and (not user_default_adapter.default_llm_adapter):@@ -189,9 +179,7 @@ class AdapterInstanceViewSet(ModelViewSet):user_default_adapter.save()except IntegrityError:- raise UniqueConstraintViolation(- f"{AdapterKeys.ADAPTER_NAME_EXISTS}"- )+ raise UniqueConstraintViolation(f"{AdapterKeys.ADAPTER_NAME_EXISTS}")except Exception as e:logger.error(f"Error saving adapter to DB: {e}")raise InternalServiceError@@ -213,18 +201,15 @@ class AdapterInstanceViewSet(ModelViewSet):)or (adapter_type == AdapterKeys.EMBEDDING- and adapter_instance- == user_default_adapter.default_embedding_adapter+ and adapter_instance == user_default_adapter.default_embedding_adapter)or (adapter_type == AdapterKeys.VECTOR_DB- and adapter_instance- == user_default_adapter.default_vector_db_adapter+ and adapter_instance == user_default_adapter.default_vector_db_adapter)or (adapter_type == AdapterKeys.X2TEXT- and adapter_instance- == user_default_adapter.default_x2text_adapter+ and adapter_instance == user_default_adapter.default_x2text_adapter)):logger.error("Cannot delete a default adapter")@@ -238,9 +223,7 @@ class AdapterInstanceViewSet(ModelViewSet):f" named {adapter_instance.adapter_name}")# TODO: Provide details of adpter usage with exception object- raise DeleteAdapterInUseError(- adapter_name=adapter_instance.adapter_name- )+ raise DeleteAdapterInUseError(adapter_name=adapter_instance.adapter_name)return Response(status=status.HTTP_204_NO_CONTENT)def partial_update(@@ -265,15 +248,9 @@ class AdapterInstanceViewSet(ModelViewSet):if user_default_adapter.default_llm_adapter == adapter:user_default_adapter.default_llm_adapter = None- elif (- user_default_adapter.default_embedding_adapter- == adapter- ):+ elif user_default_adapter.default_embedding_adapter == adapter:user_default_adapter.default_embedding_adapter = None- elif (- user_default_adapter.default_vector_db_adapter- == adapter- ):+ elif user_default_adapter.default_vector_db_adapter == adapter:user_default_adapter.default_vector_db_adapter = Noneelif user_default_adapter.default_x2text_adapter == adapter:user_default_adapter.default_x2text_adapter = None@@ -289,9 +266,7 @@ class AdapterInstanceViewSet(ModelViewSet):return super().partial_update(request, *args, **kwargs)@action(detail=True, methods=["get"])- def list_of_shared_users(- self, request: HttpRequest, pk: Any = None- ) -> Response:+ def list_of_shared_users(self, request: HttpRequest, pk: Any = None) -> Response:self.permission_classes = [IsOwnerOrSharedUser]adapter = (self.get_object()diff --git a/backend/api/deployment_helper.py b/backend/api/deployment_helper.pyindex d2136990b..8969bdb99 100644--- a/backend/api/deployment_helper.py+++ b/backend/api/deployment_helper.py@@ -44,9 +44,7 @@ class DeploymentHelper:"""@wraps(func)- def wrapper(- self: Any, request: Request, *args: Any, **kwargs: Any- ) -> Any:+ def wrapper(self: Any, request: Request, *args: Any, **kwargs: Any) -> Any:"""Wrapper to validate the inputs and key.Args:@@ -62,26 +60,18 @@ class DeploymentHelper:try:authorization_header = request.headers.get("Authorization")api_key = None- if authorization_header and authorization_header.startswith(- "Bearer "- ):+ if authorization_header and authorization_header.startswith("Bearer "):api_key = authorization_header.split(" ")[1]if not api_key:raise Forbidden("Missing api key")- org_name = kwargs.get("org_name") or request.data.get(- "org_name"- )- api_name = kwargs.get("api_name") or request.data.get(- "api_name"- )+ org_name = kwargs.get("org_name") or request.data.get("org_name")+ api_name = kwargs.get("api_name") or request.data.get("api_name")if not api_name:raise Forbidden("Missing api_name")tenant = get_tenant_model().objects.get(schema_name=org_name)with tenant_context(tenant):- api_deployment = (- DeploymentHelper.get_deployment_by_api_name(- api_name=api_name- )+ api_deployment = DeploymentHelper.get_deployment_by_api_name(+ api_name=api_name)DeploymentHelper.validate_api(api_deployment=api_deployment, api_key=api_key@@ -99,9 +89,7 @@ class DeploymentHelper:return wrapper@staticmethod- def validate_api(- api_deployment: Optional[APIDeployment], api_key: str- ) -> None:+ def validate_api(api_deployment: Optional[APIDeployment], api_key: str) -> None:"""Validating API and API key.Args:@@ -248,7 +236,7 @@ class DeploymentHelper:Returns:ReturnDict: status/result of execution"""- execution_response: ExecutionResponse = (- WorkflowHelper.get_status_of_async_task(execution_id=execution_id)+ execution_response: ExecutionResponse = WorkflowHelper.get_status_of_async_task(+ execution_id=execution_id)return execution_responsediff --git a/backend/api/exceptions.py b/backend/api/exceptions.pyindex 6d1b65ac0..f8bb97c5b 100644--- a/backend/api/exceptions.py+++ b/backend/api/exceptions.py@@ -12,9 +12,7 @@ class ApiKeyCreateException(APIException):status_code = 500default_detail = "Exception while create API key"- def __init__(- self, detail: Optional[str] = None, code: Optional[int] = None- ):+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):if detail is not None:self.detail = detailif code is not None:@@ -28,9 +26,7 @@ class Forbidden(APIException):"User is forbidden from performing this action. Please contact admin")- def __init__(- self, detail: Optional[str] = None, code: Optional[int] = None- ):+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):if detail is not None:self.detail = detailif code is not None:@@ -42,9 +38,7 @@ class APINotFound(APIException):status_code = 404default_detail = "Api not found"- def __init__(- self, detail: Optional[str] = None, code: Optional[int] = None- ):+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):if detail is not None:self.detail = detailif code is not None:@@ -56,9 +50,7 @@ class InvalidAPIRequest(APIException):status_code = 400default_detail = "Bad request"- def __init__(- self, detail: Optional[str] = None, code: Optional[int] = None- ):+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):if detail is not None:self.detail = detailif code is not None:@@ -70,9 +62,7 @@ class InactiveAPI(APIException):status_code = 404default_detail = "API not found or Inactive"- def __init__(- self, detail: Optional[str] = None, code: Optional[int] = None- ):+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):if detail is not None:self.detail = detailif code is not None:@@ -84,9 +74,7 @@ class UnauthorizedKey(APIException):status_code = 401default_detail = "Unauthorized"- def __init__(- self, detail: Optional[str] = None, code: Optional[int] = None- ):+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):if detail is not None:self.detail = detailif code is not None:diff --git a/backend/api/models.py b/backend/api/models.pyindex ac07ec74f..a2c81f4fc 100644--- a/backend/api/models.py+++ b/backend/api/models.py@@ -87,9 +87,7 @@ class APIDeployment(BaseModel):except APIDeployment.DoesNotExist:org_schema = connection.get_tenant().schema_name- self.api_endpoint = (- f"{ApiExecution.PATH}/{org_schema}/{self.api_name}/"- )+ self.api_endpoint = f"{ApiExecution.PATH}/{org_schema}/{self.api_name}/"super().save(*args, **kwargs)diff --git a/backend/api/serializers.py b/backend/api/serializers.pyindex b1622b431..1b56e0d55 100644--- a/backend/api/serializers.py+++ b/backend/api/serializers.py@@ -42,9 +42,7 @@ class APIKeySerializer(AuditSerializer):context."""context = self.context.get("context", {})deployment: APIDeployment = context.get("deployment")- representation: OrderedDict[str, Any] = super().to_representation(- instance- )+ representation: OrderedDict[str, Any] = super().to_representation(instance)if deployment:representation["api"] = deployment.idrepresentation["description"] = f"API Key for {deployment.name}"diff --git a/backend/backend/exceptions.py b/backend/backend/exceptions.pyindex ae98cdd05..01c59ad70 100644--- a/backend/backend/exceptions.py+++ b/backend/backend/exceptions.py@@ -22,8 +22,7 @@ class UnstractBaseException(APIException):detail = core_err.user_messageif detail and "Name or service not known" in str(detail):detail = (- "Failed to establish a new connection: "- "Name or service not known"+ "Failed to establish a new connection: " "Name or service not known")super().__init__(detail=detail, **kwargs)self._core_err = core_errdiff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 8284e1d9e..0f14e0b25 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -84,9 +84,7 @@ if ENV_FILE:WORKFLOW_ACTION_EXPIRATION_TIME_IN_SECOND = os.environ.get("WORKFLOW_ACTION_EXPIRATION_TIME_IN_SECOND", 10800)-WEB_APP_ORIGIN_URL = os.environ.get(- "WEB_APP_ORIGIN_URL", "http://localhost:3000"-)+WEB_APP_ORIGIN_URL = os.environ.get("WEB_APP_ORIGIN_URL", "http://localhost:3000")LOGIN_NEXT_URL = os.environ.get("LOGIN_NEXT_URL", "http://localhost:3000/org")LANDING_URL = os.environ.get("LANDING_URL", "http://localhost:3000/landing")@@ -98,9 +96,7 @@ DJANGO_APP_BACKEND_URL = os.environ.get(INTERNAL_SERVICE_API_KEY = os.environ.get("INTERNAL_SERVICE_API_KEY")GOOGLE_STORAGE_ACCESS_KEY_ID = os.environ.get("GOOGLE_STORAGE_ACCESS_KEY_ID")-GOOGLE_STORAGE_SECRET_ACCESS_KEY = os.environ.get(- "GOOGLE_STORAGE_SECRET_ACCESS_KEY"-)+GOOGLE_STORAGE_SECRET_ACCESS_KEY = os.environ.get("GOOGLE_STORAGE_SECRET_ACCESS_KEY")UNSTRACT_FREE_STORAGE_BUCKET_NAME = os.environ.get("UNSTRACT_FREE_STORAGE_BUCKET_NAME", "pandora-user-storage")@@ -337,9 +333,7 @@ CELERY_TASK_MAX_RETRIES = 3CELERY_TASK_RETRY_BACKOFF = 60 # Time in seconds before retrying the task# Feature Flag-FEATURE_FLAG_SERVICE_URL = {- "evaluate": f"{FLIPT_BASE_URL}/api/v1/flags/evaluate/"-}+FEATURE_FLAG_SERVICE_URL = {"evaluate": f"{FLIPT_BASE_URL}/api/v1/flags/evaluate/"}SCHEDULER_KWARGS = {"coalesce": True,@@ -357,16 +351,13 @@ AUTH_PASSWORD_VALIDATORS = ["UserAttributeSimilarityValidator",},{- "NAME": "django.contrib.auth.password_validation."- "MinimumLengthValidator",+ "NAME": "django.contrib.auth.password_validation." "MinimumLengthValidator",},{- "NAME": "django.contrib.auth.password_validation."- "CommonPasswordValidator",+ "NAME": "django.contrib.auth.password_validation." "CommonPasswordValidator",},{- "NAME": "django.contrib.auth.password_validation."- "NumericPasswordValidator",+ "NAME": "django.contrib.auth.password_validation." "NumericPasswordValidator",},]@@ -427,9 +418,7 @@ REDOC_SETTINGS = {}# Social Auth Settings-SOCIAL_AUTH_LOGIN_REDIRECT_URL = (- f"{WEB_APP_ORIGIN_URL}/oauth-status/?status=success"-)+SOCIAL_AUTH_LOGIN_REDIRECT_URL = f"{WEB_APP_ORIGIN_URL}/oauth-status/?status=success"SOCIAL_AUTH_LOGIN_ERROR_URL = f"{WEB_APP_ORIGIN_URL}/oauth-status/?status=error"SOCIAL_AUTH_EXTRA_DATA_EXPIRATION_TIME_IN_SECOND = os.environ.get("SOCIAL_AUTH_EXTRA_DATA_EXPIRATION_TIME_IN_SECOND", 3600diff --git a/backend/connector/connector_instance_helper.py b/backend/connector/connector_instance_helper.pyindex 2589d3c17..65a378f26 100644--- a/backend/connector/connector_instance_helper.py+++ b/backend/connector/connector_instance_helper.py@@ -106,9 +106,7 @@ class ConnectorInstanceHelper:if connector_name is not None:filter_params["connector_name"] = connector_name- connector_instances = ConnectorInstance.objects.filter(- **filter_params- ).all()+ connector_instances = ConnectorInstance.objects.filter(**filter_params).all()logger.info(f"Retrived connector instance values {connector_instances}")if values is not None:filtered_connector_instances = connector_instances.values(*values)@@ -153,13 +151,11 @@ class ConnectorInstanceHelper:filter_params["connector_name"] = connector_nametry:- connector_instance: ConnectorInstance = (- ConnectorInstance.objects.filter(**filter_params).first()- )+ connector_instance: ConnectorInstance = ConnectorInstance.objects.filter(+ **filter_params+ ).first()except Exception as exc:- logger.error(- f"Error occured while fetching connector instances {exc}"- )+ logger.error(f"Error occured while fetching connector instances {exc}")raise excreturn connector_instance@@ -331,8 +327,6 @@ class ConnectorInstanceHelper:filter_params: dict[str, Any] = {"workflow": workflow_id,}- connector_instances = ConnectorInstance.objects.filter(- **filter_params- ).all()+ connector_instances = ConnectorInstance.objects.filter(**filter_params).all()logger.info(f"Retrived connector instance values {connector_instances}")return list(connector_instances)diff --git a/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py b/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.pyindex 1e4a342c9..7c075439d 100644--- a/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py+++ b/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py@@ -24,9 +24,7 @@ class Migration(migrations.Migration):if hasattr(obj, "connector_metadata"):json_string: str = json.dumps(obj.connector_metadata)- obj.connector_metadata_b = f.encrypt(- json_string.encode("utf-8")- )+ obj.connector_metadata_b = f.encrypt(json_string.encode("utf-8"))obj.save()operations = [diff --git a/backend/connector/models.py b/backend/connector/models.pyindex eb0a3f7da..352ee3694 100644--- a/backend/connector/models.py+++ b/backend/connector/models.py@@ -48,17 +48,13 @@ class ConnectorInstance(BaseModel):null=False,blank=False,)- connector_id = models.CharField(- max_length=FLC.CONNECTOR_ID_LENGTH, default=""- )+ connector_id = models.CharField(max_length=FLC.CONNECTOR_ID_LENGTH, default="")# TODO Required to be removedconnector_metadata = ConnectorAuthJSONField(db_column="connector_metadata", null=False, blank=False, default=dict)connector_metadata_b = models.BinaryField(null=True)- connector_version = models.CharField(- max_length=VERSION_NAME_SIZE, default=""- )+ connector_version = models.CharField(max_length=VERSION_NAME_SIZE, default="")connector_type = models.CharField(choices=ConnectorType.choices)connector_auth = models.ForeignKey(ConnectorAuth, on_delete=models.SET_NULL, null=True, blank=Truediff --git a/backend/connector/serializers.py b/backend/connector/serializers.pyindex 992e2cfbf..13fea7ae8 100644--- a/backend/connector/serializers.py+++ b/backend/connector/serializers.py@@ -35,11 +35,9 @@ class ConnectorInstanceSerializer(AuditSerializer):and CIKey.CONNECTOR_METADATA in kwargs):try:- connector_oauth = (- ConnectorAuthHelper.get_or_create_connector_auth(- user=user, # type: ignore- oauth_credentials=kwargs[CIKey.CONNECTOR_METADATA],- )+ connector_oauth = ConnectorAuthHelper.get_or_create_connector_auth(+ user=user, # type: ignore+ oauth_credentials=kwargs[CIKey.CONNECTOR_METADATA],)kwargs[CIKey.CONNECTOR_AUTH] = connector_oauth(@@ -61,9 +59,7 @@ class ConnectorInstanceSerializer(AuditSerializer):if self.validated_data:self.validated_data.pop(CIKey.CONNECTOR_METADATA)- kwargs[CIKey.CONNECTOR_METADATA_B] = f.encrypt(- json_string.encode("utf-8")- )+ kwargs[CIKey.CONNECTOR_METADATA_B] = f.encrypt(json_string.encode("utf-8"))instance = super().save(**kwargs)return instance@@ -76,10 +72,8 @@ class ConnectorInstanceSerializer(AuditSerializer):if SerializerUtils.check_context_for_GET_or_POST(context=self.context):rep.pop(CIKey.CONNECTOR_AUTH)# set icon fields for UI- rep[ConnectorKeys.ICON] = (- ConnectorProcessor.get_connector_data_with_key(- instance.connector_id, ConnectorKeys.ICON- )+ rep[ConnectorKeys.ICON] = ConnectorProcessor.get_connector_data_with_key(+ instance.connector_id, ConnectorKeys.ICON)encryption_secret: str = settings.ENCRYPTION_KEYf: Fernet = Fernet(encryption_secret.encode("utf-8"))diff --git a/backend/connector/tests/connector_tests.py b/backend/connector/tests/connector_tests.pyindex 4e909f68b..cad6512f2 100644--- a/backend/connector/tests/connector_tests.py+++ b/backend/connector/tests/connector_tests.py@@ -203,9 +203,9 @@ class TestConnector(APITestCase):},}response = self.client.put(url, data, format="json")- nested_value = response.data["connector_metadata"][- "sample_metadata_json"- ]["key1"]+ nested_value = response.data["connector_metadata"]["sample_metadata_json"][+ "key1"+ ]self.assertEqual(response.status_code, status.HTTP_200_OK)self.assertEqual(nested_value, "value1")@@ -229,9 +229,9 @@ class TestConnector(APITestCase):},}response = self.client.put(url, data, format="json")- nested_value = response.data["connector_metadata"][- "sample_metadata_json"- ]["key1"]+ nested_value = response.data["connector_metadata"]["sample_metadata_json"][+ "key1"+ ]nested_list = response.data["connector_metadata"]["file_list"]last_val = nested_list.pop()@@ -296,9 +296,7 @@ class TestConnector(APITestCase):self.assertEqual(connector_id,- ConnectorInstance.objects.get(- connector_id=connector_id- ).connector_id,+ ConnectorInstance.objects.get(connector_id=connector_id).connector_id,)def test_connectors_update_json_field_patch(self) -> None:diff --git a/backend/connector/unstract_account.py b/backend/connector/unstract_account.pyindex 96d998a22..e8f496089 100644--- a/backend/connector/unstract_account.py+++ b/backend/connector/unstract_account.py@@ -35,9 +35,7 @@ class UnstractAccount:except ClientError as e:logger.info(f"{bucket_name} Folder {account_folder} does not exist")if e.response["Error"]["Code"] == "404":- logger.info(- f"Folder {account_folder} does not exist. Creating it..."- )+ logger.info(f"Folder {account_folder} does not exist. Creating it...")s3.put_object(Bucket=bucket_name, Key=account_folder)account_folder_output = f"{self.tenant}/{self.username}/output/"s3.put_object(Bucket=bucket_name, Key=account_folder_output)@@ -67,8 +65,7 @@ class UnstractAccount:folder, os.path.relpath(local_file_path, local_path))logger.info(- f"Uploading: {local_file_path} => "- f"s3://{bucket_name}/{s3_key}"+ f"Uploading: {local_file_path} => " f"s3://{bucket_name}/{s3_key}")try:s3.upload_file(local_file_path, bucket_name, s3_key)diff --git a/backend/connector/views.py b/backend/connector/views.pyindex 428a3678b..2443d6bd2 100644--- a/backend/connector/views.py+++ b/backend/connector/views.py@@ -41,9 +41,7 @@ class ConnectorInstanceViewSet(viewsets.ModelViewSet):queryset = ConnectorInstance.objects.all()return queryset- def _get_connector_metadata(- self, connector_id: str- ) -> Optional[dict[str, str]]:+ def _get_connector_metadata(self, connector_id: str) -> Optional[dict[str, str]]:"""Gets connector metadata for the ConnectorInstance.For non oauth based - obtains from request@@ -58,9 +56,7 @@ class ConnectorInstanceViewSet(viewsets.ModelViewSet):connector_metadata = Noneif ConnectorInstance.supportsOAuth(connector_id=connector_id):logger.info(f"Fetching oauth data for {connector_id}")- oauth_key = self.request.query_params.get(- ConnectorAuthKey.OAUTH_KEY- )+ oauth_key = self.request.query_params.get(ConnectorAuthKey.OAUTH_KEY)if oauth_key is None:logger.error("OAuth key missing")raise MissingParamException(param=ConnectorAuthKey.OAUTH_KEY)@@ -99,9 +95,7 @@ class ConnectorInstanceViewSet(viewsets.ModelViewSet):connector_metadata = Noneconnector_id = self.request.data.get(CIKey.CONNECTOR_ID)try:- connector_metadata = self._get_connector_metadata(- connector_id=connector_id- )+ connector_metadata = self._get_connector_metadata(connector_id=connector_id)except Exception as exc:logger.error(f"Error while obtaining ConnectorAuth: {exc}")raise OAuthTimeOutdiff --git a/backend/connector_auth/models.py b/backend/connector_auth/models.pyindex a9331e5b2..653400055 100644--- a/backend/connector_auth/models.py+++ b/backend/connector_auth/models.py@@ -57,10 +57,7 @@ class ConnectorAuth(AbstractUserSocialAuth):def set_extra_data(self, extra_data=None): # type: ignoreConnectorAuth.check_credential_format(extra_data)- if (- extra_data[SocialAuthConstants.PROVIDER]- == SocialAuthConstants.GOOGLE_OAUTH- ):+ if extra_data[SocialAuthConstants.PROVIDER] == SocialAuthConstants.GOOGLE_OAUTH:extra_data = GoogleAuthHelper.enrich_connector_metadata(extra_data)return super().set_extra_data(extra_data)@@ -73,17 +70,13 @@ class ConnectorAuth(AbstractUserSocialAuth):backend = self.get_backend_instance(strategy)if token and backend and hasattr(backend, "refresh_token"):response = backend.refresh_token(token, *args, **kwargs)- extra_data = backend.extra_data(- self, self.uid, response, self.extra_data- )+ extra_data = backend.extra_data(self, self.uid, response, self.extra_data)extra_data[SocialAuthConstants.PROVIDER] = backend.nameextra_data[SocialAuthConstants.UID] = self.uidif self.set_extra_data(extra_data): # type: ignoreself.save()- def get_and_refresh_tokens(- self, request: Request = None- ) -> tuple[JSONField, bool]:+ def get_and_refresh_tokens(self, request: Request = None) -> tuple[JSONField, bool]:"""Uses Social Auth's ability to refresh tokens if necessary.Returns:diff --git a/backend/connector_auth/pipeline/common.py b/backend/connector_auth/pipeline/common.pyindex 96d8e365c..3d82cbbf5 100644--- a/backend/connector_auth/pipeline/common.py+++ b/backend/connector_auth/pipeline/common.py@@ -13,9 +13,7 @@ from social_core.backends.oauth import BaseOAuth2logger = logging.getLogger(__name__)-def check_user_exists(- backend: BaseOAuth2, user: User, **kwargs: Any-) -> dict[str, str]:+def check_user_exists(backend: BaseOAuth2, user: User, **kwargs: Any) -> dict[str, str]:"""Checks if user is authenticated (will be handled in auth middleware,present as a fail safe)@@ -51,9 +49,7 @@ def cache_oauth_creds(settings.SOCIAL_AUTH_FIELDS_STORED_IN_SESSION[0],ConnectorAuthKey.OAUTH_KEY,)- extra_data = backend.extra_data(- user, uid, response, details, *args, **kwargs- )+ extra_data = backend.extra_data(user, uid, response, details, *args, **kwargs)extra_data[SocialAuthConstants.PROVIDER] = backend.nameextra_data[SocialAuthConstants.UID] = uiddiff --git a/backend/connector_auth/pipeline/google.py b/backend/connector_auth/pipeline/google.pyindex 6c505a31b..71a56fb33 100644--- a/backend/connector_auth/pipeline/google.py+++ b/backend/connector_auth/pipeline/google.py@@ -4,9 +4,7 @@ from connector_auth.constants import SocialAuthConstants as AuthConstantsfrom connector_auth.exceptions import EnrichConnectorMetadataExceptionfrom connector_processor.constants import ConnectorKeys-from unstract.connectors.filesystems.google_drive.constants import (- GDriveConstants,-)+from unstract.connectors.filesystems.google_drive.constants import GDriveConstantsclass GoogleAuthHelper:diff --git a/backend/connector_processor/connector_processor.py b/backend/connector_processor/connector_processor.pyindex 3edd053eb..dcababce3 100644--- a/backend/connector_processor/connector_processor.py+++ b/backend/connector_processor/connector_processor.py@@ -48,9 +48,7 @@ class ConnectorProcessor:)if len(updated_connectors) != 0:connector = updated_connectors[0]- schema_details[ConnectorKeys.OAUTH] = connector.get(- ConnectorKeys.OAUTH- )+ schema_details[ConnectorKeys.OAUTH] = connector.get(ConnectorKeys.OAUTH)schema_details[ConnectorKeys.SOCIAL_AUTH_URL] = connector.get(ConnectorKeys.SOCIAL_AUTH_URL)@@ -95,9 +93,7 @@ class ConnectorProcessor:ConnectorKeys.DESCRIPTION),ConnectorKeys.ICON: each_connector.get(ConnectorKeys.ICON),- CIKey.CONNECTOR_MODE: each_connector.get(- CIKey.CONNECTOR_MODE- ).name,+ CIKey.CONNECTOR_MODE: each_connector.get(CIKey.CONNECTOR_MODE).name,})@@ -140,13 +136,9 @@ class ConnectorProcessor:"""Generic Function to get connector data with provided key."""updated_connectors = fetch_connectors_by_key_value("id", connector_id)if len(updated_connectors) == 0:- logger.error(- f"Invalid connector ID {connector_id} while invoking utility"- )+ logger.error(f"Invalid connector ID {connector_id} while invoking utility")raise InValidConnectorId()- return fetch_connectors_by_key_value("id", connector_id)[0].get(- key_value- )+ return fetch_connectors_by_key_value("id", connector_id)[0].get(key_value)@staticmethoddef validate_connector_mode(connector_mode: str) -> ConnectorMode:diff --git a/backend/connector_processor/views.py b/backend/connector_processor/views.pyindex ada158542..55367c40d 100644--- a/backend/connector_processor/views.py+++ b/backend/connector_processor/views.py@@ -20,9 +20,7 @@ def get_connector_schema(request: HttpRequest) -> HttpResponse:connector_name = request.GET.get(ConnectorKeys.ID)if connector_name is None or connector_name == "":raise IdIsMandatory()- json_schema = ConnectorProcessor.get_json_schema(- connector_id=connector_name- )+ json_schema = ConnectorProcessor.get_json_schema(connector_id=connector_name)return Response(data=json_schema, status=status.HTTP_200_OK)@@ -42,9 +40,7 @@ def get_supported_connectors(request: HttpRequest) -> HttpResponse:connector_type = request.GET.get(ConnectorKeys.TYPE)connector_mode = request.GET.get(ConnectorKeys.CONNECTOR_MODE)if connector_mode:- connector_mode = ConnectorProcessor.validate_connector_mode(- connector_mode- )+ connector_mode = ConnectorProcessor.validate_connector_mode(connector_mode)if (connector_type == ConnectorKeys.INPUT@@ -69,9 +65,7 @@ class ConnectorViewSet(GenericViewSet):def test(self, request: Request) -> Response:"""Tests the connector against the credentials passed."""- serializer: TestConnectorSerializer = self.get_serializer(- data=request.data- )+ serializer: TestConnectorSerializer = self.get_serializer(data=request.data)serializer.is_valid(raise_exception=True)connector_id = serializer.validated_data.get(ConnectorKeys.CONNECTOR_ID)cred_string = serializer.validated_data.get(CIKey.CONNECTOR_METADATA)diff --git a/backend/feature_flag/urls.py b/backend/feature_flag/urls.pyindex 0bc4c274d..c90426d1e 100644--- a/backend/feature_flag/urls.py+++ b/backend/feature_flag/urls.py@@ -7,7 +7,5 @@ import feature_flag.views as viewsfrom django.urls import pathurlpatterns = [- path(- "evaluate/", views.evaluate_feature_flag, name="evaluate_feature_flag"- ),+ path("evaluate/", views.evaluate_feature_flag, name="evaluate_feature_flag"),]diff --git a/backend/file_management/exceptions.py b/backend/file_management/exceptions.pyindex b7bf42d22..b8cfb4700 100644--- a/backend/file_management/exceptions.py+++ b/backend/file_management/exceptions.py@@ -69,9 +69,7 @@ class ValidationError(APIException):status_code = 400default_detail = "Validation Error"- def __init__(- self, detail: Optional[str] = None, code: Optional[int] = None- ):+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):if detail is not None:self.detail = detailif code is not None:@@ -83,9 +81,7 @@ class FileDeletionFailed(APIException):status_code = 400default_detail = "Unable to delete file."- def __init__(- self, detail: Optional[str] = None, code: Optional[int] = None- ):+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):if detail is not None:self.detail = detailif code is not None:diff --git a/backend/file_management/file_management_dto.py b/backend/file_management/file_management_dto.pyindex 100efe1a4..a5cf585b7 100644--- a/backend/file_management/file_management_dto.py+++ b/backend/file_management/file_management_dto.py@@ -21,17 +21,13 @@ class FileInformation:self.type = file_info[FileInformationKey.FILE_TYPE]modified_at = file_info.get(FileInformationKey.FILE_LAST_MODIFIED)- self.modified_at = (- self.parse_datetime(modified_at) if modified_at else None- )+ self.modified_at = self.parse_datetime(modified_at) if modified_at else Noneself.content_type = file_content_typeself.size = file_info[FileInformationKey.FILE_SIZE]@staticmethod- def parse_datetime(- dt_string: Optional[Union[str, datetime]]- ) -> Optional[datetime]:+ def parse_datetime(dt_string: Optional[Union[str, datetime]]) -> Optional[datetime]:if isinstance(dt_string, str):return datetime.strptime(dt_string, "%Y-%m-%dT%H:%M:%S.%f%z")return dt_stringdiff --git a/backend/file_management/file_management_helper.py b/backend/file_management/file_management_helper.pyindex 2e904f942..0004176b2 100644--- a/backend/file_management/file_management_helper.py+++ b/backend/file_management/file_management_helper.py@@ -25,9 +25,7 @@ from fsspec import AbstractFileSystemfrom pydrive2.files import ApiRequestErrorfrom unstract.connectors.filesystems import connectors as fs_connectors-from unstract.connectors.filesystems.unstract_file_system import (- UnstractFileSystem,-)+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystemclass FileManagerHelper:@@ -38,9 +36,7 @@ class FileManagerHelper:"""Creates the `UnstractFileSystem` for the corresponding connector."""metadata = connector.metadataif connector.connector_id in fs_connectors:- connector = fs_connectors[connector.connector_id]["metadata"][- "connector"- ]+ connector = fs_connectors[connector.connector_id]["metadata"]["connector"]connector_class: UnstractFileSystem = connector(metadata)return connector_classelse:@@ -50,9 +46,7 @@ class FileManagerHelper:raise ConnectorClassNotFound@staticmethod- def list_files(- file_system: UnstractFileSystem, path: str- ) -> list[FileInformation]:+ def list_files(file_system: UnstractFileSystem, path: str) -> list[FileInformation]:fs = file_system.get_fsspec_fs()file_path = f"{path}"# TODO: Add below logic by checking each connector?@@ -73,9 +67,7 @@ class FileManagerHelper:raise FileListError()@staticmethod- def get_files(- fs: AbstractFileSystem, file_path: str- ) -> list[FileInformation]:+ def get_files(fs: AbstractFileSystem, file_path: str) -> list[FileInformation]:"""Iterate through the directories and make a list ofFileInformation."""if not file_path.endswith("/"):@@ -89,10 +81,7 @@ class FileManagerHelper:continue# Call fs.info() to get file size if its missing- if (- file_info.get("type") == "file"- and file_info.get("size") is None- ):+ if file_info.get("type") == "file" and file_info.get("size") is None:file_info = fs.info(file_name)file_content_type = file_info.get("ContentType")@@ -123,12 +112,8 @@ class FileManagerHelper:file.close()file = fs.open(file_path, "rb")- response = StreamingHttpResponse(- file, content_type=file_content_type- )- response["Content-Disposition"] = (- f"attachment; filename={base_name}"- )+ response = StreamingHttpResponse(file, content_type=file_content_type)+ response["Content-Disposition"] = f"attachment; filename={base_name}"return responseexcept ApiRequestError as exception:FileManagerHelper.logger.error(@@ -150,9 +135,7 @@ class FileManagerHelper:if fs.path and (not path or path == "/"):file_path = f"{fs.path}/"- file_path = (- file_path + "/" if not file_path.endswith("/") else file_path- )+ file_path = file_path + "/" if not file_path.endswith("/") else file_path# adding filename with pathfile_path += file_name@@ -160,9 +143,7 @@ class FileManagerHelper:remote_file.write(file.read())@staticmethod- def fetch_file_contents(- file_system: UnstractFileSystem, file_path: str- ) -> Any:+ def fetch_file_contents(file_system: UnstractFileSystem, file_path: str) -> Any:fs = file_system.get_fsspec_fs()try:file_info = fs.info(file_path)@@ -201,9 +182,7 @@ class FileManagerHelper:raise InvalidFileType@staticmethod- def delete_file(- file_system: UnstractFileSystem, path: str, file_name: str- ) -> bool:+ def delete_file(file_system: UnstractFileSystem, path: str, file_name: str) -> bool:fs = file_system.get_fsspec_fs()file_path = f"{path}"@@ -214,9 +193,7 @@ class FileManagerHelper:if fs.path and (not path or path == "/"):file_path = f"{fs.path}/"- file_path = (- file_path + "/" if not file_path.endswith("/") else file_path- )+ file_path = file_path + "/" if not file_path.endswith("/") else file_path# adding filename with pathfile_path += file_name@@ -254,8 +231,6 @@ class FileManagerHelper:os.makedirs(extract_file_path, exist_ok=True)os.makedirs(summarize_file_path, exist_ok=True)except OSError as e:- FileManagerHelper.logger.error(- f"Error while creating {file_path}: {e}"- )+ FileManagerHelper.logger.error(f"Error while creating {file_path}: {e}")raise TenantDirCreationErrorreturn str(file_path)diff --git a/backend/file_management/views.py b/backend/file_management/views.pyindex 879ead194..6cf34e456 100644--- a/backend/file_management/views.py+++ b/backend/file_management/views.py@@ -24,9 +24,7 @@ from rest_framework.response import Responsefrom rest_framework.versioning import URLPathVersioningfrom unstract.connectors.exceptions import ConnectorError-from unstract.connectors.filesystems.local_storage.local_storage import (- LocalStorageFS,-)+from unstract.connectors.filesystems.local_storage.local_storage import LocalStorageFSlogger = logging.getLogger(__name__)@@ -57,9 +55,7 @@ class FileManagementViewSet(viewsets.ModelViewSet):id: str = serializer.validated_data.get("connector_id")path: str = serializer.validated_data.get("path")try:- connector_instance: ConnectorInstance = (- ConnectorInstance.objects.get(pk=id)- )+ connector_instance: ConnectorInstance = ConnectorInstance.objects.get(pk=id)file_system = FileManagerHelper.get_file_system(connector_instance)files = FileManagerHelper.list_files(file_system, path)serializer = FileInfoSerializer(files, many=True)@@ -73,9 +69,7 @@ class FileManagementViewSet(viewsets.ModelViewSet):)raise ConnectorOAuthError()except ConnectorError as error:- logger.error(- f"ConnectorError thrown during file list, error {error}"- )+ logger.error(f"ConnectorError thrown during file list, error {error}")raise FileListError(core_err=error)except Exception as error:logger.error(f"Exception thrown from file list, error {error}")@@ -87,9 +81,7 @@ class FileManagementViewSet(viewsets.ModelViewSet):serializer.is_valid(raise_exception=True)id: str = serializer.validated_data.get("connector_id")path: str = serializer.validated_data.get("path")- connector_instance: ConnectorInstance = ConnectorInstance.objects.get(- pk=id- )+ connector_instance: ConnectorInstance = ConnectorInstance.objects.get(pk=id)file_system = FileManagerHelper.get_file_system(connector_instance)return FileManagerHelper.download_file(file_system, path)@@ -101,21 +93,15 @@ class FileManagementViewSet(viewsets.ModelViewSet):path: str = serializer.validated_data.get("path")uploaded_files: Any = serializer.validated_data.get("file")- connector_instance: ConnectorInstance = ConnectorInstance.objects.get(- pk=id- )+ connector_instance: ConnectorInstance = ConnectorInstance.objects.get(pk=id)file_system = FileManagerHelper.get_file_system(connector_instance)for uploaded_file in uploaded_files:file_name = uploaded_file.namelogger.info(- f"Uploading file: {file_name}"- if file_name- else "Uploading file"- )- FileManagerHelper.upload_file(- file_system, path, uploaded_file, file_name+ f"Uploading file: {file_name}" if file_name else "Uploading file")+ FileManagerHelper.upload_file(file_system, path, uploaded_file, file_name)return Response({"message": "Files are uploaded successfully!"})@action(detail=True, methods=["get"])diff --git a/backend/middleware/exception.py b/backend/middleware/exception.pyindex 6f4ba637f..7d4be00a6 100644--- a/backend/middleware/exception.py+++ b/backend/middleware/exception.py@@ -82,14 +82,12 @@ class ExceptionLoggingMiddleware:exception (Exception): Exception that was raised to be logged"""if hasattr(response, "status_code") and response.status_code >= 500:- message = (- "{method} {url} {status}\n\n{error}\n\n````{tb}````".format(- method=request.method,- url=request.build_absolute_uri(),- status=response.status_code,- error=repr(exception),- tb=traceback.format_exc(),- )+ message = "{method} {url} {status}\n\n{error}\n\n````{tb}````".format(+ method=request.method,+ url=request.build_absolute_uri(),+ status=response.status_code,+ error=repr(exception),+ tb=traceback.format_exc(),)else:message = "{method} {url} {status} {error}".format(diff --git a/backend/permissions/permission.py b/backend/permissions/permission.pyindex 978d2e03b..bf2f85906 100644--- a/backend/permissions/permission.py+++ b/backend/permissions/permission.py@@ -8,18 +8,14 @@ from rest_framework.views import APIViewclass IsOwner(permissions.BasePermission):"""Custom permission to only allow owners of an object."""- def has_object_permission(- self, request: Request, view: APIView, obj: Any- ) -> bool:+ def has_object_permission(self, request: Request, view: APIView, obj: Any) -> bool:return True if obj.created_by == request.user else Falseclass IsOwnerOrSharedUser(permissions.BasePermission):"""Custom permission to only allow owners and shared users of an object."""- def has_object_permission(- self, request: Request, view: APIView, obj: Any- ) -> bool:+ def has_object_permission(self, request: Request, view: APIView, obj: Any) -> bool:return (Trueif (diff --git a/backend/pipeline/manager.py b/backend/pipeline/manager.pyindex 3c6bfbc8e..bee1b0c93 100644--- a/backend/pipeline/manager.py+++ b/backend/pipeline/manager.py@@ -10,10 +10,7 @@ from pipeline.pipeline_processor import PipelineProcessorfrom rest_framework.request import Requestfrom rest_framework.response import Responsefrom utils.request.constants import RequestConstants-from workflow_manager.workflow.constants import (- WorkflowExecutionKey,- WorkflowKey,-)+from workflow_manager.workflow.constants import WorkflowExecutionKey, WorkflowKeyfrom workflow_manager.workflow.views import WorkflowViewSetfrom backend.constants import RequestHeader@@ -45,9 +42,7 @@ class PipelineManager:f"with_log = {with_log}")try:- pipeline: Pipeline = PipelineProcessor.initialize_pipeline_sync(- pipeline_id- )+ pipeline: Pipeline = PipelineProcessor.initialize_pipeline_sync(pipeline_id)# TODO: Use DRF's request and as_view() insteadrequest.data[WorkflowKey.WF_ID] = pipeline.workflow.idif execution_id is not None:@@ -69,12 +64,8 @@ class PipelineManager:) -> Optional[dict[str, Any]]:"""Gets the required data to be passed while executing a pipeline Anychanges to pipeline execution needs to be propagated here."""- callback_url = settings.DJANGO_APP_BACKEND_URL + reverse(- PipelineURL.EXECUTE- )- job_headers = {- RequestHeader.X_API_KEY: settings.INTERNAL_SERVICE_API_KEY- }+ callback_url = settings.DJANGO_APP_BACKEND_URL + reverse(PipelineURL.EXECUTE)+ job_headers = {RequestHeader.X_API_KEY: settings.INTERNAL_SERVICE_API_KEY}job_params = {WorkflowExecutionKey.WITH_LOG: "False"}job_kwargs = {RequestConstants.VERB: "POST",diff --git a/backend/pipeline/migrations/0001_initial.py b/backend/pipeline/migrations/0001_initial.pyindex 828bae01b..67c4d4ca7 100644--- a/backend/pipeline/migrations/0001_initial.py+++ b/backend/pipeline/migrations/0001_initial.py@@ -42,9 +42,7 @@ class Migration(migrations.Migration):("scheduled", models.BooleanField(default=False)),("cron_string",- models.TextField(- db_comment="UNIX cron string", max_length=256- ),+ models.TextField(db_comment="UNIX cron string", max_length=256),),("pipeline_type",diff --git a/backend/pipeline/serializers/crud.py b/backend/pipeline/serializers/crud.pyindex 3a3fdf721..56e5d1142 100644--- a/backend/pipeline/serializers/crud.py+++ b/backend/pipeline/serializers/crud.py@@ -37,9 +37,7 @@ class PipelineSerializer(AuditSerializer):)return pipeline- def _get_name_and_icon(- self, connectors: list[Any], connector_id: Any- ) -> Any:+ def _get_name_and_icon(self, connectors: list[Any], connector_id: Any) -> Any:for obj in connectors:if obj["id"] == connector_id:return obj["name"], obj["icon"]@@ -64,11 +62,9 @@ class PipelineSerializer(AuditSerializer):repr[PC.DESTINATION_NAME] = PC.NOT_CONFIGUREDfor instance in connector_instance_list:if instance.connector_type == "INPUT":- repr[PC.SOURCE_NAME], repr[PC.SOURCE_ICON] = (- self._get_name_and_icon(- connectors=connectors,- connector_id=instance.connector_id,- )+ repr[PC.SOURCE_NAME], repr[PC.SOURCE_ICON] = self._get_name_and_icon(+ connectors=connectors,+ connector_id=instance.connector_id,)if instance.connector_type == "OUTPUT":repr[PC.DESTINATION_NAME], repr[PC.DESTINATION_ICON] = (diff --git a/backend/pipeline/views.py b/backend/pipeline/views.pyindex 34c1fb05d..96cadd74f 100644--- a/backend/pipeline/views.py+++ b/backend/pipeline/views.py@@ -7,28 +7,20 @@ from django.db.models import QuerySetfrom drf_yasg import openapifrom drf_yasg.utils import swagger_auto_schemafrom permissions.permission import IsOwner-from pipeline.constants import (- PipelineConstants,- PipelineErrors,- PipelineExecutionKey,-)+from pipeline.constants import PipelineConstants, PipelineErrors, PipelineExecutionKeyfrom pipeline.constants import PipelineKey as PKfrom pipeline.manager import PipelineManagerfrom pipeline.models import Pipelinefrom pipeline.pipeline_processor import PipelineProcessorfrom pipeline.serializers.crud import PipelineSerializer-from pipeline.serializers.execute import (- PipelineExecuteSerializer as ExecuteSerializer,-)+from pipeline.serializers.execute import PipelineExecuteSerializer as ExecuteSerializerfrom pipeline.serializers.update import PipelineUpdateSerializerfrom rest_framework import serializers, status, viewsetsfrom rest_framework.request import Requestfrom rest_framework.response import Responsefrom rest_framework.versioning import URLPathVersioningfrom scheduler.helper import SchedulerHelper-from workflow_manager.workflow.constants import (- WorkflowExecutionKey as WFExecKey,-)+from workflow_manager.workflow.constants import WorkflowExecutionKey as WFExecKeylogger = logging.getLogger(__name__)@@ -99,8 +91,7 @@ class PipelineViewSet(viewsets.ModelViewSet):serializer.save()except IntegrityError:raise DuplicateData(- f"{PipelineErrors.PIPELINE_EXISTS}, "- f"{PipelineErrors.DUPLICATE_API}"+ f"{PipelineErrors.PIPELINE_EXISTS}, " f"{PipelineErrors.DUPLICATE_API}")return Response(data=serializer.data, status=status.HTTP_201_CREATED)@@ -134,6 +125,4 @@ class PipelineViewSet(viewsets.ModelViewSet):status=status.HTTP_200_OK,)else:- return Response(- serializer.errors, status=status.HTTP_400_BAD_REQUEST- )+ return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)diff --git a/backend/platform_settings/exceptions.py b/backend/platform_settings/exceptions.pyindex 4e54b58ef..538b270fd 100644--- a/backend/platform_settings/exceptions.py+++ b/backend/platform_settings/exceptions.py@@ -41,9 +41,7 @@ class DuplicateData(APIException):status_code = 400default_detail = "Duplicate Data"- def __init__(- self, detail: Optional[str] = None, code: Optional[int] = None- ):+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):if detail is not None:self.detail = detailif code is not None:diff --git a/backend/platform_settings/platform_auth_helper.py b/backend/platform_settings/platform_auth_helper.pyindex 167db67da..c3d7d2d4b 100644--- a/backend/platform_settings/platform_auth_helper.py+++ b/backend/platform_settings/platform_auth_helper.py@@ -32,9 +32,7 @@ class PlatformAuthHelper:)raise errorif not auth_controller.is_admin_by_role(member.role):- logger.error(- "User is not having right access to perform this operation."- )+ logger.error("User is not having right access to perform this operation.")raise UserForbidden()else:passdiff --git a/backend/platform_settings/platform_auth_service.py b/backend/platform_settings/platform_auth_service.pyindex 7258b982f..054291ba8 100644--- a/backend/platform_settings/platform_auth_service.py+++ b/backend/platform_settings/platform_auth_service.py@@ -73,9 +73,7 @@ class PlatformAuthenticationService:logger.info(f"platform_key is generated for {organization.id}")return resultexcept IntegrityError as error:- logger.error(- f"Integrity error - failed to generate platform key : {error}"- )+ logger.error(f"Integrity error - failed to generate platform key : {error}")raise DuplicateData(f"{ErrorMessage.KEY_EXIST}, \{ErrorMessage.DUPLICATE_API}"@@ -133,9 +131,7 @@ class PlatformAuthenticationService:logger.info(f"platform_key {id} is updated by user {user.id}")return resultexcept IntegrityError as error:- logger.error(- f"Integrity error - failed to refresh platform key : {error}"- )+ logger.error(f"Integrity error - failed to refresh platform key : {error}")raise DuplicateData(f"{ErrorMessage.KEY_EXIST}, \{ErrorMessage.DUPLICATE_API}"@@ -186,9 +182,7 @@ class PlatformAuthenticationService:{ErrorMessage.DUPLICATE_API}")except Exception as error:- logger.error(- f"Failed to activate/deactivate platform key : {error}"- )+ logger.error(f"Failed to activate/deactivate platform key : {error}")raise InternalServiceError()@staticmethoddiff --git a/backend/platform_settings/views.py b/backend/platform_settings/views.pyindex ace584262..6b4c918fa 100644--- a/backend/platform_settings/views.py+++ b/backend/platform_settings/views.py@@ -7,9 +7,7 @@ from account.models import Organization, PlatformKeyfrom django.db import connectionfrom platform_settings.constants import PlatformServiceConstantsfrom platform_settings.platform_auth_helper import PlatformAuthHelper-from platform_settings.platform_auth_service import (- PlatformAuthenticationService,-)+from platform_settings.platform_auth_service import PlatformAuthenticationServicefrom platform_settings.serializers import (PlatformKeyGenerateSerializer,PlatformKeyIDSerializer,diff --git a/backend/plugins/authentication/auth_sample/auth_service.py b/backend/plugins/authentication/auth_sample/auth_service.pyindex 3ec4e7888..499d11c5f 100644--- a/backend/plugins/authentication/auth_sample/auth_service.py+++ b/backend/plugins/authentication/auth_sample/auth_service.py@@ -7,13 +7,7 @@ from rest_framework.request import Requestfrom rest_framework.response import Responsefrom .auth_helper import AuthHelper-from .dto import (- AuthOrganization,- ResetUserPasswordDto,- TokenData,- User,- UserInfo,-)+from .dto import AuthOrganization, ResetUserPasswordDto, TokenData, User, UserInfofrom .enums import Regionfrom .exceptions import MethodNotImplemented@@ -73,9 +67,7 @@ class AuthService(ABC):def get_user_id_from_token(self, token: dict[str, Any]) -> Response:return token["userinfo"]["sub"]- def get_organization_members_by_org_id(- self, organization_id: str- ) -> Response:+ def get_organization_members_by_org_id(self, organization_id: str) -> Response:raise MethodNotImplemented()def reset_user_password(self, user: User) -> ResetUserPasswordDto:diff --git a/backend/project/urls.py b/backend/project/urls.pyindex 4af97d0d8..f015268e8 100644--- a/backend/project/urls.py+++ b/backend/project/urls.py@@ -16,9 +16,7 @@ project_detail = ProjectViewSet.as_view(project_settings = ProjectViewSet.as_view({"get": "project_settings", "put": "project_settings"})-project_settings_schema = ProjectViewSet.as_view(- {"get": "project_settings_schema"}-)+project_settings_schema = ProjectViewSet.as_view({"get": "project_settings_schema"})urlpatterns = format_suffix_patterns([diff --git a/backend/prompt/tests/test_urls.py b/backend/prompt/tests/test_urls.pyindex 94469f5c2..0dffabd36 100644--- a/backend/prompt/tests/test_urls.py+++ b/backend/prompt/tests/test_urls.py@@ -26,9 +26,7 @@ class TestPrompts(APITestCase):def test_prompts_detail_throw_404(self):"""Tests whether a 404 error is thrown on retrieving a prompt."""- url = reverse(- "prompts-detail", kwargs={"pk": 200}- ) # Prompt doesn't exist+ url = reverse("prompts-detail", kwargs={"pk": 200}) # Prompt doesn't existresponse = self.client.get(url)self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)diff --git a/backend/prompt_studio/permission.py b/backend/prompt_studio/permission.pyindex ba036172a..e1b84e86f 100644--- a/backend/prompt_studio/permission.py+++ b/backend/prompt_studio/permission.py@@ -8,9 +8,7 @@ from rest_framework.views import APIViewclass PromptAcesssToUser(permissions.BasePermission):"""Is the crud to Prompt/Notes allowed to user."""- def has_object_permission(- self, request: Request, view: APIView, obj: Any- ) -> bool:+ def has_object_permission(self, request: Request, view: APIView, obj: Any) -> bool:return (Trueif (diff --git a/backend/prompt_studio/processor_loader.py b/backend/prompt_studio/processor_loader.pyindex 28d12f8d9..2a371a458 100644--- a/backend/prompt_studio/processor_loader.py+++ b/backend/prompt_studio/processor_loader.py@@ -41,9 +41,7 @@ def load_plugins() -> list[Any]:else:continuetry:- full_module_path = (- f"{processor_package_path}.{processor_module_name}"- )+ full_module_path = f"{processor_package_path}.{processor_module_name}"module = import_module(full_module_path)metadata = getattr(module, ProcessorConfig.METADATA, {})diff --git a/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py b/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.pyindex d7be28309..482ad5f80 100644--- a/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py+++ b/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py@@ -14,9 +14,7 @@ def fill_with_default_x2text(apps, schema):encryption_secret: str = settings.ENCRYPTION_KEYf: Fernet = Fernet(encryption_secret.encode("utf-8"))- metadata = {- "url": "http://unstract-unstructured-io:8000/general/v0/general"- }+ metadata = {"url": "http://unstract-unstructured-io:8000/general/v0/general"}json_string = json.dumps(metadata)metadata_b = f.encrypt(json_string.encode("utf-8"))@@ -27,9 +25,7 @@ def fill_with_default_x2text(apps, schema):adapter_metadata_b=metadata_b,)adapter_instance.save()- ProfileManager.objects.filter(x2text__isnull=True).update(- x2text=adapter_instance- )+ ProfileManager.objects.filter(x2text__isnull=True).update(x2text=adapter_instance)def reversal_x2text(*args):@@ -38,9 +34,7 @@ def reversal_x2text(*args):def disable_triggers(apps, schema_editor):with connection.cursor() as cursor:- cursor.execute(- "ALTER TABLE adapter_adapterinstance DISABLE TRIGGER ALL;"- )+ cursor.execute("ALTER TABLE adapter_adapterinstance DISABLE TRIGGER ALL;")class Migration(migrations.Migration):diff --git a/backend/prompt_studio/prompt_profile_manager/migrations/0006_alter_profilemanager_x2text.py b/backend/prompt_studio/prompt_profile_manager/migrations/0006_alter_profilemanager_x2text.pyindex b2088ef12..d36581058 100644--- a/backend/prompt_studio/prompt_profile_manager/migrations/0006_alter_profilemanager_x2text.py+++ b/backend/prompt_studio/prompt_profile_manager/migrations/0006_alter_profilemanager_x2text.py@@ -6,9 +6,7 @@ from django.db import connection, migrations, modelsdef enable_triggers(apps, schema_editor):with connection.cursor() as cursor:- cursor.execute(- "ALTER TABLE adapter_adapterinstance ENABLE TRIGGER ALL;"- )+ cursor.execute("ALTER TABLE adapter_adapterinstance ENABLE TRIGGER ALL;")class Migration(migrations.Migration):diff --git a/backend/prompt_studio/prompt_profile_manager/migrations/0008_profilemanager_migration.py b/backend/prompt_studio/prompt_profile_manager/migrations/0008_profilemanager_migration.pyindex 33f3cca4e..e80273f38 100644--- a/backend/prompt_studio/prompt_profile_manager/migrations/0008_profilemanager_migration.py+++ b/backend/prompt_studio/prompt_profile_manager/migrations/0008_profilemanager_migration.py@@ -23,9 +23,7 @@ class Migration(migrations.Migration):def MigrateProfileManager(apps: Any, schema_editor: Any) -> None:CustomTool = apps.get_model("prompt_studio_core", "CustomTool")- ProfileManager = apps.get_model(- "prompt_profile_manager", "ProfileManager"- )+ ProfileManager = apps.get_model("prompt_profile_manager", "ProfileManager")ToolStudioPrompt = apps.get_model("prompt_studio", "ToolStudioPrompt")# Iterate over prompt studio tools@@ -55,9 +53,7 @@ class Migration(migrations.Migration):# Replicate those profile managerreplicated_profile_manager = profile_managerreplicated_profile_manager.profile_id = None- replicated_profile_manager.profile_name = (- f"Profile_{index+1}"- )+ replicated_profile_manager.profile_name = f"Profile_{index+1}""""Attach the replicated profile manager to thecorresponding prompt tool."""replicated_profile_manager.prompt_studio_tool = custom_tooldiff --git a/backend/prompt_studio/prompt_profile_manager/models.py b/backend/prompt_studio/prompt_profile_manager/models.pyindex adfffe2f1..65bdbcffd 100644--- a/backend/prompt_studio/prompt_profile_manager/models.py+++ b/backend/prompt_studio/prompt_profile_manager/models.py@@ -14,9 +14,7 @@ class ProfileManager(BaseModel):class RetrievalStrategy(models.TextChoices):SIMPLE = "simple", "Simple retrieval"- profile_id = models.UUIDField(- primary_key=True, default=uuid.uuid4, editable=False- )+ profile_id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)profile_name = models.TextField(blank=False)vector_store = models.ForeignKey(AdapterInstance,diff --git a/backend/prompt_studio/prompt_profile_manager/serializers.py b/backend/prompt_studio/prompt_profile_manager/serializers.pyindex 1f9ef62dd..4d4753561 100644--- a/backend/prompt_studio/prompt_profile_manager/serializers.py+++ b/backend/prompt_studio/prompt_profile_manager/serializers.py@@ -22,8 +22,8 @@ class ProfileManagerSerializer(AuditSerializer):vector_db = rep[ProfileManagerKeys.VECTOR_STORE]x2text = rep[ProfileManagerKeys.X2TEXT]if llm:- rep[ProfileManagerKeys.LLM] = (- AdapterProcessor.get_adapter_instance_by_id(llm)+ rep[ProfileManagerKeys.LLM] = AdapterProcessor.get_adapter_instance_by_id(+ llm)if embedding:rep[ProfileManagerKeys.EMBEDDING_MODEL] = (diff --git a/backend/prompt_studio/prompt_profile_manager/views.py b/backend/prompt_studio/prompt_profile_manager/views.pyindex 16d66ef1b..e42ed100b 100644--- a/backend/prompt_studio/prompt_profile_manager/views.py+++ b/backend/prompt_studio/prompt_profile_manager/views.py@@ -9,9 +9,7 @@ from prompt_studio.prompt_profile_manager.constants import (ProfileManagerErrors,ProfileManagerKeys,)-from prompt_studio.prompt_profile_manager.serializers import (- ProfileManagerSerializer,-)+from prompt_studio.prompt_profile_manager.serializers import ProfileManagerSerializerfrom rest_framework import status, viewsetsfrom rest_framework.response import Responsefrom rest_framework.versioning import URLPathVersioning@@ -41,9 +39,7 @@ class ProfileManagerView(viewsets.ModelViewSet):def create(self, request: HttpRequest, *args: tuple[Any], **kwargs: dict[str, Any]) -> Response:- serializer: ProfileManagerSerializer = self.get_serializer(- data=request.data- )+ serializer: ProfileManagerSerializer = self.get_serializer(data=request.data)# Overriding default exception behaviour# TO DO : Handle model related exceptions.serializer.is_valid(raise_exception=True)diff --git a/backend/prompt_studio/prompt_studio/migrations/0004_alter_toolstudioprompt_prompt.py b/backend/prompt_studio/prompt_studio/migrations/0004_alter_toolstudioprompt_prompt.pyindex 72be66dd4..5fe3253fe 100644--- a/backend/prompt_studio/prompt_studio/migrations/0004_alter_toolstudioprompt_prompt.py+++ b/backend/prompt_studio/prompt_studio/migrations/0004_alter_toolstudioprompt_prompt.py@@ -12,8 +12,6 @@ class Migration(migrations.Migration):migrations.AlterField(model_name="toolstudioprompt",name="prompt",- field=models.TextField(- blank=True, db_comment="Field to store the prompt"- ),+ field=models.TextField(blank=True, db_comment="Field to store the prompt"),),]diff --git a/backend/prompt_studio/prompt_studio/models.py b/backend/prompt_studio/prompt_studio/models.pyindex d59f5f8f9..245277278 100644--- a/backend/prompt_studio/prompt_studio/models.py+++ b/backend/prompt_studio/prompt_studio/models.py@@ -28,9 +28,7 @@ class ToolStudioPrompt(BaseModel):class Mode(models.TextChoices):DEFAULT = "Default", "Default choice for output"- prompt_id = models.UUIDField(- primary_key=True, default=uuid.uuid4, editable=False- )+ prompt_id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)prompt_key = models.TextField(blank=False,db_comment="Field to store the prompt key",diff --git a/backend/prompt_studio/prompt_studio_core/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_core/migrations/0001_initial.pyindex a22e616ce..14ae4c61a 100644--- a/backend/prompt_studio/prompt_studio_core/migrations/0001_initial.py+++ b/backend/prompt_studio/prompt_studio_core/migrations/0001_initial.py@@ -65,9 +65,7 @@ class Migration(migrations.Migration):),("preamble",- models.TextField(- blank=True, db_comment="Preamble to the prompts"- ),+ models.TextField(blank=True, db_comment="Preamble to the prompts"),),("postamble",diff --git a/backend/prompt_studio/prompt_studio_core/models.py b/backend/prompt_studio/prompt_studio_core/models.pyindex cac0a0607..e9bbff42d 100644--- a/backend/prompt_studio/prompt_studio_core/models.py+++ b/backend/prompt_studio/prompt_studio_core/models.py@@ -24,9 +24,7 @@ class CustomToolModelManager(models.Manager):class CustomTool(BaseModel):"""Model to store the custom tools designed in the tool studio."""- tool_id = models.UUIDField(- primary_key=True, default=uuid.uuid4, editable=False- )+ tool_id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)tool_name = models.TextField(unique=True, blank=False, null=False)description = models.TextField(blank=False, null=False)author = models.TextField(@@ -119,8 +117,6 @@ class CustomTool(BaseModel):# Introduced field to establish M2M relation between users and custom_tool.# This will introduce intermediary table which relates both the models.- shared_users = models.ManyToManyField(- User, related_name="shared_custom_tool"- )+ shared_users = models.ManyToManyField(User, related_name="shared_custom_tool")objects = CustomToolModelManager()diff --git a/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.py b/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.pyindex 111a891d7..0c460a048 100644--- a/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.py+++ b/backend/prompt_studio/prompt_studio_core/prompt_ide_base_tool.py@@ -1,16 +1,12 @@import os-from platform_settings.platform_auth_service import (- PlatformAuthenticationService,-)+from platform_settings.platform_auth_service import PlatformAuthenticationServicefrom prompt_studio.prompt_studio_core.constants import LogLevel, ToolStudioKeysfrom unstract.sdk.tool.stream import StreamMixinclass PromptIdeBaseTool(StreamMixin):- def __init__(- self, log_level: LogLevel = LogLevel.INFO, org_id: str = ""- ) -> None:+ def __init__(self, log_level: LogLevel = LogLevel.INFO, org_id: str = "") -> None:"""Args:tool (UnstractAbstractTool): Instance of UnstractAbstractTool@@ -34,17 +30,13 @@ class PromptIdeBaseTool(StreamMixin):"""# HACK: Adding platform key for multitenancyif env_key == ToolStudioKeys.PLATFORM_SERVICE_API_KEY:- platform_key = (- PlatformAuthenticationService.get_active_platform_key(- self.org_id- )+ platform_key = PlatformAuthenticationService.get_active_platform_key(+ self.org_id)key: str = str(platform_key.key)return keyelse:env_value = os.environ.get(env_key)if env_value is None or env_value == "":- self.stream_error_and_exit(- f"Env variable {env_key} is required"- )+ self.stream_error_and_exit(f"Env variable {env_key} is required")return env_value # type:ignorediff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.pyindex 0f137abec..154600369 100644--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py@@ -11,9 +11,7 @@ from file_management.file_management_helper import FileManagerHelperfrom prompt_studio.prompt_profile_manager.models import ProfileManagerfrom prompt_studio.prompt_studio.models import ToolStudioPromptfrom prompt_studio.prompt_studio_core.constants import LogLevels-from prompt_studio.prompt_studio_core.constants import (- ToolStudioPromptKeys as TSPKeys,-)+from prompt_studio.prompt_studio_core.constants import ToolStudioPromptKeys as TSPKeysfrom prompt_studio.prompt_studio_core.exceptions import (AnswerFetchError,DefaultProfileError,@@ -25,9 +23,7 @@ from prompt_studio.prompt_studio_core.exceptions import (ToolNotValid,)from prompt_studio.prompt_studio_core.models import CustomTool-from prompt_studio.prompt_studio_core.prompt_ide_base_tool import (- PromptIdeBaseTool,-)+from prompt_studio.prompt_studio_core.prompt_ide_base_tool import PromptIdeBaseToolfrom prompt_studio.prompt_studio_index_manager.prompt_studio_index_helper import ( # noqa: E501PromptStudioIndexHelper,)@@ -185,8 +181,8 @@ class PromptStudioHelper:Returns:List[ToolStudioPrompt]: List of instance of the model"""- prompt_instances: list[ToolStudioPrompt] = (- ToolStudioPrompt.objects.filter(tool_id=tool_id)+ prompt_instances: list[ToolStudioPrompt] = ToolStudioPrompt.objects.filter(+ tool_id=tool_id)return prompt_instances@@ -243,9 +239,7 @@ class PromptStudioHelper:)# Need to check the user who created profile manager# has access to adapters configured in profile manager- PromptStudioHelper.validate_profile_manager_owner_access(- default_profile- )+ PromptStudioHelper.validate_profile_manager_owner_access(default_profile)doc_id = PromptStudioHelper.dynamic_indexer(profile_manager=default_profile,@@ -326,9 +320,7 @@ class PromptStudioHelper:os.path.splitext(filename)[0] + ".txt",)- logger.info(- f"[{tool.tool_id}] Invoking prompt service for prompt {id}"- )+ logger.info(f"[{tool.tool_id}] Invoking prompt service for prompt {id}")PromptStudioHelper._publish_log({"tool_id": tool_id, "prompt_id": id},LogLevels.DEBUG,@@ -382,14 +374,10 @@ class PromptStudioHelper:else:prompts = PromptStudioHelper.fetch_prompt_from_tool(tool_id)prompts = [- prompt- for prompt in prompts- if prompt.prompt_type != TSPKeys.NOTES+ prompt for prompt in prompts if prompt.prompt_type != TSPKeys.NOTES]if not prompts:- logger.error(- f"[{tool_id or 'NA'}] No prompts found for id: {id}"- )+ logger.error(f"[{tool_id or 'NA'}] No prompts found for id: {id}")raise NoPromptsFound()logger.info(f"[{tool_id}] Executing prompts in single pass")@@ -444,9 +432,7 @@ class PromptStudioHelper:)raise AnswerFetchError()- logger.info(- f"[{tool.tool_id}] Single pass response fetched successfully"- )+ logger.info(f"[{tool.tool_id}] Single pass response fetched successfully")PromptStudioHelper._publish_log({"tool_id": tool_id, "prompt_id": str(id)},LogLevels.INFO,@@ -530,9 +516,7 @@ class PromptStudioHelper:)output: dict[str, Any] = {}- output[TSPKeys.ASSERTION_FAILURE_PROMPT] = (- prompt.assertion_failure_prompt- )+ output[TSPKeys.ASSERTION_FAILURE_PROMPT] = prompt.assertion_failure_promptoutput[TSPKeys.ASSERT_PROMPT] = prompt.assert_promptoutput[TSPKeys.IS_ASSERT] = prompt.is_assertoutput[TSPKeys.PROMPT] = prompt.prompt@@ -550,9 +534,7 @@ class PromptStudioHelper:output[TSPKeys.RETRIEVAL_STRATEGY] = (prompt.profile_manager.retrieval_strategy)- output[TSPKeys.SIMILARITY_TOP_K] = (- prompt.profile_manager.similarity_top_k- )+ output[TSPKeys.SIMILARITY_TOP_K] = prompt.profile_manager.similarity_top_koutput[TSPKeys.SECTION] = prompt.profile_manager.sectionoutput[TSPKeys.X2TEXT_ADAPTER] = x2text# Eval settings for the prompt@@ -709,9 +691,7 @@ class PromptStudioHelper:challenge_llm = str(default_profile.llm.id)# Need to check the user who created profile manager# has access to adapters configured in profile manager- PromptStudioHelper.validate_profile_manager_owner_access(- default_profile- )+ PromptStudioHelper.validate_profile_manager_owner_access(default_profile)default_profile.chunk_size = 0 # To retrive full contextif prompt_grammar:@@ -759,9 +739,7 @@ class PromptStudioHelper:if tool.summarize_as_source:path = Path(file_path)- file_path = str(- path.parent / TSPKeys.SUMMARIZE / (path.stem + ".txt")- )+ file_path = str(path.parent / TSPKeys.SUMMARIZE / (path.stem + ".txt"))file_hash = ToolUtils.get_hash_from_file(file_path=file_path)payload = {diff --git a/backend/prompt_studio/prompt_studio_core/urls.py b/backend/prompt_studio/prompt_studio_core/urls.pyindex becc267c0..3a32f63c4 100644--- a/backend/prompt_studio/prompt_studio_core/urls.py+++ b/backend/prompt_studio/prompt_studio_core/urls.py@@ -3,9 +3,7 @@ from rest_framework.urlpatterns import format_suffix_patternsfrom .views import PromptStudioCoreView-prompt_studio_list = PromptStudioCoreView.as_view(- {"get": "list", "post": "create"}-)+prompt_studio_list = PromptStudioCoreView.as_view({"get": "list", "post": "create"})prompt_studio_detail = PromptStudioCoreView.as_view({"get": "retrieve",@@ -14,9 +12,7 @@ prompt_studio_detail = PromptStudioCoreView.as_view("delete": "destroy",})-prompt_studio_choices = PromptStudioCoreView.as_view(- {"get": "get_select_choices"}-)+prompt_studio_choices = PromptStudioCoreView.as_view({"get": "get_select_choices"})prompt_studio_profiles = PromptStudioCoreView.as_view({"get": "list_profiles", "patch": "make_profile_default"})@@ -27,21 +23,15 @@ prompt_studio_profilemanager = PromptStudioCoreView.as_view({"post": "create_profile_manager"})-prompt_studio_prompt_index = PromptStudioCoreView.as_view(- {"post": "index_document"}-)-prompt_studio_prompt_response = PromptStudioCoreView.as_view(- {"post": "fetch_response"}-)+prompt_studio_prompt_index = PromptStudioCoreView.as_view({"post": "index_document"})+prompt_studio_prompt_response = PromptStudioCoreView.as_view({"post": "fetch_response"})prompt_studio_adapter_choices = PromptStudioCoreView.as_view({"get": "get_adapter_choices"})prompt_studio_single_pass_extraction = PromptStudioCoreView.as_view({"post": "single_pass_extraction"})-prompt_studio_users = PromptStudioCoreView.as_view(- {"get": "list_of_shared_users"}-)+prompt_studio_users = PromptStudioCoreView.as_view({"get": "list_of_shared_users"})prompt_studio_file = PromptStudioCoreView.as_view(diff --git a/backend/prompt_studio/prompt_studio_core/views.py b/backend/prompt_studio/prompt_studio_core/views.pyindex 0ce3619be..412fdb3fa 100644--- a/backend/prompt_studio/prompt_studio_core/views.py+++ b/backend/prompt_studio/prompt_studio_core/views.py@@ -11,9 +11,7 @@ from permissions.permission import IsOwner, IsOwnerOrSharedUserfrom prompt_studio.processor_loader import ProcessorConfig, load_pluginsfrom prompt_studio.prompt_profile_manager.constants import ProfileManagerErrorsfrom prompt_studio.prompt_profile_manager.models import ProfileManager-from prompt_studio.prompt_profile_manager.serializers import (- ProfileManagerSerializer,-)+from prompt_studio.prompt_profile_manager.serializers import ProfileManagerSerializerfrom prompt_studio.prompt_studio.constants import ToolStudioPromptErrorsfrom prompt_studio.prompt_studio.exceptions import FilenameMissingErrorfrom prompt_studio.prompt_studio.serializers import ToolStudioPromptSerializer@@ -27,9 +25,7 @@ from prompt_studio.prompt_studio_core.exceptions import (IndexingAPIError,ToolDeleteError,)-from prompt_studio.prompt_studio_core.prompt_studio_helper import (- PromptStudioHelper,-)+from prompt_studio.prompt_studio_core.prompt_studio_helper import PromptStudioHelperfrom prompt_studio.prompt_studio_document_manager.models import DocumentManagerfrom prompt_studio.prompt_studio_document_manager.prompt_studio_document_helper import ( # noqa: E501PromptStudioDocumentHelper,@@ -48,9 +44,7 @@ from rest_framework.response import Responsefrom rest_framework.versioning import URLPathVersioningfrom tool_instance.models import ToolInstance-from unstract.connectors.filesystems.local_storage.local_storage import (- LocalStorageFS,-)+from unstract.connectors.filesystems.local_storage.local_storage import LocalStorageFSfrom .models import CustomToolfrom .serializers import (@@ -126,12 +120,10 @@ class PromptStudioCoreView(viewsets.ModelViewSet):if summarize_llm_profile_id:prompt_tool = self.get_object()- ProfileManager.objects.filter(- prompt_studio_tool=prompt_tool- ).update(is_summarize_llm=False)- profile_manager = ProfileManager.objects.get(- pk=summarize_llm_profile_id+ ProfileManager.objects.filter(prompt_studio_tool=prompt_tool).update(+ is_summarize_llm=False)+ profile_manager = ProfileManager.objects.get(pk=summarize_llm_profile_id)profile_manager.is_summarize_llm = Trueprofile_manager.save()@@ -147,9 +139,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):Response: Reponse of dropdown dict"""try:- select_choices: dict[str, Any] = (- PromptStudioHelper.get_select_fields()- )+ select_choices: dict[str, Any] = PromptStudioHelper.get_select_fields()return Response(select_choices, status=status.HTTP_200_OK)except Exception as e:logger.error(f"Error occured while fetching select fields {e}")@@ -172,9 +162,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):return Response(serialized_instances)@action(detail=True, methods=["patch"])- def make_profile_default(- self, request: HttpRequest, pk: Any = None- ) -> Response:+ def make_profile_default(self, request: HttpRequest, pk: Any = None) -> Response:prompt_tool = (self.get_object()) # Assuming you have a get_object method in your viewset@@ -183,9 +171,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):is_default=False)- profile_manager = ProfileManager.objects.get(- pk=request.data["default_profile"]- )+ profile_manager = ProfileManager.objects.get(pk=request.data["default_profile"])profile_manager.is_default = Trueprofile_manager.save()@@ -242,9 +228,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):status=status.HTTP_200_OK,)else:- logger.error(- "Error occured while indexing. Unique ID is not valid."- )+ logger.error("Error occured while indexing. Unique ID is not valid.")raise IndexingAPIError()@action(detail=True, methods=["post"])@@ -281,9 +265,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):return Response(response, status=status.HTTP_200_OK)@action(detail=True, methods=["post"])- def single_pass_extraction(- self, request: HttpRequest, pk: uuid- ) -> Response:+ def single_pass_extraction(self, request: HttpRequest, pk: uuid) -> Response:"""API Entry point method to fetch response to prompt.Args:@@ -317,9 +299,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):return Response(response, status=status.HTTP_200_OK)@action(detail=True, methods=["get"])- def list_of_shared_users(- self, request: HttpRequest, pk: Any = None- ) -> Response:+ def list_of_shared_users(self, request: HttpRequest, pk: Any = None) -> Response:custom_tool = (self.get_object()@@ -332,9 +312,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):@action(detail=True, methods=["post"])def create_prompt(self, request: HttpRequest, pk: Any = None) -> Response:context = super().get_serializer_context()- serializer = ToolStudioPromptSerializer(- data=request.data, context=context- )+ serializer = ToolStudioPromptSerializer(data=request.data, context=context)serializer.is_valid(raise_exception=True)try:# serializer.save()@@ -347,13 +325,9 @@ class PromptStudioCoreView(viewsets.ModelViewSet):return Response(serializer.data, status=status.HTTP_201_CREATED)@action(detail=True, methods=["post"])- def create_profile_manager(- self, request: HttpRequest, pk: Any = None- ) -> Response:+ def create_profile_manager(self, request: HttpRequest, pk: Any = None) -> Response:context = super().get_serializer_context()- serializer = ProfileManagerSerializer(- data=request.data, context=context- )+ serializer = ProfileManagerSerializer(data=request.data, context=context)serializer.is_valid(raise_exception=True)try:@@ -366,9 +340,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):return Response(serializer.data, status=status.HTTP_201_CREATED)@action(detail=True, methods=["get"])- def fetch_contents_ide(- self, request: HttpRequest, pk: Any = None- ) -> Response:+ def fetch_contents_ide(self, request: HttpRequest, pk: Any = None) -> Response:custom_tool = self.get_object()serializer = FileInfoIdeSerializer(data=request.GET)serializer.is_valid(raise_exception=True)@@ -380,8 +352,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):filename_without_extension = file_name.rsplit(".", 1)[0]if view_type == FileViewTypes.EXTRACT:file_name = (- f"{FileViewTypes.EXTRACT.lower()}/"- f"{filename_without_extension}.txt"+ f"{FileViewTypes.EXTRACT.lower()}/" f"{filename_without_extension}.txt")if view_type == FileViewTypes.SUMMARIZE:file_name = (@@ -389,13 +360,11 @@ class PromptStudioCoreView(viewsets.ModelViewSet):f"{filename_without_extension}.txt")- file_path = file_path = (- FileManagerHelper.handle_sub_directory_for_tenants(- request.org_id,- is_create=True,- user_id=custom_tool.created_by.user_id,- tool_id=str(custom_tool.tool_id),- )+ file_path = file_path = FileManagerHelper.handle_sub_directory_for_tenants(+ request.org_id,+ is_create=True,+ user_id=custom_tool.created_by.user_id,+ tool_id=str(custom_tool.tool_id),)file_system = LocalStorageFS(settings={"path": file_path})if not file_path.endswith("/"):@@ -435,9 +404,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):}# Store filelogger.info(- f"Uploading file: {file_name}"- if file_name- else "Uploading file"+ f"Uploading file: {file_name}" if file_name else "Uploading file")FileManagerHelper.upload_file(file_system,@@ -489,9 +456,7 @@ class PromptStudioCoreView(viewsets.ModelViewSet):custom_tool = self.get_object()serializer = ExportToolRequestSerializer(data=request.data)serializer.is_valid(raise_exception=True)- is_shared_with_org: bool = serializer.validated_data.get(- "is_shared_with_org"- )+ is_shared_with_org: bool = serializer.validated_data.get("is_shared_with_org")user_ids = set(serializer.validated_data.get("user_id"))PromptStudioRegistryHelper.update_or_create_psr_tool(diff --git a/backend/prompt_studio/prompt_studio_document_manager/models.py b/backend/prompt_studio/prompt_studio_document_manager/models.pyindex f45ed3830..256d60e45 100644--- a/backend/prompt_studio/prompt_studio_document_manager/models.py+++ b/backend/prompt_studio/prompt_studio_document_manager/models.py@@ -9,9 +9,7 @@ from utils.models.base_model import BaseModelclass DocumentManager(BaseModel):"""Model to store the document details."""- document_id = models.UUIDField(- primary_key=True, default=uuid.uuid4, editable=False- )+ document_id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)document_name = models.CharField(db_comment="Field to store the document name",diff --git a/backend/prompt_studio/prompt_studio_index_manager/migrations/0001_initial.py b/backend/prompt_studio/prompt_studio_index_manager/migrations/0001_initial.pyindex 48841d074..e2a5bb3d6 100644--- a/backend/prompt_studio/prompt_studio_index_manager/migrations/0001_initial.py+++ b/backend/prompt_studio/prompt_studio_index_manager/migrations/0001_initial.py@@ -51,9 +51,7 @@ class Migration(migrations.Migration):),("index_ids_history",- models.JSONField(- db_comment="List of index ids", default=list- ),+ models.JSONField(db_comment="List of index ids", default=list),),("created_by",diff --git a/backend/prompt_studio/prompt_studio_index_manager/urls.py b/backend/prompt_studio/prompt_studio_index_manager/urls.pyindex cc7d45448..b082bc1d8 100644--- a/backend/prompt_studio/prompt_studio_index_manager/urls.py+++ b/backend/prompt_studio/prompt_studio_index_manager/urls.py@@ -3,9 +3,7 @@ from rest_framework.urlpatterns import format_suffix_patternsfrom .views import IndexManagerView-prompt_studio_index_list = IndexManagerView.as_view(- {"get": "list", "post": "create"}-)+prompt_studio_index_list = IndexManagerView.as_view({"get": "list", "post": "create"})prompt_studio_index_detail = IndexManagerView.as_view({diff --git a/backend/prompt_studio/prompt_studio_index_manager/views.py b/backend/prompt_studio/prompt_studio_index_manager/views.pyindex a07e6f6c8..414db28cd 100644--- a/backend/prompt_studio/prompt_studio_index_manager/views.py+++ b/backend/prompt_studio/prompt_studio_index_manager/views.py@@ -2,9 +2,7 @@ from typing import Optionalfrom django.db.models import QuerySetfrom prompt_studio.prompt_studio_index_manager.constants import IndexManagerKeys-from prompt_studio.prompt_studio_index_manager.serializers import (- IndexManagerSerializer,-)+from prompt_studio.prompt_studio_index_manager.serializers import IndexManagerSerializerfrom rest_framework import viewsetsfrom rest_framework.versioning import URLPathVersioningfrom utils.filtering import FilterHelperdiff --git a/backend/prompt_studio/prompt_studio_output_manager/migrations/0004_alter_promptstudiooutputmanager_doc_name.py b/backend/prompt_studio/prompt_studio_output_manager/migrations/0004_alter_promptstudiooutputmanager_doc_name.pyindex 76f6081b8..b51b80233 100644--- a/backend/prompt_studio/prompt_studio_output_manager/migrations/0004_alter_promptstudiooutputmanager_doc_name.py+++ b/backend/prompt_studio/prompt_studio_output_manager/migrations/0004_alter_promptstudiooutputmanager_doc_name.py@@ -15,8 +15,6 @@ class Migration(migrations.Migration):migrations.AlterField(model_name="promptstudiooutputmanager",name="doc_name",- field=models.CharField(- db_comment="Field to store the document name"- ),+ field=models.CharField(db_comment="Field to store the document name"),),]diff --git a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.pyindex 24e1f022e..4efbbcb27 100644--- a/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py+++ b/backend/prompt_studio/prompt_studio_output_manager/output_manager_helper.py@@ -7,9 +7,7 @@ from prompt_studio.prompt_studio_document_manager.models import DocumentManagerfrom prompt_studio.prompt_studio_output_manager.constants import (PromptStudioOutputManagerKeys as PSOMKeys,)-from prompt_studio.prompt_studio_output_manager.models import (- PromptStudioOutputManager,-)+from prompt_studio.prompt_studio_output_manager.models import PromptStudioOutputManagerclass OutputManagerHelper:diff --git a/backend/prompt_studio/prompt_studio_output_manager/views.py b/backend/prompt_studio/prompt_studio_output_manager/views.pyindex affcbd0d4..3be3c5888 100644--- a/backend/prompt_studio/prompt_studio_output_manager/views.py+++ b/backend/prompt_studio/prompt_studio_output_manager/views.py@@ -39,9 +39,7 @@ class PromptStudioOutputView(viewsets.ModelViewSet):)# Convert the string representation to a boolean value- is_single_pass_extract = CommonUtils.str_to_bool(- is_single_pass_extract_param- )+ is_single_pass_extract = CommonUtils.str_to_bool(is_single_pass_extract_param)filter_args[PromptStudioOutputManagerKeys.IS_SINGLE_PASS_EXTRACT] = (is_single_pass_extractdiff --git a/backend/prompt_studio/prompt_studio_registry/constants.py b/backend/prompt_studio/prompt_studio_registry/constants.pyindex 745c59975..4c8713636 100644--- a/backend/prompt_studio/prompt_studio_registry/constants.py+++ b/backend/prompt_studio/prompt_studio_registry/constants.py@@ -14,9 +14,7 @@ class PromptStudioRegistryKeys:class PromptStudioRegistryErrors:SERIALIZATION_FAILED = "Data Serialization Failed."DUPLICATE_API = "It appears that a duplicate call may have been made."- CUSTOM_TOOL_EXISTS = (- "Custom tool with similiar configuration already exists"- )+ CUSTOM_TOOL_EXISTS = "Custom tool with similiar configuration already exists"class LogLevels:diff --git a/backend/prompt_studio/prompt_studio_registry/models.py b/backend/prompt_studio/prompt_studio_registry/models.pyindex bfff30d36..2ce06af3e 100644--- a/backend/prompt_studio/prompt_studio_registry/models.py+++ b/backend/prompt_studio/prompt_studio_registry/models.py@@ -7,11 +7,7 @@ from django.db.models import QuerySetfrom prompt_studio.prompt_studio.models import CustomToolfrom utils.models.base_model import BaseModel-from .fields import (- ToolMetadataJSONField,- ToolPropertyJSONField,- ToolSpecJSONField,-)+from .fields import ToolMetadataJSONField, ToolPropertyJSONField, ToolSpecJSONFieldclass PromptStudioRegistryModelManager(models.Manager):@@ -59,9 +55,7 @@ class PromptStudioRegistry(BaseModel):blank=False,default=dict,)- icon = models.CharField(- db_comment="Tool icon in svg format", editable=False- )+ icon = models.CharField(db_comment="Tool icon in svg format", editable=False)url = models.CharField(editable=False)custom_tool = models.OneToOneField(CustomTool,@@ -92,8 +86,6 @@ class PromptStudioRegistry(BaseModel):)# Introduced field to establish M2M relation between users and tools.# This will introduce intermediary table which relates both the models.- shared_users = models.ManyToManyField(- User, related_name="shared_exported_tools"- )+ shared_users = models.ManyToManyField(User, related_name="shared_exported_tools")objects = PromptStudioRegistryModelManager()diff --git a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.pyindex ee0b3b028..bb6a85c4a 100644--- a/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py+++ b/backend/prompt_studio/prompt_studio_registry/prompt_studio_registry_helper.py@@ -7,9 +7,7 @@ from django.db import IntegrityErrorfrom prompt_studio.prompt_profile_manager.models import ProfileManagerfrom prompt_studio.prompt_studio.models import ToolStudioPromptfrom prompt_studio.prompt_studio_core.models import CustomTool-from prompt_studio.prompt_studio_core.prompt_studio_helper import (- PromptStudioHelper,-)+from prompt_studio.prompt_studio_core.prompt_studio_helper import PromptStudioHelperfrom unstract.tool_registry.dto import Properties, Spec, Toolfrom .constants import JsonSchemaKey@@ -111,14 +109,12 @@ class PromptStudioRegistryHelper:obj: PromptStudioRegistry instance that was updated or created"""try:- properties: Properties = (- PromptStudioRegistryHelper.frame_properties(tool=custom_tool)+ properties: Properties = PromptStudioRegistryHelper.frame_properties(+ tool=custom_tool)spec: Spec = PromptStudioRegistryHelper.frame_spec(tool=custom_tool)- prompts: list[ToolStudioPrompt] = (- PromptStudioHelper.fetch_prompt_from_tool(- tool_id=custom_tool.tool_id- )+ prompts: list[ToolStudioPrompt] = PromptStudioHelper.fetch_prompt_from_tool(+ tool_id=custom_tool.tool_id)metadata = PromptStudioRegistryHelper.frame_export_json(tool=custom_tool, prompts=prompts@@ -210,9 +206,7 @@ class PromptStudioRegistryHelper:output[JsonSchemaKey.VECTOR_DB] = vector_dboutput[JsonSchemaKey.EMBEDDING] = embedding_modeloutput[JsonSchemaKey.X2TEXT_ADAPTER] = x2text- output[JsonSchemaKey.CHUNK_OVERLAP] = (- prompt.profile_manager.chunk_overlap- )+ output[JsonSchemaKey.CHUNK_OVERLAP] = prompt.profile_manager.chunk_overlapoutput[JsonSchemaKey.LLM] = llmoutput[JsonSchemaKey.PREAMBLE] = tool.preambleoutput[JsonSchemaKey.POSTAMBLE] = tool.postamble@@ -248,9 +242,7 @@ class PromptStudioRegistryHelper:instance=prompt_studio_tools, many=True)except Exception as error:- logger.error(- f"Error occured while fetching tool for tool_id: {error}"- )+ logger.error(f"Error occured while fetching tool for tool_id: {error}")raise InternalError()tool_metadata: dict[str, Any] = {}tool_list = []diff --git a/backend/prompt_studio/prompt_studio_registry/serializers.py b/backend/prompt_studio/prompt_studio_registry/serializers.pyindex dd1d6ea20..d0ab1bdb8 100644--- a/backend/prompt_studio/prompt_studio_registry/serializers.py+++ b/backend/prompt_studio/prompt_studio_registry/serializers.py@@ -22,6 +22,4 @@ class PromptStudioRegistryInfoSerializer(AuditSerializer):class ExportToolRequestSerializer(serializers.Serializer):is_shared_with_org = serializers.BooleanField(default=False)- user_id = serializers.ListField(- child=serializers.IntegerField(), required=False- )+ user_id = serializers.ListField(child=serializers.IntegerField(), required=False)diff --git a/backend/prompt_studio/prompt_studio_registry/views.py b/backend/prompt_studio/prompt_studio_registry/views.pyindex 5b0bd74cc..d050088a2 100644--- a/backend/prompt_studio/prompt_studio_registry/views.py+++ b/backend/prompt_studio/prompt_studio_registry/views.py@@ -2,9 +2,7 @@ import loggingfrom typing import Optionalfrom django.db.models import QuerySet-from prompt_studio.prompt_studio_registry.constants import (- PromptStudioRegistryKeys,-)+from prompt_studio.prompt_studio_registry.constants import PromptStudioRegistryKeysfrom prompt_studio.prompt_studio_registry.serializers import (PromptStudioRegistrySerializer,)diff --git a/backend/scheduler/helper.py b/backend/scheduler/helper.pyindex 110b5398e..20a2d7d29 100644--- a/backend/scheduler/helper.py+++ b/backend/scheduler/helper.py@@ -12,9 +12,7 @@ logger = logging.getLogger(__name__)class SchedulerHelper:@staticmethod- def add_job(- pipeline_id: str, cron_string: str = SC.DEFAULT_CRON_STRING- ) -> None:+ def add_job(pipeline_id: str, cron_string: str = SC.DEFAULT_CRON_STRING) -> None:logger.info(f"Scheduling job for {pipeline_id} with {cron_string}")name = f"Pipeline job-{pipeline_id}"job_serialize_data = {diff --git a/backend/scheduler/serializer.py b/backend/scheduler/serializer.pyindex 585cba98f..2ea3c3b53 100644--- a/backend/scheduler/serializer.py+++ b/backend/scheduler/serializer.py@@ -25,9 +25,7 @@ class JobKwargsSerializer(serializers.Serializer):class SchedulerKwargsSerializer(serializers.Serializer):coalesce = serializers.BooleanField()- misfire_grace_time = serializers.IntegerField(- allow_null=True, required=False- )+ misfire_grace_time = serializers.IntegerField(allow_null=True, required=False)max_instances = serializers.IntegerField()replace_existing = serializers.BooleanField()diff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.pyindex 2f6479645..b42244576 100644--- a/backend/scheduler/tasks.py+++ b/backend/scheduler/tasks.py@@ -51,9 +51,7 @@ def create_periodic_task(task=task_path,args=task_args_json,)- logger.info(- f"Completed running task addition to beat table: {task_name}"- )+ logger.info(f"Completed running task addition to beat table: {task_name}")except Exception as e:logger.error(f"Failed to create periodic task: {e}")@@ -97,18 +95,14 @@ def execute_pipeline_task(workflow = Workflow.objects.get(id=workflow_id)try:logger.info(f"Executing workflow: {workflow}")- update_pipeline(- pipepline_id, Pipeline.PipelineStatus.INPROGRESS- )+ update_pipeline(pipepline_id, Pipeline.PipelineStatus.INPROGRESS)execution_response = WorkflowHelper.complete_execution(workflow, execution_id, pipepline_id, with_logs)logger.info(f"Execution response: {execution_response}")update_pipeline(pipepline_id, Pipeline.PipelineStatus.SUCCESS)except IntegrityError:- logger.error(- f"Failed to complete execution for workflow: {workflow}"- )+ logger.error(f"Failed to complete execution for workflow: {workflow}")update_pipeline(pipepline_id, Pipeline.PipelineStatus.FAILURE)logger.info(f"Execution completed for pipeline: {name}")except Exception as e:diff --git a/backend/scheduler/views.py b/backend/scheduler/views.pyindex 33a09d610..1e02a93d1 100644--- a/backend/scheduler/views.py+++ b/backend/scheduler/views.py@@ -7,13 +7,8 @@ from pipeline.models import Pipelinefrom pipeline.pipeline_processor import PipelineProcessorfrom rest_framework.response import Responsefrom scheduler.tasks import create_periodic_task-from workflow_manager.workflow.constants import (- WorkflowExecutionKey,- WorkflowKey,-)-from workflow_manager.workflow.exceptions import (- WorkflowExecutionBadRequestException,-)+from workflow_manager.workflow.constants import WorkflowExecutionKey, WorkflowKey+from workflow_manager.workflow.exceptions import WorkflowExecutionBadRequestExceptionfrom workflow_manager.workflow.serializers import ExecuteWorkflowSerializerfrom workflow_manager.workflow.views import WorkflowViewSet@@ -27,9 +22,7 @@ def schedule_task_job(pipeline_id: str, job_data: Any) -> Response:return Response({"error": "cron_string is required"}, status=400)if "id" not in job_data:return Response({"error": "id is required"}, status=400)- if "job_kwargs" in job_data and not isinstance(- job_data["job_kwargs"], dict- ):+ if "job_kwargs" in job_data and not isinstance(job_data["job_kwargs"], dict):return Response({"error": "job_kwargs is required to be a dict"}, status=400)@@ -57,9 +50,7 @@ def schedule_task_job(pipeline_id: str, job_data: Any) -> Response:workflow_id = serializer.get_workflow_id(serializer.validated_data)- execution_action = serializer.get_execution_action(- serializer.validated_data- )+ execution_action = serializer.get_execution_action(serializer.validated_data)org_schema = connection.tenant.schema_namecreate_periodic_task(diff --git a/backend/tenant_account/invitation_views.py b/backend/tenant_account/invitation_views.pyindex de49bd84e..8d727eb74 100644--- a/backend/tenant_account/invitation_views.py+++ b/backend/tenant_account/invitation_views.py@@ -15,10 +15,8 @@ class InvitationViewSet(viewsets.ViewSet):@action(detail=False, methods=["GET"])def list_invitations(self, request: Request) -> Response:auth_controller = AuthenticationController()- invitations: list[MemberInvitation] = (- auth_controller.get_user_invitations(- organization_id=request.org_id,- )+ invitations: list[MemberInvitation] = auth_controller.get_user_invitations(+ organization_id=request.org_id,)serialized_members = ListInvitationsResponseSerializer(invitations, many=Truediff --git a/backend/tenant_account/serializer.py b/backend/tenant_account/serializer.pyindex 305efa749..250bbdb18 100644--- a/backend/tenant_account/serializer.py+++ b/backend/tenant_account/serializer.py@@ -35,9 +35,7 @@ class OrganizationMemberSerializer(serializers.ModelSerializer):class LimitedUserEmailListSerializer(serializers.ListSerializer):def __init__(self, *args: Any, **kwargs: Any) -> None:- self.max_elements: int = kwargs.pop(- "max_elements", Common.MAX_EMAIL_IN_REQUEST- )+ self.max_elements: int = kwargs.pop("max_elements", Common.MAX_EMAIL_IN_REQUEST)super().__init__(*args, **kwargs)def validate(self, data: list[str]) -> Any:@@ -50,9 +48,7 @@ class LimitedUserEmailListSerializer(serializers.ListSerializer):class LimitedUserListSerializer(serializers.ListSerializer):def __init__(self, *args: Any, **kwargs: Any) -> None:- self.max_elements: int = kwargs.pop(- "max_elements", Common.MAX_EMAIL_IN_REQUEST- )+ self.max_elements: int = kwargs.pop("max_elements", Common.MAX_EMAIL_IN_REQUEST)super().__init__(*args, **kwargs)def validate(@@ -65,13 +61,9 @@ class LimitedUserListSerializer(serializers.ListSerializer):for item in data:if not isinstance(item, dict):- raise ValidationError(- "Each item in the list must be a dictionary."- )+ raise ValidationError("Each item in the list must be a dictionary.")if "email" not in item:- raise ValidationError(- "Each item in the list must have 'email' key."- )+ raise ValidationError("Each item in the list must have 'email' key.")if "role" not in item:item["role"] = None@@ -125,9 +117,7 @@ class ChangeUserRoleRequestSerializer(serializers.Serializer):class DeleteInvitationRequestSerializer(serializers.Serializer):id = serializers.EmailField(required=True)- def get_id(- self, validated_data: dict[str, Union[str, None]]- ) -> Optional[str]:+ def get_id(self, validated_data: dict[str, Union[str, None]]) -> Optional[str]:return validated_data.get(Common.ID)diff --git a/backend/tenant_account/users_view.py b/backend/tenant_account/users_view.pyindex 2f8852107..3e244332b 100644--- a/backend/tenant_account/users_view.py+++ b/backend/tenant_account/users_view.py@@ -78,9 +78,7 @@ class OrganizationUserViewSet(viewsets.ViewSet):try:# z_code = request.COOKIES.get(Cookie.Z_CODE)user_info = auth_controller.get_user_info(request)- role = auth_controller.get_organization_members_by_user(- request.user- )+ role = auth_controller.get_organization_members_by_user(request.user)if not user_info:return Response(status=status.HTTP_404_NOT_FOUND,@@ -112,9 +110,7 @@ class OrganizationUserViewSet(viewsets.ViewSet):admin=request.user, org_id=request.org_id, user_list=user_list)- response_serializer = UserInviteResponseSerializer(- invite_response, many=True- )+ response_serializer = UserInviteResponseSerializer(invite_response, many=True)if invite_response and len(invite_response) != 0:response = Response(@@ -160,9 +156,7 @@ class OrganizationUserViewSet(viewsets.ViewSet):members: list[OrganizationMember] = (auth_controller.get_organization_members_by_org_id())- serialized_members = OrganizationMemberSerializer(- members, many=True- ).data+ serialized_members = OrganizationMemberSerializer(members, many=True).datareturn Response(status=status.HTTP_200_OK,data={"message": "success", "members": serialized_members},diff --git a/backend/tenant_account/views.py b/backend/tenant_account/views.pyindex 598a72ce7..1f95d07ae 100644--- a/backend/tenant_account/views.py+++ b/backend/tenant_account/views.py@@ -37,9 +37,7 @@ def get_roles(request: Request) -> Response:@api_view(["POST"])def reset_password(request: Request) -> Response:auth_controller = AuthenticationController()- data: ResetUserPasswordDto = auth_controller.reset_user_password(- request.user- )+ data: ResetUserPasswordDto = auth_controller.reset_user_password(request.user)if data.status:return Response(status=status.HTTP_200_OK,diff --git a/backend/tool_instance/models.py b/backend/tool_instance/models.pyindex 7c9bb5122..7ddc73cff 100644--- a/backend/tool_instance/models.py+++ b/backend/tool_instance/models.py@@ -39,19 +39,13 @@ class ToolInstance(BaseModel):max_length=TOOL_ID_LENGTH,db_comment="Function name of the tool being used",)- input = models.JSONField(- null=True, db_comment="Provisional WF input to a tool"- )- output = models.JSONField(- null=True, db_comment="Provisional WF output to a tool"- )+ input = models.JSONField(null=True, db_comment="Provisional WF input to a tool")+ output = models.JSONField(null=True, db_comment="Provisional WF output to a tool")version = models.CharField(max_length=TOOL_VERSION_LENGTH)metadata = models.JSONField(db_comment="Stores config for a tool")step = models.IntegerField()# TODO: Make as an enum supporting fixed values once we have clarity- status = models.CharField(- max_length=TOOL_STATUS_LENGTH, default="Ready to start"- )+ status = models.CharField(max_length=TOOL_STATUS_LENGTH, default="Ready to start")created_by = models.ForeignKey(User,on_delete=models.SET_NULL,diff --git a/backend/tool_instance/serializers.py b/backend/tool_instance/serializers.pyindex 944447b3f..4ce323b49 100644--- a/backend/tool_instance/serializers.py+++ b/backend/tool_instance/serializers.py@@ -2,15 +2,8 @@ import loggingimport uuidfrom typing import Any-from prompt_studio.prompt_studio_registry.constants import (- PromptStudioRegistryKeys,-)-from rest_framework.serializers import (- ListField,- Serializer,- UUIDField,- ValidationError,-)+from prompt_studio.prompt_studio_registry.constants import PromptStudioRegistryKeys+from rest_framework.serializers import ListField, Serializer, UUIDField, ValidationErrorfrom tool_instance.constants import ToolInstanceKey as TIKeyfrom tool_instance.constants import ToolKeyfrom tool_instance.exceptions import ToolDoesNotExist@@ -72,9 +65,7 @@ class ToolInstanceSerializer(AuditSerializer):try:workflow = Workflow.objects.get(pk=workflow_id)except Workflow.DoesNotExist:- raise ValidationError(- f"Workflow with ID {workflow_id} does not exist."- )+ raise ValidationError(f"Workflow with ID {workflow_id} does not exist.")validated_data[TIKey.WORKFLOW] = workflowtool_uid = validated_data.get(TIKey.TOOL_ID)@@ -113,9 +104,7 @@ class ToolInstanceReorderSerializer(Serializer):try:workflow = Workflow.objects.get(pk=workflow_id)except Workflow.DoesNotExist:- raise ValidationError(- f"Workflow with ID {workflow_id} does not exist."- )+ raise ValidationError(f"Workflow with ID {workflow_id} does not exist.")# Check if the number of tool instances matches the actual counttool_instance_count = workflow.workflow_tool.count()@@ -128,9 +117,7 @@ class ToolInstanceReorderSerializer(Serializer):raise ValidationError(detail=msg)# Check if each tool instance exists in the workflow- existing_tool_instance_ids = workflow.workflow_tool.values_list(- "id", flat=True- )+ existing_tool_instance_ids = workflow.workflow_tool.values_list("id", flat=True)for tool_instance_id in tool_instances:if tool_instance_id not in existing_tool_instance_ids:raise ValidationError(diff --git a/backend/tool_instance/tool_instance_helper.py b/backend/tool_instance/tool_instance_helper.pyindex 167093ddb..87ef6052c 100644--- a/backend/tool_instance/tool_instance_helper.py+++ b/backend/tool_instance/tool_instance_helper.py@@ -44,13 +44,11 @@ class ToolInstanceHelper:offset_value = 0 if not offset else offsetto = offset_value + limitreturn list(- ToolInstance.objects.filter(**wf_filter)[- offset_value:to- ].order_by(order_by)+ ToolInstance.objects.filter(**wf_filter)[offset_value:to].order_by(+ order_by+ ))- return list(- ToolInstance.objects.filter(**wf_filter).all().order_by(order_by)- )+ return list(ToolInstance.objects.filter(**wf_filter).all().order_by(order_by))@staticmethoddef update_instance_metadata(@@ -60,9 +58,7 @@ class ToolInstanceHelper:JsonSchemaKey.OUTPUT_FILE_CONNECTOR in metadataand JsonSchemaKey.OUTPUT_FOLDER in metadata):- output_connector_name = metadata[- JsonSchemaKey.OUTPUT_FILE_CONNECTOR- ]+ output_connector_name = metadata[JsonSchemaKey.OUTPUT_FILE_CONNECTOR]output_connector = ConnectorInstanceHelper.get_output_connector_instance_by_name_for_workflow( # noqatool_instance.workflow_id, output_connector_name)@@ -210,9 +206,7 @@ class ToolInstanceHelper:JsonSchemaKey.OUTPUT_FILE_CONNECTOR in metadataand JsonSchemaKey.OUTPUT_FOLDER in metadata):- output_connector_name = metadata[- JsonSchemaKey.OUTPUT_FILE_CONNECTOR- ]+ output_connector_name = metadata[JsonSchemaKey.OUTPUT_FILE_CONNECTOR]output_connector = ConnectorInstanceHelper.get_output_connector_instance_by_name_for_workflow( # noqatool_instance.workflow_id, output_connector_name)@@ -350,9 +344,7 @@ class ToolInstanceHelper:tool: Tool = ToolProcessor.get_tool_by_uid(tool_uid=tool_uid)tool_name: str = (- tool.properties.display_name- if tool.properties.display_name- else tool_uid+ tool.properties.display_name if tool.properties.display_name else tool_uid)schema_json: dict[str, Any] = ToolProcessor.get_json_schema_for_tool(tool_uid=tool_uid, user=user@@ -379,51 +371,39 @@ class ToolInstanceHelper:if llm.is_enabled and llm.adapter_id:adapter_id = tool_meta[llm.adapter_id]elif llm.is_enabled:- adapter_id = tool_meta[- AdapterPropertyKey.DEFAULT_LLM_ADAPTER_ID- ]+ adapter_id = tool_meta[AdapterPropertyKey.DEFAULT_LLM_ADAPTER_ID]adapter_ids.add(adapter_id)for vdb in tool.properties.adapter.vector_stores:if vdb.is_enabled and vdb.adapter_id:adapter_id = tool_meta[vdb.adapter_id]elif vdb.is_enabled:- adapter_id = tool_meta[- AdapterPropertyKey.DEFAULT_VECTOR_DB_ADAPTER_ID- ]+ adapter_id = tool_meta[AdapterPropertyKey.DEFAULT_VECTOR_DB_ADAPTER_ID]adapter_ids.add(adapter_id)for embedding in tool.properties.adapter.embedding_services:if embedding.is_enabled and embedding.adapter_id:adapter_id = tool_meta[embedding.adapter_id]elif embedding.is_enabled:- adapter_id = tool_meta[- AdapterPropertyKey.DEFAULT_EMBEDDING_ADAPTER_ID- ]+ adapter_id = tool_meta[AdapterPropertyKey.DEFAULT_EMBEDDING_ADAPTER_ID]adapter_ids.add(adapter_id)for text_extractor in tool.properties.adapter.text_extractors:if text_extractor.is_enabled and text_extractor.adapter_id:adapter_id = tool_meta[text_extractor.adapter_id]elif text_extractor.is_enabled:- adapter_id = tool_meta[- AdapterPropertyKey.DEFAULT_X2TEXT_ADAPTER_ID- ]+ adapter_id = tool_meta[AdapterPropertyKey.DEFAULT_X2TEXT_ADAPTER_ID]adapter_ids.add(adapter_id)- ToolInstanceHelper.validate_adapter_access(- user=user, adapter_ids=adapter_ids- )+ ToolInstanceHelper.validate_adapter_access(user=user, adapter_ids=adapter_ids)@staticmethoddef validate_adapter_access(user: User,adapter_ids: set[str],) -> None:- adapter_instances = AdapterInstance.objects.filter(- id__in=adapter_ids- ).all()+ adapter_instances = AdapterInstance.objects.filter(id__in=adapter_ids).all()for adapter_instance in adapter_instances:if not (@@ -458,6 +438,4 @@ class ToolInstanceHelper:):returnelse:- raise PermissionDenied(- "You don't have permission to perform this action."- )+ raise PermissionDenied("You don't have permission to perform this action.")diff --git a/backend/tool_instance/tool_processor.py b/backend/tool_instance/tool_processor.pyindex acc0ab06f..bd0a01cf7 100644--- a/backend/tool_instance/tool_processor.py+++ b/backend/tool_instance/tool_processor.py@@ -62,9 +62,7 @@ class ToolProcessor:return schema_json@staticmethod- def update_schema_with_adapter_configurations(- schema: Spec, user: User- ) -> None:+ def update_schema_with_adapter_configurations(schema: Spec, user: User) -> None:"""Updates the JSON schema with the available adapter configurationsfor the LLM, embedding, and vector DB adapters.@@ -85,9 +83,7 @@ class ToolProcessor:AdapterTypes.LLM, user=user)for key in llm_keys:- adapter_names = map(- lambda adapter: str(adapter.adapter_name), adapters- )+ adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)schema.properties[key]["enum"] = list(adapter_names)if embedding_keys:@@ -95,9 +91,7 @@ class ToolProcessor:AdapterTypes.EMBEDDING, user=user)for key in embedding_keys:- adapter_names = map(- lambda adapter: str(adapter.adapter_name), adapters- )+ adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)schema.properties[key]["enum"] = list(adapter_names)if vector_db_keys:@@ -105,9 +99,7 @@ class ToolProcessor:AdapterTypes.VECTOR_DB, user=user)for key in vector_db_keys:- adapter_names = map(- lambda adapter: str(adapter.adapter_name), adapters- )+ adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)schema.properties[key]["enum"] = list(adapter_names)if x2text_keys:@@ -115,9 +107,7 @@ class ToolProcessor:AdapterTypes.X2TEXT, user=user)for key in x2text_keys:- adapter_names = map(- lambda adapter: str(adapter.adapter_name), adapters- )+ adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)schema.properties[key]["enum"] = list(adapter_names)if ocr_keys:@@ -125,9 +115,7 @@ class ToolProcessor:AdapterTypes.OCR, user=user)for key in ocr_keys:- adapter_names = map(- lambda adapter: str(adapter.adapter_name), adapters- )+ adapter_names = map(lambda adapter: str(adapter.adapter_name), adapters)schema.properties[key]["enum"] = list(adapter_names)@staticmethod@@ -137,9 +125,7 @@ class ToolProcessor:prompt_studio_tools: list[dict[str, Any]] = (PromptStudioRegistryHelper.fetch_json_for_registry(user))- tool_list: list[dict[str, Any]] = (- tool_registry.fetch_tools_descriptions()- )+ tool_list: list[dict[str, Any]] = tool_registry.fetch_tools_descriptions()tool_list = tool_list + prompt_studio_toolsreturn tool_listdiff --git a/backend/tool_instance/views.py b/backend/tool_instance/views.pyindex 98bf6c8a8..e6b537003 100644--- a/backend/tool_instance/views.py+++ b/backend/tool_instance/views.py@@ -13,10 +13,7 @@ from rest_framework.versioning import URLPathVersioningfrom tool_instance.constants import ToolInstanceErrorsfrom tool_instance.constants import ToolInstanceKey as TIKeyfrom tool_instance.constants import ToolKey-from tool_instance.exceptions import (- FetchToolListFailed,- ToolFunctionIsMandatory,-)+from tool_instance.exceptions import FetchToolListFailed, ToolFunctionIsMandatoryfrom tool_instance.models import ToolInstancefrom tool_instance.serializers import (ToolInstanceReorderSerializer as TIReorderSerializer,@@ -134,19 +131,13 @@ class ToolInstanceViewSet(viewsets.ModelViewSet):instance.save()return- def partial_update(- self, request: Request, *args: Any, **kwargs: Any- ) -> Response:+ def partial_update(self, request: Request, *args: Any, **kwargs: Any) -> Response:"""Allows partial updates on a tool instance."""instance: ToolInstance = self.get_object()- serializer = self.get_serializer(- instance, data=request.data, partial=True- )+ serializer = self.get_serializer(instance, data=request.data, partial=True)serializer.is_valid(raise_exception=True)if serializer.validated_data.get(TIKey.METADATA):- metadata: dict[str, Any] = serializer.validated_data.get(- TIKey.METADATA- )+ metadata: dict[str, Any] = serializer.validated_data.get(TIKey.METADATA)# TODO: Move update logic into serializerToolInstanceHelper.update_instance_metadata(@@ -168,10 +159,6 @@ class ToolInstanceViewSet(viewsets.ModelViewSet):]ToolInstanceHelper.reorder_tool_instances(instances_to_reorder)- tool_instances = ToolInstance.objects.get_instances_for_workflow(- workflow=wf_id- )- ti_serializer = ToolInstanceSerializer(- instance=tool_instances, many=True- )+ tool_instances = ToolInstance.objects.get_instances_for_workflow(workflow=wf_id)+ ti_serializer = ToolInstanceSerializer(instance=tool_instances, many=True)return Response(ti_serializer.data, status=status.HTTP_200_OK)diff --git a/backend/utils/log_events.py b/backend/utils/log_events.pyindex 0e510db53..d83cc03ee 100644--- a/backend/utils/log_events.py+++ b/backend/utils/log_events.py@@ -59,9 +59,7 @@ def _pubsub_listen_forever() -> None:message = pubsub.get_message()if message:- logger.debug(- f"[{os.getpid()}] Pub sub message received: {message}"- )+ logger.debug(f"[{os.getpid()}] Pub sub message received: {message}")if message["type"] == "pmessage":_handle_pubsub_messages(message)@@ -75,9 +73,7 @@ def _pubsub_listen_forever() -> None:def start_server(django_app: WSGIHandler, namespace: str) -> WSGIHandler:django_app = socketio.WSGIApp(sio, django_app, socketio_path=namespace)- pubsub_listener = threading.Thread(- target=_pubsub_listen_forever, daemon=True- )+ pubsub_listener = threading.Thread(target=_pubsub_listen_forever, daemon=True)pubsub_listener.start()return django_appdiff --git a/backend/utils/request/request.py b/backend/utils/request/request.pyindex e976e8274..9c05df155 100644--- a/backend/utils/request/request.py+++ b/backend/utils/request/request.py@@ -28,15 +28,11 @@ def make_http_request(if verb == HTTPMethod.GET:response = pyrequests.get(url, params=params, headers=headers)elif verb == HTTPMethod.POST:- response = pyrequests.post(- url, json=data, params=params, headers=headers- )+ response = pyrequests.post(url, json=data, params=params, headers=headers)elif verb == HTTPMethod.DELETE:response = pyrequests.delete(url, params=params, headers=headers)else:- raise ValueError(- "Invalid HTTP verb. Supported verbs: GET, POST, DELETE"- )+ raise ValueError("Invalid HTTP verb. Supported verbs: GET, POST, DELETE")response.raise_for_status()return_val: str = (diff --git a/backend/workflow_manager/endpoint/base_connector.py b/backend/workflow_manager/endpoint/base_connector.pyindex 7d4d7e6ae..e90ae03c2 100644--- a/backend/workflow_manager/endpoint/base_connector.py+++ b/backend/workflow_manager/endpoint/base_connector.py@@ -4,15 +4,11 @@ from typing import Anyfrom django.conf import settingsfrom django.db import connectionfrom fsspec import AbstractFileSystem-from unstract.workflow_execution.execution_file_handler import (- ExecutionFileHandler,-)+from unstract.workflow_execution.execution_file_handler import ExecutionFileHandlerfrom utils.constants import Commonfrom unstract.connectors.filesystems import connectors-from unstract.connectors.filesystems.unstract_file_system import (- UnstractFileSystem,-)+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystemclass BaseConnector(ExecutionFileHandler):@@ -76,9 +72,7 @@ class BaseConnector(ExecutionFileHandler):return schema@classmethod- def get_api_storage_dir_path(- cls, workflow_id: str, execution_id: str- ) -> str:+ def get_api_storage_dir_path(cls, workflow_id: str, execution_id: str) -> str:"""Get the directory path for storing api files.Parameters:diff --git a/backend/workflow_manager/endpoint/database_utils.py b/backend/workflow_manager/endpoint/database_utils.pyindex 6402b0697..f852e2002 100644--- a/backend/workflow_manager/endpoint/database_utils.py+++ b/backend/workflow_manager/endpoint/database_utils.py@@ -159,9 +159,7 @@ class DatabaseUtils:)column_types = DatabaseUtils.get_column_types_util(results)except Exception as e:- logger.error(- f"Error getting column types for {table_name}: {str(e)}"- )+ logger.error(f"Error getting column types for {table_name}: {str(e)}")raise ereturn column_types@@ -307,9 +305,7 @@ class DatabaseUtils:def get_db_class(connector_id: str, connector_settings: dict[str, Any]) -> UnstractDB:- connector = db_connectors[connector_id][Common.METADATA][- Common.CONNECTOR- ]+ connector = db_connectors[connector_id][Common.METADATA][Common.CONNECTOR]connector_class: UnstractDB = connector(connector_settings)return connector_class@@ -317,9 +313,7 @@ class DatabaseUtils:def execute_and_fetch_data(connector_id: str, connector_settings: dict[str, Any], query: str) -> Any:- connector = db_connectors[connector_id][Common.METADATA][- Common.CONNECTOR- ]+ connector = db_connectors[connector_id][Common.METADATA][Common.CONNECTOR]connector_class: UnstractDB = connector(connector_settings)return connector_class.execute(query=query)diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.pyindex f76bd6eda..a293eaf2e 100644--- a/backend/workflow_manager/endpoint/destination.py+++ b/backend/workflow_manager/endpoint/destination.py@@ -88,10 +88,7 @@ class DestinationConnector(BaseConnector):raise MissingDestinationConnectionType()if connection_type not in WorkflowEndpoint.ConnectionType.values:raise InvalidDestinationConnectionType()- if (- connection_type != WorkflowEndpoint.ConnectionType.API- and connector is None- ):+ if connection_type != WorkflowEndpoint.ConnectionType.API and connector is None:raise DestinationConnectorNotConfigured()def handle_output(@@ -107,9 +104,7 @@ class DestinationConnector(BaseConnector):result: Optional[str] = Noneif error:if connection_type == WorkflowEndpoint.ConnectionType.API:- self._handle_api_result(- file_name=file_name, error=error, result=result- )+ self._handle_api_result(file_name=file_name, error=error, result=result)returnif connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:self.copy_output_to_output_directory()@@ -117,9 +112,7 @@ class DestinationConnector(BaseConnector):self.insert_into_db(file_history)elif connection_type == WorkflowEndpoint.ConnectionType.API:result = self.get_result(file_history)- self._handle_api_result(- file_name=file_name, error=error, result=result- )+ self._handle_api_result(file_name=file_name, error=error, result=result)if not file_history:FileHistoryHelper.create_file_history(cache_key=file_hash,@@ -184,27 +177,17 @@ class DestinationConnector(BaseConnector):connector_instance: ConnectorInstance = self.endpoint.connector_instanceconnector_settings: dict[str, Any] = connector_instance.metadatadestination_configurations: dict[str, Any] = self.endpoint.configuration- table_name: str = str(- destination_configurations.get(DestinationKey.TABLE)- )+ table_name: str = str(destination_configurations.get(DestinationKey.TABLE))include_agent: bool = bool(destination_configurations.get(DestinationKey.INCLUDE_AGENT, False))include_timestamp = bool(- destination_configurations.get(- DestinationKey.INCLUDE_TIMESTAMP, False- )- )- agent_name = str(- destination_configurations.get(DestinationKey.AGENT_NAME)- )- column_mode = str(- destination_configurations.get(DestinationKey.COLUMN_MODE)+ destination_configurations.get(DestinationKey.INCLUDE_TIMESTAMP, False))+ agent_name = str(destination_configurations.get(DestinationKey.AGENT_NAME))+ column_mode = str(destination_configurations.get(DestinationKey.COLUMN_MODE))single_column_name = str(- destination_configurations.get(- DestinationKey.SINGLE_COLUMN_NAME, "data"- )+ destination_configurations.get(DestinationKey.SINGLE_COLUMN_NAME, "data"))data = self.get_result(file_history)@@ -230,14 +213,12 @@ class DestinationConnector(BaseConnector):table_name=table_name,database_entry=values,)- sql_columns_and_values = (- DatabaseUtils.get_sql_columns_and_values_for_query(- engine=engine,- connector_id=connector_instance.connector_id,- connector_settings=connector_settings,- table_name=table_name,- values=values,- )+ sql_columns_and_values = DatabaseUtils.get_sql_columns_and_values_for_query(+ engine=engine,+ connector_id=connector_instance.connector_id,+ connector_settings=connector_settings,+ table_name=table_name,+ values=values,)DatabaseUtils.execute_write_query(engine=engine,@@ -364,9 +345,7 @@ class DestinationConnector(BaseConnector):self.delete_api_storage_dir(self.workflow_id, self.execution_id)@classmethod- def delete_api_storage_dir(- cls, workflow_id: str, execution_id: str- ) -> None:+ def delete_api_storage_dir(cls, workflow_id: str, execution_id: str) -> None:"""Delete the api storage path.Returns:diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.pyindex 973655457..c97ea149f 100644--- a/backend/workflow_manager/endpoint/source.py+++ b/backend/workflow_manager/endpoint/source.py@@ -102,10 +102,7 @@ class SourceConnector(BaseConnector):raise MissingSourceConnectionType()if connection_type not in WorkflowEndpoint.ConnectionType.values:raise InvalidSourceConnectionType()- if (- connection_type != WorkflowEndpoint.ConnectionType.API- and connector is None- ):+ if connection_type != WorkflowEndpoint.ConnectionType.API and connector is None:raise SourceConnectorNotConfigured()def valid_file_patterns(self, required_patterns: list[Any]) -> list[str]:@@ -147,9 +144,7 @@ class SourceConnector(BaseConnector):connector: ConnectorInstance = self.endpoint.connector_instanceconnector_settings: dict[str, Any] = connector.connector_metadatasource_configurations: dict[str, Any] = self.endpoint.configuration- required_patterns = source_configurations.get(- SourceKey.FILE_EXTENSIONS, []- )+ required_patterns = source_configurations.get(SourceKey.FILE_EXTENSIONS, [])recursive = bool(source_configurations.get(SourceKey.PROCESS_SUB_DIRECTORIES, False))@@ -159,13 +154,9 @@ class SourceConnector(BaseConnector):))root_dir_path = connector_settings.get(ConnectorKeys.PATH, "")- input_directory = str(- source_configurations.get(SourceKey.ROOT_FOLDER, "")- )+ input_directory = str(source_configurations.get(SourceKey.ROOT_FOLDER, ""))if root_dir_path: # user needs to manually type the optional file path- input_directory = str(- Path(root_dir_path, input_directory.lstrip("/"))- )+ input_directory = str(Path(root_dir_path, input_directory.lstrip("/")))if not isinstance(required_patterns, list):required_patterns = [required_patterns]@@ -187,9 +178,7 @@ class SourceConnector(BaseConnector):) -> None:if not self.execution_service:return None- input_log = (- f"##Input folder:\n\n `{os.path.basename(input_directory)}`\n\n"- )+ input_log = f"##Input folder:\n\n `{os.path.basename(input_directory)}`\n\n"self.execution_service.publish_update_log(state=LogState.INPUT_UPDATE, message=input_log)@@ -198,13 +187,9 @@ class SourceConnector(BaseConnector):state=LogState.OUTPUT_UPDATE, message=output_log)- def publish_input_file_content(- self, input_file_path: str, input_text: str- ) -> None:+ def publish_input_file_content(self, input_file_path: str, input_text: str) -> None:if self.execution_service:- output_log_message = (- f"##Input text:\n\n```text\n{input_text}\n```\n\n"- )+ output_log_message = f"##Input text:\n\n```text\n{input_text}\n```\n\n"input_log_message = ("##Input file:\n\n```text\n"f"{os.path.basename(input_file_path)}\n```\n\n"@@ -254,9 +239,7 @@ class SourceConnector(BaseConnector):count = 0max_depth = int(SourceConstant.MAX_RECURSIVE_DEPTH) if recursive else 1- for root, dirs, files in source_fs.walk(- input_directory, maxdepth=max_depth- ):+ for root, dirs, files in source_fs.walk(input_directory, maxdepth=max_depth):if count >= limit:breakfor file in files:@@ -326,9 +309,7 @@ class SourceConnector(BaseConnector):"""connector: ConnectorInstance = self.endpoint.connector_instanceconnector_settings: dict[str, Any] = connector.connector_metadata- source_file_path = os.path.join(- self.execution_dir, WorkflowFileType.SOURCE- )+ source_file_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)source_file = f"file://{source_file_path}"@@ -346,8 +327,7 @@ class SourceConnector(BaseConnector):f": {hash_value_of_file_content}")input_log = (- file_content[:500].decode("utf-8", errors="replace")- + "...(truncated)"+ file_content[:500].decode("utf-8", errors="replace") + "...(truncated)")self.publish_input_file_content(input_file_path, input_log)@@ -356,9 +336,7 @@ class SourceConnector(BaseConnector):logger.info(f"{input_file_path} is added in to execution directory")return hash_value_of_file_content- def add_input_from_api_storage_to_volume(- self, input_file_path: str- ) -> None:+ def add_input_from_api_storage_to_volume(self, input_file_path: str) -> None:"""Add input file to execution directory from api storage."""infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)source_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)@@ -386,9 +364,7 @@ class SourceConnector(BaseConnector):input_file_path=input_file_path,)elif connection_type == WorkflowEndpoint.ConnectionType.API:- self.add_input_from_api_storage_to_volume(- input_file_path=input_file_path- )+ self.add_input_from_api_storage_to_volume(input_file_path=input_file_path)if file_name not in hash_values_of_files:raise FileHashNotFound()file_content_hash = hash_values_of_files[file_name]diff --git a/backend/workflow_manager/endpoint/urls.py b/backend/workflow_manager/endpoint/urls.pyindex 9efd62f80..1b658b27c 100644--- a/backend/workflow_manager/endpoint/urls.py+++ b/backend/workflow_manager/endpoint/urls.py@@ -14,9 +14,7 @@ endpoint_settings_detail = WorkflowEndpointViewSet.as_view(urlpatterns = [path("", endpoint_list, name="endpoint-list"),- path(- "<str:pk>/", workflow_endpoint_detail, name="workflow-endpoint-detail"- ),+ path("<str:pk>/", workflow_endpoint_detail, name="workflow-endpoint-detail"),path("<str:pk>/settings/",endpoint_settings_detail,diff --git a/backend/workflow_manager/endpoint/views.py b/backend/workflow_manager/endpoint/views.pyindex b1cc1c35c..50375d97f 100644--- a/backend/workflow_manager/endpoint/views.py+++ b/backend/workflow_manager/endpoint/views.py@@ -21,12 +21,8 @@ class WorkflowEndpointViewSet(viewsets.ModelViewSet):.select_related("workflow").filter(workflow__created_by=self.request.user))- endpoint_type_filter = self.request.query_params.get(- "endpoint_type", None- )- connection_type_filter = self.request.query_params.get(- "connection_type", None- )+ endpoint_type_filter = self.request.query_params.get("endpoint_type", None)+ connection_type_filter = self.request.query_params.get("connection_type", None)if endpoint_type_filter:queryset = queryset.filter(endpoint_type=endpoint_type_filter)if connection_type_filter:diff --git a/backend/workflow_manager/workflow/exceptions.py b/backend/workflow_manager/workflow/exceptions.pyindex 4715a9719..56f111600 100644--- a/backend/workflow_manager/workflow/exceptions.py+++ b/backend/workflow_manager/workflow/exceptions.py@@ -62,9 +62,7 @@ class WorkflowExecutionBadRequestException(APIException):status_code = 400default_detail = "Bad request"- def __init__(- self, detail: Optional[str] = None, code: Optional[int] = None- ):+ def __init__(self, detail: Optional[str] = None, code: Optional[int] = None):if detail is not None:self.detail = detailif code is not None:diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.pyindex 035c21e31..acfff7fee 100644--- a/backend/workflow_manager/workflow/execution.py+++ b/backend/workflow_manager/workflow/execution.py@@ -5,23 +5,15 @@ from typing import Optionalfrom account.constants import Commonfrom api.exceptions import InvalidAPIRequestfrom django.db import connection-from platform_settings.platform_auth_service import (- PlatformAuthenticationService,-)+from platform_settings.platform_auth_service import PlatformAuthenticationServicefrom tool_instance.constants import JsonSchemaKeyfrom tool_instance.models import ToolInstancefrom tool_instance.tool_processor import ToolProcessorfrom unstract.tool_registry.dto import Toolfrom unstract.workflow_execution import WorkflowExecutionService-from unstract.workflow_execution.dto import (- ToolInstance as ToolInstanceDataClass,-)+from unstract.workflow_execution.dto import ToolInstance as ToolInstanceDataClassfrom unstract.workflow_execution.dto import WorkflowDto-from unstract.workflow_execution.enums import (- ExecutionType,- LogComponent,- LogState,-)+from unstract.workflow_execution.enums import ExecutionType, LogComponent, LogStatefrom unstract.workflow_execution.exceptions import StopExecutionfrom utils.local_context import StateStorefrom workflow_manager.workflow.constants import WorkflowKey@@ -51,8 +43,8 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):tool_instances_as_dto.append(self.convert_tool_instance_model_to_data_class(tool_instance))- workflow_as_dto: WorkflowDto = (- self.convert_workflow_model_to_data_class(workflow=workflow)+ workflow_as_dto: WorkflowDto = self.convert_workflow_model_to_data_class(+ workflow=workflow)organization_id = organization_id or connection.tenant.schema_nameif not organization_id:@@ -71,9 +63,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):# Use pipline_id for pipelines / API deployment# since session might not be present.log_events_id = StateStore.get(Common.LOG_EVENTS_ID)- self.execution_log_id = (- log_events_id if log_events_id else pipeline_id- )+ self.execution_log_id = log_events_id if log_events_id else pipeline_idself.execution_mode = modeself.execution_method: tuple[str, str] = (WorkflowExecution.Method.SCHEDULED@@ -103,17 +93,13 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):self.set_messaging_channel(str(self.execution_log_id))project_settings = {}- project_settings[WorkflowKey.WF_PROJECT_GUID] = str(- self.execution_log_id- )+ project_settings[WorkflowKey.WF_PROJECT_GUID] = str(self.execution_log_id)self.workflow_id = workflow.idself.project_settings = project_settingsself.pipeline_id = pipeline_idself.execution_id = str(workflow_execution.id)- self.compilation_result = self.compile_workflow(- execution_id=self.execution_id- )+ self.compilation_result = self.compile_workflow(execution_id=self.execution_id)def _initiate_api_execution(self, tool_instance: ToolInstance, execution_path: Optional[str]@@ -250,18 +236,14 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):end_time = time.time()execution_time = end_time - start_timemessage = str(exception)[:EXECUTION_ERROR_LENGTH]- logger.info(- f"Execution {self.execution_id} Error {exception}"- )+ logger.info(f"Execution {self.execution_id} Error {exception}")raise self.__execution_error(error_message=message,detailed_message=message,execution_time=execution_time,)else:- error_message = (- f"Unknown Execution Method {self.execution_mode}"- )+ error_message = f"Unknown Execution Method {self.execution_mode}"raise self.__execution_error(error_message=error_message, detailed_message=error_message)@@ -282,9 +264,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):None"""self.publish_log(f"Total matched files: {total_files}")- self.publish_update_log(- LogState.BEGIN_WORKFLOW, "1", LogComponent.STATUS_BAR- )+ self.publish_update_log(LogState.BEGIN_WORKFLOW, "1", LogComponent.STATUS_BAR)self.publish_update_log(LogState.RUNNING, "Ready for execution", LogComponent.WORKFLOW)@@ -295,9 +275,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):Returns:None"""- self.publish_update_log(- LogState.END_WORKFLOW, "1", LogComponent.STATUS_BAR- )+ self.publish_update_log(LogState.END_WORKFLOW, "1", LogComponent.STATUS_BAR)self.publish_update_log(LogState.SUCCESS, "Executed successfully", LogComponent.WORKFLOW)@@ -344,9 +322,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):if single_step:execution_type = ExecutionType.STEPif not (file_history and file_history.is_completed()):- self.execute_uncached_input(- file_name=file_name, single_step=single_step- )+ self.execute_uncached_input(file_name=file_name, single_step=single_step)else:self.publish_log(f"Skipping file {file_name} as it is already processed."@@ -400,9 +376,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):execution_type = ExecutionType.COMPLETEif single_step:execution_type = ExecutionType.STEP- self.publish_initial_tool_execution_logs(- current_step, total_step, file_name- )+ self.publish_initial_tool_execution_logs(current_step, total_step, file_name)self._handle_execution_type(execution_type)source_status_message = (@@ -416,9 +390,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):self.publish_log("Trying to fetch results from cache")@staticmethod- def update_execution_status(- execution_id: str, status: ExecutionStatus- ) -> None:+ def update_execution_status(execution_id: str, status: ExecutionStatus) -> None:try:execution = WorkflowExecution.objects.get(pk=execution_id)execution.status = status.valuediff --git a/backend/workflow_manager/workflow/generator.py b/backend/workflow_manager/workflow/generator.pyindex f52a0a9ce..7c901b4bf 100644--- a/backend/workflow_manager/workflow/generator.py+++ b/backend/workflow_manager/workflow/generator.py@@ -35,9 +35,7 @@ class WorkflowGenerator:def provisional_wf(self) -> ProvisionalWorkflow:return self._provisional_wf- def _get_provisional_workflow(- self, tools: list[Tool]- ) -> ProvisionalWorkflow:+ def _get_provisional_workflow(self, tools: list[Tool]) -> ProvisionalWorkflow:"""Helper to generate the provisional workflow Gets stored as`workflow.Workflow.llm_response` eventually."""provisional_wf: ProvisionalWorkflow@@ -48,13 +46,11 @@ class WorkflowGenerator:)llm_interface = LLMInterface()- provisional_wf_dict = (- llm_interface.get_provisional_workflow_from_llm(- workflow_id=self._workflow_id,- tools=tools,- user_prompt=self._request.data.get(WorkflowKey.PROMPT_TEXT),- use_cache=True,- )+ provisional_wf_dict = llm_interface.get_provisional_workflow_from_llm(+ workflow_id=self._workflow_id,+ tools=tools,+ user_prompt=self._request.data.get(WorkflowKey.PROMPT_TEXT),+ use_cache=True,)provisional_wf = ProvisionalWorkflow(provisional_wf_dict)if provisional_wf.result != "OK":@@ -112,8 +108,6 @@ class WorkflowGenerator:}tool_instance_data_list.append(tool_instance_data)except Exception as e:- logger.error(- f"Error while getting data for {tool_function}: {e}"- )+ logger.error(f"Error while getting data for {tool_function}: {e}")raise ToolInstantiationError(tool_name=tool_function)return tool_instance_data_listdiff --git a/backend/workflow_manager/workflow/migrations/0001_initial.py b/backend/workflow_manager/workflow/migrations/0001_initial.pyindex 9b5e2a4d2..50483fe13 100644--- a/backend/workflow_manager/workflow/migrations/0001_initial.py+++ b/backend/workflow_manager/workflow/migrations/0001_initial.py@@ -267,9 +267,7 @@ class Migration(migrations.Migration):),("result",- models.TextField(- blank=True, db_comment="Result from execution"- ),+ models.TextField(blank=True, db_comment="Result from execution"),),("workflow",diff --git a/backend/workflow_manager/workflow/models/execution.py b/backend/workflow_manager/workflow/models/execution.pyindex ddba5b475..1f70df3b1 100644--- a/backend/workflow_manager/workflow/models/execution.py+++ b/backend/workflow_manager/workflow/models/execution.py@@ -52,18 +52,14 @@ class WorkflowExecution(BaseModel):execution_log_id = models.CharField(default="", editable=False, db_comment="Execution log events Id")- status = models.CharField(- default="", db_comment="Current status of execution"- )+ status = models.CharField(default="", db_comment="Current status of execution")error_message = models.CharField(max_length=EXECUTION_ERROR_LENGTH,blank=True,default="",db_comment="Details of encountered errors",)- attempts = models.IntegerField(- default=0, db_comment="number of attempts taken"- )+ attempts = models.IntegerField(default=0, db_comment="number of attempts taken")execution_time = models.FloatField(default=0, db_comment="execution time in seconds")diff --git a/backend/workflow_manager/workflow/models/file_history.py b/backend/workflow_manager/workflow/models/file_history.pyindex f518d2ba9..7bc61f480 100644--- a/backend/workflow_manager/workflow/models/file_history.py+++ b/backend/workflow_manager/workflow/models/file_history.py@@ -16,8 +16,7 @@ class FileHistory(BaseModel):bool: True if the execution status is completed, False otherwise."""return (- self.status is not None- and self.status == ExecutionStatus.COMPLETED.value+ self.status is not None and self.status == ExecutionStatus.COMPLETED.value)id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)diff --git a/backend/workflow_manager/workflow/models/workflow.py b/backend/workflow_manager/workflow/models/workflow.pyindex 3da08344b..94505da55 100644--- a/backend/workflow_manager/workflow/models/workflow.py+++ b/backend/workflow_manager/workflow/models/workflow.py@@ -37,9 +37,7 @@ class Workflow(BaseModel):)# TODO: Move prompt fields as a One-One relationship/into Prompt insteadprompt_name = models.CharField(max_length=PROMPT_NAME_LENGTH, default="")- description = models.TextField(- max_length=DESCRIPTION_FIELD_LENGTH, default=""- )+ description = models.TextField(max_length=DESCRIPTION_FIELD_LENGTH, default="")workflow_name = models.CharField(max_length=WORKFLOW_NAME_SIZE, unique=True)prompt_text = models.TextField(default="")is_active = models.BooleanField(default=False)diff --git a/backend/workflow_manager/workflow/serializers.py b/backend/workflow_manager/workflow/serializers.pyindex b9ba92db7..180884887 100644--- a/backend/workflow_manager/workflow/serializers.py+++ b/backend/workflow_manager/workflow/serializers.py@@ -14,10 +14,7 @@ from rest_framework.serializers import (from tool_instance.serializers import ToolInstanceSerializerfrom tool_instance.tool_instance_helper import ToolInstanceHelperfrom workflow_manager.endpoint.models import WorkflowEndpoint-from workflow_manager.workflow.constants import (- WorkflowExecutionKey,- WorkflowKey,-)+from workflow_manager.workflow.constants import WorkflowExecutionKey, WorkflowKeyfrom workflow_manager.workflow.exceptions import WorkflowGenerationErrorfrom workflow_manager.workflow.generator import WorkflowGeneratorfrom workflow_manager.workflow.models.workflow import Workflowdiff --git a/backend/workflow_manager/workflow/urls.py b/backend/workflow_manager/workflow/urls.pyindex 648fe6513..144ad09b7 100644--- a/backend/workflow_manager/workflow/urls.py+++ b/backend/workflow_manager/workflow/urls.py@@ -18,14 +18,10 @@ workflow_detail = WorkflowViewSet.as_view(}# fmt: on)-workflow_execute = WorkflowViewSet.as_view(- {"post": "execute", "put": "activate"}-)+workflow_execute = WorkflowViewSet.as_view({"post": "execute", "put": "activate"})execution_entity = WorkflowViewSet.as_view({"get": "get_execution"})workflow_clear_cache = WorkflowViewSet.as_view({"get": "clear_cache"})-workflow_clear_file_marker = WorkflowViewSet.as_view(- {"get": "clear_file_marker"}-)+workflow_clear_file_marker = WorkflowViewSet.as_view({"get": "clear_file_marker"})workflow_schema = WorkflowViewSet.as_view({"get": "get_schema"})can_update = WorkflowViewSet.as_view({"get": "can_update"})urlpatterns = format_suffix_patterns(diff --git a/backend/workflow_manager/workflow/views.py b/backend/workflow_manager/workflow/views.pyindex 40ae2eb88..7ce7b2cdc 100644--- a/backend/workflow_manager/workflow/views.py+++ b/backend/workflow_manager/workflow/views.py@@ -46,13 +46,9 @@ from backend.constants import RequestKeylogger = logging.getLogger(__name__)-def update_pipeline(- pipeline_guid: Optional[str], status: tuple[str, str]-) -> Any:+def update_pipeline(pipeline_guid: Optional[str], status: tuple[str, str]) -> Any:if pipeline_guid:- pipeline: Pipeline = PipelineProcessor.fetch_pipeline(- pipeline_id=pipeline_guid- )+ pipeline: Pipeline = PipelineProcessor.fetch_pipeline(pipeline_id=pipeline_guid)PipelineProcessor.update_pipeline_status(pipeline=pipeline, is_end=True, status=status)@@ -155,13 +151,9 @@ class WorkflowViewSet(viewsets.ModelViewSet):def get_execution(self, request: Request, pk: str) -> Response:execution = WorkflowHelper.get_current_execution(pk)- return Response(- make_execution_response(execution), status=status.HTTP_200_OK- )+ return Response(make_execution_response(execution), status=status.HTTP_200_OK)- def get_error_from_serializer(- self, error_details: dict[str, Any]- ) -> Optional[str]:+ def get_error_from_serializer(self, error_details: dict[str, Any]) -> Optional[str]:"""Validation error."""error_key = next(iter(error_details))# Get the first error message@@ -188,9 +180,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):if workflow_id:workflow = WorkflowHelper.get_workflow_by_id(workflow_id)elif project_id:- workflow = WorkflowHelper.get_active_workflow_by_project_id(- project_id- )+ workflow = WorkflowHelper.get_active_workflow_by_project_id(project_id)else:raise WorkflowDoesNotExistError()return workflow@@ -211,9 +201,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):workflow_id = serializer.get_workflow_id(serializer.validated_data)project_id = serializer.get_project_id(serializer.validated_data)execution_id = serializer.get_execution_id(serializer.validated_data)- execution_action = serializer.get_execution_action(- serializer.validated_data- )+ execution_action = serializer.get_execution_action(serializer.validated_data)file_objs = request.FILES.getlist("files")hashes_of_files = {}if file_objs and execution_id and workflow_id:@@ -247,10 +235,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):update_pipeline(pipeline_guid, Pipeline.PipelineStatus.FAILURE)logger.error(f"Error while executing workflow: {exception}")return Response(- {- "error": "Please check the logs for more details: "- + str(exception)- },+ {"error": "Please check the logs for more details: " + str(exception)},status=status.HTTP_400_BAD_REQUEST,)except Exception as exception:@@ -305,13 +290,9 @@ class WorkflowViewSet(viewsets.ModelViewSet):return Response(serializer.data, status=status.HTTP_200_OK)@action(detail=True, methods=["get"])- def clear_cache(- self, request: Request, *args: Any, **kwargs: Any- ) -> Response:+ def clear_cache(self, request: Request, *args: Any, **kwargs: Any) -> Response:workflow = self.get_object()- response: dict[str, Any] = WorkflowHelper.clear_cache(- workflow_id=workflow.id- )+ response: dict[str, Any] = WorkflowHelper.clear_cache(workflow_id=workflow.id)return Response(response.get("message"), status=response.get("status"))@action(detail=True, methods=["get"])@@ -330,9 +311,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):return Response(response.get("message"), status=response.get("status"))@action(detail=False, methods=["get"])- def get_schema(- self, request: Request, *args: Any, **kwargs: Any- ) -> Response:+ def get_schema(self, request: Request, *args: Any, **kwargs: Any) -> Response:"""Retrieves the JSON schema for source/destination type modules forentities file/API/DB.@@ -343,9 +322,7 @@ class WorkflowViewSet(viewsets.ModelViewSet):Response: JSON schema for the request made"""schema_type = request.query_params.get("type", SchemaType.SRC.value)- schema_entity = request.query_params.get(- "entity", SchemaEntity.FILE.value- )+ schema_entity = request.query_params.get("entity", SchemaEntity.FILE.value)WorkflowSchemaHelper.validate_request(schema_type=SchemaType(schema_type),diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.pyindex d84aafb29..db8a73d35 100644--- a/backend/workflow_manager/workflow/workflow_helper.py+++ b/backend/workflow_manager/workflow/workflow_helper.py@@ -30,11 +30,7 @@ from workflow_manager.workflow.constants import (WorkflowMessages,)from workflow_manager.workflow.dto import AsyncResultData, ExecutionResponse-from workflow_manager.workflow.enums import (- ExecutionStatus,- SchemaEntity,- SchemaType,-)+from workflow_manager.workflow.enums import ExecutionStatus, SchemaEntity, SchemaTypefrom workflow_manager.workflow.exceptions import (InvalidRequest,TaskDoesNotExistError,@@ -221,9 +217,7 @@ class WorkflowHelper:))- WorkflowHelper.validate_tool_instances_meta(- tool_instances=tool_instances- )+ WorkflowHelper.validate_tool_instances_meta(tool_instances=tool_instances)execution_mode = execution_mode or WorkflowExecution.Mode.INSTANTexecution_service = WorkflowHelper.build_workflow_execution_service(organization_id=organization_id,@@ -242,9 +236,7 @@ class WorkflowHelper:execution_id=execution_id,execution_service=execution_service,)- destination = DestinationConnector(- workflow=workflow, execution_id=execution_id- )+ destination = DestinationConnector(workflow=workflow, execution_id=execution_id)# Validating endpointssource.validate()destination.validate()@@ -422,9 +414,7 @@ class WorkflowHelper:)except IntegrityError:# Use existing instance on retry attempt- workflow_execution = WorkflowExecution.objects.get(- pk=execution_id- )+ workflow_execution = WorkflowExecution.objects.get(pk=execution_id)WorkflowExecutionServiceHelper.update_execution_task(execution_id=execution_id, task_id=task_id)@@ -462,13 +452,11 @@ class WorkflowHelper:if pipeline_id:# pipeline scheduled executionexecution_id = str(uuid.uuid4())- response: ExecutionResponse = (- WorkflowHelper.execute_workflow_async(- workflow_id=workflow.id,- pipeline_id=pipeline_id,- execution_id=execution_id,- hash_values_of_files=hash_values_of_files,- )+ response: ExecutionResponse = WorkflowHelper.execute_workflow_async(+ workflow_id=workflow.id,+ pipeline_id=pipeline_id,+ execution_id=execution_id,+ hash_values_of_files=hash_values_of_files,)return responsereturn WorkflowHelper.run_workflow(workflow=workflow)@@ -482,8 +470,7 @@ class WorkflowHelper:workflow_execution = WorkflowExecution.objects.get(pk=execution_id)if (workflow_execution.status != ExecutionStatus.PENDING.value- or workflow_execution.execution_type- != WorkflowExecution.Type.COMPLETE+ or workflow_execution.execution_type != WorkflowExecution.Type.COMPLETE):raise InvalidRequest(WorkflowErrors.INVALID_EXECUTION_ID)return WorkflowHelper.run_workflow(@@ -518,18 +505,13 @@ class WorkflowHelper:execution_id: Optional[str] = None,hash_values_of_files: dict[str, str] = {},) -> ExecutionResponse:- if (- execution_action- is Workflow.ExecutionAction.START.value # type: ignore- ):+ if execution_action is Workflow.ExecutionAction.START.value: # type: ignoreif execution_id is None:return WorkflowHelper.create_and_make_execution_response(workflow_id=workflow.id, single_step=True)try:- workflow_execution = WorkflowExecution.objects.get(- pk=execution_id- )+ workflow_execution = WorkflowExecution.objects.get(pk=execution_id)return WorkflowHelper.run_workflow(workflow=workflow,single_step=True,@@ -545,17 +527,12 @@ class WorkflowHelper:if execution_id is None:raise InvalidRequest("execution_id is missed")try:- workflow_execution = WorkflowExecution.objects.get(- pk=execution_id- )+ workflow_execution = WorkflowExecution.objects.get(pk=execution_id)except WorkflowExecution.DoesNotExist:- raise WorkflowExecutionNotExist(- WorkflowErrors.INVALID_EXECUTION_ID- )+ raise WorkflowExecutionNotExist(WorkflowErrors.INVALID_EXECUTION_ID)if (workflow_execution.status != ExecutionStatus.PENDING.value- or workflow_execution.execution_type- != WorkflowExecution.Type.STEP+ or workflow_execution.execution_type != WorkflowExecution.Type.STEP):raise InvalidRequest(WorkflowErrors.INVALID_EXECUTION_ID)current_action: Optional[str] = CacheService.get_key(execution_id)@@ -581,13 +558,11 @@ class WorkflowHelper:single_step: bool = False,mode: tuple[str, str] = WorkflowExecution.Mode.INSTANT,) -> ExecutionResponse:- workflow_execution = (- WorkflowExecutionServiceHelper.create_workflow_execution(- workflow_id=workflow_id,- single_step=single_step,- pipeline_id=pipeline_id,- mode=mode,- )+ workflow_execution = WorkflowExecutionServiceHelper.create_workflow_execution(+ workflow_id=workflow_id,+ single_step=single_step,+ pipeline_id=pipeline_id,+ mode=mode,)return ExecutionResponse(workflow_execution.workflow_id,@@ -667,9 +642,7 @@ class WorkflowHelper:raise WorkflowDoesNotExistError()used_count = Pipeline.objects.filter(workflow=workflow).count()if used_count == 0:- used_count = APIDeployment.objects.filter(- workflow=workflow- ).count()+ used_count = APIDeployment.objects.filter(workflow=workflow).count()return {"can_update": used_count == 0}except Workflow.DoesNotExist:logger.error(f"Error getting workflow: {id}")@@ -680,9 +653,7 @@ class WorkflowSchemaHelper:"""Helper class for workflow schema related methods."""@staticmethod- def validate_request(- schema_type: SchemaType, schema_entity: SchemaEntity- ) -> bool:+ def validate_request(schema_type: SchemaType, schema_entity: SchemaEntity) -> bool:"""Validates the given args for reading the JSON schema.Schema type of `src`, allows entities `file` and `api`@@ -714,9 +685,7 @@ class WorkflowSchemaHelper:f"should be one of {possible_entities}")- if (- schema_type == SchemaType.SRC and schema_entity == SchemaEntity.DB- ) or (+ if (schema_type == SchemaType.SRC and schema_entity == SchemaEntity.DB) or (schema_type == SchemaType.DEST and schema_entity != SchemaEntity.DB):raise serializers.ValidationError(@@ -743,8 +712,7 @@ class WorkflowSchemaHelper:dict[str, Any]: JSON schema for the requested entity"""schema_path = (- f"{os.path.dirname(__file__)}/static/"- f"{schema_type}/{schema_entity}.json"+ f"{os.path.dirname(__file__)}/static/" f"{schema_type}/{schema_entity}.json")with open(schema_path, encoding="utf-8") as file:schema = json.load(file)diff --git a/docker/scripts/merge_env.py b/docker/scripts/merge_env.pyindex 9d736ae49..3099da501 100644--- a/docker/scripts/merge_env.py+++ b/docker/scripts/merge_env.py@@ -49,9 +49,7 @@ def _extract_from_env_file(file_path: str) -> dict[str, str]:return env-def _merge_to_env_file(- base_env_file_path: str, target_env: dict[str, str] = {}-) -> str:+def _merge_to_env_file(base_env_file_path: str, target_env: dict[str, str] = {}) -> str:"""Generates file contents after merging input base env file path withtarget env.@@ -118,12 +116,8 @@ def merge_env(base_env_file_path: str, target_env_file_path: str, dry_run: bool = False) -> None:target_env = _extract_from_env_file(target_env_file_path)- merged_contents = _merge_to_env_file(- base_env_file_path, target_env=target_env- )- _save_merged_contents(- target_env_file_path, merged_contents, dry_run=dry_run- )+ merged_contents = _merge_to_env_file(base_env_file_path, target_env=target_env)+ _save_merged_contents(target_env_file_path, merged_contents, dry_run=dry_run)if __name__ == "__main__":diff --git a/document-service/src/unstract/document_service/main.py b/document-service/src/unstract/document_service/main.pyindex fe88c6093..731a419b9 100644--- a/document-service/src/unstract/document_service/main.py+++ b/document-service/src/unstract/document_service/main.py@@ -16,13 +16,9 @@ logging.basicConfig()UPLOAD_FOLDER = os.environ.get("UPLOAD_FOLDER", "/tmp/document_service/upload")-PROCESS_FOLDER = os.environ.get(- "PROCESS_FOLDER", "/tmp/document_service/process"-)+PROCESS_FOLDER = os.environ.get("PROCESS_FOLDER", "/tmp/document_service/process")LIBREOFFICE_PYTHON = os.environ.get("LIBREOFFICE_PYTHON", "/usr/bin/python3")-MAX_FILE_SIZE = int(- os.environ.get("MAX_FILE_SIZE", 10485760)-) # 10 * 1024 * 1024+MAX_FILE_SIZE = int(os.environ.get("MAX_FILE_SIZE", 10485760)) # 10 * 1024 * 1024SERVICE_API_TOKEN = os.environ.get("SERVICE_API_TOKEN", "")app = Flask("document_service")@@ -104,9 +100,7 @@ def upload_file():redis_host = os.environ.get("REDIS_HOST")redis_port = int(os.environ.get("REDIS_PORT"))redis_password = os.environ.get("REDIS_PASSWORD")- r = redis.Redis(- host=redis_host, port=redis_port, password=redis_password- )+ r = redis.Redis(host=redis_host, port=redis_port, password=redis_password)# TODO: Create a file reaper process to look at uploaded time and deleteredis_key = f"upload_time:{account_id}_{file_name}"current_timestamp = int(time.time())@@ -130,9 +124,7 @@ def find_and_replace():output_format = request.args.get("output_format").lower()find_and_replace_text = request.json- app.logger.info(- f"Find and replace for file {file_name} for account {account_id}"- )+ app.logger.info(f"Find and replace for file {file_name} for account {account_id}")app.logger.info(f"Output format: {output_format}")if output_format not in ["pdf"]:@@ -152,9 +144,7 @@ def find_and_replace():try:command = f"{LIBREOFFICE_PYTHON} -m unoserver.converter --convert-to odt \--filter writer8 {file_namex} {file_name_odt}"- result = subprocess.run(- command, shell=True, capture_output=True, text=True- )+ result = subprocess.run(command, shell=True, capture_output=True, text=True)app.logger.info(result)if result.returncode != 0:app.logger.error(@@ -166,9 +156,7 @@ def find_and_replace():app.logger.info(f"File converted to ODT format successfully! {file_name_odt}")- app.logger.info(- f"ODT convertion result: {result.stdout} | {result.stderr}"- )+ app.logger.info(f"ODT convertion result: {result.stdout} | {result.stderr}")except Exception as e:app.logger.error(f"Error while converting file to ODT format: {e}")return "Error while converting file to ODT format!", 500@@ -182,13 +170,9 @@ def find_and_replace():replace_str = find_and_replace_text[find_str]for element in doc.getElementsByType(text.Span):if find_str in teletype.extractText(element):- app.logger.info(- f"Found {find_str} in {teletype.extractText(element)}"- )+ app.logger.info(f"Found {find_str} in {teletype.extractText(element)}")new_element = text.Span()- new_element.setAttribute(- "stylename", element.getAttribute("stylename")- )+ new_element.setAttribute("stylename", element.getAttribute("stylename"))t = teletype.extractText(element)t = t.replace(find_str, replace_str)new_element.addText(t)@@ -205,9 +189,7 @@ def find_and_replace():f"{LIBREOFFICE_PYTHON} -m unoserver.converter --convert-to pdf "f"--filter writer_pdf_Export {file_name_odt} {file_name_output}")- result = subprocess.run(- command, shell=True, capture_output=True, text=True- )+ result = subprocess.run(command, shell=True, capture_output=True, text=True)if result.returncode != 0:app.logger.error(f"Failed to convert file to {output_format} format: "@@ -219,13 +201,9 @@ def find_and_replace():f"File converted to {output_format} format successfully! "f"{file_name_output}")- app.logger.info(- f"ODT convertion result: {result.stdout} | {result.stderr}"- )+ app.logger.info(f"ODT convertion result: {result.stdout} | {result.stderr}")except Exception as e:- app.logger.error(- f"Error while converting file to {output_format} format: {e}"- )+ app.logger.error(f"Error while converting file to {output_format} format: {e}")return f"Error while converting file to {output_format} format!", 500return send_file(file_name_output, as_attachment=True)diff --git a/platform-service/src/unstract/platform_service/exceptions.py b/platform-service/src/unstract/platform_service/exceptions.pyindex fd310e0a9..c9a0c89c7 100644--- a/platform-service/src/unstract/platform_service/exceptions.py+++ b/platform-service/src/unstract/platform_service/exceptions.py@@ -1,7 +1,5 @@class CustomException(Exception):- def __init__(- self, message: str = "An error occurred", code: int = 500- ) -> None:+ def __init__(self, message: str = "An error occurred", code: int = 500) -> None:self.message = messageself.code = codesuper().__init__(self.message)diff --git a/platform-service/src/unstract/platform_service/main.py b/platform-service/src/unstract/platform_service/main.pyindex fb3d2c5fd..702ca4238 100644--- a/platform-service/src/unstract/platform_service/main.py+++ b/platform-service/src/unstract/platform_service/main.py@@ -106,14 +106,10 @@ def authentication_middleware(func: Any) -> Any:def get_account_from_bearer_token(token: Optional[str]) -> str:- query = (- "SELECT organization_id FROM account_platformkey "- f"WHERE key='{token}'"- )+ query = "SELECT organization_id FROM account_platformkey " f"WHERE key='{token}'"organization = execute_query(query)query_org = (- "SELECT schema_name FROM account_organization "- f"WHERE id='{organization}'"+ "SELECT schema_name FROM account_organization " f"WHERE id='{organization}'")schema_name: str = execute_query(query_org)return schema_name@@ -139,9 +135,7 @@ def validate_bearer_token(token: Optional[str]) -> bool:result_row = cursor.fetchone()cursor.close()if not result_row or len(result_row) == 0:- app.logger.error(- f"Authentication failed. bearer token not found {token}"- )+ app.logger.error(f"Authentication failed. bearer token not found {token}")return Falseplatform_key = str(result_row[1])is_active = bool(result_row[2])@@ -151,9 +145,7 @@ def validate_bearer_token(token: Optional[str]) -> bool:)return Falseif platform_key != token:- app.logger.error(- f"Authentication failed. Invalid bearer token: {token}"- )+ app.logger.error(f"Authentication failed. Invalid bearer token: {token}")return Falseexcept Exception as e:@@ -224,9 +216,7 @@ def usage() -> Any:usage_value=usage_value,usage_units=usage_units,)- app.logger.info(- "Entry created with id %s for %s", usage.id, organization_id- )+ app.logger.info("Entry created with id %s for %s", usage.id, organization_id)result["status"] = "OK"result["unique_id"] = usage.idreturn result@@ -368,20 +358,16 @@ def adapter_instance() -> Any:adapter_instance_id = request.args.get("adapter_instance_id")try:- data_dict = (- AdapterInstanceRequestHelper.get_adapter_instance_from_db(- db_instance=be_db,- organization_id=organization_id,- adapter_instance_id=adapter_instance_id,- )+ data_dict = AdapterInstanceRequestHelper.get_adapter_instance_from_db(+ db_instance=be_db,+ organization_id=organization_id,+ adapter_instance_id=adapter_instance_id,)f: Fernet = Fernet(ENCRYPTION_KEY.encode("utf-8"))data_dict["adapter_metadata"] = json.loads(- f.decrypt(- bytes(data_dict.pop("adapter_metadata_b")).decode("utf-8")- )+ f.decrypt(bytes(data_dict.pop("adapter_metadata_b")).decode("utf-8")))return jsonify(data_dict)diff --git a/platform-service/src/unstract/platform_service/utils.py b/platform-service/src/unstract/platform_service/utils.pyindex 04fddd4b9..aa57b46bd 100644--- a/platform-service/src/unstract/platform_service/utils.py+++ b/platform-service/src/unstract/platform_service/utils.py@@ -35,8 +35,7 @@ class EnvManager:ValueError: Error mentioning envs which are not configured."""if cls.missing_settings:- ERROR_MESSAGE = (- "Below required settings are missing.\n"- + ",\n".join(cls.missing_settings)+ ERROR_MESSAGE = "Below required settings are missing.\n" + ",\n".join(+ cls.missing_settings)raise ValueError(ERROR_MESSAGE)diff --git a/prompt-service/src/unstract/prompt_service/authentication_middleware.py b/prompt-service/src/unstract/prompt_service/authentication_middleware.pyindex 3c91c4c16..0ad5c6aaa 100644--- a/prompt-service/src/unstract/prompt_service/authentication_middleware.py+++ b/prompt-service/src/unstract/prompt_service/authentication_middleware.py@@ -11,9 +11,7 @@ class AuthenticationMiddleware:def validate_bearer_token(cls, token: Optional[str]) -> bool:try:if token is None:- current_app.logger.error(- "Authentication failed. Empty bearer token"- )+ current_app.logger.error("Authentication failed. Empty bearer token")return Falsequery = f"SELECT * FROM account_platformkey WHERE key = '{token}'"@@ -40,9 +38,7 @@ class AuthenticationMiddleware:return Falseexcept Exception as e:- current_app.logger.error(- f"Error while validating bearer token: {e}"- )+ current_app.logger.error(f"Error while validating bearer token: {e}")return Falsereturn True@@ -61,13 +57,11 @@ class AuthenticationMiddleware:@classmethoddef get_account_from_bearer_token(cls, token: Optional[str]) -> str:query = (- "SELECT organization_id FROM account_platformkey "- f"WHERE key='{token}'"+ "SELECT organization_id FROM account_platformkey " f"WHERE key='{token}'")organization = AuthenticationMiddleware.execute_query(query)query_org = (- "SELECT schema_name FROM account_organization "- f"WHERE id='{organization}'"+ "SELECT schema_name FROM account_organization " f"WHERE id='{organization}'")schema_name: str = AuthenticationMiddleware.execute_query(query_org)return schema_namediff --git a/prompt-service/src/unstract/prompt_service/helper.py b/prompt-service/src/unstract/prompt_service/helper.pyindex ab96860f8..0ce9da214 100644--- a/prompt-service/src/unstract/prompt_service/helper.py+++ b/prompt-service/src/unstract/prompt_service/helper.py@@ -63,9 +63,7 @@ def plugin_loader(app: Flask) -> dict[str, dict[str, Any]]:metadata: dict[str, Any] = module.metadataif metadata.get("disable", False):- app.logger.info(- f"Ignore disabled plugin: {pkg} v{metadata['version']}"- )+ app.logger.info(f"Ignore disabled plugin: {pkg} v{metadata['version']}")continuetry:diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.pyindex 9c6329128..130a0b5f8 100644--- a/prompt-service/src/unstract/prompt_service/main.py+++ b/prompt-service/src/unstract/prompt_service/main.py@@ -7,9 +7,7 @@ from flask import Flask, json, requestfrom llama_index.core import VectorStoreIndexfrom llama_index.core.llms import LLMfrom llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters-from unstract.prompt_service.authentication_middleware import (- AuthenticationMiddleware,-)+from unstract.prompt_service.authentication_middleware import AuthenticationMiddlewarefrom unstract.prompt_service.constants import PromptServiceContants as PSKeysfrom unstract.prompt_service.constants import RunLevelfrom unstract.prompt_service.helper import EnvLoader, plugin_loader@@ -18,9 +16,7 @@ from unstract.sdk.constants import LogLevelfrom unstract.sdk.embedding import ToolEmbeddingfrom unstract.sdk.index import ToolIndexfrom unstract.sdk.llm import ToolLLM-from unstract.sdk.utils.callback_manager import (- CallbackManager as UNCallbackManager,-)+from unstract.sdk.utils.callback_manager import CallbackManager as UNCallbackManagerfrom unstract.sdk.vector_db import ToolVectorDBfrom werkzeug.exceptions import HTTPException@@ -148,9 +144,7 @@ def authentication_middleware(func: Any) -> Any:def wrapper(*args: Any, **kwargs: Any) -> Any:token = AuthenticationMiddleware.get_token_from_auth_header(request)# Check if bearer token exists and validate it- if not token or not AuthenticationMiddleware.validate_bearer_token(- token- ):+ if not token or not AuthenticationMiddleware.validate_bearer_token(token):return "Unauthorized", 401return func(*args, **kwargs)@@ -195,9 +189,7 @@ def prompt_processor() -> Any:name = output[PSKeys.NAME]promptx = output[PSKeys.PROMPT]chunk_size = output[PSKeys.CHUNK_SIZE]- util = PromptServiceBaseTool(- log_level=LogLevel.INFO, platform_key=platform_key- )+ util = PromptServiceBaseTool(log_level=LogLevel.INFO, platform_key=platform_key)tool_index = ToolIndex(tool=util)if active is False:@@ -347,15 +339,9 @@ def prompt_processor() -> Any:)assertion_failed = Trueanswer = ""- if (- output[PSKeys.ASSERTION_FAILURE_PROMPT]- .lower()- .startswith("@assign")- ):+ if output[PSKeys.ASSERTION_FAILURE_PROMPT].lower().startswith("@assign"):answer = "NA"- first_space_index = output[- PSKeys.ASSERTION_FAILURE_PROMPT- ].find(" ")+ first_space_index = output[PSKeys.ASSERTION_FAILURE_PROMPT].find(" ")if first_space_index > 0:answer = output[PSKeys.ASSERTION_FAILURE_PROMPT][first_space_index + 1@@ -483,11 +469,7 @@ def prompt_processor() -> Any:else:structured_output[output[PSKeys.NAME]] = Falseelif output[PSKeys.TYPE] == PSKeys.JSON:- if (- assertion_failed- or answer.lower() == "[]"- or answer.lower() == "na"- ):+ if assertion_failed or answer.lower() == "[]" or answer.lower() == "na":structured_output[output[PSKeys.NAME]] = Noneelse:# Remove any markdown code blocks@@ -500,9 +482,7 @@ def prompt_processor() -> Any:try:structured_output[output[PSKeys.NAME]] = json.loads(answer)except Exception as e:- app.logger.info(- f"JSON format error : {answer}", LogLevel.ERROR- )+ app.logger.info(f"JSON format error : {answer}", LogLevel.ERROR)app.logger.info(f"Error parsing response (to json): {e}", LogLevel.ERROR)@@ -676,9 +656,7 @@ def simple_retriver( # type:ignoreif node.score > 0.6:text += node.get_content() + "\n"else:- app.logger.info(- "Node score is less than 0.6. " f"Ignored: {node.score}"- )+ app.logger.info("Node score is less than 0.6. " f"Ignored: {node.score}")answer, usage = construct_and_run_prompt( # type:ignoreoutput,@@ -720,9 +698,7 @@ def run_completion(platform_api_key = llm_helper.tool.get_env_or_die(PSKeys.PLATFORM_SERVICE_API_KEY)- completion = llm_helper.run_completion(- llm_li, platform_api_key, prompt, 3- )+ completion = llm_helper.run_completion(llm_li, platform_api_key, prompt, 3)answer: str = completion[PSKeys.RESPONSE].textusage = {}@@ -749,8 +725,7 @@ def extract_variable()else:raise ValueError(- f"Variable {variable_name} not found "- "in structured output"+ f"Variable {variable_name} not found " "in structured output")if promptx != output[PSKeys.PROMPT]:diff --git a/pyproject.toml b/pyproject.tomlindex e693e6115..4241a031e 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -51,14 +51,14 @@ hook-check-django-migrations = [][tool.black]-line-length = 80+line-length = 88target-version = ['py39'][tool.pycln]all = true[tool.isort]-line_length = 80+line_length = 88multi_line_output = 3include_trailing_comma = trueforce_grid_wrap = 0diff --git a/tools/classifier/src/helper.py b/tools/classifier/src/helper.pyindex 8870c5e25..a57e4c2a5 100644--- a/tools/classifier/src/helper.py+++ b/tools/classifier/src/helper.py@@ -30,9 +30,7 @@ class ClassifierHelper:return self._extract_from_adapter(file, text_extraction_adapter_id)- def _extract_from_adapter(- self, file: str, adapter_id: str- ) -> Optional[str]:+ def _extract_from_adapter(self, file: str, adapter_id: str) -> Optional[str]:"""Extract text from adapter.Args:@@ -42,15 +40,12 @@ class ClassifierHelper:str: page content"""self.tool.stream_log(- "Creating text extraction adapter "- f"using adapter_id: {adapter_id}"+ "Creating text extraction adapter " f"using adapter_id: {adapter_id}")tool_extraction = X2Text(tool=self.tool)adapter = tool_extraction.get_x2text(adapter_id)- self.tool.stream_log(- "Text extraction adapter has been created successfully."- )+ self.tool.stream_log("Text extraction adapter has been created successfully.")self.tool.stream_log("Adapter created")try:@@ -112,9 +107,7 @@ class ClassifierHelper:classification = self.get_result_from_cache(cache_key=cache_key)if classification is None:- self.tool.stream_log(- "No classification found in cache, calling LLM."- )+ self.tool.stream_log("No classification found in cache, calling LLM.")classification = self.call_llm(prompt=prompt, llm=llm)if not classification:classification = "unknown"diff --git a/tools/classifier/src/main.py b/tools/classifier/src/main.pyindex 6b1e3df2c..17d13d4fb 100644--- a/tools/classifier/src/main.py+++ b/tools/classifier/src/main.py@@ -14,9 +14,7 @@ from unstract.sdk.constants import (from unstract.sdk.llm import ToolLLMfrom unstract.sdk.tool.base import BaseToolfrom unstract.sdk.tool.entrypoint import ToolEntrypoint-from unstract.sdk.utils.callback_manager import (- CallbackManager as UNCallbackManager,-)+from unstract.sdk.utils.callback_manager import CallbackManager as UNCallbackManagerclass UnstractClassifier(BaseTool):@@ -33,9 +31,7 @@ class UnstractClassifier(BaseTool):elif len(bins) < 2:self.stream_error_and_exit("At least two bins are required")if not llm_adapter_instance_id:- self.stream_error_and_exit(- "Choose an LLM to process the classifier"- )+ self.stream_error_and_exit("Choose an LLM to process the classifier")if not text_extraction_adapter_id:self.stream_error_and_exit("Choose an LLM to extract the documents")@@ -95,12 +91,8 @@ class UnstractClassifier(BaseTool):)max_tokens = tool_llm.get_max_tokens(reserved_for_output=50 + 1000)max_bytes = int(max_tokens * 1.3)- self.stream_log(- f"LLM Max tokens: {max_tokens} ==> Max bytes: {max_bytes}"- )- self.stream_log(- f"LLM Max tokens: {max_tokens} ==> Max bytes: {max_bytes}"- )+ self.stream_log(f"LLM Max tokens: {max_tokens} ==> Max bytes: {max_bytes}")+ self.stream_log(f"LLM Max tokens: {max_tokens} ==> Max bytes: {max_bytes}")limited_text = ""for byte in text.encode():if len(limited_text.encode()) < max_bytes:diff --git a/tools/structure/src/main.py b/tools/structure/src/main.pyindex 4eabd063c..770eca2c1 100644--- a/tools/structure/src/main.py+++ b/tools/structure/src/main.py@@ -34,18 +34,12 @@ class StructureTool(BaseTool):tool=self, prompt_registry_id=prompt_registry_id)tool_metadata = exported_tool[SettingsKeys.TOOL_METADATA]- self.stream_log(- f"Tool Metadata retrived succesfully: {tool_metadata}"- )+ self.stream_log(f"Tool Metadata retrived succesfully: {tool_metadata}")except Exception as e:- self.stream_error_and_exit(- f"Error loading structure definition: {e}"- )+ self.stream_error_and_exit(f"Error loading structure definition: {e}")# Update GUI- input_log = (- f"### Structure Definition:\n```json\n{tool_metadata}\n```\n\n"- )+ input_log = f"### Structure Definition:\n```json\n{tool_metadata}\n```\n\n"output_log = "### Indexing..."self.stream_update(input_log, state=LogState.INPUT_UPDATE)self.stream_update(output_log, state=LogState.OUTPUT_UPDATE)@@ -94,9 +88,7 @@ class StructureTool(BaseTool):f"### Structure Definition:\n"f"```json\n{json.dumps(tool_metadata, indent=2)}\n```\n\n")- output_log = (- f"### Parsed output:\n```json\n{structured_output}\n```\n\n"- )+ output_log = f"### Parsed output:\n```json\n{structured_output}\n```\n\n"self.stream_update(input_log, state=LogState.INPUT_UPDATE)self.stream_update(output_log, state=LogState.OUTPUT_UPDATE)diff --git a/tools/text_extractor/src/main.py b/tools/text_extractor/src/main.pyindex 390da2a04..41ec9cfc8 100644--- a/tools/text_extractor/src/main.py+++ b/tools/text_extractor/src/main.py@@ -58,20 +58,14 @@ class TextExtractor(BaseTool):text_extraction_adapter = tool_extraction.get_x2text(adapter_instance_id=text_extraction_adapter_id)- self.stream_log(- "Text extraction adapter has been created successfully."- )- extracted_text = text_extraction_adapter.process(- input_file_path=input_file- )+ self.stream_log("Text extraction adapter has been created successfully.")+ extracted_text = text_extraction_adapter.process(input_file_path=input_file)extracted_text = self.convert_to_actual_string(extracted_text)self.stream_log("Text has been extracted successfully.")first_5_lines = "\n\n".join(extracted_text.split("\n")[:5])- output_log = (- f"### Text\n\n```text\n{first_5_lines}\n```\n\n...(truncated)"- )+ output_log = f"### Text\n\n```text\n{first_5_lines}\n```\n\n...(truncated)"self.stream_update(output_log, state=LogState.OUTPUT_UPDATE)try:@@ -87,9 +81,7 @@ class TextExtractor(BaseTool):"Error creating/writing output file: source_name not found")except Exception as e:- self.stream_error_and_exit(- f"Error creating/writing output file: {e}"- )+ self.stream_error_and_exit(f"Error creating/writing output file: {e}")self.write_tool_result(data=extracted_text)def convert_to_actual_string(self, text: Any) -> str:diff --git a/unstract/connectors/src/unstract/connectors/connectorkit.py b/unstract/connectors/src/unstract/connectors/connectorkit.pyindex e53ad43e8..1ef216491 100644--- a/unstract/connectors/src/unstract/connectors/connectorkit.py+++ b/unstract/connectors/src/unstract/connectors/connectorkit.py@@ -40,23 +40,15 @@ class Connectorkit:connector_registry_metadata,) in self._connectors.items():if (- connector_registry_metadata[Common.METADATA][- Common.CONNECTOR- ].__name__+ connector_registry_metadata[Common.METADATA][Common.CONNECTOR].__name__is connector_name):- connector_class: UnstractConnector = (- connector_registry_metadata[Common.METADATA][- Common.CONNECTOR- ]- )+ connector_class: UnstractConnector = connector_registry_metadata[+ Common.METADATA+ ][Common.CONNECTOR]return connector_class- logging.error(- f">> Connector '{connector_name}' not found in connectorkit"- )- logging.error(- f">> Connectors in connectorkit : {self._connectors.keys()}"- )+ logging.error(f">> Connector '{connector_name}' not found in connectorkit")+ logging.error(f">> Connectors in connectorkit : {self._connectors.keys()}")return Nonedef get_connector_by_id(@@ -73,8 +65,8 @@ class Connectorkit:Returns:UnstractConnector: Concrete impl of the `UnstractConnector` base"""- connector_class: UnstractConnector = (- self.get_connector_class_by_connector_id(connector_id)+ connector_class: UnstractConnector = self.get_connector_class_by_connector_id(+ connector_id)return connector_class(*args, **kwargs)diff --git a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.pyindex 5661f32ff..2cd8b75f6 100644--- a/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py+++ b/unstract/connectors/src/unstract/connectors/databases/bigquery/bigquery.py@@ -13,9 +13,7 @@ from unstract.connectors.exceptions import ConnectorErrorclass BigQuery(UnstractDB):def __init__(self, settings: dict[str, Any]):super().__init__("BigQuery")- self.json_credentials = json.loads(- settings.get("json_credentials", "{}")- )+ self.json_credentials = json.loads(settings.get("json_credentials", "{}"))@staticmethoddef get_id() -> str:diff --git a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.pyindex 7b2ddf051..6025f5ee5 100644--- a/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py+++ b/unstract/connectors/src/unstract/connectors/databases/postgresql/postgresql.py@@ -21,11 +21,7 @@ class PostgreSQL(UnstractDB):if not self.schema:self.schema = "public"if not self.connection_url and not (- self.user- and self.password- and self.host- and self.port- and self.database+ self.user and self.password and self.host and self.port and self.database):raise ValueError("Either ConnectionURL or connection parameters must be provided."diff --git a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.pyindex 91389f34f..d83426e2c 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/azure_cloud_storage/azure_cloud_storage.py@@ -5,9 +5,7 @@ from typing import Anyfrom adlfs import AzureBlobFileSystemfrom unstract.connectors.exceptions import ConnectorError-from unstract.connectors.filesystems.unstract_file_system import (- UnstractFileSystem,-)+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystemlogger = logging.getLogger(__name__)diff --git a/unstract/connectors/src/unstract/connectors/filesystems/box/box.py b/unstract/connectors/src/unstract/connectors/filesystems/box/box.pyindex 7229413c1..e30cd264d 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/box/box.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/box/box.py@@ -9,9 +9,7 @@ from boxsdk import JWTAuthfrom boxsdk.exception import BoxOAuthExceptionfrom unstract.connectors.exceptions import ConnectorError-from unstract.connectors.filesystems.unstract_file_system import (- UnstractFileSystem,-)+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystemlogger = logging.getLogger(__name__)logging.getLogger("boxsdk").setLevel(logging.ERROR)@@ -80,9 +78,7 @@ class BoxFS(UnstractFileSystem):@staticmethoddef get_description() -> str:- return (- "Fetch and store data to and from the Box content management system"- )+ return "Fetch and store data to and from the Box content management system"@staticmethoddef get_icon() -> str:diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.pyindex ba1f32fbb..915cb2b2c 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_cloud_storage/google_cloud_storage.py@@ -6,9 +6,7 @@ from typing import Anyfrom gcsfs import GCSFileSystemfrom unstract.connectors.exceptions import ConnectorError-from unstract.connectors.filesystems.unstract_file_system import (- UnstractFileSystem,-)+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystemlogger = logging.getLogger(__name__)diff --git a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.pyindex 7fe9f1e62..1a2241b78 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/google_drive/google_drive.py@@ -8,12 +8,8 @@ from pydrive2.auth import GoogleAuthfrom pydrive2.fs import GDriveFileSystemfrom unstract.connectors.exceptions import ConnectorError-from unstract.connectors.filesystems.google_drive.constants import (- GDriveConstants,-)-from unstract.connectors.filesystems.unstract_file_system import (- UnstractFileSystem,-)+from unstract.connectors.filesystems.google_drive.constants import GDriveConstants+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystemfrom unstract.connectors.gcs_helper import GCSHelperlogger = logging.getLogger(__name__)@@ -39,9 +35,7 @@ class GoogleDriveFS(UnstractFileSystem):"invalid": False,"access_token": settings["access_token"],"refresh_token": settings["refresh_token"],- GDriveConstants.TOKEN_EXPIRY: settings[- GDriveConstants.TOKEN_EXPIRY- ],+ GDriveConstants.TOKEN_EXPIRY: settings[GDriveConstants.TOKEN_EXPIRY],}gauth = GoogleAuth(settings_file=f"{os.path.dirname(__file__)}/static/settings.yaml",diff --git a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py b/unstract/connectors/src/unstract/connectors/filesystems/http/http.pyindex 9169086a3..c12236e15 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/http/http.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/http/http.py@@ -6,9 +6,7 @@ import aiohttpfrom fsspec.implementations.http import HTTPFileSystemfrom unstract.connectors.exceptions import ConnectorError-from unstract.connectors.filesystems.unstract_file_system import (- UnstractFileSystem,-)+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystemlogger = logging.getLogger(__name__)@@ -25,9 +23,7 @@ class HttpFS(UnstractFileSystem):"base_url": settings["base_url"],}if all(settings.get(key) for key in ("username", "password")):- basic_auth = aiohttp.BasicAuth(- settings["username"], settings["password"]- )+ basic_auth = aiohttp.BasicAuth(settings["username"], settings["password"])client_kwargs.update({"auth": basic_auth})self.http_fs = HTTPFileSystem(client_kwargs=client_kwargs)diff --git a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.pyindex b51ab2f95..ec179e990 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/local_storage/local_storage.py@@ -4,9 +4,7 @@ from typing import Any, Optionalfrom fsspec.implementations.local import LocalFileSystem-from unstract.connectors.filesystems.unstract_file_system import (- UnstractFileSystem,-)+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystemlogger = logging.getLogger(__name__)diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.pyindex f22bf4c24..f0cf798b1 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py@@ -5,9 +5,7 @@ from typing import Anyfrom s3fs.core import S3FileSystemfrom unstract.connectors.exceptions import ConnectorError-from unstract.connectors.filesystems.unstract_file_system import (- UnstractFileSystem,-)+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystemlogger = logging.getLogger(__name__)diff --git a/unstract/connectors/src/unstract/connectors/filesystems/register.py b/unstract/connectors/src/unstract/connectors/filesystems/register.pyindex 5bcc637cd..d820e5f62 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/register.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/register.py@@ -4,9 +4,7 @@ from importlib import import_modulefrom typing import Anyfrom unstract.connectors.constants import Common-from unstract.connectors.filesystems.unstract_file_system import (- UnstractFileSystem,-)+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystemlogger = logging.getLogger(__name__)@@ -24,9 +22,7 @@ def register_connectors(connectors: dict[str, Any]) -> None:module = import_module(full_module_path)metadata = getattr(module, Common.METADATA, {})if metadata.get("is_active", False):- connector_class: UnstractFileSystem = metadata[- Common.CONNECTOR- ]+ connector_class: UnstractFileSystem = metadata[Common.CONNECTOR]connector_id = connector_class.get_id()if not connector_id or (connector_id in connectors):logger.warning(f"Duplicate Id : {connector_id}")diff --git a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.pyindex 2ac4bb0eb..31b743cbb 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/zs_dropbox/zs_dropbox.py@@ -6,9 +6,7 @@ from dropbox.exceptions import DropboxExceptionfrom dropboxdrivefs import DropboxDriveFileSystemfrom unstract.connectors.exceptions import ConnectorError-from unstract.connectors.filesystems.unstract_file_system import (- UnstractFileSystem,-)+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystemfrom .exceptions import handle_dropbox_exceptiondiff --git a/unstract/connectors/src/unstract/connectors/gcs_helper.py b/unstract/connectors/src/unstract/connectors/gcs_helper.pyindex 82c351c13..2e606adb2 100644--- a/unstract/connectors/src/unstract/connectors/gcs_helper.py+++ b/unstract/connectors/src/unstract/connectors/gcs_helper.py@@ -31,10 +31,8 @@ class GCSHelper:"GOOGLE_PROJECT_ID environment variable not set")- self.google_credentials = (- service_account.Credentials.from_service_account_info(- json.loads(self.google_service_json)- )+ self.google_credentials = service_account.Credentials.from_service_account_info(+ json.loads(self.google_service_json))def get_google_credentials(self) -> Credentials:@@ -60,30 +58,22 @@ class GCSHelper:md5_hash_bytes = base64.b64decode(blob.md5_hash)md5_hash_hex = md5_hash_bytes.hex()except Exception:- logger.error(- f"Could not get blob {object_name} from bucket {bucket_name}"- )+ logger.error(f"Could not get blob {object_name} from bucket {bucket_name}")return md5_hash_hex- def upload_file(- self, bucket_name: str, object_name: str, file_path: str- ) -> None:+ def upload_file(self, bucket_name: str, object_name: str, file_path: str) -> None:client = Client(credentials=self.google_credentials)bucket = client.bucket(bucket_name)blob = bucket.blob(object_name)blob.upload_from_filename(file_path)- def upload_text(- self, bucket_name: str, object_name: str, text: str- ) -> None:+ def upload_text(self, bucket_name: str, object_name: str, text: str) -> None:client = Client(credentials=self.google_credentials)bucket = client.bucket(bucket_name)blob = bucket.blob(object_name)blob.upload_from_string(text)- def upload_object(- self, bucket_name: str, object_name: str, object: Any- ) -> None:+ def upload_object(self, bucket_name: str, object_name: str, object: Any) -> None:client = Client(credentials=self.google_credentials)bucket = client.bucket(bucket_name)blob = bucket.blob(object_name)@@ -101,6 +91,4 @@ class GCSHelper:logger.info(f"Reading file {object_name} from bucket {bucket_name}")return objexcept Exception:- logger.error(- f"Could not get blob {object_name} from bucket {bucket_name}"- )+ logger.error(f"Could not get blob {object_name} from bucket {bucket_name}")diff --git a/unstract/connectors/tests/filesystems/test_google_drive_fs.py b/unstract/connectors/tests/filesystems/test_google_drive_fs.pyindex bfeccbac8..26b1ac59a 100644--- a/unstract/connectors/tests/filesystems/test_google_drive_fs.py+++ b/unstract/connectors/tests/filesystems/test_google_drive_fs.py@@ -1,8 +1,6 @@import unittest-from unstract.connectors.filesystems.google_drive.google_drive import (- GoogleDriveFS,-)+from unstract.connectors.filesystems.google_drive.google_drive import GoogleDriveFSclass TestGoogleDriveFS(unittest.TestCase):diff --git a/unstract/connectors/tests/filesystems/test_miniofs.py b/unstract/connectors/tests/filesystems/test_miniofs.pyindex 1ab09fcc5..0443bbde1 100644--- a/unstract/connectors/tests/filesystems/test_miniofs.py+++ b/unstract/connectors/tests/filesystems/test_miniofs.py@@ -31,9 +31,7 @@ class TestMinoFS(unittest.TestCase):access_key = os.environ.get("MINIO_ACCESS_KEY_ID")secret_key = os.environ.get("MINIO_SECRET_ACCESS_KEY")print(access_key, secret_key)- bucket_name = os.environ.get(- "FREE_STORAGE_AWS_BUCKET_NAME", "minio-test"- )+ bucket_name = os.environ.get("FREE_STORAGE_AWS_BUCKET_NAME", "minio-test")s3 = MinioFS({"key": access_key,diff --git a/unstract/core/src/unstract/core/llm_helper/config.py b/unstract/core/src/unstract/core/llm_helper/config.pyindex 2e711300f..1a09cabcd 100644--- a/unstract/core/src/unstract/core/llm_helper/config.py+++ b/unstract/core/src/unstract/core/llm_helper/config.py@@ -33,18 +33,14 @@ class AzureOpenAIConfig:@classmethoddef from_env(cls) -> "AzureOpenAIConfig":kwargs = {- "model": UnstractUtils.get_env(- OpenAIKeys.OPENAI_API_MODEL, raise_err=True- ),+ "model": UnstractUtils.get_env(OpenAIKeys.OPENAI_API_MODEL, raise_err=True),"deployment_name": UnstractUtils.get_env(OpenAIKeys.OPENAI_API_ENGINE, raise_err=True),"engine": UnstractUtils.get_env(OpenAIKeys.OPENAI_API_ENGINE, raise_err=True),- "api_key": UnstractUtils.get_env(- OpenAIKeys.OPENAI_API_KEY, raise_err=True- ),+ "api_key": UnstractUtils.get_env(OpenAIKeys.OPENAI_API_KEY, raise_err=True),"api_version": UnstractUtils.get_env(OpenAIKeys.OPENAI_API_VERSION,default=OpenAIDefaults.OPENAI_API_VERSION,diff --git a/unstract/core/src/unstract/core/llm_helper/llm_cache.py b/unstract/core/src/unstract/core/llm_helper/llm_cache.pyindex 580c51e5e..5af5b67d1 100644--- a/unstract/core/src/unstract/core/llm_helper/llm_cache.py+++ b/unstract/core/src/unstract/core/llm_helper/llm_cache.py@@ -12,9 +12,7 @@ class LLMCache:redis_host = os.environ.get("REDIS_HOST")redis_port = os.environ.get("REDIS_PORT")if redis_host is None or redis_port is None:- raise RuntimeError(- "REDIS_HOST or REDIS_PORT environment variable not set"- )+ raise RuntimeError("REDIS_HOST or REDIS_PORT environment variable not set")redis_password = os.environ.get("REDIS_PASSWORD", None)if redis_password and (redis_password == "" or redis_password.lower() == "none"@@ -107,10 +105,7 @@ class LLMCache:"""logger.info(f"Clearing cache with prefix: {self.cache_key_prefix}")keys_to_delete = [- key- for key in self.llm_cache.scan_iter(- match=self.cache_key_prefix + "*"- )+ key for key in self.llm_cache.scan_iter(match=self.cache_key_prefix + "*")]if keys_to_delete:return self.delete(*keys_to_delete)diff --git a/unstract/core/src/unstract/core/llm_helper/llm_helper.py b/unstract/core/src/unstract/core/llm_helper/llm_helper.pyindex 0912a783c..12d947cc6 100644--- a/unstract/core/src/unstract/core/llm_helper/llm_helper.py+++ b/unstract/core/src/unstract/core/llm_helper/llm_helper.py@@ -47,9 +47,7 @@ class LLMHelper:prompt_for_model = self.promptif self.prompt_context == PromptContext.GENERATE_CRON_STRING:- prompt_for_model = prompt_for_model.replace(- "{$user_prompt}", user_prompt- )+ prompt_for_model = prompt_for_model.replace("{$user_prompt}", user_prompt)return prompt_for_model@@ -71,9 +69,7 @@ class LLMHelper:if ai_service == "azure-open-ai":logger.info("Using Azure OpenAI")if use_cache:- response = self.llm_cache.get_for_prompt(- prompt=prompt_for_model- )+ response = self.llm_cache.get_for_prompt(prompt=prompt_for_model)if response:return LLMResponse(result=LLMResult.OK, output=response, cost_type="cache"@@ -108,12 +104,8 @@ class LLMHelper:logger.info(f"OpenAI Response: {resp}")time_taken = end_time - start_time- self.llm_cache.set_for_prompt(- prompt=prompt_for_model, response=resp- )- return LLMResponse(- output=resp, cost_type=ai_service, time_taken=time_taken- )+ self.llm_cache.set_for_prompt(prompt=prompt_for_model, response=resp)+ return LLMResponse(output=resp, cost_type=ai_service, time_taken=time_taken)else:logger.error(f"AI service '{ai_service}' not found")return LLMResponse(diff --git a/unstract/core/src/unstract/core/llm_workflow_generator/llm_interface.py b/unstract/core/src/unstract/core/llm_workflow_generator/llm_interface.pyindex 0a5874318..fb66a8780 100644--- a/unstract/core/src/unstract/core/llm_workflow_generator/llm_interface.py+++ b/unstract/core/src/unstract/core/llm_workflow_generator/llm_interface.py@@ -70,9 +70,7 @@ class LLMInterface:port=int(redis_port),password=redis_password,)- redis_key = (- f"cache:{workflow_id}:workflow_prompt:{hash_hex}"- )+ redis_key = f"cache:{workflow_id}:workflow_prompt:{hash_hex}"workflow_bin = r.get(redis_key)if workflow_bin is not None:logging.info("Cache hit")diff --git a/unstract/core/src/unstract/core/utilities.py b/unstract/core/src/unstract/core/utilities.pyindex 8f45dfed5..0364d28d9 100644--- a/unstract/core/src/unstract/core/utilities.py+++ b/unstract/core/src/unstract/core/utilities.py@@ -4,9 +4,7 @@ from typing import Optionalclass UnstractUtils:@staticmethod- def get_env(- env_key: str, default: Optional[str] = None, raise_err=False- ) -> str:+ def get_env(env_key: str, default: Optional[str] = None, raise_err=False) -> str:"""Returns the value of an env variable.If its empty or None, raises an errordiff --git a/unstract/core/tests/llm_helper/test_llm_cache.py b/unstract/core/tests/llm_helper/test_llm_cache.pyindex d94f74401..3f6389242 100644--- a/unstract/core/tests/llm_helper/test_llm_cache.py+++ b/unstract/core/tests/llm_helper/test_llm_cache.py@@ -18,9 +18,7 @@ class LLMCacheTests(unittest.TestCase):cache.set_for_prompt("prompt1", "response1")cache.set_for_prompt("prompt2", "response2")cache.clear_by_prefix()- self.assertEqual(- cache.get_for_prompt("prompt1"), "", "Cache is not cleared"- )+ self.assertEqual(cache.get_for_prompt("prompt1"), "", "Cache is not cleared")if __name__ == "__main__":diff --git a/unstract/core/tests/test_pubsub_helper.py b/unstract/core/tests/test_pubsub_helper.pyindex 33a70cd53..ae3e10269 100644--- a/unstract/core/tests/test_pubsub_helper.py+++ b/unstract/core/tests/test_pubsub_helper.py@@ -11,9 +11,7 @@ class PubSubHelperTestCase(unittest.TestCase):)ps2 = Log.publish(project_guid="test",- message=Log.log(- level="ERROR", stage="COMPILE", message="Compile failed"- ),+ message=Log.log(level="ERROR", stage="COMPILE", message="Compile failed"),)self.assertEqual(ps1, True)self.assertEqual(ps2, True)diff --git a/unstract/flags/src/unstract/flags/evaluation_pb2.py b/unstract/flags/src/unstract/flags/evaluation_pb2.pyindex e5883270c..bc6c6c575 100644--- a/unstract/flags/src/unstract/flags/evaluation_pb2.py+++ b/unstract/flags/src/unstract/flags/evaluation_pb2.py@@ -12,9 +12,7 @@ from google.protobuf.internal import enum_type_wrapper_sym_db = _symbol_database.Default()-from google.protobuf import (- timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2,-)+from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2DESCRIPTOR = _descriptor.FileDescriptor(name="protos/evaluation.proto",diff --git a/unstract/tool-registry/src/unstract/tool_registry/dto.py b/unstract/tool-registry/src/unstract/tool_registry/dto.pyindex 084ba0091..116d02e22 100644--- a/unstract/tool-registry/src/unstract/tool_registry/dto.py+++ b/unstract/tool-registry/src/unstract/tool_registry/dto.py@@ -67,9 +67,7 @@ class Spec:default_factory=dict[str, dict[str, Any]])- def get_adapter_properties_keys(- self, adapter_type: AdapterTypes- ) -> set[str]:+ def get_adapter_properties_keys(self, adapter_type: AdapterTypes) -> set[str]:properties = set()if self.properties is not None:properties = {@@ -255,9 +253,7 @@ class Adapter:text_extractors: list[AdapterProperties] = field(default_factory=list[AdapterProperties])- ocrs: list[AdapterProperties] = field(- default_factory=list[AdapterProperties]- )+ ocrs: list[AdapterProperties] = field(default_factory=list[AdapterProperties])@classmethoddef from_dict(cls, data_dict: dict[str, Any]) -> "Adapter":diff --git a/unstract/tool-registry/src/unstract/tool_registry/helper.py b/unstract/tool-registry/src/unstract/tool_registry/helper.pyindex 1dff798c7..a6fcf64a9 100644--- a/unstract/tool-registry/src/unstract/tool_registry/helper.py+++ b/unstract/tool-registry/src/unstract/tool_registry/helper.py@@ -104,9 +104,7 @@ class ToolRegistryHelper:image_tag=tool_meta.tag,)- tool_properties: Optional[dict[str, Any]] = (- tool_sandbox.get_properties()- )+ tool_properties: Optional[dict[str, Any]] = tool_sandbox.get_properties()if not tool_properties:return {}return tool_properties@@ -165,9 +163,7 @@ class ToolRegistryHelper:Returns:ToolData: _description_"""- tool_metadata = ToolUtils.get_tool_meta_from_tool_url(- registry_tool=image_url- )+ tool_metadata = ToolUtils.get_tool_meta_from_tool_url(registry_tool=image_url)spec = self.get_tool_spec(tool_meta=tool_metadata)properties = self.get_tool_properties(tool_meta=tool_metadata)icon = self.get_tool_icon(tool_meta=tool_metadata)@@ -202,9 +198,7 @@ class ToolRegistryHelper:dict[str, Any]: _description_"""try:- yml_data: dict[str, Any] = ToolUtils.get_registry(- self.registry_file- )+ yml_data: dict[str, Any] = ToolUtils.get_registry(self.registry_file)return yml_dataexcept FileNotFoundError:logger.error(f"File not found: {self.registry_file}")@@ -325,20 +319,14 @@ class ToolRegistryHelper:tool_data: dict[str, Any] = tools.pop(tool_uid, {})return tool_data- def add_new_tool_to_disk_by_uid(- self, uuid: str, data: dict[str, Any]- ) -> None:+ def add_new_tool_to_disk_by_uid(self, uuid: str, data: dict[str, Any]) -> None:tools = self.get_all_tools_from_disk()tools[uuid] = data- ToolUtils.save_tools_in_to_disk(- file_path=self.private_tools_file, data=tools- )+ ToolUtils.save_tools_in_to_disk(file_path=self.private_tools_file, data=tools)def add_new_tool_to_disk_by_image_url(self, image_url: str) -> None:tool_data = self.get_tool_data_by_image_url(image_url=image_url)- self.add_new_tool_to_disk_by_uid(- uuid=tool_data.uid, data=tool_data.data- )+ self.add_new_tool_to_disk_by_uid(uuid=tool_data.uid, data=tool_data.data)def remove_tools_from_disk(self, tools: dict[str, Any], tool_ids: list[str]diff --git a/unstract/tool-registry/src/unstract/tool_registry/schema_validator.py b/unstract/tool-registry/src/unstract/tool_registry/schema_validator.pyindex ef37a8911..9ededb2f7 100644--- a/unstract/tool-registry/src/unstract/tool_registry/schema_validator.py+++ b/unstract/tool-registry/src/unstract/tool_registry/schema_validator.py@@ -27,9 +27,7 @@ class JsonSchemaValidator:logger.error(f"Validation error: {e}")raise InvalidSchemaInput- def validate_and_filter(- self, data: dict[str, Any]- ) -> Optional[dict[str, Any]]:+ def validate_and_filter(self, data: dict[str, Any]) -> Optional[dict[str, Any]]:"""Validates the input data against the schema and filters the databased on the schema's properties.diff --git a/unstract/tool-registry/src/unstract/tool_registry/tool_registry.py b/unstract/tool-registry/src/unstract/tool_registry/tool_registry.pyindex 73d54e3c5..ce2cdd8e3 100644--- a/unstract/tool-registry/src/unstract/tool_registry/tool_registry.py+++ b/unstract/tool-registry/src/unstract/tool_registry/tool_registry.py@@ -81,9 +81,7 @@ class ToolRegistry:self.helper.remove_tool_from_registry(image_url=image_url)logger.info(f"Tool '{tool_id}' removed from registry successfully.")self.helper.save_tools(data=tools)- logger.info(- f"Tool '{tool_id}' removed from tools file successfully."- )+ logger.info(f"Tool '{tool_id}' removed from tools file successfully.")except Exception as e:logger.error(f"An error occurred: {str(e)}")@@ -154,14 +152,10 @@ class ToolRegistry:tools_list = []if load_from_source:for tool in self.helper.get_all_registry_tools():- tool_meta = ToolUtils.get_tool_meta_from_tool_url(- registry_tool=tool- )+ tool_meta = ToolUtils.get_tool_meta_from_tool_url(registry_tool=tool)if not tool_meta:continue- properties = self.helper.get_tool_properties(- tool_meta=tool_meta- )+ properties = self.helper.get_tool_properties(tool_meta=tool_meta)icon = self.helper.get_tool_icon(tool_meta=tool_meta)if not properties:continue@@ -169,9 +163,7 @@ class ToolRegistry:ToolKey.NAME: properties.get(PropKey.DISPLAY_NAME),ToolKey.DESCRIPTION: properties.get(PropKey.DESCRIPTION),ToolKey.ICON: icon,- ToolKey.FUNCTION_NAME: properties.get(- PropKey.FUNCTION_NAME- ),+ ToolKey.FUNCTION_NAME: properties.get(PropKey.FUNCTION_NAME),}tools_list.append(tool_data)else:@@ -206,13 +198,11 @@ class ToolRegistry:tools_list = {}if load_from_source:for tool in self.helper.get_all_registry_tools():- tool_meta = ToolUtils.get_tool_meta_from_tool_url(- registry_tool=tool- )+ tool_meta = ToolUtils.get_tool_meta_from_tool_url(registry_tool=tool)if not tool_meta:continue- properties: Optional[dict[str, Any]] = (- self.helper.get_tool_properties(tool_meta=tool_meta)+ properties: Optional[dict[str, Any]] = self.helper.get_tool_properties(+ tool_meta=tool_meta)spec: Optional[dict[str, Any]] = self.helper.get_tool_spec(tool_meta=tool_meta@@ -235,9 +225,7 @@ class ToolRegistry:"icon": icon,}else:- tools: dict[str, dict[str, Any]] = (- self.helper.get_all_tools_from_disk()- )+ tools: dict[str, dict[str, Any]] = self.helper.get_all_tools_from_disk()for tool, configuration in tools.items():properties = configuration.get("properties")spec = configuration.get("spec")@@ -287,9 +275,7 @@ class ToolRegistry:dict[str, Any]: _description_"""tools = self.helper.get_all_tools_from_disk()- properties: dict[str, Any] = tools.get(tool_id, {}).get(- "properties", {}- )+ properties: dict[str, Any] = tools.get(tool_id, {}).get("properties", {})return propertiesdef get_tool_icon_by_tool_id(self, tool_id: str) -> dict[str, Any]:diff --git a/unstract/tool-registry/src/unstract/tool_registry/tool_utils.py b/unstract/tool-registry/src/unstract/tool_registry/tool_utils.pyindex 3dcb33f04..af9ca040f 100644--- a/unstract/tool-registry/src/unstract/tool_registry/tool_utils.py+++ b/unstract/tool-registry/src/unstract/tool_registry/tool_utils.py@@ -18,17 +18,13 @@ class ToolUtils:@staticmethoddef is_valid_tool_url(image_url: str) -> bool:# Define a regular expression pattern for Docker image URLs- docker_pattern = (- r"^docker:[a-zA-Z0-9_.-]+(/[a-zA-Z0-9_.-]+)*/[a-zA-Z0-9_.-]+$"- )+ docker_pattern = r"^docker:[a-zA-Z0-9_.-]+(/[a-zA-Z0-9_.-]+)*/[a-zA-Z0-9_.-]+$"# Define a regular expression pattern for local tool URLslocal_pattern = r"^local:[a-zA-Z0-9_.-:]+$"# Check if the URL matches either the Docker image or local tool pattern- if re.match(docker_pattern, image_url) or re.match(- local_pattern, image_url- ):+ if re.match(docker_pattern, image_url) or re.match(local_pattern, image_url):return Trueelse:return False@@ -172,9 +168,7 @@ class ToolUtils:list[AdapterProperties]:A list of enabled AdapterProperties objects."""- enabled_adapters = [- adapter for adapter in adapters if adapter.is_enabled- ]+ enabled_adapters = [adapter for adapter in adapters if adapter.is_enabled]return enabled_adapters@staticmethod@@ -240,14 +234,10 @@ class ToolUtils:schema = tool.specadapter = tool.properties.adapter- language_models = ToolUtils.get_enabled_adapters(- adapter.language_models- )+ language_models = ToolUtils.get_enabled_adapters(adapter.language_models)embeddings = ToolUtils.get_enabled_adapters(adapter.embedding_services)vector_stores = ToolUtils.get_enabled_adapters(adapter.vector_stores)- text_extractors = ToolUtils.get_enabled_adapters(- adapter.text_extractors- )+ text_extractors = ToolUtils.get_enabled_adapters(adapter.text_extractors)ocrs = ToolUtils.get_enabled_adapters(adapter.ocrs)ToolUtils.process_adapter_models(diff --git a/unstract/tool-registry/tests/test_tool_registry.py b/unstract/tool-registry/tests/test_tool_registry.pyindex 4a09a459a..7fff83b06 100644--- a/unstract/tool-registry/tests/test_tool_registry.py+++ b/unstract/tool-registry/tests/test_tool_registry.py@@ -25,13 +25,9 @@ class TestToolRegistry(unittest.TestCase):"tools": [self.TEST_IMAGE_URL],}directory = os.path.dirname(os.path.abspath(__file__))- registry_file_path = os.path.join(- directory, TestToolRegistry.REGISTRY_FILE- )+ registry_file_path = os.path.join(directory, TestToolRegistry.REGISTRY_FILE)with open(registry_file_path, "w") as yaml_file:- yaml.dump(- test_registry_content, yaml_file, default_flow_style=False- )+ yaml.dump(test_registry_content, yaml_file, default_flow_style=False)# Apply the patch to 'run_tool_and_get_logs' before each test methodself.mock_run_tool_and_get_logs = patch.object(@@ -91,9 +87,7 @@ class TestToolRegistry(unittest.TestCase):def test_get_tool_properties_by_tool_id(self) -> None:tool_id = "document_indexer"- properties = self.registry.get_tool_properties_by_tool_id(- tool_id=tool_id- )+ properties = self.registry.get_tool_properties_by_tool_id(tool_id=tool_id)self.assertIsNotNone(properties)self.assertEqual(properties.get("function_name"), tool_id)diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.pyindex 9e745949a..5967f1306 100644--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py@@ -20,18 +20,14 @@ class ToolSandboxHelper:) -> None:worker_host = os.environ.get("UNSTRACT_WORKER_HOST")worker_port = os.environ.get("UNSTRACT_WORKER_PORT")- self.base_url = (- f"{worker_host}:{worker_port}{UnstractWorker.BASE_API_ENDPOINT}"- )+ self.base_url = f"{worker_host}:{worker_port}{UnstractWorker.BASE_API_ENDPOINT}"self.organization_id = str(organization_id)self.workflow_id = str(workflow_id)self.execution_id = str(execution_id)self.envs = environment_variablesself.messaging_channel = str(messaging_channel)- def convert_str_to_dict(- self, data: Union[str, dict[str, Any]]- ) -> dict[str, Any]:+ def convert_str_to_dict(self, data: Union[str, dict[str, Any]]) -> dict[str, Any]:if isinstance(data, str):output: dict[str, Any] = {}try:@@ -67,8 +63,7 @@ class ToolSandboxHelper:)else:logger.error(- f"Error while calling tool {image_name} "- f" reason: {response.reason}"+ f"Error while calling tool {image_name} " f" reason: {response.reason}")return result@@ -107,8 +102,7 @@ class ToolSandboxHelper:)else:logger.error(- f"Error while calling tool {image_name} "- f" reason: {response.reason}"+ f"Error while calling tool {image_name} " f" reason: {response.reason}")return resultdiff --git a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.pyindex e60a42ebf..48dc77a70 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py@@ -26,9 +26,7 @@ class ExecutionFileHandler:self.execution_dir = self.create_execution_dir_path(workflow_id, execution_id, organization_id)- self.source_file = os.path.join(- self.execution_dir, WorkflowFileType.SOURCE- )+ self.source_file = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)self.infile = os.path.join(self.execution_dir, WorkflowFileType.INFILE)self.metadata_file = os.path.join(self.execution_dir, WorkflowFileType.METADATA_JSON@@ -80,17 +78,13 @@ class ExecutionFileHandler:)return output_type- def get_last_tool_metadata(- self, metadata: dict[str, Any]- ) -> dict[str, Any]:+ def get_last_tool_metadata(self, metadata: dict[str, Any]) -> dict[str, Any]:tool_metadata = self.get_list_of_tool_metadata(metadata)if not tool_metadata:raise ToolMetadataNotFound()return tool_metadata[-1]- def add_metadata_to_volume(- self, input_file_path: str, source_hash: str- ) -> None:+ def add_metadata_to_volume(self, input_file_path: str, source_hash: str) -> None:"""Creating metadata for workflow. This method is responsible forcreating metadata for the workflow. It takes the input file path andthe source hash as parameters. The metadata is stored in a JSON file in@@ -120,8 +114,7 @@ class ExecutionFileHandler:with fsspec.open(f"file://{destination_path}", "w") as local_file:json.dump(content, local_file)logger.info(- f"metadata for {input_file_path} is "- "added in to execution directory"+ f"metadata for {input_file_path} is " "added in to execution directory")@classmethoddiff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.pyindex b0f8b6f65..c2f440179 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py@@ -48,18 +48,10 @@ class ToolsUtils:self.doc_processor_api_key = ToolsUtils.get_env(ToolRV.DOCUMENT_PROCESSOR_API_KEY, raise_exception=True)- self.prompt_host = ToolsUtils.get_env(- ToolRV.PROMPT_HOST, raise_exception=True- )- self.prompt_port = ToolsUtils.get_env(- ToolRV.PROMPT_PORT, raise_exception=True- )- self.x2text_host = ToolsUtils.get_env(- ToolRV.X2TEXT_HOST, raise_exception=True- )- self.x2text_port = ToolsUtils.get_env(- ToolRV.X2TEXT_PORT, raise_exception=True- )+ self.prompt_host = ToolsUtils.get_env(ToolRV.PROMPT_HOST, raise_exception=True)+ self.prompt_port = ToolsUtils.get_env(ToolRV.PROMPT_PORT, raise_exception=True)+ self.x2text_host = ToolsUtils.get_env(ToolRV.X2TEXT_HOST, raise_exception=True)+ self.x2text_port = ToolsUtils.get_env(ToolRV.X2TEXT_PORT, raise_exception=True)def set_messaging_channel(self, messaging_channel: str) -> None:self.messaging_channel = messaging_channel@@ -79,8 +71,8 @@ class ToolsUtils:dict[str, dict[str, Any]]: tools"""tool_uids = [tool_instance.tool_id for tool_instance in tool_instances]- tools: dict[str, dict[str, Any]] = (- self.tool_registry.get_available_tools(tool_uids)+ tools: dict[str, dict[str, Any]] = self.tool_registry.get_available_tools(+ tool_uids)if not (all(tool_uid in tools for tool_uid in tool_uids)diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.pyindex 2b3f1daa9..584f70243 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py@@ -20,9 +20,7 @@ from unstract.workflow_execution.exceptions import (StopExecution,ToolOutputNotFoundException,)-from unstract.workflow_execution.execution_file_handler import (- ExecutionFileHandler,-)+from unstract.workflow_execution.execution_file_handler import ExecutionFileHandlerfrom unstract.workflow_execution.tools_utils import ToolsUtilsfrom unstract.core.pubsub_helper import LogPublisher@@ -242,8 +240,7 @@ class WorkflowExecutionService:"""with self.redis_con as red:logger.info(- f"Setting single stepping flag to "- f"{ExecutionAction.START.value}"+ f"Setting single stepping flag to " f"{ExecutionAction.START.value}")red.setex(self.execution_id,@@ -272,8 +269,7 @@ class WorkflowExecutionService:execution_action = ExecutionAction(execution_value)if execution_action == ExecutionAction.NEXT:log_message = (- f"Execution '{self.execution_id}' Executing "- "NEXT step"+ f"Execution '{self.execution_id}' Executing " "NEXT step")self.publish_log(log_message)break@@ -336,17 +332,12 @@ class WorkflowExecutionService:Args:execution_type (ExecutionType): ExecutionType"""- if (- execution_type == ExecutionType.STEP- and not self.override_single_step- ):+ if execution_type == ExecutionType.STEP and not self.override_single_step:self._handling_step_execution()def validate_execution_result(self, step: int) -> bool:workflow_metadata = self.file_handler.get_workflow_metadata()- metadata_list = self.file_handler.get_list_of_tool_metadata(- workflow_metadata- )+ metadata_list = self.file_handler.get_list_of_tool_metadata(workflow_metadata)if len(metadata_list) == step:return Truereturn False@@ -406,7 +397,5 @@ class WorkflowExecutionService:if isinstance(component, LogComponent):component = component.value- log_details = LogPublisher.log_workflow_update(- state.value, message, component- )+ log_details = LogPublisher.log_workflow_update(state.value, message, component)LogPublisher.publish(self.messaging_channel, log_details)diff --git a/unstract/workflow-execution/tests/workflow_test.py b/unstract/workflow-execution/tests/workflow_test.pyindex a10922a62..906d64853 100644--- a/unstract/workflow-execution/tests/workflow_test.py+++ b/unstract/workflow-execution/tests/workflow_test.py@@ -18,16 +18,10 @@ def get_mock_tool_instances() -> list[ToolInstance]:input=item["input"],output=item["output"],metadata=item["metadata"],- input_file_connector=ConnectorInstance(- **item["input_file_connector"]- ),- output_file_connector=ConnectorInstance(- **item["output_file_connector"]- ),+ input_file_connector=ConnectorInstance(**item["input_file_connector"]),+ output_file_connector=ConnectorInstance(**item["output_file_connector"]),input_db_connector=ConnectorInstance(**item["input_db_connector"]),- output_db_connector=ConnectorInstance(- **item["output_db_connector"]- ),+ output_db_connector=ConnectorInstance(**item["output_db_connector"]),tool_settings=ToolSettings(**item["tool_settings"]),)for item in tool_instance_data@@ -91,8 +85,8 @@ class TestWorkflow(unittest.TestCase):mock_tool_instances = get_mock_tool_instances()mock_tool_utils = Mock()- mock_tool_utils.validate_tool_instance_with_tools.side_effect = (- Exception("Test error message")+ mock_tool_utils.validate_tool_instance_with_tools.side_effect = Exception(+ "Test error message")workflow = Workflow(diff --git a/worker/src/unstract/worker/worker.py b/worker/src/unstract/worker/worker.pyindex acd8a1d05..6454dd9b0 100644--- a/worker/src/unstract/worker/worker.py+++ b/worker/src/unstract/worker/worker.py@@ -76,14 +76,10 @@ class UnstractWorker:try:# Attempt to get the image informationself.client.images.get(image_name_with_tag)- logger.info(- f"Image '{image_name_with_tag}' found in the local system."- )+ logger.info(f"Image '{image_name_with_tag}' found in the local system.")return Trueexcept docker.errors.ImageNotFound: # type: ignore[attr-defined]- logger.info(- f"Image '{image_name_with_tag}' not found in the local system."- )+ logger.info(f"Image '{image_name_with_tag}' not found in the local system.")return Falseexcept docker.errors.APIError as e: # type: ignore[attr-defined]logger.error(f"An API error occurred: {e}")@@ -119,8 +115,7 @@ class UnstractWorker:return log_dictexcept json.JSONDecodeError as e:logger.warn(- f"Received invalid JSON log message: {log_message} \n "- f"error: {e}"+ f"Received invalid JSON log message: {log_message} \n " f"error: {e}")return None@@ -136,8 +131,7 @@ class UnstractWorker:log_type = self.get_log_type(log_dict)if not self.is_valid_log_type(log_type):logger.warning(- f"Received invalid logType: {log_type} with log message: "- f"{log_dict}"+ f"Received invalid logType: {log_type} with log message: " f"{log_dict}")return Noneif log_type == LogType.RESULT:@@ -301,17 +295,13 @@ class UnstractWorker:self.envs = envsself.messaging_channel = messaging_channelself.container_name = (- self.normalize_container_name(self.image_name)- + "-"- + self.workflow_id+ self.normalize_container_name(self.image_name) + "-" + self.workflow_id)container_config = self.get_container_run_config()# Add labels to container for logging with Loki.# This only required for observability.try:- labels = ast.literal_eval(- os.environ.get(Env.TOOL_CONTAINER_LABELS, "")- )+ labels = ast.literal_eval(os.environ.get(Env.TOOL_CONTAINER_LABELS, ""))container_config["labels"] = labelsexcept Exception as e:logger.info(f"Invalid labels for logging: {e}")diff --git a/x2text-service/app/authentication_middleware.py b/x2text-service/app/authentication_middleware.pyindex 5889fc115..0f03ce0d1 100644--- a/x2text-service/app/authentication_middleware.py+++ b/x2text-service/app/authentication_middleware.py@@ -8,9 +8,7 @@ def authentication_middleware(func: Any) -> Any:def wrapper(*args: Any, **kwargs: Any) -> Any:token = AuthenticationMiddleware.get_token_from_auth_header(request)# Check if bearer token exists and validate it- if not token or not AuthenticationMiddleware.validate_bearer_token(- token- ):+ if not token or not AuthenticationMiddleware.validate_bearer_token(token):return "Unauthorized", 401return func(*args, **kwargs)@@ -24,9 +22,7 @@ class AuthenticationMiddleware:def validate_bearer_token(cls, token: Optional[str]) -> bool:try:if token is None:- current_app.logger.error(- "Authentication failed. Empty bearer token"- )+ current_app.logger.error("Authentication failed. Empty bearer token")return Falsequery = f"SELECT * FROM account_platformkey WHERE key = '{token}'"@@ -53,9 +49,7 @@ class AuthenticationMiddleware:return Falseexcept Exception as e:- current_app.logger.error(- f"Error while validating bearer token: {e}"- )+ current_app.logger.error(f"Error while validating bearer token: {e}")return Falsereturn True@@ -74,13 +68,11 @@ class AuthenticationMiddleware:@classmethoddef get_account_from_bearer_token(cls, token: Optional[str]) -> str:query = (- "SELECT organization_id FROM account_platformkey "- f"WHERE key='{token}'"+ "SELECT organization_id FROM account_platformkey " f"WHERE key='{token}'")organization = AuthenticationMiddleware.execute_query(query)query_org = (- "SELECT schema_name FROM account_organization "- f"WHERE id='{organization}'"+ "SELECT schema_name FROM account_organization " f"WHERE id='{organization}'")schema_name: str = AuthenticationMiddleware.execute_query(query_org)return schema_namediff --git a/x2text-service/app/controllers/controller.py b/x2text-service/app/controllers/controller.pyindex 82f6a2dc8..16dab861e 100644--- a/x2text-service/app/controllers/controller.py+++ b/x2text-service/app/controllers/controller.py@@ -32,13 +32,9 @@ def health() -> str:@basic.route("/test-connection", methods=["POST"])@authentication_middlewaredef test_connection() -> Any:- logging.info(- "Received a test connection request from %s", request.remote_addr- )+ logging.info("Received a test connection request from %s", request.remote_addr)form_data = dict(request.form)- unstructured_api_key = X2TextUtil.get_value_for_key(- UNSTRUCTURED_API_KEY, form_data- )+ unstructured_api_key = X2TextUtil.get_value_for_key(UNSTRUCTURED_API_KEY, form_data)url = X2TextUtil.get_value_for_key(UNSTRUCTURED_URL, form_data)if not url:return {"message": "Missing or empty url in form data"}, 400@@ -79,9 +75,7 @@ def test_connection() -> Any:@basic.route("/process", methods=["POST"])@authentication_middlewaredef process() -> Any:- logging.info(- "Received a doc processing request from %s", request.remote_addr- )+ logging.info("Received a doc processing request from %s", request.remote_addr)form_data = dict(request.form)url = X2TextUtil.get_value_for_key(UNSTRUCTURED_URL, form_data)if not url:@@ -97,9 +91,7 @@ def process() -> Any:file_size_in_kb = int(request.headers["Content-Length"]) / 1024bearer_token = AuthenticationMiddleware.get_token_from_auth_header(request)- org_id = AuthenticationMiddleware.get_account_from_bearer_token(- bearer_token- )+ org_id = AuthenticationMiddleware.get_account_from_bearer_token(bearer_token)x2_text_audit: X2TextAudit = X2TextAudit.create(org_id=org_id,@@ -116,9 +108,7 @@ def process() -> Any:)}- unstructured_api_key = X2TextUtil.get_value_for_key(- UNSTRUCTURED_API_KEY, form_data- )+ unstructured_api_key = X2TextUtil.get_value_for_key(UNSTRUCTURED_API_KEY, form_data)headers = {"accept": "application/json","unstructured-api-key": unstructured_api_key,@@ -139,13 +129,9 @@ def process() -> Any:file_stream = BytesIO(response_text.encode("utf-8"))x2_text_audit.status = "Success"x2_text_audit.save()- return send_file(- file_stream, download_name="infile.txt", as_attachment=True- )+ return send_file(file_stream, download_name="infile.txt", as_attachment=True)x2_text_audit.status = "Failed"x2_text_audit.save()return_val = X2TextUtil.read_response(response=response)- logging.error(- "Text extraction failed: [%s] %s", response.status_code, return_val- )+ logging.error("Text extraction failed: [%s] %s", response.status_code, return_val)return return_val, response.status_code