LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Fix/adapter permissions (#128)
Rahul Johny0bf3e86dbaa3
diff --git a/backend/account/custom_auth_middleware.py b/backend/account/custom_auth_middleware.pyindex e26f46ad2..13533b60d 100644--- a/backend/account/custom_auth_middleware.py+++ b/backend/account/custom_auth_middleware.py@@ -9,8 +9,10 @@ from django.http import HttpRequest, HttpResponse, JsonResponsefrom tenant_account.organization_member_service import OrganizationMemberServicefrom utils.cache_service import CacheServicefrom utils.local_context import StateStore+from backend.constants import RequestHeader+class CustomAuthMiddleware:def __init__(self, get_response: HttpResponse):self.get_response = get_response@@ -42,16 +44,11 @@ class CustomAuthMiddleware:self.local_authentication(request)elif request.COOKIES:self.authenticate_with_cookies(request, tenantAccessiblePublicPath)- if (- request.user- and request.session- and "user" in request.session- ):+ if request.user and request.session and "user" in request.session:StateStore.set(Common.LOG_EVENTS_ID, request.session.session_key)- StateStore.set(Common.USER_ID, request.user)response = self.get_response(request)StateStore.clear(Common.LOG_EVENTS_ID)- StateStore.clear(Common.USER_ID)+return responsereturn JsonResponse({"message": "Unauthorized"}, status=401)diff --git a/backend/adapter_processor/adapter_processor.py b/backend/adapter_processor/adapter_processor.pyindex 159952a14..eebce0778 100644--- a/backend/adapter_processor/adapter_processor.py+++ b/backend/adapter_processor/adapter_processor.py@@ -159,7 +159,7 @@ class AdapterProcessor:if default_triad.get(AdapterKeys.X2TEXT, None):user_default_adapter.default_x2text_adapter = (AdapterInstance.objects.get(- pk=default_triad[AdapterKeys.X2TEXT]+ pk=default_triad[AdapterKeys.X2TEXT_DEFAULT]))diff --git a/backend/adapter_processor/constants.py b/backend/adapter_processor/constants.pyindex 2cd8f010b..50aa8f5b5 100644--- a/backend/adapter_processor/constants.py+++ b/backend/adapter_processor/constants.py@@ -19,6 +19,7 @@ class AdapterKeys:VECTOR_DB_DEFAULT = "vector_db_default"EMBEDDING_DEFAULT = "embedding_default"X2TEXT_DEFAULT = "x2text_default"+ SHARED_USERS = "shared_users"ADAPTER_NAME_EXISTS = ("Configuration with this ID already exists. ""Please try with a different ID"diff --git a/backend/adapter_processor/views.py b/backend/adapter_processor/views.pyindex c95aa5a3a..2ee90c41d 100644--- a/backend/adapter_processor/views.py+++ b/backend/adapter_processor/views.py@@ -232,6 +232,38 @@ class AdapterInstanceViewSet(ModelViewSet):super().perform_destroy(adapter_instance)return Response(status=status.HTTP_204_NO_CONTENT)+ def partial_update(+ self, request: Request, *args: tuple[Any], **kwargs: dict[str, Any]+ ) -> Response:+ if AdapterKeys.SHARED_USERS in request.data:+ # find the deleted users+ adapter = self.get_object()+ shared_users = {+ int(user_id) for user_id in request.data.get("shared_users", {})+ }+ current_users = {user.id for user in adapter.shared_users.all()}+ removed_users = current_users.difference(shared_users)++ # if removed user use this adapter as default+ # Remove the same from his default+ for user_id in removed_users:+ user_default_adapter = UserDefaultAdapter.objects.get(+ user_id=user_id+ )++ if user_default_adapter.default_llm_adapter == adapter:+ user_default_adapter.default_llm_adapter = None+ elif user_default_adapter.default_embedding_adapter == adapter:+ user_default_adapter.default_embedding_adapter = None+ elif user_default_adapter.default_vector_db_adapter == adapter:+ user_default_adapter.default_vector_db_adapter = None+ elif user_default_adapter.default_x2text_adapter == adapter:+ user_default_adapter.default_x2text_adapter = None++ user_default_adapter.save()++ return super().partial_update(request, *args, **kwargs)+@action(detail=True, methods=["get"])def list_of_shared_users(self, request: HttpRequest, pk: Any = Nonediff --git a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.pyindex 0405e7cc5..f1ef785e9 100644--- a/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py+++ b/backend/prompt_studio/prompt_studio_core/prompt_studio_helper.py@@ -7,6 +7,7 @@ from typing import Any, Optionalfrom account.constants import Commonfrom adapter_processor.models import AdapterInstancefrom django.conf import settings+from django.core.exceptions import PermissionDeniedfrom file_management.file_management_helper import FileManagerHelperfrom prompt_studio.prompt_profile_manager.models import ProfileManagerfrom prompt_studio.prompt_studio.models import ToolStudioPrompt@@ -39,6 +40,7 @@ from utils.local_context import StateStorefrom unstract.core.pubsub_helper import LogPublisherCHOICES_JSON = "/static/select_choices.json"+ERROR_MSG = "User %s doesn't have access to adapter %s"logger = logging.getLogger(__name__)@@ -47,6 +49,81 @@ class PromptStudioHelper:"""Helper class for Custom tool operations."""@staticmethod+ def validate_profile_manager_owner_access(+ profile_manager: ProfileManager,+ ) -> None:+ """Helper method to validate the owner's access to the profile manager.++ Args:+ profile_manager (ProfileManager): The profile manager instance to+ validate.++ Raises:+ PermissionDenied: If the owner does not have permission to perform+ the action.+ """+ profile_manager_owner = profile_manager.created_by++ is_llm_owned = (+ profile_manager.llm.created_by == profile_manager_owner+ or profile_manager.llm.shared_users.filter(+ pk=profile_manager_owner.pk+ ).exists()+ )+ is_vector_store_owned = (+ profile_manager.vector_store.created_by == profile_manager_owner+ or profile_manager.vector_store.shared_users.filter(+ pk=profile_manager_owner.pk+ ).exists()+ )+ is_embedding_model_owned = (+ profile_manager.embedding_model.created_by == profile_manager_owner+ or profile_manager.embedding_model.shared_users.filter(+ pk=profile_manager_owner.pk+ ).exists()+ )+ is_x2text_owned = (+ profile_manager.x2text.created_by == profile_manager_owner+ or profile_manager.x2text.shared_users.filter(+ pk=profile_manager_owner.pk+ ).exists()+ )++ if not (+ is_llm_owned+ and is_vector_store_owned+ and is_embedding_model_owned+ and is_x2text_owned+ ):+ if not is_llm_owned:+ logger.error(+ ERROR_MSG,+ profile_manager_owner.user_id,+ profile_manager.llm.id,+ )+ if not is_vector_store_owned:+ logger.error(+ ERROR_MSG,+ profile_manager_owner.user_id,+ profile_manager.vector_store.id,+ )+ if not is_embedding_model_owned:+ logger.error(+ ERROR_MSG,+ profile_manager_owner.user_id,+ profile_manager.embedding_model.id,+ )+ if not is_x2text_owned:+ logger.error(+ ERROR_MSG,+ profile_manager_owner.user_id,+ profile_manager.x2text.id,+ )++ raise PermissionDenied(+ "You don't have permission to perform this action."+ )+def _publish_log(component: dict[str, str], level: str, state: str, message: str) -> None:@@ -149,6 +226,11 @@ class PromptStudioHelper:LogLevels.RUN,"Indexing started",)+ # Need to check the user who created profile manager+ # has access to adapters configured in profile manager+ PromptStudioHelper.validate_profile_manager_owner_access(+ default_profile+ )doc_id = PromptStudioHelper.dynamic_indexer(profile_manager=default_profile,@@ -247,6 +329,8 @@ class PromptStudioHelper:org_id=org_id,document_id=document_id,)+ except PermissionDenied as e:+ raise eexcept Exception as exc:logger.error(f"[{tool.tool_id}] Error while fetching response for prompt {id}: {exc}" # noqa: E501@@ -293,6 +377,8 @@ class PromptStudioHelper:org_id=org_id,document_id=document_id,)+ except PermissionDenied as e:+ raise eexcept Exception as e:logger.error(f"[{tool.tool_id}] Error while fetching single pass response: {e}" # noqa: E501@@ -363,6 +449,11 @@ class PromptStudioHelper:grammar_list.append(grammer_dict)grammer_dict = {}for prompt in prompts:+ # Need to check the user who created profile manager+ # has access to adapters+ PromptStudioHelper.validate_profile_manager_owner_access(+ prompt.profile_manager+ )# Not checking reindex here as there might be# change in Profile Managervector_db = str(prompt.profile_manager.vector_store.id)@@ -536,6 +627,11 @@ class PromptStudioHelper:default_profile: ProfileManager = ProfileManager.objects.get(prompt_studio_tool=tool, is_default=True)+ # Need to check the user who created profile manager+ # has access to adapters configured in profile manager+ PromptStudioHelper.validate_profile_manager_owner_access(+ default_profile+ )default_profile.chunk_size = 0 # To retrive full contextif prompt_grammar:diff --git a/backend/tool_instance/tool_instance_helper.py b/backend/tool_instance/tool_instance_helper.pyindex 27e29829a..07b59b03c 100644--- a/backend/tool_instance/tool_instance_helper.py+++ b/backend/tool_instance/tool_instance_helper.py@@ -4,11 +4,11 @@ import uuidfrom json import JSONDecodeErrorfrom typing import Any, Optional-from account.constants import Commonfrom account.models import Userfrom adapter_processor.adapter_processor import AdapterProcessorfrom adapter_processor.models import AdapterInstancefrom connector.connector_instance_helper import ConnectorInstanceHelper+from django.core.exceptions import PermissionDeniedfrom jsonschema.exceptions import UnknownType, ValidationErrorfrom tool_instance.constants import JsonSchemaKeyfrom tool_instance.models import ToolInstance@@ -18,7 +18,6 @@ from unstract.sdk.tool.validator import DefaultsGeneratingValidatorfrom unstract.tool_registry.constants import AdapterPropertyKeyfrom unstract.tool_registry.dto import Spec, Toolfrom unstract.tool_registry.tool_utils import ToolUtils-from utils.local_context import StateStorefrom workflow_manager.workflow.constants import WorkflowKeylogger = logging.getLogger(__name__)@@ -336,22 +335,19 @@ class ToolInstanceHelper:@staticmethoddef validate_tool_settings(- tool_uid: str, tool_meta: dict[str, Any]+ user: User, tool_uid: str, tool_meta: dict[str, Any]) -> tuple[bool, str]:"""Function to validate Tools settings."""+ ToolInstanceHelper.validate_adapter_permissions(+ user=user, tool_uid=tool_uid, tool_meta=tool_meta+ )+tool: Tool = ToolProcessor.get_tool_by_uid(tool_uid=tool_uid)tool_name: str = (tool.properties.display_nameif tool.properties.display_nameelse tool_uid)- # Getting user from local_context store- user: Optional[User] = StateStore.get(Common.USER_ID)- # FIXME: Skipping tool setting validation if user is not set.- # This occurs during API deployment since the API endpoint is- # whitelisted and `user` is not set in StateStore by the middleware.- if not user:- return True, ""schema_json: dict[str, Any] = ToolProcessor.get_json_schema_for_tool(tool_uid=tool_uid, user=user)@@ -361,6 +357,78 @@ class ToolInstanceHelper:except JSONDecodeError as e:return False, str(e)except ValidationError as e:+ logger.error(e)return False, str(tool_name + ": " + e.schema["description"])except UnknownType as e:return False, str(e)++ @staticmethod+ def validate_adapter_permissions(+ user: User, tool_uid: str, tool_meta: dict[str, Any]+ ) -> None:+ tool: Tool = ToolProcessor.get_tool_by_uid(tool_uid=tool_uid)+ adapter_ids = set()++ for llm in tool.properties.adapter.language_models:+ if llm.is_enabled and llm.adapter_id:+ adapter_id = tool_meta[llm.adapter_id]+ elif llm.is_enabled:+ adapter_id = tool_meta[+ AdapterPropertyKey.DEFAULT_LLM_ADAPTER_ID+ ]++ adapter_ids.add(adapter_id)+ for vdb in tool.properties.adapter.vector_stores:+ if vdb.is_enabled and vdb.adapter_id:+ adapter_id = tool_meta[vdb.adapter_id]+ elif vdb.is_enabled:+ adapter_id = tool_meta[+ AdapterPropertyKey.DEFAULT_VECTOR_DB_ADAPTER_ID+ ]++ adapter_ids.add(adapter_id)+ for embedding in tool.properties.adapter.embedding_services:+ if embedding.is_enabled and embedding.adapter_id:+ adapter_id = tool_meta[embedding.adapter_id]+ elif embedding.is_enabled:+ adapter_id = tool_meta[+ AdapterPropertyKey.DEFAULT_EMBEDDING_ADAPTER_ID+ ]++ adapter_ids.add(adapter_id)+ for text_extractor in tool.properties.adapter.text_extractors:+ if text_extractor.is_enabled and text_extractor.adapter_id:+ adapter_id = tool_meta[text_extractor.adapter_id]+ elif text_extractor.is_enabled:+ adapter_id = tool_meta[+ AdapterPropertyKey.DEFAULT_X2TEXT_ADAPTER_ID+ ]++ adapter_ids.add(adapter_id)++ ToolInstanceHelper.validate_adapter_access(+ user=user, adapter_ids=adapter_ids+ )++ @staticmethod+ def validate_adapter_access(+ user: User,+ adapter_ids: str,+ ) -> None:+ adapter_instances = AdapterInstance.objects.filter(+ id__in=adapter_ids+ ).all()++ for adapter_instance in adapter_instances:+ if not (+ adapter_instance.created_by == user+ or adapter_instance.shared_users.filter(pk=user.pk).exists()+ ):+ logger.error(+ "User %s doesn't have access to adapter %s",+ user.user_id,+ adapter_instance.id,+ )+ raise PermissionDenied(+ "You don't have permission to perform this action."+ )diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.pyindex 7e57d1dd8..267722d8d 100644--- a/backend/workflow_manager/workflow/workflow_helper.py+++ b/backend/workflow_manager/workflow/workflow_helper.py@@ -197,7 +197,9 @@ class WorkflowHelper:) -> None:for tool in tool_instances:valid, message = ToolInstanceHelper.validate_tool_settings(- tool_uid=tool.tool_id, tool_meta=tool.metadata+ user=tool.workflow.created_by,+ tool_uid=tool.tool_id,+ tool_meta=tool.metadata,)if not valid:raise ToolValidationError(message)