LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Feat/queue connector (#371)
vishnuszipstackefc74b35c539
diff --git a/backend/account/authentication_controller.py b/backend/account/authentication_controller.pyindex 72dcce79e..f8b41e921 100644--- a/backend/account/authentication_controller.py+++ b/backend/account/authentication_controller.py@@ -184,7 +184,9 @@ class AuthenticationController:f"{ErrorMessage.ORGANIZATION_EXIST}, \{ErrorMessage.DUPLICATE_API}")- self.create_tenant_user(organization=organization, user=user)+ organization_member = self.create_tenant_user(+ organization=organization, user=user+ )if new_organization:try:@@ -222,6 +224,7 @@ class AuthenticationController:organization_id=current_organization_id,)UserSessionUtils.set_organization_id(request, organization_id)+ UserSessionUtils.set_organization_member_role(request, organization_member)OrganizationMemberService.set_user_membership_in_organization_cache(user_id=user.user_id, organization_id=organization_id)@@ -420,12 +423,14 @@ class AuthenticationController:organization_user.role = roleorganization_user.save()- def create_tenant_user(self, organization: Organization, user: User) -> None:+ def create_tenant_user(+ self, organization: Organization, user: User+ ) -> OrganizationMember:with tenant_context(organization):existing_tenant_user = OrganizationMemberService.get_user_by_id(id=user.id)if existing_tenant_user:Logger.info(f"{existing_tenant_user.user.email} Already exist")-+ return existing_tenant_userelse:account_user = self.get_or_create_user(user=user)if account_user:@@ -442,7 +447,7 @@ class AuthenticationController:is_prompt_studio_onboarding_msg=False,)tenant_user.save()-+ return tenant_userelse:raise UserNotExistError()diff --git a/backend/account/dto.py b/backend/account/dto.pyindex 1554901fb..66cf6c1aa 100644--- a/backend/account/dto.py+++ b/backend/account/dto.py@@ -59,6 +59,7 @@ class UserSessionInfo:email: strorganization_id: struser: UserInfo+ role: str@staticmethoddef from_dict(data: dict[str, Any]) -> "UserSessionInfo":@@ -67,6 +68,7 @@ class UserSessionInfo:user_id=data["user_id"],email=data["email"],organization_id=data["organization_id"],+ role=data["role"],)def to_dict(self) -> Any:@@ -75,6 +77,7 @@ class UserSessionInfo:"user_id": self.user_id,"email": self.email,"organization_id": self.organization_id,+ "role": self.role,}diff --git a/backend/account/serializer.py b/backend/account/serializer.pyindex 57d166fe7..c40c0b993 100644--- a/backend/account/serializer.py+++ b/backend/account/serializer.py@@ -116,3 +116,4 @@ class UserSessionResponseSerializer(serializers.Serializer):user_id = serializers.CharField()email = serializers.CharField()organization_id = serializers.CharField()+ role = serializers.CharField()diff --git a/backend/account/views.py b/backend/account/views.pyindex 0f7eb8719..0e6582821 100644--- a/backend/account/views.py+++ b/backend/account/views.py@@ -142,6 +142,7 @@ def make_session_response(email=request.user.email,user=auth_controller.get_user_info(request),organization_id=UserSessionUtils.get_organization_id(request),+ role=UserSessionUtils.get_organization_member_role(request),)).datadiff --git a/backend/backend/urls.py b/backend/backend/urls.pyindex e0e306b1b..fea9c6077 100644--- a/backend/backend/urls.py+++ b/backend/backend/urls.py@@ -93,3 +93,12 @@ try:]except ImportError:pass++try:+ import pluggable_apps.manual_review.urls # noqa: F401++ urlpatterns += [+ path("manual_review/", include("pluggable_apps.manual_review.urls")),+ ]+except ImportError:+ passdiff --git a/backend/connector/migrations/0004_alter_connectorinstance_connector_mode.py b/backend/connector/migrations/0004_alter_connectorinstance_connector_mode.pynew file mode 100644index 000000000..6feec997f--- /dev/null+++ b/backend/connector/migrations/0004_alter_connectorinstance_connector_mode.py@@ -0,0 +1,28 @@+# Generated by Django 4.2.1 on 2024-07-04 05:44++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("connector", "0003_alter_connectorinstance_connector_mode"),+ ]++ operations = [+ migrations.AlterField(+ model_name="connectorinstance",+ name="connector_mode",+ field=models.CharField(+ choices=[+ (0, "UNKNOWN"),+ (1, "FILE_SYSTEM"),+ (2, "DATABASE"),+ (3, "APPDEPLOYMENT"),+ (4, "MANUAL_REVIEW"),+ ],+ db_comment="Choices of connectors",+ default=0,+ ),+ ),+ ]diff --git a/backend/connector/models.py b/backend/connector/models.pyindex 19b845341..55c228e47 100644--- a/backend/connector/models.py+++ b/backend/connector/models.py@@ -30,6 +30,7 @@ class ConnectorInstance(BaseModel):FILE_SYSTEM = 1, "FILE_SYSTEM"DATABASE = 2, "DATABASE"APPDEPLOYMENT = 3, "APPDEPLOYMENT"+ MANUAL_REVIEW = 4, "MANUAL_REVIEW"id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)connector_name = models.TextField(diff --git a/backend/tenant_account/users_view.py b/backend/tenant_account/users_view.pyindex 9e8ffd21a..ebcebd14a 100644--- a/backend/tenant_account/users_view.py+++ b/backend/tenant_account/users_view.py@@ -31,8 +31,6 @@ class OrganizationUserViewSet(viewsets.ViewSet):if not (user_email and role):raise BadRequestExceptionorg_id: str = UserSessionUtils.get_organization_id(request)- auth_controller = AuthenticationController()-auth_controller = AuthenticationController()update_status = auth_controller.add_user_role(request.user, org_id, user_email, rolediff --git a/backend/utils/user_session.py b/backend/utils/user_session.pyindex d8758b25e..888c6c353 100644--- a/backend/utils/user_session.py+++ b/backend/utils/user_session.py@@ -1,6 +1,7 @@from typing import Optionalfrom django.http import HttpRequest+from tenant_account.models import OrganizationMemberfrom utils.constants import FeatureFlagfrom unstract.flags.feature_flag import check_feature_flag_status@@ -23,3 +24,13 @@ class UserSessionUtils:@staticmethoddef get_user_id(request: HttpRequest) -> Optional[str]:return request.session.get("user_id")++ @staticmethod+ def set_organization_member_role(+ request: HttpRequest, member: OrganizationMember+ ) -> None:+ request.session["role"] = member.role++ @staticmethod+ def get_organization_member_role(request: HttpRequest) -> Optional[str]:+ return request.session.get("role")diff --git a/backend/workflow_manager/endpoint/constants.py b/backend/workflow_manager/endpoint/constants.pyindex 1674c5725..d9553245d 100644--- a/backend/workflow_manager/endpoint/constants.py+++ b/backend/workflow_manager/endpoint/constants.py@@ -92,3 +92,14 @@ class BigQuery:"""TABLE_NAME_SIZE = 3+ COLUMN_TYPES = [+ "DATE",+ "DATETIME",+ "TIME",+ "TIMESTAMP",+ ]+++class QueueResultStatus:+ SUCCESS = "Success"+ FAILED = "Failed"diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.pyindex 464652d7f..6b88dd86c 100644--- a/backend/workflow_manager/endpoint/destination.py+++ b/backend/workflow_manager/endpoint/destination.py@@ -1,4 +1,5 @@import ast+import base64import jsonimport loggingimport os@@ -15,6 +16,7 @@ from workflow_manager.endpoint.base_connector import BaseConnectorfrom workflow_manager.endpoint.constants import (ApiDeploymentResultStatus,DestinationKey,+ QueueResultStatus,WorkflowFileType,)from workflow_manager.endpoint.database_utils import DatabaseUtils@@ -26,6 +28,7 @@ from workflow_manager.endpoint.exceptions import (ToolOutputTypeMismatch,)from workflow_manager.endpoint.models import WorkflowEndpoint+from workflow_manager.endpoint.queue_utils import QueueResult, QueueUtilsfrom workflow_manager.workflow.enums import ExecutionStatusfrom workflow_manager.workflow.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow.models.file_history import FileHistory@@ -55,8 +58,10 @@ class DestinationConnector(BaseConnector):organization_id = connection.tenant.schema_namesuper().__init__(workflow.id, execution_id, organization_id)self.endpoint = self._get_endpoint_for_workflow(workflow=workflow)+ self.source_endpoint = self._get_source_endpoint_for_workflow(workflow=workflow)self.execution_id = execution_idself.api_results: list[dict[str, Any]] = []+ self.queue_results: list[dict[str, Any]] = []def _get_endpoint_for_workflow(self,@@ -81,6 +86,29 @@ class DestinationConnector(BaseConnector):)return endpoint+ def _get_source_endpoint_for_workflow(+ self,+ workflow: Workflow,+ ) -> WorkflowEndpoint:+ """Get WorkflowEndpoint instance.++ Args:+ workflow (Workflow): Workflow associated with the+ destination connector.++ Returns:+ WorkflowEndpoint: WorkflowEndpoint instance.+ """+ endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(+ workflow=workflow,+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,+ )+ if endpoint.connector_instance:+ endpoint.connector_instance.connector_metadata = (+ endpoint.connector_instance.metadata+ )+ return endpoint+def validate(self) -> None:connection_type = self.endpoint.connection_typeconnector: ConnectorInstance = self.endpoint.connector_instance@@ -88,7 +116,11 @@ class DestinationConnector(BaseConnector):raise MissingDestinationConnectionType()if connection_type not in WorkflowEndpoint.ConnectionType.values:raise InvalidDestinationConnectionType()- if connection_type != WorkflowEndpoint.ConnectionType.API and connector is None:+ if (+ connection_type != WorkflowEndpoint.ConnectionType.API+ and connection_type != WorkflowEndpoint.ConnectionType.MANUALREVIEW+ and connector is None+ ):raise DestinationConnectorNotConfigured()def handle_output(@@ -98,6 +130,7 @@ class DestinationConnector(BaseConnector):workflow: Workflow,file_history: Optional[FileHistory] = None,error: Optional[str] = None,+ input_file_path: Optional[str] = None,) -> None:"""Handle the output based on the connection type."""connection_type = self.endpoint.connection_type@@ -114,10 +147,19 @@ class DestinationConnector(BaseConnector):elif connection_type == WorkflowEndpoint.ConnectionType.API:result = self.get_result(file_history)meta_data = self.get_metadata(file_history)-self._handle_api_result(file_name=file_name, error=error, result=result, meta_data=meta_data)+ elif connection_type == WorkflowEndpoint.ConnectionType.MANUALREVIEW:+ result = self.get_result(file_history)+ meta_data = self.get_metadata(file_history)+ self._push_to_queue(+ file_name=file_name,+ workflow=workflow,+ result=result,+ input_file_path=input_file_path,+ meta_data=meta_data,+ )if not file_history:FileHistoryHelper.create_file_history(cache_key=file_hash,@@ -433,3 +475,58 @@ class DestinationConnector(BaseConnector):os.path.dirname(__file__), "static", "dest", "api.json")return cls.get_json_schema(file_path=schema_path)++ def _push_to_queue(+ self,+ file_name: str,+ workflow: Workflow,+ result: Optional[str] = None,+ input_file_path: Optional[str] = None,+ meta_data: Optional[dict[str, Any]] = None,+ ) -> None:+ """Handle the Manual Review QUEUE result.++ This method is responsible for pushing the input file and result to+ review queue.+ Args:+ file_name (str): The name of the file.+ workflow (Workflow): The workflow object containing+ details about the workflow.+ result (Optional[str], optional): The result of the API call.+ Defaults to None.+ input_file_path (Optional[str], optional):+ The path to the input file.+ Defaults to None.+ meta_data (Optional[dict[str, Any]], optional):+ A dictionary containing additional+ metadata related to the file. Defaults to None.++ Returns:+ None+ """+ if not result:+ return+ connector: ConnectorInstance = self.source_endpoint.connector_instance+ connector_settings: dict[str, Any] = connector.connector_metadata++ source_fs = self.get_fsspec(+ settings=connector_settings, connector_id=connector.connector_id+ )+ with source_fs.open(input_file_path, "rb") as remote_file:+ file_content = remote_file.read()+ # Convert file content to a base64 encoded string+ file_content_base64 = base64.b64encode(file_content).decode("utf-8")+ q_name = f"review_queue_{self.organization_id}_{workflow.workflow_name}"+ queue_result = QueueResult(+ file=file_name,+ whisper_hash=meta_data["whisper-hash"],+ status=QueueResultStatus.SUCCESS,+ result=result,+ workflow_id=str(self.workflow_id),+ file_content=file_content_base64,+ )+ # Convert the result dictionary to a JSON string+ queue_result_json = json.dumps(queue_result)+ conn = QueueUtils.get_queue_inst()+ # Enqueue the JSON string+ conn.enqueue(queue_name=q_name, message=queue_result_json)diff --git a/backend/workflow_manager/endpoint/exceptions.py b/backend/workflow_manager/endpoint/exceptions.pyindex 69e332cdd..7028eec28 100644--- a/backend/workflow_manager/endpoint/exceptions.py+++ b/backend/workflow_manager/endpoint/exceptions.py@@ -77,3 +77,11 @@ class UnstractDBException(APIException):def __init__(self, detail: str = default_detail) -> None:status_code = 500super().__init__(detail=detail, code=status_code)+++class UnstractQueueException(APIException):+ default_detail = "Error creating/inserting to Queue. "++ def __init__(self, detail: str = default_detail) -> None:+ status_code = 500+ super().__init__(detail=detail, code=status_code)diff --git a/backend/workflow_manager/endpoint/migrations/0003_alter_workflowendpoint_connection_type.py b/backend/workflow_manager/endpoint/migrations/0003_alter_workflowendpoint_connection_type.pynew file mode 100644index 000000000..c2da4a302--- /dev/null+++ b/backend/workflow_manager/endpoint/migrations/0003_alter_workflowendpoint_connection_type.py@@ -0,0 +1,28 @@+# Generated by Django 4.2.1 on 2024-07-04 05:44++from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("endpoint", "0002_alter_workflowendpoint_connection_type"),+ ]++ operations = [+ migrations.AlterField(+ model_name="workflowendpoint",+ name="connection_type",+ field=models.CharField(+ blank=True,+ choices=[+ ("FILESYSTEM", "FileSystem connector"),+ ("DATABASE", "Database Connector"),+ ("API", "API Connector"),+ ("APPDEPLOYMENT", "App Deployment"),+ ("MANUALREVIEW", "Manual Review Queue Connector"),+ ],+ db_comment="Connection type (Filesystem, Database, API or Manualreview)",+ ),+ ),+ ]diff --git a/backend/workflow_manager/endpoint/models.py b/backend/workflow_manager/endpoint/models.pyindex 96f64b58c..f49ca1474 100644--- a/backend/workflow_manager/endpoint/models.py+++ b/backend/workflow_manager/endpoint/models.py@@ -16,6 +16,7 @@ class WorkflowEndpoint(BaseModel):DATABASE = "DATABASE", "Database Connector"API = "API", "API Connector"APPDEPLOYMENT = "APPDEPLOYMENT", "App Deployment"+ MANUALREVIEW = "MANUALREVIEW", "Manual Review Queue Connector"id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)workflow = models.ForeignKey(@@ -33,7 +34,7 @@ class WorkflowEndpoint(BaseModel):connection_type = models.CharField(choices=ConnectionType.choices,blank=True,- db_comment="Connection type (Filesystem, Database or API)",+ db_comment="Connection type (Filesystem, Database, API or Manualreview)",)configuration = models.JSONField(blank=True, null=True, db_comment="Configuration in JSON format"diff --git a/backend/workflow_manager/endpoint/queue_utils.py b/backend/workflow_manager/endpoint/queue_utils.pynew file mode 100644index 000000000..bbb4dddb3--- /dev/null+++ b/backend/workflow_manager/endpoint/queue_utils.py@@ -0,0 +1,41 @@+import logging+from dataclasses import dataclass+from enum import Enum+from typing import Any++from utils.constants import Common+from workflow_manager.endpoint.exceptions import UnstractQueueException++from unstract.connectors.queues import connectors as queue_connectors+from unstract.connectors.queues.unstract_queue import UnstractQueue++logger = logging.getLogger(__name__)+++class QueueResultStatus(Enum):+ SUCCESS = "success"+ FAILURE = "failure"+ # Add other statuses as needed+++class QueueUtils:+ @staticmethod+ def get_queue_inst(connector_settings: dict[str, Any] = {}) -> UnstractQueue:+ if not queue_connectors:+ raise UnstractQueueException(detail="Queue connector not exists")+ queue_connector_key = next(iter(queue_connectors))+ connector = queue_connectors[queue_connector_key][Common.METADATA][+ Common.CONNECTOR+ ]+ connector_class: UnstractQueue = connector(connector_settings)+ return connector_class+++@dataclass+class QueueResult:+ file: str+ whisper_hash: str+ status: QueueResultStatus+ result: Any+ workflow_id: str+ file_content: strdiff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.pyindex b264845aa..9f4e434cc 100644--- a/backend/workflow_manager/workflow/workflow_helper.py+++ b/backend/workflow_manager/workflow/workflow_helper.py@@ -205,6 +205,7 @@ class WorkflowHelper:workflow=workflow,file_history=file_history,error=error,+ input_file_path=input_file,)execution_service.publish_update_log(LogState.SUCCESS,diff --git a/frontend/public/icons/connector-icons/Redis.png b/frontend/public/icons/connector-icons/Redis.pngnew file mode 100644index 000000000..b8deff6ccBinary files /dev/null and b/frontend/public/icons/connector-icons/Redis.png differdiff --git a/frontend/src/components/agency/actions/Actions.jsx b/frontend/src/components/agency/actions/Actions.jsxindex fe82adaca..faa973bf0 100644--- a/frontend/src/components/agency/actions/Actions.jsx+++ b/frontend/src/components/agency/actions/Actions.jsx@@ -38,7 +38,7 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {const [openAddApiModal, setOpenAddApiModal] = useState(false);const [apiOpsPresent, setApiOpsPresent] = useState(false);const [canAddTaskPipeline, setCanAddTaskPipeline] = useState(false);- const [canAddETLPipeline, setCanAddETAPipeline] = useState(false);+ const [canAddETLPipeline, setCanAddETLPipeline] = useState(false);const [openAddTaskModal, setOpenAddTaskModal] = useState(false);const [openAddETLModal, setOpenAddETLModal] = useState(false);const [deploymentName, setDeploymentName] = useState("");@@ -76,8 +76,9 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {);// Enable Deploy as ETL Pipeline only when// destination connection_type is DATABASE and Source & Destination are Configured- setCanAddETAPipeline(- destination?.connection_type === "DATABASE" &&+ setCanAddETLPipeline(+ (destination?.connection_type === "DATABASE" ||+ destination.connection_type === "MANUALREVIEW") &&source?.connector_instance &&destination.connector_instance);@@ -302,6 +303,12 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {) {return false;}+ if (+ source?.connection_type === "FILESYSTEM" &&+ destination?.connection_type === "MANUALREVIEW"+ ) {+ return false;+ }return !source?.connector_instance || !destination?.connector_instance;};diff --git a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsxindex 0f3977546..e781869e4 100644--- a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx+++ b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx@@ -41,7 +41,8 @@ function ConfigureConnectorModal({disabled:!connectorId ||connDetails?.connector_id !== selectedId ||- connType === "DATABASE",+ connType === "DATABASE" ||+ connType === "MANUALREVIEW",},];diff --git a/frontend/src/components/agency/configure-forms-layout/ConfigureFormsLayout.jsx b/frontend/src/components/agency/configure-forms-layout/ConfigureFormsLayout.jsxindex 74a9e916d..9f6a4ed9b 100644--- a/frontend/src/components/agency/configure-forms-layout/ConfigureFormsLayout.jsx+++ b/frontend/src/components/agency/configure-forms-layout/ConfigureFormsLayout.jsx@@ -50,26 +50,28 @@ function ConfigureFormsLayout({</div></div></Col>- <Col span={12} className="conn-modal-form-left">- <div className="conn-modal-flex">- <Typography.Text strong>Configuration</Typography.Text>- <div className="conn-modal-gap" />- <div className="conn-modal-flex-1">- {!specConfig || Object.keys(specConfig)?.length === 0 ? (- <EmptyState text="Failed to load the configuration form" />- ) : (- <SettingsForm- selectedId={selectedId}- handleUpdate={handleUpdate}- spec={specConfig}- formData={formDataConfig}- setFormData={setFormDataConfig}- isLoading={isSpecConfigLoading}- />- )}+ {connType !== "MANUALREVIEW" && (+ <Col span={12} className="conn-modal-form-left">+ <div className="conn-modal-flex">+ <Typography.Text strong>Configuration</Typography.Text>+ <div className="conn-modal-gap" />+ <div className="conn-modal-flex-1">+ {!specConfig || Object.keys(specConfig)?.length === 0 ? (+ <EmptyState text="Failed to load the configuration form" />+ ) : (+ <SettingsForm+ selectedId={selectedId}+ handleUpdate={handleUpdate}+ spec={specConfig}+ formData={formDataConfig}+ setFormData={setFormDataConfig}+ isLoading={isSpecConfigLoading}+ />+ )}+ </div></div>- </div>- </Col>+ </Col>+ )}</Row>);}diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsxindex bf0d78e4d..7647f5a62 100644--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx@@ -63,7 +63,6 @@ function DsSettingsCard({ type, endpointDetails, message }) {const [formDataConfig, setFormDataConfig] = useState({});const [selectedId, setSelectedId] = useState("");const [selectedItemName, setSelectedItemName] = useState("");-const [inputOptions, setInputOptions] = useState([{value: "API",@@ -84,14 +83,32 @@ function DsSettingsCard({ type, endpointDetails, message }) {const { setAlertDetails } = useAlertStore();const axiosPrivate = useAxiosPrivate();const handleException = useExceptionHandler();-const { flags } = sessionDetails;const icons = {input: <ImportOutlined className="ds-set-icon-size" />,output: <ExportOutlined className="ds-set-icon-size" />,};-+ useEffect(() => {+ try {+ const inputOption =+ require("../../../plugins/dscard-input-options/DsSettingsCardInputOptions").inputOption;+ if (flags.manual_review && inputOption) {+ setInputOptions((prevInputOptions) => {+ // Check if inputOption already exists in prevInputOptions+ if (prevInputOptions.some((opt) => opt.value === inputOption.value)) {+ return prevInputOptions; // Return previous state unchanged+ } else {+ // Create a new array with the existing options and the new option+ const updatedInputOptions = [...prevInputOptions, inputOption];+ return updatedInputOptions;+ }+ });+ }+ } catch {+ // The component will remain null of it is not available+ }+ }, []);useEffect(() => {try {const inputOption =@@ -114,7 +131,10 @@ function DsSettingsCard({ type, endpointDetails, message }) {} else {// Filter options based on source connection typeconst filteredOptions = ["API"].includes(source?.connection_type)- ? inputOptions.filter((option) => option.value === "API")+ ? inputOptions.filter(+ (option) =>+ option.value === "API" || option.value === "MANUALREVIEW"+ ): inputOptions.filter((option) => option.value !== "API");setOptions(filteredOptions);@@ -125,7 +145,9 @@ function DsSettingsCard({ type, endpointDetails, message }) {// Remove Database from Source Dropdownconst filteredOptions = inputOptions.filter((option) =>- option.value !== "DATABASE" && option.value !== "APPDEPLOYMENT"+ option.value !== "DATABASE" &&+ option.value !== "APPDEPLOYMENT" &&+ option.value !== "MANUALREVIEW");setOptions(filteredOptions);}@@ -376,6 +398,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {disabled={!endpointDetails?.connection_type ||connType === "API" ||+ connType === "MANUALREVIEW" ||connType === "APPDEPLOYMENT"}>@@ -398,7 +421,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {</Space>) : (<>- {connType === "API" ? (+ {connType === "API" || connType === "MANUALREVIEW" ? (<Typography.TextclassName="font-size-12 display-flex-align-center"ellipsis={{ rows: 1, expandable: false }}@@ -406,7 +429,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {><CheckCircleTwoTone twoToneColor="#52c41a" /><span style={{ marginLeft: "5px" }}>- {titleCase(type)} set to API successfully+ {titleCase(type)} set to {connType} successfully</span></Typography.Text>) : (@@ -463,7 +486,6 @@ DsSettingsCard.propTypes = {type: PropTypes.string.isRequired,endpointDetails: PropTypes.object.isRequired,message: PropTypes.string,- canUpdate: PropTypes.bool.isRequired,};export { DsSettingsCard };diff --git a/frontend/src/components/helpers/auth/RequireAuth.js b/frontend/src/components/helpers/auth/RequireAuth.jsindex 424967e1f..f745aca17 100644--- a/frontend/src/components/helpers/auth/RequireAuth.js+++ b/frontend/src/components/helpers/auth/RequireAuth.js@@ -1,11 +1,11 @@import { Navigate, Outlet, useLocation } from "react-router-dom";+import { useEffect } from "react";import {getOrgNameFromPathname,onboardCompleted,} from "../../../helpers/GetStaticData";import { useSessionStore } from "../../../store/session-store";-import { useEffect } from "react";import usePostHogEvents from "../../../hooks/usePostHogEvents";let ProductFruitsManager;@@ -25,6 +25,7 @@ const RequireAuth = () => {const pathname = location?.pathname;const adapters = sessionDetails?.adapters;const currOrgName = getOrgNameFromPathname(pathname);+ const { flags } = sessionDetails;useEffect(() => {if (!sessionDetails?.isLoggedIn) {@@ -38,6 +39,14 @@ const RequireAuth = () => {if (onboardCompleted(adapters)) {navigateTo = `/${orgName}/tools`;}+ if (flags.manual_review) {+ if (+ sessionDetails.role === "unstract_reviewer" ||+ sessionDetails.role === "unstract_supervisor"+ ) {+ navigateTo = `/${orgName}/review`;+ }+ }if (!isLoggedIn) {return <Navigate to="/landing" state={{ from: location }} replace />;diff --git a/frontend/src/components/helpers/auth/RequireGuest.js b/frontend/src/components/helpers/auth/RequireGuest.jsindex ff5a48ab5..42de4f693 100644--- a/frontend/src/components/helpers/auth/RequireGuest.js+++ b/frontend/src/components/helpers/auth/RequireGuest.js@@ -5,13 +5,21 @@ import { useSessionStore } from "../../../store/session-store";const RequireGuest = () => {const { sessionDetails } = useSessionStore();- const { orgName, adapters } = sessionDetails;+ const { orgName, adapters, flags } = sessionDetails;const location = useLocation();const pathname = location.pathname;let navigateTo = `/${orgName}/onboard`;if (onboardCompleted(adapters)) {navigateTo = `/${orgName}/tools`;}+ if (flags.manual_review) {+ if (+ sessionDetails.role === "unstract_reviewer" ||+ sessionDetails.role === "unstract_supervisor"+ ) {+ navigateTo = `/${orgName}/review`;+ }+ }return !sessionDetails?.isLoggedIn && publicRoutes.includes(pathname) ? (<Outlet />diff --git a/frontend/src/components/navigations/top-nav-bar/TopNavBar.css b/frontend/src/components/navigations/top-nav-bar/TopNavBar.cssindex 511f62628..bb4095d7d 100644--- a/frontend/src/components/navigations/top-nav-bar/TopNavBar.css+++ b/frontend/src/components/navigations/top-nav-bar/TopNavBar.css@@ -58,3 +58,15 @@.switch-org-menu .ant-dropdown-menu-title-content .ant-space {display: block;}++.page-identifier {+ vertical-align: super;+ color: white;+}++.page-heading {+ color: white;+ font-size: 16px;+ font-weight: 600;+ margin-left: 10px;+}diff --git a/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsx b/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsxindex 5d5339baf..cce90112f 100644--- a/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsx+++ b/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsx@@ -8,8 +8,16 @@ import {Space,Typography,} from "antd";+import {+ UserOutlined,+ UserSwitchOutlined,+ LogoutOutlined,+ DownloadOutlined,+ FileProtectOutlined,+ LikeOutlined,+} from "@ant-design/icons";import { useEffect, useState } from "react";-import { useNavigate } from "react-router-dom";+import { useNavigate, useLocation } from "react-router-dom";import axios from "axios";import { UnstractLogo } from "../../../assets/index.js";@@ -36,19 +44,54 @@ try {function TopNavBar() {const navigate = useNavigate();const { sessionDetails } = useSessionStore();- const { orgName, remainingTrialDays, allOrganization, orgId } =+ const { orgName, remainingTrialDays, allOrganization, orgId, flags } =sessionDetails;const baseUrl = getBaseUrl();const onBoardUrl = baseUrl + `/${orgName}/onboard`;const logout = useLogout();const [showOnboardBanner, setShowOnboardBanner] = useState(false);+ const [approverStatus, setApproverStatus] = useState(false);+ const [reviewerStatus, setReviewerStatus] = useState(false);+ const [reviewPageHeader, setReviewPageHeader] = useState("");const { setAlertDetails } = useAlertStore();const handleException = useExceptionHandler();+ const location = useLocation();useEffect(() => {- setShowOnboardBanner(!onboardCompleted(sessionDetails?.adapters));+ const isUnstractReviewer = sessionDetails.role === "unstract_reviewer";+ const isUnstractSupervisor = sessionDetails.role === "unstract_supervisor";+ const isUnstractAdmin = sessionDetails.role === "unstract_admin";++ setShowOnboardBanner(+ !onboardCompleted(sessionDetails?.adapters) &&+ !isUnstractReviewer &&+ !isUnstractSupervisor+ );++ setApproverStatus(+ (isUnstractAdmin || isUnstractSupervisor) && flags.manual_review+ );+ setReviewerStatus(isUnstractReviewer && flags.manual_review);}, [sessionDetails]);+ useEffect(() => {+ if (flags.manual_review) {+ const checkReviewPage = location.pathname.split("review");++ if (checkReviewPage.length > 1) {+ if (checkReviewPage[1].includes("/approve")) {+ setReviewPageHeader("Approve");+ } else if (checkReviewPage[1].includes("/download_and_sync")) {+ setReviewPageHeader("Download and syncmanager");+ } else {+ setReviewPageHeader("Review");+ }+ } else {+ setReviewPageHeader(null);+ }+ }+ }, [location]);+const cascadeOptions = allOrganization.map((org) => {return {key: org?.id,@@ -92,7 +135,7 @@ function TopNavBar() {});};- // Dropdown items+ // Profile Dropdown itemsconst items = [{key: "1",@@ -101,15 +144,7 @@ function TopNavBar() {onClick={() => navigate(`/${orgName}/profile`)}className="logout-button">- Profile- </Button>- ),- },- {- key: "2",- label: (- <Button onClick={logout} className="logout-button">- Logout+ <UserOutlined /> Profile</Button>),},@@ -126,10 +161,54 @@ function TopNavBar() {}}placement="left">- <div>Switch Org</div>+ <div>+ {" "}+ <UserSwitchOutlined /> Switch Org+ </div></Dropdown>),},+ (reviewerStatus || approverStatus) && {+ key: "4",+ label: (+ <Button+ onClick={() => navigate(`/${orgName}/review`)}+ className="logout-button"+ >+ <FileProtectOutlined /> Review+ </Button>+ ),+ },+ approverStatus && {+ key: "5",+ label: (+ <Button+ onClick={() => navigate(`/${orgName}/review/approve`)}+ className="logout-button"+ >+ <LikeOutlined /> Approve+ </Button>+ ),+ },+ approverStatus && {+ key: "6",+ label: (+ <Button+ onClick={() => navigate(`/${orgName}/review/download_and_sync`)}+ className="logout-button"+ >+ <DownloadOutlined /> Download and Sync Manager+ </Button>+ ),+ },+ {+ key: "2",+ label: (+ <Button onClick={logout} className="logout-button">+ <LogoutOutlined /> Logout+ </Button>+ ),+ },];// Function to get the initials from the user name@@ -144,10 +223,16 @@ function TopNavBar() {return (<Row align="middle" className="topNav">- <Col span={4}>+ <Col span={6}><UnstractLogo className="topbar-logo" />+ {reviewPageHeader && (+ <span className="page-identifier">+ <span className="custom-tools-header-v-divider" />+ <span className="page-heading">{reviewPageHeader}</span>+ </span>+ )}</Col>- <Col span={16} className="top-nav-alert-col">+ <Col span={14} className="top-nav-alert-col">{showOnboardBanner && (<Alerttype="error"diff --git a/frontend/src/components/settings/invite/InviteEditUser.css b/frontend/src/components/settings/invite/InviteEditUser.cssindex 27c4c95ea..867049d32 100644--- a/frontend/src/components/settings/invite/InviteEditUser.css+++ b/frontend/src/components/settings/invite/InviteEditUser.css@@ -26,7 +26,8 @@}.role-select{- width:250px+ width:auto;+ min-width:200px;}.form-select{diff --git a/frontend/src/components/settings/invite/InviteEditUser.jsx b/frontend/src/components/settings/invite/InviteEditUser.jsxindex 1afd4f352..e7815c042 100644--- a/frontend/src/components/settings/invite/InviteEditUser.jsx+++ b/frontend/src/components/settings/invite/InviteEditUser.jsx@@ -25,6 +25,8 @@ function InviteEditUser() {const [submitLoading, setSubmitLoading] = useState(false);const [initialRole, setInitialRole] = useState();const { setPostHogCustomEvent } = usePostHogEvents();+ const { flags } = sessionDetails;+const isInvite = location.pathname.split("/").slice(-1)[0] === "invite";const validateMessages = {required: "required!",@@ -185,7 +187,7 @@ function InviteEditUser() {label: formatOptionLabel(role.name),};})}- disabled={isInvite}+ disabled={!flags.manual_review}/></Form.Item></div>diff --git a/frontend/src/helpers/GetSessionData.js b/frontend/src/helpers/GetSessionData.jsindex f5b8c288b..5ee6b9291 100644--- a/frontend/src/helpers/GetSessionData.js+++ b/frontend/src/helpers/GetSessionData.js@@ -26,6 +26,7 @@ function getSessionData(sessionData) {loginOnboardingMessage: sessionData?.loginOnboardingMessage,promptOnboardingMessage: sessionData?.promptOnboardingMessage,flags: sessionData?.flags,+ role: sessionData?.role,};}diff --git a/frontend/src/hooks/useSessionValid.js b/frontend/src/hooks/useSessionValid.jsindex 16fcf754b..fe34e91f2 100644--- a/frontend/src/hooks/useSessionValid.js+++ b/frontend/src/hooks/useSessionValid.js@@ -129,6 +129,7 @@ function useSessionValid() {if (isPlatformAdmin) {userAndOrgDetails["isPlatformAdmin"] = await isPlatformAdmin();}+ userAndOrgDetails["role"] = userSessionData.role;// Set the session detailssetSessionDetails(getSessionData(userAndOrgDetails));} catch (err) {diff --git a/frontend/src/routes/Router.jsx b/frontend/src/routes/Router.jsxindex ab99fde82..47bc7c8ba 100644--- a/frontend/src/routes/Router.jsx+++ b/frontend/src/routes/Router.jsx@@ -35,7 +35,8 @@ let PlatformAdminPage;let AppDeployments;let ChatAppPage;let ChatAppLayout;-+let ManualReviewPage;+let ReviewLayout;try {TrialRoutes =require("../plugins/subscription/trial-page/TrialEndPage.jsx").TrialEndPage;@@ -52,12 +53,18 @@ try {require("../plugins/app-deployment/AppDeployments.jsx").AppDeployments;ChatAppPage =require("../plugins/app-deployment/chat-app/ChatAppPage.jsx").ChatAppPage;- ChatAppLayout =- require("../plugins/app-deployment/chat-app/ChatAppLayout.jsx").ChatAppLayout;} catch (err) {// Do nothing, Not-found Page will be triggered.}+try {+ ManualReviewPage =+ require("../plugins/manual-review/page/ManualReviewPage.jsx").ManualReviewPage;+ ReviewLayout =+ require("../plugins/manual-review/review-layout/ReviewLayout.jsx").ReviewLayout;+} catch (err) {+ // Do nothing, Not-found Page will be triggered.+}// Import pages/components related to Simple Prompt Studio.let SimplePromptStudioHelper;let SimplePromptStudio;@@ -178,6 +185,22 @@ function Router() {</Route>)}</Route>+ {ReviewLayout && ManualReviewPage && (+ <Route path=":orgName" element={<ReviewLayout />}>+ <Route+ path="review"+ element={<ManualReviewPage type="review" />}+ ></Route>+ <Route+ path="review/download_and_sync"+ element={<ManualReviewPage type="download" />}+ />+ <Route+ path="review/approve"+ element={<ManualReviewPage type="approve" />}+ />+ </Route>+ )}</Route>{TrialRoutes && (<Route path="/trial-expired" element={<TrialRoutes />} />diff --git a/unstract/connectors/src/unstract/connectors/connectorkit.py b/unstract/connectors/src/unstract/connectors/connectorkit.pyindex 1ef216491..5b1d574c1 100644--- a/unstract/connectors/src/unstract/connectors/connectorkit.py+++ b/unstract/connectors/src/unstract/connectors/connectorkit.py@@ -7,13 +7,14 @@ from unstract.connectors.constants import Commonfrom unstract.connectors.databases import connectors as db_connectorsfrom unstract.connectors.enums import ConnectorModefrom unstract.connectors.filesystems import connectors as fs_connectors+from unstract.connectors.queues import connectors as q_connectorslogger = logging.getLogger(__name__)class Connectorkit:def __init__(self) -> None:- self._connectors: ConnectorDict = fs_connectors | db_connectors+ self._connectors: ConnectorDict = fs_connectors | db_connectors | q_connectors@propertydef connectors(self) -> ConnectorDict:diff --git a/unstract/connectors/src/unstract/connectors/enums.py b/unstract/connectors/src/unstract/connectors/enums.pyindex f28ec5cb1..e68b4b063 100644--- a/unstract/connectors/src/unstract/connectors/enums.py+++ b/unstract/connectors/src/unstract/connectors/enums.py@@ -6,3 +6,4 @@ class ConnectorMode(Enum):FILE_SYSTEM = "FILE_SYSTEM"DATABASE = "DATABASE"API = "API"+ MANUAL_REVIEW = "MANUAL_REVIEW"diff --git a/unstract/connectors/src/unstract/connectors/queues/__init__.py b/unstract/connectors/src/unstract/connectors/queues/__init__.pynew file mode 100644index 000000000..8a6d8d0bd--- /dev/null+++ b/unstract/connectors/src/unstract/connectors/queues/__init__.py@@ -0,0 +1,5 @@+from unstract.connectors import ConnectorDict+from unstract.connectors.queues.register import register_connectors++connectors: ConnectorDict = {}+register_connectors(connectors)diff --git a/unstract/connectors/src/unstract/connectors/queues/register.py b/unstract/connectors/src/unstract/connectors/queues/register.pynew file mode 100644index 000000000..4eca532c6--- /dev/null+++ b/unstract/connectors/src/unstract/connectors/queues/register.py@@ -0,0 +1,38 @@+import logging+import os+from importlib import import_module+from typing import Any++from unstract.connectors.constants import Common+from unstract.connectors.queues.unstract_queue import UnstractQueue++logger = logging.getLogger(__name__)+++def register_connectors(connectors: dict[str, Any]) -> None:+ current_directory = os.path.dirname(os.path.abspath(__file__))+ package = "unstract.connectors.queues"++ for connector in os.listdir(current_directory):+ connector_path = os.path.join(current_directory, connector)+ # Check if the item is a directory and not a special directory like __pycache__+ if os.path.isdir(connector_path) and not connector.startswith("__"):+ try:+ full_module_path = f"{package}.{connector}"+ module = import_module(full_module_path)+ metadata = getattr(module, "metadata", {})+ if metadata.get("is_active", False):+ connector_class: UnstractQueue = metadata[Common.CONNECTOR]+ connector_id = connector_class.get_id()+ if not connector_id or (connector_id in connectors):+ logger.warning(f"Duplicate Id : {connector_id}")+ else:+ connectors[connector_id] = {+ Common.MODULE: module,+ Common.METADATA: metadata,+ }+ except ModuleNotFoundError as exception:+ logger.error(f"Error while importing connectors : {exception}")++ if len(connectors) == 0:+ logger.warning("No connector found.")diff --git a/unstract/connectors/src/unstract/connectors/queues/unstract_queue.py b/unstract/connectors/src/unstract/connectors/queues/unstract_queue.pynew file mode 100644index 000000000..940bfe041--- /dev/null+++ b/unstract/connectors/src/unstract/connectors/queues/unstract_queue.py@@ -0,0 +1,95 @@+import logging+from abc import ABC, abstractmethod+from typing import Any++from unstract.connectors.base import UnstractConnector+from unstract.connectors.enums import ConnectorMode+++class UnstractQueue(UnstractConnector, ABC):+ """Abstract class for queue connector."""++ logging.basicConfig(+ level=logging.INFO,+ format="%(asctime)s - %(levelname)s - %(filename)s - %(message)s",+ )++ def __init__(self, name: str):+ super().__init__(name)+ self.name = name++ @staticmethod+ def get_id() -> str:+ return ""++ @staticmethod+ def get_name() -> str:+ return ""++ @staticmethod+ def get_description() -> str:+ return ""++ @staticmethod+ def get_icon() -> str:+ return ""++ @staticmethod+ def get_json_schema() -> str:+ return ""++ @staticmethod+ def can_write() -> bool:+ return False++ @staticmethod+ def can_read() -> bool:+ return False++ @staticmethod+ def requires_oauth() -> bool:+ return False++ @staticmethod+ def python_social_auth_backend() -> str:+ return ""++ @staticmethod+ def get_connector_mode() -> ConnectorMode:+ return ConnectorMode.MANUAL_REVIEW++ def test_credentials(self) -> bool:+ """Override to test credentials for a connector."""+ return True++ @abstractmethod+ def get_engine(self) -> Any:+ pass++ @abstractmethod+ def enqueue(self, queue_name: str, message: str) -> Any:+ pass++ @abstractmethod+ def dequeue(self, queue_name: str, timeout: int = 5) -> Any:+ pass++ @abstractmethod+ def peek(self, queue_name: str) -> Any:+ pass++ @abstractmethod+ def lset(self, queue_name: str, index: int, value: str) -> None:+ pass++ @abstractmethod+ def llen(self, queue_name: str) -> int:+ pass++ @abstractmethod+ def lindex(self, queue_name: str, index: int) -> Any:+ pass++ @abstractmethod+ def keys(self, pattern: str = "*") -> list[str]:+ pass