Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Feat/queue connector (#371)

vishnuszipstackefc74b35c539
diff --git a/backend/account/authentication_controller.py b/backend/account/authentication_controller.py
index 72dcce79e..f8b41e921 100644
--- a/backend/account/authentication_controller.py
+++ b/backend/account/authentication_controller.py
@@ -184,7 +184,9 @@ class AuthenticationController:
f"{ErrorMessage.ORGANIZATION_EXIST}, \
{ErrorMessage.DUPLICATE_API}"
)
- self.create_tenant_user(organization=organization, user=user)
+ organization_member = self.create_tenant_user(
+ organization=organization, user=user
+ )
if new_organization:
try:
@@ -222,6 +224,7 @@ class AuthenticationController:
organization_id=current_organization_id,
)
UserSessionUtils.set_organization_id(request, organization_id)
+ UserSessionUtils.set_organization_member_role(request, organization_member)
OrganizationMemberService.set_user_membership_in_organization_cache(
user_id=user.user_id, organization_id=organization_id
)
@@ -420,12 +423,14 @@ class AuthenticationController:
organization_user.role = role
organization_user.save()
- def create_tenant_user(self, organization: Organization, user: User) -> None:
+ def create_tenant_user(
+ self, organization: Organization, user: User
+ ) -> OrganizationMember:
with tenant_context(organization):
existing_tenant_user = OrganizationMemberService.get_user_by_id(id=user.id)
if existing_tenant_user:
Logger.info(f"{existing_tenant_user.user.email} Already exist")
-
+ return existing_tenant_user
else:
account_user = self.get_or_create_user(user=user)
if account_user:
@@ -442,7 +447,7 @@ class AuthenticationController:
is_prompt_studio_onboarding_msg=False,
)
tenant_user.save()
-
+ return tenant_user
else:
raise UserNotExistError()
diff --git a/backend/account/dto.py b/backend/account/dto.py
index 1554901fb..66cf6c1aa 100644
--- a/backend/account/dto.py
+++ b/backend/account/dto.py
@@ -59,6 +59,7 @@ class UserSessionInfo:
email: str
organization_id: str
user: UserInfo
+ role: str
@staticmethod
def from_dict(data: dict[str, Any]) -> "UserSessionInfo":
@@ -67,6 +68,7 @@ class UserSessionInfo:
user_id=data["user_id"],
email=data["email"],
organization_id=data["organization_id"],
+ role=data["role"],
)
def to_dict(self) -> Any:
@@ -75,6 +77,7 @@ class UserSessionInfo:
"user_id": self.user_id,
"email": self.email,
"organization_id": self.organization_id,
+ "role": self.role,
}
diff --git a/backend/account/serializer.py b/backend/account/serializer.py
index 57d166fe7..c40c0b993 100644
--- a/backend/account/serializer.py
+++ b/backend/account/serializer.py
@@ -116,3 +116,4 @@ class UserSessionResponseSerializer(serializers.Serializer):
user_id = serializers.CharField()
email = serializers.CharField()
organization_id = serializers.CharField()
+ role = serializers.CharField()
diff --git a/backend/account/views.py b/backend/account/views.py
index 0f7eb8719..0e6582821 100644
--- a/backend/account/views.py
+++ b/backend/account/views.py
@@ -142,6 +142,7 @@ def make_session_response(
email=request.user.email,
user=auth_controller.get_user_info(request),
organization_id=UserSessionUtils.get_organization_id(request),
+ role=UserSessionUtils.get_organization_member_role(request),
)
).data
diff --git a/backend/backend/urls.py b/backend/backend/urls.py
index e0e306b1b..fea9c6077 100644
--- a/backend/backend/urls.py
+++ b/backend/backend/urls.py
@@ -93,3 +93,12 @@ try:
]
except ImportError:
pass
+
+try:
+ import pluggable_apps.manual_review.urls # noqa: F401
+
+ urlpatterns += [
+ path("manual_review/", include("pluggable_apps.manual_review.urls")),
+ ]
+except ImportError:
+ pass
diff --git a/backend/connector/migrations/0004_alter_connectorinstance_connector_mode.py b/backend/connector/migrations/0004_alter_connectorinstance_connector_mode.py
new file mode 100644
index 000000000..6feec997f
--- /dev/null
+++ b/backend/connector/migrations/0004_alter_connectorinstance_connector_mode.py
@@ -0,0 +1,28 @@
+# Generated by Django 4.2.1 on 2024-07-04 05:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("connector", "0003_alter_connectorinstance_connector_mode"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="connectorinstance",
+ name="connector_mode",
+ field=models.CharField(
+ choices=[
+ (0, "UNKNOWN"),
+ (1, "FILE_SYSTEM"),
+ (2, "DATABASE"),
+ (3, "APPDEPLOYMENT"),
+ (4, "MANUAL_REVIEW"),
+ ],
+ db_comment="Choices of connectors",
+ default=0,
+ ),
+ ),
+ ]
diff --git a/backend/connector/models.py b/backend/connector/models.py
index 19b845341..55c228e47 100644
--- a/backend/connector/models.py
+++ b/backend/connector/models.py
@@ -30,6 +30,7 @@ class ConnectorInstance(BaseModel):
FILE_SYSTEM = 1, "FILE_SYSTEM"
DATABASE = 2, "DATABASE"
APPDEPLOYMENT = 3, "APPDEPLOYMENT"
+ MANUAL_REVIEW = 4, "MANUAL_REVIEW"
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
connector_name = models.TextField(
diff --git a/backend/tenant_account/users_view.py b/backend/tenant_account/users_view.py
index 9e8ffd21a..ebcebd14a 100644
--- a/backend/tenant_account/users_view.py
+++ b/backend/tenant_account/users_view.py
@@ -31,8 +31,6 @@ class OrganizationUserViewSet(viewsets.ViewSet):
if not (user_email and role):
raise BadRequestException
org_id: str = UserSessionUtils.get_organization_id(request)
- auth_controller = AuthenticationController()
-
auth_controller = AuthenticationController()
update_status = auth_controller.add_user_role(
request.user, org_id, user_email, role
diff --git a/backend/utils/user_session.py b/backend/utils/user_session.py
index d8758b25e..888c6c353 100644
--- a/backend/utils/user_session.py
+++ b/backend/utils/user_session.py
@@ -1,6 +1,7 @@
from typing import Optional
from django.http import HttpRequest
+from tenant_account.models import OrganizationMember
from utils.constants import FeatureFlag
from unstract.flags.feature_flag import check_feature_flag_status
@@ -23,3 +24,13 @@ class UserSessionUtils:
@staticmethod
def get_user_id(request: HttpRequest) -> Optional[str]:
return request.session.get("user_id")
+
+ @staticmethod
+ def set_organization_member_role(
+ request: HttpRequest, member: OrganizationMember
+ ) -> None:
+ request.session["role"] = member.role
+
+ @staticmethod
+ def get_organization_member_role(request: HttpRequest) -> Optional[str]:
+ return request.session.get("role")
diff --git a/backend/workflow_manager/endpoint/constants.py b/backend/workflow_manager/endpoint/constants.py
index 1674c5725..d9553245d 100644
--- a/backend/workflow_manager/endpoint/constants.py
+++ b/backend/workflow_manager/endpoint/constants.py
@@ -92,3 +92,14 @@ class BigQuery:
"""
TABLE_NAME_SIZE = 3
+ COLUMN_TYPES = [
+ "DATE",
+ "DATETIME",
+ "TIME",
+ "TIMESTAMP",
+ ]
+
+
+class QueueResultStatus:
+ SUCCESS = "Success"
+ FAILED = "Failed"
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.py
index 464652d7f..6b88dd86c 100644
--- a/backend/workflow_manager/endpoint/destination.py
+++ b/backend/workflow_manager/endpoint/destination.py
@@ -1,4 +1,5 @@
import ast
+import base64
import json
import logging
import os
@@ -15,6 +16,7 @@ from workflow_manager.endpoint.base_connector import BaseConnector
from workflow_manager.endpoint.constants import (
ApiDeploymentResultStatus,
DestinationKey,
+ QueueResultStatus,
WorkflowFileType,
)
from workflow_manager.endpoint.database_utils import DatabaseUtils
@@ -26,6 +28,7 @@ from workflow_manager.endpoint.exceptions import (
ToolOutputTypeMismatch,
)
from workflow_manager.endpoint.models import WorkflowEndpoint
+from workflow_manager.endpoint.queue_utils import QueueResult, QueueUtils
from workflow_manager.workflow.enums import ExecutionStatus
from workflow_manager.workflow.file_history_helper import FileHistoryHelper
from workflow_manager.workflow.models.file_history import FileHistory
@@ -55,8 +58,10 @@ class DestinationConnector(BaseConnector):
organization_id = connection.tenant.schema_name
super().__init__(workflow.id, execution_id, organization_id)
self.endpoint = self._get_endpoint_for_workflow(workflow=workflow)
+ self.source_endpoint = self._get_source_endpoint_for_workflow(workflow=workflow)
self.execution_id = execution_id
self.api_results: list[dict[str, Any]] = []
+ self.queue_results: list[dict[str, Any]] = []
def _get_endpoint_for_workflow(
self,
@@ -81,6 +86,29 @@ class DestinationConnector(BaseConnector):
)
return endpoint
+ def _get_source_endpoint_for_workflow(
+ self,
+ workflow: Workflow,
+ ) -> WorkflowEndpoint:
+ """Get WorkflowEndpoint instance.
+
+ Args:
+ workflow (Workflow): Workflow associated with the
+ destination connector.
+
+ Returns:
+ WorkflowEndpoint: WorkflowEndpoint instance.
+ """
+ endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(
+ workflow=workflow,
+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
+ )
+ if endpoint.connector_instance:
+ endpoint.connector_instance.connector_metadata = (
+ endpoint.connector_instance.metadata
+ )
+ return endpoint
+
def validate(self) -> None:
connection_type = self.endpoint.connection_type
connector: ConnectorInstance = self.endpoint.connector_instance
@@ -88,7 +116,11 @@ class DestinationConnector(BaseConnector):
raise MissingDestinationConnectionType()
if connection_type not in WorkflowEndpoint.ConnectionType.values:
raise InvalidDestinationConnectionType()
- if connection_type != WorkflowEndpoint.ConnectionType.API and connector is None:
+ if (
+ connection_type != WorkflowEndpoint.ConnectionType.API
+ and connection_type != WorkflowEndpoint.ConnectionType.MANUALREVIEW
+ and connector is None
+ ):
raise DestinationConnectorNotConfigured()
def handle_output(
@@ -98,6 +130,7 @@ class DestinationConnector(BaseConnector):
workflow: Workflow,
file_history: Optional[FileHistory] = None,
error: Optional[str] = None,
+ input_file_path: Optional[str] = None,
) -> None:
"""Handle the output based on the connection type."""
connection_type = self.endpoint.connection_type
@@ -114,10 +147,19 @@ class DestinationConnector(BaseConnector):
elif connection_type == WorkflowEndpoint.ConnectionType.API:
result = self.get_result(file_history)
meta_data = self.get_metadata(file_history)
-
self._handle_api_result(
file_name=file_name, error=error, result=result, meta_data=meta_data
)
+ elif connection_type == WorkflowEndpoint.ConnectionType.MANUALREVIEW:
+ result = self.get_result(file_history)
+ meta_data = self.get_metadata(file_history)
+ self._push_to_queue(
+ file_name=file_name,
+ workflow=workflow,
+ result=result,
+ input_file_path=input_file_path,
+ meta_data=meta_data,
+ )
if not file_history:
FileHistoryHelper.create_file_history(
cache_key=file_hash,
@@ -433,3 +475,58 @@ class DestinationConnector(BaseConnector):
os.path.dirname(__file__), "static", "dest", "api.json"
)
return cls.get_json_schema(file_path=schema_path)
+
+ def _push_to_queue(
+ self,
+ file_name: str,
+ workflow: Workflow,
+ result: Optional[str] = None,
+ input_file_path: Optional[str] = None,
+ meta_data: Optional[dict[str, Any]] = None,
+ ) -> None:
+ """Handle the Manual Review QUEUE result.
+
+ This method is responsible for pushing the input file and result to
+ review queue.
+ Args:
+ file_name (str): The name of the file.
+ workflow (Workflow): The workflow object containing
+ details about the workflow.
+ result (Optional[str], optional): The result of the API call.
+ Defaults to None.
+ input_file_path (Optional[str], optional):
+ The path to the input file.
+ Defaults to None.
+ meta_data (Optional[dict[str, Any]], optional):
+ A dictionary containing additional
+ metadata related to the file. Defaults to None.
+
+ Returns:
+ None
+ """
+ if not result:
+ return
+ connector: ConnectorInstance = self.source_endpoint.connector_instance
+ connector_settings: dict[str, Any] = connector.connector_metadata
+
+ source_fs = self.get_fsspec(
+ settings=connector_settings, connector_id=connector.connector_id
+ )
+ with source_fs.open(input_file_path, "rb") as remote_file:
+ file_content = remote_file.read()
+ # Convert file content to a base64 encoded string
+ file_content_base64 = base64.b64encode(file_content).decode("utf-8")
+ q_name = f"review_queue_{self.organization_id}_{workflow.workflow_name}"
+ queue_result = QueueResult(
+ file=file_name,
+ whisper_hash=meta_data["whisper-hash"],
+ status=QueueResultStatus.SUCCESS,
+ result=result,
+ workflow_id=str(self.workflow_id),
+ file_content=file_content_base64,
+ )
+ # Convert the result dictionary to a JSON string
+ queue_result_json = json.dumps(queue_result)
+ conn = QueueUtils.get_queue_inst()
+ # Enqueue the JSON string
+ conn.enqueue(queue_name=q_name, message=queue_result_json)
diff --git a/backend/workflow_manager/endpoint/exceptions.py b/backend/workflow_manager/endpoint/exceptions.py
index 69e332cdd..7028eec28 100644
--- a/backend/workflow_manager/endpoint/exceptions.py
+++ b/backend/workflow_manager/endpoint/exceptions.py
@@ -77,3 +77,11 @@ class UnstractDBException(APIException):
def __init__(self, detail: str = default_detail) -> None:
status_code = 500
super().__init__(detail=detail, code=status_code)
+
+
+class UnstractQueueException(APIException):
+ default_detail = "Error creating/inserting to Queue. "
+
+ def __init__(self, detail: str = default_detail) -> None:
+ status_code = 500
+ super().__init__(detail=detail, code=status_code)
diff --git a/backend/workflow_manager/endpoint/migrations/0003_alter_workflowendpoint_connection_type.py b/backend/workflow_manager/endpoint/migrations/0003_alter_workflowendpoint_connection_type.py
new file mode 100644
index 000000000..c2da4a302
--- /dev/null
+++ b/backend/workflow_manager/endpoint/migrations/0003_alter_workflowendpoint_connection_type.py
@@ -0,0 +1,28 @@
+# Generated by Django 4.2.1 on 2024-07-04 05:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("endpoint", "0002_alter_workflowendpoint_connection_type"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="workflowendpoint",
+ name="connection_type",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("FILESYSTEM", "FileSystem connector"),
+ ("DATABASE", "Database Connector"),
+ ("API", "API Connector"),
+ ("APPDEPLOYMENT", "App Deployment"),
+ ("MANUALREVIEW", "Manual Review Queue Connector"),
+ ],
+ db_comment="Connection type (Filesystem, Database, API or Manualreview)",
+ ),
+ ),
+ ]
diff --git a/backend/workflow_manager/endpoint/models.py b/backend/workflow_manager/endpoint/models.py
index 96f64b58c..f49ca1474 100644
--- a/backend/workflow_manager/endpoint/models.py
+++ b/backend/workflow_manager/endpoint/models.py
@@ -16,6 +16,7 @@ class WorkflowEndpoint(BaseModel):
DATABASE = "DATABASE", "Database Connector"
API = "API", "API Connector"
APPDEPLOYMENT = "APPDEPLOYMENT", "App Deployment"
+ MANUALREVIEW = "MANUALREVIEW", "Manual Review Queue Connector"
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
workflow = models.ForeignKey(
@@ -33,7 +34,7 @@ class WorkflowEndpoint(BaseModel):
connection_type = models.CharField(
choices=ConnectionType.choices,
blank=True,
- db_comment="Connection type (Filesystem, Database or API)",
+ db_comment="Connection type (Filesystem, Database, API or Manualreview)",
)
configuration = models.JSONField(
blank=True, null=True, db_comment="Configuration in JSON format"
diff --git a/backend/workflow_manager/endpoint/queue_utils.py b/backend/workflow_manager/endpoint/queue_utils.py
new file mode 100644
index 000000000..bbb4dddb3
--- /dev/null
+++ b/backend/workflow_manager/endpoint/queue_utils.py
@@ -0,0 +1,41 @@
+import logging
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+from utils.constants import Common
+from workflow_manager.endpoint.exceptions import UnstractQueueException
+
+from unstract.connectors.queues import connectors as queue_connectors
+from unstract.connectors.queues.unstract_queue import UnstractQueue
+
+logger = logging.getLogger(__name__)
+
+
+class QueueResultStatus(Enum):
+ SUCCESS = "success"
+ FAILURE = "failure"
+ # Add other statuses as needed
+
+
+class QueueUtils:
+ @staticmethod
+ def get_queue_inst(connector_settings: dict[str, Any] = {}) -> UnstractQueue:
+ if not queue_connectors:
+ raise UnstractQueueException(detail="Queue connector not exists")
+ queue_connector_key = next(iter(queue_connectors))
+ connector = queue_connectors[queue_connector_key][Common.METADATA][
+ Common.CONNECTOR
+ ]
+ connector_class: UnstractQueue = connector(connector_settings)
+ return connector_class
+
+
+@dataclass
+class QueueResult:
+ file: str
+ whisper_hash: str
+ status: QueueResultStatus
+ result: Any
+ workflow_id: str
+ file_content: str
diff --git a/backend/workflow_manager/workflow/workflow_helper.py b/backend/workflow_manager/workflow/workflow_helper.py
index b264845aa..9f4e434cc 100644
--- a/backend/workflow_manager/workflow/workflow_helper.py
+++ b/backend/workflow_manager/workflow/workflow_helper.py
@@ -205,6 +205,7 @@ class WorkflowHelper:
workflow=workflow,
file_history=file_history,
error=error,
+ input_file_path=input_file,
)
execution_service.publish_update_log(
LogState.SUCCESS,
diff --git a/frontend/public/icons/connector-icons/Redis.png b/frontend/public/icons/connector-icons/Redis.png
new file mode 100644
index 000000000..b8deff6cc
Binary files /dev/null and b/frontend/public/icons/connector-icons/Redis.png differ
diff --git a/frontend/src/components/agency/actions/Actions.jsx b/frontend/src/components/agency/actions/Actions.jsx
index fe82adaca..faa973bf0 100644
--- a/frontend/src/components/agency/actions/Actions.jsx
+++ b/frontend/src/components/agency/actions/Actions.jsx
@@ -38,7 +38,7 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {
const [openAddApiModal, setOpenAddApiModal] = useState(false);
const [apiOpsPresent, setApiOpsPresent] = useState(false);
const [canAddTaskPipeline, setCanAddTaskPipeline] = useState(false);
- const [canAddETLPipeline, setCanAddETAPipeline] = useState(false);
+ const [canAddETLPipeline, setCanAddETLPipeline] = useState(false);
const [openAddTaskModal, setOpenAddTaskModal] = useState(false);
const [openAddETLModal, setOpenAddETLModal] = useState(false);
const [deploymentName, setDeploymentName] = useState("");
@@ -76,8 +76,9 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {
);
// Enable Deploy as ETL Pipeline only when
// destination connection_type is DATABASE and Source & Destination are Configured
- setCanAddETAPipeline(
- destination?.connection_type === "DATABASE" &&
+ setCanAddETLPipeline(
+ (destination?.connection_type === "DATABASE" ||
+ destination.connection_type === "MANUALREVIEW") &&
source?.connector_instance &&
destination.connector_instance
);
@@ -302,6 +303,12 @@ function Actions({ statusBarMsg, initializeWfComp, stepLoader }) {
) {
return false;
}
+ if (
+ source?.connection_type === "FILESYSTEM" &&
+ destination?.connection_type === "MANUALREVIEW"
+ ) {
+ return false;
+ }
return !source?.connector_instance || !destination?.connector_instance;
};
diff --git a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx
index 0f3977546..e781869e4 100644
--- a/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx
+++ b/frontend/src/components/agency/configure-connector-modal/ConfigureConnectorModal.jsx
@@ -41,7 +41,8 @@ function ConfigureConnectorModal({
disabled:
!connectorId ||
connDetails?.connector_id !== selectedId ||
- connType === "DATABASE",
+ connType === "DATABASE" ||
+ connType === "MANUALREVIEW",
},
];
diff --git a/frontend/src/components/agency/configure-forms-layout/ConfigureFormsLayout.jsx b/frontend/src/components/agency/configure-forms-layout/ConfigureFormsLayout.jsx
index 74a9e916d..9f6a4ed9b 100644
--- a/frontend/src/components/agency/configure-forms-layout/ConfigureFormsLayout.jsx
+++ b/frontend/src/components/agency/configure-forms-layout/ConfigureFormsLayout.jsx
@@ -50,26 +50,28 @@ function ConfigureFormsLayout({
</div>
</div>
</Col>
- <Col span={12} className="conn-modal-form-left">
- <div className="conn-modal-flex">
- <Typography.Text strong>Configuration</Typography.Text>
- <div className="conn-modal-gap" />
- <div className="conn-modal-flex-1">
- {!specConfig || Object.keys(specConfig)?.length === 0 ? (
- <EmptyState text="Failed to load the configuration form" />
- ) : (
- <SettingsForm
- selectedId={selectedId}
- handleUpdate={handleUpdate}
- spec={specConfig}
- formData={formDataConfig}
- setFormData={setFormDataConfig}
- isLoading={isSpecConfigLoading}
- />
- )}
+ {connType !== "MANUALREVIEW" && (
+ <Col span={12} className="conn-modal-form-left">
+ <div className="conn-modal-flex">
+ <Typography.Text strong>Configuration</Typography.Text>
+ <div className="conn-modal-gap" />
+ <div className="conn-modal-flex-1">
+ {!specConfig || Object.keys(specConfig)?.length === 0 ? (
+ <EmptyState text="Failed to load the configuration form" />
+ ) : (
+ <SettingsForm
+ selectedId={selectedId}
+ handleUpdate={handleUpdate}
+ spec={specConfig}
+ formData={formDataConfig}
+ setFormData={setFormDataConfig}
+ isLoading={isSpecConfigLoading}
+ />
+ )}
+ </div>
</div>
- </div>
- </Col>
+ </Col>
+ )}
</Row>
);
}
diff --git a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
index bf0d78e4d..7647f5a62 100644
--- a/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
+++ b/frontend/src/components/agency/ds-settings-card/DsSettingsCard.jsx
@@ -63,7 +63,6 @@ function DsSettingsCard({ type, endpointDetails, message }) {
const [formDataConfig, setFormDataConfig] = useState({});
const [selectedId, setSelectedId] = useState("");
const [selectedItemName, setSelectedItemName] = useState("");
-
const [inputOptions, setInputOptions] = useState([
{
value: "API",
@@ -84,14 +83,32 @@ function DsSettingsCard({ type, endpointDetails, message }) {
const { setAlertDetails } = useAlertStore();
const axiosPrivate = useAxiosPrivate();
const handleException = useExceptionHandler();
-
const { flags } = sessionDetails;
const icons = {
input: <ImportOutlined className="ds-set-icon-size" />,
output: <ExportOutlined className="ds-set-icon-size" />,
};
-
+ useEffect(() => {
+ try {
+ const inputOption =
+ require("../../../plugins/dscard-input-options/DsSettingsCardInputOptions").inputOption;
+ if (flags.manual_review && inputOption) {
+ setInputOptions((prevInputOptions) => {
+ // Check if inputOption already exists in prevInputOptions
+ if (prevInputOptions.some((opt) => opt.value === inputOption.value)) {
+ return prevInputOptions; // Return previous state unchanged
+ } else {
+ // Create a new array with the existing options and the new option
+ const updatedInputOptions = [...prevInputOptions, inputOption];
+ return updatedInputOptions;
+ }
+ });
+ }
+ } catch {
+ // The component will remain null of it is not available
+ }
+ }, []);
useEffect(() => {
try {
const inputOption =
@@ -114,7 +131,10 @@ function DsSettingsCard({ type, endpointDetails, message }) {
} else {
// Filter options based on source connection type
const filteredOptions = ["API"].includes(source?.connection_type)
- ? inputOptions.filter((option) => option.value === "API")
+ ? inputOptions.filter(
+ (option) =>
+ option.value === "API" || option.value === "MANUALREVIEW"
+ )
: inputOptions.filter((option) => option.value !== "API");
setOptions(filteredOptions);
@@ -125,7 +145,9 @@ function DsSettingsCard({ type, endpointDetails, message }) {
// Remove Database from Source Dropdown
const filteredOptions = inputOptions.filter(
(option) =>
- option.value !== "DATABASE" && option.value !== "APPDEPLOYMENT"
+ option.value !== "DATABASE" &&
+ option.value !== "APPDEPLOYMENT" &&
+ option.value !== "MANUALREVIEW"
);
setOptions(filteredOptions);
}
@@ -376,6 +398,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {
disabled={
!endpointDetails?.connection_type ||
connType === "API" ||
+ connType === "MANUALREVIEW" ||
connType === "APPDEPLOYMENT"
}
>
@@ -398,7 +421,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {
</Space>
) : (
<>
- {connType === "API" ? (
+ {connType === "API" || connType === "MANUALREVIEW" ? (
<Typography.Text
className="font-size-12 display-flex-align-center"
ellipsis={{ rows: 1, expandable: false }}
@@ -406,7 +429,7 @@ function DsSettingsCard({ type, endpointDetails, message }) {
>
<CheckCircleTwoTone twoToneColor="#52c41a" />
<span style={{ marginLeft: "5px" }}>
- {titleCase(type)} set to API successfully
+ {titleCase(type)} set to {connType} successfully
</span>
</Typography.Text>
) : (
@@ -463,7 +486,6 @@ DsSettingsCard.propTypes = {
type: PropTypes.string.isRequired,
endpointDetails: PropTypes.object.isRequired,
message: PropTypes.string,
- canUpdate: PropTypes.bool.isRequired,
};
export { DsSettingsCard };
diff --git a/frontend/src/components/helpers/auth/RequireAuth.js b/frontend/src/components/helpers/auth/RequireAuth.js
index 424967e1f..f745aca17 100644
--- a/frontend/src/components/helpers/auth/RequireAuth.js
+++ b/frontend/src/components/helpers/auth/RequireAuth.js
@@ -1,11 +1,11 @@
import { Navigate, Outlet, useLocation } from "react-router-dom";
+import { useEffect } from "react";
import {
getOrgNameFromPathname,
onboardCompleted,
} from "../../../helpers/GetStaticData";
import { useSessionStore } from "../../../store/session-store";
-import { useEffect } from "react";
import usePostHogEvents from "../../../hooks/usePostHogEvents";
let ProductFruitsManager;
@@ -25,6 +25,7 @@ const RequireAuth = () => {
const pathname = location?.pathname;
const adapters = sessionDetails?.adapters;
const currOrgName = getOrgNameFromPathname(pathname);
+ const { flags } = sessionDetails;
useEffect(() => {
if (!sessionDetails?.isLoggedIn) {
@@ -38,6 +39,14 @@ const RequireAuth = () => {
if (onboardCompleted(adapters)) {
navigateTo = `/${orgName}/tools`;
}
+ if (flags.manual_review) {
+ if (
+ sessionDetails.role === "unstract_reviewer" ||
+ sessionDetails.role === "unstract_supervisor"
+ ) {
+ navigateTo = `/${orgName}/review`;
+ }
+ }
if (!isLoggedIn) {
return <Navigate to="/landing" state={{ from: location }} replace />;
diff --git a/frontend/src/components/helpers/auth/RequireGuest.js b/frontend/src/components/helpers/auth/RequireGuest.js
index ff5a48ab5..42de4f693 100644
--- a/frontend/src/components/helpers/auth/RequireGuest.js
+++ b/frontend/src/components/helpers/auth/RequireGuest.js
@@ -5,13 +5,21 @@ import { useSessionStore } from "../../../store/session-store";
const RequireGuest = () => {
const { sessionDetails } = useSessionStore();
- const { orgName, adapters } = sessionDetails;
+ const { orgName, adapters, flags } = sessionDetails;
const location = useLocation();
const pathname = location.pathname;
let navigateTo = `/${orgName}/onboard`;
if (onboardCompleted(adapters)) {
navigateTo = `/${orgName}/tools`;
}
+ if (flags.manual_review) {
+ if (
+ sessionDetails.role === "unstract_reviewer" ||
+ sessionDetails.role === "unstract_supervisor"
+ ) {
+ navigateTo = `/${orgName}/review`;
+ }
+ }
return !sessionDetails?.isLoggedIn && publicRoutes.includes(pathname) ? (
<Outlet />
diff --git a/frontend/src/components/navigations/top-nav-bar/TopNavBar.css b/frontend/src/components/navigations/top-nav-bar/TopNavBar.css
index 511f62628..bb4095d7d 100644
--- a/frontend/src/components/navigations/top-nav-bar/TopNavBar.css
+++ b/frontend/src/components/navigations/top-nav-bar/TopNavBar.css
@@ -58,3 +58,15 @@
.switch-org-menu .ant-dropdown-menu-title-content .ant-space {
display: block;
}
+
+.page-identifier {
+ vertical-align: super;
+ color: white;
+}
+
+.page-heading {
+ color: white;
+ font-size: 16px;
+ font-weight: 600;
+ margin-left: 10px;
+}
diff --git a/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsx b/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsx
index 5d5339baf..cce90112f 100644
--- a/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsx
+++ b/frontend/src/components/navigations/top-nav-bar/TopNavBar.jsx
@@ -8,8 +8,16 @@ import {
Space,
Typography,
} from "antd";
+import {
+ UserOutlined,
+ UserSwitchOutlined,
+ LogoutOutlined,
+ DownloadOutlined,
+ FileProtectOutlined,
+ LikeOutlined,
+} from "@ant-design/icons";
import { useEffect, useState } from "react";
-import { useNavigate } from "react-router-dom";
+import { useNavigate, useLocation } from "react-router-dom";
import axios from "axios";
import { UnstractLogo } from "../../../assets/index.js";
@@ -36,19 +44,54 @@ try {
function TopNavBar() {
const navigate = useNavigate();
const { sessionDetails } = useSessionStore();
- const { orgName, remainingTrialDays, allOrganization, orgId } =
+ const { orgName, remainingTrialDays, allOrganization, orgId, flags } =
sessionDetails;
const baseUrl = getBaseUrl();
const onBoardUrl = baseUrl + `/${orgName}/onboard`;
const logout = useLogout();
const [showOnboardBanner, setShowOnboardBanner] = useState(false);
+ const [approverStatus, setApproverStatus] = useState(false);
+ const [reviewerStatus, setReviewerStatus] = useState(false);
+ const [reviewPageHeader, setReviewPageHeader] = useState("");
const { setAlertDetails } = useAlertStore();
const handleException = useExceptionHandler();
+ const location = useLocation();
useEffect(() => {
- setShowOnboardBanner(!onboardCompleted(sessionDetails?.adapters));
+ const isUnstractReviewer = sessionDetails.role === "unstract_reviewer";
+ const isUnstractSupervisor = sessionDetails.role === "unstract_supervisor";
+ const isUnstractAdmin = sessionDetails.role === "unstract_admin";
+
+ setShowOnboardBanner(
+ !onboardCompleted(sessionDetails?.adapters) &&
+ !isUnstractReviewer &&
+ !isUnstractSupervisor
+ );
+
+ setApproverStatus(
+ (isUnstractAdmin || isUnstractSupervisor) && flags.manual_review
+ );
+ setReviewerStatus(isUnstractReviewer && flags.manual_review);
}, [sessionDetails]);
+ useEffect(() => {
+ if (flags.manual_review) {
+ const checkReviewPage = location.pathname.split("review");
+
+ if (checkReviewPage.length > 1) {
+ if (checkReviewPage[1].includes("/approve")) {
+ setReviewPageHeader("Approve");
+ } else if (checkReviewPage[1].includes("/download_and_sync")) {
+ setReviewPageHeader("Download and syncmanager");
+ } else {
+ setReviewPageHeader("Review");
+ }
+ } else {
+ setReviewPageHeader(null);
+ }
+ }
+ }, [location]);
+
const cascadeOptions = allOrganization.map((org) => {
return {
key: org?.id,
@@ -92,7 +135,7 @@ function TopNavBar() {
});
};
- // Dropdown items
+ // Profile Dropdown items
const items = [
{
key: "1",
@@ -101,15 +144,7 @@ function TopNavBar() {
onClick={() => navigate(`/${orgName}/profile`)}
className="logout-button"
>
- Profile
- </Button>
- ),
- },
- {
- key: "2",
- label: (
- <Button onClick={logout} className="logout-button">
- Logout
+ <UserOutlined /> Profile
</Button>
),
},
@@ -126,10 +161,54 @@ function TopNavBar() {
}}
placement="left"
>
- <div>Switch Org</div>
+ <div>
+ {" "}
+ <UserSwitchOutlined /> Switch Org
+ </div>
</Dropdown>
),
},
+ (reviewerStatus || approverStatus) && {
+ key: "4",
+ label: (
+ <Button
+ onClick={() => navigate(`/${orgName}/review`)}
+ className="logout-button"
+ >
+ <FileProtectOutlined /> Review
+ </Button>
+ ),
+ },
+ approverStatus && {
+ key: "5",
+ label: (
+ <Button
+ onClick={() => navigate(`/${orgName}/review/approve`)}
+ className="logout-button"
+ >
+ <LikeOutlined /> Approve
+ </Button>
+ ),
+ },
+ approverStatus && {
+ key: "6",
+ label: (
+ <Button
+ onClick={() => navigate(`/${orgName}/review/download_and_sync`)}
+ className="logout-button"
+ >
+ <DownloadOutlined /> Download and Sync Manager
+ </Button>
+ ),
+ },
+ {
+ key: "2",
+ label: (
+ <Button onClick={logout} className="logout-button">
+ <LogoutOutlined /> Logout
+ </Button>
+ ),
+ },
];
// Function to get the initials from the user name
@@ -144,10 +223,16 @@ function TopNavBar() {
return (
<Row align="middle" className="topNav">
- <Col span={4}>
+ <Col span={6}>
<UnstractLogo className="topbar-logo" />
+ {reviewPageHeader && (
+ <span className="page-identifier">
+ <span className="custom-tools-header-v-divider" />
+ <span className="page-heading">{reviewPageHeader}</span>
+ </span>
+ )}
</Col>
- <Col span={16} className="top-nav-alert-col">
+ <Col span={14} className="top-nav-alert-col">
{showOnboardBanner && (
<Alert
type="error"
diff --git a/frontend/src/components/settings/invite/InviteEditUser.css b/frontend/src/components/settings/invite/InviteEditUser.css
index 27c4c95ea..867049d32 100644
--- a/frontend/src/components/settings/invite/InviteEditUser.css
+++ b/frontend/src/components/settings/invite/InviteEditUser.css
@@ -26,7 +26,8 @@
}
.role-select{
- width:250px
+ width:auto;
+ min-width:200px;
}
.form-select{
diff --git a/frontend/src/components/settings/invite/InviteEditUser.jsx b/frontend/src/components/settings/invite/InviteEditUser.jsx
index 1afd4f352..e7815c042 100644
--- a/frontend/src/components/settings/invite/InviteEditUser.jsx
+++ b/frontend/src/components/settings/invite/InviteEditUser.jsx
@@ -25,6 +25,8 @@ function InviteEditUser() {
const [submitLoading, setSubmitLoading] = useState(false);
const [initialRole, setInitialRole] = useState();
const { setPostHogCustomEvent } = usePostHogEvents();
+ const { flags } = sessionDetails;
+
const isInvite = location.pathname.split("/").slice(-1)[0] === "invite";
const validateMessages = {
required: "required!",
@@ -185,7 +187,7 @@ function InviteEditUser() {
label: formatOptionLabel(role.name),
};
})}
- disabled={isInvite}
+ disabled={!flags.manual_review}
/>
</Form.Item>
</div>
diff --git a/frontend/src/helpers/GetSessionData.js b/frontend/src/helpers/GetSessionData.js
index f5b8c288b..5ee6b9291 100644
--- a/frontend/src/helpers/GetSessionData.js
+++ b/frontend/src/helpers/GetSessionData.js
@@ -26,6 +26,7 @@ function getSessionData(sessionData) {
loginOnboardingMessage: sessionData?.loginOnboardingMessage,
promptOnboardingMessage: sessionData?.promptOnboardingMessage,
flags: sessionData?.flags,
+ role: sessionData?.role,
};
}
diff --git a/frontend/src/hooks/useSessionValid.js b/frontend/src/hooks/useSessionValid.js
index 16fcf754b..fe34e91f2 100644
--- a/frontend/src/hooks/useSessionValid.js
+++ b/frontend/src/hooks/useSessionValid.js
@@ -129,6 +129,7 @@ function useSessionValid() {
if (isPlatformAdmin) {
userAndOrgDetails["isPlatformAdmin"] = await isPlatformAdmin();
}
+ userAndOrgDetails["role"] = userSessionData.role;
// Set the session details
setSessionDetails(getSessionData(userAndOrgDetails));
} catch (err) {
diff --git a/frontend/src/routes/Router.jsx b/frontend/src/routes/Router.jsx
index ab99fde82..47bc7c8ba 100644
--- a/frontend/src/routes/Router.jsx
+++ b/frontend/src/routes/Router.jsx
@@ -35,7 +35,8 @@ let PlatformAdminPage;
let AppDeployments;
let ChatAppPage;
let ChatAppLayout;
-
+let ManualReviewPage;
+let ReviewLayout;
try {
TrialRoutes =
require("../plugins/subscription/trial-page/TrialEndPage.jsx").TrialEndPage;
@@ -52,12 +53,18 @@ try {
require("../plugins/app-deployment/AppDeployments.jsx").AppDeployments;
ChatAppPage =
require("../plugins/app-deployment/chat-app/ChatAppPage.jsx").ChatAppPage;
- ChatAppLayout =
- require("../plugins/app-deployment/chat-app/ChatAppLayout.jsx").ChatAppLayout;
} catch (err) {
// Do nothing, Not-found Page will be triggered.
}
+try {
+ ManualReviewPage =
+ require("../plugins/manual-review/page/ManualReviewPage.jsx").ManualReviewPage;
+ ReviewLayout =
+ require("../plugins/manual-review/review-layout/ReviewLayout.jsx").ReviewLayout;
+} catch (err) {
+ // Do nothing, Not-found Page will be triggered.
+}
// Import pages/components related to Simple Prompt Studio.
let SimplePromptStudioHelper;
let SimplePromptStudio;
@@ -178,6 +185,22 @@ function Router() {
</Route>
)}
</Route>
+ {ReviewLayout && ManualReviewPage && (
+ <Route path=":orgName" element={<ReviewLayout />}>
+ <Route
+ path="review"
+ element={<ManualReviewPage type="review" />}
+ ></Route>
+ <Route
+ path="review/download_and_sync"
+ element={<ManualReviewPage type="download" />}
+ />
+ <Route
+ path="review/approve"
+ element={<ManualReviewPage type="approve" />}
+ />
+ </Route>
+ )}
</Route>
{TrialRoutes && (
<Route path="/trial-expired" element={<TrialRoutes />} />
diff --git a/unstract/connectors/src/unstract/connectors/connectorkit.py b/unstract/connectors/src/unstract/connectors/connectorkit.py
index 1ef216491..5b1d574c1 100644
--- a/unstract/connectors/src/unstract/connectors/connectorkit.py
+++ b/unstract/connectors/src/unstract/connectors/connectorkit.py
@@ -7,13 +7,14 @@ from unstract.connectors.constants import Common
from unstract.connectors.databases import connectors as db_connectors
from unstract.connectors.enums import ConnectorMode
from unstract.connectors.filesystems import connectors as fs_connectors
+from unstract.connectors.queues import connectors as q_connectors
logger = logging.getLogger(__name__)
class Connectorkit:
def __init__(self) -> None:
- self._connectors: ConnectorDict = fs_connectors | db_connectors
+ self._connectors: ConnectorDict = fs_connectors | db_connectors | q_connectors
@property
def connectors(self) -> ConnectorDict:
diff --git a/unstract/connectors/src/unstract/connectors/enums.py b/unstract/connectors/src/unstract/connectors/enums.py
index f28ec5cb1..e68b4b063 100644
--- a/unstract/connectors/src/unstract/connectors/enums.py
+++ b/unstract/connectors/src/unstract/connectors/enums.py
@@ -6,3 +6,4 @@ class ConnectorMode(Enum):
FILE_SYSTEM = "FILE_SYSTEM"
DATABASE = "DATABASE"
API = "API"
+ MANUAL_REVIEW = "MANUAL_REVIEW"
diff --git a/unstract/connectors/src/unstract/connectors/queues/__init__.py b/unstract/connectors/src/unstract/connectors/queues/__init__.py
new file mode 100644
index 000000000..8a6d8d0bd
--- /dev/null
+++ b/unstract/connectors/src/unstract/connectors/queues/__init__.py
@@ -0,0 +1,5 @@
+from unstract.connectors import ConnectorDict
+from unstract.connectors.queues.register import register_connectors
+
+connectors: ConnectorDict = {}
+register_connectors(connectors)
diff --git a/unstract/connectors/src/unstract/connectors/queues/register.py b/unstract/connectors/src/unstract/connectors/queues/register.py
new file mode 100644
index 000000000..4eca532c6
--- /dev/null
+++ b/unstract/connectors/src/unstract/connectors/queues/register.py
@@ -0,0 +1,38 @@
+import logging
+import os
+from importlib import import_module
+from typing import Any
+
+from unstract.connectors.constants import Common
+from unstract.connectors.queues.unstract_queue import UnstractQueue
+
+logger = logging.getLogger(__name__)
+
+
+def register_connectors(connectors: dict[str, Any]) -> None:
+ current_directory = os.path.dirname(os.path.abspath(__file__))
+ package = "unstract.connectors.queues"
+
+ for connector in os.listdir(current_directory):
+ connector_path = os.path.join(current_directory, connector)
+ # Check if the item is a directory and not a special directory like __pycache__
+ if os.path.isdir(connector_path) and not connector.startswith("__"):
+ try:
+ full_module_path = f"{package}.{connector}"
+ module = import_module(full_module_path)
+ metadata = getattr(module, "metadata", {})
+ if metadata.get("is_active", False):
+ connector_class: UnstractQueue = metadata[Common.CONNECTOR]
+ connector_id = connector_class.get_id()
+ if not connector_id or (connector_id in connectors):
+ logger.warning(f"Duplicate Id : {connector_id}")
+ else:
+ connectors[connector_id] = {
+ Common.MODULE: module,
+ Common.METADATA: metadata,
+ }
+ except ModuleNotFoundError as exception:
+ logger.error(f"Error while importing connectors : {exception}")
+
+ if len(connectors) == 0:
+ logger.warning("No connector found.")
diff --git a/unstract/connectors/src/unstract/connectors/queues/unstract_queue.py b/unstract/connectors/src/unstract/connectors/queues/unstract_queue.py
new file mode 100644
index 000000000..940bfe041
--- /dev/null
+++ b/unstract/connectors/src/unstract/connectors/queues/unstract_queue.py
@@ -0,0 +1,95 @@
+import logging
+from abc import ABC, abstractmethod
+from typing import Any
+
+from unstract.connectors.base import UnstractConnector
+from unstract.connectors.enums import ConnectorMode
+
+
+class UnstractQueue(UnstractConnector, ABC):
+ """Abstract class for queue connector."""
+
+ logging.basicConfig(
+ level=logging.INFO,
+ format="%(asctime)s - %(levelname)s - %(filename)s - %(message)s",
+ )
+
+ def __init__(self, name: str):
+ super().__init__(name)
+ self.name = name
+
+ @staticmethod
+ def get_id() -> str:
+ return ""
+
+ @staticmethod
+ def get_name() -> str:
+ return ""
+
+ @staticmethod
+ def get_description() -> str:
+ return ""
+
+ @staticmethod
+ def get_icon() -> str:
+ return ""
+
+ @staticmethod
+ def get_json_schema() -> str:
+ return ""
+
+ @staticmethod
+ def can_write() -> bool:
+ return False
+
+ @staticmethod
+ def can_read() -> bool:
+ return False
+
+ @staticmethod
+ def requires_oauth() -> bool:
+ return False
+
+ @staticmethod
+ def python_social_auth_backend() -> str:
+ return ""
+
+ @staticmethod
+ def get_connector_mode() -> ConnectorMode:
+ return ConnectorMode.MANUAL_REVIEW
+
+ def test_credentials(self) -> bool:
+ """Override to test credentials for a connector."""
+ return True
+
+ @abstractmethod
+ def get_engine(self) -> Any:
+ pass
+
+ @abstractmethod
+ def enqueue(self, queue_name: str, message: str) -> Any:
+ pass
+
+ @abstractmethod
+ def dequeue(self, queue_name: str, timeout: int = 5) -> Any:
+ pass
+
+ @abstractmethod
+ def peek(self, queue_name: str) -> Any:
+ pass
+
+ @abstractmethod
+ def lset(self, queue_name: str, index: int, value: str) -> None:
+ pass
+
+ @abstractmethod
+ def llen(self, queue_name: str) -> int:
+ pass
+
+ @abstractmethod
+ def lindex(self, queue_name: str, index: int) -> Any:
+ pass
+
+ @abstractmethod
+ def keys(self, pattern: str = "*") -> list[str]:
+ pass