Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

removed translator, PII redactor and workflow settings

ali-zipstack60f062b8ed36
diff --git a/backend/adapter_processor/migrations/0005_alter_adapterinstance_adapter_type.py b/backend/adapter_processor/migrations/0005_alter_adapterinstance_adapter_type.py
new file mode 100644
index 000000000..9d622b190
--- /dev/null
+++ b/backend/adapter_processor/migrations/0005_alter_adapterinstance_adapter_type.py
@@ -0,0 +1,27 @@
+# Generated by Django 4.2.1 on 2024-02-28 11:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("adapter_processor", "0004_alter_adapterinstance_adapter_type"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="adapterinstance",
+ name="adapter_type",
+ field=models.CharField(
+ choices=[
+ ("UNKNOWN", "UNKNOWN"),
+ ("LLM", "LLM"),
+ ("EMBEDDING", "EMBEDDING"),
+ ("VECTOR_DB", "VECTOR_DB"),
+ ("OCR", "OCR"),
+ ("X2TEXT", "X2TEXT"),
+ ],
+ db_comment="Type of adapter LLM/EMBEDDING/VECTOR_DB",
+ ),
+ ),
+ ]
diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.py
index 59388d860..5f2074e9e 100644
--- a/backend/workflow_manager/workflow/execution.py
+++ b/backend/workflow_manager/workflow/execution.py
@@ -449,7 +449,4 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
def convert_workflow_model_to_data_class(
workflow: Workflow,
) -> WorkflowDto:
- return WorkflowDto(
- id=workflow.id,
- settings=workflow.settings,
- )
+ return WorkflowDto(id=workflow.id)
diff --git a/backend/workflow_manager/workflow/migrations/0002_remove_workflow_settings.py b/backend/workflow_manager/workflow/migrations/0002_remove_workflow_settings.py
new file mode 100644
index 000000000..b42e3efa1
--- /dev/null
+++ b/backend/workflow_manager/workflow/migrations/0002_remove_workflow_settings.py
@@ -0,0 +1,16 @@
+# Generated by Django 4.2.1 on 2024-02-28 11:29
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("workflow", "0001_initial"),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name="workflow",
+ name="settings",
+ ),
+ ]
diff --git a/backend/workflow_manager/workflow/models/workflow.py b/backend/workflow_manager/workflow/models/workflow.py
index 5fdfac678..3da08344b 100644
--- a/backend/workflow_manager/workflow/models/workflow.py
+++ b/backend/workflow_manager/workflow/models/workflow.py
@@ -41,7 +41,6 @@ class Workflow(BaseModel):
max_length=DESCRIPTION_FIELD_LENGTH, default=""
)
workflow_name = models.CharField(max_length=WORKFLOW_NAME_SIZE, unique=True)
- settings = models.JSONField(null=True, db_comment="Workflow settings")
prompt_text = models.TextField(default="")
is_active = models.BooleanField(default=False)
status = models.CharField(max_length=WORKFLOW_STATUS_LENGTH, default="")
diff --git a/backend/workflow_manager/workflow/urls.py b/backend/workflow_manager/workflow/urls.py
index 4bcc4b69c..7aa4c9925 100644
--- a/backend/workflow_manager/workflow/urls.py
+++ b/backend/workflow_manager/workflow/urls.py
@@ -27,13 +27,6 @@ workflow_clear_file_marker = WorkflowViewSet.as_view(
{"get": "clear_file_marker"}
)
workflow_schema = WorkflowViewSet.as_view({"get": "get_schema"})
-workflow_settings = WorkflowViewSet.as_view(
- {"get": "workflow_settings", "put": "workflow_settings"}
-)
-workflow_settings_schema = WorkflowViewSet.as_view(
- {"get": "workflow_settings_schema"}
-)
-
urlpatterns = format_suffix_patterns(
[
path("", workflow_list, name="workflow-list"),
@@ -64,13 +57,5 @@ urlpatterns = format_suffix_patterns(
workflow_schema,
name="workflow-schema",
),
- path(
- "<uuid:pk>/settings/", workflow_settings, name="workflow-settings"
- ),
- path(
- "settings/",
- workflow_settings_schema,
- name="workflow-settings-schema",
- ),
]
)
diff --git a/backend/workflow_manager/workflow/views.py b/backend/workflow_manager/workflow/views.py
index 496c97212..5a1fb8127 100644
--- a/backend/workflow_manager/workflow/views.py
+++ b/backend/workflow_manager/workflow/views.py
@@ -1,11 +1,9 @@
import logging
from typing import Any, Optional
-from backend.constants import RequestKey
from connector.connector_instance_helper import ConnectorInstanceHelper
from django.conf import settings
from django.db.models.query import QuerySet
-from django.http import HttpRequest
from permissions.permission import IsOwner
from pipeline.models import Pipeline
from pipeline.pipeline_processor import PipelineProcessor
@@ -16,7 +14,6 @@ from rest_framework.response import Response
from rest_framework.versioning import URLPathVersioning
from tool_instance.tool_processor import ToolProcessor
from unstract.tool_registry.dto import Tool
-from unstract.tool_registry.tool_registry import ToolRegistry
from utils.filtering import FilterHelper
from workflow_manager.endpoint.destination import DestinationConnector
from workflow_manager.endpoint.endpoint_utils import WorkflowEndpointUtils
@@ -27,7 +24,6 @@ from workflow_manager.workflow.enums import SchemaEntity, SchemaType
from workflow_manager.workflow.exceptions import (
InvalidRequest,
WorkflowDoesNotExistError,
- WorkflowExecutionBadRequestException,
WorkflowExecutionError,
WorkflowGenerationError,
WorkflowRegenerationError,
@@ -44,6 +40,8 @@ from workflow_manager.workflow.workflow_helper import (
WorkflowSchemaHelper,
)
+from backend.constants import RequestKey
+
logger = logging.getLogger(__name__)
@@ -340,33 +338,3 @@ class WorkflowViewSet(viewsets.ModelViewSet):
schema_type=schema_type, schema_entity=schema_entity
)
return Response(data=json_schema, status=status.HTTP_200_OK)
-
- @action(detail=True, methods=["GET"])
- def workflow_settings_schema(self, request: HttpRequest) -> Response:
- tool_registry = ToolRegistry()
- schema = tool_registry.get_project_settings_schema()
- return Response(schema)
-
- @action(detail=True, methods=["GET", "PUT"])
- def workflow_settings(self, request: HttpRequest, pk: str) -> Response:
- workflow = self.get_object()
- tool_registry = ToolRegistry()
- if request.method == "GET":
- schema = tool_registry.get_project_settings_schema()
- return Response({"schema": schema, "settings": workflow.settings})
-
- elif request.method == "PUT":
- schema = tool_registry.get_project_settings_schema()
-
- try:
- tool_registry.validate_schema_with_data(request.data, schema)
- except Exception as e:
- logger.error(f"Invalid input data {str(e)}")
- raise WorkflowExecutionBadRequestException("Invalid input")
-
- workflow.settings = request.data
- workflow.save()
- return Response(
- {"message": "Settings updated successfully"},
- status=status.HTTP_200_OK,
- )
diff --git a/frontend/src/components/agency/configuration-modal/ConfigurationModal.css b/frontend/src/components/agency/configuration-modal/ConfigurationModal.css
deleted file mode 100644
index e69de29bb..000000000
diff --git a/frontend/src/components/agency/configuration-modal/ConfigurationModal.jsx b/frontend/src/components/agency/configuration-modal/ConfigurationModal.jsx
deleted file mode 100644
index 3e5597f76..000000000
--- a/frontend/src/components/agency/configuration-modal/ConfigurationModal.jsx
+++ /dev/null
@@ -1,25 +0,0 @@
-import { Modal } from "antd";
-import PropTypes from "prop-types";
-
-import { Configuration } from "../configuration/Configuration";
-
-function ConfigurationModal({ isOpen, setOpen }) {
- return (
- <Modal
- open={isOpen}
- onCancel={() => setOpen(false)}
- footer={false}
- closable={true}
- maskClosable={false}
- >
- <Configuration setOpen={setOpen} />
- </Modal>
- );
-}
-
-ConfigurationModal.propTypes = {
- isOpen: PropTypes.bool.isRequired,
- setOpen: PropTypes.func.isRequired,
-};
-
-export { ConfigurationModal };
diff --git a/frontend/src/components/agency/configuration/Configuration.css b/frontend/src/components/agency/configuration/Configuration.css
deleted file mode 100644
index 534b54fb2..000000000
--- a/frontend/src/components/agency/configuration/Configuration.css
+++ /dev/null
@@ -1,10 +0,0 @@
-/* Styles for Configuration */
-
-.config-head {
- padding-bottom: 10px;
-}
-
-.config-head-typo {
- font-size: 18px;
- font-weight: 600;
-}
\ No newline at end of file
diff --git a/frontend/src/components/agency/configuration/Configuration.jsx b/frontend/src/components/agency/configuration/Configuration.jsx
deleted file mode 100644
index e834e4625..000000000
--- a/frontend/src/components/agency/configuration/Configuration.jsx
+++ /dev/null
@@ -1,135 +0,0 @@
-import { Button, Space, Typography } from "antd";
-import PropTypes from "prop-types";
-import { createRef, useEffect, useState } from "react";
-import { useParams } from "react-router-dom";
-
-import { handleException } from "../../../helpers/GetStaticData";
-import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";
-import { RjsfFormLayout } from "../../../layouts/rjsf-form-layout/RjsfFormLayout";
-import { useAlertStore } from "../../../store/alert-store";
-import { useSessionStore } from "../../../store/session-store";
-import { useWorkflowStore } from "../../../store/workflow-store";
-import { CustomButton } from "../../widgets/custom-button/CustomButton";
-import "./Configuration.css";
-
-function Configuration({ setOpen }) {
- const formRef = createRef(null);
- const [schema, setSchema] = useState({});
- const [formData, setFormData] = useState({});
- const [isSchemaLoading, setSchemaLoading] = useState(false);
- const [isUpdateApiLoading, setUpdateApiLoading] = useState(false);
- const { id } = useParams();
- const axiosPrivate = useAxiosPrivate();
- const { sessionDetails } = useSessionStore();
- const { projectName } = useWorkflowStore();
- const { setAlertDetails } = useAlertStore();
-
- useEffect(() => {
- const requestOptions = {
- method: "GET",
- url: `/api/v1/unstract/${sessionDetails?.orgId}/workflow/${id}/settings/`,
- };
-
- setSchemaLoading(true);
- axiosPrivate(requestOptions)
- .then((res) => {
- const data = res?.data;
- setSchema(data?.schema || {});
- setFormData(data?.settings || {});
- })
- .catch((err) => {
- setAlertDetails(handleException(err, "Failed to load the form"));
- })
- .finally(() => {
- setSchemaLoading(false);
- });
- }, []);
-
- const isFormValid = () => {
- if (formRef) {
- formRef?.current?.validateFields((errors, values) => {
- if (errors) {
- return false;
- }
- });
- }
- return true;
- };
-
- const validateAndSubmit = (updatedFormData) => {
- if (!isFormValid()) {
- return;
- }
- handleSubmit(updatedFormData);
- };
-
- const handleSubmit = (updatedFormData) => {
- setFormData(updatedFormData);
- const body = {
- workflow_name: projectName,
- settings: updatedFormData,
- };
-
- const requestOptions = {
- method: "PUT",
- url: `/api/v1/unstract/${sessionDetails?.orgId}/workflow/${id}/`,
- headers: {
- "X-CSRFToken": sessionDetails?.csrfToken,
- "Content-Type": "application/json",
- },
- data: body,
- };
-
- setUpdateApiLoading(true);
- axiosPrivate(requestOptions)
- .then(() => {
- setAlertDetails({
- type: "success",
- content: "Saved successfully",
- });
- setOpen(false);
- })
- .catch((err) => {
- setAlertDetails(handleException(err, "Failed to save"));
- })
- .finally(() => {
- setUpdateApiLoading(false);
- });
- };
-
- return (
- <div>
- <div className="config-head">
- <Typography.Text className="config-head-typo">
- Configuration
- </Typography.Text>
- </div>
- <RjsfFormLayout
- schema={schema}
- formData={formData}
- isLoading={isSchemaLoading}
- validateAndSubmit={validateAndSubmit}
- formRef={formRef}
- >
- <div className="display-flex-right">
- <Space>
- <Button onClick={() => setOpen(false)}>Cancel</Button>
- <CustomButton
- type="primary"
- htmlType="submit"
- loading={isUpdateApiLoading}
- >
- Save
- </CustomButton>
- </Space>
- </div>
- </RjsfFormLayout>
- </div>
- );
-}
-
-Configuration.propTypes = {
- setOpen: PropTypes.func.isRequired,
-};
-
-export { Configuration };
diff --git a/frontend/src/layouts/menu-layout/MenuLayout.jsx b/frontend/src/layouts/menu-layout/MenuLayout.jsx
index e2f18f45a..893130fde 100644
--- a/frontend/src/layouts/menu-layout/MenuLayout.jsx
+++ b/frontend/src/layouts/menu-layout/MenuLayout.jsx
@@ -2,23 +2,17 @@ import { useEffect, useState, useRef } from "react";
import { useLocation, useNavigate } from "react-router-dom";
import PropTypes from "prop-types";
import { Button, Space, Typography } from "antd";
-import {
- ArrowLeftOutlined,
- ToolOutlined,
- QuestionCircleOutlined,
-} from "@ant-design/icons";
+import { ArrowLeftOutlined, QuestionCircleOutlined } from "@ant-design/icons";
import "./MenuLayout.css";
import { useSessionStore } from "../../store/session-store";
import { useWorkflowStore } from "../../store/workflow-store";
-import { ConfigurationModal } from "../../components/agency/configuration-modal/ConfigurationModal";
function MenuLayout({ children }) {
const navigate = useNavigate();
const location = useLocation();
const currentMenu = useRef();
const [activeTab, setActiveTab] = useState("");
- const [openConfigModal, setOpenConfigModal] = useState(false);
const { sessionDetails } = useSessionStore();
const { projectName } = useWorkflowStore();
@@ -51,14 +45,6 @@ function MenuLayout({ children }) {
</div>
<div>
<Space>
- <Button
- onClick={() => setOpenConfigModal((prev) => !prev)}
- key="configuration"
- icon={<ToolOutlined />}
- disabled={false}
- >
- Configuration
- </Button>
<Button
key="help"
icon={<QuestionCircleOutlined />}
@@ -73,10 +59,6 @@ function MenuLayout({ children }) {
<div className="appBody">
<div className="appBody2">{children}</div>
</div>
- <ConfigurationModal
- isOpen={openConfigModal}
- setOpen={setOpenConfigModal}
- />
</>
);
}
diff --git a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json b/unstract/tool-registry/src/unstract/tool_registry/public_tools.json
index d5e04f633..a208ef6cc 100644
--- a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json
+++ b/unstract/tool-registry/src/unstract/tool_registry/public_tools.json
@@ -134,315 +134,6 @@
"image_name": "unstract/tool-classifier",
"image_tag": "0.0.1"
},
- "translate": {
- "tool_uid": "translate",
- "properties": {
- "schemaVersion": "0.0.1",
- "displayName": "Translate",
- "functionName": "translate",
- "toolVersion": "0.0.1",
- "description": "This is a tool which can translate the given text into a target language.",
- "input": {
- "description": "File to be translated"
- },
- "output": {
- "description": "Creates a file with the translated text"
- },
- "result": {
- "type": "TXT",
- "description": "Response containing the translated text"
- },
- "ioCompatibility": {
- "api": {
- "sourceSupport": true,
- "destinationSupport": true,
- "additionalArgs": {
- "sync": true
- }
- },
- "file": {
- "sourceSupport": true,
- "destinationSupport": true,
- "additionalArgs": {}
- },
- "db": {
- "destinationSupport": false,
- "additionalArgs": {}
- }
- },
- "restrictions": {
- "maxFileSize": "10MB",
- "allowedFileTypes": [
- "*"
- ]
- }
- },
- "spec": {
- "title": "Language translation tool settings",
- "description": "Setup the translation tool",
- "type": "object",
- "required": [],
- "properties": {
- "processor": {
- "type": "string",
- "title": "Processor to use",
- "default": "Google Translate",
- "enum": [
- "Google Translate"
- ],
- "description": "Service used to translate text"
- },
- "targetLanguage": {
- "type": "string",
- "title": "Target language",
- "default": "English",
- "enum": [
- "Albanian",
- "Arabic",
- "Azerbaijani",
- "Bengali",
- "Bulgarian",
- "Chinese",
- "Croatian",
- "Czech",
- "Danish",
- "Dutch",
- "English",
- "Estonian",
- "Finnish",
- "French",
- "Georgian",
- "German",
- "Greek",
- "Hebrew",
- "Hindi",
- "Hungarian",
- "Icelandic",
- "Indonesian",
- "Italian",
- "Japanese",
- "Kazakh",
- "Korean",
- "Latvian",
- "Lithuanian",
- "Malay",
- "Malayalam",
- "Mongolian",
- "Norwegian",
- "Northern_sami",
- "Pashto",
- "Persian",
- "Polish",
- "Portuguese",
- "Romanian",
- "Russian",
- "Serbian",
- "Slovak",
- "Slovenian",
- "Spanish",
- "Swedish",
- "Tagalog",
- "Tamil",
- "Thai",
- "Turkish",
- "Ukrainian",
- "Urdu",
- "Vietnamese"
- ],
- "description": "The language to translate to"
- },
- "useCache": {
- "type": "boolean",
- "title": "Cache and use cached results",
- "default": true,
- "description": "Use cached results"
- }
- }
- },
- "variables": {
- "title": "Runtime Variables",
- "description": "Runtime Variables for translate",
- "type": "object",
- "required": [
- "GOOGLE_SERVICE_ACCOUNT"
- ],
- "properties": {
- "GOOGLE_SERVICE_ACCOUNT": {
- "type": "string",
- "title": "Google Service Account",
- "description": "Google Service account"
- }
- }
- },
- "icon": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<svg\n height=\"48\"\n viewBox=\"0 0 24 24\"\n width=\"48\"\n fill=\"#000000\"\n version=\"1.1\"\n id=\"svg256\"\n xmlns=\"http://www.w3.org/2000/svg\"\n xmlns:svg=\"http://www.w3.org/2000/svg\">\n <defs\n id=\"defs260\" />\n <path\n d=\"M 0,0 H 24 V 24 H 0 Z\"\n fill=\"none\"\n id=\"path252\" />\n <path\n d=\"m 12.87,15.07 -2.54,-2.51 0.03,-0.03 C 12.1,10.59 13.34,8.36 14.07,6 H 17 V 4 H 10 V 2 H 8 V 4 H 1 V 5.99 H 12.17 C 11.5,7.92 10.44,9.75 9,11.35 8.07,10.32 7.3,9.19 6.69,8 h -2 c 0.73,1.63 1.73,3.17 2.98,4.56 L 2.58,17.58 4,19 9,14 12.11,17.11 Z M 18.5,10 h -2 L 12,22 h 2 l 1.12,-3 h 4.75 L 21,22 h 2 z m -2.62,7 1.62,-4.33 1.62,4.33 z\"\n id=\"path254\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n</svg>\n",
- "image_url": "docker:unstract/tool-translate:0.0.1",
- "image_name": "unstract/tool-translate",
- "image_tag": "0.0.1"
- },
- "pii_redactor": {
- "tool_uid": "pii_redactor",
- "properties": {
- "schemaVersion": "0.0.1",
- "displayName": "PII Redactor",
- "functionName": "pii_redactor",
- "toolVersion": "0.0.1",
- "description": "This is a tool which can redact the PII (Personally Identifiable Information) data or information from a file.",
- "input": {
- "description": "File whose PII needs to be redacted"
- },
- "output": {
- "description": "Creates a PDF file with PII redacted from it"
- },
- "result": {
- "type": "TXT",
- "description": "Response containing the text with redacted PII"
- },
- "ioCompatibility": {
- "api": {
- "sourceSupport": true,
- "destinationSupport": true,
- "additionalArgs": {
- "sync": false
- }
- },
- "file": {
- "sourceSupport": true,
- "destinationSupport": true,
- "additionalArgs": {}
- },
- "db": {
- "destinationSupport": false,
- "additionalArgs": {}
- }
- },
- "restrictions": {
- "maxFileSize": "10MB",
- "allowedFileTypes": [
- "txt",
- "pdf",
- "doc",
- "docx",
- "odt"
- ]
- }
- },
- "spec": {
- "title": "PII Redactor tool settings",
- "description": "Setup the document PII redactor tool",
- "type": "object",
- "required": [
- "processor"
- ],
- "properties": {
- "processor": {
- "type": "string",
- "title": "Processor to use",
- "default": "Amazon Comprehend",
- "enum": [
- "Amazon Comprehend"
- ],
- "description": "Service used to detect PII entities"
- },
- "redactionElements": {
- "type": "array",
- "title": "PII elements to redact",
- "description": "PII elements to redact. Leave it empty to process all elements",
- "items": {
- "type": "string",
- "enum": [
- "Address",
- "Age",
- "Bank Account Number",
- "Bank Routing Number",
- "Canada Health Number",
- "Canada Social Insurance Number",
- "Credit Card Number",
- "Credit Card CVV",
- "Credit Card Expiration Date",
- "Date & Time",
- "Email",
- "Indian Aadhaar Number",
- "Indian PAN Number",
- "Indian NREGA Number",
- "Indian Voter ID Number",
- "IP Address",
- "MAC Address",
- "Name",
- "Passport Number",
- "Password",
- "Phone Number",
- "PIN",
- "SSN",
- "URL",
- "UK National Insurance Number",
- "UK NHS Number",
- "UK Tax ID Number",
- "US Individual Taxpayer Identification Number",
- "Username",
- "VIN"
- ]
- },
- "default": []
- },
- "useCache": {
- "type": "boolean",
- "title": "Cache and use cached results",
- "default": true,
- "description": "Use cached results"
- },
- "scoreThreshold": {
- "type": "number",
- "title": "Score Threshold",
- "default": 0.8,
- "description": "Minimum confidence needed to treat an entity as PII (from 0.0 to 1.0)"
- }
- }
- },
- "variables": {
- "title": "Runtime Variables",
- "description": "Runtime Variables for doc pii redactor",
- "type": "object",
- "required": [
- "PII_REDACT_AWS_REGION",
- "PII_REDACT_AWS_ACCESS_KEY_ID",
- "PII_REDACT_AWS_SECRET_ACCESS_KEY",
- "DOCUMENT_PROCESSOR_URL",
- "DOCUMENT_PROCESSOR_API_KEY"
- ],
- "properties": {
- "PII_REDACT_AWS_REGION": {
- "type": "string",
- "title": "AWS Comprehend's Region",
- "description": "AWS Comprehend's region used in Document PII Redactor"
- },
- "PII_REDACT_AWS_ACCESS_KEY_ID": {
- "type": "string",
- "title": "AWS Comprehend's Access Key ID",
- "description": "AWS Comprehend's access key ID used in Document PII Redactor"
- },
- "PII_REDACT_AWS_SECRET_ACCESS_KEY": {
- "type": "string",
- "title": "AWS Comprehend's Secret Access Key",
- "format": "password",
- "description": "AWS Comprehend's secret access key used in Document PII Redactor"
- },
- "DOCUMENT_PROCESSOR_URL": {
- "type": "string",
- "title": "Document Processor URL",
- "description": "Document processor service URL"
- },
- "DOCUMENT_PROCESSOR_API_KEY": {
- "type": "string",
- "title": "Document Processor API key",
- "description": "Document processor service API key"
- }
- }
- },
- "icon": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<svg\n enable-background=\"new 0 0 24 24\"\n height=\"48\"\n viewBox=\"0 0 24 24\"\n width=\"48\"\n fill=\"#000000\"\n version=\"1.1\"\n id=\"svg2702\"\n sodipodi:docname=\"no_accounts_black_48dp.svg\"\n xmlns:inkscape=\"http://www.inkscape.org/namespaces/inkscape\"\n xmlns:sodipodi=\"http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd\"\n xmlns=\"http://www.w3.org/2000/svg\"\n xmlns:svg=\"http://www.w3.org/2000/svg\">\n <defs\n id=\"defs2706\" />\n <sodipodi:namedview\n id=\"namedview2704\"\n pagecolor=\"#ffffff\"\n bordercolor=\"#000000\"\n borderopacity=\"0.25\"\n inkscape:showpageshadow=\"2\"\n inkscape:pageopacity=\"0.0\"\n inkscape:pagecheckerboard=\"0\"\n inkscape:deskcolor=\"#d1d1d1\"\n showgrid=\"false\" />\n <g\n id=\"g2692\">\n <rect\n fill=\"none\"\n height=\"24\"\n width=\"24\"\n id=\"rect2690\"\n x=\"0\"\n y=\"0\" />\n </g>\n <g\n id=\"g2700\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <g\n id=\"g2698\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <path\n d=\"M 15.18,10.94 C 15.38,10.5 15.5,10.02 15.5,9.5 15.5,7.57 13.93,6 12,6 11.48,6 11,6.12 10.56,6.32 Z\"\n id=\"path2694\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n <path\n d=\"M 12,2 C 6.48,2 2,6.48 2,12 2,17.52 6.48,22 12,22 17.52,22 22,17.52 22,12 22,6.48 17.52,2 12,2 Z M 4,12 C 4,10.15 4.63,8.45 5.69,7.1 l 2.86,2.86 c 0.21,1.56 1.43,2.79 2.99,2.99 l 2.2,2.2 C 13.17,15.05 12.59,15 12,15 9.68,15 7.55,15.8 5.86,17.12 4.7,15.73 4,13.95 4,12 Z m 8,8 C 10.26,20 8.66,19.44 7.35,18.5 8.66,17.56 10.26,17 12,17 c 1.74,0 3.34,0.56 4.65,1.5 C 15.34,19.44 13.74,20 12,20 Z M 18.31,16.9 7.1,5.69 C 8.45,4.63 10.15,4 12,4 c 4.42,0 8,3.58 8,8 0,1.85 -0.63,3.54 -1.69,4.9 z\"\n id=\"path2696\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n </g>\n </g>\n</svg>\n",
- "image_url": "docker:unstract/tool-doc-pii-redactor:0.0.1",
- "image_name": "unstract/tool-doc-pii-redactor",
- "image_tag": "0.0.1"
- },
"document_indexer": {
"tool_uid": "document_indexer",
"properties": {
@@ -647,4 +338,4 @@
"image_name": "unstract/tool-text-extractor",
"image_tag": "0.0.1"
}
-}
\ No newline at end of file
+}
diff --git a/unstract/tool-registry/src/unstract/tool_registry/tool_registry.py b/unstract/tool-registry/src/unstract/tool_registry/tool_registry.py
index f81235300..dcb6d233b 100644
--- a/unstract/tool-registry/src/unstract/tool_registry/tool_registry.py
+++ b/unstract/tool-registry/src/unstract/tool_registry/tool_registry.py
@@ -333,15 +333,6 @@ class ToolRegistry:
is_ready: bool = self.helper.is_image_ready(image_name=image_name)
return is_ready
- def get_project_settings_schema(self) -> dict[str, Any]:
- """Project tool's environmental settings schema.
-
- Returns:
- dict[str, Any]: schema
- """
- schema: dict[str, Any] = ToolUtils.get_tool_project_schema()
- return schema
-
def validate_schema_with_data(
self, schema: dict[str, Any], data: dict[str, Any]
) -> None:
diff --git a/unstract/tool-registry/src/unstract/tool_registry/tool_utils.py b/unstract/tool-registry/src/unstract/tool_registry/tool_utils.py
index 22114ecf3..c8b98586d 100644
--- a/unstract/tool-registry/src/unstract/tool_registry/tool_utils.py
+++ b/unstract/tool-registry/src/unstract/tool_registry/tool_utils.py
@@ -248,9 +248,7 @@ class ToolUtils:
text_extractors = ToolUtils.get_enabled_adapters(
adapter.text_extractors
)
- ocrs = ToolUtils.get_enabled_adapters(
- adapter.ocrs
- )
+ ocrs = ToolUtils.get_enabled_adapters(adapter.ocrs)
ToolUtils.process_adapter_models(
models=language_models,
@@ -281,43 +279,3 @@ class ToolUtils:
schema=schema,
)
return schema
-
- @staticmethod
- def get_tool_project_schema() -> dict[str, Any]:
- return {
- "title": "Project Settings",
- "description": "Project Settings for tools",
- "type": "object",
- "properties": {
- "GOOGLE_SERVICE_ACCOUNT": {
- "type": "string",
- "title": "Google Service Account",
- "format": "password",
- "description": (
- "Google Service account for OCR and Translate tools"
- ),
- },
- "GOOGLE_DOCUMENT_AI_ENDPOINT": {
- "type": "string",
- "title": "Google Document API Base URL",
- "description": "The base URL for the Google Document service used for OCR", # noqa: E501
- },
- # Used by the PII redactor tool
- "PII_REDACT_AWS_REGION": {
- "type": "string",
- "title": "AWS Comprehend's Region",
- "description": "AWS Comprehend's region used in Document PII Redactor", # noqa: E501
- },
- "PII_REDACT_AWS_ACCESS_KEY_ID": {
- "type": "string",
- "title": "AWS Comprehend's Access Key ID",
- "description": "AWS Comprehend's access key ID used in Document PII Redactor", # noqa: E501
- },
- "PII_REDACT_AWS_SECRET_ACCESS_KEY": {
- "type": "string",
- "title": "AWS Comprehend's Secret Access Key",
- "format": "password",
- "description": "AWS Comprehend's secret access key used in Document PII Redactor", # noqa: E501
- },
- },
- }
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/dto.py b/unstract/workflow-execution/src/unstract/workflow_execution/dto.py
index b4d91b9a5..0324c7e66 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/dto.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/dto.py
@@ -38,4 +38,3 @@ class ToolInstance:
@dataclass
class WorkflowDto:
id: str
- settings: dict[str, Any]
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
index d3f2988af..008df896a 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
@@ -33,7 +33,6 @@ class ToolsUtils:
self.organization_id = organization_id
self.platform_service_api_key = platform_service_api_key
self.workflow_id = workflow.id
- self.project_settings = workflow.settings
self.ignore_processed_entities = ignore_processed_entities
self.messaging_channel: Optional[str] = None
self.platform_service_host = ToolsUtils.get_env(
@@ -117,9 +116,7 @@ class ToolsUtils:
),
)
- tool_envs = self.get_tool_environment_variables(
- self.project_settings
- )
+ tool_envs = self.get_tool_environment_variables()
logger.info(f"Tool Environments are collected for tool {tool_uid}")
image_name = tool_instance.image_name
image_tag = tool_instance.image_tag
@@ -203,7 +200,7 @@ class ToolsUtils:
return None
def get_tool_environment_variables(
- self, project_settings: Optional[dict[str, Any]]
+ self, project_settings: Optional[dict[str, Any]] = None
) -> dict[str, Any]:
"""Obtain a dictionary of env variables required by a tool.