LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
removed translator, PII redactor and workflow settings
ali-zipstack60f062b8ed36
diff --git a/backend/adapter_processor/migrations/0005_alter_adapterinstance_adapter_type.py b/backend/adapter_processor/migrations/0005_alter_adapterinstance_adapter_type.pynew file mode 100644index 000000000..9d622b190--- /dev/null+++ b/backend/adapter_processor/migrations/0005_alter_adapterinstance_adapter_type.py@@ -0,0 +1,27 @@+# Generated by Django 4.2.1 on 2024-02-28 11:25++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("adapter_processor", "0004_alter_adapterinstance_adapter_type"),+ ]++ operations = [+ migrations.AlterField(+ model_name="adapterinstance",+ name="adapter_type",+ field=models.CharField(+ choices=[+ ("UNKNOWN", "UNKNOWN"),+ ("LLM", "LLM"),+ ("EMBEDDING", "EMBEDDING"),+ ("VECTOR_DB", "VECTOR_DB"),+ ("OCR", "OCR"),+ ("X2TEXT", "X2TEXT"),+ ],+ db_comment="Type of adapter LLM/EMBEDDING/VECTOR_DB",+ ),+ ),+ ]diff --git a/backend/workflow_manager/workflow/execution.py b/backend/workflow_manager/workflow/execution.pyindex 59388d860..5f2074e9e 100644--- a/backend/workflow_manager/workflow/execution.py+++ b/backend/workflow_manager/workflow/execution.py@@ -449,7 +449,4 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):def convert_workflow_model_to_data_class(workflow: Workflow,) -> WorkflowDto:- return WorkflowDto(- id=workflow.id,- settings=workflow.settings,- )+ return WorkflowDto(id=workflow.id)diff --git a/backend/workflow_manager/workflow/migrations/0002_remove_workflow_settings.py b/backend/workflow_manager/workflow/migrations/0002_remove_workflow_settings.pynew file mode 100644index 000000000..b42e3efa1--- /dev/null+++ b/backend/workflow_manager/workflow/migrations/0002_remove_workflow_settings.py@@ -0,0 +1,16 @@+# Generated by Django 4.2.1 on 2024-02-28 11:29++from django.db import migrations+++class Migration(migrations.Migration):+ dependencies = [+ ("workflow", "0001_initial"),+ ]++ operations = [+ migrations.RemoveField(+ model_name="workflow",+ name="settings",+ ),+ ]diff --git a/backend/workflow_manager/workflow/models/workflow.py b/backend/workflow_manager/workflow/models/workflow.pyindex 5fdfac678..3da08344b 100644--- a/backend/workflow_manager/workflow/models/workflow.py+++ b/backend/workflow_manager/workflow/models/workflow.py@@ -41,7 +41,6 @@ class Workflow(BaseModel):max_length=DESCRIPTION_FIELD_LENGTH, default="")workflow_name = models.CharField(max_length=WORKFLOW_NAME_SIZE, unique=True)- settings = models.JSONField(null=True, db_comment="Workflow settings")prompt_text = models.TextField(default="")is_active = models.BooleanField(default=False)status = models.CharField(max_length=WORKFLOW_STATUS_LENGTH, default="")diff --git a/backend/workflow_manager/workflow/urls.py b/backend/workflow_manager/workflow/urls.pyindex 4bcc4b69c..7aa4c9925 100644--- a/backend/workflow_manager/workflow/urls.py+++ b/backend/workflow_manager/workflow/urls.py@@ -27,13 +27,6 @@ workflow_clear_file_marker = WorkflowViewSet.as_view({"get": "clear_file_marker"})workflow_schema = WorkflowViewSet.as_view({"get": "get_schema"})-workflow_settings = WorkflowViewSet.as_view(- {"get": "workflow_settings", "put": "workflow_settings"}-)-workflow_settings_schema = WorkflowViewSet.as_view(- {"get": "workflow_settings_schema"}-)-urlpatterns = format_suffix_patterns([path("", workflow_list, name="workflow-list"),@@ -64,13 +57,5 @@ urlpatterns = format_suffix_patterns(workflow_schema,name="workflow-schema",),- path(- "<uuid:pk>/settings/", workflow_settings, name="workflow-settings"- ),- path(- "settings/",- workflow_settings_schema,- name="workflow-settings-schema",- ),])diff --git a/backend/workflow_manager/workflow/views.py b/backend/workflow_manager/workflow/views.pyindex 496c97212..5a1fb8127 100644--- a/backend/workflow_manager/workflow/views.py+++ b/backend/workflow_manager/workflow/views.py@@ -1,11 +1,9 @@import loggingfrom typing import Any, Optional-from backend.constants import RequestKeyfrom connector.connector_instance_helper import ConnectorInstanceHelperfrom django.conf import settingsfrom django.db.models.query import QuerySet-from django.http import HttpRequestfrom permissions.permission import IsOwnerfrom pipeline.models import Pipelinefrom pipeline.pipeline_processor import PipelineProcessor@@ -16,7 +14,6 @@ from rest_framework.response import Responsefrom rest_framework.versioning import URLPathVersioningfrom tool_instance.tool_processor import ToolProcessorfrom unstract.tool_registry.dto import Tool-from unstract.tool_registry.tool_registry import ToolRegistryfrom utils.filtering import FilterHelperfrom workflow_manager.endpoint.destination import DestinationConnectorfrom workflow_manager.endpoint.endpoint_utils import WorkflowEndpointUtils@@ -27,7 +24,6 @@ from workflow_manager.workflow.enums import SchemaEntity, SchemaTypefrom workflow_manager.workflow.exceptions import (InvalidRequest,WorkflowDoesNotExistError,- WorkflowExecutionBadRequestException,WorkflowExecutionError,WorkflowGenerationError,WorkflowRegenerationError,@@ -44,6 +40,8 @@ from workflow_manager.workflow.workflow_helper import (WorkflowSchemaHelper,)+from backend.constants import RequestKey+logger = logging.getLogger(__name__)@@ -340,33 +338,3 @@ class WorkflowViewSet(viewsets.ModelViewSet):schema_type=schema_type, schema_entity=schema_entity)return Response(data=json_schema, status=status.HTTP_200_OK)-- @action(detail=True, methods=["GET"])- def workflow_settings_schema(self, request: HttpRequest) -> Response:- tool_registry = ToolRegistry()- schema = tool_registry.get_project_settings_schema()- return Response(schema)-- @action(detail=True, methods=["GET", "PUT"])- def workflow_settings(self, request: HttpRequest, pk: str) -> Response:- workflow = self.get_object()- tool_registry = ToolRegistry()- if request.method == "GET":- schema = tool_registry.get_project_settings_schema()- return Response({"schema": schema, "settings": workflow.settings})-- elif request.method == "PUT":- schema = tool_registry.get_project_settings_schema()-- try:- tool_registry.validate_schema_with_data(request.data, schema)- except Exception as e:- logger.error(f"Invalid input data {str(e)}")- raise WorkflowExecutionBadRequestException("Invalid input")-- workflow.settings = request.data- workflow.save()- return Response(- {"message": "Settings updated successfully"},- status=status.HTTP_200_OK,- )diff --git a/frontend/src/components/agency/configuration-modal/ConfigurationModal.css b/frontend/src/components/agency/configuration-modal/ConfigurationModal.cssdeleted file mode 100644index e69de29bb..000000000diff --git a/frontend/src/components/agency/configuration-modal/ConfigurationModal.jsx b/frontend/src/components/agency/configuration-modal/ConfigurationModal.jsxdeleted file mode 100644index 3e5597f76..000000000--- a/frontend/src/components/agency/configuration-modal/ConfigurationModal.jsx+++ /dev/null@@ -1,25 +0,0 @@-import { Modal } from "antd";-import PropTypes from "prop-types";--import { Configuration } from "../configuration/Configuration";--function ConfigurationModal({ isOpen, setOpen }) {- return (- <Modal- open={isOpen}- onCancel={() => setOpen(false)}- footer={false}- closable={true}- maskClosable={false}- >- <Configuration setOpen={setOpen} />- </Modal>- );-}--ConfigurationModal.propTypes = {- isOpen: PropTypes.bool.isRequired,- setOpen: PropTypes.func.isRequired,-};--export { ConfigurationModal };diff --git a/frontend/src/components/agency/configuration/Configuration.css b/frontend/src/components/agency/configuration/Configuration.cssdeleted file mode 100644index 534b54fb2..000000000--- a/frontend/src/components/agency/configuration/Configuration.css+++ /dev/null@@ -1,10 +0,0 @@-/* Styles for Configuration */--.config-head {- padding-bottom: 10px;-}--.config-head-typo {- font-size: 18px;- font-weight: 600;-}\ No newline at end of filediff --git a/frontend/src/components/agency/configuration/Configuration.jsx b/frontend/src/components/agency/configuration/Configuration.jsxdeleted file mode 100644index e834e4625..000000000--- a/frontend/src/components/agency/configuration/Configuration.jsx+++ /dev/null@@ -1,135 +0,0 @@-import { Button, Space, Typography } from "antd";-import PropTypes from "prop-types";-import { createRef, useEffect, useState } from "react";-import { useParams } from "react-router-dom";--import { handleException } from "../../../helpers/GetStaticData";-import { useAxiosPrivate } from "../../../hooks/useAxiosPrivate";-import { RjsfFormLayout } from "../../../layouts/rjsf-form-layout/RjsfFormLayout";-import { useAlertStore } from "../../../store/alert-store";-import { useSessionStore } from "../../../store/session-store";-import { useWorkflowStore } from "../../../store/workflow-store";-import { CustomButton } from "../../widgets/custom-button/CustomButton";-import "./Configuration.css";--function Configuration({ setOpen }) {- const formRef = createRef(null);- const [schema, setSchema] = useState({});- const [formData, setFormData] = useState({});- const [isSchemaLoading, setSchemaLoading] = useState(false);- const [isUpdateApiLoading, setUpdateApiLoading] = useState(false);- const { id } = useParams();- const axiosPrivate = useAxiosPrivate();- const { sessionDetails } = useSessionStore();- const { projectName } = useWorkflowStore();- const { setAlertDetails } = useAlertStore();-- useEffect(() => {- const requestOptions = {- method: "GET",- url: `/api/v1/unstract/${sessionDetails?.orgId}/workflow/${id}/settings/`,- };-- setSchemaLoading(true);- axiosPrivate(requestOptions)- .then((res) => {- const data = res?.data;- setSchema(data?.schema || {});- setFormData(data?.settings || {});- })- .catch((err) => {- setAlertDetails(handleException(err, "Failed to load the form"));- })- .finally(() => {- setSchemaLoading(false);- });- }, []);-- const isFormValid = () => {- if (formRef) {- formRef?.current?.validateFields((errors, values) => {- if (errors) {- return false;- }- });- }- return true;- };-- const validateAndSubmit = (updatedFormData) => {- if (!isFormValid()) {- return;- }- handleSubmit(updatedFormData);- };-- const handleSubmit = (updatedFormData) => {- setFormData(updatedFormData);- const body = {- workflow_name: projectName,- settings: updatedFormData,- };-- const requestOptions = {- method: "PUT",- url: `/api/v1/unstract/${sessionDetails?.orgId}/workflow/${id}/`,- headers: {- "X-CSRFToken": sessionDetails?.csrfToken,- "Content-Type": "application/json",- },- data: body,- };-- setUpdateApiLoading(true);- axiosPrivate(requestOptions)- .then(() => {- setAlertDetails({- type: "success",- content: "Saved successfully",- });- setOpen(false);- })- .catch((err) => {- setAlertDetails(handleException(err, "Failed to save"));- })- .finally(() => {- setUpdateApiLoading(false);- });- };-- return (- <div>- <div className="config-head">- <Typography.Text className="config-head-typo">- Configuration- </Typography.Text>- </div>- <RjsfFormLayout- schema={schema}- formData={formData}- isLoading={isSchemaLoading}- validateAndSubmit={validateAndSubmit}- formRef={formRef}- >- <div className="display-flex-right">- <Space>- <Button onClick={() => setOpen(false)}>Cancel</Button>- <CustomButton- type="primary"- htmlType="submit"- loading={isUpdateApiLoading}- >- Save- </CustomButton>- </Space>- </div>- </RjsfFormLayout>- </div>- );-}--Configuration.propTypes = {- setOpen: PropTypes.func.isRequired,-};--export { Configuration };diff --git a/frontend/src/layouts/menu-layout/MenuLayout.jsx b/frontend/src/layouts/menu-layout/MenuLayout.jsxindex e2f18f45a..893130fde 100644--- a/frontend/src/layouts/menu-layout/MenuLayout.jsx+++ b/frontend/src/layouts/menu-layout/MenuLayout.jsx@@ -2,23 +2,17 @@ import { useEffect, useState, useRef } from "react";import { useLocation, useNavigate } from "react-router-dom";import PropTypes from "prop-types";import { Button, Space, Typography } from "antd";-import {- ArrowLeftOutlined,- ToolOutlined,- QuestionCircleOutlined,-} from "@ant-design/icons";+import { ArrowLeftOutlined, QuestionCircleOutlined } from "@ant-design/icons";import "./MenuLayout.css";import { useSessionStore } from "../../store/session-store";import { useWorkflowStore } from "../../store/workflow-store";-import { ConfigurationModal } from "../../components/agency/configuration-modal/ConfigurationModal";function MenuLayout({ children }) {const navigate = useNavigate();const location = useLocation();const currentMenu = useRef();const [activeTab, setActiveTab] = useState("");- const [openConfigModal, setOpenConfigModal] = useState(false);const { sessionDetails } = useSessionStore();const { projectName } = useWorkflowStore();@@ -51,14 +45,6 @@ function MenuLayout({ children }) {</div><div><Space>- <Button- onClick={() => setOpenConfigModal((prev) => !prev)}- key="configuration"- icon={<ToolOutlined />}- disabled={false}- >- Configuration- </Button><Buttonkey="help"icon={<QuestionCircleOutlined />}@@ -73,10 +59,6 @@ function MenuLayout({ children }) {<div className="appBody"><div className="appBody2">{children}</div></div>- <ConfigurationModal- isOpen={openConfigModal}- setOpen={setOpenConfigModal}- /></>);}diff --git a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json b/unstract/tool-registry/src/unstract/tool_registry/public_tools.jsonindex d5e04f633..a208ef6cc 100644--- a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json+++ b/unstract/tool-registry/src/unstract/tool_registry/public_tools.json@@ -134,315 +134,6 @@"image_name": "unstract/tool-classifier","image_tag": "0.0.1"},- "translate": {- "tool_uid": "translate",- "properties": {- "schemaVersion": "0.0.1",- "displayName": "Translate",- "functionName": "translate",- "toolVersion": "0.0.1",- "description": "This is a tool which can translate the given text into a target language.",- "input": {- "description": "File to be translated"- },- "output": {- "description": "Creates a file with the translated text"- },- "result": {- "type": "TXT",- "description": "Response containing the translated text"- },- "ioCompatibility": {- "api": {- "sourceSupport": true,- "destinationSupport": true,- "additionalArgs": {- "sync": true- }- },- "file": {- "sourceSupport": true,- "destinationSupport": true,- "additionalArgs": {}- },- "db": {- "destinationSupport": false,- "additionalArgs": {}- }- },- "restrictions": {- "maxFileSize": "10MB",- "allowedFileTypes": [- "*"- ]- }- },- "spec": {- "title": "Language translation tool settings",- "description": "Setup the translation tool",- "type": "object",- "required": [],- "properties": {- "processor": {- "type": "string",- "title": "Processor to use",- "default": "Google Translate",- "enum": [- "Google Translate"- ],- "description": "Service used to translate text"- },- "targetLanguage": {- "type": "string",- "title": "Target language",- "default": "English",- "enum": [- "Albanian",- "Arabic",- "Azerbaijani",- "Bengali",- "Bulgarian",- "Chinese",- "Croatian",- "Czech",- "Danish",- "Dutch",- "English",- "Estonian",- "Finnish",- "French",- "Georgian",- "German",- "Greek",- "Hebrew",- "Hindi",- "Hungarian",- "Icelandic",- "Indonesian",- "Italian",- "Japanese",- "Kazakh",- "Korean",- "Latvian",- "Lithuanian",- "Malay",- "Malayalam",- "Mongolian",- "Norwegian",- "Northern_sami",- "Pashto",- "Persian",- "Polish",- "Portuguese",- "Romanian",- "Russian",- "Serbian",- "Slovak",- "Slovenian",- "Spanish",- "Swedish",- "Tagalog",- "Tamil",- "Thai",- "Turkish",- "Ukrainian",- "Urdu",- "Vietnamese"- ],- "description": "The language to translate to"- },- "useCache": {- "type": "boolean",- "title": "Cache and use cached results",- "default": true,- "description": "Use cached results"- }- }- },- "variables": {- "title": "Runtime Variables",- "description": "Runtime Variables for translate",- "type": "object",- "required": [- "GOOGLE_SERVICE_ACCOUNT"- ],- "properties": {- "GOOGLE_SERVICE_ACCOUNT": {- "type": "string",- "title": "Google Service Account",- "description": "Google Service account"- }- }- },- "icon": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<svg\n height=\"48\"\n viewBox=\"0 0 24 24\"\n width=\"48\"\n fill=\"#000000\"\n version=\"1.1\"\n id=\"svg256\"\n xmlns=\"http://www.w3.org/2000/svg\"\n xmlns:svg=\"http://www.w3.org/2000/svg\">\n <defs\n id=\"defs260\" />\n <path\n d=\"M 0,0 H 24 V 24 H 0 Z\"\n fill=\"none\"\n id=\"path252\" />\n <path\n d=\"m 12.87,15.07 -2.54,-2.51 0.03,-0.03 C 12.1,10.59 13.34,8.36 14.07,6 H 17 V 4 H 10 V 2 H 8 V 4 H 1 V 5.99 H 12.17 C 11.5,7.92 10.44,9.75 9,11.35 8.07,10.32 7.3,9.19 6.69,8 h -2 c 0.73,1.63 1.73,3.17 2.98,4.56 L 2.58,17.58 4,19 9,14 12.11,17.11 Z M 18.5,10 h -2 L 12,22 h 2 l 1.12,-3 h 4.75 L 21,22 h 2 z m -2.62,7 1.62,-4.33 1.62,4.33 z\"\n id=\"path254\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n</svg>\n",- "image_url": "docker:unstract/tool-translate:0.0.1",- "image_name": "unstract/tool-translate",- "image_tag": "0.0.1"- },- "pii_redactor": {- "tool_uid": "pii_redactor",- "properties": {- "schemaVersion": "0.0.1",- "displayName": "PII Redactor",- "functionName": "pii_redactor",- "toolVersion": "0.0.1",- "description": "This is a tool which can redact the PII (Personally Identifiable Information) data or information from a file.",- "input": {- "description": "File whose PII needs to be redacted"- },- "output": {- "description": "Creates a PDF file with PII redacted from it"- },- "result": {- "type": "TXT",- "description": "Response containing the text with redacted PII"- },- "ioCompatibility": {- "api": {- "sourceSupport": true,- "destinationSupport": true,- "additionalArgs": {- "sync": false- }- },- "file": {- "sourceSupport": true,- "destinationSupport": true,- "additionalArgs": {}- },- "db": {- "destinationSupport": false,- "additionalArgs": {}- }- },- "restrictions": {- "maxFileSize": "10MB",- "allowedFileTypes": [- "txt",- "pdf",- "doc",- "docx",- "odt"- ]- }- },- "spec": {- "title": "PII Redactor tool settings",- "description": "Setup the document PII redactor tool",- "type": "object",- "required": [- "processor"- ],- "properties": {- "processor": {- "type": "string",- "title": "Processor to use",- "default": "Amazon Comprehend",- "enum": [- "Amazon Comprehend"- ],- "description": "Service used to detect PII entities"- },- "redactionElements": {- "type": "array",- "title": "PII elements to redact",- "description": "PII elements to redact. Leave it empty to process all elements",- "items": {- "type": "string",- "enum": [- "Address",- "Age",- "Bank Account Number",- "Bank Routing Number",- "Canada Health Number",- "Canada Social Insurance Number",- "Credit Card Number",- "Credit Card CVV",- "Credit Card Expiration Date",- "Date & Time",- "Email",- "Indian Aadhaar Number",- "Indian PAN Number",- "Indian NREGA Number",- "Indian Voter ID Number",- "IP Address",- "MAC Address",- "Name",- "Passport Number",- "Password",- "Phone Number",- "PIN",- "SSN",- "URL",- "UK National Insurance Number",- "UK NHS Number",- "UK Tax ID Number",- "US Individual Taxpayer Identification Number",- "Username",- "VIN"- ]- },- "default": []- },- "useCache": {- "type": "boolean",- "title": "Cache and use cached results",- "default": true,- "description": "Use cached results"- },- "scoreThreshold": {- "type": "number",- "title": "Score Threshold",- "default": 0.8,- "description": "Minimum confidence needed to treat an entity as PII (from 0.0 to 1.0)"- }- }- },- "variables": {- "title": "Runtime Variables",- "description": "Runtime Variables for doc pii redactor",- "type": "object",- "required": [- "PII_REDACT_AWS_REGION",- "PII_REDACT_AWS_ACCESS_KEY_ID",- "PII_REDACT_AWS_SECRET_ACCESS_KEY",- "DOCUMENT_PROCESSOR_URL",- "DOCUMENT_PROCESSOR_API_KEY"- ],- "properties": {- "PII_REDACT_AWS_REGION": {- "type": "string",- "title": "AWS Comprehend's Region",- "description": "AWS Comprehend's region used in Document PII Redactor"- },- "PII_REDACT_AWS_ACCESS_KEY_ID": {- "type": "string",- "title": "AWS Comprehend's Access Key ID",- "description": "AWS Comprehend's access key ID used in Document PII Redactor"- },- "PII_REDACT_AWS_SECRET_ACCESS_KEY": {- "type": "string",- "title": "AWS Comprehend's Secret Access Key",- "format": "password",- "description": "AWS Comprehend's secret access key used in Document PII Redactor"- },- "DOCUMENT_PROCESSOR_URL": {- "type": "string",- "title": "Document Processor URL",- "description": "Document processor service URL"- },- "DOCUMENT_PROCESSOR_API_KEY": {- "type": "string",- "title": "Document Processor API key",- "description": "Document processor service API key"- }- }- },- "icon": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<svg\n enable-background=\"new 0 0 24 24\"\n height=\"48\"\n viewBox=\"0 0 24 24\"\n width=\"48\"\n fill=\"#000000\"\n version=\"1.1\"\n id=\"svg2702\"\n sodipodi:docname=\"no_accounts_black_48dp.svg\"\n xmlns:inkscape=\"http://www.inkscape.org/namespaces/inkscape\"\n xmlns:sodipodi=\"http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd\"\n xmlns=\"http://www.w3.org/2000/svg\"\n xmlns:svg=\"http://www.w3.org/2000/svg\">\n <defs\n id=\"defs2706\" />\n <sodipodi:namedview\n id=\"namedview2704\"\n pagecolor=\"#ffffff\"\n bordercolor=\"#000000\"\n borderopacity=\"0.25\"\n inkscape:showpageshadow=\"2\"\n inkscape:pageopacity=\"0.0\"\n inkscape:pagecheckerboard=\"0\"\n inkscape:deskcolor=\"#d1d1d1\"\n showgrid=\"false\" />\n <g\n id=\"g2692\">\n <rect\n fill=\"none\"\n height=\"24\"\n width=\"24\"\n id=\"rect2690\"\n x=\"0\"\n y=\"0\" />\n </g>\n <g\n id=\"g2700\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <g\n id=\"g2698\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <path\n d=\"M 15.18,10.94 C 15.38,10.5 15.5,10.02 15.5,9.5 15.5,7.57 13.93,6 12,6 11.48,6 11,6.12 10.56,6.32 Z\"\n id=\"path2694\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n <path\n d=\"M 12,2 C 6.48,2 2,6.48 2,12 2,17.52 6.48,22 12,22 17.52,22 22,17.52 22,12 22,6.48 17.52,2 12,2 Z M 4,12 C 4,10.15 4.63,8.45 5.69,7.1 l 2.86,2.86 c 0.21,1.56 1.43,2.79 2.99,2.99 l 2.2,2.2 C 13.17,15.05 12.59,15 12,15 9.68,15 7.55,15.8 5.86,17.12 4.7,15.73 4,13.95 4,12 Z m 8,8 C 10.26,20 8.66,19.44 7.35,18.5 8.66,17.56 10.26,17 12,17 c 1.74,0 3.34,0.56 4.65,1.5 C 15.34,19.44 13.74,20 12,20 Z M 18.31,16.9 7.1,5.69 C 8.45,4.63 10.15,4 12,4 c 4.42,0 8,3.58 8,8 0,1.85 -0.63,3.54 -1.69,4.9 z\"\n id=\"path2696\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n </g>\n </g>\n</svg>\n",- "image_url": "docker:unstract/tool-doc-pii-redactor:0.0.1",- "image_name": "unstract/tool-doc-pii-redactor",- "image_tag": "0.0.1"- },"document_indexer": {"tool_uid": "document_indexer","properties": {@@ -647,4 +338,4 @@"image_name": "unstract/tool-text-extractor","image_tag": "0.0.1"}-}\ No newline at end of file+}diff --git a/unstract/tool-registry/src/unstract/tool_registry/tool_registry.py b/unstract/tool-registry/src/unstract/tool_registry/tool_registry.pyindex f81235300..dcb6d233b 100644--- a/unstract/tool-registry/src/unstract/tool_registry/tool_registry.py+++ b/unstract/tool-registry/src/unstract/tool_registry/tool_registry.py@@ -333,15 +333,6 @@ class ToolRegistry:is_ready: bool = self.helper.is_image_ready(image_name=image_name)return is_ready- def get_project_settings_schema(self) -> dict[str, Any]:- """Project tool's environmental settings schema.-- Returns:- dict[str, Any]: schema- """- schema: dict[str, Any] = ToolUtils.get_tool_project_schema()- return schema-def validate_schema_with_data(self, schema: dict[str, Any], data: dict[str, Any]) -> None:diff --git a/unstract/tool-registry/src/unstract/tool_registry/tool_utils.py b/unstract/tool-registry/src/unstract/tool_registry/tool_utils.pyindex 22114ecf3..c8b98586d 100644--- a/unstract/tool-registry/src/unstract/tool_registry/tool_utils.py+++ b/unstract/tool-registry/src/unstract/tool_registry/tool_utils.py@@ -248,9 +248,7 @@ class ToolUtils:text_extractors = ToolUtils.get_enabled_adapters(adapter.text_extractors)- ocrs = ToolUtils.get_enabled_adapters(- adapter.ocrs- )+ ocrs = ToolUtils.get_enabled_adapters(adapter.ocrs)ToolUtils.process_adapter_models(models=language_models,@@ -281,43 +279,3 @@ class ToolUtils:schema=schema,)return schema-- @staticmethod- def get_tool_project_schema() -> dict[str, Any]:- return {- "title": "Project Settings",- "description": "Project Settings for tools",- "type": "object",- "properties": {- "GOOGLE_SERVICE_ACCOUNT": {- "type": "string",- "title": "Google Service Account",- "format": "password",- "description": (- "Google Service account for OCR and Translate tools"- ),- },- "GOOGLE_DOCUMENT_AI_ENDPOINT": {- "type": "string",- "title": "Google Document API Base URL",- "description": "The base URL for the Google Document service used for OCR", # noqa: E501- },- # Used by the PII redactor tool- "PII_REDACT_AWS_REGION": {- "type": "string",- "title": "AWS Comprehend's Region",- "description": "AWS Comprehend's region used in Document PII Redactor", # noqa: E501- },- "PII_REDACT_AWS_ACCESS_KEY_ID": {- "type": "string",- "title": "AWS Comprehend's Access Key ID",- "description": "AWS Comprehend's access key ID used in Document PII Redactor", # noqa: E501- },- "PII_REDACT_AWS_SECRET_ACCESS_KEY": {- "type": "string",- "title": "AWS Comprehend's Secret Access Key",- "format": "password",- "description": "AWS Comprehend's secret access key used in Document PII Redactor", # noqa: E501- },- },- }diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/dto.py b/unstract/workflow-execution/src/unstract/workflow_execution/dto.pyindex b4d91b9a5..0324c7e66 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/dto.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/dto.py@@ -38,4 +38,3 @@ class ToolInstance:@dataclassclass WorkflowDto:id: str- settings: dict[str, Any]diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.pyindex d3f2988af..008df896a 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py@@ -33,7 +33,6 @@ class ToolsUtils:self.organization_id = organization_idself.platform_service_api_key = platform_service_api_keyself.workflow_id = workflow.id- self.project_settings = workflow.settingsself.ignore_processed_entities = ignore_processed_entitiesself.messaging_channel: Optional[str] = Noneself.platform_service_host = ToolsUtils.get_env(@@ -117,9 +116,7 @@ class ToolsUtils:),)- tool_envs = self.get_tool_environment_variables(- self.project_settings- )+ tool_envs = self.get_tool_environment_variables()logger.info(f"Tool Environments are collected for tool {tool_uid}")image_name = tool_instance.image_nameimage_tag = tool_instance.image_tag@@ -203,7 +200,7 @@ class ToolsUtils:return Nonedef get_tool_environment_variables(- self, project_settings: Optional[dict[str, Any]]+ self, project_settings: Optional[dict[str, Any]] = None) -> dict[str, Any]:"""Obtain a dictionary of env variables required by a tool.