LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
v2 changes of Filemanagement And Scheduler (#466)
ali99de0623475d
diff --git a/backend/file_management/file_management_helper.py b/backend/file_management/file_management_helper.pyindex 0c13812b4..6293f2583 100644--- a/backend/file_management/file_management_helper.py+++ b/backend/file_management/file_management_helper.py@@ -6,7 +6,6 @@ from pathlib import Pathfrom typing import Anyimport magic-from connector.models import ConnectorInstancefrom django.conf import settingsfrom django.http import StreamingHttpResponsefrom file_management.exceptions import (@@ -24,8 +23,15 @@ from file_management.file_management_dto import FileInformationfrom fsspec import AbstractFileSystemfrom pydrive2.files import ApiRequestError+from backend.constants import FeatureFlagfrom unstract.connectors.filesystems import connectors as fs_connectorsfrom unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem+from unstract.flags.feature_flag import check_feature_flag_status++if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):+ from connector_v2.models import ConnectorInstance+else:+ from connector.models import ConnectorInstanceclass FileManagerHelper:diff --git a/backend/file_management/views.py b/backend/file_management/views.pyindex 7ff72a777..3ecfd0904 100644--- a/backend/file_management/views.py+++ b/backend/file_management/views.py@@ -1,7 +1,6 @@import loggingfrom typing import Any-from connector.models import ConnectorInstancefrom django.http import HttpRequestfrom file_management.exceptions import (ConnectorInstanceNotFound,@@ -17,15 +16,24 @@ from file_management.serializer import (FileUploadSerializer,)from oauth2client.client import HttpAccessTokenRefreshError-from prompt_studio.prompt_studio_document_manager.models import DocumentManagerfrom rest_framework import serializers, status, viewsetsfrom rest_framework.decorators import actionfrom rest_framework.response import Responsefrom rest_framework.versioning import URLPathVersioningfrom utils.user_session import UserSessionUtils+from backend.constants import FeatureFlagfrom unstract.connectors.exceptions import ConnectorErrorfrom unstract.connectors.filesystems.local_storage.local_storage import LocalStorageFS+from unstract.flags.feature_flag import check_feature_flag_status++if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):+ from connector_v2.models import ConnectorInstance+ from prompt_studio.prompt_studio_document_manager_v2.models import DocumentManager++else:+ from connector.models import ConnectorInstance+ from prompt_studio.prompt_studio_document_manager.models import DocumentManagerlogger = logging.getLogger(__name__)diff --git a/backend/scheduler/helper.py b/backend/scheduler/helper.pyindex 30c479b61..7cc30e07a 100644--- a/backend/scheduler/helper.py+++ b/backend/scheduler/helper.py@@ -3,8 +3,6 @@ import uuidfrom typing import Anyfrom django.db import connection-from pipeline.models import Pipeline-from pipeline.pipeline_processor import PipelineProcessorfrom rest_framework.serializers import ValidationErrorfrom scheduler.constants import SchedulerConstants as SCfrom scheduler.exceptions import JobDeletionError, JobSchedulingError@@ -15,9 +13,21 @@ from scheduler.tasks import (disable_task,enable_task,)-from workflow_manager.workflow.constants import WorkflowExecutionKey, WorkflowKey-from workflow_manager.workflow.serializers import ExecuteWorkflowSerializer+from backend.constants import FeatureFlag+from unstract.flags.feature_flag import check_feature_flag_status++if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):+ from pipeline_v2.models import Pipeline+ from pipeline_v2.pipeline_processor import PipelineProcessor+ from utils.user_context import UserContext+ from workflow_manager.workflow_v2.constants import WorkflowExecutionKey, WorkflowKey+ from workflow_manager.workflow_v2.serializers import ExecuteWorkflowSerializer+else:+ from pipeline.models import Pipeline+ from pipeline.pipeline_processor import PipelineProcessor+ from workflow_manager.workflow.constants import WorkflowExecutionKey, WorkflowKey+ from workflow_manager.workflow.serializers import ExecuteWorkflowSerializerlogger = logging.getLogger(__name__)@@ -46,7 +56,10 @@ class SchedulerHelper:workflow_id = serializer.get_workflow_id(serializer.validated_data)# TODO: Remove unused argument in execute_pipeline_taskexecution_action = serializer.get_execution_action(serializer.validated_data)- org_schema = connection.tenant.schema_name+ if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):+ organization_id = UserContext.get_organization_identifier()+ else:+ organization_id = connection.tenant.schema_namecreate_periodic_task(cron_string=cron_string,@@ -54,7 +67,7 @@ class SchedulerHelper:task_path="scheduler.tasks.execute_pipeline_task",task_args=[str(workflow_id),- org_schema,+ organization_id,execution_action or "",execution_id,str(pipeline.pk),diff --git a/backend/scheduler/serializer.py b/backend/scheduler/serializer.pyindex ef19de11f..65c2c1910 100644--- a/backend/scheduler/serializer.py+++ b/backend/scheduler/serializer.py@@ -1,11 +1,17 @@import loggingfrom typing import Any-from pipeline.manager import PipelineManagerfrom rest_framework import serializersfrom scheduler.constants import SchedulerConstants as SC+from backend.constants import FeatureFlagfrom backend.constants import FieldLengthConstants as FieldLength+from unstract.flags.feature_flag import check_feature_flag_status++if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):+ from pipeline_v2.manager import PipelineManager+else:+ from pipeline.manager import PipelineManagerlogger = logging.getLogger(__name__)diff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.pyindex dbc6f6101..abbc7cf91 100644--- a/backend/scheduler/tasks.py+++ b/backend/scheduler/tasks.py@@ -2,15 +2,28 @@ import jsonimport loggingfrom typing import Any-from account.models import Organization-from account.subscription_loader import load_plugins, validate_etl_runfrom celery import shared_taskfrom django_celery_beat.models import CrontabSchedule, PeriodicTaskfrom django_tenants.utils import get_tenant_model, tenant_context-from pipeline.models import Pipeline-from pipeline.pipeline_processor import PipelineProcessor-from workflow_manager.workflow.models.workflow import Workflow-from workflow_manager.workflow.workflow_helper import WorkflowHelper++from backend.constants import FeatureFlag+from unstract.flags.feature_flag import check_feature_flag_status++if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):+ from account_v2.subscription_loader import load_plugins, validate_etl_run+ from pipeline_v2.models import Pipeline+ from pipeline_v2.pipeline_processor import PipelineProcessor+ from utils.user_context import UserContext+ from workflow_manager.workflow_v2.models.workflow import Workflow+ from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper+else:+ from account.models import Organization+ from account.subscription_loader import load_plugins, validate_etl_run+ from pipeline.models import Pipeline+ from pipeline.pipeline_processor import PipelineProcessor+ from workflow_manager.workflow.models.workflow import Workflow+ from workflow_manager.workflow.workflow_helper import WorkflowHelper+logger = logging.getLogger(__name__)subscription_loader = load_plugins()@@ -58,6 +71,15 @@ def execute_pipeline_task(with_logs: Any,name: Any,) -> None:+ if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):+ execute_pipeline_task_v2(+ workflow_id=workflow_id,+ organization_id=org_schema,+ execution_id=execution_id,+ pipeline_id=pipepline_id,+ pipeline_name=name,+ )+ returnlogger.info(f"Executing pipeline name: {name}")try:logger.info(f"Executing workflow id: {workflow_id}")@@ -92,6 +114,61 @@ def execute_pipeline_task(logger.error(f"Failed to execute pipeline: {name}. Error: {e}")+def execute_pipeline_task_v2(+ workflow_id: Any,+ organization_id: Any,+ execution_id: Any,+ pipeline_id: Any,+ pipeline_name: Any,+) -> None:+ """V2 of execute_pipeline method.++ Args:+ workflow_id (Any): UID of workflow entity+ org_schema (Any): Organization Identifier+ execution_id (Any): UID of execution entity+ pipeline_id (Any): UID of pipeline entity+ name (Any): pipeline name+ """+ try:+ logger.info(+ f"Executing workflow id: {workflow_id} for pipeline {pipeline_name}"+ )+ # Set organization in state store for execution+ UserContext.set_organization_identifier(organization_id)+ if (+ subscription_loader+ and subscription_loader[0]+ and not validate_etl_run(organization_id)+ ):+ try:+ logger.info(f"Disabling ETL task: {pipeline_id}")+ disable_task(pipeline_id)+ except Exception as e:+ logger.warning(f"Failed to disable task: {pipeline_id}. Error: {e}")+ return+ workflow = WorkflowHelper.get_workflow_by_id(+ id=workflow_id, organization_id=organization_id+ )+ logger.info(f"Executing workflow: {workflow}")+ PipelineProcessor.update_pipeline(+ pipeline_id, Pipeline.PipelineStatus.INPROGRESS+ )+ execution_response = WorkflowHelper.complete_execution(+ workflow, execution_id, pipeline_id+ )+ logger.info(+ f"Execution response for pipeline {pipeline_name} of organization "+ f"{organization_id}: {execution_response}"+ )+ logger.info(+ f"Execution completed for pipeline {pipeline_name} of organization: "+ f"{organization_id}"+ )+ except Exception as e:+ logger.error(f"Failed to execute pipeline: {pipeline_name}. Error: {e}")++def delete_periodic_task(task_name: str) -> None:try:task = PeriodicTask.objects.get(name=task_name)