Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

v2 changes of workflow with endpoints (#464)

ali298f4f48a549
diff --git a/backend/workflow_manager/endpoint_v2/__init__ .py b/backend/workflow_manager/endpoint_v2/__init__ .py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/workflow_manager/endpoint_v2/apps.py b/backend/workflow_manager/endpoint_v2/apps.py
new file mode 100644
index 000000000..bf6010bf0
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/apps.py
@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class WorkflowEndpointConfig(AppConfig):
+ default_auto_field = "django.db.models.BigAutoField"
+ name = "workflow_manager.endpoint_v2"
diff --git a/backend/workflow_manager/endpoint_v2/base_connector.py b/backend/workflow_manager/endpoint_v2/base_connector.py
new file mode 100644
index 000000000..3207537e6
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/base_connector.py
@@ -0,0 +1,91 @@
+import json
+from typing import Any
+
+from django.conf import settings
+from fsspec import AbstractFileSystem
+from unstract.workflow_execution.execution_file_handler import ExecutionFileHandler
+from utils.constants import Common
+from utils.user_context import UserContext
+
+from unstract.connectors.filesystems import connectors
+from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
+
+
+class BaseConnector(ExecutionFileHandler):
+ """Base class for connectors providing common methods and utilities."""
+
+ def __init__(
+ self, workflow_id: str, execution_id: str, organization_id: str
+ ) -> None:
+ """Initialize the BaseConnector class.
+
+ This class serves as a base for connectors and provides common
+ utilities.
+ """
+ if not (settings.API_STORAGE_DIR and settings.WORKFLOW_DATA_DIR):
+ raise ValueError("Missed env API_STORAGE_DIR or WORKFLOW_DATA_DIR")
+ super().__init__(workflow_id, execution_id, organization_id)
+ # Directory path for storing execution-related files for API
+ self.api_storage_dir: str = self.create_execution_dir_path(
+ workflow_id, execution_id, organization_id, settings.API_STORAGE_DIR
+ )
+
+ def get_fsspec(
+ self, settings: dict[str, Any], connector_id: str
+ ) -> AbstractFileSystem:
+ """Get an fsspec file system based on the specified connector.
+
+ Parameters:
+ - settings (dict): Connector-specific settings.
+ - connector_id (str): Identifier for the desired connector.
+
+ Returns:
+ AbstractFileSystem: An fsspec file system instance.
+
+ Raises:
+ KeyError: If the connector_id is not found in the connectors dictionary.
+ """
+ if connector_id not in connectors:
+ raise ValueError(f"Invalid connector_id: {connector_id}")
+ connector = connectors[connector_id][Common.METADATA][Common.CONNECTOR]
+ connector_class: UnstractFileSystem = connector(settings)
+ return connector_class.get_fsspec_fs()
+
+ @classmethod
+ def get_json_schema(cls, file_path: str) -> dict[str, Any]:
+ """Load and return a JSON schema from the specified file path.
+
+ Parameters:
+ - file_path (str): The path to the JSON schema file.
+
+ Returns:
+ dict: The loaded JSON schema.
+
+ Raises:
+ json.JSONDecodeError: If there is an issue decoding the JSON file.
+ """
+ try:
+ with open(file_path, encoding="utf-8") as file:
+ schema: dict[str, Any] = json.load(file)
+ except OSError:
+ schema = {}
+ return schema
+
+ @classmethod
+ def get_api_storage_dir_path(cls, workflow_id: str, execution_id: str) -> str:
+ """Get the directory path for storing api files.
+
+ Parameters:
+ - workflow_id (str): Identifier for the workflow.
+ - execution_id (str): Identifier for the execution.
+ - organization_id (Optional[str]): Identifier for the organization
+ (default: None).
+
+ Returns:
+ str: The directory path for the execution.
+ """
+ organization_id = UserContext.get_organization_identifier()
+ api_storage_dir: str = cls.create_execution_dir_path(
+ workflow_id, execution_id, organization_id, settings.API_STORAGE_DIR
+ )
+ return api_storage_dir
diff --git a/backend/workflow_manager/endpoint_v2/constants.py b/backend/workflow_manager/endpoint_v2/constants.py
new file mode 100644
index 000000000..d9553245d
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/constants.py
@@ -0,0 +1,105 @@
+class TableColumns:
+ CREATED_BY = "created_by"
+ CREATED_AT = "created_at"
+ PERMANENT_COLUMNS = ["created_by", "created_at"]
+
+
+class DBConnectionClass:
+ SNOWFLAKE = "SnowflakeDB"
+ BIGQUERY = "BigQuery"
+ MSSQL = "MSSQL"
+
+
+class Snowflake:
+ COLUMN_TYPES = [
+ "VARCHAR",
+ "CHAR",
+ "CHARACTER",
+ "STRING",
+ "TEXT",
+ "BINARY",
+ "VARBINARY",
+ "DATE",
+ "DATETIME",
+ "TIME",
+ "TIMESTAMP",
+ "TIMESTAMP_LTZ",
+ "TIMESTAMP_NTZ",
+ "TIMESTAMP_TZ",
+ "BOOLEAN",
+ ]
+
+
+class FileSystemConnector:
+ MAX_FILES = 100
+
+
+class WorkflowFileType:
+ SOURCE = "SOURCE"
+ INFILE = "INFILE"
+ METADATA_JSON = "METADATA.json"
+
+
+class SourceKey:
+ FILE_EXTENSIONS = "fileExtensions"
+ PROCESS_SUB_DIRECTORIES = "processSubDirectories"
+ MAX_FILES = "maxFiles"
+ ROOT_FOLDER = "rootFolder"
+
+
+class DestinationKey:
+ TABLE = "table"
+ INCLUDE_AGENT = "includeAgent"
+ INCLUDE_TIMESTAMP = "includeTimestamp"
+ AGENT_NAME = "agentName"
+ COLUMN_MODE = "columnMode"
+ SINGLE_COLUMN_NAME = "singleColumnName"
+ PATH = "path"
+ OUTPUT_FOLDER = "outputFolder"
+ OVERWRITE_OUTPUT_DOCUMENT = "overwriteOutput"
+
+
+class OutputJsonKey:
+ JSON_RESULT_KEY = "result"
+
+
+class FileType:
+ PDF_DOCUMENTS = "PDF documents"
+ TEXT_DOCUMENTS = "Text documents"
+ IMAGES = "Images"
+
+
+class FilePattern:
+ PDF_DOCUMENTS = ["*.pdf"]
+ TEXT_DOCUMENTS = ["*.txt"]
+ IMAGES = ["*.jpg", "*.jpeg", "*.png", "*.gif", "*.bmp"]
+
+
+class SourceConstant:
+ MAX_RECURSIVE_DEPTH = 10
+
+
+class ApiDeploymentResultStatus:
+ SUCCESS = "Success"
+ FAILED = "Failed"
+
+
+class BigQuery:
+ """In big query, table name has to be in the format {db}.{schema}.{table}
+ Throws error if any of the params not set.
+
+ When converted to list table size should be 3
+ """
+
+ TABLE_NAME_SIZE = 3
+ COLUMN_TYPES = [
+ "DATE",
+ "DATETIME",
+ "TIME",
+ "TIMESTAMP",
+ ]
+
+
+class QueueResultStatus:
+ SUCCESS = "Success"
+ FAILED = "Failed"
diff --git a/backend/workflow_manager/endpoint_v2/database_utils.py b/backend/workflow_manager/endpoint_v2/database_utils.py
new file mode 100644
index 000000000..00b9fbf73
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/database_utils.py
@@ -0,0 +1,350 @@
+import datetime
+import json
+import logging
+import uuid
+from typing import Any, Optional
+
+from utils.constants import Common
+from workflow_manager.endpoint_v2.constants import (
+ BigQuery,
+ DBConnectionClass,
+ TableColumns,
+)
+from workflow_manager.endpoint_v2.db_connector_helper import DBConnectorQueryHelper
+from workflow_manager.endpoint_v2.exceptions import (
+ BigQueryTableNotFound,
+ UnstractDBException,
+)
+from workflow_manager.workflow_v2.enums import AgentName, ColumnModes
+
+from unstract.connectors.databases import connectors as db_connectors
+from unstract.connectors.databases.exceptions import UnstractDBConnectorException
+from unstract.connectors.databases.unstract_db import UnstractDB
+from unstract.connectors.exceptions import ConnectorError
+
+logger = logging.getLogger(__name__)
+
+
+class DatabaseUtils:
+ @staticmethod
+ def get_sql_values_for_query(
+ values: dict[str, Any], column_types: dict[str, str], cls_name: str
+ ) -> dict[str, str]:
+ """Making Sql Columns and Values for Query.
+
+ Args:
+ values (dict[str, Any]): dictionary of columns and values
+ column_types (dict[str,str]): types of columns
+ cls (Any, optional): The database connection class (e.g.,
+ DBConnectionClass.SNOWFLAKE) for handling database-specific
+ queries.
+ Defaults to None.
+
+ Returns:
+ list[str]: _description_
+
+ Note:
+ - If `cls` is not provided or is None, the function assumes a
+ Default SQL database and makes values accordingly.
+ - If `cls` is provided and matches DBConnectionClass.SNOWFLAKE,
+ the function makes values using Snowflake-specific syntax.
+
+ - Unstract creates id by default if table not exists.
+ If there is column 'id' in db table, it will insert
+ 'id' as uuid into the db table.
+ Else it will GET table details from INFORMATION SCHEMA and
+ insert into the table accordingly
+ """
+ sql_values: dict[str, Any] = {}
+ for column in values:
+ if cls_name == DBConnectionClass.SNOWFLAKE:
+ col = column.lower()
+ type_x = column_types[col]
+ if type_x == "VARIANT":
+ values[column] = values[column].replace("'", "\\'")
+ sql_values[column] = f"parse_json($${values[column]}$$)"
+ else:
+ sql_values[column] = f"{values[column]}"
+ else:
+ # Default to Other SQL DBs
+ # TODO: Handle numeric types with no quotes
+ sql_values[column] = f"{values[column]}"
+ if column_types.get("id"):
+ uuid_id = str(uuid.uuid4())
+ sql_values["id"] = f"{uuid_id}"
+ return sql_values
+
+ @staticmethod
+ def get_column_types_util(columns_with_types: Any) -> dict[str, str]:
+ """Converts db results columns_with_types to dict.
+
+ Args:
+ columns_with_types (Any): _description_
+
+ Returns:
+ dict[str, str]: _description_
+ """
+ column_types: dict[str, str] = {}
+ for column_name, data_type in columns_with_types:
+ column_types[column_name] = data_type
+ return column_types
+
+ @staticmethod
+ def get_column_types(
+ cls_name: Any,
+ table_name: str,
+ connector_id: str,
+ connector_settings: dict[str, Any],
+ ) -> Any:
+ """Get db column name and types.
+
+ Args:
+ cls (Any): _description_
+ table_name (str): _description_
+ connector_id (str): _description_
+ connector_settings (dict[str, Any]): _description_
+
+ Raises:
+ ValueError: _description_
+ e: _description_
+
+ Returns:
+ Any: _description_
+ """
+ column_types: dict[str, str] = {}
+ try:
+ if cls_name == DBConnectionClass.SNOWFLAKE:
+ query = f"describe table {table_name}"
+ results = DatabaseUtils.execute_and_fetch_data(
+ connector_id=connector_id,
+ connector_settings=connector_settings,
+ query=query,
+ )
+ for column in results:
+ column_types[column[0].lower()] = column[1].split("(")[0]
+ elif cls_name == DBConnectionClass.BIGQUERY:
+ bigquery_table_name = str.lower(table_name).split(".")
+ if len(bigquery_table_name) != BigQuery.TABLE_NAME_SIZE:
+ raise BigQueryTableNotFound()
+ database = bigquery_table_name[0]
+ schema = bigquery_table_name[1]
+ table = bigquery_table_name[2]
+ query = (
+ "SELECT column_name, data_type FROM "
+ f"{database}.{schema}.INFORMATION_SCHEMA.COLUMNS WHERE "
+ f"table_name = '{table}'"
+ )
+ results = DatabaseUtils.execute_and_fetch_data(
+ connector_id=connector_id,
+ connector_settings=connector_settings,
+ query=query,
+ )
+ column_types = DatabaseUtils.get_column_types_util(results)
+ else:
+ table_name = str.lower(table_name)
+ query = (
+ "SELECT column_name, data_type FROM "
+ "information_schema.columns WHERE "
+ f"table_name = '{table_name}'"
+ )
+ results = DatabaseUtils.execute_and_fetch_data(
+ connector_id=connector_id,
+ connector_settings=connector_settings,
+ query=query,
+ )
+ column_types = DatabaseUtils.get_column_types_util(results)
+ except Exception as e:
+ logger.error(f"Error getting column types for {table_name}: {str(e)}")
+ raise e
+ return column_types
+
+ @staticmethod
+ def get_columns_and_values(
+ column_mode_str: str,
+ data: Any,
+ include_timestamp: bool = False,
+ include_agent: bool = False,
+ agent_name: Optional[str] = AgentName.UNSTRACT_DBWRITER.value,
+ single_column_name: str = "data",
+ ) -> dict[str, Any]:
+ """Generate a dictionary of columns and values based on specified
+ parameters.
+
+ Args:
+ column_mode_str (str): The string representation of the column mode,
+ which determines how data is stored in the dictionary.
+ data (Any): The data to be stored in the dictionary.
+ include_timestamp (bool, optional): Whether to include the
+ current timestamp in the dictionary. Defaults to False.
+ include_agent (bool, optional): Whether to include the agent's name
+ in the dictionary. Defaults to False.
+ agent_name (str, optional): The name of the agent when include_agent
+ is true. Defaults to AgentName.UNSTRACT_DBWRITER.
+ single_column_name (str, optional): The name of the single column
+ when using 'WRITE_JSON_TO_A_SINGLE_COLUMN' mode.
+ Defaults to "data".
+
+ Returns:
+ dict: A dictionary containing columns and values based on
+ the specified parameters.
+ """
+
+ values: dict[str, Any] = {}
+ try:
+ column_mode = ColumnModes(column_mode_str)
+ except ValueError:
+ # Handle the case where the string is not a valid enum value
+ column_mode = ColumnModes.WRITE_JSON_TO_A_SINGLE_COLUMN
+
+ if include_agent and agent_name:
+ values[TableColumns.CREATED_BY] = agent_name
+
+ if include_timestamp:
+ values[TableColumns.CREATED_AT] = datetime.datetime.now()
+
+ if column_mode == ColumnModes.WRITE_JSON_TO_A_SINGLE_COLUMN:
+ if isinstance(data, str):
+ values[single_column_name] = data
+ else:
+ values[single_column_name] = json.dumps(data)
+ if column_mode == ColumnModes.SPLIT_JSON_INTO_COLUMNS:
+ if isinstance(data, dict):
+ values.update(data)
+ elif isinstance(data, str):
+ values[single_column_name] = data
+ else:
+ values[single_column_name] = json.dumps(data)
+
+ return values
+
+ @staticmethod
+ def get_sql_query_data(
+ cls_name: str,
+ connector_id: str,
+ connector_settings: dict[str, Any],
+ table_name: str,
+ values: dict[str, Any],
+ ) -> dict[str, Any]:
+ """Generate SQL columns and values for an insert query based on the
+ provided values and table schema.
+
+ Args:
+ connector_id: The connector id of the connector provided
+ connector_settings: Connector settings provided by user
+ table_name (str): The name of the target table for the insert query.
+ values (dict[str, Any]): A dictionary containing column-value pairs
+ for the insert query.
+
+ Returns:
+ list[str]: A list of SQL values suitable for use in an insert query.
+
+ Note:
+ - This function determines the database type based on the
+ `engine` parameter.
+ - If the database is Snowflake (DBConnectionClass.SNOWFLAKE),
+ it handles Snowflake-specific SQL generation.
+ - For other SQL databases, it uses default SQL generation
+ based on column types.
+ """
+ column_types: dict[str, str] = DatabaseUtils.get_column_types(
+ cls_name=cls_name,
+ table_name=table_name,
+ connector_id=connector_id,
+ connector_settings=connector_settings,
+ )
+ sql_columns_and_values = DatabaseUtils.get_sql_values_for_query(
+ values=values,
+ column_types=column_types,
+ cls_name=cls_name,
+ )
+ return sql_columns_and_values
+
+ @staticmethod
+ def execute_write_query(
+ db_class: UnstractDB,
+ engine: Any,
+ table_name: str,
+ sql_keys: list[str],
+ sql_values: Any,
+ ) -> None:
+ """Execute Insert Query.
+
+ Args:
+ engine (Any): _description_
+ table_name (str): table name
+ sql_keys (list[str]): columns
+ sql_values (list[str]): values
+ Notes:
+ - Snowflake does not support INSERT INTO ... VALUES ...
+ syntax when VARIANT columns are present (JSON).
+ So we need to use INSERT INTO ... SELECT ... syntax
+ - sql values can contain data with single quote. It needs to
+ """
+ cls_name = db_class.__class__.__name__
+ sql = DBConnectorQueryHelper.build_sql_insert_query(
+ cls_name=cls_name, table_name=table_name, sql_keys=sql_keys
+ )
+ logger.debug(f"inserting into table {table_name} with: {sql} query")
+
+ sql_values = DBConnectorQueryHelper.prepare_sql_values(
+ cls_name=cls_name, sql_values=sql_values, sql_keys=sql_keys
+ )
+ logger.debug(f"sql_values: {sql_values}")
+
+ try:
+ db_class.execute_query(
+ engine=engine,
+ sql_query=sql,
+ sql_values=sql_values,
+ table_name=table_name,
+ sql_keys=sql_keys,
+ )
+ except UnstractDBConnectorException as e:
+ raise UnstractDBException(detail=e.detail) from e
+ logger.debug(f"sucessfully inserted into table {table_name} with: {sql} query")
+
+ @staticmethod
+ def get_db_class(
+ connector_id: str, connector_settings: dict[str, Any]
+ ) -> UnstractDB:
+ connector = db_connectors[connector_id][Common.METADATA][Common.CONNECTOR]
+ connector_class: UnstractDB = connector(connector_settings)
+ return connector_class
+
+ @staticmethod
+ def execute_and_fetch_data(
+ connector_id: str, connector_settings: dict[str, Any], query: str
+ ) -> Any:
+ connector = db_connectors[connector_id][Common.METADATA][Common.CONNECTOR]
+ connector_class: UnstractDB = connector(connector_settings)
+ try:
+ return connector_class.execute(query=query)
+ except ConnectorError as e:
+ raise UnstractDBException(detail=e.message) from e
+
+ @staticmethod
+ def create_table_if_not_exists(
+ db_class: UnstractDB,
+ engine: Any,
+ table_name: str,
+ database_entry: dict[str, Any],
+ ) -> None:
+ """Creates table if not exists.
+
+ Args:
+ class_name (UnstractDB): Type of Unstract DB connector
+ table_name (str): _description_
+ database_entry (dict[str, Any]): _description_
+
+ Raises:
+ e: _description_
+ """
+ sql = DBConnectorQueryHelper.create_table_query(
+ conn_cls=db_class, table=table_name, database_entry=database_entry
+ )
+ logger.debug(f"creating table {table_name} with: {sql} query")
+ try:
+ db_class.execute_query(engine=engine, sql_query=sql, sql_values=None)
+ except UnstractDBConnectorException as e:
+ raise UnstractDBException(detail=e.detail) from e
+ logger.debug(f"successfully created table {table_name} with: {sql} query")
diff --git a/backend/workflow_manager/endpoint_v2/db_connector_helper.py b/backend/workflow_manager/endpoint_v2/db_connector_helper.py
new file mode 100644
index 000000000..6c1e92de3
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/db_connector_helper.py
@@ -0,0 +1,77 @@
+from typing import Any
+
+from google.cloud import bigquery
+from workflow_manager.endpoint_v2.constants import DBConnectionClass, TableColumns
+
+from unstract.connectors.databases.unstract_db import UnstractDB
+
+
+class DBConnectorQueryHelper:
+ """A class that helps to generate query for connector table operations."""
+
+ @staticmethod
+ def create_table_query(
+ conn_cls: UnstractDB, table: str, database_entry: dict[str, Any]
+ ) -> Any:
+ sql_query = ""
+ """Generate a SQL query to create a table, based on the provided
+ database entry.
+
+ Args:
+ conn_cls (str): The database connector class.
+ Should be one of 'BIGQUERY', 'SNOWFLAKE', or other.
+ table (str): The name of the table to be created.
+ database_entry (dict[str, Any]):
+ A dictionary containing column names as keys
+ and their corresponding values.
+
+ These values are used to determine the data types,
+ for the columns in the table.
+
+ Returns:
+ str: A SQL query string to create a table with the specified name,
+ and column definitions.
+
+ Note:
+ - Each conn_cls have it's implementation for SQL create table query
+ Based on the implementation, a base SQL create table query will be
+ created containing Permanent columns
+ - Each conn_cls also has a mapping to convert python datatype to
+ corresponding column type (string, VARCHAR etc)
+ - keys in database_entry will be converted to column type, and
+ column values will be the valus in database_entry
+ - base SQL create table will be appended based column type and
+ values, and generates a complete SQL create table query
+ """
+ create_table_query = conn_cls.get_create_table_query(table=table)
+ sql_query += create_table_query
+
+ for key, val in database_entry.items():
+ if key not in TableColumns.PERMANENT_COLUMNS:
+ sql_type = conn_cls.sql_to_db_mapping(val)
+ sql_query += f"{key} {sql_type}, "
+
+ return sql_query.rstrip(", ") + ");"
+
+ @staticmethod
+ def build_sql_insert_query(
+ cls_name: str, table_name: str, sql_keys: list[str]
+ ) -> str:
+ keys_str = ",".join(sql_keys)
+ if cls_name == DBConnectionClass.BIGQUERY:
+ values_placeholder = ",".join(["@" + key for key in sql_keys])
+ else:
+ values_placeholder = ",".join(["%s" for _ in sql_keys])
+ return f"INSERT INTO {table_name} ({keys_str}) VALUES ({values_placeholder})"
+
+ @staticmethod
+ def prepare_sql_values(cls_name: str, sql_values: Any, sql_keys: list[str]) -> Any:
+ if cls_name == DBConnectionClass.MSSQL:
+ return tuple(sql_values)
+ elif cls_name == DBConnectionClass.BIGQUERY:
+ query_parameters = [
+ bigquery.ScalarQueryParameter(key, "STRING", value)
+ for key, value in zip(sql_keys, sql_values)
+ ]
+ return bigquery.QueryJobConfig(query_parameters=query_parameters)
+ return sql_values
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
new file mode 100644
index 000000000..c882c1b0e
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -0,0 +1,532 @@
+import ast
+import base64
+import json
+import logging
+import os
+from typing import Any, Optional
+
+import fsspec
+import magic
+from connector_v2.models import ConnectorInstance
+from fsspec.implementations.local import LocalFileSystem
+from unstract.sdk.constants import ToolExecKey
+from unstract.workflow_execution.constants import ToolOutputType
+from utils.user_context import UserContext
+from workflow_manager.endpoint_v2.base_connector import BaseConnector
+from workflow_manager.endpoint_v2.constants import (
+ ApiDeploymentResultStatus,
+ DestinationKey,
+ QueueResultStatus,
+ WorkflowFileType,
+)
+from workflow_manager.endpoint_v2.database_utils import DatabaseUtils
+from workflow_manager.endpoint_v2.exceptions import (
+ DestinationConnectorNotConfigured,
+ InvalidDestinationConnectionType,
+ InvalidToolOutputType,
+ MissingDestinationConnectionType,
+ ToolOutputTypeMismatch,
+)
+from workflow_manager.endpoint_v2.models import WorkflowEndpoint
+from workflow_manager.endpoint_v2.queue_utils import QueueResult, QueueUtils
+from workflow_manager.workflow_v2.enums import ExecutionStatus
+from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
+from workflow_manager.workflow_v2.models.file_history import FileHistory
+from workflow_manager.workflow_v2.models.workflow import Workflow
+
+logger = logging.getLogger(__name__)
+
+
+class DestinationConnector(BaseConnector):
+ """A class representing a Destination connector for a workflow.
+
+ This class extends the BaseConnector class and provides methods for
+ interacting with different types of destination connectors,
+ such as file system connectors and API connectors and DB connectors.
+
+ Attributes:
+ workflow (Workflow): The workflow associated with
+ the destination connector.
+ """
+
+ def __init__(self, workflow: Workflow, execution_id: str) -> None:
+ """Initialize a DestinationConnector object.
+
+ Args:
+ workflow (Workflow): _description_
+ """
+ organization_id = UserContext.get_organization_identifier()
+ super().__init__(workflow.id, execution_id, organization_id)
+ self.endpoint = self._get_endpoint_for_workflow(workflow=workflow)
+ self.source_endpoint = self._get_source_endpoint_for_workflow(workflow=workflow)
+ self.execution_id = execution_id
+ self.api_results: list[dict[str, Any]] = []
+ self.queue_results: list[dict[str, Any]] = []
+
+ def _get_endpoint_for_workflow(
+ self,
+ workflow: Workflow,
+ ) -> WorkflowEndpoint:
+ """Get WorkflowEndpoint instance.
+
+ Args:
+ workflow (Workflow): Workflow associated with the
+ destination connector.
+
+ Returns:
+ WorkflowEndpoint: WorkflowEndpoint instance.
+ """
+ endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(
+ workflow=workflow,
+ endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,
+ )
+ if endpoint.connector_instance:
+ endpoint.connector_instance.connector_metadata = (
+ endpoint.connector_instance.metadata
+ )
+ return endpoint
+
+ def _get_source_endpoint_for_workflow(
+ self,
+ workflow: Workflow,
+ ) -> WorkflowEndpoint:
+ """Get WorkflowEndpoint instance.
+
+ Args:
+ workflow (Workflow): Workflow associated with the
+ destination connector.
+
+ Returns:
+ WorkflowEndpoint: WorkflowEndpoint instance.
+ """
+ endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(
+ workflow=workflow,
+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
+ )
+ if endpoint.connector_instance:
+ endpoint.connector_instance.connector_metadata = (
+ endpoint.connector_instance.metadata
+ )
+ return endpoint
+
+ def validate(self) -> None:
+ connection_type = self.endpoint.connection_type
+ connector: ConnectorInstance = self.endpoint.connector_instance
+ if connection_type is None:
+ raise MissingDestinationConnectionType()
+ if connection_type not in WorkflowEndpoint.ConnectionType.values:
+ raise InvalidDestinationConnectionType()
+ if (
+ connection_type != WorkflowEndpoint.ConnectionType.API
+ and connection_type != WorkflowEndpoint.ConnectionType.MANUALREVIEW
+ and connector is None
+ ):
+ raise DestinationConnectorNotConfigured()
+
+ def handle_output(
+ self,
+ file_name: str,
+ file_hash: str,
+ workflow: Workflow,
+ file_history: Optional[FileHistory] = None,
+ error: Optional[str] = None,
+ input_file_path: Optional[str] = None,
+ ) -> None:
+ """Handle the output based on the connection type."""
+ connection_type = self.endpoint.connection_type
+ result: Optional[str] = None
+ meta_data: Optional[str] = None
+ if error:
+ if connection_type == WorkflowEndpoint.ConnectionType.API:
+ self._handle_api_result(file_name=file_name, error=error, result=result)
+ return
+ if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
+ self.copy_output_to_output_directory()
+ elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:
+ self.insert_into_db(file_history)
+ elif connection_type == WorkflowEndpoint.ConnectionType.API:
+ result = self.get_result(file_history)
+ meta_data = self.get_metadata(file_history)
+ self._handle_api_result(
+ file_name=file_name, error=error, result=result, meta_data=meta_data
+ )
+ elif connection_type == WorkflowEndpoint.ConnectionType.MANUALREVIEW:
+ result = self.get_result(file_history)
+ meta_data = self.get_metadata(file_history)
+ self._push_to_queue(
+ file_name=file_name,
+ workflow=workflow,
+ result=result,
+ input_file_path=input_file_path,
+ meta_data=meta_data,
+ )
+ if not file_history:
+ FileHistoryHelper.create_file_history(
+ cache_key=file_hash,
+ workflow=workflow,
+ status=ExecutionStatus.COMPLETED,
+ result=result,
+ metadata=meta_data,
+ file_name=file_name,
+ )
+
+ def copy_output_to_output_directory(self) -> None:
+ """Copy output to the destination directory."""
+ connector: ConnectorInstance = self.endpoint.connector_instance
+ connector_settings: dict[str, Any] = connector.connector_metadata
+ destination_configurations: dict[str, Any] = self.endpoint.configuration
+ root_path = str(connector_settings.get(DestinationKey.PATH))
+ output_folder = str(
+ destination_configurations.get(DestinationKey.OUTPUT_FOLDER, "/")
+ )
+ overwrite = bool(
+ destination_configurations.get(
+ DestinationKey.OVERWRITE_OUTPUT_DOCUMENT, False
+ )
+ )
+ output_directory = os.path.join(root_path, output_folder)
+
+ destination_volume_path = os.path.join(
+ self.execution_dir, ToolExecKey.OUTPUT_DIR
+ )
+
+ connector_fs = self.get_fsspec(
+ settings=connector_settings, connector_id=connector.connector_id
+ )
+ if not connector_fs.isdir(output_directory):
+ connector_fs.mkdir(output_directory)
+
+ # Traverse local directory and create the same structure in the
+ # output_directory
+ for root, dirs, files in os.walk(destination_volume_path):
+ for dir_name in dirs:
+ connector_fs.mkdir(
+ os.path.join(
+ output_directory,
+ os.path.relpath(root, destination_volume_path),
+ dir_name,
+ )
+ )
+
+ for file_name in files:
+ source_path = os.path.join(root, file_name)
+ destination_path = os.path.join(
+ output_directory,
+ os.path.relpath(root, destination_volume_path),
+ file_name,
+ )
+ normalized_path = os.path.normpath(destination_path)
+ with open(source_path, "rb") as source_file:
+ connector_fs.write_bytes(
+ normalized_path, source_file.read(), overwrite=overwrite
+ )
+
+ def insert_into_db(self, file_history: Optional[FileHistory]) -> None:
+ """Insert data into the database."""
+ connector_instance: ConnectorInstance = self.endpoint.connector_instance
+ connector_settings: dict[str, Any] = connector_instance.metadata
+ destination_configurations: dict[str, Any] = self.endpoint.configuration
+ table_name: str = str(destination_configurations.get(DestinationKey.TABLE))
+ include_agent: bool = bool(
+ destination_configurations.get(DestinationKey.INCLUDE_AGENT, False)
+ )
+ include_timestamp = bool(
+ destination_configurations.get(DestinationKey.INCLUDE_TIMESTAMP, False)
+ )
+ agent_name = str(destination_configurations.get(DestinationKey.AGENT_NAME))
+ column_mode = str(destination_configurations.get(DestinationKey.COLUMN_MODE))
+ single_column_name = str(
+ destination_configurations.get(DestinationKey.SINGLE_COLUMN_NAME, "data")
+ )
+
+ data = self.get_result(file_history)
+ values = DatabaseUtils.get_columns_and_values(
+ column_mode_str=column_mode,
+ data=data,
+ include_timestamp=include_timestamp,
+ include_agent=include_agent,
+ agent_name=agent_name,
+ single_column_name=single_column_name,
+ )
+ db_class = DatabaseUtils.get_db_class(
+ connector_id=connector_instance.connector_id,
+ connector_settings=connector_settings,
+ )
+ engine = db_class.get_engine()
+ # If data is None, don't execute CREATE or INSERT query
+ if data is None:
+ return
+ DatabaseUtils.create_table_if_not_exists(
+ db_class=db_class,
+ engine=engine,
+ table_name=table_name,
+ database_entry=values,
+ )
+ cls_name = db_class.__class__.__name__
+ sql_columns_and_values = DatabaseUtils.get_sql_query_data(
+ cls_name=cls_name,
+ connector_id=connector_instance.connector_id,
+ connector_settings=connector_settings,
+ table_name=table_name,
+ values=values,
+ )
+ DatabaseUtils.execute_write_query(
+ db_class=db_class,
+ engine=engine,
+ table_name=table_name,
+ sql_keys=list(sql_columns_and_values.keys()),
+ sql_values=list(sql_columns_and_values.values()),
+ )
+
+ def _handle_api_result(
+ self,
+ file_name: str,
+ error: Optional[str] = None,
+ result: Optional[str] = None,
+ meta_data: Optional[dict[str, Any]] = None,
+ ) -> None:
+ """Handle the API result.
+
+ This method is responsible for handling the API result.
+ It appends the file name and result to the 'results' list for API resp.
+
+ Args:
+ file_name (str): The name of the file.
+ result (Optional[str], optional): The result of the API call.
+ Defaults to None.
+
+ Returns:
+ None
+ """
+ api_result: dict[str, Any] = {"file": file_name}
+ if error:
+ api_result.update(
+ {"status": ApiDeploymentResultStatus.FAILED, "error": error}
+ )
+ else:
+ if result:
+ api_result.update(
+ {
+ "status": ApiDeploymentResultStatus.SUCCESS,
+ "result": result,
+ "metadata": meta_data,
+ }
+ )
+ else:
+ api_result.update(
+ {"status": ApiDeploymentResultStatus.SUCCESS, "result": ""}
+ )
+ self.api_results.append(api_result)
+
+ def parse_string(self, original_string: str) -> Any:
+ """Parse the given string, attempting to evaluate it as a Python
+ literal.
+ ex: a json string to dict method
+ Parameters:
+ - original_string (str): The input string to be parsed.
+
+ Returns:
+ - Any: The parsed result. If the string can be evaluated as a Python
+ literal, the result of the evaluation is returned.
+ If not, the original string is returned unchanged.
+
+ Note:
+ This function uses `ast.literal_eval` to attempt parsing the string as a
+ Python literal. If parsing fails due to a SyntaxError or ValueError,
+ the original string is returned.
+
+ Example:
+ >>> parser.parse_string("42")
+ 42
+ >>> parser.parse_string("[1, 2, 3]")
+ [1, 2, 3]
+ >>> parser.parse_string("Hello, World!")
+ 'Hello, World!'
+ """
+ try:
+ # Try to evaluate as a Python literal
+ python_literal = ast.literal_eval(original_string)
+ return python_literal
+ except (SyntaxError, ValueError):
+ # If evaluating as a Python literal fails,
+ # assume it's a plain string
+ return original_string
+
+ def get_result(self, file_history: Optional[FileHistory]) -> Optional[Any]:
+ """Get result data from the output file.
+
+ Returns:
+ Union[dict[str, Any], str]: Result data.
+ """
+ if file_history and file_history.result:
+ return self.parse_string(file_history.result)
+ output_file = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
+ metadata: dict[str, Any] = self.get_workflow_metadata()
+ output_type = self.get_output_type(metadata)
+ result: Optional[Any] = None
+ try:
+ # TODO: SDK handles validation; consider removing here.
+ mime = magic.Magic()
+ file_type = mime.from_file(output_file)
+ if output_type == ToolOutputType.JSON:
+ if "JSON" not in file_type:
+ logger.error(f"Output type json mismatched {file_type}")
+ raise ToolOutputTypeMismatch()
+ with open(output_file) as file:
+ result = json.load(file)
+ elif output_type == ToolOutputType.TXT:
+ if "JSON" in file_type:
+ logger.error(f"Output type txt mismatched {file_type}")
+ raise ToolOutputTypeMismatch()
+ with open(output_file) as file:
+ result = file.read()
+ result = result.encode("utf-8").decode("unicode-escape")
+ else:
+ raise InvalidToolOutputType()
+ except (FileNotFoundError, json.JSONDecodeError) as err:
+ logger.error(f"Error while getting result {err}")
+ return result
+
+ def get_metadata(
+ self, file_history: Optional[FileHistory]
+ ) -> Optional[dict[str, Any]]:
+ """Get meta_data from the output file.
+
+ Returns:
+ Union[dict[str, Any], str]: Meta data.
+ """
+ if file_history and file_history.meta_data:
+ return self.parse_string(file_history.meta_data)
+ metadata: dict[str, Any] = self.get_workflow_metadata()
+
+ return metadata
+
+ def delete_execution_directory(self) -> None:
+ """Delete the execution directory.
+
+ Returns:
+ None
+ """
+ fs: LocalFileSystem = fsspec.filesystem("file")
+ fs.rm(self.execution_dir, recursive=True)
+ self.delete_api_storage_dir(self.workflow_id, self.execution_id)
+
+ @classmethod
+ def delete_api_storage_dir(cls, workflow_id: str, execution_id: str) -> None:
+ """Delete the api storage path.
+
+ Returns:
+ None
+ """
+ api_storage_dir = cls.get_api_storage_dir_path(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ fs: LocalFileSystem = fsspec.filesystem("file")
+ fs.rm(api_storage_dir, recursive=True)
+
+ @classmethod
+ def create_endpoint_for_workflow(
+ cls,
+ workflow: Workflow,
+ ) -> None:
+ """Create a workflow endpoint for the destination.
+
+ Args:
+ workflow (Workflow): Workflow for which the endpoint is created.
+ """
+ endpoint = WorkflowEndpoint(
+ workflow=workflow,
+ endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,
+ )
+ endpoint.save()
+
+ @classmethod
+ def get_json_schema_for_database(cls) -> dict[str, Any]:
+ """Get JSON schema for the database.
+
+ Returns:
+ dict[str, Any]: JSON schema for the database.
+ """
+ schema_path = os.path.join(
+ os.path.dirname(__file__), "static", "dest", "db.json"
+ )
+ return cls.get_json_schema(file_path=schema_path)
+
+ @classmethod
+ def get_json_schema_for_file_system(cls) -> dict[str, Any]:
+ """Get JSON schema for the file system.
+
+ Returns:
+ dict[str, Any]: JSON schema for the file system.
+ """
+ schema_path = os.path.join(
+ os.path.dirname(__file__), "static", "dest", "file.json"
+ )
+ return cls.get_json_schema(file_path=schema_path)
+
+ @classmethod
+ def get_json_schema_for_api(cls) -> dict[str, Any]:
+ """Json schema for api.
+
+ Returns:
+ dict[str, Any]: _description_
+ """
+ schema_path = os.path.join(
+ os.path.dirname(__file__), "static", "dest", "api.json"
+ )
+ return cls.get_json_schema(file_path=schema_path)
+
+ def _push_to_queue(
+ self,
+ file_name: str,
+ workflow: Workflow,
+ result: Optional[str] = None,
+ input_file_path: Optional[str] = None,
+ meta_data: Optional[dict[str, Any]] = None,
+ ) -> None:
+ """Handle the Manual Review QUEUE result.
+
+ This method is responsible for pushing the input file and result to
+ review queue.
+ Args:
+ file_name (str): The name of the file.
+ workflow (Workflow): The workflow object containing
+ details about the workflow.
+ result (Optional[str], optional): The result of the API call.
+ Defaults to None.
+ input_file_path (Optional[str], optional):
+ The path to the input file.
+ Defaults to None.
+ meta_data (Optional[dict[str, Any]], optional):
+ A dictionary containing additional
+ metadata related to the file. Defaults to None.
+
+ Returns:
+ None
+ """
+ if not result:
+ return
+ connector: ConnectorInstance = self.source_endpoint.connector_instance
+ connector_settings: dict[str, Any] = connector.connector_metadata
+
+ source_fs = self.get_fsspec(
+ settings=connector_settings, connector_id=connector.connector_id
+ )
+ with source_fs.open(input_file_path, "rb") as remote_file:
+ file_content = remote_file.read()
+ # Convert file content to a base64 encoded string
+ file_content_base64 = base64.b64encode(file_content).decode("utf-8")
+ q_name = f"review_queue_{self.organization_id}_{workflow.workflow_name}"
+ queue_result = QueueResult(
+ file=file_name,
+ whisper_hash=meta_data["whisper-hash"],
+ status=QueueResultStatus.SUCCESS,
+ result=result,
+ workflow_id=str(self.workflow_id),
+ file_content=file_content_base64,
+ )
+ # Convert the result dictionary to a JSON string
+ queue_result_json = json.dumps(queue_result)
+ conn = QueueUtils.get_queue_inst()
+ # Enqueue the JSON string
+ conn.enqueue(queue_name=q_name, message=queue_result_json)
diff --git a/backend/workflow_manager/endpoint_v2/endpoint_utils.py b/backend/workflow_manager/endpoint_v2/endpoint_utils.py
new file mode 100644
index 000000000..70207b3a9
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/endpoint_utils.py
@@ -0,0 +1,30 @@
+from workflow_manager.endpoint_v2.destination import DestinationConnector
+from workflow_manager.endpoint_v2.models import WorkflowEndpoint
+from workflow_manager.endpoint_v2.source import SourceConnector
+from workflow_manager.workflow_v2.models.workflow import Workflow
+from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper
+
+
+class WorkflowEndpointUtils:
+ @staticmethod
+ def create_endpoints_for_workflow(workflow: Workflow) -> None:
+ """Create endpoints for a given workflow. This method creates both
+ source and destination endpoints for the specified workflow.
+
+ Parameters:
+ workflow (Workflow): The workflow for which
+ the endpoints need to be created.
+
+ Returns:
+ None
+ """
+ SourceConnector.create_endpoint_for_workflow(workflow)
+ DestinationConnector.create_endpoint_for_workflow(workflow)
+
+ @staticmethod
+ def get_endpoints_for_workflow(workflow_id: str) -> list[WorkflowEndpoint]:
+ workflow = WorkflowHelper.get_workflow_by_id(workflow_id)
+ endpoints: list[WorkflowEndpoint] = WorkflowEndpoint.objects.filter(
+ workflow=workflow
+ )
+ return endpoints
diff --git a/backend/workflow_manager/endpoint_v2/exceptions.py b/backend/workflow_manager/endpoint_v2/exceptions.py
new file mode 100644
index 000000000..7028eec28
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/exceptions.py
@@ -0,0 +1,87 @@
+from rest_framework.exceptions import APIException
+
+
+class InvalidInputDirectory(APIException):
+ status_code = 400
+ default_detail = "The provided directory is invalid."
+
+
+class InvalidSourceConnectionType(APIException):
+ status_code = 400
+ default_detail = "The provided source connection type is invalid."
+
+
+class InvalidDestinationConnectionType(APIException):
+ status_code = 400
+ default_detail = "The provided destination connection type is invalid."
+
+
+class MissingSourceConnectionType(APIException):
+ status_code = 400
+ default_detail = "The source connection type is missing."
+
+
+class MissingDestinationConnectionType(APIException):
+ status_code = 400
+ default_detail = "The destination connection type is missing."
+
+
+class SourceConnectorNotConfigured(APIException):
+ status_code = 400
+ default_detail = "The source connector is not configured"
+
+
+class DestinationConnectorNotConfigured(APIException):
+ status_code = 400
+ default_detail = "The destination connector is not configured"
+
+
+class FileHashNotFound(APIException):
+ status_code = 500
+ default_detail = "Internal server error: File hash not found."
+
+
+class ToolMetadataNotFound(APIException):
+ status_code = 500
+ default_detail = "Internal server error: Tool metadata not found."
+
+
+class OrganizationIdNotFound(APIException):
+ status_code = 404
+ default_detail = "The organization ID could not be found"
+
+
+class InvalidToolOutputType(APIException):
+ status_code = 500
+ default_detail = "Invalid output type is returned from tool"
+
+
+class ToolOutputTypeMismatch(APIException):
+ status_code = 400
+ default_detail = (
+ "The data type of the tool's output does not match the expected type."
+ )
+
+
+class BigQueryTableNotFound(APIException):
+ status_code = 400
+ default_detail = (
+ "Please enter correct correct bigquery table in the form "
+ "{table}.{schema}.{database}."
+ )
+
+
+class UnstractDBException(APIException):
+ default_detail = "Error creating/inserting to database. "
+
+ def __init__(self, detail: str = default_detail) -> None:
+ status_code = 500
+ super().__init__(detail=detail, code=status_code)
+
+
+class UnstractQueueException(APIException):
+ default_detail = "Error creating/inserting to Queue. "
+
+ def __init__(self, detail: str = default_detail) -> None:
+ status_code = 500
+ super().__init__(detail=detail, code=status_code)
diff --git a/backend/workflow_manager/endpoint_v2/models.py b/backend/workflow_manager/endpoint_v2/models.py
new file mode 100644
index 000000000..6b067160e
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/models.py
@@ -0,0 +1,54 @@
+import uuid
+
+from connector_v2.models import ConnectorInstance
+from django.db import models
+from utils.models.base_model import BaseModel
+from workflow_manager.workflow_v2.models.workflow import Workflow
+
+
+class WorkflowEndpoint(BaseModel):
+ class EndpointType(models.TextChoices):
+ SOURCE = "SOURCE", "Source connector"
+ DESTINATION = "DESTINATION", "Destination Connector"
+
+ class ConnectionType(models.TextChoices):
+ FILESYSTEM = "FILESYSTEM", "FileSystem connector"
+ DATABASE = "DATABASE", "Database Connector"
+ API = "API", "API Connector"
+ APPDEPLOYMENT = "APPDEPLOYMENT", "App Deployment"
+ MANUALREVIEW = "MANUALREVIEW", "Manual Review Queue Connector"
+
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ workflow = models.ForeignKey(
+ Workflow,
+ on_delete=models.CASCADE,
+ db_index=True,
+ editable=False,
+ db_comment="Foreign key from Workflow model",
+ )
+ endpoint_type = models.CharField(
+ choices=EndpointType.choices,
+ editable=False,
+ db_comment="Endpoint type (source or destination)",
+ )
+ connection_type = models.CharField(
+ choices=ConnectionType.choices,
+ blank=True,
+ db_comment="Connection type (Filesystem, Database, API or Manualreview)",
+ )
+ configuration = models.JSONField(
+ blank=True, null=True, db_comment="Configuration in JSON format"
+ )
+ connector_instance = models.ForeignKey(
+ ConnectorInstance,
+ on_delete=models.CASCADE,
+ db_index=True,
+ null=True,
+ db_comment="Foreign key from ConnectorInstance model",
+ related_name="workflow_endpoints",
+ )
+
+ class Meta:
+ verbose_name = "Workflow Endpoint"
+ verbose_name_plural = "Workflow Endpoints"
+ db_table = "workflow_endpoints_v2"
diff --git a/backend/workflow_manager/endpoint_v2/queue_utils.py b/backend/workflow_manager/endpoint_v2/queue_utils.py
new file mode 100644
index 000000000..fde790e42
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/queue_utils.py
@@ -0,0 +1,41 @@
+import logging
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+from utils.constants import Common
+from workflow_manager.endpoint_v2.exceptions import UnstractQueueException
+
+from unstract.connectors.queues import connectors as queue_connectors
+from unstract.connectors.queues.unstract_queue import UnstractQueue
+
+logger = logging.getLogger(__name__)
+
+
+class QueueResultStatus(Enum):
+ SUCCESS = "success"
+ FAILURE = "failure"
+ # Add other statuses as needed
+
+
+class QueueUtils:
+ @staticmethod
+ def get_queue_inst(connector_settings: dict[str, Any] = {}) -> UnstractQueue:
+ if not queue_connectors:
+ raise UnstractQueueException(detail="Queue connector not exists")
+ queue_connector_key = next(iter(queue_connectors))
+ connector = queue_connectors[queue_connector_key][Common.METADATA][
+ Common.CONNECTOR
+ ]
+ connector_class: UnstractQueue = connector(connector_settings)
+ return connector_class
+
+
+@dataclass
+class QueueResult:
+ file: str
+ whisper_hash: str
+ status: QueueResultStatus
+ result: Any
+ workflow_id: str
+ file_content: str
diff --git a/backend/workflow_manager/endpoint_v2/serializers.py b/backend/workflow_manager/endpoint_v2/serializers.py
new file mode 100644
index 000000000..2ce3d4a8f
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/serializers.py
@@ -0,0 +1,12 @@
+import logging
+
+from rest_framework.serializers import ModelSerializer
+from workflow_manager.endpoint_v2.models import WorkflowEndpoint
+
+logger = logging.getLogger(__name__)
+
+
+class WorkflowEndpointSerializer(ModelSerializer):
+ class Meta:
+ model = WorkflowEndpoint
+ fields = "__all__"
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
new file mode 100644
index 000000000..57ef8d3f5
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -0,0 +1,461 @@
+import fnmatch
+import logging
+import os
+import shutil
+from hashlib import md5, sha256
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Optional
+
+import fsspec
+from connector_processor.constants import ConnectorKeys
+from connector_v2.models import ConnectorInstance
+from django.core.files.uploadedfile import UploadedFile
+from unstract.workflow_execution.enums import LogState
+from utils.user_context import UserContext
+from workflow_manager.endpoint_v2.base_connector import BaseConnector
+from workflow_manager.endpoint_v2.constants import (
+ FilePattern,
+ FileSystemConnector,
+ FileType,
+ SourceConstant,
+ SourceKey,
+ WorkflowFileType,
+)
+from workflow_manager.endpoint_v2.exceptions import (
+ FileHashNotFound,
+ InvalidInputDirectory,
+ InvalidSourceConnectionType,
+ MissingSourceConnectionType,
+ OrganizationIdNotFound,
+ SourceConnectorNotConfigured,
+)
+from workflow_manager.endpoint_v2.models import WorkflowEndpoint
+from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper
+from workflow_manager.workflow_v2.models.workflow import Workflow
+
+logger = logging.getLogger(__name__)
+
+
+class SourceConnector(BaseConnector):
+ """A class representing a source connector for a workflow.
+
+ This class extends the BaseConnector class and provides methods for
+ interacting with different types of source connectors,
+ such as file system connectors and API connectors.
+ It allows listing files from the source connector,
+ adding files to the execution volume, and retrieving JSON schemas for
+ different types of connectors.
+
+ Attributes:
+ workflow (Workflow): The workflow associated with the source connector.
+ """
+
+ def __init__(
+ self,
+ workflow: Workflow,
+ execution_id: str,
+ organization_id: Optional[str] = None,
+ execution_service: Optional[WorkflowExecutionServiceHelper] = None,
+ ) -> None:
+ """Initialize a SourceConnector object.
+
+ Args:
+ workflow (Workflow): _description_
+ """
+ organization_id = organization_id or UserContext.get_organization_identifier()
+ if not organization_id:
+ raise OrganizationIdNotFound()
+ super().__init__(workflow.id, execution_id, organization_id)
+ self.endpoint = self._get_endpoint_for_workflow(workflow=workflow)
+ self.workflow = workflow
+ self.execution_id = execution_id
+ self.organization_id = organization_id
+ self.hash_value_of_file_content: Optional[str] = None
+ self.execution_service = execution_service
+
+ def _get_endpoint_for_workflow(
+ self,
+ workflow: Workflow,
+ ) -> WorkflowEndpoint:
+ """Get WorkflowEndpoint instance.
+
+ Args:
+ workflow (Workflow): Workflow
+
+ Returns:
+ WorkflowEndpoint: _description_
+ """
+ endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(
+ workflow=workflow,
+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
+ )
+ if endpoint.connector_instance:
+ endpoint.connector_instance.connector_metadata = (
+ endpoint.connector_instance.metadata
+ )
+ return endpoint
+
+ def validate(self) -> None:
+ connection_type = self.endpoint.connection_type
+ connector: ConnectorInstance = self.endpoint.connector_instance
+ if connection_type is None:
+ raise MissingSourceConnectionType()
+ if connection_type not in WorkflowEndpoint.ConnectionType.values:
+ raise InvalidSourceConnectionType()
+ if connection_type != WorkflowEndpoint.ConnectionType.API and connector is None:
+ raise SourceConnectorNotConfigured()
+
+ def valid_file_patterns(self, required_patterns: list[Any]) -> list[str]:
+ patterns = {
+ FileType.PDF_DOCUMENTS: FilePattern.PDF_DOCUMENTS,
+ FileType.TEXT_DOCUMENTS: FilePattern.TEXT_DOCUMENTS,
+ FileType.IMAGES: FilePattern.IMAGES,
+ }
+ wildcard = []
+ if not required_patterns:
+ wildcard.append("*")
+ else:
+ for pattern in required_patterns:
+ wildcard.extend(patterns.get(pattern, []))
+ return wildcard
+
+ def list_file_from_api_storage(self) -> list[str]:
+ """List all files from the api_storage_dir directory."""
+ files: list[str] = []
+ if not self.api_storage_dir:
+ return files
+ for file in os.listdir(self.api_storage_dir):
+ file_path = os.path.join(self.api_storage_dir, file)
+ if os.path.isfile(file_path):
+ files.append(file_path)
+ return files
+
+ def list_files_from_file_connector(self) -> list[str]:
+ """_summary_
+
+ Raises:
+ InvalidDirectory: _description_
+
+ Returns:
+ list[str]: _description_
+ """
+ connector: ConnectorInstance = self.endpoint.connector_instance
+ connector_settings: dict[str, Any] = connector.connector_metadata
+ source_configurations: dict[str, Any] = self.endpoint.configuration
+ required_patterns = source_configurations.get(SourceKey.FILE_EXTENSIONS, [])
+ recursive = bool(
+ source_configurations.get(SourceKey.PROCESS_SUB_DIRECTORIES, False)
+ )
+ limit = int(
+ source_configurations.get(
+ SourceKey.MAX_FILES, FileSystemConnector.MAX_FILES
+ )
+ )
+ root_dir_path = connector_settings.get(ConnectorKeys.PATH, "")
+ input_directory = str(source_configurations.get(SourceKey.ROOT_FOLDER, ""))
+ if root_dir_path: # user needs to manually type the optional file path
+ input_directory = str(Path(root_dir_path, input_directory.lstrip("/")))
+ if not isinstance(required_patterns, list):
+ required_patterns = [required_patterns]
+
+ source_fs = self.get_fsspec(
+ settings=connector_settings, connector_id=connector.connector_id
+ )
+ patterns = self.valid_file_patterns(required_patterns=required_patterns)
+ is_directory = source_fs.isdir(input_directory)
+ if not is_directory:
+ raise InvalidInputDirectory()
+ matched_files = self._get_matched_files(
+ source_fs, input_directory, patterns, recursive, limit
+ )
+ self.publish_input_output_list_file_logs(input_directory, matched_files)
+ return matched_files
+
+ def publish_input_output_list_file_logs(
+ self, input_directory: str, matched_files: list[str]
+ ) -> None:
+ if not self.execution_service:
+ return None
+ input_log = f"##Input folder:\n\n `{os.path.basename(input_directory)}`\n\n"
+ self.execution_service.publish_update_log(
+ state=LogState.INPUT_UPDATE, message=input_log
+ )
+ output_log = self._matched_files_component_log(matched_files)
+ self.execution_service.publish_update_log(
+ state=LogState.OUTPUT_UPDATE, message=output_log
+ )
+
+ def publish_input_file_content(self, input_file_path: str, input_text: str) -> None:
+ if self.execution_service:
+ output_log_message = f"##Input text:\n\n```text\n{input_text}\n```\n\n"
+ input_log_message = (
+ "##Input file:\n\n```text\n"
+ f"{os.path.basename(input_file_path)}\n```\n\n"
+ )
+ self.execution_service.publish_update_log(
+ state=LogState.INPUT_UPDATE, message=input_log_message
+ )
+ self.execution_service.publish_update_log(
+ state=LogState.OUTPUT_UPDATE, message=output_log_message
+ )
+
+ def _matched_files_component_log(self, matched_files: list[str]) -> str:
+ output_log = "### Matched files \n```text\n\n\n"
+ for file in matched_files[:20]:
+ output_log += f"{file}\n"
+ output_log += "```\n\n"
+ output_log += f"""Total matched files: {len(matched_files)}
+ \n\nPlease note that only the first 20 files are shown.\n\n"""
+ return output_log
+
+ def _get_matched_files(
+ self,
+ source_fs: Any,
+ input_directory: str,
+ patterns: list[str],
+ recursive: bool,
+ limit: int,
+ ) -> list[str]:
+ """Get a list of matched files based on patterns in a directory.
+
+ This method searches for files in the specified `input_directory` that
+ match any of the given `patterns`.
+ The search can be performed recursively if `recursive` is set to True.
+ The number of matched files returned is limited by `limit`.
+
+ Args:
+ source_fs (Any): The file system object used for searching.
+ input_directory (str): The directory to search for files.
+ patterns (list[str]): The patterns to match against file names.
+ recursive (bool): Whether to perform a recursive search.
+ limit (int): The maximum number of matched files to return.
+
+ Returns:
+ list[str]: A list of matched file paths.
+ """
+ matched_files = []
+ count = 0
+ max_depth = int(SourceConstant.MAX_RECURSIVE_DEPTH) if recursive else 1
+
+ for root, dirs, files in source_fs.walk(input_directory, maxdepth=max_depth):
+ if count >= limit:
+ break
+ for file in files:
+ if not file:
+ continue
+ if count >= limit:
+ break
+ if any(fnmatch.fnmatch(file, pattern) for pattern in patterns):
+ file_path = os.path.join(root, file)
+ file_path = f"{file_path}"
+ matched_files.append(file_path)
+ count += 1
+
+ return matched_files
+
+ def list_files_from_source(self) -> list[str]:
+ """List files from source connector.
+
+ Args:
+ api_storage_dir (Optional[str], optional): API storage directory
+ Returns:
+ list[str]: list of files
+ """
+ connection_type = self.endpoint.connection_type
+ if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
+ return self.list_files_from_file_connector()
+ elif connection_type == WorkflowEndpoint.ConnectionType.API:
+ return self.list_file_from_api_storage()
+ raise InvalidSourceConnectionType()
+
+ @classmethod
+ def hash_str(cls, string_to_hash: Any, hash_method: str = "sha256") -> str:
+ """Computes the hash for a given input string.
+
+ Useful to hash strings needed for caching and other purposes.
+ Hash method defaults to "md5"
+
+ Args:
+ string_to_hash (str): String to be hashed
+ hash_method (str): Hash hash_method to use, supported ones
+ - "md5"
+
+ Returns:
+ str: Hashed string
+ """
+ if hash_method == "md5":
+ if isinstance(string_to_hash, bytes):
+ return str(md5(string_to_hash).hexdigest())
+ return str(md5(string_to_hash.encode()).hexdigest())
+ elif hash_method == "sha256":
+ if isinstance(string_to_hash, (bytes, bytearray)):
+ return str(sha256(string_to_hash).hexdigest())
+ return str(sha256(string_to_hash.encode()).hexdigest())
+ else:
+ raise ValueError(f"Unsupported hash_method: {hash_method}")
+
+ def add_input_from_connector_to_volume(self, input_file_path: str) -> str:
+ """Add input file to execution directory.
+
+ Args:
+ input_file_path (str): The path of the input file.
+ Returns:
+ str: The hash value of the file content.
+ Raises:
+ FileHashNotFound: If the hash value of the file content
+ is not found.
+ """
+ connector: ConnectorInstance = self.endpoint.connector_instance
+ connector_settings: dict[str, Any] = connector.connector_metadata
+ source_file_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)
+ infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
+ source_file = f"file://{source_file_path}"
+
+ source_fs = self.get_fsspec(
+ settings=connector_settings, connector_id=connector.connector_id
+ )
+ with (
+ source_fs.open(input_file_path, "rb") as remote_file,
+ fsspec.open(source_file, "wb") as local_file,
+ ):
+ file_content = remote_file.read()
+ hash_value_of_file_content = self.hash_str(file_content)
+ logger.info(
+ f"hash_value_of_file {source_file} is "
+ f": {hash_value_of_file_content}"
+ )
+ input_log = (
+ file_content[:500].decode("utf-8", errors="replace") + "...(truncated)"
+ )
+ self.publish_input_file_content(input_file_path, input_log)
+
+ local_file.write(file_content)
+ shutil.copyfile(source_file_path, infile_path)
+ logger.info(f"{input_file_path} is added in to execution directory")
+ return hash_value_of_file_content
+
+ def add_input_from_api_storage_to_volume(self, input_file_path: str) -> None:
+ """Add input file to execution directory from api storage."""
+ infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
+ source_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)
+ shutil.copyfile(input_file_path, infile_path)
+ shutil.copyfile(input_file_path, source_path)
+
+ def add_file_to_volume(
+ self, input_file_path: str, hash_values_of_files: dict[str, str] = {}
+ ) -> tuple[str, str]:
+ """Add input file to execution directory.
+
+ Args:
+ input_file_path (str): source file
+
+ Raises:
+ InvalidSource: _description_
+
+ Returns:
+ str: file_name
+ """
+ connection_type = self.endpoint.connection_type
+ file_name = os.path.basename(input_file_path)
+ if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
+ file_content_hash = self.add_input_from_connector_to_volume(
+ input_file_path=input_file_path,
+ )
+ elif connection_type == WorkflowEndpoint.ConnectionType.API:
+ self.add_input_from_api_storage_to_volume(input_file_path=input_file_path)
+ if file_name not in hash_values_of_files:
+ raise FileHashNotFound()
+ file_content_hash = hash_values_of_files[file_name]
+ else:
+ raise InvalidSourceConnectionType()
+
+ self.add_metadata_to_volume(
+ input_file_path=input_file_path, source_hash=file_content_hash
+ )
+ return file_name, file_content_hash
+
+ def handle_final_result(
+ self,
+ results: list[dict[str, Any]],
+ file_name: str,
+ result: Optional[str],
+ ) -> None:
+ connection_type = self.endpoint.connection_type
+ if connection_type == WorkflowEndpoint.ConnectionType.API:
+ results.append({"file": file_name, "result": result})
+
+ def load_file(self, input_file_path: str) -> tuple[str, BytesIO]:
+ connector: ConnectorInstance = self.endpoint.connector_instance
+ connector_settings: dict[str, Any] = connector.connector_metadata
+ source_fs: fsspec.AbstractFileSystem = self.get_fsspec(
+ settings=connector_settings, connector_id=connector.connector_id
+ )
+ with source_fs.open(input_file_path, "rb") as remote_file:
+ file_content = remote_file.read()
+ file_stream = BytesIO(file_content)
+
+ return remote_file.key, file_stream
+
+ @classmethod
+ def add_input_file_to_api_storage(
+ cls, workflow_id: str, execution_id: str, file_objs: list[UploadedFile]
+ ) -> dict[str, str]:
+ """Add input file to api storage.
+
+ Args:
+ workflow_id (str): workflow id
+ execution_id (str): execution_id
+ file_objs (list[UploadedFile]): api file objects
+ """
+ api_storage_dir = cls.get_api_storage_dir_path(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ file_hashes: dict[str, str] = {}
+ for file in file_objs:
+ file_name = file.name
+ destination_path = os.path.join(api_storage_dir, file_name)
+ os.makedirs(os.path.dirname(destination_path), exist_ok=True)
+ with open(destination_path, "wb") as f:
+ buffer = bytearray()
+ for chunk in file.chunks():
+ buffer.extend(chunk)
+ f.write(buffer)
+ file_hashes.update({file_name: cls.hash_str(buffer)})
+ return file_hashes
+
+ @classmethod
+ def create_endpoint_for_workflow(
+ cls,
+ workflow: Workflow,
+ ) -> None:
+ """Creating WorkflowEndpoint entity."""
+ endpoint = WorkflowEndpoint(
+ workflow=workflow,
+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
+ )
+ endpoint.save()
+
+ @classmethod
+ def get_json_schema_for_api(cls) -> dict[str, Any]:
+ """Json schema for api.
+
+ Returns:
+ dict[str, Any]: _description_
+ """
+ schema_path = os.path.join(
+ os.path.dirname(__file__), "static", "src", "api.json"
+ )
+ return cls.get_json_schema(file_path=schema_path)
+
+ @classmethod
+ def get_json_schema_for_file_system(cls) -> dict[str, Any]:
+ """Json schema for Filesystem.
+
+ Returns:
+ dict[str, Any]: _description_
+ """
+ schema_path = os.path.join(
+ os.path.dirname(__file__), "static", "src", "file.json"
+ )
+ return cls.get_json_schema(file_path=schema_path)
diff --git a/backend/workflow_manager/endpoint_v2/static/dest/db.json b/backend/workflow_manager/endpoint_v2/static/dest/db.json
new file mode 100644
index 000000000..70286c704
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/static/dest/db.json
@@ -0,0 +1,54 @@
+{
+ "title": "Workflow DB Destination",
+ "description": "Settings for DB Destination",
+ "type": "object",
+ "required": [
+ "table",
+ "includeAgent",
+ "includeTimestamp",
+ "columnMode"
+ ],
+ "properties": {
+ "table": {
+ "type": "string",
+ "title": "Table",
+ "default": "",
+ "description": "Table to store the output. If your database supports schemas, use the format schema.table"
+ },
+ "includeAgent": {
+ "type": "boolean",
+ "title": "Include 'created_by' column",
+ "default": false,
+ "description": "Include the 'created_by' in the output row"
+ },
+ "agentName": {
+ "type": "string",
+ "title": "Agent Name",
+ "enum": [
+ "Unstract/DBWriter"
+ ],
+ "default": "Unstract/DBWriter",
+ "description": "Name of the agent to use as the 'created_by' value"
+ },
+ "includeTimestamp": {
+ "type": "boolean",
+ "title": "Include 'created_at' column",
+ "default": false,
+ "description": "Include the 'created_at' in the output row"
+ },
+ "columnMode": {
+ "type": "string",
+ "title": "Select how you want to write the output",
+ "enum": [
+ "Write JSON to a single column"
+ ],
+ "default": "Write JSON to a single column"
+ },
+ "singleColumnName": {
+ "type": "string",
+ "title": "Single Column Name",
+ "default": "data",
+ "description": "Name of the column to write the JSON to"
+ }
+ }
+}
diff --git a/backend/workflow_manager/endpoint_v2/static/dest/file.json b/backend/workflow_manager/endpoint_v2/static/dest/file.json
new file mode 100644
index 000000000..aa3a1263e
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/static/dest/file.json
@@ -0,0 +1,25 @@
+{
+ "title": "Workflow File Destination",
+ "description": "Settings for File Destination",
+ "type": "object",
+ "required": [
+ "outputFolder"
+ ],
+ "properties": {
+ "outputFolder": {
+ "type": "string",
+ "title": "Output folder",
+ "default": "output",
+ "description": "Folder to store the output",
+ "minLength": 1,
+ "maxLength": 100,
+ "format": "file-path"
+ },
+ "overwriteOutput": {
+ "type": "boolean",
+ "title": "Overwrite existing output",
+ "default": true,
+ "description": "Used to overwrite output document"
+ }
+ }
+}
diff --git a/backend/workflow_manager/endpoint_v2/static/src/api.json b/backend/workflow_manager/endpoint_v2/static/src/api.json
new file mode 100644
index 000000000..8d4a9d022
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/static/src/api.json
@@ -0,0 +1,21 @@
+{
+ "title": "Workflow API Source",
+ "description": "Settings for API Source",
+ "type": "object",
+ "required": [],
+ "properties": {
+ "fileExtensions": {
+ "type": "array",
+ "title": "File types to process",
+ "description": "Limit the file types to process. Leave it empty to process all files",
+ "items": {
+ "type": "string",
+ "enum": [
+ "PDF documents",
+ "Text documents",
+ "Images"
+ ]
+ }
+ }
+ }
+}
diff --git a/backend/workflow_manager/endpoint_v2/static/src/file.json b/backend/workflow_manager/endpoint_v2/static/src/file.json
new file mode 100644
index 000000000..d603b7e53
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/static/src/file.json
@@ -0,0 +1,39 @@
+{
+ "title": "Workflow File Source",
+ "description": "Settings for File Source",
+ "type": "object",
+ "required": [],
+ "properties": {
+ "rootFolder": {
+ "type": "string",
+ "title": "Folder to process",
+ "default": "",
+ "description": "The root folder to start processing files from. Leave it empty to use the root folder"
+ },
+ "processSubDirectories": {
+ "type": "boolean",
+ "title": "Process sub directories",
+ "default": true,
+ "description": "Process sub directories recursively"
+ },
+ "fileExtensions": {
+ "type": "array",
+ "title": "File types to process",
+ "description": "Limit the file types to process. Leave it empty to process all files",
+ "items": {
+ "type": "string",
+ "enum": [
+ "PDF documents",
+ "Text documents",
+ "Images"
+ ]
+ }
+ },
+ "maxFiles": {
+ "type": "number",
+ "title": "Max files to process",
+ "default": 100,
+ "description": "The maximum number of files to process"
+ }
+ }
+}
diff --git a/backend/workflow_manager/endpoint_v2/tests/__init__.py b/backend/workflow_manager/endpoint_v2/tests/__init__.py
new file mode 100644
index 000000000..fca2b2401
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/tests/__init__.py
@@ -0,0 +1,3 @@
+from backend.celery import app as celery_app
+
+__all__ = ["celery_app"]
diff --git a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/__init__.py b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/base_test_db.py b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/base_test_db.py
new file mode 100644
index 000000000..4040eb624
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/base_test_db.py
@@ -0,0 +1,160 @@
+import datetime
+import json
+import os
+from typing import Any
+
+import pytest # type: ignore
+from dotenv import load_dotenv
+
+from unstract.connectors.databases.bigquery import BigQuery
+from unstract.connectors.databases.mariadb import MariaDB
+from unstract.connectors.databases.mssql import MSSQL
+from unstract.connectors.databases.mysql import MySQL
+from unstract.connectors.databases.postgresql import PostgreSQL
+from unstract.connectors.databases.redshift import Redshift
+from unstract.connectors.databases.snowflake import SnowflakeDB
+from unstract.connectors.databases.unstract_db import UnstractDB
+
+load_dotenv("test.env")
+
+
+class BaseTestDB:
+ @pytest.fixture(autouse=True)
+ def base_setup(self) -> None:
+ self.postgres_creds = {
+ "user": os.getenv("DB_USER"),
+ "password": os.getenv("DB_PASSWORD"),
+ "host": os.getenv("DB_HOST"),
+ "port": os.getenv("DB_PORT"),
+ "database": os.getenv("DB_NAME"),
+ }
+ self.redshift_creds = {
+ "user": os.getenv("REDSHIFT_USER"),
+ "password": os.getenv("REDSHIFT_PASSWORD"),
+ "host": os.getenv("REDSHIFT_HOST"),
+ "port": os.getenv("REDSHIFT_PORT"),
+ "database": os.getenv("REDSHIFT_DB"),
+ }
+ self.snowflake_creds = {
+ "user": os.getenv("SNOWFLAKE_USER"),
+ "password": os.getenv("SNOWFLAKE_PASSWORD"),
+ "account": os.getenv("SNOWFLAKE_ACCOUNT"),
+ "role": os.getenv("SNOWFLAKE_ROLE"),
+ "database": os.getenv("SNOWFLAKE_DB"),
+ "schema": os.getenv("SNOWFLAKE_SCHEMA"),
+ "warehouse": os.getenv("SNOWFLAKE_WAREHOUSE"),
+ }
+ self.mssql_creds = {
+ "user": os.getenv("MSSQL_USER"),
+ "password": os.getenv("MSSQL_PASSWORD"),
+ "server": os.getenv("MSSQL_SERVER"),
+ "port": os.getenv("MSSQL_PORT"),
+ "database": os.getenv("MSSQL_DB"),
+ }
+ self.mysql_creds = {
+ "user": os.getenv("MYSQL_USER"),
+ "password": os.getenv("MYSQL_PASSWORD"),
+ "host": os.getenv("MYSQL_SERVER"),
+ "port": os.getenv("MYSQL_PORT"),
+ "database": os.getenv("MYSQL_DB"),
+ }
+ self.mariadb_creds = {
+ "user": os.getenv("MARIADB_USER"),
+ "password": os.getenv("MARIADB_PASSWORD"),
+ "host": os.getenv("MARIADB_SERVER"),
+ "port": os.getenv("MARIADB_PORT"),
+ "database": os.getenv("MARIADB_DB"),
+ }
+ self.database_entry = {
+ "created_by": "Unstract/DBWriter",
+ "created_at": datetime.datetime(2024, 5, 20, 7, 46, 57, 307998),
+ "data": '{"input_file": "simple.pdf", "result": "report"}',
+ }
+ valid_schema_name = "public"
+ invalid_schema_name = "public_1"
+ self.valid_postgres_creds = {**self.postgres_creds, "schema": valid_schema_name}
+ self.invalid_postgres_creds = {
+ **self.postgres_creds,
+ "schema": invalid_schema_name,
+ }
+ self.valid_redshift_creds = {**self.redshift_creds, "schema": valid_schema_name}
+ self.invalid_redshift_creds = {
+ **self.redshift_creds,
+ "schema": invalid_schema_name,
+ }
+ self.invalid_syntax_table_name = "invalid-syntax.name.test_output"
+ self.invalid_wrong_table_name = "database.schema.test_output"
+ self.valid_table_name = "test_output"
+ bigquery_json_str = os.getenv("BIGQUERY_CREDS", "{}")
+ self.bigquery_settings = json.loads(bigquery_json_str)
+ self.bigquery_settings["json_credentials"] = bigquery_json_str
+ self.valid_bigquery_table_name = "pandoras-tamer.bigquery_test.bigquery_output"
+ self.invalid_snowflake_db = {**self.snowflake_creds, "database": "invalid"}
+ self.invalid_snowflake_schema = {**self.snowflake_creds, "schema": "invalid"}
+ self.invalid_snowflake_warehouse = {
+ **self.snowflake_creds,
+ "warehouse": "invalid",
+ }
+
+ # Gets all valid db instances except
+ # Bigquery (table name needs to be writted separately for bigquery)
+ @pytest.fixture(
+ params=[
+ ("valid_postgres_creds", PostgreSQL),
+ ("snowflake_creds", SnowflakeDB),
+ ("mssql_creds", MSSQL),
+ ("mysql_creds", MySQL),
+ ("mariadb_creds", MariaDB),
+ ("valid_redshift_creds", Redshift),
+ ]
+ )
+ def valid_dbs_instance(self, request: Any) -> Any:
+ return self.get_db_instance(request=request)
+
+ # Gets all valid db instances except:
+ # Bigquery (table name needs to be writted separately for bigquery)
+ # Redshift (can't process more than 64KB character type)
+ @pytest.fixture(
+ params=[
+ ("valid_postgres_creds", PostgreSQL),
+ ("snowflake_creds", SnowflakeDB),
+ ("mssql_creds", MSSQL),
+ ("mysql_creds", MySQL),
+ ("mariadb_creds", MariaDB),
+ ]
+ )
+ def valid_dbs_instance_to_handle_large_doc(self, request: Any) -> Any:
+ return self.get_db_instance(request=request)
+
+ def get_db_instance(self, request: Any) -> UnstractDB:
+ creds_name, db_class = request.param
+ creds = getattr(self, creds_name)
+ if not creds:
+ pytest.fail(f"Unknown credentials: {creds_name}")
+ db_instance = db_class(settings=creds)
+ return db_instance
+
+ # Gets all invalid-db instances for postgres, redshift:
+ @pytest.fixture(
+ params=[
+ ("invalid_postgres_creds", PostgreSQL),
+ ("invalid_redshift_creds", Redshift),
+ ]
+ )
+ def invalid_dbs_instance(self, request: Any) -> Any:
+ return self.get_db_instance(request=request)
+
+ @pytest.fixture
+ def valid_bigquery_db_instance(self) -> Any:
+ return BigQuery(settings=self.bigquery_settings)
+
+ # Gets all invalid-db instances for snowflake:
+ @pytest.fixture(
+ params=[
+ ("invalid_snowflake_db", SnowflakeDB),
+ ("invalid_snowflake_schema", SnowflakeDB),
+ ("invalid_snowflake_warehouse", SnowflakeDB),
+ ]
+ )
+ def invalid_snowflake_db_instance(self, request: Any) -> Any:
+ return self.get_db_instance(request=request)
diff --git a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/static/large_doc.txt b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/static/large_doc.txt
new file mode 100644
index 000000000..3a3b67a00
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/static/large_doc.txt
@@ -0,0 +1 @@
+"\n\n UNITED STATES \n SECURITIES AND EXCHANGE COMMISSION \n Washington, D.C. 20549 \n\n FORM 10-Q \n\n(Mark One) \n [X] X QUARTERLY REPORT PURSUANT TO SECTION 13 OR 15(d) OF THE SECURITIES EXCHANGE ACT OF 1934 \n For the quarterly period ended December 30, 2023 \n or \n [ ] TRANSITION REPORT PURSUANT TO SECTION 13 OR 15(d) OF THE SECURITIES EXCHANGE ACT OF 1934 \n For the transition period from to \n Commission File Number: 001-36743 \n\n Apple Inc. \n (Exact name of Registrant as specified in its charter) \n\n California 94-2404110 \n (State or other jurisdiction (I.R.S. Employer Identification No.) \n of incorporation or organization) \n\n One Apple Park Way \n Cupertino, California 95014 \n (Address of principal executive offices) (Zip Code) \n (408) 996-1010 \n (Registrant\'s telephone number, including area code) \n\n Securities registered pursuant to Section 12(b) of the Act \n\n Title of each class symbol(s) Trading Name of each exchange on which registered \n Common Stock, $0.00001 par value per share AAPL The Nasdaq Stock Market LLC \n 0.000% Notes due 2025 The Nasdaq Stock Market LLC \n 0.875% Notes due 2025 The Nasdaq Stock Market LLC \n 1.625% Notes due 2026 The Nasdaq Stock Market LLC \n 2.000% Notes due 2027 The Nasdaq Stock Market LLC \n 1.375% Notes due 2029 The Nasdaq Stock Market LLC \n 3.050% Notes due 2029 The Nasdaq Stock Market LLC \n 0.500% Notes due 2031 The Nasdaq Stock Market LLC \n 3.600% Notes due 2042 The Nasdaq Stock Market LLC \n\nIndicate by check mark whether the Registrant (1) has filed all reports required to be filed by Section 13 or 15(d) of the Securities Exchange Act \nof 1934 during the preceding 12 months (or for such shorter period that the Registrant was required to file such reports), and (2) has been \nsubject to such filing requirements for the past 90 days. \n Yes [X] No [ ] \n\nIndicate by check mark whether the Registrant has submitted electronically every Interactive Data File required to be submitted pursuant to Rule \n405 of Regulation S-T (§232.405 of this chapter) during the preceding 12 months (or for such shorter period that the Registrant was required to \nsubmit such files). \n Yes [X] No [ ] \n<<<\n\n\nIndicate by check mark whether the Registrant is a large accelerated filer, an accelerated filer, a non-accelerated filer, a smaller reporting \ncompany, or an emerging growth company. See the definitions of "large accelerated filer," "accelerated filer," "smaller reporting company," and \n"emerging growth company" in Rule 12b-2 of the Exchange Act. \n\n Large accelerated filer [X] Accelerated filer [ ] \n Non-accelerated filer [ ] Smaller reporting company [ ] \n Emerging growth company [ ] \n\nIf an emerging growth company, indicate by check mark if the Registrant has elected not to use the extended transition period for complying with \nany new or revised financial accounting standards provided pursuant to Section 13(a) of the Exchange Act. [ ] \n\nIndicate by check mark whether the Registrant is a shell company (as defined in Rule 12b-2 of the Exchange Act). \n Yes [ ] No [X] \n\n 15,441,881,000 shares of common stock were issued and outstanding as of January 19, 2024. \n<<<\n\n\n Apple Inc. \n\n Form 10-Q \n\n For the Fiscal Quarter Ended December 30, 2023 \n TABLE OF CONTENTS \n\n Page \n Part I \nItem 1. Financial Statements 1 \nItem 2. Management\'s Discussion and Analysis of Financial Condition and Results of Operations 13 \nItem 3. Quantitative and Qualitative Disclosures About Market Risk 18 \nItem 4. Controls and Procedures 18 \n Part II \nItem 1. Legal Proceedings 19 \nItem 1A. Risk Factors 19 \nItem 2. Unregistered Sales of Equity Securities and Use of Proceeds 20 \nItem 3. Defaults Upon Senior Securities 21 \nItem 4. Mine Safety Disclosures 21 \nItem 5. Other Information 21 \nItem 6. Exhibits 21 \n<<<\n\n\nPARTI - FINANCIAL INFORMATION \n\nItem 1. Financial Statements \n\n Apple Inc. \n\n CONDENSED CONSOLIDATED STATEMENTS OF OPERATIONS (Unaudited) \n (In millions, except number of shares, which are reflected in thousands, and per-share amounts) \n\n Three Months Ended \n December 2023 30, December 2022 31, \n\n Net sales: \n Products $ 96,458 $ 96,388 \n Services 23,117 20,766 \n Total net sales 119,575 117,154 \n\n Cost of sales: \n Products 58,440 60,765 \n Services 6,280 6,057 \n Total cost of sales 64,720 66,822 \n Gross margin 54,855 50,332 \n\n Operating expenses: \n Research and development 7,696 7,709 \n Selling, general and administrative 6,786 6,607 \n Total operating expenses 14,482 14,316 \n\n Operating income 40,373 36,016 \n Other income/(expense), net (50) (393) \n Income before provision for income taxes 40,323 35,623 \n Provision for income taxes 6,407 5,625 \n Net income $ 33,916 $ 29,998 \n\n Earnings per share: \n Basic $ 2.19 $ 1.89 \n Diluted $ 2.18 $ 1.88 \n\n Shares used in computing earnings per share: \n Basic 15,509,763 15,892,723 \n Diluted 15,576,641 15,955,718 \n\n See accompanying Notes to Condensed Consolidated Financial Statements. \n\n Apple Inc. IQ1 2024 Form 10-Q | 1 \n<<<\n\n\n Apple Inc. \n\n CONDENSED CONSOLIDATED STATEMENTS OF COMPREHENSIVE INCOME (Unaudited) \n (In millions) \n\n Three Months Ended \n December 2023 30, December 2022 31, \n\nNet income $ 33,916 $ 29,998 \nOther comprehensive income/(loss): \n Change in foreign currency translation, net of tax 308 (14) \n\n Change in unrealized gains/losses on derivative instruments, net of tax: \n Change in fair value of derivative instruments (531) (988) \n Adjustment for net (gains)/losses realized and included in net income (823) (1,766) \n Total change in unrealized gains/losses on derivative instruments (1,354) (2,754) \n\n Change in unrealized gains/losses on marketable debt securities, net of tax: \n Change in fair value of marketable debt securities 3,045 900 \n Adjustment for net (gains)/losses realized and included in net income 75 65 \n Total change in unrealized gains/losses on marketable debt securities 3,120 965 \n\nTotal other comprehensive income/(loss) 2,074 (1,803) \nTotal comprehensive income $ 35,990 $ 28,195 \n\n See accompanying Notes to Condensed Consolidated Financial Statements. \n\n Apple Inc. I Q1 2024 Form 10-Q 12 \n<<<\n\n\n Apple Inc. \n\n CONDENSED CONSOLIDATED BALANCE SHEETS (Unaudited) \n (In millions, except number of shares, which are reflected in thousands, and par value) \n\n December 2023 30, September 2023 30, \n\n ASSETS: \nCurrent assets: \n Cash and cash equivalents $ 40,760 $ 29,965 \n Marketable securities 32,340 31,590 \n Accounts receivable, net 23,194 29,508 \n Vendor non-trade receivables 26,908 31,477 \n Inventories 6,511 6,331 \n Other current assets 13,979 14,695 \n Total current assets 143,692 143,566 \n\nNon-current assets: \n Marketable securities 99,475 100,544 \n Property, plant and equipment, net 43,666 43,715 \n Other non-current assets 66,681 64,758 \n Total non-current assets 209,822 209,017 \n Total assets $ 353,514 $ 352,583 \n\n LIABILITIES AND SHAREHOLDERS\' EQUITY: \nCurrent liabilities: \n Accounts payable $ 58,146 $ 62,611 \n Other current liabilities 54,611 58,829 \n Deferred revenue 8,264 8,061 \n Commercial paper 1,998 5,985 \n Term debt 10,954 9,822 \n Total current liabilities 133,973 145,308 \n\nNon-current liabilities : \n Term debt 95,088 95,281 \n Other non-current liabilities 50,353 49,848 \n Total non-current liabilities 145,441 145,129 \n Total liabilities 279,414 290,437 \n\nCommitments and contingencies \n\nShareholders\' equity: \n Common stock and additional paid-in capital, $0.00001 par value: 50,400,000 shares \n authorized; 15,460,223 and 15,550,061 shares issued and outstanding, respectively 75,236 73,812 \n Retained earnings/(Accumulated deficit) 8,242 (214) \n Accumulated other comprehensive loss (9,378) (11,452) \n Total shareholders\' equity 74,100 62,146 \n Total liabilities and shareholders\' equity $ 353,514 $ 352,583 \n\n See accompanying Notes to Condensed Consolidated Financial Statements. \n\n Apple Inc. IQ1 2024 Form 10-Q 13 \n<<<\n\n\n Apple Inc. \n\n CONDENSED CONSOLIDATED STATEMENTS OF SHAREHOLDERS\' EQUITY (Unaudited) \n (In millions, except per-share amounts) \n\n Three Months Ended \n December 2023 30, December 2022 31, \n\nTotal shareholders\' equity, beginning balances $ 62,146 $ 50,672 \n\nCommon stock and additional paid-in capital: \n Beginning balances 73,812 64,849 \n Common stock withheld related to net share settlement of equity awards (1,660) (1,434) \n Share-based compensation 3,084 2,984 \n Ending balances 75,236 66,399 \n\nRetained earnings/(Accumulated deficit): \n Beginning balances (214) (3,068) \n Net income 33,916 29,998 \n Dividends and dividend equivalents declared (3,774) (3,712) \n Common stock withheld related to net share settlement of equity awards (1,018) (978) \n Common stock repurchased (20,668) (19,000) \n Ending balances 8,242 3,240 \n\nAccumulated other comprehensive income/(loss): \n Beginning balances (11,452) (11,109) \n Other comprehensive income/(loss) 2,074 (1,803) \n Ending balances (9,378) (12,912) \n\nTotal shareholders\' equity, ending balances $ 74,100 $ 56,727 \n\nDividends and dividend equivalents declared per share or RSU $ 0.24 $ 0.23 \n\n See accompanying Notes to Condensed Consolidated Financial Statements. \n\n Apple Inc. 2024 Form 10-Q 14 \n<<<\n\n\n Apple Inc. \n\n CONDENSED CONSOLIDATED STATEMENTS OF CASH FLOWS (Unaudited) \n (In millions) \n\n Three Months Ended \n December 2023 30, December 2022 31, \n\nCash, cash equivalents and restricted cash, beginning balances $ 30,737 $ 24,977 \n\nOperating activities: \n Net income 33,916 29,998 \n Adjustments to reconcile net income to cash generated by operating activities: \n Depreciation and amortization 2,848 2,916 \n Share-based compensation expense 2,997 2,905 \n Other (989) (317) \n Changes in operating assets and liabilities: \n Accounts receivable, net 6,555 4,275 \n Vendor non-trade receivables 4,569 2,320 \n Inventories (137) (1,807) \n Other current and non-current assets (1,457) (4,099) \n Accounts payable (4,542) (6,075) \n Other current and non-current liabilities (3,865) 3,889 \n Cash generated by operating activities 39,895 34,005 \n\nInvesting activities: \n Purchases of marketable securities (9,780) (5,153) \n Proceeds from maturities of marketable securities 13,046 7,127 \n Proceeds from sales of marketable securities 1,337 509 \n Payments for acquisition of property, plant and equipment (2,392) (3,787) \n Other (284) (141) \n Cash generated by/(used in) investing activities 1,927 (1,445) \n\nFinancing activities: \n Payments for taxes related to net share settlement of equity awards (2,591) (2,316) \n Payments for dividends and dividend equivalents (3,825) (3,768) \n Repurchases of common stock (20,139) (19,475) \n Repayments of term debt (1,401) \n Repayments of commercial paper, net (3,984) (8,214) \n Other (46) (389) \n Cash used in financing activities (30,585) (35,563) \n\nIncrease/(Decrease) in cash, cash equivalents and restricted cash 11,237 (3,003) \nCash, cash equivalents and restricted cash, ending balances $ 41,974 $ 21,974 \n\nSupplemental cash flow disclosure: \n Cash paid for income taxes, net $ 7,255 $ 828 \n\n See accompanying Notes to Condensed Consolidated Financial Statements. \n\n Apple Inc. IQ1 2024 Form 10-Q 1 5 \n<<<\n\n\n Apple Inc. \n\n Notes to Condensed Consolidated Financial Statements (Unaudited) \n\nNote 1 - Summary of Significant Accounting Policies \n\nBasis of Presentation and Preparation \nThe condensed consolidated financial statements include the accounts of Apple Inc. and its wholly owned subsidiaries \n(collectively "Apple" or the "Company"). In the opinion of the Company\'s management, the condensed consolidated financial \nstatements reflect all adjustments, which are normal and recurring in nature, necessary for fair financial statement presentation. \nThe preparation of these condensed consolidated financial statements and accompanying notes in conformity with U.S. generally \naccepted accounting principles ("GAAP") requires the use of management estimates. Certain prior period amounts in the \ncondensed consolidated financial statements and accompanying notes have been reclassified to conform to the current period\'s \npresentation. These condensed consolidated financial statements and accompanying notes should be read in conjunction with \nthe Company\'s annual consolidated financial statements and accompanying notes included in its Annual Report on Form 10-K \nfor the fiscal year ended September 30, 2023 (the "2023 Form 10-K"). \n\nThe Company\'s fiscal year is the 52- or 53-week period that ends on the last Saturday of September. An additional week is \nincluded in the first fiscal quarter every five or six years to realign the Company\'s fiscal quarters with calendar quarters, which \noccurred in the first fiscal quarter of 2023. The Company\'s fiscal years 2024 and 2023 span 52 and 53 weeks, respectively. \nUnless otherwise stated, references to particular years, quarters, months and periods refer to the Company\'s fiscal years ended \nin September and the associated quarters, months and periods of those fiscal years. \n\nNote 2 - Revenue \nNet sales disaggregated by significant products and services for the three months ended December 30, 2023 and December 31, \n2022 were as follows (in millions): \n Three Months Ended \n December 2023 30, December 2022 31, \n iPhone® $ 69,702 $ 65,775 \n Mac® 7,780 7,735 \n iPad® 7,023 9,396 \n Wearables, Home and Accessories 11,953 13,482 \n Services 23,117 20,766 \n Total net sales $ 119,575 $ 117,154 \n\nTotal net sales include $3.5 billion of revenue recognized in the three months ended December 30, 2023 that was included in \ndeferred revenue as of September 30, 2023 and $3.4 billion of revenue recognized in the three months ended December 31, \n2022 that was included in deferred revenue as of September 24, 2022. \n\nThe Company\'s proportion of net sales by disaggregated revenue source was generally consistent for each reportable segment \nin Note 10, "Segment Information and Geographic Data" for the three months ended December 30, 2023 and December 31, \n2022, except in Greater China, where iPhone revenue represented a moderately higher proportion of net sales. \n\nAs of December 30, 2023 and September 30, 2023, the Company had total deferred revenue of $12.5 billion and $12.1 billion, \nrespectively. As of December 30, 2023, the Company expects 66% of total deferred revenue to be realized in less than a year, \n26% within one-to-two years, 7% within two-to-three years and 1% in greater than three years. \n\n Apple Inc. I Q1 2024 Form 10-Q | 6 \n<<<\n\n\nNote 3 - Earnings Per Share \nThe following table shows the computation of basic and diluted earnings per share for the three months ended December 30, \n2023 and December 31, 2022 (net income in millions and shares in thousands): \n Three Months Ended \n December 2023 30, December 2022 31, \n\nNumerator: \n Net income $ 33,916 $ 29,998 \n\nDenominator: \n Weighted-average basic shares outstanding 15,509,763 15,892,723 \n Effect of dilutive share-based awards 66,878 62,995 \n Weighted-average diluted shares 15,576,641 15,955,718 \n\nBasic earnings per share $ 2.19 $ 1.89 \nDiluted earnings per share $ 2.18 $ 1.88 \n\nApproximately 89 million restricted stock units ("RSUs") were excluded from the computation of diluted earnings per share for the \nthree months ended December 31, 2022 because their effect would have been antidilutive. \n\nNote 4 - Financial Instruments \n\nCash, Cash Equivalents and Marketable Securities \nThe following tables show the Company\'s cash, cash equivalents and marketable securities by significant investment category \nas of December 30, 2023 and September 30, 2023 (in millions): \n December 30, 2023 \n Cash and Current Non-Current \n Adjusted Cost Unrealized Unrealized Fair Cash Marketable Marketable \n Gains Losses Value Equivalents Securities Securities \n Cash $ 29,542 $ $ - $ 29,542 $ 29,542 $ $ \n\nLevel 1: \n Money market funds 2,000 2,000 2,000 \n Mutual funds 448 35 (11) 472 472 \n Subtotal 2,448 35 (11) 2,472 2,000 472 \n\nLevel 2 (1): \n U.S. Treasury securities 24,041 12 (920) 23,133 7,303 4,858 10,972 \n U.S. agency securities 5,791 (448) 5,343 243 98 5,002 \n Non-U.S. government securities 17,326 54 (675) 16,705 11,175 5,530 \n Certificates of deposit and time deposits 1,448 - - 1,448 1,119 329 \n Commercial paper 1,361 1,361 472 889 \n Corporate debt securities 75,360 112 (3,964) 71,508 81 13,909 57,518 \n Municipal securities 562 (14) 548 185 363 \n Mortgage- and asset-backed securities 22,369 53 (1,907) 20,515 425 20,090 \n Subtotal 148,258 231 (7,928) 140,561 9,218 31,868 99,475 \n Total (2) $ 180,248 $ 266 $ (7,939) $ 172,575 $ 40,760 $ 32,340 $ 99,475 \n\n Apple Inc. IQ1 2024 Form 10-Q 1 7 \n<<<\n\n\n September 30, 2023 \n Cash and Current Non-Current \n Adjusted Cost Unrealized Gains Unrealized Value Fair Cash Marketable Marketable Securities \n Losses Equivalents Securities \n Cash $ 28,359 $ $ $ 28,359 $ 28,359 $ $ \n Level 1: \n Money market funds 481 481 481 \n Mutual funds and equity securities 442 12 (26) 428 428 \n Subtotal 923 12 (26) 909 481 428 \n Level 2 (1): \n U.S. Treasury securities 19,406 (1,292) 18,114 35 5,468 12,611 \n U.S. agency securities 5,736 (600) 5,136 36 271 4,829 \n Non-U.S. government securities 17,533 6 (1,048) 16,491 11,332 5,159 \n Certificates of deposit and time deposits 1,354 - 1,354 1,034 320 \n Commercial paper 608 608 608 \n Corporate debt securities 76,840 6 (5,956) 70,890 20 12,627 58,243 \n Municipal securities 628 (26) 602 192 410 \n Mortgage- and asset-backed securities 22,365 6 (2,735) 19,636 344 19,292 \n Subtotal 144,470 18 (11,657) 132,831 1,125 31,162 100,544 \n Total (2) $ 173,752 $ 30 $ (11,683) $ 162,099 $ 29,965 $ 31,590 $ 100,544 \n\n (1) The valuation techniques used to measure the fair values of the Company\'s Level 2 financial instruments, which generally \n have counterparties with high credit ratings, are based on quoted market prices or model-driven valuations using significant \n inputs derived from or corroborated by observable market data. \n (2) As of December 30, 2023 and September 30, 2023, total marketable securities included $13.9 billion and $13.8 billion, \n respectively, that were restricted from general use, related to the European Commission decision finding that Ireland granted \n state aid to the Company, and other agreements. \n\nThe following table shows the fair value of the Company\'s non-current marketable debt securities, by contractual maturity, as of \nDecember 30, 2023 (in millions): \n\n Due after 1 year through 5 years $ 72,994 \nDue after 5 years through 10 years 9,368 \nDue after 10 years 17,113 \n Total fair value $ 99,475 \n\nDerivative Instruments and Hedging \nThe Company may use derivative instruments to partially offset its business exposure to foreign exchange and interest rate risk. \nHowever, the Company may choose not to hedge certain exposures for a variety of reasons, including accounting considerations \nor the prohibitive economic cost of hedging particular exposures. There can be no assurance the hedges will offset more than a \nportion of the financial impact resulting from movements in foreign exchange or interest rates. \n\nForeign Exchange Rate Risk \nTo protect gross margins from fluctuations in foreign exchange rates, the Company may use forwards, options or other \ninstruments, and may designate these instruments as cash flow hedges. The Company generally hedges portions of its \nforecasted foreign currency exposure associated with revenue and inventory purchases, typically for up to 12 months. \n\nTo protect the Company\'s foreign currency-denominated term debt or marketable securities from fluctuations in foreign \nexchange rates, the Company may use forwards, cross-currency swaps or other instruments. The Company designates these \ninstruments as either cash flow or fair value hedges. As of December 30, 2023, the maximum length of time over which the \nCompany is hedging its exposure to the variability in future cash flows for term debt-related foreign currency transactions is 19 \nyears. \n\nThe Company may also use derivative instruments that are not designated as accounting hedges to protect gross margins from \ncertain fluctuations in foreign exchange rates, as well as to offset a portion of the foreign currency gains and losses generated by \nthe remeasurement of certain assets and liabilities denominated in non-functional currencies. \n\n Apple Inc. IQ1 2024 Form 10-Q 18 \n<<<\n\n\nInterest Rate Risk \nTo protect the Company\'s term debt or marketable securities from fluctuations in interest rates, the Company may use interest \nrate swaps, options or other instruments. The Company designates these instruments as either cash flow or fair value hedges. \n\nThe notional amounts of the Company\'s outstanding derivative instruments as of December 30, 2023 and September 30, 2023 \nwere as follows (in millions): \n December 2023 30, September 2023 30, \n\n Derivative instruments designated as accounting hedges: \n Foreign exchange contracts $ 66,735 $ 74,730 \n Interest rate contracts $ 19,375 $ 19,375 \n\n Derivative instruments not designated as accounting hedges : \n Foreign exchange contracts $ 102,108 $ 104,777 \n\n The carrying amounts of the Company\'s hedged items in fair value hedges as of December 30, 2023 and September 30, 2023 \nwere as follows (in millions): \n December 2023 30, September 2023 30, \n\n Hedged assets/(liabilities): \n Current and non-current marketable securities $ 15,102 $ 14,433 \n Current and non-current term debt $ (18,661) $ (18,247) \n\nAccounts Receivable \n\n Trade Receivables \n The Company\'s third-party cellular network carriers accounted for 34% and 41% of total trade receivables as of December 30, \n2023 and September 30, 2023, respectively. The Company requires third-party credit support or collateral from certain \ncustomers to limit credit risk. \n\n Vendor Non-Trade Receivables \nThe Company has non-trade receivables from certain of its manufacturing vendors resulting from the sale of components to \nthese vendors who manufacture subassemblies or assemble final products for the Company. The Company purchases these \ncomponents directly from suppliers. The Company does not reflect the sale of these components in products net sales. Rather, \nthe Company recognizes any gain on these sales as a reduction of products cost of sales when the related final products are \nsold by the Company. As of December 30, 2023, the Company had two vendors that individually represented 10% or more of \ntotal vendor non-trade receivables, which accounted for 50% and 20%. As of September 30, 2023, the Company had two \nvendors that individually represented 10% or more of total vendor non-trade receivables, which accounted for 48% and 23%. \n\nNote 5 - Condensed Consolidated Financial Statement Details \nThe following table shows the Company\'s condensed consolidated financial statement details as of December 30, 2023 and \nSeptember 30, 2023 (in millions): \n\nProperty, Plant and Equipment, Net \n December 2023 30, September 2023 30, \n\n Gross property, plant and equipment $ 116,176 $ 114,599 \n Accumulated depreciation (72,510) (70,884) \n Total property, plant and equipment, net $ 43,666 $ 43,715 \n\n Apple Inc. IQ1 2024 Form 10-Q 19 \n<<<\n\n\nNote 6 - Debt \n\nCommercial Paper \nThe Company issues unsecured short-term promissory notes pursuant to a commercial paper program. The Company uses net \nproceeds from the commercial paper program for general corporate purposes, including dividends and share repurchases. As of \nDecember 30, 2023 and September 30, 2023, the Company had $2.0 billion and $6.0 billion of commercial paper outstanding, \nrespectively. The following table provides a summary of cash flows associated with the issuance and maturities of commercial \npaper for the three months ended December 30, 2023 and December 31, 2022 (in millions): \n Three Months Ended \n December 2023 30, December 2022 31, \n\n Maturities 90 days or less: \n Repayments of commercial paper, net $ (3,984) $ (5,569) \n\n Maturities greater than 90 days: \n Repayments of commercial paper - (2,645) \n\n Total repayments of commercial paper, net $ (3,984) $ (8,214) \n\nTerm Debt \nAs of December 30, 2023 and September 30, 2023, the Company had outstanding fixed-rate notes with varying maturities for an \naggregate carrying amount of $106.0 billion and $105.1 billion, respectively (collectively the "Notes"). As of December 30, 2023 \nand September 30, 2023, the fair value of the Company\'s Notes, based on Level 2 inputs, was $96.7 billion and $90.8 billion, \nrespectively. \n\nNote 7 - Shareholders\' Equity \n\nShare Repurchase Program \nDuring the three months ended December 30, 2023, the Company repurchased 118 million shares of its common stock for $20.5 \nbillion. The Company\'s share repurchase program does not obligate the Company to acquire a minimum amount of shares. \nUnder the program, shares may be repurchased in privately negotiated or open market transactions, including under plans \ncomplying with Rule 10b5-1 under the Securities Exchange Act of 1934, as amended (the "Exchange Act"). \n\nNote 8 - Share-Based Compensation \n\nRestricted Stock Units \nA summary of the Company\'s RSU activity and related information for the three months ended December 30, 2023 is as follows: \n Number RSUs of Weighted-Average Grant Date Fair Aggregate Fair Value \n (in thousands) Value Per RSU (in millions) \n Balance as of September 30, 2023 180,247 $ 135.91 \n RSUs granted 74,241 $ 171.58 \n RSUs vested (42,490) $ 110.75 \n RSUs canceled (3,026) $ 109.05 \n Balance as of December 30, 2023 208,972 $ 154.09 $ 40,233 \n\nThe fair value as of the respective vesting dates of RSUs was $7.7 billion and $6.8 billion for the three months ended December \n30, 2023 and December 31, 2022, respectively. \n\n Apple Inc. I Q1 2024 Form 10-Q | 10 \n<<<\n\n\nShare-Based Compensation \nThe following table shows share-based compensation expense and the related income tax benefit included in the Condensed \nConsolidated Statements of Operations for the three months ended December 30, 2023 and December 31, 2022 (in millions): \n Three Months Ended \n December 2023 30, December 2022 31, \n\nShare-based compensation expense $ 2,997 $ 2,905 \nIncome tax benefit related to share-based compensation expense $ (1,235) $ (1,178) \n\nAs of December 30, 2023, the total unrecognized compensation cost related to outstanding RSUs was $27.4 billion, which the \nCompany expects to recognize over a weighted-average period of 2.9 years. \n\nNote 9 - Contingencies \nThe Company is subject to various legal proceedings and claims that have arisen in the ordinary course of business and that \nhave not been fully resolved. The outcome of litigation is inherently uncertain. In the opinion of management, there was not at \nleast a reasonable possibility the Company may have incurred a material loss, or a material loss greater than a recorded accrual, \nconcerning loss contingencies for asserted legal and other claims. \n\nNote 10 - Segment Information and Geographic Data \nThe following table shows information by reportable segment for the three months ended December 30, 2023 and December 31, \n2022 (in millions): \n Three Months Ended \n December 2023 30, December 2022 31, \n\nAmericas: \n Net sales $ 50,430 $ 49,278 \n Operating income $ 20,357 $ 17,864 \n\n Europe: \n Net sales $ 30,397 $ 27,681 \n Operating income $ 12,711 $ 10,017 \n\nGreater China: \n Net sales $ 20,819 $ 23,905 \n Operating income $ 8,622 $ 10,437 \n\nJapan: \n Net sales $ 7,767 $ 6,755 \n Operating income $ 3,819 $ 3,236 \n\nRest of Asia Pacific: \n Net sales $ 10,162 $ 9,535 \n Operating income $ 4,579 $ 3,851 \n\n Apple Inc. I Q1 2024 Form 10-Q | 11 \n<<<\n\n\nA reconciliation of the Company\'s segment operating income to the Condensed Consolidated Statements of Operations for the \nthree months ended December 30, 2023 and December 31, 2022 is as follows (in millions): \n Three Months Ended \n December 2023 30, December 2022 31, \n\n Segment operating income $ 50,088 $ 45,405 \n Research and development expense (7,696) (7,709) \n Other corporate expenses, net (2,019) (1,680) \n Total operating income $ 40,373 $ 36,016 \n\n Apple Inc. I Q1 2024 Form 10-Q | 12 \n<<<\n\n\nItem 2. Management\'s Discussion and Analysis of Financial Condition and Results of Operations \n\nThis Item and other sections of this Quarterly Report on Form 10-Q ("Form 10-Q") contain forward-looking statements, within \nthe meaning of the Private Securities Litigation Reform Act of 1995, that involve risks and uncertainties. Forward-looking \nstatements provide current expectations of future events based on certain assumptions and include any statement that does \nnot directly relate to any historical or current fact. For example, statements in this Form 10-Q regarding the potential future \nimpact of macroeconomic conditions on the Company\'s business and results of operations are forward-looking statements. \nForward-looking statements can also be identified by words such as "future," "anticipates," "believes," "estimates," "expects," \n"intends," "plans," "predicts," "will," "would," "could," "can," "may," and similar terms. Forward-looking statements are not \nguarantees of future performance and the Company\'s actual results may differ significantly from the results discussed in the \nforward-looking statements. Factors that might cause such differences include, but are not limited to, those discussed in Part I, \nItem 1A of the 2023 Form 10-K under the heading "Risk Factors." The Company assumes no obligation to revise or update any \nforward-looking statements for any reason, except as required by law. \n\nUnless otherwise stated, all information presented herein is based on the Company\'s fiscal calendar, and references to \nparticular years, quarters, months or periods refer to the Company\'s fiscal years ended in September and the associated \nquarters, months and periods of those fiscal years. \n\nThe following discussion should be read in conjunction with the 2023 Form 10-K filed with the U.S. Securities and Exchange \nCommission (the "SEC") and the condensed consolidated financial statements and accompanying notes included in Part I, \nItem 1 of this Form 10-Q. \n\nAvailable Information \nThe Company periodically provides certain information for investors on its corporate website, www.apple.com, and its investor \nrelations website, investor.apple.com. This includes press releases and other information about financial performance, \ninformation on environmental, social and governance matters, and details related to the Company\'s annual meeting of \nshareholders. The information contained on the websites referenced in this Form 10-Q is not incorporated by reference into this \nfiling. Further, the Company\'s references to website URLs are intended to be inactive textual references only. \n\nBusiness Seasonality and Product Introductions \nThe Company has historically experienced higher net sales in its first quarter compared to other quarters in its fiscal year due in \npart to seasonal holiday demand. Additionally, new product and service introductions can significantly impact net sales, cost of \nsales and operating expenses. The timing of product introductions can also impact the Company\'s net sales to its indirect \ndistribution channels as these channels are filled with new inventory following a product launch, and channel inventory of an \nolder product often declines as the launch of a newer product approaches. Net sales can also be affected when consumers and \ndistributors anticipate a product introduction. \n\nFiscal Period \nThe Company\'s fiscal year is the 52- or 53-week period that ends on the last Saturday of September. An additional week is \nincluded in the first fiscal quarter every five or six years to realign the Company\'s fiscal quarters with calendar quarters, which \noccurred in the first quarter of 2023. The Company\'s fiscal years 2024 and 2023 span 52 and 53 weeks, respectively. \n\nQuarterly Highlights \nThe Company\'s first quarter of 2024 included 13 weeks, compared to 14 weeks during the first quarter of 2023. \n\nThe Company\'s total net sales increased 2% or $2.4 billion during the first quarter of 2024 compared to the same quarter in \n2023, driven primarily by higher net sales of iPhone and Services, partially offset by lower net sales of iPad and Wearables, \nHome and Accessories. \n\nDuring the first quarter of 2024, the Company announced an updated MacBook Pro® 14-in., MacBook Pro 16-in. and iMac®. \n\nThe Company repurchased $20.5 billion of its common stock and paid dividends and dividend equivalents of $3.8 billion during \nthe first quarter of 2024. \n\nMacroeconomic Conditions \nMacroeconomic conditions, including inflation, changes in interest rates, and currency fluctuations, have directly and indirectly \nimpacted, and could in the future materially impact, the Company\'s results of operations and financial condition. \n\n Apple Inc. I Q1 2024 Form 10-Q | 13 \n<<<\n\n\nSegment Operating Performance \nThe following table shows net sales by reportable segment for the three months ended December 30, 2023 and December 31, \n2022 (dollars in millions): \n Three Months Ended \n December 2023 30, December 2022 31, Change \n\n Net sales by reportable segment: \n Americas $ 50,430 $ 49,278 2 % \n Europe 30,397 27,681 10 % \n Greater China 20,819 23,905 (13)% \n Japan 7,767 6,755 15 % \n Rest of Asia Pacific 10,162 9,535 7 % \n Total net sales $ 119,575 $ 117,154 2 % \n\nAmericas \nAmericas net sales increased 2% or $1.2 billion during the first quarter of 2024 compared to the same quarter in 2023 due \nprimarily to higher net sales of Services and iPhone, partially offset by lower net sales of iPad. The strength in foreign currencies \nrelative to the U.S. dollar had a net favorable year-over-year impact on Americas net sales during the first quarter of 2024. \n\nEurope \nEurope net sales increased 10% or $2.7 billion during the first quarter of 2024 compared to the same quarter in 2023 due \nprimarily to higher net sales of iPhone. The strength in foreign currencies relative to the U.S. dollar had a net favorable year- \nover-year impact on Europe net sales during the first quarter of 2024. \n\nGreater China \nGreater China net sales decreased 13% or $3.1 billion during the first quarter of 2024 compared to the same quarter in 2023 due \nprimarily to lower net sales of iPhone, iPad and Wearables, Home and Accessories. The weakness in the renminbi relative to the \nU.S. dollar had an unfavorable year-over-year impact on Greater China net sales during the first quarter of 2024. \n\nJapan \nJapan net sales increased 15% or $1.0 billion during the first quarter of 2024 compared to the same quarter in 2023 due primarily \nto higher net sales of iPhone. The weakness in the yen relative to the U.S. dollar had an unfavorable year-over-year impact on \nJapan net sales during the first quarter of 2024. \n\nRest of Asia Pacific \nRest of Asia Pacific net sales increased 7% or $627 million during the first quarter of 2024 compared to the same quarter in 2023 \ndue primarily to higher net sales of iPhone, partially offset by lower net sales of Wearables, Home and Accessories. \n\n Apple Inc. I Q1 2024 Form 10-Q | 14 \n<<<\n\n\nProducts and Services Performance \nThe following table shows net sales by category for the three months ended December 30, 2023 and December 31, 2022 \n(dollars in millions): \n Three Months Ended \n December 2023 30, December 2022 31, Change \n\nNet sales by category: \n iPhone $ 69,702 $ 65,775 6 % \n Mac 7,780 7,735 1 % \n iPad 7,023 9,396 (25)% \n Wearables, Home and Accessories 11,953 13,482 (11)% \n Services 23,117 20,766 11 % \n Total net sales $ 119,575 $ 117,154 2 % \n\niPhone \niPhone net sales increased 6% or $3.9 billion during the first quarter of 2024 compared to the same quarter in 2023 due primarily \nto higher net sales of Pro models, partially offset by lower net sales of other models. \n\nMac \nMac net sales were relatively flat during the first quarter of 2024 compared to the same quarter in 2023. \n\niPad \niPad net sales decreased 25% or $2.4 billion during the first quarter of 2024 compared to the same quarter in 2023 due primarily \nto lower net sales of iPad Pro, iPad 9th generation and iPad Air. \n\nWearables, Home and Accessories \nWearables, Home and Accessories net sales decreased 11% or $1.5 billion during the first quarter of 2024 compared to the \nsame quarter in 2023 due primarily to lower net sales of Wearables and Accessories. \n\nServices \nServices net sales increased 11% or $2.4 billion during the first quarter of 2024 compared to the same quarter in 2023 due \nprimarily to higher net sales from advertising, video and cloud services. \n\n Apple Inc. I Q1 2024 Form 10-Q | 15 \n<<<\n\n\nGross Margin \nProducts and Services gross margin and gross margin percentage for the three months ended December 30, 2023 and \nDecember 31, 2022 were as follows (dollars in millions): \n Three Months Ended \n December 2023 30, December 2022 31, \n\nGross margin: \n Products $ 38,018 $ 35,623 \n Services 16,837 14,709 \n Total gross margin $ 54,855 $ 50,332 \n\n Gross margin percentage: \n Products 39.4% 37.0% \n Services 72.8% 70.8% \n Total gross margin percentage 45.9% 43.0% \n\nProducts Gross Margin \nProducts gross margin increased during the first quarter of 2024 compared to the same quarter in 2023 due primarily to cost \nsavings and a different Products mix, partially offset by the weakness in foreign currencies relative to the U.S. dollar and lower \nProducts volume. \n\nProducts gross margin percentage increased during the first quarter of 2024 compared to the same quarter in 2023 due primarily \nto cost savings and a different Products mix, partially offset by the weakness in foreign currencies relative to the U.S. dollar. \n\nServices Gross Margin \nServices gross margin increased during the first quarter of 2024 compared to the same quarter in 2023 due primarily to higher \nServices net sales and a different Services mix. \n\nServices gross margin percentage increased during the first quarter of 2024 compared to the same quarter in 2023 due primarily \nto a different Services mix. \n\nThe Company\'s future gross margins can be impacted by a variety of factors, as discussed in Part I, Item 1A of the 2023 Form \n10-K under the heading "Risk Factors." As a result, the Company believes, in general, gross margins will be subject to volatility \nand downward pressure. \n\n Apple Inc. I Q1 2024 Form 10-Q | 16 \n<<<\n\n\nOperating Expenses \nOperating expenses for the three months ended December 30, 2023 and December 31, 2022 were as follows (dollars in \nmillions): \n Three Months Ended \n December 2023 30, December 2022 31, \n\n Research and development $ 7,696 $ 7,709 \n Percentage of total net sales 6% 7% \n\n Selling, general and administrative $ 6,786 $ 6,607 \n Percentage of total net sales 6% 6% \n Total operating expenses $ 14,482 $ 14,316 \n Percentage of total net sales 12% 12% \n\nResearch and Development \nResearch and development ("R&D") expense was relatively flat during the first quarter of 2024 compared to the same quarter in \n2023. \n\nSelling, General and Administrative \nSelling, general and administrative expense increased 3% or $179 million during the first quarter of 2024 compared to the same \nquarter in 2023. \n\nProvision for Income Taxes \nProvision for income taxes, effective tax rate and statutory federal income tax rate for the three months ended December 30, \n2023 and December 31, 2022 were as follows (dollars in millions): \n Three Months Ended \n December 2023 30, December 2022 31, \n\n Provision for income taxes $ 6,407 $ 5,625 \n Effective tax rate 15.9% 15.8% \n Statutory federal income tax rate 21% 21% \n\nThe Company\'s effective tax rate for the first quarter of 2024 was lower than the statutory federal income tax rate due primarily to \na lower effective tax rate on foreign earnings, tax benefits from share-based compensation, and the impact of the U.S. federal \nR&D credit, partially offset by state income taxes. \n\nThe Company\'s effective tax rate for the first quarter of 2024 was relatively flat compared to the same quarter in 2023. \n\nLiquidity and Capital Resources \nThe Company believes its balances of cash, cash equivalents and unrestricted marketable securities, along with cash generated \nby ongoing operations and continued access to debt markets, will be sufficient to satisfy its cash requirements and capital return \nprogram over the next 12 months and beyond. \n\nThe Company\'s contractual cash requirements have not changed materially since the 2023 Form 10-K, except for manufacturing \npurchase obligations. \n\nManufacturing Purchase Obligations \nThe Company utilizes several outsourcing partners to manufacture subassemblies for the Company\'s products and to perform \nfinal assembly and testing of finished products. The Company also obtains individual components for its products from a wide \nvariety of individual suppliers. As of December 30, 2023, the Company had manufacturing purchase obligations of $38.0 billion, \nwith $37.9 billion payable within 12 months. \n\n Apple Inc. I Q1 2024 Form 10-Q | 17 \n<<<\n\n\nCapital Return Program \nIn addition to its contractual cash requirements, the Company has an authorized share repurchase program. The program does \nnot obligate the Company to acquire a minimum amount of shares. As of December 30, 2023, the Company\'s quarterly cash \ndividend was $0.24 per share. The Company intends to increase its dividend on an annual basis, subject to declaration by the \nBoard of Directors. \n\nRecent Accounting Pronouncements \n\nIncome Taxes \nIn December 2023, the Financial Accounting Standards Board (the "FASB") issued Accounting Standards Update ("ASU") No. \n2023-09, Income Taxes (Topic 740): Improvements to Income Tax Disclosures ("ASU 2023-09"), which will require the Company \nto disclose specified additional information in its income tax rate reconciliation and provide additional information for reconciling \nitems that meet a quantitative threshold. ASU 2023-09 will also require the Company to disaggregate its income taxes paid \ndisclosure by federal, state and foreign taxes, with further disaggregation required for significant individual jurisdictions. The \nCompany will adopt ASU 2023-09 in its fourth quarter of 2026. ASU 2023-09 allows for adoption using either a prospective or \nretrospective transition method. \n\nSegment Reporting \nIn November 2023, the FASB issued ASU No. 2023-07, Segment Reporting (Topic 280): Improvements to Reportable Segment \nDisclosures ("ASU 2023-07\'), which will require the Company to disclose segment expenses that are significant and regularly \nprovided to the Company\'s chief operating decision maker ("CODM"). In addition, ASU 2023-07 will require the Company to \ndisclose the title and position of its CODM and how the CODM uses segment profit or loss information in assessing segment \nperformance and deciding how to allocate resources. The Company will adopt ASU 2023-07 in its fourth quarter of 2025 using a \nretrospective transition method. \n\nCritical Accounting Estimates \nThe preparation of financial statements and related disclosures in conformity with GAAP and the Company\'s discussion and \nanalysis of its financial condition and operating results require the Company\'s management to make judgments, assumptions \nand estimates that affect the amounts reported. Note 1, "Summary of Significant Accounting Policies" of the Notes to Condensed \nConsolidated Financial Statements in Part I, Item 1 of this Form 10-Q and in the Notes to Consolidated Financial Statements in \nPart II, Item 8 of the 2023 Form 10-K describe the significant accounting policies and methods used in the preparation of the \nCompany\'s condensed consolidated financial statements. There have been no material changes to the Company\'s critical \naccounting estimates since the 2023 Form 10-K. \n\nItem 3. Quantitative and Qualitative Disclosures About Market Risk \n\nThere have been no material changes to the Company\'s market risk during the first three months of 2024. For a discussion of the \nCompany\'s exposure to market risk, refer to the Company\'s market risk disclosures set forth in Part II, Item 7A, "Quantitative and \nQualitative Disclosures About Market Risk" of the 2023 Form 10-K. \n\nItem 4. Controls and Procedures \n\nEvaluation of Disclosure Controls and Procedures \nBased on an evaluation under the supervision and with the participation of the Company\'s management, the Company\'s principal \nexecutive officer and principal financial officer have concluded that the Company\'s disclosure controls and procedures as defined \nin Rules 13a-15(e) and 15d-15(e) under the Exchange Act were effective as of December 30, 2023 to provide reasonable \nassurance that information required to be disclosed by the Company in reports that it files or submits under the Exchange Act is \n(i) recorded, processed, summarized and reported within the time periods specified in the SEC rules and forms and \n(ii) accumulated and communicated to the Company\'s management, including its principal executive officer and principal \nfinancial officer, as appropriate to allow timely decisions regarding required disclosure. \n\nChanges in Internal Control over Financial Reporting \nThere were no changes in the Company\'s internal control over financial reporting during the first quarter of 2024, which were \nidentified in connection with management\'s evaluation required by paragraph (d) of Rules 13a-15 and 15d-15 under the \nExchange Act, that have materially affected, or are reasonably likely to materially affect, the Company\'s internal control over \nfinancial reporting. \n\n Apple Inc. I Q1 2024 Form 10-Q | 18 \n<<<\n\n\nPART II - OTHER INFORMATION \n\nItem 1. Legal Proceedings \n\nEpic Games \nEpic Games, Inc. ("Epic") filed a lawsuit in the U.S. District Court for the Northern District of California (the "District Court") \nagainst the Company alleging violations of federal and state antitrust laws and California\'s unfair competition law based upon the \nCompany\'s operation of its App Store®. On September 10, 2021, the District Court ruled in favor of the Company with respect to \nnine out of the ten counts included in Epic\'s claim. The District Court found that certain provisions of the Company\'s App Store \nReview Guidelines violate California\'s unfair competition law and issued an injunction enjoining the Company from prohibiting \ndevelopers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app \npurchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS® App Store. On April 24, 2023, the U.S. \nCourt of Appeals for the Ninth Circuit (the "Circuit Court") affirmed the District Court\'s ruling. On June 7, 2023, the Company and \nEpic filed petitions with the Circuit Court requesting further review of the decision. On June 30, 2023, the Circuit Court denied \nboth petitions. On July 17, 2023, the Circuit Court granted Apple\'s motion to stay enforcement of the injunction pending appeal to \nthe U.S. Supreme Court (the "Supreme Court"). On January 16, 2024, the Supreme Court denied both the Company\'s and Epic\'s \npetitions and the stay terminated. The Supreme Court\'s denial of Epic\'s petition confirms the District Court\'s ruling in favor of the \nCompany with respect to all of the antitrust claims. Following termination of the stay, the Company implemented a plan to comply \nwith the injunction and filed a statement of compliance with the District Court. On January 31, 2024, Epic filed a notice with the \nDistrict Court indicating its intent to dispute the Company\'s compliance plan. \n\nMasimo \nMasimo Corporation and Cercacor Laboratories, Inc. (together, "Masimo") filed a complaint before the U.S. International Trade \nCommission (the "ITC") alleging infringement by the Company of five patents relating to the functionality of the blood oxygen \nfeature in Apple Watch® Series 6 and 7. In its complaint, Masimo sought a permanent exclusion order prohibiting importation to \nthe U.S. of certain Apple Watch models that include blood oxygen sensing functionality. On October 26, 2023, the ITC entered a \nlimited exclusion order (the "Order") prohibiting importation and sales in the U.S. of Apple Watch models with blood oxygen \nsensing functionality, which includes Apple Watch Series 9 and Apple Watch Ultra™ 2. The Company subsequently proposed a \nredesign of Apple Watch Series 9 and Apple Watch Ultra 2 to the U.S. Customs and Border Protection (the "CBP") and appealed \nthe Order. On January 12, 2024, the CBP found that the Company\'s proposed redesign of Apple Watch Series 9 and Apple \nWatch Ultra 2 falls outside the scope of the Order, permitting the Company to import and sell the models in the U.S. \n\nOther Legal Proceedings \nThe Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the \nordinary course of business. The Company settled certain matters during the first quarter of 2024 that did not individually or in \nthe aggregate have a material impact on the Company\'s financial condition or operating results. The outcome of litigation is \ninherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above \nmanagement\'s expectations, the Company\'s financial condition and operating results for that reporting period could be materially \nadversely affected. \n\nItem 1A. Risk Factors \n\nThe Company\'s business, reputation, results of operations, financial condition and stock price can be affected by a number of \nfactors, whether currently known or unknown, including those described in Part I, Item 1A of the 2023 Form 10-K under the \nheading "Risk Factors." When any one or more of these risks materialize from time to time, the Company\'s business, reputation, \nresults of operations, financial condition and stock price can be materially and adversely affected. Except as set forth below, \nthere have been no material changes to the Company\'s risk factors since the 2023 Form 10-K. \n\nThe technology industry, including, in some instances, the Company, is subject to intense media, political and regulatory \nscrutiny, which exposes the Company to increasing regulation, government investigations, legal actions and penalties. \nFrom time to time, the Company has made changes to its App Store, including actions taken in response to litigation, \ncompetition, market conditions and legal and regulatory requirements. The Company expects to make further business changes \nin the future. For example, in the U.S. the Company has implemented changes to how developers communicate with consumers \nwithin apps on the U.S. storefront of the iOS and iPadOS App Store regarding alternative purchasing mechanisms. \n\n Apple Inc. I Q1 2024 Form 10-Q | 19 \n<<<\n\n\n In January 2024, the Company announced changes to iOS, the App Store and Safari® in the European Union to comply with the \n Digital Markets Act (the "DMA"), including new business terms and alternative fee structures for iOS apps, alternative methods of \n distribution for iOS apps, alternative payment processing for apps across the Company\'s operating systems, and additional tools \n and application programming interfaces ("APIs") for developers. Although the Company\'s compliance plan is intended to address \n the DMA\'s obligations, it is still subject to potential challenge by the European Commission or private litigants. In addition, other \n jurisdictions may seek to require the Company to make changes to its business. While the changes introduced by the Company \n in the European Union are intended to reduce new privacy and security risks the DMA poses to European Union users, many \n risks will remain. \n\n The Company is also currently subject to antitrust investigations in various jurisdictions around the world, which can result in \n legal proceedings and claims against the Company that could, individually or in the aggregate, have a materially adverse impact \n on the Company\'s business, results of operations and financial condition. For example, the Company is the subject of \n investigations in Europe and other jurisdictions relating to App Store terms and conditions. If such investigations result in adverse \nfindings against the Company, the Company could be exposed to significant fines and may be required to make further changes \n to its App Store business, all of which could materially adversely affect the Company\'s business, results of operations and \n financial condition. \n\n Further, the Company has commercial relationships with other companies in the technology industry that are or may become \n subject to investigations and litigation that, if resolved against those other companies, could materially adversely affect the \n Company\'s commercial relationships with those business partners and materially adversely affect the Company\'s business, \n results of operations and financial condition. For example, the Company earns revenue from licensing arrangements with other \n companies to offer their search services on the Company\'s platforms and applications, and certain of these arrangements are \n currently subject to government investigations and legal proceedings. \n\n There can be no assurance the Company\'s business will not be materially adversely affected, individually or in the aggregate, by \nthe outcomes of such investigations, litigation or changes to laws and regulations in the future. Changes to the Company\'s \n business practices to comply with new laws and regulations or in connection with other legal proceedings can negatively impact \n the reputation of the Company\'s products for privacy and security and otherwise adversely affect the experience for users of the \n Company\'s products and services, and result in harm to the Company\'s reputation, loss of competitive advantage, poor market \n acceptance, reduced demand for products and services, and lost sales. \n\n Item 2. Unregistered Sales of Equity Securities and Use of Proceeds \n\n Purchases of Equity Securities by the Issuer and Affiliated Purchasers \n Share repurchase activity during the three months ended December 30, 2023 was as follows (in millions, except number of \n shares, which are reflected in thousands, and per-share amounts): \n Total of Shares Number \n Purchased as Dollar Approximate Value of \n Total Number Average Price Part Announced of Publicly Yet Shares Be Purchased That May \n of Shares Paid Per Plans or Under the Plans \n Periods Purchased Share Programs or Programs (1) \n October 1, 2023 to November 4, 2023: \n August 2023 ASRs 6,498 (2) 6,498 \n Open market and privately negotiated purchases 45,970 $ 174.03 45,970 \n\n November 5, 2023 to December 2, 2023: \n Open market and privately negotiated purchases 33,797 $ 187.14 33,797 \n\n December 3, 2023 to December 30, 2023: \n Open market and privately negotiated purchases 31,782 $ 194.29 31,782 \n Total 118,047 $ 53,569 \n\n (1) As of December 30, 2023, the Company was authorized by the Board of Directors to purchase up to $90 billion of the \n Company\'s common stock under a share repurchase program announced on May 4, 2023, of which $36.4 billion had been \n utilized. The program does not obligate the Company to acquire a minimum amount of shares. Under the program, shares \n may be repurchased in privately negotiated or open market transactions, including under plans complying with Rule 10b5-1 \n under the Exchange Act. \n (2) In August 2023, the Company entered into accelerated share repurchase agreements ("ASRs") to purchase up to a total of \n $5.0 billion of the Company\'s common stock. In October 2023, the purchase periods for these ASRs ended and an additional \n 6 million shares were delivered and retired. In total, 29 million shares were delivered under these ASRs at an average \n repurchase price of $174.93 per share. \n\n Apple Inc. I Q1 2024 Form 10-Q | 20 \n<<<\n\n\nItem 3. Defaults Upon Senior Securities \n\nNone. \n\nItem 4. Mine Safety Disclosures \n\nNot applicable. \n\nItem 5. Other Information \n\nInsider Trading Arrangements \nOn November 11, 2023 and November 27, 2023, respectively, Luca Maestri, the Company\'s Senior Vice President and Chief \nFinancial Officer, and Katherine L. Adams, the Company\'s Senior Vice President and General Counsel, each entered into a \ntrading plan intended to satisfy the affirmative defense conditions of Rule 10b5-1(c) under the Exchange Act. The plans provide \nfor the sale of all shares vested during the duration of the plans pursuant to certain equity awards granted to Mr. Maestri and Ms. \nAdams, respectively, excluding any shares withheld by the Company to satisfy income tax withholding and remittance \nobligations. Mr. Maestri\'s plan will expire on December 31, 2024, and Ms. Adams\'s plan will expire on November 1, 2024, subject \nto early termination for certain specified events set forth in the plans. \n\nItem 6. Exhibits \n Incorporated by Reference \n\n Number Exhibit Form Filing Period Date/ End \n Exhibit Description Exhibit Date \n 31.1* Rule 13a-14(a) / 15d-14(a) Certification of Chief Executive Officer. \n 31.2* Rule 13a-14(a) / 15d-14(a) Certification of Chief Financial Officer. \n 32.1 ** Section 1350 Certifications of Chief Executive Officer and Chief Financial Officer. \n 101* Inline XBRL Document Set for the condensed consolidated financial statements \n and accompanying notes in Part I, Item 1, "Financial Statements" of this \n Quarterly Report on Form 10-Q. \n 104* Inline the Exhibit XBRL for 101 the Inline cover XBRL page Document of this Quarterly Set. Report on Form 10-Q, included in \n\n * Filed herewith. \n ** Furnished herewith. \n\n Apple Inc. I Q1 2024 Form 10-Q | 21 \n<<<\n\n\n SIGNATURE \n\n Pursuant to the requirements of the Securities Exchange Act of 1934, the Registrant has duly caused this report to be signed on \nits behalf by the undersigned thereunto duly authorized. \n\n Date: February 1, 2024 Apple Inc. \n\n By: /s/ Luca Maestri \n Luca Maestri \n Senior Vice President, \n Chief Financial Officer \n\n Apple Inc. I Q1 2024 Form 10-Q | 22 \n<<<\n\n\n Exhibit 31.1 \n\n CERTIFICATION \n\nI, Timothy D. Cook, certify that: \n\n1. I have reviewed this quarterly report on Form 10-Q of Apple Inc .; \n\n2. Based on my knowledge, this report does not contain any untrue statement of a material fact or omit to state a material fact \n necessary to make the statements made, in light of the circumstances under which such statements were made, not \n misleading with respect to the period covered by this report; \n\n3. Based on my knowledge, the financial statements, and other financial information included in this report, fairly present in all \n material respects the financial condition, results of operations and cash flows of the Registrant as of, and for, the periods \n presented in this report; \n\n4. The Registrant\'s other certifying officer(s) and I are responsible for establishing and maintaining disclosure controls and \n procedures (as defined in Exchange Act Rules 13a-15(e) and 15d-15(e)) and internal control over financial reporting (as \n defined in Exchange Act Rules 13a-15(f) and 15d-15(f)) for the Registrant and have: \n\n (a) Designed such disclosure controls and procedures, or caused such disclosure controls and procedures to be \n designed under our supervision, to ensure that material information relating to the Registrant, including its \n consolidated subsidiaries, is made known to us by others within those entities, particularly during the period in \n which this report is being prepared; \n\n (b) Designed such internal control over financial reporting, or caused such internal control over financial reporting \n to be designed under our supervision, to provide reasonable assurance regarding the reliability of financial \n reporting and the preparation of financial statements for external purposes in accordance with generally \n accepted accounting principles; \n\n (c) Evaluated the effectiveness of the Registrant\'s disclosure controls and procedures and presented in this report \n our conclusions about the effectiveness of the disclosure controls and procedures, as of the end of the period \n covered by this report based on such evaluation; and \n\n (d) Disclosed in this report any change in the Registrant\'s internal control over financial reporting that occurred \n during the Registrant\'s most recent fiscal quarter (the Registrant\'s fourth fiscal quarter in the case of an annual \n report) that has materially affected, or is reasonably likely to materially affect, the Registrant\'s internal control \n over financial reporting; and \n\n5. The Registrant\'s other certifying officer(s) and I have disclosed, based on our most recent evaluation of internal control over \n financial reporting, to the Registrant\'s auditors and the audit committee of the Registrant\'s board of directors (or persons \n performing the equivalent functions): \n\n (a) All significant deficiencies and material weaknesses in the design or operation of internal control over financial \n reporting which are reasonably likely to adversely affect the Registrant\'s ability to record, process, summarize \n and report financial information; and \n\n (b) Any fraud, whether or not material, that involves management or other employees who have a significant role \n in the Registrant\'s internal control over financial reporting. \n\nDate: February 1, 2024 \n\n By: /s/ Timothy D. Cook \n Timothy D. Cook \n Chief Executive Officer \n<<<\n\n\n Exhibit 31.2 \n\n CERTIFICATION \n\nI, Luca Maestri, certify that: \n\n1. I have reviewed this quarterly report on Form 10-Q of Apple Inc .; \n\n2. Based on my knowledge, this report does not contain any untrue statement of a material fact or omit to state a material fact \n necessary to make the statements made, in light of the circumstances under which such statements were made, not \n misleading with respect to the period covered by this report; \n\n3. Based on my knowledge, the financial statements, and other financial information included in this report, fairly present in all \n material respects the financial condition, results of operations and cash flows of the Registrant as of, and for, the periods \n presented in this report; \n\n4. The Registrant\'s other certifying officer(s) and I are responsible for establishing and maintaining disclosure controls and \n procedures (as defined in Exchange Act Rules 13a-15(e) and 15d-15(e)) and internal control over financial reporting (as \n defined in Exchange Act Rules 13a-15(f) and 15d-15(f)) for the Registrant and have: \n\n (a) Designed such disclosure controls and procedures, or caused such disclosure controls and procedures to be \n designed under our supervision, to ensure that material information relating to the Registrant, including its \n consolidated subsidiaries, is made known to us by others within those entities, particularly during the period in \n which this report is being prepared; \n\n (b) Designed such internal control over financial reporting, or caused such internal control over financial reporting \n to be designed under our supervision, to provide reasonable assurance regarding the reliability of financial \n reporting and the preparation of financial statements for external purposes in accordance with generally \n accepted accounting principles; \n\n (c) Evaluated the effectiveness of the Registrant\'s disclosure controls and procedures and presented in this report \n our conclusions about the effectiveness of the disclosure controls and procedures, as of the end of the period \n covered by this report based on such evaluation; and \n\n (d) Disclosed in this report any change in the Registrant\'s internal control over financial reporting that occurred \n during the Registrant\'s most recent fiscal quarter (the Registrant\'s fourth fiscal quarter in the case of an annual \n report) that has materially affected, or is reasonably likely to materially affect, the Registrant\'s internal control \n over financial reporting; and \n\n5. The Registrant\'s other certifying officer(s) and I have disclosed, based on our most recent evaluation of internal control over \n financial reporting, to the Registrant\'s auditors and the audit committee of the Registrant\'s board of directors (or persons \n performing the equivalent functions): \n\n (a) All significant deficiencies and material weaknesses in the design or operation of internal control over financial \n reporting which are reasonably likely to adversely affect the Registrant\'s ability to record, process, summarize \n and report financial information; and \n\n (b) Any fraud, whether or not material, that involves management or other employees who have a significant role \n in the Registrant\'s internal control over financial reporting. \n\nDate: February 1, 2024 \n\n By: /s/ Luca Maestri \n Luca Maestri \n Senior Vice President, \n Chief Financial Officer \n<<<\n\n\n Exhibit 32.1 \n\n CERTIFICATIONS OF CHIEF EXECUTIVE OFFICER AND CHIEF FINANCIAL OFFICER \n PURSUANT TO \n 18 U.S.C. SECTION 1350, \n AS ADOPTED PURSUANT TO \n SECTION 906 OF THE SARBANES-OXLEY ACT OF 2002 \n\nI, Timothy D. Cook, certify, as of the date hereof, pursuant to 18 U.S.C. Section 1350, as adopted pursuant to Section 906 of the \nSarbanes-Oxley Act of 2002, that the Quarterly Report of Apple Inc. on Form 10-Q for the period ended December 30, 2023 fully \ncomplies with the requirements of Section 13(a) or 15(d) of the Securities Exchange Act of 1934 and that information contained \nin such Form 10-Q fairly presents in all material respects the financial condition and results of operations of Apple Inc. at the \ndates and for the periods indicated. \n\nDate: February 1, 2024 \n\n By: /s/ Timothy D. Cook \n Timothy D. Cook \n Chief Executive Officer \n\nI, Luca Maestri, certify, as of the date hereof, pursuant to 18 U.S.C. Section 1350, as adopted pursuant to Section 906 of the \nSarbanes-Oxley Act of 2002, that the Quarterly Report of Apple Inc. on Form 10-Q for the period ended December 30, 2023 fully \ncomplies with the requirements of Section 13(a) or 15(d) of the Securities Exchange Act of 1934 and that information contained \nin such Form 10-Q fairly presents in all material respects the financial condition and results of operations of Apple Inc. at the \ndates and for the periods indicated. \n\nDate: February 1, 2024 \n\n By: /s/ Luca Maestri \n Luca Maestri \n Senior Vice President, \n Chief Financial Officer \n\nA signed original of this written statement required by Section 906 has been provided to Apple Inc. and will be retained by Apple \nInc. and furnished to the Securities and Exchange Commission or its staff upon request. \n<<<\n"
diff --git a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_create_table_if_not_exists.py b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_create_table_if_not_exists.py
new file mode 100644
index 000000000..cbf4f1619
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_create_table_if_not_exists.py
@@ -0,0 +1,97 @@
+import pytest # type: ignore
+from workflow_manager.endpoint_v2.database_utils import DatabaseUtils
+from workflow_manager.endpoint_v2.exceptions import UnstractDBException
+
+from unstract.connectors.databases.unstract_db import UnstractDB
+
+from .base_test_db import BaseTestDB
+
+
+class TestCreateTableIfNotExists(BaseTestDB):
+ def test_create_table_if_not_exists_valid(
+ self, valid_dbs_instance: UnstractDB
+ ) -> None:
+ engine = valid_dbs_instance.get_engine()
+ result = DatabaseUtils.create_table_if_not_exists(
+ db_class=valid_dbs_instance,
+ engine=engine,
+ table_name=self.valid_table_name,
+ database_entry=self.database_entry,
+ )
+ assert result is None
+
+ def test_create_table_if_not_exists_bigquery_valid(
+ self, valid_bigquery_db_instance: UnstractDB
+ ) -> None:
+ engine = valid_bigquery_db_instance.get_engine()
+ result = DatabaseUtils.create_table_if_not_exists(
+ db_class=valid_bigquery_db_instance,
+ engine=engine,
+ table_name=self.valid_bigquery_table_name,
+ database_entry=self.database_entry,
+ )
+ assert result is None
+
+ def test_create_table_if_not_exists_invalid_schema(
+ self, invalid_dbs_instance: UnstractDB
+ ) -> None:
+ engine = invalid_dbs_instance.get_engine()
+ with pytest.raises(UnstractDBException):
+ DatabaseUtils.create_table_if_not_exists(
+ db_class=invalid_dbs_instance,
+ engine=engine,
+ table_name=self.valid_table_name,
+ database_entry=self.database_entry,
+ )
+
+ def test_create_table_if_not_exists_invalid_syntax(
+ self, valid_dbs_instance: UnstractDB
+ ) -> None:
+ engine = valid_dbs_instance.get_engine()
+ with pytest.raises(UnstractDBException):
+ DatabaseUtils.create_table_if_not_exists(
+ db_class=valid_dbs_instance,
+ engine=engine,
+ table_name=self.invalid_syntax_table_name,
+ database_entry=self.database_entry,
+ )
+
+ def test_create_table_if_not_exists_wrong_table_name(
+ self, valid_dbs_instance: UnstractDB
+ ) -> None:
+ engine = valid_dbs_instance.get_engine()
+ with pytest.raises(UnstractDBException):
+ DatabaseUtils.create_table_if_not_exists(
+ db_class=valid_dbs_instance,
+ engine=engine,
+ table_name=self.invalid_wrong_table_name,
+ database_entry=self.database_entry,
+ )
+
+ def test_create_table_if_not_exists_feature_not_supported(
+ self, invalid_dbs_instance: UnstractDB
+ ) -> None:
+ engine = invalid_dbs_instance.get_engine()
+ with pytest.raises(UnstractDBException):
+ DatabaseUtils.create_table_if_not_exists(
+ db_class=invalid_dbs_instance,
+ engine=engine,
+ table_name=self.invalid_wrong_table_name,
+ database_entry=self.database_entry,
+ )
+
+ def test_create_table_if_not_exists_invalid_snowflake_db(
+ self, invalid_snowflake_db_instance: UnstractDB
+ ) -> None:
+ engine = invalid_snowflake_db_instance.get_engine()
+ with pytest.raises(UnstractDBException):
+ DatabaseUtils.create_table_if_not_exists(
+ db_class=invalid_snowflake_db_instance,
+ engine=engine,
+ table_name=self.invalid_wrong_table_name,
+ database_entry=self.database_entry,
+ )
+
+
+if __name__ == "__main__":
+ pytest.main()
diff --git a/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_execute_write_query.py b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_execute_write_query.py
new file mode 100644
index 000000000..22c26dcfd
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/tests/test_database_utils/test_execute_write_query.py
@@ -0,0 +1,169 @@
+import os
+import uuid
+from typing import Any
+
+import pytest # type: ignore
+from workflow_manager.endpoint_v2.database_utils import DatabaseUtils
+from workflow_manager.endpoint_v2.exceptions import UnstractDBException
+
+from unstract.connectors.databases.redshift import Redshift
+from unstract.connectors.databases.unstract_db import UnstractDB
+
+from .base_test_db import BaseTestDB
+
+
+class TestExecuteWriteQuery(BaseTestDB):
+ @pytest.fixture(autouse=True)
+ def setup(self, base_setup: Any) -> None:
+ self.sql_columns_and_values = {
+ "created_by": "Unstract/DBWriter",
+ "created_at": "2024-05-20 10:36:25.362609",
+ "data": '{"input_file": "simple.pdf", "result": "report"}',
+ "id": str(uuid.uuid4()),
+ }
+
+ def test_execute_write_query_valid(self, valid_dbs_instance: Any) -> None:
+ engine = valid_dbs_instance.get_engine()
+ result = DatabaseUtils.execute_write_query(
+ db_class=valid_dbs_instance,
+ engine=engine,
+ table_name=self.valid_table_name,
+ sql_keys=list(self.sql_columns_and_values.keys()),
+ sql_values=list(self.sql_columns_and_values.values()),
+ )
+ assert result is None
+
+ def test_execute_write_query_invalid_schema(
+ self, invalid_dbs_instance: Any
+ ) -> None:
+ engine = invalid_dbs_instance.get_engine()
+ with pytest.raises(UnstractDBException):
+ DatabaseUtils.execute_write_query(
+ db_class=invalid_dbs_instance,
+ engine=engine,
+ table_name=self.valid_table_name,
+ sql_keys=list(self.sql_columns_and_values.keys()),
+ sql_values=list(self.sql_columns_and_values.values()),
+ )
+
+ def test_execute_write_query_invalid_syntax(self, valid_dbs_instance: Any) -> None:
+ engine = valid_dbs_instance.get_engine()
+ with pytest.raises(UnstractDBException):
+ DatabaseUtils.execute_write_query(
+ db_class=valid_dbs_instance,
+ engine=engine,
+ table_name=self.invalid_syntax_table_name,
+ sql_keys=list(self.sql_columns_and_values.keys()),
+ sql_values=list(self.sql_columns_and_values.values()),
+ )
+
+ def test_execute_write_query_feature_not_supported(
+ self, invalid_dbs_instance: Any
+ ) -> None:
+ engine = invalid_dbs_instance.get_engine()
+ with pytest.raises(UnstractDBException):
+ DatabaseUtils.execute_write_query(
+ db_class=invalid_dbs_instance,
+ engine=engine,
+ table_name=self.invalid_wrong_table_name,
+ sql_keys=list(self.sql_columns_and_values.keys()),
+ sql_values=list(self.sql_columns_and_values.values()),
+ )
+
+ def load_text_to_sql_values(self) -> dict[str, Any]:
+ file_path = os.path.join(os.path.dirname(__file__), "static", "large_doc.txt")
+ with open(file_path, encoding="utf-8") as file:
+ content = file.read()
+ sql_columns_and_values = self.sql_columns_and_values.copy()
+ sql_columns_and_values["data"] = content
+ return sql_columns_and_values
+
+ @pytest.fixture
+ def valid_redshift_db_instance(self) -> Any:
+ return Redshift(self.valid_redshift_creds)
+
+ def test_execute_write_query_datatype_too_large_redshift(
+ self, valid_redshift_db_instance: Any
+ ) -> None:
+ engine = valid_redshift_db_instance.get_engine()
+ sql_columns_and_values = self.load_text_to_sql_values()
+ with pytest.raises(UnstractDBException):
+ DatabaseUtils.execute_write_query(
+ db_class=valid_redshift_db_instance,
+ engine=engine,
+ table_name=self.valid_table_name,
+ sql_keys=list(sql_columns_and_values.keys()),
+ sql_values=list(sql_columns_and_values.values()),
+ )
+
+ def test_execute_write_query_bigquery_valid(
+ self, valid_bigquery_db_instance: Any
+ ) -> None:
+ engine = valid_bigquery_db_instance.get_engine()
+ result = DatabaseUtils.execute_write_query(
+ db_class=valid_bigquery_db_instance,
+ engine=engine,
+ table_name=self.valid_bigquery_table_name,
+ sql_keys=list(self.sql_columns_and_values.keys()),
+ sql_values=list(self.sql_columns_and_values.values()),
+ )
+ assert result is None
+
+ def test_execute_write_query_wrong_table_name(
+ self, valid_dbs_instance: UnstractDB
+ ) -> None:
+ engine = valid_dbs_instance.get_engine()
+ with pytest.raises(UnstractDBException):
+ DatabaseUtils.execute_write_query(
+ db_class=valid_dbs_instance,
+ engine=engine,
+ table_name=self.invalid_wrong_table_name,
+ sql_keys=list(self.sql_columns_and_values.keys()),
+ sql_values=list(self.sql_columns_and_values.values()),
+ )
+
+ def test_execute_write_query_bigquery_large_doc(
+ self, valid_bigquery_db_instance: Any
+ ) -> None:
+ engine = valid_bigquery_db_instance.get_engine()
+ sql_columns_and_values = self.load_text_to_sql_values()
+ result = DatabaseUtils.execute_write_query(
+ db_class=valid_bigquery_db_instance,
+ engine=engine,
+ table_name=self.valid_bigquery_table_name,
+ sql_keys=list(sql_columns_and_values.keys()),
+ sql_values=list(sql_columns_and_values.values()),
+ )
+ assert result is None
+
+ def test_execute_write_query_invalid_snowflake_db(
+ self, invalid_snowflake_db_instance: UnstractDB
+ ) -> None:
+ engine = invalid_snowflake_db_instance.get_engine()
+ with pytest.raises(UnstractDBException):
+ DatabaseUtils.execute_write_query(
+ db_class=invalid_snowflake_db_instance,
+ engine=engine,
+ table_name=self.invalid_wrong_table_name,
+ sql_keys=list(self.sql_columns_and_values.keys()),
+ sql_values=list(self.sql_columns_and_values.values()),
+ )
+
+ # Make this function at last to cover all large doc
+ def test_execute_write_query_large_doc(
+ self, valid_dbs_instance_to_handle_large_doc: Any
+ ) -> None:
+ engine = valid_dbs_instance_to_handle_large_doc.get_engine()
+ sql_columns_and_values = self.load_text_to_sql_values()
+ result = DatabaseUtils.execute_write_query(
+ db_class=valid_dbs_instance_to_handle_large_doc,
+ engine=engine,
+ table_name=self.valid_table_name,
+ sql_keys=list(sql_columns_and_values.keys()),
+ sql_values=list(sql_columns_and_values.values()),
+ )
+ assert result is None
+
+
+if __name__ == "__main__":
+ pytest.main()
diff --git a/backend/workflow_manager/endpoint_v2/urls.py b/backend/workflow_manager/endpoint_v2/urls.py
new file mode 100644
index 000000000..522859e23
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/urls.py
@@ -0,0 +1,23 @@
+from django.urls import path
+from workflow_manager.endpoint_v2.views import WorkflowEndpointViewSet
+
+workflow_endpoint_list = WorkflowEndpointViewSet.as_view(
+ {"get": "workflow_endpoint_list"}
+)
+endpoint_list = WorkflowEndpointViewSet.as_view({"get": "list"})
+workflow_endpoint_detail = WorkflowEndpointViewSet.as_view(
+ {"get": "retrieve", "put": "update"}
+)
+endpoint_settings_detail = WorkflowEndpointViewSet.as_view(
+ {"get": WorkflowEndpointViewSet.get_settings.__name__}
+)
+
+urlpatterns = [
+ path("", endpoint_list, name="endpoint-list"),
+ path("<str:pk>/", workflow_endpoint_detail, name="workflow-endpoint-detail"),
+ path(
+ "<str:pk>/settings/",
+ endpoint_settings_detail,
+ name="workflow-endpoint-detail",
+ ),
+]
diff --git a/backend/workflow_manager/endpoint_v2/views.py b/backend/workflow_manager/endpoint_v2/views.py
new file mode 100644
index 000000000..6e82a0e36
--- /dev/null
+++ b/backend/workflow_manager/endpoint_v2/views.py
@@ -0,0 +1,82 @@
+from django.db.models import QuerySet
+from rest_framework import status, viewsets
+from rest_framework.decorators import action
+from rest_framework.request import Request
+from rest_framework.response import Response
+from workflow_manager.endpoint_v2.destination import DestinationConnector
+from workflow_manager.endpoint_v2.endpoint_utils import WorkflowEndpointUtils
+from workflow_manager.endpoint_v2.models import WorkflowEndpoint
+from workflow_manager.endpoint_v2.source import SourceConnector
+from workflow_manager.workflow_v2.serializers import WorkflowEndpointSerializer
+
+
+class WorkflowEndpointViewSet(viewsets.ModelViewSet):
+ queryset = WorkflowEndpoint.objects.all()
+ serializer_class = WorkflowEndpointSerializer
+
+ def get_queryset(self) -> QuerySet:
+
+ queryset = (
+ WorkflowEndpoint.objects.all()
+ .select_related("workflow")
+ .filter(workflow__created_by=self.request.user)
+ )
+ endpoint_type_filter = self.request.query_params.get("endpoint_type", None)
+ connection_type_filter = self.request.query_params.get("connection_type", None)
+ if endpoint_type_filter:
+ queryset = queryset.filter(endpoint_type=endpoint_type_filter)
+ if connection_type_filter:
+ queryset = queryset.filter(connection_type=connection_type_filter)
+ return queryset
+
+ @action(detail=True, methods=["get"])
+ def get_settings(self, request: Request, pk: str) -> Response:
+ """Retrieve the settings/schema for a specific workflow endpoint.
+
+ Parameters:
+ request (Request): The HTTP request object.
+ pk (str): The primary key of the workflow endpoint.
+
+ Returns:
+ Response: The HTTP response containing the settings/schema for
+ the endpoint.
+ """
+ endpoint: WorkflowEndpoint = self.get_object()
+ connection_type = endpoint.connection_type
+ endpoint_type = endpoint.endpoint_type
+ schema = None
+ if endpoint_type == WorkflowEndpoint.EndpointType.SOURCE:
+ if connection_type == WorkflowEndpoint.ConnectionType.API:
+ schema = SourceConnector.get_json_schema_for_api()
+ if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
+ schema = SourceConnector.get_json_schema_for_file_system()
+ if endpoint_type == WorkflowEndpoint.EndpointType.DESTINATION:
+ if connection_type == WorkflowEndpoint.ConnectionType.DATABASE:
+ schema = DestinationConnector.get_json_schema_for_database()
+ if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
+ schema = DestinationConnector.get_json_schema_for_file_system()
+ if connection_type == WorkflowEndpoint.ConnectionType.API:
+ schema = DestinationConnector.get_json_schema_for_api()
+
+ return Response(
+ {
+ "status": status.HTTP_200_OK,
+ "schema": schema,
+ }
+ )
+
+ @action(detail=True, methods=["get"])
+ def workflow_endpoint_list(self, request: Request, pk: str) -> Response:
+ """Retrieve a list of endpoints for a specific workflow.
+
+ Parameters:
+ request (Request): The HTTP request object.
+ pk (str): The primary key of the workflow.
+
+ Returns:
+ Response: The HTTP response containing the serialized list of
+ endpoints.
+ """
+ endpoints = WorkflowEndpointUtils.get_endpoints_for_workflow(pk)
+ serializer = WorkflowEndpointSerializer(endpoints, many=True)
+ return Response(serializer.data)
diff --git a/backend/workflow_manager/workflow_v2/__init__.py b/backend/workflow_manager/workflow_v2/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/backend/workflow_manager/workflow_v2/admin.py b/backend/workflow_manager/workflow_v2/admin.py
new file mode 100644
index 000000000..6ab58abaf
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/admin.py
@@ -0,0 +1,4 @@
+from django.contrib import admin
+from workflow_manager.workflow_v2.models.workflow import Workflow
+
+admin.site.register(Workflow)
diff --git a/backend/workflow_manager/workflow_v2/apps.py b/backend/workflow_manager/workflow_v2/apps.py
new file mode 100644
index 000000000..255090664
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/apps.py
@@ -0,0 +1,13 @@
+from django.apps import AppConfig
+
+
+class WorkflowConfig(AppConfig):
+ default_auto_field = "django.db.models.BigAutoField"
+ name = "workflow_manager.workflow_v2"
+
+ def ready(self):
+ from workflow_manager.workflow_v2.execution_log_utils import (
+ create_log_consumer_scheduler_if_not_exists,
+ )
+
+ create_log_consumer_scheduler_if_not_exists()
diff --git a/backend/workflow_manager/workflow_v2/constants.py b/backend/workflow_manager/workflow_v2/constants.py
new file mode 100644
index 000000000..95aab13e3
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/constants.py
@@ -0,0 +1,60 @@
+class WorkflowKey:
+ """Dict keys related to workflows."""
+
+ PROMPT_TEXT = "prompt_text"
+ LLM_RESPONSE = "llm_response"
+ WF_STEPS = "steps"
+ WF_TOOL = "tool"
+ WF_INSTANCE_SETTINGS = "instance_settings"
+ WF_TOOL_INSTANCE_ID = "tool_instance_id"
+ WF_CONNECTOR_CLASS = "connector_class"
+ WF_INPUT = "input"
+ WF_OUTPUT = "output"
+ WF_TOOL_UUID = "id"
+ WF_ID = "workflow_id"
+ WF_NAME = "workflow_name"
+ WF_OWNER = "workflow_owner"
+ WF_TOOL_INSTANCES = "tool_instances"
+ WF_IS_ACTIVE = "is_active"
+ EXECUTION_ACTION = "execution_action"
+ # Keys from provisional workflow
+ PWF_RESULT = "result"
+ PWF_OUTPUT = "output"
+ PWF_COST_TYPE = "cost_type"
+ PWF_COST = "cost"
+ PWF_TIME_TAKEN = "time_taken"
+ WF_CACHE_PATTERN = r"^cache:{?\w{8}-?\w{4}-?\w{4}-?\w{4}-?\w{12}}?$"
+ WF_PROJECT_GUID = "guid"
+
+
+class WorkflowExecutionKey:
+ WORKFLOW_EXECUTION_ID_PREFIX = "workflow"
+ EXECUTION_ID = "execution_id"
+ LOG_GUID = "log_guid"
+ WITH_LOG = "with_log"
+
+
+class WorkflowErrors:
+ WORKFLOW_EXISTS = "Workflow with this configuration already exists."
+ DUPLICATE_API = "It appears that a duplicate call may have been made."
+ INVALID_EXECUTION_ID = "Invalid execution_id"
+
+
+class CeleryConfigurations:
+ INTERVAL = 2
+
+
+class Tool:
+ APIOPS = "apiops"
+
+
+class WorkflowMessages:
+ CACHE_CLEAR_SUCCESS = "Cache cleared successfully."
+ CACHE_CLEAR_FAILED = "Failed to clear cache."
+ CACHE_EMPTY = "Cache is already empty."
+ CELERY_TIMEOUT_MESSAGE = (
+ "Your request is being processed. Please wait."
+ "You can check the status using the status API."
+ )
+ FILE_MARKER_CLEAR_SUCCESS = "File marker cleared successfully."
+ FILE_MARKER_CLEAR_FAILED = "Failed to clear file marker."
diff --git a/backend/workflow_manager/workflow_v2/dto.py b/backend/workflow_manager/workflow_v2/dto.py
new file mode 100644
index 000000000..5f2e0db43
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/dto.py
@@ -0,0 +1,68 @@
+from dataclasses import dataclass
+from typing import Any, Optional
+
+from celery.result import AsyncResult
+from workflow_manager.workflow_v2.constants import WorkflowKey
+
+
+@dataclass
+class ProvisionalWorkflow:
+ result: str
+ output: dict[str, str]
+ cost_type: str
+ cost: str
+ time_taken: float
+
+ def __init__(self, input_dict: dict[str, Any]) -> None:
+ self.result = input_dict.get(WorkflowKey.PWF_RESULT, "")
+ self.output = input_dict.get(WorkflowKey.PWF_OUTPUT, {})
+ self.cost_type = input_dict.get(WorkflowKey.PWF_COST_TYPE, "")
+ self.cost = input_dict.get(WorkflowKey.PWF_COST, "")
+ self.time_taken = input_dict.get(WorkflowKey.PWF_TIME_TAKEN, 0.0)
+
+
+@dataclass
+class ExecutionResponse:
+ workflow_id: str
+ execution_id: str
+ execution_status: str
+ log_id: Optional[str] = None
+ status_api: Optional[str] = None
+ error: Optional[str] = None
+ mode: Optional[str] = None
+ result: Optional[Any] = None
+ message: Optional[str] = None
+
+ def __post_init__(self) -> None:
+ self.log_id = self.log_id or None
+ self.mode = self.mode or None
+ self.error = self.error or None
+ self.result = self.result or None
+ self.message = self.message or None
+ self.status_api = self.status_api or None
+
+
+@dataclass
+class AsyncResultData:
+ id: str
+ status: str
+ result: Any
+ is_ready: bool
+ is_failed: bool
+ info: Any
+
+ def __init__(self, async_result: AsyncResult):
+ self.id = async_result.id
+ self.status = async_result.status
+ self.result = async_result.result
+ self.is_ready = async_result.ready()
+ self.is_failed = async_result.failed()
+ self.info = async_result.info
+ if isinstance(self.result, Exception):
+ self.result = str(self.result)
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "status": self.status,
+ "result": self.result,
+ }
diff --git a/backend/workflow_manager/workflow_v2/enums.py b/backend/workflow_manager/workflow_v2/enums.py
new file mode 100644
index 000000000..cc13a1e35
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/enums.py
@@ -0,0 +1,73 @@
+from enum import Enum
+
+from utils.common_utils import ModelEnum
+
+
+class WorkflowExecutionMethod(Enum):
+ INSTANT = "INSTANT"
+ QUEUED = "QUEUED"
+
+
+class ExecutionStatus(ModelEnum):
+ """An enumeration representing the various statuses of an execution
+ process.
+
+ Statuses:
+ PENDING: The execution's entry has been created in the database.
+ QUEUED: The execution task is queued for asynchronous execution
+ INITIATED: The execution has been initiated.
+ READY: The execution is ready for the build phase.
+ EXECUTING: The execution is currently in progress.
+ COMPLETED: The execution has been successfully completed.
+ STOPPED: The execution was stopped by the user
+ (applicable to step executions).
+ ERROR: An error occurred during the execution process.
+
+ Note:
+ Intermediate statuses might not be experienced due to
+ Django's query triggering once all processes are completed.
+ """
+
+ PENDING = "PENDING"
+ INITIATED = "INITIATED"
+ QUEUED = "QUEUED"
+ READY = "READY"
+ EXECUTING = "EXECUTING"
+ COMPLETED = "COMPLETED"
+ STOPPED = "STOPPED"
+ ERROR = "ERROR"
+
+
+class SchemaType(Enum):
+ """Possible types for workflow module's JSON schema.
+
+ Values:
+ src: Refers to the source module's schema
+ dest: Refers to the destination module's schema
+ """
+
+ SRC = "src"
+ DEST = "dest"
+
+
+class SchemaEntity(Enum):
+ """Possible entities for workflow module's JSON schema.
+
+ Values:
+ file: Refers to schema for file based sources
+ api: Refers to schema for API based sources
+ db: Refers to schema for DB based destinations
+ """
+
+ FILE = "file"
+ API = "api"
+ DB = "db"
+
+
+class ColumnModes(Enum):
+ WRITE_JSON_TO_A_SINGLE_COLUMN = "Write JSON to a single column"
+ SPLIT_JSON_INTO_COLUMNS = "Split JSON into columns"
+
+
+class AgentName(Enum):
+ UNSTRACT_DBWRITER = "Unstract/DBWriter"
diff --git a/backend/workflow_manager/workflow_v2/exceptions.py b/backend/workflow_manager/workflow_v2/exceptions.py
new file mode 100644
index 000000000..39087ab8e
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/exceptions.py
@@ -0,0 +1,56 @@
+from rest_framework.exceptions import APIException
+
+
+class WorkflowGenerationError(APIException):
+ status_code = 500
+ default_detail = "Error generating workflow."
+
+
+class WorkflowRegenerationError(APIException):
+ status_code = 500
+ default_detail = "Error regenerating workflow."
+
+
+class WorkflowExecutionError(APIException):
+ status_code = 500
+ default_detail = "Error executing workflow."
+
+
+class WorkflowDoesNotExistError(APIException):
+ status_code = 404
+ default_detail = "Workflow does not exist"
+
+
+class TaskDoesNotExistError(APIException):
+ status_code = 404
+ default_detail = "Task does not exist"
+
+
+class DuplicateActionError(APIException):
+ status_code = 400
+ default_detail = "Action is running"
+
+
+class InvalidRequest(APIException):
+ status_code = 400
+ default_detail = "Invalid Request"
+
+
+class MissingEnvException(APIException):
+ status_code = 500
+ default_detail = "At least one active platform key should be available."
+
+
+class InternalException(APIException):
+ """Internal Error.
+
+ Args:
+ APIException (_type_): _description_
+ """
+
+ status_code = 500
+
+
+class WorkflowExecutionNotExist(APIException):
+ status_code = 404
+ default_detail = "Workflow execution does not exist"
diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.py
new file mode 100644
index 000000000..fd64cba50
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/execution.py
@@ -0,0 +1,420 @@
+import logging
+import time
+from typing import Optional
+
+from account_v2.constants import Common
+from api_v2.exceptions import InvalidAPIRequest
+from platform_settings_v2.platform_auth_service import PlatformAuthenticationService
+from tool_instance_v2.constants import JsonSchemaKey
+from tool_instance_v2.models import ToolInstance
+from tool_instance_v2.tool_processor import ToolProcessor
+from unstract.tool_registry.dto import Tool
+from unstract.workflow_execution import WorkflowExecutionService
+from unstract.workflow_execution.dto import ToolInstance as ToolInstanceDataClass
+from unstract.workflow_execution.dto import WorkflowDto
+from unstract.workflow_execution.enums import ExecutionType, LogComponent, LogState
+from unstract.workflow_execution.exceptions import StopExecution
+from utils.local_context import StateStore
+from utils.user_context import UserContext
+from workflow_manager.workflow_v2.constants import WorkflowKey
+from workflow_manager.workflow_v2.enums import ExecutionStatus
+from workflow_manager.workflow_v2.exceptions import WorkflowExecutionError
+from workflow_manager.workflow_v2.models import Workflow, WorkflowExecution
+from workflow_manager.workflow_v2.models.execution import EXECUTION_ERROR_LENGTH
+from workflow_manager.workflow_v2.models.file_history import FileHistory
+
+logger = logging.getLogger(__name__)
+
+
+class WorkflowExecutionServiceHelper(WorkflowExecutionService):
+ def __init__(
+ self,
+ workflow: Workflow,
+ tool_instances: list[ToolInstance],
+ organization_id: Optional[str] = None,
+ pipeline_id: Optional[str] = None,
+ single_step: bool = False,
+ scheduled: bool = False,
+ mode: tuple[str, str] = WorkflowExecution.Mode.INSTANT,
+ workflow_execution: Optional[WorkflowExecution] = None,
+ include_metadata: bool = False,
+ ) -> None:
+ tool_instances_as_dto = []
+ for tool_instance in tool_instances:
+ tool_instances_as_dto.append(
+ self.convert_tool_instance_model_to_data_class(tool_instance)
+ )
+ workflow_as_dto: WorkflowDto = self.convert_workflow_model_to_data_class(
+ workflow=workflow
+ )
+ organization_id = organization_id or UserContext.get_organization_identifier()
+ if not organization_id:
+ raise WorkflowExecutionError(detail="invalid Organization ID")
+
+ platform_key = PlatformAuthenticationService.get_active_platform_key()
+ super().__init__(
+ organization_id=organization_id,
+ workflow_id=workflow.id,
+ workflow=workflow_as_dto,
+ tool_instances=tool_instances_as_dto,
+ platform_service_api_key=str(platform_key.key),
+ ignore_processed_entities=False,
+ include_metadata=include_metadata,
+ )
+ if not workflow_execution:
+ # Use pipline_id for pipelines / API deployment
+ # since session might not be present.
+ log_events_id = StateStore.get(Common.LOG_EVENTS_ID)
+ self.execution_log_id = log_events_id if log_events_id else pipeline_id
+ self.execution_mode = mode
+ self.execution_method: tuple[str, str] = (
+ WorkflowExecution.Method.SCHEDULED
+ if scheduled
+ else WorkflowExecution.Method.DIRECT
+ )
+ self.execution_type: tuple[str, str] = (
+ WorkflowExecution.Type.STEP
+ if single_step
+ else WorkflowExecution.Type.COMPLETE
+ )
+ workflow_execution = WorkflowExecution(
+ pipeline_id=pipeline_id,
+ workflow_id=workflow.id,
+ execution_mode=mode,
+ execution_method=self.execution_method,
+ execution_type=self.execution_type,
+ status=ExecutionStatus.INITIATED.value,
+ execution_log_id=self.execution_log_id,
+ )
+ workflow_execution.save()
+ else:
+ self.execution_mode = workflow_execution.execution_mode
+ self.execution_method = workflow_execution.execution_method
+ self.execution_type = workflow_execution.execution_type
+ self.execution_log_id = workflow_execution.execution_log_id
+
+ self.set_messaging_channel(str(self.execution_log_id))
+ project_settings = {}
+ project_settings[WorkflowKey.WF_PROJECT_GUID] = str(self.execution_log_id)
+ self.workflow_id = workflow.id
+ self.project_settings = project_settings
+ self.pipeline_id = pipeline_id
+ self.execution_id = str(workflow_execution.id)
+ logger.info(
+ f"Executing for Pipeline ID: {pipeline_id}, "
+ f"workflow ID: {self.workflow_id}, execution ID: {self.execution_id}, "
+ f"web socket messaging channel ID: {self.execution_log_id}"
+ )
+
+ self.compilation_result = self.compile_workflow(execution_id=self.execution_id)
+
+ def _initiate_api_execution(
+ self, tool_instance: ToolInstance, execution_path: Optional[str]
+ ) -> None:
+ if not execution_path:
+ raise InvalidAPIRequest("File shouldn't be empty")
+ tool_instance.metadata[JsonSchemaKey.ROOT_FOLDER] = execution_path
+
+ @staticmethod
+ def create_workflow_execution(
+ workflow_id: str,
+ pipeline_id: Optional[str] = None,
+ single_step: bool = False,
+ scheduled: bool = False,
+ log_events_id: Optional[str] = None,
+ execution_id: Optional[str] = None,
+ mode: tuple[str, str] = WorkflowExecution.Mode.INSTANT,
+ ) -> WorkflowExecution:
+ execution_method: tuple[str, str] = (
+ WorkflowExecution.Method.SCHEDULED
+ if scheduled
+ else WorkflowExecution.Method.DIRECT
+ )
+ execution_type: tuple[str, str] = (
+ WorkflowExecution.Type.STEP
+ if single_step
+ else WorkflowExecution.Type.COMPLETE
+ )
+ execution_log_id = log_events_id if log_events_id else pipeline_id
+ # TODO: Using objects.create() instead
+ workflow_execution = WorkflowExecution(
+ pipeline_id=pipeline_id,
+ workflow_id=workflow_id,
+ execution_mode=mode,
+ execution_method=execution_method,
+ execution_type=execution_type,
+ status=ExecutionStatus.PENDING.value,
+ execution_log_id=execution_log_id,
+ )
+ if execution_id:
+ workflow_execution.id = execution_id
+ workflow_execution.save()
+ return workflow_execution
+
+ def update_execution(
+ self,
+ status: Optional[ExecutionStatus] = None,
+ execution_time: Optional[float] = None,
+ error: Optional[str] = None,
+ increment_attempt: bool = False,
+ ) -> None:
+ execution = WorkflowExecution.objects.get(pk=self.execution_id)
+
+ if status is not None:
+ execution.status = status.value
+ if execution_time is not None:
+ execution.execution_time = execution_time
+ if error:
+ execution.error_message = error
+ if increment_attempt:
+ execution.attempts += 1
+
+ execution.save()
+
+ def has_successful_compilation(self) -> bool:
+ return self.compilation_result["success"] is True
+
+ def get_execution_instance(self) -> WorkflowExecution:
+ execution: WorkflowExecution = WorkflowExecution.objects.get(
+ pk=self.execution_id
+ )
+ return execution
+
+ def build(self) -> None:
+ if self.compilation_result["success"] is True:
+ self.build_workflow()
+ self.update_execution(status=ExecutionStatus.READY)
+ else:
+ logger.error(
+ "Errors while compiling workflow "
+ f"{self.compilation_result['problems']}"
+ )
+ self.update_execution(
+ status=ExecutionStatus.ERROR,
+ error=self.compilation_result["problems"][0],
+ )
+ raise WorkflowExecutionError(self.compilation_result["problems"][0])
+
+ def execute(self, single_step: bool = False) -> None:
+ execution_type = ExecutionType.COMPLETE
+ if single_step:
+ execution_type = ExecutionType.STEP
+
+ if self.compilation_result["success"] is False:
+ error_message = (
+ f"Errors while compiling workflow "
+ f"{self.compilation_result['problems'][0]}"
+ )
+ raise WorkflowExecutionError(error_message)
+
+ if self.execution_mode not in (
+ WorkflowExecution.Mode.INSTANT,
+ WorkflowExecution.Mode.QUEUE,
+ ):
+ error_message = f"Unknown Execution Method {self.execution_mode}"
+ raise WorkflowExecutionError(error_message)
+
+ start_time = time.time()
+ try:
+ self.execute_workflow(execution_type=execution_type)
+ end_time = time.time()
+ execution_time = end_time - start_time
+ except StopExecution as exception:
+ end_time = time.time()
+ execution_time = end_time - start_time
+ logger.info(f"Execution {self.execution_id} stopped")
+ raise exception
+ except Exception as exception:
+ end_time = time.time()
+ execution_time = end_time - start_time
+ message = str(exception)[:EXECUTION_ERROR_LENGTH]
+ logger.info(
+ f"Execution {self.execution_id} ran for {execution_time:.4f}s, "
+ f" Error {exception}"
+ )
+ raise WorkflowExecutionError(message) from exception
+
+ def publish_initial_workflow_logs(self, total_files: int) -> None:
+ """Publishes the initial logs for the workflow.
+
+ Args:
+ total_files (int): The total number of matched files.
+
+ Returns:
+ None
+ """
+ self.publish_log(f"Total matched files: {total_files}")
+ self.publish_update_log(LogState.BEGIN_WORKFLOW, "1", LogComponent.STATUS_BAR)
+ self.publish_update_log(
+ LogState.RUNNING, "Ready for execution", LogComponent.WORKFLOW
+ )
+
+ def publish_final_workflow_logs(
+ self, total_files: int, processed_files: int
+ ) -> None:
+ """Publishes the final logs for the workflow.
+
+ Returns:
+ None
+ """
+ self.publish_update_log(LogState.END_WORKFLOW, "1", LogComponent.STATUS_BAR)
+ self.publish_update_log(
+ LogState.SUCCESS, "Executed successfully", LogComponent.WORKFLOW
+ )
+ self.publish_log(
+ f"Execution completed for {processed_files} files out of {total_files}"
+ )
+
+ def publish_initial_tool_execution_logs(
+ self, current_file_idx: int, total_files: int, file_name: str
+ ) -> None:
+ """Publishes the initial logs for tool execution.
+
+ Args:
+ current_file_idx (int): 1-based index for the current file being processed
+ total_files (int): The total number of files to process
+ file_name (str): The name of the file being processed.
+
+ Returns:
+ None
+ """
+ self.publish_update_log(
+ component=LogComponent.STATUS_BAR,
+ state=LogState.MESSAGE,
+ message=f"Processing file {file_name} {current_file_idx}/{total_files}",
+ )
+ self.publish_log(f"Processing file {file_name}")
+
+ def execute_input_file(
+ self,
+ file_name: str,
+ single_step: bool,
+ file_history: Optional[FileHistory] = None,
+ ) -> bool:
+ """Executes the input file.
+
+ Args:
+ file_name (str): The name of the file to be executed.
+ single_step (bool): Flag indicating whether to execute in
+ single step mode.
+ file_history (Optional[FileHistory], optional):
+ The file history object. Defaults to None.
+ Returns:
+ bool: Flag indicating whether the file was executed.
+ """
+ execution_type = ExecutionType.COMPLETE
+ is_executed = False
+ if single_step:
+ execution_type = ExecutionType.STEP
+ if not (file_history and file_history.is_completed()):
+ self.execute_uncached_input(file_name=file_name, single_step=single_step)
+ self.publish_log(f"Tool executed successfully for {file_name}")
+ is_executed = True
+ else:
+ self.publish_log(
+ f"Skipping file {file_name} as it is already processed."
+ "Clear the cache to process it again"
+ )
+ self._handle_execution_type(execution_type)
+ return is_executed
+
+ def execute_uncached_input(self, file_name: str, single_step: bool) -> None:
+ """Executes the uncached input file.
+
+ Args:
+ file_name (str): The name of the file to be executed.
+ single_step (bool): Flag indicating whether to execute in
+ single step mode.
+
+ Returns:
+ None
+ """
+ self.publish_log("No entries found in cache, executing the tools")
+ self.publish_update_log(
+ state=LogState.SUCCESS,
+ message=f"{file_name} Sent for execution",
+ component=LogComponent.SOURCE,
+ )
+ self.execute(single_step)
+
+ def initiate_tool_execution(
+ self,
+ current_file_idx: int,
+ total_files: int,
+ file_name: str,
+ single_step: bool,
+ ) -> None:
+ """Initiates the execution of a tool for a specific file in the
+ workflow.
+
+ Args:
+ current_file_idx (int): 1-based index for the current file being processed
+ total_step (int): The total number of files to process in the workflow
+ file_name (str): The name of the file being processed
+ single_step (bool): Flag indicating whether the execution is in
+ single-step mode
+
+ Returns:
+ None
+
+ Raises:
+ None
+ """
+ execution_type = ExecutionType.COMPLETE
+ if single_step:
+ execution_type = ExecutionType.STEP
+ self.publish_initial_tool_execution_logs(
+ current_file_idx, total_files, file_name
+ )
+ self._handle_execution_type(execution_type)
+
+ source_status_message = (
+ f"({current_file_idx}/{total_files})Processing file {file_name}"
+ )
+ self.publish_update_log(
+ state=LogState.RUNNING,
+ message=source_status_message,
+ component=LogComponent.SOURCE,
+ )
+ self.publish_log("Trying to fetch results from cache")
+
+ @staticmethod
+ def update_execution_status(execution_id: str, status: ExecutionStatus) -> None:
+ try:
+ execution = WorkflowExecution.objects.get(pk=execution_id)
+ execution.status = status.value
+ execution.save()
+ except WorkflowExecution.DoesNotExist:
+ logger.error(f"execution doesn't exist {execution_id}")
+
+ @staticmethod
+ def update_execution_task(execution_id: str, task_id: str) -> None:
+ try:
+ execution = WorkflowExecution.objects.get(pk=execution_id)
+ execution.task_id = task_id
+ execution.save()
+ except WorkflowExecution.DoesNotExist:
+ logger.error(f"execution doesn't exist {execution_id}")
+
+ @staticmethod
+ def convert_tool_instance_model_to_data_class(
+ tool_instance: ToolInstance,
+ ) -> ToolInstanceDataClass:
+ tool: Tool = ToolProcessor.get_tool_by_uid(tool_instance.tool_id)
+ tool_dto = ToolInstanceDataClass(
+ id=tool_instance.id,
+ tool_id=tool_instance.tool_id,
+ workflow=tool_instance.workflow.id,
+ metadata=tool_instance.metadata,
+ step=tool_instance.step,
+ properties=tool.properties,
+ image_name=tool.image_name,
+ image_tag=tool.image_tag,
+ )
+ return tool_dto
+
+ @staticmethod
+ def convert_workflow_model_to_data_class(
+ workflow: Workflow,
+ ) -> WorkflowDto:
+ return WorkflowDto(id=workflow.id)
diff --git a/backend/workflow_manager/workflow_v2/execution_log_utils.py b/backend/workflow_manager/workflow_v2/execution_log_utils.py
new file mode 100644
index 000000000..5b1ef6079
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/execution_log_utils.py
@@ -0,0 +1,107 @@
+import logging
+import sys
+from collections import defaultdict
+
+from celery import shared_task
+from django.db import IntegrityError
+from django.db.utils import ProgrammingError
+from django_celery_beat.models import IntervalSchedule, PeriodicTask
+from utils.cache_service import CacheService
+from utils.constants import ExecutionLogConstants
+from utils.dto import LogDataDTO
+from workflow_manager.workflow_v2.models.execution_log import ExecutionLog
+
+logger = logging.getLogger(__name__)
+
+
+@shared_task(bind=True, name=ExecutionLogConstants.TASK_V2)
+def consume_log_history(self):
+ organization_logs = defaultdict(list)
+ logs_count = 0
+
+ while logs_count < ExecutionLogConstants.LOGS_BATCH_LIMIT:
+ log = CacheService.lpop(ExecutionLogConstants.LOG_QUEUE_NAME)
+ if not log:
+ break
+
+ log_data = LogDataDTO.from_json(log)
+ if not log_data:
+ continue
+
+ organization_id = log_data.organization_id
+ organization_logs[organization_id].append(
+ ExecutionLog(
+ execution_id=log_data.execution_id,
+ data=log_data.data,
+ event_time=log_data.event_time,
+ )
+ )
+ logs_count += 1
+ logger.info(f"Logs count: {logs_count}")
+ for organization_id, logs in organization_logs.items():
+ store_to_db(organization_id, logs)
+
+
+def create_log_consumer_scheduler_if_not_exists() -> None:
+ try:
+ interval, _ = IntervalSchedule.objects.get_or_create(
+ every=ExecutionLogConstants.CONSUMER_INTERVAL,
+ period=IntervalSchedule.SECONDS,
+ )
+ except ProgrammingError as error:
+ logger.warning(
+ "ProgrammingError occurred while creating "
+ "log consumer scheduler. If you are currently running "
+ "migrations for new environment, you can ignore this warning"
+ )
+ if all(arg not in sys.argv for arg in ("migrate", "makemigrations")):
+ logger.warning(f"ProgrammingError details: {error}")
+ return
+ except IntervalSchedule.MultipleObjectsReturned as error:
+ logger.error(f"Error occurred while getting interval schedule: {error}")
+ interval = IntervalSchedule.objects.filter(
+ every=ExecutionLogConstants.CONSUMER_INTERVAL,
+ period=IntervalSchedule.SECONDS,
+ ).first()
+ try:
+ # Create the scheduler
+ task, created = PeriodicTask.objects.get_or_create(
+ name=ExecutionLogConstants.PERIODIC_TASK_NAME_V2,
+ task=ExecutionLogConstants.TASK_V2,
+ defaults={
+ "interval": interval,
+ "queue": ExecutionLogConstants.CELERY_QUEUE_NAME,
+ "enabled": ExecutionLogConstants.IS_ENABLED,
+ },
+ )
+ if not created:
+ task.enabled = ExecutionLogConstants.IS_ENABLED
+ task.interval = interval
+ task.queue = ExecutionLogConstants.CELERY_QUEUE_NAME
+ task.save()
+ logger.info("Log consumer scheduler updated successfully.")
+ else:
+ logger.info("Log consumer scheduler created successfully.")
+ except IntegrityError as error:
+ logger.error(f"Error occurred while creating log consumer scheduler: {error}")
+
+
+def store_to_db(organization_id: str, execution_logs: list[ExecutionLog]) -> None:
+
+ # Store the log data in the database within tenant context
+ ExecutionLog.objects.bulk_create(objs=execution_logs, ignore_conflicts=True)
+
+
+class ExecutionLogUtils:
+
+ @staticmethod
+ def get_execution_logs_by_execution_id(execution_id) -> list[ExecutionLog]:
+ """Get all execution logs for a given execution ID.
+
+ Args:
+ execution_id (int): The ID of the execution.
+
+ Returns:
+ list[ExecutionLog]: A list of ExecutionLog objects.
+ """
+ return ExecutionLog.objects.filter(execution_id=execution_id)
diff --git a/backend/workflow_manager/workflow_v2/execution_log_view.py b/backend/workflow_manager/workflow_v2/execution_log_view.py
new file mode 100644
index 000000000..96f557b77
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/execution_log_view.py
@@ -0,0 +1,27 @@
+import logging
+
+from permissions.permission import IsOwner
+from rest_framework import viewsets
+from rest_framework.versioning import URLPathVersioning
+from utils.pagination import CustomPagination
+from workflow_manager.workflow_v2.models.execution_log import ExecutionLog
+from workflow_manager.workflow_v2.serializers import WorkflowExecutionLogSerializer
+
+logger = logging.getLogger(__name__)
+
+
+class WorkflowExecutionLogViewSet(viewsets.ModelViewSet):
+ versioning_class = URLPathVersioning
+ permission_classes = [IsOwner]
+ serializer_class = WorkflowExecutionLogSerializer
+ pagination_class = CustomPagination
+
+ EVENT_TIME_FELID_ASC = "event_time"
+
+ def get_queryset(self):
+ # Get the execution_id:pk from the URL path
+ execution_id = self.kwargs.get("pk")
+ queryset = ExecutionLog.objects.filter(execution_id=execution_id).order_by(
+ self.EVENT_TIME_FELID_ASC
+ )
+ return queryset
diff --git a/backend/workflow_manager/workflow_v2/execution_view.py b/backend/workflow_manager/workflow_v2/execution_view.py
new file mode 100644
index 000000000..190a72617
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/execution_view.py
@@ -0,0 +1,25 @@
+import logging
+
+from permissions.permission import IsOwner
+from rest_framework import viewsets
+from rest_framework.versioning import URLPathVersioning
+from workflow_manager.workflow_v2.models.execution import WorkflowExecution
+from workflow_manager.workflow_v2.serializers import WorkflowExecutionSerializer
+
+logger = logging.getLogger(__name__)
+
+
+class WorkflowExecutionViewSet(viewsets.ModelViewSet):
+ versioning_class = URLPathVersioning
+ permission_classes = [IsOwner]
+ serializer_class = WorkflowExecutionSerializer
+
+ CREATED_AT_FIELD_DESC = "-created_at"
+
+ def get_queryset(self):
+ # Get the uuid:pk from the URL path
+ workflow_id = self.kwargs.get("pk")
+ queryset = WorkflowExecution.objects.filter(workflow_id=workflow_id).order_by(
+ self.CREATED_AT_FIELD_DESC
+ )
+ return queryset
diff --git a/backend/workflow_manager/workflow_v2/file_history_helper.py b/backend/workflow_manager/workflow_v2/file_history_helper.py
new file mode 100644
index 000000000..7d84cf0da
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/file_history_helper.py
@@ -0,0 +1,89 @@
+import logging
+from typing import Any, Optional
+
+from django.db.utils import IntegrityError
+from workflow_manager.workflow_v2.enums import ExecutionStatus
+from workflow_manager.workflow_v2.models.file_history import FileHistory
+from workflow_manager.workflow_v2.models.workflow import Workflow
+
+logger = logging.getLogger(__name__)
+
+
+class FileHistoryHelper:
+ """A helper class for managing file history related operations."""
+
+ @staticmethod
+ def get_file_history(
+ workflow: Workflow, cache_key: Optional[str] = None
+ ) -> Optional[FileHistory]:
+ """Retrieve a file history record based on the cache key.
+
+ Args:
+ cache_key (Optional[str]): The cache key to search for.
+
+ Returns:
+ Optional[FileHistory]: The matching file history record, if found.
+ """
+ if not cache_key:
+ return None
+ try:
+ file_history: FileHistory = FileHistory.objects.get(
+ cache_key=cache_key, workflow=workflow
+ )
+ except FileHistory.DoesNotExist:
+ return None
+ return file_history
+
+ @staticmethod
+ def create_file_history(
+ cache_key: str,
+ workflow: Workflow,
+ status: ExecutionStatus,
+ result: Any,
+ metadata: Any,
+ error: Optional[str] = None,
+ file_name: Optional[str] = None,
+ ) -> FileHistory:
+ """Create a new file history record.
+
+ Args:
+ cache_key (str): The cache key for the file.
+ workflow (Workflow): The associated workflow.
+ status (ExecutionStatus): The execution status.
+ result (Any): The result from the execution.
+
+ Returns:
+ FileHistory: The newly created file history record.
+ """
+ try:
+ file_history: FileHistory = FileHistory.objects.create(
+ workflow=workflow,
+ cache_key=cache_key,
+ status=status.value,
+ result=str(result),
+ meta_data=str(metadata),
+ error=str(error) if error else "",
+ )
+ except IntegrityError:
+ # TODO: Need to find why duplicate insert is coming
+ logger.warning(
+ "Trying to insert duplication data for filename %s for workflow %s",
+ file_name,
+ workflow,
+ )
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=cache_key
+ )
+
+ return file_history
+
+ @staticmethod
+ def clear_history_for_workflow(
+ workflow: Workflow,
+ ) -> None:
+ """Clear all file history records associated with a workflow.
+
+ Args:
+ workflow (Workflow): The workflow to clear the history for.
+ """
+ FileHistory.objects.filter(workflow=workflow).delete()
diff --git a/backend/workflow_manager/workflow_v2/generator.py b/backend/workflow_manager/workflow_v2/generator.py
new file mode 100644
index 000000000..6543aae7f
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/generator.py
@@ -0,0 +1,113 @@
+import logging
+import uuid
+from typing import Any
+
+from rest_framework.request import Request
+from tool_instance_v2.constants import ToolInstanceKey as TIKey
+from tool_instance_v2.exceptions import ToolInstantiationError
+from tool_instance_v2.tool_processor import ToolProcessor
+from unstract.tool_registry.dto import Tool
+from workflow_manager.workflow_v2.constants import WorkflowKey
+from workflow_manager.workflow_v2.dto import ProvisionalWorkflow
+from workflow_manager.workflow_v2.exceptions import WorkflowGenerationError
+from workflow_manager.workflow_v2.models.workflow import Workflow as WorkflowModel
+
+from unstract.core.llm_workflow_generator.llm_interface import LLMInterface
+
+logger = logging.getLogger(__name__)
+
+
+class WorkflowGenerator:
+ """Helps with generating a workflow using the LLM."""
+
+ def __init__(self, workflow_id: str = str(uuid.uuid4())) -> None:
+ self._request: Request = {}
+ self._llm_response = ""
+ self._workflow_id = workflow_id
+ self._provisional_wf: ProvisionalWorkflow
+
+ @property
+ def llm_response(self) -> dict[str, Any]:
+ output: dict[str, str] = self._provisional_wf.output
+ return output
+
+ @property
+ def provisional_wf(self) -> ProvisionalWorkflow:
+ return self._provisional_wf
+
+ def _get_provisional_workflow(self, tools: list[Tool]) -> ProvisionalWorkflow:
+ """Helper to generate the provisional workflow Gets stored as
+ `workflow.Workflow.llm_response` eventually."""
+ provisional_wf: ProvisionalWorkflow
+ try:
+ if not self._request:
+ raise WorkflowGenerationError(
+ "Unable to generate a workflow: missing request"
+ )
+ llm_interface = LLMInterface()
+
+ provisional_wf_dict = llm_interface.get_provisional_workflow_from_llm(
+ workflow_id=self._workflow_id,
+ tools=tools,
+ user_prompt=self._request.data.get(WorkflowKey.PROMPT_TEXT),
+ use_cache=True,
+ )
+ provisional_wf = ProvisionalWorkflow(provisional_wf_dict)
+ if provisional_wf.result != "OK":
+ raise WorkflowGenerationError(
+ f"Unable to generate a workflow: {provisional_wf.output}"
+ )
+ except Exception as e:
+ logger.error(f"{e}")
+ raise WorkflowGenerationError
+ return provisional_wf
+
+ def set_request(self, request: Request) -> None:
+ self._request = request
+
+ def generate_workflow(self, tools: list[Tool]) -> None:
+ """Used to talk to the GPT model through core and obtain a provisional
+ workflow for the user to work with."""
+ self._provisional_wf = self._get_provisional_workflow(tools)
+
+ @staticmethod
+ def get_tool_instance_data_from_llm(
+ workflow: WorkflowModel,
+ ) -> list[dict[str, str]]:
+ """Used to generate the dict of tool instances for a given workflow.
+
+ Call with ToolInstanceSerializer(data=tool_instance_data_list,many=True)
+ """
+ tool_instance_data_list = []
+ for step, tool_step in enumerate(
+ workflow.llm_response.get(WorkflowKey.WF_STEPS, [])
+ ):
+ step = step + 1
+ logger.info(f"Building tool instance data for step: {step}")
+ tool_function: str = tool_step[WorkflowKey.WF_TOOL]
+ wf_input: str = tool_step[WorkflowKey.WF_INPUT]
+ wf_output: str = tool_step[WorkflowKey.WF_OUTPUT]
+ try:
+ tool: Tool = ToolProcessor.get_tool_by_uid(tool_function)
+ # TODO: Mark optional fields in model and handle in ToolInstance serializer # noqa
+ tool_instance_data = {
+ TIKey.PK: tool_step[WorkflowKey.WF_TOOL_UUID],
+ TIKey.WORKFLOW: workflow.id,
+ # Added to support changes for UN-154
+ WorkflowKey.WF_ID: workflow.id,
+ TIKey.TOOL_ID: tool_function,
+ TIKey.METADATA: {
+ WorkflowKey.WF_TOOL_INSTANCE_ID: tool_step[
+ WorkflowKey.WF_TOOL_UUID
+ ],
+ **ToolProcessor.get_default_settings(tool),
+ },
+ TIKey.STEP: str(step),
+ TIKey.INPUT: wf_input,
+ TIKey.OUTPUT: wf_output,
+ }
+ tool_instance_data_list.append(tool_instance_data)
+ except Exception as e:
+ logger.error(f"Error while getting data for {tool_function}: {e}")
+ raise ToolInstantiationError(tool_name=tool_function)
+ return tool_instance_data_list
diff --git a/backend/workflow_manager/workflow_v2/models/__init__.py b/backend/workflow_manager/workflow_v2/models/__init__.py
new file mode 100644
index 000000000..938907762
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/models/__init__.py
@@ -0,0 +1,4 @@
+from .execution import WorkflowExecution # noqa: F401
+from .execution_log import ExecutionLog # noqa: F401
+from .file_history import FileHistory # noqa: F401
+from .workflow import Workflow # noqa: F401
diff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.py
new file mode 100644
index 000000000..23ceb2bbd
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/models/execution.py
@@ -0,0 +1,81 @@
+import uuid
+
+from django.db import models
+from utils.models.base_model import BaseModel
+
+EXECUTION_ERROR_LENGTH = 256
+
+
+class WorkflowExecution(BaseModel):
+ class Mode(models.TextChoices):
+ INSTANT = "INSTANT", "will be executed immediately"
+ QUEUE = "QUEUE", "will be placed in a queue"
+
+ class Method(models.TextChoices):
+ DIRECT = "DIRECT", " Execution triggered manually"
+ SCHEDULED = "SCHEDULED", "Scheduled execution"
+
+ class Type(models.TextChoices):
+ COMPLETE = "COMPLETE", "For complete execution"
+ STEP = "STEP", "For step-by-step execution "
+
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ # TODO: Make as foreign key to access the instance directly
+ pipeline_id = models.UUIDField(
+ editable=False,
+ null=True,
+ db_comment="ID of the associated pipeline, if applicable",
+ )
+ task_id = models.UUIDField(
+ editable=False,
+ null=True,
+ db_comment="task id of asynchronous execution",
+ )
+ # We can remove workflow_id if it not required
+ workflow_id = models.UUIDField(
+ editable=False, db_comment="Id of workflow to be executed"
+ )
+ project_settings_id = models.UUIDField(
+ editable=False,
+ default=uuid.uuid4,
+ db_comment="Id of project settings used while execution",
+ )
+ execution_mode = models.CharField(
+ choices=Mode.choices, db_comment="Mode of execution"
+ )
+ execution_method = models.CharField(
+ choices=Method.choices, db_comment="Method of execution"
+ )
+ execution_type = models.CharField(
+ choices=Type.choices, db_comment="Type of execution"
+ )
+ execution_log_id = models.CharField(
+ default="", editable=False, db_comment="Execution log events Id"
+ )
+ # TODO: Restrict with an enum
+ status = models.CharField(default="", db_comment="Current status of execution")
+ error_message = models.CharField(
+ max_length=EXECUTION_ERROR_LENGTH,
+ blank=True,
+ default="",
+ db_comment="Details of encountered errors",
+ )
+ attempts = models.IntegerField(default=0, db_comment="number of attempts taken")
+ execution_time = models.FloatField(
+ default=0, db_comment="execution time in seconds"
+ )
+
+ def __str__(self) -> str:
+ return (
+ f"Workflow execution: {self.id} ("
+ f"pipeline ID: {self.pipeline_id}, "
+ f"workflow iD: {self.workflow_id}, "
+ f"execution method: {self.execution_method}, "
+ f"status: {self.status}, "
+ f"error message: {self.error_message})"
+ )
+
+ class Meta:
+ verbose_name = "Workflow Execution"
+ verbose_name_plural = "Workflow Executions"
+ db_table = "workflow_execution_v2"
diff --git a/backend/workflow_manager/workflow_v2/models/execution_log.py b/backend/workflow_manager/workflow_v2/models/execution_log.py
new file mode 100644
index 000000000..6bec96009
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/models/execution_log.py
@@ -0,0 +1,22 @@
+import uuid
+
+from django.db import models
+from utils.models.base_model import BaseModel
+
+
+class ExecutionLog(BaseModel):
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ execution_id = models.UUIDField(
+ editable=False,
+ db_comment="Execution ID",
+ )
+ data = models.JSONField(db_comment="Execution log data")
+ event_time = models.DateTimeField(db_comment="Execution log event time")
+
+ def __str__(self):
+ return f"Execution ID: {self.execution_id}, Message: {self.data}"
+
+ class Meta:
+ verbose_name = "Execution Log"
+ verbose_name_plural = "Execution Logs"
+ db_table = "execution_log_v2"
diff --git a/backend/workflow_manager/workflow_v2/models/file_history.py b/backend/workflow_manager/workflow_v2/models/file_history.py
new file mode 100644
index 000000000..5ad521cd0
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/models/file_history.py
@@ -0,0 +1,52 @@
+import uuid
+
+from django.db import models
+from utils.models.base_model import BaseModel
+from workflow_manager.workflow_v2.enums import ExecutionStatus
+from workflow_manager.workflow_v2.models.workflow import Workflow
+
+HASH_LENGTH = 64
+
+
+class FileHistory(BaseModel):
+ def is_completed(self) -> bool:
+ """Check if the execution status is completed.
+
+ Returns:
+ bool: True if the execution status is completed, False otherwise.
+ """
+ return (
+ self.status is not None and self.status == ExecutionStatus.COMPLETED.value
+ )
+
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ cache_key = models.CharField(
+ max_length=HASH_LENGTH,
+ db_comment="Hash value of file contents, WF and tool modified times",
+ )
+ workflow = models.ForeignKey(
+ Workflow,
+ on_delete=models.CASCADE,
+ related_name="file_histories",
+ )
+ status = models.TextField(
+ choices=ExecutionStatus.choices(),
+ db_comment="Latest status of execution",
+ )
+ error = models.TextField(
+ blank=True,
+ default="",
+ db_comment="Error message",
+ )
+ result = models.TextField(blank=True, db_comment="Result from execution")
+
+ class Meta:
+ verbose_name = "File History"
+ verbose_name_plural = "File Histories"
+ db_table = "file_history_v2"
+ constraints = [
+ models.UniqueConstraint(
+ fields=["workflow", "cache_key"],
+ name="unique_workflow_cacheKey",
+ ),
+ ]
diff --git a/backend/workflow_manager/workflow_v2/models/workflow.py b/backend/workflow_manager/workflow_v2/models/workflow.py
new file mode 100644
index 000000000..dcdaa5dae
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/models/workflow.py
@@ -0,0 +1,95 @@
+import uuid
+
+from account_v2.models import User
+from django.db import models
+from utils.models.base_model import BaseModel
+from utils.models.organization_mixin import (
+ DefaultOrganizationManagerMixin,
+ DefaultOrganizationMixin,
+)
+
+PROMPT_NAME_LENGTH = 32
+WORKFLOW_STATUS_LENGTH = 16
+EXECUTION_ERROR_LENGTH = 256
+DESCRIPTION_FIELD_LENGTH = 490
+WORKFLOW_NAME_SIZE = 128
+
+
+class WorkflowModelManager(DefaultOrganizationManagerMixin, models.Manager):
+ pass
+
+
+class Workflow(DefaultOrganizationMixin, BaseModel):
+ class WorkflowType(models.TextChoices):
+ DEFAULT = "DEFAULT", "Not ready yet"
+ ETL = "ETL", "ETL pipeline"
+ TASK = "TASK", "TASK pipeline"
+ API = "API", "API deployment"
+ APP = "APP", "App deployment"
+
+ class ExecutionAction(models.TextChoices):
+ START = "START", "Start the Execution"
+ NEXT = "NEXT", "Execute next tool"
+ STOP = "STOP", "Stop the execution"
+ CONTINUE = "CONTINUE", "Continue to full execution"
+
+ # TODO Make this guid as primaryId instaed of current id bigint
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ # TODO: Move prompt fields as a One-One relationship/into Prompt instead
+ prompt_name = models.CharField(max_length=PROMPT_NAME_LENGTH, default="")
+ description = models.TextField(max_length=DESCRIPTION_FIELD_LENGTH, default="")
+ workflow_name = models.CharField(max_length=WORKFLOW_NAME_SIZE)
+ prompt_text = models.TextField(default="")
+ is_active = models.BooleanField(default=False)
+ status = models.CharField(max_length=WORKFLOW_STATUS_LENGTH, default="")
+ llm_response = models.TextField()
+ workflow_owner = models.ForeignKey(
+ User,
+ on_delete=models.SET_NULL,
+ related_name="workflows_owned",
+ null=True,
+ blank=True,
+ )
+ deployment_type = models.CharField(
+ choices=WorkflowType.choices,
+ db_comment="Type of workflow deployment",
+ default=WorkflowType.DEFAULT,
+ )
+ source_settings = models.JSONField(
+ null=True, db_comment="Settings for the Source module"
+ )
+ destination_settings = models.JSONField(
+ null=True, db_comment="Settings for the Destination module"
+ )
+
+ created_by = models.ForeignKey(
+ User,
+ on_delete=models.SET_NULL,
+ related_name="workflows_created",
+ null=True,
+ blank=True,
+ )
+ modified_by = models.ForeignKey(
+ User,
+ on_delete=models.SET_NULL,
+ related_name="workflows_modified",
+ null=True,
+ blank=True,
+ )
+
+ # Manager
+ objects = WorkflowModelManager()
+
+ def __str__(self) -> str:
+ return f"{self.id}, name: {self.workflow_name}"
+
+ class Meta:
+ verbose_name = "Workflow"
+ verbose_name_plural = "Workflows"
+ db_table = "workflow_v2"
+ constraints = [
+ models.UniqueConstraint(
+ fields=["workflow_name", "organization"],
+ name="unique_workflow_name",
+ ),
+ ]
diff --git a/backend/workflow_manager/workflow_v2/serializers.py b/backend/workflow_manager/workflow_v2/serializers.py
new file mode 100644
index 000000000..0f4c8c3d6
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/serializers.py
@@ -0,0 +1,165 @@
+import logging
+from typing import Any, Optional, Union
+
+from project.constants import ProjectKey
+from rest_framework.serializers import (
+ CharField,
+ ChoiceField,
+ JSONField,
+ ModelSerializer,
+ Serializer,
+ UUIDField,
+ ValidationError,
+)
+from tool_instance_v2.serializers import ToolInstanceSerializer
+from tool_instance_v2.tool_instance_helper import ToolInstanceHelper
+from workflow_manager.endpoint_v2.models import WorkflowEndpoint
+from workflow_manager.workflow_v2.constants import WorkflowExecutionKey, WorkflowKey
+from workflow_manager.workflow_v2.exceptions import WorkflowGenerationError
+from workflow_manager.workflow_v2.generator import WorkflowGenerator
+from workflow_manager.workflow_v2.models.execution import WorkflowExecution
+from workflow_manager.workflow_v2.models.execution_log import ExecutionLog
+from workflow_manager.workflow_v2.models.workflow import Workflow
+
+from backend.constants import RequestKey
+from backend.serializers import AuditSerializer
+
+logger = logging.getLogger(__name__)
+
+
+class WorkflowSerializer(AuditSerializer):
+ tool_instances = ToolInstanceSerializer(many=True, read_only=True)
+
+ class Meta:
+ model = Workflow
+ fields = "__all__"
+ extra_kwargs = {
+ WorkflowKey.LLM_RESPONSE: {
+ "required": False,
+ },
+ }
+
+ def to_representation(self, instance: Workflow) -> dict[str, str]:
+ representation: dict[str, str] = super().to_representation(instance)
+ representation[WorkflowKey.WF_NAME] = instance.workflow_name
+ representation[WorkflowKey.WF_TOOL_INSTANCES] = ToolInstanceSerializer(
+ ToolInstanceHelper.get_tool_instances_by_workflow(
+ workflow_id=instance.id, order_by="step"
+ ),
+ many=True,
+ context=self.context,
+ ).data
+ representation["created_by_email"] = instance.created_by.email
+ return representation
+
+ def create(self, validated_data: dict[str, Any]) -> Any:
+ if self.context.get(RequestKey.REQUEST):
+ validated_data[WorkflowKey.WF_OWNER] = self.context.get(
+ RequestKey.REQUEST
+ ).user
+ return super().create(validated_data)
+
+ def update(self, instance: Any, validated_data: dict[str, Any]) -> Any:
+ if validated_data.get(WorkflowKey.PROMPT_TEXT):
+ instance.workflow_tool.all().delete()
+ return super().update(instance, validated_data)
+
+ def save(self, **kwargs: Any) -> Workflow:
+ workflow: Workflow = super().save(**kwargs)
+ if self.validated_data.get(WorkflowKey.PROMPT_TEXT):
+ try:
+ tool_serializer = ToolInstanceSerializer(
+ data=WorkflowGenerator.get_tool_instance_data_from_llm(
+ workflow=workflow
+ ),
+ many=True,
+ context=self.context,
+ )
+ tool_serializer.is_valid(raise_exception=True)
+ tool_serializer.save()
+ except Exception as exc:
+ logger.error(f"Error while generating tool instances: {exc}")
+ raise WorkflowGenerationError
+
+ request = self.context.get("request")
+ if not request:
+ return workflow
+ return workflow
+
+
+class ExecuteWorkflowSerializer(Serializer):
+ workflow_id = UUIDField(required=False)
+ project_id = UUIDField(required=False)
+ execution_action = ChoiceField(
+ choices=Workflow.ExecutionAction.choices, required=False
+ )
+ execution_id = UUIDField(required=False)
+ log_guid = UUIDField(required=False)
+ # TODO: Add other fields to handle WFExecution method, mode .etc.
+
+ def get_workflow_id(
+ self, validated_data: dict[str, Union[str, None]]
+ ) -> Optional[str]:
+ return validated_data.get(WorkflowKey.WF_ID)
+
+ def get_project_id(
+ self, validated_data: dict[str, Union[str, None]]
+ ) -> Optional[str]:
+ return validated_data.get(ProjectKey.PROJECT_ID)
+
+ def get_execution_id(
+ self, validated_data: dict[str, Union[str, None]]
+ ) -> Optional[str]:
+ return validated_data.get(WorkflowExecutionKey.EXECUTION_ID)
+
+ def get_log_guid(
+ self, validated_data: dict[str, Union[str, None]]
+ ) -> Optional[str]:
+ return validated_data.get(WorkflowExecutionKey.LOG_GUID)
+
+ def get_execution_action(
+ self, validated_data: dict[str, Union[str, None]]
+ ) -> Optional[str]:
+ return validated_data.get(WorkflowKey.EXECUTION_ACTION)
+
+ def validate(
+ self, data: dict[str, Union[str, None]]
+ ) -> dict[str, Union[str, None]]:
+ workflow_id = data.get(WorkflowKey.WF_ID)
+ project_id = data.get(ProjectKey.PROJECT_ID)
+
+ if not workflow_id and not project_id:
+ raise ValidationError(
+ "At least one of 'workflow_id' or 'project_id' is required."
+ )
+
+ return data
+
+
+class ExecuteWorkflowResponseSerializer(Serializer):
+ workflow_id = UUIDField()
+ execution_id = UUIDField()
+ execution_status = CharField()
+ log_id = CharField()
+ error = CharField()
+ result = JSONField()
+
+
+class WorkflowEndpointSerializer(ModelSerializer):
+ workflow_name = CharField(source="workflow.workflow_name", read_only=True)
+
+ class Meta:
+ model = WorkflowEndpoint
+ fields = "__all__"
+
+
+class WorkflowExecutionSerializer(ModelSerializer):
+ class Meta:
+ model = WorkflowExecution
+ fields = "__all__"
+
+
+class WorkflowExecutionLogSerializer(ModelSerializer):
+ class Meta:
+ model = ExecutionLog
+ fields = "__all__"
diff --git a/backend/workflow_manager/workflow_v2/urls.py b/backend/workflow_manager/workflow_v2/urls.py
new file mode 100644
index 000000000..78e1b9aff
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/urls.py
@@ -0,0 +1,77 @@
+from django.urls import path
+from rest_framework.urlpatterns import format_suffix_patterns
+from workflow_manager.workflow_v2.execution_log_view import WorkflowExecutionLogViewSet
+from workflow_manager.workflow_v2.execution_view import WorkflowExecutionViewSet
+from workflow_manager.workflow_v2.views import WorkflowViewSet
+
+workflow_list = WorkflowViewSet.as_view(
+ {
+ "get": "list",
+ "post": "create",
+ }
+)
+workflow_detail = WorkflowViewSet.as_view(
+ # fmt: off
+ {
+ 'get': 'retrieve',
+ 'put': 'update',
+ 'patch': 'partial_update',
+ 'delete': 'destroy'
+ }
+ # fmt: on
+)
+workflow_execute = WorkflowViewSet.as_view({"post": "execute", "put": "activate"})
+execution_entity = WorkflowExecutionViewSet.as_view({"get": "retrieve"})
+execution_list = WorkflowExecutionViewSet.as_view({"get": "list"})
+execution_log_list = WorkflowExecutionLogViewSet.as_view({"get": "list"})
+workflow_clear_cache = WorkflowViewSet.as_view({"get": "clear_cache"})
+workflow_clear_file_marker = WorkflowViewSet.as_view({"get": "clear_file_marker"})
+workflow_schema = WorkflowViewSet.as_view({"get": "get_schema"})
+can_update = WorkflowViewSet.as_view({"get": "can_update"})
+urlpatterns = format_suffix_patterns(
+ [
+ path("", workflow_list, name="workflow-list"),
+ path("<uuid:pk>/", workflow_detail, name="workflow-detail"),
+ path(
+ "<uuid:pk>/clear-cache/",
+ workflow_clear_cache,
+ name="clear-cache",
+ ),
+ path(
+ "<uuid:pk>/clear-file-marker/",
+ workflow_clear_file_marker,
+ name="clear-file-marker",
+ ),
+ path(
+ "<uuid:pk>/can-update/",
+ can_update,
+ name="can-update",
+ ),
+ path("execute/", workflow_execute, name="execute-workflow"),
+ path(
+ "active/<uuid:pk>/",
+ workflow_execute,
+ name="active-workflow",
+ ),
+ path(
+ "<uuid:pk>/execution/",
+ execution_list,
+ name="execution-list",
+ ),
+ path(
+ "execution/<uuid:pk>/",
+ execution_entity,
+ name="workflow-detail",
+ ),
+ path(
+ "execution/<uuid:pk>/logs/",
+ execution_log_list,
+ name="execution-log",
+ ),
+ path(
+ "schema/",
+ workflow_schema,
+ name="workflow-schema",
+ ),
+ ]
+)
diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.py
new file mode 100644
index 000000000..f9b9c50f4
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/views.py
@@ -0,0 +1,296 @@
+import logging
+from typing import Any, Optional
+
+from connector_v2.connector_instance_helper import ConnectorInstanceHelper
+from django.conf import settings
+from django.db.models.query import QuerySet
+from permissions.permission import IsOwner
+from pipeline_v2.models import Pipeline
+from pipeline_v2.pipeline_processor import PipelineProcessor
+from rest_framework import serializers, status, viewsets
+from rest_framework.decorators import action
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.versioning import URLPathVersioning
+from tool_instance_v2.tool_processor import ToolProcessor
+from unstract.tool_registry.dto import Tool
+from utils.filtering import FilterHelper
+from workflow_manager.endpoint_v2.destination import DestinationConnector
+from workflow_manager.endpoint_v2.endpoint_utils import WorkflowEndpointUtils
+from workflow_manager.endpoint_v2.source import SourceConnector
+from workflow_manager.workflow_v2.constants import WorkflowKey
+from workflow_manager.workflow_v2.dto import ExecutionResponse
+from workflow_manager.workflow_v2.enums import SchemaEntity, SchemaType
+from workflow_manager.workflow_v2.exceptions import (
+ WorkflowDoesNotExistError,
+ WorkflowGenerationError,
+ WorkflowRegenerationError,
+)
+from workflow_manager.workflow_v2.generator import WorkflowGenerator
+from workflow_manager.workflow_v2.models.workflow import Workflow
+from workflow_manager.workflow_v2.serializers import (
+ ExecuteWorkflowResponseSerializer,
+ ExecuteWorkflowSerializer,
+ WorkflowSerializer,
+)
+from workflow_manager.workflow_v2.workflow_helper import (
+ WorkflowHelper,
+ WorkflowSchemaHelper,
+)
+
+from backend.constants import RequestKey
+
+logger = logging.getLogger(__name__)
+
+
+def make_execution_response(response: ExecutionResponse) -> Any:
+ return ExecuteWorkflowResponseSerializer(response).data
+
+
+class WorkflowViewSet(viewsets.ModelViewSet):
+ versioning_class = URLPathVersioning
+ permission_classes = [IsOwner]
+ queryset = Workflow.objects.all()
+
+ def get_queryset(self) -> QuerySet:
+ filter_args = FilterHelper.build_filter_args(
+ self.request,
+ RequestKey.PROJECT,
+ WorkflowKey.WF_OWNER,
+ WorkflowKey.WF_IS_ACTIVE,
+ )
+ queryset = (
+ Workflow.objects.filter(created_by=self.request.user, **filter_args)
+ if filter_args
+ else Workflow.objects.filter(created_by=self.request.user)
+ )
+ order_by = self.request.query_params.get("order_by")
+ if order_by == "desc":
+ queryset = queryset.order_by("-modified_at")
+ elif order_by == "asc":
+ queryset = queryset.order_by("modified_at")
+
+ return queryset
+
+ def get_serializer_class(self) -> serializers.Serializer:
+ if self.action == "execute":
+ return ExecuteWorkflowSerializer
+ else:
+ return WorkflowSerializer
+
+ def _generate_workflow(self, workflow_id: str) -> WorkflowGenerator:
+ registry_tools: list[Tool] = ToolProcessor.get_registry_tools()
+ generator = WorkflowGenerator(workflow_id=workflow_id)
+ generator.set_request(self.request)
+ generator.generate_workflow(registry_tools)
+ return generator
+
+ def perform_update(self, serializer: WorkflowSerializer) -> Workflow:
+ """To edit a workflow. Regenerates the tool instances for a new prompt.
+
+ Raises: WorkflowGenerationError
+ """
+ kwargs = {}
+ if serializer.validated_data.get(WorkflowKey.PROMPT_TEXT):
+ workflow: Workflow = self.get_object()
+ generator = self._generate_workflow(workflow_id=workflow.id)
+ kwargs = {
+ WorkflowKey.LLM_RESPONSE: generator.llm_response,
+ WorkflowKey.WF_IS_ACTIVE: True,
+ }
+ try:
+ workflow = serializer.save(**kwargs)
+ return workflow
+ except Exception as e:
+ logger.error(f"Error saving workflow to DB: {e}")
+ raise WorkflowRegenerationError
+
+ def perform_create(self, serializer: WorkflowSerializer) -> Workflow:
+ """To create a new workflow. Creates the Workflow instance first and
+ uses it to generate the tool instances.
+
+ Raises: WorkflowGenerationError
+ """
+ try:
+ workflow = serializer.save(
+ is_active=True,
+ )
+ WorkflowEndpointUtils.create_endpoints_for_workflow(workflow)
+
+ # Enable GCS configurations to create GCS while creating a workflow
+ if (
+ settings.GOOGLE_STORAGE_ACCESS_KEY_ID
+ and settings.UNSTRACT_FREE_STORAGE_BUCKET_NAME
+ ):
+ ConnectorInstanceHelper.create_default_gcs_connector(
+ workflow, self.request.user
+ )
+
+ except Exception as e:
+ logger.error(f"Error saving workflow to DB: {e}")
+ raise WorkflowGenerationError
+ return workflow
+
+ def get_execution(self, request: Request, pk: str) -> Response:
+ execution = WorkflowHelper.get_current_execution(pk)
+ return Response(make_execution_response(execution), status=status.HTTP_200_OK)
+
+ def get_workflow_by_id_or_project_id(
+ self,
+ workflow_id: Optional[str] = None,
+ project_id: Optional[str] = None,
+ ) -> Workflow:
+ """Retrieve workflow by workflow id or project Id.
+
+ Args:
+ workflow_id (Optional[str], optional): workflow Id.
+ project_id (Optional[str], optional): Project Id.
+
+ Raises:
+ WorkflowDoesNotExistError: _description_
+
+ Returns:
+ Workflow: workflow
+ """
+ if workflow_id:
+ workflow = WorkflowHelper.get_workflow_by_id(workflow_id)
+ elif project_id:
+ workflow = WorkflowHelper.get_active_workflow_by_project_id(project_id)
+ else:
+ raise WorkflowDoesNotExistError()
+ return workflow
+
+ def execute(
+ self,
+ request: Request,
+ pipeline_guid: Optional[str] = None,
+ ) -> Response:
+ self.serializer_class = ExecuteWorkflowSerializer
+ serializer = ExecuteWorkflowSerializer(data=request.data)
+ serializer.is_valid(raise_exception=True)
+ workflow_id = serializer.get_workflow_id(serializer.validated_data)
+ project_id = serializer.get_project_id(serializer.validated_data)
+ execution_id = serializer.get_execution_id(serializer.validated_data)
+ execution_action = serializer.get_execution_action(serializer.validated_data)
+ file_objs = request.FILES.getlist("files")
+ include_metadata = (
+ request.data.get("include_metadata", "false").lower() == "true"
+ )
+ hashes_of_files = {}
+ if file_objs and execution_id and workflow_id:
+ hashes_of_files = SourceConnector.add_input_file_to_api_storage(
+ workflow_id=workflow_id,
+ execution_id=execution_id,
+ file_objs=file_objs,
+ )
+
+ try:
+ workflow = self.get_workflow_by_id_or_project_id(
+ workflow_id=workflow_id, project_id=project_id
+ )
+ execution_response = self.execute_workflow(
+ workflow=workflow,
+ execution_action=execution_action,
+ execution_id=execution_id,
+ pipeline_guid=pipeline_guid,
+ hash_values_of_files=hashes_of_files,
+ include_metadata=include_metadata,
+ )
+ return Response(
+ make_execution_response(execution_response),
+ status=status.HTTP_200_OK,
+ )
+ except Exception as exception:
+ logger.error(f"Error while executing workflow: {exception}")
+ if file_objs and execution_id and workflow_id:
+ DestinationConnector.delete_api_storage_dir(
+ workflow_id=workflow_id, execution_id=execution_id
+ )
+ raise exception
+
+ def execute_workflow(
+ self,
+ workflow: Workflow,
+ execution_action: Optional[str] = None,
+ execution_id: Optional[str] = None,
+ pipeline_guid: Optional[str] = None,
+ hash_values_of_files: dict[str, str] = {},
+ include_metadata: bool = False,
+ ) -> ExecutionResponse:
+ if execution_action is not None:
+ # Step execution
+ execution_response = WorkflowHelper.step_execution(
+ workflow,
+ execution_action,
+ execution_id=execution_id,
+ hash_values_of_files=hash_values_of_files,
+ include_metadata=include_metadata,
+ )
+ elif pipeline_guid:
+ # pipeline execution
+ PipelineProcessor.update_pipeline(
+ pipeline_guid, Pipeline.PipelineStatus.INPROGRESS
+ )
+ execution_response = WorkflowHelper.complete_execution(
+ workflow=workflow,
+ execution_id=execution_id,
+ pipeline_id=pipeline_guid,
+ hash_values_of_files=hash_values_of_files,
+ )
+ else:
+ execution_response = WorkflowHelper.complete_execution(
+ workflow=workflow,
+ execution_id=execution_id,
+ hash_values_of_files=hash_values_of_files,
+ include_metadata=include_metadata,
+ )
+ return execution_response
+
+ def activate(self, request: Request, pk: str) -> Response:
+ workflow = WorkflowHelper.active_project_workflow(pk)
+ serializer = WorkflowSerializer(workflow)
+ return Response(serializer.data, status=status.HTTP_200_OK)
+
+ @action(detail=True, methods=["get"])
+ def clear_cache(self, request: Request, *args: Any, **kwargs: Any) -> Response:
+ workflow = self.get_object()
+ response: dict[str, Any] = WorkflowHelper.clear_cache(workflow_id=workflow.id)
+ return Response(response.get("message"), status=response.get("status"))
+
+ @action(detail=True, methods=["get"])
+ def can_update(self, request: Request, pk: str) -> Response:
+ response: dict[str, Any] = WorkflowHelper.can_update_workflow(pk)
+ return Response(response, status=status.HTTP_200_OK)
+
+ @action(detail=True, methods=["get"])
+ def clear_file_marker(
+ self, request: Request, *args: Any, **kwargs: Any
+ ) -> Response:
+ workflow = self.get_object()
+ response: dict[str, Any] = WorkflowHelper.clear_file_marker(
+ workflow_id=workflow.id
+ )
+ return Response(response.get("message"), status=response.get("status"))
+
+ @action(detail=False, methods=["get"])
+ def get_schema(self, request: Request, *args: Any, **kwargs: Any) -> Response:
+ """Retrieves the JSON schema for source/destination type modules for
+ entities file/API/DB.
+
+ Takes query params `type` (defaults to "src") and
+ `entity` (defaults to "file").
+
+ Returns:
+ Response: JSON schema for the request made
+ """
+ schema_type = request.query_params.get("type", SchemaType.SRC.value)
+ schema_entity = request.query_params.get("entity", SchemaEntity.FILE.value)
+
+ WorkflowSchemaHelper.validate_request(
+ schema_type=SchemaType(schema_type),
+ schema_entity=SchemaEntity(schema_entity),
+ )
+ json_schema = WorkflowSchemaHelper.get_json_schema(
+ schema_type=schema_type, schema_entity=schema_entity
+ )
+ return Response(data=json_schema, status=status.HTTP_200_OK)
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
new file mode 100644
index 000000000..89b4babf0
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -0,0 +1,819 @@
+import json
+import logging
+import os
+import traceback
+import uuid
+from typing import Any, Optional
+
+from account_v2.constants import Common
+from api_v2.models import APIDeployment
+from celery import current_task
+from celery import exceptions as celery_exceptions
+from celery import shared_task
+from celery.result import AsyncResult
+from django.db import IntegrityError
+from pipeline_v2.models import Pipeline
+from pipeline_v2.pipeline_processor import PipelineProcessor
+from rest_framework import serializers
+from tool_instance_v2.constants import ToolInstanceKey
+from tool_instance_v2.models import ToolInstance
+from tool_instance_v2.tool_instance_helper import ToolInstanceHelper
+from unstract.workflow_execution.enums import LogComponent, LogLevel, LogState
+from unstract.workflow_execution.exceptions import StopExecution
+from utils.cache_service import CacheService
+from utils.constants import Account
+from utils.local_context import StateStore
+from utils.user_context import UserContext
+from workflow_manager.endpoint_v2.destination import DestinationConnector
+from workflow_manager.endpoint_v2.source import SourceConnector
+from workflow_manager.workflow_v2.constants import (
+ CeleryConfigurations,
+ WorkflowErrors,
+ WorkflowExecutionKey,
+ WorkflowMessages,
+)
+from workflow_manager.workflow_v2.dto import AsyncResultData, ExecutionResponse
+from workflow_manager.workflow_v2.enums import ExecutionStatus, SchemaEntity, SchemaType
+from workflow_manager.workflow_v2.exceptions import (
+ InvalidRequest,
+ TaskDoesNotExistError,
+ WorkflowDoesNotExistError,
+ WorkflowExecutionNotExist,
+)
+from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper
+from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
+from workflow_manager.workflow_v2.models.execution import WorkflowExecution
+from workflow_manager.workflow_v2.models.workflow import Workflow
+
+logger = logging.getLogger(__name__)
+
+
+class WorkflowHelper:
+ @staticmethod
+ def get_workflow_by_id(id: str) -> Workflow:
+ try:
+ workflow: Workflow = Workflow.objects.get(pk=id)
+ if not workflow or workflow is None:
+ raise WorkflowDoesNotExistError()
+ return workflow
+ except Workflow.DoesNotExist:
+ logger.error(f"Error getting workflow: {id}")
+ raise WorkflowDoesNotExistError()
+
+ @staticmethod
+ def get_active_workflow_by_project_id(project_id: str) -> Workflow:
+ try:
+ workflow: Workflow = Workflow.objects.filter(
+ project_id=project_id, is_active=True
+ ).first()
+ if not workflow or workflow is None:
+ raise WorkflowDoesNotExistError()
+ return workflow
+ except Workflow.DoesNotExist:
+ raise WorkflowDoesNotExistError()
+
+ @staticmethod
+ def active_project_workflow(workflow_id: str) -> Workflow:
+ workflow: Workflow = WorkflowHelper.get_workflow_by_id(workflow_id)
+ workflow.is_active = True
+ workflow.save()
+ return workflow
+
+ @staticmethod
+ def build_workflow_execution_service(
+ organization_id: Optional[str],
+ workflow: Workflow,
+ tool_instances: list[ToolInstance],
+ pipeline_id: Optional[str],
+ single_step: bool,
+ scheduled: bool,
+ execution_mode: tuple[str, str],
+ workflow_execution: Optional[WorkflowExecution],
+ include_metadata: bool = False,
+ ) -> WorkflowExecutionServiceHelper:
+ workflow_execution_service = WorkflowExecutionServiceHelper(
+ organization_id=organization_id,
+ workflow=workflow,
+ tool_instances=tool_instances,
+ pipeline_id=pipeline_id,
+ single_step=single_step,
+ scheduled=scheduled,
+ mode=execution_mode,
+ workflow_execution=workflow_execution,
+ include_metadata=include_metadata,
+ )
+ workflow_execution_service.build()
+ return workflow_execution_service
+
+ @staticmethod
+ def process_input_files(
+ workflow: Workflow,
+ source: SourceConnector,
+ destination: DestinationConnector,
+ execution_service: WorkflowExecutionServiceHelper,
+ single_step: bool,
+ hash_values_of_files: dict[str, str] = {},
+ ) -> WorkflowExecution:
+ input_files = source.list_files_from_source()
+ total_files = len(input_files)
+ processed_files = 0
+ error_raised = 0
+ execution_service.publish_initial_workflow_logs(total_files)
+ execution_service.update_execution(
+ ExecutionStatus.EXECUTING, increment_attempt=True
+ )
+ for index, input_file in enumerate(input_files):
+ file_number = index + 1
+ try:
+ is_executed, error = WorkflowHelper.process_file(
+ current_file_idx=file_number,
+ total_files=total_files,
+ input_file=input_file,
+ workflow=workflow,
+ source=source,
+ destination=destination,
+ execution_service=execution_service,
+ single_step=single_step,
+ hash_values_of_files=hash_values_of_files,
+ )
+ if is_executed:
+ processed_files += 1
+ if error:
+ error_raised += 1
+ except StopExecution as exception:
+ execution_service.update_execution(
+ ExecutionStatus.STOPPED, error=str(exception)
+ )
+ break
+ if error_raised and error_raised == total_files:
+ execution_service.update_execution(ExecutionStatus.ERROR)
+ else:
+ execution_service.update_execution(ExecutionStatus.COMPLETED)
+
+ execution_service.publish_final_workflow_logs(
+ total_files=total_files, processed_files=processed_files
+ )
+ return execution_service.get_execution_instance()
+
+ @staticmethod
+ def process_file(
+ current_file_idx: int,
+ total_files: int,
+ input_file: str,
+ workflow: Workflow,
+ source: SourceConnector,
+ destination: DestinationConnector,
+ execution_service: WorkflowExecutionServiceHelper,
+ single_step: bool,
+ hash_values_of_files: dict[str, str],
+ ) -> tuple[bool, Optional[str]]:
+ file_history = None
+ error = None
+ is_executed = False
+ file_name, file_hash = source.add_file_to_volume(
+ input_file_path=input_file,
+ hash_values_of_files=hash_values_of_files,
+ )
+ try:
+ execution_service.initiate_tool_execution(
+ current_file_idx, total_files, file_name, single_step
+ )
+ file_history = FileHistoryHelper.get_file_history(
+ workflow=workflow, cache_key=file_hash
+ )
+ is_executed = execution_service.execute_input_file(
+ file_name=file_name,
+ single_step=single_step,
+ file_history=file_history,
+ )
+ except StopExecution:
+ raise
+ except Exception as e:
+ execution_service.publish_log(
+ f"Error processing file {input_file}: {str(e)}",
+ level=LogLevel.ERROR,
+ )
+ error = str(e)
+ execution_service.publish_update_log(
+ LogState.RUNNING,
+ f"Processing output for {file_name}",
+ LogComponent.DESTINATION,
+ )
+ destination.handle_output(
+ file_name=file_name,
+ file_hash=file_hash,
+ workflow=workflow,
+ file_history=file_history,
+ error=error,
+ input_file_path=input_file,
+ )
+ execution_service.publish_update_log(
+ LogState.SUCCESS,
+ f"{file_name}'s output is processed successfully",
+ LogComponent.DESTINATION,
+ )
+ return is_executed, error
+
+ @staticmethod
+ def validate_tool_instances_meta(
+ tool_instances: list[ToolInstance],
+ ) -> None:
+ for tool in tool_instances:
+ ToolInstanceHelper.validate_tool_settings(
+ user=tool.workflow.created_by,
+ tool_uid=tool.tool_id,
+ tool_meta=tool.metadata,
+ )
+
+ @staticmethod
+ def run_workflow(
+ workflow: Workflow,
+ hash_values_of_files: dict[str, str] = {},
+ organization_id: Optional[str] = None,
+ pipeline_id: Optional[str] = None,
+ scheduled: bool = False,
+ single_step: bool = False,
+ workflow_execution: Optional[WorkflowExecution] = None,
+ execution_mode: Optional[tuple[str, str]] = None,
+ include_metadata: bool = False,
+ ) -> ExecutionResponse:
+ tool_instances: list[ToolInstance] = (
+ ToolInstanceHelper.get_tool_instances_by_workflow(
+ workflow.id, ToolInstanceKey.STEP
+ )
+ )
+
+ WorkflowHelper.validate_tool_instances_meta(tool_instances=tool_instances)
+ execution_mode = execution_mode or WorkflowExecution.Mode.INSTANT
+ execution_service = WorkflowHelper.build_workflow_execution_service(
+ organization_id=organization_id,
+ workflow=workflow,
+ tool_instances=tool_instances,
+ pipeline_id=pipeline_id,
+ single_step=single_step,
+ scheduled=scheduled,
+ execution_mode=execution_mode,
+ workflow_execution=workflow_execution,
+ include_metadata=include_metadata,
+ )
+ execution_id = execution_service.execution_id
+ source = SourceConnector(
+ organization_id=organization_id,
+ workflow=workflow,
+ execution_id=execution_id,
+ execution_service=execution_service,
+ )
+ destination = DestinationConnector(workflow=workflow, execution_id=execution_id)
+ # Validating endpoints
+ source.validate()
+ destination.validate()
+ # Execution Process
+ try:
+ workflow_execution = WorkflowHelper.process_input_files(
+ workflow,
+ source,
+ destination,
+ execution_service,
+ single_step=single_step,
+ hash_values_of_files=hash_values_of_files,
+ )
+ # TODO: Update through signals
+ WorkflowHelper._update_pipeline_status(
+ pipeline_id=pipeline_id, workflow_execution=workflow_execution
+ )
+ return ExecutionResponse(
+ str(workflow.id),
+ str(workflow_execution.id),
+ workflow_execution.status,
+ log_id=str(execution_service.execution_log_id),
+ error=workflow_execution.error_message,
+ mode=workflow_execution.execution_mode,
+ result=destination.api_results,
+ )
+ finally:
+ destination.delete_execution_directory()
+
+ @staticmethod
+ def _update_pipeline_status(
+ pipeline_id: Optional[str], workflow_execution: WorkflowExecution
+ ) -> None:
+ try:
+ if pipeline_id:
+ # Update pipeline status
+ if workflow_execution.status != ExecutionStatus.ERROR.value:
+ PipelineProcessor.update_pipeline(
+ pipeline_id, Pipeline.PipelineStatus.SUCCESS
+ )
+ else:
+ PipelineProcessor.update_pipeline(
+ pipeline_id, Pipeline.PipelineStatus.FAILURE
+ )
+ # Expected exception since API deployments are not tracked in Pipeline
+ except Pipeline.DoesNotExist:
+ pass
+ except Exception as e:
+ logger.warning(
+ f"Error updating pipeline {pipeline_id} status: {e}, "
+ f"with workflow execution: {workflow_execution}"
+ )
+
+ @staticmethod
+ def get_status_of_async_task(
+ execution_id: str,
+ ) -> ExecutionResponse:
+ """Get celery task status.
+
+ Args:
+ execution_id (str): workflow execution id
+
+ Raises:
+ TaskDoesNotExistError: Not found exception
+
+ Returns:
+ ExecutionResponse: _description_
+ """
+ execution = WorkflowExecution.objects.get(id=execution_id)
+
+ if not execution.task_id:
+ raise TaskDoesNotExistError()
+
+ result = AsyncResult(str(execution.task_id))
+
+ task = AsyncResultData(async_result=result)
+ return ExecutionResponse(
+ execution.workflow_id,
+ execution_id,
+ task.status,
+ result=task.result,
+ )
+
+ @staticmethod
+ def execute_workflow_async(
+ workflow_id: str,
+ execution_id: str,
+ hash_values_of_files: dict[str, str],
+ timeout: int = -1,
+ pipeline_id: Optional[str] = None,
+ include_metadata: bool = False,
+ ) -> ExecutionResponse:
+ """Adding a workflow to the queue for execution.
+
+ Args:
+ workflow_id (str): workflowId
+ execution_id (str): Execution ID
+ timeout (int): Celery timeout (timeout -1 : async execution)
+ pipeline_id (Optional[str], optional): Optional pipeline. Defaults to None.
+ include_metadata (bool): Whether to include metadata in the prompt output
+
+ Returns:
+ ExecutionResponse: Existing status of execution
+ """
+ try:
+ org_schema = UserContext.get_organization_identifier()
+ log_events_id = StateStore.get(Common.LOG_EVENTS_ID)
+ async_execution = WorkflowHelper.execute_bin.delay(
+ org_schema,
+ workflow_id,
+ hash_values_of_files=hash_values_of_files,
+ execution_id=execution_id,
+ pipeline_id=pipeline_id,
+ log_events_id=log_events_id,
+ include_metadata=include_metadata,
+ )
+ if timeout > -1:
+ async_execution.wait(
+ timeout=timeout,
+ interval=CeleryConfigurations.INTERVAL,
+ )
+ task = AsyncResultData(async_result=async_execution)
+ logger.info(f"Job {async_execution} enqueued.")
+ celery_result = task.to_dict()
+ task_result = celery_result.get("result")
+ return ExecutionResponse(
+ workflow_id,
+ execution_id,
+ task.status,
+ result=task_result,
+ )
+ except celery_exceptions.TimeoutError:
+ return ExecutionResponse(
+ workflow_id,
+ execution_id,
+ async_execution.status,
+ message=WorkflowMessages.CELERY_TIMEOUT_MESSAGE,
+ )
+ except Exception as error:
+ WorkflowExecutionServiceHelper.update_execution_status(
+ execution_id, ExecutionStatus.ERROR
+ )
+ logger.error(f"Errors while job enqueueing {str(error)}")
+ logger.error(f"Error {traceback.format_exc()}")
+ return ExecutionResponse(
+ workflow_id,
+ execution_id,
+ ExecutionStatus.ERROR.value,
+ error=str(error),
+ )
+
+ @staticmethod
+ @shared_task(
+ name="async_execute_bin",
+ acks_late=True,
+ autoretry_for=(Exception,),
+ max_retries=1,
+ retry_backoff=True,
+ retry_backoff_max=500,
+ retry_jitter=True,
+ )
+ def execute_bin(
+ schema_name: str,
+ workflow_id: str,
+ execution_id: str,
+ hash_values_of_files: dict[str, str],
+ scheduled: bool = False,
+ execution_mode: Optional[tuple[str, str]] = None,
+ pipeline_id: Optional[str] = None,
+ include_metadata: bool = False,
+ **kwargs: dict[str, Any],
+ ) -> Optional[list[Any]]:
+ """Asynchronous Execution By celery.
+
+ Args:
+ schema_name (str): schema name to get Data
+ workflow_id (str): Workflow Id
+ execution_id (str): Id of the execution
+ scheduled (bool, optional): Represents if it is a scheduled execution
+ Defaults to False
+ execution_mode (Optional[WorkflowExecution.Mode]): WorkflowExecution Mode
+ Defaults to None
+ pipeline_id (Optional[str], optional): Id of pipeline. Defaults to None
+ include_metadata (bool): Whether to include metadata in the prompt output
+
+ Kwargs:
+ log_events_id (str): Session ID of the user,
+ helps establish WS connection for streaming logs to the FE
+
+ Returns:
+ dict[str, list[Any]]: Returns a dict with result from workflow execution
+ """
+ task_id = current_task.request.id
+ # Set organization in state store for execution
+ StateStore.set(Account.ORGANIZATION_ID, schema_name)
+ return WorkflowHelper.execute_workflow(
+ organization_id=schema_name,
+ task_id=task_id,
+ workflow_id=workflow_id,
+ execution_id=execution_id,
+ hash_values_of_files=hash_values_of_files,
+ scheduled=scheduled,
+ execution_mode=execution_mode,
+ pipeline_id=pipeline_id,
+ include_metadata=include_metadata,
+ **kwargs,
+ )
+
+ @staticmethod
+ def execute_workflow(
+ organization_id: str,
+ task_id: str,
+ workflow_id: str,
+ execution_id: str,
+ hash_values_of_files: dict[str, str],
+ scheduled: bool = False,
+ execution_mode: Optional[tuple[str, str]] = None,
+ pipeline_id: Optional[str] = None,
+ include_metadata: bool = False,
+ **kwargs: dict[str, Any],
+ ) -> Optional[list[Any]]:
+ """Asynchronous Execution By celery.
+
+ Args:
+ schema_name (str): schema name to get Data
+ workflow_id (str): Workflow Id
+ execution_id (Optional[str], optional): Id of the execution.
+ Defaults to None.
+ scheduled (bool, optional): Represents if it is a scheduled
+ execution. Defaults to False.
+ execution_mode (Optional[WorkflowExecution.Mode]):
+ WorkflowExecution Mode. Defaults to None.
+ pipeline_id (Optional[str], optional): Id of pipeline.
+ Defaults to None.
+ include_metadata (bool): Whether to include metadata in the prompt output
+
+ Kwargs:
+ log_events_id (str): Session ID of the user, helps establish
+ WS connection for streaming logs to the FE
+
+ Returns:
+ dict[str, list[Any]]: Returns a dict with result from
+ workflow execution
+ """
+ workflow = Workflow.objects.get(id=workflow_id)
+ try:
+ workflow_execution = (
+ WorkflowExecutionServiceHelper.create_workflow_execution(
+ workflow_id=workflow_id,
+ single_step=False,
+ pipeline_id=pipeline_id,
+ mode=WorkflowExecution.Mode.QUEUE,
+ execution_id=execution_id,
+ **kwargs, # type: ignore
+ )
+ )
+ except IntegrityError:
+ # Use existing instance on retry attempt
+ workflow_execution = WorkflowExecution.objects.get(pk=execution_id)
+ WorkflowExecutionServiceHelper.update_execution_task(
+ execution_id=execution_id, task_id=task_id
+ )
+ result = WorkflowHelper.run_workflow(
+ workflow=workflow,
+ organization_id=organization_id,
+ pipeline_id=pipeline_id,
+ scheduled=scheduled,
+ workflow_execution=workflow_execution,
+ execution_mode=execution_mode,
+ hash_values_of_files=hash_values_of_files,
+ include_metadata=include_metadata,
+ ).result
+ return result
+
+ @staticmethod
+ def complete_execution(
+ workflow: Workflow,
+ execution_id: Optional[str] = None,
+ pipeline_id: Optional[str] = None,
+ hash_values_of_files: dict[str, str] = {},
+ include_metadata: bool = False,
+ ) -> ExecutionResponse:
+ if pipeline_id:
+ logger.info(f"Executing pipeline: {pipeline_id}")
+ response: ExecutionResponse = WorkflowHelper.execute_workflow_async(
+ workflow_id=workflow.id,
+ pipeline_id=pipeline_id,
+ execution_id=str(uuid.uuid4()),
+ hash_values_of_files=hash_values_of_files,
+ )
+ return response
+ if execution_id is None:
+ # Creating execution entity and return
+ return WorkflowHelper.create_and_make_execution_response(
+ workflow_id=workflow.id, pipeline_id=pipeline_id
+ )
+ try:
+ # Normal execution
+ workflow_execution = WorkflowExecution.objects.get(pk=execution_id)
+ if (
+ workflow_execution.status != ExecutionStatus.PENDING.value
+ or workflow_execution.execution_type != WorkflowExecution.Type.COMPLETE
+ ):
+ raise InvalidRequest(WorkflowErrors.INVALID_EXECUTION_ID)
+ return WorkflowHelper.run_workflow(
+ workflow=workflow,
+ workflow_execution=workflow_execution,
+ hash_values_of_files=hash_values_of_files,
+ include_metadata=include_metadata,
+ )
+ except WorkflowExecution.DoesNotExist:
+ return WorkflowHelper.create_and_make_execution_response(
+ workflow_id=workflow.id, pipeline_id=pipeline_id
+ )
+
+ @staticmethod
+ def get_current_execution(execution_id: str) -> ExecutionResponse:
+ try:
+ workflow_execution = WorkflowExecution.objects.get(pk=execution_id)
+ return ExecutionResponse(
+ workflow_execution.workflow_id,
+ workflow_execution.id,
+ workflow_execution.status,
+ log_id=workflow_execution.execution_log_id,
+ error=workflow_execution.error_message,
+ mode=workflow_execution.execution_mode,
+ )
+ except WorkflowExecution.DoesNotExist:
+ raise WorkflowExecutionNotExist()
+
+ @staticmethod
+ def step_execution(
+ workflow: Workflow,
+ execution_action: str,
+ execution_id: Optional[str] = None,
+ hash_values_of_files: dict[str, str] = {},
+ include_metadata: bool = False,
+ ) -> ExecutionResponse:
+ if execution_action is Workflow.ExecutionAction.START.value: # type: ignore
+ if execution_id is None:
+ return WorkflowHelper.create_and_make_execution_response(
+ workflow_id=workflow.id, single_step=True
+ )
+ try:
+ workflow_execution = WorkflowExecution.objects.get(pk=execution_id)
+ return WorkflowHelper.run_workflow(
+ workflow=workflow,
+ single_step=True,
+ workflow_execution=workflow_execution,
+ hash_values_of_files=hash_values_of_files,
+ include_metadata=include_metadata,
+ )
+ except WorkflowExecution.DoesNotExist:
+ return WorkflowHelper.create_and_make_execution_response(
+ workflow_id=workflow.id, single_step=True
+ )
+
+ else:
+ if execution_id is None:
+ raise InvalidRequest("execution_id is missed")
+ try:
+ workflow_execution = WorkflowExecution.objects.get(pk=execution_id)
+ except WorkflowExecution.DoesNotExist:
+ raise WorkflowExecutionNotExist(WorkflowErrors.INVALID_EXECUTION_ID)
+ if (
+ workflow_execution.status != ExecutionStatus.PENDING.value
+ or workflow_execution.execution_type != WorkflowExecution.Type.STEP
+ ):
+ raise InvalidRequest(WorkflowErrors.INVALID_EXECUTION_ID)
+ current_action: Optional[str] = CacheService.get_key(execution_id)
+ logger.info(f"workflow_execution.current_action {current_action}")
+ if current_action is None:
+ raise InvalidRequest(WorkflowErrors.INVALID_EXECUTION_ID)
+ CacheService.set_key(execution_id, execution_action)
+ workflow_execution = WorkflowExecution.objects.get(pk=execution_id)
+
+ return ExecutionResponse(
+ workflow.id,
+ execution_id,
+ workflow_execution.status,
+ log_id=workflow_execution.execution_log_id,
+ error=workflow_execution.error_message,
+ mode=workflow_execution.execution_mode,
+ )
+
+ @staticmethod
+ def create_and_make_execution_response(
+ workflow_id: str,
+ pipeline_id: Optional[str] = None,
+ single_step: bool = False,
+ mode: tuple[str, str] = WorkflowExecution.Mode.INSTANT,
+ ) -> ExecutionResponse:
+ log_events_id = StateStore.get(Common.LOG_EVENTS_ID)
+ workflow_execution = WorkflowExecutionServiceHelper.create_workflow_execution(
+ workflow_id=workflow_id,
+ single_step=single_step,
+ pipeline_id=pipeline_id,
+ mode=mode,
+ log_events_id=log_events_id,
+ )
+ return ExecutionResponse(
+ workflow_execution.workflow_id,
+ workflow_execution.id,
+ workflow_execution.status,
+ log_id=workflow_execution.execution_log_id,
+ error=workflow_execution.error_message,
+ mode=workflow_execution.execution_mode,
+ )
+
+ # TODO: Access cache through a manager
+ @staticmethod
+ def clear_cache(workflow_id: str) -> dict[str, Any]:
+ """Function to clear cache with a specific pattern."""
+ response: dict[str, Any] = {}
+ try:
+ key_pattern = f"*:cache:{workflow_id}:*"
+ CacheService.clear_cache(key_pattern)
+ response["message"] = WorkflowMessages.CACHE_CLEAR_SUCCESS
+ response["status"] = 200
+ return response
+ except Exception as exc:
+ logger.error(f"Error occurred while clearing cache : {exc}")
+ response["message"] = WorkflowMessages.CACHE_CLEAR_FAILED
+ response["status"] = 400
+ return response
+
+ @staticmethod
+ def clear_file_marker(workflow_id: str) -> dict[str, Any]:
+ """Function to clear file marker from the cache."""
+ # Clear file history from the table
+ response: dict[str, Any] = {}
+ workflow = Workflow.objects.get(id=workflow_id)
+ try:
+ FileHistoryHelper.clear_history_for_workflow(workflow=workflow)
+ response["message"] = WorkflowMessages.FILE_MARKER_CLEAR_SUCCESS
+ response["status"] = 200
+ return response
+ except Exception as exc:
+ logger.error(f"Error occurred while clearing file marker : {exc}")
+ response["message"] = WorkflowMessages.FILE_MARKER_CLEAR_FAILED
+ response["status"] = 400
+ return response
+
+ @staticmethod
+ def get_workflow_execution_id(execution_id: str) -> str:
+ wf_exec_prefix = WorkflowExecutionKey.WORKFLOW_EXECUTION_ID_PREFIX
+ workflow_execution_id = f"{wf_exec_prefix}-{execution_id}"
+ return workflow_execution_id
+
+ @staticmethod
+ def get_execution_by_id(execution_id: str) -> WorkflowExecution:
+ try:
+ execution: WorkflowExecution = WorkflowExecution.objects.get(
+ id=execution_id
+ )
+ return execution
+ except WorkflowExecution.DoesNotExist:
+ raise WorkflowDoesNotExistError()
+
+ @staticmethod
+ def make_async_result(obj: AsyncResult) -> dict[str, Any]:
+ return {
+ "id": obj.id,
+ "status": obj.status,
+ "result": obj.result,
+ "is_ready": obj.ready(),
+ "is_failed": obj.failed(),
+ "info": obj.info,
+ }
+
+ @staticmethod
+ def can_update_workflow(workflow_id: str) -> dict[str, Any]:
+ try:
+ workflow: Workflow = Workflow.objects.get(pk=workflow_id)
+ if not workflow or workflow is None:
+ raise WorkflowDoesNotExistError()
+ used_count = Pipeline.objects.filter(workflow=workflow).count()
+ if used_count == 0:
+ used_count = APIDeployment.objects.filter(workflow=workflow).count()
+ return {"can_update": used_count == 0}
+ except Workflow.DoesNotExist:
+ logger.error(f"Error getting workflow: {id}")
+ raise WorkflowDoesNotExistError()
+
+
+class WorkflowSchemaHelper:
+ """Helper class for workflow schema related methods."""
+
+ @staticmethod
+ def validate_request(schema_type: SchemaType, schema_entity: SchemaEntity) -> bool:
+ """Validates the given args for reading the JSON schema.
+
+ Schema type of `src`, allows entities `file` and `api`
+ Schema type of `dest`, allows entities `db`
+
+ Args:
+ schema_type (SchemaType): Enum with values `src`, `dest`
+ schema_entity (SchemaEntity): Enum with values `file`, `api`, `db`
+
+ Raises:
+ serializers.ValidationError: If invalid values/
+ combination is requested
+
+ Returns:
+ bool: _description_
+ """
+ possible_types = [e.value for e in SchemaType]
+ possible_entities = [e.value for e in SchemaEntity]
+
+ if schema_type.value not in possible_types:
+ raise serializers.ValidationError(
+ f"Invalid value for 'type': {schema_type.value}, "
+ f"should be one of {possible_types}"
+ )
+
+ if schema_entity.value not in possible_entities:
+ raise serializers.ValidationError(
+ f"Invalid value for 'entity': {schema_entity.value}, "
+ f"should be one of {possible_entities}"
+ )
+
+ if (schema_type == SchemaType.SRC and schema_entity == SchemaEntity.DB) or (
+ schema_type == SchemaType.DEST and schema_entity != SchemaEntity.DB
+ ):
+ raise serializers.ValidationError(
+ f"Invalid values for 'type': {schema_type.value}, "
+ f"'entity': {schema_entity.value}."
+ f"Param 'type': {SchemaType.SRC.value} allows "
+ f"{SchemaEntity.FILE.value} and {SchemaEntity.API.value}"
+ f"'type': {SchemaType.DEST.value} allows "
+ f"{SchemaEntity.DB.value}."
+ )
+ return True
+
+ @staticmethod
+ def get_json_schema(
+ schema_type: SchemaType, schema_entity: SchemaEntity
+ ) -> dict[str, Any]:
+ """Reads and returns the JSON schema for the given args.
+
+ Args:
+ schema_type (SchemaType): Enum with values `src`, `dest`
+ schema_entity (SchemaEntity): Enum with values `file`, `api`, `db`
+
+ Returns:
+ dict[str, Any]: JSON schema for the requested entity
+ """
+ schema_path = (
+ f"{os.path.dirname(__file__)}/static/" f"{schema_type}/{schema_entity}.json"
+ )
+ with open(schema_path, encoding="utf-8") as file:
+ schema = json.load(file)
+ return schema # type: ignore