Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Feat/remote storage workflow (#837)

alif2a55f2c0401
diff --git a/backend/backend/constants.py b/backend/backend/constants.py
index 26d944d9a..53126cebe 100644
--- a/backend/backend/constants.py
+++ b/backend/backend/constants.py
@@ -34,3 +34,4 @@ class FeatureFlag:
"""Temporary feature flags."""
APP_DEPLOYMENT = "app_deployment"
+ REMOTE_FILE_STORAGE = "remote_file_storage"
diff --git a/backend/sample.env b/backend/sample.env
index f91496da3..cd3419cf1 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -144,3 +144,22 @@ TOOL_REGISTRY_CONFIG_PATH="/data/tool_registry_config"
# Flipt Service
FLIPT_SERVICE_AVAILABLE=False
+
+
+# File System Configuration for Workflow and API Execution
+
+# Directory Prefixes for storing execution files
+WORKFLOW_EXECUTION_DIR_PREFIX="unstract/execution"
+API_EXECUTION_DIR_PREFIX="unstract/api"
+
+# Storage Provider for Workflow Execution
+# Valid options: MINIO, S3, etc..
+WORKFLOW_EXECUTION_FS_PROVIDER="MINIO"
+WORKFLOW_EXECUTION_FS_CREDENTIAL='{"endpoint_url": "", "key": "", "secret": ""}'
+
+# Storage Provider for API Execution
+API_STORAGE_FS_PROVIDER="MINIO"
+API_STORAGE_FS_CREDENTIAL='{"endpoint_url": "", "key": "", "secret": ""}'
+
+# Optional: Legacy storage path (if applicable)
+LEGACY_STORAGE_PATH="/path/to/legacy/storage"
diff --git a/backend/utils/constants.py b/backend/utils/constants.py
index a92f397a4..377c21da6 100644
--- a/backend/utils/constants.py
+++ b/backend/utils/constants.py
@@ -15,12 +15,6 @@ class Account:
ORGANIZATION_ID = "organization_id"
-class FeatureFlag:
- """Temporary feature flags."""
-
- pass
-
-
class Common:
METADATA = "metadata"
MODULE = "module"
diff --git a/backend/workflow_manager/endpoint_v2/base_connector.py b/backend/workflow_manager/endpoint_v2/base_connector.py
index 2523523e3..8b34910d7 100644
--- a/backend/workflow_manager/endpoint_v2/base_connector.py
+++ b/backend/workflow_manager/endpoint_v2/base_connector.py
@@ -7,8 +7,13 @@ from unstract.workflow_execution.execution_file_handler import ExecutionFileHand
from utils.constants import Common
from utils.user_context import UserContext
+from backend.constants import FeatureFlag
from unstract.connectors.filesystems import connectors
from unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem
+from unstract.flags.feature_flag import check_feature_flag_status
+
+if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ from unstract.filesystem import FileStorageType, FileSystem
class BaseConnector(ExecutionFileHandler):
@@ -22,13 +27,14 @@ class BaseConnector(ExecutionFileHandler):
This class serves as a base for connectors and provides common
utilities.
"""
- if not (settings.API_STORAGE_DIR and settings.WORKFLOW_DATA_DIR):
- raise ValueError("Missed env API_STORAGE_DIR or WORKFLOW_DATA_DIR")
super().__init__(workflow_id, execution_id, organization_id)
- # Directory path for storing execution-related files for API
- self.api_storage_dir: str = self.create_execution_dir_path(
- workflow_id, execution_id, organization_id, settings.API_STORAGE_DIR
- )
+ if not check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ if not (settings.API_STORAGE_DIR and settings.WORKFLOW_DATA_DIR):
+ raise ValueError("Missed env API_STORAGE_DIR or WORKFLOW_DATA_DIR")
+ # Directory path for storing execution-related files for API
+ self.api_storage_dir: str = self.create_execution_dir_path(
+ workflow_id, execution_id, organization_id, settings.API_STORAGE_DIR
+ )
def get_fsspec(
self, settings: dict[str, Any], connector_id: str
@@ -80,8 +86,14 @@ class BaseConnector(ExecutionFileHandler):
json.JSONDecodeError: If there is an issue decoding the JSON file.
"""
try:
- with open(file_path, encoding="utf-8") as file:
- schema: dict[str, Any] = json.load(file)
+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
+ file_storage = file_system.get_file_storage()
+ file_contents = file_storage.read(path=file_path, encoding="utf-8")
+ schema: dict[str, Any] = json.load(file_contents)
+ else:
+ with open(file_path, encoding="utf-8") as file:
+ schema: dict[str, Any] = json.load(file)
except OSError:
schema = {}
return schema
@@ -100,7 +112,12 @@ class BaseConnector(ExecutionFileHandler):
str: The directory path for the execution.
"""
organization_id = UserContext.get_organization_identifier()
- api_storage_dir: str = cls.create_execution_dir_path(
- workflow_id, execution_id, organization_id, settings.API_STORAGE_DIR
- )
+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ api_storage_dir: str = cls.get_api_execution_dir(
+ workflow_id, execution_id, organization_id
+ )
+ else:
+ api_storage_dir: str = cls.create_execution_dir_path(
+ workflow_id, execution_id, organization_id, settings.API_STORAGE_DIR
+ )
return api_storage_dir
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index 88f45b312..036cece50 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -10,6 +10,7 @@ import magic
from connector_v2.models import ConnectorInstance
from fsspec.implementations.local import LocalFileSystem
from unstract.sdk.constants import ToolExecKey
+from unstract.sdk.tool.mime_types import EXT_MIME_MAP
from unstract.workflow_execution.constants import ToolOutputType
from utils.user_context import UserContext
from workflow_manager.endpoint_v2.base_connector import BaseConnector
@@ -36,8 +37,13 @@ from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
from workflow_manager.workflow_v2.models.file_history import FileHistory
from workflow_manager.workflow_v2.models.workflow import Workflow
+from backend.constants import FeatureFlag
from backend.exceptions import UnstractFSException
from unstract.connectors.exceptions import ConnectorError
+from unstract.flags.feature_flag import check_feature_flag_status
+
+if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ from unstract.filesystem import FileStorageType, FileSystem
logger = logging.getLogger(__name__)
@@ -405,6 +411,8 @@ class DestinationConnector(BaseConnector):
Returns:
Union[dict[str, Any], str]: Result data.
"""
+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ return self.get_result_with_file_storage(file_history=file_history)
if file_history and file_history.result:
return self.parse_string(file_history.result)
output_file = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
@@ -434,6 +442,43 @@ class DestinationConnector(BaseConnector):
logger.error(f"Error while getting result {err}")
return result
+ def get_result_with_file_storage(
+ self, file_history: Optional[FileHistory] = None
+ ) -> Optional[Any]:
+ """Get result data from the output file.
+
+ Returns:
+ Union[dict[str, Any], str]: Result data.
+ """
+ if file_history and file_history.result:
+ return self.parse_string(file_history.result)
+ output_file = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
+ metadata: dict[str, Any] = self.get_workflow_metadata()
+ output_type = self.get_output_type(metadata)
+ result: Union[dict[str, Any], str] = ""
+ file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
+ file_storage = file_system.get_file_storage()
+ try:
+ # TODO: SDK handles validation; consider removing here.
+ file_type = file_storage.mime_type(output_file)
+ if output_type == ToolOutputType.JSON:
+ if file_type != EXT_MIME_MAP[ToolOutputType.JSON.lower()]:
+ logger.error(f"Output type json mismatched {file_type}")
+ raise ToolOutputTypeMismatch()
+ file_content = file_storage.read(output_file, mode="r")
+ result = json.loads(file_content)
+ elif output_type == ToolOutputType.TXT:
+ if file_type == EXT_MIME_MAP[ToolOutputType.JSON.lower()]:
+ logger.error(f"Output type txt mismatched {file_type}")
+ raise ToolOutputTypeMismatch()
+ file_content = file_storage.read(output_file, mode="r")
+ result = file_content.encode("utf-8").decode("unicode-escape")
+ else:
+ raise InvalidToolOutputType()
+ except (FileNotFoundError, json.JSONDecodeError) as err:
+ logger.error(f"Error while getting result {err}")
+ return result
+
def get_metadata(
self, file_history: Optional[FileHistory] = None
) -> Optional[dict[str, Any]]:
@@ -454,8 +499,13 @@ class DestinationConnector(BaseConnector):
Returns:
None
"""
- fs: LocalFileSystem = fsspec.filesystem("file")
- fs.rm(self.execution_dir, recursive=True)
+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
+ file_storage = file_system.get_file_storage()
+ file_storage.rm(self.execution_dir, recursive=True)
+ else:
+ fs: LocalFileSystem = fsspec.filesystem("file")
+ fs.rm(self.execution_dir, recursive=True)
self.delete_api_storage_dir(self.workflow_id, self.execution_id)
@classmethod
@@ -468,8 +518,13 @@ class DestinationConnector(BaseConnector):
api_storage_dir = cls.get_api_storage_dir_path(
workflow_id=workflow_id, execution_id=execution_id
)
- fs: LocalFileSystem = fsspec.filesystem("file")
- fs.rm(api_storage_dir, recursive=True)
+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ file_system = FileSystem(FileStorageType.API_EXECUTION)
+ file_storage = file_system.get_file_storage()
+ file_storage.rm(api_storage_dir, recursive=True)
+ else:
+ fs: LocalFileSystem = fsspec.filesystem("file")
+ fs.rm(api_storage_dir, recursive=True)
@classmethod
def create_endpoint_for_workflow(
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index 59b51306d..c004f3397 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -37,6 +37,12 @@ from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelpe
from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelper
from workflow_manager.workflow_v2.models.workflow import Workflow
+from backend.constants import FeatureFlag
+from unstract.flags.feature_flag import check_feature_flag_status
+
+if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ from unstract.filesystem import FileStorageType, FileSystem
+
logger = logging.getLogger(__name__)
@@ -475,11 +481,17 @@ class SourceConnector(BaseConnector):
)
self.publish_input_file_content(input_file_path, input_log)
- with fsspec.open(source_file, "wb") as local_file:
- local_file.write(file_content)
+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
+ file_storage = file_system.get_file_storage()
+ file_storage.write(path=source_file_path, mode="wb", data=file_content)
+ file_storage.write(path=infile_path, mode="wb", data=file_content)
+ else:
+ with fsspec.open(source_file, "wb") as local_file:
+ local_file.write(file_content)
- # Copy file to infile directory
- self.copy_file_to_infile_dir(source_file_path, infile_path)
+ # Copy file to infile directory
+ self.copy_file_to_infile_dir(source_file_path, infile_path)
logger.info(f"{input_file_path} is added to execution directory")
return hash_value_of_file_content
@@ -488,8 +500,76 @@ class SourceConnector(BaseConnector):
"""Add input file to execution directory from api storage."""
infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
source_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)
- shutil.copyfile(input_file_path, infile_path)
- shutil.copyfile(input_file_path, source_path)
+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ api_file_system = FileSystem(FileStorageType.API_EXECUTION)
+ api_file_storage = api_file_system.get_file_storage()
+ workflow_file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
+ workflow_file_storage = workflow_file_system.get_file_storage()
+ self._copy_file_to_destination(
+ source_storage=api_file_storage,
+ destination_storage=workflow_file_storage,
+ source_path=input_file_path,
+ destination_paths=[infile_path, source_path],
+ )
+ else:
+ shutil.copyfile(input_file_path, infile_path)
+ shutil.copyfile(input_file_path, source_path)
+
+ # TODO: replace it with method from SDK Utils
+ def _copy_file_to_destination(
+ self,
+ source_storage: Any,
+ destination_storage: Any,
+ source_path: str,
+ destination_paths: list[str],
+ chunk_size: int = 4096,
+ ) -> None:
+ """
+ Copy a file from a source storage to one or more paths in a
+ destination storage.
+
+ This function reads the source file in chunks and writes each chunk to
+ the specified destination paths. The function will continue until the
+ entire source file is copied.
+
+ Args:
+ source_storage (FileStorage): The storage object from which
+ the file is read.
+ destination_storage (FileStorage): The storage object to which
+ the file is written.
+ source_path (str): The path of the file in the source storage.
+ destination_paths (list[str]): A list of paths where the file will be
+ copied in the destination storage.
+ chunk_size (int, optional): The number of bytes to read per chunk.
+ Default is 4096 bytes.
+ """
+ seek_position = 0 # Start from the beginning
+ end_of_file = False
+
+ # Loop to read and write in chunks until the end of the file
+ while not end_of_file:
+ # Read a chunk from the source file
+ chunk = source_storage.read(
+ path=source_path,
+ mode="rb",
+ seek_position=seek_position,
+ length=chunk_size,
+ )
+ # Check if the end of the file has been reached
+ if not chunk:
+ end_of_file = True
+ else:
+ # Write the chunk to each destination path
+ for destination_file in destination_paths:
+ destination_storage.write(
+ path=destination_file,
+ mode="ab",
+ seek_position=seek_position,
+ data=chunk,
+ )
+
+ # Update the seek position
+ seek_position += len(chunk)
def add_file_to_volume(self, input_file_path: str, file_hash: FileHash) -> str:
"""Add input file to execution directory.
@@ -584,12 +664,20 @@ class SourceConnector(BaseConnector):
for file in file_objs:
file_name = file.name
destination_path = os.path.join(api_storage_dir, file_name)
- os.makedirs(os.path.dirname(destination_path), exist_ok=True)
- with open(destination_path, "wb") as f:
+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ file_system = FileSystem(FileStorageType.API_EXECUTION)
+ file_storage = file_system.get_file_storage()
buffer = bytearray()
for chunk in file.chunks():
buffer.extend(chunk)
- f.write(buffer)
+ file_storage.write(path=destination_path, mode="wb", data=buffer)
+ else:
+ os.makedirs(os.path.dirname(destination_path), exist_ok=True)
+ with open(destination_path, "wb") as f:
+ buffer = bytearray()
+ for chunk in file.chunks():
+ buffer.extend(chunk)
+ f.write(buffer)
file_hash = cls.hash_str(buffer)
connection_type = WorkflowEndpoint.ConnectionType.API
diff --git a/tox.ini b/tox.ini
index 7ca6d4ac8..3aac163bc 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,6 +8,7 @@ env_list = py{39,310,311}, worker
changedir = worker
setenv =
PDM_IGNORE_SAVED_PYTHON="1"
+ PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
deps = pdm
allowlist_externals=
sh
diff --git a/unstract/filesystem/README.md b/unstract/filesystem/README.md
new file mode 100644
index 000000000..b4b530ef7
--- /dev/null
+++ b/unstract/filesystem/README.md
@@ -0,0 +1,70 @@
+# Filesystem
+
+The **Filesystem** module manages file storage interactions within the project. It is built on top of the **Unstract SDK**, ensuring consistency and flexibility across different storage backends.
+
+
+## Key Components
+### FileSystem Class
+The `FileSystem` class provides methods to configure, authenticate, and initialize storage mechanisms based on predefined types. It uses mappings and environment variables to customize its behavior dynamically.
+
+
+### Storage Types
+- **Workflow Execution Storage**: Temporary storage for workflow-related files.
+- **API Execution Storage**: Temporary storage for API-related files.
+
+### Mappings
+- **Storage Mapping**: Defines the storage class for each `FileStorageType`.
+- **Credentials Mapping**: Associates credentials (from environment variables) with specific `FileStorageType`.
+- **Provider Mapping**: Maps environment-based storage providers to `FileStorageType`.
+
+### Environment Variables
+The module relies on environment variables for configuration. Below is a detailed list of required variables:
+
+#### Provider Configuration
+
+`WORKFLOW_EXECUTION_FS_PROVIDER`: The provider for workflow execution storage (default: minio).
+`API_STORAGE_FS_PROVIDER`: The provider for API execution storage (default: minio).
+
+#### Credential Configuration
+
+`WORKFLOW_EXECUTION_FS_CREDENTIAL`: JSON string containing credentials for workflow execution storage.
+`API_STORAGE_FS_CREDENTIAL`: JSON string containing credentials for API execution storage.
+
+#### Additional Configuration
+
+`LEGACY_STORAGE_PATH` (*optional*): Used specifically with **PermanentFileStorage** for legacy data handling.
+
+
+### Example Environment File (.env)
+```env
+# Workflow Execution File Storage
+WORKFLOW_EXECUTION_FS_PROVIDER=minio
+WORKFLOW_EXECUTION_FS_CREDENTIAL={"access_key": "your-access-key", "secret_key": "your-secret-key"}
+
+# API Execution File Storage
+API_STORAGE_FS_PROVIDER=minio
+API_STORAGE_FS_CREDENTIAL={"access_key": "api-access-key", "secret_key": "api-secret-key"}
+
+# Legacy Storage Path (Optional)
+LEGACY_STORAGE_PATH=/path/to/legacy/storage
+```
+
+### How It Works
+1. **Storage Class Initialization**: Based on `FileStorageType`, the appropriate storage class is chosen from `STORAGE_MAPPING`.
+2. **Credential Injection**: Credentials are fetched from environment variables and injected into the storage class.
+3. **Provider Selection**: Providers are dynamically resolved using the `get_provider` method and mapped to the relevant `FileStorageType`.
+4. **Storage Instance Creation**: A storage instance is created and returned, ready for use.
+
+
+## Extending the System
+To add a new `FileStorageType`:
+
+1. Update the `FileStorageType` enum with the new type.
+2. Add the new type to:
+ - `STORAGE_MAPPING`: Map the type to a storage class.
+ - `FILE_STORAGE_CREDENTIALS_MAPPING`: Define its credentials.
+ - `FILE_STORAGE_PROVIDER_MAPPING`: Specify its provider.
+
+# Error Handling
+- If a provider is not found in the `FileStorageProvider` enum, a `ProviderNotFound` exception is raised.
+- If no valid credentials or mappings are found, the system raises appropriate exceptions (`ValueError`).
diff --git a/unstract/filesystem/pdm.lock b/unstract/filesystem/pdm.lock
new file mode 100644
index 000000000..29b4418ca
--- /dev/null
+++ b/unstract/filesystem/pdm.lock
@@ -0,0 +1,8 @@
+# This file is @generated by PDM.
+# It is not intended for manual editing.
+
+[metadata]
+groups = ["default"]
+strategy = ["cross_platform", "inherit_metadata"]
+lock_version = "4.4.2"
+content_hash = "sha256:761e9d2e66433c05e581c1ee8664065959079372805516912f6a3d14a3b2cec7"
diff --git a/unstract/filesystem/pyproject.toml b/unstract/filesystem/pyproject.toml
new file mode 100644
index 000000000..a4b2082c9
--- /dev/null
+++ b/unstract/filesystem/pyproject.toml
@@ -0,0 +1,18 @@
+[build-system]
+requires = ["pdm-backend"]
+build-backend = "pdm.backend"
+
+[project]
+name = "unstract-filesystem"
+version = "0.0.1"
+description = "Unstract filesystem package"
+authors = [
+ {name = "Zipstack Inc.", email = "[email protected]"},
+]
+dependencies = []
+requires-python = ">=3.9,<3.11.1"
+readme = "README.md"
+
+[tool.pdm.build]
+includes = ["src"]
+package-dir = "src"
diff --git a/unstract/filesystem/src/unstract/filesystem/__init__.py b/unstract/filesystem/src/unstract/filesystem/__init__.py
new file mode 100644
index 000000000..3cfdc80eb
--- /dev/null
+++ b/unstract/filesystem/src/unstract/filesystem/__init__.py
@@ -0,0 +1,10 @@
+"""
+This module exposes the main classes for file storage handling.
+"""
+
+from unstract.sdk.file_storage.fs_shared_temporary import SharedTemporaryFileStorage
+
+from .file_storage_types import FileStorageType
+from .filesystem import FileSystem
+
+__all__ = ["FileSystem", "SharedTemporaryFileStorage", "FileStorageType"]
diff --git a/unstract/filesystem/src/unstract/filesystem/exceptions.py b/unstract/filesystem/src/unstract/filesystem/exceptions.py
new file mode 100644
index 000000000..d51241478
--- /dev/null
+++ b/unstract/filesystem/src/unstract/filesystem/exceptions.py
@@ -0,0 +1,17 @@
+class FilesystemError(Exception):
+ """
+ Base exception class for filesystem-related errors.
+ """
+
+ default_message = "An error occurred in the filesystem operation."
+
+ def __init__(self, message: str = None):
+ self.message = message or self.default_message
+ super().__init__(self.message)
+
+ def __str__(self) -> str:
+ return self.message
+
+
+class ProviderNotFound(FilesystemError):
+ default_message = "The requested provider was not found."
diff --git a/unstract/filesystem/src/unstract/filesystem/file_storage_config.py b/unstract/filesystem/src/unstract/filesystem/file_storage_config.py
new file mode 100644
index 000000000..b0b497b7b
--- /dev/null
+++ b/unstract/filesystem/src/unstract/filesystem/file_storage_config.py
@@ -0,0 +1,51 @@
+import json
+import logging
+import os
+
+from unstract.sdk.file_storage import FileStorageProvider
+from unstract.sdk.file_storage.fs_shared_temporary import SharedTemporaryFileStorage
+
+from .exceptions import ProviderNotFound
+from .file_storage_types import FileStorageType # Import the shared enum
+
+logger = logging.getLogger(__name__)
+
+
+def get_provider(var_name: str, default: str = "minio") -> FileStorageProvider:
+ """Retrieve the file storage provider based on an environment variable."""
+ provider_name = os.environ.get(var_name, default).capitalize()
+ try:
+ # Attempt to map the provider name to an enum value, case-insensitively
+ return FileStorageProvider[provider_name]
+ except KeyError:
+ allowed_providers = ", ".join(
+ [provider.name for provider in FileStorageProvider]
+ )
+ logger.error(
+ f"Invalid provider '{provider_name}'. Allowed providers: "
+ f"{allowed_providers}"
+ )
+ raise ProviderNotFound(f"Provider '{provider_name}' not found")
+
+
+# Mappings for storage types, credentials, and providers
+STORAGE_MAPPING = {
+ FileStorageType.WORKFLOW_EXECUTION: SharedTemporaryFileStorage,
+ FileStorageType.API_EXECUTION: SharedTemporaryFileStorage,
+}
+
+FILE_STORAGE_CREDENTIALS_MAPPING = {
+ FileStorageType.WORKFLOW_EXECUTION: json.loads(
+ os.environ.get("WORKFLOW_EXECUTION_FS_CREDENTIAL", "{}")
+ ),
+ FileStorageType.API_EXECUTION: json.loads(
+ os.environ.get("API_STORAGE_FS_CREDENTIAL", "{}")
+ ),
+}
+
+FILE_STORAGE_PROVIDER_MAPPING = {
+ FileStorageType.WORKFLOW_EXECUTION: get_provider(
+ "WORKFLOW_EXECUTION_FS_PROVIDER", "minio"
+ ),
+ FileStorageType.API_EXECUTION: get_provider("API_STORAGE_FS_PROVIDER", "minio"),
+}
diff --git a/unstract/filesystem/src/unstract/filesystem/file_storage_types.py b/unstract/filesystem/src/unstract/filesystem/file_storage_types.py
new file mode 100644
index 000000000..be9889c89
--- /dev/null
+++ b/unstract/filesystem/src/unstract/filesystem/file_storage_types.py
@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class FileStorageType(Enum):
+ WORKFLOW_EXECUTION = "WORKFLOW_EXECUTION"
+ API_EXECUTION = "API_EXECUTION"
diff --git a/unstract/filesystem/src/unstract/filesystem/filesystem.py b/unstract/filesystem/src/unstract/filesystem/filesystem.py
new file mode 100644
index 000000000..094d2448a
--- /dev/null
+++ b/unstract/filesystem/src/unstract/filesystem/filesystem.py
@@ -0,0 +1,51 @@
+import logging
+import os
+from typing import Any
+
+from unstract.sdk.file_storage import FileStorageProvider
+from unstract.sdk.file_storage.fs_impl import FileStorage
+from unstract.sdk.file_storage.fs_permanent import PermanentFileStorage
+
+from unstract.filesystem.file_storage_types import FileStorageType
+
+from .file_storage_config import (
+ FILE_STORAGE_CREDENTIALS_MAPPING,
+ FILE_STORAGE_PROVIDER_MAPPING,
+ STORAGE_MAPPING,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class FileSystem:
+ def __init__(self, file_storage_type: FileStorageType):
+ self.file_storage_type = file_storage_type
+
+ def _get_credentials(self) -> dict[str, Any]:
+ """Retrieve storage credentials based on the storage type."""
+ return FILE_STORAGE_CREDENTIALS_MAPPING.get(self.file_storage_type, {})
+
+ def _get_provider(self) -> FileStorageProvider:
+ """Retrieve provider based on the storage type."""
+ provider_name = FILE_STORAGE_PROVIDER_MAPPING.get(self.file_storage_type)
+ if not provider_name:
+ raise ValueError(f"No provider found for {self.file_storage_type}.")
+ return provider_name
+
+ def get_file_storage(self) -> FileStorage:
+ """Initialize and return the appropriate file storage instance."""
+ # Get the storage class based on the FileStorageType
+ storage_class = STORAGE_MAPPING.get(self.file_storage_type)
+
+ if not storage_class:
+ raise ValueError(f"Unsupported FileStorageType: {self.file_storage_type}")
+
+ # Base configuration for all storage classes
+ credentials = self._get_credentials()
+
+ # Add specific parameters based on the storage class type
+ if storage_class == PermanentFileStorage:
+ credentials["legacy_storage_path"] = os.getenv("LEGACY_STORAGE_PATH", None)
+
+ # Return an instance of the storage class with the appropriate configuration
+ return storage_class(provider=self._get_provider(), **credentials)
diff --git a/unstract/workflow-execution/pyproject.toml b/unstract/workflow-execution/pyproject.toml
index b442540e8..511515644 100644
--- a/unstract/workflow-execution/pyproject.toml
+++ b/unstract/workflow-execution/pyproject.toml
@@ -23,6 +23,7 @@ dependencies = [
"unstract-core",
"unstract-tool-registry",
"unstract-tool-sandbox",
+ "unstract-flags"
]
requires-python = ">=3.9"
readme = "README.md"
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py b/unstract/workflow-execution/src/unstract/workflow_execution/constants.py
index 172d12769..c00145620 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/constants.py
@@ -21,6 +21,8 @@ class ToolRuntimeVariable:
ADAPTER_LLMW_POLL_INTERVAL = "ADAPTER_LLMW_POLL_INTERVAL"
ADAPTER_LLMW_MAX_POLLS = "ADAPTER_LLMW_MAX_POLLS"
EXECUTION_BY_TOOL = "EXECUTION_BY_TOOL"
+ WORKFLOW_EXECUTION_DIR_PREFIX = "WORKFLOW_EXECUTION_DIR_PREFIX"
+ API_EXECUTION_DIR_PREFIX = "API_EXECUTION_DIR_PREFIX"
class WorkflowFileType:
@@ -47,3 +49,9 @@ class ToolMetadataKey:
class ToolOutputType:
TXT = "TXT"
JSON = "JSON"
+
+
+class FeatureFlag:
+ """Temporary feature flags."""
+
+ REMOTE_FILE_STORAGE = "remote_file_storage"
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py
index 48dc77a70..84af11a47 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py
@@ -6,12 +6,20 @@ from typing import Any, Optional
import fsspec
from unstract.workflow_execution.constants import (
+ FeatureFlag,
MetaDataKey,
ToolMetadataKey,
ToolOutputType,
+ ToolRuntimeVariable,
WorkflowFileType,
)
from unstract.workflow_execution.exceptions import ToolMetadataNotFound
+from unstract.workflow_execution.tools_utils import ToolsUtils
+
+from unstract.flags.feature_flag import check_feature_flag_status
+
+if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ from unstract.filesystem import FileStorageType, FileSystem
logger = logging.getLogger(__name__)
@@ -23,9 +31,14 @@ class ExecutionFileHandler:
self.organization_id = organization_id
self.workflow_id = workflow_id
self.execution_id = execution_id
- self.execution_dir = self.create_execution_dir_path(
- workflow_id, execution_id, organization_id
- )
+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ self.execution_dir = self.get_execution_dir(
+ workflow_id, execution_id, organization_id
+ )
+ else:
+ self.execution_dir = self.create_execution_dir_path(
+ workflow_id, execution_id, organization_id
+ )
self.source_file = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)
self.infile = os.path.join(self.execution_dir, WorkflowFileType.INFILE)
self.metadata_file = os.path.join(
@@ -38,8 +51,14 @@ class ExecutionFileHandler:
Returns:
dict[str, Any]: Workflow metadata.
"""
- with open(self.metadata_file) as file:
- metadata: dict[str, Any] = json.load(file)
+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
+ file_storage = file_system.get_file_storage()
+ metadata_content = file_storage.read(path=self.metadata_file, mode="r")
+ metadata = json.loads(metadata_content)
+ else:
+ with open(self.metadata_file) as file:
+ metadata: dict[str, Any] = json.load(file)
return metadata
def get_list_of_tool_metadata(
@@ -100,9 +119,7 @@ class ExecutionFileHandler:
Raises:
None
"""
- destination_path = os.path.join(
- self.execution_dir, WorkflowFileType.METADATA_JSON
- )
+ metadata_path = os.path.join(self.execution_dir, WorkflowFileType.METADATA_JSON)
filename = os.path.basename(input_file_path)
content = {
MetaDataKey.SOURCE_NAME: filename,
@@ -111,8 +128,14 @@ class ExecutionFileHandler:
MetaDataKey.WORKFLOW_ID: str(self.workflow_id),
MetaDataKey.EXECUTION_ID: str(self.execution_id),
}
- with fsspec.open(f"file://{destination_path}", "w") as local_file:
- json.dump(content, local_file)
+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):
+ file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
+ file_storage = file_system.get_file_storage()
+ file_storage.write(path=metadata_path, mode="w", data=json.dumps(content))
+ else:
+ with fsspec.open(f"file://{metadata_path}", "w") as local_file:
+ json.dump(content, local_file)
+
logger.info(
f"metadata for {input_file_path} is " "added in to execution directory"
)
@@ -145,3 +168,44 @@ class ExecutionFileHandler:
)
execution_dir.mkdir(parents=True, exist_ok=True)
return str(execution_dir)
+
+ @classmethod
+ def get_execution_dir(
+ cls, workflow_id: str, execution_id: str, organization_id: str
+ ) -> str:
+ """Create the directory path for storing execution-related files.
+
+ Parameters:
+ - workflow_id (str): Identifier for the workflow.
+ - execution_id (str): Identifier for the execution.
+ - organization_id (Optional[str]):
+ Identifier for the organization (default: None).
+
+ Returns:
+ str: The directory path for the execution.
+ """
+ path_prefix = ToolsUtils.get_env(
+ ToolRuntimeVariable.WORKFLOW_EXECUTION_DIR_PREFIX
+ )
+ execution_dir = Path(path_prefix) / organization_id / workflow_id / execution_id
+
+ return str(execution_dir)
+
+ @classmethod
+ def get_api_execution_dir(
+ cls, workflow_id: str, execution_id: str, organization_id: str
+ ) -> str:
+ """Create the directory path for storing execution-related files.
+
+ Parameters:
+ - workflow_id (str): Identifier for the workflow.
+ - execution_id (str): Identifier for the execution.
+ - organization_id (Optional[str]):
+ Identifier for the organization (default: None).
+
+ Returns:
+ str: The directory path for the execution.
+ """
+ path_prefix = ToolsUtils.get_env(ToolRuntimeVariable.API_EXECUTION_DIR_PREFIX)
+ execution_dir = Path(path_prefix) / organization_id / workflow_id / execution_id
+ return str(execution_dir)
diff --git a/worker/src/unstract/worker/clients/test_docker.py b/worker/src/unstract/worker/clients/test_docker.py
index d6d52fef0..6c587a78d 100644
--- a/worker/src/unstract/worker/clients/test_docker.py
+++ b/worker/src/unstract/worker/clients/test_docker.py
@@ -109,7 +109,6 @@ def test_get_image(docker_client, mocker):
def test_get_container_run_config(docker_client, mocker):
"""Test the get_container_run_config method."""
os.environ[Env.WORKFLOW_DATA_DIR] = "/source"
- os.environ[Env.FLIPT_SERVICE_AVAILABLE] = "False"
os.environ[Env.EXECUTION_RUN_DATA_FOLDER_PREFIX] = "/app/workflow_data"
command = ["echo", "hello"]
organization_id = "org123"