LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Feat/remote storage workflow (#837)
alif2a55f2c0401
diff --git a/backend/backend/constants.py b/backend/backend/constants.pyindex 26d944d9a..53126cebe 100644--- a/backend/backend/constants.py+++ b/backend/backend/constants.py@@ -34,3 +34,4 @@ class FeatureFlag:"""Temporary feature flags."""APP_DEPLOYMENT = "app_deployment"+ REMOTE_FILE_STORAGE = "remote_file_storage"diff --git a/backend/sample.env b/backend/sample.envindex f91496da3..cd3419cf1 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -144,3 +144,22 @@ TOOL_REGISTRY_CONFIG_PATH="/data/tool_registry_config"# Flipt ServiceFLIPT_SERVICE_AVAILABLE=False+++# File System Configuration for Workflow and API Execution++# Directory Prefixes for storing execution files+WORKFLOW_EXECUTION_DIR_PREFIX="unstract/execution"+API_EXECUTION_DIR_PREFIX="unstract/api"++# Storage Provider for Workflow Execution+# Valid options: MINIO, S3, etc..+WORKFLOW_EXECUTION_FS_PROVIDER="MINIO"+WORKFLOW_EXECUTION_FS_CREDENTIAL='{"endpoint_url": "", "key": "", "secret": ""}'++# Storage Provider for API Execution+API_STORAGE_FS_PROVIDER="MINIO"+API_STORAGE_FS_CREDENTIAL='{"endpoint_url": "", "key": "", "secret": ""}'++# Optional: Legacy storage path (if applicable)+LEGACY_STORAGE_PATH="/path/to/legacy/storage"diff --git a/backend/utils/constants.py b/backend/utils/constants.pyindex a92f397a4..377c21da6 100644--- a/backend/utils/constants.py+++ b/backend/utils/constants.py@@ -15,12 +15,6 @@ class Account:ORGANIZATION_ID = "organization_id"-class FeatureFlag:- """Temporary feature flags."""-- pass--class Common:METADATA = "metadata"MODULE = "module"diff --git a/backend/workflow_manager/endpoint_v2/base_connector.py b/backend/workflow_manager/endpoint_v2/base_connector.pyindex 2523523e3..8b34910d7 100644--- a/backend/workflow_manager/endpoint_v2/base_connector.py+++ b/backend/workflow_manager/endpoint_v2/base_connector.py@@ -7,8 +7,13 @@ from unstract.workflow_execution.execution_file_handler import ExecutionFileHandfrom utils.constants import Commonfrom utils.user_context import UserContext+from backend.constants import FeatureFlagfrom unstract.connectors.filesystems import connectorsfrom unstract.connectors.filesystems.unstract_file_system import UnstractFileSystem+from unstract.flags.feature_flag import check_feature_flag_status++if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ from unstract.filesystem import FileStorageType, FileSystemclass BaseConnector(ExecutionFileHandler):@@ -22,13 +27,14 @@ class BaseConnector(ExecutionFileHandler):This class serves as a base for connectors and provides commonutilities."""- if not (settings.API_STORAGE_DIR and settings.WORKFLOW_DATA_DIR):- raise ValueError("Missed env API_STORAGE_DIR or WORKFLOW_DATA_DIR")super().__init__(workflow_id, execution_id, organization_id)- # Directory path for storing execution-related files for API- self.api_storage_dir: str = self.create_execution_dir_path(- workflow_id, execution_id, organization_id, settings.API_STORAGE_DIR- )+ if not check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ if not (settings.API_STORAGE_DIR and settings.WORKFLOW_DATA_DIR):+ raise ValueError("Missed env API_STORAGE_DIR or WORKFLOW_DATA_DIR")+ # Directory path for storing execution-related files for API+ self.api_storage_dir: str = self.create_execution_dir_path(+ workflow_id, execution_id, organization_id, settings.API_STORAGE_DIR+ )def get_fsspec(self, settings: dict[str, Any], connector_id: str@@ -80,8 +86,14 @@ class BaseConnector(ExecutionFileHandler):json.JSONDecodeError: If there is an issue decoding the JSON file."""try:- with open(file_path, encoding="utf-8") as file:- schema: dict[str, Any] = json.load(file)+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)+ file_storage = file_system.get_file_storage()+ file_contents = file_storage.read(path=file_path, encoding="utf-8")+ schema: dict[str, Any] = json.load(file_contents)+ else:+ with open(file_path, encoding="utf-8") as file:+ schema: dict[str, Any] = json.load(file)except OSError:schema = {}return schema@@ -100,7 +112,12 @@ class BaseConnector(ExecutionFileHandler):str: The directory path for the execution."""organization_id = UserContext.get_organization_identifier()- api_storage_dir: str = cls.create_execution_dir_path(- workflow_id, execution_id, organization_id, settings.API_STORAGE_DIR- )+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ api_storage_dir: str = cls.get_api_execution_dir(+ workflow_id, execution_id, organization_id+ )+ else:+ api_storage_dir: str = cls.create_execution_dir_path(+ workflow_id, execution_id, organization_id, settings.API_STORAGE_DIR+ )return api_storage_dirdiff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex 88f45b312..036cece50 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -10,6 +10,7 @@ import magicfrom connector_v2.models import ConnectorInstancefrom fsspec.implementations.local import LocalFileSystemfrom unstract.sdk.constants import ToolExecKey+from unstract.sdk.tool.mime_types import EXT_MIME_MAPfrom unstract.workflow_execution.constants import ToolOutputTypefrom utils.user_context import UserContextfrom workflow_manager.endpoint_v2.base_connector import BaseConnector@@ -36,8 +37,13 @@ from workflow_manager.workflow_v2.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow_v2.models.file_history import FileHistoryfrom workflow_manager.workflow_v2.models.workflow import Workflow+from backend.constants import FeatureFlagfrom backend.exceptions import UnstractFSExceptionfrom unstract.connectors.exceptions import ConnectorError+from unstract.flags.feature_flag import check_feature_flag_status++if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ from unstract.filesystem import FileStorageType, FileSystemlogger = logging.getLogger(__name__)@@ -405,6 +411,8 @@ class DestinationConnector(BaseConnector):Returns:Union[dict[str, Any], str]: Result data."""+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ return self.get_result_with_file_storage(file_history=file_history)if file_history and file_history.result:return self.parse_string(file_history.result)output_file = os.path.join(self.execution_dir, WorkflowFileType.INFILE)@@ -434,6 +442,43 @@ class DestinationConnector(BaseConnector):logger.error(f"Error while getting result {err}")return result+ def get_result_with_file_storage(+ self, file_history: Optional[FileHistory] = None+ ) -> Optional[Any]:+ """Get result data from the output file.++ Returns:+ Union[dict[str, Any], str]: Result data.+ """+ if file_history and file_history.result:+ return self.parse_string(file_history.result)+ output_file = os.path.join(self.execution_dir, WorkflowFileType.INFILE)+ metadata: dict[str, Any] = self.get_workflow_metadata()+ output_type = self.get_output_type(metadata)+ result: Union[dict[str, Any], str] = ""+ file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)+ file_storage = file_system.get_file_storage()+ try:+ # TODO: SDK handles validation; consider removing here.+ file_type = file_storage.mime_type(output_file)+ if output_type == ToolOutputType.JSON:+ if file_type != EXT_MIME_MAP[ToolOutputType.JSON.lower()]:+ logger.error(f"Output type json mismatched {file_type}")+ raise ToolOutputTypeMismatch()+ file_content = file_storage.read(output_file, mode="r")+ result = json.loads(file_content)+ elif output_type == ToolOutputType.TXT:+ if file_type == EXT_MIME_MAP[ToolOutputType.JSON.lower()]:+ logger.error(f"Output type txt mismatched {file_type}")+ raise ToolOutputTypeMismatch()+ file_content = file_storage.read(output_file, mode="r")+ result = file_content.encode("utf-8").decode("unicode-escape")+ else:+ raise InvalidToolOutputType()+ except (FileNotFoundError, json.JSONDecodeError) as err:+ logger.error(f"Error while getting result {err}")+ return result+def get_metadata(self, file_history: Optional[FileHistory] = None) -> Optional[dict[str, Any]]:@@ -454,8 +499,13 @@ class DestinationConnector(BaseConnector):Returns:None"""- fs: LocalFileSystem = fsspec.filesystem("file")- fs.rm(self.execution_dir, recursive=True)+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)+ file_storage = file_system.get_file_storage()+ file_storage.rm(self.execution_dir, recursive=True)+ else:+ fs: LocalFileSystem = fsspec.filesystem("file")+ fs.rm(self.execution_dir, recursive=True)self.delete_api_storage_dir(self.workflow_id, self.execution_id)@classmethod@@ -468,8 +518,13 @@ class DestinationConnector(BaseConnector):api_storage_dir = cls.get_api_storage_dir_path(workflow_id=workflow_id, execution_id=execution_id)- fs: LocalFileSystem = fsspec.filesystem("file")- fs.rm(api_storage_dir, recursive=True)+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ file_system = FileSystem(FileStorageType.API_EXECUTION)+ file_storage = file_system.get_file_storage()+ file_storage.rm(api_storage_dir, recursive=True)+ else:+ fs: LocalFileSystem = fsspec.filesystem("file")+ fs.rm(api_storage_dir, recursive=True)@classmethoddef create_endpoint_for_workflow(diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex 59b51306d..c004f3397 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -37,6 +37,12 @@ from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelpefrom workflow_manager.workflow_v2.file_history_helper import FileHistoryHelperfrom workflow_manager.workflow_v2.models.workflow import Workflow+from backend.constants import FeatureFlag+from unstract.flags.feature_flag import check_feature_flag_status++if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ from unstract.filesystem import FileStorageType, FileSystem+logger = logging.getLogger(__name__)@@ -475,11 +481,17 @@ class SourceConnector(BaseConnector):)self.publish_input_file_content(input_file_path, input_log)- with fsspec.open(source_file, "wb") as local_file:- local_file.write(file_content)+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)+ file_storage = file_system.get_file_storage()+ file_storage.write(path=source_file_path, mode="wb", data=file_content)+ file_storage.write(path=infile_path, mode="wb", data=file_content)+ else:+ with fsspec.open(source_file, "wb") as local_file:+ local_file.write(file_content)- # Copy file to infile directory- self.copy_file_to_infile_dir(source_file_path, infile_path)+ # Copy file to infile directory+ self.copy_file_to_infile_dir(source_file_path, infile_path)logger.info(f"{input_file_path} is added to execution directory")return hash_value_of_file_content@@ -488,8 +500,76 @@ class SourceConnector(BaseConnector):"""Add input file to execution directory from api storage."""infile_path = os.path.join(self.execution_dir, WorkflowFileType.INFILE)source_path = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)- shutil.copyfile(input_file_path, infile_path)- shutil.copyfile(input_file_path, source_path)+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ api_file_system = FileSystem(FileStorageType.API_EXECUTION)+ api_file_storage = api_file_system.get_file_storage()+ workflow_file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)+ workflow_file_storage = workflow_file_system.get_file_storage()+ self._copy_file_to_destination(+ source_storage=api_file_storage,+ destination_storage=workflow_file_storage,+ source_path=input_file_path,+ destination_paths=[infile_path, source_path],+ )+ else:+ shutil.copyfile(input_file_path, infile_path)+ shutil.copyfile(input_file_path, source_path)++ # TODO: replace it with method from SDK Utils+ def _copy_file_to_destination(+ self,+ source_storage: Any,+ destination_storage: Any,+ source_path: str,+ destination_paths: list[str],+ chunk_size: int = 4096,+ ) -> None:+ """+ Copy a file from a source storage to one or more paths in a+ destination storage.++ This function reads the source file in chunks and writes each chunk to+ the specified destination paths. The function will continue until the+ entire source file is copied.++ Args:+ source_storage (FileStorage): The storage object from which+ the file is read.+ destination_storage (FileStorage): The storage object to which+ the file is written.+ source_path (str): The path of the file in the source storage.+ destination_paths (list[str]): A list of paths where the file will be+ copied in the destination storage.+ chunk_size (int, optional): The number of bytes to read per chunk.+ Default is 4096 bytes.+ """+ seek_position = 0 # Start from the beginning+ end_of_file = False++ # Loop to read and write in chunks until the end of the file+ while not end_of_file:+ # Read a chunk from the source file+ chunk = source_storage.read(+ path=source_path,+ mode="rb",+ seek_position=seek_position,+ length=chunk_size,+ )+ # Check if the end of the file has been reached+ if not chunk:+ end_of_file = True+ else:+ # Write the chunk to each destination path+ for destination_file in destination_paths:+ destination_storage.write(+ path=destination_file,+ mode="ab",+ seek_position=seek_position,+ data=chunk,+ )++ # Update the seek position+ seek_position += len(chunk)def add_file_to_volume(self, input_file_path: str, file_hash: FileHash) -> str:"""Add input file to execution directory.@@ -584,12 +664,20 @@ class SourceConnector(BaseConnector):for file in file_objs:file_name = file.namedestination_path = os.path.join(api_storage_dir, file_name)- os.makedirs(os.path.dirname(destination_path), exist_ok=True)- with open(destination_path, "wb") as f:+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ file_system = FileSystem(FileStorageType.API_EXECUTION)+ file_storage = file_system.get_file_storage()buffer = bytearray()for chunk in file.chunks():buffer.extend(chunk)- f.write(buffer)+ file_storage.write(path=destination_path, mode="wb", data=buffer)+ else:+ os.makedirs(os.path.dirname(destination_path), exist_ok=True)+ with open(destination_path, "wb") as f:+ buffer = bytearray()+ for chunk in file.chunks():+ buffer.extend(chunk)+ f.write(buffer)file_hash = cls.hash_str(buffer)connection_type = WorkflowEndpoint.ConnectionType.APIdiff --git a/tox.ini b/tox.iniindex 7ca6d4ac8..3aac163bc 100644--- a/tox.ini+++ b/tox.ini@@ -8,6 +8,7 @@ env_list = py{39,310,311}, workerchangedir = workersetenv =PDM_IGNORE_SAVED_PYTHON="1"+ PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=pythondeps = pdmallowlist_externals=shdiff --git a/unstract/filesystem/README.md b/unstract/filesystem/README.mdnew file mode 100644index 000000000..b4b530ef7--- /dev/null+++ b/unstract/filesystem/README.md@@ -0,0 +1,70 @@+# Filesystem++The **Filesystem** module manages file storage interactions within the project. It is built on top of the **Unstract SDK**, ensuring consistency and flexibility across different storage backends.+++## Key Components+### FileSystem Class+The `FileSystem` class provides methods to configure, authenticate, and initialize storage mechanisms based on predefined types. It uses mappings and environment variables to customize its behavior dynamically.+++### Storage Types+- **Workflow Execution Storage**: Temporary storage for workflow-related files.+- **API Execution Storage**: Temporary storage for API-related files.++### Mappings+- **Storage Mapping**: Defines the storage class for each `FileStorageType`.+- **Credentials Mapping**: Associates credentials (from environment variables) with specific `FileStorageType`.+- **Provider Mapping**: Maps environment-based storage providers to `FileStorageType`.++### Environment Variables+The module relies on environment variables for configuration. Below is a detailed list of required variables:++#### Provider Configuration++`WORKFLOW_EXECUTION_FS_PROVIDER`: The provider for workflow execution storage (default: minio).+`API_STORAGE_FS_PROVIDER`: The provider for API execution storage (default: minio).++#### Credential Configuration++`WORKFLOW_EXECUTION_FS_CREDENTIAL`: JSON string containing credentials for workflow execution storage.+`API_STORAGE_FS_CREDENTIAL`: JSON string containing credentials for API execution storage.++#### Additional Configuration++`LEGACY_STORAGE_PATH` (*optional*): Used specifically with **PermanentFileStorage** for legacy data handling.+++### Example Environment File (.env)+```env+# Workflow Execution File Storage+WORKFLOW_EXECUTION_FS_PROVIDER=minio+WORKFLOW_EXECUTION_FS_CREDENTIAL={"access_key": "your-access-key", "secret_key": "your-secret-key"}++# API Execution File Storage+API_STORAGE_FS_PROVIDER=minio+API_STORAGE_FS_CREDENTIAL={"access_key": "api-access-key", "secret_key": "api-secret-key"}++# Legacy Storage Path (Optional)+LEGACY_STORAGE_PATH=/path/to/legacy/storage+```++### How It Works+1. **Storage Class Initialization**: Based on `FileStorageType`, the appropriate storage class is chosen from `STORAGE_MAPPING`.+2. **Credential Injection**: Credentials are fetched from environment variables and injected into the storage class.+3. **Provider Selection**: Providers are dynamically resolved using the `get_provider` method and mapped to the relevant `FileStorageType`.+4. **Storage Instance Creation**: A storage instance is created and returned, ready for use.+++## Extending the System+To add a new `FileStorageType`:++1. Update the `FileStorageType` enum with the new type.+2. Add the new type to:+ - `STORAGE_MAPPING`: Map the type to a storage class.+ - `FILE_STORAGE_CREDENTIALS_MAPPING`: Define its credentials.+ - `FILE_STORAGE_PROVIDER_MAPPING`: Specify its provider.++# Error Handling+- If a provider is not found in the `FileStorageProvider` enum, a `ProviderNotFound` exception is raised.+- If no valid credentials or mappings are found, the system raises appropriate exceptions (`ValueError`).diff --git a/unstract/filesystem/pdm.lock b/unstract/filesystem/pdm.locknew file mode 100644index 000000000..29b4418ca--- /dev/null+++ b/unstract/filesystem/pdm.lock@@ -0,0 +1,8 @@+# This file is @generated by PDM.+# It is not intended for manual editing.++[metadata]+groups = ["default"]+strategy = ["cross_platform", "inherit_metadata"]+lock_version = "4.4.2"+content_hash = "sha256:761e9d2e66433c05e581c1ee8664065959079372805516912f6a3d14a3b2cec7"diff --git a/unstract/filesystem/pyproject.toml b/unstract/filesystem/pyproject.tomlnew file mode 100644index 000000000..a4b2082c9--- /dev/null+++ b/unstract/filesystem/pyproject.toml@@ -0,0 +1,18 @@+[build-system]+requires = ["pdm-backend"]+build-backend = "pdm.backend"++[project]+name = "unstract-filesystem"+version = "0.0.1"+description = "Unstract filesystem package"+authors = [+ {name = "Zipstack Inc.", email = "[email protected]"},+]+dependencies = []+requires-python = ">=3.9,<3.11.1"+readme = "README.md"++[tool.pdm.build]+includes = ["src"]+package-dir = "src"diff --git a/unstract/filesystem/src/unstract/filesystem/__init__.py b/unstract/filesystem/src/unstract/filesystem/__init__.pynew file mode 100644index 000000000..3cfdc80eb--- /dev/null+++ b/unstract/filesystem/src/unstract/filesystem/__init__.py@@ -0,0 +1,10 @@+"""+This module exposes the main classes for file storage handling.+"""++from unstract.sdk.file_storage.fs_shared_temporary import SharedTemporaryFileStorage++from .file_storage_types import FileStorageType+from .filesystem import FileSystem++__all__ = ["FileSystem", "SharedTemporaryFileStorage", "FileStorageType"]diff --git a/unstract/filesystem/src/unstract/filesystem/exceptions.py b/unstract/filesystem/src/unstract/filesystem/exceptions.pynew file mode 100644index 000000000..d51241478--- /dev/null+++ b/unstract/filesystem/src/unstract/filesystem/exceptions.py@@ -0,0 +1,17 @@+class FilesystemError(Exception):+ """+ Base exception class for filesystem-related errors.+ """++ default_message = "An error occurred in the filesystem operation."++ def __init__(self, message: str = None):+ self.message = message or self.default_message+ super().__init__(self.message)++ def __str__(self) -> str:+ return self.message+++class ProviderNotFound(FilesystemError):+ default_message = "The requested provider was not found."diff --git a/unstract/filesystem/src/unstract/filesystem/file_storage_config.py b/unstract/filesystem/src/unstract/filesystem/file_storage_config.pynew file mode 100644index 000000000..b0b497b7b--- /dev/null+++ b/unstract/filesystem/src/unstract/filesystem/file_storage_config.py@@ -0,0 +1,51 @@+import json+import logging+import os++from unstract.sdk.file_storage import FileStorageProvider+from unstract.sdk.file_storage.fs_shared_temporary import SharedTemporaryFileStorage++from .exceptions import ProviderNotFound+from .file_storage_types import FileStorageType # Import the shared enum++logger = logging.getLogger(__name__)+++def get_provider(var_name: str, default: str = "minio") -> FileStorageProvider:+ """Retrieve the file storage provider based on an environment variable."""+ provider_name = os.environ.get(var_name, default).capitalize()+ try:+ # Attempt to map the provider name to an enum value, case-insensitively+ return FileStorageProvider[provider_name]+ except KeyError:+ allowed_providers = ", ".join(+ [provider.name for provider in FileStorageProvider]+ )+ logger.error(+ f"Invalid provider '{provider_name}'. Allowed providers: "+ f"{allowed_providers}"+ )+ raise ProviderNotFound(f"Provider '{provider_name}' not found")+++# Mappings for storage types, credentials, and providers+STORAGE_MAPPING = {+ FileStorageType.WORKFLOW_EXECUTION: SharedTemporaryFileStorage,+ FileStorageType.API_EXECUTION: SharedTemporaryFileStorage,+}++FILE_STORAGE_CREDENTIALS_MAPPING = {+ FileStorageType.WORKFLOW_EXECUTION: json.loads(+ os.environ.get("WORKFLOW_EXECUTION_FS_CREDENTIAL", "{}")+ ),+ FileStorageType.API_EXECUTION: json.loads(+ os.environ.get("API_STORAGE_FS_CREDENTIAL", "{}")+ ),+}++FILE_STORAGE_PROVIDER_MAPPING = {+ FileStorageType.WORKFLOW_EXECUTION: get_provider(+ "WORKFLOW_EXECUTION_FS_PROVIDER", "minio"+ ),+ FileStorageType.API_EXECUTION: get_provider("API_STORAGE_FS_PROVIDER", "minio"),+}diff --git a/unstract/filesystem/src/unstract/filesystem/file_storage_types.py b/unstract/filesystem/src/unstract/filesystem/file_storage_types.pynew file mode 100644index 000000000..be9889c89--- /dev/null+++ b/unstract/filesystem/src/unstract/filesystem/file_storage_types.py@@ -0,0 +1,6 @@+from enum import Enum+++class FileStorageType(Enum):+ WORKFLOW_EXECUTION = "WORKFLOW_EXECUTION"+ API_EXECUTION = "API_EXECUTION"diff --git a/unstract/filesystem/src/unstract/filesystem/filesystem.py b/unstract/filesystem/src/unstract/filesystem/filesystem.pynew file mode 100644index 000000000..094d2448a--- /dev/null+++ b/unstract/filesystem/src/unstract/filesystem/filesystem.py@@ -0,0 +1,51 @@+import logging+import os+from typing import Any++from unstract.sdk.file_storage import FileStorageProvider+from unstract.sdk.file_storage.fs_impl import FileStorage+from unstract.sdk.file_storage.fs_permanent import PermanentFileStorage++from unstract.filesystem.file_storage_types import FileStorageType++from .file_storage_config import (+ FILE_STORAGE_CREDENTIALS_MAPPING,+ FILE_STORAGE_PROVIDER_MAPPING,+ STORAGE_MAPPING,+)++logger = logging.getLogger(__name__)+++class FileSystem:+ def __init__(self, file_storage_type: FileStorageType):+ self.file_storage_type = file_storage_type++ def _get_credentials(self) -> dict[str, Any]:+ """Retrieve storage credentials based on the storage type."""+ return FILE_STORAGE_CREDENTIALS_MAPPING.get(self.file_storage_type, {})++ def _get_provider(self) -> FileStorageProvider:+ """Retrieve provider based on the storage type."""+ provider_name = FILE_STORAGE_PROVIDER_MAPPING.get(self.file_storage_type)+ if not provider_name:+ raise ValueError(f"No provider found for {self.file_storage_type}.")+ return provider_name++ def get_file_storage(self) -> FileStorage:+ """Initialize and return the appropriate file storage instance."""+ # Get the storage class based on the FileStorageType+ storage_class = STORAGE_MAPPING.get(self.file_storage_type)++ if not storage_class:+ raise ValueError(f"Unsupported FileStorageType: {self.file_storage_type}")++ # Base configuration for all storage classes+ credentials = self._get_credentials()++ # Add specific parameters based on the storage class type+ if storage_class == PermanentFileStorage:+ credentials["legacy_storage_path"] = os.getenv("LEGACY_STORAGE_PATH", None)++ # Return an instance of the storage class with the appropriate configuration+ return storage_class(provider=self._get_provider(), **credentials)diff --git a/unstract/workflow-execution/pyproject.toml b/unstract/workflow-execution/pyproject.tomlindex b442540e8..511515644 100644--- a/unstract/workflow-execution/pyproject.toml+++ b/unstract/workflow-execution/pyproject.toml@@ -23,6 +23,7 @@ dependencies = ["unstract-core","unstract-tool-registry","unstract-tool-sandbox",+ "unstract-flags"]requires-python = ">=3.9"readme = "README.md"diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py b/unstract/workflow-execution/src/unstract/workflow_execution/constants.pyindex 172d12769..c00145620 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/constants.py@@ -21,6 +21,8 @@ class ToolRuntimeVariable:ADAPTER_LLMW_POLL_INTERVAL = "ADAPTER_LLMW_POLL_INTERVAL"ADAPTER_LLMW_MAX_POLLS = "ADAPTER_LLMW_MAX_POLLS"EXECUTION_BY_TOOL = "EXECUTION_BY_TOOL"+ WORKFLOW_EXECUTION_DIR_PREFIX = "WORKFLOW_EXECUTION_DIR_PREFIX"+ API_EXECUTION_DIR_PREFIX = "API_EXECUTION_DIR_PREFIX"class WorkflowFileType:@@ -47,3 +49,9 @@ class ToolMetadataKey:class ToolOutputType:TXT = "TXT"JSON = "JSON"+++class FeatureFlag:+ """Temporary feature flags."""++ REMOTE_FILE_STORAGE = "remote_file_storage"diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.pyindex 48dc77a70..84af11a47 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/execution_file_handler.py@@ -6,12 +6,20 @@ from typing import Any, Optionalimport fsspecfrom unstract.workflow_execution.constants import (+ FeatureFlag,MetaDataKey,ToolMetadataKey,ToolOutputType,+ ToolRuntimeVariable,WorkflowFileType,)from unstract.workflow_execution.exceptions import ToolMetadataNotFound+from unstract.workflow_execution.tools_utils import ToolsUtils++from unstract.flags.feature_flag import check_feature_flag_status++if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ from unstract.filesystem import FileStorageType, FileSystemlogger = logging.getLogger(__name__)@@ -23,9 +31,14 @@ class ExecutionFileHandler:self.organization_id = organization_idself.workflow_id = workflow_idself.execution_id = execution_id- self.execution_dir = self.create_execution_dir_path(- workflow_id, execution_id, organization_id- )+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ self.execution_dir = self.get_execution_dir(+ workflow_id, execution_id, organization_id+ )+ else:+ self.execution_dir = self.create_execution_dir_path(+ workflow_id, execution_id, organization_id+ )self.source_file = os.path.join(self.execution_dir, WorkflowFileType.SOURCE)self.infile = os.path.join(self.execution_dir, WorkflowFileType.INFILE)self.metadata_file = os.path.join(@@ -38,8 +51,14 @@ class ExecutionFileHandler:Returns:dict[str, Any]: Workflow metadata."""- with open(self.metadata_file) as file:- metadata: dict[str, Any] = json.load(file)+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)+ file_storage = file_system.get_file_storage()+ metadata_content = file_storage.read(path=self.metadata_file, mode="r")+ metadata = json.loads(metadata_content)+ else:+ with open(self.metadata_file) as file:+ metadata: dict[str, Any] = json.load(file)return metadatadef get_list_of_tool_metadata(@@ -100,9 +119,7 @@ class ExecutionFileHandler:Raises:None"""- destination_path = os.path.join(- self.execution_dir, WorkflowFileType.METADATA_JSON- )+ metadata_path = os.path.join(self.execution_dir, WorkflowFileType.METADATA_JSON)filename = os.path.basename(input_file_path)content = {MetaDataKey.SOURCE_NAME: filename,@@ -111,8 +128,14 @@ class ExecutionFileHandler:MetaDataKey.WORKFLOW_ID: str(self.workflow_id),MetaDataKey.EXECUTION_ID: str(self.execution_id),}- with fsspec.open(f"file://{destination_path}", "w") as local_file:- json.dump(content, local_file)+ if check_feature_flag_status(FeatureFlag.REMOTE_FILE_STORAGE):+ file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)+ file_storage = file_system.get_file_storage()+ file_storage.write(path=metadata_path, mode="w", data=json.dumps(content))+ else:+ with fsspec.open(f"file://{metadata_path}", "w") as local_file:+ json.dump(content, local_file)+logger.info(f"metadata for {input_file_path} is " "added in to execution directory")@@ -145,3 +168,44 @@ class ExecutionFileHandler:)execution_dir.mkdir(parents=True, exist_ok=True)return str(execution_dir)++ @classmethod+ def get_execution_dir(+ cls, workflow_id: str, execution_id: str, organization_id: str+ ) -> str:+ """Create the directory path for storing execution-related files.++ Parameters:+ - workflow_id (str): Identifier for the workflow.+ - execution_id (str): Identifier for the execution.+ - organization_id (Optional[str]):+ Identifier for the organization (default: None).++ Returns:+ str: The directory path for the execution.+ """+ path_prefix = ToolsUtils.get_env(+ ToolRuntimeVariable.WORKFLOW_EXECUTION_DIR_PREFIX+ )+ execution_dir = Path(path_prefix) / organization_id / workflow_id / execution_id++ return str(execution_dir)++ @classmethod+ def get_api_execution_dir(+ cls, workflow_id: str, execution_id: str, organization_id: str+ ) -> str:+ """Create the directory path for storing execution-related files.++ Parameters:+ - workflow_id (str): Identifier for the workflow.+ - execution_id (str): Identifier for the execution.+ - organization_id (Optional[str]):+ Identifier for the organization (default: None).++ Returns:+ str: The directory path for the execution.+ """+ path_prefix = ToolsUtils.get_env(ToolRuntimeVariable.API_EXECUTION_DIR_PREFIX)+ execution_dir = Path(path_prefix) / organization_id / workflow_id / execution_id+ return str(execution_dir)diff --git a/worker/src/unstract/worker/clients/test_docker.py b/worker/src/unstract/worker/clients/test_docker.pyindex d6d52fef0..6c587a78d 100644--- a/worker/src/unstract/worker/clients/test_docker.py+++ b/worker/src/unstract/worker/clients/test_docker.py@@ -109,7 +109,6 @@ def test_get_image(docker_client, mocker):def test_get_container_run_config(docker_client, mocker):"""Test the get_container_run_config method."""os.environ[Env.WORKFLOW_DATA_DIR] = "/source"- os.environ[Env.FLIPT_SERVICE_AVAILABLE] = "False"os.environ[Env.EXECUTION_RUN_DATA_FOLDER_PREFIX] = "/app/workflow_data"command = ["echo", "hello"]organization_id = "org123"