LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2297 [FEAT] Support AWS S3 storage authentication via IRSA (#1852)
Kirtiman Mishrafa357640b548
diff --git a/unstract/sdk1/src/unstract/sdk1/file_storage/helper.py b/unstract/sdk1/src/unstract/sdk1/file_storage/helper.pyindex 17e614b00..8fdf7e106 100644--- a/unstract/sdk1/src/unstract/sdk1/file_storage/helper.py+++ b/unstract/sdk1/src/unstract/sdk1/file_storage/helper.py@@ -1,5 +1,4 @@import logging-from typing import Anyimport fsspecfrom fsspec import AbstractFileSystem@@ -12,7 +11,7 @@ logger = logging.getLogger(__name__)class FileStorageHelper:@staticmethoddef file_storage_init(- provider: FileStorageProvider, **storage_config: dict[str, Any]+ provider: FileStorageProvider, **storage_config: object) -> AbstractFileSystem:"""Initialises file storage based on provider.@@ -28,24 +27,44 @@ class FileStorageHelper:protocol = provider.valueif provider == FileStorageProvider.LOCAL:# Hard set auto_mkdir to True as default- storage_config.update({"auto_mkdir": True}) # type: ignore+ storage_config.update({"auto_mkdir": True})elif provider in [FileStorageProvider.MINIO]:# Initialise using s3 for Minioprotocol = FileStorageProvider.S3.value+ if provider in (FileStorageProvider.S3, FileStorageProvider.MINIO):+ # Strip empty string values so boto3's credential chain+ # can work (e.g., IRSA on EKS)+ storage_config = {+ k: v+ for k, v in storage_config.items()+ if not (isinstance(v, str) and v.strip() == "")+ }+ # s3fs expects region_name inside client_kwargs,+ # not as a top-level arg+ region_name = storage_config.pop("region_name", None)+ if region_name:+ existing_kwargs = storage_config.get("client_kwargs", {})+ if not isinstance(existing_kwargs, dict):+ existing_kwargs = {}+ client_kwargs: dict[str, object] = existing_kwargs+ client_kwargs["region_name"] = region_name+ storage_config["client_kwargs"] = client_kwargs+fs = fsspec.filesystem(protocol=protocol,**storage_config,)- logger.debug(f"Connected to {provider.value} file system")+ logger.debug("Connected to %s file system", provider.value)except KeyError as e:logger.error(- f"Error in initialising {provider.value} "- f"file system because of missing config {e}"+ "Error in initialising %s file system because of missing config %s",+ provider.value,+ e,)raise FileStorageError(str(e)) from eexcept Exception as e:- logger.error(f"Error in initialising {provider.value} file system {e}")+ logger.error("Error in initialising %s file system %s", provider.value, e)raise FileStorageError(str(e)) from ereturn fs@@ -58,11 +77,13 @@ class FileStorageHelper:"""try:fs = fsspec.filesystem(protocol=FileStorageProvider.LOCAL.value)- logger.debug(f"Connected to {FileStorageProvider.LOCAL.value} file system")+ logger.debug("Connected to %s file system", FileStorageProvider.LOCAL.value)return fsexcept Exception as e:logger.error(- f"Error in initialising {FileStorageProvider.GCS.value} file system {e}"+ "Error in initialising %s file system %s",+ FileStorageProvider.LOCAL.value,+ e,)raise FileStorageError(str(e)) from ediff --git a/unstract/sdk1/tests/file_storage/__init__.py b/unstract/sdk1/tests/file_storage/__init__.pynew file mode 100644index 000000000..e69de29bb