Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-3343 [FEAT] Support AWS S3 connector authentication via IRSA (#1866)

Kirtiman Mishrad657e197cbb2
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/exceptions.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/exceptions.py
index 0cc1a8977..05e7ed67d 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/exceptions.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/exceptions.py
@@ -1,12 +1,41 @@
from unstract.connectors.exceptions import ConnectorError
-S3FS_EXC_TO_UNSTRACT_EXC = {
+S3FS_EXC_TO_UNSTRACT_EXC: dict[str, str] = {
+ # Auth errors
"The AWS Access Key Id you provided does not exist in our records": (
"Invalid Key (Access Key ID) provided, please provide a valid one."
),
"The request signature we calculated does not match the signature you provided": (
"Invalid Secret (Secret Access Key) provided, please provide a valid one."
),
+ "Unable to locate credentials": (
+ "No AWS credentials found. Provide a valid access key/secret or ensure "
+ "the instance/pod has an IAM role attached."
+ ),
+ "AssumeRoleWithWebIdentity": (
+ "Failed to assume IAM role via web identity. Verify the IAM role exists "
+ "and has the correct trust policy."
+ ),
+ "InvalidIdentityToken": (
+ "The identity token provided is invalid. Verify the OIDC provider "
+ "and ServiceAccount configuration."
+ ),
+ "ExpiredToken": (
+ "AWS security token has expired. Refresh your credentials or ensure "
+ "IAM role session duration is sufficient."
+ ),
+ # Permission errors
+ "AccessDenied": (
+ "Access denied. The IAM user or role does not have sufficient S3 "
+ "permissions. Ensure the policy grants the required S3 actions "
+ "(s3:ListAllMyBuckets, s3:ListBucket, s3:GetObject, s3:PutObject) "
+ "on the target bucket."
+ ),
+ # Bucket errors
+ "NoSuchBucket": (
+ "The specified bucket does not exist. Please check the bucket name."
+ ),
+ # Endpoint / connectivity errors
"[Errno 22] S3 API Requests must be made to API port": ( # Minio only
"Request made to invalid port, please check the port of the endpoint URL."
),
@@ -14,6 +43,21 @@ S3FS_EXC_TO_UNSTRACT_EXC = {
"Could not connect to the endpoint URL. Please check if the URL is correct "
"and accessible."
),
+ "timed out": (
+ "Connection timed out. Check network connectivity and the endpoint URL."
+ ),
+ "SSL: CERTIFICATE_VERIFY_FAILED": (
+ "SSL certificate verification failed. If using a self-signed certificate "
+ "(e.g. MinIO), check your endpoint configuration."
+ ),
+ "Name or service not known": (
+ "Could not resolve the endpoint hostname. Please check the endpoint URL."
+ ),
+ # Clock / request errors
+ "RequestTimeTooSkewed": (
+ "The system clock is out of sync with AWS. Ensure the host's clock "
+ "is accurate (max allowed skew is 15 minutes)."
+ ),
}
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
index bf99e5f74..d6253348a 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py
@@ -16,22 +16,28 @@ logger = logging.getLogger(__name__)
class MinioFS(UnstractFileSystem):
def __init__(self, settings: dict[str, Any]):
super().__init__("MinioFS/S3")
- key = settings.get("key", "")
- secret = settings.get("secret", "")
- endpoint_url = settings.get("endpoint_url", "")
+ key = (settings.get("key") or "").strip()
+ secret = (settings.get("secret") or "").strip()
+ endpoint_url = (settings.get("endpoint_url") or "").strip()
client_kwargs = {}
if "region_name" in settings and settings["region_name"] != "":
client_kwargs = {"region_name": settings["region_name"]}
+
+ creds: dict[str, str] = {}
+ if key and secret:
+ creds["key"] = key
+ creds["secret"] = secret
+ if endpoint_url:
+ creds["endpoint_url"] = endpoint_url
+
self.s3 = S3FileSystem(
anon=False,
- key=key,
- secret=secret,
use_listings_cache=False,
default_fill_cache=False,
default_cache_type="none",
skip_instance_cache=True,
- endpoint_url=endpoint_url,
client_kwargs=client_kwargs,
+ **creds,
)
@staticmethod
@@ -87,17 +93,18 @@ class MinioFS(UnstractFileSystem):
Optional[str]: The file hash in hexadecimal format or None if not found.
"""
# Extracts ETag for MinIO
- file_hash = metadata.get("ETag")
+ file_hash: str | None = metadata.get("ETag")
if file_hash:
file_hash = file_hash.strip('"')
if "-" in file_hash:
logger.warning(
- f"[S3/MinIO] Multipart upload detected. ETag may not be an "
- f"MD5 hash. Full metadata: {metadata}"
+ "[S3/MinIO] Multipart upload detected. ETag may not be an "
+ "MD5 hash. Full metadata: %s",
+ metadata,
)
return None
return file_hash
- logger.error(f"[MinIO] File hash not found for the metadata: {metadata}")
+ logger.error("[MinIO] File hash not found for the metadata: %s", metadata)
return None
def is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:
@@ -119,7 +126,8 @@ class MinioFS(UnstractFileSystem):
return last_modified
logger.debug(
- f"[S3/MinIO] No modified date found in metadata keys: {list(metadata.keys())}"
+ "[S3/MinIO] No modified date found in metadata keys: %s",
+ list(metadata.keys()),
)
return None
@@ -146,7 +154,9 @@ class MinioFS(UnstractFileSystem):
return dt.astimezone(UTC)
except (ValueError, TypeError):
logger.warning(
- f"[S3/MinIO] Failed to parse datetime '{date_str}' from metadata keys: {metadata_keys}"
+ "[S3/MinIO] Failed to parse datetime '%s' from metadata keys: %s",
+ date_str,
+ metadata_keys,
)
return None
@@ -155,7 +165,7 @@ class MinioFS(UnstractFileSystem):
try:
return datetime.fromtimestamp(timestamp, tz=UTC)
except (ValueError, OSError):
- logger.warning(f"[S3/MinIO] Invalid epoch timestamp: {timestamp}")
+ logger.warning("[S3/MinIO] Invalid epoch timestamp: %s", timestamp)
return None
def extract_modified_date(self, metadata: dict[str, Any]) -> datetime | None:
@@ -183,7 +193,9 @@ class MinioFS(UnstractFileSystem):
return self._parse_numeric_timestamp(last_modified)
logger.debug(
- f"[S3/MinIO] Unsupported datetime type '{type(last_modified)}' in metadata keys: {list(metadata.keys())}"
+ "[S3/MinIO] Unsupported datetime type '%s' in metadata keys: %s",
+ type(last_modified),
+ list(metadata.keys()),
)
return None
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json
index e357f4805..d0ba5ed91 100644
--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json
+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json
@@ -4,8 +4,6 @@
"type": "object",
"required": [
"connectorName",
- "key",
- "secret",
"endpoint_url",
"region_name"
],
@@ -19,25 +17,25 @@
"type": "string",
"title": "Key",
"default": "",
- "description": "Access Key ID"
+ "description": "Access Key ID (leave blank to use IAM role / instance profile)"
},
"secret": {
"type": "string",
"title": "Secret",
"format": "password",
- "description": "Secret Access Key"
+ "description": "Secret Access Key (leave blank to use IAM role / instance profile)"
},
"endpoint_url": {
"type": "string",
"title": "Endpoint URL",
"default": "https://s3.amazonaws.com",
- "description": "Endpoint URL to connect to. (example `https://s3.amazonaws.com`)"
+ "description": "Endpoint URL (leave blank for default AWS S3)"
},
"region_name": {
"type": "string",
"title": "Region Name",
"default": "ap-south",
- "description": "Region of the AWS S3 account. For Minio, leave it blank"
+ "description": "Region of the AWS S3 account (leave blank for Minio)"
}
}
}