LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3343 [FEAT] Support AWS S3 connector authentication via IRSA (#1866)
Kirtiman Mishrad657e197cbb2
diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/exceptions.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/exceptions.pyindex 0cc1a8977..05e7ed67d 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/exceptions.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/exceptions.py@@ -1,12 +1,41 @@from unstract.connectors.exceptions import ConnectorError-S3FS_EXC_TO_UNSTRACT_EXC = {+S3FS_EXC_TO_UNSTRACT_EXC: dict[str, str] = {+ # Auth errors"The AWS Access Key Id you provided does not exist in our records": ("Invalid Key (Access Key ID) provided, please provide a valid one."),"The request signature we calculated does not match the signature you provided": ("Invalid Secret (Secret Access Key) provided, please provide a valid one."),+ "Unable to locate credentials": (+ "No AWS credentials found. Provide a valid access key/secret or ensure "+ "the instance/pod has an IAM role attached."+ ),+ "AssumeRoleWithWebIdentity": (+ "Failed to assume IAM role via web identity. Verify the IAM role exists "+ "and has the correct trust policy."+ ),+ "InvalidIdentityToken": (+ "The identity token provided is invalid. Verify the OIDC provider "+ "and ServiceAccount configuration."+ ),+ "ExpiredToken": (+ "AWS security token has expired. Refresh your credentials or ensure "+ "IAM role session duration is sufficient."+ ),+ # Permission errors+ "AccessDenied": (+ "Access denied. The IAM user or role does not have sufficient S3 "+ "permissions. Ensure the policy grants the required S3 actions "+ "(s3:ListAllMyBuckets, s3:ListBucket, s3:GetObject, s3:PutObject) "+ "on the target bucket."+ ),+ # Bucket errors+ "NoSuchBucket": (+ "The specified bucket does not exist. Please check the bucket name."+ ),+ # Endpoint / connectivity errors"[Errno 22] S3 API Requests must be made to API port": ( # Minio only"Request made to invalid port, please check the port of the endpoint URL."),@@ -14,6 +43,21 @@ S3FS_EXC_TO_UNSTRACT_EXC = {"Could not connect to the endpoint URL. Please check if the URL is correct ""and accessible."),+ "timed out": (+ "Connection timed out. Check network connectivity and the endpoint URL."+ ),+ "SSL: CERTIFICATE_VERIFY_FAILED": (+ "SSL certificate verification failed. If using a self-signed certificate "+ "(e.g. MinIO), check your endpoint configuration."+ ),+ "Name or service not known": (+ "Could not resolve the endpoint hostname. Please check the endpoint URL."+ ),+ # Clock / request errors+ "RequestTimeTooSkewed": (+ "The system clock is out of sync with AWS. Ensure the host's clock "+ "is accurate (max allowed skew is 15 minutes)."+ ),}diff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.pyindex bf99e5f74..d6253348a 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/minio.py@@ -16,22 +16,28 @@ logger = logging.getLogger(__name__)class MinioFS(UnstractFileSystem):def __init__(self, settings: dict[str, Any]):super().__init__("MinioFS/S3")- key = settings.get("key", "")- secret = settings.get("secret", "")- endpoint_url = settings.get("endpoint_url", "")+ key = (settings.get("key") or "").strip()+ secret = (settings.get("secret") or "").strip()+ endpoint_url = (settings.get("endpoint_url") or "").strip()client_kwargs = {}if "region_name" in settings and settings["region_name"] != "":client_kwargs = {"region_name": settings["region_name"]}++ creds: dict[str, str] = {}+ if key and secret:+ creds["key"] = key+ creds["secret"] = secret+ if endpoint_url:+ creds["endpoint_url"] = endpoint_url+self.s3 = S3FileSystem(anon=False,- key=key,- secret=secret,use_listings_cache=False,default_fill_cache=False,default_cache_type="none",skip_instance_cache=True,- endpoint_url=endpoint_url,client_kwargs=client_kwargs,+ **creds,)@staticmethod@@ -87,17 +93,18 @@ class MinioFS(UnstractFileSystem):Optional[str]: The file hash in hexadecimal format or None if not found."""# Extracts ETag for MinIO- file_hash = metadata.get("ETag")+ file_hash: str | None = metadata.get("ETag")if file_hash:file_hash = file_hash.strip('"')if "-" in file_hash:logger.warning(- f"[S3/MinIO] Multipart upload detected. ETag may not be an "- f"MD5 hash. Full metadata: {metadata}"+ "[S3/MinIO] Multipart upload detected. ETag may not be an "+ "MD5 hash. Full metadata: %s",+ metadata,)return Nonereturn file_hash- logger.error(f"[MinIO] File hash not found for the metadata: {metadata}")+ logger.error("[MinIO] File hash not found for the metadata: %s", metadata)return Nonedef is_dir_by_metadata(self, metadata: dict[str, Any]) -> bool:@@ -119,7 +126,8 @@ class MinioFS(UnstractFileSystem):return last_modifiedlogger.debug(- f"[S3/MinIO] No modified date found in metadata keys: {list(metadata.keys())}"+ "[S3/MinIO] No modified date found in metadata keys: %s",+ list(metadata.keys()),)return None@@ -146,7 +154,9 @@ class MinioFS(UnstractFileSystem):return dt.astimezone(UTC)except (ValueError, TypeError):logger.warning(- f"[S3/MinIO] Failed to parse datetime '{date_str}' from metadata keys: {metadata_keys}"+ "[S3/MinIO] Failed to parse datetime '%s' from metadata keys: %s",+ date_str,+ metadata_keys,)return None@@ -155,7 +165,7 @@ class MinioFS(UnstractFileSystem):try:return datetime.fromtimestamp(timestamp, tz=UTC)except (ValueError, OSError):- logger.warning(f"[S3/MinIO] Invalid epoch timestamp: {timestamp}")+ logger.warning("[S3/MinIO] Invalid epoch timestamp: %s", timestamp)return Nonedef extract_modified_date(self, metadata: dict[str, Any]) -> datetime | None:@@ -183,7 +193,9 @@ class MinioFS(UnstractFileSystem):return self._parse_numeric_timestamp(last_modified)logger.debug(- f"[S3/MinIO] Unsupported datetime type '{type(last_modified)}' in metadata keys: {list(metadata.keys())}"+ "[S3/MinIO] Unsupported datetime type '%s' in metadata keys: %s",+ type(last_modified),+ list(metadata.keys()),)return Nonediff --git a/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json b/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.jsonindex e357f4805..d0ba5ed91 100644--- a/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json+++ b/unstract/connectors/src/unstract/connectors/filesystems/minio/static/json_schema.json@@ -4,8 +4,6 @@"type": "object","required": ["connectorName",- "key",- "secret","endpoint_url","region_name"],@@ -19,25 +17,25 @@"type": "string","title": "Key","default": "",- "description": "Access Key ID"+ "description": "Access Key ID (leave blank to use IAM role / instance profile)"},"secret": {"type": "string","title": "Secret","format": "password",- "description": "Secret Access Key"+ "description": "Secret Access Key (leave blank to use IAM role / instance profile)"},"endpoint_url": {"type": "string","title": "Endpoint URL","default": "https://s3.amazonaws.com",- "description": "Endpoint URL to connect to. (example `https://s3.amazonaws.com`)"+ "description": "Endpoint URL (leave blank for default AWS S3)"},"region_name": {"type": "string","title": "Region Name","default": "ap-south",- "description": "Region of the AWS S3 account. For Minio, leave it blank"+ "description": "Region of the AWS S3 account (leave blank for Minio)"}}}