OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
"""Amazon S3 / S3-compatible storage connector for OpenRAG."""
import mimetypes
import os
from datetime import UTC, datetime
from posixpath import basename
from typing import Any
from config.settings import IBM_AUTH_ENABLED, is_dev_aws_s3_enabled
from connectors.base import (
CONTENT_ETAG_METADATA_KEY,
BaseConnector,
ConnectorDocument,
DocumentACL,
normalize_etag,
)
from utils.logging_config import get_logger
from .auth import create_s3_client, create_s3_resource
logger = get_logger(__name__)
# Separator used in composite file IDs: "<bucket>::<key>"
_ID_SEPARATOR = "::"
def _make_file_id(bucket: str, key: str) -> str:
return f"{bucket}{_ID_SEPARATOR}{key}"
def _split_file_id(file_id: str):
"""Split a composite file ID into (bucket, key). Raises ValueError if invalid."""
if _ID_SEPARATOR not in file_id:
raise ValueError(f"Invalid S3 file ID (missing separator): {file_id!r}")
bucket, key = file_id.split(_ID_SEPARATOR, 1)
return bucket, key
class S3Connector(BaseConnector):
"""Connector for Amazon S3 and S3-compatible object storage.
Uses HMAC (Access Key + Secret Key) authentication. Supports AWS S3,
MinIO, Cloudflare R2, and any service that speaks the S3 API.
Config dict keys:
access_key (str): Overrides AWS_ACCESS_KEY_ID.
secret_key (str): Overrides AWS_SECRET_ACCESS_KEY.
endpoint_url (str): Optional; overrides AWS_S3_ENDPOINT. Leave empty for AWS S3.
region (str): Optional; overrides AWS_REGION. Default: us-east-1.
bucket_names (list[str]): Buckets to ingest from. If empty, all accessible buckets are used.
connection_id (str): Connection identifier used for logging.
"""
CONNECTOR_TYPE = "aws_s3"
CONNECTOR_KIND = "bucket"
CHANGE_DETECTION = "timestamp"
CONNECTOR_NAME = "Amazon S3"
CONNECTOR_DESCRIPTION = "Add knowledge from Amazon S3 or any S3-compatible storage"
CONNECTOR_ICON = "aws-s3"
SECRET_CONFIG_KEYS = ("aws_secret_access_key",)
CLIENT_ID_ENV_VAR = "AWS_ACCESS_KEY_ID"
CLIENT_SECRET_ENV_VAR = "AWS_SECRET_ACCESS_KEY"
@classmethod
def is_available(cls, manager, user_id=None) -> bool:
# Enterprise/SaaS gate is IBM_AUTH_ENABLED, like the other bucket
# connectors (ibm_cos, azure_blob). OPENRAG_DEV_AWS_S3=true bypasses it
# for local dev (e.g. against MinIO; never in production).
return IBM_AUTH_ENABLED or is_dev_aws_s3_enabled()
@classmethod
def register_routes(cls, app) -> None:
from .api import s3_bucket_status, s3_configure, s3_defaults, s3_list_buckets
# Registered before generic /{connector_type}/... to avoid shadowing.
app.add_api_route(
"/connectors/aws_s3/defaults", s3_defaults, methods=["GET"], tags=["internal"]
)
app.add_api_route(
"/connectors/aws_s3/configure", s3_configure, methods=["POST"], tags=["internal"]
)
app.add_api_route(
"/connectors/aws_s3/{connection_id}/buckets",
s3_list_buckets,
methods=["GET"],
tags=["internal"],
)
app.add_api_route(
"/connectors/aws_s3/{connection_id}/bucket-status",
s3_bucket_status,
methods=["GET"],
tags=["internal"],
)
def get_client_id(self) -> str:
"""Return access key from config dict, or AWS_ACCESS_KEY_ID env var as fallback."""
val = self.config.get("access_key") or os.getenv("AWS_ACCESS_KEY_ID")
if val:
return val
raise ValueError(
"S3 credentials not set. Provide 'access_key' in the connector config "
"or set the AWS_ACCESS_KEY_ID environment variable."
)
def get_client_secret(self) -> str:
"""Return secret key from config dict, or AWS_SECRET_ACCESS_KEY env var as fallback."""
val = self.config.get("secret_key") or os.getenv("AWS_SECRET_ACCESS_KEY")
if val:
return val
raise ValueError(
"S3 credentials not set. Provide 'secret_key' in the connector config "
"or set the AWS_SECRET_ACCESS_KEY environment variable."
)
def __init__(self, config: dict[str, Any]):
if config is None:
config = {}
super().__init__(config)
self.bucket_names: list[str] = config.get("bucket_names") or []
self.prefix: str = config.get("prefix", "")
self.connection_id: str = config.get("connection_id", "default")
self._resource = None # Lazy-initialised on first use
self._client = None
def _get_resource(self):
if self._resource is None:
self._resource = create_s3_resource(self.config)
return self._resource
def _get_client(self):
if self._client is None:
self._client = create_s3_client(self.config)
return self._client
# ------------------------------------------------------------------
# BaseConnector abstract method implementations
# ------------------------------------------------------------------
async def authenticate(self) -> bool:
"""Validate credentials by listing accessible buckets."""
try:
resource = self._get_resource()
list(resource.buckets.all())
self._authenticated = True
logger.debug(f"S3 authenticated for connection {self.connection_id}")
return True
except Exception as exc:
logger.warning(f"S3 authentication failed: {exc}")
self._authenticated = False
return False
def _resolve_bucket_names(self) -> list[str]:
"""Return configured bucket names, or auto-discover all accessible buckets."""
if self.bucket_names:
return self.bucket_names
try:
resource = self._get_resource()
buckets = [b.name for b in resource.buckets.all()]
logger.debug("S3 auto-discovered %d bucket(s)", len(buckets))
return buckets
except Exception as exc:
logger.warning(f"S3 could not auto-discover buckets: {exc}")
return []
async def list_files(
self,
page_token: str | None = None,
max_files: int | None = None,
**kwargs,
) -> dict[str, Any]:
"""List objects across all configured (or auto-discovered) buckets.
Uses the boto3 resource API: Bucket.objects.all() handles pagination
internally so all objects are returned without manual continuation tokens.
Returns:
dict with keys:
"files": list of file dicts (id, name, bucket, size, modified_time)
"next_page_token": always None (SDK handles pagination internally)
"""
resource = self._get_resource()
files: list[dict[str, Any]] = []
bucket_names = self._resolve_bucket_names()
for bucket_name in bucket_names:
try:
bucket = resource.Bucket(bucket_name)
objects = (
bucket.objects.filter(Prefix=self.prefix)
if self.prefix
else bucket.objects.all()
)
for obj in objects:
if obj.key.endswith("/"):
continue
files.append(
{
"id": _make_file_id(bucket_name, obj.key),
"name": basename(obj.key) or obj.key,
"bucket": bucket_name,
"key": obj.key,
"size": obj.size,
"modified_time": obj.last_modified.isoformat()
if obj.last_modified
else None,
# Change detection compares this against the tag
# stored at ingest; it moves on every overwrite.
"etag": normalize_etag(getattr(obj, "e_tag", None)),
}
)
if max_files and len(files) >= max_files:
return {"files": files, "next_page_token": None}
except Exception as exc:
logger.error("Failed to list objects in S3 bucket: %s", exc)
continue
return {"files": files, "next_page_token": None}
async def get_file_content(self, file_id: str) -> ConnectorDocument:
"""Download an object from S3 and return a ConnectorDocument.
Args:
file_id: Composite ID in the form "<bucket>::<key>".
Returns:
ConnectorDocument with content bytes, ACL, and metadata.
"""
bucket_name, key = _split_file_id(file_id)
resource = self._get_resource()
response = resource.Object(bucket_name, key).get()
content: bytes = response["Body"].read()
last_modified: datetime = response.get("LastModified") or datetime.now(UTC)
size: int = response.get("ContentLength", len(content))
# Prefer filename extension over generic S3 content-type (often application/octet-stream)
raw_content_type = response.get("ContentType", "")
if raw_content_type and raw_content_type != "application/octet-stream":
mime_type: str = raw_content_type
else:
mime_type = mimetypes.guess_type(key)[0] or "application/octet-stream"
filename = basename(key) or key
acl = await self._extract_acl(bucket_name, key)
return ConnectorDocument(
id=file_id,
filename=filename,
mimetype=mime_type,
content=content,
source_url=f"s3://{bucket_name}/{key}",
acl=acl,
modified_time=last_modified,
created_time=last_modified, # S3 does not expose creation time
metadata={
"s3_bucket": bucket_name,
"s3_key": key,
"size": size,
# Persisted at ingest so a later sync can tell an overwritten
# object from an untouched one without trusting clocks.
CONTENT_ETAG_METADATA_KEY: normalize_etag(response.get("ETag")),
},
)
async def _extract_acl(self, bucket: str, key: str) -> DocumentACL:
"""Fetch object ACL from S3 and map it to DocumentACL.
Falls back to a minimal ACL on failure (e.g. ACLs disabled on the bucket).
"""
try:
client = self._get_client()
acl_response = client.get_object_acl(Bucket=bucket, Key=key)
owner_id: str = (
acl_response.get("Owner", {}).get("DisplayName")
or acl_response.get("Owner", {}).get("ID")
or ""
)
allowed_users: list[str] = []
for grant in acl_response.get("Grants", []):
grantee = grant.get("Grantee", {})
permission = grant.get("Permission", "")
if permission in ("FULL_CONTROL", "READ"):
user_id = (
grantee.get("DisplayName")
or grantee.get("ID")
or grantee.get("EmailAddress")
)
if user_id and user_id not in allowed_users:
allowed_users.append(user_id)
return DocumentACL(
owner=owner_id or None,
allowed_users=allowed_users,
allowed_groups=[],
)
except Exception as exc:
logger.warning("Could not fetch S3 object ACL, using fallback: %s", exc)
return DocumentACL(owner=None, allowed_users=[], allowed_groups=[])
# ------------------------------------------------------------------
# Webhook / subscription stubs (S3 event notifications are out of scope)
# ------------------------------------------------------------------
async def setup_subscription(self) -> str:
"""No-op: S3 event notifications are out of scope for this connector."""
return ""
async def handle_webhook(self, payload: dict[str, Any]) -> list[str]:
"""No-op: webhooks are not supported in this connector version."""
return []
def extract_webhook_channel_id(
self, payload: dict[str, Any], headers: dict[str, str]
) -> str | None:
return None
async def cleanup_subscription(self, subscription_id: str) -> bool:
"""No-op: no subscription to clean up."""
return True