Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

Refactor S3 and IBM COS connector APIs

Edwin Jose28694aba4e4d
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 53ded30db..6d2431f9b 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -652,469 +652,6 @@ async def connector_disconnect(
)
-# ---------------------------------------------------------------------------
-# IBM COS-specific endpoints
-# ---------------------------------------------------------------------------
-
-class IBMCOSConfigureBody(BaseModel):
- auth_mode: str # "iam" or "hmac"
- endpoint: str
- # IAM fields
- api_key: Optional[str] = None
- service_instance_id: Optional[str] = None
- auth_endpoint: Optional[str] = None
- # HMAC fields
- hmac_access_key: Optional[str] = None
- hmac_secret_key: Optional[str] = None
- # Optional bucket selection
- bucket_names: Optional[List[str]] = None
- # Optional: update an existing connection
- connection_id: Optional[str] = None
-
-
-async def ibm_cos_defaults(
- connector_service=Depends(get_connector_service),
- user: User = Depends(get_current_user),
-):
- """Return current IBM COS env-var defaults for pre-filling the config dialog.
-
- Sensitive values (API key, HMAC secret) are masked — only whether they are
- set is returned, not the actual values.
- """
- import os
-
- api_key = os.getenv("IBM_COS_API_KEY", "")
- service_instance_id = os.getenv("IBM_COS_SERVICE_INSTANCE_ID", "")
- endpoint = os.getenv("IBM_COS_ENDPOINT", "")
- hmac_access_key = os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID", "")
- hmac_secret_key = os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY", "")
- disable_iam = os.getenv("OPENRAG_IBM_COS_IAM_UI", "").lower() not in ("1", "true", "yes")
-
- # Try to read existing connection config for this user too
- connections = await connector_service.connection_manager.list_connections(
- user_id=user.user_id, connector_type="ibm_cos"
- )
- conn_config = {}
- if connections:
- conn_config = connections[0].config or {}
-
- def _pick(conn_key, env_val):
- """Prefer connection config value over env var."""
- return conn_config.get(conn_key) or env_val
-
- return JSONResponse({
- "api_key_set": bool(api_key or conn_config.get("api_key")),
- "service_instance_id": _pick("service_instance_id", service_instance_id),
- "endpoint": _pick("endpoint_url", endpoint),
- "hmac_access_key_set": bool(hmac_access_key or conn_config.get("hmac_access_key")),
- "hmac_secret_key_set": bool(hmac_secret_key or conn_config.get("hmac_secret_key")),
- # Return which auth mode was previously used; default to hmac when IAM is disabled
- "auth_mode": conn_config.get("auth_mode", "hmac" if (disable_iam or not (api_key or conn_config.get("api_key"))) else "iam"),
- "disable_iam": disable_iam,
- # Return bucket_names from existing connection (if any)
- "bucket_names": conn_config.get("bucket_names", []),
- # Return connection_id if an existing connection exists
- "connection_id": connections[0].connection_id if connections else None,
- })
-
-
-async def ibm_cos_configure(
- body: IBMCOSConfigureBody,
- connector_service=Depends(get_connector_service),
- user: User = Depends(get_current_user),
-):
- """Create or update an IBM COS connection with explicit credentials.
-
- Tests the credentials by listing buckets, then persists the connection.
- Credentials are stored in the connection config dict (not env vars) so
- the connector works even without system-level env vars.
- """
- import os
- from connectors.ibm_cos.auth import create_ibm_cos_client, create_ibm_cos_resource
-
- # Build the config dict that will be stored in the connection
- conn_config: dict = {
- "auth_mode": body.auth_mode,
- "endpoint_url": body.endpoint,
- }
-
- if body.auth_mode == "iam":
- # Resolve: use supplied value, fall back to env var, fall back to existing connection
- api_key = body.api_key or os.getenv("IBM_COS_API_KEY")
- svc_id = body.service_instance_id or os.getenv("IBM_COS_SERVICE_INSTANCE_ID")
-
- # If still empty, pull from existing connection config
- existing_connections = await connector_service.connection_manager.list_connections(
- user_id=user.user_id, connector_type="ibm_cos"
- )
- if not api_key and existing_connections:
- api_key = existing_connections[0].config.get("api_key")
- if not svc_id and existing_connections:
- svc_id = existing_connections[0].config.get("service_instance_id")
-
- if not api_key or not svc_id:
- return JSONResponse(
- {"error": "IAM mode requires api_key and service_instance_id"},
- status_code=400,
- )
- conn_config["api_key"] = api_key
- conn_config["service_instance_id"] = svc_id
- if body.auth_endpoint:
- conn_config["auth_endpoint"] = body.auth_endpoint
- else:
- # HMAC mode
- hmac_access = body.hmac_access_key or os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID")
- hmac_secret = body.hmac_secret_key or os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY")
-
- existing_connections = await connector_service.connection_manager.list_connections(
- user_id=user.user_id, connector_type="ibm_cos"
- )
- if not hmac_access and existing_connections:
- hmac_access = existing_connections[0].config.get("hmac_access_key")
- if not hmac_secret and existing_connections:
- hmac_secret = existing_connections[0].config.get("hmac_secret_key")
-
- if not hmac_access or not hmac_secret:
- return JSONResponse(
- {"error": "HMAC mode requires hmac_access_key and hmac_secret_key"},
- status_code=400,
- )
- conn_config["hmac_access_key"] = hmac_access
- conn_config["hmac_secret_key"] = hmac_secret
-
- if body.bucket_names is not None:
- conn_config["bucket_names"] = body.bucket_names
-
- # Test credentials — IAM uses client (avoids ibm_botocore discovery-call bug),
- # HMAC uses resource (S3-compatible, works with MinIO).
- try:
- if conn_config.get("auth_mode", "iam") == "hmac":
- cos = create_ibm_cos_resource(conn_config)
- list(cos.buckets.all())
- else:
- cos = create_ibm_cos_client(conn_config)
- cos.list_buckets()
- except Exception as exc:
- logger.exception("Failed to connect to IBM COS during credential test.")
- return JSONResponse(
- {"error": "Could not connect to IBM COS with the provided configuration."},
- status_code=400,
- )
-
- # Persist: update existing connection or create a new one
- if body.connection_id:
- existing = await connector_service.connection_manager.get_connection(body.connection_id)
- if existing and existing.user_id == user.user_id:
- await connector_service.connection_manager.update_connection(
- connection_id=body.connection_id,
- config=conn_config,
- )
- # Evict cached connector so next call gets a fresh instance
- connector_service.connection_manager.active_connectors.pop(body.connection_id, None)
- return JSONResponse({"connection_id": body.connection_id, "status": "connected"})
-
- # Create a fresh connection
- connection_id = await connector_service.connection_manager.create_connection(
- connector_type="ibm_cos",
- name="IBM Cloud Object Storage",
- config=conn_config,
- user_id=user.user_id,
- )
- return JSONResponse({"connection_id": connection_id, "status": "connected"})
-
-
-async def ibm_cos_list_buckets(
- connection_id: str,
- connector_service=Depends(get_connector_service),
- user: User = Depends(get_current_user),
-):
- """List all buckets accessible with the stored IBM COS credentials."""
- from connectors.ibm_cos.auth import create_ibm_cos_client, create_ibm_cos_resource
-
- connection = await connector_service.connection_manager.get_connection(connection_id)
- if not connection or connection.user_id != user.user_id:
- return JSONResponse({"error": "Connection not found"}, status_code=404)
- if connection.connector_type != "ibm_cos":
- return JSONResponse({"error": "Not an IBM COS connection"}, status_code=400)
-
- try:
- cfg = connection.config
- if cfg.get("auth_mode", "iam") == "hmac":
- cos = create_ibm_cos_resource(cfg)
- buckets = [b.name for b in cos.buckets.all()]
- else:
- cos = create_ibm_cos_client(cfg)
- buckets = [b["Name"] for b in cos.list_buckets().get("Buckets", [])]
- return JSONResponse({"buckets": buckets})
- except Exception as exc:
- return JSONResponse({"error": "Failed to list buckets"}, status_code=500)
-
-
-
-async def ibm_cos_bucket_status(
- connection_id: str,
- connector_service=Depends(get_connector_service),
- session_manager=Depends(get_session_manager),
- user: User = Depends(get_current_user),
-):
- """Return all buckets for an IBM COS connection with their ingestion status.
-
- Each entry includes the bucket name, whether it has been ingested (is_synced),
- and the count of indexed documents from that bucket.
- """
- from connectors.ibm_cos.auth import create_ibm_cos_resource
-
- connection = await connector_service.connection_manager.get_connection(connection_id)
- if not connection or connection.user_id != user.user_id:
- return JSONResponse({"error": "Connection not found"}, status_code=404)
- if connection.connector_type != "ibm_cos":
- return JSONResponse({"error": "Not an IBM COS connection"}, status_code=400)
-
- # 1. List all buckets from COS
- try:
- cfg = connection.config
- cos = create_ibm_cos_resource(cfg)
- all_buckets = [b.name for b in cos.buckets.all()]
- except Exception as exc:
- logger.exception("Failed to list IBM COS buckets for connection %s", connection_id)
- return JSONResponse({"error": f"Failed to list buckets: {exc}"}, status_code=500)
-
- # 2. Count indexed documents per bucket from OpenSearch
- ingested_counts: dict = {}
- try:
- opensearch_client = session_manager.get_user_opensearch_client(
- user.user_id, user.jwt_token
- )
- query_body = {
- "size": 0,
- "query": {"term": {"connector_type": "ibm_cos"}},
- "aggs": {
- "doc_ids": {
- "terms": {"field": "document_id", "size": 50000}
- }
- },
- }
- index_name = get_index_name(user.user_id)
- os_resp = opensearch_client.search(index=index_name, body=query_body)
- for bucket_entry in os_resp.get("aggregations", {}).get("doc_ids", {}).get("buckets", []):
- doc_id = bucket_entry["key"]
- if "::" in doc_id:
- bucket_name = doc_id.split("::")[0]
- ingested_counts[bucket_name] = ingested_counts.get(bucket_name, 0) + 1
- except Exception:
- pass # OpenSearch unavailable — show zero counts
-
- result = [
- {
- "name": bucket,
- "ingested_count": ingested_counts.get(bucket, 0),
- "is_synced": ingested_counts.get(bucket, 0) > 0,
- }
- for bucket in all_buckets
- ]
- return JSONResponse({"buckets": result})
-
-
-# ---------------------------------------------------------------------------
-# Amazon S3 / S3-compatible endpoints
-# ---------------------------------------------------------------------------
-
-class S3ConfigureBody(BaseModel):
- access_key: Optional[str] = None
- secret_key: Optional[str] = None
- endpoint_url: Optional[str] = None
- region: Optional[str] = None
- bucket_names: Optional[List[str]] = None
- connection_id: Optional[str] = None
-
-
-async def s3_defaults(
- connector_service=Depends(get_connector_service),
- user: User = Depends(get_current_user),
-):
- """Return current S3 env-var defaults for pre-filling the config dialog.
-
- Sensitive values (secret key) are masked — only whether they are set is returned.
- """
- import os
-
- access_key = os.getenv("AWS_ACCESS_KEY_ID", "")
- secret_key = os.getenv("AWS_SECRET_ACCESS_KEY", "")
- endpoint_url = os.getenv("AWS_S3_ENDPOINT", "")
- region = os.getenv("AWS_REGION", "")
-
- connections = await connector_service.connection_manager.list_connections(
- user_id=user.user_id, connector_type="aws_s3"
- )
- conn_config = {}
- if connections:
- conn_config = connections[0].config or {}
-
- def _pick(conn_key, env_val):
- return conn_config.get(conn_key) or env_val
-
- return JSONResponse({
- "access_key_set": bool(access_key or conn_config.get("access_key")),
- "secret_key_set": bool(secret_key or conn_config.get("secret_key")),
- "endpoint": _pick("endpoint_url", endpoint_url),
- "region": _pick("region", region),
- "bucket_names": conn_config.get("bucket_names", []),
- "connection_id": connections[0].connection_id if connections else None,
- })
-
-
-async def s3_configure(
- body: S3ConfigureBody,
- connector_service=Depends(get_connector_service),
- user: User = Depends(get_current_user),
-):
- """Create or update an S3 connection with explicit credentials.
-
- Tests the credentials by listing buckets, then persists the connection.
- """
- import os
- from connectors.aws_s3.auth import create_s3_resource
-
- access_key = body.access_key or os.getenv("AWS_ACCESS_KEY_ID")
- secret_key = body.secret_key or os.getenv("AWS_SECRET_ACCESS_KEY")
-
- # Fall back to existing connection config
- existing_connections = await connector_service.connection_manager.list_connections(
- user_id=user.user_id, connector_type="aws_s3"
- )
- if not access_key and existing_connections:
- access_key = existing_connections[0].config.get("access_key")
- if not secret_key and existing_connections:
- secret_key = existing_connections[0].config.get("secret_key")
-
- if not access_key or not secret_key:
- return JSONResponse(
- {"error": "access_key and secret_key are required"},
- status_code=400,
- )
-
- conn_config: dict = {
- "access_key": access_key.strip(),
- "secret_key": secret_key.strip(),
- }
- if body.endpoint_url:
- conn_config["endpoint_url"] = body.endpoint_url.strip()
- if body.region:
- conn_config["region"] = body.region.strip()
- if body.bucket_names is not None:
- conn_config["bucket_names"] = body.bucket_names
-
- # Test credentials
- try:
- s3 = create_s3_resource(conn_config)
- list(s3.buckets.all())
- except Exception as exc:
- return JSONResponse(
- {"error": f"Could not connect to S3: {exc}"},
- status_code=400,
- )
-
- # Persist: update existing connection or create a new one
- if body.connection_id:
- existing = await connector_service.connection_manager.get_connection(body.connection_id)
- if existing and existing.user_id == user.user_id:
- await connector_service.connection_manager.update_connection(
- connection_id=body.connection_id,
- config=conn_config,
- )
- connector_service.connection_manager.active_connectors.pop(body.connection_id, None)
- return JSONResponse({"connection_id": body.connection_id, "status": "connected"})
-
- connection_id = await connector_service.connection_manager.create_connection(
- connector_type="aws_s3",
- name="Amazon S3",
- config=conn_config,
- user_id=user.user_id,
- )
- return JSONResponse({"connection_id": connection_id, "status": "connected"})
-
-
-async def s3_list_buckets(
- connection_id: str,
- connector_service=Depends(get_connector_service),
- user: User = Depends(get_current_user),
-):
- """List all buckets accessible with the stored S3 credentials."""
- from connectors.aws_s3.auth import create_s3_resource
-
- connection = await connector_service.connection_manager.get_connection(connection_id)
- if not connection or connection.user_id != user.user_id:
- return JSONResponse({"error": "Connection not found"}, status_code=404)
- if connection.connector_type != "aws_s3":
- return JSONResponse({"error": "Not an S3 connection"}, status_code=400)
-
- try:
- s3 = create_s3_resource(connection.config)
- buckets = [b.name for b in s3.buckets.all()]
- return JSONResponse({"buckets": buckets})
- except Exception as exc:
- return JSONResponse({"error": f"Failed to list buckets: {exc}"}, status_code=500)
-
-
-async def s3_bucket_status(
- connection_id: str,
- connector_service=Depends(get_connector_service),
- session_manager=Depends(get_session_manager),
- user: User = Depends(get_current_user),
-):
- """Return all buckets for an S3 connection with their ingestion status."""
- from connectors.aws_s3.auth import create_s3_resource
-
- connection = await connector_service.connection_manager.get_connection(connection_id)
- if not connection or connection.user_id != user.user_id:
- return JSONResponse({"error": "Connection not found"}, status_code=404)
- if connection.connector_type != "aws_s3":
- return JSONResponse({"error": "Not an S3 connection"}, status_code=400)
-
- # 1. List all buckets from S3
- try:
- s3 = create_s3_resource(connection.config)
- all_buckets = [b.name for b in s3.buckets.all()]
- except Exception as exc:
- logger.exception("Failed to list buckets from S3 for connection %s", connection_id)
- return JSONResponse({"error": "Failed to list buckets"}, status_code=500)
-
- # 2. Count indexed documents per bucket from OpenSearch
- ingested_counts: dict = {}
- try:
- opensearch_client = session_manager.get_user_opensearch_client(
- user.user_id, user.jwt_token
- )
- query_body = {
- "size": 0,
- "query": {"term": {"connector_type": "aws_s3"}},
- "aggs": {
- "doc_ids": {
- "terms": {"field": "document_id", "size": 50000}
- }
- },
- }
- index_name = get_index_name(user.user_id)
- os_resp = opensearch_client.search(index=index_name, body=query_body)
- for bucket_entry in os_resp.get("aggregations", {}).get("doc_ids", {}).get("buckets", []):
- doc_id = bucket_entry["key"]
- if "::" in doc_id:
- bucket_name = doc_id.split("::")[0]
- ingested_counts[bucket_name] = ingested_counts.get(bucket_name, 0) + 1
- except Exception:
- pass # OpenSearch unavailable — show zero counts
-
- result = [
- {
- "name": bucket,
- "ingested_count": ingested_counts.get(bucket, 0),
- "is_synced": ingested_counts.get(bucket, 0) > 0,
- }
- for bucket in all_buckets
- ]
- return JSONResponse({"buckets": result})
-
-
# ---------------------------------------------------------------------------
async def sync_all_connectors(
diff --git a/src/connectors/aws_s3/__init__.py b/src/connectors/aws_s3/__init__.py
index ee9a5ccab..32c63a878 100644
--- a/src/connectors/aws_s3/__init__.py
+++ b/src/connectors/aws_s3/__init__.py
@@ -1,5 +1,19 @@
"""Amazon S3 / S3-compatible connector for OpenRAG."""
from .connector import S3Connector
+from .models import S3ConfigureBody
+from .api import (
+ s3_defaults,
+ s3_configure,
+ s3_list_buckets,
+ s3_bucket_status,
+)
-__all__ = ["S3Connector"]
+__all__ = [
+ "S3Connector",
+ "S3ConfigureBody",
+ "s3_defaults",
+ "s3_configure",
+ "s3_list_buckets",
+ "s3_bucket_status",
+]
diff --git a/src/connectors/aws_s3/api.py b/src/connectors/aws_s3/api.py
new file mode 100644
index 000000000..2612ca698
--- /dev/null
+++ b/src/connectors/aws_s3/api.py
@@ -0,0 +1,173 @@
+"""FastAPI route handlers for AWS S3-specific endpoints."""
+
+import os
+
+from fastapi import Depends
+from fastapi.responses import JSONResponse
+
+from config.settings import get_index_name
+from dependencies import get_connector_service, get_session_manager, get_current_user
+from session_manager import User
+from utils.logging_config import get_logger
+
+from .auth import create_s3_resource
+from .models import S3ConfigureBody
+from .support import build_s3_config
+
+logger = get_logger(__name__)
+
+
+async def s3_defaults(
+ connector_service=Depends(get_connector_service),
+ user: User = Depends(get_current_user),
+):
+ """Return current S3 env-var defaults for pre-filling the config dialog.
+
+ Sensitive values (secret key) are masked — only whether they are set is returned.
+ """
+ access_key = os.getenv("AWS_ACCESS_KEY_ID", "")
+ secret_key = os.getenv("AWS_SECRET_ACCESS_KEY", "")
+ endpoint_url = os.getenv("AWS_S3_ENDPOINT", "")
+ region = os.getenv("AWS_REGION", "")
+
+ connections = await connector_service.connection_manager.list_connections(
+ user_id=user.user_id, connector_type="aws_s3"
+ )
+ conn_config = connections[0].config or {} if connections else {}
+
+ def _pick(conn_key, env_val):
+ return conn_config.get(conn_key) or env_val
+
+ return JSONResponse({
+ "access_key_set": bool(access_key or conn_config.get("access_key")),
+ "secret_key_set": bool(secret_key or conn_config.get("secret_key")),
+ "endpoint": _pick("endpoint_url", endpoint_url),
+ "region": _pick("region", region),
+ "bucket_names": conn_config.get("bucket_names", []),
+ "connection_id": connections[0].connection_id if connections else None,
+ })
+
+
+async def s3_configure(
+ body: S3ConfigureBody,
+ connector_service=Depends(get_connector_service),
+ user: User = Depends(get_current_user),
+):
+ """Create or update an S3 connection with explicit credentials.
+
+ Tests the credentials by listing buckets, then persists the connection.
+ """
+ existing_connections = await connector_service.connection_manager.list_connections(
+ user_id=user.user_id, connector_type="aws_s3"
+ )
+ existing_config = existing_connections[0].config if existing_connections else {}
+
+ conn_config, error = build_s3_config(body, existing_config)
+ if error:
+ return JSONResponse({"error": error}, status_code=400)
+
+ # Test credentials
+ try:
+ s3 = create_s3_resource(conn_config)
+ list(s3.buckets.all())
+ except Exception as exc:
+ return JSONResponse(
+ {"error": f"Could not connect to S3: {exc}"},
+ status_code=400,
+ )
+
+ # Persist: update existing connection or create a new one
+ if body.connection_id:
+ existing = await connector_service.connection_manager.get_connection(body.connection_id)
+ if existing and existing.user_id == user.user_id:
+ await connector_service.connection_manager.update_connection(
+ connection_id=body.connection_id,
+ config=conn_config,
+ )
+ connector_service.connection_manager.active_connectors.pop(body.connection_id, None)
+ return JSONResponse({"connection_id": body.connection_id, "status": "connected"})
+
+ connection_id = await connector_service.connection_manager.create_connection(
+ connector_type="aws_s3",
+ name="Amazon S3",
+ config=conn_config,
+ user_id=user.user_id,
+ )
+ return JSONResponse({"connection_id": connection_id, "status": "connected"})
+
+
+async def s3_list_buckets(
+ connection_id: str,
+ connector_service=Depends(get_connector_service),
+ user: User = Depends(get_current_user),
+):
+ """List all buckets accessible with the stored S3 credentials."""
+ connection = await connector_service.connection_manager.get_connection(connection_id)
+ if not connection or connection.user_id != user.user_id:
+ return JSONResponse({"error": "Connection not found"}, status_code=404)
+ if connection.connector_type != "aws_s3":
+ return JSONResponse({"error": "Not an S3 connection"}, status_code=400)
+
+ try:
+ s3 = create_s3_resource(connection.config)
+ buckets = [b.name for b in s3.buckets.all()]
+ return JSONResponse({"buckets": buckets})
+ except Exception as exc:
+ return JSONResponse({"error": f"Failed to list buckets: {exc}"}, status_code=500)
+
+
+async def s3_bucket_status(
+ connection_id: str,
+ connector_service=Depends(get_connector_service),
+ session_manager=Depends(get_session_manager),
+ user: User = Depends(get_current_user),
+):
+ """Return all buckets for an S3 connection with their ingestion status."""
+ connection = await connector_service.connection_manager.get_connection(connection_id)
+ if not connection or connection.user_id != user.user_id:
+ return JSONResponse({"error": "Connection not found"}, status_code=404)
+ if connection.connector_type != "aws_s3":
+ return JSONResponse({"error": "Not an S3 connection"}, status_code=400)
+
+ # 1. List all buckets from S3
+ try:
+ s3 = create_s3_resource(connection.config)
+ all_buckets = [b.name for b in s3.buckets.all()]
+ except Exception as exc:
+ logger.exception("Failed to list buckets from S3 for connection %s", connection_id)
+ return JSONResponse({"error": "Failed to list buckets"}, status_code=500)
+
+ # 2. Count indexed documents per bucket from OpenSearch
+ ingested_counts: dict = {}
+ try:
+ opensearch_client = session_manager.get_user_opensearch_client(
+ user.user_id, user.jwt_token
+ )
+ query_body = {
+ "size": 0,
+ "query": {"term": {"connector_type": "aws_s3"}},
+ "aggs": {
+ "doc_ids": {
+ "terms": {"field": "document_id", "size": 50000}
+ }
+ },
+ }
+ index_name = get_index_name()
+ os_resp = opensearch_client.search(index=index_name, body=query_body)
+ for bucket_entry in os_resp.get("aggregations", {}).get("doc_ids", {}).get("buckets", []):
+ doc_id = bucket_entry["key"]
+ if "::" in doc_id:
+ bucket_name = doc_id.split("::")[0]
+ ingested_counts[bucket_name] = ingested_counts.get(bucket_name, 0) + 1
+ except Exception:
+ pass # OpenSearch unavailable — show zero counts
+
+ result = [
+ {
+ "name": bucket,
+ "ingested_count": ingested_counts.get(bucket, 0),
+ "is_synced": ingested_counts.get(bucket, 0) > 0,
+ }
+ for bucket in all_buckets
+ ]
+ return JSONResponse({"buckets": result})
diff --git a/src/connectors/aws_s3/models.py b/src/connectors/aws_s3/models.py
new file mode 100644
index 000000000..582111bd2
--- /dev/null
+++ b/src/connectors/aws_s3/models.py
@@ -0,0 +1,13 @@
+"""Pydantic request/response models for AWS S3 API endpoints."""
+
+from typing import List, Optional
+from pydantic import BaseModel
+
+
+class S3ConfigureBody(BaseModel):
+ access_key: Optional[str] = None
+ secret_key: Optional[str] = None
+ endpoint_url: Optional[str] = None
+ region: Optional[str] = None
+ bucket_names: Optional[List[str]] = None
+ connection_id: Optional[str] = None
diff --git a/src/connectors/aws_s3/support.py b/src/connectors/aws_s3/support.py
new file mode 100644
index 000000000..ccaa069b7
--- /dev/null
+++ b/src/connectors/aws_s3/support.py
@@ -0,0 +1,51 @@
+"""Support helpers for AWS S3 API endpoints.
+
+Contains pure (non-async) business logic for credential resolution and
+config dict construction, keeping the route handlers thin.
+"""
+
+import os
+from typing import Dict, Optional, Tuple
+
+from .models import S3ConfigureBody
+
+
+def build_s3_config(
+ body: S3ConfigureBody,
+ existing_config: Dict,
+) -> Tuple[Dict, Optional[str]]:
+ """Resolve S3 credentials and build the connection config dict.
+
+ Resolution order for each credential: request body → environment variable
+ → existing connection config.
+
+ Returns:
+ (conn_config, None) on success
+ ({}, error_message) on validation failure
+ """
+ access_key = (
+ body.access_key
+ or os.getenv("AWS_ACCESS_KEY_ID")
+ or existing_config.get("access_key")
+ )
+ secret_key = (
+ body.secret_key
+ or os.getenv("AWS_SECRET_ACCESS_KEY")
+ or existing_config.get("secret_key")
+ )
+
+ if not access_key or not secret_key:
+ return {}, "access_key and secret_key are required"
+
+ conn_config: Dict = {
+ "access_key": access_key.strip(),
+ "secret_key": secret_key.strip(),
+ }
+ if body.endpoint_url:
+ conn_config["endpoint_url"] = body.endpoint_url.strip()
+ if body.region:
+ conn_config["region"] = body.region.strip()
+ if body.bucket_names is not None:
+ conn_config["bucket_names"] = body.bucket_names
+
+ return conn_config, None
diff --git a/src/connectors/ibm_cos/__init__.py b/src/connectors/ibm_cos/__init__.py
index 32f19defc..5bf01ad43 100644
--- a/src/connectors/ibm_cos/__init__.py
+++ b/src/connectors/ibm_cos/__init__.py
@@ -1,3 +1,17 @@
from .connector import IBMCOSConnector
+from .models import IBMCOSConfigureBody
+from .api import (
+ ibm_cos_defaults,
+ ibm_cos_configure,
+ ibm_cos_list_buckets,
+ ibm_cos_bucket_status,
+)
-__all__ = ["IBMCOSConnector"]
+__all__ = [
+ "IBMCOSConnector",
+ "IBMCOSConfigureBody",
+ "ibm_cos_defaults",
+ "ibm_cos_configure",
+ "ibm_cos_list_buckets",
+ "ibm_cos_bucket_status",
+]
diff --git a/src/connectors/ibm_cos/api.py b/src/connectors/ibm_cos/api.py
new file mode 100644
index 000000000..e6793f2f0
--- /dev/null
+++ b/src/connectors/ibm_cos/api.py
@@ -0,0 +1,201 @@
+"""FastAPI route handlers for IBM COS-specific endpoints."""
+
+import os
+
+from fastapi import Depends
+from fastapi.responses import JSONResponse
+
+from config.settings import get_index_name
+from dependencies import get_connector_service, get_session_manager, get_current_user
+from session_manager import User
+from utils.logging_config import get_logger
+
+from .auth import create_ibm_cos_client, create_ibm_cos_resource
+from .models import IBMCOSConfigureBody
+from .support import build_ibm_cos_config
+
+logger = get_logger(__name__)
+
+
+async def ibm_cos_defaults(
+ connector_service=Depends(get_connector_service),
+ user: User = Depends(get_current_user),
+):
+ """Return current IBM COS env-var defaults for pre-filling the config dialog.
+
+ Sensitive values (API key, HMAC secret) are masked — only whether they are
+ set is returned, not the actual values.
+ """
+ api_key = os.getenv("IBM_COS_API_KEY", "")
+ service_instance_id = os.getenv("IBM_COS_SERVICE_INSTANCE_ID", "")
+ endpoint = os.getenv("IBM_COS_ENDPOINT", "")
+ hmac_access_key = os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID", "")
+ hmac_secret_key = os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY", "")
+ disable_iam = os.getenv("OPENRAG_IBM_COS_IAM_UI", "").lower() not in ("1", "true", "yes")
+
+ connections = await connector_service.connection_manager.list_connections(
+ user_id=user.user_id, connector_type="ibm_cos"
+ )
+ conn_config = connections[0].config or {} if connections else {}
+
+ def _pick(conn_key, env_val):
+ """Prefer connection config value over env var."""
+ return conn_config.get(conn_key) or env_val
+
+ return JSONResponse({
+ "api_key_set": bool(api_key or conn_config.get("api_key")),
+ "service_instance_id": _pick("service_instance_id", service_instance_id),
+ "endpoint": _pick("endpoint_url", endpoint),
+ "hmac_access_key_set": bool(hmac_access_key or conn_config.get("hmac_access_key")),
+ "hmac_secret_key_set": bool(hmac_secret_key or conn_config.get("hmac_secret_key")),
+ "auth_mode": conn_config.get(
+ "auth_mode",
+ "hmac" if (disable_iam or not (api_key or conn_config.get("api_key"))) else "iam",
+ ),
+ "disable_iam": disable_iam,
+ "bucket_names": conn_config.get("bucket_names", []),
+ "connection_id": connections[0].connection_id if connections else None,
+ })
+
+
+async def ibm_cos_configure(
+ body: IBMCOSConfigureBody,
+ connector_service=Depends(get_connector_service),
+ user: User = Depends(get_current_user),
+):
+ """Create or update an IBM COS connection with explicit credentials.
+
+ Tests the credentials by listing buckets, then persists the connection.
+ Credentials are stored in the connection config dict (not env vars) so
+ the connector works even without system-level env vars.
+ """
+ existing_connections = await connector_service.connection_manager.list_connections(
+ user_id=user.user_id, connector_type="ibm_cos"
+ )
+ existing_config = existing_connections[0].config if existing_connections else {}
+
+ conn_config, error = build_ibm_cos_config(body, existing_config)
+ if error:
+ return JSONResponse({"error": error}, status_code=400)
+
+ # Test credentials — IAM uses client (avoids ibm_botocore discovery-call bug),
+ # HMAC uses resource (S3-compatible, works with MinIO).
+ try:
+ if conn_config.get("auth_mode", "iam") == "hmac":
+ cos = create_ibm_cos_resource(conn_config)
+ list(cos.buckets.all())
+ else:
+ cos = create_ibm_cos_client(conn_config)
+ cos.list_buckets()
+ except Exception:
+ logger.exception("Failed to connect to IBM COS during credential test.")
+ return JSONResponse(
+ {"error": "Could not connect to IBM COS with the provided configuration."},
+ status_code=400,
+ )
+
+ # Persist: update existing connection or create a new one
+ if body.connection_id:
+ existing = await connector_service.connection_manager.get_connection(body.connection_id)
+ if existing and existing.user_id == user.user_id:
+ await connector_service.connection_manager.update_connection(
+ connection_id=body.connection_id,
+ config=conn_config,
+ )
+ connector_service.connection_manager.active_connectors.pop(body.connection_id, None)
+ return JSONResponse({"connection_id": body.connection_id, "status": "connected"})
+
+ connection_id = await connector_service.connection_manager.create_connection(
+ connector_type="ibm_cos",
+ name="IBM Cloud Object Storage",
+ config=conn_config,
+ user_id=user.user_id,
+ )
+ return JSONResponse({"connection_id": connection_id, "status": "connected"})
+
+
+async def ibm_cos_list_buckets(
+ connection_id: str,
+ connector_service=Depends(get_connector_service),
+ user: User = Depends(get_current_user),
+):
+ """List all buckets accessible with the stored IBM COS credentials."""
+ connection = await connector_service.connection_manager.get_connection(connection_id)
+ if not connection or connection.user_id != user.user_id:
+ return JSONResponse({"error": "Connection not found"}, status_code=404)
+ if connection.connector_type != "ibm_cos":
+ return JSONResponse({"error": "Not an IBM COS connection"}, status_code=400)
+
+ try:
+ cfg = connection.config
+ if cfg.get("auth_mode", "iam") == "hmac":
+ cos = create_ibm_cos_resource(cfg)
+ buckets = [b.name for b in cos.buckets.all()]
+ else:
+ cos = create_ibm_cos_client(cfg)
+ buckets = [b["Name"] for b in cos.list_buckets().get("Buckets", [])]
+ return JSONResponse({"buckets": buckets})
+ except Exception as exc:
+ return JSONResponse({"error": "Failed to list buckets"}, status_code=500)
+
+
+async def ibm_cos_bucket_status(
+ connection_id: str,
+ connector_service=Depends(get_connector_service),
+ session_manager=Depends(get_session_manager),
+ user: User = Depends(get_current_user),
+):
+ """Return all buckets for an IBM COS connection with their ingestion status.
+
+ Each entry includes the bucket name, whether it has been ingested (is_synced),
+ and the count of indexed documents from that bucket.
+ """
+ connection = await connector_service.connection_manager.get_connection(connection_id)
+ if not connection or connection.user_id != user.user_id:
+ return JSONResponse({"error": "Connection not found"}, status_code=404)
+ if connection.connector_type != "ibm_cos":
+ return JSONResponse({"error": "Not an IBM COS connection"}, status_code=400)
+
+ # 1. List all buckets from COS
+ try:
+ cfg = connection.config
+ cos = create_ibm_cos_resource(cfg)
+ all_buckets = [b.name for b in cos.buckets.all()]
+ except Exception as exc:
+ logger.exception("Failed to list IBM COS buckets for connection %s", connection_id)
+ return JSONResponse({"error": f"Failed to list buckets: {exc}"}, status_code=500)
+
+ # 2. Count indexed documents per bucket from OpenSearch
+ ingested_counts: dict = {}
+ try:
+ opensearch_client = session_manager.get_user_opensearch_client(
+ user.user_id, user.jwt_token
+ )
+ query_body = {
+ "size": 0,
+ "query": {"term": {"connector_type": "ibm_cos"}},
+ "aggs": {
+ "doc_ids": {
+ "terms": {"field": "document_id", "size": 50000}
+ }
+ },
+ }
+ index_name = get_index_name()
+ os_resp = opensearch_client.search(index=index_name, body=query_body)
+ for bucket_entry in os_resp.get("aggregations", {}).get("doc_ids", {}).get("buckets", []):
+ doc_id = bucket_entry["key"]
+ if "::" in doc_id:
+ bucket_name = doc_id.split("::")[0]
+ ingested_counts[bucket_name] = ingested_counts.get(bucket_name, 0) + 1
+ except Exception:
+ pass # OpenSearch unavailable — show zero counts
+
+ result = [
+ {
+ "name": bucket,
+ "ingested_count": ingested_counts.get(bucket, 0),
+ "is_synced": ingested_counts.get(bucket, 0) > 0,
+ }
+ for bucket in all_buckets
+ ]
+ return JSONResponse({"buckets": result})
diff --git a/src/connectors/ibm_cos/models.py b/src/connectors/ibm_cos/models.py
new file mode 100644
index 000000000..59f2324f9
--- /dev/null
+++ b/src/connectors/ibm_cos/models.py
@@ -0,0 +1,20 @@
+"""Pydantic request/response models for IBM COS API endpoints."""
+
+from typing import List, Optional
+from pydantic import BaseModel
+
+
+class IBMCOSConfigureBody(BaseModel):
+ auth_mode: str # "iam" or "hmac"
+ endpoint: str
+ # IAM fields
+ api_key: Optional[str] = None
+ service_instance_id: Optional[str] = None
+ auth_endpoint: Optional[str] = None
+ # HMAC fields
+ hmac_access_key: Optional[str] = None
+ hmac_secret_key: Optional[str] = None
+ # Optional bucket selection
+ bucket_names: Optional[List[str]] = None
+ # Optional: update an existing connection
+ connection_id: Optional[str] = None
diff --git a/src/connectors/ibm_cos/support.py b/src/connectors/ibm_cos/support.py
new file mode 100644
index 000000000..25e5e5bcb
--- /dev/null
+++ b/src/connectors/ibm_cos/support.py
@@ -0,0 +1,68 @@
+"""Support helpers for IBM COS API endpoints.
+
+Contains pure (non-async) business logic for credential resolution and
+config dict construction, keeping the route handlers thin.
+"""
+
+import os
+from typing import Dict, Optional, Tuple
+
+from .models import IBMCOSConfigureBody
+
+
+def build_ibm_cos_config(
+ body: IBMCOSConfigureBody,
+ existing_config: Dict,
+) -> Tuple[Dict, Optional[str]]:
+ """Resolve IBM COS credentials and build the connection config dict.
+
+ Resolution order for each credential: request body → environment variable
+ → existing connection config.
+
+ Returns:
+ (conn_config, None) on success
+ ({}, error_message) on validation failure
+ """
+ conn_config: Dict = {
+ "auth_mode": body.auth_mode,
+ "endpoint_url": body.endpoint,
+ }
+
+ if body.auth_mode == "iam":
+ api_key = (
+ body.api_key
+ or os.getenv("IBM_COS_API_KEY")
+ or existing_config.get("api_key")
+ )
+ svc_id = (
+ body.service_instance_id
+ or os.getenv("IBM_COS_SERVICE_INSTANCE_ID")
+ or existing_config.get("service_instance_id")
+ )
+ if not api_key or not svc_id:
+ return {}, "IAM mode requires api_key and service_instance_id"
+ conn_config["api_key"] = api_key
+ conn_config["service_instance_id"] = svc_id
+ if body.auth_endpoint:
+ conn_config["auth_endpoint"] = body.auth_endpoint
+ else:
+ # HMAC mode
+ hmac_access = (
+ body.hmac_access_key
+ or os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID")
+ or existing_config.get("hmac_access_key")
+ )
+ hmac_secret = (
+ body.hmac_secret_key
+ or os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY")
+ or existing_config.get("hmac_secret_key")
+ )
+ if not hmac_access or not hmac_secret:
+ return {}, "HMAC mode requires hmac_access_key and hmac_secret_key"
+ conn_config["hmac_access_key"] = hmac_access
+ conn_config["hmac_secret_key"] = hmac_secret
+
+ if body.bucket_names is not None:
+ conn_config["bucket_names"] = body.bucket_names
+
+ return conn_config, None
diff --git a/src/main.py b/src/main.py
index 01f5c479f..e48c41744 100644
--- a/src/main.py
+++ b/src/main.py
@@ -38,6 +38,18 @@ from api import (
)
from api.connector_router import ConnectorRouter
+from connectors.ibm_cos.api import (
+ ibm_cos_defaults,
+ ibm_cos_configure,
+ ibm_cos_list_buckets,
+ ibm_cos_bucket_status,
+)
+from connectors.aws_s3.api import (
+ s3_defaults,
+ s3_configure,
+ s3_list_buckets,
+ s3_bucket_status,
+)
from services.api_key_service import APIKeyService
from api import keys as api_keys
from api.v1 import chat as v1_chat, search as v1_search, documents as v1_documents, settings as v1_settings, models as v1_models, knowledge_filters as v1_knowledge_filters
@@ -756,14 +768,15 @@ async def create_app():
# Connector endpoints
app.add_api_route("/connectors", connectors.list_connectors, methods=["GET"], tags=["internal"])
# IBM COS-specific routes (registered before generic /{connector_type}/... to avoid shadowing)
- app.add_api_route("/connectors/ibm_cos/defaults", connectors.ibm_cos_defaults, methods=["GET"], tags=["internal"])
- app.add_api_route("/connectors/ibm_cos/configure", connectors.ibm_cos_configure, methods=["POST"], tags=["internal"])
- app.add_api_route("/connectors/ibm_cos/{connection_id}/buckets", connectors.ibm_cos_list_buckets, methods=["GET"], tags=["internal"])
- app.add_api_route("/connectors/ibm_cos/{connection_id}/bucket-status", connectors.ibm_cos_bucket_status, methods=["GET"], tags=["internal"])
- app.add_api_route("/connectors/aws_s3/defaults", connectors.s3_defaults, methods=["GET"], tags=["internal"])
- app.add_api_route("/connectors/aws_s3/configure", connectors.s3_configure, methods=["POST"], tags=["internal"])
- app.add_api_route("/connectors/aws_s3/{connection_id}/buckets", connectors.s3_list_buckets, methods=["GET"], tags=["internal"])
- app.add_api_route("/connectors/aws_s3/{connection_id}/bucket-status", connectors.s3_bucket_status, methods=["GET"], tags=["internal"])
+ app.add_api_route("/connectors/ibm_cos/defaults", ibm_cos_defaults, methods=["GET"], tags=["internal"])
+ app.add_api_route("/connectors/ibm_cos/configure", ibm_cos_configure, methods=["POST"], tags=["internal"])
+ app.add_api_route("/connectors/ibm_cos/{connection_id}/buckets", ibm_cos_list_buckets, methods=["GET"], tags=["internal"])
+ app.add_api_route("/connectors/ibm_cos/{connection_id}/bucket-status", ibm_cos_bucket_status, methods=["GET"], tags=["internal"])
+ # AWS S3-specific routes (registered before generic /{connector_type}/... to avoid shadowing)
+ app.add_api_route("/connectors/aws_s3/defaults", s3_defaults, methods=["GET"], tags=["internal"])
+ app.add_api_route("/connectors/aws_s3/configure", s3_configure, methods=["POST"], tags=["internal"])
+ app.add_api_route("/connectors/aws_s3/{connection_id}/buckets", s3_list_buckets, methods=["GET"], tags=["internal"])
+ app.add_api_route("/connectors/aws_s3/{connection_id}/bucket-status", s3_bucket_status, methods=["GET"], tags=["internal"])
app.add_api_route("/connectors/{connector_type}/sync", connectors.connector_sync, methods=["POST"], tags=["internal"])
app.add_api_route("/connectors/sync-all", connectors.sync_all_connectors, methods=["POST"], tags=["internal"])
app.add_api_route("/connectors/{connector_type}/status", connectors.connector_status, methods=["GET"], tags=["internal"])