OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
Refactor S3 and IBM COS connector APIs
Edwin Jose28694aba4e4d
diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 53ded30db..6d2431f9b 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -652,469 +652,6 @@ async def connector_disconnect()-# ----------------------------------------------------------------------------# IBM COS-specific endpoints-# -----------------------------------------------------------------------------class IBMCOSConfigureBody(BaseModel):- auth_mode: str # "iam" or "hmac"- endpoint: str- # IAM fields- api_key: Optional[str] = None- service_instance_id: Optional[str] = None- auth_endpoint: Optional[str] = None- # HMAC fields- hmac_access_key: Optional[str] = None- hmac_secret_key: Optional[str] = None- # Optional bucket selection- bucket_names: Optional[List[str]] = None- # Optional: update an existing connection- connection_id: Optional[str] = None---async def ibm_cos_defaults(- connector_service=Depends(get_connector_service),- user: User = Depends(get_current_user),-):- """Return current IBM COS env-var defaults for pre-filling the config dialog.-- Sensitive values (API key, HMAC secret) are masked — only whether they are- set is returned, not the actual values.- """- import os-- api_key = os.getenv("IBM_COS_API_KEY", "")- service_instance_id = os.getenv("IBM_COS_SERVICE_INSTANCE_ID", "")- endpoint = os.getenv("IBM_COS_ENDPOINT", "")- hmac_access_key = os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID", "")- hmac_secret_key = os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY", "")- disable_iam = os.getenv("OPENRAG_IBM_COS_IAM_UI", "").lower() not in ("1", "true", "yes")-- # Try to read existing connection config for this user too- connections = await connector_service.connection_manager.list_connections(- user_id=user.user_id, connector_type="ibm_cos"- )- conn_config = {}- if connections:- conn_config = connections[0].config or {}-- def _pick(conn_key, env_val):- """Prefer connection config value over env var."""- return conn_config.get(conn_key) or env_val-- return JSONResponse({- "api_key_set": bool(api_key or conn_config.get("api_key")),- "service_instance_id": _pick("service_instance_id", service_instance_id),- "endpoint": _pick("endpoint_url", endpoint),- "hmac_access_key_set": bool(hmac_access_key or conn_config.get("hmac_access_key")),- "hmac_secret_key_set": bool(hmac_secret_key or conn_config.get("hmac_secret_key")),- # Return which auth mode was previously used; default to hmac when IAM is disabled- "auth_mode": conn_config.get("auth_mode", "hmac" if (disable_iam or not (api_key or conn_config.get("api_key"))) else "iam"),- "disable_iam": disable_iam,- # Return bucket_names from existing connection (if any)- "bucket_names": conn_config.get("bucket_names", []),- # Return connection_id if an existing connection exists- "connection_id": connections[0].connection_id if connections else None,- })---async def ibm_cos_configure(- body: IBMCOSConfigureBody,- connector_service=Depends(get_connector_service),- user: User = Depends(get_current_user),-):- """Create or update an IBM COS connection with explicit credentials.-- Tests the credentials by listing buckets, then persists the connection.- Credentials are stored in the connection config dict (not env vars) so- the connector works even without system-level env vars.- """- import os- from connectors.ibm_cos.auth import create_ibm_cos_client, create_ibm_cos_resource-- # Build the config dict that will be stored in the connection- conn_config: dict = {- "auth_mode": body.auth_mode,- "endpoint_url": body.endpoint,- }-- if body.auth_mode == "iam":- # Resolve: use supplied value, fall back to env var, fall back to existing connection- api_key = body.api_key or os.getenv("IBM_COS_API_KEY")- svc_id = body.service_instance_id or os.getenv("IBM_COS_SERVICE_INSTANCE_ID")-- # If still empty, pull from existing connection config- existing_connections = await connector_service.connection_manager.list_connections(- user_id=user.user_id, connector_type="ibm_cos"- )- if not api_key and existing_connections:- api_key = existing_connections[0].config.get("api_key")- if not svc_id and existing_connections:- svc_id = existing_connections[0].config.get("service_instance_id")-- if not api_key or not svc_id:- return JSONResponse(- {"error": "IAM mode requires api_key and service_instance_id"},- status_code=400,- )- conn_config["api_key"] = api_key- conn_config["service_instance_id"] = svc_id- if body.auth_endpoint:- conn_config["auth_endpoint"] = body.auth_endpoint- else:- # HMAC mode- hmac_access = body.hmac_access_key or os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID")- hmac_secret = body.hmac_secret_key or os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY")-- existing_connections = await connector_service.connection_manager.list_connections(- user_id=user.user_id, connector_type="ibm_cos"- )- if not hmac_access and existing_connections:- hmac_access = existing_connections[0].config.get("hmac_access_key")- if not hmac_secret and existing_connections:- hmac_secret = existing_connections[0].config.get("hmac_secret_key")-- if not hmac_access or not hmac_secret:- return JSONResponse(- {"error": "HMAC mode requires hmac_access_key and hmac_secret_key"},- status_code=400,- )- conn_config["hmac_access_key"] = hmac_access- conn_config["hmac_secret_key"] = hmac_secret-- if body.bucket_names is not None:- conn_config["bucket_names"] = body.bucket_names-- # Test credentials — IAM uses client (avoids ibm_botocore discovery-call bug),- # HMAC uses resource (S3-compatible, works with MinIO).- try:- if conn_config.get("auth_mode", "iam") == "hmac":- cos = create_ibm_cos_resource(conn_config)- list(cos.buckets.all())- else:- cos = create_ibm_cos_client(conn_config)- cos.list_buckets()- except Exception as exc:- logger.exception("Failed to connect to IBM COS during credential test.")- return JSONResponse(- {"error": "Could not connect to IBM COS with the provided configuration."},- status_code=400,- )-- # Persist: update existing connection or create a new one- if body.connection_id:- existing = await connector_service.connection_manager.get_connection(body.connection_id)- if existing and existing.user_id == user.user_id:- await connector_service.connection_manager.update_connection(- connection_id=body.connection_id,- config=conn_config,- )- # Evict cached connector so next call gets a fresh instance- connector_service.connection_manager.active_connectors.pop(body.connection_id, None)- return JSONResponse({"connection_id": body.connection_id, "status": "connected"})-- # Create a fresh connection- connection_id = await connector_service.connection_manager.create_connection(- connector_type="ibm_cos",- name="IBM Cloud Object Storage",- config=conn_config,- user_id=user.user_id,- )- return JSONResponse({"connection_id": connection_id, "status": "connected"})---async def ibm_cos_list_buckets(- connection_id: str,- connector_service=Depends(get_connector_service),- user: User = Depends(get_current_user),-):- """List all buckets accessible with the stored IBM COS credentials."""- from connectors.ibm_cos.auth import create_ibm_cos_client, create_ibm_cos_resource-- connection = await connector_service.connection_manager.get_connection(connection_id)- if not connection or connection.user_id != user.user_id:- return JSONResponse({"error": "Connection not found"}, status_code=404)- if connection.connector_type != "ibm_cos":- return JSONResponse({"error": "Not an IBM COS connection"}, status_code=400)-- try:- cfg = connection.config- if cfg.get("auth_mode", "iam") == "hmac":- cos = create_ibm_cos_resource(cfg)- buckets = [b.name for b in cos.buckets.all()]- else:- cos = create_ibm_cos_client(cfg)- buckets = [b["Name"] for b in cos.list_buckets().get("Buckets", [])]- return JSONResponse({"buckets": buckets})- except Exception as exc:- return JSONResponse({"error": "Failed to list buckets"}, status_code=500)----async def ibm_cos_bucket_status(- connection_id: str,- connector_service=Depends(get_connector_service),- session_manager=Depends(get_session_manager),- user: User = Depends(get_current_user),-):- """Return all buckets for an IBM COS connection with their ingestion status.-- Each entry includes the bucket name, whether it has been ingested (is_synced),- and the count of indexed documents from that bucket.- """- from connectors.ibm_cos.auth import create_ibm_cos_resource-- connection = await connector_service.connection_manager.get_connection(connection_id)- if not connection or connection.user_id != user.user_id:- return JSONResponse({"error": "Connection not found"}, status_code=404)- if connection.connector_type != "ibm_cos":- return JSONResponse({"error": "Not an IBM COS connection"}, status_code=400)-- # 1. List all buckets from COS- try:- cfg = connection.config- cos = create_ibm_cos_resource(cfg)- all_buckets = [b.name for b in cos.buckets.all()]- except Exception as exc:- logger.exception("Failed to list IBM COS buckets for connection %s", connection_id)- return JSONResponse({"error": f"Failed to list buckets: {exc}"}, status_code=500)-- # 2. Count indexed documents per bucket from OpenSearch- ingested_counts: dict = {}- try:- opensearch_client = session_manager.get_user_opensearch_client(- user.user_id, user.jwt_token- )- query_body = {- "size": 0,- "query": {"term": {"connector_type": "ibm_cos"}},- "aggs": {- "doc_ids": {- "terms": {"field": "document_id", "size": 50000}- }- },- }- index_name = get_index_name(user.user_id)- os_resp = opensearch_client.search(index=index_name, body=query_body)- for bucket_entry in os_resp.get("aggregations", {}).get("doc_ids", {}).get("buckets", []):- doc_id = bucket_entry["key"]- if "::" in doc_id:- bucket_name = doc_id.split("::")[0]- ingested_counts[bucket_name] = ingested_counts.get(bucket_name, 0) + 1- except Exception:- pass # OpenSearch unavailable — show zero counts-- result = [- {- "name": bucket,- "ingested_count": ingested_counts.get(bucket, 0),- "is_synced": ingested_counts.get(bucket, 0) > 0,- }- for bucket in all_buckets- ]- return JSONResponse({"buckets": result})---# ----------------------------------------------------------------------------# Amazon S3 / S3-compatible endpoints-# -----------------------------------------------------------------------------class S3ConfigureBody(BaseModel):- access_key: Optional[str] = None- secret_key: Optional[str] = None- endpoint_url: Optional[str] = None- region: Optional[str] = None- bucket_names: Optional[List[str]] = None- connection_id: Optional[str] = None---async def s3_defaults(- connector_service=Depends(get_connector_service),- user: User = Depends(get_current_user),-):- """Return current S3 env-var defaults for pre-filling the config dialog.-- Sensitive values (secret key) are masked — only whether they are set is returned.- """- import os-- access_key = os.getenv("AWS_ACCESS_KEY_ID", "")- secret_key = os.getenv("AWS_SECRET_ACCESS_KEY", "")- endpoint_url = os.getenv("AWS_S3_ENDPOINT", "")- region = os.getenv("AWS_REGION", "")-- connections = await connector_service.connection_manager.list_connections(- user_id=user.user_id, connector_type="aws_s3"- )- conn_config = {}- if connections:- conn_config = connections[0].config or {}-- def _pick(conn_key, env_val):- return conn_config.get(conn_key) or env_val-- return JSONResponse({- "access_key_set": bool(access_key or conn_config.get("access_key")),- "secret_key_set": bool(secret_key or conn_config.get("secret_key")),- "endpoint": _pick("endpoint_url", endpoint_url),- "region": _pick("region", region),- "bucket_names": conn_config.get("bucket_names", []),- "connection_id": connections[0].connection_id if connections else None,- })---async def s3_configure(- body: S3ConfigureBody,- connector_service=Depends(get_connector_service),- user: User = Depends(get_current_user),-):- """Create or update an S3 connection with explicit credentials.-- Tests the credentials by listing buckets, then persists the connection.- """- import os- from connectors.aws_s3.auth import create_s3_resource-- access_key = body.access_key or os.getenv("AWS_ACCESS_KEY_ID")- secret_key = body.secret_key or os.getenv("AWS_SECRET_ACCESS_KEY")-- # Fall back to existing connection config- existing_connections = await connector_service.connection_manager.list_connections(- user_id=user.user_id, connector_type="aws_s3"- )- if not access_key and existing_connections:- access_key = existing_connections[0].config.get("access_key")- if not secret_key and existing_connections:- secret_key = existing_connections[0].config.get("secret_key")-- if not access_key or not secret_key:- return JSONResponse(- {"error": "access_key and secret_key are required"},- status_code=400,- )-- conn_config: dict = {- "access_key": access_key.strip(),- "secret_key": secret_key.strip(),- }- if body.endpoint_url:- conn_config["endpoint_url"] = body.endpoint_url.strip()- if body.region:- conn_config["region"] = body.region.strip()- if body.bucket_names is not None:- conn_config["bucket_names"] = body.bucket_names-- # Test credentials- try:- s3 = create_s3_resource(conn_config)- list(s3.buckets.all())- except Exception as exc:- return JSONResponse(- {"error": f"Could not connect to S3: {exc}"},- status_code=400,- )-- # Persist: update existing connection or create a new one- if body.connection_id:- existing = await connector_service.connection_manager.get_connection(body.connection_id)- if existing and existing.user_id == user.user_id:- await connector_service.connection_manager.update_connection(- connection_id=body.connection_id,- config=conn_config,- )- connector_service.connection_manager.active_connectors.pop(body.connection_id, None)- return JSONResponse({"connection_id": body.connection_id, "status": "connected"})-- connection_id = await connector_service.connection_manager.create_connection(- connector_type="aws_s3",- name="Amazon S3",- config=conn_config,- user_id=user.user_id,- )- return JSONResponse({"connection_id": connection_id, "status": "connected"})---async def s3_list_buckets(- connection_id: str,- connector_service=Depends(get_connector_service),- user: User = Depends(get_current_user),-):- """List all buckets accessible with the stored S3 credentials."""- from connectors.aws_s3.auth import create_s3_resource-- connection = await connector_service.connection_manager.get_connection(connection_id)- if not connection or connection.user_id != user.user_id:- return JSONResponse({"error": "Connection not found"}, status_code=404)- if connection.connector_type != "aws_s3":- return JSONResponse({"error": "Not an S3 connection"}, status_code=400)-- try:- s3 = create_s3_resource(connection.config)- buckets = [b.name for b in s3.buckets.all()]- return JSONResponse({"buckets": buckets})- except Exception as exc:- return JSONResponse({"error": f"Failed to list buckets: {exc}"}, status_code=500)---async def s3_bucket_status(- connection_id: str,- connector_service=Depends(get_connector_service),- session_manager=Depends(get_session_manager),- user: User = Depends(get_current_user),-):- """Return all buckets for an S3 connection with their ingestion status."""- from connectors.aws_s3.auth import create_s3_resource-- connection = await connector_service.connection_manager.get_connection(connection_id)- if not connection or connection.user_id != user.user_id:- return JSONResponse({"error": "Connection not found"}, status_code=404)- if connection.connector_type != "aws_s3":- return JSONResponse({"error": "Not an S3 connection"}, status_code=400)-- # 1. List all buckets from S3- try:- s3 = create_s3_resource(connection.config)- all_buckets = [b.name for b in s3.buckets.all()]- except Exception as exc:- logger.exception("Failed to list buckets from S3 for connection %s", connection_id)- return JSONResponse({"error": "Failed to list buckets"}, status_code=500)-- # 2. Count indexed documents per bucket from OpenSearch- ingested_counts: dict = {}- try:- opensearch_client = session_manager.get_user_opensearch_client(- user.user_id, user.jwt_token- )- query_body = {- "size": 0,- "query": {"term": {"connector_type": "aws_s3"}},- "aggs": {- "doc_ids": {- "terms": {"field": "document_id", "size": 50000}- }- },- }- index_name = get_index_name(user.user_id)- os_resp = opensearch_client.search(index=index_name, body=query_body)- for bucket_entry in os_resp.get("aggregations", {}).get("doc_ids", {}).get("buckets", []):- doc_id = bucket_entry["key"]- if "::" in doc_id:- bucket_name = doc_id.split("::")[0]- ingested_counts[bucket_name] = ingested_counts.get(bucket_name, 0) + 1- except Exception:- pass # OpenSearch unavailable — show zero counts-- result = [- {- "name": bucket,- "ingested_count": ingested_counts.get(bucket, 0),- "is_synced": ingested_counts.get(bucket, 0) > 0,- }- for bucket in all_buckets- ]- return JSONResponse({"buckets": result})--# ---------------------------------------------------------------------------async def sync_all_connectors(diff --git a/src/connectors/aws_s3/__init__.py b/src/connectors/aws_s3/__init__.pyindex ee9a5ccab..32c63a878 100644--- a/src/connectors/aws_s3/__init__.py+++ b/src/connectors/aws_s3/__init__.py@@ -1,5 +1,19 @@"""Amazon S3 / S3-compatible connector for OpenRAG."""from .connector import S3Connector+from .models import S3ConfigureBody+from .api import (+ s3_defaults,+ s3_configure,+ s3_list_buckets,+ s3_bucket_status,+)-__all__ = ["S3Connector"]+__all__ = [+ "S3Connector",+ "S3ConfigureBody",+ "s3_defaults",+ "s3_configure",+ "s3_list_buckets",+ "s3_bucket_status",+]diff --git a/src/connectors/aws_s3/api.py b/src/connectors/aws_s3/api.pynew file mode 100644index 000000000..2612ca698--- /dev/null+++ b/src/connectors/aws_s3/api.py@@ -0,0 +1,173 @@+"""FastAPI route handlers for AWS S3-specific endpoints."""++import os++from fastapi import Depends+from fastapi.responses import JSONResponse++from config.settings import get_index_name+from dependencies import get_connector_service, get_session_manager, get_current_user+from session_manager import User+from utils.logging_config import get_logger++from .auth import create_s3_resource+from .models import S3ConfigureBody+from .support import build_s3_config++logger = get_logger(__name__)+++async def s3_defaults(+ connector_service=Depends(get_connector_service),+ user: User = Depends(get_current_user),+):+ """Return current S3 env-var defaults for pre-filling the config dialog.++ Sensitive values (secret key) are masked — only whether they are set is returned.+ """+ access_key = os.getenv("AWS_ACCESS_KEY_ID", "")+ secret_key = os.getenv("AWS_SECRET_ACCESS_KEY", "")+ endpoint_url = os.getenv("AWS_S3_ENDPOINT", "")+ region = os.getenv("AWS_REGION", "")++ connections = await connector_service.connection_manager.list_connections(+ user_id=user.user_id, connector_type="aws_s3"+ )+ conn_config = connections[0].config or {} if connections else {}++ def _pick(conn_key, env_val):+ return conn_config.get(conn_key) or env_val++ return JSONResponse({+ "access_key_set": bool(access_key or conn_config.get("access_key")),+ "secret_key_set": bool(secret_key or conn_config.get("secret_key")),+ "endpoint": _pick("endpoint_url", endpoint_url),+ "region": _pick("region", region),+ "bucket_names": conn_config.get("bucket_names", []),+ "connection_id": connections[0].connection_id if connections else None,+ })+++async def s3_configure(+ body: S3ConfigureBody,+ connector_service=Depends(get_connector_service),+ user: User = Depends(get_current_user),+):+ """Create or update an S3 connection with explicit credentials.++ Tests the credentials by listing buckets, then persists the connection.+ """+ existing_connections = await connector_service.connection_manager.list_connections(+ user_id=user.user_id, connector_type="aws_s3"+ )+ existing_config = existing_connections[0].config if existing_connections else {}++ conn_config, error = build_s3_config(body, existing_config)+ if error:+ return JSONResponse({"error": error}, status_code=400)++ # Test credentials+ try:+ s3 = create_s3_resource(conn_config)+ list(s3.buckets.all())+ except Exception as exc:+ return JSONResponse(+ {"error": f"Could not connect to S3: {exc}"},+ status_code=400,+ )++ # Persist: update existing connection or create a new one+ if body.connection_id:+ existing = await connector_service.connection_manager.get_connection(body.connection_id)+ if existing and existing.user_id == user.user_id:+ await connector_service.connection_manager.update_connection(+ connection_id=body.connection_id,+ config=conn_config,+ )+ connector_service.connection_manager.active_connectors.pop(body.connection_id, None)+ return JSONResponse({"connection_id": body.connection_id, "status": "connected"})++ connection_id = await connector_service.connection_manager.create_connection(+ connector_type="aws_s3",+ name="Amazon S3",+ config=conn_config,+ user_id=user.user_id,+ )+ return JSONResponse({"connection_id": connection_id, "status": "connected"})+++async def s3_list_buckets(+ connection_id: str,+ connector_service=Depends(get_connector_service),+ user: User = Depends(get_current_user),+):+ """List all buckets accessible with the stored S3 credentials."""+ connection = await connector_service.connection_manager.get_connection(connection_id)+ if not connection or connection.user_id != user.user_id:+ return JSONResponse({"error": "Connection not found"}, status_code=404)+ if connection.connector_type != "aws_s3":+ return JSONResponse({"error": "Not an S3 connection"}, status_code=400)++ try:+ s3 = create_s3_resource(connection.config)+ buckets = [b.name for b in s3.buckets.all()]+ return JSONResponse({"buckets": buckets})+ except Exception as exc:+ return JSONResponse({"error": f"Failed to list buckets: {exc}"}, status_code=500)+++async def s3_bucket_status(+ connection_id: str,+ connector_service=Depends(get_connector_service),+ session_manager=Depends(get_session_manager),+ user: User = Depends(get_current_user),+):+ """Return all buckets for an S3 connection with their ingestion status."""+ connection = await connector_service.connection_manager.get_connection(connection_id)+ if not connection or connection.user_id != user.user_id:+ return JSONResponse({"error": "Connection not found"}, status_code=404)+ if connection.connector_type != "aws_s3":+ return JSONResponse({"error": "Not an S3 connection"}, status_code=400)++ # 1. List all buckets from S3+ try:+ s3 = create_s3_resource(connection.config)+ all_buckets = [b.name for b in s3.buckets.all()]+ except Exception as exc:+ logger.exception("Failed to list buckets from S3 for connection %s", connection_id)+ return JSONResponse({"error": "Failed to list buckets"}, status_code=500)++ # 2. Count indexed documents per bucket from OpenSearch+ ingested_counts: dict = {}+ try:+ opensearch_client = session_manager.get_user_opensearch_client(+ user.user_id, user.jwt_token+ )+ query_body = {+ "size": 0,+ "query": {"term": {"connector_type": "aws_s3"}},+ "aggs": {+ "doc_ids": {+ "terms": {"field": "document_id", "size": 50000}+ }+ },+ }+ index_name = get_index_name()+ os_resp = opensearch_client.search(index=index_name, body=query_body)+ for bucket_entry in os_resp.get("aggregations", {}).get("doc_ids", {}).get("buckets", []):+ doc_id = bucket_entry["key"]+ if "::" in doc_id:+ bucket_name = doc_id.split("::")[0]+ ingested_counts[bucket_name] = ingested_counts.get(bucket_name, 0) + 1+ except Exception:+ pass # OpenSearch unavailable — show zero counts++ result = [+ {+ "name": bucket,+ "ingested_count": ingested_counts.get(bucket, 0),+ "is_synced": ingested_counts.get(bucket, 0) > 0,+ }+ for bucket in all_buckets+ ]+ return JSONResponse({"buckets": result})diff --git a/src/connectors/aws_s3/models.py b/src/connectors/aws_s3/models.pynew file mode 100644index 000000000..582111bd2--- /dev/null+++ b/src/connectors/aws_s3/models.py@@ -0,0 +1,13 @@+"""Pydantic request/response models for AWS S3 API endpoints."""++from typing import List, Optional+from pydantic import BaseModel+++class S3ConfigureBody(BaseModel):+ access_key: Optional[str] = None+ secret_key: Optional[str] = None+ endpoint_url: Optional[str] = None+ region: Optional[str] = None+ bucket_names: Optional[List[str]] = None+ connection_id: Optional[str] = Nonediff --git a/src/connectors/aws_s3/support.py b/src/connectors/aws_s3/support.pynew file mode 100644index 000000000..ccaa069b7--- /dev/null+++ b/src/connectors/aws_s3/support.py@@ -0,0 +1,51 @@+"""Support helpers for AWS S3 API endpoints.++Contains pure (non-async) business logic for credential resolution and+config dict construction, keeping the route handlers thin.+"""++import os+from typing import Dict, Optional, Tuple++from .models import S3ConfigureBody+++def build_s3_config(+ body: S3ConfigureBody,+ existing_config: Dict,+) -> Tuple[Dict, Optional[str]]:+ """Resolve S3 credentials and build the connection config dict.++ Resolution order for each credential: request body → environment variable+ → existing connection config.++ Returns:+ (conn_config, None) on success+ ({}, error_message) on validation failure+ """+ access_key = (+ body.access_key+ or os.getenv("AWS_ACCESS_KEY_ID")+ or existing_config.get("access_key")+ )+ secret_key = (+ body.secret_key+ or os.getenv("AWS_SECRET_ACCESS_KEY")+ or existing_config.get("secret_key")+ )++ if not access_key or not secret_key:+ return {}, "access_key and secret_key are required"++ conn_config: Dict = {+ "access_key": access_key.strip(),+ "secret_key": secret_key.strip(),+ }+ if body.endpoint_url:+ conn_config["endpoint_url"] = body.endpoint_url.strip()+ if body.region:+ conn_config["region"] = body.region.strip()+ if body.bucket_names is not None:+ conn_config["bucket_names"] = body.bucket_names++ return conn_config, Nonediff --git a/src/connectors/ibm_cos/__init__.py b/src/connectors/ibm_cos/__init__.pyindex 32f19defc..5bf01ad43 100644--- a/src/connectors/ibm_cos/__init__.py+++ b/src/connectors/ibm_cos/__init__.py@@ -1,3 +1,17 @@from .connector import IBMCOSConnector+from .models import IBMCOSConfigureBody+from .api import (+ ibm_cos_defaults,+ ibm_cos_configure,+ ibm_cos_list_buckets,+ ibm_cos_bucket_status,+)-__all__ = ["IBMCOSConnector"]+__all__ = [+ "IBMCOSConnector",+ "IBMCOSConfigureBody",+ "ibm_cos_defaults",+ "ibm_cos_configure",+ "ibm_cos_list_buckets",+ "ibm_cos_bucket_status",+]diff --git a/src/connectors/ibm_cos/api.py b/src/connectors/ibm_cos/api.pynew file mode 100644index 000000000..e6793f2f0--- /dev/null+++ b/src/connectors/ibm_cos/api.py@@ -0,0 +1,201 @@+"""FastAPI route handlers for IBM COS-specific endpoints."""++import os++from fastapi import Depends+from fastapi.responses import JSONResponse++from config.settings import get_index_name+from dependencies import get_connector_service, get_session_manager, get_current_user+from session_manager import User+from utils.logging_config import get_logger++from .auth import create_ibm_cos_client, create_ibm_cos_resource+from .models import IBMCOSConfigureBody+from .support import build_ibm_cos_config++logger = get_logger(__name__)+++async def ibm_cos_defaults(+ connector_service=Depends(get_connector_service),+ user: User = Depends(get_current_user),+):+ """Return current IBM COS env-var defaults for pre-filling the config dialog.++ Sensitive values (API key, HMAC secret) are masked — only whether they are+ set is returned, not the actual values.+ """+ api_key = os.getenv("IBM_COS_API_KEY", "")+ service_instance_id = os.getenv("IBM_COS_SERVICE_INSTANCE_ID", "")+ endpoint = os.getenv("IBM_COS_ENDPOINT", "")+ hmac_access_key = os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID", "")+ hmac_secret_key = os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY", "")+ disable_iam = os.getenv("OPENRAG_IBM_COS_IAM_UI", "").lower() not in ("1", "true", "yes")++ connections = await connector_service.connection_manager.list_connections(+ user_id=user.user_id, connector_type="ibm_cos"+ )+ conn_config = connections[0].config or {} if connections else {}++ def _pick(conn_key, env_val):+ """Prefer connection config value over env var."""+ return conn_config.get(conn_key) or env_val++ return JSONResponse({+ "api_key_set": bool(api_key or conn_config.get("api_key")),+ "service_instance_id": _pick("service_instance_id", service_instance_id),+ "endpoint": _pick("endpoint_url", endpoint),+ "hmac_access_key_set": bool(hmac_access_key or conn_config.get("hmac_access_key")),+ "hmac_secret_key_set": bool(hmac_secret_key or conn_config.get("hmac_secret_key")),+ "auth_mode": conn_config.get(+ "auth_mode",+ "hmac" if (disable_iam or not (api_key or conn_config.get("api_key"))) else "iam",+ ),+ "disable_iam": disable_iam,+ "bucket_names": conn_config.get("bucket_names", []),+ "connection_id": connections[0].connection_id if connections else None,+ })+++async def ibm_cos_configure(+ body: IBMCOSConfigureBody,+ connector_service=Depends(get_connector_service),+ user: User = Depends(get_current_user),+):+ """Create or update an IBM COS connection with explicit credentials.++ Tests the credentials by listing buckets, then persists the connection.+ Credentials are stored in the connection config dict (not env vars) so+ the connector works even without system-level env vars.+ """+ existing_connections = await connector_service.connection_manager.list_connections(+ user_id=user.user_id, connector_type="ibm_cos"+ )+ existing_config = existing_connections[0].config if existing_connections else {}++ conn_config, error = build_ibm_cos_config(body, existing_config)+ if error:+ return JSONResponse({"error": error}, status_code=400)++ # Test credentials — IAM uses client (avoids ibm_botocore discovery-call bug),+ # HMAC uses resource (S3-compatible, works with MinIO).+ try:+ if conn_config.get("auth_mode", "iam") == "hmac":+ cos = create_ibm_cos_resource(conn_config)+ list(cos.buckets.all())+ else:+ cos = create_ibm_cos_client(conn_config)+ cos.list_buckets()+ except Exception:+ logger.exception("Failed to connect to IBM COS during credential test.")+ return JSONResponse(+ {"error": "Could not connect to IBM COS with the provided configuration."},+ status_code=400,+ )++ # Persist: update existing connection or create a new one+ if body.connection_id:+ existing = await connector_service.connection_manager.get_connection(body.connection_id)+ if existing and existing.user_id == user.user_id:+ await connector_service.connection_manager.update_connection(+ connection_id=body.connection_id,+ config=conn_config,+ )+ connector_service.connection_manager.active_connectors.pop(body.connection_id, None)+ return JSONResponse({"connection_id": body.connection_id, "status": "connected"})++ connection_id = await connector_service.connection_manager.create_connection(+ connector_type="ibm_cos",+ name="IBM Cloud Object Storage",+ config=conn_config,+ user_id=user.user_id,+ )+ return JSONResponse({"connection_id": connection_id, "status": "connected"})+++async def ibm_cos_list_buckets(+ connection_id: str,+ connector_service=Depends(get_connector_service),+ user: User = Depends(get_current_user),+):+ """List all buckets accessible with the stored IBM COS credentials."""+ connection = await connector_service.connection_manager.get_connection(connection_id)+ if not connection or connection.user_id != user.user_id:+ return JSONResponse({"error": "Connection not found"}, status_code=404)+ if connection.connector_type != "ibm_cos":+ return JSONResponse({"error": "Not an IBM COS connection"}, status_code=400)++ try:+ cfg = connection.config+ if cfg.get("auth_mode", "iam") == "hmac":+ cos = create_ibm_cos_resource(cfg)+ buckets = [b.name for b in cos.buckets.all()]+ else:+ cos = create_ibm_cos_client(cfg)+ buckets = [b["Name"] for b in cos.list_buckets().get("Buckets", [])]+ return JSONResponse({"buckets": buckets})+ except Exception as exc:+ return JSONResponse({"error": "Failed to list buckets"}, status_code=500)+++async def ibm_cos_bucket_status(+ connection_id: str,+ connector_service=Depends(get_connector_service),+ session_manager=Depends(get_session_manager),+ user: User = Depends(get_current_user),+):+ """Return all buckets for an IBM COS connection with their ingestion status.++ Each entry includes the bucket name, whether it has been ingested (is_synced),+ and the count of indexed documents from that bucket.+ """+ connection = await connector_service.connection_manager.get_connection(connection_id)+ if not connection or connection.user_id != user.user_id:+ return JSONResponse({"error": "Connection not found"}, status_code=404)+ if connection.connector_type != "ibm_cos":+ return JSONResponse({"error": "Not an IBM COS connection"}, status_code=400)++ # 1. List all buckets from COS+ try:+ cfg = connection.config+ cos = create_ibm_cos_resource(cfg)+ all_buckets = [b.name for b in cos.buckets.all()]+ except Exception as exc:+ logger.exception("Failed to list IBM COS buckets for connection %s", connection_id)+ return JSONResponse({"error": f"Failed to list buckets: {exc}"}, status_code=500)++ # 2. Count indexed documents per bucket from OpenSearch+ ingested_counts: dict = {}+ try:+ opensearch_client = session_manager.get_user_opensearch_client(+ user.user_id, user.jwt_token+ )+ query_body = {+ "size": 0,+ "query": {"term": {"connector_type": "ibm_cos"}},+ "aggs": {+ "doc_ids": {+ "terms": {"field": "document_id", "size": 50000}+ }+ },+ }+ index_name = get_index_name()+ os_resp = opensearch_client.search(index=index_name, body=query_body)+ for bucket_entry in os_resp.get("aggregations", {}).get("doc_ids", {}).get("buckets", []):+ doc_id = bucket_entry["key"]+ if "::" in doc_id:+ bucket_name = doc_id.split("::")[0]+ ingested_counts[bucket_name] = ingested_counts.get(bucket_name, 0) + 1+ except Exception:+ pass # OpenSearch unavailable — show zero counts++ result = [+ {+ "name": bucket,+ "ingested_count": ingested_counts.get(bucket, 0),+ "is_synced": ingested_counts.get(bucket, 0) > 0,+ }+ for bucket in all_buckets+ ]+ return JSONResponse({"buckets": result})diff --git a/src/connectors/ibm_cos/models.py b/src/connectors/ibm_cos/models.pynew file mode 100644index 000000000..59f2324f9--- /dev/null+++ b/src/connectors/ibm_cos/models.py@@ -0,0 +1,20 @@+"""Pydantic request/response models for IBM COS API endpoints."""++from typing import List, Optional+from pydantic import BaseModel+++class IBMCOSConfigureBody(BaseModel):+ auth_mode: str # "iam" or "hmac"+ endpoint: str+ # IAM fields+ api_key: Optional[str] = None+ service_instance_id: Optional[str] = None+ auth_endpoint: Optional[str] = None+ # HMAC fields+ hmac_access_key: Optional[str] = None+ hmac_secret_key: Optional[str] = None+ # Optional bucket selection+ bucket_names: Optional[List[str]] = None+ # Optional: update an existing connection+ connection_id: Optional[str] = Nonediff --git a/src/connectors/ibm_cos/support.py b/src/connectors/ibm_cos/support.pynew file mode 100644index 000000000..25e5e5bcb--- /dev/null+++ b/src/connectors/ibm_cos/support.py@@ -0,0 +1,68 @@+"""Support helpers for IBM COS API endpoints.++Contains pure (non-async) business logic for credential resolution and+config dict construction, keeping the route handlers thin.+"""++import os+from typing import Dict, Optional, Tuple++from .models import IBMCOSConfigureBody+++def build_ibm_cos_config(+ body: IBMCOSConfigureBody,+ existing_config: Dict,+) -> Tuple[Dict, Optional[str]]:+ """Resolve IBM COS credentials and build the connection config dict.++ Resolution order for each credential: request body → environment variable+ → existing connection config.++ Returns:+ (conn_config, None) on success+ ({}, error_message) on validation failure+ """+ conn_config: Dict = {+ "auth_mode": body.auth_mode,+ "endpoint_url": body.endpoint,+ }++ if body.auth_mode == "iam":+ api_key = (+ body.api_key+ or os.getenv("IBM_COS_API_KEY")+ or existing_config.get("api_key")+ )+ svc_id = (+ body.service_instance_id+ or os.getenv("IBM_COS_SERVICE_INSTANCE_ID")+ or existing_config.get("service_instance_id")+ )+ if not api_key or not svc_id:+ return {}, "IAM mode requires api_key and service_instance_id"+ conn_config["api_key"] = api_key+ conn_config["service_instance_id"] = svc_id+ if body.auth_endpoint:+ conn_config["auth_endpoint"] = body.auth_endpoint+ else:+ # HMAC mode+ hmac_access = (+ body.hmac_access_key+ or os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID")+ or existing_config.get("hmac_access_key")+ )+ hmac_secret = (+ body.hmac_secret_key+ or os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY")+ or existing_config.get("hmac_secret_key")+ )+ if not hmac_access or not hmac_secret:+ return {}, "HMAC mode requires hmac_access_key and hmac_secret_key"+ conn_config["hmac_access_key"] = hmac_access+ conn_config["hmac_secret_key"] = hmac_secret++ if body.bucket_names is not None:+ conn_config["bucket_names"] = body.bucket_names++ return conn_config, Nonediff --git a/src/main.py b/src/main.pyindex 01f5c479f..e48c41744 100644--- a/src/main.py+++ b/src/main.py@@ -38,6 +38,18 @@ from api import ()from api.connector_router import ConnectorRouter+from connectors.ibm_cos.api import (+ ibm_cos_defaults,+ ibm_cos_configure,+ ibm_cos_list_buckets,+ ibm_cos_bucket_status,+)+from connectors.aws_s3.api import (+ s3_defaults,+ s3_configure,+ s3_list_buckets,+ s3_bucket_status,+)from services.api_key_service import APIKeyServicefrom api import keys as api_keysfrom api.v1 import chat as v1_chat, search as v1_search, documents as v1_documents, settings as v1_settings, models as v1_models, knowledge_filters as v1_knowledge_filters@@ -756,14 +768,15 @@ async def create_app():# Connector endpointsapp.add_api_route("/connectors", connectors.list_connectors, methods=["GET"], tags=["internal"])# IBM COS-specific routes (registered before generic /{connector_type}/... to avoid shadowing)- app.add_api_route("/connectors/ibm_cos/defaults", connectors.ibm_cos_defaults, methods=["GET"], tags=["internal"])- app.add_api_route("/connectors/ibm_cos/configure", connectors.ibm_cos_configure, methods=["POST"], tags=["internal"])- app.add_api_route("/connectors/ibm_cos/{connection_id}/buckets", connectors.ibm_cos_list_buckets, methods=["GET"], tags=["internal"])- app.add_api_route("/connectors/ibm_cos/{connection_id}/bucket-status", connectors.ibm_cos_bucket_status, methods=["GET"], tags=["internal"])- app.add_api_route("/connectors/aws_s3/defaults", connectors.s3_defaults, methods=["GET"], tags=["internal"])- app.add_api_route("/connectors/aws_s3/configure", connectors.s3_configure, methods=["POST"], tags=["internal"])- app.add_api_route("/connectors/aws_s3/{connection_id}/buckets", connectors.s3_list_buckets, methods=["GET"], tags=["internal"])- app.add_api_route("/connectors/aws_s3/{connection_id}/bucket-status", connectors.s3_bucket_status, methods=["GET"], tags=["internal"])+ app.add_api_route("/connectors/ibm_cos/defaults", ibm_cos_defaults, methods=["GET"], tags=["internal"])+ app.add_api_route("/connectors/ibm_cos/configure", ibm_cos_configure, methods=["POST"], tags=["internal"])+ app.add_api_route("/connectors/ibm_cos/{connection_id}/buckets", ibm_cos_list_buckets, methods=["GET"], tags=["internal"])+ app.add_api_route("/connectors/ibm_cos/{connection_id}/bucket-status", ibm_cos_bucket_status, methods=["GET"], tags=["internal"])+ # AWS S3-specific routes (registered before generic /{connector_type}/... to avoid shadowing)+ app.add_api_route("/connectors/aws_s3/defaults", s3_defaults, methods=["GET"], tags=["internal"])+ app.add_api_route("/connectors/aws_s3/configure", s3_configure, methods=["POST"], tags=["internal"])+ app.add_api_route("/connectors/aws_s3/{connection_id}/buckets", s3_list_buckets, methods=["GET"], tags=["internal"])+ app.add_api_route("/connectors/aws_s3/{connection_id}/bucket-status", s3_bucket_status, methods=["GET"], tags=["internal"])app.add_api_route("/connectors/{connector_type}/sync", connectors.connector_sync, methods=["POST"], tags=["internal"])app.add_api_route("/connectors/sync-all", connectors.sync_all_connectors, methods=["POST"], tags=["internal"])app.add_api_route("/connectors/{connector_type}/status", connectors.connector_status, methods=["GET"], tags=["internal"])