Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

Add COS env vars and improve connector logging

Edwin Jose0d3d07430817
diff --git a/docker-compose.yml b/docker-compose.yml
index 0aef3a172..4f97d288b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -80,6 +80,14 @@ services:
- WEBHOOK_BASE_URL=${WEBHOOK_BASE_URL}
- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
- AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+ - AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT}
+ - AWS_REGION=${AWS_REGION}
+ - IBM_COS_API_KEY=${IBM_COS_API_KEY}
+ - IBM_COS_SERVICE_INSTANCE_ID=${IBM_COS_SERVICE_INSTANCE_ID}
+ - IBM_COS_ENDPOINT=${IBM_COS_ENDPOINT}
+ - IBM_COS_HMAC_ACCESS_KEY_ID=${IBM_COS_HMAC_ACCESS_KEY_ID}
+ - IBM_COS_HMAC_SECRET_ACCESS_KEY=${IBM_COS_HMAC_SECRET_ACCESS_KEY}
+ - IBM_COS_AUTH_ENDPOINT=${IBM_COS_AUTH_ENDPOINT}
- OPENSEARCH_INDEX_NAME=${OPENSEARCH_INDEX_NAME:-documents}
- LANGFLOW_KEY=${LANGFLOW_KEY}
- LANGFLOW_KEY_RETRIES=${LANGFLOW_KEY_RETRIES:-15}
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 32ec5008a..53ded30db 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -862,7 +862,7 @@ async def ibm_cos_bucket_status(
Each entry includes the bucket name, whether it has been ingested (is_synced),
and the count of indexed documents from that bucket.
"""
- from connectors.ibm_cos.auth import create_ibm_cos_client, create_ibm_cos_resource
+ from connectors.ibm_cos.auth import create_ibm_cos_resource
connection = await connector_service.connection_manager.get_connection(connection_id)
if not connection or connection.user_id != user.user_id:
diff --git a/src/connectors/aws_s3/auth.py b/src/connectors/aws_s3/auth.py
index 5542debf7..78894eca5 100644
--- a/src/connectors/aws_s3/auth.py
+++ b/src/connectors/aws_s3/auth.py
@@ -67,10 +67,7 @@ def create_s3_resource(config: Dict[str, Any]):
creds = _resolve_credentials(config)
kwargs = _build_boto3_kwargs(creds)
- logger.debug(
- "Creating S3 resource with HMAC authentication (boto3), "
- f"endpoint={creds['endpoint_url']!r}, region={creds['region']!r}"
- )
+ logger.debug("Creating S3 resource with HMAC authentication (boto3)")
return boto3.resource("s3", **kwargs)
@@ -89,8 +86,5 @@ def create_s3_client(config: Dict[str, Any]):
creds = _resolve_credentials(config)
kwargs = _build_boto3_kwargs(creds)
- logger.debug(
- "Creating S3 client with HMAC authentication (boto3), "
- f"endpoint={creds['endpoint_url']!r}, region={creds['region']!r}"
- )
+ logger.debug("Creating S3 client with HMAC authentication (boto3)")
return boto3.client("s3", **kwargs)
diff --git a/src/connectors/aws_s3/connector.py b/src/connectors/aws_s3/connector.py
index 0b5ec749a..13929e41b 100644
--- a/src/connectors/aws_s3/connector.py
+++ b/src/connectors/aws_s3/connector.py
@@ -117,7 +117,7 @@ class S3Connector(BaseConnector):
try:
resource = self._get_resource()
buckets = [b.name for b in resource.buckets.all()]
- logger.debug(f"S3 auto-discovered {len(buckets)} bucket(s): {buckets}")
+ logger.debug("S3 auto-discovered %d bucket(s)", len(buckets))
return buckets
except Exception as exc:
logger.warning(f"S3 could not auto-discover buckets: {exc}")
@@ -169,7 +169,7 @@ class S3Connector(BaseConnector):
if max_files and len(files) >= max_files:
return {"files": files, "next_page_token": None}
except Exception as exc:
- logger.error(f"Failed to list objects in S3 bucket {bucket_name!r}: {exc}")
+ logger.error("Failed to list objects in S3 bucket: %s", exc)
continue
return {"files": files, "next_page_token": None}
@@ -252,10 +252,7 @@ class S3Connector(BaseConnector):
allowed_groups=[],
)
except Exception as exc:
- logger.warning(
- f"Could not fetch ACL for s3://{bucket}/{key}: {exc}. "
- "Using fallback ACL."
- )
+ logger.warning("Could not fetch S3 object ACL, using fallback: %s", exc)
return DocumentACL(owner=None, allowed_users=[], allowed_groups=[])
# ------------------------------------------------------------------
diff --git a/src/connectors/ibm_cos/connector.py b/src/connectors/ibm_cos/connector.py
index 4a41a5b7d..46d21a2ee 100644
--- a/src/connectors/ibm_cos/connector.py
+++ b/src/connectors/ibm_cos/connector.py
@@ -58,7 +58,12 @@ class IBMCOSConnector(BaseConnector):
def get_client_id(self) -> str:
"""Return IAM API key, or HMAC access key ID as fallback."""
- val = os.getenv("IBM_COS_API_KEY") or os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID")
+ val = (
+ self.config.get("api_key")
+ or self.config.get("hmac_access_key")
+ or os.getenv("IBM_COS_API_KEY")
+ or os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID")
+ )
if val:
return val
raise ValueError(
@@ -68,7 +73,12 @@ class IBMCOSConnector(BaseConnector):
def get_client_secret(self) -> str:
"""Return IAM service instance ID, or HMAC secret key as fallback."""
- val = os.getenv("IBM_COS_SERVICE_INSTANCE_ID") or os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY")
+ val = (
+ self.config.get("service_instance_id")
+ or self.config.get("hmac_secret_key")
+ or os.getenv("IBM_COS_SERVICE_INSTANCE_ID")
+ or os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY")
+ )
if val:
return val
raise ValueError(
@@ -142,7 +152,7 @@ class IBMCOSConnector(BaseConnector):
else:
resp = handle.list_buckets()
buckets = [b["Name"] for b in resp.get("Buckets", [])]
- logger.debug(f"IBM COS auto-discovered {len(buckets)} bucket(s): {buckets}")
+ logger.debug("IBM COS auto-discovered %d bucket(s)", len(buckets))
return buckets
except Exception as exc:
logger.warning(f"IBM COS could not auto-discover buckets: {exc}")
@@ -228,7 +238,7 @@ class IBMCOSConnector(BaseConnector):
break
except Exception as exc:
- logger.error(f"Failed to list objects in bucket {bucket_name!r}: {exc}")
+ logger.error("Failed to list objects in IBM COS bucket: %s", exc)
continue
return {"files": files, "next_page_token": None}
@@ -326,10 +336,7 @@ class IBMCOSConnector(BaseConnector):
allowed_groups=[],
)
except Exception as exc:
- logger.warning(
- f"Could not fetch ACL for cos://{bucket}/{key}: {exc}. "
- "Using fallback ACL."
- )
+ logger.warning("Could not fetch IBM COS object ACL, using fallback: %s", exc)
return DocumentACL(
owner=self._service_instance_id or None,
allowed_users=[],