OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
Add COS env vars and improve connector logging
Edwin Jose0d3d07430817
diff --git a/docker-compose.yml b/docker-compose.ymlindex 0aef3a172..4f97d288b 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -80,6 +80,14 @@ services:- WEBHOOK_BASE_URL=${WEBHOOK_BASE_URL}- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}- AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}+ - AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT}+ - AWS_REGION=${AWS_REGION}+ - IBM_COS_API_KEY=${IBM_COS_API_KEY}+ - IBM_COS_SERVICE_INSTANCE_ID=${IBM_COS_SERVICE_INSTANCE_ID}+ - IBM_COS_ENDPOINT=${IBM_COS_ENDPOINT}+ - IBM_COS_HMAC_ACCESS_KEY_ID=${IBM_COS_HMAC_ACCESS_KEY_ID}+ - IBM_COS_HMAC_SECRET_ACCESS_KEY=${IBM_COS_HMAC_SECRET_ACCESS_KEY}+ - IBM_COS_AUTH_ENDPOINT=${IBM_COS_AUTH_ENDPOINT}- OPENSEARCH_INDEX_NAME=${OPENSEARCH_INDEX_NAME:-documents}- LANGFLOW_KEY=${LANGFLOW_KEY}- LANGFLOW_KEY_RETRIES=${LANGFLOW_KEY_RETRIES:-15}diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 32ec5008a..53ded30db 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -862,7 +862,7 @@ async def ibm_cos_bucket_status(Each entry includes the bucket name, whether it has been ingested (is_synced),and the count of indexed documents from that bucket."""- from connectors.ibm_cos.auth import create_ibm_cos_client, create_ibm_cos_resource+ from connectors.ibm_cos.auth import create_ibm_cos_resourceconnection = await connector_service.connection_manager.get_connection(connection_id)if not connection or connection.user_id != user.user_id:diff --git a/src/connectors/aws_s3/auth.py b/src/connectors/aws_s3/auth.pyindex 5542debf7..78894eca5 100644--- a/src/connectors/aws_s3/auth.py+++ b/src/connectors/aws_s3/auth.py@@ -67,10 +67,7 @@ def create_s3_resource(config: Dict[str, Any]):creds = _resolve_credentials(config)kwargs = _build_boto3_kwargs(creds)- logger.debug(- "Creating S3 resource with HMAC authentication (boto3), "- f"endpoint={creds['endpoint_url']!r}, region={creds['region']!r}"- )+ logger.debug("Creating S3 resource with HMAC authentication (boto3)")return boto3.resource("s3", **kwargs)@@ -89,8 +86,5 @@ def create_s3_client(config: Dict[str, Any]):creds = _resolve_credentials(config)kwargs = _build_boto3_kwargs(creds)- logger.debug(- "Creating S3 client with HMAC authentication (boto3), "- f"endpoint={creds['endpoint_url']!r}, region={creds['region']!r}"- )+ logger.debug("Creating S3 client with HMAC authentication (boto3)")return boto3.client("s3", **kwargs)diff --git a/src/connectors/aws_s3/connector.py b/src/connectors/aws_s3/connector.pyindex 0b5ec749a..13929e41b 100644--- a/src/connectors/aws_s3/connector.py+++ b/src/connectors/aws_s3/connector.py@@ -117,7 +117,7 @@ class S3Connector(BaseConnector):try:resource = self._get_resource()buckets = [b.name for b in resource.buckets.all()]- logger.debug(f"S3 auto-discovered {len(buckets)} bucket(s): {buckets}")+ logger.debug("S3 auto-discovered %d bucket(s)", len(buckets))return bucketsexcept Exception as exc:logger.warning(f"S3 could not auto-discover buckets: {exc}")@@ -169,7 +169,7 @@ class S3Connector(BaseConnector):if max_files and len(files) >= max_files:return {"files": files, "next_page_token": None}except Exception as exc:- logger.error(f"Failed to list objects in S3 bucket {bucket_name!r}: {exc}")+ logger.error("Failed to list objects in S3 bucket: %s", exc)continuereturn {"files": files, "next_page_token": None}@@ -252,10 +252,7 @@ class S3Connector(BaseConnector):allowed_groups=[],)except Exception as exc:- logger.warning(- f"Could not fetch ACL for s3://{bucket}/{key}: {exc}. "- "Using fallback ACL."- )+ logger.warning("Could not fetch S3 object ACL, using fallback: %s", exc)return DocumentACL(owner=None, allowed_users=[], allowed_groups=[])# ------------------------------------------------------------------diff --git a/src/connectors/ibm_cos/connector.py b/src/connectors/ibm_cos/connector.pyindex 4a41a5b7d..46d21a2ee 100644--- a/src/connectors/ibm_cos/connector.py+++ b/src/connectors/ibm_cos/connector.py@@ -58,7 +58,12 @@ class IBMCOSConnector(BaseConnector):def get_client_id(self) -> str:"""Return IAM API key, or HMAC access key ID as fallback."""- val = os.getenv("IBM_COS_API_KEY") or os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID")+ val = (+ self.config.get("api_key")+ or self.config.get("hmac_access_key")+ or os.getenv("IBM_COS_API_KEY")+ or os.getenv("IBM_COS_HMAC_ACCESS_KEY_ID")+ )if val:return valraise ValueError(@@ -68,7 +73,12 @@ class IBMCOSConnector(BaseConnector):def get_client_secret(self) -> str:"""Return IAM service instance ID, or HMAC secret key as fallback."""- val = os.getenv("IBM_COS_SERVICE_INSTANCE_ID") or os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY")+ val = (+ self.config.get("service_instance_id")+ or self.config.get("hmac_secret_key")+ or os.getenv("IBM_COS_SERVICE_INSTANCE_ID")+ or os.getenv("IBM_COS_HMAC_SECRET_ACCESS_KEY")+ )if val:return valraise ValueError(@@ -142,7 +152,7 @@ class IBMCOSConnector(BaseConnector):else:resp = handle.list_buckets()buckets = [b["Name"] for b in resp.get("Buckets", [])]- logger.debug(f"IBM COS auto-discovered {len(buckets)} bucket(s): {buckets}")+ logger.debug("IBM COS auto-discovered %d bucket(s)", len(buckets))return bucketsexcept Exception as exc:logger.warning(f"IBM COS could not auto-discover buckets: {exc}")@@ -228,7 +238,7 @@ class IBMCOSConnector(BaseConnector):breakexcept Exception as exc:- logger.error(f"Failed to list objects in bucket {bucket_name!r}: {exc}")+ logger.error("Failed to list objects in IBM COS bucket: %s", exc)continuereturn {"files": files, "next_page_token": None}@@ -326,10 +336,7 @@ class IBMCOSConnector(BaseConnector):allowed_groups=[],)except Exception as exc:- logger.warning(- f"Could not fetch ACL for cos://{bucket}/{key}: {exc}. "- "Using fallback ACL."- )+ logger.warning("Could not fetch IBM COS object ACL, using fallback: %s", exc)return DocumentACL(owner=self._service_instance_id or None,allowed_users=[],