Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: add encryption to the config files and add IBM secrets manager (#1179)

Lucas Oliveirad09c3a142059
diff --git a/.env.example b/.env.example
index dce12c465..db6eb53d4 100644
--- a/.env.example
+++ b/.env.example
@@ -47,6 +47,22 @@ LANGFLOW_URL_INGEST_FLOW_ID=72c3d17c-2dac-4a73-b48a-6518473d7830
# LANGFLOW_INGEST_FLOW_ID=1402618b-e6d1-4ff2-9a11-d6ce71186915
NUDGES_FLOW_ID=ebc01d31-1976-46ce-a385-b0240327226c
+# Base64 AES-256-GCM Master Encryption Key
+# If not provided, OpenRAG will automatically generate a secure 32-byte key when you run the TUI or CLI wizard.
+# If completely unset during normal execution, the master encryption system will remain inactive.
+# This key is required to decrypt API credentials stored in local databases and provider indices.
+OPENRAG_ENCRYPTION_KEY=
+
+# Identifier for AAD tenant binding.
+# Used to verify that encrypted files belong to the expected tenant.
+# Default: openrag
+OPENRAG_TENANT_ID=openrag
+
+# If set to true, the application will fail to start if any critical
+# prerequisites are not met (e.g. master encryption key missing).
+# Default: false
+OPENRAG_ENFORCE_PREREQUISITES=false
+
# Set a strong admin password for OpenSearch; a bcrypt hash is generated at
# container startup from this value. Do not commit real secrets.
# must match the hashed password in secureconfig, must change for secure deployment!!!
diff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdx
index e2ba0470f..65484f2df 100644
--- a/docs/docs/reference/configuration.mdx
+++ b/docs/docs/reference/configuration.mdx
@@ -130,6 +130,16 @@ For more information, see [Configure connectors](/knowledge-connectors).
| `MICROSOFT_GRAPH_OAUTH_CLIENT_ID`<br/>`MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET` | Not set | Enable the [Microsoft Graph OAuth client](https://learn.microsoft.com/en-us/onedrive/developer/rest-api/getting-started/graph-oauth) integration by providing [Azure application registration credentials for SharePoint and OneDrive](https://learn.microsoft.com/en-us/onedrive/developer/rest-api/getting-started/app-registration?view=odsp-graph-online). |
| `WEBHOOK_BASE_URL` | Not set | Optional base URL for OAuth connector webhook endpoints. If not set, a default base URL is used. This variable is required to enable automatic cloud storage ingestion. |
+## Security and Encryption settings
+
+Configure cryptographic parameters and connection data isolation protocols.
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `OPENRAG_ENCRYPTION_KEY` | Not set | 32-byte Base64 AES-256-GCM master key locking all database index payloads. Automatically generated when you run the TUI or CLI configuration wizard. If left completely unset, the master encryption wrapper remains inactive and credentials remain in plaintext matrices. Warning: Cannot be lost or rotated without risking data-loss. |
+| `OPENRAG_TENANT_ID` | `openrag` | Identifier for AAD (Additional Authenticated Data) tenant binding. Ensures that encrypted files are correctly associated with the expected tenant environment. |
+| `IBM_AUTH_ENABLED` | `false` | Enabling this feature queries the IBM Secrets Manager API for your decryption keys natively. If the IBM API request is unauthorized, misconfigured, or fails, the system automatically falls back to evaluating your local `OPENRAG_ENCRYPTION_KEY`. |
+
## OpenSearch settings
Configure OpenSearch database authentication.
diff --git a/pyproject.toml b/pyproject.toml
index d6a869206..f91227fff 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,7 +42,8 @@ dependencies = [
"python-dotenv>=1.0.0",
"textual-fspicker>=0.6.0",
"structlog>=25.4.0",
- "zxcvbn>=4.5.0"
+ "zxcvbn>=4.5.0",
+ "ibm-secrets-manager-sdk>=2.1.19",
]
[dependency-groups]
diff --git a/src/config/config_manager.py b/src/config/config_manager.py
index 5f7954715..37fe089f9 100644
--- a/src/config/config_manager.py
+++ b/src/config/config_manager.py
@@ -116,12 +116,22 @@ class OpenRAGConfig:
def from_dict(cls, data: Dict[str, Any]) -> "OpenRAGConfig":
"""Create config from dictionary."""
providers_data = data.get("providers", {})
+
+ # Import inside to avoid circular dependencies if any
+ from utils.encryption import decrypt_secret
+
+ def _decrypt_provider(p_data: dict) -> dict:
+ new_data = dict(p_data)
+ if "api_key" in new_data:
+ new_data["api_key"] = decrypt_secret(new_data["api_key"])
+ return new_data
+
return cls(
providers=ProvidersConfig(
- openai=OpenAIConfig(**providers_data.get("openai", {})),
- anthropic=AnthropicConfig(**providers_data.get("anthropic", {})),
- watsonx=WatsonXConfig(**providers_data.get("watsonx", {})),
- ollama=OllamaConfig(**providers_data.get("ollama", {})),
+ openai=OpenAIConfig(**_decrypt_provider(providers_data.get("openai", {}))),
+ anthropic=AnthropicConfig(**_decrypt_provider(providers_data.get("anthropic", {}))),
+ watsonx=WatsonXConfig(**_decrypt_provider(providers_data.get("watsonx", {}))),
+ ollama=OllamaConfig(**_decrypt_provider(providers_data.get("ollama", {}))),
),
knowledge=KnowledgeConfig(**data.get("knowledge", {})),
agent=AgentConfig(**data.get("agent", {})),
@@ -154,6 +164,8 @@ class ConfigManager:
self.config_file = Path(config_file) if config_file else Path("config/config.yaml")
self._config: Optional[OpenRAGConfig] = None
+
+
def load_config(self) -> OpenRAGConfig:
"""Load configuration from environment variables and config file.
@@ -177,6 +189,9 @@ class ConfigManager:
"agent": {},
"onboarding": {},
}
+
+ needs_encryption_upgrade = False
+ from utils.encryption import get_master_secret
# Load from config file if it exists
if self.config_file.exists():
@@ -188,10 +203,12 @@ class ConfigManager:
if "providers" in file_config:
for provider in ["openai", "anthropic", "watsonx", "ollama"]:
if provider in file_config["providers"]:
- config_data["providers"][provider].update(
- file_config["providers"][provider]
- )
-
+ provider_data = file_config["providers"][provider]
+ # Check if api_key is unencrypted and we have a key
+ if "api_key" in provider_data and isinstance(provider_data["api_key"], str) and provider_data["api_key"]:
+ if get_master_secret() is not None:
+ needs_encryption_upgrade = True
+ config_data["providers"][provider].update(provider_data)
for section in ["knowledge", "agent", "onboarding"]:
if section in file_config:
config_data[section].update(file_config[section])
@@ -211,6 +228,10 @@ class ConfigManager:
# Create config object
self._config = OpenRAGConfig.from_dict(config_data)
+ if needs_encryption_upgrade:
+ logger.info("Upgrading unencrypted secrets in config.yaml to AES-256-GCM")
+ self.save_config_file(self._config, preserve_edited=True)
+
logger.debug("Configuration loaded", config=self._config.to_dict())
return self._config
@@ -285,11 +306,12 @@ class ConfigManager:
self._config = None
return self.load_config()
- def save_config_file(self, config: Optional[OpenRAGConfig] = None) -> bool:
+ def save_config_file(self, config: Optional[OpenRAGConfig] = None, preserve_edited: bool = False) -> bool:
"""Save configuration to file.
Args:
config: Configuration to save. If None, uses current config.
+ preserve_edited: If True, do not forcefully set the 'edited' flag upon saving.
Returns:
True if saved successfully, False otherwise.
@@ -297,15 +319,25 @@ class ConfigManager:
if config is None:
config = self.get_config()
- # Mark config as edited when saving
- config.edited = True
+ # Mark config as edited when saving manually
+ if not preserve_edited:
+ config.edited = True
try:
# Ensure directory exists
self.config_file.parent.mkdir(parents=True, exist_ok=True)
+ config_dict = config.to_dict()
+
+ # Encrypt provider API keys before saving
+ from utils.encryption import encrypt_secret
+ providers = config_dict.get("providers", {})
+ for provider_name, provider_config in providers.items():
+ if "api_key" in provider_config:
+ provider_config["api_key"] = encrypt_secret(provider_config["api_key"])
+
with open(self.config_file, "w") as f:
- yaml.dump(config.to_dict(), f, default_flow_style=False, indent=2)
+ yaml.dump(config_dict, f, default_flow_style=False, indent=2)
# Update cached config to reflect the edited flags
self._config = config
diff --git a/src/connectors/connection_manager.py b/src/connectors/connection_manager.py
index 65674bda6..bbf6bb203 100644
--- a/src/connectors/connection_manager.py
+++ b/src/connectors/connection_manager.py
@@ -48,11 +48,35 @@ class ConnectionManager:
async def load_connections(self):
"""Load connections from persistent storage"""
+ from utils.encryption import decrypt_secret, get_master_secret
+
+ needs_encryption_upgrade = False
+ decryption_failed = False
+ secret_keys = {
+ "api_key", "hmac_secret_key", "secret_key", "client_secret",
+ "aws_secret_access_key", "ibm_api_key", "access_token", "refresh_token",
+ "access_key", "hmac_access_key", "service_instance_id"
+ }
+
if self.connections_file.exists():
async with aiofiles.open(self.connections_file, "r") as f:
data = json.loads(await f.read())
for conn_data in data.get("connections", []):
+ # Decrypt sensitive fields
+ if "config" in conn_data and isinstance(conn_data["config"], dict):
+ for k, v in conn_data["config"].items():
+ if isinstance(v, dict) and v.get("algorithm") == "AES-256-GCM":
+ try:
+ tenant = conn_data.get("user_id") or "openrag"
+ conn_data["config"][k] = decrypt_secret(v, expected_tenant_id=tenant)
+ except ValueError as e:
+ logger.error(f"Failed to decrypt connection secret {k}: {e}")
+ decryption_failed = True
+ elif k in secret_keys and isinstance(v, str) and v:
+ if get_master_secret() is not None:
+ needs_encryption_upgrade = True
+
# Convert datetime strings back to datetime objects
if conn_data.get("created_at"):
conn_data["created_at"] = datetime.fromisoformat(
@@ -65,16 +89,45 @@ class ConnectionManager:
config = ConnectionConfig(**conn_data)
self.connections[config.connection_id] = config
+
+ if needs_encryption_upgrade:
+ if decryption_failed:
+ logger.warning(
+ "Detected unencrypted connection secrets in %s but skipped "
+ "encryption upgrade because some secrets failed to decrypt.",
+ self.connections_file,
+ )
+ else:
+ logger.info(
+ "Upgrading unencrypted connection secrets in %s to AES-256-GCM",
+ self.connections_file,
+ )
+ await self.save_connections()
# Now that connections are loaded, clean up duplicates
await self.cleanup_duplicate_connections(remove_duplicates=True)
async def save_connections(self):
"""Save connections to persistent storage"""
+ from utils.encryption import encrypt_secret
+ secret_keys = {
+ "api_key", "hmac_secret_key", "secret_key", "client_secret",
+ "aws_secret_access_key", "ibm_api_key", "access_token", "refresh_token",
+ "access_key", "hmac_access_key", "service_instance_id"
+ }
+
data = {"connections": []}
for config in self.connections.values():
conn_data = asdict(config)
+
+ # Encrypt sensitive fields in config
+ if "config" in conn_data and isinstance(conn_data["config"], dict):
+ for k, v in conn_data["config"].items():
+ if k in secret_keys and isinstance(v, str):
+ tenant_id = conn_data.get("user_id") or "openrag"
+ conn_data["config"][k] = encrypt_secret(v, tenant_id=tenant_id)
+
# Convert datetime objects to strings
if conn_data.get("created_at"):
conn_data["created_at"] = conn_data["created_at"].isoformat()
diff --git a/src/connectors/google_drive/oauth.py b/src/connectors/google_drive/oauth.py
index e9beb0a36..8a46be1e8 100644
--- a/src/connectors/google_drive/oauth.py
+++ b/src/connectors/google_drive/oauth.py
@@ -5,7 +5,7 @@ from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import Flow
from googleapiclient.discovery import build
-import aiofiles
+
class GoogleDriveOAuth:
@@ -35,28 +35,41 @@ class GoogleDriveOAuth:
async def load_credentials(self) -> Optional[Credentials]:
"""Load existing credentials from token file"""
- if os.path.exists(self.token_file):
- async with aiofiles.open(self.token_file, "r") as f:
- token_data = json.loads(await f.read())
-
- # Create credentials from token data
- self.creds = Credentials(
- token=token_data.get("token"),
- refresh_token=token_data.get("refresh_token"),
- id_token=token_data.get("id_token"),
- token_uri="https://oauth2.googleapis.com/token",
- client_id=self.client_id,
- client_secret=self.client_secret, # Need for refresh
- scopes=token_data.get("scopes", self.SCOPES),
- )
-
- # Set expiry if available (ensure timezone-naive for Google auth compatibility)
- if token_data.get("expiry"):
- from datetime import datetime
-
- expiry_dt = datetime.fromisoformat(token_data["expiry"])
- # Remove timezone info to make it naive (Google auth expects naive datetimes)
- self.creds.expiry = expiry_dt.replace(tzinfo=None)
+ from utils.encryption import read_encrypted_file
+
+ raw_data, needs_upgrade = await read_encrypted_file(self.token_file)
+ if raw_data is None:
+ return None
+
+ try:
+ token_data = json.loads(raw_data)
+ except Exception:
+ # Corrupted file, fallback to missing
+ if os.path.exists(self.token_file):
+ os.remove(self.token_file)
+ return None
+
+ # Create credentials from token data
+ self.creds = Credentials(
+ token=token_data.get("token"),
+ refresh_token=token_data.get("refresh_token"),
+ id_token=token_data.get("id_token"),
+ token_uri="https://oauth2.googleapis.com/token",
+ client_id=self.client_id,
+ client_secret=self.client_secret, # Need for refresh
+ scopes=token_data.get("scopes", self.SCOPES),
+ )
+
+ # Set expiry if available (ensure timezone-naive for Google auth compatibility)
+ if token_data.get("expiry"):
+ from datetime import datetime
+
+ expiry_dt = datetime.fromisoformat(token_data["expiry"])
+ # Remove timezone info to make it naive (Google auth expects naive datetimes)
+ self.creds.expiry = expiry_dt.replace(tzinfo=None)
+
+ if needs_upgrade and self.creds:
+ await self.save_credentials()
# If credentials are expired, refresh them
if self.creds and self.creds.expired and self.creds.refresh_token:
@@ -95,9 +108,9 @@ class GoogleDriveOAuth:
# Add expiry if available
if self.creds.expiry:
token_data["expiry"] = self.creds.expiry.isoformat()
-
- async with aiofiles.open(self.token_file, "w") as f:
- await f.write(json.dumps(token_data, indent=2))
+
+ from utils.encryption import write_encrypted_file
+ await write_encrypted_file(self.token_file, json.dumps(token_data))
def create_authorization_url(self, redirect_uri: str) -> str:
"""Create authorization URL for OAuth flow"""
diff --git a/src/connectors/onedrive/oauth.py b/src/connectors/onedrive/oauth.py
index 0b05d98df..e60feb5e3 100644
--- a/src/connectors/onedrive/oauth.py
+++ b/src/connectors/onedrive/oauth.py
@@ -3,7 +3,7 @@ import json
import logging
from typing import Optional, Dict, Any
-import aiofiles
+
import msal
logger = logging.getLogger(__name__)
@@ -80,59 +80,57 @@ class OneDriveOAuth:
async def load_credentials(self) -> bool:
"""Load existing credentials from token file (async)."""
try:
+ from utils.encryption import read_encrypted_file
+
logger.debug(f"OneDrive OAuth loading credentials from: {self.token_file}")
- if os.path.exists(self.token_file):
- logger.debug(f"Token file exists, reading: {self.token_file}")
-
- # Read the token file
- async with aiofiles.open(self.token_file, "r") as f:
- cache_data = await f.read()
- logger.debug(f"Read {len(cache_data)} chars from token file")
-
- if cache_data.strip():
- # 1) Try legacy flat JSON first
- try:
- json_data = json.loads(cache_data)
- if isinstance(json_data, dict) and "refresh_token" in json_data:
- if self.allow_json_refresh:
- logger.debug(
- "Found legacy JSON refresh_token and allow_json_refresh=True; attempting migration refresh"
- )
- return await self._refresh_from_json_token(json_data)
- else:
- logger.warning(
- "Token file contains a legacy JSON refresh_token, but allow_json_refresh=False. "
- "Delete the file and re-auth."
- )
- return False
- except json.JSONDecodeError:
- logger.debug("Token file is not flat JSON; attempting MSAL cache format")
-
- # 2) Try MSAL cache format
- logger.debug("Attempting MSAL cache deserialization")
- self.token_cache.deserialize(cache_data)
-
- # Get accounts from loaded cache
- accounts = self.app.get_accounts()
- logger.debug(f"Found {len(accounts)} accounts in MSAL cache")
- if accounts:
- self._current_account = accounts[0]
- logger.debug(f"Set current account: {self._current_account.get('username', 'no username')}")
-
- # Use RESOURCE_SCOPES (no reserved scopes) for silent acquisition
- result = self.app.acquire_token_silent(self.RESOURCE_SCOPES, account=self._current_account)
- logger.debug(f"Silent token acquisition result keys: {list(result.keys()) if result else 'None'}")
- if result and "access_token" in result:
- logger.debug("Silent token acquisition successful")
- await self.save_cache()
- return True
+ cache_data, needs_upgrade = await read_encrypted_file(self.token_file)
+
+ if cache_data and cache_data.strip():
+ # 1) Try legacy flat JSON first
+ try:
+ json_data = json.loads(cache_data)
+ if isinstance(json_data, dict) and "refresh_token" in json_data:
+ if self.allow_json_refresh:
+ logger.debug(
+ "Found legacy JSON refresh_token and allow_json_refresh=True; attempting migration refresh"
+ )
+ return await self._refresh_from_json_token(json_data)
else:
- error_msg = (result or {}).get("error") or "No result"
- logger.warning(f"Silent token acquisition failed: {error_msg}")
- else:
- logger.debug(f"Token file {self.token_file} is empty")
+ logger.warning(
+ "Token file contains a legacy JSON refresh_token, but allow_json_refresh=False. "
+ "Delete the file and re-auth."
+ )
+ return False
+ except json.JSONDecodeError:
+ logger.debug("Token file is not flat JSON; attempting MSAL cache format")
+
+ # 2) Try MSAL cache format
+ logger.debug("Attempting MSAL cache deserialization")
+ self.token_cache.deserialize(cache_data)
+
+ # Get accounts from loaded cache
+ accounts = self.app.get_accounts()
+ logger.debug(f"Found {len(accounts)} accounts in MSAL cache")
+ if accounts:
+ self._current_account = accounts[0]
+ logger.debug(f"Set current account: {self._current_account.get('username', 'no username')}")
+
+ if needs_upgrade:
+ await self.save_cache()
+
+ # Use RESOURCE_SCOPES (no reserved scopes) for silent acquisition
+ result = self.app.acquire_token_silent(self.RESOURCE_SCOPES, account=self._current_account)
+ logger.debug(f"Silent token acquisition result keys: {list(result.keys()) if result else 'None'}")
+ if result and "access_token" in result:
+ logger.debug("Silent token acquisition successful")
+ if getattr(self.token_cache, "has_state_changed", False):
+ await self.save_cache()
+ return True
+ else:
+ error_msg = (result or {}).get("error") or "No result"
+ logger.warning(f"Silent token acquisition failed: {error_msg}")
else:
- logger.debug(f"Token file does not exist: {self.token_file}")
+ logger.debug(f"Token file {self.token_file} is empty or does not exist")
return False
@@ -202,8 +200,9 @@ class OneDriveOAuth:
cache_data = self.token_cache.serialize()
if cache_data:
- async with aiofiles.open(self.token_file, "w") as f:
- await f.write(cache_data)
+ from utils.encryption import write_encrypted_file
+ await write_encrypted_file(self.token_file, cache_data)
+
logger.debug(f"Token cache saved to {self.token_file}")
except Exception as e:
logger.error(f"Failed to save token cache: {e}")
@@ -276,6 +275,8 @@ class OneDriveOAuth:
result = self.app.acquire_token_silent(self.RESOURCE_SCOPES, account=self._current_account)
if result and "access_token" in result:
logger.info("OneDrive is_authenticated: Successfully acquired token with account")
+ if getattr(self.token_cache, "has_state_changed", False):
+ await self.save_cache()
return True
else:
error_msg = (result or {}).get("error") or (result or {}).get("error_description") or "No result returned"
@@ -290,6 +291,8 @@ class OneDriveOAuth:
logger.info(f"OneDrive is_authenticated: Fallback succeeded, found {len(accounts)} accounts")
if accounts:
self._current_account = accounts[0]
+ if getattr(self.token_cache, "has_state_changed", False):
+ await self.save_cache()
return True
logger.warning(f"OneDrive is_authenticated: Fallback also failed, result: {result}")
diff --git a/src/connectors/sharepoint/oauth.py b/src/connectors/sharepoint/oauth.py
index 2a162b913..1bb81f786 100644
--- a/src/connectors/sharepoint/oauth.py
+++ b/src/connectors/sharepoint/oauth.py
@@ -3,7 +3,7 @@ import json
import logging
from typing import Optional, Dict, Any
-import aiofiles
+
import msal
logger = logging.getLogger(__name__)
@@ -80,59 +80,57 @@ class SharePointOAuth:
async def load_credentials(self) -> bool:
"""Load existing credentials from token file (async)."""
try:
+ from utils.encryption import read_encrypted_file
+
logger.debug(f"SharePoint OAuth loading credentials from: {self.token_file}")
- if os.path.exists(self.token_file):
- logger.debug(f"Token file exists, reading: {self.token_file}")
-
- # Read the token file
- async with aiofiles.open(self.token_file, "r") as f:
- cache_data = await f.read()
- logger.debug(f"Read {len(cache_data)} chars from token file")
-
- if cache_data.strip():
- # 1) Try legacy flat JSON first
- try:
- json_data = json.loads(cache_data)
- if isinstance(json_data, dict) and "refresh_token" in json_data:
- if self.allow_json_refresh:
- logger.debug(
- "Found legacy JSON refresh_token and allow_json_refresh=True; attempting migration refresh"
- )
- return await self._refresh_from_json_token(json_data)
- else:
- logger.warning(
- "Token file contains a legacy JSON refresh_token, but allow_json_refresh=False. "
- "Delete the file and re-auth."
- )
- return False
- except json.JSONDecodeError:
- logger.debug("Token file is not flat JSON; attempting MSAL cache format")
-
- # 2) Try MSAL cache format
- logger.debug("Attempting MSAL cache deserialization")
- self.token_cache.deserialize(cache_data)
-
- # Get accounts from loaded cache
- accounts = self.app.get_accounts()
- logger.debug(f"Found {len(accounts)} accounts in MSAL cache")
- if accounts:
- self._current_account = accounts[0]
- logger.debug(f"Set current account: {self._current_account.get('username', 'no username')}")
-
- # IMPORTANT: Use RESOURCE_SCOPES (no reserved scopes) for silent acquisition
- result = self.app.acquire_token_silent(self.RESOURCE_SCOPES, account=self._current_account)
- logger.debug(f"Silent token acquisition result keys: {list(result.keys()) if result else 'None'}")
- if result and "access_token" in result:
- logger.debug("Silent token acquisition successful")
- await self.save_cache()
- return True
+ cache_data, needs_upgrade = await read_encrypted_file(self.token_file)
+
+ if cache_data and cache_data.strip():
+ # 1) Try legacy flat JSON first
+ try:
+ json_data = json.loads(cache_data)
+ if isinstance(json_data, dict) and "refresh_token" in json_data:
+ if self.allow_json_refresh:
+ logger.debug(
+ "Found legacy JSON refresh_token and allow_json_refresh=True; attempting migration refresh"
+ )
+ return await self._refresh_from_json_token(json_data)
else:
- error_msg = (result or {}).get("error") or "No result"
- logger.warning(f"Silent token acquisition failed: {error_msg}")
- else:
- logger.debug(f"Token file {self.token_file} is empty")
+ logger.warning(
+ "Token file contains a legacy JSON refresh_token, but allow_json_refresh=False. "
+ "Delete the file and re-auth."
+ )
+ return False
+ except json.JSONDecodeError:
+ logger.debug("Token file is not flat JSON; attempting MSAL cache format")
+
+ # 2) Try MSAL cache format
+ logger.debug("Attempting MSAL cache deserialization")
+ self.token_cache.deserialize(cache_data)
+
+ # Get accounts from loaded cache
+ accounts = self.app.get_accounts()
+ logger.debug(f"Found {len(accounts)} accounts in MSAL cache")
+ if accounts:
+ self._current_account = accounts[0]
+ logger.debug(f"Set current account: {self._current_account.get('username', 'no username')}")
+
+ if needs_upgrade:
+ await self.save_cache()
+
+ # IMPORTANT: Use RESOURCE_SCOPES (no reserved scopes) for silent acquisition
+ result = self.app.acquire_token_silent(self.RESOURCE_SCOPES, account=self._current_account)
+ logger.debug(f"Silent token acquisition result keys: {list(result.keys()) if result else 'None'}")
+ if result and "access_token" in result:
+ logger.debug("Silent token acquisition successful")
+ if getattr(self.token_cache, "has_state_changed", False):
+ await self.save_cache()
+ return True
+ else:
+ error_msg = (result or {}).get("error") or "No result"
+ logger.warning(f"Silent token acquisition failed: {error_msg}")
else:
- logger.debug(f"Token file does not exist: {self.token_file}")
+ logger.debug(f"Token file {self.token_file} is empty or does not exist")
return False
@@ -205,8 +203,9 @@ class SharePointOAuth:
cache_data = self.token_cache.serialize()
if cache_data:
- async with aiofiles.open(self.token_file, "w") as f:
- await f.write(cache_data)
+ from utils.encryption import write_encrypted_file
+ await write_encrypted_file(self.token_file, cache_data)
+
logger.debug(f"Token cache saved to {self.token_file}")
except Exception as e:
logger.error(f"Failed to save token cache: {e}")
@@ -282,6 +281,8 @@ class SharePointOAuth:
result = self.app.acquire_token_silent(self.RESOURCE_SCOPES, account=self._current_account)
if result and "access_token" in result:
logger.info("SharePoint is_authenticated: Successfully acquired token with account")
+ if getattr(self.token_cache, "has_state_changed", False):
+ await self.save_cache()
return True
else:
error_msg = (result or {}).get("error") or (result or {}).get("error_description") or "No result returned"
@@ -297,6 +298,8 @@ class SharePointOAuth:
logger.info(f"SharePoint is_authenticated: Fallback succeeded, found {len(accounts)} accounts")
if accounts:
self._current_account = accounts[0]
+ if getattr(self.token_cache, "has_state_changed", False):
+ await self.save_cache()
return True
logger.warning(f"SharePoint is_authenticated: Fallback also failed, result: {result}")
diff --git a/src/main.py b/src/main.py
index c168f6c6a..041c573cc 100644
--- a/src/main.py
+++ b/src/main.py
@@ -17,6 +17,7 @@ from services.flows_service import FlowsService
from utils.container_utils import detect_container_environment
from utils.embeddings import create_dynamic_index_body
from utils.logging_config import configure_from_env, get_logger
+from utils.encryption import enforce_startup_prerequisites
from utils.telemetry import TelemetryClient, Category, MessageId
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
@@ -105,6 +106,7 @@ from services.task_service import TaskService
from session_manager import SessionManager
configure_from_env()
+enforce_startup_prerequisites()
logger = get_logger(__name__)
# Files to exclude from startup ingestion
diff --git a/src/services/auth_service.py b/src/services/auth_service.py
index b9bdc9044..ae34b0066 100644
--- a/src/services/auth_service.py
+++ b/src/services/auth_service.py
@@ -1,7 +1,7 @@
import uuid
import json
import httpx
-import aiofiles
+
import logging
from datetime import datetime, timedelta
from typing import Optional
@@ -314,8 +314,8 @@ class AuthService:
# Save tokens to file
token_file_path = connection_config.config["token_file"]
- async with aiofiles.open(token_file_path, "w") as f:
- await f.write(json.dumps(token_file_data, indent=2))
+ from utils.encryption import write_encrypted_file
+ await write_encrypted_file(token_file_path, json.dumps(token_file_data))
# Route based on purpose
purpose = connection_config.config.get("purpose", "data_source")
diff --git a/src/tui/config_fields.py b/src/tui/config_fields.py
index 184365a32..73bdcf37d 100644
--- a/src/tui/config_fields.py
+++ b/src/tui/config_fields.py
@@ -43,6 +43,27 @@ class ConfigSection:
CONFIG_SECTIONS: list[ConfigSection] = [
+ # ── Security ────────────────────────────────────────────────
+ ConfigSection("Security", [
+ ConfigField(
+ "openrag_encryption_key", "OPENRAG_ENCRYPTION_KEY", "OpenRAG Master Key",
+ placeholder="Auto-generated secure Base64 key",
+ secret=True, required=True,
+ helper_text="32-byte Base64 key for securing your database credentials (auto-generates if empty)",
+ ),
+ ConfigField(
+ "openrag_tenant_id", "OPENRAG_TENANT_ID", "Tenant ID",
+ placeholder="openrag", default="openrag",
+ helper_text="Identifier for AAD tenant binding (default: openrag)",
+ ),
+ ConfigField(
+ "openrag_enforce_prerequisites", "OPENRAG_ENFORCE_PREREQUISITES", "Enforce Prerequisites",
+ placeholder="false", default="false",
+ advanced=True,
+ helper_text="If true, application will fail to start if the encryption key is missing",
+ ),
+ ]),
+
# ── OpenSearch ──────────────────────────────────────────────
ConfigSection("OpenSearch", [
ConfigField(
diff --git a/src/tui/managers/env_manager.py b/src/tui/managers/env_manager.py
index 1a7780b65..791507885 100644
--- a/src/tui/managers/env_manager.py
+++ b/src/tui/managers/env_manager.py
@@ -29,6 +29,9 @@ class EnvConfig:
# Core settings
openai_api_key: str = ""
+ openrag_encryption_key: str = ""
+ openrag_tenant_id: str = "openrag"
+ openrag_enforce_prerequisites: str = "false"
opensearch_password: str = ""
opensearch_username: str = "admin"
opensearch_host: str = "opensearch"
@@ -159,6 +162,11 @@ class EnvManager:
"""Generate a secure secret key for Langflow."""
return secrets.token_urlsafe(32)
+ def generate_openrag_encryption_key(self) -> str:
+ """Generate a secure AES-256 base64 master key for OpenRAG."""
+ import base64
+ return base64.b64encode(secrets.token_bytes(32)).decode("ascii")
+
def _quote_env_value(self, value: str) -> str:
"""Single quote all environment variable values for consistency."""
if not value:
@@ -177,6 +185,9 @@ class EnvManager:
"WATSONX_API_KEY": "watsonx_api_key", # pragma: allowlist secret
"WATSONX_ENDPOINT": "watsonx_endpoint",
"WATSONX_PROJECT_ID": "watsonx_project_id",
+ "OPENRAG_ENCRYPTION_KEY": "openrag_encryption_key", # pragma: allowlist secret
+ "OPENRAG_TENANT_ID": "openrag_tenant_id",
+ "OPENRAG_ENFORCE_PREREQUISITES": "openrag_enforce_prerequisites",
"OPENSEARCH_PASSWORD": "opensearch_password", # pragma: allowlist secret
"OPENSEARCH_USERNAME": "opensearch_username",
"OPENSEARCH_HOST": "opensearch_host",
@@ -232,19 +243,17 @@ class EnvManager:
preserved_lines: list[str] = []
try:
+ is_managed_block = True
for raw_line in self.env_file.read_text().splitlines():
- stripped = raw_line.strip()
- if not stripped or stripped.startswith("#"):
- continue
-
match = EnvManager.assignment_pattern.match(raw_line)
- if not match:
- continue
-
- env_var = match.group(1)
- if env_var in managed_vars:
- continue
- preserved_lines.append(raw_line)
+ if match:
+ env_var = match.group(1)
+ is_managed_block = env_var in managed_vars
+ if not is_managed_block:
+ preserved_lines.append(raw_line)
+ elif not is_managed_block:
+ # Preserves multi-line backslash strings, empty splits and unmanaged comments natively inline
+ preserved_lines.append(raw_line)
except Exception:
logger.warning(
f"Failed to preserve custom .env lines from {self.env_file}",
@@ -297,6 +306,9 @@ class EnvManager:
if not self.config.langflow_secret_key:
self.config.langflow_secret_key = self.generate_langflow_secret_key()
+ if not self.config.openrag_encryption_key:
+ self.config.openrag_encryption_key = self.generate_openrag_encryption_key()
+
# Set OPENRAG_VERSION to TUI version if not already set
if not self.config.openrag_version:
try:
@@ -455,6 +467,9 @@ class EnvManager:
)
f.write(f"LANGFLOW_URL_INGEST_FLOW_ID={self._quote_env_value(self.config.langflow_url_ingest_flow_id)}\n")
f.write(f"NUDGES_FLOW_ID={self._quote_env_value(self.config.nudges_flow_id)}\n")
+ f.write(f"OPENRAG_ENCRYPTION_KEY={self._quote_env_value(self.config.openrag_encryption_key)}\n")
+ f.write(f"OPENRAG_TENANT_ID={self._quote_env_value(self.config.openrag_tenant_id)}\n")
+ f.write(f"OPENRAG_ENFORCE_PREREQUISITES={self._quote_env_value(self.config.openrag_enforce_prerequisites)}\n")
f.write(f"OPENSEARCH_PASSWORD={self._quote_env_value(self.config.opensearch_password)}\n")
if self.config.opensearch_username and self.config.opensearch_username != "admin":
f.write(f"OPENSEARCH_USERNAME={self._quote_env_value(self.config.opensearch_username)}\n")
@@ -635,6 +650,12 @@ class EnvManager:
"""Get fields required for no-auth setup mode. Returns (field_name, display_name, placeholder, can_generate)."""
return [
("openai_api_key", "OpenAI API Key", "sk-... or leave empty", False),
+ (
+ "openrag_encryption_key",
+ "OpenRAG Encryption Key",
+ "Will be auto-generated if empty",
+ True,
+ ),
(
"opensearch_password",
"OpenSearch Password",
diff --git a/src/tui/screens/config.py b/src/tui/screens/config.py
index 14639b034..54873b00d 100644
--- a/src/tui/screens/config.py
+++ b/src/tui/screens/config.py
@@ -1,7 +1,6 @@
"""Configuration screen for OpenRAG TUI."""
import os
-import re
from zxcvbn import zxcvbn
from textual.app import ComposeResult
from textual.containers import Container, Vertical, Horizontal, ScrollableContainer
@@ -476,6 +475,10 @@ class ConfigScreen(Screen):
if opensearch_input:
self.env_manager.config.opensearch_password = opensearch_input.value
+ encryption_key_input = self.inputs.get("openrag_encryption_key")
+ if encryption_key_input:
+ self.env_manager.config.openrag_encryption_key = encryption_key_input.value
+
# Only generate OpenSearch password if empty
if not self.env_manager.config.opensearch_password:
self.env_manager.config.opensearch_password = self.env_manager.generate_secure_password()
@@ -484,11 +487,17 @@ class ConfigScreen(Screen):
if not self.env_manager.config.langflow_secret_key:
self.env_manager.config.langflow_secret_key = self.env_manager.generate_langflow_secret_key()
+ if not self.env_manager.config.openrag_encryption_key:
+ self.env_manager.config.openrag_encryption_key = self.env_manager.generate_openrag_encryption_key()
+
# Update input fields with generated values
if opensearch_input:
opensearch_input.value = self.env_manager.config.opensearch_password
- self.notify("Generated secure password for OpenSearch", severity="information")
+ if encryption_key_input:
+ encryption_key_input.value = self.env_manager.config.openrag_encryption_key
+
+ self.notify("Generated secure passwords and encryption keys", severity="information")
def action_save(self) -> None:
"""Save the configuration."""
diff --git a/src/utils/encryption.py b/src/utils/encryption.py
new file mode 100644
index 000000000..6c43513f1
--- /dev/null
+++ b/src/utils/encryption.py
@@ -0,0 +1,257 @@
+import os
+import secrets
+import base64
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+from typing import Union, Dict, Any, Tuple, Optional
+import aiofiles
+import json
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+# KDF Constants
+ENCRYPTION_ALGORITHM = "AES-256-GCM"
+KDF_ALGORITHM = hashes.SHA256()
+KDF_ITERATIONS = 100000
+
+_cached_master_secret: Optional[str] = None
+
+def get_master_secret() -> str | None:
+ """Retrieve the master secret string from IBM Secrets Manager or local environment."""
+ global _cached_master_secret
+ if _cached_master_secret is not None:
+ return _cached_master_secret
+
+ secret_str = None
+ ibm_auth_enabled = os.environ.get("IBM_AUTH_ENABLED", "false").lower() in ("true", "1", "yes")
+
+ if ibm_auth_enabled:
+ ibm_api_key = os.environ.get("IBM_CLOUD_API_KEY")
+ ibm_secret_id = os.environ.get("IBM_SECRETS_MANAGER_SECRET_ID")
+
+ trusted_profile_id = os.environ.get("IBM_CLOUD_TRUSTED_PROFILE_ID")
+ instance_id = os.environ.get("SECRET_MANAGER_INSTANCE_ID")
+ region = os.environ.get("SECRET_MANAGER_REGION", "us-south")
+ cr_token_file = os.environ.get("IBM_CLOUD_CR_TOKEN_FILE", "/var/run/secrets/tokens/sa-token")
+
+ if instance_id and ibm_secret_id:
+ try:
+ from ibm_secrets_manager_sdk.secrets_manager_v2 import SecretsManagerV2
+
+ authenticator = None
+ if ibm_api_key:
+ from ibm_cloud_sdk_core.authenticators import IAMAuthenticator
+ authenticator = IAMAuthenticator(ibm_api_key)
+ elif trusted_profile_id:
+ from ibm_cloud_sdk_core.authenticators import ContainerAuthenticator
+ authenticator = ContainerAuthenticator(
+ cr_token_filename=cr_token_file,
+ iam_profile_id=trusted_profile_id
+ )
+ else:
+ raise ValueError("Neither IBM_CLOUD_API_KEY nor IBM_CLOUD_TRUSTED_PROFILE_ID provided")
+
+ secrets_manager = SecretsManagerV2(authenticator=authenticator)
+ secrets_manager.set_service_url(f"https://{instance_id}.{region}.secrets-manager.appdomain.cloud")
+
+ # Retrieve the secret
+ response = secrets_manager.get_secret(id=ibm_secret_id).get_result()
+
+ secret_data = response.get("secret_data")
+ if not secret_data and "resources" in response and len(response["resources"]) > 0:
+ secret_data = response["resources"][0].get("secret_data")
+
+ if secret_data and "payload" in secret_data:
+ secret_str = secret_data["payload"]
+ logger.debug("Successfully retrieved master secret from IBM Secrets Manager.")
+ else:
+ logger.warning("IBM Secrets Manager: 'payload' not found in secret_data.")
+ except Exception as e:
+ logger.warning(f"Failed to retrieve encryption key from IBM Secrets Manager: {e}. Falling back to OPENRAG_ENCRYPTION_KEY.")
+
+ if not secret_str:
+ secret_str = os.environ.get("OPENRAG_ENCRYPTION_KEY")
+
+ if not secret_str:
+ if os.environ.get("OPENRAG_ENFORCE_PREREQUISITES", "false").lower() in ("true", "1", "yes"):
+ raise RuntimeError(
+ "CRITICAL: OPENRAG_ENFORCE_PREREQUISITES is enabled but no master encryption key "
+ "could be retrieved from IBM Secrets Manager or OPENRAG_ENCRYPTION_KEY. "
+ "Application will not start in unencrypted mode."
+ )
+ return None
+
+ _cached_master_secret = secret_str
+ return secret_str
+
+
+def enforce_startup_prerequisites():
+ """
+ Validates that the encryption master secret is available if ENFORCE_PREREQUISITES is set.
+ This should be called early during application startup.
+ """
+ try:
+ get_master_secret()
+ except RuntimeError as e:
+ logger.critical(str(e))
+ import sys
+ sys.exit(1)
+
+
+
+def encrypt_secret(plaintext: str, tenant_id: str = "openrag") -> Union[Dict[str, Any], str]:
+ """
+ Encrypt a plaintext secret using AES-256-GCM and PBKDF2HMAC.
+ Returns a JSON-serializable dictionary with the ciphertext and metadata.
+ If master secret is not set, returns the plaintext string for backward compatibility.
+ """
+ if not isinstance(plaintext, str) or not plaintext:
+ return plaintext
+
+ master_secret = get_master_secret()
+ if not master_secret:
+ return plaintext
+
+ try:
+ salt = secrets.token_bytes(16)
+ kdf = PBKDF2HMAC(
+ algorithm=KDF_ALGORITHM,
+ length=32,
+ salt=salt,
+ iterations=KDF_ITERATIONS,
+ )
+ derived_key = kdf.derive(master_secret.encode("utf-8"))
+
+ aesgcm = AESGCM(derived_key)
+ nonce = secrets.token_bytes(12)
+ plaintext_bytes = plaintext.encode("utf-8")
+ aad = f"tenant_id:{tenant_id}".encode("utf-8")
+
+ ciphertext = aesgcm.encrypt(nonce, plaintext_bytes, aad)
+
+ return {
+ "version": "1.0",
+ "algorithm": ENCRYPTION_ALGORITHM,
+ "kdf": "PBKDF2HMAC-SHA256",
+ "tenant_id": tenant_id,
+ "salt": base64.b64encode(salt).decode("ascii"),
+ "nonce": base64.b64encode(nonce).decode("ascii"),
+ "ciphertext": base64.b64encode(ciphertext).decode("ascii"),
+ }
+ except Exception as e:
+ logger.error(f"Failed to encrypt secret: {e}")
+ # If encryption fails, fallback to plaintext so we don't lose data
+ return plaintext
+
+
+def decrypt_secret(payload: Union[Dict[str, Any], str], expected_tenant_id: Optional[str] = None) -> str:
+ """
+ Decrypt a secret payload using AES-256-GCM.
+ Supports backward compatibility with non-KDF base64 raw keys.
+ If expected_tenant_id is provided, it is used as the authoritative tenant identifier
+ for constructing the AES-GCM AAD, and the payload's tenant_id (if present) must match it.
+ """
+ if not isinstance(payload, dict):
+ return payload
+
+ if payload.get("algorithm") != ENCRYPTION_ALGORITHM or "ciphertext" not in payload:
+ raise ValueError(
+ "Invalid encrypted secret payload: expected AES-256-GCM envelope with 'algorithm' "
+ "set to 'AES-256-GCM' and a 'ciphertext' field."
+ )
+
+ master_secret = get_master_secret()
+ if not master_secret:
+ raise ValueError(
+ "Master secret not found in environment, but encrypted secret detected in config."
+ )
+
+ try:
+ # Backward compatibility for originally raw base64 32-byte keys
+ if "salt" in payload:
+ salt = base64.b64decode(payload["salt"])
+ kdf = PBKDF2HMAC(
+ algorithm=KDF_ALGORITHM,
+ length=32,
+ salt=salt,
+ iterations=KDF_ITERATIONS,
+ )
+ key = kdf.derive(master_secret.encode("utf-8"))
+ else:
+ # Legacy assumption: master_secret was exactly 32 bytes of raw base64 data
+ key = base64.b64decode(master_secret)
+
+ aesgcm = AESGCM(key)
+ nonce = base64.b64decode(payload["nonce"])
+ ciphertext = base64.b64decode(payload["ciphertext"])
+
+ # Determine tenant_id for AAD using a trusted expected value when available.
+ payload_tenant_id = payload.get("tenant_id")
+ if expected_tenant_id is not None:
+ if payload_tenant_id is not None and payload_tenant_id != expected_tenant_id:
+ raise ValueError(
+ f"Tenant ID in payload ('{payload_tenant_id}') does not match expected tenant ID."
+ )
+ tenant_id = expected_tenant_id
+ else:
+ # Backwards-compatible behaviour when no external tenant binding is configured.
+ tenant_id = payload_tenant_id or "openrag"
+
+ aad = f"tenant_id:{tenant_id}".encode("utf-8")
+
+ plaintext_bytes = aesgcm.decrypt(nonce, ciphertext, aad)
+ return plaintext_bytes.decode("utf-8")
+ except Exception as e:
+ logger.error(f"Failed to decrypt secret: {e}")
+ raise ValueError(f"Failed to decrypt secret: {e}")
+
+async def read_encrypted_file(file_path: str) -> Tuple[Optional[str], bool]:
+ """
+ Reads an encrypted or plaintext JSON/string file.
+ Returns a tuple: (file_content_as_string, needs_upgrade_boolean)
+ """
+ if not os.path.exists(file_path):
+ return None, False
+
+ try:
+ async with aiofiles.open(file_path, "r") as f:
+ raw_data = await f.read()
+
+ if not raw_data.strip():
+ return raw_data, False
+
+ file_json = json.loads(raw_data)
+ if isinstance(file_json, dict) and file_json.get("algorithm") == ENCRYPTION_ALGORITHM:
+ expected_tenant_id = os.getenv("OPENRAG_TENANT_ID")
+ decrypted_str = decrypt_secret(file_json, expected_tenant_id=expected_tenant_id)
+ return decrypted_str, False
+ else:
+ # It's plaintext
+ needs_upgrade = get_master_secret() is not None
+ return raw_data, needs_upgrade
+ except json.JSONDecodeError:
+ # Not a JSON dict, could be MSAL plaintext string or something else
+ needs_upgrade = get_master_secret() is not None
+ return raw_data, needs_upgrade
+ except Exception as e:
+ logger.error(f"Failed to read encrypted file {file_path}: {e}")
+ return None, False
+
+async def write_encrypted_file(file_path: str, data: str):
+ """
+ Encrypts string data (if key is present) and writes to file.
+ """
+ tenant_id = os.getenv("OPENRAG_TENANT_ID") or "openrag"
+ encrypted = encrypt_secret(data, tenant_id=tenant_id)
+ payload_to_write = json.dumps(encrypted, indent=2) if isinstance(encrypted, dict) else data
+
+ # Ensure parent dir exists
+ parent = os.path.dirname(os.path.abspath(file_path))
+ if parent and not os.path.exists(parent):
+ os.makedirs(parent, exist_ok=True)
+
+ async with aiofiles.open(file_path, "w") as f:
+ await f.write(payload_to_write)
+
diff --git a/tests/unit/test_encryption.py b/tests/unit/test_encryption.py
new file mode 100644
index 000000000..33219183d
--- /dev/null
+++ b/tests/unit/test_encryption.py
@@ -0,0 +1,162 @@
+import base64
+
+import pytest
+
+from config.config_manager import ConfigManager
+from connectors.connection_manager import ConnectionManager
+from utils.encryption import decrypt_secret, encrypt_secret
+
[email protected](autouse=True)
+def setup_encryption_env(monkeypatch):
+ import utils.encryption
+ utils.encryption._cached_master_secret = None
+ monkeypatch.setenv("OPENRAG_ENCRYPTION_KEY", base64.b64encode(b"0123456789abcdef0123456789abcdef").decode("ascii"))
+
+def test_encryption_utility():
+ print("Testing encryption utility...")
+ plaintext = "super-secret-api-key"
+ payload = encrypt_secret(plaintext, "tenant-1")
+ assert isinstance(payload, dict)
+ assert payload["algorithm"] == "AES-256-GCM"
+ assert payload["tenant_id"] == "tenant-1"
+
+ # Passing explicitly matches original AAD
+ decrypted = decrypt_secret(payload, expected_tenant_id="tenant-1")
+ assert decrypted == plaintext
+
+ # Missing explicit bound works generically using payload lookup
+ decrypted_fallback = decrypt_secret(payload)
+ assert decrypted_fallback == plaintext
+
+ # Spoofing the identity dynamically rejects the AES-GCM tags!
+ try:
+ decrypt_secret(payload, expected_tenant_id="wrong-tenant-id")
+ assert False, "Should have thrown ValueError from AESGCM AAD mismatch"
+ except ValueError:
+ pass
+ print("OK")
+
+def test_config_manager(tmp_path):
+ print("Testing config manager encryption...")
+ # Create an initial config manager with a temporary file
+ test_yaml = tmp_path / "test_openrag_config.yaml"
+ if test_yaml.exists():
+ test_yaml.unlink()
+
+ cm = ConfigManager(str(test_yaml))
+ config = cm.get_config()
+ config.providers.openai.api_key = "openai-api-key-plaintext"
+
+ # Save the config
+ cm.save_config_file(config)
+
+ import yaml
+ with open(test_yaml, "r") as f:
+ saved_data = yaml.safe_load(f)
+
+ # Verify it was encrypted on disk
+ assert isinstance(saved_data["providers"]["openai"]["api_key"], dict)
+ assert saved_data["providers"]["openai"]["api_key"]["algorithm"] == "AES-256-GCM"
+
+ # Verify it can be loaded correctly
+ cm_new = ConfigManager(str(test_yaml))
+ config_new = cm_new.get_config()
+ assert config_new.providers.openai.api_key == "openai-api-key-plaintext"
+ print("OK")
+
+async def test_connection_manager(tmp_path):
+ print("Testing connection manager encryption...")
+ test_json = tmp_path / "test_openrag_connections.json"
+ if test_json.exists():
+ test_json.unlink()
+
+ cm = ConnectionManager(str(test_json))
+ await cm.create_connection(
+ connector_type="google_drive",
+ name="Test Drive",
+ config={"client_secret": "my-client-secret-plaintext", "other_setting": "not-secret"},
+ user_id="user-1"
+ )
+ await cm.create_connection(
+ connector_type="aws_s3",
+ name="Test S3",
+ config={"secret_key": "aws-secret", "access_key": "aws-access", "bucket_names": ["foo"]},
+ user_id="user-1"
+ )
+ await cm.create_connection(
+ connector_type="ibm_cos",
+ name="Test IBM",
+ config={"api_key": "ibm-api-key", "service_instance_id": "ibm-service", "bucket_names": ["bar"]},
+ user_id="user-1"
+ )
+ # Should be saved encrypted
+ import json
+ with open(test_json, "r") as f:
+ data = json.load(f)
+
+ found_gd = False
+ found_s3 = False
+ found_ibm = False
+ for c in data["connections"]:
+ if c["connector_type"] == "google_drive":
+ found_gd = True
+ assert isinstance(c["config"]["client_secret"], dict)
+ assert c["config"]["client_secret"]["algorithm"] == "AES-256-GCM"
+ assert c["config"]["other_setting"] == "not-secret"
+ if c["connector_type"] == "aws_s3":
+ found_s3 = True
+ assert isinstance(c["config"]["secret_key"], dict)
+ assert c["config"]["secret_key"]["algorithm"] == "AES-256-GCM"
+ assert isinstance(c["config"]["access_key"], dict)
+ assert c["config"]["bucket_names"] == ["foo"]
+ if c["connector_type"] == "ibm_cos":
+ found_ibm = True
+ assert isinstance(c["config"]["api_key"], dict)
+ assert c["config"]["api_key"]["algorithm"] == "AES-256-GCM"
+ assert isinstance(c["config"]["service_instance_id"], dict)
+ assert found_gd and found_s3 and found_ibm
+
+ # Reloading should decrypt
+ cm2 = ConnectionManager(str(test_json))
+ await cm2.load_connections()
+
+ found_gd = False
+ found_s3 = False
+ found_ibm = False
+ for c in cm2.connections.values():
+ if c.connector_type == "google_drive":
+ found_gd = True
+ assert c.config["client_secret"] == "my-client-secret-plaintext"
+ assert c.config["other_setting"] == "not-secret"
+ elif c.connector_type == "aws_s3":
+ found_s3 = True
+ assert c.config["secret_key"] == "aws-secret"
+ assert c.config["access_key"] == "aws-access"
+ elif c.connector_type == "ibm_cos":
+ found_ibm = True
+ assert c.config["api_key"] == "ibm-api-key"
+ assert c.config["service_instance_id"] == "ibm-service"
+ assert found_gd and found_s3 and found_ibm
+ print("OK")
+
+def test_auto_upgrade_features(tmp_path):
+ test_yaml = tmp_path / "test_openrag_config_upgrade.yaml"
+ import yaml
+ # Write purely plaintext config
+ with open(test_yaml, "w") as f:
+ yaml.dump({
+ "providers": {
+ "openai": {"api_key": "raw-unencrypted-openai-key-from-past"}
+ }
+ }, f)
+
+ cm = ConfigManager(str(test_yaml))
+ cm.get_config()
+ # Upon loading, the auto-upgrade should save the file over itself with the encrypted key.
+ with open(test_yaml, "r") as f:
+ upgraded_data = yaml.safe_load(f)
+ assert isinstance(upgraded_data["providers"]["openai"]["api_key"], dict), "Failed to auto-upgrade config"
+ assert upgraded_data["providers"]["openai"]["api_key"]["algorithm"] == "AES-256-GCM"
+ assert cm.get_config().providers.openai.api_key == "raw-unencrypted-openai-key-from-past"
+ print("Auto-upgrade OK")
diff --git a/tests/unit/test_encryption_enforcement.py b/tests/unit/test_encryption_enforcement.py
new file mode 100644
index 000000000..9cf599a1c
--- /dev/null
+++ b/tests/unit/test_encryption_enforcement.py
@@ -0,0 +1,31 @@
+import os
+import pytest
+from unittest.mock import patch
+from utils.encryption import get_master_secret
+
[email protected](autouse=True)
+def reset_encryption_cache():
+ """Reset the encryption cache before and after each test."""
+ import utils.encryption
+ utils.encryption._cached_master_secret = None
+ yield
+ utils.encryption._cached_master_secret = None
+
+def test_enforce_prerequisites_fail():
+ """Test that get_master_secret raises RuntimeError when enforcement is enabled and key is missing."""
+ with patch.dict(os.environ, {"OPENRAG_ENFORCE_PREREQUISITES": "true", "OPENRAG_ENCRYPTION_KEY": "", "IBM_AUTH_ENABLED": "false"}):
+ with pytest.raises(RuntimeError) as excinfo:
+ get_master_secret()
+ assert "OPENRAG_ENFORCE_PREREQUISITES is enabled but no master encryption key could be retrieved" in str(excinfo.value)
+
+def test_enforce_prerequisites_success():
+ """Test that get_master_secret succeeds when enforcement is enabled and key is present."""
+ with patch.dict(os.environ, {"OPENRAG_ENFORCE_PREREQUISITES": "true", "OPENRAG_ENCRYPTION_KEY": "some-key", "IBM_AUTH_ENABLED": "false"}):
+ secret = get_master_secret()
+ assert secret == "some-key"
+
+def test_no_enforce_prerequisites_none():
+ """Test that get_master_secret returns None when enforcement is disabled and key is missing."""
+ with patch.dict(os.environ, {"OPENRAG_ENFORCE_PREREQUISITES": "false", "OPENRAG_ENCRYPTION_KEY": "", "IBM_AUTH_ENABLED": "false"}):
+ secret = get_master_secret()
+ assert secret is None
diff --git a/tests/unit/test_env_manager.py b/tests/unit/test_env_manager.py
index e07eb51d8..91dc61f8d 100644
--- a/tests/unit/test_env_manager.py
+++ b/tests/unit/test_env_manager.py
@@ -133,8 +133,8 @@ class TestSaveEnvFilePermissions:
assert multiline_block in content
assert content.count("UNMANAGED_MULTILINE=") == 1
# Managed password should be updated, not duplicated.
- assert 'OPENSEARCH_PASSWORD="NewSecurePass!456"' in content
- assert 'OPENSEARCH_PASSWORD="old-password"' not in content
+ assert "OPENSEARCH_PASSWORD='NewSecurePass!456'" in content
+ assert "OPENSEARCH_PASSWORD='old-password'" not in content
def test_preserves_unmanaged_continued_line(self, env_manager, tmp_path):
"""Unmanaged values using backslash continuation are preserved."""
diff --git a/tests/unit/test_ibm_encryption.py b/tests/unit/test_ibm_encryption.py
new file mode 100644
index 000000000..906118e24
--- /dev/null
+++ b/tests/unit/test_ibm_encryption.py
@@ -0,0 +1,59 @@
+import base64
+
+import pytest
+
+from utils import encryption
+
+def run_env(monkeypatch):
+ monkeypatch.setenv("OPENRAG_ENCRYPTION_KEY", base64.b64encode(b"B" * 32).decode("ascii"))
+ monkeypatch.delenv("IBM_CLOUD_API_KEY", raising=False)
+ monkeypatch.delenv("IBM_CLOUD_TRUSTED_PROFILE_ID", raising=False)
+ monkeypatch.setattr(encryption, "_cached_master_secret", None)
+ return monkeypatch
+
+def test_fallback_when_credentials_missing(run_env):
+ # No IBM keys set here
+ key = encryption.get_master_secret()
+ assert key == base64.b64encode(b"B" * 32).decode("ascii")
+ print("Fallback with missing IBM creds successful")
+
+def test_cache_mechanism(run_env):
+ k1 = encryption.get_master_secret()
+ assert encryption._cached_master_secret is not None
+
+ # Remove local env ensuring it hits cache
+ run_env.delenv("OPENRAG_ENCRYPTION_KEY", raising=False)
+ k2 = encryption.get_master_secret()
+ assert k2 == k1
+ print("Caching optimization successful")
+
+def test_ibm_exception_handling(run_env):
+ run_env.setenv("IBM_AUTH_ENABLED", "true")
+ run_env.setenv("IBM_CLOUD_API_KEY", "fake")
+ run_env.setenv("SECRET_MANAGER_INSTANCE_ID", "fake")
+ run_env.setenv("IBM_SECRETS_MANAGER_SECRET_ID", "fake")
+ run_env.setenv("OPENRAG_ENCRYPTION_KEY", base64.b64encode(b"C" * 32).decode("ascii"))
+
+ from unittest.mock import patch
+ with patch("ibm_secrets_manager_sdk.secrets_manager_v2.SecretsManagerV2", side_effect=ValueError("Simulating SDK crash")):
+ key = encryption.get_master_secret()
+ assert key == base64.b64encode(b"C" * 32).decode("ascii")
+ print("IBM SDK exception intercept successful")
+
+def test_trusted_profiles_fallback(run_env):
+ run_env.delenv("IBM_CLOUD_API_KEY", raising=False)
+
+ run_env.setenv("IBM_AUTH_ENABLED", "true")
+ run_env.setenv("SECRET_MANAGER_INSTANCE_ID", "fake")
+ run_env.setenv("IBM_SECRETS_MANAGER_SECRET_ID", "fake")
+ run_env.setenv("IBM_CLOUD_TRUSTED_PROFILE_ID", "my-profile")
+ run_env.setenv("SECRET_MANAGER_REGION", "us-east")
+ run_env.setenv("OPENRAG_ENCRYPTION_KEY", base64.b64encode(b"D" * 32).decode("ascii"))
+
+ from unittest.mock import patch
+ with patch("ibm_secrets_manager_sdk.secrets_manager_v2.SecretsManagerV2", side_effect=ValueError("Simulating SDK crash")):
+ key = encryption.get_master_secret()
+ assert key == base64.b64encode(b"D" * 32).decode("ascii")
+ print("Trusted Profile exception intercept successful")
+
diff --git a/tests/unit/test_oauth_encryption.py b/tests/unit/test_oauth_encryption.py
new file mode 100644
index 000000000..2d0b6c0d0
--- /dev/null
+++ b/tests/unit/test_oauth_encryption.py
@@ -0,0 +1,66 @@
+import json
+
+import pytest
+from connectors.google_drive.oauth import GoogleDriveOAuth
+from connectors.onedrive.oauth import OneDriveOAuth
+
+async def test_google_drive_auth_upgrade(tmp_path, monkeypatch):
+ """Test that Google Drive credentials are encrypted on load if not already."""
+ import utils.encryption
+ # Reset internal cache to ensure it reads from env
+ utils.encryption._cached_master_secret = None
+ monkeypatch.setenv("OPENRAG_ENCRYPTION_KEY", "dGVzdC1rZXktMzItYnl0ZXMtZm9yLXVuaXQtdGVzdGluZy1vbmx5") # 32-byte base64
+
+ print("Testing Google Drive Auth Upgrade...")
+ g_token_path = tmp_path / "fake_gdrive_token.json"
+ g_plain = {
+ "token": "fake-google-token",
+ "refresh_token": "fake-refresh",
+ "scopes": ["some-scope"],
+ "expiry": "2028-03-02T15:46:11.368083"
+ }
+ with open(g_token_path, "w") as f:
+ json.dump(g_plain, f)
+
+ g_oauth = GoogleDriveOAuth(client_id="abc", client_secret="def", token_file=str(g_token_path))
+ try:
+ await g_oauth.load_credentials()
+ except ValueError:
+ print("Expected network validation error caught. Proceeding to verify disk encryption...")
+
+ # Check if upgraded
+ with open(g_token_path, "r") as f:
+ upgraded = json.load(f)
+ assert "algorithm" in upgraded and upgraded["algorithm"] == "AES-256-GCM"
+ print("Google Drive Auth OK")
+
+async def test_msal_auth_upgrade(tmp_path, monkeypatch):
+ """Test that MSAL credentials are encrypted on load if not already."""
+ import utils.encryption
+ # Reset internal cache to ensure it reads from env
+ utils.encryption._cached_master_secret = None
+ monkeypatch.setenv("OPENRAG_ENCRYPTION_KEY", "dGVzdC1rZXktMzItYnl0ZXMtZm9yLXVuaXQtdGVzdGluZy1vbmx5") # 32-byte base64
+
+ print("Testing MSAL Auth Upgrade...")
+ m_token_path = tmp_path / "fake_msal_token.json"
+ m_plain = {
+ "refresh_token": "legacy-flat-refresh-token"
+ }
+ with open(m_token_path, "w") as f:
+ json.dump(m_plain, f)
+
+ m_oauth = OneDriveOAuth(client_id="abc", client_secret="def", token_file=str(m_token_path))
+
+ # We monkey patch MSAL acquire_token_by_refresh_token to pretend it worked
+ m_oauth.app.acquire_token_by_refresh_token = lambda refresh_token, scopes: {"access_token": "new-access-token"}
+ m_oauth.app.get_accounts = lambda: [{"username": "fake-user"}]
+
+ result = await m_oauth.load_credentials()
+ assert result
+
+ with open(m_token_path, "r") as f:
+ upgraded_m = json.load(f)
+ assert "algorithm" in upgraded_m and upgraded_m["algorithm"] == "AES-256-GCM"
+ print("MSAL Auth OK")
diff --git a/uv.lock b/uv.lock
index f523b6195..a16475cdb 100644
--- a/uv.lock
+++ b/uv.lock
@@ -819,6 +819,21 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d5/ae/2f6d96b4e6c5478d87d606a1934b5d436c4a2bce6bb7c6fdece891c128e3/huggingface_hub-1.4.1-py3-none-any.whl", hash = "sha256:9931d075fb7a79af5abc487106414ec5fba2c0ae86104c0c62fd6cae38873d18", size = 553326, upload-time = "2026-02-06T09:20:00.728Z" },
]
+[[package]]
+name = "ibm-cloud-sdk-core"
+version = "3.24.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "pyjwt" },
+ { name = "python-dateutil" },
+ { name = "requests" },
+ { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/4a/0c/965f4f4b34f1a9a2210bec224ac489c631153f18ef75c6e694750efda654/ibm_cloud_sdk_core-3.24.4.tar.gz", hash = "sha256:20498959ce0177a58938e1855ee09f08b9712e43cc3209e95010c046e8c6d2a6", size = 81778, upload-time = "2026-02-03T08:44:42.217Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/5e/8d/90cf967468fd81f9fd22bd42dbf3b1fbb1234485d1f81b6f17c8de138fff/ibm_cloud_sdk_core-3.24.4-py3-none-any.whl", hash = "sha256:4c3487ed5eb5180770ea2d07d95cfed1e69002ed249bbbc7d155226985c0b785", size = 75786, upload-time = "2026-02-03T08:44:41.094Z" },
+]
+
[[package]]
name = "ibm-cos-sdk"
version = "2.16.0"
@@ -851,6 +866,21 @@ dependencies = [
]
sdist = { url = "https://files.pythonhosted.org/packages/b3/6f/660a6e0b7015a512304487c17c4ac87c280180b0a6142ee196570efc221e/ibm_cos_sdk_s3transfer-2.16.0.tar.gz", hash = "sha256:5cca69c48dcb7a1442b39cadb3a635162fbc2cf002820c8cd6cb170056d27c5c", size = 141138, upload-time = "2026-01-06T10:30:06.375Z" }
+[[package]]
+name = "ibm-secrets-manager-sdk"
+version = "2.1.19"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "ibm-cloud-sdk-core" },
+ { name = "python-dateutil" },
+ { name = "requests" },
+ { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/21/67/7427f602cf32fb5eeff72ab5d05b8805235ef16b0c49ea92e65c3470534a/ibm_secrets_manager_sdk-2.1.19.tar.gz", hash = "sha256:8f945c3ec2245aced6956510c891355b6f28ef4f1c84c1aea2ec138d08b8694d", size = 251807, upload-time = "2026-03-17T19:33:56.075Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/be/73/2ba3e270f1d095b5dfba7d2e32b125e70f842a4438f5bacf04d2c8d04a60/ibm_secrets_manager_sdk-2.1.19-py3-none-any.whl", hash = "sha256:82293fa1c7357b5fe203afc676db654a807696064c7ff5807f56044dd19f8e5a", size = 150219, upload-time = "2026-03-17T19:33:55.118Z" },
+]
+
[[package]]
name = "idna"
version = "3.11"
@@ -1298,6 +1328,7 @@ dependencies = [
{ name = "google-auth-oauthlib" },
{ name = "httpx" },
{ name = "ibm-cos-sdk" },
+ { name = "ibm-secrets-manager-sdk" },
{ name = "msal" },
{ name = "opensearch-py", extra = ["async"] },
{ name = "psutil" },
@@ -1332,6 +1363,7 @@ requires-dist = [
{ name = "google-auth-oauthlib", specifier = ">=1.2.0" },
{ name = "httpx", specifier = ">=0.27.0" },
{ name = "ibm-cos-sdk", specifier = ">=2.13.0" },
+ { name = "ibm-secrets-manager-sdk", specifier = ">=2.1.19" },
{ name = "msal", specifier = ">=1.29.0" },
{ name = "opensearch-py", extras = ["async"], specifier = ">=3.0.0" },
{ name = "psutil", specifier = ">=7.0.0" },