OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: Data ingestion fails due to OpenSearch "documents" index not found exception
Mike Pawlowskiedea93f3e2e9
diff --git a/.env.example b/.env.exampleindex a18d9b934..a1484525b 100644--- a/.env.example+++ b/.env.example@@ -123,6 +123,15 @@ NEXT_ALLOWED_DEV_ORIGINS=# LANGFLOW_WORKERS=2 # Langflow workers (default: 1)# DOCLING_WORKERS=2 # Docling workers (default: 1)+# OPTIONAL: Set the logging level for the Backend service+# - Supported values: DEBUG, INFO, WARNING, ERROR, CRITICAL+# - Default: INFO+# LOG_LEVEL=DEBUG++# OPTIONAL: Disable colored log output (https://no-color.org)+# - When set (to any value), ANSI color codes are omitted from log output+# NO_COLOR=1+# OPTIONAL: Enable or disable HTTP access logging events# - e.g. INFO: 127.0.0.1:45132 - "GET /tasks HTTP/1.1" 200 OK# - Default: truediff --git a/flows/components/opensearch_multimodal.py b/flows/components/opensearch_multimodal.pyindex 5d8d96f87..dad23c56b 100644--- a/flows/components/opensearch_multimodal.py+++ b/flows/components/opensearch_multimodal.py@@ -462,7 +462,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponengine: str = "jvector",space_type: str = "l2",ef_search: int = 512,- ef_construction: int = 100,+ ef_construction: int = 512,m: int = 16,vector_field: str = "vector_field",) -> dict[str, Any]:diff --git a/scripts/migrate_embedding_model_field.py b/scripts/migrate_embedding_model_field.pyindex ec7f093cf..254da6580 100644--- a/scripts/migrate_embedding_model_field.py+++ b/scripts/migrate_embedding_model_field.py@@ -55,6 +55,8 @@ from opensearchpy._async.http_aiohttp import AIOHttpConnectionsys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))from config.settings import (+ KNN_EF_CONSTRUCTION,+ KNN_M,OPENSEARCH_HOST,OPENSEARCH_PORT,OPENSEARCH_USERNAME,@@ -83,7 +85,7 @@ async def ensure_new_field_exists("name": "disk_ann","engine": "jvector","space_type": "l2",- "parameters": {"ef_construction": 100, "m": 16},+ "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},},},"embedding_model": {diff --git a/src/api/settings.py b/src/api/settings.pyindex dedba0f7c..99a2e1283 100644--- a/src/api/settings.py+++ b/src/api/settings.py@@ -20,6 +20,7 @@ from config.settings import (is_no_auth_mode,)from api.provider_validation import validate_provider_setup+from utils.langflow_utils import LangflowNotReadyError, wait_for_langflowlogger = get_logger(__name__)@@ -438,7 +439,7 @@ async def update_settings(request, session_manager):current_config.agent.llm_model = body["llm_model"]config_updated = Trueawait TelemetryClient.send_event(- Category.SETTINGS_OPERATIONS,+ Category.SETTINGS_OPERATIONS,MessageId.ORB_SETTINGS_LLM_MODEL)logger.info(f"LLM model changed from {old_model} to {body['llm_model']}")@@ -448,7 +449,7 @@ async def update_settings(request, session_manager):current_config.agent.llm_provider = body["llm_provider"]config_updated = Trueawait TelemetryClient.send_event(- Category.SETTINGS_OPERATIONS,+ Category.SETTINGS_OPERATIONS,MessageId.ORB_SETTINGS_LLM_PROVIDER)logger.info(f"LLM provider changed from {old_provider} to {body['llm_provider']}")@@ -457,7 +458,7 @@ async def update_settings(request, session_manager):current_config.agent.system_prompt = body["system_prompt"]config_updated = Trueawait TelemetryClient.send_event(- Category.SETTINGS_OPERATIONS,+ Category.SETTINGS_OPERATIONS,MessageId.ORB_SETTINGS_SYSTEM_PROMPT)@@ -477,7 +478,7 @@ async def update_settings(request, session_manager):current_config.knowledge.embedding_model = new_embedding_modelconfig_updated = Trueawait TelemetryClient.send_event(- Category.SETTINGS_OPERATIONS,+ Category.SETTINGS_OPERATIONS,MessageId.ORB_SETTINGS_EMBED_MODEL)logger.info(f"Embedding model changed from {old_model} to {new_embedding_model}")@@ -487,7 +488,7 @@ async def update_settings(request, session_manager):current_config.knowledge.embedding_provider = body["embedding_provider"]config_updated = Trueawait TelemetryClient.send_event(- Category.SETTINGS_OPERATIONS,+ Category.SETTINGS_OPERATIONS,MessageId.ORB_SETTINGS_EMBED_PROVIDER)logger.info(f"Embedding provider changed from {old_provider} to {body['embedding_provider']}")@@ -496,7 +497,7 @@ async def update_settings(request, session_manager):current_config.knowledge.table_structure = body["table_structure"]config_updated = Trueawait TelemetryClient.send_event(- Category.SETTINGS_OPERATIONS,+ Category.SETTINGS_OPERATIONS,MessageId.ORB_SETTINGS_DOCLING_UPDATED)@@ -511,7 +512,7 @@ async def update_settings(request, session_manager):current_config.knowledge.ocr = body["ocr"]config_updated = Trueawait TelemetryClient.send_event(- Category.SETTINGS_OPERATIONS,+ Category.SETTINGS_OPERATIONS,MessageId.ORB_SETTINGS_DOCLING_UPDATED)@@ -526,7 +527,7 @@ async def update_settings(request, session_manager):current_config.knowledge.picture_descriptions = body["picture_descriptions"]config_updated = Trueawait TelemetryClient.send_event(- Category.SETTINGS_OPERATIONS,+ Category.SETTINGS_OPERATIONS,MessageId.ORB_SETTINGS_DOCLING_UPDATED)@@ -541,7 +542,7 @@ async def update_settings(request, session_manager):current_config.knowledge.chunk_size = body["chunk_size"]config_updated = Trueawait TelemetryClient.send_event(- Category.SETTINGS_OPERATIONS,+ Category.SETTINGS_OPERATIONS,MessageId.ORB_SETTINGS_CHUNK_UPDATED)@@ -561,7 +562,7 @@ async def update_settings(request, session_manager):current_config.knowledge.chunk_overlap = body["chunk_overlap"]config_updated = Trueawait TelemetryClient.send_event(- Category.SETTINGS_OPERATIONS,+ Category.SETTINGS_OPERATIONS,MessageId.ORB_SETTINGS_CHUNK_UPDATED)@@ -583,7 +584,7 @@ async def update_settings(request, session_manager):current_config.knowledge.index_name = new_index_nameconfig_updated = Trueawait TelemetryClient.send_event(- Category.SETTINGS_OPERATIONS,+ Category.SETTINGS_OPERATIONS,MessageId.ORB_SETTINGS_INDEX_NAME_UPDATED)logger.info(f"Index name changed from {old_index_name} to {new_index_name}")@@ -637,10 +638,10 @@ async def update_settings(request, session_manager):current_config.providers.ollama.configured = Trueconfig_updated = Trueprovider_updated = True-+if provider_updated:await TelemetryClient.send_event(- Category.SETTINGS_OPERATIONS,+ Category.SETTINGS_OPERATIONS,MessageId.ORB_SETTINGS_PROVIDER_CREDS)@@ -668,7 +669,7 @@ async def update_settings(request, session_manager):if any(key in body for key in provider_fields_to_check):try:flows_service = _get_flows_service()-+# Update global variablesawait _update_langflow_global_variables(current_config)@@ -677,7 +678,7 @@ async def update_settings(request, session_manager):await _update_mcp_servers_with_provider_credentials(current_config, session_manager)-+# Update model values if provider or model changedif "llm_provider" in body or "llm_model" in body or "embedding_provider" in body or "embedding_model" in body:await _update_langflow_model_values(current_config, flows_service)@@ -692,7 +693,7 @@ async def update_settings(request, session_manager):"Configuration updated successfully", updated_fields=list(body.keys()))await TelemetryClient.send_event(- Category.SETTINGS_OPERATIONS,+ Category.SETTINGS_OPERATIONS,MessageId.ORB_SETTINGS_UPDATED)return JSONResponse({"message": "Configuration updated successfully"})@@ -700,7 +701,7 @@ async def update_settings(request, session_manager):except Exception as e:logger.error("Failed to update settings", error=str(e))await TelemetryClient.send_event(- Category.SETTINGS_OPERATIONS,+ Category.SETTINGS_OPERATIONS,MessageId.ORB_SETTINGS_UPDATE_FAILED)return JSONResponse(@@ -712,7 +713,7 @@ async def onboarding(request, flows_service, session_manager=None):"""Handle onboarding configuration setup"""try:await TelemetryClient.send_event(Category.ONBOARDING, MessageId.ORB_ONBOARD_START)-+# Get current configurationcurrent_config = get_openrag_config()@@ -756,7 +757,7 @@ async def onboarding(request, flows_service, session_manager=None):# Update agent settings (LLM)llm_model_selected = Nonellm_provider_selected = None-+if "llm_model" in body:if not isinstance(body["llm_model"], str) or not body["llm_model"].strip():return JSONResponse(@@ -766,7 +767,7 @@ async def onboarding(request, flows_service, session_manager=None):current_config.agent.llm_model = llm_model_selectedconfig_updated = Trueawait TelemetryClient.send_event(- Category.ONBOARDING,+ Category.ONBOARDING,MessageId.ORB_ONBOARD_LLM_MODEL,metadata={"llm_model": llm_model_selected})@@ -790,7 +791,7 @@ async def onboarding(request, flows_service, session_manager=None):current_config.agent.llm_provider = llm_provider_selectedconfig_updated = Trueawait TelemetryClient.send_event(- Category.ONBOARDING,+ Category.ONBOARDING,MessageId.ORB_ONBOARD_LLM_PROVIDER,metadata={"llm_provider": llm_provider_selected})@@ -799,7 +800,7 @@ async def onboarding(request, flows_service, session_manager=None):# Update knowledge settings (embedding)embedding_model_selected = Noneembedding_provider_selected = None-+if "embedding_model" in body and not DISABLE_INGEST_WITH_LANGFLOW:if (not isinstance(body["embedding_model"], str)@@ -813,7 +814,7 @@ async def onboarding(request, flows_service, session_manager=None):current_config.knowledge.embedding_model = embedding_model_selectedconfig_updated = Trueawait TelemetryClient.send_event(- Category.ONBOARDING,+ Category.ONBOARDING,MessageId.ORB_ONBOARD_EMBED_MODEL,metadata={"embedding_model": embedding_model_selected})@@ -838,7 +839,7 @@ async def onboarding(request, flows_service, session_manager=None):current_config.knowledge.embedding_provider = embedding_provider_selectedconfig_updated = Trueawait TelemetryClient.send_event(- Category.ONBOARDING,+ Category.ONBOARDING,MessageId.ORB_ONBOARD_EMBED_PROVIDER,metadata={"embedding_provider": embedding_provider_selected})@@ -923,7 +924,7 @@ async def onboarding(request, flows_service, session_manager=None):should_ingest_sample_data = INGEST_SAMPLE_DATAif should_ingest_sample_data:await TelemetryClient.send_event(- Category.ONBOARDING,+ Category.ONBOARDING,MessageId.ORB_ONBOARD_SAMPLE_DATA)logger.info("Sample data ingestion enabled via environment variable")@@ -976,6 +977,20 @@ async def onboarding(request, flows_service, session_manager=None):status_code=400,)+ # Ensure the Langflow service is ready before attempting to configure it+ try:+ await wait_for_langflow()+ except LangflowNotReadyError as e:+ message: str = "Aborted the Langflow service configuration process. The Langflow service is not ready."+ logger.error(message, error=str(e))++ return JSONResponse(+ {+ "error": message+ },+ status_code=503,+ )+# Set Langflow global variables and model values based on provider configurationtry:# Check if any provider-related fields were provided@@ -984,14 +999,14 @@ async def onboarding(request, flows_service, session_manager=None):"watsonx_api_key", "watsonx_endpoint", "watsonx_project_id","ollama_endpoint"])-+# Update global variables if any provider fields were provided# or if existing config has values (for OpenAI/Anthropic that might already be set)- if (provider_fields_provided or- current_config.providers.openai.api_key != "" or+ if (provider_fields_provided or+ current_config.providers.openai.api_key != "" orcurrent_config.providers.anthropic.api_key != ""):await _update_langflow_global_variables(current_config)-+if "embedding_provider" in body or "embedding_model" in body:await _update_mcp_servers_with_provider_credentials(current_config, session_manager)@@ -1010,12 +1025,12 @@ async def onboarding(request, flows_service, session_manager=None):if "embedding_model" in body or "embedding_provider" in body:try:# Import here to avoid circular imports- from main import init_index+ from main import init_index_when_readylogger.info("Initializing OpenSearch index after onboarding configuration")- await init_index()+ await init_index_when_ready()logger.info("OpenSearch index initialization completed successfully")except Exception as e:logger.error(@@ -1054,9 +1069,9 @@ async def onboarding(request, flows_service, session_manager=None):logger.error("Failed to complete sample data ingestion", error=str(e))-+return JSONResponse(- {"error": f"Failed to ingest sample documents: {str(e)}"},+ {"error": f"Failed to ingest sample documents: {str(e)}"},status_code=500)@@ -1066,10 +1081,10 @@ async def onboarding(request, flows_service, session_manager=None):"Onboarding configuration updated successfully",updated_fields=updated_fields,)-+# Mark config as edited and send telemetry with model informationcurrent_config.edited = True-+# Build metadata with selected modelsonboarding_metadata = {}if llm_provider_selected:@@ -1080,14 +1095,14 @@ async def onboarding(request, flows_service, session_manager=None):onboarding_metadata["embedding_provider"] = embedding_provider_selectedif embedding_model_selected:onboarding_metadata["embedding_model"] = embedding_model_selected-+await TelemetryClient.send_event(- Category.ONBOARDING,+ Category.ONBOARDING,MessageId.ORB_ONBOARD_CONFIG_EDITED,metadata=onboarding_metadata)await TelemetryClient.send_event(- Category.ONBOARDING,+ Category.ONBOARDING,MessageId.ORB_ONBOARD_COMPLETE,metadata=onboarding_metadata)@@ -1095,7 +1110,7 @@ async def onboarding(request, flows_service, session_manager=None):else:await TelemetryClient.send_event(- Category.ONBOARDING,+ Category.ONBOARDING,MessageId.ORB_ONBOARD_FAILED)return JSONResponse(@@ -1140,7 +1155,7 @@ async def onboarding(request, flows_service, session_manager=None):except Exception as e:logger.error("Failed to update onboarding settings", error=str(e))await TelemetryClient.send_event(- Category.ONBOARDING,+ Category.ONBOARDING,MessageId.ORB_ONBOARD_FAILED)return JSONResponse(@@ -1279,33 +1294,33 @@ async def _update_mcp_servers_with_provider_credentials(config, session_managertry:from services.langflow_mcp_service import LangflowMCPServicefrom utils.langflow_headers import build_mcp_global_vars_from_config-+mcp_service = LangflowMCPService()-+# Build global vars using utility functionmcp_global_vars = build_mcp_global_vars_from_config(config)-+# In no-auth mode, add the anonymous JWT token and user detailsif is_no_auth_mode() and session_manager:from session_manager import AnonymousUser-+# Create/get anonymous JWT for no-auth modeanonymous_jwt = session_manager.get_effective_jwt_token(None, None)if anonymous_jwt:mcp_global_vars["JWT"] = anonymous_jwt-+# Add anonymous user detailsanonymous_user = AnonymousUser()mcp_global_vars["OWNER"] = anonymous_user.user_id # "anonymous"mcp_global_vars["OWNER_NAME"] = f'"{anonymous_user.name}"' # "Anonymous User" (quoted)mcp_global_vars["OWNER_EMAIL"] = anonymous_user.email # "anonymous@localhost"-+logger.debug("Added anonymous JWT and user details to MCP servers for no-auth mode")-+if mcp_global_vars:result = await mcp_service.update_mcp_servers_with_global_vars(mcp_global_vars)logger.info("Updated MCP servers with provider credentials after settings change", **result)-+except Exception as mcp_error:logger.warning(f"Failed to update MCP servers after settings change: {str(mcp_error)}")# Don't fail the entire settings update if MCP update fails@@ -1392,10 +1407,10 @@ async def update_onboarding_state(request):"""Update onboarding state in configuration"""try:await TelemetryClient.send_event(Category.ONBOARDING, MessageId.ORB_ONBOARD_START)-+# Parse request bodybody = await request.json()-+# Validate allowed fieldsallowed_fields = {"current_step",@@ -1406,7 +1421,7 @@ async def update_onboarding_state(request):"openrag_docs_filter_id","user_doc_filter_id",}-+# Check for invalid fieldsinvalid_fields = set(body.keys()) - allowed_fieldsif invalid_fields:@@ -1416,25 +1431,25 @@ async def update_onboarding_state(request):},status_code=400,)-+# Update onboarding state using config managersuccess = config_manager.update_onboarding_state(**body)-+if not success:return JSONResponse({"error": "Failed to update onboarding state"},status_code=500,)-+logger.info(f"Onboarding state updated: {body}")-+return JSONResponse({"message": "Onboarding state updated successfully","updated_fields": list(body.keys()),})-+except json.JSONDecodeError:return JSONResponse({"error": "Invalid JSON in request body"}, status_code=400@@ -1499,7 +1514,7 @@ async def reapply_all_settings(session_manager = None):async def rollback_onboarding(request, session_manager, task_service):"""Rollback onboarding configuration when sample data files fail.-+This will:1. Cancel all active tasks2. Delete successfully ingested knowledge documents@@ -1522,15 +1537,15 @@ async def rollback_onboarding(request, session_manager, task_service):# Get all tasks for the userall_tasks = task_service.get_all_tasks(user.user_id)-+cancelled_tasks = []deleted_files = []-+# Cancel all active tasks and collect successfully ingested filesfor task_data in all_tasks:task_id = task_data.get("task_id")task_status = task_data.get("status")-+# Cancel active tasks (pending, running, processing)if task_status in ["pending", "running", "processing"]:try:@@ -1540,7 +1555,7 @@ async def rollback_onboarding(request, session_manager, task_service):logger.info(f"Cancelled task {task_id}")except Exception as e:logger.error(f"Failed to cancel task {task_id}: {str(e)}")-+# For completed tasks, find successfully ingested files and delete themelif task_status == "completed":files = task_data.get("files", {})@@ -1550,26 +1565,26 @@ async def rollback_onboarding(request, session_manager, task_service):if isinstance(file_info, dict):file_status = file_info.get("status")filename = file_info.get("filename") or file_path.split("/")[-1]-+if file_status == "completed" and filename:try:# Get user's OpenSearch clientopensearch_client = session_manager.get_user_opensearch_client(user.user_id, jwt_token)-+# Delete documents by filenamefrom utils.opensearch_queries import build_filename_delete_bodyfrom config.settings import get_index_name-+delete_query = build_filename_delete_body(filename)-+result = await opensearch_client.delete_by_query(index=get_index_name(),body=delete_query,conflicts="proceed")-+deleted_count = result.get("deleted", 0)if deleted_count > 0:deleted_files.append(filename)@@ -1580,7 +1595,7 @@ async def rollback_onboarding(request, session_manager, task_service):# Clear embedding provider and model settingscurrent_config.knowledge.embedding_provider = "openai" # Reset to defaultcurrent_config.knowledge.embedding_model = ""-+# Mark config as not edited so user can go through onboarding againcurrent_config.edited = False@@ -1588,17 +1603,17 @@ async def rollback_onboarding(request, session_manager, task_service):try:import yamlconfig_file = config_manager.config_file-+# Ensure directory existsconfig_file.parent.mkdir(parents=True, exist_ok=True)-+# Save config with edited=Falsewith open(config_file, "w") as f:yaml.dump(current_config.to_dict(), f, default_flow_style=False, indent=2)-+# Update cached configconfig_manager._config = current_config-+logger.info("Successfully saved rolled back configuration with edited=False")except Exception as e:logger.error(f"Failed to save rolled back configuration: {e}")@@ -1611,10 +1626,10 @@ async def rollback_onboarding(request, session_manager, task_service):f"Cancelled {len(cancelled_tasks)} tasks, deleted {len(deleted_files)} files")await TelemetryClient.send_event(- Category.ONBOARDING,+ Category.ONBOARDING,MessageId.ORB_ONBOARD_ROLLBACK)-+return JSONResponse({"message": "Onboarding configuration rolled back successfully",diff --git a/src/config/settings.py b/src/config/settings.pyindex 94a61adf8..51a7906f9 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -92,6 +92,8 @@ WEBHOOK_BASE_URL = os.getenv(# OpenSearch configurationVECTOR_DIM = 1536+KNN_EF_CONSTRUCTION = 512+KNN_M = 16EMBED_MODEL = "text-embedding-3-small"OPENAI_EMBEDDING_DIMENSIONS = {@@ -135,7 +137,7 @@ INDEX_BODY = {"name": "disk_ann","engine": "jvector","space_type": "l2",- "parameters": {"ef_construction": 100, "m": 16},+ "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},},},# Track which embedding model was used for this chunk@@ -309,9 +311,6 @@ class AppClients:self.converter = Noneasync def initialize(self):- # Generate Langflow API key first- await get_langflow_api_key()-# Initialize OpenSearch clientself.opensearch = AsyncOpenSearch(hosts=[{"host": OPENSEARCH_HOST, "port": OPENSEARCH_PORT}],@@ -324,26 +323,6 @@ class AppClients:http_compress=True,)- # Initialize Langflow client with generated/provided API key- if LANGFLOW_KEY and self.langflow_client is None:- try:- if not OPENSEARCH_PASSWORD:- raise ValueError("OPENSEARCH_PASSWORD is not set")- else:- await self.ensure_langflow_client()- # Note: OPENSEARCH_PASSWORD global variable should be created automatically- # via LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT in docker-compose- logger.info(- "Langflow client initialized - OPENSEARCH_PASSWORD should be available via environment variables"- )- except Exception as e:- logger.warning("Failed to initialize Langflow client", error=str(e))- self.langflow_client = None- if self.langflow_client is None:- logger.warning(- "No Langflow client initialized yet, will attempt later on first use"- )-# Initialize patched OpenAI client if API key is available# This allows the app to start even if OPENAI_API_KEY is not set yet# (e.g., when it will be provided during onboarding)@@ -358,6 +337,7 @@ class AppClients:self.converter = create_document_converter(ocr_engine=DOCLING_OCR_ENGINE)# Initialize Langflow HTTP client with extended timeouts for large documents+ # Must be created before wait_for_langflow / get_langflow_api_key# Use explicit timeout configuration to handle large PDF ingestion (300+ pages)self.langflow_http_client = httpx.AsyncClient(base_url=LANGFLOW_URL,@@ -375,6 +355,34 @@ class AppClients:connect_timeout_seconds=LANGFLOW_CONNECT_TIMEOUT,)+ # Wait for Langflow to be healthy before generating API key+ # Deferred import to avoid circular dependency (langflow_utils imports config.settings.clients)+ from utils.langflow_utils import wait_for_langflow+ await wait_for_langflow()++ # Generate Langflow API key now that Langflow is confirmed ready+ await get_langflow_api_key()++ # Initialize Langflow client with generated/provided API key+ if LANGFLOW_KEY and self.langflow_client is None:+ try:+ if not OPENSEARCH_PASSWORD:+ raise ValueError("OPENSEARCH_PASSWORD is not set")+ else:+ await self.ensure_langflow_client()+ # Note: OPENSEARCH_PASSWORD global variable should be created automatically+ # via LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT in docker-compose+ logger.info(+ "Langflow client initialized - OPENSEARCH_PASSWORD should be available via environment variables"+ )+ except Exception as e:+ logger.warning("Failed to initialize Langflow client", error=str(e))+ self.langflow_client = None+ if self.langflow_client is None:+ logger.warning(+ "No Langflow client initialized yet, will attempt later on first use"+ )+return selfasync def ensure_langflow_client(self):diff --git a/src/main.py b/src/main.pyindex b51ee9b0b..7de3c13b3 100644--- a/src/main.py+++ b/src/main.py@@ -283,6 +283,12 @@ async def init_index():raise e+async def init_index_when_ready():+ """Wait for the OpenSearch service to be ready and then initialize the OpenSearch index."""+ await wait_for_opensearch()+ await init_index()++def generate_jwt_keys():"""Generate RSA keys for JWT signing if they don't exist"""keys_dir = "keys"@@ -338,19 +344,6 @@ def generate_jwt_keys():logger.warning("Failed to set permissions on existing keys", error=str(e))-async def init_index_when_ready():- """Initialize OpenSearch index when it becomes available"""- try:- await init_index()- logger.info("OpenSearch index initialization completed successfully")- except Exception as e:- logger.error("OpenSearch index initialization failed", error=str(e))- await TelemetryClient.send_event(Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_INIT_FAIL)- logger.warning(- "OIDC endpoints will still work, but document operations may fail until OpenSearch is ready"- )--def _get_documents_dir():"""Get the documents directory path, handling both Docker and local environments."""# In Docker, the volume is mounted at /app/openrag-documents@@ -580,6 +573,33 @@ async def startup_tasks(services):if DISABLE_INGEST_WITH_LANGFLOW:await _ensure_opensearch_index()+ # Ensure that the OpenSearch index exists if onboarding was already completed+ # - Handles the case where OpenSearch is reset (e.g., volume deleted) after onboarding+ embedding_model = None+ try:+ config = get_openrag_config()+ embedding_model = config.knowledge.embedding_model++ if config.edited and embedding_model:+ logger.info(+ "Ensuring that the OpenSearch index exists (after onboarding)...",+ embedding_model=embedding_model,+ )++ await init_index()++ logger.info(+ "Successfully ensured that the OpenSearch index exists (after onboarding).",+ embedding_model=embedding_model,+ )+ except Exception as e:+ logger.error(+ "Failed to ensure that the OpenSearch index exists (after onboarding).",+ embedding_model=embedding_model,+ error=str(e),+ )+ raise+# Configure alerting securityawait configure_alerting_security()diff --git a/src/utils/embedding_fields.py b/src/utils/embedding_fields.pyindex 02342407b..27469f374 100644--- a/src/utils/embedding_fields.py+++ b/src/utils/embedding_fields.py@@ -86,7 +86,7 @@ async def ensure_embedding_field_exists(Raises:Exception: If unable to add the field mapping"""- from config.settings import get_index_name+ from config.settings import KNN_EF_CONSTRUCTION, KNN_M, get_index_namefrom utils.embeddings import get_embedding_dimensionsif index_name is None:@@ -134,7 +134,7 @@ async def ensure_embedding_field_exists("name": "disk_ann","engine": "jvector","space_type": "l2",- "parameters": {"ef_construction": 100, "m": 16},+ "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},},},# Also ensure the embedding_model tracking field exists as keyworddiff --git a/src/utils/embeddings.py b/src/utils/embeddings.pyindex 598bf1016..1e2944b94 100644--- a/src/utils/embeddings.py+++ b/src/utils/embeddings.py@@ -1,5 +1,5 @@import httpx-from config.settings import OPENAI_EMBEDDING_DIMENSIONS, VECTOR_DIM, WATSONX_EMBEDDING_DIMENSIONS+from config.settings import KNN_EF_CONSTRUCTION, KNN_M, OPENAI_EMBEDDING_DIMENSIONS, VECTOR_DIM, WATSONX_EMBEDDING_DIMENSIONSfrom utils.container_utils import transform_localhost_urlfrom utils.logging_config import get_logger@@ -176,7 +176,7 @@ async def create_dynamic_index_body("name": "disk_ann","engine": "jvector","space_type": "l2",- "parameters": {"ef_construction": 100, "m": 16},+ "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},},},# Track which embedding model was used for this chunkdiff --git a/src/utils/langflow_utils.py b/src/utils/langflow_utils.pynew file mode 100644index 000000000..f48f33e70--- /dev/null+++ b/src/utils/langflow_utils.py@@ -0,0 +1,76 @@+import asyncio+import random++from config.settings import clients+from utils.logging_config import get_logger++logger = get_logger(__name__)+++class LangflowNotReadyError(Exception):+ """Raised when Langflow fails to become ready within the retry limit."""+++async def wait_for_langflow(max_retries: int = 10, base_delay: float = 1.0, max_delay: float = 20.0) -> None:+ """Wait for Langflow to be ready with exponential backoff and jitter.++ Args:+ max_retries: Maximum number of retry attempts.+ base_delay: Initial delay in seconds before the first retry.+ max_delay: Upper bound in seconds for the retry delay.++ Raises:+ LangflowNotReadyError: If Langflow fails to become ready within the retry limit.+ """+ for attempt in range(max_retries):+ display_attempt: int = attempt + 1++ logger.info(+ "Verifying whether the Langflow service is ready...",+ attempt=display_attempt,+ max_retries=max_retries,+ )++ try:+ response = await clients.langflow_http_client.get("/health", timeout=5.0)+ status_code = response.status_code++ if status_code == 200:+ logger.info(+ "Successfully verified that the Langflow service is ready.",+ attempt=display_attempt,+ max_retries=max_retries,+ status_code=status_code,+ )+ return+ else:+ logger.warning(+ "The Langflow service is not ready. Encountered a non-200 HTTP status_code.",+ attempt=display_attempt,+ max_retries=max_retries,+ status_code=status_code,+ )+ except Exception as e:+ logger.warning(+ "The Langflow service is not ready.",+ attempt=display_attempt,+ max_retries=max_retries,+ error=str(e),+ )++ if attempt < max_retries - 1:+ delay = min(base_delay * (2 ** attempt), max_delay)+ delay = random.uniform(delay / 2, delay)++ logger.debug(+ "Retry the Langflow service readiness check after a delay (seconds).",+ attempt=display_attempt,+ max_retries=max_retries,+ delay=delay,+ )++ await asyncio.sleep(delay)++ message: str = "Failed to verify whether the Langflow service is ready."+ logger.error(message)+ raise LangflowNotReadyError(message)diff --git a/src/utils/logging_config.py b/src/utils/logging_config.pyindex d516909b4..b4bd596f1 100644--- a/src/utils/logging_config.py+++ b/src/utils/logging_config.py@@ -5,6 +5,20 @@ import structlogfrom structlog import processors+LOC_WIDTH_SHORT = 30+LOC_WIDTH_LONG = 60++LEVEL_COLORS = {+ "DEBUG": "\033[36m", # Cyan+ "INFO": "\033[32m", # Green+ "WARNING": "\033[33m", # Yellow+ "ERROR": "\033[31m", # Red+ "CRITICAL": "\033[1;31m", # Bold red+}+DIM = "\033[38;5;244m" # Medium grey+RESET = "\033[0m"++def configure_logging(log_level: str = "INFO",json_logs: bool = False,@@ -48,8 +62,7 @@ def configure_logging(console_renderer = structlog.processors.JSONRenderer()else:# Custom clean format: timestamp path/file:loc logentry- LOC_WIDTH_SHORT = 30- LOC_WIDTH_LONG = 60+ use_colors = "NO_COLOR" not in os.environ and hasattr(sys.stderr, "isatty") and sys.stderr.isatty()def custom_formatter(logger, log_method, event_dict):timestamp = event_dict.pop("timestamp", "")@@ -80,12 +93,22 @@ def configure_logging(if event:message_parts.append(event)- header = f"[{timestamp}] [{level:<7}] [{location:<{loc_width}}] "+ if use_colors:+ colored_timestamp = f"{DIM}{timestamp}{RESET}"+ color = LEVEL_COLORS.get(level, "")+ colored_level = f"{color}{level:<7}{RESET}"+ else:+ colored_timestamp = timestamp+ colored_level = f"{level:<7}"++ header = f"[{colored_timestamp}] [{colored_level}] [{location:<{loc_width}}] "+ # Visible width excludes ANSI escape codes for correct padding+ visible_header = f"[{timestamp}] [{level:<7}] [{location:<{loc_width}}] "# Add any remaining context as indented multi-line fieldsextra = {k: v for k, v in event_dict.items() if k not in ["service", "func_name"]}if extra:- padding = " " * len(header)+ padding = " " * len(visible_header)for key, value in extra.items():message_parts.append(f"\n{padding}- {key}: {value}")diff --git a/tests/unit/conftest.py b/tests/unit/conftest.pynew file mode 100644index 000000000..878ba1997--- /dev/null+++ b/tests/unit/conftest.py@@ -0,0 +1,13 @@+"""Unit test configuration.++Overrides the session-scoped onboard_system fixture from the root conftest+so that unit tests don't require running infrastructure (Langflow, OpenSearch, etc.).+"""++import pytest_asyncio+++@pytest_asyncio.fixture(scope="session", autouse=True)+async def onboard_system():+ """No-op override — unit tests mock their own dependencies."""+ yielddiff --git a/tests/unit/test_langflow_utils.py b/tests/unit/test_langflow_utils.pynew file mode 100644index 000000000..e009e0f15--- /dev/null+++ b/tests/unit/test_langflow_utils.py@@ -0,0 +1,202 @@+"""+Tests for utils/langflow_utils.py+Validates wait_for_langflow retry logic, backoff behavior, and error handling.+All external dependencies (HTTP client, sleep, logging) are fully mocked.+"""++import pytest+from unittest.mock import AsyncMock, MagicMock, patch++from utils.langflow_utils import LangflowNotReadyError, wait_for_langflow+++def _make_response(status_code: int) -> MagicMock:+ """Create a mock HTTP response with the given status code."""+ resp = MagicMock()+ resp.status_code = status_code+ return resp+++def mock_langflow_client():+ """Provide a mocked langflow HTTP client and patch it into settings."""+ client = AsyncMock()+ with patch("utils.langflow_utils.clients") as mock_clients:+ mock_clients.langflow_http_client = client+ yield client++[email protected](autouse=True)+def no_sleep():+ """Patch asyncio.sleep so tests run instantly."""+ with patch("utils.langflow_utils.asyncio.sleep", new_callable=AsyncMock) as mock_sleep:+ yield mock_sleep+++# ── Success on first attempt ──────────────────────────────────────────+++async def test_ready_on_first_attempt(mock_langflow_client, no_sleep):+ """Returns immediately when health check responds 200 on the first try."""+ mock_langflow_client.get.return_value = _make_response(200)++ await wait_for_langflow(max_retries=3)++ mock_langflow_client.get.assert_called_once_with("/health", timeout=5.0)+ no_sleep.assert_not_called()+++# ── Success after transient failures ─────────────────────────────────+++async def test_ready_after_non_200_then_200(mock_langflow_client, no_sleep):+ """Retries on non-200 responses and succeeds when 200 is returned."""+ mock_langflow_client.get.side_effect = [+ _make_response(503),+ _make_response(200),+ ]++ await wait_for_langflow(max_retries=3)++ assert mock_langflow_client.get.call_count == 2+ assert no_sleep.call_count == 1+++async def test_ready_after_exception_then_200(mock_langflow_client, no_sleep):+ """Retries on connection errors and succeeds when 200 is returned."""+ mock_langflow_client.get.side_effect = [+ ConnectionError("refused"),+ _make_response(200),+ ]++ await wait_for_langflow(max_retries=3)++ assert mock_langflow_client.get.call_count == 2+ assert no_sleep.call_count == 1+++async def test_ready_after_mixed_failures(mock_langflow_client, no_sleep):+ """Handles a mix of exceptions and non-200 responses before success."""+ mock_langflow_client.get.side_effect = [+ ConnectionError("refused"),+ _make_response(500),+ TimeoutError("timed out"),+ _make_response(200),+ ]++ await wait_for_langflow(max_retries=5)++ assert mock_langflow_client.get.call_count == 4+ assert no_sleep.call_count == 3+++# ── Exhausted retries ────────────────────────────────────────────────+++async def test_raises_after_all_retries_exhausted_non_200(mock_langflow_client):+ """Raises LangflowNotReadyError when every attempt returns non-200."""+ mock_langflow_client.get.return_value = _make_response(503)++ with pytest.raises(LangflowNotReadyError):+ await wait_for_langflow(max_retries=3)++ assert mock_langflow_client.get.call_count == 3+++async def test_raises_after_all_retries_exhausted_exception(mock_langflow_client):+ """Raises LangflowNotReadyError when every attempt raises an exception."""+ mock_langflow_client.get.side_effect = ConnectionError("refused")++ with pytest.raises(LangflowNotReadyError):+ await wait_for_langflow(max_retries=2)++ assert mock_langflow_client.get.call_count == 2+++async def test_single_retry_no_sleep_before_raise(mock_langflow_client, no_sleep):+ """With max_retries=1, fails immediately without sleeping."""+ mock_langflow_client.get.return_value = _make_response(500)++ with pytest.raises(LangflowNotReadyError):+ await wait_for_langflow(max_retries=1)++ mock_langflow_client.get.assert_called_once()+ no_sleep.assert_not_called()+++# ── Backoff behavior ─────────────────────────────────────────────────+++async def test_sleep_delay_respects_bounds(mock_langflow_client, no_sleep):+ """Sleep delay stays within [delay/2, delay] and never exceeds max_delay."""+ mock_langflow_client.get.return_value = _make_response(503)+ base_delay = 1.0+ max_delay = 4.0++ with pytest.raises(LangflowNotReadyError):+ await wait_for_langflow(max_retries=5, base_delay=base_delay, max_delay=max_delay)++ # 4 sleeps for 5 retries (no sleep after the last attempt)+ assert no_sleep.call_count == 4++ for call in no_sleep.call_args_list:+ delay = call.args[0]+ assert 0 <= delay <= max_delay+++async def test_exponential_backoff_increases(mock_langflow_client, no_sleep):+ """Backoff upper bound doubles each attempt (before capping at max_delay)."""+ mock_langflow_client.get.return_value = _make_response(503)++ with patch("utils.langflow_utils.random.uniform", side_effect=lambda lo, hi: hi):+ with pytest.raises(LangflowNotReadyError):+ await wait_for_langflow(max_retries=4, base_delay=1.0, max_delay=100.0)++ # With jitter pinned to the upper bound, delays should be 1, 2, 4+ delays = [call.args[0] for call in no_sleep.call_args_list]+ assert delays == [1.0, 2.0, 4.0]+++async def test_max_delay_cap(mock_langflow_client, no_sleep):+ """Delay is capped at max_delay even when exponential growth exceeds it."""+ mock_langflow_client.get.return_value = _make_response(503)++ with patch("utils.langflow_utils.random.uniform", side_effect=lambda lo, hi: hi):+ with pytest.raises(LangflowNotReadyError):+ await wait_for_langflow(max_retries=6, base_delay=2.0, max_delay=5.0)++ delays = [call.args[0] for call in no_sleep.call_args_list]+ # base_delay * 2^attempt: 2, 4, 5(cap), 5(cap), 5(cap)+ assert delays == [2.0, 4.0, 5.0, 5.0, 5.0]+++# ── Edge cases ───────────────────────────────────────────────────────+++async def test_default_parameters(mock_langflow_client, no_sleep):+ """Works correctly with default parameter values."""+ mock_langflow_client.get.return_value = _make_response(200)++ await wait_for_langflow()++ mock_langflow_client.get.assert_called_once()+++async def test_error_message_content(mock_langflow_client):+ """LangflowNotReadyError contains a meaningful message."""+ mock_langflow_client.get.return_value = _make_response(503)++ with pytest.raises(LangflowNotReadyError, match="Failed to verify"):+ await wait_for_langflow(max_retries=1)