Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: Data ingestion fails due to OpenSearch "documents" index not found exception

Mike Pawlowskiedea93f3e2e9
diff --git a/.env.example b/.env.example
index a18d9b934..a1484525b 100644
--- a/.env.example
+++ b/.env.example
@@ -123,6 +123,15 @@ NEXT_ALLOWED_DEV_ORIGINS=
# LANGFLOW_WORKERS=2 # Langflow workers (default: 1)
# DOCLING_WORKERS=2 # Docling workers (default: 1)
+# OPTIONAL: Set the logging level for the Backend service
+# - Supported values: DEBUG, INFO, WARNING, ERROR, CRITICAL
+# - Default: INFO
+# LOG_LEVEL=DEBUG
+
+# OPTIONAL: Disable colored log output (https://no-color.org)
+# - When set (to any value), ANSI color codes are omitted from log output
+# NO_COLOR=1
+
# OPTIONAL: Enable or disable HTTP access logging events
# - e.g. INFO: 127.0.0.1:45132 - "GET /tasks HTTP/1.1" 200 OK
# - Default: true
diff --git a/flows/components/opensearch_multimodal.py b/flows/components/opensearch_multimodal.py
index 5d8d96f87..dad23c56b 100644
--- a/flows/components/opensearch_multimodal.py
+++ b/flows/components/opensearch_multimodal.py
@@ -462,7 +462,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
engine: str = "jvector",
space_type: str = "l2",
ef_search: int = 512,
- ef_construction: int = 100,
+ ef_construction: int = 512,
m: int = 16,
vector_field: str = "vector_field",
) -> dict[str, Any]:
diff --git a/scripts/migrate_embedding_model_field.py b/scripts/migrate_embedding_model_field.py
index ec7f093cf..254da6580 100644
--- a/scripts/migrate_embedding_model_field.py
+++ b/scripts/migrate_embedding_model_field.py
@@ -55,6 +55,8 @@ from opensearchpy._async.http_aiohttp import AIOHttpConnection
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
from config.settings import (
+ KNN_EF_CONSTRUCTION,
+ KNN_M,
OPENSEARCH_HOST,
OPENSEARCH_PORT,
OPENSEARCH_USERNAME,
@@ -83,7 +85,7 @@ async def ensure_new_field_exists(
"name": "disk_ann",
"engine": "jvector",
"space_type": "l2",
- "parameters": {"ef_construction": 100, "m": 16},
+ "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},
},
},
"embedding_model": {
diff --git a/src/api/settings.py b/src/api/settings.py
index dedba0f7c..99a2e1283 100644
--- a/src/api/settings.py
+++ b/src/api/settings.py
@@ -20,6 +20,7 @@ from config.settings import (
is_no_auth_mode,
)
from api.provider_validation import validate_provider_setup
+from utils.langflow_utils import LangflowNotReadyError, wait_for_langflow
logger = get_logger(__name__)
@@ -438,7 +439,7 @@ async def update_settings(request, session_manager):
current_config.agent.llm_model = body["llm_model"]
config_updated = True
await TelemetryClient.send_event(
- Category.SETTINGS_OPERATIONS,
+ Category.SETTINGS_OPERATIONS,
MessageId.ORB_SETTINGS_LLM_MODEL
)
logger.info(f"LLM model changed from {old_model} to {body['llm_model']}")
@@ -448,7 +449,7 @@ async def update_settings(request, session_manager):
current_config.agent.llm_provider = body["llm_provider"]
config_updated = True
await TelemetryClient.send_event(
- Category.SETTINGS_OPERATIONS,
+ Category.SETTINGS_OPERATIONS,
MessageId.ORB_SETTINGS_LLM_PROVIDER
)
logger.info(f"LLM provider changed from {old_provider} to {body['llm_provider']}")
@@ -457,7 +458,7 @@ async def update_settings(request, session_manager):
current_config.agent.system_prompt = body["system_prompt"]
config_updated = True
await TelemetryClient.send_event(
- Category.SETTINGS_OPERATIONS,
+ Category.SETTINGS_OPERATIONS,
MessageId.ORB_SETTINGS_SYSTEM_PROMPT
)
@@ -477,7 +478,7 @@ async def update_settings(request, session_manager):
current_config.knowledge.embedding_model = new_embedding_model
config_updated = True
await TelemetryClient.send_event(
- Category.SETTINGS_OPERATIONS,
+ Category.SETTINGS_OPERATIONS,
MessageId.ORB_SETTINGS_EMBED_MODEL
)
logger.info(f"Embedding model changed from {old_model} to {new_embedding_model}")
@@ -487,7 +488,7 @@ async def update_settings(request, session_manager):
current_config.knowledge.embedding_provider = body["embedding_provider"]
config_updated = True
await TelemetryClient.send_event(
- Category.SETTINGS_OPERATIONS,
+ Category.SETTINGS_OPERATIONS,
MessageId.ORB_SETTINGS_EMBED_PROVIDER
)
logger.info(f"Embedding provider changed from {old_provider} to {body['embedding_provider']}")
@@ -496,7 +497,7 @@ async def update_settings(request, session_manager):
current_config.knowledge.table_structure = body["table_structure"]
config_updated = True
await TelemetryClient.send_event(
- Category.SETTINGS_OPERATIONS,
+ Category.SETTINGS_OPERATIONS,
MessageId.ORB_SETTINGS_DOCLING_UPDATED
)
@@ -511,7 +512,7 @@ async def update_settings(request, session_manager):
current_config.knowledge.ocr = body["ocr"]
config_updated = True
await TelemetryClient.send_event(
- Category.SETTINGS_OPERATIONS,
+ Category.SETTINGS_OPERATIONS,
MessageId.ORB_SETTINGS_DOCLING_UPDATED
)
@@ -526,7 +527,7 @@ async def update_settings(request, session_manager):
current_config.knowledge.picture_descriptions = body["picture_descriptions"]
config_updated = True
await TelemetryClient.send_event(
- Category.SETTINGS_OPERATIONS,
+ Category.SETTINGS_OPERATIONS,
MessageId.ORB_SETTINGS_DOCLING_UPDATED
)
@@ -541,7 +542,7 @@ async def update_settings(request, session_manager):
current_config.knowledge.chunk_size = body["chunk_size"]
config_updated = True
await TelemetryClient.send_event(
- Category.SETTINGS_OPERATIONS,
+ Category.SETTINGS_OPERATIONS,
MessageId.ORB_SETTINGS_CHUNK_UPDATED
)
@@ -561,7 +562,7 @@ async def update_settings(request, session_manager):
current_config.knowledge.chunk_overlap = body["chunk_overlap"]
config_updated = True
await TelemetryClient.send_event(
- Category.SETTINGS_OPERATIONS,
+ Category.SETTINGS_OPERATIONS,
MessageId.ORB_SETTINGS_CHUNK_UPDATED
)
@@ -583,7 +584,7 @@ async def update_settings(request, session_manager):
current_config.knowledge.index_name = new_index_name
config_updated = True
await TelemetryClient.send_event(
- Category.SETTINGS_OPERATIONS,
+ Category.SETTINGS_OPERATIONS,
MessageId.ORB_SETTINGS_INDEX_NAME_UPDATED
)
logger.info(f"Index name changed from {old_index_name} to {new_index_name}")
@@ -637,10 +638,10 @@ async def update_settings(request, session_manager):
current_config.providers.ollama.configured = True
config_updated = True
provider_updated = True
-
+
if provider_updated:
await TelemetryClient.send_event(
- Category.SETTINGS_OPERATIONS,
+ Category.SETTINGS_OPERATIONS,
MessageId.ORB_SETTINGS_PROVIDER_CREDS
)
@@ -668,7 +669,7 @@ async def update_settings(request, session_manager):
if any(key in body for key in provider_fields_to_check):
try:
flows_service = _get_flows_service()
-
+
# Update global variables
await _update_langflow_global_variables(current_config)
@@ -677,7 +678,7 @@ async def update_settings(request, session_manager):
await _update_mcp_servers_with_provider_credentials(
current_config, session_manager
)
-
+
# Update model values if provider or model changed
if "llm_provider" in body or "llm_model" in body or "embedding_provider" in body or "embedding_model" in body:
await _update_langflow_model_values(current_config, flows_service)
@@ -692,7 +693,7 @@ async def update_settings(request, session_manager):
"Configuration updated successfully", updated_fields=list(body.keys())
)
await TelemetryClient.send_event(
- Category.SETTINGS_OPERATIONS,
+ Category.SETTINGS_OPERATIONS,
MessageId.ORB_SETTINGS_UPDATED
)
return JSONResponse({"message": "Configuration updated successfully"})
@@ -700,7 +701,7 @@ async def update_settings(request, session_manager):
except Exception as e:
logger.error("Failed to update settings", error=str(e))
await TelemetryClient.send_event(
- Category.SETTINGS_OPERATIONS,
+ Category.SETTINGS_OPERATIONS,
MessageId.ORB_SETTINGS_UPDATE_FAILED
)
return JSONResponse(
@@ -712,7 +713,7 @@ async def onboarding(request, flows_service, session_manager=None):
"""Handle onboarding configuration setup"""
try:
await TelemetryClient.send_event(Category.ONBOARDING, MessageId.ORB_ONBOARD_START)
-
+
# Get current configuration
current_config = get_openrag_config()
@@ -756,7 +757,7 @@ async def onboarding(request, flows_service, session_manager=None):
# Update agent settings (LLM)
llm_model_selected = None
llm_provider_selected = None
-
+
if "llm_model" in body:
if not isinstance(body["llm_model"], str) or not body["llm_model"].strip():
return JSONResponse(
@@ -766,7 +767,7 @@ async def onboarding(request, flows_service, session_manager=None):
current_config.agent.llm_model = llm_model_selected
config_updated = True
await TelemetryClient.send_event(
- Category.ONBOARDING,
+ Category.ONBOARDING,
MessageId.ORB_ONBOARD_LLM_MODEL,
metadata={"llm_model": llm_model_selected}
)
@@ -790,7 +791,7 @@ async def onboarding(request, flows_service, session_manager=None):
current_config.agent.llm_provider = llm_provider_selected
config_updated = True
await TelemetryClient.send_event(
- Category.ONBOARDING,
+ Category.ONBOARDING,
MessageId.ORB_ONBOARD_LLM_PROVIDER,
metadata={"llm_provider": llm_provider_selected}
)
@@ -799,7 +800,7 @@ async def onboarding(request, flows_service, session_manager=None):
# Update knowledge settings (embedding)
embedding_model_selected = None
embedding_provider_selected = None
-
+
if "embedding_model" in body and not DISABLE_INGEST_WITH_LANGFLOW:
if (
not isinstance(body["embedding_model"], str)
@@ -813,7 +814,7 @@ async def onboarding(request, flows_service, session_manager=None):
current_config.knowledge.embedding_model = embedding_model_selected
config_updated = True
await TelemetryClient.send_event(
- Category.ONBOARDING,
+ Category.ONBOARDING,
MessageId.ORB_ONBOARD_EMBED_MODEL,
metadata={"embedding_model": embedding_model_selected}
)
@@ -838,7 +839,7 @@ async def onboarding(request, flows_service, session_manager=None):
current_config.knowledge.embedding_provider = embedding_provider_selected
config_updated = True
await TelemetryClient.send_event(
- Category.ONBOARDING,
+ Category.ONBOARDING,
MessageId.ORB_ONBOARD_EMBED_PROVIDER,
metadata={"embedding_provider": embedding_provider_selected}
)
@@ -923,7 +924,7 @@ async def onboarding(request, flows_service, session_manager=None):
should_ingest_sample_data = INGEST_SAMPLE_DATA
if should_ingest_sample_data:
await TelemetryClient.send_event(
- Category.ONBOARDING,
+ Category.ONBOARDING,
MessageId.ORB_ONBOARD_SAMPLE_DATA
)
logger.info("Sample data ingestion enabled via environment variable")
@@ -976,6 +977,20 @@ async def onboarding(request, flows_service, session_manager=None):
status_code=400,
)
+ # Ensure the Langflow service is ready before attempting to configure it
+ try:
+ await wait_for_langflow()
+ except LangflowNotReadyError as e:
+ message: str = "Aborted the Langflow service configuration process. The Langflow service is not ready."
+ logger.error(message, error=str(e))
+
+ return JSONResponse(
+ {
+ "error": message
+ },
+ status_code=503,
+ )
+
# Set Langflow global variables and model values based on provider configuration
try:
# Check if any provider-related fields were provided
@@ -984,14 +999,14 @@ async def onboarding(request, flows_service, session_manager=None):
"watsonx_api_key", "watsonx_endpoint", "watsonx_project_id",
"ollama_endpoint"
])
-
+
# Update global variables if any provider fields were provided
# or if existing config has values (for OpenAI/Anthropic that might already be set)
- if (provider_fields_provided or
- current_config.providers.openai.api_key != "" or
+ if (provider_fields_provided or
+ current_config.providers.openai.api_key != "" or
current_config.providers.anthropic.api_key != ""):
await _update_langflow_global_variables(current_config)
-
+
if "embedding_provider" in body or "embedding_model" in body:
await _update_mcp_servers_with_provider_credentials(current_config, session_manager)
@@ -1010,12 +1025,12 @@ async def onboarding(request, flows_service, session_manager=None):
if "embedding_model" in body or "embedding_provider" in body:
try:
# Import here to avoid circular imports
- from main import init_index
+ from main import init_index_when_ready
logger.info(
"Initializing OpenSearch index after onboarding configuration"
)
- await init_index()
+ await init_index_when_ready()
logger.info("OpenSearch index initialization completed successfully")
except Exception as e:
logger.error(
@@ -1054,9 +1069,9 @@ async def onboarding(request, flows_service, session_manager=None):
logger.error(
"Failed to complete sample data ingestion", error=str(e)
)
-
+
return JSONResponse(
- {"error": f"Failed to ingest sample documents: {str(e)}"},
+ {"error": f"Failed to ingest sample documents: {str(e)}"},
status_code=500
)
@@ -1066,10 +1081,10 @@ async def onboarding(request, flows_service, session_manager=None):
"Onboarding configuration updated successfully",
updated_fields=updated_fields,
)
-
+
# Mark config as edited and send telemetry with model information
current_config.edited = True
-
+
# Build metadata with selected models
onboarding_metadata = {}
if llm_provider_selected:
@@ -1080,14 +1095,14 @@ async def onboarding(request, flows_service, session_manager=None):
onboarding_metadata["embedding_provider"] = embedding_provider_selected
if embedding_model_selected:
onboarding_metadata["embedding_model"] = embedding_model_selected
-
+
await TelemetryClient.send_event(
- Category.ONBOARDING,
+ Category.ONBOARDING,
MessageId.ORB_ONBOARD_CONFIG_EDITED,
metadata=onboarding_metadata
)
await TelemetryClient.send_event(
- Category.ONBOARDING,
+ Category.ONBOARDING,
MessageId.ORB_ONBOARD_COMPLETE,
metadata=onboarding_metadata
)
@@ -1095,7 +1110,7 @@ async def onboarding(request, flows_service, session_manager=None):
else:
await TelemetryClient.send_event(
- Category.ONBOARDING,
+ Category.ONBOARDING,
MessageId.ORB_ONBOARD_FAILED
)
return JSONResponse(
@@ -1140,7 +1155,7 @@ async def onboarding(request, flows_service, session_manager=None):
except Exception as e:
logger.error("Failed to update onboarding settings", error=str(e))
await TelemetryClient.send_event(
- Category.ONBOARDING,
+ Category.ONBOARDING,
MessageId.ORB_ONBOARD_FAILED
)
return JSONResponse(
@@ -1279,33 +1294,33 @@ async def _update_mcp_servers_with_provider_credentials(config, session_manager
try:
from services.langflow_mcp_service import LangflowMCPService
from utils.langflow_headers import build_mcp_global_vars_from_config
-
+
mcp_service = LangflowMCPService()
-
+
# Build global vars using utility function
mcp_global_vars = build_mcp_global_vars_from_config(config)
-
+
# In no-auth mode, add the anonymous JWT token and user details
if is_no_auth_mode() and session_manager:
from session_manager import AnonymousUser
-
+
# Create/get anonymous JWT for no-auth mode
anonymous_jwt = session_manager.get_effective_jwt_token(None, None)
if anonymous_jwt:
mcp_global_vars["JWT"] = anonymous_jwt
-
+
# Add anonymous user details
anonymous_user = AnonymousUser()
mcp_global_vars["OWNER"] = anonymous_user.user_id # "anonymous"
mcp_global_vars["OWNER_NAME"] = f'"{anonymous_user.name}"' # "Anonymous User" (quoted)
mcp_global_vars["OWNER_EMAIL"] = anonymous_user.email # "anonymous@localhost"
-
+
logger.debug("Added anonymous JWT and user details to MCP servers for no-auth mode")
-
+
if mcp_global_vars:
result = await mcp_service.update_mcp_servers_with_global_vars(mcp_global_vars)
logger.info("Updated MCP servers with provider credentials after settings change", **result)
-
+
except Exception as mcp_error:
logger.warning(f"Failed to update MCP servers after settings change: {str(mcp_error)}")
# Don't fail the entire settings update if MCP update fails
@@ -1392,10 +1407,10 @@ async def update_onboarding_state(request):
"""Update onboarding state in configuration"""
try:
await TelemetryClient.send_event(Category.ONBOARDING, MessageId.ORB_ONBOARD_START)
-
+
# Parse request body
body = await request.json()
-
+
# Validate allowed fields
allowed_fields = {
"current_step",
@@ -1406,7 +1421,7 @@ async def update_onboarding_state(request):
"openrag_docs_filter_id",
"user_doc_filter_id",
}
-
+
# Check for invalid fields
invalid_fields = set(body.keys()) - allowed_fields
if invalid_fields:
@@ -1416,25 +1431,25 @@ async def update_onboarding_state(request):
},
status_code=400,
)
-
+
# Update onboarding state using config manager
success = config_manager.update_onboarding_state(**body)
-
+
if not success:
return JSONResponse(
{"error": "Failed to update onboarding state"},
status_code=500,
)
-
+
logger.info(f"Onboarding state updated: {body}")
-
+
return JSONResponse(
{
"message": "Onboarding state updated successfully",
"updated_fields": list(body.keys()),
}
)
-
+
except json.JSONDecodeError:
return JSONResponse(
{"error": "Invalid JSON in request body"}, status_code=400
@@ -1499,7 +1514,7 @@ async def reapply_all_settings(session_manager = None):
async def rollback_onboarding(request, session_manager, task_service):
"""Rollback onboarding configuration when sample data files fail.
-
+
This will:
1. Cancel all active tasks
2. Delete successfully ingested knowledge documents
@@ -1522,15 +1537,15 @@ async def rollback_onboarding(request, session_manager, task_service):
# Get all tasks for the user
all_tasks = task_service.get_all_tasks(user.user_id)
-
+
cancelled_tasks = []
deleted_files = []
-
+
# Cancel all active tasks and collect successfully ingested files
for task_data in all_tasks:
task_id = task_data.get("task_id")
task_status = task_data.get("status")
-
+
# Cancel active tasks (pending, running, processing)
if task_status in ["pending", "running", "processing"]:
try:
@@ -1540,7 +1555,7 @@ async def rollback_onboarding(request, session_manager, task_service):
logger.info(f"Cancelled task {task_id}")
except Exception as e:
logger.error(f"Failed to cancel task {task_id}: {str(e)}")
-
+
# For completed tasks, find successfully ingested files and delete them
elif task_status == "completed":
files = task_data.get("files", {})
@@ -1550,26 +1565,26 @@ async def rollback_onboarding(request, session_manager, task_service):
if isinstance(file_info, dict):
file_status = file_info.get("status")
filename = file_info.get("filename") or file_path.split("/")[-1]
-
+
if file_status == "completed" and filename:
try:
# Get user's OpenSearch client
opensearch_client = session_manager.get_user_opensearch_client(
user.user_id, jwt_token
)
-
+
# Delete documents by filename
from utils.opensearch_queries import build_filename_delete_body
from config.settings import get_index_name
-
+
delete_query = build_filename_delete_body(filename)
-
+
result = await opensearch_client.delete_by_query(
index=get_index_name(),
body=delete_query,
conflicts="proceed"
)
-
+
deleted_count = result.get("deleted", 0)
if deleted_count > 0:
deleted_files.append(filename)
@@ -1580,7 +1595,7 @@ async def rollback_onboarding(request, session_manager, task_service):
# Clear embedding provider and model settings
current_config.knowledge.embedding_provider = "openai" # Reset to default
current_config.knowledge.embedding_model = ""
-
+
# Mark config as not edited so user can go through onboarding again
current_config.edited = False
@@ -1588,17 +1603,17 @@ async def rollback_onboarding(request, session_manager, task_service):
try:
import yaml
config_file = config_manager.config_file
-
+
# Ensure directory exists
config_file.parent.mkdir(parents=True, exist_ok=True)
-
+
# Save config with edited=False
with open(config_file, "w") as f:
yaml.dump(current_config.to_dict(), f, default_flow_style=False, indent=2)
-
+
# Update cached config
config_manager._config = current_config
-
+
logger.info("Successfully saved rolled back configuration with edited=False")
except Exception as e:
logger.error(f"Failed to save rolled back configuration: {e}")
@@ -1611,10 +1626,10 @@ async def rollback_onboarding(request, session_manager, task_service):
f"Cancelled {len(cancelled_tasks)} tasks, deleted {len(deleted_files)} files"
)
await TelemetryClient.send_event(
- Category.ONBOARDING,
+ Category.ONBOARDING,
MessageId.ORB_ONBOARD_ROLLBACK
)
-
+
return JSONResponse(
{
"message": "Onboarding configuration rolled back successfully",
diff --git a/src/config/settings.py b/src/config/settings.py
index 94a61adf8..51a7906f9 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -92,6 +92,8 @@ WEBHOOK_BASE_URL = os.getenv(
# OpenSearch configuration
VECTOR_DIM = 1536
+KNN_EF_CONSTRUCTION = 512
+KNN_M = 16
EMBED_MODEL = "text-embedding-3-small"
OPENAI_EMBEDDING_DIMENSIONS = {
@@ -135,7 +137,7 @@ INDEX_BODY = {
"name": "disk_ann",
"engine": "jvector",
"space_type": "l2",
- "parameters": {"ef_construction": 100, "m": 16},
+ "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},
},
},
# Track which embedding model was used for this chunk
@@ -309,9 +311,6 @@ class AppClients:
self.converter = None
async def initialize(self):
- # Generate Langflow API key first
- await get_langflow_api_key()
-
# Initialize OpenSearch client
self.opensearch = AsyncOpenSearch(
hosts=[{"host": OPENSEARCH_HOST, "port": OPENSEARCH_PORT}],
@@ -324,26 +323,6 @@ class AppClients:
http_compress=True,
)
- # Initialize Langflow client with generated/provided API key
- if LANGFLOW_KEY and self.langflow_client is None:
- try:
- if not OPENSEARCH_PASSWORD:
- raise ValueError("OPENSEARCH_PASSWORD is not set")
- else:
- await self.ensure_langflow_client()
- # Note: OPENSEARCH_PASSWORD global variable should be created automatically
- # via LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT in docker-compose
- logger.info(
- "Langflow client initialized - OPENSEARCH_PASSWORD should be available via environment variables"
- )
- except Exception as e:
- logger.warning("Failed to initialize Langflow client", error=str(e))
- self.langflow_client = None
- if self.langflow_client is None:
- logger.warning(
- "No Langflow client initialized yet, will attempt later on first use"
- )
-
# Initialize patched OpenAI client if API key is available
# This allows the app to start even if OPENAI_API_KEY is not set yet
# (e.g., when it will be provided during onboarding)
@@ -358,6 +337,7 @@ class AppClients:
self.converter = create_document_converter(ocr_engine=DOCLING_OCR_ENGINE)
# Initialize Langflow HTTP client with extended timeouts for large documents
+ # Must be created before wait_for_langflow / get_langflow_api_key
# Use explicit timeout configuration to handle large PDF ingestion (300+ pages)
self.langflow_http_client = httpx.AsyncClient(
base_url=LANGFLOW_URL,
@@ -375,6 +355,34 @@ class AppClients:
connect_timeout_seconds=LANGFLOW_CONNECT_TIMEOUT,
)
+ # Wait for Langflow to be healthy before generating API key
+ # Deferred import to avoid circular dependency (langflow_utils imports config.settings.clients)
+ from utils.langflow_utils import wait_for_langflow
+ await wait_for_langflow()
+
+ # Generate Langflow API key now that Langflow is confirmed ready
+ await get_langflow_api_key()
+
+ # Initialize Langflow client with generated/provided API key
+ if LANGFLOW_KEY and self.langflow_client is None:
+ try:
+ if not OPENSEARCH_PASSWORD:
+ raise ValueError("OPENSEARCH_PASSWORD is not set")
+ else:
+ await self.ensure_langflow_client()
+ # Note: OPENSEARCH_PASSWORD global variable should be created automatically
+ # via LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT in docker-compose
+ logger.info(
+ "Langflow client initialized - OPENSEARCH_PASSWORD should be available via environment variables"
+ )
+ except Exception as e:
+ logger.warning("Failed to initialize Langflow client", error=str(e))
+ self.langflow_client = None
+ if self.langflow_client is None:
+ logger.warning(
+ "No Langflow client initialized yet, will attempt later on first use"
+ )
+
return self
async def ensure_langflow_client(self):
diff --git a/src/main.py b/src/main.py
index b51ee9b0b..7de3c13b3 100644
--- a/src/main.py
+++ b/src/main.py
@@ -283,6 +283,12 @@ async def init_index():
raise e
+async def init_index_when_ready():
+ """Wait for the OpenSearch service to be ready and then initialize the OpenSearch index."""
+ await wait_for_opensearch()
+ await init_index()
+
+
def generate_jwt_keys():
"""Generate RSA keys for JWT signing if they don't exist"""
keys_dir = "keys"
@@ -338,19 +344,6 @@ def generate_jwt_keys():
logger.warning("Failed to set permissions on existing keys", error=str(e))
-async def init_index_when_ready():
- """Initialize OpenSearch index when it becomes available"""
- try:
- await init_index()
- logger.info("OpenSearch index initialization completed successfully")
- except Exception as e:
- logger.error("OpenSearch index initialization failed", error=str(e))
- await TelemetryClient.send_event(Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_INIT_FAIL)
- logger.warning(
- "OIDC endpoints will still work, but document operations may fail until OpenSearch is ready"
- )
-
-
def _get_documents_dir():
"""Get the documents directory path, handling both Docker and local environments."""
# In Docker, the volume is mounted at /app/openrag-documents
@@ -580,6 +573,33 @@ async def startup_tasks(services):
if DISABLE_INGEST_WITH_LANGFLOW:
await _ensure_opensearch_index()
+ # Ensure that the OpenSearch index exists if onboarding was already completed
+ # - Handles the case where OpenSearch is reset (e.g., volume deleted) after onboarding
+ embedding_model = None
+ try:
+ config = get_openrag_config()
+ embedding_model = config.knowledge.embedding_model
+
+ if config.edited and embedding_model:
+ logger.info(
+ "Ensuring that the OpenSearch index exists (after onboarding)...",
+ embedding_model=embedding_model,
+ )
+
+ await init_index()
+
+ logger.info(
+ "Successfully ensured that the OpenSearch index exists (after onboarding).",
+ embedding_model=embedding_model,
+ )
+ except Exception as e:
+ logger.error(
+ "Failed to ensure that the OpenSearch index exists (after onboarding).",
+ embedding_model=embedding_model,
+ error=str(e),
+ )
+ raise
+
# Configure alerting security
await configure_alerting_security()
diff --git a/src/utils/embedding_fields.py b/src/utils/embedding_fields.py
index 02342407b..27469f374 100644
--- a/src/utils/embedding_fields.py
+++ b/src/utils/embedding_fields.py
@@ -86,7 +86,7 @@ async def ensure_embedding_field_exists(
Raises:
Exception: If unable to add the field mapping
"""
- from config.settings import get_index_name
+ from config.settings import KNN_EF_CONSTRUCTION, KNN_M, get_index_name
from utils.embeddings import get_embedding_dimensions
if index_name is None:
@@ -134,7 +134,7 @@ async def ensure_embedding_field_exists(
"name": "disk_ann",
"engine": "jvector",
"space_type": "l2",
- "parameters": {"ef_construction": 100, "m": 16},
+ "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},
},
},
# Also ensure the embedding_model tracking field exists as keyword
diff --git a/src/utils/embeddings.py b/src/utils/embeddings.py
index 598bf1016..1e2944b94 100644
--- a/src/utils/embeddings.py
+++ b/src/utils/embeddings.py
@@ -1,5 +1,5 @@
import httpx
-from config.settings import OPENAI_EMBEDDING_DIMENSIONS, VECTOR_DIM, WATSONX_EMBEDDING_DIMENSIONS
+from config.settings import KNN_EF_CONSTRUCTION, KNN_M, OPENAI_EMBEDDING_DIMENSIONS, VECTOR_DIM, WATSONX_EMBEDDING_DIMENSIONS
from utils.container_utils import transform_localhost_url
from utils.logging_config import get_logger
@@ -176,7 +176,7 @@ async def create_dynamic_index_body(
"name": "disk_ann",
"engine": "jvector",
"space_type": "l2",
- "parameters": {"ef_construction": 100, "m": 16},
+ "parameters": {"ef_construction": KNN_EF_CONSTRUCTION, "m": KNN_M},
},
},
# Track which embedding model was used for this chunk
diff --git a/src/utils/langflow_utils.py b/src/utils/langflow_utils.py
new file mode 100644
index 000000000..f48f33e70
--- /dev/null
+++ b/src/utils/langflow_utils.py
@@ -0,0 +1,76 @@
+import asyncio
+import random
+
+from config.settings import clients
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class LangflowNotReadyError(Exception):
+ """Raised when Langflow fails to become ready within the retry limit."""
+
+
+async def wait_for_langflow(max_retries: int = 10, base_delay: float = 1.0, max_delay: float = 20.0) -> None:
+ """Wait for Langflow to be ready with exponential backoff and jitter.
+
+ Args:
+ max_retries: Maximum number of retry attempts.
+ base_delay: Initial delay in seconds before the first retry.
+ max_delay: Upper bound in seconds for the retry delay.
+
+ Raises:
+ LangflowNotReadyError: If Langflow fails to become ready within the retry limit.
+ """
+ for attempt in range(max_retries):
+ display_attempt: int = attempt + 1
+
+ logger.info(
+ "Verifying whether the Langflow service is ready...",
+ attempt=display_attempt,
+ max_retries=max_retries,
+ )
+
+ try:
+ response = await clients.langflow_http_client.get("/health", timeout=5.0)
+ status_code = response.status_code
+
+ if status_code == 200:
+ logger.info(
+ "Successfully verified that the Langflow service is ready.",
+ attempt=display_attempt,
+ max_retries=max_retries,
+ status_code=status_code,
+ )
+ return
+ else:
+ logger.warning(
+ "The Langflow service is not ready. Encountered a non-200 HTTP status_code.",
+ attempt=display_attempt,
+ max_retries=max_retries,
+ status_code=status_code,
+ )
+ except Exception as e:
+ logger.warning(
+ "The Langflow service is not ready.",
+ attempt=display_attempt,
+ max_retries=max_retries,
+ error=str(e),
+ )
+
+ if attempt < max_retries - 1:
+ delay = min(base_delay * (2 ** attempt), max_delay)
+ delay = random.uniform(delay / 2, delay)
+
+ logger.debug(
+ "Retry the Langflow service readiness check after a delay (seconds).",
+ attempt=display_attempt,
+ max_retries=max_retries,
+ delay=delay,
+ )
+
+ await asyncio.sleep(delay)
+
+ message: str = "Failed to verify whether the Langflow service is ready."
+ logger.error(message)
+ raise LangflowNotReadyError(message)
diff --git a/src/utils/logging_config.py b/src/utils/logging_config.py
index d516909b4..b4bd596f1 100644
--- a/src/utils/logging_config.py
+++ b/src/utils/logging_config.py
@@ -5,6 +5,20 @@ import structlog
from structlog import processors
+LOC_WIDTH_SHORT = 30
+LOC_WIDTH_LONG = 60
+
+LEVEL_COLORS = {
+ "DEBUG": "\033[36m", # Cyan
+ "INFO": "\033[32m", # Green
+ "WARNING": "\033[33m", # Yellow
+ "ERROR": "\033[31m", # Red
+ "CRITICAL": "\033[1;31m", # Bold red
+}
+DIM = "\033[38;5;244m" # Medium grey
+RESET = "\033[0m"
+
+
def configure_logging(
log_level: str = "INFO",
json_logs: bool = False,
@@ -48,8 +62,7 @@ def configure_logging(
console_renderer = structlog.processors.JSONRenderer()
else:
# Custom clean format: timestamp path/file:loc logentry
- LOC_WIDTH_SHORT = 30
- LOC_WIDTH_LONG = 60
+ use_colors = "NO_COLOR" not in os.environ and hasattr(sys.stderr, "isatty") and sys.stderr.isatty()
def custom_formatter(logger, log_method, event_dict):
timestamp = event_dict.pop("timestamp", "")
@@ -80,12 +93,22 @@ def configure_logging(
if event:
message_parts.append(event)
- header = f"[{timestamp}] [{level:<7}] [{location:<{loc_width}}] "
+ if use_colors:
+ colored_timestamp = f"{DIM}{timestamp}{RESET}"
+ color = LEVEL_COLORS.get(level, "")
+ colored_level = f"{color}{level:<7}{RESET}"
+ else:
+ colored_timestamp = timestamp
+ colored_level = f"{level:<7}"
+
+ header = f"[{colored_timestamp}] [{colored_level}] [{location:<{loc_width}}] "
+ # Visible width excludes ANSI escape codes for correct padding
+ visible_header = f"[{timestamp}] [{level:<7}] [{location:<{loc_width}}] "
# Add any remaining context as indented multi-line fields
extra = {k: v for k, v in event_dict.items() if k not in ["service", "func_name"]}
if extra:
- padding = " " * len(header)
+ padding = " " * len(visible_header)
for key, value in extra.items():
message_parts.append(f"\n{padding}- {key}: {value}")
diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py
new file mode 100644
index 000000000..878ba1997
--- /dev/null
+++ b/tests/unit/conftest.py
@@ -0,0 +1,13 @@
+"""Unit test configuration.
+
+Overrides the session-scoped onboard_system fixture from the root conftest
+so that unit tests don't require running infrastructure (Langflow, OpenSearch, etc.).
+"""
+
+import pytest_asyncio
+
+
+@pytest_asyncio.fixture(scope="session", autouse=True)
+async def onboard_system():
+ """No-op override — unit tests mock their own dependencies."""
+ yield
diff --git a/tests/unit/test_langflow_utils.py b/tests/unit/test_langflow_utils.py
new file mode 100644
index 000000000..e009e0f15
--- /dev/null
+++ b/tests/unit/test_langflow_utils.py
@@ -0,0 +1,202 @@
+"""
+Tests for utils/langflow_utils.py
+Validates wait_for_langflow retry logic, backoff behavior, and error handling.
+All external dependencies (HTTP client, sleep, logging) are fully mocked.
+"""
+
+import pytest
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from utils.langflow_utils import LangflowNotReadyError, wait_for_langflow
+
+
+def _make_response(status_code: int) -> MagicMock:
+ """Create a mock HTTP response with the given status code."""
+ resp = MagicMock()
+ resp.status_code = status_code
+ return resp
+
+
+def mock_langflow_client():
+ """Provide a mocked langflow HTTP client and patch it into settings."""
+ client = AsyncMock()
+ with patch("utils.langflow_utils.clients") as mock_clients:
+ mock_clients.langflow_http_client = client
+ yield client
+
+
[email protected](autouse=True)
+def no_sleep():
+ """Patch asyncio.sleep so tests run instantly."""
+ with patch("utils.langflow_utils.asyncio.sleep", new_callable=AsyncMock) as mock_sleep:
+ yield mock_sleep
+
+
+# ── Success on first attempt ──────────────────────────────────────────
+
+
+async def test_ready_on_first_attempt(mock_langflow_client, no_sleep):
+ """Returns immediately when health check responds 200 on the first try."""
+ mock_langflow_client.get.return_value = _make_response(200)
+
+ await wait_for_langflow(max_retries=3)
+
+ mock_langflow_client.get.assert_called_once_with("/health", timeout=5.0)
+ no_sleep.assert_not_called()
+
+
+# ── Success after transient failures ─────────────────────────────────
+
+
+async def test_ready_after_non_200_then_200(mock_langflow_client, no_sleep):
+ """Retries on non-200 responses and succeeds when 200 is returned."""
+ mock_langflow_client.get.side_effect = [
+ _make_response(503),
+ _make_response(200),
+ ]
+
+ await wait_for_langflow(max_retries=3)
+
+ assert mock_langflow_client.get.call_count == 2
+ assert no_sleep.call_count == 1
+
+
+async def test_ready_after_exception_then_200(mock_langflow_client, no_sleep):
+ """Retries on connection errors and succeeds when 200 is returned."""
+ mock_langflow_client.get.side_effect = [
+ ConnectionError("refused"),
+ _make_response(200),
+ ]
+
+ await wait_for_langflow(max_retries=3)
+
+ assert mock_langflow_client.get.call_count == 2
+ assert no_sleep.call_count == 1
+
+
+async def test_ready_after_mixed_failures(mock_langflow_client, no_sleep):
+ """Handles a mix of exceptions and non-200 responses before success."""
+ mock_langflow_client.get.side_effect = [
+ ConnectionError("refused"),
+ _make_response(500),
+ TimeoutError("timed out"),
+ _make_response(200),
+ ]
+
+ await wait_for_langflow(max_retries=5)
+
+ assert mock_langflow_client.get.call_count == 4
+ assert no_sleep.call_count == 3
+
+
+# ── Exhausted retries ────────────────────────────────────────────────
+
+
+async def test_raises_after_all_retries_exhausted_non_200(mock_langflow_client):
+ """Raises LangflowNotReadyError when every attempt returns non-200."""
+ mock_langflow_client.get.return_value = _make_response(503)
+
+ with pytest.raises(LangflowNotReadyError):
+ await wait_for_langflow(max_retries=3)
+
+ assert mock_langflow_client.get.call_count == 3
+
+
+async def test_raises_after_all_retries_exhausted_exception(mock_langflow_client):
+ """Raises LangflowNotReadyError when every attempt raises an exception."""
+ mock_langflow_client.get.side_effect = ConnectionError("refused")
+
+ with pytest.raises(LangflowNotReadyError):
+ await wait_for_langflow(max_retries=2)
+
+ assert mock_langflow_client.get.call_count == 2
+
+
+async def test_single_retry_no_sleep_before_raise(mock_langflow_client, no_sleep):
+ """With max_retries=1, fails immediately without sleeping."""
+ mock_langflow_client.get.return_value = _make_response(500)
+
+ with pytest.raises(LangflowNotReadyError):
+ await wait_for_langflow(max_retries=1)
+
+ mock_langflow_client.get.assert_called_once()
+ no_sleep.assert_not_called()
+
+
+# ── Backoff behavior ─────────────────────────────────────────────────
+
+
+async def test_sleep_delay_respects_bounds(mock_langflow_client, no_sleep):
+ """Sleep delay stays within [delay/2, delay] and never exceeds max_delay."""
+ mock_langflow_client.get.return_value = _make_response(503)
+ base_delay = 1.0
+ max_delay = 4.0
+
+ with pytest.raises(LangflowNotReadyError):
+ await wait_for_langflow(max_retries=5, base_delay=base_delay, max_delay=max_delay)
+
+ # 4 sleeps for 5 retries (no sleep after the last attempt)
+ assert no_sleep.call_count == 4
+
+ for call in no_sleep.call_args_list:
+ delay = call.args[0]
+ assert 0 <= delay <= max_delay
+
+
+async def test_exponential_backoff_increases(mock_langflow_client, no_sleep):
+ """Backoff upper bound doubles each attempt (before capping at max_delay)."""
+ mock_langflow_client.get.return_value = _make_response(503)
+
+ with patch("utils.langflow_utils.random.uniform", side_effect=lambda lo, hi: hi):
+ with pytest.raises(LangflowNotReadyError):
+ await wait_for_langflow(max_retries=4, base_delay=1.0, max_delay=100.0)
+
+ # With jitter pinned to the upper bound, delays should be 1, 2, 4
+ delays = [call.args[0] for call in no_sleep.call_args_list]
+ assert delays == [1.0, 2.0, 4.0]
+
+
+async def test_max_delay_cap(mock_langflow_client, no_sleep):
+ """Delay is capped at max_delay even when exponential growth exceeds it."""
+ mock_langflow_client.get.return_value = _make_response(503)
+
+ with patch("utils.langflow_utils.random.uniform", side_effect=lambda lo, hi: hi):
+ with pytest.raises(LangflowNotReadyError):
+ await wait_for_langflow(max_retries=6, base_delay=2.0, max_delay=5.0)
+
+ delays = [call.args[0] for call in no_sleep.call_args_list]
+ # base_delay * 2^attempt: 2, 4, 5(cap), 5(cap), 5(cap)
+ assert delays == [2.0, 4.0, 5.0, 5.0, 5.0]
+
+
+# ── Edge cases ───────────────────────────────────────────────────────
+
+
+async def test_default_parameters(mock_langflow_client, no_sleep):
+ """Works correctly with default parameter values."""
+ mock_langflow_client.get.return_value = _make_response(200)
+
+ await wait_for_langflow()
+
+ mock_langflow_client.get.assert_called_once()
+
+
+async def test_error_message_content(mock_langflow_client):
+ """LangflowNotReadyError contains a meaningful message."""
+ mock_langflow_client.get.return_value = _make_response(503)
+
+ with pytest.raises(LangflowNotReadyError, match="Failed to verify"):
+ await wait_for_langflow(max_retries=1)