OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
Configurable index name (#906)
matanor648f11bccfa5
diff --git a/scripts/migrate_embedding_model_field.py b/scripts/migrate_embedding_model_field.pyindex d90f270ea..ec7f093cf 100644--- a/scripts/migrate_embedding_model_field.py+++ b/scripts/migrate_embedding_model_field.py@@ -59,7 +59,7 @@ from config.settings import (OPENSEARCH_PORT,OPENSEARCH_USERNAME,OPENSEARCH_PASSWORD,- INDEX_NAME,+ get_index_name,)from utils.logging_config import get_loggerfrom utils.embedding_fields import get_embedding_field_name@@ -231,7 +231,7 @@ async def migrate_legacy_embeddings() -> bool:"""Main migration function."""if index_name is None:- index_name = INDEX_NAME+ index_name = get_index_name()new_field_name = get_embedding_field_name(model_name)@@ -359,7 +359,7 @@ Examples:parser.add_argument('--index',default=None,- help=f'Index name (default: {INDEX_NAME})'+ help=f'Index name (default: {get_index_name()})')args = parser.parse_args()diff --git a/src/api/connector_router.py b/src/api/connector_router.pyindex 98e12f334..1dbe5770b 100644--- a/src/api/connector_router.py+++ b/src/api/connector_router.py@@ -5,7 +5,6 @@ from starlette.requests import Requestfrom config.settings import (DISABLE_INGEST_WITH_LANGFLOW,clients,- INDEX_NAME,INDEX_BODY,)from utils.logging_config import get_loggerdiff --git a/src/api/connectors.py b/src/api/connectors.pyindex 579fd4211..da76541d2 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -1,9 +1,9 @@from starlette.requests import Requestfrom starlette.responses import JSONResponse, PlainTextResponsefrom connectors.sharepoint.utils import is_valid_sharepoint_url+from config.settings import get_index_namefrom utils.logging_config import get_loggerfrom utils.telemetry import TelemetryClient, Category, MessageId-from config.settings import INDEX_NAMElogger = get_logger(__name__)@@ -49,7 +49,7 @@ async def get_synced_file_ids_for_connector(}result = await opensearch_client.search(- index=INDEX_NAME,+ index=get_index_name(),body=query_body)diff --git a/src/api/documents.py b/src/api/documents.pyindex 048f746a3..8d7d386ec 100644--- a/src/api/documents.py+++ b/src/api/documents.py@@ -1,7 +1,7 @@from starlette.requests import Requestfrom starlette.responses import JSONResponsefrom utils.logging_config import get_logger-from config.settings import INDEX_NAME+from config.settings import get_index_namelogger = get_logger(__name__)@@ -27,10 +27,10 @@ async def check_filename_exists(request: Request, document_service, session_manasearch_body = build_filename_search_body(filename, size=1, source=["filename"])- logger.debug(f"Checking filename existence: {filename}")+ logger.debug(f"Checking filename existence", filename=filename, index_name=get_index_name())response = await opensearch_client.search(- index=INDEX_NAME,+ index=get_index_name(),body=search_body)@@ -79,7 +79,7 @@ async def delete_documents_by_filename(request: Request, document_service, sessilogger.debug(f"Deleting documents with filename: {filename}")result = await opensearch_client.delete_by_query(- index=INDEX_NAME,+ index=get_index_name(),body=delete_query,conflicts="proceed")diff --git a/src/api/settings.py b/src/api/settings.pyindex 2b88f0bbe..36b558fe7 100644--- a/src/api/settings.py+++ b/src/api/settings.py@@ -13,6 +13,7 @@ from config.settings import (LANGFLOW_PUBLIC_URL,LOCALHOST_URL,clients,+ get_index_name,get_openrag_config,config_manager,is_no_auth_mode,@@ -105,6 +106,7 @@ async def get_settings(request, session_manager):"table_structure": knowledge_config.table_structure,"ocr": knowledge_config.ocr,"picture_descriptions": knowledge_config.picture_descriptions,+ "index_name": knowledge_config.index_name,},"agent": {"llm_model": agent_config.llm_model,@@ -219,6 +221,7 @@ async def update_settings(request, session_manager):"picture_descriptions","embedding_model","embedding_provider",+ "index_name",# Provider-specific fields (structured as provider_name.field_name)"openai_api_key","anthropic_api_key",@@ -278,6 +281,16 @@ async def update_settings(request, session_manager):status_code=400,)+ if "index_name" in body:+ if (+ not isinstance(body["index_name"], str)+ or not body["index_name"].strip()+ ):+ return JSONResponse(+ {"error": "index_name must be a non-empty string"},+ status_code=400,+ )+if "llm_provider" in body:if (not isinstance(body["llm_provider"], str)@@ -563,6 +576,27 @@ async def update_settings(request, session_manager):except Exception as e:logger.error(f"Failed to update ingest flow chunk overlap: {str(e)}")# Don't fail the entire settings update if flow update fails+ if "index_name" in body:+ old_index_name = current_config.knowledge.index_name+ new_index_name = body["index_name"].strip()+ current_config.knowledge.index_name = new_index_name+ config_updated = True+ await TelemetryClient.send_event(+ Category.SETTINGS_OPERATIONS,+ MessageId.ORB_SETTINGS_INDEX_NAME_UPDATED+ )+ logger.info(f"Index name changed from {old_index_name} to {new_index_name}")++ # Also update global variable with new index name+ try:+ await clients._create_langflow_global_variable("OPENSEARCH_INDEX_NAME", new_index_name, modify=True)+ logger.info(+ f"Successfully updated global variable with new index name {new_index_name}"+ )+ except Exception as e:+ logger.error(f"Failed to update global variable with new index name: {str(e)}")+ # Don't fail the entire settings update if flow update fails+# The config will still be saved# Update provider-specific settings@@ -1539,12 +1573,12 @@ async def rollback_onboarding(request, session_manager, task_service):# Delete documents by filenamefrom utils.opensearch_queries import build_filename_delete_body- from config.settings import INDEX_NAME+ from config.settings import get_index_namedelete_query = build_filename_delete_body(filename)result = await opensearch_client.delete_by_query(- index=INDEX_NAME,+ index=get_index_name(),body=delete_query,conflicts="proceed")diff --git a/src/api/v1/documents.py b/src/api/v1/documents.pyindex f2a30c3cb..66b91ea97 100644--- a/src/api/v1/documents.py+++ b/src/api/v1/documents.py@@ -115,7 +115,7 @@ async def delete_document_endpoint(request: Request, document_service, session_muser = request.state.usertry:- from config.settings import INDEX_NAME+ from config.settings import get_index_namefrom utils.opensearch_queries import build_filename_delete_body# Get OpenSearch client (API key auth uses internal client)@@ -127,7 +127,7 @@ async def delete_document_endpoint(request: Request, document_service, session_mdelete_query = build_filename_delete_body(filename)result = await opensearch_client.delete_by_query(- index=INDEX_NAME,+ index=get_index_name(),body=delete_query,conflicts="proceed")diff --git a/src/config/config_manager.py b/src/config/config_manager.pyindex 6af07db4d..8b3bb38b8 100644--- a/src/config/config_manager.py+++ b/src/config/config_manager.py@@ -74,6 +74,7 @@ class KnowledgeConfig:table_structure: bool = Trueocr: bool = Falsepicture_descriptions: bool = False+ index_name: str = "documents" # OpenSearch index name@dataclass@@ -249,6 +250,8 @@ class ConfigManager:config_data["knowledge"]["chunk_size"] = int(os.getenv("CHUNK_SIZE"))if os.getenv("CHUNK_OVERLAP"):config_data["knowledge"]["chunk_overlap"] = int(os.getenv("CHUNK_OVERLAP"))+ if os.getenv("OPENSEARCH_INDEX_NAME"):+ config_data["knowledge"]["index_name"] = os.getenv("OPENSEARCH_INDEX_NAME")if os.getenv("OCR_ENABLED"):config_data["knowledge"]["ocr"] = os.getenv("OCR_ENABLED").lower() in ("true",diff --git a/src/config/settings.py b/src/config/settings.pyindex f4784e5ed..c2be7bb90 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -86,7 +86,6 @@ WEBHOOK_BASE_URL = os.getenv() # No default - must be explicitly configured# OpenSearch configuration-INDEX_NAME = os.getenv("OPENSEARCH_INDEX_NAME", "documents")VECTOR_DIM = 1536EMBED_MODEL = "text-embedding-3-small"@@ -825,3 +824,8 @@ def get_agent_config():def get_embedding_model() -> str:"""Return the currently configured embedding model."""return get_openrag_config().knowledge.embedding_model or EMBED_MODEL if DISABLE_INGEST_WITH_LANGFLOW else ""+++def get_index_name() -> str:+ """Return the currently configured index name."""+ return get_openrag_config().knowledge.index_namediff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.pyindex 4202051f5..f80df0b02 100644--- a/src/connectors/langflow_connector_service.py+++ b/src/connectors/langflow_connector_service.py@@ -79,10 +79,10 @@ class LangflowConnectorService:# This prevents duplicate chunks when syncing filesif self.session_manager:try:- from config.settings import INDEX_NAME+ from config.settings import get_index_nameopensearch_client = self.session_manager.get_user_opensearch_client(owner_user_id, jwt_token)delete_body = {"query": {"term": {"filename": processed_filename}}}- delete_result = await opensearch_client.delete_by_query(index=INDEX_NAME, body=delete_body)+ delete_result = await opensearch_client.delete_by_query(index=get_index_name(), body=delete_body)deleted_count = delete_result.get("deleted", 0)logger.info("Deleted existing chunks before re-ingestion", filename=processed_filename, deleted_count=deleted_count)except Exception as delete_err:diff --git a/src/main.py b/src/main.pyindex 5168209c7..677601396 100644--- a/src/main.py+++ b/src/main.py@@ -70,10 +70,10 @@ from config.settings import (DISABLE_INGEST_WITH_LANGFLOW,INGESTION_TIMEOUT,INDEX_BODY,- INDEX_NAME,SESSION_SECRET,clients,get_embedding_model,+ get_index_name,is_no_auth_mode,get_openrag_config,)@@ -155,16 +155,17 @@ async def configure_alerting_security():async def _ensure_opensearch_index():"""Ensure OpenSearch index exists when using traditional connector service."""try:+ index_name = get_index_name()# Check if index already exists- if await clients.opensearch.indices.exists(index=INDEX_NAME):- logger.debug("OpenSearch index already exists", index_name=INDEX_NAME)+ if await clients.opensearch.indices.exists(index=index_name):+ logger.debug("OpenSearch index already exists", index_name=index_name)return# Create the index with hard-coded INDEX_BODY (uses OpenAI embedding dimensions)- await clients.opensearch.indices.create(index=INDEX_NAME, body=INDEX_BODY)+ await clients.opensearch.indices.create(index=index_name, body=INDEX_BODY)logger.info("Created OpenSearch index for traditional connector service",- index_name=INDEX_NAME,+ index_name=index_name,vector_dimensions=INDEX_BODY["mappings"]["properties"]["chunk_embedding"]["dimension"],@@ -175,7 +176,7 @@ async def _ensure_opensearch_index():logger.error("Failed to initialize OpenSearch index for traditional connector service",error=str(e),- index_name=INDEX_NAME,+ index_name=get_index_name(),)await TelemetryClient.send_event(Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_CREATE_FAIL)# Don't raise the exception to avoid breaking the initialization@@ -201,20 +202,21 @@ async def init_index():)# Create documents index- if not await clients.opensearch.indices.exists(index=INDEX_NAME):+ index_name = get_index_name()+ if not await clients.opensearch.indices.exists(index=index_name):await clients.opensearch.indices.create(- index=INDEX_NAME, body=dynamic_index_body+ index=index_name, body=dynamic_index_body)logger.info("Created OpenSearch index",- index_name=INDEX_NAME,+ index_name=index_name,embedding_model=embedding_model,)await TelemetryClient.send_event(Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_CREATED)else:logger.info("Index already exists, skipping creation",- index_name=INDEX_NAME,+ index_name=index_name,embedding_model=embedding_model,)await TelemetryClient.send_event(Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_EXISTS)@@ -647,7 +649,7 @@ async def initialize_services():patched_async_client=clients, # Pass the clients object itselfprocess_pool=process_pool,embed_model=get_embedding_model(),- index_name=INDEX_NAME,+ index_name=get_index_name(),task_service=task_service,session_manager=session_manager,)diff --git a/src/models/processors.py b/src/models/processors.pyindex 098328d1a..9c857a0b9 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -21,7 +21,7 @@ class TaskProcessor:Check if a document with the given hash already exists in OpenSearch.Consolidated hash checking for all processors."""- from config.settings import INDEX_NAME+ from config.settings import get_index_nameimport asynciomax_retries = 3@@ -29,7 +29,7 @@ class TaskProcessor:for attempt in range(max_retries):try:- exists = await opensearch_client.exists(index=INDEX_NAME, id=file_hash)+ exists = await opensearch_client.exists(index=get_index_name(), id=file_hash)return existsexcept (asyncio.TimeoutError, Exception) as e:if attempt == max_retries - 1:@@ -65,7 +65,7 @@ class TaskProcessor:Check if a document with the given filename already exists in OpenSearch.Returns True if any chunks with this filename exist."""- from config.settings import INDEX_NAME+ from config.settings import get_index_namefrom utils.opensearch_queries import build_filename_search_bodyimport asyncio@@ -78,7 +78,7 @@ class TaskProcessor:search_body = build_filename_search_body(filename, size=1, source=False)response = await opensearch_client.search(- index=INDEX_NAME,+ index=get_index_name(),body=search_body)@@ -119,7 +119,7 @@ class TaskProcessor:"""Delete all chunks of a document with the given filename from OpenSearch."""- from config.settings import INDEX_NAME+ from config.settings import get_index_namefrom utils.opensearch_queries import build_filename_delete_bodytry:@@ -127,7 +127,7 @@ class TaskProcessor:delete_body = build_filename_delete_body(filename)response = await opensearch_client.delete_by_query(- index=INDEX_NAME,+ index=get_index_name(),body=delete_body)@@ -171,7 +171,7 @@ class TaskProcessor:acl: DocumentACL instance with access control information"""import datetime- from config.settings import INDEX_NAME, clients, get_embedding_model+ from config.settings import clients, get_embedding_model, get_index_namefrom services.document_service import chunk_texts_for_embeddingsfrom utils.document_processing import extract_relevantfrom utils.embedding_fields import get_embedding_field_name, ensure_embedding_field_exists@@ -190,7 +190,7 @@ class TaskProcessor:# Ensure the embedding field exists for this modelembedding_field_name = await ensure_embedding_field_exists(- opensearch_client, embedding_model, INDEX_NAME+ opensearch_client, embedding_model, get_index_name())logger.info(@@ -279,7 +279,7 @@ class TaskProcessor:chunk_id = f"{file_hash}_{i}"try:await opensearch_client.index(- index=INDEX_NAME, id=chunk_id, body=chunk_doc+ index=get_index_name(), id=chunk_id, body=chunk_doc)except Exception as e:logger.error(@@ -622,7 +622,7 @@ class S3FileProcessor(TaskProcessor):import timeimport asyncioimport datetime- from config.settings import INDEX_NAME, clients, get_embedding_model+ from config.settings import clients, get_embedding_model, get_index_namefrom services.document_service import chunk_texts_for_embeddingsfrom utils.document_processing import process_document_syncdiff --git a/src/services/document_service.py b/src/services/document_service.pyindex f40c3d823..f010b6c5a 100644--- a/src/services/document_service.py+++ b/src/services/document_service.py@@ -12,7 +12,7 @@ from utils.logging_config import get_loggerlogger = get_logger(__name__)-from config.settings import clients, INDEX_NAME, get_embedding_model+from config.settings import clients, get_embedding_model, get_index_namefrom utils.document_processing import extract_relevant, process_document_syncfrom utils.telemetry import TelemetryClient, Category, MessageId@@ -153,7 +153,7 @@ class DocumentService:)try:- exists = await opensearch_client.exists(index=INDEX_NAME, id=file_hash)+ exists = await opensearch_client.exists(index=get_index_name(), id=file_hash)except Exception as e:logger.error("OpenSearch exists check failed", file_hash=file_hash, error=str(e)diff --git a/src/services/search_service.py b/src/services/search_service.pyindex 9ef240acc..ef88bb9d4 100644--- a/src/services/search_service.py+++ b/src/services/search_service.py@@ -2,7 +2,7 @@ import copyimport jsonfrom typing import Any, Dictfrom agentd.tool_decorator import tool-from config.settings import EMBED_MODEL, clients, INDEX_NAME, get_embedding_model, WATSONX_EMBEDDING_DIMENSIONS+from config.settings import EMBED_MODEL, clients, get_embedding_model, get_index_name, WATSONX_EMBEDDING_DIMENSIONSfrom auth_context import get_auth_contextfrom utils.logging_config import get_logger@@ -121,7 +121,7 @@ class SearchService:}agg_result = await opensearch_client.search(- index=INDEX_NAME, body=agg_query, params={"terminate_after": 0}+ index=get_index_name(), body=agg_query, params={"terminate_after": 0})buckets = agg_result.get("aggregations", {}).get("embedding_models", {}).get("buckets", [])available_models = [b["key"] for b in buckets if b["key"]]@@ -396,8 +396,10 @@ class SearchService:search_params = {"terminate_after": 0}try:+ index_name = get_index_name()+ logger.info(f"Sending query to index '{index_name}'..")results = await opensearch_client.search(- index=INDEX_NAME, body=search_body, params=search_params+ index=index_name, body=search_body, params=search_params)except RequestError as e:error_message = str(e)@@ -410,7 +412,7 @@ class SearchService:)try:results = await opensearch_client.search(- index=INDEX_NAME,+ index=get_index_name(),body=fallback_search_body,params=search_params,)diff --git a/src/utils/embedding_fields.py b/src/utils/embedding_fields.pyindex 990cb116d..02342407b 100644--- a/src/utils/embedding_fields.py+++ b/src/utils/embedding_fields.py@@ -78,7 +78,7 @@ async def ensure_embedding_field_exists(Args:opensearch_client: OpenSearch client instancemodel_name: The embedding model name- index_name: OpenSearch index name (defaults to INDEX_NAME from settings)+ index_name: OpenSearch index name (defaults to get_index_name() from settings)Returns:The field name that was ensured to exist@@ -86,11 +86,11 @@ async def ensure_embedding_field_exists(Raises:Exception: If unable to add the field mapping"""- from config.settings import INDEX_NAME+ from config.settings import get_index_namefrom utils.embeddings import get_embedding_dimensionsif index_name is None:- index_name = INDEX_NAME+ index_name = get_index_name()field_name = get_embedding_field_name(model_name)dimensions = await get_embedding_dimensions(model_name)diff --git a/src/utils/telemetry/message_id.py b/src/utils/telemetry/message_id.pyindex c00e5eb35..a5b17656e 100644--- a/src/utils/telemetry/message_id.py+++ b/src/utils/telemetry/message_id.py@@ -176,6 +176,8 @@ class MessageId:ORB_SETTINGS_CHUNK_UPDATED = "ORB_SETTINGS_CHUNK_UPDATED"# Message: Docling settings updatedORB_SETTINGS_DOCLING_UPDATED = "ORB_SETTINGS_DOCLING_UPDATED"+ # Message: Index name updated+ ORB_SETTINGS_INDEX_NAME_UPDATED = "ORB_SETTINGS_INDEX_NAME_UPDATED"# Message: Provider credentials updatedORB_SETTINGS_PROVIDER_CREDS = "ORB_SETTINGS_PROVIDER_CREDS"diff --git a/tests/integration/test_api_endpoints.py b/tests/integration/test_api_endpoints.pyindex ddad3535a..bbc633aa7 100644--- a/tests/integration/test_api_endpoints.py+++ b/tests/integration/test_api_endpoints.py@@ -143,12 +143,12 @@ async def test_upload_and_search_endpoint(tmp_path: Path, disable_langflow_ingessys.modules.pop(mod, None)from src.main import create_app, startup_tasksimport src.api.router as upload_router- from src.config.settings import clients, INDEX_NAME, DISABLE_INGEST_WITH_LANGFLOW+ from src.config.settings import clients, get_index_name, DISABLE_INGEST_WITH_LANGFLOW# Ensure a clean index before startupawait clients.initialize()try:- await clients.opensearch.indices.delete(index=INDEX_NAME)+ await clients.opensearch.indices.delete(index=get_index_name())# Wait for deletion to completeawait asyncio.sleep(1)except Exception:@@ -164,7 +164,7 @@ async def test_upload_and_search_endpoint(tmp_path: Path, disable_langflow_inges# Verify index is truly empty after startuptry:- count_response = await clients.opensearch.count(index=INDEX_NAME)+ count_response = await clients.opensearch.count(index=get_index_name())doc_count = count_response.get('count', 0)assert doc_count == 0, f"Index should be empty after startup but contains {doc_count} documents"except Exception as e:@@ -480,11 +480,11 @@ async def test_search_multi_embedding_models(sys.modules.pop(mod, None)from src.main import create_app, startup_tasks- from src.config.settings import clients, INDEX_NAME+ from src.config.settings import clients, get_index_nameawait clients.initialize()try:- await clients.opensearch.indices.delete(index=INDEX_NAME)+ await clients.opensearch.indices.delete(index=get_index_name())await asyncio.sleep(1)except Exception:pass@@ -626,12 +626,12 @@ async def test_router_upload_ingest_traditional(tmp_path: Path, disable_langflowsys.modules.pop(mod, None)from src.main import create_app, startup_tasksimport src.api.router as upload_router- from src.config.settings import clients, INDEX_NAME, DISABLE_INGEST_WITH_LANGFLOW+ from src.config.settings import clients, get_index_name, DISABLE_INGEST_WITH_LANGFLOW# Ensure a clean index before startupawait clients.initialize()try:- await clients.opensearch.indices.delete(index=INDEX_NAME)+ await clients.opensearch.indices.delete(index=get_index_name())# Wait for deletion to completeawait asyncio.sleep(1)except Exception:@@ -646,7 +646,7 @@ async def test_router_upload_ingest_traditional(tmp_path: Path, disable_langflow# Verify index is truly empty after startuptry:- count_response = await clients.opensearch.count(index=INDEX_NAME)+ count_response = await clients.opensearch.count(index=get_index_name())doc_count = count_response.get('count', 0)assert doc_count == 0, f"Index should be empty after startup but contains {doc_count} documents"except Exception as e:diff --git a/tests/integration/test_startup_ingest.py b/tests/integration/test_startup_ingest.pyindex 78402392a..4c43cd546 100644--- a/tests/integration/test_startup_ingest.py+++ b/tests/integration/test_startup_ingest.py@@ -60,12 +60,12 @@ async def test_startup_ingest_creates_task(disable_langflow_ingest: bool):sys.modules.pop(mod, None)from src.main import create_app, startup_tasks- from src.config.settings import clients, INDEX_NAME+ from src.config.settings import clients, get_index_name# Ensure a clean index before startupawait clients.initialize()try:- await clients.opensearch.indices.delete(index=INDEX_NAME)+ await clients.opensearch.indices.delete(index=get_index_name())except Exception:pass