OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
Add ACL support and harden OpenSearch handling
Edwin Josebdea290445f0
diff --git a/flows/components/opensearch.py b/flows/components/opensearch.pyindex 69d650439..aa063478b 100644--- a/flows/components/opensearch.py+++ b/flows/components/opensearch.py@@ -1556,7 +1556,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon}},"aggs": {- "data_sources": {"terms": {"field": "filename", "size": 20}},+ "data_sources": {"terms": {"field": "filename.keyword", "size": 20}},"document_types": {"terms": {"field": "mimetype", "size": 10}},"owners": {"terms": {"field": "owner", "size": 10}},"embedding_models": {"terms": {"field": "embedding_model", "size": 10}},diff --git a/flows/components/opensearch_multimodel.py b/flows/components/opensearch_multimodel.pyindex 5a75e846a..4c9af8937 100644--- a/flows/components/opensearch_multimodel.py+++ b/flows/components/opensearch_multimodel.py@@ -11,8 +11,6 @@ from opensearchpy import OpenSearch, helpersfrom opensearchpy.exceptions import OpenSearchException, RequestErrorfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store-from lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection-from lfx.inputs.inputs import DictInputfrom lfx.io import (BoolInput,DropdownInput,@@ -64,7 +62,6 @@ def get_embedding_field_name(model_name: str) -> str:return f"chunk_embedding_{normalize_model_name(model_name)}"-@vector_store_connectionclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):"""OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.@@ -95,33 +92,10 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompondisplay_name: str = "OpenSearch (Multi-Model Multi-Embedding)"icon: str = "OpenSearch"description: str = (- "Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search."- "To search use the tools search_documents and raw_search. Search documents takes a query for vector search, for example\n"- " {search_query: \"components in openrag\"}"- "\n"- "you can also override the filter_expression to limit the hybrid query in search_documents by also passing filter_expression\n"- "for example:\n"- " {search_query: \"components in openrag\", filter_expression: {\"data_sources\":[\"my_doc.md\"],\"document_types\":[\"*\"],\"owners\":[\"*\"],\"connector_types\":[\"*\"]},\"limit\":10,\"scoreThreshold\":0}"- "\n"- "raw_search takes actual opensearch queries for example:"- " {"- " \"size\": 100,"- " \"query\": {"- " \"term\": {\"filename\": \"my_doc.md\"}"- " }"- " \"_source\": [\"filename\", \"text\", \"page\"]"- " }"- "\n"- "or:"- "\n"- " {"- " \"size\": 0,"- " \"aggs\": {"- " \"distinct_filenames\": {"- " \"cardinality\": {\"field\": \"filename\"}"- " }"- " },"- " }"+ "Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. "+ "To search use the tools search_documents and raw_search. "+ "Search documents takes a query for vector search, for example\n"+ ' {search_query: "components in openrag"}')# Keys we consider baseline@@ -370,7 +344,6 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponname="search_results",method="search_documents",),- Output(display_name="DataFrame", name="dataframe", method="as_dataframe"),Output(display_name="Raw Search", name="raw_search", method="raw_search"),]@@ -386,22 +359,26 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponRaises:ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict)."""- query = self.search_query- if isinstance(query, str):- query = json.loads(query)+ raw_query = query if query is not None else self.search_query+ if isinstance(raw_query, str):+ raw_query = json.loads(raw_query)client = self.build_client()- logger.info(f"query: {query}")+ logger.info(f"query: {raw_query}")resp = client.search(index=self.index_name,- body=query,+ body=raw_query,params={"terminate_after": 0},)# Remove any _source keys whose value is a list of floats (embedding vectors)+ # Minimum length threshold to identify embedding vectors+ min_vector_length = 100+def is_vector(val):- # Accepts if it's a list of numbers (float or int) and has reasonable vector length (>3)+ # Accepts if it's a list of numbers (float or int) and has reasonable vector lengthreturn (- isinstance(val, list) and len(val) > 100 and all(isinstance(x, (float, int)) for x in val)+ isinstance(val, list) and len(val) > min_vector_length and all(isinstance(x, (float, int)) for x in val))+if "hits" in resp and "hits" in resp["hits"]:for hit in resp["hits"]["hits"]:source = hit.get("_source")@@ -540,6 +517,11 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponThis allows adding new embedding models without recreating the entire index.Also ensures the embedding_model tracking field exists.+ Note: Some OpenSearch versions/configurations have issues with dynamically adding+ knn_vector mappings (NullPointerException). This method checks if the field+ already exists before attempting to add it, and gracefully skips if the field+ is already properly configured.+Args:client: OpenSearch client instanceindex_name: Target index name@@ -550,6 +532,24 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponef_construction: Construction parameterm: HNSW parameter"""+ # First, check if the field already exists and is properly mapped+ properties = self._get_index_properties(client)+ if self._is_knn_vector_field(properties, field_name):+ # Field already exists as knn_vector - verify dimensions match+ existing_dim = self._get_field_dimension(properties, field_name)+ if existing_dim is not None and existing_dim != dim:+ logger.warning(+ f"Field '{field_name}' exists with dimension {existing_dim}, "+ f"but current embedding has dimension {dim}. Using existing mapping."+ )+ else:+ logger.info(+ f"[OpenSearchMultimodal] Field '{field_name}' already exists"+ f"as knn_vector with matching dimensions - skipping mapping update"+ )+ return++ # Field doesn't exist, try to add the mappingtry:mapping = {"properties": {@@ -571,13 +571,26 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponclient.indices.put_mapping(index=index_name, body=mapping)logger.info(f"Added/updated embedding field mapping: {field_name}")except Exception as e:- logger.warning(f"Could not add embedding field mapping for {field_name}: {e}")+ # Check if this is the known OpenSearch k-NN NullPointerException issue+ error_str = str(e).lower()+ if "null" in error_str or "nullpointerexception" in error_str:+ logger.warning(+ f"[OpenSearchMultimodal] Could not add embedding field mapping for {field_name}"+ f"due to OpenSearch k-NN plugin issue: {e}. "+ f"This is a known issue with some OpenSearch versions. "+ f"[OpenSearchMultimodal] Skipping mapping update. "+ f"Please ensure the index has the correct mapping for KNN search to work."+ )+ # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist+ return+ logger.warning(f"[OpenSearchMultimodal] Could not add embedding field mapping for {field_name}: {e}")raise+ # Verify the field was added correctlyproperties = self._get_index_properties(client)if not self._is_knn_vector_field(properties, field_name):msg = f"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}"- logger.aerror(msg)+ logger.error(msg)raise ValueError(msg)def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:@@ -647,6 +660,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponReturns:List of document IDs that were successfully ingested"""+ logger.debug(f"[OpenSearchMultimodal] Bulk ingesting embeddings for {index_name}")if not mapping:mapping = {}@@ -713,6 +727,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponReturns:Configured OpenSearch client ready for operations"""+ logger.debug("[OpenSearchMultimodal] Building OpenSearch client")auth_kwargs = self._build_auth_kwargs()return OpenSearch(hosts=[self.opensearch_url],@@ -731,10 +746,10 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon# Check if we're in ingestion-only mode (no search query)has_search_query = bool((self.search_query or "").strip())if not has_search_query:- logger.debug("Ingestion-only mode activated: search operations will be skipped")- logger.debug("Starting ingestion mode...")+ logger.debug("[OpenSearchMultimodal] Ingestion-only mode activated: search operations will be skipped")+ logger.debug("[OpenSearchMultimodal] Starting ingestion mode...")- logger.warning(f"Embedding: {self.embedding}")+ logger.debug(f"[OpenSearchMultimodal] Embedding: {self.embedding}")self._add_documents_to_vector_store(client=client)return client@@ -751,16 +766,16 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponArgs:client: OpenSearch client for performing operations"""- logger.debug("[INGESTION] _add_documents_to_vector_store called")+ logger.debug("[OpenSearchMultimodal][INGESTION] _add_documents_to_vector_store called")# Convert DataFrame to Data if needed using parent's methodself.ingest_data = self._prepare_ingest_data()logger.debug(- f"[INGESTION] ingest_data type: "+ f"[OpenSearchMultimodal][INGESTION] ingest_data type: "f"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}")logger.debug(- f"[INGESTION] ingest_data content: "+ f"[OpenSearchMultimodal][INGESTION] ingest_data content: "f"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}")@@ -785,8 +800,8 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponself.log("Embedding returned None (fail-safe mode enabled). Skipping document ingestion.")return- logger.debug(f"[INGESTION] Valid embeddings after filtering: {len(embeddings_list)}")- self.log(f"Available embedding models: {len(embeddings_list)}")+ logger.debug(f"[OpenSearchMultimodal][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}")+ self.log(f"[OpenSearchMultimodal][INGESTION] Available embedding models: {len(embeddings_list)}")# Select the embedding to use for ingestionselected_embedding = None@@ -950,57 +965,96 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponmetadatas.append(data_copy)self.log(metadatas)- # Generate embeddings (threaded for concurrency) with retries- def embed_chunk(chunk_text: str) -> list[float]:- return selected_embedding.embed_documents([chunk_text])[0]+ # Generate embeddings with rate-limit-aware retry logic using tenacity+ from tenacity import (+ retry,+ retry_if_exception,+ stop_after_attempt,+ wait_exponential,+ )++ def is_rate_limit_error(exception: Exception) -> bool:+ """Check if exception is a rate limit error (429)."""+ error_str = str(exception).lower()+ return "429" in error_str or "rate_limit" in error_str or "rate limit" in error_str++ def is_other_retryable_error(exception: Exception) -> bool:+ """Check if exception is retryable but not a rate limit error."""+ # Retry on most exceptions except for specific non-retryable ones+ # Add other non-retryable exceptions here if needed+ return not is_rate_limit_error(exception)++ # Create retry decorator for rate limit errors (longer backoff)+ retry_on_rate_limit = retry(+ retry=retry_if_exception(is_rate_limit_error),+ stop=stop_after_attempt(5),+ wait=wait_exponential(multiplier=2, min=2, max=30),+ reraise=True,+ before_sleep=lambda retry_state: logger.warning(+ f"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), "+ f"backing off for {retry_state.next_action.sleep:.1f}s"+ ),+ )++ # Create retry decorator for other errors (shorter backoff)+ retry_on_other_errors = retry(+ retry=retry_if_exception(is_other_retryable_error),+ stop=stop_after_attempt(3),+ wait=wait_exponential(multiplier=1, min=1, max=8),+ reraise=True,+ before_sleep=lambda retry_state: logger.warning(+ f"Error embedding chunk (attempt {retry_state.attempt_number}/3), "+ f"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}"+ ),+ )++ def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:+ """Embed a single chunk with rate-limit-aware retry logic."""- vectors: list[list[float]] | None = None- last_exception: Exception | None = None- delay = 1.0- attempts = 0- max_attempts = 3+ @retry_on_rate_limit+ @retry_on_other_errors+ def _embed(text: str) -> list[float]:+ return selected_embedding.embed_documents([text])[0]- while attempts < max_attempts:- attempts += 1try:- # Restrict concurrency for IBM/Watsonx models to avoid rate limits- is_ibm = (embedding_model and "ibm" in str(embedding_model).lower()) or (- selected_embedding and "watsonx" in type(selected_embedding).__name__.lower()- )- logger.debug(f"Is IBM: {is_ibm}")- max_workers = 1 if is_ibm else min(max(len(texts), 1), 8)-- with ThreadPoolExecutor(max_workers=max_workers) as executor:- futures = {executor.submit(embed_chunk, chunk): idx for idx, chunk in enumerate(texts)}- vectors = [None] * len(texts)- for future in as_completed(futures):- idx = futures[future]- vectors[idx] = future.result()- break- except Exception as exc:- last_exception = exc- if attempts >= max_attempts:- logger.error(- f"Embedding generation failed for model {embedding_model} after retries",- error=str(exc),- )- raise- logger.warning(- "Threaded embedding generation failed for model %s (attempt %s/%s), retrying in %.1fs",- embedding_model,- attempts,- max_attempts,- delay,+ return _embed(chunk_text)+ except Exception as e:+ logger.error(+ f"Failed to embed chunk {chunk_idx} after all retries: {e}",+ error=str(e),)- time.sleep(delay)- delay = min(delay * 2, 8.0)-- if vectors is None:- raise RuntimeError(- f"Embedding generation failed for {embedding_model}: {last_exception}"- if last_exception- else f"Embedding generation failed for {embedding_model}"- )+ raise++ # Restrict concurrency for IBM/Watsonx models to avoid rate limits+ is_ibm = (embedding_model and "ibm" in str(embedding_model).lower()) or (+ selected_embedding and "watsonx" in type(selected_embedding).__name__.lower()+ )+ logger.debug(f"Is IBM: {is_ibm}")++ # For IBM models, use sequential processing with rate limiting+ # For other models, use parallel processing+ vectors: list[list[float]] = [None] * len(texts)++ if is_ibm:+ # Sequential processing with inter-request delay for IBM models+ inter_request_delay = 0.6 # ~1.67 req/s, safely under 2 req/s limit+ logger.info(f"Using sequential processing for IBM model with {inter_request_delay}s delay between requests")++ for idx, chunk in enumerate(texts):+ if idx > 0:+ # Add delay between requests (but not before the first one)+ time.sleep(inter_request_delay)+ vectors[idx] = embed_chunk_with_retry(chunk, idx)+ else:+ # Parallel processing for non-IBM models+ max_workers = min(max(len(texts), 1), 8)+ logger.debug(f"Using parallel processing with {max_workers} workers")++ with ThreadPoolExecutor(max_workers=max_workers) as executor:+ futures = {executor.submit(embed_chunk_with_retry, chunk, idx): idx for idx, chunk in enumerate(texts)}+ for future in as_completed(futures):+ idx = futures[future]+ vectors[idx] = future.result()if not vectors:self.log(f"No vectors generated from documents for model {embedding_model}.")@@ -1600,7 +1654,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon}},"aggs": {- "data_sources": {"terms": {"field": "filename", "size": 20}},+ "data_sources": {"terms": {"field": "filename.keyword", "size": 20}},"document_types": {"terms": {"field": "mimetype", "size": 10}},"owners": {"terms": {"field": "owner", "size": 10}},"embedding_models": {"terms": {"field": "embedding_model", "size": 10}},@@ -1776,4 +1830,4 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponexcept (KeyError, ValueError) as e:self.log(f"update_build_config error: {e}")- return build_config+ return build_config\ No newline at end of filediff --git a/frontend/app/api/queries/useGetSearchQuery.ts b/frontend/app/api/queries/useGetSearchQuery.tsindex 972f8f16d..c77be72a3 100644--- a/frontend/app/api/queries/useGetSearchQuery.ts+++ b/frontend/app/api/queries/useGetSearchQuery.ts@@ -33,6 +33,8 @@ export interface ChunkResult {embedding_model?: string;embedding_dimensions?: number;index?: number;+ allowed_users?: string[];+ allowed_groups?: string[];}export interface File {@@ -57,6 +59,8 @@ export interface File {| "sync";error?: string;chunks?: ChunkResult[];+ allowed_users?: string[];+ allowed_groups?: string[];}export const useGetSearchQuery = (@@ -150,6 +154,8 @@ export const useGetSearchQuery = (connector_type?: string;embedding_model?: string;embedding_dimensions?: number;+ allowed_users?: string[];+ allowed_groups?: string[];}>();@@ -181,6 +187,8 @@ export const useGetSearchQuery = (connector_type: chunk.connector_type,embedding_model: chunk.embedding_model,embedding_dimensions: chunk.embedding_dimensions,+ allowed_users: chunk.allowed_users || [],+ allowed_groups: chunk.allowed_groups || [],});}});@@ -199,6 +207,8 @@ export const useGetSearchQuery = (embedding_model: file.embedding_model,embedding_dimensions: file.embedding_dimensions,chunks: file.chunks,+ allowed_users: file.allowed_users || [],+ allowed_groups: file.allowed_groups || [],}));return files;diff --git a/frontend/app/knowledge/chunks/page.tsx b/frontend/app/knowledge/chunks/page.tsxindex 4d8495060..f4271faf4 100644--- a/frontend/app/knowledge/chunks/page.tsx+++ b/frontend/app/knowledge/chunks/page.tsx@@ -316,6 +316,67 @@ function ChunksPageContent() {</div> */}</dl></div>+ {(fileData?.allowed_users?.length ?? 0) > 0 ||+ (fileData?.allowed_groups?.length ?? 0) > 0 ? (+ <div className="mb-4">+ <h2 className="text-xl font-semibold mt-2 mb-3">+ Access Control+ </h2>+ <dl>+ {fileData?.allowed_users &&+ fileData.allowed_users.length > 0 && (+ <div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5">+ <dt className="text-sm/6 text-muted-foreground">+ Allowed users+ </dt>+ <dd className="mt-1 text-sm/6 text-gray-800 dark:text-gray-100 sm:col-span-2 sm:mt-0">+ <div className="space-y-1">+ {fileData.allowed_users.map((user, idx) => (+ <div+ key={user ?? idx}+ className="flex items-center gap-2"+ >+ <span className="inline-flex h-6 w-6 items-center justify-center rounded-full bg-blue-100 dark:bg-blue-900">+ <span className="text-xs font-medium text-blue-800 dark:text-blue-200">+ {user?.charAt(0).toUpperCase()}+ </span>+ </span>+ <span className="text-sm break-all">{user}</span>+ </div>+ ))}+ </div>+ </dd>+ </div>+ )}++ {fileData?.allowed_groups &&+ fileData.allowed_groups.length > 0 && (+ <div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5">+ <dt className="text-sm/6 text-muted-foreground">+ Allowed groups+ </dt>+ <dd className="mt-1 text-sm/6 text-gray-800 dark:text-gray-100 sm:col-span-2 sm:mt-0">+ <div className="space-y-1">+ {fileData.allowed_groups.map((group, idx) => (+ <div+ key={group ?? idx}+ className="flex items-center gap-2"+ >+ <span className="inline-flex h-6 w-6 items-center justify-center rounded-full bg-green-100 dark:bg-green-900">+ <span className="text-xs font-medium text-green-800 dark:text-green-200">+ {group?.charAt(0).toUpperCase()}+ </span>+ </span>+ <span className="text-sm break-all">{group}</span>+ </div>+ ))}+ </div>+ </dd>+ </div>+ )}+ </dl>+ </div>+ ) : null}</div>)}</div>diff --git a/src/services/search_service.py b/src/services/search_service.pyindex b0927d0f8..67192936d 100644--- a/src/services/search_service.py+++ b/src/services/search_service.py@@ -329,7 +329,7 @@ class SearchService:search_body = {"query": query_block,"aggs": {- "data_sources": {"terms": {"field": "filename", "size": 20}},+ "data_sources": {"terms": {"field": "filename.keyword", "size": 20}},"document_types": {"terms": {"field": "mimetype", "size": 10}},"owners": {"terms": {"field": "owner_name.keyword", "size": 10}},"connector_types": {"terms": {"field": "connector_type", "size": 10}},@@ -435,21 +435,25 @@ class SearchService:# Transform results (keep for backward compatibility)chunks = []for hit in results["hits"]["hits"]:+ source = hit.get("_source", {})chunks.append({- "filename": hit["_source"].get("filename"),- "mimetype": hit["_source"].get("mimetype"),- "page": hit["_source"].get("page"),- "text": hit["_source"].get("text"),+ "filename": source.get("filename"),+ "mimetype": source.get("mimetype"),+ "page": source.get("page"),+ "text": source.get("text"),"score": hit.get("_score"),- "source_url": hit["_source"].get("source_url"),- "owner": hit["_source"].get("owner"),- "owner_name": hit["_source"].get("owner_name"),- "owner_email": hit["_source"].get("owner_email"),- "file_size": hit["_source"].get("file_size"),- "connector_type": hit["_source"].get("connector_type"),- "embedding_model": hit["_source"].get("embedding_model"), # Include in results- "embedding_dimensions": hit["_source"].get("embedding_dimensions"),+ "source_url": source.get("source_url"),+ "owner": source.get("owner"),+ "owner_name": source.get("owner_name"),+ "owner_email": source.get("owner_email"),+ "file_size": source.get("file_size"),+ "connector_type": source.get("connector_type"),+ "embedding_model": source.get("embedding_model"), # Include in results+ "embedding_dimensions": source.get("embedding_dimensions"),+ # ACL fields (may be missing for some documents)+ "allowed_users": source.get("allowed_users", []),+ "allowed_groups": source.get("allowed_groups", []),})